Tentative improvement on sequence_length error for position_ids

in RobertaEmbeddings config (Should we extend to all position_embeddings
?)

src/transformers/modeling_roberta.py

@@ -115,6 +115,11 @@ class RobertaEmbeddings(nn.Module):
 
         if inputs_embeds is None:
             inputs_embeds = self.word_embeddings(input_ids)
+
+        max_position_embeddings = self.position_embeddings.num_embeddings
+        if position_ids.max() > max_position_embeddings:
+            raise ValueError("Position ids are too large, the max is {}.".format(max_position_embeddings))
+
         position_embeddings = self.position_embeddings(position_ids)
         token_type_embeddings = self.token_type_embeddings(token_type_ids)
 

tests/test_modeling_roberta.py

@@ -371,6 +371,20 @@ class RobertaModelTest(ModelTesterMixin, unittest.TestCase):
         self.assertEqual(position_ids.shape, expected_positions.shape)
         self.assertTrue(torch.all(torch.eq(position_ids, expected_positions)))
 
+    def test_handling_too_long_sequences_for_position_ids(self):
+        config = self.model_tester.prepare_config_and_inputs()[0]
+        model = RobertaEmbeddings(config=config)
+
+        input_ids = torch.zeros((1, config.max_position_embeddings + 1)).long()
+        expected_positions = torch.as_tensor([list(range(config.max_position_embeddings + 1))]) + model.padding_idx + 1
+
+        position_ids = create_position_ids_from_input_ids(input_ids, model.padding_idx)
+        self.assertEqual(position_ids.shape, expected_positions.shape)
+        self.assertTrue(torch.all(torch.eq(position_ids, expected_positions)))
+
+        with self.assertRaises(ValueError):
+            model.forward(input_ids)
+
     def test_create_position_ids_from_inputs_embeds(self):
         """Ensure that the default position ids only assign a sequential . This is a regression
         test for https://github.com/huggingface/transformers/issues/1761