overflowing_tokens do not really make sense here, let's just return a number
Co-Authored-By: Lysandre Debut <lysandre.debut@reseau.eseo.fr>
This commit is contained in:
1 parent
f5bcde0b2f
commit
b350662955
2 files changed
+3
-2
No files matched your search
@@ -338,7 +338,7 @@ def convert_examples_to_features(
|
||||
max_length=max_length,
|
||||
truncate_both_sequences=True
|
||||
)
|
||||
if 'overflowing_tokens' in inputs and len(inputs['overflowing_tokens']) > 0:
|
||||
if 'num_truncated_tokens' in inputs and inputs['num_truncated_tokens'] > 0:
|
||||
logger.info('Attention! you are cropping tokens (swag task is ok). '
|
||||
'If you are training ARC and RACE and you are poping question + options,'
|
||||
'you need to try to use a bigger max seq length!')
|
||||
|
||||
Reference in new issue
Block a user