adding max_lengths for single sentences and sentences pairs

This commit is contained in:
thomwolf committed 2019-08-23 17:31:11 +02:00
1 parent 90dcd8c05d
commit 47d6853439
5 files changed
+40

No files matched your search

@@ -139,6 +139,14 @@ class BertTokenizer(PreTrainedTokenizer):
tokenize_chinese_chars=tokenize_chinese_chars)
self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=self.unk_token)
@property
def max_len_single_sentence(self):
return self.max_len - 2 # take into account special tokens
@property
def max_len_sentences_pair(self):
return self.max_len - 3 # take into account special tokens
@property
def vocab_size(self):
return len(self.vocab)