update doc and tests

This commit is contained in:
thomwolf committed 2019-08-05 18:48:16 +02:00
1 parent ed4e542260
commit 13936a9621
4 files changed
+72 -4

No files matched your search

+1
View File
@@ -1,4 +1,5 @@
__version__ = "1.0.0"
from .tokenization_auto import AutoTokenizer
from .tokenization_bert import BertTokenizer, BasicTokenizer, WordpieceTokenizer
from .tokenization_openai import OpenAIGPTTokenizer
from .tokenization_transfo_xl import (TransfoXLTokenizer, TransfoXLCorpus)