update doc and tests
This commit is contained in:
1 parent
ed4e542260
commit
13936a9621
4 files changed
+72
-4
No files matched your search
@@ -1,4 +1,5 @@
|
||||
__version__ = "1.0.0"
|
||||
from .tokenization_auto import AutoTokenizer
|
||||
from .tokenization_bert import BertTokenizer, BasicTokenizer, WordpieceTokenizer
|
||||
from .tokenization_openai import OpenAIGPTTokenizer
|
||||
from .tokenization_transfo_xl import (TransfoXLTokenizer, TransfoXLCorpus)
|
||||
|
||||
Reference in new issue
Block a user