add tokenizer and tests

2019-06-21 11:09:51 +02:00
1 parent 45709d7532
commit 32da75486b
@@ -54,7 +54,8 @@ setup(
                      'boto3',
                      'requests',
                      'tqdm',
-                      'regex'],
+                      'regex',
+                      'sentencepiece'],
    entry_points={
      'console_scripts': [
        "pytorch_pretrained_bert=pytorch_pretrained_bert.__main__:main",