unified tokenizer api and serialization + tests

pytorch_transformers/tests/tokenization_utils_test.py

@@ -17,6 +17,7 @@ from __future__ import division
 from __future__ import print_function
 
 import unittest
+import six
 
 from pytorch_transformers import PreTrainedTokenizer
 from pytorch_transformers.tokenization_gpt2 import GPT2Tokenizer
@@ -27,8 +28,17 @@ class TokenizerUtilsTest(unittest.TestCase):
         for model_name in s3_models[:1]:
             tokenizer = tokenizer_class.from_pretrained(model_name)
             self.assertIsNotNone(tokenizer)
+            self.assertIsInstance(tokenizer, tokenizer_class)
             self.assertIsInstance(tokenizer, PreTrainedTokenizer)
 
+            for special_tok in tokenizer.all_special_tokens:
+                if six.PY2:
+                    self.assertIsInstance(special_tok, unicode)
+                else:
+                    self.assertIsInstance(special_tok, str)
+                special_tok_id = tokenizer.convert_tokens_to_ids(special_tok)
+                self.assertIsInstance(special_tok_id, int)
+
     def test_pretrained_tokenizers(self):
         self.check_tokenizer_from_pretrained(GPT2Tokenizer)