Bumping tokenizers dependency to 0.8.0.dev1 to check tests are passing.

Unfortunately i accidentally orphaned my other PR

model_cards/ViktorAlm/electra-base-norwegian-uncased-discriminator/README.md

@@ -1,19 +1,25 @@
+---
+language: norwegian
+thumbnail: https://i.imgur.com/QqSEC5I.png
+---
+
 # Norwegian Electra
-Image incoming, im going to have som fun with this one.
+![Image of norwegian electra](https://i.imgur.com/QqSEC5I.png)
 
 Trained on Oscar + wikipedia + opensubtitles + some other data I had with the awesome power of TPUs(V3-8)
 
 Use with caution. I have no downstream tasks in Norwegian to test on so I have no idea of its performance yet.
-
+# Model
+## Electra: Pre-training Text Encoders as Discriminators Rather Than Generators
+Kevin Clark and Minh-Thang Luong and Quoc V. Le and Christopher D. Manning
+- https://openreview.net/pdf?id=r1xMH1BtvB
+- https://github.com/google-research/electra
 # Acknowledgments
-
 ### TensorFlow Research Cloud
 Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). Thanks for providing access to the TFRC ❤️
 - https://www.tensorflow.org/tfrc
-
 #### OSCAR corpus
 - https://oscar-corpus.com/
-
 #### OPUS
 - http://opus.nlpl.eu/
 - http://www.opensubtitles.org/

model_cards/mrm8488/gpt2-imdb-neg/README.md

@@ -20,6 +20,9 @@ A few examples of the model response to a query before and after optimisation:
 |I have watched 3 episodes |with this guy and he is such a talented actor...|	but the show is just plain awful and there ne...|	2.681171|	-4.512792|
 |We know that firefighters and|	police officers are forced to become populari...|	other chains have going to get this disaster ...|	1.367811|	-3.34017|
 
+## Training logs and metrics <img src="https://gblobscdn.gitbook.com/spaces%2F-Lqya5RvLedGEWPhtkjU%2Favatar.png?alt=media" width="25" height="25">
+Watch the whole training logs and metrics on [W&B](https://app.wandb.ai/mrm8488/gpt2-sentiment-negative?workspace=user-mrm8488)
+
 
 
 > Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488)

setup.py

@@ -78,7 +78,7 @@ extras["testing"] = ["pytest", "pytest-xdist", "timeout-decorator"]
 extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme"]
 extras["quality"] = [
     "black",
-    "isort",
+    "isort @ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort",
     "flake8==3.7.9",
 ]
 extras["dev"] = extras["testing"] + extras["quality"] + ["mecab-python3", "scikit-learn", "tensorflow", "torch"]
@@ -98,7 +98,7 @@ setup(
     packages=find_packages("src"),
     install_requires=[
         "numpy",
-        "tokenizers == 0.7.0",
+        "tokenizers == 0.8.0.dev1",
         # dataclasses for Python versions that don't have it
         "dataclasses;python_version<'3.7'",
         # filesystem locks e.g. to prevent parallel downloads

src/transformers/tokenization_utils.py

@@ -185,6 +185,15 @@ class BatchEncoding(UserDict):
 
         self._encodings = encoding
 
+    @property
+    def is_fast(self):
+        """
+        Indicate if this BatchEncoding was generated from the result of a PreTrainedTokenizerFast
+        Returns: True if generated from subclasses of PreTrainedTokenizerFast, else otherwise
+
+        """
+        return self._encodings is not None
+
     def __getitem__(self, item: Union[int, str]) -> EncodingFast:
         """ If the key is a string, get the value of the dict associated to `key` ('input_ids', 'attention_mask'...)
             If the key is an integer, get the EncodingFast for batch item with index `key`
@@ -202,6 +211,16 @@ class BatchEncoding(UserDict):
     def __getattr__(self, item: str):
         return self.data[item]
 
+    def __getstate__(self):
+        return {"data": self.data, "encodings": self._encodings}
+
+    def __setstate__(self, state):
+        if "data" in state:
+            self.data = state["data"]
+
+        if "encodings" in state:
+            self._encodings = state["encodings"]
+
     def keys(self):
         return self.data.keys()
 
@@ -224,7 +243,7 @@ class BatchEncoding(UserDict):
         """
         return self._encodings
 
-    def tokens(self, batch_index: int = 0) -> List[int]:
+    def tokens(self, batch_index: int = 0) -> List[str]:
         if not self._encodings:
             raise ValueError("tokens() is not available when using Python based tokenizers")
         return self._encodings[batch_index].tokens

src/transformers/trainer.py

@@ -542,9 +542,28 @@ class Trainer:
 
         Will only save from the master process.
         """
-        if self.is_world_master():
+
+        if is_tpu_available():
+            self._save_tpu(output_dir)
+        elif self.is_world_master():
             self._save(output_dir)
 
+    def _save_tpu(self, output_dir: Optional[str] = None):
+        output_dir = output_dir if output_dir is not None else self.args.output_dir
+        logger.info("Saving model checkpoint to %s", output_dir)
+
+        if xm.is_master_ordinal():
+            os.makedirs(output_dir, exist_ok=True)
+            torch.save(self.args, os.path.join(output_dir, "training_args.bin"))
+
+        # Save a trained model and configuration using `save_pretrained()`.
+        # They can then be reloaded using `from_pretrained()`
+        if not isinstance(self.model, PreTrainedModel):
+            raise ValueError("Trainer.model appears to not be a PreTrainedModel")
+
+        xm.rendezvous("saving_checkpoint")
+        self.model.save_pretrained(output_dir)
+
     def _save(self, output_dir: Optional[str] = None):
         output_dir = output_dir if output_dir is not None else self.args.output_dir
         os.makedirs(output_dir, exist_ok=True)

tests/test_tokenization_utils.py

@@ -16,7 +16,7 @@
 
 import unittest
 
-from transformers import PreTrainedTokenizer
+from transformers import BertTokenizer, BertTokenizerFast, PreTrainedTokenizer
 from transformers.tokenization_gpt2 import GPT2Tokenizer
 
 from .utils import slow
@@ -39,3 +39,34 @@ class TokenizerUtilsTest(unittest.TestCase):
     @slow
     def test_pretrained_tokenizers(self):
         self.check_tokenizer_from_pretrained(GPT2Tokenizer)
+
+    def test_batch_encoding_pickle(self):
+        from pickle import loads, dumps
+
+        # Get a slow & a fast tokenizer
+        tok_slow = BertTokenizer.from_pretrained("bert-base-cased")
+        tok_fast = BertTokenizerFast.from_pretrained("bert-base-cased")
+
+        # Encode a sentence
+        be_slow = tok_slow.encode_plus("This is a dummy input sentence")
+        be_fast = tok_fast.encode_plus("This is a dummy input sentence")
+
+        # Make sure both are pickable
+        be_slow_data = dumps(be_slow)
+        be_fast_data = dumps(be_fast)
+
+        # Try to restore
+        be_slow_pickled = loads(be_slow_data)
+        be_fast_pickled = loads(be_fast_data)
+
+        # Ensure pickled objects keeps the is_fast attribute
+        self.assertFalse(be_slow_pickled.is_fast)
+        self.assertTrue(be_fast_pickled.is_fast)
+
+        # Ensure .data match
+        self.assertDictEqual(be_slow_pickled.data, be_slow.data)
+        self.assertDictEqual(be_fast_pickled.data, be_fast.data)
+
+        # Ensure .encodings match
+        self.assertIsNone(be_slow_pickled.encodings)
+        self.assertEqual(len(be_fast_pickled.encodings), len(be_fast.encodings))