Compare commits

..
Author SHA1 Message Date
Morgan Funtowicz f22083c575 Fix TransfoXL constructor vocab_file / pretrained_vocab_file mixin.
Signed-off-by: Morgan Funtowicz <morgan@huggingface.co>
2020-02-25 00:08:20 +01:00
Morgan Funtowicz 5b1b723e2f Making sure there is really a bug in unittest 2020-02-24 23:39:39 +01:00
Morgan Funtowicz 2903c72076 flake8
Signed-off-by: Morgan Funtowicz <morgan@huggingface.co>
2020-02-24 23:28:31 +01:00
Morgan Funtowicz 9fc55e44d1 Quality and format.
Signed-off-by: Morgan Funtowicz <morgan@huggingface.co>
2020-02-24 23:11:48 +01:00
Morgan Funtowicz 88f709a601 Update tokenizers to 0.5.2
Signed-off-by: Morgan Funtowicz <morgan@huggingface.co>
2020-02-24 23:06:52 +01:00
Morgan Funtowicz 0cc4c5453d Added some save_pretrained / from_pretrained unittests.
Signed-off-by: Morgan Funtowicz <morgan@huggingface.co>
2020-02-24 23:02:03 +01:00
Morgan Funtowicz e0d888d3e8 TransfoXLTokenizerFast uses a json vocabulary file + warning about incompatibility between Python and Rust
Signed-off-by: Morgan Funtowicz <morgan@huggingface.co>
2020-02-24 22:57:01 +01:00
Morgan Funtowicz 11269fb6de Bump tokenizers dependency to 0.5.1
Signed-off-by: Morgan Funtowicz <morgan@huggingface.co>
2020-02-24 22:57:01 +01:00
Morgan Funtowicz 39134cc9b4 Remove python quick and dirty fix for clean Rust impl.
Signed-off-by: Morgan Funtowicz <morgan@huggingface.co>
2020-02-24 22:57:01 +01:00
Morgan Funtowicz d6a2454995 Added save_vocabulary tests.
Signed-off-by: Morgan Funtowicz <morgan@huggingface.co>
2020-02-24 22:57:01 +01:00
Morgan Funtowicz 7421592037 Renamed file generate by tokenizers when calling save_pretrained to match python.
Signed-off-by: Morgan Funtowicz <morgan@huggingface.co>
2020-02-24 22:57:01 +01:00
6 changed files with 8 additions and 8 deletions

No files matched your search

+1 -1
View File
@@ -26,7 +26,7 @@ author = u'huggingface'
# The short X.Y version
version = u''
# The full version, including alpha/beta/rc tags
release = u'2.5.1'
release = u'2.5.0'
# -- General configuration ---------------------------------------------------
+1 -1
View File
@@ -79,7 +79,7 @@ extras["dev"] = extras["testing"] + extras["quality"] + ["mecab-python3", "sciki
setup(
name="transformers",
version="2.5.1",
version="2.5.0",
author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Sam Shleifer, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors",
author_email="thomas@huggingface.co",
description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch",
+1 -1
View File
@@ -2,7 +2,7 @@
# There's no way to ignore "F401 '...' imported but unused" warnings in this
# module, but to preserve other warnings. So, don't check this module at all.
__version__ = "2.5.1"
__version__ = "2.5.0"
# Work around to update TensorFlow's absl.logging threshold which alters the
# default Python logging output behavior when present.
+1 -1
View File
@@ -645,7 +645,7 @@ class TransfoXLModel(TransfoXLPreTrainedModel):
else:
return None
def _update_mems(self, hids, mems, mlen, qlen):
def _update_mems(self, hids, mems, qlen, mlen):
# does not deal with None
if mems is None:
return None
+2 -2
View File
@@ -157,7 +157,7 @@ class AutoTokenizer:
A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.
The proxies are used on each request.
use_fast: (`optional`) boolean, default False:
use_fast: (`optional`) boolean, default True:
Indicate if transformers should try to load the fast version of the tokenizer (True) or use the Python one (False).
inputs: (`optional`) positional arguments: will be passed to the Tokenizer ``__init__`` method.
@@ -183,7 +183,7 @@ class AutoTokenizer:
if "bert-base-japanese" in pretrained_model_name_or_path:
return BertJapaneseTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
use_fast = kwargs.pop("use_fast", False)
use_fast = kwargs.pop("use_fast", True)
for config_class, (tokenizer_class_py, tokenizer_class_fast) in TOKENIZER_MAPPING.items():
if isinstance(config, config_class):
if tokenizer_class_fast and use_fast:
+2 -2
View File
@@ -101,5 +101,5 @@ class AutoTokenizerTest(unittest.TestCase):
self.assertFalse(issubclass(child_model_fast, parent_model_fast))
def test_from_pretrained_use_fast_toggle(self):
self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"), BertTokenizer)
self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased", use_fast=True), BertTokenizerFast)
self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"), BertTokenizerFast)
self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased", use_fast=False), BertTokenizer)