diff --git a/src/transformers/tokenization_blenderbot.py b/src/transformers/tokenization_blenderbot.py index 6e4582f1..b337c74c 100644 --- a/src/transformers/tokenization_blenderbot.py +++ b/src/transformers/tokenization_blenderbot.py @@ -60,7 +60,6 @@ BLENDERBOT_90M_PRETRAINED_VOCAB_FILES_MAP = { def get_pairs(word): - """Return set of symbol pairs in a word. Word is represented as tuple of symbols (symbols being variable-length strings). diff --git a/tests/test_modeling_blenderbot.py b/tests/test_modeling_blenderbot.py index cf328939..402d6a2b 100644 --- a/tests/test_modeling_blenderbot.py +++ b/tests/test_modeling_blenderbot.py @@ -115,6 +115,8 @@ class BlenderbotTesterMixin(ModelTesterMixin, unittest.TestCase): model.model.decoder.embed_positions.weight.shape == expected_shape + + @unittest.skipUnless(torch_device != "cpu", "3B test too slow on CPU.") @require_torch class Blenderbot3BIntegrationTests(unittest.TestCase): @@ -131,104 +133,7 @@ class Blenderbot3BIntegrationTests(unittest.TestCase): def tokenizer(self): return BlenderbotTokenizer.from_pretrained(self.ckpt) - @slow - def test_tokenization_same_as_parlai(self): - # TODO(SS): this can run on CPU - self.assertListEqual(self.tokenizer("sam").input_ids, [268, 343, 2]) - @slow - @unittest.skip("This fails.") - def test_forward_3B_same_as_parlai(self): - torch.manual_seed(0) - config = BlenderbotConfig( - d_model=16, - vocab_size=50, - encoder_attention_heads=2, - decoder_attention_heads=2, - encoder_layers=2, - decoder_layers=2, - encoder_ffn_dim=4, - decoder_ffn_dim=4, - variant="prelayernorm", - scale_embedding=True, - normalize_embedding=True, - max_position_embeddings=50, - activation_function="gelu", - static_position_embeddings=True, - dropout=0.1, - ) - input_ids = torch.tensor( - [[49, 12, 38, 24, 13, 25, 10, 28, 37, 7, 44, 7, 2, 3]], - dtype=torch.long, - device=torch_device, - ) - model = BlenderbotForConditionalGeneration(config).to(torch_device) - model.eval() - - # output from parlai model after copying the same blenderbot weight in parlai and setting a manual_seed - expected_logits = torch.tensor( - [ - [ - [ - -1.0000e20, - 0.0000e00, - 2.6462e-02, - 9.7588e-02, - -5.6271e-02, - -1.1409e-01, - -3.3294e-02, - -1.0423e-01, - -4.8363e-02, - -1.2610e-01, - -3.4125e-02, - -2.9841e-02, - 8.6975e-02, - 2.5547e-02, - 2.0425e-03, - -5.9153e-02, - 1.4392e-02, - -1.4324e-02, - 1.2774e-01, - -5.3284e-02, - -1.5876e-02, - 9.1752e-02, - -3.0166e-02, - -3.1726e-02, - 9.9600e-02, - 1.0991e-01, - 8.0946e-03, - 3.5396e-03, - -2.5164e-02, - -4.0277e-02, - -3.6360e-02, - 7.5158e-02, - 4.3379e-02, - 1.3465e-01, - -1.3209e-01, - -1.1706e-01, - 5.6180e-02, - -3.6239e-02, - 6.6490e-02, - 4.9879e-02, - 1.0979e-02, - -2.7895e-02, - -8.4691e-02, - 4.5857e-02, - 2.0233e-02, - 9.2533e-02, - -6.4260e-02, - -5.0988e-02, - -4.0852e-02, - 1.9867e-02, - ] - ] - ], - device=torch_device, - ) - - decoder_inputs = torch.LongTensor([1]).expand(1, 1).to(torch_device) - logits = model(input_ids, decoder_input_ids=decoder_inputs, return_dict=True)["logits"] - assert torch.allclose(expected_logits, logits, atol=1e-4) @unittest.skip("This fails.") @slow @@ -257,6 +162,7 @@ class Blenderbot3BIntegrationTests(unittest.TestCase): # 9/24: __start__ I have social anxiety. I feel like I'm going to throw up and I'm sweating and blushing.__end__ + @require_torch class Blenderbot90MIntegrationTests(unittest.TestCase): ckpt = "facebook/blenderbot-90M" @@ -272,8 +178,6 @@ class Blenderbot90MIntegrationTests(unittest.TestCase): def tokenizer(self): return BlenderbotSmallTokenizer.from_pretrained(self.ckpt) - def test_tokenization_same_as_parlai(self): - assert self.tokenizer("sam").input_ids == [1384] @unittest.skip("This does not pass. It should be deleted") def test_forward_90M_same_as_parlai(self): @@ -377,6 +281,7 @@ class Blenderbot90MIntegrationTests(unittest.TestCase): ] # tgt_text = "i ' m not sure . i just feel like i ' m going to throw up ." tgt_text = "i don't know. i just feel like i'm going to throw up. it's not fun." + # FP16: " i don't know. i feel like i'm going to throw up. it's hard for me." model_inputs = self.tokenizer(src_text, return_tensors="pt").to(torch_device) generated_ids = self.model.generate(**model_inputs, early_stopping=True)[0] diff --git a/tests/test_tokenization_blenderbot.py b/tests/test_tokenization_blenderbot.py index 665a754c..02b238f9 100644 --- a/tests/test_tokenization_blenderbot.py +++ b/tests/test_tokenization_blenderbot.py @@ -18,7 +18,8 @@ import json import os import unittest -from transformers.tokenization_blenderbot import VOCAB_FILES_NAMES, BlenderbotSmallTokenizer +from transformers.tokenization_blenderbot import VOCAB_FILES_NAMES, BlenderbotSmallTokenizer, BlenderbotTokenizer +from transformers.file_utils import cached_property from .test_tokenization_common import TokenizerTesterMixin @@ -66,8 +67,27 @@ class BlenderbotSmallTokenizerTest(TokenizerTesterMixin, unittest.TestCase): def test_special_tokens_small_tok(self): tok = BlenderbotSmallTokenizer.from_pretrained("facebook/blenderbot-90M") + assert tok("sam").input_ids == [1384] src_text = "I am a small frog." encoded = tok([src_text], padding=False, truncation=False)["input_ids"] decoded = tok.batch_decode(encoded, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] assert src_text != decoded # I wish it did! assert decoded == "i am a small frog ." + +class Blenderbot3BTokenizerTests(unittest.TestCase): + @cached_property + def tokenizer_3b(self): + return BlenderbotTokenizer.from_pretrained("facebook/blenderbot-3B") + + def test_special_tokens_3B_tok(self): + tok = self.tokenizer_3b + src_text = " I am a small frog." + encoded = tok([src_text], padding=False, truncation=False)["input_ids"] + decoded = tok.batch_decode(encoded, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + assert src_text == decoded # I wish it did! + import ipdb; ipdb.set_trace() + + def test_3B_tokenization_same_as_parlai(self): + # TODO(SS): this can run on CPU + tok = self.tokenizer_3b + self.assertListEqual(tok(["sam"]).input_ids[0], [268, 343, 2])