c-my/transformers
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make language generation task pass

Browse Source
This commit is contained in:
Patrick von Platen
2020-03-17 10:02:32 +01:00
parent 6caade97c2
commit 2095fade87
6 changed files with 40 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/transformers/__init__.py
+3 -6
View File
@@ -176,9 +176,7 @@ if is_torch_available():
load_tf_weights_in_transfo_xl,
TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP,
)
from .modeling_transfo_xl_utilities import (
LogUniformSampler,
)
from .modeling_transfo_xl_utilities import LogUniformSampler
from .modeling_gpt2 import (
GPT2PreTrainedModel,
GPT2Model,
@@ -347,11 +345,10 @@ if is_tf_available():
TFTransfoXLModel,
TFTransfoXLLMHeadModel,
TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP,
TFAdaptiveEmbedding,
)
from .modeling_tf_transfo_xl_utilities import (
TFLogUniformSampler,
)
from .modeling_tf_transfo_xl_utilities import TFLogUniformSampler
from .modeling_tf_xlnet import (
TFXLNetPreTrainedModel,
src/transformers/modeling_tf_transfo_xl.py
+22 -2
View File
@@ -23,7 +23,7 @@ import tensorflow as tf
from .configuration_transfo_xl import TransfoXLConfig
from .file_utils import add_start_docstrings, add_start_docstrings_to_callable
from .modeling_tf_transfo_xl_utilities import TFAdaptiveSoftmaxMask
from .modeling_tf_transfo_xl_utilities import TFAdaptiveSoftmaxMask, TFLogUniformSampler
from .modeling_tf_utils import TFPreTrainedModel, get_initializer, keras_serializable, shape_list
@@ -745,13 +745,25 @@ class TFTransfoXLLMHeadModel(TFTransfoXLPreTrainedModel):
self.sample_softmax = config.sample_softmax
# use sampled softmax
if config.sample_softmax > 0:
raise NotImplementedError
# self.out_layer = nn.Linear(config.d_model, config.vocab_size)
self.sampler = TFLogUniformSampler(config.vocab_size, config.sample_softmax)
# use adaptive softmax (including standard softmax)
else:
self.crit = TFAdaptiveSoftmaxMask(
config.vocab_size, config.d_embed, config.d_model, config.cutoffs, div_val=config.div_val, name="crit"
)
def get_output_embeddings(self):
""" Double-check if you are using adaptive softmax.
"""
if self.sample_softmax > 0:
raise NotImplementedError
# return self.out_layer
else:
if len(self.crit.out_layers) > 0:
return self.crit.out_layers[-1]
return None
def reset_length(self, tgt_len, ext_len, mem_len):
self.transformer.reset_length(tgt_len, ext_len, mem_len)
@@ -822,6 +834,14 @@ class TFTransfoXLLMHeadModel(TFTransfoXLPreTrainedModel):
outputs = transformer_outputs[1:]
if self.sample_softmax > 0 and training:
raise NotImplementedError
# assert self.config.tie_weight
# logit = sample_logits(self.transformer.word_emb, self.out_layer.bias, labels, pred_hid, self.sampler)
# softmax_output = -tf.nn.log_softmax(logit, -1)[:, :, 0]
# outputs = [softmax_output] + outputs
# if labels is not None:
# TODO: This is not implemented
# raise NotImplementedError
else:
# pred_hid = tf.reshape(pred_hid, (-1, shape_list(pred_hid)[-1]))
softmax_output = self.crit([pred_hid, labels], training=training)
src/transformers/modeling_tf_transfo_xl_utilities.py
+5 -3
View File
@@ -194,7 +194,9 @@ class TFLogUniformSampler(object):
log_indices = tf.math.log(tf.range(1.0, range_max + 2.0))
self.dist = (log_indices[1:] - log_indices[:-1]) / log_indices[-1]
self.log_q = tf.cast(tf.math.log(-tf.math.expm1(-tf.math.log1p(tf.cast(self.dist, tf.double)) * 2 * n_sample)), tf.float32)
self.log_q = tf.cast(
tf.math.log(-tf.math.expm1(-tf.math.log1p(tf.cast(self.dist, tf.double)) * 2 * n_sample)), tf.float32
)
self.n_sample = n_sample
@@ -211,6 +213,6 @@ class TFLogUniformSampler(object):
n_tries = 2 * n_sample
neg_samples = tf.unique(tf.squeeze(tf.random.categorical(tf.expand_dims(self.dist, 0), n_tries), 0))[0]
true_log_probs = self.log_q[labels]
samp_log_probs = self.log_q[neg_samples]
true_log_probs = tf.gather_nd(self.log_q, labels)
samp_log_probs = tf.gather(self.log_q, neg_samples)
return true_log_probs, samp_log_probs, neg_samples
tests/test_modeling_tf_common.py
+2 -2
View File
@@ -30,7 +30,7 @@ if is_tf_available():
import tensorflow as tf
import numpy as np
from transformers import tf_top_k_top_p_filtering
from transformers import tf_top_k_top_p_filtering, TFAdaptiveEmbedding
if _tf_gpu_memory_limit is not None:
gpus = tf.config.list_physical_devices("GPU")
@@ -348,7 +348,7 @@ class TFModelTesterMixin:
for model_class in self.all_model_classes:
model = model_class(config)
assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer)
assert isinstance(model.get_input_embeddings(), (tf.keras.layers.Layer, TFAdaptiveEmbedding))
x = model.get_output_embeddings()
assert x is None or isinstance(x, tf.keras.layers.Layer)
tests/test_modeling_tf_transfo_xl.py
+4 -7
View File
@@ -26,7 +26,7 @@ from .utils import CACHE_DIR, require_tf, slow
if is_tf_available():
import tensorflow as tf
from transformers.modeling_tf_transfo_xl import (
from transformers import (
TFTransfoXLModel,
TFTransfoXLLMHeadModel,
TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP,
@@ -364,7 +364,7 @@ class TFTransfoXLModelLanguageGenerationTest(unittest.TestCase):
0,
]
],
dtype=tf.int31,
dtype=tf.int32,
)
# In 1991 , the remains of Russian Tsar Nicholas II and his family
# ( except for Alexei and Maria ) are discovered .
@@ -570,8 +570,5 @@ class TFTransfoXLModelLanguageGenerationTest(unittest.TestCase):
# Nicholas II and his family were discovered. The voice of <unk> young son,
# Tsarevich Alexei Nikolaevich, narrates the remainder of the story.<eos>
# TODO: add this test when trasnfo-xl-lmhead is implemented
with self.assertRaises(NotImplementedError):
model.generate(input_ids, max_length=200, do_sample=False)
print(expected_output_ids)
# self.assertListEqual(output_ids[0].numpy().tolist(), expected_output_ids) TODO: (PVP) to add when transfo-xl is implemented
output_ids = model.generate(input_ids, max_length=200, do_sample=False)
self.assertListEqual(output_ids[0].numpy().tolist(), expected_output_ids)
tests/test_modeling_transfo_xl.py
+4 -8
View File
@@ -28,7 +28,6 @@ if is_torch_available():
import torch
from transformers import TransfoXLConfig, TransfoXLModel, TransfoXLLMHeadModel
from transformers.modeling_transfo_xl import TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP
from transformers.modeling_transfo_xl_utilities import LogUniformSampler
@require_torch
@@ -130,10 +129,10 @@ class TransfoXLModelTest(ModelTesterMixin, unittest.TestCase):
def check_transfo_xl_model_output(self, result):
self.parent.assertListEqual(
list(result["hidden_states_1"].size()), [self.batch_size, self.seq_length, self.hidden_size]
list(result["hidden_states_1"].size()), [self.batch_size, self.seq_length, self.hidden_size],
)
self.parent.assertListEqual(
list(result["hidden_states_2"].size()), [self.batch_size, self.seq_length, self.hidden_size]
list(result["hidden_states_2"].size()), [self.batch_size, self.seq_length, self.hidden_size],
)
self.parent.assertListEqual(
list(list(mem.size()) for mem in result["mems_1"]),
@@ -167,7 +166,7 @@ class TransfoXLModelTest(ModelTesterMixin, unittest.TestCase):
def check_transfo_xl_lm_head_output(self, result):
self.parent.assertListEqual(list(result["loss_1"].size()), [self.batch_size, self.seq_length])
self.parent.assertListEqual(
list(result["lm_logits_1"].size()), [self.batch_size, self.seq_length, self.vocab_size]
list(result["lm_logits_1"].size()), [self.batch_size, self.seq_length, self.vocab_size],
)
self.parent.assertListEqual(
list(list(mem.size()) for mem in result["mems_1"]),
@@ -176,7 +175,7 @@ class TransfoXLModelTest(ModelTesterMixin, unittest.TestCase):
self.parent.assertListEqual(list(result["loss_2"].size()), [self.batch_size, self.seq_length])
self.parent.assertListEqual(
list(result["lm_logits_2"].size()), [self.batch_size, self.seq_length, self.vocab_size]
list(result["lm_logits_2"].size()), [self.batch_size, self.seq_length, self.vocab_size],
)
self.parent.assertListEqual(
list(list(mem.size()) for mem in result["mems_2"]),
@@ -208,9 +207,6 @@ class TransfoXLModelTest(ModelTesterMixin, unittest.TestCase):
output_result = self.model_tester.create_transfo_xl_lm_head(*config_and_inputs)
self.model_tester.check_transfo_xl_lm_head_output(output_result)
def test_log_uniform_sampler(self):
@slow
def test_model_from_pretrained(self):
for model_name in list(TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: