45 KiB
45 KiB
In [1]:
from extract_features import *In [2]:
data_dir="/Users/thomaswolf/Documents/Thomas/Code/HF/BERT/data/glue_data/MRPC/"
vocab_file="/Users/thomaswolf/Documents/Thomas/Code/HF/BERT/google_models/uncased_L-12_H-768_A-12/vocab.txt"
bert_config_file="/Users/thomaswolf/Documents/Thomas/Code/HF/BERT/google_models/uncased_L-12_H-768_A-12/bert_config.json"
init_checkpoint="/Users/thomaswolf/Documents/Thomas/Code/HF/BERT/google_models/uncased_L-12_H-768_A-12/bert_model.ckpt"
max_seq_length=128
input_file="/Users/thomaswolf/Documents/Thomas/Code/HF/BERT/pytorch-pretrained-BERT/input.txt"
layer_indexes = [-1]
bert_config = modeling.BertConfig.from_json_file(bert_config_file)
tokenizer = tokenization.FullTokenizer(
vocab_file=vocab_file, do_lower_case=True)
examples = read_examples(input_file)
features = convert_examples_to_features(
examples=examples, seq_length=max_seq_length, tokenizer=tokenizer)
unique_id_to_feature = {}
for feature in features:
unique_id_to_feature[feature.unique_id] = featureINFO:tensorflow:*** Example *** INFO:tensorflow:unique_id: 0 INFO:tensorflow:tokens: [CLS] who was jim henson ? [SEP] jim henson was a puppet ##eer [SEP] INFO:tensorflow:input_ids: 101 2040 2001 3958 27227 1029 102 3958 27227 2001 1037 13997 11510 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 INFO:tensorflow:input_type_ids: 0 0 0 0 0 0 0 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
In [3]:
is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
run_config = tf.contrib.tpu.RunConfig(
master=None,
tpu_config=tf.contrib.tpu.TPUConfig(
num_shards=1,
per_host_input_for_training=is_per_host))
model_fn = model_fn_builder(
bert_config=bert_config,
init_checkpoint=init_checkpoint,
layer_indexes=layer_indexes,
use_tpu=False,
use_one_hot_embeddings=False)
# If TPU is not available, this will fall back to normal Estimator on CPU
# or GPU.
estimator = tf.contrib.tpu.TPUEstimator(
use_tpu=False,
model_fn=model_fn,
config=run_config,
predict_batch_size=1)
input_fn = input_fn_builder(
features=features, seq_length=max_seq_length)WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x12b266ae8>) includes params argument, but params are not passed to Estimator.
WARNING:tensorflow:Using temporary folder as model directory: /var/folders/yx/cw8n_njx3js5jksyw_qlp8p00000gn/T/tmphrjfnoqh
INFO:tensorflow:Using config: {'_model_dir': '/var/folders/yx/cw8n_njx3js5jksyw_qlp8p00000gn/T/tmphrjfnoqh', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true
graph_options {
rewrite_options {
meta_optimizer_iterations: ONE
}
}
, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x12e2c1160>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=2, num_shards=1, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': None}
WARNING:tensorflow:Setting TPUConfig.num_shards==1 is an unsupported behavior. Please fix as soon as possible (leaving num_shards as None.
INFO:tensorflow:_TPUContext: eval_on_tpu True
WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.
In [4]:
all_out = []
for result in estimator.predict(input_fn, yield_single_examples=True):
unique_id = int(result["unique_id"])
feature = unique_id_to_feature[unique_id]
output_json = collections.OrderedDict()
output_json["linex_index"] = unique_id
all_out_features = []
for (i, token) in enumerate(feature.tokens):
all_layers = []
for (j, layer_index) in enumerate(layer_indexes):
layer_output = result["layer_output_%d" % j]
layers = collections.OrderedDict()
layers["index"] = layer_index
layers["values"] = [
round(float(x), 6) for x in layer_output[i:(i + 1)].flat
]
all_layers.append(layers)
out_features = collections.OrderedDict()
out_features["token"] = token
out_features["layers"] = all_layers
all_out_features.append(out_features)
output_json["features"] = all_out_features
all_out.append(output_json)INFO:tensorflow:Could not find trained model in model_dir: /var/folders/yx/cw8n_njx3js5jksyw_qlp8p00000gn/T/tmphrjfnoqh, running initialization to predict. INFO:tensorflow:Calling model_fn. INFO:tensorflow:Running infer on CPU INFO:tensorflow:Done calling model_fn. INFO:tensorflow:Graph was finalized. INFO:tensorflow:Running local_init_op. INFO:tensorflow:Done running local_init_op. INFO:tensorflow:prediction_loop marked as finished INFO:tensorflow:prediction_loop marked as finished
In [5]:
print(len(all_out))
print(len(all_out[0]))
print(all_out[0].keys())
print(len(all_out[0]['features']))1 2 odict_keys(['linex_index', 'features']) 14
In [6]:
tensorflow_output = all_out[0]['features'][0]['layers'][0]['values']
tensorflow_output[:10]Out [6]:
[-0.628111, 0.193215, -0.75185, -0.040464, -0.875331, 0.15654, 1.385444, 1.066997, -0.349549, 0.270686]
In [7]:
from extract_features_pytorch import *In [8]:
init_checkpoint_pt="/Users/thomaswolf/Documents/Thomas/Code/HF/BERT/google_models/uncased_L-12_H-768_A-12/pytorch_model.bin"
device = torch.device("cpu")
model = BertModel(bert_config)
model.load_state_dict(torch.load(init_checkpoint_pt, map_location='cpu'))
model.to(device)Out [8]:
BertModel(
(embeddings): BERTEmbeddings(
(word_embeddings): Embedding(30522, 768)
(position_embeddings): Embedding(512, 768)
(token_type_embeddings): Embedding(2, 768)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
(encoder): BERTEncoder(
(layer): ModuleList(
(0): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(1): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(2): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(3): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(4): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(5): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(6): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(7): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(8): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(9): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(10): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(11): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
)
)
(pooler): BERTPooler(
(dense): Linear(in_features=768, out_features=768, bias=True)
(activation): Tanh()
)
)In [9]:
all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask, all_example_index)
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=1)
model.eval()Out [9]:
BertModel(
(embeddings): BERTEmbeddings(
(word_embeddings): Embedding(30522, 768)
(position_embeddings): Embedding(512, 768)
(token_type_embeddings): Embedding(2, 768)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
(encoder): BERTEncoder(
(layer): ModuleList(
(0): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(1): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(2): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(3): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(4): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(5): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(6): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(7): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(8): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(9): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(10): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(11): BERTLayer(
(attention): BERTAttention(
(self): BERTSelfAttention(
(query): Linear(in_features=768, out_features=768, bias=True)
(key): Linear(in_features=768, out_features=768, bias=True)
(value): Linear(in_features=768, out_features=768, bias=True)
(dropout): Dropout(p=0.1)
)
(output): BERTSelfOutput(
(dense): Linear(in_features=768, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
(intermediate): BERTIntermediate(
(dense): Linear(in_features=768, out_features=3072, bias=True)
)
(output): BERTOutput(
(dense): Linear(in_features=3072, out_features=768, bias=True)
(LayerNorm): BERTLayerNorm()
(dropout): Dropout(p=0.1)
)
)
)
)
(pooler): BERTPooler(
(dense): Linear(in_features=768, out_features=768, bias=True)
(activation): Tanh()
)
)In [10]:
pytorch_all_out = []
for input_ids, input_mask, example_indices in eval_dataloader:
input_ids = input_ids.to(device)
input_mask = input_mask.float().to(device)
all_encoder_layers, _ = model(input_ids, token_type_ids=None, attention_mask=input_mask)
for enc_layers, example_index in zip(all_encoder_layers, example_indices):
feature = features[example_index.item()]
unique_id = int(feature.unique_id)
# feature = unique_id_to_feature[unique_id]
output_json = collections.OrderedDict()
output_json["linex_index"] = unique_id
all_out_features = []
for (i, token) in enumerate(feature.tokens):
all_layers = []
for (j, layer_index) in enumerate(layer_indexes):
layer_output = enc_layers[int(layer_index)].detach().cpu().numpy()
layers = collections.OrderedDict()
layers["index"] = layer_index
layers["values"] = [
round(float(x), 6) for x in layer_output[i:(i + 1)].flat
]
all_layers.append(layers)
out_features = collections.OrderedDict()
out_features["token"] = token
out_features["layers"] = all_layers
all_out_features.append(out_features)
output_json["features"] = all_out_features
pytorch_all_out.append(output_json)In [11]:
print(len(pytorch_all_out))
print(len(pytorch_all_out[0]))
print(pytorch_all_out[0].keys())
print(len(pytorch_all_out[0]['features']))1 2 odict_keys(['linex_index', 'features']) 14
In [13]:
pytorch_output = pytorch_all_out[0]['features'][0]['layers'][0]['values']
pytorch_output[:10]Out [13]:
[-0.016153, -0.697252, -0.298296, -0.167194, -0.219306, 0.061712, -0.006953, 0.366519, -0.031027, -0.33547]
In [14]:
tensorflow_output[:10]Out [14]:
[-0.628111, 0.193215, -0.75185, -0.040464, -0.875331, 0.15654, 1.385444, 1.066997, -0.349549, 0.270686]
In [ ]: