90 KiB
90 KiB
In [1]:
import os
import sys
model_cls = 'BertModel'
model_typ = 'bert-base-uncased'
token_cls = 'BertTokenizer'
max_seq = 12
CLS = "[CLS]"
SEP = "[SEP]"
MASK = "[MASK]"
CLS_IDX = 0
layer_idxs = tuple(range(12))
input_text = "jim henson was a puppeteer"
pt_model_dir = "/home/ubuntu/.pytorch-pretrained-BERT-cache/{}".format(model_typ)
tf_bert_dir = "/home/ubuntu/bert"
pt_vocab_file = os.path.join(pt_model_dir, "vocab.txt")
pt_init_ckpt = os.path.join(pt_model_dir, model_typ.replace("-", "_") + ".bin")
tf_model_dir = os.path.join(pt_model_dir, 'tf')
tf_vocab_file = os.path.join(tf_model_dir, "vocab.txt")
tf_init_ckpt = os.path.join(tf_model_dir, model_typ.replace("-", "_") + ".ckpt")
tf_config_file = os.path.join(tf_model_dir, "bert_config.json")
if not os.path.isdir(tf_model_dir):
os.makedirs(tf_model_dir, exist_ok=True)In [2]:
def tokenize(text, tokenizer):
text = text.strip().lower()
tok_ids = tokenizer.tokenize(text)
if len(tok_ids) > max_seq - 2:
tok_ids = tok_ids[:max_seq - 2]
tok_ids.insert(CLS_IDX, CLS)
tok_ids.append(SEP)
input_ids = tokenizer.convert_tokens_to_ids(tok_ids)
mask_ids = [1] * len(input_ids)
seg_ids = [0] * len(input_ids)
padding = [0] * (max_seq - len(input_ids))
input_ids += padding
mask_ids += padding
seg_ids += padding
return input_ids, mask_ids, seg_idsIn [3]:
import numpy as np
import torch
from pytorch_pretrained_bert import (BertConfig,
BertModel,
BertTokenizer,
BertForSequenceClassification)
# Save Vocab
pt_tokenizer = BertTokenizer.from_pretrained(
pretrained_model_name_or_path=model_typ,
cache_dir=pt_model_dir)
pt_tokenizer.save_vocabulary(pt_model_dir)
pt_tokenizer.save_vocabulary(tf_model_dir)
# Save Model
pt_model = BertModel.from_pretrained(
pretrained_model_name_or_path=model_typ,
cache_dir=pt_model_dir).to('cpu')
pt_model.eval()
pt_model.config.hidden_dropout_prob = 0.0
pt_model.config.attention_probs_dropout_prob = 0.0
pt_model.config.to_json_file(tf_config_file)
torch.save(pt_model.state_dict(), pt_init_ckpt)
# Inputs
input_ids_pt, mask_ids_pt, seg_ids_pt = tokenize(input_text, pt_tokenizer)
# PT Embedding
tok_tensor = torch.tensor(input_ids_pt).to('cpu').unsqueeze(0)
seg_tensor = torch.tensor(seg_ids_pt).to('cpu').unsqueeze(0)
msk_tensor = torch.tensor(mask_ids_pt).to('cpu').unsqueeze(0)
attn_blks, nsp_logits = pt_model(tok_tensor, seg_tensor, msk_tensor)
pt_embedding = nsp_logits.detach().numpy()
print("Pytorch embedding shape: {}".format(pt_embedding.shape))100%|██████████| 231508/231508 [00:00<00:00, 41092464.26B/s] 100%|██████████| 407873900/407873900 [00:07<00:00, 58092479.52B/s]
Pytorch embedding shape: (1, 768)
In [4]:
from pytorch_pretrained_bert.convert_pytorch_checkpoint_to_tf import main
main([
'--model_name', model_typ,
'--pytorch_model_path', pt_init_ckpt,
'--tf_cache_dir', tf_model_dir,
'--cache_dir', pt_model_dir
])WARNING:tensorflow:From /home/ubuntu/anaconda3/envs/nlp/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version. Instructions for updating: Colocations handled automatically by placer. bert/embeddings/word_embeddings initialized bert/embeddings/position_embeddings initialized bert/embeddings/token_type_embeddings initialized bert/embeddings/LayerNorm/gamma initialized bert/embeddings/LayerNorm/beta initialized bert/encoder/layer_0/attention/self/query/kernel initialized bert/encoder/layer_0/attention/self/query/bias initialized bert/encoder/layer_0/attention/self/key/kernel initialized bert/encoder/layer_0/attention/self/key/bias initialized bert/encoder/layer_0/attention/self/value/kernel initialized bert/encoder/layer_0/attention/self/value/bias initialized bert/encoder/layer_0/attention/output/dense/kernel initialized bert/encoder/layer_0/attention/output/dense/bias initialized bert/encoder/layer_0/attention/output/LayerNorm/gamma initialized bert/encoder/layer_0/attention/output/LayerNorm/beta initialized bert/encoder/layer_0/intermediate/dense/kernel initialized bert/encoder/layer_0/intermediate/dense/bias initialized bert/encoder/layer_0/output/dense/kernel initialized bert/encoder/layer_0/output/dense/bias initialized bert/encoder/layer_0/output/LayerNorm/gamma initialized bert/encoder/layer_0/output/LayerNorm/beta initialized bert/encoder/layer_1/attention/self/query/kernel initialized bert/encoder/layer_1/attention/self/query/bias initialized bert/encoder/layer_1/attention/self/key/kernel initialized bert/encoder/layer_1/attention/self/key/bias initialized bert/encoder/layer_1/attention/self/value/kernel initialized bert/encoder/layer_1/attention/self/value/bias initialized bert/encoder/layer_1/attention/output/dense/kernel initialized bert/encoder/layer_1/attention/output/dense/bias initialized bert/encoder/layer_1/attention/output/LayerNorm/gamma initialized bert/encoder/layer_1/attention/output/LayerNorm/beta initialized bert/encoder/layer_1/intermediate/dense/kernel initialized bert/encoder/layer_1/intermediate/dense/bias initialized bert/encoder/layer_1/output/dense/kernel initialized bert/encoder/layer_1/output/dense/bias initialized bert/encoder/layer_1/output/LayerNorm/gamma initialized bert/encoder/layer_1/output/LayerNorm/beta initialized bert/encoder/layer_2/attention/self/query/kernel initialized bert/encoder/layer_2/attention/self/query/bias initialized bert/encoder/layer_2/attention/self/key/kernel initialized bert/encoder/layer_2/attention/self/key/bias initialized bert/encoder/layer_2/attention/self/value/kernel initialized bert/encoder/layer_2/attention/self/value/bias initialized bert/encoder/layer_2/attention/output/dense/kernel initialized bert/encoder/layer_2/attention/output/dense/bias initialized bert/encoder/layer_2/attention/output/LayerNorm/gamma initialized bert/encoder/layer_2/attention/output/LayerNorm/beta initialized bert/encoder/layer_2/intermediate/dense/kernel initialized bert/encoder/layer_2/intermediate/dense/bias initialized bert/encoder/layer_2/output/dense/kernel initialized bert/encoder/layer_2/output/dense/bias initialized bert/encoder/layer_2/output/LayerNorm/gamma initialized bert/encoder/layer_2/output/LayerNorm/beta initialized bert/encoder/layer_3/attention/self/query/kernel initialized bert/encoder/layer_3/attention/self/query/bias initialized bert/encoder/layer_3/attention/self/key/kernel initialized bert/encoder/layer_3/attention/self/key/bias initialized bert/encoder/layer_3/attention/self/value/kernel initialized bert/encoder/layer_3/attention/self/value/bias initialized bert/encoder/layer_3/attention/output/dense/kernel initialized bert/encoder/layer_3/attention/output/dense/bias initialized bert/encoder/layer_3/attention/output/LayerNorm/gamma initialized bert/encoder/layer_3/attention/output/LayerNorm/beta initialized bert/encoder/layer_3/intermediate/dense/kernel initialized bert/encoder/layer_3/intermediate/dense/bias initialized bert/encoder/layer_3/output/dense/kernel initialized bert/encoder/layer_3/output/dense/bias initialized bert/encoder/layer_3/output/LayerNorm/gamma initialized bert/encoder/layer_3/output/LayerNorm/beta initialized bert/encoder/layer_4/attention/self/query/kernel initialized bert/encoder/layer_4/attention/self/query/bias initialized bert/encoder/layer_4/attention/self/key/kernel initialized bert/encoder/layer_4/attention/self/key/bias initialized bert/encoder/layer_4/attention/self/value/kernel initialized bert/encoder/layer_4/attention/self/value/bias initialized bert/encoder/layer_4/attention/output/dense/kernel initialized bert/encoder/layer_4/attention/output/dense/bias initialized bert/encoder/layer_4/attention/output/LayerNorm/gamma initialized bert/encoder/layer_4/attention/output/LayerNorm/beta initialized bert/encoder/layer_4/intermediate/dense/kernel initialized bert/encoder/layer_4/intermediate/dense/bias initialized bert/encoder/layer_4/output/dense/kernel initialized bert/encoder/layer_4/output/dense/bias initialized bert/encoder/layer_4/output/LayerNorm/gamma initialized bert/encoder/layer_4/output/LayerNorm/beta initialized bert/encoder/layer_5/attention/self/query/kernel initialized bert/encoder/layer_5/attention/self/query/bias initialized bert/encoder/layer_5/attention/self/key/kernel initialized bert/encoder/layer_5/attention/self/key/bias initialized bert/encoder/layer_5/attention/self/value/kernel initialized bert/encoder/layer_5/attention/self/value/bias initialized bert/encoder/layer_5/attention/output/dense/kernel initialized bert/encoder/layer_5/attention/output/dense/bias initialized bert/encoder/layer_5/attention/output/LayerNorm/gamma initialized bert/encoder/layer_5/attention/output/LayerNorm/beta initialized bert/encoder/layer_5/intermediate/dense/kernel initialized bert/encoder/layer_5/intermediate/dense/bias initialized bert/encoder/layer_5/output/dense/kernel initialized bert/encoder/layer_5/output/dense/bias initialized bert/encoder/layer_5/output/LayerNorm/gamma initialized bert/encoder/layer_5/output/LayerNorm/beta initialized bert/encoder/layer_6/attention/self/query/kernel initialized bert/encoder/layer_6/attention/self/query/bias initialized bert/encoder/layer_6/attention/self/key/kernel initialized bert/encoder/layer_6/attention/self/key/bias initialized bert/encoder/layer_6/attention/self/value/kernel initialized bert/encoder/layer_6/attention/self/value/bias initialized bert/encoder/layer_6/attention/output/dense/kernel initialized bert/encoder/layer_6/attention/output/dense/bias initialized bert/encoder/layer_6/attention/output/LayerNorm/gamma initialized bert/encoder/layer_6/attention/output/LayerNorm/beta initialized bert/encoder/layer_6/intermediate/dense/kernel initialized bert/encoder/layer_6/intermediate/dense/bias initialized bert/encoder/layer_6/output/dense/kernel initialized bert/encoder/layer_6/output/dense/bias initialized bert/encoder/layer_6/output/LayerNorm/gamma initialized bert/encoder/layer_6/output/LayerNorm/beta initialized bert/encoder/layer_7/attention/self/query/kernel initialized bert/encoder/layer_7/attention/self/query/bias initialized bert/encoder/layer_7/attention/self/key/kernel initialized bert/encoder/layer_7/attention/self/key/bias initialized bert/encoder/layer_7/attention/self/value/kernel initialized bert/encoder/layer_7/attention/self/value/bias initialized bert/encoder/layer_7/attention/output/dense/kernel initialized bert/encoder/layer_7/attention/output/dense/bias initialized bert/encoder/layer_7/attention/output/LayerNorm/gamma initialized bert/encoder/layer_7/attention/output/LayerNorm/beta initialized bert/encoder/layer_7/intermediate/dense/kernel initialized bert/encoder/layer_7/intermediate/dense/bias initialized bert/encoder/layer_7/output/dense/kernel initialized bert/encoder/layer_7/output/dense/bias initialized bert/encoder/layer_7/output/LayerNorm/gamma initialized bert/encoder/layer_7/output/LayerNorm/beta initialized bert/encoder/layer_8/attention/self/query/kernel initialized bert/encoder/layer_8/attention/self/query/bias initialized bert/encoder/layer_8/attention/self/key/kernel initialized bert/encoder/layer_8/attention/self/key/bias initialized bert/encoder/layer_8/attention/self/value/kernel initialized bert/encoder/layer_8/attention/self/value/bias initialized bert/encoder/layer_8/attention/output/dense/kernel initialized bert/encoder/layer_8/attention/output/dense/bias initialized bert/encoder/layer_8/attention/output/LayerNorm/gamma initialized bert/encoder/layer_8/attention/output/LayerNorm/beta initialized bert/encoder/layer_8/intermediate/dense/kernel initialized bert/encoder/layer_8/intermediate/dense/bias initialized bert/encoder/layer_8/output/dense/kernel initialized bert/encoder/layer_8/output/dense/bias initialized bert/encoder/layer_8/output/LayerNorm/gamma initialized bert/encoder/layer_8/output/LayerNorm/beta initialized bert/encoder/layer_9/attention/self/query/kernel initialized bert/encoder/layer_9/attention/self/query/bias initialized bert/encoder/layer_9/attention/self/key/kernel initialized bert/encoder/layer_9/attention/self/key/bias initialized bert/encoder/layer_9/attention/self/value/kernel initialized bert/encoder/layer_9/attention/self/value/bias initialized bert/encoder/layer_9/attention/output/dense/kernel initialized bert/encoder/layer_9/attention/output/dense/bias initialized bert/encoder/layer_9/attention/output/LayerNorm/gamma initialized bert/encoder/layer_9/attention/output/LayerNorm/beta initialized bert/encoder/layer_9/intermediate/dense/kernel initialized bert/encoder/layer_9/intermediate/dense/bias initialized bert/encoder/layer_9/output/dense/kernel initialized bert/encoder/layer_9/output/dense/bias initialized bert/encoder/layer_9/output/LayerNorm/gamma initialized bert/encoder/layer_9/output/LayerNorm/beta initialized bert/encoder/layer_10/attention/self/query/kernel initialized bert/encoder/layer_10/attention/self/query/bias initialized bert/encoder/layer_10/attention/self/key/kernel initialized bert/encoder/layer_10/attention/self/key/bias initialized bert/encoder/layer_10/attention/self/value/kernel initialized bert/encoder/layer_10/attention/self/value/bias initialized bert/encoder/layer_10/attention/output/dense/kernel initialized bert/encoder/layer_10/attention/output/dense/bias initialized bert/encoder/layer_10/attention/output/LayerNorm/gamma initialized bert/encoder/layer_10/attention/output/LayerNorm/beta initialized bert/encoder/layer_10/intermediate/dense/kernel initialized bert/encoder/layer_10/intermediate/dense/bias initialized bert/encoder/layer_10/output/dense/kernel initialized bert/encoder/layer_10/output/dense/bias initialized bert/encoder/layer_10/output/LayerNorm/gamma initialized bert/encoder/layer_10/output/LayerNorm/beta initialized bert/encoder/layer_11/attention/self/query/kernel initialized bert/encoder/layer_11/attention/self/query/bias initialized bert/encoder/layer_11/attention/self/key/kernel initialized bert/encoder/layer_11/attention/self/key/bias initialized bert/encoder/layer_11/attention/self/value/kernel initialized bert/encoder/layer_11/attention/self/value/bias initialized bert/encoder/layer_11/attention/output/dense/kernel initialized bert/encoder/layer_11/attention/output/dense/bias initialized bert/encoder/layer_11/attention/output/LayerNorm/gamma initialized bert/encoder/layer_11/attention/output/LayerNorm/beta initialized bert/encoder/layer_11/intermediate/dense/kernel initialized bert/encoder/layer_11/intermediate/dense/bias initialized bert/encoder/layer_11/output/dense/kernel initialized bert/encoder/layer_11/output/dense/bias initialized bert/encoder/layer_11/output/LayerNorm/gamma initialized bert/encoder/layer_11/output/LayerNorm/beta initialized bert/pooler/dense/kernel initialized bert/pooler/dense/bias initialized
In [5]:
import tensorflow as tf
sys.path.insert(0, tf_bert_dir)
import modeling
import tokenization
tf.reset_default_graph()
# Process text
tf_tokenizer = tokenization.FullTokenizer(vocab_file=tf_vocab_file)
# Graph inputs
input_ids_tf, mask_ids_tf, seg_ids_tf = tokenize(input_text, tf_tokenizer)
config = modeling.BertConfig.from_json_file(
os.path.join(tf_model_dir, 'bert_config.json'))
input_tensor = tf.placeholder(
dtype=tf.int32,
shape=[1, None],
name='input_ids')
mask_tensor = tf.placeholder(
dtype=tf.int32,
shape=[1, None],
name='mask_ids')
seg_tensor = tf.placeholder(
dtype=tf.int32,
shape=[1, None],
name='seg_ids')
tf_model = modeling.BertModel(
config=config,
is_training=False,
input_ids=input_tensor,
input_mask=mask_tensor,
token_type_ids=seg_tensor,
use_one_hot_embeddings=False)
output_layer = tf_model.get_pooled_output()
# Load tf model
session = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
vars_to_load = [v for v in tf.global_variables()]
session.run(tf.variables_initializer(var_list=vars_to_load))
saver = tf.train.Saver(vars_to_load)
saver.restore(session, save_path=tf_init_ckpt)
# TF Embedding
fetches = output_layer
feed_dict = {
input_tensor: [input_ids_tf],
mask_tensor: [mask_ids_tf],
seg_tensor: [seg_ids_tf]
}
tf_embedding = session.run(fetches=fetches, feed_dict=feed_dict)
print("Tensorflow embedding shape: {}".format(tf_embedding.shape))WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0. For more information, please see: * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md * https://github.com/tensorflow/addons If you depend on functionality not listed there, please file an issue. WARNING:tensorflow:From /home/ubuntu/bert/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.dense instead. WARNING:tensorflow:From /home/ubuntu/anaconda3/envs/nlp/lib/python3.6/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version. Instructions for updating: Use standard file APIs to check for files with this prefix. INFO:tensorflow:Restoring parameters from /home/ubuntu/.pytorch-pretrained-BERT-cache/bert-base-uncased/tf/bert_base_uncased.ckpt Tensorflow embedding shape: (1, 768)
In [6]:
print("TOKEN_IDS_PT: {}".format(input_ids_pt))
print("TOKEN_IDS_TF: {}".format(input_ids_tf))
print("SEG_IDS_PT: {}".format(seg_ids_pt))
print("SEG_IDS_TF: {}".format(seg_ids_tf))
print("MASK_IDS_PT: {}".format(mask_ids_pt))
print("MASK_IDS_TF: {}".format(mask_ids_tf))TOKEN_IDS_PT: [101, 3958, 27227, 2001, 1037, 13997, 11510, 102, 0, 0, 0, 0] TOKEN_IDS_TF: [101, 3958, 27227, 2001, 1037, 13997, 11510, 102, 0, 0, 0, 0] SEG_IDS_PT: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] SEG_IDS_TF: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] MASK_IDS_PT: [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0] MASK_IDS_TF: [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0]
In [7]:
tensors_to_transopse = (
"dense.weight",
"attention.self.query",
"attention.self.key",
"attention.self.value"
)
var_map = (
('layer.', 'layer_'),
('word_embeddings.weight', 'word_embeddings'),
('position_embeddings.weight', 'position_embeddings'),
('token_type_embeddings.weight', 'token_type_embeddings'),
('.', '/'),
('LayerNorm/weight', 'LayerNorm/gamma'),
('LayerNorm/bias', 'LayerNorm/beta'),
('weight', 'kernel')
)
def to_tf_var_name(name:str):
for patt, repl in iter(var_map):
name = name.replace(patt, repl)
return 'bert/{}'.format(name)
tf_vars = {v.name: session.run(fetches=v) for v in tf.global_variables()}
pt_vars = {}
for v, T in pt_model.state_dict().items():
T = T.detach().numpy()
if any([x in v for x in tensors_to_transopse]):
T = T.T
pt_vars.update({to_tf_var_name(v): T})
for var_name in tf_vars:
pt = pt_vars[var_name.strip(":0")]
tf = tf_vars[var_name]
print(var_name.strip(":0"))
# Assert equivalence
print("|sum(pt_wts - tf_wts)| = {}".format(
np.abs(np.sum(pt - tf, keepdims=False))
))
assert not np.sum(pt - tf, keepdims=False)
if len(pt.shape) == 2:
print("PT: shape: {0} values: {1}".format(pt.shape, pt[0, :5]))
print("TF: shape: {0} values: {1}".format(tf.shape, tf[0, :5]))
else:
print("PT: shape: {0} values: {1}".format(pt.shape, pt[:5]))
print("TF: shape: {0} values: {1}".format(tf.shape, tf[:5]))
print()bert/embeddings/word_embeddings |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (30522, 768) values: [-0.01018257 -0.06154883 -0.02649689 -0.0420608 0.00116716] TF: shape: (30522, 768) values: [-0.01018257 -0.06154883 -0.02649689 -0.0420608 0.00116716] bert/embeddings/token_type_embeddings |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (2, 768) values: [0.00043164 0.01098826 0.00370439 0.00150542 0.00057812] TF: shape: (2, 768) values: [0.00043164 0.01098826 0.00370439 0.00150542 0.00057812] bert/embeddings/position_embeddings |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (512, 768) values: [ 0.01750538 -0.02563101 -0.03664156 -0.02528613 0.00797095] TF: shape: (512, 768) values: [ 0.01750538 -0.02563101 -0.03664156 -0.02528613 0.00797095] bert/embeddings/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.02591471 -0.0195513 0.02423946 0.08904593 -0.06281059] TF: shape: (768,) values: [-0.02591471 -0.0195513 0.02423946 0.08904593 -0.06281059] bert/embeddings/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.9260566 0.8851115 0.85807985 0.8616906 0.8937205 ] TF: shape: (768,) values: [0.9260566 0.8851115 0.85807985 0.8616906 0.8937205 ] bert/encoder/layer_0/attention/self/query/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.01640572 -0.03257025 0.01046295 -0.04442816 -0.02256124] TF: shape: (768, 768) values: [-0.01640572 -0.03257025 0.01046295 -0.04442816 -0.02256124] bert/encoder/layer_0/attention/self/query/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.58488506 -0.3312432 -0.43010172 0.37446147 -0.29811692] TF: shape: (768,) values: [ 0.58488506 -0.3312432 -0.43010172 0.37446147 -0.29811692] bert/encoder/layer_0/attention/self/key/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.00807745 0.02652155 -0.01866494 0.01797846 0.00450485] TF: shape: (768, 768) values: [ 0.00807745 0.02652155 -0.01866494 0.01797846 0.00450485] bert/encoder/layer_0/attention/self/key/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.00104306 0.00035106 -0.0024626 -0.00010567 -0.00119283] TF: shape: (768,) values: [ 0.00104306 0.00035106 -0.0024626 -0.00010567 -0.00119283] bert/encoder/layer_0/attention/self/value/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.01144261 -0.02663044 0.01911472 -0.02206182 -0.00287949] TF: shape: (768, 768) values: [ 0.01144261 -0.02663044 0.01911472 -0.02206182 -0.00287949] bert/encoder/layer_0/attention/self/value/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.01184616 -0.01596605 -0.00251847 0.01736802 0.00449983] TF: shape: (768,) values: [-0.01184616 -0.01596605 -0.00251847 0.01736802 0.00449983] bert/encoder/layer_0/attention/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.00581949 0.03170148 -0.06135742 -0.01706108 -0.00759045] TF: shape: (768, 768) values: [ 0.00581949 0.03170148 -0.06135742 -0.01706108 -0.00759045] bert/encoder/layer_0/attention/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.00511063 -0.0166625 0.02812938 -0.01166061 0.01942627] TF: shape: (768,) values: [ 0.00511063 -0.0166625 0.02812938 -0.01166061 0.01942627] bert/encoder/layer_0/attention/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.25779155 -0.03077853 -0.2772697 -0.38847703 0.36841765] TF: shape: (768,) values: [ 0.25779155 -0.03077853 -0.2772697 -0.38847703 0.36841765] bert/encoder/layer_0/attention/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.9803408 0.959969 0.96368986 0.9603653 0.9801324 ] TF: shape: (768,) values: [0.9803408 0.959969 0.96368986 0.9603653 0.9801324 ] bert/encoder/layer_0/intermediate/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 3072) values: [-0.01010427 -0.060398 -0.01468864 0.00311493 0.02862451] TF: shape: (768, 3072) values: [-0.01010427 -0.060398 -0.01468864 0.00311493 0.02862451] bert/encoder/layer_0/intermediate/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072,) values: [-0.11498757 -0.09629171 -0.12399033 -0.129036 -0.06369043] TF: shape: (3072,) values: [-0.11498757 -0.09629171 -0.12399033 -0.129036 -0.06369043] bert/encoder/layer_0/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072, 768) values: [-0.03710171 0.0648794 0.00758566 -0.05224452 -0.04348791] TF: shape: (3072, 768) values: [-0.03710171 0.0648794 0.00758566 -0.05224452 -0.04348791] bert/encoder/layer_0/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.04801027 0.19766568 0.02154854 0.02880666 0.0444298 ] TF: shape: (768,) values: [-0.04801027 0.19766568 0.02154854 0.02880666 0.0444298 ] bert/encoder/layer_0/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.10142924 -0.00499344 0.04274083 0.09324206 -0.10700516] TF: shape: (768,) values: [-0.10142924 -0.00499344 0.04274083 0.09324206 -0.10700516] bert/encoder/layer_0/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.7835125 0.8072406 0.7670588 0.73706394 0.76303864] TF: shape: (768,) values: [0.7835125 0.8072406 0.7670588 0.73706394 0.76303864] bert/encoder/layer_1/attention/self/query/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.03132744 -0.01340016 -0.07761582 0.0655639 -0.00337808] TF: shape: (768, 768) values: [ 0.03132744 -0.01340016 -0.07761582 0.0655639 -0.00337808] bert/encoder/layer_1/attention/self/query/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.27827993 0.17387655 -0.2497937 -0.8809636 0.41262135] TF: shape: (768,) values: [-0.27827993 0.17387655 -0.2497937 -0.8809636 0.41262135] bert/encoder/layer_1/attention/self/key/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.03353037 0.04007257 0.05320328 -0.02166729 -0.03581231] TF: shape: (768, 768) values: [-0.03353037 0.04007257 0.05320328 -0.02166729 -0.03581231] bert/encoder/layer_1/attention/self/key/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.00504407 0.00136887 -0.00394336 0.00646125 -0.00148919] TF: shape: (768,) values: [-0.00504407 0.00136887 -0.00394336 0.00646125 -0.00148919] bert/encoder/layer_1/attention/self/value/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.00464159 0.06674305 -0.00970626 -0.0276653 -0.01597566] TF: shape: (768, 768) values: [-0.00464159 0.06674305 -0.00970626 -0.0276653 -0.01597566] bert/encoder/layer_1/attention/self/value/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.00381288 0.02650839 -0.0059689 -0.00508269 -0.01293722] TF: shape: (768,) values: [ 0.00381288 0.02650839 -0.0059689 -0.00508269 -0.01293722] bert/encoder/layer_1/attention/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.01390745 -0.01100563 0.01303005 -0.01969771 0.0125082 ] TF: shape: (768, 768) values: [-0.01390745 -0.01100563 0.01303005 -0.01969771 0.0125082 ] bert/encoder/layer_1/attention/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.02946591 0.05715097 0.01293636 0.01920356 0.00805334] TF: shape: (768,) values: [0.02946591 0.05715097 0.01293636 0.01920356 0.00805334] bert/encoder/layer_1/attention/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.08583715 0.14199966 -0.0856637 -0.18797271 0.21056814] TF: shape: (768,) values: [ 0.08583715 0.14199966 -0.0856637 -0.18797271 0.21056814] bert/encoder/layer_1/attention/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.896962 0.87148863 0.8531161 0.8690647 0.9488987 ] TF: shape: (768,) values: [0.896962 0.87148863 0.8531161 0.8690647 0.9488987 ] bert/encoder/layer_1/intermediate/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 3072) values: [ 0.01841293 -0.02650284 -0.09708428 -0.01734244 -0.05529237] TF: shape: (768, 3072) values: [ 0.01841293 -0.02650284 -0.09708428 -0.01734244 -0.05529237] bert/encoder/layer_1/intermediate/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072,) values: [-0.15203774 -0.10449131 -0.08440229 -0.09323178 -0.08511415] TF: shape: (3072,) values: [-0.15203774 -0.10449131 -0.08440229 -0.09323178 -0.08511415] bert/encoder/layer_1/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072, 768) values: [-0.02372648 0.03326349 0.08291997 -0.01519038 0.01868557] TF: shape: (3072, 768) values: [-0.02372648 0.03326349 0.08291997 -0.01519038 0.01868557] bert/encoder/layer_1/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.02514724 0.09868994 -0.027811 0.03749462 0.01086514] TF: shape: (768,) values: [-0.02514724 0.09868994 -0.027811 0.03749462 0.01086514] bert/encoder/layer_1/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.07662535 -0.10506564 0.03191236 0.07633785 -0.11187791] TF: shape: (768,) values: [-0.07662535 -0.10506564 0.03191236 0.07633785 -0.11187791] bert/encoder/layer_1/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.9017883 0.8868776 0.8862677 0.85865664 0.87496454] TF: shape: (768,) values: [0.9017883 0.8868776 0.8862677 0.85865664 0.87496454] bert/encoder/layer_2/attention/self/query/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.08433672 0.09580533 0.07543895 -0.01126779 -0.01354045] TF: shape: (768, 768) values: [ 0.08433672 0.09580533 0.07543895 -0.01126779 -0.01354045] bert/encoder/layer_2/attention/self/query/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.0371241 0.03406003 0.27713948 -0.21613775 -0.05275448] TF: shape: (768,) values: [ 0.0371241 0.03406003 0.27713948 -0.21613775 -0.05275448] bert/encoder/layer_2/attention/self/key/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.04794507 0.02517631 -0.01319554 -0.02094732 0.09073472] TF: shape: (768, 768) values: [ 0.04794507 0.02517631 -0.01319554 -0.02094732 0.09073472] bert/encoder/layer_2/attention/self/key/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.00037404 -0.00125881 -0.00114734 -0.00157741 0.00037122] TF: shape: (768,) values: [-0.00037404 -0.00125881 -0.00114734 -0.00157741 0.00037122] bert/encoder/layer_2/attention/self/value/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.01119406 -0.01488636 -0.02960914 0.04746444 0.00428481] TF: shape: (768, 768) values: [-0.01119406 -0.01488636 -0.02960914 0.04746444 0.00428481] bert/encoder/layer_2/attention/self/value/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.02728729 0.04979054 0.08326469 0.04150949 0.600959 ] TF: shape: (768,) values: [-0.02728729 0.04979054 0.08326469 0.04150949 0.600959 ] bert/encoder/layer_2/attention/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.00517425 0.01197957 0.0393172 -0.0063884 -0.02673388] TF: shape: (768, 768) values: [ 0.00517425 0.01197957 0.0393172 -0.0063884 -0.02673388] bert/encoder/layer_2/attention/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.01754025 0.1226335 -0.05733554 0.06844623 0.00879776] TF: shape: (768,) values: [ 0.01754025 0.1226335 -0.05733554 0.06844623 0.00879776] bert/encoder/layer_2/attention/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.1490809 0.12386955 -0.19382021 -0.26515856 0.32723007] TF: shape: (768,) values: [ 0.1490809 0.12386955 -0.19382021 -0.26515856 0.32723007] bert/encoder/layer_2/attention/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.8983343 0.88877076 0.86283594 0.8584952 0.9587886 ] TF: shape: (768,) values: [0.8983343 0.88877076 0.86283594 0.8584952 0.9587886 ] bert/encoder/layer_2/intermediate/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 3072) values: [-0.01619919 0.00662888 0.01492284 -0.01280748 0.01318596] TF: shape: (768, 3072) values: [-0.01619919 0.00662888 0.01492284 -0.01280748 0.01318596] bert/encoder/layer_2/intermediate/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072,) values: [-0.08474881 -0.12850781 -0.11550345 -0.09513011 -0.02519853] TF: shape: (3072,) values: [-0.08474881 -0.12850781 -0.11550345 -0.09513011 -0.02519853] bert/encoder/layer_2/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072, 768) values: [-0.07225161 -0.0129784 0.00618811 -0.01593373 -0.02160194] TF: shape: (3072, 768) values: [-0.07225161 -0.0129784 0.00618811 -0.01593373 -0.02160194] bert/encoder/layer_2/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.06319264 0.06169628 -0.03041368 0.00924282 0.06277442] TF: shape: (768,) values: [-0.06319264 0.06169628 -0.03041368 0.00924282 0.06277442] bert/encoder/layer_2/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.1139038 -0.11665309 0.07883061 0.07796711 -0.14219187] TF: shape: (768,) values: [-0.1139038 -0.11665309 0.07883061 0.07796711 -0.14219187] bert/encoder/layer_2/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.8813261 0.85744697 0.8511922 0.85261875 0.8329574 ] TF: shape: (768,) values: [0.8813261 0.85744697 0.8511922 0.85261875 0.8329574 ] bert/encoder/layer_3/attention/self/query/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.05855456 -0.00111438 -0.00828963 0.04117409 -0.07591715] TF: shape: (768, 768) values: [ 0.05855456 -0.00111438 -0.00828963 0.04117409 -0.07591715] bert/encoder/layer_3/attention/self/query/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.09740101 -0.19290674 0.04332267 0.17937997 -0.08023558] TF: shape: (768,) values: [ 0.09740101 -0.19290674 0.04332267 0.17937997 -0.08023558] bert/encoder/layer_3/attention/self/key/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.02562077 0.02507281 -0.03361562 0.05613289 -0.05435724] TF: shape: (768, 768) values: [ 0.02562077 0.02507281 -0.03361562 0.05613289 -0.05435724] bert/encoder/layer_3/attention/self/key/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.00188639 -0.00379197 -0.01020415 0.00969649 -0.00094182] TF: shape: (768,) values: [ 0.00188639 -0.00379197 -0.01020415 0.00969649 -0.00094182] bert/encoder/layer_3/attention/self/value/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.00539032 0.00959642 0.01325458 0.00490616 0.0129908 ] TF: shape: (768, 768) values: [-0.00539032 0.00959642 0.01325458 0.00490616 0.0129908 ] bert/encoder/layer_3/attention/self/value/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.04573824 0.05405985 0.00681163 0.00655945 0.01141771] TF: shape: (768,) values: [0.04573824 0.05405985 0.00681163 0.00655945 0.01141771] bert/encoder/layer_3/attention/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.01850341 0.03148198 0.02705758 -0.0004669 0.01367511] TF: shape: (768, 768) values: [ 0.01850341 0.03148198 0.02705758 -0.0004669 0.01367511] bert/encoder/layer_3/attention/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.01981483 0.03566506 -0.05016088 0.02958186 0.04989756] TF: shape: (768,) values: [ 0.01981483 0.03566506 -0.05016088 0.02958186 0.04989756] bert/encoder/layer_3/attention/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.09815404 0.00063774 -0.01257733 -0.26485074 0.22568701] TF: shape: (768,) values: [ 0.09815404 0.00063774 -0.01257733 -0.26485074 0.22568701] bert/encoder/layer_3/attention/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.91457725 0.88453823 0.8340887 0.84203583 0.95247847] TF: shape: (768,) values: [0.91457725 0.88453823 0.8340887 0.84203583 0.95247847] bert/encoder/layer_3/intermediate/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 3072) values: [-0.02733567 0.03307878 -0.01331292 -0.00032527 0.03252084] TF: shape: (768, 3072) values: [-0.02733567 0.03307878 -0.01331292 -0.00032527 0.03252084] bert/encoder/layer_3/intermediate/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072,) values: [-0.11436842 -0.15038085 -0.07842971 0.01335877 -0.09492484] TF: shape: (3072,) values: [-0.11436842 -0.15038085 -0.07842971 0.01335877 -0.09492484] bert/encoder/layer_3/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072, 768) values: [-0.01751153 0.01631314 -0.02660011 0.03569947 -0.01394763] TF: shape: (3072, 768) values: [-0.01751153 0.01631314 -0.02660011 0.03569947 -0.01394763] bert/encoder/layer_3/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.03873252 0.08414765 -0.0399323 0.01997361 0.12924597] TF: shape: (768,) values: [-0.03873252 0.08414765 -0.0399323 0.01997361 0.12924597] bert/encoder/layer_3/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.08049371 -0.06923949 -0.03357155 0.05231095 -0.09717073] TF: shape: (768,) values: [-0.08049371 -0.06923949 -0.03357155 0.05231095 -0.09717073] bert/encoder/layer_3/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.827748 0.83012533 0.82399255 0.81772 0.80794513] TF: shape: (768,) values: [0.827748 0.83012533 0.82399255 0.81772 0.80794513] bert/encoder/layer_4/attention/self/query/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.08296382 0.02076941 0.06525186 -0.02659729 0.03491377] TF: shape: (768, 768) values: [ 0.08296382 0.02076941 0.06525186 -0.02659729 0.03491377] bert/encoder/layer_4/attention/self/query/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.07045844 -0.13412629 -0.0514146 0.00061329 0.1248519 ] TF: shape: (768,) values: [ 0.07045844 -0.13412629 -0.0514146 0.00061329 0.1248519 ] bert/encoder/layer_4/attention/self/key/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.06941643 0.08133814 -0.0453992 0.0668715 -0.06014847] TF: shape: (768, 768) values: [ 0.06941643 0.08133814 -0.0453992 0.0668715 -0.06014847] bert/encoder/layer_4/attention/self/key/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.00588725 -0.00235185 0.00281131 0.00173088 -0.00546653] TF: shape: (768,) values: [-0.00588725 -0.00235185 0.00281131 0.00173088 -0.00546653] bert/encoder/layer_4/attention/self/value/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.06889665 0.06645385 0.01232084 0.0132611 -0.01595679] TF: shape: (768, 768) values: [ 0.06889665 0.06645385 0.01232084 0.0132611 -0.01595679] bert/encoder/layer_4/attention/self/value/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.01126871 -0.02704018 0.0301532 0.02332082 -0.04233487] TF: shape: (768,) values: [-0.01126871 -0.02704018 0.0301532 0.02332082 -0.04233487] bert/encoder/layer_4/attention/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.02285513 -0.04172142 -0.0146292 0.04862929 -0.0442014 ] TF: shape: (768, 768) values: [ 0.02285513 -0.04172142 -0.0146292 0.04862929 -0.0442014 ] bert/encoder/layer_4/attention/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.03054528 0.00479777 -0.02729505 -0.0325212 -0.00525727] TF: shape: (768,) values: [ 0.03054528 0.00479777 -0.02729505 -0.0325212 -0.00525727] bert/encoder/layer_4/attention/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.00903359 0.0052285 -0.02841488 -0.22355485 0.28281343] TF: shape: (768,) values: [ 0.00903359 0.0052285 -0.02841488 -0.22355485 0.28281343] bert/encoder/layer_4/attention/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.8849676 0.86927813 0.8114595 0.80269504 0.94864094] TF: shape: (768,) values: [0.8849676 0.86927813 0.8114595 0.80269504 0.94864094] bert/encoder/layer_4/intermediate/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 3072) values: [-0.00639783 0.06198016 -0.03184223 0.00485356 -0.02453273] TF: shape: (768, 3072) values: [-0.00639783 0.06198016 -0.03184223 0.00485356 -0.02453273] bert/encoder/layer_4/intermediate/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072,) values: [-0.08770327 -0.11779705 -0.11764182 -0.00192611 -0.1335473 ] TF: shape: (3072,) values: [-0.08770327 -0.11779705 -0.11764182 -0.00192611 -0.1335473 ] bert/encoder/layer_4/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072, 768) values: [-0.05421264 0.0221118 -0.02674172 0.03672203 -0.02399626] TF: shape: (3072, 768) values: [-0.05421264 0.0221118 -0.02674172 0.03672203 -0.02399626] bert/encoder/layer_4/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.05068972 0.04838871 0.01156022 0.05381602 0.08857913] TF: shape: (768,) values: [-0.05068972 0.04838871 0.01156022 0.05381602 0.08857913] bert/encoder/layer_4/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.04338909 -0.0781464 -0.01518662 0.04936362 -0.12378412] TF: shape: (768,) values: [-0.04338909 -0.0781464 -0.01518662 0.04936362 -0.12378412] bert/encoder/layer_4/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.8734387 0.8576282 0.8339444 0.8450325 0.8105372] TF: shape: (768,) values: [0.8734387 0.8576282 0.8339444 0.8450325 0.8105372] bert/encoder/layer_5/attention/self/query/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.00858843 -0.03920127 0.02552994 -0.02786552 0.02436485] TF: shape: (768, 768) values: [-0.00858843 -0.03920127 0.02552994 -0.02786552 0.02436485] bert/encoder/layer_5/attention/self/query/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.00859117 -0.01642405 -0.04391079 0.01085692 0.02925887] TF: shape: (768,) values: [-0.00859117 -0.01642405 -0.04391079 0.01085692 0.02925887] bert/encoder/layer_5/attention/self/key/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.00352847 0.02330176 -0.00369894 -0.03904612 0.00294574] TF: shape: (768, 768) values: [ 0.00352847 0.02330176 -0.00369894 -0.03904612 0.00294574] bert/encoder/layer_5/attention/self/key/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.01087186 -0.01176561 0.00016575 -0.01163023 0.00946616] TF: shape: (768,) values: [-0.01087186 -0.01176561 0.00016575 -0.01163023 0.00946616] bert/encoder/layer_5/attention/self/value/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.06134222 0.04238288 0.02796064 -0.01284983 0.03683741] TF: shape: (768, 768) values: [ 0.06134222 0.04238288 0.02796064 -0.01284983 0.03683741] bert/encoder/layer_5/attention/self/value/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.05061118 -0.02954445 -0.0034053 -0.00025261 0.0437019 ] TF: shape: (768,) values: [ 0.05061118 -0.02954445 -0.0034053 -0.00025261 0.0437019 ] bert/encoder/layer_5/attention/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.00739815 0.0533964 -0.03736389 -0.04999201 0.01693069] TF: shape: (768, 768) values: [-0.00739815 0.0533964 -0.03736389 -0.04999201 0.01693069] bert/encoder/layer_5/attention/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.0021682 0.01711399 -0.04201518 0.01605333 0.00552063] TF: shape: (768,) values: [-0.0021682 0.01711399 -0.04201518 0.01605333 0.00552063] bert/encoder/layer_5/attention/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.06841327 -0.0146848 0.09792476 -0.23284538 0.2785602 ] TF: shape: (768,) values: [-0.06841327 -0.0146848 0.09792476 -0.23284538 0.2785602 ] bert/encoder/layer_5/attention/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.8908311 0.87884724 0.81637293 0.8047641 0.96539867] TF: shape: (768,) values: [0.8908311 0.87884724 0.81637293 0.8047641 0.96539867] bert/encoder/layer_5/intermediate/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 3072) values: [-0.03246041 0.07251058 -0.08201726 0.00772481 0.02532209] TF: shape: (768, 3072) values: [-0.03246041 0.07251058 -0.08201726 0.00772481 0.02532209] bert/encoder/layer_5/intermediate/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072,) values: [-0.09689714 -0.27696273 -0.13047501 -0.10892326 -0.1057625 ] TF: shape: (3072,) values: [-0.09689714 -0.27696273 -0.13047501 -0.10892326 -0.1057625 ] bert/encoder/layer_5/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072, 768) values: [ 0.0642072 -0.01738782 -0.05095377 0.00523853 0.04425264] TF: shape: (3072, 768) values: [ 0.0642072 -0.01738782 -0.05095377 0.00523853 0.04425264] bert/encoder/layer_5/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.0007217 0.06006297 0.0016595 0.03848181 0.06703516] TF: shape: (768,) values: [-0.0007217 0.06006297 0.0016595 0.03848181 0.06703516] bert/encoder/layer_5/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.00278729 -0.05594506 -0.0631047 0.06023621 -0.18672828] TF: shape: (768,) values: [-0.00278729 -0.05594506 -0.0631047 0.06023621 -0.18672828] bert/encoder/layer_5/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.8621183 0.8515807 0.82654256 0.81729776 0.7985204 ] TF: shape: (768,) values: [0.8621183 0.8515807 0.82654256 0.81729776 0.7985204 ] bert/encoder/layer_6/attention/self/query/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.02527807 -0.01429243 0.01467054 0.08624706 -0.00188593] TF: shape: (768, 768) values: [-0.02527807 -0.01429243 0.01467054 0.08624706 -0.00188593] bert/encoder/layer_6/attention/self/query/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.17319514 0.27564248 0.16801168 -0.10946485 0.1643271 ] TF: shape: (768,) values: [-0.17319514 0.27564248 0.16801168 -0.10946485 0.1643271 ] bert/encoder/layer_6/attention/self/key/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.05886372 0.00706217 0.0398422 0.00882155 -0.04571463] TF: shape: (768, 768) values: [ 0.05886372 0.00706217 0.0398422 0.00882155 -0.04571463] bert/encoder/layer_6/attention/self/key/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.00424696 -0.0001192 0.0046079 -0.00315606 0.00434314] TF: shape: (768,) values: [-0.00424696 -0.0001192 0.0046079 -0.00315606 0.00434314] bert/encoder/layer_6/attention/self/value/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.01720381 0.01170722 0.02346902 -0.02284313 -0.03173028] TF: shape: (768, 768) values: [-0.01720381 0.01170722 0.02346902 -0.02284313 -0.03173028] bert/encoder/layer_6/attention/self/value/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.03492057 0.01813157 -0.00182878 -0.01420629 -0.00508944] TF: shape: (768,) values: [-0.03492057 0.01813157 -0.00182878 -0.01420629 -0.00508944] bert/encoder/layer_6/attention/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.0323688 -0.00689882 0.07379091 0.01121114 -0.02059202] TF: shape: (768, 768) values: [ 0.0323688 -0.00689882 0.07379091 0.01121114 -0.02059202] bert/encoder/layer_6/attention/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.00648672 -0.05935453 -0.05673229 -0.01152384 -0.02766573] TF: shape: (768,) values: [-0.00648672 -0.05935453 -0.05673229 -0.01152384 -0.02766573] bert/encoder/layer_6/attention/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.06793639 0.03157783 0.15647687 -0.15025291 0.14727171] TF: shape: (768,) values: [-0.06793639 0.03157783 0.15647687 -0.15025291 0.14727171] bert/encoder/layer_6/attention/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.8882361 0.8704905 0.80289173 0.77365315 0.92333615] TF: shape: (768,) values: [0.8882361 0.8704905 0.80289173 0.77365315 0.92333615] bert/encoder/layer_6/intermediate/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 3072) values: [ 0.04492201 0.05160861 0.09041415 -0.00742628 0.048133 ] TF: shape: (768, 3072) values: [ 0.04492201 0.05160861 0.09041415 -0.00742628 0.048133 ] bert/encoder/layer_6/intermediate/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072,) values: [-0.09301704 -0.158612 -0.10633879 -0.09706812 -0.17319229] TF: shape: (3072,) values: [-0.09301704 -0.158612 -0.10633879 -0.09706812 -0.17319229] bert/encoder/layer_6/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072, 768) values: [-0.00085372 -0.00974195 0.00684915 0.00038686 0.06610142] TF: shape: (3072, 768) values: [-0.00085372 -0.00974195 0.00684915 0.00038686 0.06610142] bert/encoder/layer_6/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.03254414 0.05681704 0.03720434 0.01936359 0.09134153] TF: shape: (768,) values: [-0.03254414 0.05681704 0.03720434 0.01936359 0.09134153] bert/encoder/layer_6/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.0117129 -0.03209404 -0.08646043 0.03760341 -0.13841423] TF: shape: (768,) values: [-0.0117129 -0.03209404 -0.08646043 0.03760341 -0.13841423] bert/encoder/layer_6/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.8674175 0.8657014 0.8151861 0.82301307 0.8305737 ] TF: shape: (768,) values: [0.8674175 0.8657014 0.8151861 0.82301307 0.8305737 ] bert/encoder/layer_7/attention/self/query/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.00075523 -0.01501983 0.04090893 0.01884826 0.04670674] TF: shape: (768, 768) values: [-0.00075523 -0.01501983 0.04090893 0.01884826 0.04670674] bert/encoder/layer_7/attention/self/query/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.0010344 -0.00423982 0.3117479 0.04494623 -0.01260845] TF: shape: (768,) values: [ 0.0010344 -0.00423982 0.3117479 0.04494623 -0.01260845] bert/encoder/layer_7/attention/self/key/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.02781927 -0.00906972 0.02121989 0.0298591 0.05854786] TF: shape: (768, 768) values: [ 0.02781927 -0.00906972 0.02121989 0.0298591 0.05854786] bert/encoder/layer_7/attention/self/key/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.00074918 0.00731079 0.00089338 0.00345652 0.00043817] TF: shape: (768,) values: [-0.00074918 0.00731079 0.00089338 0.00345652 0.00043817] bert/encoder/layer_7/attention/self/value/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.01080035 -0.03468366 0.03167168 0.01583073 0.0327719 ] TF: shape: (768, 768) values: [-0.01080035 -0.03468366 0.03167168 0.01583073 0.0327719 ] bert/encoder/layer_7/attention/self/value/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.02824226 0.01605172 0.00067929 -0.04553111 0.0076044 ] TF: shape: (768,) values: [-0.02824226 0.01605172 0.00067929 -0.04553111 0.0076044 ] bert/encoder/layer_7/attention/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.05496112 0.01006968 0.02206531 -0.01873116 0.02149118] TF: shape: (768, 768) values: [-0.05496112 0.01006968 0.02206531 -0.01873116 0.02149118] bert/encoder/layer_7/attention/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.00349772 -0.05831751 -0.0594084 -0.0342187 0.02965918] TF: shape: (768,) values: [ 0.00349772 -0.05831751 -0.0594084 -0.0342187 0.02965918] bert/encoder/layer_7/attention/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.02826844 0.04427591 0.05678326 -0.0475907 0.16136196] TF: shape: (768,) values: [-0.02826844 0.04427591 0.05678326 -0.0475907 0.16136196] bert/encoder/layer_7/attention/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.8742141 0.870608 0.79147685 0.7595279 0.9223656 ] TF: shape: (768,) values: [0.8742141 0.870608 0.79147685 0.7595279 0.9223656 ] bert/encoder/layer_7/intermediate/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 3072) values: [ 0.03598932 -0.12225644 0.03019998 0.05691092 0.03717208] TF: shape: (768, 3072) values: [ 0.03598932 -0.12225644 0.03019998 0.05691092 0.03717208] bert/encoder/layer_7/intermediate/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072,) values: [-0.12465011 -0.08639494 -0.06206005 -0.08012587 -0.08773767] TF: shape: (3072,) values: [-0.12465011 -0.08639494 -0.06206005 -0.08012587 -0.08773767] bert/encoder/layer_7/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072, 768) values: [-0.02190432 -0.02279165 0.03279508 0.01011065 -0.07793335] TF: shape: (3072, 768) values: [-0.02190432 -0.02279165 0.03279508 0.01011065 -0.07793335] bert/encoder/layer_7/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.04282642 0.03700675 0.06142357 -0.04787201 0.02958163] TF: shape: (768,) values: [-0.04282642 0.03700675 0.06142357 -0.04787201 0.02958163] bert/encoder/layer_7/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.03142036 -0.04358427 -0.05132087 -0.01788123 -0.16399944] TF: shape: (768,) values: [-0.03142036 -0.04358427 -0.05132087 -0.01788123 -0.16399944] bert/encoder/layer_7/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.83858097 0.8179645 0.80693793 0.81225365 0.7844832 ] TF: shape: (768,) values: [0.83858097 0.8179645 0.80693793 0.81225365 0.7844832 ] bert/encoder/layer_8/attention/self/query/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [0.0448719 0.02289526 0.03083764 0.03048073 0.02436891] TF: shape: (768, 768) values: [0.0448719 0.02289526 0.03083764 0.03048073 0.02436891] bert/encoder/layer_8/attention/self/query/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.25132924 -0.23753347 0.02581017 0.00901509 0.18424493] TF: shape: (768,) values: [-0.25132924 -0.23753347 0.02581017 0.00901509 0.18424493] bert/encoder/layer_8/attention/self/key/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.01999719 0.00711403 0.03949134 -0.0102224 0.03152475] TF: shape: (768, 768) values: [-0.01999719 0.00711403 0.03949134 -0.0102224 0.03152475] bert/encoder/layer_8/attention/self/key/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 5.5668897e-05 3.4638541e-03 -1.7605867e-03 -6.1321147e-03 -4.4074579e-04] TF: shape: (768,) values: [ 5.5668897e-05 3.4638541e-03 -1.7605867e-03 -6.1321147e-03 -4.4074579e-04] bert/encoder/layer_8/attention/self/value/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.00736056 -0.01795213 0.00104576 -0.00034653 0.03190543] TF: shape: (768, 768) values: [-0.00736056 -0.01795213 0.00104576 -0.00034653 0.03190543] bert/encoder/layer_8/attention/self/value/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.02892835 0.00642501 -0.03608712 0.00264269 -0.0245198 ] TF: shape: (768,) values: [ 0.02892835 0.00642501 -0.03608712 0.00264269 -0.0245198 ] bert/encoder/layer_8/attention/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.03971623 0.05307067 -0.01298818 0.00946693 -0.00121235] TF: shape: (768, 768) values: [ 0.03971623 0.05307067 -0.01298818 0.00946693 -0.00121235] bert/encoder/layer_8/attention/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.01468131 -0.05406622 -0.06289103 0.004484 0.0240819 ] TF: shape: (768,) values: [ 0.01468131 -0.05406622 -0.06289103 0.004484 0.0240819 ] bert/encoder/layer_8/attention/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.06004262 0.0457275 0.08688109 -0.14416659 -0.05500487] TF: shape: (768,) values: [-0.06004262 0.0457275 0.08688109 -0.14416659 -0.05500487] bert/encoder/layer_8/attention/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.8907534 0.89116573 0.811639 0.7810443 0.9045574 ] TF: shape: (768,) values: [0.8907534 0.89116573 0.811639 0.7810443 0.9045574 ] bert/encoder/layer_8/intermediate/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 3072) values: [-0.01962814 -0.01482586 -0.02292624 0.03397145 0.02457482] TF: shape: (768, 3072) values: [-0.01962814 -0.01482586 -0.02292624 0.03397145 0.02457482] bert/encoder/layer_8/intermediate/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072,) values: [-0.08129632 -0.1691108 -0.10681771 -0.10392351 -0.13120006] TF: shape: (3072,) values: [-0.08129632 -0.1691108 -0.10681771 -0.10392351 -0.13120006] bert/encoder/layer_8/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072, 768) values: [-0.04683433 -0.02690669 0.02979059 0.02223369 -0.00130287] TF: shape: (3072, 768) values: [-0.04683433 -0.02690669 0.02979059 0.02223369 -0.00130287] bert/encoder/layer_8/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.09155537 -0.04465394 0.05649116 -0.09628641 0.11875238] TF: shape: (768,) values: [-0.09155537 -0.04465394 0.05649116 -0.09628641 0.11875238] bert/encoder/layer_8/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.06043394 -0.06657387 -0.05341128 -0.00374733 -0.10855272] TF: shape: (768,) values: [-0.06043394 -0.06657387 -0.05341128 -0.00374733 -0.10855272] bert/encoder/layer_8/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.84467345 0.84421015 0.82582206 0.84553087 0.8207573 ] TF: shape: (768,) values: [0.84467345 0.84421015 0.82582206 0.84553087 0.8207573 ] bert/encoder/layer_9/attention/self/query/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.08004542 -0.0143706 -0.04219061 -0.05175152 -0.01147588] TF: shape: (768, 768) values: [ 0.08004542 -0.0143706 -0.04219061 -0.05175152 -0.01147588] bert/encoder/layer_9/attention/self/query/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.14508031 0.40926442 -0.3281781 -0.02869792 -0.26104516] TF: shape: (768,) values: [-0.14508031 0.40926442 -0.3281781 -0.02869792 -0.26104516] bert/encoder/layer_9/attention/self/key/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.01337681 0.00615428 -0.0455939 0.03379053 -0.01992556] TF: shape: (768, 768) values: [-0.01337681 0.00615428 -0.0455939 0.03379053 -0.01992556] bert/encoder/layer_9/attention/self/key/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.0051302 0.0083288 0.00377641 0.00928865 -0.00418182] TF: shape: (768,) values: [-0.0051302 0.0083288 0.00377641 0.00928865 -0.00418182] bert/encoder/layer_9/attention/self/value/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.02485976 -0.0301923 0.00984638 -0.02495162 0.01074037] TF: shape: (768, 768) values: [-0.02485976 -0.0301923 0.00984638 -0.02495162 0.01074037] bert/encoder/layer_9/attention/self/value/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.04229928 -0.02636711 0.0060447 0.00222829 0.04979481] TF: shape: (768,) values: [-0.04229928 -0.02636711 0.0060447 0.00222829 0.04979481] bert/encoder/layer_9/attention/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.01258144 0.00871274 0.00482882 -0.00675888 -0.04390825] TF: shape: (768, 768) values: [-0.01258144 0.00871274 0.00482882 -0.00675888 -0.04390825] bert/encoder/layer_9/attention/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.02457753 0.05051134 -0.06890804 -0.00962795 0.00864793] TF: shape: (768,) values: [ 0.02457753 0.05051134 -0.06890804 -0.00962795 0.00864793] bert/encoder/layer_9/attention/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.08963391 -0.06362236 0.0676669 -0.09895685 0.08318913] TF: shape: (768,) values: [-0.08963391 -0.06362236 0.0676669 -0.09895685 0.08318913] bert/encoder/layer_9/attention/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.85100883 0.82569736 0.7927931 0.7660444 0.8912934 ] TF: shape: (768,) values: [0.85100883 0.82569736 0.7927931 0.7660444 0.8912934 ] bert/encoder/layer_9/intermediate/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 3072) values: [ 0.06290598 0.0203122 -0.05384256 0.05442941 0.00484769] TF: shape: (768, 3072) values: [ 0.06290598 0.0203122 -0.05384256 0.05442941 0.00484769] bert/encoder/layer_9/intermediate/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072,) values: [-0.10818483 -0.00169527 -0.08962701 -0.10280421 -0.14310956] TF: shape: (3072,) values: [-0.10818483 -0.00169527 -0.08962701 -0.10280421 -0.14310956] bert/encoder/layer_9/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072, 768) values: [ 0.05487705 0.01644666 0.00436198 -0.00490768 -0.03238423] TF: shape: (3072, 768) values: [ 0.05487705 0.01644666 0.00436198 -0.00490768 -0.03238423] bert/encoder/layer_9/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.08755219 -0.01910074 -0.02988298 -0.08150438 0.09897955] TF: shape: (768,) values: [-0.08755219 -0.01910074 -0.02988298 -0.08150438 0.09897955] bert/encoder/layer_9/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.04136161 -0.02113917 -0.07581077 -0.00809791 -0.09790538] TF: shape: (768,) values: [-0.04136161 -0.02113917 -0.07581077 -0.00809791 -0.09790538] bert/encoder/layer_9/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.8250572 0.83477134 0.7794141 0.81264955 0.7827918 ] TF: shape: (768,) values: [0.8250572 0.83477134 0.7794141 0.81264955 0.7827918 ] bert/encoder/layer_10/attention/self/query/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.00071212 -0.00853064 0.01776993 0.03189976 0.02183623] TF: shape: (768, 768) values: [ 0.00071212 -0.00853064 0.01776993 0.03189976 0.02183623] bert/encoder/layer_10/attention/self/query/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.03667567 -0.01449654 -0.03822913 0.00118343 -0.05489838] TF: shape: (768,) values: [-0.03667567 -0.01449654 -0.03822913 0.00118343 -0.05489838] bert/encoder/layer_10/attention/self/key/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.0494106 0.05531096 -0.02459413 -0.06019118 -0.02829785] TF: shape: (768, 768) values: [-0.0494106 0.05531096 -0.02459413 -0.06019118 -0.02829785] bert/encoder/layer_10/attention/self/key/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.00692997 0.00855893 0.00670777 -0.0052475 -0.00017074] TF: shape: (768,) values: [-0.00692997 0.00855893 0.00670777 -0.0052475 -0.00017074] bert/encoder/layer_10/attention/self/value/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.01911842 0.04858809 -0.02608485 0.00794924 -0.02246636] TF: shape: (768, 768) values: [ 0.01911842 0.04858809 -0.02608485 0.00794924 -0.02246636] bert/encoder/layer_10/attention/self/value/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.0133503 -0.01224133 -0.0051834 -0.00232528 0.00148614] TF: shape: (768,) values: [-0.0133503 -0.01224133 -0.0051834 -0.00232528 0.00148614] bert/encoder/layer_10/attention/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.05904732 0.02616 0.00794104 -0.02889086 -0.03692576] TF: shape: (768, 768) values: [-0.05904732 0.02616 0.00794104 -0.02889086 -0.03692576] bert/encoder/layer_10/attention/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.02089205 0.01458059 0.05217785 0.0324267 0.00907548] TF: shape: (768,) values: [0.02089205 0.01458059 0.05217785 0.0324267 0.00907548] bert/encoder/layer_10/attention/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.10986238 -0.04332284 0.02603893 -0.06236923 0.14469369] TF: shape: (768,) values: [-0.10986238 -0.04332284 0.02603893 -0.06236923 0.14469369] bert/encoder/layer_10/attention/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.8515822 0.81392974 0.836747 0.78040504 0.88091415] TF: shape: (768,) values: [0.8515822 0.81392974 0.836747 0.78040504 0.88091415] bert/encoder/layer_10/intermediate/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 3072) values: [-0.07061081 0.06997397 0.01433633 0.04150929 0.02865192] TF: shape: (768, 3072) values: [-0.07061081 0.06997397 0.01433633 0.04150929 0.02865192] bert/encoder/layer_10/intermediate/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072,) values: [-0.13879126 -0.06401426 -0.1408043 -0.15043251 -0.10193057] TF: shape: (3072,) values: [-0.13879126 -0.06401426 -0.1408043 -0.15043251 -0.10193057] bert/encoder/layer_10/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072, 768) values: [ 0.02918765 0.02609882 -0.02259856 0.01636725 -0.00038442] TF: shape: (3072, 768) values: [ 0.02918765 0.02609882 -0.02259856 0.01636725 -0.00038442] bert/encoder/layer_10/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.01799502 0.10970547 -0.02384165 -0.03350981 0.10491351] TF: shape: (768,) values: [-0.01799502 0.10970547 -0.02384165 -0.03350981 0.10491351] bert/encoder/layer_10/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.00999107 -0.0217309 -0.0854177 -0.01109101 -0.07902174] TF: shape: (768,) values: [ 0.00999107 -0.0217309 -0.0854177 -0.01109101 -0.07902174] bert/encoder/layer_10/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.8272796 0.8597452 0.79116803 0.81267637 0.8273501 ] TF: shape: (768,) values: [0.8272796 0.8597452 0.79116803 0.81267637 0.8273501 ] bert/encoder/layer_11/attention/self/query/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.04141425 -0.06491017 -0.03202523 0.06226195 0.02193764] TF: shape: (768, 768) values: [-0.04141425 -0.06491017 -0.03202523 0.06226195 0.02193764] bert/encoder/layer_11/attention/self/query/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.0501296 0.11886728 0.2186807 0.08720991 -0.20476632] TF: shape: (768,) values: [ 0.0501296 0.11886728 0.2186807 0.08720991 -0.20476632] bert/encoder/layer_11/attention/self/key/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.02634268 -0.01357682 -0.06076496 0.04210597 0.01783857] TF: shape: (768, 768) values: [ 0.02634268 -0.01357682 -0.06076496 0.04210597 0.01783857] bert/encoder/layer_11/attention/self/key/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.0007798 -0.00065806 -0.00010521 0.00119144 -0.00180091] TF: shape: (768,) values: [-0.0007798 -0.00065806 -0.00010521 0.00119144 -0.00180091] bert/encoder/layer_11/attention/self/value/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.03520973 -0.00678078 -0.02883583 -0.01011515 0.04519828] TF: shape: (768, 768) values: [ 0.03520973 -0.00678078 -0.02883583 -0.01011515 0.04519828] bert/encoder/layer_11/attention/self/value/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.01502306 -0.00530942 0.00023572 0.00205218 -0.00578036] TF: shape: (768,) values: [ 0.01502306 -0.00530942 0.00023572 0.00205218 -0.00578036] bert/encoder/layer_11/attention/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [ 0.02361419 0.03112707 -0.00063031 0.04209773 -0.02434015] TF: shape: (768, 768) values: [ 0.02361419 0.03112707 -0.00063031 0.04209773 -0.02434015] bert/encoder/layer_11/attention/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [ 0.02566087 0.0028438 -0.00475678 0.02149458 -0.01755187] TF: shape: (768,) values: [ 0.02566087 0.0028438 -0.00475678 0.02149458 -0.01755187] bert/encoder/layer_11/attention/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.03134411 0.01207957 -0.04636396 -0.03013046 0.07944281] TF: shape: (768,) values: [-0.03134411 0.01207957 -0.04636396 -0.03013046 0.07944281] bert/encoder/layer_11/attention/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.85203767 0.8020145 0.8554237 0.8150477 0.8441815 ] TF: shape: (768,) values: [0.85203767 0.8020145 0.8554237 0.8150477 0.8441815 ] bert/encoder/layer_11/intermediate/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 3072) values: [ 0.05871898 -0.01124212 0.00206979 -0.04366514 -0.00716808] TF: shape: (768, 3072) values: [ 0.05871898 -0.01124212 0.00206979 -0.04366514 -0.00716808] bert/encoder/layer_11/intermediate/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072,) values: [-0.09762521 -0.06175711 -0.05153917 -0.08580919 -0.09734315] TF: shape: (3072,) values: [-0.09762521 -0.06175711 -0.05153917 -0.08580919 -0.09734315] bert/encoder/layer_11/output/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (3072, 768) values: [-0.022382 0.01073206 -0.01357213 0.02484621 0.01403091] TF: shape: (3072, 768) values: [-0.022382 0.01073206 -0.01357213 0.02484621 0.01403091] bert/encoder/layer_11/output/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.06574099 0.04207807 0.01201084 0.00229322 0.05551811] TF: shape: (768,) values: [-0.06574099 0.04207807 0.01201084 0.00229322 0.05551811] bert/encoder/layer_11/output/LayerNorm/beta |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.00634605 -0.01989403 0.04628465 0.01585056 -0.04256899] TF: shape: (768,) values: [-0.00634605 -0.01989403 0.04628465 0.01585056 -0.04256899] bert/encoder/layer_11/output/LayerNorm/gamma |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [0.6384234 0.6300364 0.66570055 0.6126921 0.63756436] TF: shape: (768,) values: [0.6384234 0.6300364 0.66570055 0.6126921 0.63756436] bert/pooler/dense/kernel |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768, 768) values: [-0.00127425 0.00199868 -0.03863145 -0.00139355 0.00691627] TF: shape: (768, 768) values: [-0.00127425 0.00199868 -0.03863145 -0.00139355 0.00691627] bert/pooler/dense/bias |sum(pt_wts - tf_wts)| = 0.0 PT: shape: (768,) values: [-0.03597581 -0.00389536 0.05181352 0.02224747 -0.00493723] TF: shape: (768,) values: [-0.03597581 -0.00389536 0.05181352 0.02224747 -0.00493723]
In [8]:
# Mean Squared Error (MSE) between last projection of each model
MSE = np.mean((pt_embedding - tf_embedding) ** 2, keepdims=False)
print("MSE: {}".format(MSE))
print("PT-values: {}".format(pt_embedding[0, :5]))
print("TF-values: {}".format(tf_embedding[0, :5]))MSE: 2.7155439966009e-05 PT-values: [-0.876663 -0.41088238 -0.12200808 0.44941 0.19445966] TF-values: [-0.8742865 -0.40621698 -0.10585472 0.444904 0.1825743 ]