make save_pretrained work with added tokens

right now it's dumping the *decoder* when it should be dumping the *encoder*. this fixes that.
This commit is contained in:
Joel Grus authored and GitHub committed 2019-07-24 16:54:48 -07:00
1 parent 067923d326
commit ae152cec09
1 file changed
+1 -1
+1 -1
View File
@@ -266,7 +266,7 @@ class PreTrainedTokenizer(object):
with open(added_tokens_file, 'w', encoding='utf-8') as f:
if self.added_tokens_encoder:
out_str = json.dumps(self.added_tokens_decoder, ensure_ascii=False)
out_str = json.dumps(self.added_tokens_encoder, ensure_ascii=False)
else:
out_str = u"{}"
f.write(out_str)