c-my/transformers
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #42 from weiyumou/master

Browse Source 
Fixed UnicodeDecodeError: 'ascii' codec can't decode byte 0xc2
This commit is contained in:
Thomas Wolf authored and GitHub committed 2018-11-20 10:09:50 +01:00
2 parents eed255a58d 9ff2b7d86d
commit fd32ebed81
2 files changed
+2 -2
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

README.md
+1 -1
View File
@@ -99,7 +99,7 @@ from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
# Tokenized input # Tokenized input
tokenized_text = "Who was Jim Henson ? Jim Henson was a puppeteer" text = "Who was Jim Henson ? Jim Henson was a puppeteer"
tokenized_text = tokenizer.tokenize(text) tokenized_text = tokenizer.tokenize(text)
# Mask a token that we will try to predict back with `BertForMaskedLM` # Mask a token that we will try to predict back with `BertForMaskedLM`
pytorch_pretrained_bert/tokenization.py
+1 -1
View File
@@ -65,7 +65,7 @@ def load_vocab(vocab_file):
"""Loads a vocabulary file into a dictionary.""" """Loads a vocabulary file into a dictionary."""
vocab = collections.OrderedDict() vocab = collections.OrderedDict()
index = 0 index = 0
with open(vocab_file, "r") as reader: with open(vocab_file, "r", encoding="utf8") as reader:
while True: while True:
token = convert_to_unicode(reader.readline()) token = convert_to_unicode(reader.readline())
if not token: if not token: