Merge pull request #42 from weiyumou/master

Fixed UnicodeDecodeError: 'ascii' codec can't decode byte 0xc2
This commit is contained in:
Thomas Wolf authored and GitHub committed 2018-11-20 10:09:50 +01:00
commit fd32ebed81
2 files changed
+2 -2

No files matched your search

+1 -1
View File
@@ -99,7 +99,7 @@ from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
# Tokenized input
tokenized_text = "Who was Jim Henson ? Jim Henson was a puppeteer"
text = "Who was Jim Henson ? Jim Henson was a puppeteer"
tokenized_text = tokenizer.tokenize(text)
# Mask a token that we will try to predict back with `BertForMaskedLM`