Remove '\n' before adding token into vocab
This commit is contained in:
Yiqing-Zhou authored and GitHub committed 2019-07-22 22:30:49 +08:00
1 parent 897d0841be
commit bef0c629ca
1 file changed
+1
@@ -69,6 +69,7 @@ def load_vocab(vocab_file):
with open(vocab_file, "r", encoding="utf-8") as reader:
tokens = reader.readlines()
for index, token in enumerate(tokens):
token = token[:-1]
vocab[token] = index
return vocab