diff --git a/README.md b/README.md index 4e57de58..dc51629b 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ for model_class, tokenizer_class, pretrained_weights in MODELS: model = model_class.from_pretrained(pretrained_weights) # Encode text - input_ids = torch.tensor([tokenizer.encode("Here is some text to encode")]) + input_ids = torch.tensor([tokenizer.encode("Here is some text to encode", add_special_tokens=True)]) # Add special tokens takes care of adding [CLS], [SEP], ... tokens in the right way for each model. with torch.no_grad(): last_hidden_states = model(input_ids)[0] # Models outputs are now tuples