diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 0bc2cbb7..9d814f70 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -641,9 +641,9 @@ class PreTrainedTokenizer(object): filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens) text = self.convert_tokens_to_string(filtered_tokens) - if self.sep_token is not None and self.sep_token in text: - text = text.replace(self.cls_token, self.sep_token) - split_text = list(filter(lambda sentence: len(sentence) > 0, text.split(self.sep_token))) + if self._sep_token is not None and self._sep_token in text: + text = text.replace(self._cls_token, self._sep_token) + split_text = list(filter(lambda sentence: len(sentence) > 0, text.split(self._sep_token))) if clean_up_tokenization_spaces: clean_text = [self.clean_up_tokenization(text) for text in split_text] return clean_text