[blenderbot] regex fix (#8282)
Fixing:
```
src/transformers/tokenization_blenderbot.py:163: DeprecationWarning: invalid escape sequence \s
token = re.sub("\s{2,}", " ", token)
```
This commit is contained in:
@@ -160,7 +160,7 @@ class BlenderbotSmallTokenizer(PreTrainedTokenizer):
|
||||
return self.cache[token]
|
||||
token = re.sub("([.,!?()])", r" \1", token)
|
||||
token = re.sub("(')", r" \1 ", token)
|
||||
token = re.sub("\s{2,}", " ", token)
|
||||
token = re.sub(r"\s{2,}", " ", token)
|
||||
if "\n" in token:
|
||||
token = token.replace("\n", " __newln__")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user