Merge pull request #2081 from pglock/patch-1

handle string with only whitespaces as empty
2019-12-12 08:20:43 +01:00
parent 137e20a846 21451ec6ba
commit 707f9e9241
1 changed files with 1 additions and 1 deletions
@@ -669,7 +669,7 @@ class PreTrainedTokenizer(object):
            return result

        def split_on_tokens(tok_list, text):
-            if not text:
+            if not text.strip():
                return []
            if not tok_list:
                return self._tokenize(text, **kwargs)