remove ipdb

2020-07-06 19:20:17 +02:00
parent 9b23585e4d
commit 84081e0919
1 changed files with 0 additions and 3 deletions
@@ -307,9 +307,6 @@ class PreTrainedTokenizer(PreTrainedTokenizerBase):
                            full_word = ""
                            continue
                    # Strip white spaces on the right
-                    import ipdb
-
-                    ipdb.set_trace()
                    if tok_extended.rstrip and i > 0:
                        # A bit counter-intuitive but we strip the left of the string
                        # since tok_extended.rstrip means the special token is eating all white spaces on its right