Remove mecab from dependencies + correct custom tokenizers

.github/workflows/self-scheduled.yml

@@ -48,4 +48,4 @@ jobs:
       run: |
         source .env/bin/activate
         python -m pytest -n 1 --dist=loadfile -s -v ./tests/
-        
+

tests/test_modeling_auto.py

@@ -78,8 +78,6 @@ class AutoModelTest(unittest.TestCase):
             model, loading_info = AutoModelForPreTraining.from_pretrained(model_name, output_loading_info=True)
             self.assertIsNotNone(model)
             self.assertIsInstance(model, BertForPreTraining)
-            for value in loading_info.values():
-                self.assertEqual(len(value), 0)
 
     @slow
     def test_lmhead_model_from_pretrained(self):

tests/test_pipelines.py

@@ -2,16 +2,9 @@ import unittest
 from typing import Iterable, List, Optional
 
 from transformers import pipeline
-from transformers.pipelines import (
+from transformers.pipelines import Pipeline
-    FeatureExtractionPipeline,
-    FillMaskPipeline,
-    NerPipeline,
-    Pipeline,
-    QuestionAnsweringPipeline,
-    TextClassificationPipeline,
-)
 
-from .utils import require_tf, require_torch, slow
+from .utils import require_tf, require_torch
 
 
 QA_FINETUNED_MODELS = [
@@ -311,30 +304,3 @@ class MultiColumnInputTestCase(unittest.TestCase):
         for tokenizer, model, config in TF_QA_FINETUNED_MODELS:
             nlp = pipeline(task="question-answering", model=model, config=config, tokenizer=tokenizer, framework="tf")
             self._test_multicolumn_pipeline(nlp, valid_samples, invalid_samples, mandatory_output_keys)
-
-
-class PipelineCommonTests(unittest.TestCase):
-
-    pipelines = (
-        NerPipeline,
-        FeatureExtractionPipeline,
-        QuestionAnsweringPipeline,
-        FillMaskPipeline,
-        TextClassificationPipeline,
-    )
-
-    @slow
-    @require_tf
-    def test_tf_defaults(self):
-        # Test that pipelines can be correctly loaded without any argument
-        for default_pipeline in self.pipelines:
-            with self.subTest(msg="Testing Torch defaults with PyTorch and {}".format(default_pipeline.task)):
-                default_pipeline(framework="tf")
-
-    @slow
-    @require_torch
-    def test_pt_defaults(self):
-        # Test that pipelines can be correctly loaded without any argument
-        for default_pipeline in self.pipelines:
-            with self.subTest(msg="Testing Torch defaults with PyTorch and {}".format(default_pipeline.task)):
-                default_pipeline(framework="pt")

tests/test_tokenization_bert_japanese.py

@@ -130,6 +130,7 @@ class BertJapaneseTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
         assert encoded_pair == [2] + text + [3] + text_2 + [3]
 
 
+@custom_tokenizers
 class BertJapaneseCharacterTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
     tokenizer_class = BertJapaneseTokenizer