c-my/transformers
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Address CI needs

Browse Source
This commit is contained in:
LysandreJik
2020-12-10 14:55:17 -05:00
parent d240f471a0
commit 29d7e4a1ae
5 changed files with 54 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+3
View File
@@ -102,6 +102,7 @@ jobs:
- v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip
- run: pip install .[sklearn,tf-cpu,torch,testing,sentencepiece]
- run: pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.7.0+cpu.html
- save_cache:
key: v0.4-{{ checksum "setup.py" }}
paths:
@@ -129,6 +130,7 @@ jobs:
- v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip
- run: pip install .[sklearn,torch,testing,sentencepiece]
- run: pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.7.0+cpu.html
- save_cache:
key: v0.4-torch-{{ checksum "setup.py" }}
paths:
@@ -210,6 +212,7 @@ jobs:
- v0.4-{{ checksum "setup.py" }}
- run: pip install --upgrade pip
- run: pip install .[sklearn,torch,testing,sentencepiece]
- run: pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.7.0+cpu.html
- save_cache:
key: v0.4-torch-{{ checksum "setup.py" }}
paths:
src/transformers/file_utils.py
+13
View File
@@ -216,6 +216,15 @@ except ImportError:
_tokenizers_available = False
try:
import pandas # noqa: F401
_pandas_available = True
except ImportError:
_pandas_available = False
try:
import torch_scatter
@@ -343,6 +352,10 @@ def is_scatter_available():
return _scatter_available
def is_pandas_available():
return _pandas_available
def torch_only_method(fn):
def wrapper(*args, **kwargs):
if not _torch_available:
src/transformers/models/tapas/tokenization_tapas.py
+18 -15
View File
@@ -26,11 +26,11 @@ import unicodedata
from dataclasses import dataclass
from typing import Callable, Dict, Generator, List, Optional, Text, Tuple, Union
import pandas as pd
import torch
import numpy as np
from transformers import add_end_docstrings
from ...file_utils import is_pandas_available
from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...tokenization_utils_base import (
ENCODE_KWARGS_DOCSTRING,
@@ -45,6 +45,9 @@ from ...tokenization_utils_base import (
from ...utils import logging
if is_pandas_available():
import pandas as pd
logger = logging.get_logger(__name__)
@@ -307,6 +310,9 @@ class TapasTokenizer(PreTrainedTokenizer):
additional_special_tokens: Optional[List[str]] = None,
**kwargs
):
if not is_pandas_available():
raise ImportError("Pandas is required for the TAPAS tokenizer.")
if additional_special_tokens is not None:
if empty_token not in additional_special_tokens:
additional_special_tokens.append(empty_token)
@@ -539,7 +545,7 @@ class TapasTokenizer(PreTrainedTokenizer):
@add_end_docstrings(TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
def __call__(
self,
table: pd.DataFrame,
table: "pd.DataFrame",
queries: Optional[
Union[
TextInput,
@@ -663,7 +669,7 @@ class TapasTokenizer(PreTrainedTokenizer):
@add_end_docstrings(ENCODE_KWARGS_DOCSTRING, TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
def batch_encode_plus(
self,
table: pd.DataFrame,
table: "pd.DataFrame",
queries: Optional[
Union[
List[TextInput],
@@ -812,7 +818,7 @@ class TapasTokenizer(PreTrainedTokenizer):
def _batch_prepare_for_model(
self,
raw_table: pd.DataFrame,
raw_table: "pd.DataFrame",
raw_queries: Union[
List[TextInput],
List[PreTokenizedInput],
@@ -884,7 +890,7 @@ class TapasTokenizer(PreTrainedTokenizer):
@add_end_docstrings(ENCODE_KWARGS_DOCSTRING)
def encode(
self,
table: pd.DataFrame,
table: "pd.DataFrame",
query: Optional[
Union[
TextInput,
@@ -927,7 +933,7 @@ class TapasTokenizer(PreTrainedTokenizer):
@add_end_docstrings(ENCODE_KWARGS_DOCSTRING, TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
def encode_plus(
self,
table: pd.DataFrame,
table: "pd.DataFrame",
query: Optional[
Union[
TextInput,
@@ -1010,7 +1016,7 @@ class TapasTokenizer(PreTrainedTokenizer):
def _encode_plus(
self,
table: pd.DataFrame,
table: "pd.DataFrame",
query: Union[
TextInput,
PreTokenizedInput,
@@ -1066,7 +1072,7 @@ class TapasTokenizer(PreTrainedTokenizer):
@add_end_docstrings(ENCODE_KWARGS_DOCSTRING, TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
def prepare_for_model(
self,
raw_table: pd.DataFrame,
raw_table: "pd.DataFrame",
raw_query: Union[
TextInput,
PreTokenizedInput,
@@ -1884,7 +1890,7 @@ class TapasTokenizer(PreTrainedTokenizer):
col = column_ids[i] - 1
row = row_ids[i] - 1
coords_to_probs[(col, row)].append(prob)
return {coords: torch.as_tensor(cell_probs).mean() for coords, cell_probs in coords_to_probs.items()}
return {coords: np.array(cell_probs).mean() for coords, cell_probs in coords_to_probs.items()}
def convert_logits_to_predictions(self, data, logits, logits_agg=None, cell_classification_threshold=0.5):
"""
@@ -1912,11 +1918,8 @@ class TapasTokenizer(PreTrainedTokenizer):
of length ``batch_size``: Predicted aggregation operator indices of the aggregation head.
"""
# compute probabilities from token logits
dist_per_token = torch.distributions.Bernoulli(logits=logits)
probabilities = dist_per_token.probs * data["attention_mask"].type(torch.float32).to(
dist_per_token.probs.device
)
# DO sigmoid here
probabilities = 1 / (1 + np.exp(-logits)) * data["attention_mask"]
token_types = [
"segment_ids",
"column_ids",
src/transformers/testing_utils.py
+14
View File
@@ -28,6 +28,7 @@ from .file_utils import (
_datasets_available,
_faiss_available,
_flax_available,
_pandas_available,
_scatter_available,
_sentencepiece_available,
_tf_available,
@@ -222,6 +223,19 @@ def require_tokenizers(test_case):
return test_case
def require_pandas(test_case):
"""
Decorator marking a test that requires pandas.
These tests are skipped when pandas isn't installed.
"""
if not _pandas_available:
return unittest.skip("test requires pandas")(test_case)
else:
return test_case
def require_scatter(test_case):
"""
Decorator marking a test that requires PyTorch Scatter. These tests are skipped when PyTorch Scatter isn't
tests/test_tokenization_tapas.py
+6 -1
View File
@@ -32,12 +32,13 @@ from transformers.models.tapas.tokenization_tapas import (
_is_punctuation,
_is_whitespace,
)
from transformers.testing_utils import is_pt_tf_cross_test, require_tokenizers, require_torch, slow
from transformers.testing_utils import is_pt_tf_cross_test, require_pandas, require_tokenizers, require_torch, slow
from .test_tokenization_common import TokenizerTesterMixin, filter_non_english, merge_model_tokenizer_mappings
@require_tokenizers
@require_pandas
class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
tokenizer_class = TapasTokenizer
test_rust_tokenizer = False
@@ -3418,3 +3419,7 @@ class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
self.assertListEqual(segment_ids.tolist(), expected_results["segment_ids"])
self.assertListEqual(column_ids.tolist(), expected_results["column_ids"])
self.assertListEqual(row_ids.tolist(), expected_results["row_ids"])
@unittest.skip("Skip this test while all models are still to be uploaded.")
def test_pretrained_model_lists(self):
pass