import torch

Workaround braces
don't waste resources
2020-12-11 13:32:08 -05:00 · 2020-12-11 13:29:58 -05:00 · 2020-12-11 13:23:24 -05:00 · 2020-12-11 13:16:47 -05:00 · 2020-12-11 13:08:25 -05:00 · 2020-12-11 10:20:56 -05:00
3 changed files with 3 additions and 235 deletions
@@ -3,47 +3,6 @@ orbs:
    gcp-gke: circleci/gcp-gke@1.0.4
    go: circleci/go@1.3.0

-commands:
-  skip-job-on-doc-only-changes:
-    description: "Do not continue this job and exit with success for PRs with only doc changes"
-    steps:
-
-      - run:
-          name: docs-only changes skip check
-          command: |
-            if test -n "$CIRCLE_PR_NUMBER"
-            then
-                echo $CIRCLE_PR_NUMBER
-                resp=$(curl -Ls https://api.github.com/repos/huggingface/transformers/pulls/${CIRCLE_PR_NUMBER})
-                user=$(jq -r .user.login \<<< $resp)   # PR creator username
-                head_ref=$(jq -r .head.ref \<<< $resp) # PR user's branch name
-                echo head_ref=$head_ref, user=$user
-            fi
-            
-            if test -n "$user" && test -n "$head_ref"
-            then
-                git clone https://github.com/$user/transformers user-clone
-                cd user-clone
-                git checkout $head_ref
-                fork_point_sha=$(git merge-base --fork-point master)
-                cd -
-            fi
-
-            if test -n "$fork_point_sha" && test -n "$(git diff --name-only $fork_point_sha)"
-            then
-                git --no-pager diff --name-only $fork_point_sha
-                if git diff --name-only $fork_point_sha | egrep -qv '\.(md|rst)$'
-                then
-                    echo "Non-docs were modified in this PR, proceeding normally"
-                else
-                    echo "Only docs were modified in this PR, quitting this job"
-                    # enable skipping once we get this sorted out
-                    # circleci step halt
-                fi
-            else
-                echo "Not enough data to perform a skipping check - continuing the job"
-            fi
-

 # TPU REFERENCES
 references:
@@ -114,7 +73,6 @@ jobs:
        parallelism: 1
        steps:
            - checkout
-            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-torch_and_tf-{{ checksum "setup.py" }}
@@ -141,7 +99,6 @@ jobs:
        parallelism: 1
        steps:
            - checkout
-            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-torch-{{ checksum "setup.py" }}
@@ -168,7 +125,6 @@ jobs:
        parallelism: 1
        steps:
            - checkout
-            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-tf-{{ checksum "setup.py" }}
@@ -195,7 +151,6 @@ jobs:
        parallelism: 1
        steps:
            - checkout
-            - skip-job-on-doc-only-changes
            - restore_cache:
                keys:
                    - v0.4-flax-{{ checksum "setup.py" }}
@@ -222,7 +177,6 @@ jobs:
        parallelism: 1
        steps:
            - checkout
-            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-torch-{{ checksum "setup.py" }}
@@ -249,7 +203,6 @@ jobs:
        parallelism: 1
        steps:
            - checkout
-            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-tf-{{ checksum "setup.py" }}
@@ -274,7 +227,6 @@ jobs:
            RUN_CUSTOM_TOKENIZERS: yes
        steps:
            - checkout
-            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-custom_tokenizers-{{ checksum "setup.py" }}
@@ -302,7 +254,6 @@ jobs:
        parallelism: 1
        steps:
            - checkout
-            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-torch_examples-{{ checksum "setup.py" }}
@@ -2,6 +2,8 @@ name: Model templates runner

 on:
  push:
+    branches:
+      - master
    paths:
      - "src/**"
      - "tests/**"
@@ -50,6 +50,7 @@ jobs:
          pip install --upgrade pip
          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
          pip install git+https://github.com/huggingface/datasets
+          pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.6.0+$(python -c "import torch; print(''.join(torch.version.cuda.split('.')))").html

      - name: Are GPUs recognized by our DL frameworks
        run: |
@@ -85,189 +86,3 @@ jobs:
          path: reports
                  

-  run_tests_tf_gpu:
-    runs-on: [self-hosted, gpu, single-gpu]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Python version
-        run: |
-          which python
-          python --version
-          pip --version
-      - name: Current dir
-        run: pwd
-      - run: nvidia-smi
-
-      - name: Loading cache.
-        uses: actions/cache@v2
-        id: cache
-        with:
-          path: .env
-          key: v1.1-tests_tf_gpu-${{ hashFiles('setup.py') }}
-
-      - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
-        run: |
-          python -m venv .env
-          source .env/bin/activate
-          which python
-          python --version
-          pip --version
-
-      - name: Install dependencies
-        run: |
-          source .env/bin/activate
-          pip install --upgrade pip
-          pip install .[tf,sklearn,testing,onnxruntime,sentencepiece]
-          pip install git+https://github.com/huggingface/datasets
-
-      - name: Are GPUs recognized by our DL frameworks
-        run: |
-          source .env/bin/activate
-          TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))"
-          TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))"
-
-      - name: Create model files
-        run: |
-          source .env/bin/activate
-#          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-#          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/pt-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-#          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/standalone.json --path=templates/adding_a_new_model
-#          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/tf-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-
-      - name: Run all non-slow tests on GPU
-        env:
-          OMP_NUM_THREADS: 1
-          CUDA_VISIBLE_DEVICES: 0
-        run: |
-          source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_tf_gpu tests
-
-      - name: Failure short reports
-        if: ${{ always() }}
-        run: cat reports/tests_tf_gpu_failures_short.txt
-        
-      - name: Test suite reports artifacts
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: run_all_tests_tf_gpu_test_reports
-          path: reports
-
-  run_tests_torch_multi_gpu:
-    runs-on: [self-hosted, gpu, multi-gpu]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Python version
-        run: |
-          which python
-          python --version
-          pip --version
-
-      - name: Current dir
-        run: pwd
-      - run: nvidia-smi
-
-      - name: Loading cache.
-        uses: actions/cache@v2
-        id: cache
-        with:
-          path: .env
-          key: v1.1-tests_torch_multi_gpu-${{ hashFiles('setup.py') }}
-
-      - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
-        run: |
-          python -m venv .env
-          source .env/bin/activate
-          which python
-          python --version
-          pip --version
-      - name: Install dependencies
-        run: |
-          source .env/bin/activate
-          pip install --upgrade pip
-          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
-          pip install git+https://github.com/huggingface/datasets
-
-      - name: Are GPUs recognized by our DL frameworks
-        run: |
-          source .env/bin/activate
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
-
-      - name: Run all non-slow tests on GPU
-        env:
-          OMP_NUM_THREADS: 1
-        run: |
-          source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_torch_multi_gpu tests
-
-      - name: Failure short reports
-        if: ${{ always() }}
-        run: cat reports/tests_torch_multi_gpu_failures_short.txt          
-
-      - name: Test suite reports artifacts
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: run_all_tests_torch_multi_gpu_test_reports
-          path: reports
-
-  run_tests_tf_multi_gpu:
-    runs-on: [self-hosted, gpu, multi-gpu]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Python version
-        run: |
-          which python
-          python --version
-          pip --version
-
-      - name: Current dir
-        run: pwd
-      - run: nvidia-smi
-
-      - name: Loading cache.
-        uses: actions/cache@v2
-        id: cache
-        with:
-          path: .env
-          key: v1.1-tests_tf_multi_gpu-${{ hashFiles('setup.py') }}
-
-      - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
-        run: |
-          python -m venv .env
-          source .env/bin/activate
-          which python
-          python --version
-          pip --version
-      - name: Install dependencies
-        run: |
-          source .env/bin/activate
-          pip install --upgrade pip
-          pip install .[tf,sklearn,testing,onnxruntime,sentencepiece]
-          pip install git+https://github.com/huggingface/datasets
-
-      - name: Are GPUs recognized by our DL frameworks
-        run: |
-          source .env/bin/activate
-          TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))"
-          TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))"
-
-      - name: Run all non-slow tests on GPU
-        env:
-          OMP_NUM_THREADS: 1
-        run: |
-          source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_tf_multi_gpu tests
-
-      - name: Failure short reports
-        if: ${{ always() }}
-        run: cat reports/tests_tf_multi_gpu_failures_short.txt
-
-      - name: Test suite reports artifacts
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: run_all_tests_tf_multi_gpu_test_reports
-          path: reports
-
Author	SHA1	Message	Date
Lysandre	a90468ec8e	import torch	2020-12-11 13:32:08 -05:00
Lysandre	a73721c47d	Workaround braces	2020-12-11 13:29:58 -05:00
Lysandre	312524f6e2	don't waste resources	2020-12-11 13:23:24 -05:00
Lysandre	bc120d02bb	via torch instead of awk	2020-12-11 13:16:47 -05:00
Lysandre	abb4699e45	Install pytorch-scatter with specific cuda version	2020-12-11 13:08:25 -05:00
Lysandre	944e23a941	Remove docs only check	2020-12-11 10:20:56 -05:00