diff --git a/.circleci/config.yml b/.circleci/config.yml
index 1dbb5374..a1ab5e2f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -150,7 +150,7 @@ jobs:
             - skip-job-on-doc-only-changes
             - restore_cache:
                   keys:
-                      - v0.4-torch-{{ checksum "setup.py" }}
+                      - v0.4-torch-1-3-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
             - run: pip install .[sklearn,testing,sentencepiece]
@@ -172,7 +172,7 @@ jobs:
             - skip-job-on-doc-only-changes
             - restore_cache:
                   keys:
-                      - v0.4-torch-{{ checksum "setup.py" }}
+                      - v0.4-torch-1-4-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
             - run: pip install .[sklearn,testing,sentencepiece]
@@ -187,6 +187,50 @@ jobs:
             - store_artifacts:
                   path: ~/transformers/reports
 
+    run_tests_torch-1-5:
+        <<: *setup
+        steps:
+            - checkout
+            - skip-job-on-doc-only-changes
+            - restore_cache:
+                  keys:
+                      - v0.4-torch-1-5-{{ checksum "setup.py" }}
+                      - v0.4-{{ checksum "setup.py" }}
+            - run: pip install --upgrade pip
+            - run: pip install .[sklearn,testing,sentencepiece]
+            - run: pip install torch==1.5
+            - save_cache:
+                  key: v0.4-torch-1-5-{{ checksum "setup.py" }}
+                  paths:
+                      - '~/.cache/pip'
+            - run: python -m pytest -n 8 --dist=loadfile -s --make-reports=tests_torch ./tests/ | tee tests_output.txt
+            - store_artifacts:
+                  path: ~/transformers/tests_output.txt
+            - store_artifacts:
+                  path: ~/transformers/reports
+
+    run_tests_torch-1-6:
+        <<: *setup
+        steps:
+            - checkout
+            - skip-job-on-doc-only-changes
+            - restore_cache:
+                  keys:
+                      - v0.4-torch-1-6-{{ checksum "setup.py" }}
+                      - v0.4-{{ checksum "setup.py" }}
+            - run: pip install --upgrade pip
+            - run: pip install .[sklearn,testing,sentencepiece]
+            - run: pip install torch==1.6
+            - save_cache:
+                  key: v0.4-torch-1-6-{{ checksum "setup.py" }}
+                  paths:
+                      - '~/.cache/pip'
+            - run: python -m pytest -n 8 --dist=loadfile -s --make-reports=tests_torch ./tests/ | tee tests_output.txt
+            - store_artifacts:
+                  path: ~/transformers/tests_output.txt
+            - store_artifacts:
+                  path: ~/transformers/reports
+
     run_tests_tf:
         <<: *setup
         steps:
@@ -421,6 +465,10 @@ workflows:
             - run_tests_custom_tokenizers
             - run_tests_torch_and_tf
             - run_tests_torch
+            - run_tests_torch-1-3
+            - run_tests_torch-1-4
+            - run_tests_torch-1-5
+            - run_tests_torch-1-6
             - run_tests_tf
             - run_tests_flax
             - run_tests_pipelines_torch