diff --git a/examples/run_glue.py b/examples/run_glue.py
index a2167ed8..45924c92 100644
--- a/examples/run_glue.py
+++ b/examples/run_glue.py
@@ -494,7 +494,7 @@ def main():
 
 
     # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
-    if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
+    if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and not args.tpu:
         # Create output directory if needed
         if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
             os.makedirs(args.output_dir)