diff --git a/examples/run_classifier.py b/examples/run_classifier.py
index 112be6fb..49941184 100644
--- a/examples/run_classifier.py
+++ b/examples/run_classifier.py
@@ -859,6 +859,7 @@ def main():
                     optimizer.zero_grad()
                     global_step += 1
 
+    if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
         # Save a trained model, configuration and tokenizer
         model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self
 
diff --git a/examples/run_squad.py b/examples/run_squad.py
index cd85219f..410fd852 100644
--- a/examples/run_squad.py
+++ b/examples/run_squad.py
@@ -1020,7 +1020,7 @@ def main():
                     optimizer.zero_grad()
                     global_step += 1
 
-    if args.do_train:
+    if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
         # Save a trained model, configuration and tokenizer
         model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self