diff --git a/examples/run_glue.py b/examples/run_glue.py
index ea5cc9f4..979c6444 100644
--- a/examples/run_glue.py
+++ b/examples/run_glue.py
@@ -166,6 +166,9 @@ def train(args, train_dataset, model, tokenizer):
             train_iterator.close()
             break
 
+    if args.local_rank in [-1, 0]:
+        tb_writer.close()
+
     return global_step, tr_loss / global_step
 
 
@@ -230,9 +233,6 @@ def evaluate(args, model, tokenizer, prefix=""):
                 logger.info("  %s = %s", key, str(result[key]))
                 writer.write("%s = %s\n" % (key, str(result[key])))
 
-    if args.local_rank in [-1, 0]:
-        tb_writer.close()
-
     return results