diff --git a/examples/run_bert_squad.py b/examples/run_bert_squad.py
index b35a9175..9aaa711c 100644
--- a/examples/run_bert_squad.py
+++ b/examples/run_bert_squad.py
@@ -313,7 +313,8 @@ def main():
                     optimizer.zero_grad()
                     global_step += 1
                     if args.local_rank in [-1, 0]:
-                        tb_writer.add_scalar('lr', optimizer.get_lr()[0], global_step)
+                        if not args.fp16:
+                            tb_writer.add_scalar('lr', optimizer.get_lr()[0], global_step)
                         tb_writer.add_scalar('loss', loss.item(), global_step)
 
     if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
diff --git a/examples/run_xlnet_classifier.py b/examples/run_xlnet_classifier.py
index 0278b40c..23098159 100644
--- a/examples/run_xlnet_classifier.py
+++ b/examples/run_xlnet_classifier.py
@@ -319,7 +319,8 @@ def main():
                     optimizer.zero_grad()
                     global_step += 1
                     if args.local_rank in [-1, 0] and (args.log_every <= 0 or (step + 1) % args.log_every == 0):
-                        tb_writer.add_scalar('lr', optimizer.get_lr()[0], global_step)
+                        if not args.fp16:
+                            tb_writer.add_scalar('lr', optimizer.get_lr()[0], global_step)
                         tb_writer.add_scalar('loss', loss.item(), global_step)
 
     ### Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
diff --git a/examples/run_xlnet_squad.py b/examples/run_xlnet_squad.py
index a72d648f..927668c5 100644
--- a/examples/run_xlnet_squad.py
+++ b/examples/run_xlnet_squad.py
@@ -313,7 +313,8 @@ def main():
                     optimizer.zero_grad()
                     global_step += 1
                     if args.local_rank in [-1, 0]:
-                        tb_writer.add_scalar('lr', optimizer.get_lr()[0], global_step)
+                        if not args.fp16:
+                            tb_writer.add_scalar('lr', optimizer.get_lr()[0], global_step)
                         tb_writer.add_scalar('loss', loss.item(), global_step)
 
     if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):