c-my/transformers
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixing param.grad is None in fp16 examples

Browse Source
This commit is contained in:
thomwolf committed 2018-11-20 10:01:21 +01:00
1 parent da73925f6a
commit 2f21497d3e
2 files changed
+4 -2
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

examples/run_classifier.py
+2 -1
View File
@@ -555,7 +555,8 @@ def main():
if args.fp16 and args.loss_scale != 1.0: if args.fp16 and args.loss_scale != 1.0:
# scale down gradients for fp16 training # scale down gradients for fp16 training
for param in model.parameters(): for param in model.parameters():
param.grad.data = param.grad.data / args.loss_scale if param.grad is not None:
param.grad.data = param.grad.data / args.loss_scale
is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True) is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True)
if is_nan: if is_nan:
logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling") logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling")
examples/run_squad.py
+2 -1
View File
@@ -898,7 +898,8 @@ def main():
if args.fp16 and args.loss_scale != 1.0: if args.fp16 and args.loss_scale != 1.0:
# scale down gradients for fp16 training # scale down gradients for fp16 training
for param in model.parameters(): for param in model.parameters():
param.grad.data = param.grad.data / args.loss_scale if param.grad is not None:
param.grad.data = param.grad.data / args.loss_scale
is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True) is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True)
if is_nan: if is_nan:
logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling") logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling")