{"record":{"id":"b01f171b717bdf53","repo":"tensorflow/models","slug":"total-loss-is-nan","errorCode":null,"errorMessage":"total loss is NaN.","messagePattern":"total loss is NaN\\.","errorType":"exception","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"official/legacy/detection/executor/distributed_executor.py","lineNumber":500,"sourceCode":"\n    logging.info('Training started')\n    last_save_checkpoint_step = current_step\n    while current_step < total_steps:\n\n      num_steps = _steps_to_run(current_step, total_steps, iterations_per_loop)\n      _run_callbacks_on_batch_begin(current_step)\n      train_loss = train_step(train_iterator,\n                              tf.convert_to_tensor(num_steps, dtype=tf.int32))\n      current_step += num_steps\n\n      train_loss = tf.nest.map_structure(lambda x: x.numpy().astype(float),\n                                         train_loss)\n\n      _run_callbacks_on_batch_end(current_step - 1)\n      if not isinstance(train_loss, dict):\n        train_loss = {'total_loss': train_loss}\n      if np.isnan(train_loss['total_loss']):\n        raise ValueError('total loss is NaN.')\n\n      if train_metric:\n        train_metric_result = metric_results(train_metric)\n        train_metric_result.update(train_loss)\n      else:\n        train_metric_result = train_loss\n      if callable(optimizer.lr):\n        train_metric_result.update(\n            {'learning_rate': optimizer.lr(current_step).numpy()})  # pyrefly: ignore[missing-attribute]\n      else:\n        train_metric_result.update({'learning_rate': optimizer.lr.numpy()})\n      logging.info('Train Step: %d/%d  / loss = %s / training metric = %s',\n                   current_step, total_steps, train_loss, train_metric_result)\n\n      train_summary_writer(\n          metrics=train_metric_result, step=optimizer.iterations)\n\n      # Saves model checkpoints and run validation steps at every","sourceCodeStart":482,"sourceCodeEnd":518,"githubUrl":"https://github.com/tensorflow/models/blob/e006f5f0d534913e49c1f1dae87364039fa607e2/official/legacy/detection/executor/distributed_executor.py#L482-L518","documentation":"Error \"total loss is NaN.\" thrown in tensorflow/models.","triggerScenarios":"Thrown at official/legacy/detection/executor/distributed_executor.py:500 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"e006f5f0d534913e49c1f1dae87364039fa607e2","analyzedAt":"2026-08-24T14:09:15.576Z","schemaVersion":2},"datasetVersion":"2026-08-24T17:17:21.512Z"}