{"record":{"id":"0d9bf64053416e5c","repo":"tensorflow/models","slug":"the-loss-value-is-nan-after-training-loop-and-it-h","errorCode":null,"errorMessage":"The loss value is NaN after training loop and it happens %d times.","messagePattern":"The loss value is NaN after training loop and it happens (.+?) times\\.","errorType":"exception","errorClass":"RuntimeError","httpStatus":null,"severity":"error","filePath":"official/core/actions.py","lineNumber":163,"sourceCode":"  \"\"\"Recovery Condition.\"\"\"\n\n  def __init__(self,\n               global_step: tf.Variable,\n               loss_upper_bound: float,\n               recovery_begin_steps: int = 0,\n               recovery_max_trials: int = 3):\n    self.recover_counter = 0\n    self.recovery_begin_steps = recovery_begin_steps\n    self.recovery_max_trials = recovery_max_trials\n    self.loss_upper_bound = loss_upper_bound\n    self.global_step = global_step\n\n  def __call__(self, outputs: orbit.runner.Output):\n    loss_value = outputs['training_loss']\n    if tf.math.is_nan(loss_value):\n      self.recover_counter += 1\n      if self.recover_counter > self.recovery_max_trials:\n        raise RuntimeError(\n            'The loss value is NaN after training loop and it happens %d times.'\n            % self.recover_counter)\n      return True\n    if (self.global_step >= self.recovery_begin_steps and\n        loss_value > self.loss_upper_bound):  # pyrefly: ignore[unsupported-operation]\n      self.recover_counter += 1\n      if self.recover_counter > self.recovery_max_trials:\n        raise RuntimeError(\n            f'The loss value is {loss_value}, which is larger than the bound {self.loss_upper_bound}, happens {self.recover_counter} times.'\n        )\n      return True\n    return False\n\n\n@gin.configurable\ndef get_eval_actions(params: config_definitions.ExperimentConfig,\n                     trainer: base_trainer.Trainer,\n                     model_dir: str) -> List[orbit.Action]:","sourceCodeStart":145,"sourceCodeEnd":181,"githubUrl":"https://github.com/tensorflow/models/blob/e006f5f0d534913e49c1f1dae87364039fa607e2/official/core/actions.py#L145-L181","documentation":"Error \"The loss value is NaN after training loop and it happens %d times.\" thrown in tensorflow/models.","triggerScenarios":"Thrown at official/core/actions.py:163 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"e006f5f0d534913e49c1f1dae87364039fa607e2","analyzedAt":"2026-08-24T14:09:15.576Z","schemaVersion":2},"datasetVersion":"2026-08-24T17:17:21.512Z"}