{"record":{"id":"78fd7b238fa2bdbf","repo":"tensorflow/models","slug":"the-loss-value-is-loss-value-which-is-larger-th","errorCode":null,"errorMessage":"The loss value is {loss_value}, which is larger than the bound {self.loss_upper_bound}, happens {self.recover_counter} times.","messagePattern":"The loss value is (.+?), which is larger than the bound (.+?), happens (.+?) times\\.","errorType":"exception","errorClass":"RuntimeError","httpStatus":null,"severity":"error","filePath":"official/core/actions.py","lineNumber":171,"sourceCode":"    self.recovery_begin_steps = recovery_begin_steps\n    self.recovery_max_trials = recovery_max_trials\n    self.loss_upper_bound = loss_upper_bound\n    self.global_step = global_step\n\n  def __call__(self, outputs: orbit.runner.Output):\n    loss_value = outputs['training_loss']\n    if tf.math.is_nan(loss_value):\n      self.recover_counter += 1\n      if self.recover_counter > self.recovery_max_trials:\n        raise RuntimeError(\n            'The loss value is NaN after training loop and it happens %d times.'\n            % self.recover_counter)\n      return True\n    if (self.global_step >= self.recovery_begin_steps and\n        loss_value > self.loss_upper_bound):  # pyrefly: ignore[unsupported-operation]\n      self.recover_counter += 1\n      if self.recover_counter > self.recovery_max_trials:\n        raise RuntimeError(\n            f'The loss value is {loss_value}, which is larger than the bound {self.loss_upper_bound}, happens {self.recover_counter} times.'\n        )\n      return True\n    return False\n\n\n@gin.configurable\ndef get_eval_actions(params: config_definitions.ExperimentConfig,\n                     trainer: base_trainer.Trainer,\n                     model_dir: str) -> List[orbit.Action]:\n  \"\"\"Gets eval actions for TFM trainer.\"\"\"\n  eval_actions = []\n  # Adds ema checkpointing action to save the average weights under\n  # ema_checkpoints subdir.\n  if isinstance(trainer.optimizer, optimization.ExponentialMovingAverage):\n    eval_actions.append(\n        EMACheckpointing(\n            export_dir=model_dir,","sourceCodeStart":153,"sourceCodeEnd":189,"githubUrl":"https://github.com/tensorflow/models/blob/e006f5f0d534913e49c1f1dae87364039fa607e2/official/core/actions.py#L153-L189","documentation":"Error \"The loss value is {loss_value}, which is larger than the bound {self.loss_upper_bound}, happens {self.recover_counter} times.\" thrown in tensorflow/models.","triggerScenarios":"Thrown at official/core/actions.py:171 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"e006f5f0d534913e49c1f1dae87364039fa607e2","analyzedAt":"2026-08-24T14:09:15.576Z","schemaVersion":2},"datasetVersion":"2026-08-24T17:17:21.512Z"}