{"record":{"id":"88adc2499d64c870","repo":"huggingface/transformers","slug":"deepspeed-failed-to-resume-from-checkpoint-chec","errorCode":null,"errorMessage":"[deepspeed] failed to resume from checkpoint {checkpoint_path}","messagePattern":"\\[deepspeed\\] failed to resume from checkpoint (.+?)","errorType":"exception","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"src/transformers/integrations/deepspeed.py","lineNumber":669,"sourceCode":"    # it's possible that the user is trying to resume from model_path, which doesn't necessarily\n    # contain a deepspeed checkpoint. e.g. examples just check if the dir exists and assume it's\n    # a resume from a checkpoint and not just a local pretrained weight. So we check here if the\n    # path contains what looks like a deepspeed checkpoint\n    import glob\n\n    deepspeed_checkpoint_dirs = sorted(glob.glob(f\"{checkpoint_path}/global_step*\"))\n\n    if len(deepspeed_checkpoint_dirs) > 0:\n        logger.info(f\"Attempting to resume from {checkpoint_path}\")\n        # this magically updates self.optimizer and self.lr_scheduler\n        load_path, _ = deepspeed_engine.load_checkpoint(\n            checkpoint_path,\n            load_module_strict=load_module_strict,\n            load_optimizer_states=True,\n            load_lr_scheduler_states=True,\n        )\n        if load_path is None:\n            raise ValueError(f\"[deepspeed] failed to resume from checkpoint {checkpoint_path}\")\n    else:\n        raise ValueError(f\"Can't find a valid checkpoint at {checkpoint_path}\")\n\n\ndef propagate_args_to_deepspeed(accelerator, args, auto_find_batch_size=False):\n    \"\"\"\n    Sets values in the deepspeed plugin based on the TrainingArguments.\n\n    Args:\n        accelerator (`Accelerator`): The Accelerator object.\n        args (`TrainingArguments`): The training arguments to propagate to DeepSpeed config.\n        auto_find_batch_size (`bool`, *optional*, defaults to `False`):\n            Whether batch size was auto-discovered by trying increasingly smaller sizes.\n    \"\"\"\n    ds_plugin = accelerator.state.deepspeed_plugin\n\n    ds_plugin.hf_ds_config = HfTrainerDeepSpeedConfig(ds_plugin.hf_ds_config.config)\n    ds_plugin.deepspeed_config = ds_plugin.hf_ds_config.config","sourceCodeStart":651,"sourceCodeEnd":687,"githubUrl":"https://github.com/huggingface/transformers/blob/a597f974857b3d92939971296bc0deb93d33d780/src/transformers/integrations/deepspeed.py#L651-L687","documentation":"Error \"[deepspeed] failed to resume from checkpoint {checkpoint_path}\" thrown in huggingface/transformers.","triggerScenarios":"Raised when DeepSpeed fails to resume training from the given checkpoint path.","commonSituations":"Corrupted, incomplete, or mismatched DeepSpeed checkpoint passed via resume_from_checkpoint.","solutions":["Verify the checkpoint path contains a valid DeepSpeed checkpoint.","Re-run saving to produce a complete checkpoint before resuming."],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"a597f974857b3d92939971296bc0deb93d33d780","analyzedAt":"2026-08-14T18:24:08.354Z","schemaVersion":2},"datasetVersion":"2026-08-15T17:31:12.345Z"}