{"record":{"id":"32d3f1726559ca4c","repo":"hiyouga/LlamaFactory","slug":"evaluation-will-be-deprecated-in-the-future","errorCode":null,"errorMessage":"Evaluation will be deprecated in the future.","messagePattern":"Evaluation will be deprecated in the future\\.","errorType":"exception","errorClass":"NotImplementedError","httpStatus":null,"severity":"warning","filePath":"src/llamafactory/launcher.py","lineNumber":147,"sourceCode":"                .split(),\n                env=env,\n                check=True,\n            )\n\n        sys.exit(process.returncode)\n\n    elif command == \"api\":\n        from .api.app import run_api\n\n        run_api()\n\n    elif command == \"chat\":\n        from .chat.chat_model import run_chat\n\n        run_chat()\n\n    elif command == \"eval\":\n        raise NotImplementedError(\"Evaluation will be deprecated in the future.\")\n\n    elif command == \"export\":\n        from .train.tuner import export_model\n\n        export_model()\n\n    elif command == \"train\":\n        from .train.tuner import run_exp\n\n        run_exp()\n\n    elif command == \"webchat\":\n        from .webui.interface import run_web_demo\n\n        run_web_demo()\n\n    elif command == \"webui\":\n        from .webui.interface import run_web_ui","sourceCodeStart":129,"sourceCodeEnd":165,"githubUrl":"https://github.com/hiyouga/LlamaFactory/blob/f28afaf6355af515454dfb16c97d728307c93897/src/llamafactory/launcher.py#L129-L165","documentation":"Raised by the launcher for the 'eval' subcommand as a deliberate NotImplementedError: the standalone evaluation flow is being deprecated in LlamaFactory and the command intentionally does nothing. It is a planned-removal signal, not a runtime failure.","triggerScenarios":"Executing llamafactory-cli eval with any arguments.","commonSituations":"Following outdated tutorials or docs that reference the eval subcommand; scripts written against older LlamaFactory versions.","solutions":["Remove the eval step from your workflow; use the API/chat path or an external benchmark harness (e.g. lm-evaluation-harness) for evaluation.","Pin/consult a newer LlamaFactory version's docs for the supported evaluation alternative."],"exampleFix":"# before\nllamafactory-cli eval eval_config.yaml\n\n# after\n# use an external evaluator, e.g.\nlm_eval --model hf --model_args pretrained=...,dtype=bfloat16 --tasks mmlu","handlingStrategy":"validation","validationCode":"if command == \"eval\":\n    raise SystemExit(\"'eval' is deprecated in LlamaFactory; use the API/chat path or lm-evaluation-harness\")","typeGuard":null,"tryCatchPattern":null,"preventionTips":["Audit launch scripts for the eval subcommand when upgrading LlamaFactory.","Prefer external benchmark harnesses for reproducible evaluation."],"tags":["deprecated","eval","launcher","not-implemented"],"backgroundTag":null,"analyzedSha":"f28afaf6355af515454dfb16c97d728307c93897","analyzedAt":"2026-08-14T21:57:28.298Z","schemaVersion":2},"datasetVersion":"2026-08-15T22:17:37.221Z"}