{"record":{"id":"c762ea46c631d6ae","repo":"hiyouga/LlamaFactory","slug":"ppotrainer-does-not-support-eval-dataset-yet","errorCode":null,"errorMessage":"PPOTrainer does not support eval dataset yet.","messagePattern":"PPOTrainer does not support eval dataset yet\\.","errorType":"exception","errorClass":"NotImplementedError","httpStatus":null,"severity":"error","filePath":"src/llamafactory/train/ppo/trainer.py","lineNumber":84,"sourceCode":"\n    def __init__(\n        self,\n        model_args: \"ModelArguments\",\n        training_args: \"Seq2SeqTrainingArguments\",\n        finetuning_args: \"FinetuningArguments\",\n        generating_args: \"GeneratingArguments\",\n        callbacks: Optional[list[\"TrainerCallback\"]],\n        model: \"AutoModelForCausalLMWithValueHead\",\n        reward_model: Optional[\"AutoModelForCausalLMWithValueHead\"],\n        ref_model: Optional[\"AutoModelForCausalLMWithValueHead\"],\n        tokenizer: \"PreTrainedTokenizer\",\n        processor: Optional[\"ProcessorMixin\"],\n        data_collator: \"DataCollatorWithPadding\",\n        train_dataset: Optional[\"Dataset\"] = None,\n        eval_dataset: Optional[\"Dataset\"] = None,\n    ) -> None:\n        if eval_dataset is not None:\n            raise NotImplementedError(\"PPOTrainer does not support eval dataset yet.\")\n\n        # Check if TRL version is compatible (0.8.6 <= version <= 0.9.6)\n        try:\n            from transformers.utils.versions import require_version\n\n            require_version(\n                \"trl>=0.8.6,<=0.9.6\",\n                \"Incompatible TRL version detected. LLaMA-Factory ppo requires TRL version >=0.8.6,<=0.9.6. \"\n                f\"Found version {trl_version}. Please install the correct version with: `pip install trl>=0.8.6,<=0.9.6`\\n\"\n                \"To fix: run `DISABLE_VERSION_CHECK=1 llamafactory-cli train example_ppo.yaml`\\n\",\n            )\n        except ImportError as e:\n            raise e\n\n        backward_batch_size = training_args.per_device_train_batch_size * training_args.gradient_accumulation_steps\n        ppo_config = PPOConfig(\n            model_name=model_args.model_name_or_path,\n            learning_rate=training_args.learning_rate,","sourceCodeStart":66,"sourceCodeEnd":102,"githubUrl":"https://github.com/hiyouga/LlamaFactory/blob/f28afaf6355af515454dfb16c97d728307c93897/src/llamafactory/train/ppo/trainer.py#L66-L102","documentation":"Error \"PPOTrainer does not support eval dataset yet.\" thrown in hiyouga/LlamaFactory.","triggerScenarios":"Thrown at src/llamafactory/train/ppo/trainer.py:84 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":["Remove eval_dataset/val_size from the PPO config; evaluate the model separately using the SFT stage instead."],"exampleFix":"# remove eval_dataset and val_size from the PPO YAML, then run a separate SFT eval job","handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"f28afaf6355af515454dfb16c97d728307c93897","analyzedAt":"2026-08-14T21:57:28.298Z","schemaVersion":2},"datasetVersion":"2026-08-15T22:17:37.221Z"}