{"record":{"id":"59cd552cdb409814","repo":"rohitg00/ai-engineering-from-scratch","slug":"fixture-validation-failed-errors","errorCode":null,"errorMessage":"fixture validation failed: {errors}","messagePattern":"fixture validation failed: (.+?)","errorType":"exception","errorClass":"RuntimeError","httpStatus":null,"severity":"error","filePath":"phases/19-capstone-projects/75-end-to-end-eval-runner/code/main.py","lineNumber":345,"sourceCode":"            if task.post_process == \"extract_letter\":\n                text = \"Answer: B\"\n            if task.metric_name == \"code_exec\":\n                text = \"```python\\ndef add(a, b):\\n    return None\\n```\"\n            confidence = 0.4\n            nll_per_token = 3.0\n        token_count = max(1, len(text.split()))\n        nll = token_count * nll_per_token\n        self.token_stats.append((nll, token_count))\n        return Generation(text=text, confidence=confidence, token_nll=nll, token_count=token_count)\n\n\ndef _load_fixture_tasks() -> list:\n    import tempfile\n    out_dir = tempfile.mkdtemp(prefix=\"aie_l75_\")\n    good, _bad = spec_mod.load_fixtures(out_dir)\n    tasks, errors = spec_mod.validate_file(good)\n    if errors:\n        raise RuntimeError(f\"fixture validation failed: {errors}\")\n    return tasks\n\n\ndef demo() -> int:\n    tasks = _load_fixture_tasks()\n    adapters = [RuleBasedAdapter(), NoisyAdapter(seed=1), BiasedAdapter(good_category=\"arithmetic\")]\n    t0 = time.time()\n    results, calibration_buf = run_eval(adapters, tasks, parallel=True, max_workers=6, code_exec_timeout_s=2.0)\n    wall = time.time() - t0\n\n    adapter_token_stats = {\n        a.model_id: list(getattr(a, \"token_stats\", [])) for a in adapters\n    }\n    report = render_report(adapters, tasks, results, calibration_buf, adapter_token_stats, wall)\n\n    print(render_markdown_block(report))\n    print()\n    print(f\"summary: {json.dumps(report.summary, indent=2)}\")","sourceCodeStart":327,"sourceCodeEnd":363,"githubUrl":"https://github.com/rohitg00/ai-engineering-from-scratch/blob/39ea8a1c6d0b61f071226eff7ede4d4105fed820/phases/19-capstone-projects/75-end-to-end-eval-runner/code/main.py#L327-L363","documentation":"Error \"fixture validation failed: {errors}\" thrown in rohitg00/ai-engineering-from-scratch.","triggerScenarios":"Thrown at phases/19-capstone-projects/75-end-to-end-eval-runner/code/main.py:345 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"39ea8a1c6d0b61f071226eff7ede4d4105fed820","analyzedAt":"2026-08-26T03:13:46.626Z","schemaVersion":2},"datasetVersion":"2026-08-26T07:17:17.940Z"}