{"record":{"id":"8d8de547520595b8","repo":"JuliusBrussee/caveman","slug":"caveman-agent-eval-needs-at-least-one-quality-gra","errorCode":null,"errorMessage":"caveman agent: eval needs at least one quality grader","messagePattern":"caveman agent: eval needs at least one quality grader","errorType":"validation","errorClass":"Error","httpStatus":null,"severity":"error","filePath":"packages/agent/src/primitives.ts","lineNumber":444,"sourceCode":"  readonly approved: boolean;\n  readonly required: boolean;\n  readonly input: unknown;\n  readonly tools: { mode: \"fixture\" | \"live\"; sandbox?: string };\n  readonly quality: readonly QualityGrader[];\n  readonly guardrails: readonly EvalGuardrail[];\n}\n\nexport function evalFixture(options: {\n  id: string;\n  approved?: boolean;\n  required?: boolean;\n  input: unknown;\n  tools?: { mode: \"fixture\" | \"live\"; sandbox?: string };\n  quality: QualityGrader[];\n  guardrails?: EvalGuardrail[];\n}): EvalDefinition {\n  if (options.id.trim() === \"\") throw new Error(\"caveman agent: eval id is required\");\n  if (options.quality.length === 0) throw new Error(\"caveman agent: eval needs at least one quality grader\");\n  const known = new Set([\"contains\", \"tool_called\", \"exact_match\", \"json_schema\"]);\n  for (const grader of options.quality) {\n    if (!known.has(grader.type)) {\n      throw new Error(`caveman agent: unknown grader ${(grader as { type: string }).type}`);\n    }\n  }\n  for (const guardrail of options.guardrails ?? []) {\n    if (guardrail.type === \"latency_threshold\" &&\n        (!Number.isSafeInteger(guardrail.p95_ms) || guardrail.p95_ms <= 0)) {\n      throw new Error(\"caveman agent: latency guardrail requires positive integer p95_ms\");\n    }\n    if (guardrail.type === \"error_rate\" &&\n        (!Number.isFinite(guardrail.max) || guardrail.max < 0 || guardrail.max > 1)) {\n      throw new Error(\"caveman agent: error-rate guardrail max must be in [0,1]\");\n    }\n  }\n  const tools = options.tools ?? { mode: \"fixture\" as const };\n  if (tools.mode === \"live\" && !tools.sandbox) {","sourceCodeStart":426,"sourceCodeEnd":462,"githubUrl":"https://github.com/JuliusBrussee/caveman/blob/766dce6b1394ebb56a3090748d5a0240a5aefb36/packages/agent/src/primitives.ts#L426-L462","documentation":"evalFixture requires at least one quality grader in options.quality; an empty grader list fails this guard right after the id check. Evals must declare how output quality will be judged.","triggerScenarios":"Thrown at packages/agent/src/primitives.ts:444 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":["Add at least one grader (contains, tool_called, exact_match, or json_schema) to quality","Seed a trivial 'contains' grader while authoring, then refine","Skip empty eval entries when generating fixtures from data"],"exampleFix":null,"handlingStrategy":"validation","validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"766dce6b1394ebb56a3090748d5a0240a5aefb36","analyzedAt":"2026-08-18T03:14:35.516Z","contentChangedAt":"2026-08-18T03:14:35.516Z","schemaVersion":2},"datasetVersion":"2026-09-14T00:17:10.932Z"}