{"record":{"id":"bd8c1ce745f989b3","repo":"JuliusBrussee/caveman","slug":"where-replay-trial-proof-id-replays-a-case-o","errorCode":null,"errorMessage":"${where} replay trial ${proof.id} replays a case outside the composed dataset","messagePattern":"(.+?) replay trial (.+?) replays a case outside the composed dataset","errorType":"exception","errorClass":"Error","httpStatus":null,"severity":"error","filePath":"packages/shared/contracts/scripts/validate-continuous-improvement.mjs","lineNumber":323,"sourceCode":"    if (roleCounts.boundary > 1) throw new Error(`${where} dataset carries ${roleCounts.boundary} boundary cases, want at most one`);\n    if (roleCounts.adversarial > (confidenceGuard ? 2 : 1)) throw new Error(`${where} dataset carries ${roleCounts.adversarial} adversarial cases`);\n    if (roleCounts.boundary !== dataset.boundary_cases.length) throw new Error(`${where} boundary case list disagrees with the composed dataset`);\n    if (roleCounts.adversarial !== dataset.generated_cases.length) throw new Error(`${where} generated case list disagrees with the composed dataset`);\n    const composed = roleCounts.target_failure + roleCounts.prior_success + roleCounts.boundary + roleCounts.adversarial;\n    if (composed !== dataset.cases.length) throw new Error(`${where} dataset roles (${composed}) do not account for its ${dataset.cases.length} cases`);\n    if (dataset.replay_case_ids.length !== datasetCaseIDs.size) throw new Error(`${where} replay manifest carries ${dataset.replay_case_ids.length} ids for ${datasetCaseIDs.size} dataset cases`);\n    for (const datasetCaseID of dataset.replay_case_ids) {\n      if (!datasetCaseIDs.has(datasetCaseID)) throw new Error(`${where} replay manifest case ${datasetCaseID} is not a composed dataset case`);\n    }\n\n    // The guard grader exists exactly when there is a perturbed case to catch a\n    // candidate on; a required grader with no case behind it is decoration.\n    const perturbed = dataset.cases.some((entry) => entry.perturbation !== \"none\");\n    if (perturbed !== item.eval_pack.graders.includes(\"guard_respected\")) {\n      throw new Error(`${where} guard_respected grader ${perturbed ? \"missing for\" : \"declared without\"} perturbed dataset cases`);\n    }\n    for (const proof of item.replay.trial_proofs) {\n      if (!datasetCaseIDs.has(proof.dataset_case_id)) throw new Error(`${where} replay trial ${proof.id} replays a case outside the composed dataset`);\n      for (const arm of [proof.baseline, proof.candidate]) {\n        const graders = arm.grader_results.map((grader) => grader.grader).sort();\n        const required = [...item.eval_pack.graders].sort();\n        if (graders.join(\",\") !== required.join(\",\")) {\n          throw new Error(`${where} replay trial ${proof.id} graded ${graders.join(\",\")}, want exactly ${required.join(\",\")}`);\n        }\n      }\n    }\n  }\n}\nif (!Array.isArray(spans) || spans.length === 0) throw new Error(\"canonical span fixture is empty\");\nfor (const [index, span] of spans.entries()) {\n  if (!validateSpan(span)) throw new Error(`canonical span ${index}: ${ajv.errorsText(validateSpan.errors)}`);\n}\nif (relationshipCount === 0) throw new Error(\"no theme relationship reached the shared-analysis-unit threshold in any fixture\");\nif (motifCount === 0) throw new Error(\"no workflow motif recurred across two variants of one task family in any fixture\");\nconsole.log(`validated ${reports.length} generated improvement report(s), ${relationshipCount} theme relationship(s), ${motifCount} workflow motif(s) and ${spans.length} canonical spans`);\n","sourceCodeStart":305,"sourceCodeEnd":341,"githubUrl":"https://github.com/JuliusBrussee/caveman/blob/27d5a3981a347890211bb1bf2439e5c821a63bc9/packages/shared/contracts/scripts/validate-continuous-improvement.mjs#L305-L341","documentation":"Error \"${where} replay trial ${proof.id} replays a case outside the composed dataset\" thrown in JuliusBrussee/caveman.","triggerScenarios":"Thrown at packages/shared/contracts/scripts/validate-continuous-improvement.mjs:323 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":["Replay only cases that belong to the composed dataset."],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"27d5a3981a347890211bb1bf2439e5c821a63bc9","analyzedAt":"2026-08-15T09:26:11.751Z","schemaVersion":2},"datasetVersion":"2026-08-15T22:17:37.221Z"}