{"record":{"id":"a8c55cdff7e42cb3","repo":"JuliusBrussee/caveman","slug":"runner-did-not-return-input-output-token-usage","errorCode":null,"errorMessage":"runner did not return input/output token usage","messagePattern":"runner did not return input/output token usage","errorType":"validation","errorClass":null,"httpStatus":null,"severity":"error","filePath":"engine/evals/quality.go","lineNumber":218,"sourceCode":"\tif err != nil {\n\t\treturn QualityReport{}, fmt.Errorf(\"compressed grading: %w\", err)\n\t}\n\treport.BaselinePassed = len(report.BaselineFailures) == 0\n\treport.CompressedPassed = len(report.CompressedFailures) == 0\n\tif report.BaselinePassed && report.CompressedPassed {\n\t\treport.Retention = 1\n\t}\n\treturn report, nil\n}\n\nfunc fillUsage(counter tokens.Counter, prompt []byte, resp ModelResponse, requireRunnerUsage bool) (ModelUsage, error) {\n\tusage := resp.Usage\n\tif usage.InputTokens > 0 && usage.OutputTokens > 0 {\n\t\tusage.Source = \"runner\"\n\t\treturn usage, nil\n\t}\n\tif requireRunnerUsage {\n\t\treturn ModelUsage{}, fmt.Errorf(\"runner did not return input/output token usage\")\n\t}\n\tif usage.InputTokens == 0 {\n\t\tusage.InputTokens = counter.Count(prompt)\n\t}\n\tif usage.OutputTokens == 0 {\n\t\tusage.OutputTokens = counter.Count(resp.Output)\n\t}\n\tusage.Source = \"local_tokenizer\"\n\treturn usage, nil\n}\n\nfunc gradeFailures(ctx context.Context, runner AnswerGrader, graders []Grader, subject Subject) ([]string, error) {\n\tvar failures []string\n\tfor _, g := range graders {\n\t\tv, err := runner.Grade(ctx, g, subject)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}","sourceCodeStart":200,"sourceCodeEnd":236,"githubUrl":"https://github.com/JuliusBrussee/caveman/blob/27d5a3981a347890211bb1bf2439e5c821a63bc9/engine/evals/quality.go#L200-L236","documentation":"Error \"runner did not return input/output token usage\" thrown in JuliusBrussee/caveman.","triggerScenarios":"Thrown at engine/evals/quality.go:218 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":["Return input/output token usage from the runner."],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"27d5a3981a347890211bb1bf2439e5c821a63bc9","analyzedAt":"2026-08-15T09:26:11.751Z","schemaVersion":2},"datasetVersion":"2026-08-15T17:31:12.345Z"}