{"record":{"id":"2500ce2612671bc3","repo":"headroomlabs-ai/headroom","slug":"no-valid-samples-processed","errorCode":null,"errorMessage":"No valid samples processed","messagePattern":"No valid samples processed","errorType":"exception","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"headroom/evals/html_oss_benchmarks.py","lineNumber":376,"sourceCode":"        em_original.append(1.0 if em_orig else 0.0)\n        em_extracted.append(1.0 if em_ext else 0.0)\n\n        question_results.append(\n            {\n                \"question\": question,\n                \"ground_truth\": ground_truth,\n                \"answer_original\": answer_original,\n                \"answer_extracted\": answer_extracted,\n                \"f1_original\": f1_orig,\n                \"f1_extracted\": f1_ext,\n            }\n        )\n\n        if (i + 1) % 10 == 0:\n            logger.info(f\"  Processed {i + 1}/{len(samples)} questions\")\n\n    if not f1_original:\n        raise ValueError(\"No valid samples processed\")\n\n    avg_f1_orig = sum(f1_original) / len(f1_original)\n    avg_f1_ext = sum(f1_extracted) / len(f1_extracted)\n    avg_em_orig = sum(em_original) / len(em_original)\n    avg_em_ext = sum(em_extracted) / len(em_extracted)\n\n    # Accuracy is preserved if extracted is within 2% of original\n    accuracy_preserved = avg_f1_ext >= avg_f1_orig - 0.02\n\n    return QAAccuracyResult(\n        total_questions=len(f1_original),\n        accuracy_original_html=avg_f1_orig,\n        accuracy_extracted=avg_f1_ext,\n        accuracy_preserved=accuracy_preserved,\n        avg_f1_original=avg_f1_orig,\n        avg_f1_extracted=avg_f1_ext,\n        exact_match_original=avg_em_orig,\n        exact_match_extracted=avg_em_ext,","sourceCodeStart":358,"sourceCodeEnd":394,"githubUrl":"https://github.com/headroomlabs-ai/headroom/blob/322425c43bffde1ed0b64fecf3cf5951565dd82b/headroom/evals/html_oss_benchmarks.py#L358-L394","documentation":"Error \"No valid samples processed\" thrown in headroomlabs-ai/headroom.","triggerScenarios":"Raised at the end of an HTML OSS benchmark run when every sample failed processing or was filtered out, leaving nothing to score.","commonSituations":"See trigger scenarios.","solutions":["Verify the input dataset actually contains samples matching the benchmark filters","Check earlier log lines for per-sample errors that caused every sample to be skipped","Relax sample filters (min/max length, required fields) if they exclude everything","Verify the LLM/API used for processing is reachable so samples are not all failing"],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"322425c43bffde1ed0b64fecf3cf5951565dd82b","analyzedAt":"2026-08-15T01:03:05.481Z","schemaVersion":2},"datasetVersion":"2026-08-15T22:17:37.221Z"}