{"record":{"id":"fea73c3dd17e6cf4","repo":"rohitg00/ai-engineering-from-scratch","slug":"neg-log-probs-and-token-counts-must-align","errorCode":null,"errorMessage":"neg_log_probs and token_counts must align","messagePattern":"neg_log_probs and token_counts must align","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"phases/19-capstone-projects/73-perplexity-calibration/code/main.py","lineNumber":37,"sourceCode":"\n\n@dataclass\nclass PerplexityResult:\n    perplexity: float\n    avg_neg_log_likelihood: float\n    total_tokens: int\n\n    def to_dict(self) -> dict:\n        return {\n            \"perplexity\": self.perplexity,\n            \"avg_neg_log_likelihood\": self.avg_neg_log_likelihood,\n            \"total_tokens\": self.total_tokens,\n        }\n\n    @classmethod\n    def from_token_nll(cls, neg_log_probs: Sequence[float], token_counts: Sequence[int]) -> \"PerplexityResult\":\n        if len(neg_log_probs) != len(token_counts):\n            raise ValueError(\"neg_log_probs and token_counts must align\")\n        total_nll = 0.0\n        total_tokens = 0\n        for nll, n in zip(neg_log_probs, token_counts):\n            if nll < 0:\n                raise ValueError(\"neg_log_probs must be non-negative (did you forget the negation?)\")\n            if n < 0:\n                raise ValueError(\"token_counts must be non-negative\")\n            total_nll += float(nll)\n            total_tokens += int(n)\n        if total_tokens == 0:\n            return cls(perplexity=float(\"nan\"), avg_neg_log_likelihood=0.0, total_tokens=0)\n        avg_nll = total_nll / total_tokens\n        return cls(perplexity=math.exp(avg_nll), avg_neg_log_likelihood=avg_nll, total_tokens=total_tokens)\n\n\ndef perplexity(neg_log_probs: Sequence[float], token_counts: Sequence[int]) -> float:\n    return PerplexityResult.from_token_nll(neg_log_probs, token_counts).perplexity\n","sourceCodeStart":19,"sourceCodeEnd":55,"githubUrl":"https://github.com/rohitg00/ai-engineering-from-scratch/blob/39ea8a1c6d0b61f071226eff7ede4d4105fed820/phases/19-capstone-projects/73-perplexity-calibration/code/main.py#L19-L55","documentation":"Error \"neg_log_probs and token_counts must align\" thrown in rohitg00/ai-engineering-from-scratch.","triggerScenarios":"Thrown at phases/19-capstone-projects/73-perplexity-calibration/code/main.py:37 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"39ea8a1c6d0b61f071226eff7ede4d4105fed820","analyzedAt":"2026-08-26T03:13:46.626Z","schemaVersion":2},"datasetVersion":"2026-08-26T07:17:17.940Z"}