{"record":{"id":"357816e4866ff899","repo":"rohitg00/ai-engineering-from-scratch","slug":"hidden-must-divide-heads-357816","errorCode":null,"errorMessage":"hidden must divide heads","messagePattern":"hidden must divide heads","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"phases/19-capstone-projects/41-eval-pipeline/code/main.py","lineNumber":77,"sourceCode":"        if len(ids) > max_len:\n            ids = ids[:max_len]\n        return ids\n\n    def encode_text(self, text: str, max_len: int) -> List[int]:\n        ids = list(text.encode(\"utf-8\", errors=\"ignore\"))\n        if len(ids) > max_len:\n            ids = ids[:max_len]\n        return ids\n\n    def decode_response(self, ids: Sequence[int]) -> str:\n        return bytes(i for i in ids if i < 256).decode(\"utf-8\", errors=\"replace\")\n\n\nclass CausalSelfAttention(nn.Module):\n    def __init__(self, hidden: int, heads: int, max_len: int):\n        super().__init__()\n        if hidden % heads != 0:\n            raise ValueError(\"hidden must divide heads\")\n        self.heads = heads\n        self.head_dim = hidden // heads\n        self.qkv = nn.Linear(hidden, hidden * 3, bias=False)\n        self.out = nn.Linear(hidden, hidden, bias=False)\n        mask = torch.tril(torch.ones(max_len, max_len, dtype=torch.bool))\n        self.register_buffer(\"causal_mask\", mask, persistent=False)\n\n    def forward(self, x: torch.Tensor, key_pad_mask: Optional[torch.Tensor] = None) -> torch.Tensor:\n        B, T, D = x.shape\n        qkv = self.qkv(x).view(B, T, 3, self.heads, self.head_dim).permute(2, 0, 3, 1, 4)\n        q, k, v = qkv[0], qkv[1], qkv[2]\n        att = (q @ k.transpose(-2, -1)) / math.sqrt(self.head_dim)\n        causal = self.causal_mask[:T, :T].view(1, 1, T, T)\n        att = att.masked_fill(~causal, float(\"-inf\"))\n        if key_pad_mask is not None:\n            km = key_pad_mask.view(B, 1, 1, T).to(torch.bool)\n            att = att.masked_fill(~km, float(\"-inf\"))\n        weights = F.softmax(att, dim=-1)","sourceCodeStart":59,"sourceCodeEnd":95,"githubUrl":"https://github.com/rohitg00/ai-engineering-from-scratch/blob/39ea8a1c6d0b61f071226eff7ede4d4105fed820/phases/19-capstone-projects/41-eval-pipeline/code/main.py#L59-L95","documentation":"Error \"hidden must divide heads\" thrown in rohitg00/ai-engineering-from-scratch.","triggerScenarios":"Thrown at phases/19-capstone-projects/41-eval-pipeline/code/main.py:77 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"39ea8a1c6d0b61f071226eff7ede4d4105fed820","analyzedAt":"2026-08-26T03:13:46.626Z","schemaVersion":2},"datasetVersion":"2026-08-26T07:17:17.940Z"}