{"record":{"id":"b549ffafb60daa40","repo":"rohitg00/ai-engineering-from-scratch","slug":"d-model-cfg-d-model-must-be-divisible-by-num-h","errorCode":null,"errorMessage":"d_model ({cfg.d_model}) must be divisible by num_heads ({cfg.num_heads})","messagePattern":"d_model \\((.+?)\\) must be divisible by num_heads \\((.+?)\\)","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"phases/19-capstone-projects/34-transformer-block/code/main.py","lineNumber":68,"sourceCode":"\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        mean = x.mean(dim=-1, keepdim=True)\n        var = x.var(dim=-1, keepdim=True, unbiased=False)\n        return self.scale * (x - mean) / torch.sqrt(var + self.eps) + self.shift\n\n\nclass MultiHeadAttention(nn.Module):\n    \"\"\"Multi head causal self attention with a fused QKV projection.\n\n    Fused QKV: one linear of width 3 * d_model instead of three linears, one\n    kernel launch, one matmul. The causal mask is registered as a buffer so it\n    is allocated once at construction and sliced per forward.\n    \"\"\"\n\n    def __init__(self, cfg: BlockConfig) -> None:\n        super().__init__()\n        if cfg.d_model % cfg.num_heads != 0:\n            raise ValueError(\n                f\"d_model ({cfg.d_model}) must be divisible by num_heads ({cfg.num_heads})\"\n            )\n        self.d_model = cfg.d_model\n        self.num_heads = cfg.num_heads\n        self.head_dim = cfg.d_model // cfg.num_heads\n        self.context_length = cfg.context_length\n\n        self.qkv = nn.Linear(cfg.d_model, 3 * cfg.d_model, bias=cfg.use_bias)\n        self.out_proj = nn.Linear(cfg.d_model, cfg.d_model, bias=cfg.use_bias)\n        self.attn_dropout = nn.Dropout(cfg.attn_dropout)\n        self.resid_dropout = nn.Dropout(cfg.residual_dropout)\n\n        mask = torch.triu(\n            torch.ones(cfg.context_length, cfg.context_length, dtype=torch.bool),\n            diagonal=1,\n        )\n        self.register_buffer(\"causal_mask\", mask, persistent=False)\n","sourceCodeStart":50,"sourceCodeEnd":86,"githubUrl":"https://github.com/rohitg00/ai-engineering-from-scratch/blob/39ea8a1c6d0b61f071226eff7ede4d4105fed820/phases/19-capstone-projects/34-transformer-block/code/main.py#L50-L86","documentation":"Error \"d_model ({cfg.d_model}) must be divisible by num_heads ({cfg.num_heads})\" thrown in rohitg00/ai-engineering-from-scratch.","triggerScenarios":"Thrown at phases/19-capstone-projects/34-transformer-block/code/main.py:68 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"39ea8a1c6d0b61f071226eff7ede4d4105fed820","analyzedAt":"2026-08-26T03:13:46.626Z","schemaVersion":2},"datasetVersion":"2026-08-26T07:17:17.940Z"}