{"record":{"id":"94a90f3215fc31da","repo":"rohitg00/ai-engineering-from-scratch","slug":"sequence-length-seq-exceeds-context-length-self","errorCode":null,"errorMessage":"sequence length {seq} exceeds context length {self.context_length}","messagePattern":"sequence length (.+?) exceeds context length (.+?)","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"phases/19-capstone-projects/34-transformer-block/code/main.py","lineNumber":90,"sourceCode":"        self.num_heads = cfg.num_heads\n        self.head_dim = cfg.d_model // cfg.num_heads\n        self.context_length = cfg.context_length\n\n        self.qkv = nn.Linear(cfg.d_model, 3 * cfg.d_model, bias=cfg.use_bias)\n        self.out_proj = nn.Linear(cfg.d_model, cfg.d_model, bias=cfg.use_bias)\n        self.attn_dropout = nn.Dropout(cfg.attn_dropout)\n        self.resid_dropout = nn.Dropout(cfg.residual_dropout)\n\n        mask = torch.triu(\n            torch.ones(cfg.context_length, cfg.context_length, dtype=torch.bool),\n            diagonal=1,\n        )\n        self.register_buffer(\"causal_mask\", mask, persistent=False)\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        batch, seq, dim = x.shape\n        if seq > self.context_length:\n            raise ValueError(\n                f\"sequence length {seq} exceeds context length {self.context_length}\"\n            )\n\n        qkv = self.qkv(x)\n        q, k, v = qkv.split(self.d_model, dim=-1)\n\n        q = q.view(batch, seq, self.num_heads, self.head_dim).transpose(1, 2)\n        k = k.view(batch, seq, self.num_heads, self.head_dim).transpose(1, 2)\n        v = v.view(batch, seq, self.num_heads, self.head_dim).transpose(1, 2)\n\n        scores = q @ k.transpose(-2, -1) / math.sqrt(self.head_dim)\n        mask = self.causal_mask[:seq, :seq]\n        scores = scores.masked_fill(mask, float(\"-inf\"))\n        attn = F.softmax(scores, dim=-1)\n        attn = self.attn_dropout(attn)\n\n        out = attn @ v\n        out = out.transpose(1, 2).contiguous().view(batch, seq, dim)","sourceCodeStart":72,"sourceCodeEnd":108,"githubUrl":"https://github.com/rohitg00/ai-engineering-from-scratch/blob/39ea8a1c6d0b61f071226eff7ede4d4105fed820/phases/19-capstone-projects/34-transformer-block/code/main.py#L72-L108","documentation":"Error \"sequence length {seq} exceeds context length {self.context_length}\" thrown in rohitg00/ai-engineering-from-scratch.","triggerScenarios":"Thrown at phases/19-capstone-projects/34-transformer-block/code/main.py:90 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"39ea8a1c6d0b61f071226eff7ede4d4105fed820","analyzedAt":"2026-08-26T03:13:46.626Z","schemaVersion":2},"datasetVersion":"2026-08-26T07:17:17.940Z"}