{"record":{"id":"d49fc7f68d27da99","repo":"rohitg00/ai-engineering-from-scratch","slug":"token-ids-too-short-for-the-requested-context-leng","errorCode":null,"errorMessage":"token_ids too short for the requested context_length","messagePattern":"token_ids too short for the requested context_length","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"phases/19-capstone-projects/36-training-loop-eval/code/main.py","lineNumber":195,"sourceCode":"            x = block(x)\n        return self.lm_head(self.final_ln(x))\n\n\ndef make_batches(\n    token_ids: torch.Tensor,\n    batch_size: int,\n    context_length: int,\n    seed: int = 0,\n) -> Iterator[tuple[torch.Tensor, torch.Tensor]]:\n    \"\"\"Yield (input, target) batches where target is input shifted by one position.\n\n    Sampling is uniform random over valid start positions. With a fixed seed the\n    sequence of batches is reproducible across runs.\n    \"\"\"\n    if token_ids.dim() != 1:\n        raise ValueError(\"token_ids must be a 1D tensor\")\n    if token_ids.numel() < context_length + 1:\n        raise ValueError(\"token_ids too short for the requested context_length\")\n\n    generator = torch.Generator().manual_seed(seed)\n    max_start = token_ids.numel() - context_length - 1\n\n    while True:\n        starts = torch.randint(0, max_start + 1, (batch_size,), generator=generator)\n        inputs = torch.stack([token_ids[s : s + context_length] for s in starts.tolist()])\n        targets = torch.stack(\n            [token_ids[s + 1 : s + 1 + context_length] for s in starts.tolist()]\n        )\n        yield inputs, targets\n\n\ndef calc_loss_batch(\n    model: GPTModel,\n    inputs: torch.Tensor,\n    targets: torch.Tensor,\n) -> torch.Tensor:","sourceCodeStart":177,"sourceCodeEnd":213,"githubUrl":"https://github.com/rohitg00/ai-engineering-from-scratch/blob/39ea8a1c6d0b61f071226eff7ede4d4105fed820/phases/19-capstone-projects/36-training-loop-eval/code/main.py#L177-L213","documentation":"Error \"token_ids too short for the requested context_length\" thrown in rohitg00/ai-engineering-from-scratch.","triggerScenarios":"Thrown at phases/19-capstone-projects/36-training-loop-eval/code/main.py:195 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"39ea8a1c6d0b61f071226eff7ede4d4105fed820","analyzedAt":"2026-08-26T03:13:46.626Z","schemaVersion":2},"datasetVersion":"2026-08-26T07:17:17.940Z"}