{"record":{"id":"edb275dfd6ea52dc","repo":"rohitg00/ai-engineering-from-scratch","slug":"max-len-must-be-3-to-fit-inst-resp-and-one-re","errorCode":null,"errorMessage":"max_len must be >= 3 to fit INST, RESP, and one response token","messagePattern":"max_len must be >= 3 to fit INST, RESP, and one response token","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"phases/19-capstone-projects/39-instruction-tuning-sft/code/main.py","lineNumber":51,"sourceCode":"# Tokeniser\n# ---------------------------------------------------------------------------\n\n\nclass InstructionTokenizer:\n    \"\"\"Byte-level tokenizer with INST, RESP, PAD specials.\"\"\"\n\n    INST_ID = 256\n    RESP_ID = 257\n    PAD_ID = 258\n    VOCAB = 260\n    IGNORE_INDEX = -100\n\n    def encode_pair(self, instruction: str, response: str, max_len: int) -> Tuple[List[int], int]:\n        \"\"\"Return (token_ids, response_start_index). Truncates to max_len if\n        needed but always keeps the RESP marker plus at least one response\n        token so SFT collation never produces fully-masked labels.\"\"\"\n        if max_len < 3:\n            raise ValueError(\"max_len must be >= 3 to fit INST, RESP, and one response token\")\n        inst_bytes = list(instruction.encode(\"utf-8\", errors=\"ignore\"))\n        resp_bytes = list(response.encode(\"utf-8\", errors=\"ignore\"))\n        # Reserve 2 control tokens + at least 1 response byte.\n        max_inst = max_len - 3\n        inst_bytes = inst_bytes[:max_inst]\n        ids = [self.INST_ID] + inst_bytes + [self.RESP_ID]\n        resp_start = len(ids)\n        ids.extend(resp_bytes[: max_len - len(ids)])\n        return ids, resp_start\n\n    def encode_prefix(self, instruction: str, max_len: int) -> List[int]:\n        \"\"\"Encode just the instruction prefix for generation. Always keeps the\n        RESP marker so the model sees the same boundary as during training.\"\"\"\n        if max_len < 2:\n            raise ValueError(\"max_len must be >= 2 to fit INST and RESP\")\n        inst_bytes = list(instruction.encode(\"utf-8\", errors=\"ignore\"))[: max_len - 2]\n        ids = [self.INST_ID] + inst_bytes + [self.RESP_ID]\n        return ids","sourceCodeStart":33,"sourceCodeEnd":69,"githubUrl":"https://github.com/rohitg00/ai-engineering-from-scratch/blob/39ea8a1c6d0b61f071226eff7ede4d4105fed820/phases/19-capstone-projects/39-instruction-tuning-sft/code/main.py#L33-L69","documentation":"Error \"max_len must be >= 3 to fit INST, RESP, and one response token\" thrown in rohitg00/ai-engineering-from-scratch.","triggerScenarios":"Thrown at phases/19-capstone-projects/39-instruction-tuning-sft/code/main.py:51 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"39ea8a1c6d0b61f071226eff7ede4d4105fed820","analyzedAt":"2026-08-26T03:13:46.626Z","schemaVersion":2},"datasetVersion":"2026-08-26T07:17:17.940Z"}