{"record":{"id":"11fb6a782ec8ca43","repo":"rohitg00/ai-engineering-from-scratch","slug":"unknown-token-id-token-id","errorCode":null,"errorMessage":"unknown token id: {token_id}","messagePattern":"unknown token id: (.+?)","errorType":"exception","errorClass":"KeyError","httpStatus":null,"severity":"error","filePath":"phases/19-capstone-projects/30-bpe-tokenizer-from-scratch/code/main.py","lineNumber":228,"sourceCode":"    return out\n\n\ndef _encode_pretokenized(tokenizer: BPETokenizer, text: str) -> list[int]:\n    out: list[int] = []\n    for chunk in _pretokenize(text):\n        out.extend(_encode_chunk(tokenizer, chunk))\n    return out\n\n\ndef decode(tokenizer: BPETokenizer, ids: list[int]) -> str:\n    \"\"\"Decode `ids` back to a string. Inverse of `encode` for round-trip safe input.\"\"\"\n    pieces: list[bytes] = []\n    for token_id in ids:\n        if token_id in tokenizer.id_to_special:\n            pieces.append(tokenizer.id_to_special[token_id].encode(\"utf-8\"))\n            continue\n        if token_id not in tokenizer.vocab:\n            raise KeyError(f\"unknown token id: {token_id}\")\n        pieces.append(tokenizer.vocab[token_id])\n    return b\"\".join(pieces).decode(\"utf-8\", errors=\"replace\")\n\n\ndef save(tokenizer: BPETokenizer, path: str) -> None:\n    \"\"\"Serialize the tokenizer to a JSON file.\"\"\"\n    payload = {\n        \"vocab\": {\n            str(token_id): list(token_bytes)\n            for token_id, token_bytes in tokenizer.vocab.items()\n        },\n        \"merges\": [\n            [list(pair), new_id]\n            for pair, new_id in tokenizer.merges.items()\n        ],\n        \"specials\": tokenizer.special_to_id,\n    }\n    with open(path, \"w\", encoding=\"utf-8\") as f:","sourceCodeStart":210,"sourceCodeEnd":246,"githubUrl":"https://github.com/rohitg00/ai-engineering-from-scratch/blob/39ea8a1c6d0b61f071226eff7ede4d4105fed820/phases/19-capstone-projects/30-bpe-tokenizer-from-scratch/code/main.py#L210-L246","documentation":"Error \"unknown token id: {token_id}\" thrown in rohitg00/ai-engineering-from-scratch.","triggerScenarios":"Thrown at phases/19-capstone-projects/30-bpe-tokenizer-from-scratch/code/main.py:228 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"39ea8a1c6d0b61f071226eff7ede4d4105fed820","analyzedAt":"2026-08-26T03:13:46.626Z","schemaVersion":2},"datasetVersion":"2026-08-26T07:17:17.940Z"}