{"record":{"id":"71a7030c6b5c6a63","repo":"sgl-project/sglang","slug":"unknown-inkling-special-token-token-r","errorCode":null,"errorMessage":"unknown Inkling special token: {token!r}","messagePattern":"unknown Inkling special token: (.+?)","errorType":"validation","errorClass":"KeyError","httpStatus":null,"severity":"error","filePath":"python/sglang/srt/parser/inkling_tokenizer.py","lineNumber":80,"sourceCode":"    for token in INKLING_SPECIAL_TOKENS\n}\n\nROLE_MESSAGE_TOKENS: dict[str, str] = {\n    \"user\": MESSAGE_USER,\n    \"assistant\": MESSAGE_MODEL,\n    \"system\": MESSAGE_SYSTEM,\n    \"tool\": MESSAGE_TOOL,\n}\n\n\ndef normalize_special_token(token: str) -> str:\n    \"\"\"Accept either message_user or <|message_user|> spellings.\"\"\"\n    if token in INKLING_SPECIAL_TOKENS:\n        return token\n    try:\n        return INKLING_SPECIAL_TOKEN_NAMES[token]\n    except KeyError as exc:\n        raise KeyError(f\"unknown Inkling special token: {token!r}\") from exc\n\n\n@dataclass(frozen=True)\nclass InklingTokenizer:\n    \"\"\"Small wrapper around a base text tokenizer plus Inkling framing IDs.\n\n    Plain text is encoded by the base tokenizer, while the minimal chat\n    framing tokens are inserted from the fixed overlay map.\n    \"\"\"\n\n    tokenizer: Any\n    special_token_ids: Mapping[str, int] | None = None\n\n    def encode_text(self, text: str) -> list[int]:\n        if not isinstance(text, str):\n            raise TypeError(f\"text must be str, got {type(text).__name__}\")\n        return list(self.tokenizer.encode(text, add_special_tokens=False))\n","sourceCodeStart":62,"sourceCodeEnd":98,"githubUrl":"https://github.com/sgl-project/sglang/blob/0132848349585cfe6aae51c4941cbae872505f8a/python/sglang/srt/parser/inkling_tokenizer.py#L62-L98","documentation":"normalize_special_token only accepts known Inkling special tokens, in either the bare ('message_user') or angle-bracket ('<|message_user|>') spelling. Anything else raises KeyError with the offending token repr.","triggerScenarios":"Calling encode_special('message_assistant_typo') or encode_special('<|unknown_token|>') — any token not present in INKLING_SPECIAL_TOKENS / INKLING_SPECIAL_TOKEN_NAMES.","commonSituations":"Typos in token names, version drift where newer/older token sets differ, or passing ordinary text instead of a special token name.","solutions":["Check the exact spelling against INKLING_SPECIAL_TOKENS defined in the same module","Use the bare name form first ('message_user'); fall back to the '<|message_user|>' form","Don't pass regular text through encode_special — use encode_text for that"],"exampleFix":"// before\nids = tok.encode_special(\"<|msg_user|>\")\n// after\nids = tok.encode_special(\"message_user\")","handlingStrategy":"type-guard","validationCode":"from sglang.srt.parser.inkling_tokenizer import INKLING_SPECIAL_TOKENS\n\ndef safe_encode_special(tok, token):\n    if token not in INKLING_SPECIAL_TOKENS:\n        raise ValueError(f\"bad token {token!r}; valid: {sorted(INKLING_SPECIAL_TOKENS)}\")\n    return tok.encode_special(token)","typeGuard":"def is_inkling_special(token: str) -> TypeGuard[str]:\n    from sglang.srt.parser.inkling_tokenizer import INKLING_SPECIAL_TOKENS\n    return token in INKLING_SPECIAL_TOKENS or token.strip(\"<|>\") in INKLING_SPECIAL_TOKENS","tryCatchPattern":"try:\n    tid = tok.encode_special(token)\nexcept KeyError as e:\n    logger.warning(\"skipping unknown special token %s\", token)\n    tid = None","preventionTips":["Constant-fold token names instead of string-building them","Assert membership in INKLING_SPECIAL_TOKENS before encoding","Use encode_text for anything that isn't a known special token"],"tags":["inkling","special-tokens","tokenizer","key-error"],"backgroundTag":"unknown-special-token","analyzedSha":"0132848349585cfe6aae51c4941cbae872505f8a","analyzedAt":"2026-08-28T05:10:05.995Z","schemaVersion":2},"datasetVersion":"2026-08-28T06:17:29.519Z"}