{"record":{"id":"7e3ec51b04474928","repo":"microsoft/vscode","slug":"could-not-load-tokenizer","errorCode":null,"errorMessage":"Could not load tokenizer","messagePattern":"Could not load tokenizer","errorType":"exception","errorClass":"CopilotPromptLoadFailure","httpStatus":null,"severity":"error","filePath":"extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/tokenization/tokenizer.ts","lineNumber":164,"sourceCode":"\t */\n\ttakeLastLinesTokens(text: string, n: number): string;\n}\n\nexport class TTokenizer implements Tokenizer {\n\tconstructor(private readonly _tokenizer: TikTokenizer) { }\n\n\tstatic async create(encoder: TokenizerName): Promise<TTokenizer> {\n\t\ttry {\n\t\t\tconst tokenizer = createTokenizer(\n\t\t\t\tparseTikTokenBinary(locateFile(`${encoder}.tiktoken`)),\n\t\t\t\tgetSpecialTokensByEncoder(encoder),\n\t\t\t\tgetRegexByEncoder(encoder),\n\t\t\t\t32768\n\t\t\t);\n\t\t\treturn new TTokenizer(tokenizer);\n\t\t} catch (e: unknown) {\n\t\t\tif (e instanceof Error) {\n\t\t\t\tthrow new CopilotPromptLoadFailure(`Could not load tokenizer`, e);\n\t\t\t}\n\t\t\tthrow e;\n\t\t}\n\t}\n\n\ttokenize(text: string): number[] {\n\t\treturn this._tokenizer.encode(text);\n\t}\n\n\tdetokenize(tokens: number[]): string {\n\t\treturn this._tokenizer.decode(tokens);\n\t}\n\n\ttokenLength(text: string): number {\n\t\treturn this.tokenize(text).length;\n\t}\n\n\ttokenizeStrings(text: string): string[] {","sourceCodeStart":146,"sourceCodeEnd":182,"githubUrl":"https://github.com/microsoft/vscode/blob/a94b963a32aa9749d69504576791fa29700ae46d/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/tokenization/tokenizer.ts#L146-L182","documentation":"Error \"Could not load tokenizer\" thrown in microsoft/vscode.","triggerScenarios":"Thrown at extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/tokenization/tokenizer.ts:164 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"analyzedSha":"a94b963a32aa9749d69504576791fa29700ae46d","analyzedAt":"2026-08-12T01:08:56.794Z","schemaVersion":2},"datasetVersion":"2026-08-12T06:17:24.410Z"}