{"record":{"id":"f02cd5b1f0d06a33","repo":"diegosouzapw/OmniRoute","slug":"js-tiktoken-not-available-cannot-create-encoder-f","errorCode":null,"errorMessage":"js-tiktoken not available: cannot create encoder for ${encoding}","messagePattern":"js-tiktoken not available: cannot create encoder for (.+?)","errorType":"exception","errorClass":"Error","httpStatus":null,"severity":"error","filePath":"src/shared/utils/tiktokenCounter.ts","lineNumber":77,"sourceCode":"    model.includes(\"codex\")\n  );\n}\n\nexport function resolveTokenizerEncoding(context?: TokenizerContext): TokenizerEncoding {\n  return isCodexTokenizerContext(context) ? \"o200k_base\" : \"cl100k_base\";\n}\n\nfunction getEncoder(encoding: TokenizerEncoding): Tiktoken {\n  const cached = encoders.get(encoding);\n  if (cached) return cached;\n  let tiktoken: { getEncoding: (name: string) => Tiktoken } | null = null;\n  try {\n    tiktoken ??= _require(\"js-tiktoken\") as { getEncoding: (name: string) => Tiktoken };\n    const created = tiktoken.getEncoding(encoding);\n    encoders.set(encoding, created);\n    return created;\n  } catch {\n    throw new Error(`js-tiktoken not available: cannot create encoder for ${encoding}`);\n  }\n}\n\n/**\n * Exact token count for a string using the selected offline tokenizer.\n * Existing callers retain cl100k_base; Codex callers may pass provider/model context\n * to use o200k_base.\n * Defensive: never throws in a counting path — falls back to a char heuristic.\n * Oversized inputs (over 50k chars) and base64 image data URIs are never\n * tokenized: the encoder is near-quadratic on large strings and would block the\n * event loop (worker wedge regression).\n */\nexport function countTextTokens(text: string, context?: TokenizerContext): number {\n  if (!text || typeof text !== \"string\") return 0;\n  const stripped = stripBase64DataUris(text);\n  if (stripped.length > MAX_EXACT_TOKEN_COUNT_CHARS) {\n    return Math.ceil(stripped.length / 4);\n  }","sourceCodeStart":59,"sourceCodeEnd":95,"githubUrl":"https://github.com/diegosouzapw/OmniRoute/blob/a179ffed5bb2e0b883b9ae7214ce8717b2a94c4d/src/shared/utils/tiktokenCounter.ts#L59-L95","documentation":"Error \"js-tiktoken not available: cannot create encoder for ${encoding}\" thrown in diegosouzapw/OmniRoute.","triggerScenarios":"Thrown at src/shared/utils/tiktokenCounter.ts:77 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"a179ffed5bb2e0b883b9ae7214ce8717b2a94c4d","analyzedAt":"2026-08-25T18:35:09.898Z","schemaVersion":2},"datasetVersion":"2026-08-25T21:54:21.419Z"}