{"record":{"id":"ea6da9bd5d9a2d35","repo":"run-llama/llama_index","slug":"single-token-exceeded-chunk-size","errorCode":null,"errorMessage":"Single token exceeded chunk size","messagePattern":"Single token exceeded chunk size","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"llama-index-core/llama_index/core/node_parser/text/sentence.py","lineNumber":280,"sourceCode":"            new_chunk = True\n\n            # add overlap to the next chunk using the last one first\n            if len(last_chunk) > 0:\n                last_index = len(last_chunk) - 1\n                while (\n                    last_index >= 0\n                    and cur_chunk_len + last_chunk[last_index][1] <= self.chunk_overlap\n                ):\n                    overlap_text, overlap_length = last_chunk[last_index]\n                    cur_chunk_len += overlap_length\n                    cur_chunk.insert(0, (overlap_text, overlap_length))\n                    last_index -= 1\n\n        split_idx = 0\n        while split_idx < len(splits):\n            cur_split = splits[split_idx]\n            if cur_split.token_size > chunk_size:\n                raise ValueError(\"Single token exceeded chunk size\")\n            if cur_chunk_len + cur_split.token_size > chunk_size and not new_chunk:\n                # if adding split to current chunk exceeds chunk size: close out chunk\n                close_chunk()\n            else:\n                # If this is a new chunk with overlap, and adding the split would\n                # exceed chunk_size, remove overlap to make room\n                if new_chunk and cur_chunk_len + cur_split.token_size > chunk_size:\n                    # Remove overlap from the beginning until split fits\n                    while (\n                        len(cur_chunk) > 0\n                        and cur_chunk_len + cur_split.token_size > chunk_size\n                    ):\n                        _, length = cur_chunk.pop(0)\n                        cur_chunk_len -= length\n\n                if (\n                    cur_split.is_sentence\n                    or cur_chunk_len + cur_split.token_size <= chunk_size","sourceCodeStart":262,"sourceCodeEnd":298,"githubUrl":"https://github.com/run-llama/llama_index/blob/afd0fef371831f9bda13e5af7167cf4e981278ab/llama-index-core/llama_index/core/node_parser/text/sentence.py#L262-L298","documentation":"Error \"Single token exceeded chunk size\" thrown in run-llama/llama_index.","triggerScenarios":"Thrown at llama-index-core/llama_index/core/node_parser/text/sentence.py:280 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":["Increase chunk_size so the largest single token/word fits.","Pre-split very long unbroken strings in the input text before parsing."],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"afd0fef371831f9bda13e5af7167cf4e981278ab","analyzedAt":"2026-08-15T05:42:58.429Z","schemaVersion":2},"datasetVersion":"2026-08-15T17:31:12.345Z"}