{"record":{"id":"8cac5bad6a38bed0","repo":"rohitg00/ai-engineering-from-scratch","slug":"max-tokens-must-be-greater-than-1","errorCode":null,"errorMessage":"max_tokens must be greater than 1","messagePattern":"max_tokens must be greater than 1","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"phases/19-capstone-projects/43-hdf5-tokenized-corpus/code/main.py","lineNumber":423,"sourceCode":"                    self.dropped_lines += 1\n                    continue\n                yield value\n\n\ndef pack_documents(\n    tokenizer: Tokenizer,\n    documents: Iterable[str],\n    max_tokens: int,\n) -> Iterator[list[int]]:\n    \"\"\"Pack tokenized documents into fixed-length groups with boundary tokens.\n\n    Yields lists of exactly max_tokens token ids. Long documents are split\n    across groups; short documents share a group separated by BOUNDARY_TOKEN_ID.\n    The final group may be shorter than max_tokens and is yielded as-is.\n    \"\"\"\n\n    if max_tokens <= 1:\n        raise ValueError(\"max_tokens must be greater than 1\")\n    buffer: list[int] = []\n    for text in documents:\n        token_ids = tokenizer.encode(text)\n        if buffer:\n            buffer.append(BOUNDARY_TOKEN_ID)\n        buffer.extend(token_ids)\n        while len(buffer) >= max_tokens:\n            yield buffer[:max_tokens]\n            buffer = buffer[max_tokens:]\n    if buffer:\n        yield buffer\n\n\ndef tokenize_jsonl_path(\n    jsonl_path: Path,\n    output_dir: Path,\n    shard_id: str,\n    chunk_size: int = DEFAULT_CHUNK_SIZE,","sourceCodeStart":405,"sourceCodeEnd":441,"githubUrl":"https://github.com/rohitg00/ai-engineering-from-scratch/blob/39ea8a1c6d0b61f071226eff7ede4d4105fed820/phases/19-capstone-projects/43-hdf5-tokenized-corpus/code/main.py#L405-L441","documentation":"Error \"max_tokens must be greater than 1\" thrown in rohitg00/ai-engineering-from-scratch.","triggerScenarios":"Thrown at phases/19-capstone-projects/43-hdf5-tokenized-corpus/code/main.py:423 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"39ea8a1c6d0b61f071226eff7ede4d4105fed820","analyzedAt":"2026-08-26T03:13:46.626Z","schemaVersion":2},"datasetVersion":"2026-08-26T07:17:17.940Z"}