{"record":{"id":"88cd07f231ee86b3","repo":"rohitg00/ai-engineering-from-scratch","slug":"store-has-only-store-total-tokens-tokens-need-m","errorCode":null,"errorMessage":"store has only {store.total_tokens} tokens; need more than {window_size}","messagePattern":"store has only (.+?) tokens; need more than (.+?)","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"phases/19-capstone-projects/43-hdf5-tokenized-corpus/code/main.py","lineNumber":348,"sourceCode":"        return out\n\n\nclass SlidingWindowDataloader:\n    \"\"\"Random sliding-window sampler over a flat token stream.\"\"\"\n\n    def __init__(\n        self,\n        store: MmapTokenStore,\n        window_size: int = DEFAULT_WINDOW_SIZE,\n        batch_size: int = 4,\n        seed: int = 0,\n    ) -> None:\n        if window_size <= 1:\n            raise ValueError(\"window_size must be greater than 1\")\n        if batch_size <= 0:\n            raise ValueError(\"batch_size must be positive\")\n        if store.total_tokens <= window_size:\n            raise ValueError(\n                f\"store has only {store.total_tokens} tokens; need more than {window_size}\"\n            )\n        self.store = store\n        self.window_size = window_size\n        self.batch_size = batch_size\n        self._random = random.Random(seed)\n        self._max_start = store.total_tokens - window_size - 1\n\n    def _sample_window(self) -> tuple[np.ndarray, np.ndarray]:\n        start = self._random.randint(0, self._max_start)\n        chunk = self.store.get_slice(start, start + self.window_size + 1)\n        return chunk[:-1], chunk[1:]\n\n    def __iter__(self) -> Iterator[tuple[np.ndarray, np.ndarray]]:\n        while True:\n            inputs = np.empty((self.batch_size, self.window_size), dtype=TOKEN_DTYPE)\n            targets = np.empty((self.batch_size, self.window_size), dtype=TOKEN_DTYPE)\n            for row in range(self.batch_size):","sourceCodeStart":330,"sourceCodeEnd":366,"githubUrl":"https://github.com/rohitg00/ai-engineering-from-scratch/blob/39ea8a1c6d0b61f071226eff7ede4d4105fed820/phases/19-capstone-projects/43-hdf5-tokenized-corpus/code/main.py#L330-L366","documentation":"Error \"store has only {store.total_tokens} tokens; need more than {window_size}\" thrown in rohitg00/ai-engineering-from-scratch.","triggerScenarios":"Thrown at phases/19-capstone-projects/43-hdf5-tokenized-corpus/code/main.py:348 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"39ea8a1c6d0b61f071226eff7ede4d4105fed820","analyzedAt":"2026-08-26T03:13:46.626Z","schemaVersion":2},"datasetVersion":"2026-08-26T07:17:17.940Z"}