{"record":{"id":"26041c42e85e169f","repo":"rohitg00/ai-engineering-from-scratch","slug":"stop-stop-exceeds-total-tokens-self-total-t","errorCode":null,"errorMessage":"stop ({stop}) exceeds total tokens ({self._total_tokens})","messagePattern":"stop \\((.+?)\\) exceeds total tokens \\((.+?)\\)","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"phases/19-capstone-projects/43-hdf5-tokenized-corpus/code/main.py","lineNumber":307,"sourceCode":"        for file in self._files:\n            try:\n                file.close()\n            except Exception:\n                pass\n        self._files = []\n        self._datasets = []\n\n    def __enter__(self) -> \"MmapTokenStore\":\n        return self\n\n    def __exit__(self, exc_type, exc, tb) -> None:\n        self.close()\n\n    def get_slice(self, start: int, stop: int) -> np.ndarray:\n        if start < 0 or stop < 0 or stop < start:\n            raise ValueError(f\"bad slice: start={start} stop={stop}\")\n        if stop > self._total_tokens:\n            raise ValueError(f\"stop ({stop}) exceeds total tokens ({self._total_tokens})\")\n        if stop == start:\n            return np.empty((0,), dtype=TOKEN_DTYPE)\n        out = np.empty((stop - start,), dtype=TOKEN_DTYPE)\n        cursor = 0\n        for entry, dataset in zip(self._entries, self._datasets):\n            shard_start = entry.global_start\n            shard_stop = shard_start + entry.token_count\n            if stop <= shard_start:\n                break\n            if start >= shard_stop:\n                continue\n            local_start = max(0, start - shard_start)\n            local_stop = min(entry.token_count, stop - shard_start)\n            length = local_stop - local_start\n            if length <= 0:\n                continue\n            out[cursor : cursor + length] = dataset[local_start:local_stop]\n            cursor += length","sourceCodeStart":289,"sourceCodeEnd":325,"githubUrl":"https://github.com/rohitg00/ai-engineering-from-scratch/blob/39ea8a1c6d0b61f071226eff7ede4d4105fed820/phases/19-capstone-projects/43-hdf5-tokenized-corpus/code/main.py#L289-L325","documentation":"Error \"stop ({stop}) exceeds total tokens ({self._total_tokens})\" thrown in rohitg00/ai-engineering-from-scratch.","triggerScenarios":"Thrown at phases/19-capstone-projects/43-hdf5-tokenized-corpus/code/main.py:307 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"39ea8a1c6d0b61f071226eff7ede4d4105fed820","analyzedAt":"2026-08-26T03:13:46.626Z","schemaVersion":2},"datasetVersion":"2026-08-26T07:17:17.940Z"}