{"record":{"id":"d9338b1e629bef63","repo":"rohitg00/ai-engineering-from-scratch","slug":"target-vocab-size-must-be-min-vocab-size-got","errorCode":null,"errorMessage":"target_vocab_size must be >= {min_vocab_size}, got {target_vocab_size}","messagePattern":"target_vocab_size must be >= (.+?), got (.+?)","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"phases/19-capstone-projects/31-tokenized-dataset-sliding-window/code/main.py","lineNumber":99,"sourceCode":"        out: list[int] = []\n        i = 0\n        a, b = pair\n        while i < len(symbols):\n            if i < len(symbols) - 1 and symbols[i] == a and symbols[i + 1] == b:\n                out.append(new_id)\n                i += 2\n            else:\n                out.append(symbols[i])\n                i += 1\n        merged = tuple(out)\n        new_units[merged] = new_units.get(merged, 0) + count\n    return new_units\n\n\ndef train_bpe(tokenizer: MiniBPE, corpus: str, target_vocab_size: int) -> None:\n    min_vocab_size = BYTE_ALPHABET_SIZE + len(DEFAULT_SPECIALS)\n    if target_vocab_size < min_vocab_size:\n        raise ValueError(\n            f\"target_vocab_size must be >= {min_vocab_size}, got {target_vocab_size}\"\n        )\n    tokenizer.initialize(DEFAULT_SPECIALS)\n    chunks = _pretokenize(corpus)\n    units: dict[tuple[int, ...], int] = {}\n    for chunk in chunks:\n        symbols = tuple(chunk.encode(\"utf-8\"))\n        units[symbols] = units.get(symbols, 0) + 1\n    while tokenizer.vocab_size < target_vocab_size:\n        pairs = _count_pairs(units)\n        if not pairs:\n            break\n        max_count = max(pairs.values())\n        candidates = sorted(p for p, c in pairs.items() if c == max_count)\n        best = candidates[0]\n        if pairs[best] < 2:\n            break\n        new_id = len(tokenizer.vocab)","sourceCodeStart":81,"sourceCodeEnd":117,"githubUrl":"https://github.com/rohitg00/ai-engineering-from-scratch/blob/39ea8a1c6d0b61f071226eff7ede4d4105fed820/phases/19-capstone-projects/31-tokenized-dataset-sliding-window/code/main.py#L81-L117","documentation":"Error \"target_vocab_size must be >= {min_vocab_size}, got {target_vocab_size}\" thrown in rohitg00/ai-engineering-from-scratch.","triggerScenarios":"Thrown at phases/19-capstone-projects/31-tokenized-dataset-sliding-window/code/main.py:99 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"39ea8a1c6d0b61f071226eff7ede4d4105fed820","analyzedAt":"2026-08-26T03:13:46.626Z","schemaVersion":2},"datasetVersion":"2026-08-26T07:17:17.940Z"}