{"record":{"id":"b6d6aa5c10770eca","repo":"Comfy-Org/ComfyUI","slug":"invalid-tokenizer","errorCode":null,"errorMessage":"invalid tokenizer","messagePattern":"invalid tokenizer","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"comfy/text_encoders/spiece_tokenizer.py","lineNumber":21,"sourceCode":"\nclass SPieceTokenizer:\n    @staticmethod\n    def from_pretrained(path, **kwargs):\n        return SPieceTokenizer(path, **kwargs)\n\n    def __init__(self, tokenizer_path, add_bos=False, add_eos=True, special_tokens=None):\n        self.add_bos = add_bos\n        self.add_eos = add_eos\n        self.special_tokens = special_tokens\n        import sentencepiece\n        if torch.is_tensor(tokenizer_path):\n            tokenizer_path = tokenizer_path.numpy().tobytes()\n\n        if isinstance(tokenizer_path, bytes):\n            self.tokenizer = sentencepiece.SentencePieceProcessor(model_proto=tokenizer_path, add_bos=self.add_bos, add_eos=self.add_eos)\n        else:\n            if not os.path.isfile(tokenizer_path):\n                raise ValueError(\"invalid tokenizer\")\n            self.tokenizer = sentencepiece.SentencePieceProcessor(model_file=tokenizer_path, add_bos=self.add_bos, add_eos=self.add_eos)\n\n    def get_vocab(self):\n        out = {}\n        for i in range(self.tokenizer.get_piece_size()):\n            out[self.tokenizer.id_to_piece(i)] = i\n        return out\n\n    def __call__(self, string):\n        if self.special_tokens is not None:\n            import re\n            special_tokens_pattern = '|'.join(re.escape(token) for token in self.special_tokens.keys())\n            if special_tokens_pattern and re.search(special_tokens_pattern, string):\n                parts = re.split(f'({special_tokens_pattern})', string)\n                result = []\n                for part in parts:\n                    if not part:\n                        continue","sourceCodeStart":3,"sourceCodeEnd":39,"githubUrl":"https://github.com/Comfy-Org/ComfyUI/blob/1c6d8d45b3693bfbb32385b410d813a7fd6be216/comfy/text_encoders/spiece_tokenizer.py#L3-L39","documentation":"SPieceTokenizer accepts either raw SentencePiece model bytes (from a tensor) or a filesystem path to a .model file. When given a string that is not an existing file (bad path, folder, or typo), it raises 'invalid tokenizer' before constructing the SentencePieceProcessor. It is a load-time path validation, not a format check.","triggerScenarios":"Passing a str tokenizer_path that fails os.path.isfile — wrong path, missing tokenizer.model in models/text_encoders, path with typos, or the string path of a directory.","commonSituations":"Model folder renamed or tokenizer.model not downloaded; embedding_directory misconfigured in a custom node; passing a tokenizer path from a different machine with different layout.","solutions":["Verify the SentencePiece .model file exists at the path you pass (os.path.isfile check) and fix the path","Ensure the tokenizer file was downloaded/placed alongside the text encoder weights in models/text_encoders","If you have the tokenizer as bytes/tensor, pass bytes instead of a path so the isfile branch is skipped"],"exampleFix":"// before\ntok = SPieceTokenizer('/models/llava/spiece.model')  # file missing\n\n# after\npath = os.path.join(embedding_dir, 'spiece.model')\nif not os.path.isfile(path):\n    raise FileNotFoundError(path)\ntok = SPieceTokenizer(path)","handlingStrategy":"validation","validationCode":"import os\nif isinstance(path, str) and not os.path.isfile(path):\n    raise FileNotFoundError(f'tokenizer model not found: {path}')\ntok = SPieceTokenizer(path)","typeGuard":"def is_spiece_source(p) -> bool:\n    return isinstance(p, (bytes, bytearray)) or (isinstance(p, str) and os.path.isfile(p))","tryCatchPattern":null,"preventionTips":["Resolve tokenizer paths through folder_paths getters rather than hardcoding","Ensure tokenizer files are downloaded alongside encoder weights"],"tags":["tokenizer","sentencepiece","file-not-found","text-encoder"],"backgroundTag":null,"analyzedSha":"1c6d8d45b3693bfbb32385b410d813a7fd6be216","analyzedAt":"2026-08-14T19:37:18.893Z","schemaVersion":2},"datasetVersion":"2026-08-16T03:17:38.424Z"}