{"record":{"id":"bbc602222584537c","repo":"oobabooga/textgen","slug":"no-tokenizer-is-loaded","errorCode":null,"errorMessage":"No tokenizer is loaded","messagePattern":"No tokenizer is loaded","errorType":"exception","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"modules/text_generation.py","lineNumber":135,"sourceCode":"                if (cur_time - last_update) > latency_threshold:\n                    yield reply\n                last_update = time.monotonic()\n\n        stop_event = state.get('stop_event')\n        if stop_found or shared.stop_everything or (stop_event and stop_event.is_set()):\n            break\n\n    if not is_chat:\n        reply = apply_extensions('output', reply, state)\n\n    yield reply\n\n\ndef encode(prompt, add_special_tokens=True, add_bos_token=True, truncation_length=None):\n    if shared.tokenizer is None:\n        models.load_model_if_idle_unloaded()\n        if shared.tokenizer is None:\n            raise ValueError('No tokenizer is loaded')\n\n    # llama.cpp case\n    if shared.model.__class__.__name__ == 'LlamaServer':\n        input_ids = shared.tokenizer.encode(str(prompt), add_bos_token=add_bos_token)\n        input_ids = np.array(input_ids).reshape(1, len(input_ids))\n\n        if truncation_length is not None:\n            input_ids = input_ids[:, -truncation_length:]\n\n        return input_ids\n\n    # All other model types\n    else:\n        import torch\n\n        from modules.torch_utils import get_device\n\n        if shared.model.__class__.__name__ in ['Exllamav3Model', 'TensorRTLLMModel']:","sourceCodeStart":117,"sourceCodeEnd":153,"githubUrl":"https://github.com/oobabooga/textgen/blob/ed888c71f221df552750e1834b3654abab8ae345/modules/text_generation.py#L117-L153","documentation":"Error \"No tokenizer is loaded\" thrown in oobabooga/textgen.","triggerScenarios":"Raised when text generation or tokenization is requested but no model/tokenizer is currently loaded in the web UI. Triggers when calling generate before loading a model, or after the model was unloaded.","commonSituations":"See trigger scenarios.","solutions":["Load a model (with its tokenizer) via the UI or API before requesting tokenization or generation."],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"ed888c71f221df552750e1834b3654abab8ae345","analyzedAt":"2026-08-15T05:24:21.000Z","schemaVersion":2},"datasetVersion":"2026-08-15T17:31:12.345Z"}