{"record":{"id":"52a6d516b81ae68f","repo":"tensorflow/models","slug":"eos-token-not-in-tokenizer-vocab-please-make-sure-52a6d5","errorCode":null,"errorMessage":"EOS token not in tokenizer vocab.Please make sure the tokenizer generates a single token for an empty string.","messagePattern":"EOS token not in tokenizer vocab\\.Please make sure the tokenizer generates a single token for an empty string\\.","errorType":"exception","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"official/nlp/tasks/translation.py","lineNumber":193,"sourceCode":"class TranslationTask(base_task.Task):\n  \"\"\"A single-replica view of training procedure.\n\n  Tasks provide artifacts for training/evalution procedures, including\n  loading/iterating over Datasets, initializing the model, calculating the loss\n  and customized metrics with reduction.\n  \"\"\"\n\n  def __init__(self, params: cfg.TaskConfig, logging_dir=None, name=None):\n    super().__init__(params, logging_dir, name=name)\n    self._sentencepiece_model_path = params.sentencepiece_model_path\n    if params.sentencepiece_model_path:\n      self._sp_tokenizer = tftxt.SentencepieceTokenizer(\n          model=tf.io.gfile.GFile(params.sentencepiece_model_path, \"rb\").read(),\n          add_eos=True)\n      try:\n        empty_str_tokenized = self._sp_tokenizer.tokenize(\"\").numpy()\n      except tf.errors.InternalError:\n        raise ValueError(\n            \"EOS token not in tokenizer vocab.\"\n            \"Please make sure the tokenizer generates a single token for an \"\n            \"empty string.\")\n      self._eos_id = empty_str_tokenized.item()\n      self._vocab_size = self._sp_tokenizer.vocab_size().numpy()\n    else:\n      raise ValueError(\"Setencepiece model path not provided.\")\n    if (params.validation_data.input_path or\n        params.validation_data.tfds_name) and self._logging_dir:\n      self._references, self._tf_record_input_path = write_test_record(\n          params.validation_data, self.logging_dir)\n\n  def build_model(self) -> tf_keras.Model:\n    \"\"\"Creates model architecture.\n\n    Returns:\n      A model instance.\n    \"\"\"","sourceCodeStart":175,"sourceCodeEnd":211,"githubUrl":"https://github.com/tensorflow/models/blob/e006f5f0d534913e49c1f1dae87364039fa607e2/official/nlp/tasks/translation.py#L175-L211","documentation":"Error \"EOS token not in tokenizer vocab.Please make sure the tokenizer generates a single token for an empty string.\" thrown in tensorflow/models.","triggerScenarios":"Thrown at official/nlp/tasks/translation.py:193 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"e006f5f0d534913e49c1f1dae87364039fa607e2","analyzedAt":"2026-08-24T14:09:15.576Z","schemaVersion":2},"datasetVersion":"2026-08-24T17:17:21.512Z"}