{"record":{"id":"0d0e7d6334fb2820","repo":"tensorflow/models","slug":"unsupported-tokenization-method","errorCode":null,"errorMessage":"Unsupported tokenization method: {}","messagePattern":"Unsupported tokenization method: (.+?)","errorType":"exception","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"official/nlp/tasks/question_answering.py","lineNumber":91,"sourceCode":"  class_logits: Optional[float] = None\n\n\n@task_factory.register_task_cls(QuestionAnsweringConfig)\nclass QuestionAnsweringTask(base_task.Task):\n  \"\"\"Task object for question answering.\"\"\"\n\n  def __init__(self, params: cfg.TaskConfig, logging_dir=None, name=None):\n    super().__init__(params, logging_dir, name=name)\n\n    if params.validation_data is None:\n      return\n\n    if params.validation_data.tokenization == 'WordPiece':\n      self.squad_lib = squad_lib_wp\n    elif params.validation_data.tokenization == 'SentencePiece':\n      self.squad_lib = squad_lib_sp\n    else:\n      raise ValueError('Unsupported tokenization method: {}'.format(\n          params.validation_data.tokenization))\n\n    if params.validation_data.input_path:\n      self._tf_record_input_path, self._eval_examples, self._eval_features = (\n          self._preprocess_eval_data(params.validation_data))\n\n  def set_preprocessed_eval_input_path(self, eval_input_path):\n    \"\"\"Sets the path to the preprocessed eval data.\"\"\"\n    self._tf_record_input_path = eval_input_path\n\n  def build_model(self):\n    if self.task_config.hub_module_url and self.task_config.init_checkpoint:\n      raise ValueError('At most one of `hub_module_url` and '\n                       '`init_checkpoint` can be specified.')\n    if self.task_config.hub_module_url:\n      encoder_network = utils.get_encoder_from_hub(\n          self.task_config.hub_module_url)\n    else:","sourceCodeStart":73,"sourceCodeEnd":109,"githubUrl":"https://github.com/tensorflow/models/blob/e006f5f0d534913e49c1f1dae87364039fa607e2/official/nlp/tasks/question_answering.py#L73-L109","documentation":"Error \"Unsupported tokenization method: {}\" thrown in tensorflow/models.","triggerScenarios":"Thrown at official/nlp/tasks/question_answering.py:91 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"e006f5f0d534913e49c1f1dae87364039fa607e2","analyzedAt":"2026-08-24T14:09:15.576Z","schemaVersion":2},"datasetVersion":"2026-08-24T17:17:21.512Z"}