{"record":{"id":"70af46b76e7afb53","repo":"tensorflow/models","slug":"unexpected-tokenization-s","errorCode":null,"errorMessage":"Unexpected tokenization: %s","messagePattern":"Unexpected tokenization: (.+?)","errorType":"exception","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"official/nlp/tasks/question_answering.py","lineNumber":175,"sourceCode":"        examples=eval_examples,\n        max_seq_length=params.seq_length,\n        doc_stride=params.doc_stride,\n        max_query_length=params.query_length,\n        is_training=False,\n        output_fn=_append_feature,\n        batch_size=params.global_batch_size,\n        xlnet_format=xlnet_ordering)\n\n    if params.tokenization == 'SentencePiece':\n      # squad_lib_sp requires one more argument 'do_lower_case'.\n      kwargs['do_lower_case'] = params.do_lower_case\n      kwargs['tokenizer'] = tokenization.FullSentencePieceTokenizer(\n          sp_model_file=params.vocab_file)\n    elif params.tokenization == 'WordPiece':\n      kwargs['tokenizer'] = tokenization.FullTokenizer(\n          vocab_file=params.vocab_file, do_lower_case=params.do_lower_case)\n    else:\n      raise ValueError('Unexpected tokenization: %s' % params.tokenization)\n\n    eval_dataset_size = self.squad_lib.convert_examples_to_features(**kwargs)\n    eval_writer.close()\n\n    logging.info('***** Evaluation input stats *****')\n    logging.info('  Num orig examples = %d', len(eval_examples))\n    logging.info('  Num split examples = %d', len(eval_features))\n    logging.info('  Batch size = %d', params.global_batch_size)\n    logging.info('  Dataset size = %d', eval_dataset_size)\n\n    return eval_writer.filename, eval_examples, eval_features\n\n  def _dummy_data(self, params, _):\n    \"\"\"Returns dummy data.\"\"\"\n    dummy_ids = tf.zeros((1, params.seq_length), dtype=tf.int32)\n    x = dict(\n        input_word_ids=dummy_ids,\n        input_mask=dummy_ids,","sourceCodeStart":157,"sourceCodeEnd":193,"githubUrl":"https://github.com/tensorflow/models/blob/e006f5f0d534913e49c1f1dae87364039fa607e2/official/nlp/tasks/question_answering.py#L157-L193","documentation":"Error \"Unexpected tokenization: %s\" thrown in tensorflow/models.","triggerScenarios":"Thrown at official/nlp/tasks/question_answering.py:175 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"e006f5f0d534913e49c1f1dae87364039fa607e2","analyzedAt":"2026-08-24T14:09:15.576Z","schemaVersion":2},"datasetVersion":"2026-08-24T17:17:21.512Z"}