{"record":{"id":"1180533fc50cc6e7","repo":"tensorflow/models","slug":"unsupported-tokenization-s","errorCode":null,"errorMessage":"Unsupported tokenization: %s","messagePattern":"Unsupported tokenization: (.+?)","errorType":"exception","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"official/nlp/data/create_finetuning_data.py","lineNumber":398,"sourceCode":"          functools.partial(\n              tagging_data_lib.UdposProcessor,\n              only_use_en_train=FLAGS.tagging_only_use_en_train,\n              only_use_en_dev=FLAGS.only_use_en_dev),\n  }\n  task_name = FLAGS.tagging_task_name.lower()\n  if task_name not in processors:\n    raise ValueError(\"Task not found: %s\" % task_name)\n\n  if FLAGS.tokenization == \"WordPiece\":\n    tokenizer = tokenization.FullTokenizer(\n        vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case)\n    processor_text_fn = tokenization.convert_to_unicode\n  elif FLAGS.tokenization == \"SentencePiece\":\n    tokenizer = tokenization.FullSentencePieceTokenizer(FLAGS.sp_model_file)\n    processor_text_fn = functools.partial(\n        tokenization.preprocess_text, lower=FLAGS.do_lower_case)\n  else:\n    raise ValueError(\"Unsupported tokenization: %s\" % FLAGS.tokenization)\n\n  processor = processors[task_name]()\n  return tagging_data_lib.generate_tf_record_from_data_file(\n      processor, FLAGS.input_data_dir, tokenizer, FLAGS.max_seq_length,\n      FLAGS.train_data_output_path, FLAGS.eval_data_output_path,\n      FLAGS.test_data_output_path, processor_text_fn)\n\n\ndef main(_):\n  if FLAGS.tokenization == \"WordPiece\":\n    if not FLAGS.vocab_file:\n      raise ValueError(\n          \"FLAG vocab_file for word-piece tokenizer is not specified.\")\n  else:\n    assert FLAGS.tokenization == \"SentencePiece\"\n    if not FLAGS.sp_model_file:\n      raise ValueError(\n          \"FLAG sp_model_file for sentence-piece tokenizer is not specified.\")","sourceCodeStart":380,"sourceCodeEnd":416,"githubUrl":"https://github.com/tensorflow/models/blob/e006f5f0d534913e49c1f1dae87364039fa607e2/official/nlp/data/create_finetuning_data.py#L380-L416","documentation":"Error \"Unsupported tokenization: %s\" thrown in tensorflow/models.","triggerScenarios":"Thrown at official/nlp/data/create_finetuning_data.py:398 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"e006f5f0d534913e49c1f1dae87364039fa607e2","analyzedAt":"2026-08-24T14:09:15.576Z","schemaVersion":2},"datasetVersion":"2026-08-24T17:17:21.512Z"}