{"record":{"id":"cd496c2cebd03c27","repo":"tensorflow/models","slug":"flag-vocab-file-for-word-piece-tokenizer-is-not-sp","errorCode":null,"errorMessage":"FLAG vocab_file for word-piece tokenizer is not specified.","messagePattern":"FLAG vocab_file for word-piece tokenizer is not specified\\.","errorType":"exception","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"official/nlp/data/create_finetuning_data.py","lineNumber":410,"sourceCode":"    processor_text_fn = tokenization.convert_to_unicode\n  elif FLAGS.tokenization == \"SentencePiece\":\n    tokenizer = tokenization.FullSentencePieceTokenizer(FLAGS.sp_model_file)\n    processor_text_fn = functools.partial(\n        tokenization.preprocess_text, lower=FLAGS.do_lower_case)\n  else:\n    raise ValueError(\"Unsupported tokenization: %s\" % FLAGS.tokenization)\n\n  processor = processors[task_name]()\n  return tagging_data_lib.generate_tf_record_from_data_file(\n      processor, FLAGS.input_data_dir, tokenizer, FLAGS.max_seq_length,\n      FLAGS.train_data_output_path, FLAGS.eval_data_output_path,\n      FLAGS.test_data_output_path, processor_text_fn)\n\n\ndef main(_):\n  if FLAGS.tokenization == \"WordPiece\":\n    if not FLAGS.vocab_file:\n      raise ValueError(\n          \"FLAG vocab_file for word-piece tokenizer is not specified.\")\n  else:\n    assert FLAGS.tokenization == \"SentencePiece\"\n    if not FLAGS.sp_model_file:\n      raise ValueError(\n          \"FLAG sp_model_file for sentence-piece tokenizer is not specified.\")\n\n  if FLAGS.fine_tuning_task_type != \"retrieval\":\n    flags.mark_flag_as_required(\"train_data_output_path\")\n\n  if FLAGS.fine_tuning_task_type == \"classification\":\n    input_meta_data = generate_classifier_dataset()\n  elif FLAGS.fine_tuning_task_type == \"regression\":\n    input_meta_data = generate_regression_dataset()\n  elif FLAGS.fine_tuning_task_type == \"retrieval\":\n    input_meta_data = generate_retrieval_dataset()\n  elif FLAGS.fine_tuning_task_type == \"squad\":\n    input_meta_data = generate_squad_dataset()","sourceCodeStart":392,"sourceCodeEnd":428,"githubUrl":"https://github.com/tensorflow/models/blob/e006f5f0d534913e49c1f1dae87364039fa607e2/official/nlp/data/create_finetuning_data.py#L392-L428","documentation":"Error \"FLAG vocab_file for word-piece tokenizer is not specified.\" thrown in tensorflow/models.","triggerScenarios":"Thrown at official/nlp/data/create_finetuning_data.py:410 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"e006f5f0d534913e49c1f1dae87364039fa607e2","analyzedAt":"2026-08-24T14:09:15.576Z","schemaVersion":2},"datasetVersion":"2026-08-24T17:17:21.512Z"}