{"record":{"id":"ce741316161de226","repo":"tensorflow/models","slug":"unsupported-tokenization-s-ce7413","errorCode":null,"errorMessage":"Unsupported tokenization: %s","messagePattern":"Unsupported tokenization: (.+?)","errorType":"exception","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"official/nlp/data/sentence_prediction_dataloader.py","lineNumber":180,"sourceCode":"               preprocessing_hub_module_url: Optional[str] = None):\n    if preprocessing_hub_module_url:\n      self._preprocessing_hub_module = hub.load(preprocessing_hub_module_url)\n      self._tokenizer = self._preprocessing_hub_module.tokenize\n      self._pack_inputs = functools.partial(\n          self._preprocessing_hub_module.bert_pack_inputs,\n          seq_length=seq_length)\n      return\n\n    if tokenization == 'WordPiece':\n      self._tokenizer = modeling.layers.BertTokenizer(\n          vocab_file=vocab_file, lower_case=lower_case)\n    elif tokenization == 'SentencePiece':\n      self._tokenizer = modeling.layers.SentencepieceTokenizer(\n          model_file_path=vocab_file,\n          lower_case=lower_case,\n          strip_diacritics=True)  # Strip diacritics to follow ALBERT model\n    else:\n      raise ValueError('Unsupported tokenization: %s' % tokenization)\n\n    self._pack_inputs = modeling.layers.BertPackInputs(\n        seq_length=seq_length,\n        special_tokens_dict=self._tokenizer.get_special_tokens_dict())\n\n  def __call__(self, segments):\n    segments = [self._tokenizer(s) for s in segments]\n    # BertTokenizer returns a RaggedTensor with shape [batch, word, subword],\n    # and SentencepieceTokenizer returns a RaggedTensor with shape\n    # [batch, sentencepiece],\n    segments = [\n        tf.cast(x.merge_dims(1, -1) if x.shape.rank > 2 else x, tf.int32)\n        for x in segments\n    ]\n    return self._pack_inputs(segments)\n\n\n@data_loader_factory.register_data_loader_cls(SentencePredictionTextDataConfig)","sourceCodeStart":162,"sourceCodeEnd":198,"githubUrl":"https://github.com/tensorflow/models/blob/e006f5f0d534913e49c1f1dae87364039fa607e2/official/nlp/data/sentence_prediction_dataloader.py#L162-L198","documentation":"Error \"Unsupported tokenization: %s\" thrown in tensorflow/models.","triggerScenarios":"Thrown at official/nlp/data/sentence_prediction_dataloader.py:180 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"e006f5f0d534913e49c1f1dae87364039fa607e2","analyzedAt":"2026-08-24T14:09:15.576Z","schemaVersion":2},"datasetVersion":"2026-08-24T17:17:21.512Z"}