{"record":{"id":"9cd5a41939bf0bd7","repo":"tensorflow/models","slug":"tokenize-with-offsets-is-not-supported-when-str","errorCode":null,"errorMessage":"`tokenize_with_offsets` is not supported when `strip_diacritics` is set to True.","messagePattern":"`tokenize_with_offsets` is not supported when `strip_diacritics` is set to True\\.","errorType":"exception","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"official/nlp/modeling/layers/text_layers.py","lineNumber":303,"sourceCode":"        diacritics requires additional text normalization and dropping bytes,\n        which makes it impossible to keep track of the offsets now. Hence\n        when `strip_diacritics` is set to True, we don't yet support\n        `tokenize_with_offsets`. NOTE: New models are encouraged to put this\n        into custom normalization rules for the Sentencepiece model itself to\n        avoid this extra step and the limitation regarding offsets.\n      **kwargs: standard arguments to `Layer()`.\n\n    Raises:\n      ImportError: if importing tensorflow_text failed.\n    \"\"\"\n    _check_if_tf_text_installed()\n    super().__init__(**kwargs)\n    if bool(model_file_path) == bool(model_serialized_proto):\n      raise ValueError(\"Exact one of `model_file_path` and \"\n                       \"`model_serialized_proto` can be specified.\")\n    # TODO(b/181866850): Support tokenize_with_offsets for strip_diacritics=True\n    if tokenize_with_offsets and strip_diacritics:\n      raise ValueError(\"`tokenize_with_offsets` is not supported when \"\n                       \"`strip_diacritics` is set to True.\")\n    if model_file_path:\n      self._model_serialized_proto = tf.io.gfile.GFile(model_file_path,\n                                                       \"rb\").read()\n    else:\n      self._model_serialized_proto = model_serialized_proto\n\n    self._lower_case = lower_case\n    self.tokenize_with_offsets = tokenize_with_offsets\n    self._nbest_size = nbest_size\n    self._alpha = alpha\n    self._strip_diacritics = strip_diacritics\n    self._tokenizer = self._create_tokenizer()\n    self._special_tokens_dict = self._create_special_tokens_dict()\n\n  def _create_tokenizer(self):\n    return text.SentencepieceTokenizer(  # pyrefly: ignore[missing-attribute]\n        model=self._model_serialized_proto,","sourceCodeStart":285,"sourceCodeEnd":321,"githubUrl":"https://github.com/tensorflow/models/blob/e006f5f0d534913e49c1f1dae87364039fa607e2/official/nlp/modeling/layers/text_layers.py#L285-L321","documentation":"Error \"`tokenize_with_offsets` is not supported when `strip_diacritics` is set to True.\" thrown in tensorflow/models.","triggerScenarios":"Thrown at official/nlp/modeling/layers/text_layers.py:303 when the library encounters an invalid state.","commonSituations":"See trigger scenarios.","solutions":[],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"e006f5f0d534913e49c1f1dae87364039fa607e2","analyzedAt":"2026-08-24T14:09:15.576Z","schemaVersion":2},"datasetVersion":"2026-08-24T17:17:21.512Z"}