{"record":{"id":"40a43057a892b3ef","repo":"weaviate/weaviate","slug":"tokenization-is-not-allowed-for-data-type-s","errorCode":null,"errorMessage":"tokenization is not allowed for data type '%s'","messagePattern":"tokenization is not allowed for data type '(.+?)'","errorType":"validation","errorClass":null,"httpStatus":null,"severity":"error","filePath":"usecases/schema/class.go","lineNumber":1272,"sourceCode":"\t\t\t\t\treturn fmt.Errorf(\"the Chinese tokenizer is not enabled; set 'ENABLE_TOKENIZER_GSE_CH' to 'true' to enable\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\tcase models.PropertyTokenizationKagomeKr:\n\t\t\t\tif !entcfg.Enabled(os.Getenv(\"ENABLE_TOKENIZER_KAGOME_KR\")) {\n\t\t\t\t\treturn fmt.Errorf(\"the Korean tokenizer is not enabled; set 'ENABLE_TOKENIZER_KAGOME_KR' to 'true' to enable\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\tcase models.PropertyTokenizationKagomeJa:\n\t\t\t\tif !entcfg.Enabled(os.Getenv(\"ENABLE_TOKENIZER_KAGOME_JA\")) {\n\t\t\t\t\treturn fmt.Errorf(\"the Japanese tokenizer is not enabled; set 'ENABLE_TOKENIZER_KAGOME_JA' to 'true' to enable\")\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}\n\t\tdefault:\n\t\t\tif tokenization == \"\" {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"tokenization is not allowed for data type '%s'\", primitiveDataType)\n\t\t}\n\t\treturn fmt.Errorf(\"tokenization '%s' is not allowed for data type '%s'\", tokenization, primitiveDataType)\n\t}\n\n\tif tokenization == \"\" {\n\t\treturn nil\n\t}\n\n\tif propertyDataType.IsNested() {\n\t\treturn fmt.Errorf(\"tokenization is not allowed for object/object[] data types\")\n\t}\n\treturn fmt.Errorf(\"tokenization is not allowed for reference data type\")\n}\n\nfunc (h *Handler) validatePropertyIndexing(prop *models.Property) error {\n\tif prop.IndexInverted != nil {\n\t\tif prop.IndexFilterable != nil || prop.IndexSearchable != nil || prop.IndexRangeFilters != nil {\n\t\t\treturn fmt.Errorf(\"`indexInverted` is deprecated and can not be set together with `indexFilterable`, \" + \"`indexSearchable` or `indexRangeFilters`\")","sourceCodeStart":1254,"sourceCodeEnd":1290,"githubUrl":"https://github.com/weaviate/weaviate/blob/75aa4b6d11f8818305aafd4440b4e32794f7ca04/usecases/schema/class.go#L1254-L1290","documentation":"Tokenization settings only apply to text-like properties (string/text). When a property has a non-text primitive dataType (int, number, boolean, date, etc.) and a non-empty tokenization value, validatePropertyTokenization rejects the schema with this error.","triggerScenarios":"Creating/updating a class where a property with dataType such as 'int', 'number', 'boolean', 'date', 'uuid', or a reference type carries a tokenization field (even 'word' or 'whitespace').","commonSituations":"Copy-pasting a full property JSON including tokenization from a text property onto a numeric/date property; code generators that always emit tokenization; bulk schema templates with tokenization on every property.","solutions":["Remove the tokenization field from the non-text property","If tokenization is truly needed, change the property dataType to 'text' or 'string'","Normalize schema templates to strip tokenization for non-text dataTypes before submitting"],"exampleFix":"// before\n{\"name\": \"age\", \"dataType\": [\"int\"], \"tokenization\": \"word\"} // fails\n// after\n{\"name\": \"age\", \"dataType\": [\"int\"]}","handlingStrategy":"validation","validationCode":"textual := map[string]bool{\"text\": true, \"string\": true}\nif prop.Tokenization != \"\" && !textual[prop.DataType[0]] {\n\treturn fmt.Errorf(\"property %s: tokenization not allowed for dataType %s\", prop.Name, prop.DataType[0])\n}","typeGuard":"func tokenizationAllowed(p *models.Property) bool {\n\tswitch p.DataType[0] {\n\tcase \"text\", \"string\":\n\t\treturn true\n\tdefault:\n\t\treturn p.Tokenization == \"\"\n\t}\n}","tryCatchPattern":"err := client.Schema().ClassCreator().WithClass(class).Do(ctx)\nif err != nil && strings.Contains(err.Error(), \"tokenization is not allowed for data type\") {\n\t// strip tokenization from non-text properties and retry\n}","preventionTips":["Emit tokenization only for text/string dataTypes in schema generators","Strip the tokenization field when copying properties between classes","Add schema template linting that flags tokenization on non-text properties"],"tags":["schema","tokenization","validation","properties"],"backgroundTag":"tokenization-not-allowed-for-datatype","analyzedSha":"75aa4b6d11f8818305aafd4440b4e32794f7ca04","analyzedAt":"2026-09-04T14:58:20.392Z","contentChangedAt":"2026-09-04T14:58:20.392Z","schemaVersion":2},"datasetVersion":"2026-09-11T21:17:09.523Z"}