{"record":{"id":"51861bfcfdcfd86a","repo":"sgl-project/sglang","slug":"unsupported-dtype-dtype-supported-float16-bfl-51861b","errorCode":null,"errorMessage":"Unsupported dtype {dtype}. Supported: float16, bfloat16, float32","messagePattern":"Unsupported dtype (.+?)\\. Supported: float16, bfloat16, float32","errorType":"validation","errorClass":"RuntimeError","httpStatus":null,"severity":"error","filePath":"python/sglang/kernels/ops/quantization/per_token_quant_fp8.py","lineNumber":22,"sourceCode":"\nimport torch\n\nfrom sglang.kernels.jit.utils import (\n    cache_once,\n    get_jit_cuda_arch,\n    load_jit,\n    make_cpp_args,\n)\nfrom sglang.srt.utils.custom_op import register_custom_op\n\nif TYPE_CHECKING:\n    from tvm_ffi.module import Module\n\n\n@cache_once\ndef _jit_per_token_quant_fp8_module(dtype: torch.dtype) -> Module:\n    if dtype not in (torch.float16, torch.bfloat16, torch.float32):\n        raise RuntimeError(\n            f\"Unsupported dtype {dtype}. Supported: float16, bfloat16, float32\"\n        )\n    arch = get_jit_cuda_arch()\n    use_fast_math = (arch.major, arch.minor) == (9, 0)\n    math_mode = \"fast_math\" if use_fast_math else \"precise_math\"\n    args = make_cpp_args(dtype)\n    return load_jit(\n        \"per_token_quant_fp8\",\n        math_mode,\n        *args,\n        cuda_files=[\"gemm/per_token_quant_fp8.cuh\"],\n        cuda_wrappers=[(\"per_token_quant_fp8\", f\"per_token_quant_fp8<{args}>\")],\n        extra_cuda_cflags=[\"--use_fast_math\"] if use_fast_math else [],\n    )\n\n\n@register_custom_op(\n    op_name=\"per_token_quant_fp8\",","sourceCodeStart":4,"sourceCodeEnd":40,"githubUrl":"https://github.com/sgl-project/sglang/blob/0132848349585cfe6aae51c4941cbae872505f8a/python/sglang/kernels/ops/quantization/per_token_quant_fp8.py#L4-L40","documentation":"The per-token FP8 quant kernel is JIT-compiled per input dtype via tvm_ffi; only float16, bfloat16 and float32 inputs have generated CUDA source. Any other dtype (fp8, int, fp64) hits this RuntimeError during module compilation lookup.","triggerScenarios":"Calling per_token_quant_fp8 with an input tensor whose dtype is not in {float16, bfloat16, float32}, e.g. already-quantized fp8 data or int32 activations.","commonSituations":"Double-quantizing (feeding an FP8 tensor back into the quant op), stray .double() casts in preprocessing, or test fixtures creating tensors with default dtypes.","solutions":["Cast input to float16/bfloat16/float32 before calling per_token_quant_fp8","Skip the call if the tensor is already fp8"],"exampleFix":"// before\nq, s = per_token_quant_fp8(x_fp8)\n// after\nq, s = per_token_quant_fp8(x.to(torch.bfloat16))","handlingStrategy":"type-guard","validationCode":"if x.dtype not in (torch.float16, torch.bfloat16, torch.float32):\n    x = x.to(torch.bfloat16)","typeGuard":"def quantizable_dtype(t): return t.dtype in (torch.float16, torch.bfloat16, torch.float32)","tryCatchPattern":null,"preventionTips":["Skip quantization for already-fp8 tensors","Avoid double-casting to double/int in preprocessing"],"tags":["fp8","quantization","jit","unsupported-dtype"],"backgroundTag":"unsupported-dtype","analyzedSha":"0132848349585cfe6aae51c4941cbae872505f8a","analyzedAt":"2026-08-28T05:10:05.995Z","schemaVersion":2},"datasetVersion":"2026-08-28T06:17:29.519Z"}