{"record":{"id":"02f0ece240160f1e","repo":"huggingface/transformers","slug":"cannot-use-prefill-chunking-without-a-cache","errorCode":null,"errorMessage":"Cannot use prefill chunking without a cache","messagePattern":"Cannot use prefill chunking without a cache","errorType":"exception","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"src/transformers/generation/utils.py","lineNumber":3960,"sourceCode":"            model_inputs = self.prepare_inputs_for_generation(\n                input_ids,\n                next_sequence_length=next_sequence_length,\n                is_first_iteration=is_first_iteration,\n                **model_kwargs,\n            )\n            return self(**model_inputs, return_dict=True)\n\n        # Chunked prefill (for very large contexts)\n        else:\n            # Even if we are not compiling the forward, flex is always compiled when used. With chunked prefill, we may\n            # end up needing just a bit more graphs than the default (which is 8). Doing this avoids very cryptic warnings\n            getattr(torch, \"_dynamo\").config.cache_size_limit = 64\n\n            chunk_size = generation_config.prefill_chunk_size\n            input_chunks = torch.split(input_ids, chunk_size, dim=-1)\n\n            if \"past_key_values\" not in model_kwargs:\n                raise ValueError(\"Cannot use prefill chunking without a cache\")\n\n            model_forward = (\n                self.get_compiled_call(generation_config.compile_config)\n                if self._valid_auto_compile_criteria(model_kwargs, generation_config)\n                else self.__call__\n            )\n\n            attention_mask = model_kwargs.pop(\"attention_mask\", None)\n            position_ids = model_kwargs.pop(\"position_ids\", None)\n            past_length = 0\n            for input_chunk in input_chunks:\n                current_length = past_length + input_chunk.shape[-1]\n                if attention_mask is not None:\n                    model_kwargs[\"attention_mask\"] = attention_mask[:, :current_length]\n                if position_ids is not None:\n                    model_kwargs[\"position_ids\"] = position_ids[:, past_length:current_length]\n                model_inputs = self.prepare_inputs_for_generation(input_chunk, **model_kwargs)\n","sourceCodeStart":3942,"sourceCodeEnd":3978,"githubUrl":"https://github.com/huggingface/transformers/blob/a597f974857b3d92939971296bc0deb93d33d780/src/transformers/generation/utils.py#L3942-L3978","documentation":"Error \"Cannot use prefill chunking without a cache\" thrown in huggingface/transformers.","triggerScenarios":"Raised in generate() when prefill chunking is enabled but no cache is in use.","commonSituations":"Setting a prefill chunk size while use_cache=False or on a model without cache support.","solutions":["Enable `use_cache=True` before using prefill chunking.","Disable prefill chunking if caching is unavailable."],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"a597f974857b3d92939971296bc0deb93d33d780","analyzedAt":"2026-08-14T18:24:08.354Z","schemaVersion":2},"datasetVersion":"2026-08-15T22:17:37.221Z"}