{"record":{"id":"0fcb07bf38e0e03b","repo":"huggingface/transformers","slug":"assisted-generate-is-only-supported-for-batch-size","errorCode":null,"errorMessage":"assisted generate is only supported for batch_size = 1","messagePattern":"assisted generate is only supported for batch_size = 1","errorType":"exception","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"src/transformers/generation/utils.py","lineNumber":3670,"sourceCode":"\n        # init attention / hidden states / scores tuples\n        scores = () if (return_dict_in_generate and output_scores) else None\n        raw_logits = () if (return_dict_in_generate and output_logits) else None\n        decoder_attentions = () if (return_dict_in_generate and output_attentions) else None\n        cross_attentions = () if (return_dict_in_generate and output_attentions) else None\n        decoder_hidden_states = () if (return_dict_in_generate and output_hidden_states) else None\n\n        # if model is an encoder-decoder, retrieve encoder attention weights and hidden states\n        if return_dict_in_generate and self.config.is_encoder_decoder:\n            encoder_attentions = model_kwargs[\"encoder_outputs\"].get(\"attentions\") if output_attentions else None\n            encoder_hidden_states = (\n                model_kwargs[\"encoder_outputs\"].get(\"hidden_states\") if output_hidden_states else None\n            )\n\n        # keep track of which sequences are already finished\n        batch_size, cur_len = input_ids.shape[:2]\n        if batch_size > 1:\n            raise ValueError(\"assisted generate is only supported for batch_size = 1\")\n        unfinished_sequences = torch.ones(batch_size, dtype=torch.long, device=input_ids.device)\n\n        this_peer_finished = False\n        is_first_iteration = True  # to preserve the same API in the output as other generation methods\n        outputs = None\n        n_matches = 0\n        while self._has_unfinished_sequences(this_peer_finished, synced_gpus, device=input_ids.device):\n            cur_len = input_ids.shape[1]\n\n            #  1. Fetch candidate sequences from a `CandidateGenerator` and move to the correct device\n            if candidate_generator.requires_model_outputs:\n                candidate_input_ids, candidate_logits = candidate_generator.get_candidates(\n                    input_ids,\n                    model_kwargs=model_kwargs,\n                    model_outputs=outputs,\n                    is_first_iteration=is_first_iteration,\n                    n_last_matches=n_matches,\n                )","sourceCodeStart":3652,"sourceCodeEnd":3688,"githubUrl":"https://github.com/huggingface/transformers/blob/a597f974857b3d92939971296bc0deb93d33d780/src/transformers/generation/utils.py#L3652-L3688","documentation":"Error \"assisted generate is only supported for batch_size = 1\" thrown in huggingface/transformers.","triggerScenarios":"Raised in assisted generation when the batch size is greater than 1.","commonSituations":"Passing multiple input sequences to generate() together with assistant_model; assisted decoding only supports batch_size=1.","solutions":["Run assisted generation with batch size 1.","Loop over batch items individually if you need batched inputs."],"exampleFix":null,"handlingStrategy":null,"validationCode":null,"typeGuard":null,"tryCatchPattern":null,"preventionTips":[],"tags":[],"backgroundTag":null,"analyzedSha":"a597f974857b3d92939971296bc0deb93d33d780","analyzedAt":"2026-08-14T18:24:08.354Z","schemaVersion":2},"datasetVersion":"2026-08-15T17:31:12.345Z"}