TF_upgrade

Signed-off-by: Mohit Soni <[email protected]>

Author: Mohit Soni

QEfficient/transformers/models/pytorch_transforms.py

@@ -139,6 +139,18 @@
     Qwen2Model,
     Qwen2RMSNorm,
 )
+from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import (
+    Qwen2_5_VisionTransformerPretrainedModel,
+    Qwen2_5_VLAttention,
+    Qwen2_5_VLDecoderLayer,
+    Qwen2_5_VLForConditionalGeneration,
+    Qwen2_5_VLModel,
+    Qwen2_5_VLTextModel,
+    Qwen2_5_VLVisionAttention,
+)
+from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import (
+    Qwen2RMSNorm as Qwen2_5RMSNorm,
+)
 from transformers.models.qwen3.modeling_qwen3 import (
     Qwen3Attention,
     Qwen3DecoderLayer,
@@ -155,17 +167,6 @@
     Qwen3MoeRotaryEmbedding,
     Qwen3MoeSparseMoeBlock,
 )
-from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import (
-    Qwen2_5_VisionTransformerPretrainedModel,
-    Qwen2_5_VLAttention,
-    Qwen2_5_VLDecoderLayer,
-    Qwen2_5_VLForConditionalGeneration,
-    Qwen2_5_VLModel,
-    Qwen2_5_VLVisionAttention,
-)
-from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import (
-    Qwen2RMSNorm as Qwen2_5RMSNorm,
-)
 from transformers.models.starcoder2.modeling_starcoder2 import (
     Starcoder2Attention,
     Starcoder2DecoderLayer,
@@ -336,6 +337,15 @@
     QEffQwen2ForCausalLM,
     QEffQwen2Model,
 )
+from QEfficient.transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import (
+    QEffQwen2_5_VisionTransformerPretrainedModel,
+    QEffQwen2_5_VLAttention,
+    QEffQwen2_5_VLDecoderLayer,
+    QEffQwen2_5_VLModel,
+    QEffQwen2_5_VLTextModel,
+    QEffQwen2_5_VLVisionAttention,
+    QEffQwen_2_5_vl_ForConditionalGeneration,
+)
 from QEfficient.transformers.models.qwen3.modeling_qwen3 import (
     QEffQwen3Attention,
     QEffQwen3DecoderLayer,
@@ -350,14 +360,6 @@
     QEffQwen3MoeRotaryEmbedding,
     QEffQwen3MoeSparseMoeBlock,
 )
-from QEfficient.transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import (
-    QEffQwen2_5_VisionTransformerPretrainedModel,
-    QEffQwen2_5_VLAttention,
-    QEffQwen2_5_VLDecoderLayer,
-    QEffQwen2_5_VLModel,
-    QEffQwen2_5_VLVisionAttention,
-    QEffQwen_2_5_vl_ForConditionalGeneration,
-)
 from QEfficient.transformers.models.starcoder2.modeling_starcoder2 import (
     QEffStarcoder2Attention,
     QEFFStarcoder2DecoderLayer,
@@ -532,6 +534,7 @@ class KVCacheTransform(ModuleMappingTransform):
         Qwen2_5_VLDecoderLayer: QEffQwen2_5_VLDecoderLayer,
         Qwen2_5_VisionTransformerPretrainedModel: QEffQwen2_5_VisionTransformerPretrainedModel,
         Qwen2_5_VLVisionAttention: QEffQwen2_5_VLVisionAttention,
+        Qwen2_5_VLTextModel: QEffQwen2_5_VLTextModel,
         # Starcoder2
         Starcoder2Attention: QEffStarcoder2Attention,
         Starcoder2DecoderLayer: QEFFStarcoder2DecoderLayer,

QEfficient/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py

@@ -14,7 +14,9 @@
     Qwen2_5_VLAttention,
     Qwen2_5_VLConfig,
     Qwen2_5_VLDecoderLayer,
+    Qwen2_5_VLModelOutputWithPast,
     Qwen2_5_VLRotaryEmbedding,
+    Qwen2_5_VLTextModel,
     Qwen2_5_VLVisionAttention,
     apply_rotary_pos_emb_vision,
     repeat_kv,
@@ -393,6 +395,7 @@ def forward(
         batch_index: Optional[torch.LongTensor] = None,
         output_attentions: bool = False,
         use_cache: bool = False,
+        cache_position: Optional[torch.LongTensor] = None,
         **kwargs,
     ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
         bsz, q_len, _ = hidden_states.size()
@@ -406,7 +409,7 @@ def forward(
         value_states = value_states.view(bsz, q_len, -1, self.head_dim).transpose(1, 2)
 
         kv_seq_len = key_states.shape[-2]
-        kv_seq_len = past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
+        kv_seq_len = past_key_value.get_seq_length(self.layer_idx, cache_position)
 
         cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
 
@@ -490,7 +493,7 @@ def forward(
             output_attentions=output_attentions,
             use_cache=use_cache,
             cache_position=cache_position,
-            # position_embeddings=position_embeddings,
+            **kwargs,
         )
         hidden_states = residual + hidden_states
 
@@ -511,7 +514,7 @@ def forward(
         return outputs
 
 
-class QEffQwen2_5_VLModel(Qwen2_5_VLModel):
+class QEffQwen2_5_VLTextModel(Qwen2_5_VLTextModel):
     def forward(
         self,
         input_ids: Optional[torch.LongTensor] = None,
@@ -525,6 +528,7 @@ def forward(
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.LongTensor] = None,
+        **kwargs,
     ) -> Union[Tuple, BaseModelOutputWithPast]:
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
@@ -571,6 +575,7 @@ def forward(
                 output_attentions=output_attentions,
                 use_cache=use_cache,
                 cache_position=cache_position,
+                **kwargs,
             )
 
             hidden_states = layer_outputs[0]
@@ -587,13 +592,66 @@ def forward(
         if return_legacy_cache:
             past_key_values = past_key_values.to_legacy_cache()
 
-        # Cast to INT32 to avoid issue while running in ONNXRT
-        logit_index = position_ids[0].to(torch.int32).argmax(1, keepdim=True)
-        hidden_states = hidden_states[torch.arange(position_ids[0].shape[0]).view(-1, 1), logit_index]
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=past_key_values,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attns,
+        )
 
         return (hidden_states, past_key_values)
 
 
+class QEffQwen2_5_VLModel(Qwen2_5_VLModel):
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        batch_index: Optional[torch.LongTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        **kwargs,
+    ) -> Union[Tuple, BaseModelOutputWithPast]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if inputs_embeds is None:
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+        outputs = self.language_model(
+            input_ids=None,
+            position_ids=position_ids,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            batch_index=batch_index,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=True,
+            cache_position=cache_position,
+            **kwargs,
+        )
+
+        output = Qwen2_5_VLModelOutputWithPast(
+            last_hidden_state=outputs.last_hidden_state,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            rope_deltas=self.rope_deltas,
+        )
+        return output if return_dict else output.to_tuple()
+
+
 class QEffQwen_2_5_vl_EncoderWrapper(nn.Module):
     def __init__(self, model):
         super().__init__()
@@ -613,7 +671,7 @@ class QEffQwen_2_5_vl_DecoderWrapper(nn.Module):
     def __init__(self, model):
         super().__init__()
         self.model = model
-        self.language_model = self.model.model
+        self.language_model = self.model.model.language_model
 
     def forward(self, input_ids, vision_embeds, position_ids, image_idx, past_key_values):
         inputs_embeds = self.model.get_input_embeddings()(input_ids)
@@ -628,10 +686,13 @@ def forward(self, input_ids, vision_embeds, position_ids, image_idx, past_key_va
         outputs = self.model.model(
             inputs_embeds=inputs_embeds, position_ids=position_ids, past_key_values=past_key_values, use_cache=True
         )
-        logits = self.model.lm_head(outputs[0])
+
+        logit_index = position_ids[0].to(torch.int32).argmax(1, keepdim=True)
+        hidden_states = outputs.last_hidden_state[torch.arange(position_ids[0].shape[0]).view(-1, 1), logit_index]
+        logits = self.model.lm_head(hidden_states)
         image_idx = (indices1.max() + 1).unsqueeze(0).unsqueeze(0)
 
-        return logits, vision_embeds, image_idx, outputs[1]
+        return logits, vision_embeds, image_idx, outputs.past_key_values
 
 
 class QEffQwen_2_5_vl_ForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):

README.md

@@ -7,6 +7,9 @@
 
 *Latest news* :fire: <br>
 
+- [10/2025] Added support for Qwen2.5VL Multi-Model [Qwen/Qwen2.5-VL-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct)
+- [10/2025] Added support for Mistral3 Multi-Model [mistralai/Mistral-Small-3.1-24B-Instruct-2503](https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503)
+- [10/2025] Added support for Molmo Multi-Model [allenai/Molmo-7B-D-0924](https://huggingface.co/allenai/Molmo-7B-D-0924)
 - [06/2025] Added support for Llama4 Multi-Model [meta-llama/Llama-4-Scout-17B-16E-Instruct](https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct)
 - [06/2025] Added support for Gemma3 Multi-Modal-Model [google/gemma-3-4b-it](https://huggingface.co/google/gemma-3-4b-it)
 - [06/2025] Added support of model `hpcai-tech/grok-1` [hpcai-tech/grok-1](https://huggingface.co/hpcai-tech/grok-1)

examples/qwen2_5_vl_example.py

@@ -15,11 +15,11 @@
 
 from QEfficient import QEFFAutoModelForImageTextToText
 
+## For AWQ model update pytorch version to 2.8.*
 model_id = "Qwen/Qwen2.5-VL-32B-Instruct"
 config = AutoConfig.from_pretrained(model_id)
 
-# For Testing Purpose Only
-config.num_hidden_layers = 1
+## Use complete model without changing num_hidden_layers as it will not work for TF version 4.55.0 for Qwen2.5VL model
 
 qeff_model = QEFFAutoModelForImageTextToText.from_pretrained(
     model_id, attn_implementation="eager", kv_offload=True, config=config
@@ -28,7 +28,7 @@
 processor = AutoProcessor.from_pretrained(model_id)
 
 ### use skip_vision=Ture, if want to run only text, ow false ###
-skip_vision = True
+skip_vision = False
 
 if skip_vision:
     ## Only Text ##
@@ -152,7 +152,7 @@
 
     inputs["position_ids"] = torch.arange(input_ids_length).view(1, 1, input_ids_length).expand(-1, batch_size, -1)
 
-    pos_ids, rope_deltas = qeff_model.model.get_rope_index(
+    pos_ids, rope_deltas = qeff_model.model.model.get_rope_index(
         inputs["input_ids"],
         inputs["image_grid_thw"],
         video_grid_thw=None,

tests/transformers/models/qnn_config.json

@@ -0,0 +1,10 @@
+{
+    "SKIP_QNN_CONVERTER_STEP":false,
+    "context_binary_generator_args_extension":"--log_level debug",
+    "converter_args_extension":"--onnx_defer_loading",
+    "qnn_compilation_backend":{
+        "compiler_enable_depth_first":true,
+        "compiler_printDDRStats":false,
+        "compiler_printPerfMetrics":false
+    }
+}