feat: check before rope type adjustment and small refactors

huggingface · Feb 5, 2025 · 76d526d · 76d526d
1 parent 1f58577
commit 76d526d
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 6 deletions.
diff --git a/launcher/src/main.rs b/launcher/src/main.rs
@@ -2053,7 +2053,7 @@ fn main() -> Result<(), LauncherError> {
                 // this is a short term temporary fix to enable vlms to avoid rejecting images
                 let default_optimal = match config {
                     Some(ref config) => match config.model_type.as_deref() {
-                        Some("qwen2_vl") => 10_000,
+                        Some("qwen2_vl") | Some("qwen2_5_vl") => 10_000,
                         _ => 4096,
                     },
                     None => 4096,

diff --git a/server/text_generation_server/layers/rotary.py b/server/text_generation_server/layers/rotary.py
@@ -86,7 +86,6 @@ def static(cls, config, dim, base, device):
             # `rope_type` is now standard in transformers, but some existing models
             # have `type` instead.
             rope_type = rope_scaling.get("rope_type", rope_scaling.get("type", None))
-            mrope_section = rope_scaling.get("mrope_section", None)
 
             if rope_type == "linear":
                 pass

diff --git a/server/text_generation_server/models/custom_modeling/qwen2_5_vl.py b/server/text_generation_server/models/custom_modeling/qwen2_5_vl.py
@@ -475,7 +475,12 @@ def __init__(self, prefix, config, weights):
         config.vision_config.speculator = config.speculator
         # set rope_scaling.type == "mrope" since AutoConfig.from_pretrained incorrectly
         # returns rope_scaling.type == "default" for Qwen2_5-VL model at the moment
-        config.rope_scaling.update({"rope_type": "mrope"})
+        if (
+            hasattr(config, "rope_scaling")
+            and config.rope_scaling is not None
+            and config.rope_scaling.get("type", None) == "default"
+        ):
+            config.rope_scaling.update({"rope_type": "mrope"})
         self.hidden_size = config.hidden_size
         self.vision_start_token_id = config.vision_start_token_id
         self.vision_end_token_id = config.vision_end_token_id
@@ -616,7 +621,6 @@ def forward(
 
         # apply the visual model to the pixel values if they are provided
         if pixel_values is not None and len(pixel_values) > 0:
-            pixel_values = pixel_values.to(inputs_embeds.dtype)
             if pixel_values is not None:
                 image_embeds = self.visual(
                     pixel_values, grid_thw=image_grid_thw

diff --git a/server/text_generation_server/models/custom_modeling/qwen2_vl.py b/server/text_generation_server/models/custom_modeling/qwen2_vl.py
@@ -379,7 +379,12 @@ def __init__(self, prefix, config, weights):
         config.vision_config.speculator = config.speculator
         # set rope_scaling.type == "mrope" since AutoConfig.from_pretrained incorrectly
         # returns rope_scaling.type == "default" for Qwen2-VL model at the moment
-        config.rope_scaling.update({"rope_type": "mrope"})
+        if (
+            hasattr(config, "rope_scaling")
+            and config.rope_scaling is not None
+            and config.rope_scaling.get("type", None) == "default"
+        ):
+            config.rope_scaling.update({"rope_type": "mrope"})
         self.hidden_size = config.hidden_size
         self.vision_start_token_id = config.vision_start_token_id
         self.vision_end_token_id = config.vision_end_token_id
@@ -520,7 +525,6 @@ def forward(
 
         # apply the visual model to the pixel values if they are provided
         if pixel_values is not None and len(pixel_values) > 0:
-            pixel_values = pixel_values.to(inputs_embeds.dtype)
             if pixel_values is not None:
                 image_embeds = self.visual(
                     pixel_values, grid_thw=image_grid_thw