decouple vision tower

Signed-off-by: Kyle Sayers <[email protected]>

Author: kylesayrs

src/llmcompressor/pipelines/sequential/helpers.py

@@ -277,14 +277,14 @@ def topological_partition(graph: GraphModule, targets: Set[Module]) -> List[List
     while len(queue) > 0:
         node = queue.popleft()
 
-        # assign to partition
-        partitions[partition_index].append(node)
-
         # guarantee targets are assigned to disjoint partitions
-        if node in target_nodes:
+        if node in target_nodes and len(partitions[partition_index]) > 0:
             partition_index += 1
             partitions.append([])
 
+        # assign to partition
+        partitions[partition_index].append(node)
+
         # recurse on last indegree only in order to guarantee that
         # the node is assigned to maximal partition
         for user in node.users: