ENH Small speedups to adapter injection (#2785)

See
huggingface/diffusers#11816 (comment)

This PR implements two small improvements to the speed of adapter
injection. On a benchmark based on the linked issue, the first change
leads to a speedup of 21% and the second change of another 3%. It's not
that much, but as the changes don't make the code more complicated,
there is really no reason not to take them.

The optimizations don't add any functional change but are simply based
on not recomputing the same values multiple times. Therefore, unless I'm
missing something, they should strictly improve runtime.

Author: BenjaminBossan

src/peft/tuners/tuners_utils.py

@@ -668,10 +668,9 @@ def inject_adapter(
             and (len(peft_config.target_modules) >= MIN_TARGET_MODULES_FOR_OPTIMIZATION)
             and (peft_config.peft_type != PeftType.IA3)
         ):
+            suffixes = tuple("." + suffix for suffix in peft_config.target_modules)
             names_no_target = [
-                name
-                for name in key_list
-                if not any((name == suffix) or name.endswith("." + suffix) for suffix in peft_config.target_modules)
+                name for name in key_list if (name not in peft_config.target_modules) and not name.endswith(suffixes)
             ]
             new_target_modules = _find_minimal_target_modules(peft_config.target_modules, names_no_target)
             if len(new_target_modules) < len(peft_config.target_modules):
@@ -681,10 +680,10 @@ def inject_adapter(
         # MATCHING & CREATING MODULES #
         ###############################
 
-        existing_adapter_map = {}
+        existing_adapter_prefixes = []
         for key, module in named_modules:
             if isinstance(module, BaseTunerLayer):
-                existing_adapter_map[key] = module
+                existing_adapter_prefixes.append(key + ".")
 
         # TODO: check if this the most robust way
         module_names: set[str] = set()
@@ -698,8 +697,8 @@ def inject_adapter(
 
             # It is possible that we're adding an additional adapter, so if we encounter a key that clearly belongs to a
             # previous adapter we can skip here since we don't want to interfere with adapter internals.
-            for adapter_key in existing_adapter_map:
-                if key.startswith(adapter_key + "."):
+            for adapter_key in existing_adapter_prefixes:
+                if key.startswith(adapter_key):
                     excluded_modules.append(key)
                     break