QuantLinear is rocm compatible.

huggingface · Oct 23, 2024 · 849d882 · 849d882
1 parent 82a6cb8
commit 849d882
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/server/text_generation_server/layers/gptq/__init__.py b/server/text_generation_server/layers/gptq/__init__.py
@@ -10,7 +10,7 @@
 
 if SYSTEM == "ipex":
     from .ipex import QuantLinear
-elif SYSTEM == "cuda":
+elif SYSTEM in {"cuda", "rocm"}:
     from .cuda import QuantLinear