Do not convert weight scale to e4m3fnuz on CUDA (#2917)

danieldk · web-flow · commit 5f78ec32a5e8 · 2025-01-16T13:44:32.000+01:00
diff --git a/server/text_generation_server/layers/compressed_tensors/w8an_fp.py b/server/text_generation_server/layers/compressed_tensors/w8an_fp.py
@@ -147,7 +147,7 @@ def get_multi_weights_col(self, weights: "Weights", prefixes: List[str], dim: in
                 else None
             )
 
-        if self.load_weight_scale or SYSTEM == "rocm":
+        if self.load_weight_scale and SYSTEM == "rocm":
             w, weight_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
                 w, weight_scale, input_scale
             )

Original file line number	Diff line number	Diff line change
`@@ -147,7 +147,7 @@ def get_multi_weights_col(self, weights: "Weights", prefixes: List[str], dim: in`
`147`	`147`	`else None`
`148`	`148`	`)`
`149`	`149`
`150`		`- if self.load_weight_scale or SYSTEM == "rocm":`
	`150`	`+ if self.load_weight_scale and SYSTEM == "rocm":`
`151`	`151`	`w, weight_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(`
`152`	`152`	`w, weight_scale, input_scale`
`153`	`153`	`)`