Skip to content

Commit d67a464

Browse files
authored
gguf: add custom Q_K_XL quants (#1452)
As reported by @unslothai , these quant types are custom (not officially exist in llama.cpp). Adding these to the list so it will be recognized via registry API After this PR is merged, we will also push the change to main code base (aka moon-landing)
1 parent 29150f4 commit d67a464

File tree

1 file changed

+18
-3
lines changed

1 file changed

+18
-3
lines changed

packages/tasks/src/gguf.ts

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,15 @@ export enum GGMLFileQuantizationType {
4040
Q4_0_8_8 = 35,
4141
TQ1_0 = 36,
4242
TQ2_0 = 37,
43+
44+
// custom quants used by unsloth
45+
// they are not officially a scheme enum value in GGUF, but only here for naming
46+
Q2_K_XL = 1000,
47+
Q3_K_XL = 1001,
48+
Q4_K_XL = 1002,
49+
Q5_K_XL = 1003,
50+
Q6_K_XL = 1004,
51+
Q8_K_XL = 1005,
4352
}
4453

4554
const ggufQuants = Object.values(GGMLFileQuantizationType).filter((v): v is string => typeof v === "string");
@@ -58,32 +67,37 @@ export const GGUF_QUANT_ORDER: GGMLFileQuantizationType[] = [
5867
GGMLFileQuantizationType.F32,
5968
GGMLFileQuantizationType.BF16,
6069
GGMLFileQuantizationType.F16,
70+
GGMLFileQuantizationType.Q8_K_XL,
6171
GGMLFileQuantizationType.Q8_0,
6272

6373
// 6-bit quantizations
74+
GGMLFileQuantizationType.Q6_K_XL,
6475
GGMLFileQuantizationType.Q6_K,
6576

6677
// 5-bit quantizations
67-
GGMLFileQuantizationType.Q5_0,
68-
GGMLFileQuantizationType.Q5_1,
78+
GGMLFileQuantizationType.Q5_K_XL,
6979
GGMLFileQuantizationType.Q5_K_M,
7080
GGMLFileQuantizationType.Q5_K_S,
81+
GGMLFileQuantizationType.Q5_0,
82+
GGMLFileQuantizationType.Q5_1,
7183

7284
// 4-bit quantizations
85+
GGMLFileQuantizationType.Q4_K_XL,
7386
GGMLFileQuantizationType.Q4_K_M,
7487
GGMLFileQuantizationType.Q4_K_S,
7588
GGMLFileQuantizationType.IQ4_NL,
7689
GGMLFileQuantizationType.IQ4_XS,
7790
GGMLFileQuantizationType.Q4_0_4_4,
7891
GGMLFileQuantizationType.Q4_0_4_8,
7992
GGMLFileQuantizationType.Q4_0_8_8,
80-
GGMLFileQuantizationType.Q4_0,
8193
GGMLFileQuantizationType.Q4_1_SOME_F16,
94+
GGMLFileQuantizationType.Q4_0,
8295
GGMLFileQuantizationType.Q4_1,
8396
GGMLFileQuantizationType.Q4_2,
8497
GGMLFileQuantizationType.Q4_3,
8598

8699
// 3-bit quantizations
100+
GGMLFileQuantizationType.Q3_K_XL,
87101
GGMLFileQuantizationType.Q3_K_L,
88102
GGMLFileQuantizationType.Q3_K_M,
89103
GGMLFileQuantizationType.Q3_K_S,
@@ -93,6 +107,7 @@ export const GGUF_QUANT_ORDER: GGMLFileQuantizationType[] = [
93107
GGMLFileQuantizationType.IQ3_XXS,
94108

95109
// 2-bit quantizations
110+
GGMLFileQuantizationType.Q2_K_XL,
96111
GGMLFileQuantizationType.Q2_K,
97112
GGMLFileQuantizationType.Q2_K_S,
98113
GGMLFileQuantizationType.IQ2_M,

0 commit comments

Comments
 (0)