Upload patch_llamacpp_hpcq_seg.py
Browse files
patch_llamacpp_hpcq_seg.py
CHANGED
|
@@ -282,7 +282,6 @@ Q2K_DEQ_NEW = f""" const int nb = k / QK_K;
|
|
| 282 |
// {BEGIN}: stack-based per-block scatter for segmented Q2_K
|
| 283 |
int64_t hpcq_b0 = 0;
|
| 284 |
const struct hpcq_seg_entry * hpcq_seg = ggml_hpcq_seg_find(x, (int64_t) sizeof(block_q2_K), &hpcq_b0);
|
| 285 |
-
const size_t hpcq_stride = (hpcq_seg && hpcq_seg->enc != 0) ? 256u : 128u;
|
| 286 |
uint16_t hpcq_ord[QK_K];
|
| 287 |
// {END}
|
| 288 |
|
|
@@ -290,7 +289,7 @@ Q2K_DEQ_NEW = f""" const int nb = k / QK_K;
|
|
| 290 |
|
| 291 |
float * y_out = y + i * QK_K;
|
| 292 |
if (hpcq_seg) {{
|
| 293 |
-
|
| 294 |
}}
|
| 295 |
|
| 296 |
const float d = GGML_FP16_TO_FP32(x[i].d);
|
|
@@ -371,7 +370,6 @@ Q3K_DEQ_OLD = """ for (int i = 0; i < nb; i++) {
|
|
| 371 |
Q3K_DEQ_NEW = f""" // {BEGIN}: stack-based per-block scatter for segmented Q3_K
|
| 372 |
int64_t hpcq_b0 = 0;
|
| 373 |
const struct hpcq_seg_entry * hpcq_seg = ggml_hpcq_seg_find(x, (int64_t) sizeof(block_q3_K), &hpcq_b0);
|
| 374 |
-
const size_t hpcq_stride = (hpcq_seg && hpcq_seg->enc != 0) ? 256u : 128u;
|
| 375 |
uint16_t hpcq_ord[QK_K];
|
| 376 |
// {END}
|
| 377 |
|
|
@@ -379,7 +377,7 @@ Q3K_DEQ_NEW = f""" // {BEGIN}: stack-based per-block scatter for segmented Q3
|
|
| 379 |
|
| 380 |
float * y_out = y + i * QK_K;
|
| 381 |
if (hpcq_seg) {{
|
| 382 |
-
|
| 383 |
}}
|
| 384 |
|
| 385 |
const float d_all = GGML_FP16_TO_FP32(x[i].d);
|
|
|
|
| 282 |
// {BEGIN}: stack-based per-block scatter for segmented Q2_K
|
| 283 |
int64_t hpcq_b0 = 0;
|
| 284 |
const struct hpcq_seg_entry * hpcq_seg = ggml_hpcq_seg_find(x, (int64_t) sizeof(block_q2_K), &hpcq_b0);
|
|
|
|
| 285 |
uint16_t hpcq_ord[QK_K];
|
| 286 |
// {END}
|
| 287 |
|
|
|
|
| 289 |
|
| 290 |
float * y_out = y + i * QK_K;
|
| 291 |
if (hpcq_seg) {{
|
| 292 |
+
hpcq_seg_make_order_at(hpcq_seg->packed, hpcq_seg->enc, hpcq_b0 + i, hpcq_ord);
|
| 293 |
}}
|
| 294 |
|
| 295 |
const float d = GGML_FP16_TO_FP32(x[i].d);
|
|
|
|
| 370 |
Q3K_DEQ_NEW = f""" // {BEGIN}: stack-based per-block scatter for segmented Q3_K
|
| 371 |
int64_t hpcq_b0 = 0;
|
| 372 |
const struct hpcq_seg_entry * hpcq_seg = ggml_hpcq_seg_find(x, (int64_t) sizeof(block_q3_K), &hpcq_b0);
|
|
|
|
| 373 |
uint16_t hpcq_ord[QK_K];
|
| 374 |
// {END}
|
| 375 |
|
|
|
|
| 377 |
|
| 378 |
float * y_out = y + i * QK_K;
|
| 379 |
if (hpcq_seg) {{
|
| 380 |
+
hpcq_seg_make_order_at(hpcq_seg->packed, hpcq_seg->enc, hpcq_b0 + i, hpcq_ord);
|
| 381 |
}}
|
| 382 |
|
| 383 |
const float d_all = GGML_FP16_TO_FP32(x[i].d);
|