|
1 | 1 | // Hopper (sm_90a) wgmma MMQ path for Q1_0: dequant-in-SMEM + int8 wgmma with exact per-block scaling. |
2 | | -// Active by default in builds configured with GGML_CUDA_HOPPER_Q1; set GGML_HOPPER_Q1_DISABLE to |
3 | | -// fall back to the standard MMQ path. Dispatched only when M,N,K % 128 == 0 and cc >= 900 (Hopper |
4 | | -// only: not Ada, not Blackwell); every other shape falls through to standard MMQ. |
5 | | -// |
6 | | -// Activations are quantized fp32 -> int8 with a per-128-K absmax scale, coarser than q8_1's |
7 | | -// per-32, so this path is NOT bit-identical to standard MMQ. The deviation has been measured |
8 | | -// against the same build with the path disabled, using a same-path control to establish the |
9 | | -// floor: it is small, the tail is bounded, and only a small fraction of tokens change their |
10 | | -// argmax. Figures are recorded in the internal notes. If you need bit-exact output, set |
11 | | -// GGML_HOPPER_Q1_DISABLE. |
| 2 | +// Active by default when built with GGML_CUDA_HOPPER_Q1; set GGML_HOPPER_Q1_DISABLE for standard MMQ. |
| 3 | +// Not bit-identical to standard MMQ: activations use a per-128-K int8 absmax scale, coarser than q8_1's per-32. |
12 | 4 |
|
13 | 5 | #include "common.cuh" |
14 | 6 |
|
@@ -457,7 +449,8 @@ bool ggml_cuda_mul_mat_q1_hopper(ggml_backend_cuda_context & ctx, |
457 | 449 | const int64_t K = src0->ne[0], N = src0->ne[1], M = src1->ne[1]; |
458 | 450 | const bool is_q1 = src0->type == GGML_TYPE_Q1_0; |
459 | 451 | const bool is_q2 = src0->type == GGML_TYPE_PQ2_0; |
460 | | - if (cc < 900 || cc >= GGML_CUDA_CC_BLACKWELL || // sm_90a wgmma only: not Ada, not Blackwell (no wgmma; needs the tcgen05 path) // 900 = Hopper; no GGML_CUDA_CC_HOPPER macro in this tree |
| 452 | + // sm_90a wgmma only; Blackwell spans CC 1000-1200 and needs the tcgen05 path |
| 453 | + if (cc < GGML_CUDA_CC_HOPPER || cc >= 1000 || |
461 | 454 | (!is_q1 && !is_q2) || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || |
462 | 455 | src1->ne[2] * src1->ne[3] != 1 || src0->ne[2] * src0->ne[3] != 1 || (M % 128) || (N % 128) || (K % 128) || |
463 | 456 | !ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { |
|
0 commit comments