Skip to content

fix/feat(ds4): ROCmFPX byte-safe decode + fused-decode path + ds4 AR optimizations #503

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
davide221 wants to merge 3 commits into
base: codex/rocmfpx-ggml-vendor
Choose a base branch
from
Open

fix/feat(ds4): ROCmFPX byte-safe decode + fused-decode path + ds4 AR optimizations #503

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
42f1e10
fix(deepseek4): ROCmFPX byte-safe decode + fused-decode path
davide221 Jul 8, 2026
6c7ee19
feat(deepseek4): add opt-in expert top-k knobs
Jul 9, 2026
e943f2a
perf(ggml): specialize ROCmFPX AR decode on gfx1151
davide221 Jul 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
288 changes: 245 additions & 43 deletions server/deps/llama.cpp/ggml/src/ggml-cuda/mmvq.cu
View file
Open in desktop

Large diffs are not rendered by default.

30 changes: 24 additions & 6 deletions server/deps/llama.cpp/ggml/src/ggml-cuda/vecdotq.cuh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -417,6 +417,29 @@ static __device__ __forceinline__ int rocmfpx_pack4_fp6_bits24_vec_cuda(const ui
return *((const int *) &v);
}

static __device__ __forceinline__ int rocmfpx_pack4_fp3_bits12_vec_cuda(const uint32_t bits12) {
#if defined(GGML_USE_HIP)
// Byte tables for codes {0, 1, 2, 4} and {0, -1, -2, -4}.
// v_perm_b32 selects a byte with each 3-bit selector, replacing four
// scalar code decodes without changing the packed int8 values.
constexpr uint32_t values_low = 0x04020100u;
constexpr uint32_t values_high = 0xFCFEFF00u;
const uint32_t selectors =
((bits12 >> 0) & 7u) |
(((bits12 >> 3) & 7u) << 8) |
(((bits12 >> 6) & 7u) << 16) |
(((bits12 >> 9) & 7u) << 24);
return (int) __builtin_amdgcn_perm(values_high, values_low, selectors);
#else
const char4 v = make_char4(
(int8_t) rocmfpx_decode_fp3_code_vec_cuda(bits12 & 7u),
(int8_t) rocmfpx_decode_fp3_code_vec_cuda((bits12 >> 3) & 7u),
(int8_t) rocmfpx_decode_fp3_code_vec_cuda((bits12 >> 6) & 7u),
(int8_t) rocmfpx_decode_fp3_code_vec_cuda((bits12 >> 9) & 7u));
return *((const int *) &v);
#endif
}

static __device__ __forceinline__ int rocmfpx_pack4_fp3_vec_cuda(const uint8_t * qs, const int base) {
const char4 v = make_char4(
(int8_t) rocmfpx_decode_fp3_code_vec_cuda(rocmfpx_get_bits_vec_cuda(qs, (base + 0)*3, 3)),
Expand Down Expand Up @@ -591,12 +614,7 @@ static __device__ __forceinline__ float vec_dot_rocmfpx_fp3_q8_1(
const uint32_t val_high = qs[reg_idx + 1];
const uint32_t bits12 = (reg_shift == 0) ? (val_low & 0xFFFu) : (((val_low >> reg_shift) | (val_high << (32 - reg_shift))) & 0xFFFu);

const char4 v = make_char4(
(int8_t) rocmfpx_decode_fp3_code_vec_cuda(bits12 & 7u),
(int8_t) rocmfpx_decode_fp3_code_vec_cuda((bits12 >> 3) & 7u),
(int8_t) rocmfpx_decode_fp3_code_vec_cuda((bits12 >> 6) & 7u),
(int8_t) rocmfpx_decode_fp3_code_vec_cuda((bits12 >> 9) & 7u));
const int val_packed = *((const int *) &v);
const int val_packed = rocmfpx_pack4_fp3_bits12_vec_cuda(bits12);

const int u = get_int_b4(bq8_1->qs, iqs + i);

Expand Down
67 changes: 54 additions & 13 deletions server/src/deepseek4/deepseek4_backend.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -535,9 +535,26 @@ int DeepSeek4Backend::do_prefill(const std::vector<int32_t> & tokens,
int kv_offset) {
// Hybrid currently implements HC for single-token steps only; keep prefill
// token-by-token so the first sampled token is seeded from the correct HC state.
const int chunk = moe_hybrid_ ? 1 : (cfg_.chunk > 0 ? cfg_.chunk : 512);
//
// Chunked prefill (n_tokens > 1) is only exact while the whole prompt fits
// inside the raw SWA ring: the chunk graph writes ring slots pos % n_swa
// while attention reads the ring in the same graph, and the learned
// compressor only runs for the chunk's last token. Prompts beyond n_swa
// degrade into incoherence. Token-by-token prefill matches the reference
// semantics at any length; chunked stays available via
// DFLASH_DS4_CHUNKED_PREFILL=1 for short-prompt benchmarking only.
const bool unsafe_chunked = env_flag_enabled("DFLASH_DS4_CHUNKED_PREFILL");
const int chunk = (moe_hybrid_ || !unsafe_chunked) ? 1 : (cfg_.chunk > 0 ? cfg_.chunk : 512);
const int n_total = (int)tokens.size();
int pos = kv_offset;
// New sequence: clear the cache buffer so compressor state double-buffers
// and compressed-KV rows start from zeros, exactly like a fresh server.
// Without this, the first flush windows of a request pool over the
// previous request's leftover state rows and outputs from the 2nd/3rd
// request on can drift by a token or two.
if (kv_offset == 0 && cache_.buf) {
ggml_backend_buffer_clear(cache_.buf, 0);

@cubic-dev-ai cubic-dev-ai Bot Jul 9, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1: New local-server requests can still see a stale compressed-cache topology after the buffer clear: ggml_backend_buffer_clear(cache_.buf, 0) zeros the tensor memory but leaves cache_.layers[*].n_comp and n_index_comp from the previous request. Since those counters control how many compressed rows attention reads, request 2+ can attend over zeroed-but-counted compressed rows instead of matching a fresh cache. It would be safer to reset cache_.cur_pos, n_comp, and n_index_comp alongside the buffer clear, matching the layer-split reset paths.

Prompt for AI agents 
Check if this issue is valid — if so, understand the root cause and fix it. At server/src/deepseek4/deepseek4_backend.cpp, line 556:

<comment>New local-server requests can still see a stale compressed-cache topology after the buffer clear: `ggml_backend_buffer_clear(cache_.buf, 0)` zeros the tensor memory but leaves `cache_.layers[*].n_comp` and `n_index_comp` from the previous request. Since those counters control how many compressed rows attention reads, request 2+ can attend over zeroed-but-counted compressed rows instead of matching a fresh cache. It would be safer to reset `cache_.cur_pos`, `n_comp`, and `n_index_comp` alongside the buffer clear, matching the layer-split reset paths.</comment>

<file context>
@@ -535,9 +535,26 @@ int DeepSeek4Backend::do_prefill(const std::vector<int32_t> & tokens,
+    // previous request's leftover state rows and outputs from the 2nd/3rd
+    // request on can drift by a token or two.
+    if (kv_offset == 0 && cache_.buf) {
+        ggml_backend_buffer_clear(cache_.buf, 0);
+    }
     last_logits_.clear();
</file context>
Suggested change
ggml_backend_buffer_clear(cache_.buf, 0);
ggml_backend_buffer_clear(cache_.buf, 0);
cache_.cur_pos = 0;
for (auto & lc : cache_.layers) {
lc.n_comp = 0;
lc.n_index_comp = 0;
}

}
last_logits_.clear();
const bool timing = env_flag_enabled("DFLASH_DS4_TIMING");
const auto phase_t0 = Clock::now();
Expand All @@ -556,13 +573,26 @@ int DeepSeek4Backend::do_prefill(const std::vector<int32_t> & tokens,
DeepSeek4StepTelemetry step_tel;
if (timing) step_tel.embed_us = elapsed_us(embed_t0, Clock::now());

// Run forward pass
// Run forward pass. The non-hybrid (all-hot) placement must use the
// HC-complete layer-range path; deepseek4_step's non-hybrid branch is
// an HC-less stub and produces garbage on this box.
std::vector<float> logits;
if (!deepseek4_step(backend_, w_, cache_, embed.data(), n_tok, pos, logits,
moe_hybrid_.get(), tokens.data() + i,
moe_hybrid_ ? &stream_engine_ : nullptr,
timing ? &step_tel : nullptr,
routing_stats_.get())) {
bool ok = false;
if (moe_hybrid_) {
ok = deepseek4_step(backend_, w_, cache_, embed.data(), n_tok, pos, logits,
moe_hybrid_.get(), tokens.data() + i,
&stream_engine_,
timing ? &step_tel : nullptr,
routing_stats_.get());
} else {
std::vector<float> hc_state;
ok = deepseek4_step_layer_range(backend_, w_, cache_, hc_state,
embed.data(), n_tok, pos,
0, w_.n_layer, &logits,
tokens.data() + i,
timing ? &step_tel : nullptr);
}
if (!ok) {
std::fprintf(stderr, "[deepseek4] prefill step failed at pos=%d\n", pos);
return -1;
}
Expand Down Expand Up @@ -619,12 +649,23 @@ bool DeepSeek4Backend::do_decode(int committed, int n_gen,
if (timing) step_tel.embed_us = elapsed_us(embed_t0, Clock::now());

const int pos = std::max(0, committed + generated - 1);
if (!deepseek4_step(backend_, w_, cache_, embed.data(), 1,
pos, logits,
moe_hybrid_.get(), &tok_to_eval,
moe_hybrid_ ? &stream_engine_ : nullptr,
timing ? &step_tel : nullptr,
routing_stats_.get())) {
bool ok = false;
if (moe_hybrid_) {
ok = deepseek4_step(backend_, w_, cache_, embed.data(), 1,
pos, logits,
moe_hybrid_.get(), &tok_to_eval,
&stream_engine_,
timing ? &step_tel : nullptr,
routing_stats_.get());
} else {
std::vector<float> hc_state;
ok = deepseek4_step_layer_range(backend_, w_, cache_, hc_state,
embed.data(), 1, pos,
0, w_.n_layer, &logits,
&tok_to_eval,
timing ? &step_tel : nullptr);
}
if (!ok) {
std::fprintf(stderr, "[deepseek4] decode step failed\n");
return false;
}
Expand Down
Loading