From 36c6a91957a2a0134d08e09a4668e5d3cba58053 Mon Sep 17 00:00:00 2001 From: Osamu Watanabe Date: Thu, 2 Jul 2026 16:54:17 +0900 Subject: [PATCH 1/4] refactor(decoder): split HT cleanup decode into setup / N-way step-1 / finish Factor the HT block decoder around its existing two-phase seam so the serial step-1 chain (MEL + CxtVLC + UVLC) can later be interleaved across codeblocks: - ht_block_decoding.hpp gains ht_cleanup_step1_nway, a lockstep N-lane transcription of the former phase-1 loop shared by all ISA files. Per-lane state lives in N-arrays and every sub-step is an unrolled for-lane loop (OpenJPH's own AVX2-decoder structure); the per-lane operation sequence is unchanged, so output is byte-identical. MEL_dec / rev_buf get default ctors + init() for lane arrays. - The AVX2 and scalar files' ht_cleanup_decode monoliths become ht_cleanup_step2 (MagSgn only); htj2k_decode is now htj2k_dec_setup (validation + modDcup) -> step1 at N=1 -> htj2k_dec_finish (step-2 dispatch + SigProp/MagRef + dequantize). - New htj2k_decode_batch entry point declared in block_decoding.hpp with trivial per-block-loop definitions in all four ISA files and a link-time htj2k_dec_batch_lanes constant (1 everywhere for now). No driver changes and no behavioral change. Verified byte-identical output on five bench fixtures (8r/16r/8i/8r_512x8/kdu_8mp multipass) for both AVX2 and scalar builds; full ctest passes on the AVX2 build. Same-moment cycle A/B against the unsplit baseline is within noise on every fixture for both builds (instructions flat to 0.02%), i.e. the N=1 instantiation collapses back to the original code. (The two security_threaded_decode_abort*_mt failures on scalar-only builds predate this change: those x86-gated fixtures rely on the AVX2 reader throwing, which the scalar reader does not do.) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01M9cKpzqghsLxJUu7kkyRAw --- source/core/coding/block_decoding.hpp | 14 + source/core/coding/ht_block_decoding.cpp | 514 +++++-------- source/core/coding/ht_block_decoding.hpp | 674 ++++++++++++------ source/core/coding/ht_block_decoding_avx2.cpp | 656 +++++++---------- source/core/coding/ht_block_decoding_neon.cpp | 180 ++--- source/core/coding/ht_block_decoding_wasm.cpp | 260 +++---- 6 files changed, 1180 insertions(+), 1118 deletions(-) diff --git a/source/core/coding/block_decoding.hpp b/source/core/coding/block_decoding.hpp index f6939d95..f21f17d3 100644 --- a/source/core/coding/block_decoding.hpp +++ b/source/core/coding/block_decoding.hpp @@ -33,3 +33,17 @@ void j2k_decode(j2k_codeblock *block, uint8_t ROIshift); bool htj2k_decode(j2k_codeblock *block, uint8_t ROIshift); + +// Decode n HT codeblocks, interleaving the serial step-1 (MEL/VLC/UVLC) +// dependency chains of same-sized neighbours when the active ISA provides a +// batched kernel (htj2k_dec_batch_lanes > 1). Caller guarantees every block +// is an HT block with num_passes > 0 and non-null compressed data. +// results[i] receives what htj2k_decode(blocks[i], ROIshift) would have +// returned; the decoded output is byte-identical to per-block decoding. +// Returns true iff every block decoded successfully. +bool htj2k_decode_batch(j2k_codeblock *const *blocks, uint32_t n, uint8_t ROIshift, bool *results); + +// Number of step-1 lanes the active ISA's batched kernel interleaves; +// 1 means htj2k_decode_batch is a plain per-block loop (link-time constant — +// SIMD dispatch is compile-time in this project). +extern const uint32_t htj2k_dec_batch_lanes; diff --git a/source/core/coding/ht_block_decoding.cpp b/source/core/coding/ht_block_decoding.cpp index 2abcc060..f5d3dc49 100644 --- a/source/core/coding/ht_block_decoding.cpp +++ b/source/core/coding/ht_block_decoding.cpp @@ -26,7 +26,8 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#if !defined(OPENHTJ2K_ENABLE_ARM_NEON) && (!defined(__AVX2__) || !defined(OPENHTJ2K_TRY_AVX2)) && !defined(OPENHTJ2K_ENABLE_WASM_SIMD) +#if !defined(OPENHTJ2K_ENABLE_ARM_NEON) && (!defined(__AVX2__) || !defined(OPENHTJ2K_TRY_AVX2)) \ + && !defined(OPENHTJ2K_ENABLE_WASM_SIMD) #include "coding_units.hpp" #include "dec_CxtVLC_tables.hpp" #include "ht_block_decoding.hpp" @@ -67,216 +68,32 @@ static FORCE_INLINE void dequant_store_scalar(void *dst, int32_t val, uint8_t tr } else { int32_t sign = val & INT32_MIN; float f = static_cast(val & INT32_MAX) * fscale_direct; - if (sign) f = -f; + if (sign) f = -f; *reinterpret_cast(dst) = f; } } -template -void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t Lcup, const int32_t Pcup, - const int32_t Scup) { +// Step-2 of the HT cleanup pass: MagSgn decoding over the (tv, u) scratch +// written by ht_cleanup_step1_nway (the former phase 1 of this function). +// Kept per-block: the fwd_buf destuff scratch is thread-local (constructing a +// second fwd_buf on the same thread invalidates the first), and step-2 is +// throughput-bound — there is nothing to gain from interleaving it. +template +static void ht_cleanup_step2(j2k_codeblock *block, const uint8_t pLSB, const int32_t Pcup, + uint16_t *scratch, const int32_t sstr) { uint8_t *compressed_data = block->get_compressed_data(); - const uint16_t QW = static_cast(ceil_int(static_cast(block->size.x), 2)); - const uint16_t QH = static_cast(ceil_int(static_cast(block->size.y), 2)); - - uint16_t scratch[8 * 513]; - int32_t sstr = static_cast(((block->size.x + 2) + 7u) & ~7u); // multiples of 8 - uint16_t *sp; - int32_t qx; - /*******************************************************************************************************************/ - // VLC, UVLC and MEL decoding - /*******************************************************************************************************************/ - MEL_dec MEL(compressed_data, Lcup, Scup); - rev_buf VLC_dec(compressed_data, Lcup, Scup); - auto sp0 = block->block_states + 1 + block->blkstate_stride; - auto sp1 = block->block_states + 1 + 2 * block->blkstate_stride; - uint32_t u_off0, u_off1; - uint32_t u0, u1; - uint32_t context = 0; - uint32_t vlcval; - - const uint16_t *dec_table; - // Initial line-pair - dec_table = dec_CxtVLC_table0_fast_16; - sp = scratch; - int32_t mel_run = MEL.get_run(); - for (qx = QW; qx > 0; qx -= 2, sp += 4) { - // Decoding of significance and EMB patterns and unsigned residual offsets - vlcval = VLC_dec.fetch(); - uint16_t tv0 = dec_table[(vlcval & 0x7F) + context]; - { - // Branchless context-0 MEL handling: replace unpredictable branch with mask - int32_t cm = -static_cast(context == 0); - mel_run -= cm & 2; - tv0 &= static_cast(-(mel_run == -1) | ~cm); - if (mel_run < 0) mel_run = MEL.get_run(); - } - sp[0] = tv0; - - // calculate context for the next quad, Eq. (1) in the spec - context = ((tv0 & 0xE0U) << 2) | ((tv0 & 0x10U) << 3); // = context << 7 - - // Decoding of significance and EMB patterns and unsigned residual offsets - vlcval = VLC_dec.advance((tv0 & 0x000F) >> 1); - uint16_t tv1 = dec_table[(vlcval & 0x7F) + context]; - { - int32_t cm = -static_cast((context == 0) & (qx > 1)); - mel_run -= cm & 2; - tv1 &= static_cast(-(mel_run == -1) | ~cm); - if (mel_run < 0) mel_run = MEL.get_run(); - } - tv1 = (qx > 1) ? tv1 : 0; - sp[2] = tv1; - - // store sigma - if (!skip_sigma) { - *sp0++ = ((tv0 >> 4) >> 0) & 1; - *sp0++ = ((tv0 >> 4) >> 2) & 1; - *sp0++ = ((tv1 >> 4) >> 0) & 1; - *sp0++ = ((tv1 >> 4) >> 2) & 1; - *sp1++ = ((tv0 >> 4) >> 1) & 1; - *sp1++ = ((tv0 >> 4) >> 3) & 1; - *sp1++ = ((tv1 >> 4) >> 1) & 1; - *sp1++ = ((tv1 >> 4) >> 3) & 1; - } - - // calculate context for the next quad, Eq. (1) in the spec - context = ((tv1 & 0xE0U) << 2) | ((tv1 & 0x10U) << 3); // = context << 7 - - vlcval = VLC_dec.advance((tv1 & 0x000F) >> 1); - u_off0 = tv0 & 1; - u_off1 = tv1 & 1; - - // Branchless MEL offset: replace compound branch with mask - uint32_t both_off = u_off0 & u_off1; - int32_t om = -static_cast(both_off); - mel_run -= om & 2; - uint32_t mel_offset = static_cast(-(mel_run == -1) & om) & 0x40; - if (mel_run < 0) mel_run = MEL.get_run(); - - // UVLC decoding - uint32_t idx = (vlcval & 0x3F) + (u_off0 << 6U) + (u_off1 << 7U) + mel_offset; - uint32_t uvlc_result = uvlc_dec_0[idx]; - // remove total prefix length - vlcval = VLC_dec.advance(uvlc_result & 0x7); - uvlc_result >>= 3; - // extract suffixes for quad 0 and 1 - uint32_t len = uvlc_result & 0xF; // suffix length for 2 quads (up to 10 = 5 + 5) - // ((1U << len) - 1U) can be replaced with _bzhi_u32(UINT32_MAX, len); not fast - uint32_t tmp = vlcval & ((1U << len) - 1U); // suffix value for 2 quads - vlcval = VLC_dec.advance(len); - uvlc_result >>= 4; - // quad 0 length - len = uvlc_result & 0x7; // quad 0 suffix length - uvlc_result >>= 3; - // U = 1+ u - u0 = 1 + (uvlc_result & 7) + (tmp & ~(0xFFU << len)); // always kappa = 1 in initial line pair - u1 = 1 + (uvlc_result >> 3) + (tmp >> len); // always kappa = 1 in initial line pair - - sp[1] = static_cast(u0); - sp[3] = static_cast(u1); - } - std::memset(sp, 0, sizeof(uint16_t) * 4); // zero 8-byte guard for MagSgn context reads - - // Non-initial line-pair - dec_table = dec_CxtVLC_table1_fast_16; - for (uint16_t row = 1; row < QH; row++) { - sp0 = block->block_states + (row * 2U + 1U) * block->blkstate_stride + 1U; - sp1 = sp0 + block->blkstate_stride; - - sp = scratch + row * sstr; - // calculate context for the next quad: w, sw, nw are always 0 at the head of a row - context = ((sp[0 - sstr] & 0xA0U) << 2) | ((sp[2 - sstr] & 0x20U) << 4); - for (qx = QW; qx > 0; qx -= 2, sp += 4) { - // Decoding of significance and EMB patterns and unsigned residual offsets - vlcval = VLC_dec.fetch(); - uint16_t tv0 = dec_table[(vlcval & 0x7F) + context]; - if (context == 0) { - mel_run -= 2; - tv0 = (mel_run == -1) ? tv0 : 0; - if (mel_run < 0) { - mel_run = MEL.get_run(); - } - } - // calculate context for the next quad, Eq. (2) in the spec - context = ((tv0 & 0x40U) << 2) | ((tv0 & 0x80U) << 1); // (w | sw) << 8 - context |= (sp[0 - sstr] & 0x80U) | ((sp[2 - sstr] & 0xA0U) << 2); // ((nw | n) << 7) | (ne << 9) - context |= (sp[4 - sstr] & 0x20U) << 4; // ( nf) << 9 - - sp[0] = tv0; - - vlcval = VLC_dec.advance((tv0 & 0x000F) >> 1); - - // Decoding of significance and EMB patterns and unsigned residual offsets - uint16_t tv1 = dec_table[(vlcval & 0x7F) + context]; - if (context == 0 && qx > 1) { - mel_run -= 2; - tv1 = (mel_run == -1) ? tv1 : 0; - if (mel_run < 0) { - mel_run = MEL.get_run(); - } - } - tv1 = (qx > 1) ? tv1 : 0; - // calculate context for the next quad, Eq. (2) in the spec - context = ((tv1 & 0x40U) << 2) | ((tv1 & 0x80U) << 1); // (w | sw) << 8 - context |= (sp[2 - sstr] & 0x80U) | ((sp[4 - sstr] & 0xA0U) << 2); // ((nw | n) << 7) | (ne << 9) - context |= (sp[6 - sstr] & 0x20U) << 4; // ( nf) << 9 - - sp[2] = tv1; - - // store sigma - if (!skip_sigma) { - *sp0++ = ((tv0 >> 4) >> 0) & 1; - *sp0++ = ((tv0 >> 4) >> 2) & 1; - *sp0++ = ((tv1 >> 4) >> 0) & 1; - *sp0++ = ((tv1 >> 4) >> 2) & 1; - *sp1++ = ((tv0 >> 4) >> 1) & 1; - *sp1++ = ((tv0 >> 4) >> 3) & 1; - *sp1++ = ((tv1 >> 4) >> 1) & 1; - *sp1++ = ((tv1 >> 4) >> 3) & 1; - } - - vlcval = VLC_dec.advance((tv1 & 0x000F) >> 1); - - // UVLC decoding - u_off0 = tv0 & 1; - u_off1 = tv1 & 1; - uint32_t idx = (vlcval & 0x3F) + (u_off0 << 6U) + (u_off1 << 7U); - - uint32_t uvlc_result = uvlc_dec_1[idx]; - // remove total prefix length - vlcval = VLC_dec.advance(uvlc_result & 0x7); - uvlc_result >>= 3; - // extract suffixes for quad 0 and 1 - uint32_t len = uvlc_result & 0xF; // suffix length for 2 quads (up to 10 = 5 + 5) - // ((1U << len) - 1U) can be replaced with _bzhi_u32(UINT32_MAX, len); not fast - uint32_t tmp = vlcval & ((1U << len) - 1U); // suffix value for 2 quads - vlcval = VLC_dec.advance(len); - uvlc_result >>= 4; - // quad 0 length - len = uvlc_result & 0x7; // quad 0 suffix length - uvlc_result >>= 3; - u0 = (uvlc_result & 7) + (tmp & ~(0xFFU << len)); - u1 = (uvlc_result >> 3) + (tmp >> len); - - sp[1] = static_cast(u0); - sp[3] = static_cast(u1); - } - std::memset(sp, 0, sizeof(uint16_t) * 4); // zero 8-byte guard per row - } - /*******************************************************************************************************************/ // MagSgn decoding /*******************************************************************************************************************/ { // Fused dequantize setup - int32_t pLSB_dq = 0; + int32_t pLSB_dq = 0; float fscale_direct = 0.0f; uint32_t out_stride = block->blksampl_stride; if constexpr (fuse_dequant) { const int32_t M_b_val = block->get_Mb(); pLSB_dq = 31 - M_b_val; - out_stride = block->band_stride; + out_stride = block->band_stride; if (block->transformation != 1) { // lossy path (transformation==0 for irrev97, transformation>=2 for ATK irrev) fscale_direct = block->stepsize; @@ -410,9 +227,8 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t for (uint32_t y = 2; y < block->size.y; y += 2) { uint16_t *sp = scratch + (y >> 1) * static_cast(sstr); uint32_t *vp = v_n_scratch; - int32_t *dp = fuse_dequant - ? reinterpret_cast(block->band_buf) + y * block->band_stride - : block->sample_buf + y * block->blksampl_stride; + int32_t *dp = fuse_dequant ? reinterpret_cast(block->band_buf) + y * block->band_stride + : block->sample_buf + y * block->blksampl_stride; prev_v_n = 0; for (uint32_t x = 0; x < block->size.x; sp += 2, ++vp) { @@ -555,10 +371,10 @@ void ht_cleanup_decode2(j2k_codeblock *block, const uint8_t &pLSB, const int32_t alignas(32) uint32_t rholine[516]; // QW_max + 4, QW_max = 512 std::memset(rholine, 0, (QW + 4U) * sizeof(uint32_t)); - uint32_t *rho_p = rholine + 1; - alignas(32) int32_t Eline[1032]; // 2 * QW_max + 8, QW_max = 512 + uint32_t *rho_p = rholine + 1; + alignas(32) int32_t Eline[1032]; // 2 * QW_max + 8, QW_max = 512 std::memset(Eline, 0, (2U * QW + 8U) * sizeof(int32_t)); - int32_t *E_p = Eline + 1; + int32_t *E_p = Eline + 1; uint32_t context = 0; uint32_t vlcval; @@ -1108,13 +924,32 @@ void j2k_codeblock::dequantize(uint8_t ROIshift) const { } } -bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { +// Per-block decode setup: segment validation, Lcup/Lref/Scup/Pcup derivation +// and the modDcup buffer mutation. Factored out of htj2k_decode so the +// 1-way and batched entries share one source of truth. +struct ht_dec_setup { + int32_t Lcup, Pcup, Scup; + uint32_t Lref; + uint8_t *Dref; // nullptr when single-segment + uint8_t S_blk; + uint8_t num_ht_passes; + bool ok; // false → htj2k_decode returns false + bool empty; // num_ht_passes == 0 → success with no work +}; + +static ht_dec_setup htj2k_dec_setup(j2k_codeblock *block) { + ht_dec_setup su; + su.Lcup = 0; + su.Pcup = 0; + su.Scup = 0; + su.Lref = 0; + su.Dref = nullptr; + su.S_blk = 0; + su.ok = false; + su.empty = false; + // number of placeholder pass uint8_t P0 = 0; - // length of HT Cleanup segment - int32_t Lcup = 0; - // length of HT Refinement segment - uint32_t Lref = 0; // number of HT Sets preceding the given(this) HT Set const uint8_t S_skip = 0; @@ -1135,125 +970,168 @@ bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { if (block->num_passes < empty_passes) { printf("WARNING: number of passes %d exceeds number of empty passes %d", block->num_passes, empty_passes); - return false; + return su; } // number of ht coding pass (Z_blk in the spec) - const uint8_t num_ht_passes = static_cast(block->num_passes - empty_passes); - // pointer to buffer for HT Cleanup segment - uint8_t *Dcup; - // pointer to buffer for HT Refinement segment - uint8_t *Dref; - - if (num_ht_passes > 0) { - // HT defines at most two segments per codeblock (Cleanup + optional - // Refinement); `all_segments[4]` is over-provisioned. A malformed - // input with pass_length_count > 4 non-zero entries used to write - // past the array and smash the stack — guard the write and the - // later `all_segments[0]` read. Reported by IM JUN SEO (KISIA) and - // OH HAN GUEL (SANGMYUNG UNIVERSITY). - uint8_t all_segments[4]; - uint32_t num_segments = 0; - for (uint32_t i = 0; i < block->pass_length_count; i++) { - if (block->pass_length[i] != 0) { - if (num_segments >= 4) { - printf("WARNING: too many HT coding-pass segments (>4) — malformed input.\n"); - return false; - } - all_segments[num_segments++] = static_cast(i); + su.num_ht_passes = static_cast(block->num_passes - empty_passes); + if (su.num_ht_passes == 0) { + su.ok = true; + su.empty = true; + return su; + } + + // HT defines at most two segments per codeblock (Cleanup + optional + // Refinement); `all_segments[4]` is over-provisioned. A malformed + // input with pass_length_count > 4 non-zero entries used to write + // past the array and smash the stack — guard the write and the + // later `all_segments[0]` read. Reported by IM JUN SEO (KISIA) and + // OH HAN GUEL (SANGMYUNG UNIVERSITY). + uint8_t all_segments[4]; + uint32_t num_segments = 0; + for (uint32_t i = 0; i < block->pass_length_count; i++) { + if (block->pass_length[i] != 0) { + if (num_segments >= 4) { + printf("WARNING: too many HT coding-pass segments (>4) — malformed input.\n"); + return su; } + all_segments[num_segments++] = static_cast(i); } - if (num_segments == 0) { - printf("WARNING: no non-empty HT coding-pass segments.\n"); - return false; - } - Lcup += static_cast(block->pass_length[all_segments[0]]); - if (Lcup < 2) { - printf("WARNING: Cleanup pass length must be at least 2 bytes in length.\n"); - return false; - } - // Bound the attacker-controlled cleanup length by the (already clamped) - // codeblock byte count before any Dcup[Lcup-1] access / modDcup write. - if (static_cast(Lcup) > block->length) { - printf("WARNING: HT cleanup pass length %d exceeds codeblock bytes %u — malformed input.\n", Lcup, - block->length); - return false; - } - for (uint32_t i = 1; i < num_segments; i++) { - Lref += block->pass_length[all_segments[i]]; - } - // The refinement segments are read from Dcup + Lcup; keep them in the buffer. - if (static_cast(Lref) > block->length - static_cast(Lcup)) { - printf("WARNING: HT refinement length exceeds remaining codeblock bytes %u — malformed input.\n", - block->length - static_cast(Lcup)); - return false; - } - Dcup = block->get_compressed_data(); + } + if (num_segments == 0) { + printf("WARNING: no non-empty HT coding-pass segments.\n"); + return su; + } + su.Lcup += static_cast(block->pass_length[all_segments[0]]); + if (su.Lcup < 2) { + printf("WARNING: Cleanup pass length must be at least 2 bytes in length.\n"); + return su; + } + // Bound the attacker-controlled cleanup length by the (already clamped) + // codeblock byte count before any Dcup[Lcup-1] access / modDcup write. + if (static_cast(su.Lcup) > block->length) { + printf("WARNING: HT cleanup pass length %d exceeds codeblock bytes %u — malformed input.\n", su.Lcup, + block->length); + return su; + } + for (uint32_t i = 1; i < num_segments; i++) { + su.Lref += block->pass_length[all_segments[i]]; + } + // The refinement segments are read from Dcup + Lcup; keep them in the buffer. + if (static_cast(su.Lref) > block->length - static_cast(su.Lcup)) { + printf("WARNING: HT refinement length exceeds remaining codeblock bytes %u — malformed input.\n", + block->length - static_cast(su.Lcup)); + return su; + } + uint8_t *Dcup = block->get_compressed_data(); - if (block->num_passes > 1 && num_segments > 1) { - Dref = block->get_compressed_data() + Lcup; - } else { - Dref = nullptr; - } - // number of (skipped) magnitude bitplanes - const uint8_t S_blk = static_cast(P0 + block->num_ZBP + S_skip); - if (S_blk >= 30) { - printf("WARNING: Number of skipped mag bitplanes %d is too large.\n", S_blk); - return false; - } - // Suffix length (=MEL + VLC) of HT Cleanup pass - const int32_t Scup = static_cast((Dcup[Lcup - 1] << 4) + (Dcup[Lcup - 2] & 0x0F)); - if (Scup < 2 || Scup > Lcup || Scup > 4079) { - printf("WARNING: cleanup pass suffix length %d is invalid.\n", Scup); - return false; - } - // modDcup (shall be done before the creation of state_VLC instance) - Dcup[Lcup - 1] = 0xFF; - Dcup[Lcup - 2] |= 0x0F; - const int32_t Pcup = static_cast(Lcup - Scup); - - // HT block decoding - bool dequant_done = false; - // The fused-dequant scalar/NEON/AVX2/WASM kernels process samples in - // 2-row quad pairs and write both rows of every pair unconditionally to - // `block->band_buf`. When `block->size.y` is odd the last pair's - // second-row write lands `band_stride` bytes past the block's final - // row — into the NEXT block's band_buf region when codeblocks tile - // vertically inside the subband (e.g. 1×1 blocks stacked in a narrow - // subband from a horizontally-subsampled component). Under single- - // threaded decode the overflow is harmlessly overwritten by the next - // block's legitimate row-0 write, but under multi-threaded dispatch - // the blocks finish out-of-order and the stale overflow clobbers - // adjacent blocks' output. Fall back to the non-fused path (which - // writes into the block-local sample_buf scratch, not band_buf) when - // height is odd; the separate dequant pass below handles bounds - // correctly. - const bool fuseable = (num_ht_passes == 1) && (ROIshift == 0) - && ((block->size.y & 1u) == 0u); - if (fuseable) { - if (block->dequant_i32) - ht_cleanup_decode(block, static_cast(30 - S_blk), Lcup, Pcup, Scup); - else - ht_cleanup_decode(block, static_cast(30 - S_blk), Lcup, Pcup, Scup); - dequant_done = true; - } else if (num_ht_passes == 1) { - ht_cleanup_decode(block, static_cast(30 - S_blk), Lcup, Pcup, Scup); - } else { - ht_cleanup_decode(block, static_cast(30 - S_blk), Lcup, Pcup, Scup); - } - if (num_ht_passes > 1) { - ht_sigprop_decode(block, Dref, Lref, static_cast(30 - (S_blk + 1))); - } - if (num_ht_passes > 2) { - ht_magref_decode(block, Dref, Lref, static_cast(30 - (S_blk + 1))); - } + if (block->num_passes > 1 && num_segments > 1) { + su.Dref = block->get_compressed_data() + su.Lcup; + } else { + su.Dref = nullptr; + } + // number of (skipped) magnitude bitplanes + su.S_blk = static_cast(P0 + block->num_ZBP + S_skip); + if (su.S_blk >= 30) { + printf("WARNING: Number of skipped mag bitplanes %d is too large.\n", su.S_blk); + return su; + } + // Suffix length (=MEL + VLC) of HT Cleanup pass + su.Scup = static_cast((Dcup[su.Lcup - 1] << 4) + (Dcup[su.Lcup - 2] & 0x0F)); + if (su.Scup < 2 || su.Scup > su.Lcup || su.Scup > 4079) { + printf("WARNING: cleanup pass suffix length %d is invalid.\n", su.Scup); + return su; + } + // modDcup (shall be done before the creation of state_VLC instance) + Dcup[su.Lcup - 1] = 0xFF; + Dcup[su.Lcup - 2] |= 0x0F; + su.Pcup = static_cast(su.Lcup - su.Scup); + su.ok = true; + return su; +} - // dequantization (skipped when already fused into MagSgn output) - if (!dequant_done) { - block->dequantize(ROIshift); - } +// Everything after step-1: step-2 (MagSgn) variant dispatch, SigProp/MagRef +// refinement passes, and dequantization. scratch/sstr hold the step-1 output. +static bool htj2k_dec_finish(j2k_codeblock *block, const ht_dec_setup &su, const uint8_t ROIshift, + uint16_t *scratch, const int32_t sstr) { + const uint8_t pLSB = static_cast(30 - su.S_blk); + + // HT block decoding + bool dequant_done = false; + // The fused-dequant scalar/NEON/AVX2/WASM kernels process samples in + // 2-row quad pairs and write both rows of every pair unconditionally to + // `block->band_buf`. When `block->size.y` is odd the last pair's + // second-row write lands `band_stride` bytes past the block's final + // row — into the NEXT block's band_buf region when codeblocks tile + // vertically inside the subband (e.g. 1×1 blocks stacked in a narrow + // subband from a horizontally-subsampled component). Under single- + // threaded decode the overflow is harmlessly overwritten by the next + // block's legitimate row-0 write, but under multi-threaded dispatch + // the blocks finish out-of-order and the stale overflow clobbers + // adjacent blocks' output. Fall back to the non-fused path (which + // writes into the block-local sample_buf scratch, not band_buf) when + // height is odd; the separate dequant pass below handles bounds + // correctly. + const bool fuseable = (su.num_ht_passes == 1) && (ROIshift == 0) && ((block->size.y & 1u) == 0u); + if (fuseable) { + if (block->dequant_i32) + ht_cleanup_step2(block, pLSB, su.Pcup, scratch, sstr); + else + ht_cleanup_step2(block, pLSB, su.Pcup, scratch, sstr); + dequant_done = true; + } else { + ht_cleanup_step2<>(block, pLSB, su.Pcup, scratch, sstr); + } + if (su.num_ht_passes > 1) { + ht_sigprop_decode(block, su.Dref, su.Lref, static_cast(30 - (su.S_blk + 1))); + } + if (su.num_ht_passes > 2) { + ht_magref_decode(block, su.Dref, su.Lref, static_cast(30 - (su.S_blk + 1))); + } - } // block decoding end + // dequantization (skipped when already fused into MagSgn output) + if (!dequant_done) { + block->dequantize(ROIshift); + } return true; } + +bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { + const ht_dec_setup su = htj2k_dec_setup(block); + if (!su.ok) return false; + if (su.empty) return true; + + const uint16_t QW = static_cast(ceil_int(static_cast(block->size.x), 2)); + const uint16_t QH = static_cast(ceil_int(static_cast(block->size.y), 2)); + const int32_t sstr = static_cast(((block->size.x + 2) + 7u) & ~7u); // multiples of 8 + + uint16_t scratch[8 * 513]; + ht_step1_lane ln; + ln.Dcup = block->get_compressed_data(); + ln.Lcup = su.Lcup; + ln.Scup = su.Scup; + ln.scratch = scratch; + ln.block_states = block->block_states; + ln.blkstate_stride = block->blkstate_stride; + if (su.num_ht_passes == 1) { + ht_cleanup_step1_nway<1, true>(&ln, QW, QH, sstr); + } else { + ht_cleanup_step1_nway<1, false>(&ln, QW, QH, sstr); + } + + return htj2k_dec_finish(block, su, ROIshift, scratch, sstr); +} + +// Batched entry point (see block_decoding.hpp). The scalar build has no +// N-way step-1 kernel wired yet: plain per-block loop. +bool htj2k_decode_batch(j2k_codeblock *const *blocks, uint32_t n, uint8_t ROIshift, bool *results) { + bool all_ok = true; + for (uint32_t i = 0; i < n; ++i) { + results[i] = htj2k_decode(blocks[i], ROIshift); + all_ok &= results[i]; + } + return all_ok; +} + +const uint32_t htj2k_dec_batch_lanes = 1; #endif \ No newline at end of file diff --git a/source/core/coding/ht_block_decoding.hpp b/source/core/coding/ht_block_decoding.hpp index 15adbb64..b6807a00 100644 --- a/source/core/coding/ht_block_decoding.hpp +++ b/source/core/coding/ht_block_decoding.hpp @@ -34,6 +34,8 @@ #include #include +#include "dec_CxtVLC_tables.hpp" + #if __GNUC__ || __has_attribute(always_inline) #define FORCE_INLINE inline __attribute__((always_inline)) #define openhtj2k_arm_clzll(x) __builtin_clzll((x)) @@ -45,6 +47,20 @@ #define openhtj2k_arm_clzll(x) __builtin_clzll((x)) #endif +#if defined(__clang__) + #define OPENHTJ2K_UNROLL _Pragma("clang loop unroll(full)") +#elif defined(__GNUC__) + #define OPENHTJ2K_UNROLL _Pragma("GCC unroll 8") +#else + #define OPENHTJ2K_UNROLL +#endif + +#if defined(_MSC_VER) && !defined(__clang__) + #define OPENHTJ2K_NOINLINE __declspec(noinline) +#else + #define OPENHTJ2K_NOINLINE __attribute__((noinline)) +#endif + // LUT for UVLC decoding in initial line-pair // index (8bits) : [bit 7] u_off_1 (1bit) // [bit 6] u_off_0 (1bit) @@ -128,16 +144,22 @@ class MEL_dec { uint64_t runs; public: - MEL_dec(const uint8_t *Dcup, int32_t Lcup, int32_t Scup) - : buf(Dcup + Lcup - Scup), - tmp(0), - bits(0), - length(Scup - 1), // length is the length of MEL+VLC-1 - unstuff(false), - MEL_k(0), - num_runs(0), - runs(0) { - int num = 4 - static_cast(reinterpret_cast(buf) & 0x3); + // Default ctor leaves the state uninitialized; init() must be called + // before use. Needed for per-lane arrays in the N-way step-1 kernel. + MEL_dec() {} + + MEL_dec(const uint8_t *Dcup, int32_t Lcup, int32_t Scup) { init(Dcup, Lcup, Scup); } + + void init(const uint8_t *Dcup, int32_t Lcup, int32_t Scup) { + buf = Dcup + Lcup - Scup; + tmp = 0; + bits = 0; + length = Scup - 1; // length is the length of MEL+VLC-1 + unstuff = false; + MEL_k = 0; + num_runs = 0; + runs = 0; + int num = 4 - static_cast(reinterpret_cast(buf) & 0x3); for (int32_t i = 0; i < num; ++i) { uint64_t d = (length > 0) ? *buf : 0xFF; // if buffer is exhausted, set data to 0xFF if (length == 1) { @@ -270,8 +292,15 @@ class rev_buf { uint32_t unstuff; public: - rev_buf(uint8_t *Dcup, int32_t Lcup, int32_t Scup) - : buf(Dcup + Lcup - 2), Creg(0), bits(0), length(Scup - 2), unstuff(0) { + // Default ctor leaves the state uninitialized; init() must be called + // before use. Needed for per-lane arrays in the N-way step-1 kernel. + rev_buf() {} + + rev_buf(uint8_t *Dcup, int32_t Lcup, int32_t Scup) { init(Dcup, Lcup, Scup); } + + void init(uint8_t *Dcup, int32_t Lcup, int32_t Scup) { + buf = Dcup + Lcup - 2; + length = Scup - 2; uint32_t d = *buf--; // read a byte (only use it's half byte) Creg = d >> 4; bits = 4 - ((Creg & 0x07) == 0x07); @@ -364,6 +393,259 @@ class rev_buf { } }; +/******************************************************************************** + * ht_cleanup_step1_nway: N-way lockstep step-1 (MEL + CxtVLC + UVLC) decoding + *******************************************************************************/ +// Per-lane inputs for the N-way step-1 kernel; all fields are computed by the +// per-block decode setup (including the modDcup buffer mutation) before the call. +struct ht_step1_lane { + uint8_t *Dcup; // block->get_compressed_data() + int32_t Lcup; // HT cleanup segment length + int32_t Scup; // MEL + VLC suffix length + uint16_t *scratch; // this lane's uint16_t[8 * 513] (tv, u) output + uint8_t *block_states; // written only when !skip_sigma + size_t blkstate_stride; +}; + +// Step-1 of the HT cleanup pass for N codeblocks in lockstep: the serial +// MEL + CxtVLC + UVLC dependency chains of the N blocks are interleaved +// sub-step by sub-step so the out-of-order engine can overlap them +// (codeblocks are independent). All lanes must share the same codeblock +// dimensions (QW, QH, sstr). The per-lane operation sequence is identical +// to the former single-block phase-1 loop — only the cross-lane instruction +// scheduling differs — so the decoded output is byte-identical to N +// sequential single-block decodes. +// Deliberately not inlined: keeps the kernel's register allocation isolated +// from the caller's step-2 code. +template +OPENHTJ2K_NOINLINE void ht_cleanup_step1_nway(const ht_step1_lane *ln, const uint16_t QW, const uint16_t QH, + const int32_t sstr) { + constexpr size_t NL = static_cast(N); + MEL_dec MEL[NL]; + rev_buf VLC_dec[NL]; + int32_t mel_run[NL]; + uint32_t context[NL]; + uint32_t vlcval[NL]; + uint16_t tv0[NL], tv1[NL]; + uint16_t *sp[NL]; + uint8_t *sp0[NL], *sp1[NL]; + + OPENHTJ2K_UNROLL + for (size_t s = 0; s < NL; ++s) { + MEL[s].init(ln[s].Dcup, ln[s].Lcup, ln[s].Scup); + VLC_dec[s].init(ln[s].Dcup, ln[s].Lcup, ln[s].Scup); + mel_run[s] = MEL[s].get_run(); + context[s] = 0; + sp[s] = ln[s].scratch; + if (!skip_sigma) { + sp0[s] = ln[s].block_states + 1 + ln[s].blkstate_stride; + sp1[s] = ln[s].block_states + 1 + 2 * ln[s].blkstate_stride; + } + } + + // Initial line-pair + for (int32_t qx = QW; qx > 0; qx -= 2) { + OPENHTJ2K_UNROLL + for (size_t s = 0; s < NL; ++s) { + // Decoding of significance and EMB patterns and unsigned residual offsets + vlcval[s] = VLC_dec[s].fetch(); + uint16_t t0 = dec_CxtVLC_table0_fast_16[(vlcval[s] & 0x7F) + context[s]]; + { + // Branchless context-0 MEL handling: replace unpredictable branch with mask + int32_t cm = -static_cast(context[s] == 0); + mel_run[s] -= cm & 2; + t0 &= static_cast(-(mel_run[s] == -1) | ~cm); + if (mel_run[s] < 0) mel_run[s] = MEL[s].get_run(); + } + sp[s][0] = t0; + tv0[s] = t0; + + // calculate context for the next quad, Eq. (1) in the spec + context[s] = ((t0 & 0xE0U) << 2) | ((t0 & 0x10U) << 3); // = context << 7 + + vlcval[s] = VLC_dec[s].advance((t0 & 0x000F) >> 1); + } + OPENHTJ2K_UNROLL + for (size_t s = 0; s < NL; ++s) { + // Decoding of significance and EMB patterns and unsigned residual offsets + uint16_t t1 = dec_CxtVLC_table0_fast_16[(vlcval[s] & 0x7F) + context[s]]; + { + int32_t cm = -static_cast((context[s] == 0) & (qx > 1)); + mel_run[s] -= cm & 2; + t1 &= static_cast(-(mel_run[s] == -1) | ~cm); + if (mel_run[s] < 0) mel_run[s] = MEL[s].get_run(); + } + t1 = (qx > 1) ? t1 : 0; + sp[s][2] = t1; + tv1[s] = t1; + + // store sigma + if (!skip_sigma) { + uint16_t t0s = tv0[s]; + *sp0[s]++ = ((t0s >> 4) >> 0) & 1; + *sp0[s]++ = ((t0s >> 4) >> 2) & 1; + *sp0[s]++ = ((t1 >> 4) >> 0) & 1; + *sp0[s]++ = ((t1 >> 4) >> 2) & 1; + *sp1[s]++ = ((t0s >> 4) >> 1) & 1; + *sp1[s]++ = ((t0s >> 4) >> 3) & 1; + *sp1[s]++ = ((t1 >> 4) >> 1) & 1; + *sp1[s]++ = ((t1 >> 4) >> 3) & 1; + } + + // calculate context for the next quad, Eq. (1) in the spec + context[s] = ((t1 & 0xE0U) << 2) | ((t1 & 0x10U) << 3); // = context << 7 + + vlcval[s] = VLC_dec[s].advance((t1 & 0x000F) >> 1); + } + OPENHTJ2K_UNROLL + for (size_t s = 0; s < NL; ++s) { + uint32_t u_off0 = tv0[s] & 1; + uint32_t u_off1 = tv1[s] & 1; + + // Branchless MEL offset: replace compound branch with mask + uint32_t both_off = u_off0 & u_off1; + int32_t om = -static_cast(both_off); + mel_run[s] -= om & 2; + uint32_t mel_offset = static_cast(-(mel_run[s] == -1) & om) & 0x40; + if (mel_run[s] < 0) mel_run[s] = MEL[s].get_run(); + + // UVLC decoding + uint32_t idx = (vlcval[s] & 0x3F) + (u_off0 << 6U) + (u_off1 << 7U) + mel_offset; + uint32_t uvlc_result = uvlc_dec_0[idx]; + // remove total prefix length + vlcval[s] = VLC_dec[s].advance(uvlc_result & 0x7); + uvlc_result >>= 3; + // extract suffixes for quad 0 and 1 + uint32_t len = uvlc_result & 0xF; // suffix length for 2 quads (up to 10 = 5 + 5) + uint32_t tmp = vlcval[s] & ((1U << len) - 1U); // suffix value for 2 quads + vlcval[s] = VLC_dec[s].advance(len); + uvlc_result >>= 4; + // quad 0 length + len = uvlc_result & 0x7; // quad 0 suffix length + uvlc_result >>= 3; + // U = 1 + u; always kappa = 1 in initial line pair + uint32_t u0 = 1 + (uvlc_result & 7) + (tmp & ~(0xFFU << len)); + uint32_t u1 = 1 + (uvlc_result >> 3) + (tmp >> len); + + sp[s][1] = static_cast(u0); + sp[s][3] = static_cast(u1); + sp[s] += 4; + } + } + OPENHTJ2K_UNROLL + for (size_t s = 0; s < NL; ++s) { + // Zero the 8-byte guard past the last VLC row: the MagSgn step-2 reads + // up to 4 extra uint16_t beyond the written region in each row. + std::memset(sp[s], 0, sizeof(uint16_t) * 4); + } + + // Non-initial line-pairs + for (uint16_t row = 1; row < QH; row++) { + OPENHTJ2K_UNROLL + for (size_t s = 0; s < NL; ++s) { + if (!skip_sigma) { + sp0[s] = ln[s].block_states + (row * 2U + 1U) * ln[s].blkstate_stride + 1U; + sp1[s] = sp0[s] + ln[s].blkstate_stride; + } + sp[s] = ln[s].scratch + row * sstr; + // calculate context for the next quad: w, sw, nw are always 0 at the head of a row + context[s] = ((sp[s][0 - sstr] & 0xA0U) << 2) | ((sp[s][2 - sstr] & 0x20U) << 4); + } + for (int32_t qx = QW; qx > 0; qx -= 2) { + OPENHTJ2K_UNROLL + for (size_t s = 0; s < NL; ++s) { + // Decoding of significance and EMB patterns and unsigned residual offsets + vlcval[s] = VLC_dec[s].fetch(); + uint16_t t0 = dec_CxtVLC_table1_fast_16[(vlcval[s] & 0x7F) + context[s]]; + if (context[s] == 0) { + mel_run[s] -= 2; + t0 = (mel_run[s] == -1) ? t0 : 0; + if (mel_run[s] < 0) { + mel_run[s] = MEL[s].get_run(); + } + } + // calculate context for the next quad, Eq. (2) in the spec + context[s] = ((t0 & 0x40U) << 2) | ((t0 & 0x80U) << 1); // (w | sw) << 8 + context[s] |= + (sp[s][0 - sstr] & 0x80U) | ((sp[s][2 - sstr] & 0xA0U) << 2); // ((nw | n) << 7) | (ne << 9) + context[s] |= (sp[s][4 - sstr] & 0x20U) << 4; // ( nf) << 9 + + sp[s][0] = t0; + tv0[s] = t0; + + vlcval[s] = VLC_dec[s].advance((t0 & 0x000F) >> 1); + } + OPENHTJ2K_UNROLL + for (size_t s = 0; s < NL; ++s) { + // Decoding of significance and EMB patterns and unsigned residual offsets + uint16_t t1 = dec_CxtVLC_table1_fast_16[(vlcval[s] & 0x7F) + context[s]]; + if (context[s] == 0 && qx > 1) { + mel_run[s] -= 2; + t1 = (mel_run[s] == -1) ? t1 : 0; + if (mel_run[s] < 0) { + mel_run[s] = MEL[s].get_run(); + } + } + t1 = (qx > 1) ? t1 : 0; + // calculate context for the next quad, Eq. (2) in the spec + context[s] = ((t1 & 0x40U) << 2) | ((t1 & 0x80U) << 1); // (w | sw) << 8 + context[s] |= + (sp[s][2 - sstr] & 0x80U) | ((sp[s][4 - sstr] & 0xA0U) << 2); // ((nw | n) << 7) | (ne << 9) + context[s] |= (sp[s][6 - sstr] & 0x20U) << 4; // ( nf) << 9 + + sp[s][2] = t1; + tv1[s] = t1; + + // store sigma + if (!skip_sigma) { + uint16_t t0s = tv0[s]; + *sp0[s]++ = ((t0s >> 4) >> 0) & 1; + *sp0[s]++ = ((t0s >> 4) >> 2) & 1; + *sp0[s]++ = ((t1 >> 4) >> 0) & 1; + *sp0[s]++ = ((t1 >> 4) >> 2) & 1; + *sp1[s]++ = ((t0s >> 4) >> 1) & 1; + *sp1[s]++ = ((t0s >> 4) >> 3) & 1; + *sp1[s]++ = ((t1 >> 4) >> 1) & 1; + *sp1[s]++ = ((t1 >> 4) >> 3) & 1; + } + + vlcval[s] = VLC_dec[s].advance((t1 & 0x000F) >> 1); + } + OPENHTJ2K_UNROLL + for (size_t s = 0; s < NL; ++s) { + // UVLC decoding + uint32_t u_off0 = tv0[s] & 1; + uint32_t u_off1 = tv1[s] & 1; + uint32_t idx = (vlcval[s] & 0x3F) + (u_off0 << 6U) + (u_off1 << 7U); + + uint32_t uvlc_result = uvlc_dec_1[idx]; + // remove total prefix length + vlcval[s] = VLC_dec[s].advance(uvlc_result & 0x7); + uvlc_result >>= 3; + // extract suffixes for quad 0 and 1 + uint32_t len = uvlc_result & 0xF; // suffix length for 2 quads (up to 10 = 5 + 5) + uint32_t tmp = vlcval[s] & ((1U << len) - 1U); // suffix value for 2 quads + vlcval[s] = VLC_dec[s].advance(len); + uvlc_result >>= 4; + // quad 0 length + len = uvlc_result & 0x7; // quad 0 suffix length + uvlc_result >>= 3; + uint32_t u0 = (uvlc_result & 7) + (tmp & ~(0xFFU << len)); + uint32_t u1 = (uvlc_result >> 3) + (tmp >> len); + + sp[s][1] = static_cast(u0); + sp[s][3] = static_cast(u1); + sp[s] += 4; + } + } + OPENHTJ2K_UNROLL + for (size_t s = 0; s < NL; ++s) { + // Zero the 8-byte guard: same reason as initial row. + std::memset(sp[s], 0, sizeof(uint16_t) * 4); + } + } +} + /******************************************************************************** * fwd_buf: *******************************************************************************/ @@ -705,47 +987,43 @@ class fwd_buf { // once per codeblock (or per row) and passed by reference to avoid // per-call L1 cache loads. struct DecodeConstants { - int16x8_t flag_mask; - int16x8_t mul_mask; + int16x8_t flag_mask; + int16x8_t mul_mask; uint8x16_t dup_lo; uint8x16_t add_01; uint8x16_t bit_tab; DecodeConstants() { - alignas(16) static const int16_t fm[8] = { - (int16_t)0x1110, 0x2220, 0x4440, (int16_t)0x8880, - (int16_t)0x1110, 0x2220, 0x4440, (int16_t)0x8880}; - alignas(16) static const int16_t mm[8] = {8, 4, 2, 1, 8, 4, 2, 1}; - alignas(16) static const uint8_t dl[16] = { - 0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14}; - alignas(16) static const uint8_t a01[16] = { - 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1}; - alignas(16) static const uint8_t bt[16] = { - 0xFF, 127, 63, 31, 15, 7, 3, 1, 0xFF, 127, 63, 31, 15, 7, 3, 1}; - flag_mask = vld1q_s16(fm); - mul_mask = vld1q_s16(mm); - dup_lo = vld1q_u8(dl); - add_01 = vld1q_u8(a01); - bit_tab = vld1q_u8(bt); + alignas(16) static const int16_t fm[8] = {(int16_t)0x1110, 0x2220, 0x4440, (int16_t)0x8880, + (int16_t)0x1110, 0x2220, 0x4440, (int16_t)0x8880}; + alignas(16) static const int16_t mm[8] = {8, 4, 2, 1, 8, 4, 2, 1}; + alignas(16) static const uint8_t dl[16] = {0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14}; + alignas(16) static const uint8_t a01[16] = {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1}; + alignas(16) static const uint8_t bt[16] = {0xFF, 127, 63, 31, 15, 7, 3, 1, + 0xFF, 127, 63, 31, 15, 7, 3, 1}; + flag_mask = vld1q_s16(fm); + mul_mask = vld1q_s16(mm); + dup_lo = vld1q_u8(dl); + add_01 = vld1q_u8(a01); + bit_tab = vld1q_u8(bt); } }; - FORCE_INLINE int16x8_t decode_two_quads_16bit(uint16_t tv0, uint16_t tv1, - uint16_t U0, uint16_t U1, - uint8_t pLSB_adj, int16x4_t &v_n, - const DecodeConstants &c) { + FORCE_INLINE int16x8_t decode_two_quads_16bit(uint16_t tv0, uint16_t tv1, uint16_t U0, uint16_t U1, + uint8_t pLSB_adj, int16x4_t &v_n, + const DecodeConstants &c) { const int16x8_t vone16 = vdupq_n_s16(1); const int16x8_t vtwo16 = vdupq_n_s16(2); const int16x8_t vzero16 = vdupq_n_s16(0); int16x8_t row = vzero16; // Broadcast inf words: tv0 to lanes 0-3, tv1 to lanes 4-7. - int16x8_t w0 = vcombine_s16(vdup_n_s16(static_cast(tv0)), - vdup_n_s16(static_cast(tv1))); + int16x8_t w0 = + vcombine_s16(vdup_n_s16(static_cast(tv0)), vdup_n_s16(static_cast(tv1))); // Extract per-sample significance/EMB flags. - int16x8_t flags = vandq_s16(w0, c.flag_mask); - uint16x8_t insig = vceqq_s16(flags, vzero16); + int16x8_t flags = vandq_s16(w0, c.flag_mask); + uint16x8_t insig = vceqq_s16(flags, vzero16); // Early exit if all 8 samples are insignificant. if (vmaxvq_u16(vreinterpretq_u16_s16(flags)) == 0) { @@ -753,8 +1031,8 @@ class fwd_buf { } // Broadcast U values: U0 to lanes 0-3, U1 to lanes 4-7. - int16x8_t U_vec = vcombine_s16(vdup_n_s16(static_cast(U0)), - vdup_n_s16(static_cast(U1))); + int16x8_t U_vec = + vcombine_s16(vdup_n_s16(static_cast(U0)), vdup_n_s16(static_cast(U1))); // Normalize flags: multiply by {8,4,2,1}. flags = vmulq_s16(flags, c.mul_mask); @@ -766,10 +1044,10 @@ class fwd_buf { // Inclusive prefix sum of m_n (8 x 16-bit). int16x8_t inc_sum = m_n; - inc_sum = vaddq_s16(inc_sum, vextq_s16(vzero16, inc_sum, 7)); - inc_sum = vaddq_s16(inc_sum, vextq_s16(vzero16, inc_sum, 6)); - inc_sum = vaddq_s16(inc_sum, vextq_s16(vzero16, inc_sum, 4)); - int total_mn = vgetq_lane_s16(inc_sum, 7); + inc_sum = vaddq_s16(inc_sum, vextq_s16(vzero16, inc_sum, 7)); + inc_sum = vaddq_s16(inc_sum, vextq_s16(vzero16, inc_sum, 6)); + inc_sum = vaddq_s16(inc_sum, vextq_s16(vzero16, inc_sum, 4)); + int total_mn = vgetq_lane_s16(inc_sum, 7); // Fetch raw MagSgn bits (gated on total_mn > 0). uint8x16_t ms_raw = vdupq_n_u8(0); @@ -798,33 +1076,33 @@ class fwd_buf { // bit_shift is ready by now (computed in parallel above). uint16x8_t bit_shift16 = vaddq_u16(vreinterpretq_u16_u8(bit_shift), vdupq_n_u16(0x0101)); - uint16x8_t d0_16 = vmulq_u16(vreinterpretq_u16_u8(d0), bit_shift16); - d0_16 = vshrq_n_u16(d0_16, 8); - uint16x8_t d1_16 = vmulq_u16(vreinterpretq_u16_u8(d1), bit_shift16); - d1_16 = vandq_u16(d1_16, vdupq_n_u16(0xFF00)); + uint16x8_t d0_16 = vmulq_u16(vreinterpretq_u16_u8(d0), bit_shift16); + d0_16 = vshrq_n_u16(d0_16, 8); + uint16x8_t d1_16 = vmulq_u16(vreinterpretq_u16_u8(d1), bit_shift16); + d1_16 = vandq_u16(d1_16, vdupq_n_u16(0xFF00)); uint16x8_t ms_vec = vorrq_u16(d0_16, d1_16); // Compute 2^m_n = (2 - e_k) << (U_q - 1). NEON per-lane variable shift. - int16x8_t w0_val = vsubq_s16(vtwo16, vreinterpretq_s16_u16(emb_k)); - int16x8_t Uq_m1 = vsubq_s16(U_vec, vone16); + int16x8_t w0_val = vsubq_s16(vtwo16, vreinterpretq_s16_u16(emb_k)); + int16x8_t Uq_m1 = vsubq_s16(U_vec, vone16); uint16x8_t shift_v = vshlq_u16(vreinterpretq_u16_s16(w0_val), Uq_m1); // Mask ms_vec to m_n magnitude bits. ms_vec = vandq_u16(ms_vec, vsubq_u16(shift_v, vreinterpretq_u16_s16(vone16))); // Place e_1 at bit position m_n: OR shift where e_1 flag is set. - uint16x8_t emb1_absent = vceqq_u16( - vandq_u16(vreinterpretq_u16_s16(flags), vdupq_n_u16(0x800)), vdupq_n_u16(0)); + uint16x8_t emb1_absent = + vceqq_u16(vandq_u16(vreinterpretq_u16_s16(flags), vdupq_n_u16(0x800)), vdupq_n_u16(0)); ms_vec = vorrq_u16(ms_vec, vbicq_u16(shift_v, emb1_absent)); // Build final sample: sign at bit 15, magnitude at pLSB_adj - 1. - uint16x8_t tvn = vbicq_u16(ms_vec, insig); // save for v_n - uint16x8_t w_sign = vshlq_n_u16(ms_vec, 15); // sign bit -> bit 15 + uint16x8_t tvn = vbicq_u16(ms_vec, insig); // save for v_n + uint16x8_t w_sign = vshlq_n_u16(ms_vec, 15); // sign bit -> bit 15 ms_vec = vorrq_u16(ms_vec, vreinterpretq_u16_s16(vone16)); // bin center ms_vec = vaddq_u16(ms_vec, vreinterpretq_u16_s16(vtwo16)); // + 2 - ms_vec = vshlq_u16(ms_vec, vdupq_n_s16(pLSB_adj - 1)); // runtime shift - ms_vec = vorrq_u16(ms_vec, w_sign); // sign - row = vreinterpretq_s16_u16(vbicq_u16(ms_vec, insig)); + ms_vec = vshlq_u16(ms_vec, vdupq_n_s16(pLSB_adj - 1)); // runtime shift + ms_vec = vorrq_u16(ms_vec, w_sign); // sign + row = vreinterpretq_s16_u16(vbicq_u16(ms_vec, insig)); // Update v_n: extract row-1 magnitudes (odd elements) per column. int16x4_t tvn_lo = vget_low_s16(vreinterpretq_s16_u16(tvn)); @@ -903,19 +1181,22 @@ class fwd_buf { FORCE_INLINE v128_t fetch(const v128_t &m) { v128_t t = fetch_raw(); - uint64_t lo = (uint64_t)wasm_i64x2_extract_lane(t, 0); - uint64_t hi = (uint64_t)wasm_i64x2_extract_lane(t, 1); + uint64_t lo = (uint64_t)wasm_i64x2_extract_lane(t, 0); + uint64_t hi = (uint64_t)wasm_i64x2_extract_lane(t, 1); __uint128_t v128i = ((__uint128_t)hi << 64) | lo; v128_t vtmp; int32_t m0 = wasm_i32x4_extract_lane(m, 0); int32_t m1 = wasm_i32x4_extract_lane(m, 1); int32_t m2 = wasm_i32x4_extract_lane(m, 2); int32_t m3 = wasm_i32x4_extract_lane(m, 3); - int32_t r0 = (int32_t)(v128i & 0xFFFFFFFFU); v128i >>= m0; - int32_t r1 = (int32_t)(v128i & 0xFFFFFFFFU); v128i >>= m1; - int32_t r2 = (int32_t)(v128i & 0xFFFFFFFFU); v128i >>= m2; + int32_t r0 = (int32_t)(v128i & 0xFFFFFFFFU); + v128i >>= m0; + int32_t r1 = (int32_t)(v128i & 0xFFFFFFFFU); + v128i >>= m1; + int32_t r2 = (int32_t)(v128i & 0xFFFFFFFFU); + v128i >>= m2; int32_t r3 = (int32_t)(v128i & 0xFFFFFFFFU); - vtmp = wasm_i32x4_make(r0, r1, r2, r3); + vtmp = wasm_i32x4_make(r0, r1, r2, r3); advance((uint32_t)(m0 + m1 + m2 + m3)); return vtmp; } @@ -930,40 +1211,38 @@ class fwd_buf { * @return v128_t [r0c0,r1c0,r0c1,r1c1,r0c2,r1c2,r0c3,r1c3] as int16 * sign at bit 15; expand to int32 via shl 16 + extend */ - FORCE_INLINE v128_t decode_two_quads_16bit_wasm(uint16_t tv0, uint16_t tv1, - uint16_t U0, uint16_t U1, - uint8_t pLSB_adj, v128_t &v_n) { + FORCE_INLINE v128_t decode_two_quads_16bit_wasm(uint16_t tv0, uint16_t tv1, uint16_t U0, uint16_t U1, + uint8_t pLSB_adj, v128_t &v_n) { const v128_t vone16 = wasm_i16x8_const_splat(1); const v128_t vtwo16 = wasm_i16x8_const_splat(2); const v128_t vzero16 = wasm_i16x8_const_splat(0); v128_t row = vzero16; // Broadcast tv0 to lanes 0-3, tv1 to lanes 4-7. - v128_t w0 = wasm_i16x8_shuffle(wasm_i16x8_splat((int16_t)tv0), - wasm_i16x8_splat((int16_t)tv1), 0, 1, 2, 3, 8, 9, 10, 11); + v128_t w0 = wasm_i16x8_shuffle(wasm_i16x8_splat((int16_t)tv0), wasm_i16x8_splat((int16_t)tv1), 0, 1, 2, + 3, 8, 9, 10, 11); // Extract per-sample significance/EMB flags. - alignas(16) static const int16_t flag_mask_arr[8] = { - (int16_t)0x1110, 0x2220, 0x4440, (int16_t)0x8880, - (int16_t)0x1110, 0x2220, 0x4440, (int16_t)0x8880}; - v128_t flags = wasm_v128_and(w0, wasm_v128_load(flag_mask_arr)); + alignas(16) static const int16_t flag_mask_arr[8] = {(int16_t)0x1110, 0x2220, 0x4440, (int16_t)0x8880, + (int16_t)0x1110, 0x2220, 0x4440, (int16_t)0x8880}; + v128_t flags = wasm_v128_and(w0, wasm_v128_load(flag_mask_arr)); v128_t insig = wasm_i16x8_eq(flags, vzero16); // all-1 where insignificant // Early exit if all 8 samples are insignificant. if (!wasm_v128_any_true(flags)) return row; // Broadcast U values: U0 to lanes 0-3, U1 to lanes 4-7. - v128_t U_vec = wasm_i16x8_shuffle(wasm_i16x8_splat((int16_t)U0), - wasm_i16x8_splat((int16_t)U1), 0, 1, 2, 3, 8, 9, 10, 11); + v128_t U_vec = wasm_i16x8_shuffle(wasm_i16x8_splat((int16_t)U0), wasm_i16x8_splat((int16_t)U1), 0, 1, 2, + 3, 8, 9, 10, 11); // Normalize flags: emb_k → bit 15, emb_1 → bit 11, rho → bit 7. alignas(16) static const int16_t mul_arr[8] = {8, 4, 2, 1, 8, 4, 2, 1}; - flags = wasm_i16x8_mul(flags, wasm_v128_load(mul_arr)); + flags = wasm_i16x8_mul(flags, wasm_v128_load(mul_arr)); // m_n = U - e_k; zero inactive lanes. - v128_t emb_k = wasm_u16x8_shr(flags, 15); // 0 or 1 + v128_t emb_k = wasm_u16x8_shr(flags, 15); // 0 or 1 v128_t m_n = wasm_i16x8_sub(U_vec, emb_k); - m_n = wasm_v128_andnot(m_n, insig); // zero where insignificant + m_n = wasm_v128_andnot(m_n, insig); // zero where insignificant // Inclusive prefix sum of m_n (8 × int16). v128_t inc_sum = m_n; @@ -987,25 +1266,22 @@ class fwd_buf { v128_t bit_idx = wasm_v128_and(ex_sum, wasm_i16x8_const_splat(7)); // Duplicate low byte of each 16-bit byte_idx to both bytes of the pair. - alignas(16) static const uint8_t dup_lo[16] = { - 0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14}; - v128_t bidx = wasm_i8x16_swizzle(byte_idx, wasm_v128_load(dup_lo)); - alignas(16) static const uint8_t add_01[16] = { - 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1}; - bidx = wasm_i8x16_add(bidx, wasm_v128_load(add_01)); - v128_t d0 = wasm_i8x16_swizzle(ms_raw, bidx); - bidx = wasm_i8x16_add(bidx, wasm_i8x16_const_splat(1)); - v128_t d1 = wasm_i8x16_swizzle(ms_raw, bidx); + alignas(16) static const uint8_t dup_lo[16] = {0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14}; + v128_t bidx = wasm_i8x16_swizzle(byte_idx, wasm_v128_load(dup_lo)); + alignas(16) static const uint8_t add_01[16] = {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1}; + bidx = wasm_i8x16_add(bidx, wasm_v128_load(add_01)); + v128_t d0 = wasm_i8x16_swizzle(ms_raw, bidx); + bidx = wasm_i8x16_add(bidx, wasm_i8x16_const_splat(1)); + v128_t d1 = wasm_i8x16_swizzle(ms_raw, bidx); // Bit-level alignment: multiply-shift to align bits to bit 0. - alignas(16) static const uint8_t bit_tab[16] = { - 0xFF, 127, 63, 31, 15, 7, 3, 1, 0xFF, 127, 63, 31, 15, 7, 3, 1}; - v128_t bit_shift = wasm_i8x16_swizzle(wasm_v128_load(bit_tab), bit_idx); + alignas(16) static const uint8_t bit_tab[16] = {0xFF, 127, 63, 31, 15, 7, 3, 1, + 0xFF, 127, 63, 31, 15, 7, 3, 1}; + v128_t bit_shift = wasm_i8x16_swizzle(wasm_v128_load(bit_tab), bit_idx); v128_t bit_shift16 = wasm_i16x8_add(bit_shift, wasm_i16x8_const_splat(0x0101)); v128_t d0_16 = wasm_u16x8_shr(wasm_i16x8_mul(d0, bit_shift16), 8); - v128_t d1_16 = wasm_v128_and(wasm_i16x8_mul(d1, bit_shift16), - wasm_i16x8_const_splat((int16_t)0xFF00)); + v128_t d1_16 = wasm_v128_and(wasm_i16x8_mul(d1, bit_shift16), wasm_i16x8_const_splat((int16_t)0xFF00)); v128_t ms_vec = wasm_v128_or(d0_16, d1_16); // shift_v = (2 - e_k) << (U - 1): U0 for lanes 0-3, U1 for lanes 4-7. @@ -1018,22 +1294,22 @@ class fwd_buf { ms_vec = wasm_v128_and(ms_vec, wasm_i16x8_sub(shift_v, vone16)); // Place e_1 at bit position m_n. - v128_t emb1_absent = wasm_i16x8_eq( - wasm_v128_and(flags, wasm_i16x8_const_splat((int16_t)0x800)), vzero16); + v128_t emb1_absent = + wasm_i16x8_eq(wasm_v128_and(flags, wasm_i16x8_const_splat((int16_t)0x800)), vzero16); ms_vec = wasm_v128_or(ms_vec, wasm_v128_andnot(shift_v, emb1_absent)); // Build final sample: sign at bit 15, magnitude at pLSB_adj-1. - v128_t tvn = wasm_v128_andnot(ms_vec, insig); // save pre-shift for v_n - v128_t w_sign = wasm_i16x8_shl(ms_vec, 15); // sign bit → bit 15 - ms_vec = wasm_v128_or(ms_vec, vone16); // bin center - ms_vec = wasm_i16x8_add(ms_vec, vtwo16); // +2 - ms_vec = wasm_i16x8_shl(ms_vec, pLSB_adj - 1); // runtime shift - ms_vec = wasm_v128_or(ms_vec, w_sign); // apply sign - row = wasm_v128_andnot(ms_vec, insig); // zero inactive + v128_t tvn = wasm_v128_andnot(ms_vec, insig); // save pre-shift for v_n + v128_t w_sign = wasm_i16x8_shl(ms_vec, 15); // sign bit → bit 15 + ms_vec = wasm_v128_or(ms_vec, vone16); // bin center + ms_vec = wasm_i16x8_add(ms_vec, vtwo16); // +2 + ms_vec = wasm_i16x8_shl(ms_vec, pLSB_adj - 1); // runtime shift + ms_vec = wasm_v128_or(ms_vec, w_sign); // apply sign + row = wasm_v128_andnot(ms_vec, insig); // zero inactive // Update v_n: odd lanes (1,3,5,7) → low 4 lanes of v_n. v128_t tvn_odd = wasm_i16x8_shuffle(tvn, vzero16, 1, 3, 5, 7, 8, 8, 8, 8); - v_n = wasm_v128_or(v_n, tvn_odd); + v_n = wasm_v128_or(v_n, tvn_odd); return row; } @@ -1281,21 +1557,18 @@ class fwd_buf { // Returns __m256i: lower 128 = mu for quad 0, upper 128 = mu for quad 1. // v_n is updated with the magnitude-bit positions of both quads. // Fetches and advances the MagSgn bit-stream sequentially (one call per quad). -#if defined(__AVX2__) - FORCE_INLINE __m256i decode_two_quads(__m256i qinf256, __m256i U_q256, uint8_t pLSB, - __m128i &v_n) { + #if defined(__AVX2__) + FORCE_INLINE __m256i decode_two_quads(__m256i qinf256, __m256i U_q256, uint8_t pLSB, __m128i &v_n) { const __m256i vone256 = _mm256_set1_epi32(1); __m256i row256 = _mm256_setzero_si256(); // Significance flags for all 8 samples (both quads) at once. __m256i flags = _mm256_and_si256( - qinf256, _mm256_set_epi32(0x8880, 0x4440, 0x2220, 0x1110, - 0x8880, 0x4440, 0x2220, 0x1110)); + qinf256, _mm256_set_epi32(0x8880, 0x4440, 0x2220, 0x1110, 0x8880, 0x4440, 0x2220, 0x1110)); __m256i insig = _mm256_cmpeq_epi32(flags, _mm256_setzero_si256()); if ((uint32_t)_mm256_movemask_epi8(insig) != 0xFFFFFFFFu) { - flags = _mm256_mullo_epi16( - flags, _mm256_set_epi16(1, 1, 2, 2, 4, 4, 8, 8, 1, 1, 2, 2, 4, 4, 8, 8)); + flags = _mm256_mullo_epi16(flags, _mm256_set_epi16(1, 1, 2, 2, 4, 4, 8, 8, 1, 1, 2, 2, 4, 4, 8, 8)); __m256i emb_k = _mm256_srli_epi32(flags, 15); __m256i m_n = _mm256_andnot_si256(insig, _mm256_sub_epi32(U_q256, emb_k)); @@ -1326,13 +1599,13 @@ class fwd_buf { // quad 0 reads from ms_vec0 and quad 1 from ms_vec1 independently. __m256i byte_idx = _mm256_srli_epi32(ex_sum, 3); __m256i bit_idx = _mm256_and_si256(ex_sum, _mm256_set1_epi32(7)); - byte_idx = _mm256_shuffle_epi8( - byte_idx, _mm256_set_epi32(0x0C0C0C0C, 0x08080808, 0x04040404, 0x00000000, - 0x0C0C0C0C, 0x08080808, 0x04040404, 0x00000000)); - byte_idx = _mm256_add_epi32(byte_idx, _mm256_set1_epi32(0x03020100)); - __m256i d0 = _mm256_shuffle_epi8(ms_vec, byte_idx); - byte_idx = _mm256_add_epi32(byte_idx, _mm256_set1_epi32(0x01010101)); - __m256i d1 = _mm256_shuffle_epi8(ms_vec, byte_idx); + byte_idx = + _mm256_shuffle_epi8(byte_idx, _mm256_set_epi32(0x0C0C0C0C, 0x08080808, 0x04040404, 0x00000000, + 0x0C0C0C0C, 0x08080808, 0x04040404, 0x00000000)); + byte_idx = _mm256_add_epi32(byte_idx, _mm256_set1_epi32(0x03020100)); + __m256i d0 = _mm256_shuffle_epi8(ms_vec, byte_idx); + byte_idx = _mm256_add_epi32(byte_idx, _mm256_set1_epi32(0x01010101)); + __m256i d1 = _mm256_shuffle_epi8(ms_vec, byte_idx); bit_idx = _mm256_or_si256(bit_idx, _mm256_slli_epi32(bit_idx, 16)); __m128i tab128 = _mm_set_epi8(1, 3, 7, 15, 31, 63, 127, -1, 1, 3, 7, 15, 31, 63, 127, -1); @@ -1360,13 +1633,10 @@ class fwd_buf { tvn = _mm256_shuffle_epi8( tvn, _mm256_set_epi8( // Upper lane (quad 1, N=1): lanes 1,3 → offsets 8..15 - 0x0F, 0x0E, 0x0D, 0x0C, 0x07, 0x06, 0x05, 0x04, - -1, -1, -1, -1, -1, -1, -1, -1, + 0x0F, 0x0E, 0x0D, 0x0C, 0x07, 0x06, 0x05, 0x04, -1, -1, -1, -1, -1, -1, -1, -1, // Lower lane (quad 0, N=0): lanes 1,3 → offsets 0..7 - -1, -1, -1, -1, -1, -1, -1, -1, - 0x0F, 0x0E, 0x0D, 0x0C, 0x07, 0x06, 0x05, 0x04)); - v_n = _mm_or_si128(v_n, _mm_or_si128(_mm256_castsi256_si128(tvn), - _mm256_extracti128_si256(tvn, 1))); + -1, -1, -1, -1, -1, -1, -1, -1, 0x0F, 0x0E, 0x0D, 0x0C, 0x07, 0x06, 0x05, 0x04)); + v_n = _mm_or_si128(v_n, _mm_or_si128(_mm256_castsi256_si128(tvn), _mm256_extracti128_si256(tvn, 1))); } return row256; } @@ -1381,8 +1651,7 @@ class fwd_buf { // Returns __m256i of 16 × int16_t: interleaved row0/row1 per quad column. // Sign bit is in bit 15 of each int16_t. // Caller must expand to int32_t via zero-extend then slli_epi32(x, 16). - FORCE_INLINE __m256i decode_four_quads(__m128i inf_u_q, __m128i U_q, uint8_t pLSB_adj, - __m128i &v_n) { + FORCE_INLINE __m256i decode_four_quads(__m128i inf_u_q, __m128i U_q, uint8_t pLSB_adj, __m128i &v_n) { const __m256i vone16 = _mm256_set1_epi16(1); const __m256i vtwo16 = _mm256_set1_epi16(2); __m256i row256 = _mm256_setzero_si256(); @@ -1390,31 +1659,28 @@ class fwd_buf { // Broadcast each quad's inf word to all 4 sample slots in its 64-bit group. // inf_u_q = [inf0(16b), u0(16b), inf1(16b), u1(16b), inf2(16b), u2(16b), inf3(16b), u3(16b)] // Step 1: duplicate each inf to both 16-bit positions of its 32-bit slot. - __m128i ddd = _mm_shuffle_epi8( - inf_u_q, _mm_set_epi32(0x0D0C0D0C, 0x09080908, 0x05040504, 0x01000100)); + __m128i ddd = _mm_shuffle_epi8(inf_u_q, _mm_set_epi32(0x0D0C0D0C, 0x09080908, 0x05040504, 0x01000100)); // ddd (8 × 16-bit): [inf0,inf0, inf1,inf1, inf2,inf2, inf3,inf3] // Step 2: broadcast each 32-bit pair to two 32-bit slots in the 256-bit register. - __m256i w0 = _mm256_permutevar8x32_epi32( - _mm256_castsi128_si256(ddd), _mm256_setr_epi32(0, 0, 1, 1, 2, 2, 3, 3)); + __m256i w0 = + _mm256_permutevar8x32_epi32(_mm256_castsi128_si256(ddd), _mm256_setr_epi32(0, 0, 1, 1, 2, 2, 3, 3)); // w0 (16 × 16-bit): [inf0 ×4, inf1 ×4, inf2 ×4, inf3 ×4] // Extract significance/EMB flags per sample. - __m256i flags = _mm256_and_si256( - w0, _mm256_set_epi16((int16_t)0x8880, 0x4440, 0x2220, 0x1110, - (int16_t)0x8880, 0x4440, 0x2220, 0x1110, - (int16_t)0x8880, 0x4440, 0x2220, 0x1110, - (int16_t)0x8880, 0x4440, 0x2220, 0x1110)); + __m256i flags = + _mm256_and_si256(w0, _mm256_set_epi16((int16_t)0x8880, 0x4440, 0x2220, 0x1110, (int16_t)0x8880, + 0x4440, 0x2220, 0x1110, (int16_t)0x8880, 0x4440, 0x2220, + 0x1110, (int16_t)0x8880, 0x4440, 0x2220, 0x1110)); __m256i insig = _mm256_cmpeq_epi16(flags, _mm256_setzero_si256()); if ((uint32_t)_mm256_movemask_epi8(insig) != 0xFFFFFFFFu) { // Broadcast each U_q value (uint32_t → uint16_t pair) to 4 samples per quad. - ddd = _mm_or_si128(_mm_bslli_si128(U_q, 2), U_q); - __m256i U_q_avx = _mm256_permutevar8x32_epi32( - _mm256_castsi128_si256(ddd), _mm256_setr_epi32(0, 0, 1, 1, 2, 2, 3, 3)); + ddd = _mm_or_si128(_mm_bslli_si128(U_q, 2), U_q); + __m256i U_q_avx = _mm256_permutevar8x32_epi32(_mm256_castsi128_si256(ddd), + _mm256_setr_epi32(0, 0, 1, 1, 2, 2, 3, 3)); // U_q_avx (16 × 16-bit): [U0 ×4, U1 ×4, U2 ×4, U3 ×4] - flags = _mm256_mullo_epi16( - flags, _mm256_set_epi16(1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8)); + flags = _mm256_mullo_epi16(flags, _mm256_set_epi16(1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8)); // After mullo: e_k at bit 15, e_1 at bit 11, rho at bit 7. __m256i emb_k = _mm256_srli_epi16(flags, 15); @@ -1423,11 +1689,11 @@ class fwd_buf { // Inclusive prefix sum of m_n within each 128-bit lane (= one quad-pair). __m256i inc_sum = m_n; - inc_sum = _mm256_add_epi16(inc_sum, _mm256_bslli_epi128(inc_sum, 2)); - inc_sum = _mm256_add_epi16(inc_sum, _mm256_bslli_epi128(inc_sum, 4)); - inc_sum = _mm256_add_epi16(inc_sum, _mm256_bslli_epi128(inc_sum, 8)); - int total_mn0 = _mm256_extract_epi16(inc_sum, 7); // quads 0+1 total bits - int total_mn1 = _mm256_extract_epi16(inc_sum, 15); // quads 2+3 total bits + inc_sum = _mm256_add_epi16(inc_sum, _mm256_bslli_epi128(inc_sum, 2)); + inc_sum = _mm256_add_epi16(inc_sum, _mm256_bslli_epi128(inc_sum, 4)); + inc_sum = _mm256_add_epi16(inc_sum, _mm256_bslli_epi128(inc_sum, 8)); + int total_mn0 = _mm256_extract_epi16(inc_sum, 7); // quads 0+1 total bits + int total_mn1 = _mm256_extract_epi16(inc_sum, 15); // quads 2+3 total bits __m128i ms_vec0 = _mm_setzero_si128(); __m128i ms_vec1 = _mm_setzero_si128(); @@ -1447,22 +1713,21 @@ class fwd_buf { __m256i byte_idx = _mm256_srli_epi16(ex_sum, 3); __m256i bit_idx = _mm256_and_si256(ex_sum, _mm256_set1_epi16(7)); byte_idx = _mm256_shuffle_epi8( - byte_idx, _mm256_set_epi16(0x0E0E, 0x0C0C, 0x0A0A, 0x0808, 0x0606, 0x0404, - 0x0202, 0x0000, 0x0E0E, 0x0C0C, 0x0A0A, 0x0808, - 0x0606, 0x0404, 0x0202, 0x0000)); - byte_idx = _mm256_add_epi16(byte_idx, _mm256_set1_epi16(0x0100)); - __m256i d0 = _mm256_shuffle_epi8(ms_vec, byte_idx); - byte_idx = _mm256_add_epi16(byte_idx, _mm256_set1_epi16(0x0101)); - __m256i d1 = _mm256_shuffle_epi8(ms_vec, byte_idx); + byte_idx, _mm256_set_epi16(0x0E0E, 0x0C0C, 0x0A0A, 0x0808, 0x0606, 0x0404, 0x0202, 0x0000, 0x0E0E, + 0x0C0C, 0x0A0A, 0x0808, 0x0606, 0x0404, 0x0202, 0x0000)); + byte_idx = _mm256_add_epi16(byte_idx, _mm256_set1_epi16(0x0100)); + __m256i d0 = _mm256_shuffle_epi8(ms_vec, byte_idx); + byte_idx = _mm256_add_epi16(byte_idx, _mm256_set1_epi16(0x0101)); + __m256i d1 = _mm256_shuffle_epi8(ms_vec, byte_idx); // For 16-bit elements, bit_idx high byte must stay 0 so vpshufb maps it to // table[0]=0xFF; after +0x0101 the high byte wraps to 0x00, giving the correct // single-byte multiplier for mullo_epi16. (The 32-bit decode_two_quads path // duplicates across 16-bit halves of a 32-bit element, which is different.) - __m256i bit_shift = _mm256_shuffle_epi8( - _mm256_set_epi8(1, 3, 7, 15, 31, 63, 127, -1, 1, 3, 7, 15, 31, 63, 127, -1, - 1, 3, 7, 15, 31, 63, 127, -1, 1, 3, 7, 15, 31, 63, 127, -1), - bit_idx); + __m256i bit_shift = + _mm256_shuffle_epi8(_mm256_set_epi8(1, 3, 7, 15, 31, 63, 127, -1, 1, 3, 7, 15, 31, 63, 127, -1, 1, + 3, 7, 15, 31, 63, 127, -1, 1, 3, 7, 15, 31, 63, 127, -1), + bit_idx); bit_shift = _mm256_add_epi16(bit_shift, _mm256_set1_epi16(0x0101)); d0 = _mm256_mullo_epi16(d0, bit_shift); d0 = _mm256_srli_epi16(d0, 8); @@ -1474,23 +1739,21 @@ class fwd_buf { // AVX2 has no _mm256_sllv_epi16; split into four _mm_sll_epi16 calls, // one per quad (each quad has a uniform shift = U_q - 1). // w0 = (2 - e_k): 1 when e_k=1, 2 when e_k=0. - w0 = _mm256_sub_epi16(vtwo16, emb_k); - __m256i Uq_m1 = _mm256_sub_epi16(U_q_avx, vone16); + w0 = _mm256_sub_epi16(vtwo16, emb_k); + __m256i Uq_m1 = _mm256_sub_epi16(U_q_avx, vone16); // Shift count for even quads (0 and 2) within each 128-bit lane. - __m256i Uq_evn = _mm256_and_si256(Uq_m1, _mm256_set_epi32(0, 0, 0, 0x1F, 0, 0, 0, 0x1F)); + __m256i Uq_evn = _mm256_and_si256(Uq_m1, _mm256_set_epi32(0, 0, 0, 0x1F, 0, 0, 0, 0x1F)); // Shift count for odd quads (1 and 3) — move to element 0 of each lane. - __m256i Uq_odd = _mm256_bsrli_epi128(Uq_m1, 14); - __m256i t_evn = _mm256_and_si256(w0, _mm256_set_epi64x(0, -1, 0, -1)); - __m256i t_odd = _mm256_and_si256(w0, _mm256_set_epi64x(-1, 0, -1, 0)); + __m256i Uq_odd = _mm256_bsrli_epi128(Uq_m1, 14); + __m256i t_evn = _mm256_and_si256(w0, _mm256_set_epi64x(0, -1, 0, -1)); + __m256i t_odd = _mm256_and_si256(w0, _mm256_set_epi64x(-1, 0, -1, 0)); { // no _mm256_sllv_epi16 in AVX2 — use four _mm_sll_epi16 calls instead __m128i lo, hi; lo = _mm_sll_epi16(_mm256_castsi256_si128(t_evn), _mm256_castsi256_si128(Uq_evn)); - hi = _mm_sll_epi16(_mm256_extracti128_si256(t_evn, 1), - _mm256_extracti128_si256(Uq_evn, 1)); + hi = _mm_sll_epi16(_mm256_extracti128_si256(t_evn, 1), _mm256_extracti128_si256(Uq_evn, 1)); t_evn = _mm256_inserti128_si256(_mm256_castsi128_si256(lo), hi, 1); lo = _mm_sll_epi16(_mm256_castsi256_si128(t_odd), _mm256_castsi256_si128(Uq_odd)); - hi = _mm_sll_epi16(_mm256_extracti128_si256(t_odd, 1), - _mm256_extracti128_si256(Uq_odd, 1)); + hi = _mm_sll_epi16(_mm256_extracti128_si256(t_odd, 1), _mm256_extracti128_si256(Uq_odd, 1)); t_odd = _mm256_inserti128_si256(_mm256_castsi128_si256(lo), hi, 1); } __m256i shift = _mm256_or_si256(t_evn, t_odd); // = (2 - e_k) << (U_q - 1) = 2^m_n @@ -1498,34 +1761,33 @@ class fwd_buf { ms_vec = _mm256_and_si256(ms_vec, _mm256_sub_epi16(shift, vone16)); // m_n magnitude bits // Place e_1 at bit position m_n. - __m256i emb1_mask = _mm256_cmpeq_epi16( - _mm256_and_si256(flags, _mm256_set1_epi16(0x800)), _mm256_setzero_si256()); + __m256i emb1_mask = + _mm256_cmpeq_epi16(_mm256_and_si256(flags, _mm256_set1_epi16(0x800)), _mm256_setzero_si256()); ms_vec = _mm256_or_si256(ms_vec, _mm256_andnot_si256(emb1_mask, shift)); // Build decoded sample: sign at bit 15, magnitude shifted by (pLSB_adj - 1). - __m256i tvn = ms_vec; // save for v_n update (before |=1) - __m256i w_sign = _mm256_slli_epi16(ms_vec, 15); // sign bit → bit 15 - ms_vec = _mm256_or_si256(ms_vec, vone16); // bin center - ms_vec = _mm256_add_epi16(ms_vec, vtwo16); // + 2 + __m256i tvn = ms_vec; // save for v_n update (before |=1) + __m256i w_sign = _mm256_slli_epi16(ms_vec, 15); // sign bit → bit 15 + ms_vec = _mm256_or_si256(ms_vec, vone16); // bin center + ms_vec = _mm256_add_epi16(ms_vec, vtwo16); // + 2 ms_vec = _mm256_slli_epi16(ms_vec, pLSB_adj - 1); - ms_vec = _mm256_or_si256(ms_vec, w_sign); // sign + ms_vec = _mm256_or_si256(ms_vec, w_sign); // sign row256 = _mm256_andnot_si256(insig, ms_vec); // Update v_n (8 × int16_t): pairwise OR of row-0 and row-1 magnitudes per column. // Gather odd-indexed 16-bit elements (row 1 only) into lower 64 bits of each 128-bit lane. tvn = _mm256_andnot_si256(insig, tvn); - __m256i vn256 = _mm256_shuffle_epi8( - tvn, _mm256_set_epi8(-1,-1,-1,-1,-1,-1,-1,-1, - 0x0F,0x0E,0x0B,0x0A,0x07,0x06,0x03,0x02, - -1,-1,-1,-1,-1,-1,-1,-1, - 0x0F,0x0E,0x0B,0x0A,0x07,0x06,0x03,0x02)); + __m256i vn256 = + _mm256_shuffle_epi8(tvn, _mm256_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, 0x0F, 0x0E, 0x0B, 0x0A, + 0x07, 0x06, 0x03, 0x02, -1, -1, -1, -1, -1, -1, -1, -1, + 0x0F, 0x0E, 0x0B, 0x0A, 0x07, 0x06, 0x03, 0x02)); // Combine: lower 64 bits of each lane into the lower 128 bits. vn256 = _mm256_permute4x64_epi64(vn256, _MM_SHUFFLE(2, 0, 2, 0)); v_n = _mm_or_si128(v_n, _mm256_castsi256_si128(vn256)); } return row256; } -#endif // defined(__AVX2__) + #endif // defined(__AVX2__) }; #else template @@ -1817,7 +2079,7 @@ class MR_dec { // } //} // -//OPENHTJ2K_MAYBE_UNUSED uint8_t state_MS_dec::importMagSgnBit() { +// OPENHTJ2K_MAYBE_UNUSED uint8_t state_MS_dec::importMagSgnBit() { // uint8_t val; // if (bits == 0) { // bits = (last == 0xFF) ? 7 : 8; @@ -1842,7 +2104,7 @@ class MR_dec { // return val; //} // -//OPENHTJ2K_MAYBE_UNUSED int32_t state_MS_dec::decodeMagSgnValue(int32_t m_n, int32_t i_n) { +// OPENHTJ2K_MAYBE_UNUSED int32_t state_MS_dec::decodeMagSgnValue(int32_t m_n, int32_t i_n) { // int32_t val = 0; // // uint8_t bit; // if (m_n > 0) { @@ -1997,7 +2259,7 @@ class MR_dec { //} // #endif // -//OPENHTJ2K_MAYBE_UNUSED void state_VLC_dec::decodeCxtVLC(const uint16_t &context, uint8_t (&u_off)[2], +// OPENHTJ2K_MAYBE_UNUSED void state_VLC_dec::decodeCxtVLC(const uint16_t &context, uint8_t (&u_off)[2], // uint8_t (&rho)[2], uint8_t (&emb_k)[2], // uint8_t (&emb_1)[2], const uint8_t &first_or_second, // const uint16_t *dec_CxtVLC_table) { @@ -2034,7 +2296,7 @@ class MR_dec { // #endif //} // -//OPENHTJ2K_MAYBE_UNUSED uint8_t state_VLC_dec::decodeUPrefix() { +// OPENHTJ2K_MAYBE_UNUSED uint8_t state_VLC_dec::decodeUPrefix() { // if (getbitfunc == 1) { // return 1; // } @@ -2048,7 +2310,7 @@ class MR_dec { // } //} // -//OPENHTJ2K_MAYBE_UNUSED uint8_t state_VLC_dec::decodeUSuffix(const uint32_t &u_pfx) { +// OPENHTJ2K_MAYBE_UNUSED uint8_t state_VLC_dec::decodeUSuffix(const uint32_t &u_pfx) { // uint8_t bit, val; // if (u_pfx < 3) { // return 0; @@ -2063,7 +2325,7 @@ class MR_dec { // } // return val; //} -//OPENHTJ2K_MAYBE_UNUSED uint8_t state_VLC_dec::decodeUExtension(const uint32_t &u_sfx) { +// OPENHTJ2K_MAYBE_UNUSED uint8_t state_VLC_dec::decodeUExtension(const uint32_t &u_sfx) { // uint8_t bit, val; // if (u_sfx < 28) { // return 0; @@ -2096,7 +2358,7 @@ FORCE_INLINE uint8_t SP_dec::importSigPropBit() { last = tmp; } uint8_t val = tmp & 1; - tmp = static_cast(tmp >> 1); + tmp = static_cast(tmp >> 1); bits--; return val; } @@ -2119,31 +2381,31 @@ FORCE_INLINE uint8_t MR_dec::importMagRefBit() { last = tmp; } uint8_t val = tmp & 1; - tmp = static_cast(tmp >> 1); + tmp = static_cast(tmp >> 1); bits--; return val; } -//OPENHTJ2K_MAYBE_UNUSED auto decodeSigEMB = [](state_MEL_decoder &MEL_decoder, rev_buf &VLC_dec, -// const uint16_t &context, uint8_t (&u_off)[2], uint8_t (&rho)[2], -// uint8_t (&emb_k)[2], uint8_t (&emb_1)[2], -// const uint8_t &first_or_second, const uint16_t *dec_CxtVLC_table) -// { -// uint8_t sym; -// if (context == 0) { -// sym = MEL_decoder.decodeMELSym(); -// if (sym == 0) { -// rho[first_or_second] = u_off[first_or_second] = emb_k[first_or_second] = emb_1[first_or_second] = 0; -// return; -// } -// } -// uint32_t vlcval = VLC_dec.fetch(); -// uint16_t value = dec_CxtVLC_table[(vlcval & 0x7F) + (context << 7)]; -// u_off[first_or_second] = value & 1; -// uint32_t len = static_cast((value & 0x000F) >> 1); -// rho[first_or_second] = static_cast((value & 0x00F0) >> 4); -// emb_k[first_or_second] = static_cast((value & 0x0F00) >> 8); -// emb_1[first_or_second] = static_cast((value & 0xF000) >> 12); -// VLC_dec.advance(len); -// // VLC_dec.decodeCxtVLC(context, u_off, rho, emb_k, emb_1, first_or_second, dec_CxtVLC_table); -//}; +// OPENHTJ2K_MAYBE_UNUSED auto decodeSigEMB = [](state_MEL_decoder &MEL_decoder, rev_buf &VLC_dec, +// const uint16_t &context, uint8_t (&u_off)[2], uint8_t (&rho)[2], +// uint8_t (&emb_k)[2], uint8_t (&emb_1)[2], +// const uint8_t &first_or_second, const uint16_t *dec_CxtVLC_table) +// { +// uint8_t sym; +// if (context == 0) { +// sym = MEL_decoder.decodeMELSym(); +// if (sym == 0) { +// rho[first_or_second] = u_off[first_or_second] = emb_k[first_or_second] = emb_1[first_or_second] = +// 0; return; +// } +// } +// uint32_t vlcval = VLC_dec.fetch(); +// uint16_t value = dec_CxtVLC_table[(vlcval & 0x7F) + (context << 7)]; +// u_off[first_or_second] = value & 1; +// uint32_t len = static_cast((value & 0x000F) >> 1); +// rho[first_or_second] = static_cast((value & 0x00F0) >> 4); +// emb_k[first_or_second] = static_cast((value & 0x0F00) >> 8); +// emb_1[first_or_second] = static_cast((value & 0xF000) >> 12); +// VLC_dec.advance(len); +// // VLC_dec.decodeCxtVLC(context, u_off, rho, emb_k, emb_1, first_or_second, dec_CxtVLC_table); +// }; diff --git a/source/core/coding/ht_block_decoding_avx2.cpp b/source/core/coding/ht_block_decoding_avx2.cpp index c68796b3..e43bf6c2 100644 --- a/source/core/coding/ht_block_decoding_avx2.cpp +++ b/source/core/coding/ht_block_decoding_avx2.cpp @@ -39,8 +39,8 @@ #include #endif -static FORCE_INLINE uint8_t calc_mbr_inline(const uint8_t *block_states, size_t blkstate_stride, - uint32_t i, uint32_t j, uint8_t causal_cond) { +static FORCE_INLINE uint8_t calc_mbr_inline(const uint8_t *block_states, size_t blkstate_stride, uint32_t i, + uint32_t j, uint8_t causal_cond) { const uint8_t *p0 = block_states + static_cast(i) * blkstate_stride + j; const uint8_t *p1 = p0 + blkstate_stride; const uint8_t *p2 = p1 + blkstate_stride; @@ -113,8 +113,8 @@ static FORCE_INLINE void dequant_store_256(int32_t *dst, __m256i val, uint8_t tr int32_t pLSB_dq, __m256 vfscale, __m256i vmagmask, __m256i vsignmask) { if (transformation == 1) { - __m256i mag = _mm256_and_si256(val, vmagmask); - __m256i res = _mm256_sign_epi32(_mm256_srai_epi32(mag, pLSB_dq), val); + __m256i mag = _mm256_and_si256(val, vmagmask); + __m256i res = _mm256_sign_epi32(_mm256_srai_epi32(mag, pLSB_dq), val); if constexpr (StoreI32) _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), res); else @@ -146,202 +146,19 @@ static FORCE_INLINE void dequant_store_128(int32_t *dst, __m128i val, uint8_t tr } } -template -void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t Lcup, const int32_t Pcup, - const int32_t Scup) { +// Step-2 of the HT cleanup pass: MagSgn decoding over the (tv, u) scratch +// written by ht_cleanup_step1_nway (the former phase 1 of this function). +// Kept per-block: the fwd_buf destuff scratch is thread-local (constructing a +// second fwd_buf on the same thread invalidates the first), and step-2 is +// throughput-bound — there is nothing to gain from interleaving it. +template +static void ht_cleanup_step2(j2k_codeblock *block, const uint8_t pLSB, const int32_t Pcup, + uint16_t *scratch, const int32_t sstr) { uint8_t *compressed_data = block->get_compressed_data(); const uint16_t QW = static_cast(ceil_int(static_cast(block->size.x), 2)); const uint16_t QH = static_cast(ceil_int(static_cast(block->size.y), 2)); - - uint16_t scratch[8 * 513]; - int32_t sstr = static_cast(((block->size.x + 2) + 7u) & ~7u); // multiples of 8 uint16_t *sp; int32_t qx; - /*******************************************************************************************************************/ - // VLC, UVLC and MEL decoding - /*******************************************************************************************************************/ - MEL_dec MEL(compressed_data, Lcup, Scup); - rev_buf VLC_dec(compressed_data, Lcup, Scup); - auto sp0 = block->block_states + 1 + block->blkstate_stride; - auto sp1 = block->block_states + 1 + 2 * block->blkstate_stride; - uint32_t u_off0, u_off1; - uint32_t u0, u1; - uint32_t context = 0; - uint32_t vlcval; - - const uint16_t *dec_table; - // Initial line-pair - dec_table = dec_CxtVLC_table0_fast_16; - sp = scratch; - int32_t mel_run = MEL.get_run(); - for (qx = QW; qx > 0; qx -= 2, sp += 4) { - // Decoding of significance and EMB patterns and unsigned residual offsets - vlcval = VLC_dec.fetch(); - uint16_t tv0 = dec_table[(vlcval & 0x7F) + context]; - { - // Branchless context-0 MEL handling: replace unpredictable branch with mask - int32_t cm = -static_cast(context == 0); - mel_run -= cm & 2; - tv0 &= static_cast(-(mel_run == -1) | ~cm); - if (mel_run < 0) mel_run = MEL.get_run(); - } - sp[0] = tv0; - - // calculate context for the next quad, Eq. (1) in the spec - context = ((tv0 & 0xE0U) << 2) | ((tv0 & 0x10U) << 3); // = context << 7 - - // Decoding of significance and EMB patterns and unsigned residual offsets - vlcval = VLC_dec.advance((tv0 & 0x000F) >> 1); - uint16_t tv1 = dec_table[(vlcval & 0x7F) + context]; - { - int32_t cm = -static_cast((context == 0) & (qx > 1)); - mel_run -= cm & 2; - tv1 &= static_cast(-(mel_run == -1) | ~cm); - if (mel_run < 0) mel_run = MEL.get_run(); - } - tv1 = (qx > 1) ? tv1 : 0; - sp[2] = tv1; - - // store sigma - if (!skip_sigma) { - *sp0++ = ((tv0 >> 4) >> 0) & 1; - *sp0++ = ((tv0 >> 4) >> 2) & 1; - *sp0++ = ((tv1 >> 4) >> 0) & 1; - *sp0++ = ((tv1 >> 4) >> 2) & 1; - *sp1++ = ((tv0 >> 4) >> 1) & 1; - *sp1++ = ((tv0 >> 4) >> 3) & 1; - *sp1++ = ((tv1 >> 4) >> 1) & 1; - *sp1++ = ((tv1 >> 4) >> 3) & 1; - } - - // calculate context for the next quad, Eq. (1) in the spec - context = ((tv1 & 0xE0U) << 2) | ((tv1 & 0x10U) << 3); // = context << 7 - - vlcval = VLC_dec.advance((tv1 & 0x000F) >> 1); - u_off0 = tv0 & 1; - u_off1 = tv1 & 1; - - // Branchless MEL offset: replace compound branch with mask - uint32_t both_off = u_off0 & u_off1; - int32_t om = -static_cast(both_off); - mel_run -= om & 2; - uint32_t mel_offset = static_cast(-(mel_run == -1) & om) & 0x40; - if (mel_run < 0) mel_run = MEL.get_run(); - - // UVLC decoding - uint32_t idx = (vlcval & 0x3F) + (u_off0 << 6U) + (u_off1 << 7U) + mel_offset; - uint32_t uvlc_result = uvlc_dec_0[idx]; - // remove total prefix length - vlcval = VLC_dec.advance(uvlc_result & 0x7); - uvlc_result >>= 3; - // extract suffixes for quad 0 and 1 - uint32_t len = uvlc_result & 0xF; // suffix length for 2 quads (up to 10 = 5 + 5) - // ((1U << len) - 1U) can be replaced with _bzhi_u32(UINT32_MAX, len); not fast - uint32_t tmp = vlcval & ((1U << len) - 1U); // suffix value for 2 quads - vlcval = VLC_dec.advance(len); - uvlc_result >>= 4; - // quad 0 length - len = uvlc_result & 0x7; // quad 0 suffix length - uvlc_result >>= 3; - // U = 1+ u - u0 = 1 + (uvlc_result & 7) + (tmp & ~(0xFFU << len)); // always kappa = 1 in initial line pair - u1 = 1 + (uvlc_result >> 3) + (tmp >> len); // always kappa = 1 in initial line pair - - sp[1] = static_cast(u0); - sp[3] = static_cast(u1); - } - // Zero the 8-byte guard past the last VLC row: the MagSgn SSE load reads 4 extra - // uint16_t beyond the written region in the final iteration of each row. - std::memset(sp, 0, sizeof(uint16_t) * 4); - - // Non-initial line-pair - dec_table = dec_CxtVLC_table1_fast_16; - for (uint16_t row = 1; row < QH; row++) { - sp0 = block->block_states + (row * 2U + 1U) * block->blkstate_stride + 1U; - sp1 = sp0 + block->blkstate_stride; - - sp = scratch + row * sstr; - // calculate context for the next quad: w, sw, nw are always 0 at the head of a row - context = ((sp[0 - sstr] & 0xA0U) << 2) | ((sp[2 - sstr] & 0x20U) << 4); - for (qx = QW; qx > 0; qx -= 2, sp += 4) { - // Decoding of significance and EMB patterns and unsigned residual offsets - vlcval = VLC_dec.fetch(); - uint16_t tv0 = dec_table[(vlcval & 0x7F) + context]; - if (context == 0) { - mel_run -= 2; - tv0 = (mel_run == -1) ? tv0 : 0; - if (mel_run < 0) { - mel_run = MEL.get_run(); - } - } - // calculate context for the next quad, Eq. (2) in the spec - context = ((tv0 & 0x40U) << 2) | ((tv0 & 0x80U) << 1); // (w | sw) << 8 - context |= (sp[0 - sstr] & 0x80U) | ((sp[2 - sstr] & 0xA0U) << 2); // ((nw | n) << 7) | (ne << 9) - context |= (sp[4 - sstr] & 0x20U) << 4; // ( nf) << 9 - - sp[0] = tv0; - - vlcval = VLC_dec.advance((tv0 & 0x000F) >> 1); - - // Decoding of significance and EMB patterns and unsigned residual offsets - uint16_t tv1 = dec_table[(vlcval & 0x7F) + context]; - if (context == 0 && qx > 1) { - mel_run -= 2; - tv1 = (mel_run == -1) ? tv1 : 0; - if (mel_run < 0) { - mel_run = MEL.get_run(); - } - } - tv1 = (qx > 1) ? tv1 : 0; - // calculate context for the next quad, Eq. (2) in the spec - context = ((tv1 & 0x40U) << 2) | ((tv1 & 0x80U) << 1); // (w | sw) << 8 - context |= (sp[2 - sstr] & 0x80U) | ((sp[4 - sstr] & 0xA0U) << 2); // ((nw | n) << 7) | (ne << 9) - context |= (sp[6 - sstr] & 0x20U) << 4; // ( nf) << 9 - - sp[2] = tv1; - - // store sigma - if (!skip_sigma) { - *sp0++ = ((tv0 >> 4) >> 0) & 1; - *sp0++ = ((tv0 >> 4) >> 2) & 1; - *sp0++ = ((tv1 >> 4) >> 0) & 1; - *sp0++ = ((tv1 >> 4) >> 2) & 1; - *sp1++ = ((tv0 >> 4) >> 1) & 1; - *sp1++ = ((tv0 >> 4) >> 3) & 1; - *sp1++ = ((tv1 >> 4) >> 1) & 1; - *sp1++ = ((tv1 >> 4) >> 3) & 1; - } - - vlcval = VLC_dec.advance((tv1 & 0x000F) >> 1); - - // UVLC decoding - u_off0 = tv0 & 1; - u_off1 = tv1 & 1; - uint32_t idx = (vlcval & 0x3F) + (u_off0 << 6U) + (u_off1 << 7U); - - uint32_t uvlc_result = uvlc_dec_1[idx]; - // remove total prefix length - vlcval = VLC_dec.advance(uvlc_result & 0x7); - uvlc_result >>= 3; - // extract suffixes for quad 0 and 1 - uint32_t len = uvlc_result & 0xF; // suffix length for 2 quads (up to 10 = 5 + 5) - // ((1U << len) - 1U) can be replaced with _bzhi_u32(UINT32_MAX, len); not fast - uint32_t tmp = vlcval & ((1U << len) - 1U); // suffix value for 2 quads - vlcval = VLC_dec.advance(len); - uvlc_result >>= 4; - // quad 0 length - len = uvlc_result & 0x7; // quad 0 suffix length - uvlc_result >>= 3; - u0 = (uvlc_result & 7) + (tmp & ~(0xFFU << len)); - u1 = (uvlc_result >> 3) + (tmp >> len); - - sp[1] = static_cast(u0); - sp[3] = static_cast(u1); - } - // Zero the 8-byte guard: same reason as initial row. - std::memset(sp, 0, sizeof(uint16_t) * 4); - } - /*******************************************************************************************************************/ // MagSgn decoding /*******************************************************************************************************************/ @@ -349,16 +166,16 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t // Fused dequantize setup: when fuse_dequant is true, we write dequantized float values // directly to band_buf, eliminating the separate dequantize pass. // pLSB_dq is the dequantization shift (31 - M_b), distinct from the MagSgn pLSB. - int32_t pLSB_dq = 0; - float fscale_direct = 0.0f; - __m256 vfscale = _mm256_setzero_ps(); - __m256i vsignmask_dq = _mm256_setzero_si256(); - __m256i vmagmask_dq = _mm256_setzero_si256(); + int32_t pLSB_dq = 0; + float fscale_direct = 0.0f; + __m256 vfscale = _mm256_setzero_ps(); + __m256i vsignmask_dq = _mm256_setzero_si256(); + __m256i vmagmask_dq = _mm256_setzero_si256(); if constexpr (fuse_dequant) { const int32_t M_b_val = block->get_Mb(); pLSB_dq = 31 - M_b_val; - vmagmask_dq = _mm256_set1_epi32(0x7FFFFFFF); - vsignmask_dq = _mm256_set1_epi32(INT32_MIN); + vmagmask_dq = _mm256_set1_epi32(0x7FFFFFFF); + vsignmask_dq = _mm256_set1_epi32(INT32_MIN); if (block->transformation != 1) { // lossy path (transformation==0 for irrev97, transformation>=2 for ATK irrev) fscale_direct = block->stepsize; @@ -391,12 +208,12 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t // matches OpenHTJ2K's fixed-point format (sign at bit 31). // shuffle_r0 extracts even-indexed 16-bit elements (row 0 samples). // shuffle_r1 extracts odd-indexed 16-bit elements (row 1 samples). - const __m256i shuffle_r0 = _mm256_set_epi8( - 0x0D, 0x0C, -1, -1, 0x09, 0x08, -1, -1, 0x05, 0x04, -1, -1, 0x01, 0x00, -1, -1, - 0x0D, 0x0C, -1, -1, 0x09, 0x08, -1, -1, 0x05, 0x04, -1, -1, 0x01, 0x00, -1, -1); - const __m256i shuffle_r1 = _mm256_set_epi8( - 0x0F, 0x0E, -1, -1, 0x0B, 0x0A, -1, -1, 0x07, 0x06, -1, -1, 0x03, 0x02, -1, -1, - 0x0F, 0x0E, -1, -1, 0x0B, 0x0A, -1, -1, 0x07, 0x06, -1, -1, 0x03, 0x02, -1, -1); + const __m256i shuffle_r0 = + _mm256_set_epi8(0x0D, 0x0C, -1, -1, 0x09, 0x08, -1, -1, 0x05, 0x04, -1, -1, 0x01, 0x00, -1, -1, 0x0D, + 0x0C, -1, -1, 0x09, 0x08, -1, -1, 0x05, 0x04, -1, -1, 0x01, 0x00, -1, -1); + const __m256i shuffle_r1 = + _mm256_set_epi8(0x0F, 0x0E, -1, -1, 0x0B, 0x0A, -1, -1, 0x07, 0x06, -1, -1, 0x03, 0x02, -1, -1, 0x0F, + 0x0E, -1, -1, 0x0B, 0x0A, -1, -1, 0x07, 0x06, -1, -1, 0x03, 0x02, -1, -1); // When pLSB > 16 (mmsbp2 = 32 - pLSB < 16), decoded sample values fit in 16 bits // so we use the faster 4-quad 16-bit path which processes twice as many quads per @@ -407,24 +224,24 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t // Initial line-pair — 4 quads at a time sp = scratch; for (qx = QW; qx >= 4; qx -= 4, sp += 8, mp0 += 8, mp1 += 8) { - v_n = _mm_setzero_si128(); - __m128i qinf128 = _mm_loadu_si128((__m128i *)sp); - __m128i U_q128 = _mm_srli_epi32(qinf128, 16); - __m256i row256 = MagSgn.decode_four_quads(qinf128, U_q128, pLSB_adj, v_n); + v_n = _mm_setzero_si128(); + __m128i qinf128 = _mm_loadu_si128((__m128i *)sp); + __m128i U_q128 = _mm_srli_epi32(qinf128, 16); + __m256i row256 = MagSgn.decode_four_quads(qinf128, U_q128, pLSB_adj, v_n); if constexpr (fuse_dequant) { - dequant_store_256(mp0, _mm256_shuffle_epi8(row256, shuffle_r0), block->transformation, pLSB_dq, - vfscale, vmagmask_dq, vsignmask_dq); - dequant_store_256(mp1, _mm256_shuffle_epi8(row256, shuffle_r1), block->transformation, pLSB_dq, - vfscale, vmagmask_dq, vsignmask_dq); + dequant_store_256(mp0, _mm256_shuffle_epi8(row256, shuffle_r0), block->transformation, + pLSB_dq, vfscale, vmagmask_dq, vsignmask_dq); + dequant_store_256(mp1, _mm256_shuffle_epi8(row256, shuffle_r1), block->transformation, + pLSB_dq, vfscale, vmagmask_dq, vsignmask_dq); } else { _mm256_storeu_si256((__m256i *)mp0, _mm256_shuffle_epi8(row256, shuffle_r0)); _mm256_storeu_si256((__m256i *)mp1, _mm256_shuffle_epi8(row256, shuffle_r1)); } __m256i vn32 = _mm256_cvtepu16_epi32(v_n); - vn32 = avx2_lzcnt_epi32(vn32); - vn32 = _mm256_sub_epi32(_mm256_set1_epi32(32), vn32); + vn32 = avx2_lzcnt_epi32(vn32); + vn32 = _mm256_sub_epi32(_mm256_set1_epi32(32), vn32); _mm256_storeu_si256((__m256i *)E_p, vn32); E_p += 8; } @@ -444,9 +261,11 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t mu1_n = _mm_unpackhi_epi32(t0, t1); if constexpr (fuse_dequant) { dequant_store_128(mp0, mu0_n, block->transformation, pLSB_dq, vfscale, - _mm256_castsi256_si128(vmagmask_dq), _mm256_castsi256_si128(vsignmask_dq)); + _mm256_castsi256_si128(vmagmask_dq), + _mm256_castsi256_si128(vsignmask_dq)); dequant_store_128(mp1, mu1_n, block->transformation, pLSB_dq, vfscale, - _mm256_castsi256_si128(vmagmask_dq), _mm256_castsi256_si128(vsignmask_dq)); + _mm256_castsi256_si128(vmagmask_dq), + _mm256_castsi256_si128(vsignmask_dq)); } else { _mm_storeu_si128((__m128i *)mp0, mu0_n); _mm_storeu_si128((__m128i *)mp1, mu1_n); @@ -467,18 +286,17 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t mp0 = sample_buf + (row * 2U) * block->blksampl_stride; mp1 = mp0 + block->blksampl_stride; } - sp = scratch + row * sstr; + sp = scratch + row * sstr; // Vectorized Emax: sliding max over 4-column windows using AVX2. // Emax[q] = max(E_p[2q-1], E_p[2q], E_p[2q+1], E_p[2q+2]). // Two 8-wide loads offset by 2 → max → per-lane shift → pairwise max → permute. const __m256i perm_emax = _mm256_set_epi32(0, 0, 0, 0, 6, 4, 2, 0); - __m256i elo = _mm256_loadu_si256((__m256i *)(E_p - 1)); - __m256i ehi = _mm256_loadu_si256((__m256i *)(E_p + 1)); - __m256i emx = _mm256_max_epi32(elo, ehi); - __m256i epr = _mm256_max_epi32(emx, _mm256_srli_si256(emx, 4)); - __m128i emax128 = _mm256_castsi256_si128( - _mm256_permutevar8x32_epi32(epr, perm_emax)); + __m256i elo = _mm256_loadu_si256((__m256i *)(E_p - 1)); + __m256i ehi = _mm256_loadu_si256((__m256i *)(E_p + 1)); + __m256i emx = _mm256_max_epi32(elo, ehi); + __m256i epr = _mm256_max_epi32(emx, _mm256_srli_si256(emx, 4)); + __m128i emax128 = _mm256_castsi256_si128(_mm256_permutevar8x32_epi32(epr, perm_emax)); for (qx = QW; qx >= 4; qx -= 4, sp += 8, mp0 += 8, mp1 += 8) { v_n = _mm_setzero_si128(); @@ -496,10 +314,10 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t __m256i row256 = MagSgn.decode_four_quads(qinf128, U_q128, pLSB_adj, v_n); if constexpr (fuse_dequant) { - dequant_store_256(mp0, _mm256_shuffle_epi8(row256, shuffle_r0), block->transformation, pLSB_dq, - vfscale, vmagmask_dq, vsignmask_dq); - dequant_store_256(mp1, _mm256_shuffle_epi8(row256, shuffle_r1), block->transformation, pLSB_dq, - vfscale, vmagmask_dq, vsignmask_dq); + dequant_store_256(mp0, _mm256_shuffle_epi8(row256, shuffle_r0), block->transformation, + pLSB_dq, vfscale, vmagmask_dq, vsignmask_dq); + dequant_store_256(mp1, _mm256_shuffle_epi8(row256, shuffle_r1), block->transformation, + pLSB_dq, vfscale, vmagmask_dq, vsignmask_dq); } else { _mm256_storeu_si256((__m256i *)mp0, _mm256_shuffle_epi8(row256, shuffle_r0)); _mm256_storeu_si256((__m256i *)mp1, _mm256_shuffle_epi8(row256, shuffle_r1)); @@ -510,12 +328,11 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t ehi = _mm256_loadu_si256((__m256i *)(E_p + 9)); emx = _mm256_max_epi32(elo, ehi); epr = _mm256_max_epi32(emx, _mm256_srli_si256(emx, 4)); - emax128 = _mm256_castsi256_si128( - _mm256_permutevar8x32_epi32(epr, perm_emax)); + emax128 = _mm256_castsi256_si128(_mm256_permutevar8x32_epi32(epr, perm_emax)); __m256i vn32 = _mm256_cvtepu16_epi32(v_n); - vn32 = avx2_lzcnt_epi32(vn32); - vn32 = _mm256_sub_epi32(_mm256_set1_epi32(32), vn32); + vn32 = avx2_lzcnt_epi32(vn32); + vn32 = _mm256_sub_epi32(_mm256_set1_epi32(32), vn32); _mm256_storeu_si256((__m256i *)E_p, vn32); E_p += 8; } @@ -538,10 +355,10 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t __m256i kappa = _mm256_max_epi32(emax256, _mm256_set1_epi32(1)); U_q256 = _mm256_add_epi32(_mm256_srli_epi32(qinf256, 16), kappa); } - __m128i U_q128 = _mm_unpacklo_epi32(_mm256_castsi256_si128(U_q256), - _mm256_extracti128_si256(U_q256, 1)); - mu0_n = MagSgn.decode_one_quad<0>(qinf128, U_q128, pLSB, v_n); - mu1_n = MagSgn.decode_one_quad<1>(qinf128, U_q128, pLSB, v_n); + __m128i U_q128 = + _mm_unpacklo_epi32(_mm256_castsi256_si128(U_q256), _mm256_extracti128_si256(U_q256, 1)); + mu0_n = MagSgn.decode_one_quad<0>(qinf128, U_q128, pLSB, v_n); + mu1_n = MagSgn.decode_one_quad<1>(qinf128, U_q128, pLSB, v_n); auto t0 = _mm_unpacklo_epi32(mu0_n, mu1_n); auto t1 = _mm_unpackhi_epi32(mu0_n, mu1_n); @@ -549,9 +366,11 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t mu1_n = _mm_unpackhi_epi32(t0, t1); if constexpr (fuse_dequant) { dequant_store_128(mp0, mu0_n, block->transformation, pLSB_dq, vfscale, - _mm256_castsi256_si128(vmagmask_dq), _mm256_castsi256_si128(vsignmask_dq)); + _mm256_castsi256_si128(vmagmask_dq), + _mm256_castsi256_si128(vsignmask_dq)); dequant_store_128(mp1, mu1_n, block->transformation, pLSB_dq, vfscale, - _mm256_castsi256_si128(vmagmask_dq), _mm256_castsi256_si128(vsignmask_dq)); + _mm256_castsi256_si128(vmagmask_dq), + _mm256_castsi256_si128(vsignmask_dq)); } else { _mm_storeu_si128((__m128i *)mp0, mu0_n); _mm_storeu_si128((__m128i *)mp1, mu1_n); @@ -561,9 +380,9 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t __m128i ehi128 = _mm_loadu_si128((__m128i *)(E_p + 5)); __m128i emx128 = _mm_max_epi32(elo128, ehi128); __m128i epr128 = _mm_max_epi32(emx128, _mm_srli_si128(emx128, 4)); - emax128 = _mm_shuffle_epi32(epr128, _MM_SHUFFLE(3, 3, 2, 0)); - v_n = sse_lzcnt_epi32(v_n); - v_n = _mm_sub_epi32(_mm_set1_epi32(32), v_n); + emax128 = _mm_shuffle_epi32(epr128, _MM_SHUFFLE(3, 3, 2, 0)); + v_n = sse_lzcnt_epi32(v_n); + v_n = _mm_sub_epi32(_mm_set1_epi32(32), v_n); _mm_storeu_si128((__m128i *)E_p, v_n); E_p += 4; } @@ -587,9 +406,11 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t mu1_n = _mm_unpackhi_epi32(t0, t1); if constexpr (fuse_dequant) { dequant_store_128(mp0, mu0_n, block->transformation, pLSB_dq, vfscale, - _mm256_castsi256_si128(vmagmask_dq), _mm256_castsi256_si128(vsignmask_dq)); + _mm256_castsi256_si128(vmagmask_dq), + _mm256_castsi256_si128(vsignmask_dq)); dequant_store_128(mp1, mu1_n, block->transformation, pLSB_dq, vfscale, - _mm256_castsi256_si128(vmagmask_dq), _mm256_castsi256_si128(vsignmask_dq)); + _mm256_castsi256_si128(vmagmask_dq), + _mm256_castsi256_si128(vsignmask_dq)); } else { _mm_storeu_si128((__m128i *)mp0, mu0_n); _mm_storeu_si128((__m128i *)mp1, mu1_n); @@ -609,7 +430,7 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t mp0 = sample_buf + (row * 2U) * block->blksampl_stride; mp1 = mp0 + block->blksampl_stride; } - sp = scratch + row * sstr; + sp = scratch + row * sstr; // Vectorized Emax for 2-quad path: sliding max over 4-column windows. // epr128 positions {0, 2} hold {Emax[0], Emax[1]}; kept as vector to avoid scalar extracts. @@ -644,9 +465,11 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t mu1_n = _mm_unpackhi_epi32(t0, t1); if constexpr (fuse_dequant) { dequant_store_128(mp0, mu0_n, block->transformation, pLSB_dq, vfscale, - _mm256_castsi256_si128(vmagmask_dq), _mm256_castsi256_si128(vsignmask_dq)); + _mm256_castsi256_si128(vmagmask_dq), + _mm256_castsi256_si128(vsignmask_dq)); dequant_store_128(mp1, mu1_n, block->transformation, pLSB_dq, vfscale, - _mm256_castsi256_si128(vmagmask_dq), _mm256_castsi256_si128(vsignmask_dq)); + _mm256_castsi256_si128(vmagmask_dq), + _mm256_castsi256_si128(vsignmask_dq)); } else { _mm_storeu_si128((__m128i *)mp0, mu0_n); _mm_storeu_si128((__m128i *)mp1, mu1_n); @@ -701,12 +524,12 @@ void ht_sigprop_decode(j2k_codeblock *block, uint8_t *HT_magref_segment, uint32_ const uint8_t &pLSB, uint16_t *sigma, uint32_t mstr) { if (pLSB == 0) return; // no plane below the LSB; mirrors ht_magref_decode (avoids 1 << (pLSB-1) UB) SP_dec SigProp(HT_magref_segment, magref_length); - const uint32_t height = block->size.y; - const uint32_t width = block->size.x; - const size_t sstride = block->blksampl_stride; - int32_t *samples = block->sample_buf; - const bool non_causal = (block->Cmodes & CAUSAL) == 0; - const int32_t spp_mask = 3 << (pLSB - 1); + const uint32_t height = block->size.y; + const uint32_t width = block->size.x; + const size_t sstride = block->blksampl_stride; + int32_t *samples = block->sample_buf; + const bool non_causal = (block->Cmodes & CAUSAL) == 0; + const int32_t spp_mask = 3 << (pLSB - 1); uint16_t prev_row_sig[264]; memset(prev_row_sig, 0, sizeof(prev_row_sig)); @@ -750,7 +573,7 @@ void ht_sigprop_decode(j2k_codeblock *block, uint8_t *HT_magref_segment, uint32_ uint32_t new_sig = 0; if (mbr) { static const uint32_t row_masks[4] = {0x33u, 0x76u, 0xECu, 0xC8u}; - uint32_t inv_sig = ~cs & pat; + uint32_t inv_sig = ~cs & pat; // CTZ iteration: process only set mbr bits in column-major order // (nibble layout guarantees ascending bit positions = column-major). // Branchless bit-result handling eliminates the unpredictable @@ -808,7 +631,7 @@ void ht_magref_decode(j2k_codeblock *block, uint8_t *HT_magref_segment, uint32_t int32_t *samples = block->sample_buf; for (uint32_t y = 0; y < height; y += 4) { - const uint16_t *csig = sigma + (y >> 2) * mstr; + const uint16_t *csig = sigma + (y >> 2) * mstr; for (uint32_t x = 0; x < width; x += 4) { uint16_t sig = csig[x >> 2]; @@ -864,12 +687,12 @@ void j2k_codeblock::dequantize(uint8_t ROIshift) const { } } else { for (; len >= 16; len -= 16) { - v0 = _mm256_load_si256((__m256i *)val); - v1 = _mm256_load_si256((__m256i *)(val + 8)); - s0 = v0; - s1 = v1; - v0 = _mm256_and_si256(v0, magmask); - v1 = _mm256_and_si256(v1, magmask); + v0 = _mm256_load_si256((__m256i *)val); + v1 = _mm256_load_si256((__m256i *)(val + 8)); + s0 = v0; + s1 = v1; + v0 = _mm256_and_si256(v0, magmask); + v1 = _mm256_and_si256(v1, magmask); vROImask = _mm256_and_si256(v0, vmask); vROImask = _mm256_cmpeq_epi32(vROImask, zero); vROImask = _mm256_and_si256(vROImask, shift); @@ -878,8 +701,8 @@ void j2k_codeblock::dequantize(uint8_t ROIshift) const { vROImask = _mm256_cmpeq_epi32(vROImask, zero); vROImask = _mm256_and_si256(vROImask, shift); v1 = _mm256_sllv_epi32(v1, vROImask); - vdst0 = _mm256_sign_epi32(_mm256_srai_epi32(v0, pLSB), s0); - vdst1 = _mm256_sign_epi32(_mm256_srai_epi32(v1, pLSB), s1); + vdst0 = _mm256_sign_epi32(_mm256_srai_epi32(v0, pLSB), s0); + vdst1 = _mm256_sign_epi32(_mm256_srai_epi32(v1, pLSB), s1); simd_store(dst, vdst0, vdst1); val += 16; dst += 16; @@ -927,7 +750,7 @@ void j2k_codeblock::dequantize(uint8_t ROIshift) const { if (ROIshift == 0) { // Common case: no ROI — direct float multiply, sign via XOR. - const __m256 vfscale = _mm256_set1_ps(fscale_direct); + const __m256 vfscale = _mm256_set1_ps(fscale_direct); const __m256i vsignmask = _mm256_set1_epi32(INT32_MIN); for (size_t i = 0; i < static_cast(this->size.y); i++) { int32_t *val = this->sample_buf + i * this->blksampl_stride; @@ -948,10 +771,10 @@ void j2k_codeblock::dequantize(uint8_t ROIshift) const { __m256 f1 = _mm256_mul_ps(_mm256_cvtepi32_ps(m1), vfscale); __m256 f2 = _mm256_mul_ps(_mm256_cvtepi32_ps(m2), vfscale); __m256 f3 = _mm256_mul_ps(_mm256_cvtepi32_ps(m3), vfscale); - f0 = _mm256_xor_ps(f0, _mm256_castsi256_ps(_mm256_and_si256(a0, vsignmask))); - f1 = _mm256_xor_ps(f1, _mm256_castsi256_ps(_mm256_and_si256(a1, vsignmask))); - f2 = _mm256_xor_ps(f2, _mm256_castsi256_ps(_mm256_and_si256(a2, vsignmask))); - f3 = _mm256_xor_ps(f3, _mm256_castsi256_ps(_mm256_and_si256(a3, vsignmask))); + f0 = _mm256_xor_ps(f0, _mm256_castsi256_ps(_mm256_and_si256(a0, vsignmask))); + f1 = _mm256_xor_ps(f1, _mm256_castsi256_ps(_mm256_and_si256(a1, vsignmask))); + f2 = _mm256_xor_ps(f2, _mm256_castsi256_ps(_mm256_and_si256(a2, vsignmask))); + f3 = _mm256_xor_ps(f3, _mm256_castsi256_ps(_mm256_and_si256(a3, vsignmask))); _mm256_storeu_ps(dst, f0); _mm256_storeu_ps(dst + 8, f1); _mm256_storeu_ps(dst + 16, f2); @@ -963,7 +786,7 @@ void j2k_codeblock::dequantize(uint8_t ROIshift) const { __m256i a0 = _mm256_load_si256((__m256i *)val); __m256i m0 = _mm256_and_si256(a0, magmask); __m256 f0 = _mm256_mul_ps(_mm256_cvtepi32_ps(m0), vfscale); - f0 = _mm256_xor_ps(f0, _mm256_castsi256_ps(_mm256_and_si256(a0, vsignmask))); + f0 = _mm256_xor_ps(f0, _mm256_castsi256_ps(_mm256_and_si256(a0, vsignmask))); _mm256_storeu_ps(dst, f0); val += 8; dst += 8; @@ -971,14 +794,14 @@ void j2k_codeblock::dequantize(uint8_t ROIshift) const { for (; len > 0; --len) { int32_t sign = *val & INT32_MIN; float f = static_cast(*val & INT32_MAX) * fscale_direct; - if (sign) f = -f; + if (sign) f = -f; *dst++ = f; val++; } } } else { // ROI path — rarely used; keep integer-arithmetic approach for correctness. - float fscale = fscale_direct; + float fscale = fscale_direct; constexpr int32_t downshift = 15; fscale *= static_cast(1 << 16) * static_cast(1 << downshift); const auto scale = static_cast(fscale + 0.5f); @@ -1035,13 +858,32 @@ void j2k_codeblock::dequantize(uint8_t ROIshift) const { } } -bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { +// Per-block decode setup: segment validation, Lcup/Lref/Scup/Pcup derivation +// and the modDcup buffer mutation. Factored out of htj2k_decode so the +// 1-way and batched entries share one source of truth. +struct ht_dec_setup { + int32_t Lcup, Pcup, Scup; + uint32_t Lref; + uint8_t *Dref; // nullptr when single-segment + uint8_t S_blk; + uint8_t num_ht_passes; + bool ok; // false → htj2k_decode returns false + bool empty; // num_ht_passes == 0 → success with no work +}; + +static ht_dec_setup htj2k_dec_setup(j2k_codeblock *block) { + ht_dec_setup su; + su.Lcup = 0; + su.Pcup = 0; + su.Scup = 0; + su.Lref = 0; + su.Dref = nullptr; + su.S_blk = 0; + su.ok = false; + su.empty = false; + // number of placeholder pass uint8_t P0 = 0; - // length of HT Cleanup segment - int32_t Lcup = 0; - // length of HT Refinement segment - uint32_t Lref = 0; // number of HT Sets preceding the given(this) HT Set const uint8_t S_skip = 0; @@ -1062,122 +904,166 @@ bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { if (block->num_passes < empty_passes) { printf("WARNING: number of passes %d exceeds number of empty passes %d", block->num_passes, empty_passes); - return false; + return su; } // number of ht coding pass (Z_blk in the spec) - const uint8_t num_ht_passes = static_cast(block->num_passes - empty_passes); - // pointer to buffer for HT Cleanup segment - uint8_t *Dcup; - // pointer to buffer for HT Refinement segment - uint8_t *Dref; - - if (num_ht_passes > 0) { - // HT defines at most two segments per codeblock (Cleanup + optional - // Refinement); `all_segments[4]` is over-provisioned. A malformed - // input with pass_length_count > 4 non-zero entries used to write - // past the array and smash the stack — guard the write and the - // later `all_segments[0]` read. Reported by IM JUN SEO (KISIA) and - // OH HAN GUEL (SANGMYUNG UNIVERSITY). - uint8_t all_segments[4] = {}; - uint32_t num_segments = 0; - for (uint32_t i = 0; i < block->pass_length_count; i++) { - if (block->pass_length[i] != 0) { - if (num_segments >= 4) { - printf("WARNING: too many HT coding-pass segments (>4) — malformed input.\n"); - return false; - } - all_segments[num_segments++] = static_cast(i); - } - } - if (num_segments == 0) { - printf("WARNING: no non-empty HT coding-pass segments.\n"); - return false; - } - Lcup += static_cast(block->pass_length[all_segments[0]]); - if (Lcup < 2) { - printf("WARNING: Cleanup pass length must be at least 2 bytes in length.\n"); - return false; - } - // Bound the attacker-controlled cleanup length by the (already clamped) - // codeblock byte count before any Dcup[Lcup-1] access / modDcup write. - if (static_cast(Lcup) > block->length) { - printf("WARNING: HT cleanup pass length %d exceeds codeblock bytes %u — malformed input.\n", Lcup, - block->length); - return false; - } - for (uint32_t i = 1; i < num_segments; i++) { - Lref += block->pass_length[all_segments[i]]; - } - // The refinement segments are read from Dcup + Lcup; keep them in the buffer. - if (static_cast(Lref) > block->length - static_cast(Lcup)) { - printf("WARNING: HT refinement length exceeds remaining codeblock bytes %u — malformed input.\n", - block->length - static_cast(Lcup)); - return false; - } - Dcup = block->get_compressed_data(); + su.num_ht_passes = static_cast(block->num_passes - empty_passes); + if (su.num_ht_passes == 0) { + su.ok = true; + su.empty = true; + return su; + } - if (block->num_passes > 1 && num_segments > 1) { - Dref = block->get_compressed_data() + Lcup; - } else { - Dref = nullptr; - } - // number of (skipped) magnitude bitplanes - const uint8_t S_blk = static_cast(P0 + block->num_ZBP + S_skip); - if (S_blk >= 30) { - printf("WARNING: Number of skipped mag bitplanes %d is too large.\n", S_blk); - return false; - } - // Suffix length (=MEL + VLC) of HT Cleanup pass - const int32_t Scup = static_cast((Dcup[Lcup - 1] << 4) + (Dcup[Lcup - 2] & 0x0F)); - if (Scup < 2 || Scup > Lcup || Scup > 4079) { - printf("WARNING: cleanup pass suffix length %d is invalid.\n", Scup); - return false; - } - // modDcup (shall be done before the creation of state_VLC instance) - Dcup[Lcup - 1] = 0xFF; - Dcup[Lcup - 2] |= 0x0F; - const int32_t Pcup = static_cast(Lcup - Scup); - - // Single HT pass with no ROI: use fused dequantize path to eliminate - // the separate dequantize pass over sample_buf. - bool dequant_done = false; - // Fused dequant: the MagSgn SIMD stores write in units of 4 (128-bit) or 8 (256-bit) - // elements. When block width is not a multiple of 4, the extra elements overflow into - // adjacent blocks' column range in the shared output buffer (subband or ring buffer). - // This is safe in single-threaded decode (sequential order overwrites correctly) but - // causes a data race in multi-threaded decode. Gate on width % 4 == 0 to avoid this. - if (num_ht_passes == 1 && ROIshift == 0 && (block->size.x & 3) == 0 - && (block->size.y & 1u) == 0) { - if (block->dequant_i32) - ht_cleanup_decode(block, static_cast(30 - S_blk), Lcup, Pcup, Scup); - else - ht_cleanup_decode(block, static_cast(30 - S_blk), Lcup, Pcup, Scup); - dequant_done = true; - } else if (num_ht_passes == 1) { - ht_cleanup_decode(block, static_cast(30 - S_blk), Lcup, Pcup, Scup); - } else { - ht_cleanup_decode(block, static_cast(30 - S_blk), Lcup, Pcup, Scup); - - // Pack block_states SIGMA bits into nibble-packed sigma array - const uint32_t qw = (block->size.x + 3) >> 2; - const uint32_t mstr = ((qw + 2) + 7u) & ~7u; - uint16_t sigma_buf[(17 + 1) * 24] = {}; - pack_sigma(block->block_states, block->blkstate_stride, block->size.x, block->size.y, - sigma_buf, mstr); - - ht_sigprop_decode(block, Dref, Lref, static_cast(30 - (S_blk + 1)), sigma_buf, mstr); - if (num_ht_passes > 2) { - ht_magref_decode(block, Dref, Lref, static_cast(30 - (S_blk + 1)), sigma_buf, mstr); + // HT defines at most two segments per codeblock (Cleanup + optional + // Refinement); `all_segments[4]` is over-provisioned. A malformed + // input with pass_length_count > 4 non-zero entries used to write + // past the array and smash the stack — guard the write and the + // later `all_segments[0]` read. Reported by IM JUN SEO (KISIA) and + // OH HAN GUEL (SANGMYUNG UNIVERSITY). + uint8_t all_segments[4] = {}; + uint32_t num_segments = 0; + for (uint32_t i = 0; i < block->pass_length_count; i++) { + if (block->pass_length[i] != 0) { + if (num_segments >= 4) { + printf("WARNING: too many HT coding-pass segments (>4) — malformed input.\n"); + return su; } + all_segments[num_segments++] = static_cast(i); } + } + if (num_segments == 0) { + printf("WARNING: no non-empty HT coding-pass segments.\n"); + return su; + } + su.Lcup += static_cast(block->pass_length[all_segments[0]]); + if (su.Lcup < 2) { + printf("WARNING: Cleanup pass length must be at least 2 bytes in length.\n"); + return su; + } + // Bound the attacker-controlled cleanup length by the (already clamped) + // codeblock byte count before any Dcup[Lcup-1] access / modDcup write. + if (static_cast(su.Lcup) > block->length) { + printf("WARNING: HT cleanup pass length %d exceeds codeblock bytes %u — malformed input.\n", su.Lcup, + block->length); + return su; + } + for (uint32_t i = 1; i < num_segments; i++) { + su.Lref += block->pass_length[all_segments[i]]; + } + // The refinement segments are read from Dcup + Lcup; keep them in the buffer. + if (static_cast(su.Lref) > block->length - static_cast(su.Lcup)) { + printf("WARNING: HT refinement length exceeds remaining codeblock bytes %u — malformed input.\n", + block->length - static_cast(su.Lcup)); + return su; + } + uint8_t *Dcup = block->get_compressed_data(); - // dequantization (skipped when already fused into MagSgn output) - if (!dequant_done) { - block->dequantize(ROIshift); + if (block->num_passes > 1 && num_segments > 1) { + su.Dref = block->get_compressed_data() + su.Lcup; + } else { + su.Dref = nullptr; + } + // number of (skipped) magnitude bitplanes + su.S_blk = static_cast(P0 + block->num_ZBP + S_skip); + if (su.S_blk >= 30) { + printf("WARNING: Number of skipped mag bitplanes %d is too large.\n", su.S_blk); + return su; + } + // Suffix length (=MEL + VLC) of HT Cleanup pass + su.Scup = static_cast((Dcup[su.Lcup - 1] << 4) + (Dcup[su.Lcup - 2] & 0x0F)); + if (su.Scup < 2 || su.Scup > su.Lcup || su.Scup > 4079) { + printf("WARNING: cleanup pass suffix length %d is invalid.\n", su.Scup); + return su; + } + // modDcup (shall be done before the creation of state_VLC instance) + Dcup[su.Lcup - 1] = 0xFF; + Dcup[su.Lcup - 2] |= 0x0F; + su.Pcup = static_cast(su.Lcup - su.Scup); + su.ok = true; + return su; +} + +// Everything after step-1: step-2 (MagSgn) variant dispatch, SigProp/MagRef +// refinement passes, and dequantization. scratch/sstr hold the step-1 output. +static bool htj2k_dec_finish(j2k_codeblock *block, const ht_dec_setup &su, const uint8_t ROIshift, + uint16_t *scratch, const int32_t sstr) { + const uint8_t pLSB = static_cast(30 - su.S_blk); + + // Single HT pass with no ROI: use fused dequantize path to eliminate + // the separate dequantize pass over sample_buf. + bool dequant_done = false; + // Fused dequant: the MagSgn SIMD stores write in units of 4 (128-bit) or 8 (256-bit) + // elements. When block width is not a multiple of 4, the extra elements overflow into + // adjacent blocks' column range in the shared output buffer (subband or ring buffer). + // This is safe in single-threaded decode (sequential order overwrites correctly) but + // causes a data race in multi-threaded decode. Gate on width % 4 == 0 to avoid this. + if (su.num_ht_passes == 1 && ROIshift == 0 && (block->size.x & 3) == 0 && (block->size.y & 1u) == 0) { + if (block->dequant_i32) + ht_cleanup_step2(block, pLSB, su.Pcup, scratch, sstr); + else + ht_cleanup_step2(block, pLSB, su.Pcup, scratch, sstr); + dequant_done = true; + } else if (su.num_ht_passes == 1) { + ht_cleanup_step2<>(block, pLSB, su.Pcup, scratch, sstr); + } else { + ht_cleanup_step2<>(block, pLSB, su.Pcup, scratch, sstr); + + // Pack block_states SIGMA bits into nibble-packed sigma array + const uint32_t qw = (block->size.x + 3) >> 2; + const uint32_t mstr = ((qw + 2) + 7u) & ~7u; + uint16_t sigma_buf[(17 + 1) * 24] = {}; + pack_sigma(block->block_states, block->blkstate_stride, block->size.x, block->size.y, sigma_buf, mstr); + + ht_sigprop_decode(block, su.Dref, su.Lref, static_cast(30 - (su.S_blk + 1)), sigma_buf, mstr); + if (su.num_ht_passes > 2) { + ht_magref_decode(block, su.Dref, su.Lref, static_cast(30 - (su.S_blk + 1)), sigma_buf, mstr); } + } - } // end + // dequantization (skipped when already fused into MagSgn output) + if (!dequant_done) { + block->dequantize(ROIshift); + } return true; } + +bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { + const ht_dec_setup su = htj2k_dec_setup(block); + if (!su.ok) return false; + if (su.empty) return true; + + const uint16_t QW = static_cast(ceil_int(static_cast(block->size.x), 2)); + const uint16_t QH = static_cast(ceil_int(static_cast(block->size.y), 2)); + const int32_t sstr = static_cast(((block->size.x + 2) + 7u) & ~7u); // multiples of 8 + + uint16_t scratch[8 * 513]; + ht_step1_lane ln; + ln.Dcup = block->get_compressed_data(); + ln.Lcup = su.Lcup; + ln.Scup = su.Scup; + ln.scratch = scratch; + ln.block_states = block->block_states; + ln.blkstate_stride = block->blkstate_stride; + if (su.num_ht_passes == 1) { + ht_cleanup_step1_nway<1, true>(&ln, QW, QH, sstr); + } else { + ht_cleanup_step1_nway<1, false>(&ln, QW, QH, sstr); + } + + return htj2k_dec_finish(block, su, ROIshift, scratch, sstr); +} + +// Batched entry point (see block_decoding.hpp). Phase 1: plain per-block +// loop; the N-way group formation lands with the batch kernel enablement. +bool htj2k_decode_batch(j2k_codeblock *const *blocks, uint32_t n, uint8_t ROIshift, bool *results) { + bool all_ok = true; + for (uint32_t i = 0; i < n; ++i) { + results[i] = htj2k_decode(blocks[i], ROIshift); + all_ok &= results[i]; + } + return all_ok; +} + +const uint32_t htj2k_dec_batch_lanes = 1; #endif \ No newline at end of file diff --git a/source/core/coding/ht_block_decoding_neon.cpp b/source/core/coding/ht_block_decoding_neon.cpp index 72b579bc..65efdf17 100644 --- a/source/core/coding/ht_block_decoding_neon.cpp +++ b/source/core/coding/ht_block_decoding_neon.cpp @@ -114,10 +114,9 @@ static FORCE_INLINE void dequant_store_neon(int32_t *dst, int32x4_t val, uint8_t else vst1q_f32(reinterpret_cast(dst), vcvtq_f32_s32(res)); } else { - int32x4_t mag = vandq_s32(val, vmagmask); - float32x4_t f = vmulq_f32(vcvtq_f32_s32(mag), vfscale); - f = vreinterpretq_f32_s32( - veorq_s32(vreinterpretq_s32_f32(f), vandq_s32(val, vsignmask))); + int32x4_t mag = vandq_s32(val, vmagmask); + float32x4_t f = vmulq_f32(vcvtq_f32_s32(mag), vfscale); + f = vreinterpretq_f32_s32(veorq_s32(vreinterpretq_s32_f32(f), vandq_s32(val, vsignmask))); vst1q_f32(reinterpret_cast(dst), f); } } @@ -142,15 +141,15 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t // Fused dequantize setup: when fuse_dequant is true, we write dequantized float values // directly to band_buf, eliminating the separate dequantize pass. // pLSB_dq is the dequantization shift (31 - M_b), distinct from the MagSgn pLSB. - int32_t pLSB_dq = 0; - float32x4_t vfscale_dq = vdupq_n_f32(0.0f); - int32x4_t vmagmask_dq = vdupq_n_s32(0); - int32x4_t vsignmask_dq = vdupq_n_s32(0); + int32_t pLSB_dq = 0; + float32x4_t vfscale_dq = vdupq_n_f32(0.0f); + int32x4_t vmagmask_dq = vdupq_n_s32(0); + int32x4_t vsignmask_dq = vdupq_n_s32(0); if constexpr (fuse_dequant) { const int32_t M_b_val = block->get_Mb(); pLSB_dq = 31 - M_b_val; - vmagmask_dq = vdupq_n_s32(0x7FFFFFFF); - vsignmask_dq = vdupq_n_s32(INT32_MIN); + vmagmask_dq = vdupq_n_s32(0x7FFFFFFF); + vsignmask_dq = vdupq_n_s32(INT32_MIN); if (block->transformation != 1) { // lossy path (transformation==0 for irrev97, transformation>=2 for ATK irrev) float fscale_direct = block->stepsize; @@ -164,10 +163,10 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t } int32_t *const sample_buf = block->sample_buf; - int32_t *mp0 = fuse_dequant ? reinterpret_cast(block->band_buf) : sample_buf; - int32_t *mp1 = mp0 + (fuse_dequant ? block->band_stride : block->blksampl_stride); - auto sp0 = block->block_states + 1 + block->blkstate_stride; - auto sp1 = block->block_states + 1 + 2 * block->blkstate_stride; + int32_t *mp0 = fuse_dequant ? reinterpret_cast(block->band_buf) : sample_buf; + int32_t *mp1 = mp0 + (fuse_dequant ? block->band_stride : block->blksampl_stride); + auto sp0 = block->block_states + 1 + block->blkstate_stride; + auto sp1 = block->block_states + 1 + 2 * block->blkstate_stride; uint32_t rho0, rho1; uint32_t u_off0, u_off1; @@ -183,10 +182,10 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t alignas(32) uint32_t rholine[516]; // QW_max + 4, QW_max = 512 std::memset(rholine, 0, (QW + 4U) * sizeof(uint32_t)); - uint32_t *rho_p = rholine + 1; - alignas(32) int32_t Eline[1032]; // 2 * QW_max + 8, QW_max = 512 + uint32_t *rho_p = rholine + 1; + alignas(32) int32_t Eline[1032]; // 2 * QW_max + 8, QW_max = 512 std::memset(Eline, 0, (2U * QW + 8U) * sizeof(int32_t)); - int32_t *E_p = Eline + 1; + int32_t *E_p = Eline + 1; uint32_t context = 0; uint32_t vlcval; @@ -280,9 +279,8 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t // 16-bit fast path: batch bit extraction via vqtbl1q_u8 const uint8_t pLSB_adj = pLSB - 16; int16x4_t vn_16 = vdup_n_s16(0); - int16x8_t row16 = - MagSgn.decode_two_quads_16bit(tv0, tv1, static_cast(U0), - static_cast(U1), pLSB_adj, vn_16, dc); + int16x8_t row16 = MagSgn.decode_two_quads_16bit(tv0, tv1, static_cast(U0), + static_cast(U1), pLSB_adj, vn_16, dc); // Deinterleave row0/row1 and expand int16 -> int32 (sign bit 15 -> bit 31). int16x4_t lo = vget_low_s16(row16); int16x4_t hi = vget_high_s16(row16); @@ -290,9 +288,9 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t int16x4_t row1_16 = vuzp2_s16(lo, hi); if constexpr (fuse_dequant) { dequant_store_neon(mp0, vshll_n_s16(row0_16, 16), block->transformation, pLSB_dq, - vfscale_dq, vmagmask_dq, vsignmask_dq); + vfscale_dq, vmagmask_dq, vsignmask_dq); dequant_store_neon(mp1, vshll_n_s16(row1_16, 16), block->transformation, pLSB_dq, - vfscale_dq, vmagmask_dq, vsignmask_dq); + vfscale_dq, vmagmask_dq, vsignmask_dq); } else { vst1q_s32(mp0, vshll_n_s16(row0_16, 16)); vst1q_s32(mp1, vshll_n_s16(row1_16, 16)); @@ -308,8 +306,8 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t // Existing 32-bit path int32x4_t vmask1, sig0, sig1, vtmp, m_n_0, m_n_1, msvec, v_n_0, v_n_1, mu0, mu1; - sig0 = vdupq_n_u32(rho0); - sig0 = vtstq_s32(sig0, vm); + sig0 = vdupq_n_u32(rho0); + sig0 = vtstq_s32(sig0, vm); vtmp = vandq_s32(vtstq_s32(vdupq_n_u32(emb_k_0), vm), vone); m_n_0 = vsubq_s32(vandq_s32(sig0, vdupq_n_u32(U0)), vtmp); sig1 = vdupq_n_u32(rho1); @@ -340,10 +338,10 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t mu1 = vandq_u32(mu1, sig1); if constexpr (fuse_dequant) { - dequant_store_neon(mp0, vuzp1q_s32(mu0, mu1), block->transformation, pLSB_dq, - vfscale_dq, vmagmask_dq, vsignmask_dq); - dequant_store_neon(mp1, vuzp2q_s32(mu0, mu1), block->transformation, pLSB_dq, - vfscale_dq, vmagmask_dq, vsignmask_dq); + dequant_store_neon(mp0, vuzp1q_s32(mu0, mu1), block->transformation, pLSB_dq, vfscale_dq, + vmagmask_dq, vsignmask_dq); + dequant_store_neon(mp1, vuzp2q_s32(mu0, mu1), block->transformation, pLSB_dq, vfscale_dq, + vmagmask_dq, vsignmask_dq); } else { vst1q_s32(mp0, vuzp1q_s32(mu0, mu1)); vst1q_s32(mp1, vuzp2q_s32(mu0, mu1)); @@ -372,18 +370,18 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t mp0 = sample_buf + (row * 2U) * block->blksampl_stride; mp1 = sample_buf + (row * 2U + 1U) * block->blksampl_stride; } - sp0 = block->block_states + (row * 2U + 1U) * block->blkstate_stride + 1U; - sp1 = block->block_states + (row * 2U + 2U) * block->blkstate_stride + 1U; - rho1 = 0; + sp0 = block->block_states + (row * 2U + 1U) * block->blkstate_stride + 1U; + sp1 = block->block_states + (row * 2U + 2U) * block->blkstate_stride + 1U; + rho1 = 0; // Pre-compute Emax for first 4 quads (vectorized sliding max). int32x4_t vEmax4; { int32x2_t m01 = vpmax_s32(vget_low_s32(vmaxq_s32(vld1q_s32(E_p - 1), vld1q_s32(E_p + 1))), - vget_high_s32(vmaxq_s32(vld1q_s32(E_p - 1), vld1q_s32(E_p + 1)))); + vget_high_s32(vmaxq_s32(vld1q_s32(E_p - 1), vld1q_s32(E_p + 1)))); int32x2_t m23 = vpmax_s32(vget_low_s32(vmaxq_s32(vld1q_s32(E_p + 3), vld1q_s32(E_p + 5))), - vget_high_s32(vmaxq_s32(vld1q_s32(E_p + 3), vld1q_s32(E_p + 5)))); - vEmax4 = vcombine_s32(m01, m23); + vget_high_s32(vmaxq_s32(vld1q_s32(E_p + 3), vld1q_s32(E_p + 5)))); + vEmax4 = vcombine_s32(m01, m23); } // calculate context for the next quad @@ -493,18 +491,17 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t // 16-bit fast path: batch bit extraction via vqtbl1q_u8 const uint8_t pLSB_adj = pLSB - 16; int16x4_t vn_16 = vdup_n_s16(0); - int16x8_t row16 = - MagSgn.decode_two_quads_16bit(tv0, tv1, static_cast(U0), - static_cast(U1), pLSB_adj, vn_16, dc); - int16x4_t lo = vget_low_s16(row16); - int16x4_t hi = vget_high_s16(row16); - int16x4_t row0_16 = vuzp1_s16(lo, hi); - int16x4_t row1_16 = vuzp2_s16(lo, hi); + int16x8_t row16 = MagSgn.decode_two_quads_16bit(tv0, tv1, static_cast(U0), + static_cast(U1), pLSB_adj, vn_16, dc); + int16x4_t lo = vget_low_s16(row16); + int16x4_t hi = vget_high_s16(row16); + int16x4_t row0_16 = vuzp1_s16(lo, hi); + int16x4_t row1_16 = vuzp2_s16(lo, hi); if constexpr (fuse_dequant) { dequant_store_neon(mp0, vshll_n_s16(row0_16, 16), block->transformation, pLSB_dq, - vfscale_dq, vmagmask_dq, vsignmask_dq); + vfscale_dq, vmagmask_dq, vsignmask_dq); dequant_store_neon(mp1, vshll_n_s16(row1_16, 16), block->transformation, pLSB_dq, - vfscale_dq, vmagmask_dq, vsignmask_dq); + vfscale_dq, vmagmask_dq, vsignmask_dq); } else { vst1q_s32(mp0, vshll_n_s16(row0_16, 16)); vst1q_s32(mp1, vshll_n_s16(row1_16, 16)); @@ -518,10 +515,10 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t // are spent. Reload from E_p+3 (which still holds the previous row's values). { int32x2_t nm01 = vpmax_s32(vget_low_s32(vmaxq_s32(vld1q_s32(E_p + 3), vld1q_s32(E_p + 5))), - vget_high_s32(vmaxq_s32(vld1q_s32(E_p + 3), vld1q_s32(E_p + 5)))); + vget_high_s32(vmaxq_s32(vld1q_s32(E_p + 3), vld1q_s32(E_p + 5)))); int32x2_t nm23 = vpmax_s32(vget_low_s32(vmaxq_s32(vld1q_s32(E_p + 7), vld1q_s32(E_p + 9))), - vget_high_s32(vmaxq_s32(vld1q_s32(E_p + 7), vld1q_s32(E_p + 9)))); - vEmax4 = vcombine_s32(nm01, nm23); + vget_high_s32(vmaxq_s32(vld1q_s32(E_p + 7), vld1q_s32(E_p + 9)))); + vEmax4 = vcombine_s32(nm01, nm23); } int32x4_t vn32 = vreinterpretq_s32_u32(vmovl_u16(vreinterpret_u16_s16(vn_16))); @@ -532,8 +529,8 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t // Existing 32-bit path int32x4_t vmask1, sig0, sig1, vtmp, m_n_0, m_n_1, msvec, v_n_0, v_n_1, mu0, mu1; - sig0 = vdupq_n_u32(rho0); - sig0 = vtstq_s32(sig0, vm); + sig0 = vdupq_n_u32(rho0); + sig0 = vtstq_s32(sig0, vm); vtmp = vandq_s32(vtstq_s32(vdupq_n_u32(emb_k_0), vm), vone); m_n_0 = vsubq_s32(vandq_s32(sig0, vdupq_n_u32(U0)), vtmp); sig1 = vdupq_n_u32(rho1); @@ -565,9 +562,9 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t if constexpr (fuse_dequant) { dequant_store_neon(mp0, vuzp1q_s32(mu0, mu1), block->transformation, pLSB_dq, - vfscale_dq, vmagmask_dq, vsignmask_dq); + vfscale_dq, vmagmask_dq, vsignmask_dq); dequant_store_neon(mp1, vuzp2q_s32(mu0, mu1), block->transformation, pLSB_dq, - vfscale_dq, vmagmask_dq, vsignmask_dq); + vfscale_dq, vmagmask_dq, vsignmask_dq); } else { vst1q_s32(mp0, vuzp1q_s32(mu0, mu1)); vst1q_s32(mp1, vuzp2q_s32(mu0, mu1)); @@ -577,10 +574,10 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t { int32x2_t nm01 = vpmax_s32(vget_low_s32(vmaxq_s32(vld1q_s32(E_p + 3), vld1q_s32(E_p + 5))), - vget_high_s32(vmaxq_s32(vld1q_s32(E_p + 3), vld1q_s32(E_p + 5)))); + vget_high_s32(vmaxq_s32(vld1q_s32(E_p + 3), vld1q_s32(E_p + 5)))); int32x2_t nm23 = vpmax_s32(vget_low_s32(vmaxq_s32(vld1q_s32(E_p + 7), vld1q_s32(E_p + 9))), - vget_high_s32(vmaxq_s32(vld1q_s32(E_p + 7), vld1q_s32(E_p + 9)))); - vEmax4 = vcombine_s32(nm01, nm23); + vget_high_s32(vmaxq_s32(vld1q_s32(E_p + 7), vld1q_s32(E_p + 9)))); + vEmax4 = vcombine_s32(nm01, nm23); } vExp = vsubq_s32(vdupq_n_s32(32), vclzq_s32(vuzp2q_s32(v_n_0, v_n_1))); @@ -626,12 +623,12 @@ void ht_sigprop_decode(j2k_codeblock *block, uint8_t *HT_magref_segment, uint32_ const uint8_t &pLSB, uint16_t *sigma, uint32_t mstr) { if (pLSB == 0) return; // no plane below the LSB; mirrors ht_magref_decode (avoids 1 << (pLSB-1) UB) SP_dec SigProp(HT_magref_segment, magref_length); - const uint32_t height = block->size.y; - const uint32_t width = block->size.x; - const size_t sstride = block->blksampl_stride; - int32_t *samples = block->sample_buf; - const bool non_causal = (block->Cmodes & CAUSAL) == 0; - const int32_t spp_mask = 3 << (pLSB - 1); + const uint32_t height = block->size.y; + const uint32_t width = block->size.x; + const size_t sstride = block->blksampl_stride; + int32_t *samples = block->sample_buf; + const bool non_causal = (block->Cmodes & CAUSAL) == 0; + const int32_t spp_mask = 3 << (pLSB - 1); uint16_t prev_row_sig[264]; memset(prev_row_sig, 0, sizeof(prev_row_sig)); @@ -671,10 +668,10 @@ void ht_sigprop_decode(j2k_codeblock *block, uint8_t *HT_magref_segment, uint32_ uint32_t new_sig = 0; if (mbr) { static const uint32_t row_masks[4] = {0x33u, 0x76u, 0xECu, 0xC8u}; - uint32_t inv_sig = ~cs & pat; + uint32_t inv_sig = ~cs & pat; while (mbr) { - uint32_t pos = static_cast(openhtj2k_ctz32(mbr)); - uint32_t smask = 1u << pos; + uint32_t pos = static_cast(openhtj2k_ctz32(mbr)); + uint32_t smask = 1u << pos; mbr &= ~smask; uint32_t bit = SigProp.importSigPropBit(); uint32_t bit_mask = static_cast(-static_cast(bit)); @@ -762,12 +759,12 @@ void j2k_codeblock::dequantize(uint8_t ROIshift) const { sprec_t *dst = this->band_buf + i * this->band_stride; size_t len = this->size.x; for (; len >= 8; len -= 8) { - v0 = vld1q_s32(val); - v1 = vld1q_s32(val + 4); - s0 = vshrq_n_s32(v0, 31); - s1 = vshrq_n_s32(v1, 31); - v0 = vandq_s32(v0, vmagmask); - v1 = vandq_s32(v1, vmagmask); + v0 = vld1q_s32(val); + v1 = vld1q_s32(val + 4); + s0 = vshrq_n_s32(v0, 31); + s1 = vshrq_n_s32(v1, 31); + v0 = vandq_s32(v0, vmagmask); + v1 = vandq_s32(v1, vmagmask); vROImask = vandq_s32(v0, vmask); vROImask = vceqzq_s32(vROImask); vROImask = vandq_s32(vROImask, vROIshift); @@ -776,8 +773,8 @@ void j2k_codeblock::dequantize(uint8_t ROIshift) const { vROImask = vceqzq_s32(vROImask); vROImask = vandq_s32(vROImask, vROIshift); v1 = vshlq_s32(v1, vsubq_s32(vROImask, vpLSB)); - vdst0 = vbslq_s32(vreinterpretq_u32_s32(s0), vnegq_s32(v0), v0); - vdst1 = vbslq_s32(vreinterpretq_u32_s32(s1), vnegq_s32(v1), v1); + vdst0 = vbslq_s32(vreinterpretq_u32_s32(s0), vnegq_s32(v0), v0); + vdst1 = vbslq_s32(vreinterpretq_u32_s32(s1), vnegq_s32(v1), v1); simd_store(dst, vdst0, vdst1); val += 8; dst += 8; @@ -827,8 +824,8 @@ void j2k_codeblock::dequantize(uint8_t ROIshift) const { if (ROIshift == 0) { // Common case: no ROI — direct float multiply, sign via XOR. // Eliminates integer truncate→mul→shift pipeline: saves ~5 ops per 4 elements. - const float32x4_t vfscale = vdupq_n_f32(fscale_direct); - const int32x4_t vsignmask = vdupq_n_s32(INT32_MIN); + const float32x4_t vfscale = vdupq_n_f32(fscale_direct); + const int32x4_t vsignmask = vdupq_n_s32(INT32_MIN); for (size_t i = 0; i < static_cast(this->size.y); i++) { int32_t *val = this->sample_buf + i * this->blksampl_stride; sprec_t *dst = this->band_buf + i * this->band_stride; @@ -842,10 +839,10 @@ void j2k_codeblock::dequantize(uint8_t ROIshift) const { float32x4_t f0 = vmulq_f32(vcvtq_f32_s32(m0), vfscale); float32x4_t f1 = vmulq_f32(vcvtq_f32_s32(m1), vfscale); // XOR sign bit from input integer into float result. - f0 = vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(f0), - vreinterpretq_u32_s32(vandq_s32(a0, vsignmask)))); - f1 = vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(f1), - vreinterpretq_u32_s32(vandq_s32(a1, vsignmask)))); + f0 = vreinterpretq_f32_u32( + veorq_u32(vreinterpretq_u32_f32(f0), vreinterpretq_u32_s32(vandq_s32(a0, vsignmask)))); + f1 = vreinterpretq_f32_u32( + veorq_u32(vreinterpretq_u32_f32(f1), vreinterpretq_u32_s32(vandq_s32(a1, vsignmask)))); vst1q_f32(dst, f0); vst1q_f32(dst + 4, f1); val += 8; @@ -854,14 +851,14 @@ void j2k_codeblock::dequantize(uint8_t ROIshift) const { for (; len > 0; --len) { int32_t sign = *val & INT32_MIN; float f = static_cast(*val & INT32_MAX) * fscale_direct; - if (sign) f = -f; + if (sign) f = -f; *dst++ = f; val++; } } } else { // ROI path — rarely used; keep integer-arithmetic approach for correctness. - float fscale = fscale_direct; + float fscale = fscale_direct; constexpr int32_t downshift = 15; fscale *= static_cast(1 << 16) * static_cast(1 << downshift); const auto scale = static_cast(fscale + 0.5f); @@ -968,7 +965,7 @@ bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { // past the array and smash the stack — guard the write and the // later `all_segments[0]` read. Reported by IM JUN SEO (KISIA) and // OH HAN GUEL (SANGMYUNG UNIVERSITY). - uint8_t all_segments[4]; + uint8_t all_segments[4]; uint32_t num_segments = 0; for (uint32_t i = 0; i < block->pass_length_count; i++) { if (block->pass_length[i] != 0) { @@ -1031,8 +1028,7 @@ bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { // is not a multiple of 4, the overshoot corrupts adjacent blocks in parallel decode. // Also gate on even height: the kernel writes row-pairs unconditionally and odd // height overflows one row into the next block's region. - if (num_ht_passes == 1 && ROIshift == 0 && (block->size.x & 3) == 0 - && (block->size.y & 1u) == 0) { + if (num_ht_passes == 1 && ROIshift == 0 && (block->size.x & 3) == 0 && (block->size.y & 1u) == 0) { if (block->dequant_i32) ht_cleanup_decode(block, static_cast(30 - S_blk), Lcup, Pcup, Scup); else @@ -1043,11 +1039,11 @@ bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { } else { ht_cleanup_decode(block, static_cast(30 - S_blk), Lcup, Pcup, Scup); - const uint32_t qw = (block->size.x + 3) >> 2; - const uint32_t mstr = ((qw + 2) + 7u) & ~7u; + const uint32_t qw = (block->size.x + 3) >> 2; + const uint32_t mstr = ((qw + 2) + 7u) & ~7u; uint16_t sigma_buf[(17 + 1) * 24] = {}; - pack_sigma(block->block_states, block->blkstate_stride, block->size.x, block->size.y, - sigma_buf, mstr); + pack_sigma(block->block_states, block->blkstate_stride, block->size.x, block->size.y, sigma_buf, + mstr); ht_sigprop_decode(block, Dref, Lref, static_cast(30 - (S_blk + 1)), sigma_buf, mstr); if (num_ht_passes > 2) { ht_magref_decode(block, Dref, Lref, static_cast(30 - (S_blk + 1)), sigma_buf, mstr); @@ -1063,4 +1059,18 @@ bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { return true; } + +// Batched entry point (see block_decoding.hpp). The NEON decoder is a fused +// single-pass kernel with no separate step-1, so no N-way interleave is wired +// yet: plain per-block loop. +bool htj2k_decode_batch(j2k_codeblock *const *blocks, uint32_t n, uint8_t ROIshift, bool *results) { + bool all_ok = true; + for (uint32_t i = 0; i < n; ++i) { + results[i] = htj2k_decode(blocks[i], ROIshift); + all_ok &= results[i]; + } + return all_ok; +} + +const uint32_t htj2k_dec_batch_lanes = 1; #endif diff --git a/source/core/coding/ht_block_decoding_wasm.cpp b/source/core/coding/ht_block_decoding_wasm.cpp index 2769d5b4..e4bb5df1 100644 --- a/source/core/coding/ht_block_decoding_wasm.cpp +++ b/source/core/coding/ht_block_decoding_wasm.cpp @@ -46,14 +46,11 @@ static inline v128_t wasm_i32x4_shlv(v128_t a, v128_t b) { } static inline v128_t wasm_i32x4_vshl(v128_t a, v128_t b) { - auto shift_lane = [](int32_t val, int32_t n) -> int32_t { - return n >= 0 ? val << n : val >> (-n); - }; - return wasm_i32x4_make( - shift_lane(wasm_i32x4_extract_lane(a, 0), wasm_i32x4_extract_lane(b, 0)), - shift_lane(wasm_i32x4_extract_lane(a, 1), wasm_i32x4_extract_lane(b, 1)), - shift_lane(wasm_i32x4_extract_lane(a, 2), wasm_i32x4_extract_lane(b, 2)), - shift_lane(wasm_i32x4_extract_lane(a, 3), wasm_i32x4_extract_lane(b, 3))); + auto shift_lane = [](int32_t val, int32_t n) -> int32_t { return n >= 0 ? val << n : val >> (-n); }; + return wasm_i32x4_make(shift_lane(wasm_i32x4_extract_lane(a, 0), wasm_i32x4_extract_lane(b, 0)), + shift_lane(wasm_i32x4_extract_lane(a, 1), wasm_i32x4_extract_lane(b, 1)), + shift_lane(wasm_i32x4_extract_lane(a, 2), wasm_i32x4_extract_lane(b, 2)), + shift_lane(wasm_i32x4_extract_lane(a, 3), wasm_i32x4_extract_lane(b, 3))); } static inline int32_t wasm_i32x4_reduce_max(v128_t a) { @@ -65,10 +62,10 @@ static inline int32_t wasm_i32x4_reduce_max(v128_t a) { // Branchless vectorized CLZ via float exponent: clz32(a) = min(158 - (float_bits(a) >> 23), 32). // Valid for non-negative int32 inputs; correctly returns 32 for a == 0. static inline v128_t wasm_u32x4_clz(v128_t a) { - v128_t vf = wasm_f32x4_convert_i32x4(a); // int32 → float (non-negative) - v128_t exp = wasm_u32x4_shr(vf, 23); // extract biased exponent - v128_t clz = wasm_i32x4_sub(wasm_i32x4_const_splat(158), exp); // 158 - biased_exp = clz - return wasm_i32x4_min(clz, wasm_i32x4_const_splat(32)); // clamp: handles a == 0 + v128_t vf = wasm_f32x4_convert_i32x4(a); // int32 → float (non-negative) + v128_t exp = wasm_u32x4_shr(vf, 23); // extract biased exponent + v128_t clz = wasm_i32x4_sub(wasm_i32x4_const_splat(158), exp); // 158 - biased_exp = clz + return wasm_i32x4_min(clz, wasm_i32x4_const_splat(32)); // clamp: handles a == 0 } uint8_t j2k_codeblock::calc_mbr(const uint32_t i, const uint32_t j, const uint8_t causal_cond) const { @@ -134,7 +131,7 @@ static inline void dequant_store_wasm(int32_t *dst, v128_t val, uint8_t transfor } else { v128_t mag = wasm_v128_and(val, vmagmask); v128_t f = wasm_f32x4_mul(wasm_f32x4_convert_i32x4(mag), vfscale); - f = wasm_v128_xor(f, wasm_v128_and(val, vsignmask)); + f = wasm_v128_xor(f, wasm_v128_and(val, vsignmask)); wasm_v128_store(dst, f); } } @@ -150,15 +147,15 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t const uint16_t QH = static_cast(ceil_int(static_cast(block->size.y), 2)); // Fused dequantize setup - int32_t pLSB_dq = 0; - v128_t vfscale_dq = wasm_f32x4_const_splat(0.0f); - v128_t vmagmask_dq = wasm_i32x4_const_splat(0); + int32_t pLSB_dq = 0; + v128_t vfscale_dq = wasm_f32x4_const_splat(0.0f); + v128_t vmagmask_dq = wasm_i32x4_const_splat(0); v128_t vsignmask_dq = wasm_i32x4_const_splat(0); if constexpr (fuse_dequant) { const int32_t M_b_val = block->get_Mb(); pLSB_dq = 31 - M_b_val; - vmagmask_dq = wasm_i32x4_const_splat(0x7FFFFFFF); - vsignmask_dq = wasm_i32x4_const_splat((int32_t)0x80000000u); + vmagmask_dq = wasm_i32x4_const_splat(0x7FFFFFFF); + vsignmask_dq = wasm_i32x4_const_splat((int32_t)0x80000000u); if (block->transformation != 1) { float fscale_direct = block->stepsize; fscale_direct *= static_cast(1 << FRACBITS); @@ -196,10 +193,10 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t alignas(32) uint32_t rholine[516]; // QW_max + 4, QW_max = 512 std::memset(rholine, 0, (QW + 4U) * sizeof(uint32_t)); - uint32_t *rho_p = rholine + 1; - alignas(32) int32_t Eline[1032]; // 2 * QW_max + 8, QW_max = 512 + uint32_t *rho_p = rholine + 1; + alignas(32) int32_t Eline[1032]; // 2 * QW_max + 8, QW_max = 512 std::memset(Eline, 0, (2U * QW + 8U) * sizeof(int32_t)); - int32_t *E_p = Eline + 1; + int32_t *E_p = Eline + 1; uint32_t context = 0; uint32_t vlcval; @@ -283,17 +280,19 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t const uint8_t pLSB_adj = static_cast(pLSB - 16); const v128_t zero16 = wasm_i32x4_const_splat(0); v128_t vn_16 = zero16; - v128_t row16 = MagSgn.decode_two_quads_16bit_wasm( - tv0, tv1, static_cast(U0), static_cast(U1), pLSB_adj, vn_16); + v128_t row16 = MagSgn.decode_two_quads_16bit_wasm(tv0, tv1, static_cast(U0), + static_cast(U1), pLSB_adj, vn_16); // Deinterleave: even lanes → row0, odd lanes → row1 v128_t row0_16 = wasm_i16x8_shuffle(row16, zero16, 0, 2, 4, 6, 8, 8, 8, 8); v128_t row1_16 = wasm_i16x8_shuffle(row16, zero16, 1, 3, 5, 7, 8, 8, 8, 8); // Expand int16 → int32 in sign-magnitude format (sign at bit 31) - v128_t mu0_32 = wasm_i32x4_shl(wasm_i32x4_extend_low_i16x8(row0_16), 16); - v128_t mu1_32 = wasm_i32x4_shl(wasm_i32x4_extend_low_i16x8(row1_16), 16); + v128_t mu0_32 = wasm_i32x4_shl(wasm_i32x4_extend_low_i16x8(row0_16), 16); + v128_t mu1_32 = wasm_i32x4_shl(wasm_i32x4_extend_low_i16x8(row1_16), 16); if constexpr (fuse_dequant) { - dequant_store_wasm(mp0, mu0_32, block->transformation, pLSB_dq, vfscale_dq, vmagmask_dq, vsignmask_dq); - dequant_store_wasm(mp1, mu1_32, block->transformation, pLSB_dq, vfscale_dq, vmagmask_dq, vsignmask_dq); + dequant_store_wasm(mp0, mu0_32, block->transformation, pLSB_dq, vfscale_dq, vmagmask_dq, + vsignmask_dq); + dequant_store_wasm(mp1, mu1_32, block->transformation, pLSB_dq, vfscale_dq, vmagmask_dq, + vsignmask_dq); } else { wasm_v128_store(mp0, mu0_32); wasm_v128_store(mp1, mu1_32); @@ -309,26 +308,25 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t // 32-bit path v128_t vmask1, sig0, sig1, vtmp, m_n_0, m_n_1, msvec, v_n_0, v_n_1, mu0, mu1; - sig0 = wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)rho0), vm), wasm_i32x4_const_splat(0)); - vtmp = wasm_v128_and(wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)emb_k_0), vm), - wasm_i32x4_const_splat(0)), - vone); + sig0 = wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)rho0), vm), wasm_i32x4_const_splat(0)); + vtmp = wasm_v128_and( + wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)emb_k_0), vm), wasm_i32x4_const_splat(0)), + vone); m_n_0 = wasm_i32x4_sub(wasm_v128_and(sig0, wasm_i32x4_splat((int32_t)U0)), vtmp); sig1 = wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)rho1), vm), wasm_i32x4_const_splat(0)); - vtmp = wasm_v128_and(wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)emb_k_1), vm), - wasm_i32x4_const_splat(0)), - vone); + vtmp = wasm_v128_and( + wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)emb_k_1), vm), wasm_i32x4_const_splat(0)), + vone); m_n_1 = wasm_i32x4_sub(wasm_v128_and(sig1, wasm_i32x4_splat((int32_t)U1)), vtmp); - vmask1 = wasm_i32x4_make((1 << wasm_i32x4_extract_lane(m_n_0, 0)) - 1, - (1 << wasm_i32x4_extract_lane(m_n_0, 1)) - 1, - (1 << wasm_i32x4_extract_lane(m_n_0, 2)) - 1, - (1 << wasm_i32x4_extract_lane(m_n_0, 3)) - 1); + vmask1 = wasm_i32x4_make( + (1 << wasm_i32x4_extract_lane(m_n_0, 0)) - 1, (1 << wasm_i32x4_extract_lane(m_n_0, 1)) - 1, + (1 << wasm_i32x4_extract_lane(m_n_0, 2)) - 1, (1 << wasm_i32x4_extract_lane(m_n_0, 3)) - 1); msvec = MagSgn.fetch(m_n_0); v_n_0 = wasm_v128_and(msvec, vmask1); - vtmp = wasm_v128_and(wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)emb_1_0), vm), - wasm_i32x4_const_splat(0)), - vone); + vtmp = wasm_v128_and( + wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)emb_1_0), vm), wasm_i32x4_const_splat(0)), + vone); v_n_0 = wasm_v128_or(v_n_0, wasm_i32x4_shlv(vtmp, m_n_0)); mu0 = wasm_i32x4_add(v_n_0, vtwo); mu0 = wasm_v128_or(mu0, vone); @@ -336,15 +334,14 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t mu0 = wasm_v128_or(mu0, wasm_i32x4_shl(v_n_0, 31)); mu0 = wasm_v128_and(mu0, sig0); - vmask1 = wasm_i32x4_make((1 << wasm_i32x4_extract_lane(m_n_1, 0)) - 1, - (1 << wasm_i32x4_extract_lane(m_n_1, 1)) - 1, - (1 << wasm_i32x4_extract_lane(m_n_1, 2)) - 1, - (1 << wasm_i32x4_extract_lane(m_n_1, 3)) - 1); + vmask1 = wasm_i32x4_make( + (1 << wasm_i32x4_extract_lane(m_n_1, 0)) - 1, (1 << wasm_i32x4_extract_lane(m_n_1, 1)) - 1, + (1 << wasm_i32x4_extract_lane(m_n_1, 2)) - 1, (1 << wasm_i32x4_extract_lane(m_n_1, 3)) - 1); msvec = MagSgn.fetch(m_n_1); v_n_1 = wasm_v128_and(msvec, vmask1); - vtmp = wasm_v128_and(wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)emb_1_1), vm), - wasm_i32x4_const_splat(0)), - vone); + vtmp = wasm_v128_and( + wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)emb_1_1), vm), wasm_i32x4_const_splat(0)), + vone); v_n_1 = wasm_v128_or(v_n_1, wasm_i32x4_shlv(vtmp, m_n_1)); mu1 = wasm_i32x4_add(v_n_1, vtwo); mu1 = wasm_v128_or(mu1, vone); @@ -353,10 +350,10 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t mu1 = wasm_v128_and(mu1, sig1); if constexpr (fuse_dequant) { - dequant_store_wasm(mp0, wasm_i32x4_shuffle(mu0, mu1, 0, 2, 4, 6), block->transformation, pLSB_dq, - vfscale_dq, vmagmask_dq, vsignmask_dq); - dequant_store_wasm(mp1, wasm_i32x4_shuffle(mu0, mu1, 1, 3, 5, 7), block->transformation, pLSB_dq, - vfscale_dq, vmagmask_dq, vsignmask_dq); + dequant_store_wasm(mp0, wasm_i32x4_shuffle(mu0, mu1, 0, 2, 4, 6), block->transformation, + pLSB_dq, vfscale_dq, vmagmask_dq, vsignmask_dq); + dequant_store_wasm(mp1, wasm_i32x4_shuffle(mu0, mu1, 1, 3, 5, 7), block->transformation, + pLSB_dq, vfscale_dq, vmagmask_dq, vsignmask_dq); } else { wasm_v128_store(mp0, wasm_i32x4_shuffle(mu0, mu1, 0, 2, 4, 6)); wasm_v128_store(mp1, wasm_i32x4_shuffle(mu0, mu1, 1, 3, 5, 7)); @@ -386,9 +383,9 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t mp0 = block->sample_buf + (row * 2U) * block->blksampl_stride; mp1 = block->sample_buf + (row * 2U + 1U) * block->blksampl_stride; } - sp0 = block->block_states + (row * 2U + 1U) * block->blkstate_stride + 1U; - sp1 = block->block_states + (row * 2U + 2U) * block->blkstate_stride + 1U; - rho1 = 0; + sp0 = block->block_states + (row * 2U + 1U) * block->blkstate_stride + 1U; + sp1 = block->block_states + (row * 2U + 2U) * block->blkstate_stride + 1U; + rho1 = 0; int32_t Emax0, Emax1; Emax0 = wasm_i32x4_reduce_max(wasm_v128_load(E_p - 1)); @@ -451,14 +448,14 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t vlcval = VLC_dec.advance((tv1 & 0x000F) >> 1); - u_off0 = tv0 & 1; - u_off1 = tv1 & 1; - uint32_t idx = (vlcval & 0x3F) + (u_off0 << 6U) + (u_off1 << 7U); + u_off0 = tv0 & 1; + u_off1 = tv1 & 1; + uint32_t idx = (vlcval & 0x3F) + (u_off0 << 6U) + (u_off1 << 7U); uint32_t uvlc_result = uvlc_dec_1[idx]; - vlcval = VLC_dec.advance(uvlc_result & 0x7); + vlcval = VLC_dec.advance(uvlc_result & 0x7); uvlc_result >>= 3; - uint32_t len = uvlc_result & 0xF; - uint32_t tmp = vlcval & ((1U << len) - 1U); + uint32_t len = uvlc_result & 0xF; + uint32_t tmp = vlcval & ((1U << len) - 1U); VLC_dec.advance(len); uvlc_result >>= 4; len = uvlc_result & 0x7; @@ -479,15 +476,17 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t const uint8_t pLSB_adj = static_cast(pLSB - 16); const v128_t zero16 = wasm_i32x4_const_splat(0); v128_t vn_16 = zero16; - v128_t row16 = MagSgn.decode_two_quads_16bit_wasm( - tv0, tv1, static_cast(U0), static_cast(U1), pLSB_adj, vn_16); - v128_t row0_16 = wasm_i16x8_shuffle(row16, zero16, 0, 2, 4, 6, 8, 8, 8, 8); - v128_t row1_16 = wasm_i16x8_shuffle(row16, zero16, 1, 3, 5, 7, 8, 8, 8, 8); - v128_t mu0_32 = wasm_i32x4_shl(wasm_i32x4_extend_low_i16x8(row0_16), 16); - v128_t mu1_32 = wasm_i32x4_shl(wasm_i32x4_extend_low_i16x8(row1_16), 16); + v128_t row16 = MagSgn.decode_two_quads_16bit_wasm(tv0, tv1, static_cast(U0), + static_cast(U1), pLSB_adj, vn_16); + v128_t row0_16 = wasm_i16x8_shuffle(row16, zero16, 0, 2, 4, 6, 8, 8, 8, 8); + v128_t row1_16 = wasm_i16x8_shuffle(row16, zero16, 1, 3, 5, 7, 8, 8, 8, 8); + v128_t mu0_32 = wasm_i32x4_shl(wasm_i32x4_extend_low_i16x8(row0_16), 16); + v128_t mu1_32 = wasm_i32x4_shl(wasm_i32x4_extend_low_i16x8(row1_16), 16); if constexpr (fuse_dequant) { - dequant_store_wasm(mp0, mu0_32, block->transformation, pLSB_dq, vfscale_dq, vmagmask_dq, vsignmask_dq); - dequant_store_wasm(mp1, mu1_32, block->transformation, pLSB_dq, vfscale_dq, vmagmask_dq, vsignmask_dq); + dequant_store_wasm(mp0, mu0_32, block->transformation, pLSB_dq, vfscale_dq, + vmagmask_dq, vsignmask_dq); + dequant_store_wasm(mp1, mu1_32, block->transformation, pLSB_dq, vfscale_dq, + vmagmask_dq, vsignmask_dq); } else { wasm_v128_store(mp0, mu0_32); wasm_v128_store(mp1, mu1_32); @@ -505,26 +504,25 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t // 32-bit path v128_t vmask1, sig0, sig1, vtmp, m_n_0, m_n_1, msvec, v_n_0, v_n_1, mu0, mu1; - sig0 = wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)rho0), vm), wasm_i32x4_const_splat(0)); - vtmp = wasm_v128_and(wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)emb_k_0), vm), - wasm_i32x4_const_splat(0)), - vone); + sig0 = wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)rho0), vm), wasm_i32x4_const_splat(0)); + vtmp = wasm_v128_and( + wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)emb_k_0), vm), wasm_i32x4_const_splat(0)), + vone); m_n_0 = wasm_i32x4_sub(wasm_v128_and(sig0, wasm_i32x4_splat((int32_t)U0)), vtmp); - sig1 = wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)rho1), vm), wasm_i32x4_const_splat(0)); - vtmp = wasm_v128_and(wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)emb_k_1), vm), - wasm_i32x4_const_splat(0)), - vone); + sig1 = wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)rho1), vm), wasm_i32x4_const_splat(0)); + vtmp = wasm_v128_and( + wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)emb_k_1), vm), wasm_i32x4_const_splat(0)), + vone); m_n_1 = wasm_i32x4_sub(wasm_v128_and(sig1, wasm_i32x4_splat((int32_t)U1)), vtmp); - vmask1 = wasm_i32x4_make((1 << wasm_i32x4_extract_lane(m_n_0, 0)) - 1, - (1 << wasm_i32x4_extract_lane(m_n_0, 1)) - 1, - (1 << wasm_i32x4_extract_lane(m_n_0, 2)) - 1, - (1 << wasm_i32x4_extract_lane(m_n_0, 3)) - 1); + vmask1 = wasm_i32x4_make( + (1 << wasm_i32x4_extract_lane(m_n_0, 0)) - 1, (1 << wasm_i32x4_extract_lane(m_n_0, 1)) - 1, + (1 << wasm_i32x4_extract_lane(m_n_0, 2)) - 1, (1 << wasm_i32x4_extract_lane(m_n_0, 3)) - 1); msvec = MagSgn.fetch(m_n_0); v_n_0 = wasm_v128_and(msvec, vmask1); - vtmp = wasm_v128_and(wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)emb_1_0), vm), - wasm_i32x4_const_splat(0)), - vone); + vtmp = wasm_v128_and( + wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)emb_1_0), vm), wasm_i32x4_const_splat(0)), + vone); v_n_0 = wasm_v128_or(v_n_0, wasm_i32x4_shlv(vtmp, m_n_0)); mu0 = wasm_i32x4_add(v_n_0, vtwo); mu0 = wasm_v128_or(mu0, vone); @@ -532,15 +530,14 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t mu0 = wasm_v128_or(mu0, wasm_i32x4_shl(v_n_0, 31)); mu0 = wasm_v128_and(mu0, sig0); - vmask1 = wasm_i32x4_make((1 << wasm_i32x4_extract_lane(m_n_1, 0)) - 1, - (1 << wasm_i32x4_extract_lane(m_n_1, 1)) - 1, - (1 << wasm_i32x4_extract_lane(m_n_1, 2)) - 1, - (1 << wasm_i32x4_extract_lane(m_n_1, 3)) - 1); + vmask1 = wasm_i32x4_make( + (1 << wasm_i32x4_extract_lane(m_n_1, 0)) - 1, (1 << wasm_i32x4_extract_lane(m_n_1, 1)) - 1, + (1 << wasm_i32x4_extract_lane(m_n_1, 2)) - 1, (1 << wasm_i32x4_extract_lane(m_n_1, 3)) - 1); msvec = MagSgn.fetch(m_n_1); v_n_1 = wasm_v128_and(msvec, vmask1); - vtmp = wasm_v128_and(wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)emb_1_1), vm), - wasm_i32x4_const_splat(0)), - vone); + vtmp = wasm_v128_and( + wasm_i32x4_ne(wasm_v128_and(wasm_i32x4_splat((int32_t)emb_1_1), vm), wasm_i32x4_const_splat(0)), + vone); v_n_1 = wasm_v128_or(v_n_1, wasm_i32x4_shlv(vtmp, m_n_1)); mu1 = wasm_i32x4_add(v_n_1, vtwo); mu1 = wasm_v128_or(mu1, vone); @@ -549,10 +546,12 @@ void ht_cleanup_decode(j2k_codeblock *block, const uint8_t &pLSB, const int32_t mu1 = wasm_v128_and(mu1, sig1); if constexpr (fuse_dequant) { - dequant_store_wasm(mp0, wasm_i32x4_shuffle(mu0, mu1, 0, 2, 4, 6), block->transformation, - pLSB_dq, vfscale_dq, vmagmask_dq, vsignmask_dq); - dequant_store_wasm(mp1, wasm_i32x4_shuffle(mu0, mu1, 1, 3, 5, 7), block->transformation, - pLSB_dq, vfscale_dq, vmagmask_dq, vsignmask_dq); + dequant_store_wasm(mp0, wasm_i32x4_shuffle(mu0, mu1, 0, 2, 4, 6), + block->transformation, pLSB_dq, vfscale_dq, vmagmask_dq, + vsignmask_dq); + dequant_store_wasm(mp1, wasm_i32x4_shuffle(mu0, mu1, 1, 3, 5, 7), + block->transformation, pLSB_dq, vfscale_dq, vmagmask_dq, + vsignmask_dq); } else { wasm_v128_store(mp0, wasm_i32x4_shuffle(mu0, mu1, 0, 2, 4, 6)); wasm_v128_store(mp1, wasm_i32x4_shuffle(mu0, mu1, 1, 3, 5, 7)); @@ -575,12 +574,12 @@ void ht_sigprop_decode(j2k_codeblock *block, uint8_t *HT_magref_segment, uint32_ const uint8_t &pLSB, uint16_t *sigma, uint32_t mstr) { if (pLSB == 0) return; // no plane below the LSB; mirrors ht_magref_decode (avoids 1 << (pLSB-1) UB) SP_dec SigProp(HT_magref_segment, magref_length); - const uint32_t height = block->size.y; - const uint32_t width = block->size.x; - const size_t sstride = block->blksampl_stride; - int32_t *samples = block->sample_buf; - const bool non_causal = (block->Cmodes & CAUSAL) == 0; - const int32_t spp_mask = 3 << (pLSB - 1); + const uint32_t height = block->size.y; + const uint32_t width = block->size.x; + const size_t sstride = block->blksampl_stride; + int32_t *samples = block->sample_buf; + const bool non_causal = (block->Cmodes & CAUSAL) == 0; + const int32_t spp_mask = 3 << (pLSB - 1); uint16_t prev_row_sig[264]; memset(prev_row_sig, 0, sizeof(prev_row_sig)); @@ -620,7 +619,7 @@ void ht_sigprop_decode(j2k_codeblock *block, uint8_t *HT_magref_segment, uint32_ uint32_t new_sig = 0; if (mbr) { static const uint32_t row_masks[4] = {0x33u, 0x76u, 0xECu, 0xC8u}; - uint32_t inv_sig = ~cs & pat; + uint32_t inv_sig = ~cs & pat; while (mbr) { uint32_t pos = static_cast(openhtj2k_ctz32(mbr)); uint32_t smask = 1u << pos; @@ -671,7 +670,7 @@ void ht_magref_decode(j2k_codeblock *block, uint8_t *HT_magref_segment, uint32_t int32_t *samples = block->sample_buf; for (uint32_t y = 0; y < height; y += 4) { - const uint16_t *csig = sigma + (y >> 2) * mstr; + const uint16_t *csig = sigma + (y >> 2) * mstr; for (uint32_t x = 0; x < width; x += 4) { uint16_t sig = csig[x >> 2]; @@ -695,8 +694,8 @@ void ht_magref_decode(j2k_codeblock *block, uint8_t *HT_magref_segment, uint32_t void j2k_codeblock::dequantize(uint8_t ROIshift) const { const int32_t pLSB = 31 - M_b; - const uint32_t mask = UINT32_MAX >> (M_b + 1); - const v128_t vmask = wasm_i32x4_splat(static_cast(~mask)); + const uint32_t mask = UINT32_MAX >> (M_b + 1); + const v128_t vmask = wasm_i32x4_splat(static_cast(~mask)); const v128_t vROIshift = wasm_i32x4_splat(ROIshift); v128_t v0, v1, s0, s1, vROImask, vmagmask, vdst0, vdst1, vpLSB; @@ -789,7 +788,7 @@ void j2k_codeblock::dequantize(uint8_t ROIshift) const { v128_t f0 = wasm_f32x4_mul(wasm_f32x4_convert_i32x4(m0), vfscale); v128_t f1 = wasm_f32x4_mul(wasm_f32x4_convert_i32x4(m1), vfscale); // apply sign via XOR of sign bit into float sign bit - wasm_v128_store(dst, wasm_v128_xor(f0, wasm_v128_and(v0, vsignmask))); + wasm_v128_store(dst, wasm_v128_xor(f0, wasm_v128_and(v0, vsignmask))); wasm_v128_store(dst + 4, wasm_v128_xor(f1, wasm_v128_and(v1, vsignmask))); val += 8; dst += 8; @@ -829,11 +828,11 @@ void j2k_codeblock::dequantize(uint8_t ROIshift) const { vROImask = wasm_v128_and(vROImask, vROIshift); v1 = wasm_i32x4_vshl(v1, vROImask); // to prevent overflow, truncate to int16_t range - v0 = wasm_i32x4_shr(wasm_i32x4_add(v0, wasm_i32x4_const_splat(1 << 15)), 16); - v1 = wasm_i32x4_shr(wasm_i32x4_add(v1, wasm_i32x4_const_splat(1 << 15)), 16); + v0 = wasm_i32x4_shr(wasm_i32x4_add(v0, wasm_i32x4_const_splat(1 << 15)), 16); + v1 = wasm_i32x4_shr(wasm_i32x4_add(v1, wasm_i32x4_const_splat(1 << 15)), 16); // dequantization - v0 = wasm_i32x4_mul(v0, wasm_i32x4_splat(scale)); - v1 = wasm_i32x4_mul(v1, wasm_i32x4_splat(scale)); + v0 = wasm_i32x4_mul(v0, wasm_i32x4_splat(scale)); + v1 = wasm_i32x4_mul(v1, wasm_i32x4_splat(scale)); // downshift v0 = wasm_i32x4_shr(wasm_i32x4_add(v0, wasm_i32x4_const_splat(1 << (downshift - 1))), downshift); v1 = wasm_i32x4_shr(wasm_i32x4_add(v1, wasm_i32x4_const_splat(1 << (downshift - 1))), downshift); @@ -866,9 +865,9 @@ void j2k_codeblock::dequantize(uint8_t ROIshift) const { } bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { - uint8_t P0 = 0; - int32_t Lcup = 0; - uint32_t Lref = 0; + uint8_t P0 = 0; + int32_t Lcup = 0; + uint32_t Lref = 0; const uint8_t S_skip = 0; if (block->num_passes > 3) { @@ -907,7 +906,7 @@ bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { // past the array and smash the stack — guard the write and the // later `all_segments[0]` read. Reported by IM JUN SEO (KISIA) and // OH HAN GUEL (SANGMYUNG UNIVERSITY). - uint8_t all_segments[4]; + uint8_t all_segments[4]; uint32_t num_segments = 0; for (uint32_t i = 0; i < block->pass_length_count; i++) { if (block->pass_length[i] != 0) { @@ -934,10 +933,10 @@ bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { block->length); return false; } - Dcup = block->get_compressed_data(); - const auto Scup = static_cast((Dcup[Lcup - 1] << 4) + (Dcup[Lcup - 2] & 0x0F)); - Dcup[Lcup - 1] = 0xFF; - Dcup[Lcup - 2] |= 0x0F; + Dcup = block->get_compressed_data(); + const auto Scup = static_cast((Dcup[Lcup - 1] << 4) + (Dcup[Lcup - 2] & 0x0F)); + Dcup[Lcup - 1] = 0xFF; + Dcup[Lcup - 2] |= 0x0F; if (Scup < 2 || Scup > Lcup || Scup > 4079) { printf("WARNING: cleanup pass suffix length %d is invalid.\n", Scup); @@ -965,8 +964,7 @@ bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { // is not a multiple of 4, the overshoot corrupts adjacent blocks in parallel decode. // Also gate on even height: the kernel writes row-pairs unconditionally and odd // height overflows one row into the next block's region. - if (num_ht_passes == 1 && ROIshift == 0 && (block->size.x & 3) == 0 - && (block->size.y & 1u) == 0) { + if (num_ht_passes == 1 && ROIshift == 0 && (block->size.x & 3) == 0 && (block->size.y & 1u) == 0) { if (block->dequant_i32) ht_cleanup_decode(block, static_cast(30 - S_blk), Lcup, Pcup, Scup); else @@ -978,11 +976,11 @@ bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { ht_cleanup_decode(block, static_cast(30 - S_blk), Lcup, Pcup, Scup); // Pack block_states SIGMA bits into nibble-packed sigma array - const uint32_t qw = (block->size.x + 3) >> 2; - const uint32_t mstr = ((qw + 2) + 7u) & ~7u; + const uint32_t qw = (block->size.x + 3) >> 2; + const uint32_t mstr = ((qw + 2) + 7u) & ~7u; uint16_t sigma_buf[(17 + 1) * 24] = {}; - pack_sigma(block->block_states, block->blkstate_stride, block->size.x, block->size.y, - sigma_buf, mstr); + pack_sigma(block->block_states, block->blkstate_stride, block->size.x, block->size.y, sigma_buf, + mstr); ht_sigprop_decode(block, Dref, Lref, static_cast(30 - (S_blk + 1)), sigma_buf, mstr); if (num_ht_passes > 2) { @@ -998,4 +996,18 @@ bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { return true; } + +// Batched entry point (see block_decoding.hpp). The WASM decoder is a fused +// single-pass kernel with no separate step-1, so no N-way interleave is wired +// yet: plain per-block loop. +bool htj2k_decode_batch(j2k_codeblock *const *blocks, uint32_t n, uint8_t ROIshift, bool *results) { + bool all_ok = true; + for (uint32_t i = 0; i < n; ++i) { + results[i] = htj2k_decode(blocks[i], ROIshift); + all_ok &= results[i]; + } + return all_ok; +} + +const uint32_t htj2k_dec_batch_lanes = 1; #endif From 76f57e9b03872f7d999f4a943b175ee44102199a Mon Sep 17 00:00:00 2001 From: Osamu Watanabe Date: Thu, 2 Jul 2026 17:04:57 +0900 Subject: [PATCH 2/4] perf(decoder): 2-way lockstep step-1 for the AVX2 HT cleanup decoder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enable the N-way step-1 kernel at N=2 on AVX2 and wire the serial decode driver to feed it: - htj2k_decode_batch (AVX2) now walks the handed-in blocks and decodes pairs of consecutive codeblocks with ht_cleanup_step1_nway<2> whenever they share dimensions and pass-class and both setups validate; step-2 and the refinement passes stay per block in list order, so output is byte-identical. Setup is not idempotent (modDcup mutates the buffer), so every setup is consumed exactly once; on a mid-lockstep throw from malformed input the group is redone lane-by-lane at N=1 from the saved setups, reproducing per-block 1-way results and exceptions. - decode_strip_core's serial path forms greedy runs of equal-sized decodable HT codeblocks (up to htj2k_dec_batch_lanes) and provisions the grow-on-demand scratch buffers with per-lane offsets. ISAs without a batch kernel report lanes=1 and keep the per-block path. Whole-decode cycles vs main (Zen 5, -num_threads 1, min-of-15, same-moment A/B of separately built trees): 8r -7.7%, 16r -5.0%, 8i -6.6%, 8r_512x8 -2.1% (pairing rate ~50% by design), kdu_8mp multipass -0.6%. The split-only refactor accounts for +0.5% of that (within its ±1% gate); the interleave more than pays it back. Byte-identical output on all five fixtures at 1/2/4/6 threads; full ctest passes (463/463) on the AVX2 build. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01M9cKpzqghsLxJUu7kkyRAw --- source/core/coding/ht_block_decoding.cpp | 1 + source/core/coding/ht_block_decoding_avx2.cpp | 115 ++++++- source/core/coding/ht_block_decoding_neon.cpp | 1 + source/core/coding/ht_block_decoding_wasm.cpp | 1 + source/core/coding/subband_row_buf.cpp | 309 ++++++++++-------- 5 files changed, 279 insertions(+), 148 deletions(-) diff --git a/source/core/coding/ht_block_decoding.cpp b/source/core/coding/ht_block_decoding.cpp index f5d3dc49..968eaa9e 100644 --- a/source/core/coding/ht_block_decoding.cpp +++ b/source/core/coding/ht_block_decoding.cpp @@ -31,6 +31,7 @@ #include "coding_units.hpp" #include "dec_CxtVLC_tables.hpp" #include "ht_block_decoding.hpp" + #include "block_decoding.hpp" #include "coding_local.hpp" #include "utils.hpp" diff --git a/source/core/coding/ht_block_decoding_avx2.cpp b/source/core/coding/ht_block_decoding_avx2.cpp index e43bf6c2..efb5f3aa 100644 --- a/source/core/coding/ht_block_decoding_avx2.cpp +++ b/source/core/coding/ht_block_decoding_avx2.cpp @@ -30,6 +30,7 @@ #include "coding_units.hpp" #include "dec_CxtVLC_tables.hpp" #include "ht_block_decoding.hpp" + #include "block_decoding.hpp" #include "coding_local.hpp" #include "utils.hpp" @@ -1028,8 +1029,10 @@ static bool htj2k_dec_finish(j2k_codeblock *block, const ht_dec_setup &su, const return true; } -bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { - const ht_dec_setup su = htj2k_dec_setup(block); +// Decode one block from an already-computed setup. htj2k_dec_setup is NOT +// idempotent (modDcup mutates the compressed buffer, so a re-run reads a +// corrupted Scup) — every setup must be consumed by exactly one decode. +static bool htj2k_decode_su(j2k_codeblock *block, const ht_dec_setup &su, const uint8_t ROIshift) { if (!su.ok) return false; if (su.empty) return true; @@ -1054,16 +1057,110 @@ bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { return htj2k_dec_finish(block, su, ROIshift, scratch, sstr); } -// Batched entry point (see block_decoding.hpp). Phase 1: plain per-block -// loop; the N-way group formation lands with the batch kernel enablement. +bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { + return htj2k_decode_su(block, htj2k_dec_setup(block), ROIshift); +} + + // Number of codeblocks whose step-1 chains are decoded in lockstep. + // 2 is the Zen-class AVX2 sweet spot (N=4's step-1 working set starts to + // pressure L1d and GPRs); override at configure time to experiment. + #ifndef OPENHTJ2K_HT_DEC_BATCH_N + #define OPENHTJ2K_HT_DEC_BATCH_N 2 + #endif + +// Batched entry point (see block_decoding.hpp). Walks the block list and +// decodes OPENHTJ2K_HT_DEC_BATCH_N consecutive blocks with the N-way +// lockstep step-1 kernel whenever they share dimensions and pass-class and +// all validate; everything else falls back to the 1-way path. Output is +// byte-identical to per-block decoding (the lockstep kernel preserves each +// lane's operation sequence, and step-2 runs per block in list order). bool htj2k_decode_batch(j2k_codeblock *const *blocks, uint32_t n, uint8_t ROIshift, bool *results) { - bool all_ok = true; - for (uint32_t i = 0; i < n; ++i) { - results[i] = htj2k_decode(blocks[i], ROIshift); - all_ok &= results[i]; + constexpr uint32_t BN = OPENHTJ2K_HT_DEC_BATCH_N; + bool all_ok = true; + uint32_t i = 0; + while (i < n) { + // A group needs BN consecutive blocks of equal dimensions (⇒ shared + // QW/QH/sstr) — check the cheap key before running any setup. + bool key = (i + BN <= n); + for (uint32_t k = 1; key && k < BN; ++k) { + key = blocks[i + k]->size.x == blocks[i]->size.x && blocks[i + k]->size.y == blocks[i]->size.y; + } + if (!key) { + results[i] = htj2k_decode(blocks[i], ROIshift); + all_ok &= results[i]; + ++i; + continue; + } + + // Setup mutates each block's buffer (modDcup): from here on, every one + // of the BN setups is consumed below, batched or not. + ht_dec_setup su[BN]; + bool group = true; + for (uint32_t k = 0; k < BN; ++k) { + su[k] = htj2k_dec_setup(blocks[i + k]); + group &= su[k].ok && !su[k].empty; + } + // skip_sigma is a step-1 template parameter: lanes must share pass-class. + for (uint32_t k = 1; group && k < BN; ++k) { + group &= (su[k].num_ht_passes == 1) == (su[0].num_ht_passes == 1); + } + if (!group) { + for (uint32_t k = 0; k < BN; ++k) { + results[i + k] = htj2k_decode_su(blocks[i + k], su[k], ROIshift); + all_ok &= results[i + k]; + } + i += BN; + continue; + } + + const j2k_codeblock *b0 = blocks[i]; + const uint16_t QW = static_cast(ceil_int(static_cast(b0->size.x), 2)); + const uint16_t QH = static_cast(ceil_int(static_cast(b0->size.y), 2)); + const int32_t sstr = static_cast(((b0->size.x + 2) + 7u) & ~7u); // multiples of 8 + const bool skip_sigma = su[0].num_ht_passes == 1; + + uint16_t scratch[BN][8 * 513]; + ht_step1_lane ln[BN]; + for (uint32_t k = 0; k < BN; ++k) { + ln[k].Dcup = blocks[i + k]->get_compressed_data(); + ln[k].Lcup = su[k].Lcup; + ln[k].Scup = su[k].Scup; + ln[k].scratch = scratch[k]; + ln[k].block_states = blocks[i + k]->block_states; + ln[k].blkstate_stride = blocks[i + k]->blkstate_stride; + } + + bool step1_done = true; + try { + if (skip_sigma) { + ht_cleanup_step1_nway(ln, QW, QH, sstr); + } else { + ht_cleanup_step1_nway(ln, QW, QH, sstr); + } + } catch (...) { + // Malformed input: one lane's rev_buf underflowed mid-lockstep. Redo + // every lane 1-way from its saved setup so per-block results and + // exceptions match the non-batched path exactly (per-lane step-1 work + // is idempotent: scratch is fully rewritten, sigma stores are pure + // assignments). This path never runs on valid streams. + step1_done = false; + } + for (uint32_t k = 0; k < BN; ++k) { + if (!step1_done) { + ht_step1_lane l1 = ln[k]; + if (skip_sigma) { + ht_cleanup_step1_nway<1, true>(&l1, QW, QH, sstr); // may throw, like the 1-way path + } else { + ht_cleanup_step1_nway<1, false>(&l1, QW, QH, sstr); + } + } + results[i + k] = htj2k_dec_finish(blocks[i + k], su[k], ROIshift, scratch[k], sstr); + all_ok &= results[i + k]; + } + i += BN; } return all_ok; } -const uint32_t htj2k_dec_batch_lanes = 1; +const uint32_t htj2k_dec_batch_lanes = OPENHTJ2K_HT_DEC_BATCH_N; #endif \ No newline at end of file diff --git a/source/core/coding/ht_block_decoding_neon.cpp b/source/core/coding/ht_block_decoding_neon.cpp index 65efdf17..20b16972 100644 --- a/source/core/coding/ht_block_decoding_neon.cpp +++ b/source/core/coding/ht_block_decoding_neon.cpp @@ -30,6 +30,7 @@ #include "coding_units.hpp" #include "dec_CxtVLC_tables.hpp" #include "ht_block_decoding.hpp" + #include "block_decoding.hpp" #include "coding_local.hpp" #include "utils.hpp" diff --git a/source/core/coding/ht_block_decoding_wasm.cpp b/source/core/coding/ht_block_decoding_wasm.cpp index e4bb5df1..4f1aef50 100644 --- a/source/core/coding/ht_block_decoding_wasm.cpp +++ b/source/core/coding/ht_block_decoding_wasm.cpp @@ -30,6 +30,7 @@ #include "coding_units.hpp" #include "dec_CxtVLC_tables.hpp" #include "ht_block_decoding.hpp" + #include "block_decoding.hpp" #include "coding_local.hpp" #include "utils.hpp" #include diff --git a/source/core/coding/subband_row_buf.cpp b/source/core/coding/subband_row_buf.cpp index 6ba38673..e88f5707 100644 --- a/source/core/coding/subband_row_buf.cpp +++ b/source/core/coding/subband_row_buf.cpp @@ -78,30 +78,29 @@ static inline void spin_wait(std::atomic_int &cnt) { // ─── init / free ────────────────────────────────────────────────────────────── - -void j2k_subband_row_buf::init(j2k_resolution *resolution, uint8_t b_idx, - int32_t codeblock_height, uint8_t roi_shift, bool use_ring) { - res = resolution; - band_idx = b_idx; - ROIshift = roi_shift; - cb_h = codeblock_height; - strip_y0 = -1; - strip_y1 = -1; - ring_mode = use_ring; - ring_buf = nullptr; - ring_y0 = -1; +void j2k_subband_row_buf::init(j2k_resolution *resolution, uint8_t b_idx, int32_t codeblock_height, + uint8_t roi_shift, bool use_ring) { + res = resolution; + band_idx = b_idx; + ROIshift = roi_shift; + cb_h = codeblock_height; + strip_y0 = -1; + strip_y1 = -1; + ring_mode = use_ring; + ring_buf = nullptr; + ring_y0 = -1; #ifdef OPENHTJ2K_THREAD prefetch_buf = nullptr; combined_buf = nullptr; prefetch_y0 = -1; prefetch_y1 = -1; - par_spool = nullptr; - par_stpool = nullptr; - par_ctxpool = nullptr; - par_spool_cap = 0; - par_stpool_cap = 0; - par_ctxpool_cap = 0; + par_spool = nullptr; + par_stpool = nullptr; + par_ctxpool = nullptr; + par_spool_cap = 0; + par_stpool_cap = 0; + par_ctxpool_cap = 0; par_cnt.store(0, std::memory_order_relaxed); #endif @@ -122,9 +121,9 @@ void j2k_subband_row_buf::init(j2k_resolution *resolution, uint8_t b_idx, // combined_buf holds the base pointer (ring_buf/prefetch_buf swap on prefetch hit, // so ring_buf may become an interior pointer — always free combined_buf). sprec_t *combined = static_cast(aligned_mem_alloc(buf_bytes * 2, 32)); - combined_buf = combined; - ring_buf = combined; - prefetch_buf = combined + buf_floats; + combined_buf = combined; + ring_buf = combined; + prefetch_buf = combined + buf_floats; #else ring_buf = static_cast(aligned_mem_alloc(buf_bytes, 32)); #endif @@ -133,13 +132,13 @@ void j2k_subband_row_buf::init(j2k_resolution *resolution, uint8_t b_idx, // Pre-allocate scratch for a 64×64 codeblock (grow-on-demand). // 32-byte alignment allows _mm256_load_si256 in the dequantize hot path. - cb_sample_cap = static_cast(round_up(64, 8) * round_up(64, 8)); - cb_state_cap = static_cast((round_up(64, 8) + 2) * (round_up(64, 8) + 2)); + cb_sample_cap = static_cast(round_up(64, 8) * round_up(64, 8)); + cb_state_cap = static_cast((round_up(64, 8) + 2) * (round_up(64, 8) + 2)); // block_contexts worst-case per codeblock: 1024×4 shape ⇒ (8/4+2)×(1024+2)=4104 uint32_t. - cb_ctx_cap = 4104u; - cb_sample_buf = static_cast(aligned_mem_alloc(cb_sample_cap * sizeof(int32_t), 32)); - cb_state_buf = static_cast(aligned_mem_alloc(cb_state_cap, 16)); - cb_ctx_buf = static_cast(aligned_mem_alloc(cb_ctx_cap * sizeof(uint32_t), 32)); + cb_ctx_cap = 4104u; + cb_sample_buf = static_cast(aligned_mem_alloc(cb_sample_cap * sizeof(int32_t), 32)); + cb_state_buf = static_cast(aligned_mem_alloc(cb_state_cap, 16)); + cb_ctx_buf = static_cast(aligned_mem_alloc(cb_ctx_cap * sizeof(uint32_t), 32)); #ifdef OPENHTJ2K_THREAD // par_spool / par_stpool start at zero capacity; decode_strip_core will @@ -183,19 +182,33 @@ void j2k_subband_row_buf::free_resources() { // swapped with prefetch_buf and could be an interior pointer after prefetch hits. aligned_mem_free(combined_buf); combined_buf = ring_buf = prefetch_buf = nullptr; - ring_y0 = -1; - aligned_mem_free(par_spool); par_spool = nullptr; par_spool_cap = 0; - aligned_mem_free(par_stpool); par_stpool = nullptr; par_stpool_cap = 0; - aligned_mem_free(par_ctxpool); par_ctxpool = nullptr; par_ctxpool_cap = 0; + ring_y0 = -1; + aligned_mem_free(par_spool); + par_spool = nullptr; + par_spool_cap = 0; + aligned_mem_free(par_stpool); + par_stpool = nullptr; + par_stpool_cap = 0; + aligned_mem_free(par_ctxpool); + par_ctxpool = nullptr; + par_ctxpool_cap = 0; par_tasks.clear(); par_tasks.shrink_to_fit(); strip_cache_.clear(); strip_cache_.shrink_to_fit(); #endif - aligned_mem_free(ring_buf); ring_buf = nullptr; ring_y0 = -1; - aligned_mem_free(cb_sample_buf); cb_sample_buf = nullptr; cb_sample_cap = 0; - aligned_mem_free(cb_state_buf); cb_state_buf = nullptr; cb_state_cap = 0; - aligned_mem_free(cb_ctx_buf); cb_ctx_buf = nullptr; cb_ctx_cap = 0; + aligned_mem_free(ring_buf); + ring_buf = nullptr; + ring_y0 = -1; + aligned_mem_free(cb_sample_buf); + cb_sample_buf = nullptr; + cb_sample_cap = 0; + aligned_mem_free(cb_state_buf); + cb_state_buf = nullptr; + cb_state_cap = 0; + aligned_mem_free(cb_ctx_buf); + cb_ctx_buf = nullptr; + cb_ctx_cap = 0; strip_y0 = strip_y1 = -1; } @@ -208,7 +221,7 @@ void j2k_subband_row_buf::decode_strip_core(sprec_t *target_buf, int32_t y0, int const uint32_t np = res->npw * res->nph; // Precompute constant used to offset into ring/prefetch buffer. const int32_t sb_x0 = static_cast(sb->get_pos0().x); - const auto stride = static_cast(sb->stride); + const auto stride = static_cast(sb->stride); #ifdef OPENHTJ2K_THREAD { @@ -225,19 +238,18 @@ void j2k_subband_row_buf::decode_strip_core(sprec_t *target_buf, int32_t y0, int size_t total_s = 0, total_st = 0, total_ctx = 0; for (uint32_t p = 0; p < np; ++p) { - j2k_precinct *cp = res->access_precinct(p); + j2k_precinct *cp = res->access_precinct(p); j2k_precinct_subband *cpb = cp->access_pband(band_idx); - const uint32_t ncx = cpb->num_codeblock_x; - const uint32_t ncy = cpb->num_codeblock_y; + const uint32_t ncx = cpb->num_codeblock_x; + const uint32_t ncy = cpb->num_codeblock_y; if (ncx == 0 || ncy == 0) continue; // Skip precincts that don't overlap with [y0, y1). const int32_t cpb_y0_i = static_cast(cpb->get_pos0().y); const int32_t cpb_y1_i = static_cast(cpb->get_pos1().y); if (cpb_y1_i <= y0 || cpb_y0_i >= y1) continue; // Jump directly to the first potentially-overlapping codeblock row. - const uint32_t r0 = (y0 > cpb_y0_i) - ? static_cast((y0 - cpb_y0_i) / static_cast(cb_h)) - : 0u; + const uint32_t r0 = + (y0 > cpb_y0_i) ? static_cast((y0 - cpb_y0_i) / static_cast(cb_h)) : 0u; for (uint32_t r = r0; r < ncy; ++r) { j2k_codeblock *row_first = cpb->access_codeblock(r * ncx); if (static_cast(row_first->get_pos1().y) <= y0) continue; @@ -252,10 +264,9 @@ void j2k_subband_row_buf::decode_strip_core(sprec_t *target_buf, int32_t y0, int // Empty block: dequantize never runs, so zero its region in target_buf // explicitly (replaces the bulk ring_buf pre-zero in decode_strip). if (ring_mode && target_buf) { - const ptrdiff_t roff = - (static_cast(block->get_pos0().y) - y0) * stride; + const ptrdiff_t roff = (static_cast(block->get_pos0().y) - y0) * stride; const ptrdiff_t coff = static_cast(block->get_pos0().x) - sb_x0; - sprec_t *dst = target_buf + roff + coff; + sprec_t *dst = target_buf + roff + coff; for (uint32_t row = 0; row < block->size.y; row++) std::memset(dst + row * stride, 0, block->size.x * sizeof(sprec_t)); } @@ -270,14 +281,11 @@ void j2k_subband_row_buf::decode_strip_core(sprec_t *target_buf, int32_t y0, int bt.sample_off = total_s; bt.state_off = total_st; bt.ctx_off = total_ctx; - bt.row_off = (ring_mode && target_buf) - ? (static_cast(block->get_pos0().y) - y0) * stride - : 0; - bt.col_off = (ring_mode && target_buf) - ? static_cast(block->get_pos0().x) - sb_x0 - : 0; - total_s += static_cast(QWx2 * QHx2); - total_st += static_cast((QWx2 + 2) * (QHx2 + 2)); + bt.row_off = + (ring_mode && target_buf) ? (static_cast(block->get_pos0().y) - y0) * stride : 0; + bt.col_off = (ring_mode && target_buf) ? static_cast(block->get_pos0().x) - sb_x0 : 0; + total_s += static_cast(QWx2 * QHx2); + total_st += static_cast((QWx2 + 2) * (QHx2 + 2)); total_ctx += static_cast((QHx2 / 4 + 2) * (QWx2 + 2)); par_tasks.push_back(bt); } @@ -310,14 +318,13 @@ void j2k_subband_row_buf::decode_strip_core(sprec_t *target_buf, int32_t y0, int par_error.store(false, std::memory_order_relaxed); par_cnt.store(static_cast(par_tasks.size()), std::memory_order_relaxed); for (auto &bt : par_tasks) { - bt.block->sample_buf = par_spool + bt.sample_off; - bt.block->blksampl_stride = bt.QWx2; - bt.block->block_states = par_stpool + bt.state_off; - bt.block->blkstate_stride = bt.QWx2 + 2; - bt.block->block_contexts = par_ctxpool + bt.ctx_off; + bt.block->sample_buf = par_spool + bt.sample_off; + bt.block->blksampl_stride = bt.QWx2; + bt.block->block_states = par_stpool + bt.state_off; + bt.block->blkstate_stride = bt.QWx2 + 2; + bt.block->block_contexts = par_ctxpool + bt.ctx_off; bt.block->block_contexts_stride = bt.QWx2 + 2; - if (ring_mode && target_buf) - bt.block->band_buf = target_buf + bt.row_off + bt.col_off; + if (ring_mode && target_buf) bt.block->band_buf = target_buf + bt.row_off + bt.col_off; const bool is_ht = (bt.block->Cmodes & HT) >> 6; if (!is_ht) { std::memset(bt.block->sample_buf, 0, static_cast(bt.QWx2) * bt.QHx2 * sizeof(int32_t)); @@ -342,8 +349,7 @@ void j2k_subband_row_buf::decode_strip_core(sprec_t *target_buf, int32_t y0, int // rejects malformed input at a bounds/segment guard. Treat that // like a thrown error so the driver re-throws after the barrier // instead of silently emitting a garbage (or uninitialised) block. - if (!htj2k_decode(blk, ROIshift)) - par_error.store(true, std::memory_order_relaxed); + if (!htj2k_decode(blk, ROIshift)) par_error.store(true, std::memory_order_relaxed); } else { j2k_decode(blk, ROIshift); } @@ -369,10 +375,10 @@ void j2k_subband_row_buf::decode_strip_core(sprec_t *target_buf, int32_t y0, int // ── Serial path ──────────────────────────────────────────────────────────── for (uint32_t p = 0; p < np; ++p) { - j2k_precinct *cp = res->access_precinct(p); + j2k_precinct *cp = res->access_precinct(p); j2k_precinct_subband *cpb = cp->access_pband(band_idx); - const uint32_t ncx = cpb->num_codeblock_x; - const uint32_t ncy = cpb->num_codeblock_y; + const uint32_t ncx = cpb->num_codeblock_x; + const uint32_t ncy = cpb->num_codeblock_y; if (ncx == 0 || ncy == 0) continue; // Skip precincts that don't overlap with [y0, y1). @@ -380,15 +386,14 @@ void j2k_subband_row_buf::decode_strip_core(sprec_t *target_buf, int32_t y0, int const int32_t cpb_y1_i = static_cast(cpb->get_pos1().y); if (cpb_y1_i <= y0 || cpb_y0_i >= y1) continue; // Jump directly to the first potentially-overlapping codeblock row. - const uint32_t r0 = (y0 > cpb_y0_i) - ? static_cast((y0 - cpb_y0_i) / static_cast(cb_h)) - : 0u; + const uint32_t r0 = + (y0 > cpb_y0_i) ? static_cast((y0 - cpb_y0_i) / static_cast(cb_h)) : 0u; for (uint32_t r = r0; r < ncy; ++r) { j2k_codeblock *row_first = cpb->access_codeblock(r * ncx); if (static_cast(row_first->get_pos1().y) <= y0) continue; if (static_cast(row_first->get_pos0().y) >= y1) break; - for (uint32_t c = 0; c < ncx; ++c) { + for (uint32_t c = 0; c < ncx;) { j2k_codeblock *block = cpb->access_codeblock(r * ncx + c); // JPIP-masked codeblocks (num_passes > 0 but no compressed data) take @@ -396,76 +401,105 @@ void j2k_subband_row_buf::decode_strip_core(sprec_t *target_buf, int32_t y0, int if (!block->num_passes || block->get_compressed_data() == nullptr) { // Empty block: zero its region in target_buf (replaces bulk pre-zero). if (ring_mode && target_buf) { - const ptrdiff_t roff = - (static_cast(block->get_pos0().y) - y0) * stride; + const ptrdiff_t roff = (static_cast(block->get_pos0().y) - y0) * stride; const ptrdiff_t coff = static_cast(block->get_pos0().x) - sb_x0; - sprec_t *dst = target_buf + roff + coff; + sprec_t *dst = target_buf + roff + coff; for (uint32_t row = 0; row < block->size.y; row++) std::memset(dst + static_cast(row) * stride, 0, block->size.x * sizeof(sprec_t)); } + ++c; continue; } - const uint32_t QWx2 = round_up(block->size.x, 8U); - const uint32_t QHx2 = round_up(block->size.y, 8U); - const size_t need_s = static_cast(QWx2 * QHx2); - const size_t need_st = static_cast((QWx2 + 2) * (QHx2 + 2)); - const size_t need_ctx = static_cast((QHx2 / 4 + 2) * (QWx2 + 2)); + const bool is_ht = (block->Cmodes & HT) >> 6; - if (need_s > cb_sample_cap) { + // Greedy run formation for the batched HT decoder: extend over the + // following decodable HT codeblocks of identical dimensions (equal + // dims ⇒ equal scratch needs, so lanes can share the grow-on-demand + // buffers at fixed offsets). htj2k_dec_batch_lanes is 1 on ISAs + // without an N-way step-1 kernel, keeping this path per-block there. + uint32_t run = 1; + if (is_ht && htj2k_dec_batch_lanes > 1) { + const uint32_t max_run = htj2k_dec_batch_lanes < 8u ? htj2k_dec_batch_lanes : 8u; + while (run < max_run && c + run < ncx) { + j2k_codeblock *nb = cpb->access_codeblock(r * ncx + c + run); + if (!nb->num_passes || nb->get_compressed_data() == nullptr) break; + if (!((nb->Cmodes & HT) >> 6)) break; + if (nb->size.x != block->size.x || nb->size.y != block->size.y) break; + ++run; + } + } + + const uint32_t QWx2 = round_up(block->size.x, 8U); + const uint32_t QHx2 = round_up(block->size.y, 8U); + const size_t need_s = static_cast(QWx2 * QHx2); + const size_t need_st = static_cast((QWx2 + 2) * (QHx2 + 2)); + const size_t need_ctx = static_cast((QHx2 / 4 + 2) * (QWx2 + 2)); + + if (need_s * run > cb_sample_cap) { aligned_mem_free(cb_sample_buf); - cb_sample_buf = static_cast(aligned_mem_alloc(need_s * sizeof(int32_t), 32)); - cb_sample_cap = need_s; + cb_sample_buf = static_cast(aligned_mem_alloc(need_s * run * sizeof(int32_t), 32)); + cb_sample_cap = need_s * run; } - if (need_st > cb_state_cap) { + if (need_st * run > cb_state_cap) { aligned_mem_free(cb_state_buf); - cb_state_buf = static_cast(aligned_mem_alloc(need_st, 16)); - cb_state_cap = need_st; + cb_state_buf = static_cast(aligned_mem_alloc(need_st * run, 16)); + cb_state_cap = need_st * run; } - if (need_ctx > cb_ctx_cap) { + if (need_ctx * run > cb_ctx_cap) { aligned_mem_free(cb_ctx_buf); - cb_ctx_buf = static_cast(aligned_mem_alloc(need_ctx * sizeof(uint32_t), 32)); - cb_ctx_cap = need_ctx; + cb_ctx_buf = static_cast(aligned_mem_alloc(need_ctx * run * sizeof(uint32_t), 32)); + cb_ctx_cap = need_ctx * run; } - // ht_cleanup_decode writes every position before reading, so no - // pre-zeroing is needed for single-pass HT blocks. For EBCOT and - // multi-pass HT blocks, zero the necessary regions. - const bool is_ht = (block->Cmodes & HT) >> 6; - if (!is_ht) { - std::memset(cb_sample_buf, 0, need_s * sizeof(int32_t)); - std::memset(cb_state_buf, 0, need_st); - std::memset(cb_ctx_buf, 0, need_ctx * sizeof(uint32_t)); - } else if (block->num_passes > 1) { - std::memset(cb_state_buf, 0, need_st); - } + j2k_codeblock *grp[8]; // run ≤ htj2k_dec_batch_lanes ≤ 8 + for (uint32_t k = 0; k < run; ++k) { + j2k_codeblock *bk = cpb->access_codeblock(r * ncx + c + k); + grp[k] = bk; + + // ht_cleanup_decode writes every position before reading, so no + // pre-zeroing is needed for single-pass HT blocks. For EBCOT and + // multi-pass HT blocks, zero the necessary regions. + if (!is_ht) { + std::memset(cb_sample_buf, 0, need_s * sizeof(int32_t)); + std::memset(cb_state_buf, 0, need_st); + std::memset(cb_ctx_buf, 0, need_ctx * sizeof(uint32_t)); + } else if (bk->num_passes > 1) { + std::memset(cb_state_buf + k * need_st, 0, need_st); + } - block->sample_buf = cb_sample_buf; - block->blksampl_stride = QWx2; - block->block_states = cb_state_buf; - block->blkstate_stride = QWx2 + 2; - block->block_contexts = cb_ctx_buf; - block->block_contexts_stride = QWx2 + 2; - - if (ring_mode && target_buf != nullptr) { - const ptrdiff_t row_off = - (static_cast(block->get_pos0().y) - y0) * stride; - const ptrdiff_t col_off = static_cast(block->get_pos0().x) - sb_x0; - block->band_buf = target_buf + row_off + col_off; + bk->sample_buf = cb_sample_buf + k * need_s; + bk->blksampl_stride = QWx2; + bk->block_states = cb_state_buf + k * need_st; + bk->blkstate_stride = QWx2 + 2; + bk->block_contexts = cb_ctx_buf + k * need_ctx; + bk->block_contexts_stride = QWx2 + 2; + + if (ring_mode && target_buf != nullptr) { + const ptrdiff_t row_off = (static_cast(bk->get_pos0().y) - y0) * stride; + const ptrdiff_t col_off = static_cast(bk->get_pos0().x) - sb_x0; + bk->band_buf = target_buf + row_off + col_off; + } + + bk->dequant_i32 = this->dequant_i32; } - block->dequant_i32 = this->dequant_i32; - if ((block->Cmodes & HT) >> 6) { - // A false return means htj2k_decode rejected malformed input at a + if (is_ht) { + // A false result means htj2k_decode rejected malformed input at a // bounds/segment guard; fail fast like the threaded path (and like // j2k_decode, which throws) rather than continue with a garbage block. - if (!htj2k_decode(block, ROIshift)) { - printf("ERROR: HT code-block decoding reported failure — malformed input.\n"); - throw std::exception(); + bool results[8]; + htj2k_decode_batch(grp, run, ROIshift, results); + for (uint32_t k = 0; k < run; ++k) { + if (!results[k]) { + printf("ERROR: HT code-block decoding reported failure — malformed input.\n"); + throw std::exception(); + } } } else { j2k_decode(block, ROIshift); } + c += run; } } } @@ -529,24 +563,23 @@ void j2k_subband_row_buf::trigger_prefetch(int32_t next_y0) { const int32_t sb_y0 = static_cast(sb->get_pos0().y); const int32_t strip_idx = (prefetch_y0 - sb_y0) / cb_h; - StripCacheEntry *entry = (strip_idx >= 0 - && static_cast(strip_idx) < strip_cache_.size()) - ? &strip_cache_[static_cast(strip_idx)] - : nullptr; + StripCacheEntry *entry = (strip_idx >= 0 && static_cast(strip_idx) < strip_cache_.size()) + ? &strip_cache_[static_cast(strip_idx)] + : nullptr; // ── Cold miss: walk the tree once and capture the per-strip block list ── if (entry != nullptr && !entry->built) { - const uint32_t np = res->npw * res->nph; - const int32_t sb_x0 = static_cast(sb->get_pos0().x); - const auto stride_cold = static_cast(sb->stride); + const uint32_t np = res->npw * res->nph; + const int32_t sb_x0 = static_cast(sb->get_pos0().x); + const auto stride_cold = static_cast(sb->stride); entry->blocks.clear(); for (uint32_t p = 0; p < np; ++p) { - j2k_precinct *cp = res->access_precinct(p); + j2k_precinct *cp = res->access_precinct(p); j2k_precinct_subband *cpb = cp->access_pband(band_idx); - const uint32_t ncx = cpb->num_codeblock_x; - const uint32_t ncy = cpb->num_codeblock_y; + const uint32_t ncx = cpb->num_codeblock_x; + const uint32_t ncy = cpb->num_codeblock_y; if (ncx == 0 || ncy == 0) continue; const int32_t cpb_y0_i = static_cast(cpb->get_pos0().y); const int32_t cpb_y1_i = static_cast(cpb->get_pos1().y); @@ -591,8 +624,7 @@ void j2k_subband_row_buf::trigger_prefetch(int32_t next_y0) { if (prefetch_buf != nullptr) { sprec_t *dst = prefetch_buf + ce.row_off + ce.col_off; for (uint32_t row = 0; row < ce.size_y; ++row) { - std::memset(dst + static_cast(row) * stride, 0, - ce.size_x * sizeof(sprec_t)); + std::memset(dst + static_cast(row) * stride, 0, ce.size_x * sizeof(sprec_t)); } } return; @@ -606,8 +638,8 @@ void j2k_subband_row_buf::trigger_prefetch(int32_t next_y0) { pb.ctx_off = total_ctx; pb.row_off = ce.row_off; pb.col_off = ce.col_off; - total_s += static_cast(ce.QWx2 * ce.QHx2); - total_st += static_cast((ce.QWx2 + 2) * (ce.QHx2 + 2)); + total_s += static_cast(ce.QWx2 * ce.QHx2); + total_st += static_cast((ce.QWx2 + 2) * (ce.QHx2 + 2)); total_ctx += static_cast((ce.QHx2 / 4 + 2) * (ce.QWx2 + 2)); par_tasks.push_back(pb); }; @@ -618,13 +650,13 @@ void j2k_subband_row_buf::trigger_prefetch(int32_t next_y0) { // Fallback tree walk: populate a thread-local list then process it. static thread_local std::vector fb_blocks; fb_blocks.clear(); - const uint32_t np = res->npw * res->nph; - const int32_t sb_x0 = static_cast(sb->get_pos0().x); + const uint32_t np = res->npw * res->nph; + const int32_t sb_x0 = static_cast(sb->get_pos0().x); for (uint32_t p = 0; p < np; ++p) { - j2k_precinct *cp = res->access_precinct(p); + j2k_precinct *cp = res->access_precinct(p); j2k_precinct_subband *cpb = cp->access_pband(band_idx); - const uint32_t ncx = cpb->num_codeblock_x; - const uint32_t ncy = cpb->num_codeblock_y; + const uint32_t ncx = cpb->num_codeblock_x; + const uint32_t ncy = cpb->num_codeblock_y; if (ncx == 0 || ncy == 0) continue; const int32_t cpb_y0_i = static_cast(cpb->get_pos0().y); const int32_t cpb_y1_i = static_cast(cpb->get_pos1().y); @@ -689,8 +721,8 @@ void j2k_subband_row_buf::trigger_prefetch(int32_t next_y0) { pb.block->blkstate_stride = pb.QWx2 + 2; pb.block->block_contexts = par_ctxpool + pb.ctx_off; pb.block->block_contexts_stride = pb.QWx2 + 2; - pb.block->band_buf = pbuf + pb.row_off + pb.col_off; - const bool is_ht = (pb.block->Cmodes & HT) >> 6; + pb.block->band_buf = pbuf + pb.row_off + pb.col_off; + const bool is_ht = (pb.block->Cmodes & HT) >> 6; if (!is_ht) { std::memset(pb.block->sample_buf, 0, static_cast(pb.QWx2) * pb.QHx2 * sizeof(int32_t)); std::memset(pb.block->block_states, 0, static_cast(pb.QWx2 + 2) * (pb.QHx2 + 2)); @@ -710,8 +742,7 @@ void j2k_subband_row_buf::trigger_prefetch(int32_t next_y0) { if ((blk->Cmodes & HT) >> 6) { // A false return is htj2k_decode's non-throwing malformed-input signal; // record it like a thrown error so the prefetch-consume barrier re-throws. - if (!htj2k_decode(blk, ROIshift)) - par_error.store(true, std::memory_order_relaxed); + if (!htj2k_decode(blk, ROIshift)) par_error.store(true, std::memory_order_relaxed); } else { j2k_decode(blk, ROIshift); } @@ -745,7 +776,7 @@ const sprec_t *j2k_subband_row_buf::row_ptr(int32_t abs_row) { } std::swap(ring_buf, prefetch_buf); strip_y0 = ring_y0 = prefetch_y0; - strip_y1 = prefetch_y1; + strip_y1 = prefetch_y1; prefetch_y0 = prefetch_y1 = -1; trigger_prefetch(strip_y1); } else { From cacbfcf7a00a43589d79e7ce1572723fc766a31f Mon Sep 17 00:00:00 2001 From: Osamu Watanabe Date: Thu, 2 Jul 2026 17:07:17 +0900 Subject: [PATCH 3/4] perf(decoder): enable the 2-way lockstep step-1 kernel on the scalar build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirror the AVX2 batched entry in the scalar TU: htj2k_decode_batch now forms pairs of equal-dimension, equal-pass-class codeblocks and decodes their step-1 chains with ht_cleanup_step1_nway<2>, with the same setup-consumed-once and lane-redo-on-throw semantics. The scalar step-2 is untouched. This also repays the split cost the shared-kernel refactor put on the scalar build (+0.6..2.7% cycles): vs main, whole-decode is now 8r -3.8%, 16r -3.9%, 8i -3.9%, 8r_512x8 -2.7%, kdu_8mp -1.8% (same-moment A/B, min-of-15). Byte-identical output on all five fixtures; ctest matches main (the two security_threaded_decode_abort*_mt failures are pre-existing on AVX2-less builds — their fixtures rely on the AVX2 reader throwing). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01M9cKpzqghsLxJUu7kkyRAw --- source/core/coding/ht_block_decoding.cpp | 112 +++++++++++++++++++++-- 1 file changed, 103 insertions(+), 9 deletions(-) diff --git a/source/core/coding/ht_block_decoding.cpp b/source/core/coding/ht_block_decoding.cpp index 968eaa9e..3271e007 100644 --- a/source/core/coding/ht_block_decoding.cpp +++ b/source/core/coding/ht_block_decoding.cpp @@ -1097,8 +1097,10 @@ static bool htj2k_dec_finish(j2k_codeblock *block, const ht_dec_setup &su, const return true; } -bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { - const ht_dec_setup su = htj2k_dec_setup(block); +// Decode one block from an already-computed setup. htj2k_dec_setup is NOT +// idempotent (modDcup mutates the compressed buffer, so a re-run reads a +// corrupted Scup) — every setup must be consumed by exactly one decode. +static bool htj2k_decode_su(j2k_codeblock *block, const ht_dec_setup &su, const uint8_t ROIshift) { if (!su.ok) return false; if (su.empty) return true; @@ -1123,16 +1125,108 @@ bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { return htj2k_dec_finish(block, su, ROIshift, scratch, sstr); } -// Batched entry point (see block_decoding.hpp). The scalar build has no -// N-way step-1 kernel wired yet: plain per-block loop. +bool htj2k_decode(j2k_codeblock *block, const uint8_t ROIshift) { + return htj2k_decode_su(block, htj2k_dec_setup(block), ROIshift); +} + +// Number of codeblocks whose step-1 chains are decoded in lockstep. + #ifndef OPENHTJ2K_HT_DEC_BATCH_N + #define OPENHTJ2K_HT_DEC_BATCH_N 2 + #endif + +// Batched entry point (see block_decoding.hpp). Walks the block list and +// decodes OPENHTJ2K_HT_DEC_BATCH_N consecutive blocks with the N-way +// lockstep step-1 kernel whenever they share dimensions and pass-class and +// all validate; everything else falls back to the 1-way path. Output is +// byte-identical to per-block decoding (the lockstep kernel preserves each +// lane's operation sequence, and step-2 runs per block in list order). bool htj2k_decode_batch(j2k_codeblock *const *blocks, uint32_t n, uint8_t ROIshift, bool *results) { - bool all_ok = true; - for (uint32_t i = 0; i < n; ++i) { - results[i] = htj2k_decode(blocks[i], ROIshift); - all_ok &= results[i]; + constexpr uint32_t BN = OPENHTJ2K_HT_DEC_BATCH_N; + bool all_ok = true; + uint32_t i = 0; + while (i < n) { + // A group needs BN consecutive blocks of equal dimensions (⇒ shared + // QW/QH/sstr) — check the cheap key before running any setup. + bool key = (i + BN <= n); + for (uint32_t k = 1; key && k < BN; ++k) { + key = blocks[i + k]->size.x == blocks[i]->size.x && blocks[i + k]->size.y == blocks[i]->size.y; + } + if (!key) { + results[i] = htj2k_decode(blocks[i], ROIshift); + all_ok &= results[i]; + ++i; + continue; + } + + // Setup mutates each block's buffer (modDcup): from here on, every one + // of the BN setups is consumed below, batched or not. + ht_dec_setup su[BN]; + bool group = true; + for (uint32_t k = 0; k < BN; ++k) { + su[k] = htj2k_dec_setup(blocks[i + k]); + group &= su[k].ok && !su[k].empty; + } + // skip_sigma is a step-1 template parameter: lanes must share pass-class. + for (uint32_t k = 1; group && k < BN; ++k) { + group &= (su[k].num_ht_passes == 1) == (su[0].num_ht_passes == 1); + } + if (!group) { + for (uint32_t k = 0; k < BN; ++k) { + results[i + k] = htj2k_decode_su(blocks[i + k], su[k], ROIshift); + all_ok &= results[i + k]; + } + i += BN; + continue; + } + + const j2k_codeblock *b0 = blocks[i]; + const uint16_t QW = static_cast(ceil_int(static_cast(b0->size.x), 2)); + const uint16_t QH = static_cast(ceil_int(static_cast(b0->size.y), 2)); + const int32_t sstr = static_cast(((b0->size.x + 2) + 7u) & ~7u); // multiples of 8 + const bool skip_sigma = su[0].num_ht_passes == 1; + + uint16_t scratch[BN][8 * 513]; + ht_step1_lane ln[BN]; + for (uint32_t k = 0; k < BN; ++k) { + ln[k].Dcup = blocks[i + k]->get_compressed_data(); + ln[k].Lcup = su[k].Lcup; + ln[k].Scup = su[k].Scup; + ln[k].scratch = scratch[k]; + ln[k].block_states = blocks[i + k]->block_states; + ln[k].blkstate_stride = blocks[i + k]->blkstate_stride; + } + + bool step1_done = true; + try { + if (skip_sigma) { + ht_cleanup_step1_nway(ln, QW, QH, sstr); + } else { + ht_cleanup_step1_nway(ln, QW, QH, sstr); + } + } catch (...) { + // Malformed input: one lane's rev_buf underflowed mid-lockstep. Redo + // every lane 1-way from its saved setup so per-block results and + // exceptions match the non-batched path exactly (per-lane step-1 work + // is idempotent: scratch is fully rewritten, sigma stores are pure + // assignments). This path never runs on valid streams. + step1_done = false; + } + for (uint32_t k = 0; k < BN; ++k) { + if (!step1_done) { + ht_step1_lane l1 = ln[k]; + if (skip_sigma) { + ht_cleanup_step1_nway<1, true>(&l1, QW, QH, sstr); // may throw, like the 1-way path + } else { + ht_cleanup_step1_nway<1, false>(&l1, QW, QH, sstr); + } + } + results[i + k] = htj2k_dec_finish(blocks[i + k], su[k], ROIshift, scratch[k], sstr); + all_ok &= results[i + k]; + } + i += BN; } return all_ok; } -const uint32_t htj2k_dec_batch_lanes = 1; +const uint32_t htj2k_dec_batch_lanes = OPENHTJ2K_HT_DEC_BATCH_N; #endif \ No newline at end of file From 4378ac4004164819ca774a7c58ea3744f895acf2 Mon Sep 17 00:00:00 2001 From: Osamu Watanabe Date: Thu, 2 Jul 2026 17:14:32 +0900 Subject: [PATCH 4/4] perf(decoder): feed the MT strip and prefetch paths through the batched HT decoder Group adjacent equal-sized HT codeblock tasks (up to htj2k_dec_batch_lanes) in decode_strip_core's parallel path and trigger_prefetch, and push one thread-pool task per group calling htj2k_decode_batch. par_cnt now counts groups; each group task decrements it exactly once, so the spin_wait / drain_prefetch semantics (including the PR #445 prefetch-drain invariant) are unchanged. Per-block scratch offsets into the shared pools were already disjoint, so grouping only changes task granularity. Group descriptors are packed into a uint64 so the pushed closure stays within std::function's small-buffer optimisation. Wall clock vs main (-iter 8, min-of-12+): 8r -3.7/-2.8/-3.6% and 8i -9.6/-3.0/-4.9% at -num_threads 2/4/6. Byte-identical output at 1/2/4/6 threads; 463/463 ctest; ThreadSanitizer clean on both a 4-thread normal decode and the single-tile-reuse loop (region_decode_bench) at 4 and 6 threads. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01M9cKpzqghsLxJUu7kkyRAw --- source/core/coding/subband_row_buf.cpp | 128 +++++++++++++++---------- source/core/coding/subband_row_buf.hpp | 85 +++++++++------- 2 files changed, 127 insertions(+), 86 deletions(-) diff --git a/source/core/coding/subband_row_buf.cpp b/source/core/coding/subband_row_buf.cpp index e88f5707..ac76f272 100644 --- a/source/core/coding/subband_row_buf.cpp +++ b/source/core/coding/subband_row_buf.cpp @@ -76,6 +76,66 @@ static inline void spin_wait(std::atomic_int &cnt) { } #endif +#ifdef OPENHTJ2K_THREAD +// Fill par_groups from par_tasks: adjacent equal-sized HT tasks are grouped +// (up to htj2k_dec_batch_lanes, which is 1 on ISAs without an N-way step-1 +// kernel) so one pool task decodes the whole group with htj2k_decode_batch. +// Scratch offsets are per-block and already disjoint, so grouping only +// changes task granularity, not memory layout. +void j2k_subband_row_buf::build_par_groups() { + par_groups.clear(); + const size_t n = par_tasks.size(); + for (size_t idx = 0; idx < n;) { + uint32_t cnt = 1; + const j2k_codeblock *b0 = par_tasks[idx].block; + if (((b0->Cmodes & HT) >> 6) && htj2k_dec_batch_lanes > 1) { + const uint32_t max_run = htj2k_dec_batch_lanes < 8u ? htj2k_dec_batch_lanes : 8u; + while (cnt < max_run && idx + cnt < n) { + const j2k_codeblock *nb = par_tasks[idx + cnt].block; + if (!((nb->Cmodes & HT) >> 6)) break; + if (nb->size.x != b0->size.x || nb->size.y != b0->size.y) break; + ++cnt; + } + } + par_groups.push_back((static_cast(idx) << 8) | cnt); + idx += cnt; + } +} + +// Pool-task body shared by decode_strip_core's parallel path and +// trigger_prefetch: decode one group, record failures, decrement par_cnt. +void j2k_subband_row_buf::run_par_group(uint64_t packed) { + const size_t idx = static_cast(packed >> 8); + const uint32_t cnt = static_cast(packed & 0xFF); + j2k_codeblock *grp[8]; + bool results[8]; + for (uint32_t k = 0; k < cnt; ++k) { + grp[k] = par_tasks[idx + k].block; + grp[k]->dequant_i32 = this->dequant_i32; + } + try { + if ((grp[0]->Cmodes & HT) >> 6) { + // htj2k_decode returns false (rather than throwing) when it rejects + // malformed input at a bounds/segment guard. Treat that like a thrown + // error so the driver re-throws after the barrier instead of silently + // emitting a garbage (or uninitialised) block. + htj2k_decode_batch(grp, cnt, ROIshift, results); + for (uint32_t k = 0; k < cnt; ++k) { + if (!results[k]) par_error.store(true, std::memory_order_relaxed); + } + } else { + j2k_decode(grp[0], ROIshift); + } + } catch (...) { + // Never let a decode error escape the task: it would std::terminate the + // pool worker or unwind through try_run_one. Record it; the driver + // thread re-throws after the barrier that consumes this batch's output. + par_error.store(true, std::memory_order_relaxed); + } + par_cnt.fetch_sub(1, std::memory_order_release); +} +#endif + // ─── init / free ────────────────────────────────────────────────────────────── void j2k_subband_row_buf::init(j2k_resolution *resolution, uint8_t b_idx, int32_t codeblock_height, @@ -316,7 +376,6 @@ void j2k_subband_row_buf::decode_strip_core(sprec_t *target_buf, int32_t y0, int // multi-pass sigprop/magref read the block_states border written only by // the cleanup interior pass, leaving the border uninitialised). par_error.store(false, std::memory_order_relaxed); - par_cnt.store(static_cast(par_tasks.size()), std::memory_order_relaxed); for (auto &bt : par_tasks) { bt.block->sample_buf = par_spool + bt.sample_off; bt.block->blksampl_stride = bt.QWx2; @@ -335,32 +394,15 @@ void j2k_subband_row_buf::decode_strip_core(sprec_t *target_buf, int32_t y0, int std::memset(bt.block->block_states, 0, static_cast(bt.QWx2 + 2) * (bt.QHx2 + 2)); } } - // Batch-push all tasks under a single mutex lock + notify_all. - // Use [blk, this] (16 bytes) instead of [blk, roi, this] (≥24 bytes) so the - // closure fits within std::function's 16-byte small-buffer optimisation and - // avoids a heap allocation per task. - pool->push_batch(par_tasks, [this](const CblkTask &bt) { - auto *blk = bt.block; - return [blk, this]() { - blk->dequant_i32 = this->dequant_i32; - try { - if ((blk->Cmodes & HT) >> 6) { - // htj2k_decode returns false (rather than throwing) when it - // rejects malformed input at a bounds/segment guard. Treat that - // like a thrown error so the driver re-throws after the barrier - // instead of silently emitting a garbage (or uninitialised) block. - if (!htj2k_decode(blk, ROIshift)) par_error.store(true, std::memory_order_relaxed); - } else { - j2k_decode(blk, ROIshift); - } - } catch (...) { - // Never let a decode error escape the task: it would std::terminate - // the pool worker or unwind through try_run_one. Record it; the - // driver thread re-throws after spin_wait below. - par_error.store(true, std::memory_order_relaxed); - } - par_cnt.fetch_sub(1, std::memory_order_release); - }; + // Group adjacent equal-sized HT tasks for the batched decoder (one + // pool task per group; scratch offsets are already disjoint per block). + build_par_groups(); + par_cnt.store(static_cast(par_groups.size()), std::memory_order_relaxed); + // Batch-push all group tasks under a single mutex lock + notify_all. + // [this, packed] captures 16 bytes total — fits std::function's + // small-buffer optimisation, no heap allocation per task. + pool->push_batch(par_groups, [this](const uint64_t &packed) { + return [this, packed]() { run_par_group(packed); }; }); spin_wait(par_cnt); if (par_error.load(std::memory_order_relaxed)) { @@ -708,7 +750,6 @@ void j2k_subband_row_buf::trigger_prefetch(int32_t next_y0) { } par_error.store(false, std::memory_order_relaxed); - par_cnt.store(static_cast(par_tasks.size()), std::memory_order_relaxed); // Setup pass: assign scratch/output pointers and selectively zero buffers. // HT single-pass blocks need no pre-zeroing (ht_cleanup_decode writes all @@ -732,28 +773,15 @@ void j2k_subband_row_buf::trigger_prefetch(int32_t next_y0) { std::memset(pb.block->block_states, 0, static_cast(pb.QWx2 + 2) * (pb.QHx2 + 2)); } } - // Batch-push all tasks under a single mutex lock + notify_all. - // [blk, this] captures 16 bytes total — fits std::function's SBO, no heap alloc. - pool->push_batch(par_tasks, [this](const CblkTask &pb) { - auto *blk = pb.block; - return [blk, this]() { - blk->dequant_i32 = this->dequant_i32; - try { - if ((blk->Cmodes & HT) >> 6) { - // A false return is htj2k_decode's non-throwing malformed-input signal; - // record it like a thrown error so the prefetch-consume barrier re-throws. - if (!htj2k_decode(blk, ROIshift)) par_error.store(true, std::memory_order_relaxed); - } else { - j2k_decode(blk, ROIshift); - } - } catch (...) { - // See decode_strip_core: record decode errors instead of throwing out of - // the task; the prefetch-consume barrier (row_ptr) re-throws on the driver. - par_error.store(true, std::memory_order_relaxed); - } - par_cnt.fetch_sub(1, std::memory_order_release); - }; - }); + // Group adjacent equal-sized HT tasks for the batched decoder, then + // batch-push one task per group under a single mutex lock + notify_all. + // [this, packed] captures 16 bytes total — fits std::function's SBO, no + // heap alloc. Groups are counted in par_cnt and drained by spin_wait / + // drain_prefetch exactly as the per-block tasks were. + build_par_groups(); + par_cnt.store(static_cast(par_groups.size()), std::memory_order_relaxed); + pool->push_batch(par_groups, + [this](const uint64_t &packed) { return [this, packed]() { run_par_group(packed); }; }); } #endif diff --git a/source/core/coding/subband_row_buf.hpp b/source/core/coding/subband_row_buf.hpp index c5fed117..f504c5f1 100644 --- a/source/core/coding/subband_row_buf.hpp +++ b/source/core/coding/subband_row_buf.hpp @@ -54,30 +54,29 @@ // const sprec_t *p = rb.row_ptr(abs_row); // decode strip if needed // ───────────────────────────────────────────────────────────────────────────── struct j2k_subband_row_buf { - j2k_subband *sb; // geometry, i_samples, decode params - j2k_resolution *res; // to enumerate precincts - uint8_t band_idx; // index within resolution's subbands (0=HL,1=LH,2=HH) - uint8_t ROIshift; - bool dequant_i32 = false; - - int32_t cb_h; // codeblock height for this resolution (max across precincts) - int32_t strip_y0; // y-start of the currently-decoded codeblock strip (-1 = none) - int32_t strip_y1; // y-end of the currently-decoded codeblock strip (exclusive) + j2k_subband *sb; // geometry, i_samples, decode params + j2k_resolution *res; // to enumerate precincts + uint8_t band_idx; // index within resolution's subbands (0=HL,1=LH,2=HH) + uint8_t ROIshift; + bool dequant_i32 = false; + int32_t cb_h; // codeblock height for this resolution (max across precincts) + int32_t strip_y0; // y-start of the currently-decoded codeblock strip (-1 = none) + int32_t strip_y1; // y-end of the currently-decoded codeblock strip (exclusive) // Ring buffer for line-based mode. // When ring_mode=true, decoded samples go here instead of sb->i_samples. - bool ring_mode; // use ring buffer instead of sb->i_samples - sprec_t *ring_buf; // cb_h × sb->stride floats (one strip wide) - int32_t ring_y0; // first row of current strip in ring_buf (= strip_y0) + bool ring_mode; // use ring buffer instead of sb->i_samples + sprec_t *ring_buf; // cb_h × sb->stride floats (one strip wide) + int32_t ring_y0; // first row of current strip in ring_buf (= strip_y0) // Scratch buffers reused across codeblocks (serial decode; one block at a time). - int32_t *cb_sample_buf; - uint8_t *cb_state_buf; + int32_t *cb_sample_buf; + uint8_t *cb_state_buf; uint32_t *cb_ctx_buf; - size_t cb_sample_cap; // current capacity in elements - size_t cb_state_cap; - size_t cb_ctx_cap; // block_contexts capacity in uint32_t + size_t cb_sample_cap; // current capacity in elements + size_t cb_state_cap; + size_t cb_ctx_cap; // block_contexts capacity in uint32_t #ifdef OPENHTJ2K_THREAD // Double-buffer for strip prefetch: while IDWT consumes ring_buf (current strip), @@ -85,10 +84,10 @@ struct j2k_subband_row_buf { // ring_buf and prefetch_buf are the two halves of a single combined aligned allocation // (combined_buf). std::swap(ring_buf, prefetch_buf) happens on prefetch hit, so after // a swap ring_buf may point to the upper half. Always free combined_buf, not ring_buf. - sprec_t *prefetch_buf; // decode target for next strip (upper half of combined alloc) - sprec_t *combined_buf; // base pointer of the ring+prefetch combined allocation - int32_t prefetch_y0; // strip bounds of pending prefetch task (-1 = none) - int32_t prefetch_y1; + sprec_t *prefetch_buf; // decode target for next strip (upper half of combined alloc) + sprec_t *combined_buf; // base pointer of the ring+prefetch combined allocation + int32_t prefetch_y0; // strip bounds of pending prefetch task (-1 = none) + int32_t prefetch_y1; // Unified codeblock task descriptor used by both decode_strip_core() (parallel path) // and trigger_prefetch(). The two paths are mutually exclusive — decode_strip_core @@ -96,23 +95,29 @@ struct j2k_subband_row_buf { // only after that — so a single set of scratch resources serves both. struct CblkTask { j2k_codeblock *block; - uint32_t QWx2, QHx2; - size_t sample_off, state_off, ctx_off; - ptrdiff_t row_off, col_off; // ring/prefetch target offset; 0 in non-ring mode + uint32_t QWx2, QHx2; + size_t sample_off, state_off, ctx_off; + ptrdiff_t row_off, col_off; // ring/prefetch target offset; 0 in non-ring mode }; // Grow-only scratch pools (never freed until free_resources()). // Sized at init() from a per-subband strip pre-scan; only realloc'd on growth. - int32_t *par_spool; - uint8_t *par_stpool; + int32_t *par_spool; + uint8_t *par_stpool; uint32_t *par_ctxpool; - size_t par_spool_cap; - size_t par_stpool_cap; - size_t par_ctxpool_cap; + size_t par_spool_cap; + size_t par_stpool_cap; + size_t par_ctxpool_cap; // Task list pre-reserved to max codeblocks per strip (from init() pre-scan). std::vector par_tasks; + // Group descriptors for the batched HT decoder: each entry packs + // (index into par_tasks << 8) | block count, so a [this, packed] closure + // stays within std::function's 16-byte small-buffer optimisation. One + // pool task is pushed per group; par_cnt counts groups. + std::vector par_groups; + // In-flight counter shared by decode_strip_core and trigger_prefetch. // Replaces both the local 'remaining' atomic and the old shared_ptr prefetch_cnt. std::atomic par_cnt; @@ -140,21 +145,20 @@ struct j2k_subband_row_buf { // Indexed by strip_idx = (next_y0 - sb_y0) / cb_h. struct CachedBlock { j2k_codeblock *block; - uint32_t QWx2, QHx2; // round_up(size.x, 8), round_up(size.y, 8) - ptrdiff_t row_off, col_off; // strip-relative offsets into prefetch_buf - uint32_t size_x, size_y; // block->size, captured for empty memset + uint32_t QWx2, QHx2; // round_up(size.x, 8), round_up(size.y, 8) + ptrdiff_t row_off, col_off; // strip-relative offsets into prefetch_buf + uint32_t size_x, size_y; // block->size, captured for empty memset }; struct StripCacheEntry { - bool built = false; - std::vector blocks; + bool built = false; + std::vector blocks; }; std::vector strip_cache_; #endif // Initialise. cb_h is the maximum codeblock height for this resolution level. // When use_ring=true, allocates a ring buffer (cb_h rows) instead of using sb->i_samples. - void init(j2k_resolution *res, uint8_t band_idx, int32_t cb_h, uint8_t ROIshift, - bool use_ring = false); + void init(j2k_resolution *res, uint8_t band_idx, int32_t cb_h, uint8_t ROIshift, bool use_ring = false); // Release scratch buffers. void free_resources(); @@ -187,5 +191,14 @@ struct j2k_subband_row_buf { #ifdef OPENHTJ2K_THREAD // Submit a background task to decode the strip starting at next_y0 into prefetch_buf. void trigger_prefetch(int32_t next_y0); + + // Fill par_groups from par_tasks: adjacent equal-sized HT tasks are grouped + // (up to htj2k_dec_batch_lanes) for the batched decoder; everything else is + // a group of one. + void build_par_groups(); + + // Pool-task body: decode one par_groups entry (packed = (index << 8) | count), + // recording failures in par_error and decrementing par_cnt exactly once. + void run_par_group(uint64_t packed); #endif };