Swept the stack-heavy measurement across placements, and qualified its result (#638)

#636 reported that a stack in BTCM gave a threefold tighter spread than DRAM0
for stack-heavy work. That measurement used a single code placement, which is
the methodology #631 and #633 exist to correct: the cache benchmark got an
alignment sweep and the interrupt handler got one, and this measurement never
did. It was noticed when #637 added two threads to the same image and the figure
moved -- both spreads came out near 6500 and the minima rose 15%.

The recursive body is now generated at four placements and all four are
measured, per placement, in one image.

    placement    BTCM min / spread     DRAM0 min / spread
    offset 0      41854 / 6850          42036 / 6880
    offset 16     48670 / 1946          48792 / 1978
    offset 32     42388 / 6914          42752 / 7018
    offset 48     48914 / 1860          49198 / 6786

Reproducible across runs to within a few hundred cycles.

Spread is dominated by code placement rather than by the memory holding the
stack. It ranges from 1860 to 6914 depending on where the body falls in a cache
line, and placement also moves the minimum by 17%, from 41854 to 49214. Against
that, the memory contributes a consistent but small advantage: BTCM's minimum is
lower at all four placements, by 0.4% to 0.9%.

BTCM's spread beats DRAM0's decisively at one placement of the four, offset 48,
at 1860 against 6786. At the other three the two are within 2% of each other.
So the effect #636 reported is real where it occurs and is not a property of the
part: quoting it as one invited the reader to expect it everywhere.

#636's claim should be read as qualified by this. A stack in BTCM buys a small
consistent improvement in the best case and a large improvement in spread at
some code placements and not others. Anyone building a determinism argument on
it needs the placement sweep in the loop, not a single figure.

The pad nops that displace each placement execute on every recursion level
rather than once, so each placement carries a slightly different constant cost,
about 0.6% at the widest. That cancels in the BTCM against DRAM0 comparison,
which is made at the same placement, and does not affect spread within one.

Assisted-by: Claude Code (Opus 5) <noreply@anthropic.com>

Author: fdesbiens

ports/cortex_r52/gnu/example_build/s32z280_evb/demo_s32z280.c

@@ -289,6 +289,7 @@ static void ctx_partner_entry(ULONG which)
 #define DEEP_SAMPLES    64U
 #define DEEP_DEPTH      24U
 #define DEEP_FRAME      16U     /* words: 64 bytes, one cache line per frame */
+#define DEEP_PLACEMENTS 4U
 
 static TX_THREAD    deep_btcm_thread;
 static TX_THREAD    deep_dram_thread;
@@ -301,36 +302,65 @@ static unsigned char deep_btcm_stack[4096];
 static unsigned char deep_dram_stack[4096];
 
 static unsigned int deep_samples[CTX_PAIRS][DEEP_SAMPLES];
-static unsigned int deep_lo[CTX_PAIRS];
-static unsigned int deep_hi[CTX_PAIRS];
-static unsigned int deep_mean[CTX_PAIRS];
-static unsigned int deep_over[CTX_PAIRS];
+static unsigned int deep_lo[CTX_PAIRS][DEEP_PLACEMENTS];
+static unsigned int deep_hi[CTX_PAIRS][DEEP_PLACEMENTS];
+static unsigned int deep_mean[CTX_PAIRS][DEEP_PLACEMENTS];
+static unsigned int deep_over[CTX_PAIRS][DEEP_PLACEMENTS];
 static volatile unsigned int deep_done[CTX_PAIRS];
 
-__attribute__((noinline))
-static unsigned int deep_touch(unsigned int depth)
-{
-    volatile unsigned int   frame[DEEP_FRAME];
-    unsigned int            acc = 0U;
-    unsigned int            i;
-
-    for (i = 0U; i < DEEP_FRAME; i++)
-    {
-        frame[i] = depth + i;
-    }
-
-    if (depth > 0U)
-    {
-        acc = deep_touch(depth - 1U);
-    }
+/* Four placements of the recursive body, each 64-byte aligned and then
+   displaced within its line, and the measurement sweeps all four.
+
+   The first version of this measurement had one placement, which is the
+   methodology this example spent two changes correcting elsewhere.  It showed
+   BTCM with a threefold tighter spread than DRAM0; that did not survive an
+   image with two more threads in it, where both spreads came out near 6500 and
+   the minima moved 15%.  A single placement was reporting where the linker had
+   put the code.
+
+   The pad nops execute on every recursion level rather than once, so each
+   placement carries a slightly different constant cost -- about 0.6% at the
+   widest pad.  That cancels in the comparison that matters, because BTCM and
+   DRAM0 are measured at the same placement, and it does not affect the spread
+   within a placement.  */
+
+#define MAKE_DEEP_TOUCH(name, pad_words)                                      \
+__attribute__((aligned(64), noinline))                                        \
+static unsigned int name(unsigned int depth)                                   \
+{                                                                             \
+    volatile unsigned int   frame[DEEP_FRAME];                                \
+    unsigned int            acc = 0U;                                         \
+    unsigned int            i;                                                \
+                                                                              \
+    __asm__ volatile(".rept " #pad_words "\n\tnop\n\t.endr");                 \
+                                                                              \
+    for (i = 0U; i < DEEP_FRAME; i++)                                         \
+    {                                                                         \
+        frame[i] = depth + i;                                                 \
+    }                                                                         \
+                                                                              \
+    if (depth > 0U)                                                           \
+    {                                                                         \
+        acc = name(depth - 1U);                                               \
+    }                                                                         \
+                                                                              \
+    for (i = 0U; i < DEEP_FRAME; i++)                                         \
+    {                                                                         \
+        acc += frame[i];                                                      \
+    }                                                                         \
+                                                                              \
+    return acc;                                                               \
+}
 
-    for (i = 0U; i < DEEP_FRAME; i++)
-    {
-        acc += frame[i];
-    }
+MAKE_DEEP_TOUCH(deep_touch_a, 0)
+MAKE_DEEP_TOUCH(deep_touch_b, 4)
+MAKE_DEEP_TOUCH(deep_touch_c, 8)
+MAKE_DEEP_TOUCH(deep_touch_d, 12)
 
-    return acc;
-}
+static unsigned int (*const deep_touches[DEEP_PLACEMENTS])(unsigned int) =
+{
+    deep_touch_a, deep_touch_b, deep_touch_c, deep_touch_d
+};
 
 static void deep_entry(ULONG which)
 {
@@ -345,47 +375,67 @@ static void deep_entry(ULONG which)
     {
         unsigned int before;
         unsigned int after;
+        unsigned int slot = i / (DEEP_SAMPLES / DEEP_PLACEMENTS);
+
+        if (slot >= DEEP_PLACEMENTS)
+        {
+            slot = DEEP_PLACEMENTS - 1U;
+        }
 
         cache_clean_all();
         cache_invalidate_dcache_all();
 
         before = timer_read_cycles();
-        (void) deep_touch(DEEP_DEPTH);
+        (void) deep_touches[slot](DEEP_DEPTH);
         after  = timer_read_cycles();
 
         deep_samples[which][i] = after - before;
     }
 
-    /* Post-process: the minimum sets the scale, and anything past twice it was
-       interrupted rather than slow.  */
+    /* Post-processed per placement.  A figure spanning placements would mix the
+       four constant costs and hide exactly what this sweep exists to show.  */
 
-    lo = 0xFFFFFFFFU;
-    for (i = 0U; i < DEEP_SAMPLES; i++)
-    {
-        if (deep_samples[which][i] < lo) { lo = deep_samples[which][i]; }
-    }
-
-    hi = 0U; sum = 0UL; counted = 0U; over = 0U;
-    for (i = 0U; i < DEEP_SAMPLES; i++)
     {
-        unsigned int v = deep_samples[which][i];
+        unsigned int slot;
+        unsigned int each = DEEP_SAMPLES / DEEP_PLACEMENTS;
 
-        if (v > (lo * 2U))
+        for (slot = 0U; slot < DEEP_PLACEMENTS; slot++)
         {
-            over++;
-        }
-        else
-        {
-            if (v > hi) { hi = v; }
-            sum += (unsigned long) v;
-            counted++;
+            unsigned int    s;
+            unsigned int    lo = 0xFFFFFFFFU;
+            unsigned int    hi = 0U;
+            unsigned long   sum = 0UL;
+            unsigned int    counted = 0U;
+            unsigned int    over = 0U;
+
+            for (s = slot * each; s < ((slot + 1U) * each); s++)
+            {
+                if (deep_samples[which][s] < lo) { lo = deep_samples[which][s]; }
+            }
+
+            for (s = slot * each; s < ((slot + 1U) * each); s++)
+            {
+                unsigned int v = deep_samples[which][s];
+
+                if (v > (lo * 2U))
+                {
+                    over++;
+                }
+                else
+                {
+                    if (v > hi) { hi = v; }
+                    sum += (unsigned long) v;
+                    counted++;
+                }
+            }
+
+            deep_lo[which][slot]   = lo;
+            deep_hi[which][slot]   = hi;
+            deep_mean[which][slot] = (counted > 0U) ? (unsigned int) (sum / counted) : 0U;
+            deep_over[which][slot] = over;
         }
     }
 
-    deep_lo[which]   = lo;
-    deep_hi[which]   = hi;
-    deep_mean[which] = (counted > 0U) ? (unsigned int) (sum / counted) : 0U;
-    deep_over[which] = over;
     deep_done[which] = 1U;
 }
 
@@ -551,32 +601,38 @@ static void judge_entry(ULONG input)
         linflexd_puts("\n");
     }
 
-    linflexd_puts("stack-heavy work, cycles (24 frames, cold cache)\n");
+    linflexd_puts("stack-heavy work, cycles (24 frames, cold cache, 4 placements)\n");
     {
-        static const char *const dwhere[CTX_PAIRS] = { "stack in BTCM ",
-                                                       "stack in DRAM0" };
+        static const char *const dwhere[CTX_PAIRS] = { "BTCM ", "DRAM0" };
         unsigned int q;
+        unsigned int slot;
 
         for (q = 0U; q < CTX_PAIRS; q++)
         {
-            linflexd_puts("  ");
-            linflexd_puts(dwhere[q]);
             if (deep_done[q] == 0U)
             {
+                linflexd_puts("  ");
+                linflexd_puts(dwhere[q]);
                 linflexd_puts(": did not finish\n");
+                continue;
             }
-            else
+
+            for (slot = 0U; slot < DEEP_PLACEMENTS; slot++)
             {
+                linflexd_puts("  ");
+                linflexd_puts(dwhere[q]);
+                linflexd_puts(" offset ");
+                demo_dec((unsigned long) (slot * 16U));
                 linflexd_puts(": min ");
-                demo_dec(deep_lo[q]);
+                demo_dec(deep_lo[q][slot]);
                 linflexd_puts("  mean ");
-                demo_dec(deep_mean[q]);
+                demo_dec(deep_mean[q][slot]);
                 linflexd_puts("  max ");
-                demo_dec(deep_hi[q]);
-                linflexd_puts("  interrupted ");
-                demo_dec(deep_over[q]);
-                linflexd_puts(" of ");
-                demo_dec(DEEP_SAMPLES);
+                demo_dec(deep_hi[q][slot]);
+                linflexd_puts("  spread ");
+                demo_dec(deep_hi[q][slot] - deep_lo[q][slot]);
+                linflexd_puts("  cut ");
+                demo_dec(deep_over[q][slot]);
                 linflexd_puts("\n");
             }
         }