diff --git a/.gitignore b/.gitignore index c24d75f61b1..474a2f3ed7e 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,10 @@ __pycache__ # Backups + local notes *.bak keywords + +# research/cosmic-web +research/cosmic-web/.venv/ +research/cosmic-web/artifacts/ +.claude/launch.json +research/cosmic-web/data/ +research/cosmic-web/paper/main.pdf diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000000..c061a623c12 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,10 @@ +cff-version: 1.2.0 +title: "The correction beyond Zel'dovich is transverse: code and experiments" +message: "If you use this code or the experiment suite, please cite it." +type: software +authors: + - family-names: Moiseev + given-names: Igor +repository-code: "https://github.com/moiseevigor/moiseevigor.github.io" +url: "https://github.com/moiseevigor/moiseevigor.github.io/tree/research/geometry-of-cosmic-web/research/cosmic-web" +license: MIT diff --git a/_posts/2026-05-01-geometry-of-seeing-A1-lie-groups.md b/_appendices/2026-05-01-geometry-of-seeing-A1-lie-groups.md similarity index 91% rename from _posts/2026-05-01-geometry-of-seeing-A1-lie-groups.md rename to _appendices/2026-05-01-geometry-of-seeing-A1-lie-groups.md index def3949434c..eaf2e9a447f 100644 --- a/_posts/2026-05-01-geometry-of-seeing-A1-lie-groups.md +++ b/_appendices/2026-05-01-geometry-of-seeing-A1-lie-groups.md @@ -21,6 +21,7 @@ series_part: A1 arxiv: "0807.4731" coauthors: "Yu. L. Sachkov" comments: true +permalink: /mathematics/2026/05/01/geometry-of-seeing-A1-lie-groups/ published: false --- @@ -32,7 +33,7 @@ published: false Part 1 of the series uses the language of Lie groups and Lie algebras as if it were standard furniture: $\mathrm{SE}(2)$, $\mathfrak{se}(2)$, left-invariant vector fields $X_1 = \cos\theta\,\partial_x + -\sin\theta\,\partial_y$, the bracket $[X_1, X_2] = X_3$, the exponential map. +\sin\theta\,\partial_y$, the bracket $[X_1, X_2] = -X_3$, the exponential map. This appendix builds those objects from scratch. Read it once and Part 1 becomes a calmer text. All three figures below are powered by the same $\mathrm{SE}(2)$ matrix exponential routine that the @@ -124,31 +125,40 @@ $G$ tangent to $E_i$ at the identity, multiply it on the left by $g$, and read off its velocity at $g$. -There is a canonical way to push the Lie algebra around the group: at any -$g \in G$ define +There is a canonical way to push any algebra element $E \in \mathfrak g$ +around the group: at any $g \in G$ define -$$X_i(g) \;:=\; (dL_g)_e (E_i),$$ +$$\widetilde E(g) \;:=\; (dL_g)_e (E),$$ -where $L_g(h) = gh$ is left-multiplication. $X_i(g)$ is the pushforward of -the abstract algebra element $E_i$ to the tangent space at $g$ along the -left-translation. These are the **left-invariant vector fields**. They -satisfy $X_i(gh) = (dL_g)_h X_i(h)$, i.e. they look the same in every -left-translated frame. +where $L_g(h) = gh$ is left-multiplication. $\widetilde E(g)$ is the +pushforward of +$E$ to the tangent space at $g$ along the left-translation. The vector +fields obtained this way are the **left-invariant vector fields**; they +satisfy $\widetilde E(gh) = (dL_g)_h \widetilde E(h)$, i.e. they look the +same in every left-translated frame. -Computing them in the chart $(x, y, \theta)$ is mechanical: +Computing the pushforwards in the chart $(x, y, \theta)$ is mechanical: $L_g(x', y', \theta') = (\,x + x'\cos\theta - y'\sin\theta,\;\; y + x'\sin\theta + y'\cos\theta,\;\; - \theta + \theta')$, so + \theta + \theta')$. Following Part 1's index +convention — $X_1$ forward, $X_2$ rotation, $X_3$ sideways — the three +basis fields are $$\boxed{\; - X_1 \;=\; \cos\theta\,\partial_x + \sin\theta\,\partial_y, \qquad - X_2 \;=\; \partial_\theta, \qquad - X_3 \;=\; -\sin\theta\,\partial_x + \cos\theta\,\partial_y .\;}$$ + X_1 = (dL_g)_e E_1 = \cos\theta\,\partial_x + \sin\theta\,\partial_y, \quad + X_2 = (dL_g)_e E_3 = \partial_\theta, \quad + X_3 = (dL_g)_e E_2 = -\sin\theta\,\partial_x + \cos\theta\,\partial_y .\;}$$ + +Note the index shuffle: $X_2$ comes from the *rotation* generator $E_3$ and +$X_3$ from the *translation* generator $E_2$, because Part 1 numbers the +frame (forward, rotation, sideways) while the $E$-basis is numbered +(translate-$x$, translate-$y$, rotate). These are the same three vector fields Part 1 §3 introduced. Now you know -where they come from: they are the basis of $\mathfrak{se}(2)$, parallel- -transported across the group by left-multiplication. They form an -orthonormal frame for the *Cartan-Killing geometry* on $\mathrm{SE}(2)$. +where they come from: they are the basis of $\mathfrak{se}(2)$, +parallel-transported across the group by left-multiplication. Declaring +them orthonormal is exactly how Part 1 puts its left-invariant metric on +$\mathrm{SE}(2)$. @@ -229,14 +239,16 @@ Two warnings worth absorbing. - This basis $\{E_1, E_2, E_3\}$ of $\mathfrak{se}(2)$ is *different* from the left-invariant frame $\{X_1, X_2, X_3\}$ used in Part 1. At the - identity $X_i(e) = E_i$, but at a generic $g \in \mathrm{SE}(2)$, - $X_i(g) \neq E_i$ — the LI vector fields are not constant in + identity $X_1(e) = E_1$, $X_2(e) = E_3$, $X_3(e) = E_2$ (the index + shuffle above); and at a generic $g \in \mathrm{SE}(2)$ the $X_i(g)$ are + no longer the constant matrices $E_j$ — the left-invariant vector fields vary in coordinates. -- For the LI vector fields the relevant bracket is the **vector-field - commutator** below, *not* the matrix commutator of their constant - identity-values. The vector-field bracket of $X_1$ with $X_2$ produces - $\pm X_3$ — the missing sideways direction — even though the matrix - bracket of $E_1$ with $E_2$ vanishes. +- For left-invariant fields the vector-field commutator and the matrix + commutator *agree*: $[X_i, X_j]_{\text{v.f.}}(e) = [\,X_i(e), X_j(e)\,]$ + in $\mathrm{Mat}_3$. What you must *not* do is take the matrix commutator + of the coordinate expressions $X_i(g)$ as if they were constant — they + are not. Done correctly, $[X_1, X_2] = -X_3$ picks out the missing + sideways direction, matching the matrix bracket $[E_1, E_3] = -E_2$. **(ii) Vector-field commutator.** For two vector fields acting on smooth functions $f$, @@ -257,10 +269,10 @@ $$[X_1, X_2] f \;=\; -(\partial_\theta\cos\theta)\partial_x f \;=\; \sin\theta\,\partial_x f - \cos\theta\,\partial_y f \;=\; -X_3 f.$$ -So $[X_1, X_2] = -X_3$ as left-invariant vector fields. The sign is a -convention: Part 1 used the opposite sign convention to land at -$[X_1, X_2] = +X_3$, and we will switch to that in §4 below where it matters. -Either way, the bracket is $\pm X_3$ — non-zero, in the missing sideways +So $[X_1, X_2] = -X_3$ as left-invariant vector fields — the same sign +Part 1 carries. (Writing it the other way round, $[X_2, X_1] = +X_3$, is +the only freedom here; it is a bookkeeping choice, not a real one.) Either +way the bracket is non-zero and points along $X_3$, the missing sideways direction. **(iii) Closing-defect interpretation.** Flow along $X$ for time @@ -271,7 +283,7 @@ doesn't, and the residual is $$\Phi^Y_{-\varepsilon} \circ \Phi^X_{-\varepsilon} \circ \Phi^Y_{\varepsilon} \circ \Phi^X_{\varepsilon}\,(g) \;=\; g + \varepsilon^2 [X, Y]_g + O(\varepsilon^3).$$ -For the V1 cortex this is the four-step manoeuvre Part 1 §3.4 illustrated: +For the primary visual cortex (V1) this is the four-step manoeuvre Part 1 §3.4 illustrated: two hops of "slide along your orientation" interleaved with two hops of "rotate the orientation" produce a sideways nudge of order $\varepsilon^2$. The Lie bracket is exactly the leading coefficient of that nudge. @@ -326,7 +338,7 @@ The Lie bracket is exactly the leading coefficient of that nudge. universal. Toggle the $O(\varepsilon^3)$ residuals: after subtracting the predicted $\varepsilon^2 [X, Y]$ term, the orange (SE(2)) and pink (SO(3)) residuals fall on slope $\approx 3$ — the - next BCH contribution, also universal. Two grey reference lines + next Baker–Campbell–Hausdorff (BCH) contribution, also universal. Two grey reference lines have slopes 2 and 3 exactly; the four data traces all track them. Punchline: the algebra determines the leading order; the @@ -355,8 +367,8 @@ It does two things at once: so $\exp$ is a local diffeomorphism near the origin. It is *not* a global diffeomorphism: for SE(2) the exponential map is surjective but not injective, and we will need to be careful in Appendix A5 when we - talk about the *sub-Riemannian* exponential map (which is a different - beast — see A5). + talk about the *sub-Riemannian* (SR) exponential map (which is a + different beast). For SE(2), $\exp(t(a_1 E_1 + a_2 E_2 + a_3 E_3))$ has a closed form. Write $X = T + \omega E_3$ with $T = a_1 E_1 + a_2 E_2$ (translation part) and @@ -377,16 +389,18 @@ through Figure A1.1: $\omega = 0$ gives a straight line, $\omega \neq 0$ gives a circle whose centre is offset from the origin by the screw "axis" $a / \omega$. -### The 1-parameter subgroups *are* the geodesics of the Cartan-Killing metric +### 1-parameter subgroups and bi-invariant geodesics -If you put a left-invariant Riemannian metric on $G$ that is also right- -invariant (a "bi-invariant" metric, which on $\mathrm{SE}(2)$ exists), then -the geodesics through $e$ are exactly the 1-parameter subgroups -$t \mapsto \exp(tX)$. This is **not** the situation in Part 1: Part 1 uses -a *sub-Riemannian* metric that is left-invariant but not right-invariant, -and in that geometry the geodesics are **not** generally -$\exp(tX)$ — they are Euler's elastica. Appendix A5 explains how the SR -exponential map differs from this group exponential. +On a group that carries a *bi-invariant* metric — one invariant under both +left and right translation — the geodesics through $e$ are exactly the +1-parameter subgroups $t \mapsto \exp(tX)$. Compact groups and $\mathbb R^n$ +have such metrics; $\mathrm{SE}(2)$ does **not** — its adjoint action is +non-compact, which is the standard obstruction. So even a Riemannian story +on $\mathrm{SE}(2)$ would not make $\exp(tX)$ geodesic. Part 1 goes further +still: its metric is *sub-Riemannian*, left-invariant but not +right-invariant, and its geodesics are not $\exp(tX)$ at all — they are +Euler's elastica. Appendix A5 explains how the SR exponential map differs +from this group exponential. ## Adjoint and coadjoint actions @@ -396,7 +410,7 @@ $$\mathrm{Ad}_g : \mathfrak g \to \mathfrak g, \qquad \mathrm{Ad}_g(X) \;:=\; g X g^{-1}.$$ Differentiating at $g = e$ recovers the bracket: -$\frac{d}{dt}\bigr|_{t=0} \mathrm{Ad}_{\exp(tX)}(Y) = [X, Y] =: \mathrm{ad}_X(Y)$. +$$\frac{d}{dt}\bigr|_{t=0} \mathrm{Ad}_{\exp(tX)}(Y) = [X, Y] =: \mathrm{ad}_X(Y)$$. The functions are **doubly-periodic** when extended to the complex plane: @@ -158,7 +159,7 @@ $$\boxed{\;\mathrm{sn}'(u) = \mathrm{cn}(u)\,\mathrm{dn}(u), \quad Squaring the first identity and using the Pythagorean relations $\mathrm{sn}^2 + \mathrm{cn}^2 = 1$ and $m\,\mathrm{sn}^2 + \mathrm{dn}^2 = 1$ -yields the cleanest single ODE: +yields the cleanest single ordinary differential equation (ODE): $$(\mathrm{sn}')^2 \;=\; (1 - \mathrm{sn}^2)(1 - m\,\mathrm{sn}^2),$$ @@ -180,11 +181,8 @@ so $\ddot\varphi + \sin\varphi = 0$. ✓ This is **the** identity behind the elastica: the pendulum solution is literally the Jacobi-am function, and the curvature -$\kappa(s) = \dot\varphi(s) = 2k\,\mathrm{cn}(s)$ is one Jacobi function's -worth — exactly the formula Part 2 uses. (An alternative convention takes -$\kappa = 2k\,\mathrm{sn}$, related by a quarter-period shift $s \to s + K$; -either describes the same family of geodesics, just starting at a different -arc-length offset.) +$\kappa(s) = \dot\varphi(s) = 2k\,\mathrm{cn}(s\mid k^2)$ is one Jacobi +function's worth — exactly the formula Part 2 uses. ## The complete elliptic integrals @@ -217,9 +215,9 @@ $$a_{n+1} \;=\; \tfrac12 (a_n + b_n), \qquad b_{n+1} \;=\; \sqrt{a_n b_n}.$$ The two sequences converge — quadratically — to a common limit, the **arithmetic–geometric mean** $\mathrm{AGM}(a_0, b_0)$. Quadratic convergence means each iteration *roughly doubles the number of correct -digits*: starting from a $10^{-1}$-precision pair, six iterations land at -$\sim 10^{-32}$ precision — hence the elliptic package's "16-digit accuracy -in 6 iterations" claim. +digits*: starting from a $10^{-1}$-precision pair, five iterations land at +$\sim 10^{-32}$ precision — comfortably past the elliptic package's +"16-digit accuracy in 6 iterations" claim. Gauss (1799, unpublished) proved the miracle: @@ -228,8 +226,7 @@ Gauss (1799, unpublished) proved the miracle: $$K(m) \;=\; \frac{\pi}{2\,\mathrm{AGM}\!\bigl(1,\;\sqrt{1 - m}\bigr)}.$$ -Equivalently $K(m) = \pi / (2\,\mathrm{AGM}(\sqrt{1+\sqrt{1-m}}, -\sqrt{1-\sqrt{1-m}}))/\sqrt 2$ via the descending Landen transformation. +Six AGM steps from $(1, \sqrt{1-m})$ pin $K(m)$ down to full double precision. This is what `ellipticK` in the Python `elliptic` package computes — and @@ -302,7 +299,7 @@ under the rescaling. energy $E$, blue solid; logarithmic divergence at $E \to 1^-$ (red dashed). At small amplitude $E \to -1$, $T \to 2\pi$ (the harmonic limit); at $E = 0$, $T \approx 7.42$, already noticeably longer than - $2\pi$. Asymptotic prediction $T \sim 2\log(16/(1-E))$ near + $2\pi$. Asymptotic prediction $T \sim 2\log(32/(1-E))$ near separatrix overlaid (grey dotted). This is the same plot drawn by the elliptic project's physical-pendulum example — modulo axis labels, the @@ -314,20 +311,20 @@ under the rescaling. ## Identities used in Part 2 §4 -Part 2 uses the closed-form integral +Part 2 uses the closed-form heading integral $$\theta(s) \;=\; \theta_0 + 2\arcsin\bigl(k\,\mathrm{sn}(s\mid k^2)\bigr).$$ -Differentiating with respect to $s$: -$\dot\theta = 2k\,\mathrm{cn}\,\mathrm{dn} / \sqrt{1 - k^2 \mathrm{sn}^2} - = 2k\,\mathrm{cn}\,\mathrm{dn} / \mathrm{dn} - = 2k\,\mathrm{cn}$. -But Part 2 writes $\kappa = 2k\,\mathrm{sn}$. The reconciliation: there -are *two* parametrisations of the inflectional family by Jacobi functions, -related by a quarter-period shift $s \to s + K(k^2)$. Under that shift -$\mathrm{sn}(s + K) = \mathrm{cn}(s) / \mathrm{dn}(s)$ and the two -conventions translate. Both the Sachkov closed form (with sn) and the -"angle-of-pendulum" form (with cn) are used in the literature. +Differentiating with respect to $s$, and using +$\sqrt{1 - k^2\,\mathrm{sn}^2} = \mathrm{dn}$, + +$$\kappa = \dot\theta + = \frac{2k\,\mathrm{cn}\,\mathrm{dn}}{\sqrt{1 - k^2\,\mathrm{sn}^2}} + = \frac{2k\,\mathrm{cn}\,\mathrm{dn}}{\mathrm{dn}} + = 2k\,\mathrm{cn}(s\mid k^2),$$ + +exactly the boxed curvature of Part 2 §3 — the heading integral and the +curvature formula are one statement, differentiated once. The plane curve integration uses the **second-kind incomplete integral** @@ -349,15 +346,15 @@ Every formula in this appendix is implemented in | Formula here | Function in `elliptic` | |---|---| -| $K(m)$ via AGM (§4) | `ellipticK(m)` | -| sn/cn/dn via descending Landen (§3) | `ellipj(u, m)` | +| $K(m)$ via AGM (§5) | `ellipticK(m)` | +| sn/cn/dn via descending Landen (Fig. A4.1) | `ellipj(u, m)` | | $E(\phi\mid m), F(\phi\mid m)$ (§7) | `elliptic12(phi, m)` | -| Pendulum period $4K((E+1)/2)$ (§5) | `ellipticK((E+1)/2) * 4` | -| Carlson form $R_F$, $R_D$ (§ omitted) | `carlsonRF`, `carlsonRD` | +| Pendulum period $4K((E+1)/2)$ (§6) | `ellipticK((E+1)/2) * 4` | +| Carlson form $R_F$, $R_D$ (not covered here) | `carlsonRF`, `carlsonRD` | The browser figures on this page use `elliptic-core.js`, which is a -hand-port of the AGM and Landen routines from the Python package. Line -13–20 of `elliptic-core.js` is identical (modulo language) to the AGM +hand-port of the AGM and Landen routines from the Python package. Lines +13–20 of `elliptic-core.js` are identical (modulo language) to the AGM loop in `elliptic/_AGM.py`. ## Code @@ -377,15 +374,17 @@ from scipy.integrate import solve_ivp def pend(t, y): return [y[1], -np.sin(y[0])] +def downcross(t, y): + return y[0] # phi passing through 0 ... +downcross.direction = -1 # ... going downward: happens at t = T/2 + T_numeric = [] for E in E_vals: phi0 = 0.0 - phidot0 = np.sqrt(2 * (E - np.cos(phi0))) # initial velocity from E + phidot0 = np.sqrt(2 * (E + np.cos(phi0))) # from E = ½φ̇² − cos φ sol = solve_ivp(pend, [0, 30], [phi0, phidot0], rtol=1e-12, atol=1e-14, - dense_output=True) - # Find first return to phi = 0 with phidot > 0 - # ... (implementation omitted) - T_numeric.append(...) # matches T_closed to 1e-10 + events=downcross) + T_numeric.append(2 * sol.t_events[0][0]) # matches T_closed to ~1e-10 ``` ```python @@ -644,8 +643,8 @@ function drawPeriod() { .attr('font-family', 'Source Sans 3').attr('font-size', 11).attr('fill', '#888') .text('T = 2π (harmonic)'); - // Asymptotic prediction T ~ 2 log(16/(1-E)) at E ~ 1 - const asy = Earr.filter(E => E > 0).map(E => ({ E, T: 2 * Math.log(16 / Math.max(1e-3, 1 - E)) })); + // Asymptotic prediction T ~ 2 log(32/(1-E)) at E ~ 1 + const asy = Earr.filter(E => E > 0).map(E => ({ E, T: 2 * Math.log(32 / Math.max(1e-3, 1 - E)) })); g.append('path') .attr('d', d3.line().x(p => xS(p.E)).y(p => yS(Math.min(p.T, 60)))(asy)) .attr('fill', 'none').attr('stroke', '#888').attr('stroke-width', 1).attr('stroke-dasharray', '2,3'); diff --git a/_posts/2026-05-05-geometry-of-seeing-A5-sr-exponential.md b/_appendices/2026-05-05-geometry-of-seeing-A5-sr-exponential.md similarity index 88% rename from _posts/2026-05-05-geometry-of-seeing-A5-sr-exponential.md rename to _appendices/2026-05-05-geometry-of-seeing-A5-sr-exponential.md index 300961ef053..7b2e7cd4e22 100644 --- a/_posts/2026-05-05-geometry-of-seeing-A5-sr-exponential.md +++ b/_appendices/2026-05-05-geometry-of-seeing-A5-sr-exponential.md @@ -22,6 +22,7 @@ series_part: A5 arxiv: "0807.4731" coauthors: "Yu. L. Sachkov" comments: true +permalink: /mathematics/2026/05/05/geometry-of-seeing-A5-sr-exponential/ published: false --- @@ -38,8 +39,8 @@ Appendix A1 and this SR exponential — they are different functions on the same manifold. The interactive figures borrow the geodesic-family visualisation directly from the elliptic -project's Dubins–visual-cortex example; the Maxwell-pair and conjugate- -locus figures formalise what that figure was already showing. +project's Dubins–visual-cortex example; the Maxwell-pair and +conjugate-locus figures formalise what that figure was already showing. @@ -50,10 +51,12 @@ Lie-group $\exp$ (Appendix A1): $$\exp_{\mathrm{group}}: \mathfrak{se}(2) \to \mathrm{SE}(2), \qquad X \mapsto \exp(X) \cdot e.$$ -The 1-parameter subgroups $t \mapsto \exp(tX) \cdot e$ are the geodesics -of any **bi-invariant** Riemannian metric on $\mathrm{SE}(2)$ — and -$\mathrm{SE}(2)$ does carry such a metric. But that is *not* the V1 -sub-Riemannian metric of Part 1. +The 1-parameter subgroups $t \mapsto \exp(tX) \cdot e$ would be the +geodesics of a **bi-invariant** Riemannian metric — but Appendix A1 showed +$\mathrm{SE}(2)$ carries *no* such metric (its adjoint action is +non-compact). They remain the natural "group-straight" curves on +$\mathrm{SE}(2)$; they are certainly not the geodesics of the +sub-Riemannian (SR) metric of Part 1's primary-visual-cortex (V1) model. Sub-Riemannian $\mathrm{Exp}$: @@ -71,8 +74,8 @@ Two parametrisations of the same costate space are convenient: - **Cylinder coordinates** $(c, \omega_0, \phi_0)$: $h_1 = \sqrt c \cos\phi_0, h_2 = \sqrt c \sin\phi_0, h_3 = \omega_0$. By rescaling, set $c = 1$ on the energy surface. -- **Pendulum-energy coordinates** $(E, \phi_0)$: $E = \tfrac12 \omega_0^2 - - \cos\phi_0$ (with the rescaling of A3 §6). Here $E < 1$ is libration +- **Pendulum-energy coordinates** $(E, \phi_0)$: $E = 2C - 1$ with the + Casimir $C = h_1^2 + h_3^2$ (Appendix A3). Here $E < 1$ is libration (inflectional), $E = 1$ is the separatrix (Euler spiral), $E > 1$ is rotation (non-inflectional). @@ -84,7 +87,7 @@ whole story of cut/conjugate analysis. ## Closed-form geodesic endpoints For the inflectional family, parametrise by $k \in (0, 1)$ via -$\sin(\varphi_0/2) = k\,\mathrm{sn}(s\mid k^2)$ — the substitution that +$\sin(\varphi/2) = k\,\mathrm{sn}(s\mid k^2)$ — the substitution that A4 §3 derived from the half-angle. Then Sachkov (2011) showed that the plane projection $(x(s), y(s))$ of the SR geodesic is @@ -96,7 +99,8 @@ $$\boxed{\;y(s) \;=\; 2k\bigl(1 - \mathrm{cn}(s\mid k^2)\bigr).\;}$$ form depends on the chosen sign of the curvature and the position of $s = 0$ relative to the inflection. The point is: each component is a combination of $\mathrm{sn}, \mathrm{cn}$ and incomplete elliptic -integrals $E, F$. No numerical ODE integration is needed. Appendix A4 §7 +integrals $E, F$. No numerical integration of an ordinary differential +equation (ODE) is needed. Appendix A4 §7 gives the precise relations.) Three takeaways: @@ -122,9 +126,9 @@ Three takeaways:
- γ_A: κ = +2k·sn — γ_B: κ = −2k·sn — coincidence requires y_A(s) = 0 + γ_A: κ = +2k·cn — γ_B: κ = −2k·cn — coincidence requires y_A(s) = 0
@@ -252,7 +259,7 @@ Part 3.
Figure A5.2. The σ-symmetric pair: two geodesics $\gamma_A, \gamma_B$ leaving the origin with curvatures - $\pm 2k\,\mathrm{sn}(s\mid k^2)$ respectively. By the + $\pm 2k\,\mathrm{cn}(s\mid k^2)$ respectively. By the $y \to -y$ reflection symmetry of the pendulum equation, $\gamma_B(s) = (x_A(s),\, -y_A(s),\, -\theta_A(s))$ — they trace mirror-image curves. As SE(2) configurations, they coincide exactly @@ -260,11 +267,15 @@ Part 3. The right panel plots $|\gamma_A(s) - \gamma_B(s)|$ over $s \in [0, T]$; its first zero crossing (red marker, if any) is the first Maxwell time of this pair. - For "lemniscate" values of $k$ (where the elastica closes into a - figure-eight — numerically $k_c \approx 0.84$) this first zero - coincides with the curvature period $4K(k^2)$. For generic $k$ the - Maxwell time is k-dependent and given by Sachkov's transcendental - equation in the modulus. Slide $k$ to find lemniscate values. + Because $y_A(s) = 2k\bigl(1 - \mathrm{cn}(s\mid k^2)\bigr) \ge 0$ returns to + zero only at $s = 4K(k^2)$ (and its multiples), the pair first re-coincides + in full — position and heading — at $s = 4K(k^2)$, for every + $k$: exactly the first Maxwell time $t_{\mathrm{Maxwell}}^{(1)} = 4K(k^2)/\omega_0$ + boxed above. At the special "figure-eight" modulus + $k_c \approx 0.909$ (root of $2E(k^2) = K(k^2)$) that shared endpoint sits + back at the origin, so the closed curve is itself a single self-crossing + lemniscate; for other $k$ the two curves still meet at $s = 4K(k^2)$, just + away from the origin.
@@ -272,18 +283,19 @@ Part 3. ## How a wavefront forms from a family of geodesics -Fix $T$ and vary the initial costate over a 1-parameter slice of the unit- -energy surface — concretely, signed initial curvature $k \in [-0.95, 0.95]$ -in $\kappa(s) = 2k\,\mathrm{sn}(s\mid k^2)$. Each $k$ launches a distinct +Fix $T$ and vary the initial costate over a 1-parameter slice of the +unit-energy surface — concretely, signed initial curvature $k \in [-0.95, 0.95]$ +in $\kappa(s) = 2k\,\mathrm{cn}(s\mid k^2)$. Each $k$ launches a distinct geodesic from the origin. The set of *positions* reached at exact arc length $T$ — one position per geodesic — is the **wavefront** at time $T$: -$$\mathcal W_T \;:=\; \bigl\{\,(x(T;k),\; y(T;k)) : k \in [-1, 1]\,\bigr\} \;\subset\; \mathbb R^2.$$ +$$\mathcal W_T \;:=\; \bigl\{\,(x(T;k),\; y(T;k)) : k \in [-0.95, 0.95]\,\bigr\} \;\subset\; \mathbb R^2.$$ It is a *continuous curve* in the plane (because $k \mapsto $ trajectory is -continuous), and as $T$ grows it sweeps outward. At small $T$ the -wavefront is approximately a circle of radius $T$ around the origin -(everything moves at unit speed in approximately straight lines). As $T$ +continuous), and as $T$ grows it sweeps outward. At small $T$ — since every +geodesic leaves the origin heading the same way and curves only gently — +the wavefront is a short, almost-straight arc near $(T, 0)$, transverse to +the launch direction. As $T$ approaches the first Maxwell time $T_{\mathrm M} = 4K(k^2)/\omega_0$, neighbouring trajectories begin to converge and the wavefront develops **cusps** — these are the projections of conjugate points, where @@ -318,7 +330,7 @@ be locally surjective along a critical curve.
- Figure A5.3. A wavefront forming. 41 trajectories, + Figure A5.3. A wavefront forming. 39 trajectories, one per signed modulus $k$, are drawn in faint blue (forward-curving, $k > 0$) and faint red ($k < 0$) up to the current arc length $T$. The thick coloured curve passing through their endpoints is the @@ -327,12 +339,13 @@ be locally surjective along a critical curve. $\mathcal W_T$ moves outward and reshapes as time grows. Slide $T$ forward and watch:
    -
  1. at small $T$ the wavefront is nearly a circle;
    2. +
  2. at small $T$ the wavefront is a short arc near $(T, 0)$;
  3. at $T \approx \pi$ it lengthens and starts to flatten;
    4. -
  4. at $T \approx 4K(k_c^2) \approx 6.5$–$7$ cusps appear at - the four corners of the wavefront — these are the first conjugate - points (marked with red rings);
    5. -
  5. past the cusps, the wavefront self-intersects: those crossings +
  6. around $T \approx 2\pi$ — the smallest period $4K(0)$ in the swept + family (the near-straight $k \to 0$ geodesics) — the first + cusps appear at the corners of the wavefront: these are the + first conjugate points (marked with red rings);
    7. +
  7. as $T$ grows the wavefront self-intersects: those crossings are the Maxwell stratum drawn in Figure A5.2.
Press play to animate $T$ continuously. The four-fold @@ -360,8 +373,8 @@ The Dubins-back-wheel cuspidal trajectories of the elliptic project parking-style curves with cusps) are also relevant here: they are projections of SE(2) geodesics in the regime where the rear-axle forward velocity changes sign. The cusps in those trajectories are the -geometric analogue of the wavefront cusps in Figure A5.3 — only the rear- -axle parametrisation makes them visible. +geometric analogue of the wavefront cusps in Figure A5.3 — only the +rear-axle parametrisation makes them visible. ## Code @@ -378,10 +391,8 @@ def inflectional_geodesic(k, omega0, phi0, T, N=1500): m = k * k s = np.linspace(0, T, N) sn, cn, dn = ellipj(s, m) # vectorised - # Pendulum: sin(φ/2) = k·sn(s|m), so: - half_phi = np.arcsin(k * sn) - phi = 2 * half_phi - # Curvature κ = dθ/ds = ω0 sin(φ) — but normalise + # Heading half-angle: sin(θ/2) = k·sn(s|m), so θ = 2·arcsin(k·sn). + # Curvature κ = dθ/ds = 2k·cn·dn / sqrt(1 − k²sn²) = 2k·cn (since dn = sqrt). kappa = 2 * k * cn * dn / np.sqrt(1 - k * k * sn * sn) theta = np.zeros_like(s) x = np.zeros_like(s) @@ -436,7 +447,7 @@ that controls curvature period. What remains for Parts 3 and 4 of the blog series: - **Part 3** will give the proof that, for the inflectional family, - $t_{\mathrm{cut}} = t_{\mathrm{Maxwell}}^{(1)} = 4K(k^2)/\omega_0$, + $$t_{\mathrm{cut}} = t_{\mathrm{Maxwell}}^{(1)} = 4K(k^2)/\omega_0$$, and characterise the full Maxwell stratum via the $\mathbb Z_2 \times \mathbb Z_2$ symmetries. - **Part 4** will discuss the open question: extending this to the @@ -444,9 +455,9 @@ What remains for Parts 3 and 4 of the blog series: on the global structure of the cut locus. The five appendices A1–A5 supply every prerequisite: Lie groups (A1), -distributions and contact structures (A2), the PMP and Lie–Poisson -reduction (A3), Jacobi elliptic functions and the AGM (A4), and the SR -exponential map (A5). With them in hand, Parts 1 and 2 should read +distributions and contact structures (A2), the Pontryagin Maximum +Principle and Lie–Poisson reduction (A3), Jacobi elliptic functions and +the arithmetic–geometric mean (A4), and the SR exponential map (A5). With them in hand, Parts 1 and 2 should read fluently, and Parts 3 and 4 will be approachable when they ship. @@ -508,7 +519,7 @@ fluently, and Parts 3 and 4 will be approachable when they ship. // ── shared helpers ───────────────────────────────────────────────────── function inflectionalGeodesic(k, T, N) { - // Curvature κ(s) = 2k·sn(s|k²) (Part 2 convention) + // Curvature κ(s) = 2k·cn(s|k²) (Part 2 convention) // dθ/ds = κ, dx/ds = cos θ, dy/ds = sin θ const m = k * k; const ds = T / N; @@ -517,7 +528,7 @@ function inflectionalGeodesic(k, T, N) { for (let i = 0; i < N; i++) { const s = i * ds; const j = ellipj(s + ds / 2, m); - const k1 = 2 * k * j.sn; + const k1 = 2 * k * j.cn; const tmid = theta + 0.5 * k1 * ds; x += Math.cos(tmid) * ds; y += Math.sin(tmid) * ds; @@ -633,7 +644,7 @@ function drawExpMap() { } else { const m = Math.max(0.01, Math.min(0.99, 2 - kVal)); const Km = ellipticK(m); - info = `non-inflectional m = ${m.toFixed(2)} period 2√m·K = ${(2*Math.sqrt(m)*Km).toFixed(2)} T = ${T.toFixed(2)}`; + info = `non-inflectional m = ${m.toFixed(2)} period 2K(m) = ${(2*Km).toFixed(2)} T = ${T.toFixed(2)}`; } g.append('text').attr('x', margin.l).attr('y', margin.t + 12) .attr('font-family', 'JetBrains Mono').attr('font-size', 11).attr('fill', '#555') @@ -674,7 +685,7 @@ function drawMaxwell() { for (let i = 0; i < N; i++) { const s = i * ds; const j = ellipj(s + ds / 2, m); - const k1 = 2 * k * j.sn; + const k1 = 2 * k * j.cn; const tMid = theta + 0.5 * k1 * ds; x += Math.cos(tMid) * ds; y += Math.sin(tMid) * ds; @@ -691,17 +702,11 @@ function drawMaxwell() { // Per-s separation: |γA(s) − γB(s)| = 2|y_A(s)| const sep = ptsA.map(p => ({ s: p.s, d: 2 * Math.abs(p.y) })); - // Find first zero crossing of y_A after s = ds (skip the initial s = 0) - let firstZero = null; - for (let i = 8; i < ptsA.length; i++) { - if ((ptsA[i - 1].y > 0) !== (ptsA[i].y > 0)) { - // Linear interpolation - const y0 = ptsA[i - 1].y, y1 = ptsA[i].y; - const t0 = ptsA[i - 1].s, t1 = ptsA[i].s; - firstZero = t0 + (t1 - t0) * (-y0) / (y1 - y0); - break; - } - } + // The pair coincides in full (position AND heading) at s = 4K(k²): there + // y_A = 2k(1 − cn) returns to 0 and θ_A returns to 0 (mod 2π). Since + // y_A ≥ 0 only *touches* zero (never crosses), a sign-change search would + // miss it — use the analytic first Maxwell time directly. + const firstZero = (4 * Km <= T_full) ? 4 * Km : null; // Layout: left panel = trajectories up to T; right panel = sep(s) plot const split = W * 0.55; @@ -822,7 +827,7 @@ function drawMaxwell() { // ── Figure A5.3 — Wavefront formation ────────────────────────────────── // -// 41 trajectories sweep signed k ∈ [-0.95, 0.95]. Each trajectory is +// 39 trajectories sweep signed k ∈ [-0.95, 0.95]. Each trajectory is // integrated up to a fixed maximum arc length sMax (≥ the slider's max T). // The wavefront W_T is the curve through {(x(T;k), y(T;k)) : k ∈ [-1, 1]}. const conjState = { @@ -850,7 +855,7 @@ function buildConjTrajectories() { for (let i = 0; i < conjState.N; i++) { const s = i * ds; const j = ellipj(s + ds / 2, m); - const kappaMid = sign * 2 * ka * j.sn; + const kappaMid = sign * 2 * ka * j.cn; const tMid = theta + 0.5 * kappaMid * ds; x += Math.cos(tMid) * ds; y += Math.sin(tMid) * ds; @@ -994,11 +999,11 @@ function drawConjugate() { g.append('circle').attr('cx', xS(0)).attr('cy', yS(0)).attr('r', 4).attr('fill', '#222'); // Status text + scale legend - // Use k ≈ 0.85 as a reference for "first Maxwell time" - const Tmax_ref = 4 * ellipticK(0.85 * 0.85); + // Reference: the figure-eight modulus k_c ≈ 0.909 (root of 2E(k²) = K(k²)). + const Tmax_ref = 4 * ellipticK(0.9089 * 0.9089); g.append('text').attr('x', margin.l).attr('y', margin.t + 12) .attr('font-family', 'JetBrains Mono').attr('font-size', 11).attr('fill', '#555') - .text(`T = ${conjState.T.toFixed(2)} (1st Maxwell @ k = 0.85: T₁ = ${Tmax_ref.toFixed(2)})`); + .text(`T = ${conjState.T.toFixed(2)} (1st Maxwell @ k = 0.909: T₁ = ${Tmax_ref.toFixed(2)})`); // Colour-bar mini-legend const lx = margin.l + 4, ly = H - margin.b - 14; diff --git a/_appendices/2026-07-06-cosmic-web-B1-transport-models.md b/_appendices/2026-07-06-cosmic-web-B1-transport-models.md new file mode 100644 index 00000000000..f2e355201af --- /dev/null +++ b/_appendices/2026-07-06-cosmic-web-B1-transport-models.md @@ -0,0 +1,220 @@ +--- +layout: distill +title: "Appendix B1 — How the Universe Moves Its Matter: Transport Models" +subtitle: > + The Zel'dovich approximation as free motion in growth-factor time, why + second-order perturbation theory overshoots at shell-crossing, the adhesion + model as Hopf–Lax optimal transport on a flat metric, MUSCLE's spherical + collapse — and the transverse-damping recipe the research program froze. +date: 2026-07-06 09:00:00 +categories: [mathematics] +tags: [cosmic-web, zeldovich, adhesion, optimal-transport, cosmology] +description: > + Self-contained tour of the fast transport models of large-scale structure: + the Zel'dovich approximation, 2LPT and its shell-crossing overshoot, the + adhesion/Burgers model and the Hopf–Lax formula, MUSCLE, the flat + optimal-transport theorem chain, and the frozen transverse-damping recipe. + Companion appendix to the Geometry of the Cosmic Web series. +series: geometry-of-cosmic-web +series_title: "Geometry of the Cosmic Web" +series_part: B1 +permalink: /mathematics/2026/07/06/cosmic-web-B1-transport-models/ +published: true +comments: true +--- + +
+ +
+
What this appendix covers
+The series compares fast transport models — recipes that move the +early Universe's matter to its present positions without a full gravity +simulation. This appendix defines each from scratch: the Zel'dovich +approximation as free motion, 2LPT and its shell-crossing overshoot, the +adhesion model and its Hopf–Lax solution, the optimal-transport chain that +identifies filaments as shocks of a flat map, and MUSCLE — ending +with the precise transverse-damping recipe the program distilled from all of +them. Numbers cite the reports in research/cosmic-web/docs/ by +name, e.g. "(E9)". +
+ +## The problem every transport model solves + +The input is the infant Universe: a nearly uniform matter distribution with +tiny density ripples, summarised by an initial gravitational +potential $$\Phi_0(\mathbf{q})$$, where $$\mathbf{q}$$ labels each parcel of +matter by its starting (Lagrangian) position. The output is the parcel's final +(Eulerian) position $$\mathbf{x}$$ today, in comoving coordinates — coordinates +that expand with the Universe, so the overall expansion is factored out. A full N-body simulation integrates gravity step by step; a transport +model replaces that with a formula or a near-free evolution rule. Time is +best measured not in years but +by the **linear growth factor** $$D(t)$$, the overall factor by which small +density ripples have grown, because the equations below are simplest in that +variable. The shared obstacle is **shell-crossing**: the moment two streams of +matter first try to occupy the same place, where the density is formally +infinite (a caustic). How the models cope with that moment is where they differ. + +## The Zel'dovich approximation: straight lines in growth-factor time + +The Zel'dovich approximation (ZA; Zel'dovich 1970) is first-order Lagrangian +perturbation theory: each parcel moves ballistically along the initial force +direction. Differentiating the map with respect to $$D$$ shows the velocity is +constant along each trajectory: + +$$ +\mathbf{x}(\mathbf{q}, D) \;=\; \mathbf{q} \;-\; D\,\nabla_q \Phi_0(\mathbf{q}), +\qquad +\frac{d\mathbf{x}}{dD} \;=\; -\nabla_q \Phi_0(\mathbf{q}) \;=\; \mathrm{const}. +$$ + +Every trajectory is a **straight line traversed at constant velocity** in the +variables $$(\mathbf{x}, D)$$. Such paths extremise the free action + +$$ +S[\mathbf{x}] \;=\; \int \Bigl|\frac{d\mathbf{x}}{dD}\Bigr|^{2}\, dD , +$$ + +so they are geodesics of the **flat Euclidean metric**. The gravitational +potential exerts no force during the evolution at all: in $$D$$-time it is +absorbed entirely into the initial velocities $$\mathbf{v}_0 = -\nabla\Phi_0$$. +This constructively refutes any Jacobi-type "effective metric" picture in which +paths would be cheap along potential valleys (T1): a Jacobi metric reweights +paths by the potential *along* the path, and here there is no potential along +the path left to reweight. + +The ZA's failure mode follows from its virtue: straight lines do not stop — a +parcel falling into a forming wall sails straight *through* it. Measured +against particle-mesh (PM) N-body truth from identical initial conditions, the +ZA's median per-particle transport error is 4.98–5.00 voxels, where one voxel +is one grid cell of 1 h⁻¹Mpc ≈ 1.4 megaparsecs (E5, E9). + +## Why 2LPT overshoots at shell-crossing + +Second-order Lagrangian perturbation theory (2LPT; Bouchet et al. 1995, +Scoccimarro 1998) adds the next term of the displacement series, sourced by a +second-order potential: + +$$ +\boldsymbol{\Psi} \;=\; -\,D\,\nabla_q \Phi^{(1)} + D_2\,\nabla_q \Phi^{(2)}, \qquad D_2 \simeq -\tfrac{3}{7} D^{2}, \qquad +\nabla_q^{2} \Phi^{(2)} \;=\; \sum_{i>j} \Bigl[ \Phi^{(1)}_{,ii}\,\Phi^{(1)}_{,jj} - \bigl(\Phi^{(1)}_{,ij}\bigr)^{2} \Bigr]. +$$ + +On large, still-linear scales this is a genuine improvement. But the series is +an expansion around uncollapsed flow: it has no mechanism for *stopping*. Near +shell-crossing the second-order term keeps accelerating parcels into and +through the caustic — the known 2LPT overshoot in collapsed regions. Measured +on identical initial conditions and identical PM truth, 2LPT's median +transport error is **8.07 ± 0.39 voxels** against the plain ZA's 4.97 ± 0.27 +(E9) — the "better" perturbative term makes transport *worse* by 62%. The +pattern — arresting particles beats perturbing them — is the empirical thread +of the whole series. + +## The adhesion model: Burgers viscosity and the Hopf–Lax formula + +The adhesion model (Gurbatov, Saichev & Shandarin 1989) repairs the ZA's +sail-through with an infinitesimal viscosity, so that streams stick instead +of crossing. The velocity field obeys Burgers' equation, + +$$ +\partial_D \mathbf{v} + (\mathbf{v}\cdot\nabla)\,\mathbf{v} \;=\; \nu\,\Delta \mathbf{v}, \qquad \nu \to 0^{+}, \qquad \mathbf{v} = \nabla\psi , +$$ + +with $$\psi$$ the velocity potential; the Hopf–Cole transformation solves it +exactly, and the vanishing-viscosity limit is the **Hopf–Lax formula** (Hopf +1950; Lax 1957): + +$$ +\psi(\mathbf{x}, D) \;=\; \min_{\mathbf{q}} \left[ \psi_0(\mathbf{q}) + \frac{|\mathbf{x}-\mathbf{q}|^{2}}{2D} \right]. +$$ + +Matter still travels on straight rays, but where rays collide it sticks — +sheets, filaments and nodes persist instead of dissolving. The classical +particle-form proxy is **isotropic sticking**: at shell-crossing, damp *all* +velocity components. Measured, it overcorrects — 5.34 ± 0.15 voxels, worse +than the plain ZA's 5.00 (E5) — and destroys small-scale phase fidelity, +because it kills the along-filament motion real gravity preserves (E8). + +## The theorem chain: flat optimal transport, filaments as shocks + +The Hopf–Lax formula identifies the flow's exact variational principle: a +minimisation over straight-line paths with quadratic +cost $$|\mathbf{x}-\mathbf{q}|^{2}$$ plus initial data is precisely the +**Monge–Kantorovich optimal-transport problem**. The chain (T1): Brenier's polar factorisation theorem (Brenier 1991) guarantees the +quadratic-cost optimal map is the gradient of a convex potential; Frisch, +Matarrese, Mohayaee & Sobolevski (2002) turned this into cosmology's MAK +reconstruction — recovering initial conditions from final positions *is* +quadratic-cost optimal transport. The geometry governing cosmic-web transport +is therefore **flat metric, straight rays, plus a Legendre-type +convexification of the initial potential**. Filaments and nodes are the places +where the Hopf–Lax minimiser jumps between branches — the **shock set +(caustics) of the transport map** — not geodesics of any curved or lifted +metric: structure lives in the singularities of the map, not in curved paths. + +This derivation predicts the two dynamical signatures the simulations found +(E2, E4): matter arrives at a filament along straight rays that terminate *on* +the shock set — generically **transverse** to it (direction statistic +0.21–0.30 versus the isotropic 1/3) — and after absorption only the tangential +pre-shock velocity survives: weak along-filament streaming inside the tube +(0.364 ± 0.010 within ~4 voxels of spines). + +## MUSCLE: spherical collapse as the sticking rule + +MUSCLE (MUltiscale Spherical-CoLlapse Evolution; Neyrinck 2016) keeps the ZA's +displacement structure but replaces its linear divergence $$\psi_{\mathrm{div}} = -D\,\delta_0$$ +(which never knows that collapse has happened, $$\delta_0$$ being the initial +density contrast) with the spherical-collapse remapping + +$$ +\psi_{\mathrm{sc}}(\mathbf{q}, D) \;=\; 3\left[ \sqrt{1 - \tfrac{2}{3}\,D\,\delta_0(\mathbf{q})\,} \;-\; 1 \right], +$$ + +which reaches the collapse value −3 at *finite* linear density, so collapsed +parcels stop compressing; MUSCLE evaluates the rule on the initial field +smoothed at a hierarchy of scales and declares a parcel collapsed if the +threshold is crossed at *any* scale. On identical initial conditions MUSCLE +statistically ties the frozen transverse-damping model (4.49 ± 0.12 vs +4.52 ± 0.18 voxels), winning small-scale phase fidelity while the damping +model wins mid-scale amplitudes (E9). The mechanism reading: MUSCLE-class +prescriptions work *because* they implement transverse arrest — the +correction budget that E4 measured directly. + +## The transverse-damping recipe, stated precisely + +The program's own model adds exactly one rule to the ZA, every parameter +frozen by the declared optimisation campaign (E5c, E5d; model card: docs/MODEL-CARD.md): + +- **Rays.** Evolve particles on straight ZA rays in growth-factor steps + $$\Delta D = 0.05$$ (about 18 steps; only cheap density deposits). +- **Trigger.** At each step, deposit the model's own particles to a density + grid. A particle "crosses" the first time its local density exceeds + $$\rho_c = 5$$ (in units of the mean). +- **Damping.** At first crossing, remove $$\beta = 0.6$$ (60%) of the velocity + components perpendicular to the local filament axis $$e_3$$ (minor + eigenvector of the tidal tensor of the model's own density, smoothed at + 2 h⁻¹Mpc, refreshed every third step); never touch the along-axis component. + +Three fixed numbers ($$\beta$$, $$\rho_c$$, the smoothing scale). Held-out: +4.52 ± 0.18 voxels against the ZA's 4.98 ± 0.27 (−9%), within 0.05 of the +oracle bound 4.47 (E5d, E5b). Appendix B4 walks the full evidence chain; +Appendix B2 explains the tidal frame the rule lives in. + +## Back to the series + +Back to the series: [The Geometry of the Cosmic Web: A Research Program](/mathematics/2026/07/03/geometry-of-cosmic-web-research-program/) · [Two Ways to See a Cosmic Filament](/mathematics/2026/07/04/cosmic-web-two-models/). + +
+ +
+
+

References

+
    +
  1. Ya. B. Zel'dovich (1970). "Gravitational instability: an approximate theory for large density perturbations." Astron. Astrophys. 5, 84–89.
    2. +
  2. F. R. Bouchet, S. Colombi, E. Hivon & R. Juszkiewicz (1995). "Perturbative Lagrangian approach to gravitational instability." A&A 296, 575; R. Scoccimarro (1998). "Transients from initial conditions: a perturbative analysis." MNRAS 299, 1097–1118.
    3. +
  3. S. N. Gurbatov, A. I. Saichev & S. F. Shandarin (1989). "The large-scale structure of the universe in the frame of the model equation of non-linear diffusion." MNRAS 236, 385–402.
    4. +
  4. E. Hopf (1950). "The partial differential equation ut + uux = μuxx." Comm. Pure Appl. Math. 3, 201–230; P. D. Lax (1957). "Hyperbolic systems of conservation laws II." Comm. Pure Appl. Math. 10, 537–566.
    5. +
  5. Y. Brenier (1991). "Polar factorization and monotone rearrangement of vector-valued functions." Comm. Pure Appl. Math. 44, 375–417; U. Frisch, S. Matarrese, R. Mohayaee & A. Sobolevski (2002). "A reconstruction of the initial conditions of the Universe by optimal mass transportation." Nature 417, 260–262.
    6. +
  6. M. C. Neyrinck (2016). "Truthing the stretch: non-perturbative cosmological realizations with multiscale spherical collapse (MUSCLE)." MNRAS 455, 1204.
    7. +
  7. Experiment reports T1, E2, E4, E5, E8, E9 and the model card, in research/cosmic-web/docs/ of the repository.
    8. +
+
+
diff --git a/_appendices/2026-07-07-cosmic-web-B2-tidal-frame.md b/_appendices/2026-07-07-cosmic-web-B2-tidal-frame.md new file mode 100644 index 00000000000..8981e9173e6 --- /dev/null +++ b/_appendices/2026-07-07-cosmic-web-B2-tidal-frame.md @@ -0,0 +1,204 @@ +--- +layout: distill +title: "Appendix B2 — The Tidal Frame: How Collapse Chooses Directions" +subtitle: > + The tidal tensor and its eigenframe, the ordered collapse from sheets to + filaments to nodes, the T-web and V-web classifications, why filaments point + along the weakest tidal axis — and the E4 measurement showing the correction + to ballistic transport points across that axis, not along it. +date: 2026-07-07 09:00:00 +categories: [mathematics] +tags: [cosmic-web, tidal-tensor, zeldovich, cosmology, filaments] +description: > + Self-contained primer on the tidal eigenframe of large-scale structure: + definition of the tidal tensor, anisotropic collapse and Zel'dovich + pancakes, the T-web/V-web classifications, the full E4 residual + measurement in the tidal frame, and the tidal-alignment descriptor result + on BOSS data. Companion appendix to the Geometry of the Cosmic Web series. +series: geometry-of-cosmic-web +series_title: "Geometry of the Cosmic Web" +series_part: B2 +permalink: /mathematics/2026/07/07/cosmic-web-B2-tidal-frame/ +published: true +comments: true +--- + +
+ +
+
What this appendix covers
+Everywhere in the series, directions are measured "in the tidal frame" and +filaments "point along e₃". This appendix builds that frame from scratch: +the tidal tensor and its eigenvalues, why gravitational collapse proceeds +one axis at a time (sheet → filament → node), how the T-web and V-web +classifications turn the eigenframe into a map of the web, the full E4 +measurement of the beyond-Zel'dovich correction resolved in this frame, and +the one statistic on which the orientation-lift method beats the simpler +Hessian detector — on simulations and on the real sky. Numbers cite the +experiment reports in research/cosmic-web/docs/ by name. +
+ +## The tidal tensor and its eigenframe + +Let $$\Phi(\mathbf{x})$$ be the peculiar gravitational potential — the part +of the potential sourced by density *fluctuations* $$\delta$$ about the +cosmic mean, via the Poisson equation $$\nabla^2 \Phi \propto \delta$$. +The **tidal tensor** is its Hessian, the matrix of second derivatives: + +$$ +T_{ij}(\mathbf{x}) \;=\; \frac{\partial^{2} \Phi}{\partial x_i\, \partial x_j}. +$$ + +It records how the gravitational pull *differs* across a small region — the +stretching and squeezing that a cloud of test particles feels. Being +symmetric, it has an orthonormal **eigenframe**: eigenvectors $$e_1, e_2, e_3$$ +with eigenvalues ordered $$\lambda_1 \ge \lambda_2 \ge \lambda_3$$. +A positive eigenvalue means +matter is being compressed along that eigenvector; the ordering says the +squeeze is strongest along $$e_1$$ and weakest along $$e_3$$. + +## Collapse happens one axis at a time + +The Zel'dovich approximation (Appendix B1) makes the consequence exact. Its +map $$\mathbf{x} = \mathbf{q} - D\,\nabla_q \Phi_0(\mathbf{q})$$ has Jacobian + +$$ +\frac{\partial x_i}{\partial q_j} \;=\; \delta_{ij} \;-\; D\,\frac{\partial^{2} \Phi_0}{\partial q_i\, \partial q_j}, +$$ + +whose determinant — the inverse of the local density — first vanishes +when $$D\,\lambda_1 = 1$$. Collapse therefore happens **first +along** $$e_1$$, the strongest-squeeze axis, flattening the cloud into a sheet (a +Zel'dovich "pancake"); then along $$e_2$$, draining the sheet into a +filament; and last along $$e_3$$, pooling the filament into a node. A +filament is the structure that exists *after* two collapses and *before* +the third: it is extended along $$e_3$$, the axis gravity squeezes last. +Bond, Kofman & Pogosyan (1996) showed the resulting filamentary pattern is +already encoded in the initial tidal field around proto-clusters — hence +"cosmic web". + +## The T-web and V-web classifications + +The eigenframe gives a pointwise map of the web. The **T-web** (Hahn et +al. 2007; Forero-Romero et al. 2009) counts how many eigenvalues +of $$T_{ij}$$ exceed a threshold $$\lambda_{\mathrm{th}}$$: + +| eigenvalues above threshold | environment | axis carried | +|---|---|---| +| 0 | void | — | +| 1 | sheet | normal along e₁ | +| 2 | filament | spine along e₃ | +| 3 | node | — | + +The **V-web** (Hoffman et al. 2012) applies the same counting to the +velocity shear tensor + +$$ +\Sigma_{ij} \;=\; -\frac{1}{2H_0} \left( \frac{\partial v_i}{\partial x_j} + \frac{\partial v_j}{\partial x_i} \right), +$$ + +with $$H_0$$ the Hubble constant, which resolves finer structure because +the velocity field is smoother than the density but responds to the same +tides. Libeskind et al. (2018) compared twelve web finders on one +simulation and found substantial disagreement at filament boundaries and +junctions — the observation that motivated the series' benchmark in the +first place. + +## Why the measurement had to be made in this frame + +The series' physical hypothesis (H4 and its refinement H4') claimed the +lifted geometry supplies a correction to standard transport that is *aligned +with filaments*: cheap motion along $$e_3$$. The tidal frame is the only +frame in which that claim is falsifiable — an isotropic statistic would +average the signature away. Experiment E4 therefore measured, for every +tracked particle, the residual between truth and the particle's own +Zel'dovich prediction from the same initial conditions, + +$$ +\mathbf{R} \;=\; \mathbf{x}_{\mathrm{true}}(a{=}1) \;-\; \mathbf{x}_{\mathrm{ZA}}(a{=}1), +$$ + +and its velocity analogue $$\mathbf{R}_v$$, projected both onto the local +eigenframe, and binned by distance to the filament spine network. The +direction statistic is the squared projection of the unit residual on the +filament axis, with isotropic null 1/3: values above 1/3 mean the +correction points *along* filaments (the H4' prediction), below it +*across* them. + +## The E4 measurement in full + +Six particle-mesh (PM) N-body simulations (128³, Einstein–de Sitter), +50,000 tracked particles each; mean ± across-seed standard deviation; +distances in voxels of 1 h⁻¹Mpc (E4): + +| d to spine (vox) | n per seed | ⟨(R̂·e₃)²⟩ | ⟨(R̂ᵥ·e₃)²⟩ | RMS R∥ (vox) | RMS R⊥ per axis (vox) | R∥/R⊥ | +|---|---|---|---|---|---|---| +| 0–2 | 20,658 | 0.298 ± 0.002 | 0.312 ± 0.003 | 4.36 | 4.92 | 0.89 | +| 2–4 | 7,269 | 0.297 ± 0.004 | 0.300 ± 0.004 | 3.77 | 4.19 | 0.90 | +| 4–8 | 6,632 | 0.264 ± 0.004 | 0.280 ± 0.003 | 2.40 | 3.07 | 0.78 | +| 8–16 | 10,320 | 0.218 ± 0.002 | 0.218 ± 0.003 | 1.86 | 2.91 | 0.64 | +| 16–64 | 5,120 | 0.208 ± 0.003 | 0.203 ± 0.003 | 1.77 | 2.83 | 0.63 | + +Column key: ⟨(R̂·e₃)²⟩ and ⟨(R̂ᵥ·e₃)²⟩ are the mean squared projections of +the unit position and velocity residuals on the filament axis (isotropic +null 1/3); RMS R∥ is the root-mean-square residual component along e₃; +RMS R⊥ per axis is the same for each perpendicular axis; R∥/R⊥ is their +ratio (1 = isotropic correction, above 1 = along-filament, below 1 = +across-filament). + +Three facts follow. **The correction is large where structure forms** — +RMS ≈ 4.4 voxels within 2 voxels of spines. **It is organised by the tidal +frame** — every entry deviates strongly from isotropy. **And it points +across the filament axis at every distance**: the direction statistic sits +at 0.21–0.30, below the 1/3 null everywhere, and R∥/R⊥ = 0.63–0.89 with no +bin reaching 1. Physically: ballistic transport overshoots through forming +walls and filaments, and the correction real gravity applies is +**transverse arrest at the web** — the adhesion model's viscosity made +anisotropic by the tidal frame — not enhanced transport along it. H4' is +refuted as stated, and the measured form of the true correction became the +model of Appendix B4. The result is consistent with the model-independent +precursor (E2: bulk deviations-from-chord at 0.21–0.23 versus 1/3) and +with the theory note's shock picture (T1). + +## The tidal frame as an anisotropy judge + +The eigenframe also grades filament *finders*. Define the alignment +statistic (M3): for spine samples with unit tangent $$\hat{t}_i$$ at +points $$\mathbf{x}_i$$, + +$$ +A \;=\; \bigl\langle\, \lvert \hat{t}_i \cdot e_3(\mathbf{x}_i) \rvert \,\bigr\rangle, \qquad \mathbb{E}[A] = \tfrac{1}{2} \ \text{under an isotropic null}. +$$ + +Neither detector is shown the tidal field; A measures how well each one's +spines *recover* the frame from density data alone. On simulations, the +orientation-lift spines align at 0.734–0.759 versus the Hessian detector's +0.665–0.674 (E1). On the real Universe the separation replicates: across +18 independent tiles of BOSS CMASS galaxies stacked in the E3 campaign, +the lift reaches **0.677 ± 0.016** versus the Hessian's **0.617 ± 0.016** +(null 0.5) — a clean, tile-replicated win (E3). This is the lifted +geometry's one surviving advantage, and the theory explains why it is a +*descriptor* rather than a mechanism: the orientation manifold captures the +tangent structure of the already-formed shock set (T1), even though the +transport that builds it flows across, not along, the filaments. + +## Back to the series + +Back to the series: [The Geometry of the Cosmic Web: A Research Program](/mathematics/2026/07/03/geometry-of-cosmic-web-research-program/) · [Two Ways to See a Cosmic Filament](/mathematics/2026/07/04/cosmic-web-two-models/). + +
+ +
+
+

References

+
    +
  1. Ya. B. Zel'dovich (1970). "Gravitational instability: an approximate theory for large density perturbations." Astron. Astrophys. 5, 84–89.
    2. +
  2. J. R. Bond, L. Kofman & D. Pogosyan (1996). "How filaments of galaxies are woven into the cosmic web." Nature 380, 603–606.
    3. +
  3. O. Hahn, C. Porciani, C. M. Carollo & A. Dekel (2007). "Properties of dark matter haloes in clusters, filaments, sheets and voids." MNRAS 375, 489–499.
    4. +
  4. J. E. Forero-Romero et al. (2009). "A dynamical classification of the cosmic web." MNRAS 396, 1815–1824.
    5. +
  5. Y. Hoffman et al. (2012). "A kinematic classification of the cosmic web." MNRAS 425, 2049–2057.
    6. +
  6. N. I. Libeskind et al. (2018). "Tracing the cosmic web." MNRAS 473, 1195–1217.
    7. +
  7. Experiment reports E1, E2, E3, E4 and the theory note T1, in research/cosmic-web/docs/ of the repository.
    8. +
+
+
diff --git a/_appendices/2026-07-08-cosmic-web-B3-honest-benchmarks.md b/_appendices/2026-07-08-cosmic-web-B3-honest-benchmarks.md new file mode 100644 index 00000000000..a1ddfddd1c5 --- /dev/null +++ b/_appendices/2026-07-08-cosmic-web-B3-honest-benchmarks.md @@ -0,0 +1,197 @@ +--- +layout: distill +title: "Appendix B3 — How to Grade a Model Honestly" +subtitle: > + Every evaluation failure the research program hit, told technically: the + matched-length artifact and its sign flip, reference circularity, held-out + seed discipline, jackknife versus pair bootstrap, physically-null controls, + oracle bounds as information limits — and two byte-identical datasets + caught by hashing. +date: 2026-07-08 09:00:00 +categories: [mathematics] +tags: [cosmic-web, methodology, statistics, benchmarks, data-analysis] +description: > + The evaluation-methodology appendix of the Geometry of the Cosmic Web + series: proxy matching that manufactured a p ≈ 1e-14 false positive, + circular reference skeletons, competitor-favouring calibration, honest + error bars for overlapping data, null controls that exposed beam leakage, + oracle bounds, and dataset provenance checks. +series: geometry-of-cosmic-web +series_title: "Geometry of the Cosmic Web" +series_part: B3 +permalink: /mathematics/2026/07/08/cosmic-web-B3-honest-benchmarks/ +published: true +comments: true +--- + +
+ +
+
What this appendix covers
+The research program's most transferable products are not cosmological. +They are the evaluation failures it committed, caught, and repaired — each +one a general lesson in how a plausible protocol manufactures a discovery. +This appendix tells them technically: the matched-length artifact (a +p ≈ 10⁻¹⁴ result with the wrong sign), reference circularity in skeleton +scoring, seed discipline and competitor-favouring calibration, why pair +bootstrap overstated a detection by a factor of 1.6, physically-null +controls that exposed a 4.7σ instrumental ghost, oracle bounds that +separate physics from estimation error, and a dataset-provenance check +that caught two "independent" simulations being byte-identical. Numbers +cite the experiment reports in research/cosmic-web/docs/. +
+ +## Match on the quantity that buys the score + +The program's benchmark compared two filament detectors by +**completeness**: the fraction of the true filament network lying within +2 voxels of the recovered skeleton. Completeness has a mechanical +confounder — a longer skeleton covers more of the truth regardless of +quality — so both methods were required to output skeletons of equal total +length. The first implementation enforced that match on a **proxy**: the +volume of the hysteresis mask from which each method's skeleton is then +extracted, on the assumption that realised skeleton lengths would follow. + +They did not. At sparse sampling the orientation-lift's realised skeletons +came out about 19% longer than the Hessian baseline's — **1,468 versus +1,239 voxels** — and the resulting "+0.07 completeness, 48/50 seeds, +p ≈ 10⁻¹⁴" advantage was pure length. Under the corrected extractor, which +iterates until the realised skeleton itself hits the target length, the +**sign flips**: the Hessian wins completeness at every sampling level from +2,500 galaxies up (Δ = −0.045 to −0.066, 50/50 seeds, p ≈ 10⁻¹⁵), with a +statistical tie only at ultra-sparse sampling (SYNTHESIS; PAPER-DRAFT +§5.1). The artifact was detected only because a later instrument change +retroactively shifted archived scores, and the archived length fields +confirmed the mismatch. + +The lesson generalises beyond skeletons: *a matched comparison must +enforce the match on the quantity that determines the score, not on a +proxy upstream of it — and statistical strength (p ≈ 10⁻¹⁴) certifies +nothing about whether the comparison was constructed correctly.* + +## Reference circularity + +When no analytic truth exists, the temptation is to score each method's +sparse-data skeleton against a *reference skeleton* extracted from the +clean, noise-free field. The program's E1 ran the full 2×2 design — each +method's sparse skeleton against each method's clean reference — and found +each method scores **0.07–0.19 higher against the reference produced by +its own family**; the two clean references agree with each other only at +completeness ≈ 0.60 (E1, 12 seeds). Any single-reference comparison is +therefore circular: the choice of reference decides the winner before the +data are consulted. + +The repair is a **method-neutral criterion** that needs no reference at +all: which skeleton's spines capture more mass (total, and in the +filament-density band) at matched length (E1b). Under that criterion the +verdicts stabilised across field types and resolutions. + +## Held-out seeds and competitor-favouring calibration + +Every tunable choice is an opportunity to overfit the benchmark. The +transport-model campaign used a three-tier seed protocol: the crossing +threshold $$\rho_c$$ was calibrated on seed 1 **by optimising the +isotropic competitor**, not the proposed model; configuration selection +used seeds {2, 3}; every reported number comes from held-out seeds +({4, 5, 6}, or the five held-out seeds of E5), with jackknife-over-seed +error bars (E5–E5d). Calibrating on the competitor is the cheap trick that +makes a win conservative: any residual bias in the shared knob favours the +opponent. When the frozen model then beats the ZA by −9% on seeds it never +saw, the number means something. + +## Jackknife versus pair bootstrap + +The observational campaign stacked Compton-y maps at 876,639 galaxy pairs. +Bootstrap over pairs treats each pair as independent — but pairs share +galaxies and overlap on the sky, so their map noise is correlated, and the +bootstrap error is too small. The validity battery (E3e) replaced it with +a **jackknife over 50 right-ascension patches**: delete one sky patch at a +time, remeasure, and read the variance from the spread. The deflation was +a factor 1.6–2.4 across instruments: the headline bridge significance on +ACT dropped from **5.24σ (pair bootstrap) to 3.30σ (jackknife)**, and +Planck's from 19.3σ to 7.99σ. Rule: resample units that are actually +exchangeable — sky patches, not overlapping pairs. + +## Physically-null controls + +An estimator can pass every statistical test and still measure an +instrumental artifact. The decisive check is a **control sample in which +the physics is absent by construction but every systematic is present**. +For the pair-bridge measurement: pairs with the same transverse separation +window but line-of-sight separation 25–40 h⁻¹Mpc — same sky geometry, same +halos, no possible physical bridge between them. On ACT the null pairs are +consistent with zero (1.65σ). On Planck they show a **4.7σ "bridge"** +(1.80×10⁻⁸) — direct, quantitative confirmation of second-halo beam +leakage at Planck's 10′ resolution (E3e; Appendix B5 explains the +mechanism). Without the null, that leakage would have been booked as +astrophysics. The physically meaningful number is the null-subtracted one: +~1.2–1.4×10⁻⁸ at ≈2σ per instrument. + +The same discipline applied on the sky in an earlier form: the spine-stack +controls were rejection-matched to the spine points' galactic-latitude +histogram, so a latitude-dependent foreground could not inflate the +detection (E3b) — and the stricter control *strengthened* the result, from +4.8σ to 9.0σ. Stricter controls do not always shrink signals; they shrink +lies. + +## Oracle bounds as information limits + +When a model underperforms, is the physics wrong or the estimation noisy? +The program answered with an **oracle**: rerun the transverse-damping +model with tidal frames computed from the *true* final N-body field +instead of the model's own evolving estimate (E5b). With oracle frames the +model beats the Zel'dovich baseline in **every** distance bin (−20% at the +web; 4.47 versus 5.00 overall), so the damping physics is uniformly +correct, and the practical model's small outskirts deficit was frame +estimation, not wrong physics. The oracle also *prices* the remaining +headroom — 0.46 voxels of recoverable error, more than any other +refinement lever — and grades the final model: the frozen recipe's 4.52 +closes 89% of the gap to the 4.47 bound (E5d). An oracle bound converts +"could do better" into a number. + +## Trust the bytes: the CAMELS twins + +The referee battery required validating the program's own N-body truth +against two independent production codes. The second dataset chosen, +SIMBA_DM CV_0 from the public CAMELS suite, was checked before use with a +range-request hash comparison — and found **byte-identical** to the first, +IllustrisTNG_DM CV_0: the dark-matter-only "CV_0" runs of those two suites +are one shared simulation on the server (E12). Astrid_DM CV_0 (run with +MP-Gadget) was genuinely distinct and became the second code; the two real +codes agree with each other to 0.029 Mpc/h median per particle, and the +program's particle-mesh truth sits 0.41 Mpc/h from that consensus — an +order of magnitude below the 4–5 Mpc/h effects measured (E12). The lesson: +"independent dataset" is a hypothesis, and it is testable for pennies — +hash before you validate. + +## The checklist + +- Match comparisons on the score-determining quantity itself, never a proxy. +- Never score against a reference one contestant produced; prefer + reference-free criteria. +- Calibrate shared knobs by optimising the *competitor*; select and + validate on disjoint held-out seeds. +- Resample exchangeable units (sky patches), not overlapping ones (pairs). +- Run a control in which the physics is absent by construction; subtract + what survives it. +- Buy an oracle bound: it separates wrong physics from noisy estimation + and prices the remaining headroom. +- Hash "independent" datasets before trusting their independence. + +## Back to the series + +Back to the series: [The Geometry of the Cosmic Web: A Research Program](/mathematics/2026/07/03/geometry-of-cosmic-web-research-program/) · [Two Ways to See a Cosmic Filament](/mathematics/2026/07/04/cosmic-web-two-models/). + +
+ +
+
+

References

+
    +
  1. F. Wilcoxon (1945). "Individual comparisons by ranking methods." Biometrics Bulletin 1, 80–83. The paired signed-rank test used throughout the benchmark.
    2. +
  2. J. W. Tukey (1958). "Bias and confidence in not-quite large samples." Ann. Math. Statist. 29, 614; B. Efron (1979). "Bootstrap methods: another look at the jackknife." Ann. Statist. 7, 1–26.
    3. +
  3. F. Villaescusa-Navarro et al. (2021). "The CAMELS project." ApJ 915, 71. Source of the external-validation simulations, including the byte-identical CV_0 twins.
    4. +
  4. Experiment reports SYNTHESIS, E1, E1b, E5–E5d, E3b, E3e, E12 and the paper draft, in research/cosmic-web/docs/ of the repository.
    5. +
+
+
diff --git a/_appendices/2026-07-09-cosmic-web-B4-transverse-damping.md b/_appendices/2026-07-09-cosmic-web-B4-transverse-damping.md new file mode 100644 index 00000000000..21144970691 --- /dev/null +++ b/_appendices/2026-07-09-cosmic-web-B4-transverse-damping.md @@ -0,0 +1,220 @@ +--- +layout: distill +title: "Appendix B4 — The Transverse-Damping Model, in Full" +subtitle: > + The complete evidence chain behind the program's constructive result: the + measured correction (E4), the first model and its oracle (E5, E5b), the + frozen recipe (E5c/E5d), transfer with frozen knobs (E6, E7), field-level + fidelity (E8), literature baselines (E9), and the two-code truth + validation — with limitations and a reproduction map. +date: 2026-07-09 09:00:00 +categories: [mathematics] +tags: [cosmic-web, zeldovich, adhesion, effective-models, cosmology] +description: > + Full specification and evidence chain of the anisotropic-adhesion + ("transverse damping") transport model: frozen recipe, model ladder against + the oracle bound, MUSCLE and 2LPT baselines, transfer across clustering, + resolution and cosmology, field-level r(k)/T(k), truth validation against + Arepo and MP-Gadget, limitations, and Makefile reproduction targets. +series: geometry-of-cosmic-web +series_title: "Geometry of the Cosmic Web" +series_part: B4 +permalink: /mathematics/2026/07/09/cosmic-web-B4-transverse-damping/ +published: true +comments: true +--- + +
+ +
+
What this appendix covers
+The series' constructive product is a one-ingredient transport model: +Zel'dovich rays plus partial damping of the velocity components +perpendicular to the local filament axis at first shell-crossing. This +appendix states the frozen recipe precisely and walks the entire evidence +chain — E4 → E5 → E5b → E5c/E5d → E6/E7 → E8 → E9 — then the validation of +the truth itself against two production codes, the model's limitations, and +how to reproduce every number. All reports live in +research/cosmic-web/docs/; the model card is +MODEL-CARD.md. +
+ +## The recipe, frozen + +Input: a linear initial density field (equivalently its Zel'dovich +displacement field) on a periodic grid. Output: approximate comoving +particle positions at the present epoch, at a small fraction of an N-body +run's cost (~18 cheap steps versus 90 force-solving steps). The rule +(E5d; MODEL-CARD): + +- Evolve particles on straight Zel'dovich rays in growth-factor steps + $$\Delta D = 0.05$$; no Poisson solves beyond cheap density deposits. +- At each step, deposit the model's own particles to a density grid. +- The first time a particle's local density exceeds $$\rho_c = 5$$, remove + $$\beta = 60\%$$ of its velocity components perpendicular to the local + filament axis $$e_3$$ — the minor eigenvector of the tidal tensor of the + model's own density, smoothed at 2 h⁻¹Mpc and refreshed every third + step. The along-axis component is never touched. + +Three fixed numbers ($$\beta$$, $$\rho_c$$, the smoothing scale). Two +structural findings (E5c): partial damping ($$\beta < 1$$) is *required* +for self-estimated frames to pay — at full damping the model's own density +feedback over-triggers crossings and cancels the frame gain — and +pancake-ordered sequential damping underperforms both-perpendicular damping. + +## The evidence chain, experiment by experiment + +- **E4 — the measurement.** The residual between true particle-mesh (PM) + N-body transport and each particle's own Zel'dovich (ZA) prediction is + large near the web (RMS ≈ 4.4 voxels within 2 voxels of spines) and + points *across* the filament axis at every distance: the needed + correction is transverse arrest (full table in Appendix B2). +- **E5 — the first model.** ZA rays + full transverse damping with proxy + frames: best of three models overall (4.91 versus ZA 5.00 and isotropic + sticking 5.34), clearest at the web, with a small 2–4-voxel deficit. +- **E5b — the oracle.** Same model with tidal frames from the *true* final + field: beats ZA in every distance bin (4.47 overall, −20% at the web). + The E5 deficit was frame-estimation error; 0.46 voxels of headroom priced. +- **E5c/E5d — declared optimisation.** Selection on seeds {2, 3}, + validation on held-out {4, 5, 6}. Winner: self-density frames, β = 0.6, + 2 h⁻¹Mpc smoothing — 4.52 ± 0.18, within 0.05 of the oracle bound. +- **E6/E7 — transfer.** Frozen knobs across voxel size, clustering + amplitude and cosmology: the advantage persists everywhere and grows + with clustering. +- **E8 — field level.** Extends the usable wavenumber range of a ZA-based + mock on both phases and amplitudes; isotropic sticking destroys phases. +- **E9 — baselines.** On identical initial conditions: statistical tie + with MUSCLE, 2LPT degrades badly. The contribution is the mechanism, + not a better engine. + +## The model ladder + +Median per-particle transport error against PM truth, in voxels (= h⁻¹Mpc), +held-out seeds — ZA reference 4.98 ± 0.27 on seeds {4, 5, 6} (E5–E5d, E9): + +| stage | configuration | overall error (vox) | vs ZA | +|---|---|---|---| +| isotropic sticking (adhesion proxy) | full damping, all directions | 5.34 ± 0.15 | +7% | +| ZA baseline | straight rays, no damping | 4.98–5.00 | — | +| E5: first transverse model | β = 1, ZA-proxy frames | 4.91 ± 0.14 | −2% | +| E5c: refined | β = 0.75, self-density frames | 4.64 ± 0.18 | −7% | +| E5d: frozen recipe | β = 0.6, 2 h⁻¹Mpc frames | **4.52 ± 0.18** | **−9%** | +| E5b: oracle bound | true final-field frames | 4.47 ± 0.12 | −10% | +| MUSCLE (E9) | multiscale spherical collapse | 4.49 ± 0.12 | tie with frozen recipe | +| 2LPT (E9) | second-order perturbation theory | 8.07 ± 0.39 | +62% | + +The frozen recipe closes 89% of the recoverable gap to the oracle. The +largest gains sit where the correction was measured: at 0–2 voxels from +spines the model reaches 5.77 ± 0.16 against ZA's 6.92 ± 0.31 (−17%), with +the oracle showing −20% available (E5b, E9). MUSCLE is slightly ahead at +the web (5.57 ± 0.07); the reading of the tie is the mechanism result — +a model with *only* the measured directional ingredient reproduces +MUSCLE-class transport, so shell-crossing prescriptions work because they +implement transverse arrest. + +## Transfer with frozen knobs + +No re-calibration anywhere; resolution rows in physical h⁻¹Mpc (E6, E7, +E7b): + +| condition | seeds | ZA all | model all | Δ all | ZA web | model web | Δ web | +|---|---|---|---|---|---|---|---| +| base (EdS, σ₈ = 0.8, 1 h⁻¹Mpc vox) | 3 | 4.98 | 4.52 | −9% | 6.93 | 5.78 | −17% | +| coarse (2 h⁻¹Mpc voxels) | 3 | 2.09 | 2.04 | −2% | 3.37 | 3.08 | −9% | +| σ₈ = 0.6 | 3 | 2.98 | 2.89 | −3% | 4.73 | 4.26 | −10% | +| σ₈ = 1.0 | 3 | 7.20 | 6.23 | −13% | 9.36 | 7.33 | −22% | +| flat ΛCDM, Ωm = 0.31 | 3 | 4.93 | 4.45 | −10% | 6.90 | 5.73 | −17% | +| 0.5 h⁻¹Mpc voxels (EdS) | 1 (+2 in E7b) | 4.95 | 4.45 | −10% | 6.76 | 5.72 | −15% | + +Here σ₈ is the clustering amplitude of the initial conditions, "EdS" is +Einstein–de Sitter expansion, and "web" means within 2 h⁻¹Mpc of the spine +network. The advantage grows monotonically with clustering (−3% → −9% → +−13% overall) — the behaviour of a physical shell-crossing correction, +since higher σ₈ means more crossings — and is nearly identical between EdS +and ΛCDM, as expected for a growth-factor-parametrised geometric term; +physical-unit errors at 0.5 and 1 h⁻¹Mpc voxels match, so the error scale +is set by the physics, not the grid. + +## Field-level fidelity + +Density fields against PM truth, 3 seeds (E8, E9). The cross-correlation r(k) +measures phase/structure fidelity at wavenumber k, in h/Mpc (1 = perfect); +the transfer function T(k) measures amplitude fidelity (1 = unbiased). + +| k | r: ZA | r: 2LPT | r: MUSCLE | r: damp | T: ZA | T: 2LPT | T: MUSCLE | T: damp | +|---|---|---|---|---|---|---|---|---| +| 0.14 | 0.975 | 0.933 | 0.956 | 0.962 | 0.693 | 0.607 | 0.669 | 0.739 | +| 0.34 | 0.710 | 0.691 | 0.771 | 0.716 | 0.229 | 0.202 | 0.433 | 0.410 | +| 0.53 | 0.288 | 0.430 | 0.527 | 0.388 | 0.098 | 0.095 | 0.305 | 0.262 | +| 0.82 | 0.057 | 0.144 | 0.228 | 0.095 | 0.052 | 0.048 | 0.210 | 0.143 | + +The damping model beats ZA on both statistics in the nonlinear regime; +isotropic sticking (not shown) collapses to r = 0.089 at k = 0.53 (E8). +Against MUSCLE the strengths are complementary — MUSCLE wins small-scale +phases, the damping model mid-scale amplitudes — so a field-level hybrid is +the natural future work; the naive trigger-level hybrid was tested and +refuted (4.67, worse than both parents; E11). + +## Is the truth true? Two-code validation + +All errors above are measured against the program's own PM integrator, so +the truth itself was audited. Time convergence: doubling to 180 steps +shifts the median final position by 0.008 voxels and leaves the comparison +invariant (E10). External validation: evolved from the CAMELS CV_0 initial +conditions, the PM integrator reproduces the official final particle +positions to a median matched-ID offset of **0.41 Mpc/h** against both +Arepo and MP-Gadget — and those two production codes agree with each other +to **0.029 Mpc/h** median (E12). The hierarchy is what matters: code +consensus 0.03 ≪ our PM offset 0.41 ≪ measured effects 4–5 Mpc/h. + +## Limitations + +From the model card (MODEL-CARD), quantified: + +- **One-shot damping.** Only the first crossing is treated; multi-stream + interiors are out of scope by design. Residual web error (5.5–6 voxels + even at the oracle bound) is post-crossing physics; do not interpret + positions within ~2 voxels of density peaks as resolved halo structure. +- **Verified scope.** Frozen knobs verified for voxels 0.5–2 h⁻¹Mpc, + σ₈ ∈ [0.6, 1.0], EdS and flat ΛCDM (Ωm = 0.31). Beyond that, and at + other box sizes or tracer densities, re-calibrate β and ρ_c against a + small PM truth, optimising the competitor first (Appendix B3); the β + optimum is a flat minimum possibly slightly below 0.6. +- **Web-region correction, not global.** Gains far from the web are −2%. +- **Large-scale amplitude.** ~7% T(k) deficit in the largest-scale bin; + rescale before use in mocks. + +## Reproduce + +Every table above regenerates from research/cosmic-web/ +(Python virtualenv per requirements.txt): + +```bash +cd research/cosmic-web +make residual # E4: the measured correction (Appendix B2 table) +make model-ladder # E5, E5b, E5c, E5d: the ladder and the oracle +make baselines # E9: 2LPT and MUSCLE on identical ICs +make transfer # E6, E7, E7b: frozen-knob transfer matrix +make field-level # E8: r(k) and T(k) +make truth-validation # E10 convergence; E12 external checks +``` + +## Back to the series + +Back to the series: [The Geometry of the Cosmic Web: A Research Program](/mathematics/2026/07/03/geometry-of-cosmic-web-research-program/) · [Two Ways to See a Cosmic Filament](/mathematics/2026/07/04/cosmic-web-two-models/). + +
+ +
+
+

References

+
    +
  1. Ya. B. Zel'dovich (1970). "Gravitational instability: an approximate theory for large density perturbations." Astron. Astrophys. 5, 84–89.
    2. +
  2. S. N. Gurbatov, A. I. Saichev & S. F. Shandarin (1989). "The large-scale structure of the universe in the frame of the model equation of non-linear diffusion." MNRAS 236, 385–402.
    3. +
  3. M. C. Neyrinck (2016). "Truthing the stretch: non-perturbative cosmological realizations with multiscale spherical collapse (MUSCLE)." MNRAS 455, 1204.
    4. +
  4. F. R. Bouchet, S. Colombi, E. Hivon & R. Juszkiewicz (1995). "Perturbative Lagrangian approach to gravitational instability." A&A 296, 575.
    5. +
  5. F. Villaescusa-Navarro et al. (2021). "The CAMELS project." ApJ 915, 71.
    6. +
  6. Model card (MODEL-CARD.md), experiment reports E4–E12 and the paper draft, in research/cosmic-web/docs/ of the repository.
    7. +
+
+
diff --git a/_appendices/2026-07-10-cosmic-web-B5-reading-gas-maps.md b/_appendices/2026-07-10-cosmic-web-B5-reading-gas-maps.md new file mode 100644 index 00000000000..db0679334b3 --- /dev/null +++ b/_appendices/2026-07-10-cosmic-web-B5-reading-gas-maps.md @@ -0,0 +1,192 @@ +--- +layout: distill +title: "Appendix B5 — Reading the Sky's Hot Gas" +subtitle: > + The thermal Sunyaev–Zel'dovich effect and Compton-y maps, stacking as a + detection strategy, controls that match the sky's systematics, the 9σ spine + detection and its halo/WHIM decomposition, pair-bridge estimators that + cancel halos by symmetry, beam leakage — and the honest final numbers. +date: 2026-07-10 09:00:00 +categories: [mathematics] +tags: [cosmic-web, tsz, compton-y, planck, act, observations] +description: > + Observational appendix of the Geometry of the Cosmic Web series: how + Compton-y maps encode hot gas, how stacking extracts signals of 1e-8 from + 1e-6 maps, why controls must match galactic-latitude exposure, the 9-sigma + spine-stack detection and its decomposition, the halo-symmetric pair-bridge + estimator, the Planck beam-leakage lesson, and the calibrated bridge + amplitude of about 1.2–1.4e-8 at roughly 2 sigma per instrument. +series: geometry-of-cosmic-web +series_title: "Geometry of the Cosmic Web" +series_part: B5 +permalink: /mathematics/2026/07/10/cosmic-web-B5-reading-gas-maps/ +published: true +comments: true +--- + +
+ +
+
What this appendix covers
+The series' observational campaign asked whether the filament networks +extracted from galaxy surveys sit on real hot gas. This appendix supplies +everything needed to read those results: the physics of the thermal +Sunyaev–Zel'dovich effect in one paragraph, stacking as a +signal-to-noise strategy, why control samples must match the sky's +systematics (the galactic-latitude lesson), the 9σ spine detection and its +decomposition into halo gas versus gas between halos, the pair-bridge +estimator that cancels halos by symmetry, the beam-leakage lesson from +comparing Planck and ACT, and the final calibrated numbers. Numbers cite +the experiment reports (E3, E3b, E3c, E3d, E3e) in +research/cosmic-web/docs/. +
+ +## The thermal Sunyaev–Zel'dovich effect, in one paragraph + +Photons of the cosmic microwave background (CMB) crossing a pocket of hot +ionised gas occasionally scatter off its free electrons and pick up a +little energy — inverse Compton scattering. The distortion this imprints +on the CMB spectrum is the **thermal Sunyaev–Zel'dovich (tSZ) effect** +(Sunyaev & Zel'dovich 1972), and its amplitude at each sky position is the +**Compton-y parameter**: the line-of-sight integral of the electron +pressure, + +$$ +y \;=\; \frac{\sigma_{\mathrm{T}}}{m_e c^{2}} \int P_e \, dl , \qquad P_e = n_e k_{\mathrm{B}} T_e , +$$ + +with $$\sigma_{\mathrm{T}}$$ the Thomson cross-section, $$m_e c^2$$ the +electron rest energy, $$n_e$$ the electron density and $$T_e$$ the electron +temperature. y is dimensionless and, usefully, independent of redshift: a +parcel of hot gas contributes the same y whether nearby or distant. Galaxy +clusters reach y ~ 10⁻⁴–10⁻⁵; the warm–hot intergalactic medium (WHIM) +expected in filaments sits near y ~ 10⁻⁸ — a hundred times below the noise +per pixel of the all-sky Planck y map. Both maps used in the series are +component-separated y maps: Planck (10′ beam) and the Atacama Cosmology +Telescope's ACT DR6 (1.6′ beam), where the beam is the instrument's +angular resolution. + +## Stacking: seeing 10⁻⁸ in a 10⁻⁶ map + +No single filament is visible, so the campaign averages the map over many +sky positions selected by an external catalogue — **stacking**. Averaging +N roughly independent positions beats the noise down by $$\sqrt{N}$$, but +the zero point of a y map is not trustworthy, so the measurement is always +*differential*: the stacked value minus the same stack on **control +positions**, with significance defined as + +$$ +\mathrm{SNR} \;=\; \frac{\text{stack} - \langle \text{stack}_{\mathrm{ctrl}} \rangle}{\sigma_{\mathrm{ctrl}}} +$$ + +over hundreds of control realisations. Everything then hinges on whether +the controls are exposed to the same systematics as the signal positions. + +## Controls must match the sky's systematics + +The first pass (E3) stacked spine networks extracted from 274,075 BOSS +CMASS-North galaxies (18 tiles of 512 h⁻¹Mpc, spines at matched total +length per method) on the Planck y map, against 200 footprint-constrained +controls built by rotating in right ascension and reflecting in +declination: 4.8σ (Hessian spines) and 4.1σ (orientation-lift spines). +But those transformations change **galactic latitude** — and galactic +foregrounds (dust, and the residual it leaves in a y map) vary with +latitude, so signal and controls sampled different foreground exposure. +The second pass (E3b) drew 200 control sets rejection-matched to the spine +points' galactic-latitude histogram in 2° bins, so no latitude-dependent +foreground can contribute. The detection **strengthened**: 8.99σ (Hessian) +and 6.86σ (lift). A stricter control does not necessarily shrink a real +signal — it removes a variance term that was diluting it. + +## The 9σ spine detection, decomposed + +At 8.99σ the extracted web demonstrably sits on hot gas — but *whose* gas? +The survey galaxies themselves carry hot halos, so E3b masked a 7′ disc +around every catalogue galaxy, identically in signal and controls. The +excess collapsed to 1.95σ (Hessian) and 0.07σ (lift): the spine-stack +signal is **dominantly the tracers' own halo gas**, with at most a hint of +a between-halos (WHIM) component. The radial profile extends to ≥60′ but +cannot discriminate extended filament gas from clustered halo +contributions at a 10′ beam (E3b). Two conclusions: the spine networks are +physically real (they trace hot gas at high significance), and isolating +*filament* gas needs a sharper instrument and a different estimator. + +## The beam sets what you can see + +The obvious sharper instrument, ACT DR6 at 1.6′, at first made things +*worse*: the same spine stack dropped to 2.95σ unmasked (versus Planck's +9σ on the overlapping footprint), and the masked residual stayed below the +pre-registered 3σ gate (1.77σ at 3′ masking; E3c). The diagnosis, not a +mere excuse, is a scale mismatch: the spines are projected through a +~250 h⁻¹Mpc redshift shell, so their stacked signal is coherent on +*degree* scales — exactly the scales a ground-based map's filtering +suppresses, while it excels at arcminutes. The lesson: an analysis design +must place its signal at the angular scales its instrument preserves. +E3c's prescription — go small-scale, individual structures instead of +shell projections — became the pair-bridge design. + +## Pair bridges: cancel the halos by symmetry + +Take close galaxy pairs — transverse separation 6–14 h⁻¹Mpc, +line-of-sight separation < 6 h⁻¹Mpc — which simulations expect to be +connected by filament bridges. The estimator (E3d, refined in E3d-v2): +average y in a 2′ disc at the pair **midpoint**, and subtract the mean of +the same disc at **ring control points** at ±60°, ±90° and ±120° around +each galaxy, at identical angular distance from that galaxy. Any +circularly symmetric halo profile contributes *exactly zero* by +construction — the control points sit at the same distance from the halo +as the midpoint does. What survives is gas that lives preferentially *on* +the inter-pair axis: the bridge. Over 876,639 CMASS pairs this gave +2.28×10⁻⁸ at 5.24σ on ACT and 2.97×10⁻⁸ at 19.3σ on Planck with +per-pair bootstrap errors (E3d-v2). + +## The validity battery and the final numbers + +Two upgrades of rigour (E3e; Appendix B3 discusses both as general +methods). First, pairs overlap on the sky, so bootstrap over pairs is too +optimistic: a jackknife over 50 right-ascension patches deflates the +significances by 1.6–2.4×. Second, a **physically-null control**: pairs +with the same transverse window but line-of-sight separation 25–40 h⁻¹Mpc +— same sky geometry, no possible bridge. The nulls came back zero on ACT +(1.65σ) but **4.7σ nonzero on Planck** (1.80×10⁻⁸): at a 10′ beam, the +second galaxy's smeared halo contributes more at the midpoint than at the +ring controls (which sit farther from it), manufacturing a fake bridge — +**beam leakage**, now measured rather than suspected. + +| control stage | ACT DR6 | Planck | +|---|---|---| +| pair bootstrap errors (E3d-v2) | 5.24σ (2.28×10⁻⁸) | 19.3σ (2.97×10⁻⁸) | +| sky-patch jackknife (E3e) | 3.30σ | 7.99σ | +| null-pair subtracted (E3e) | ≈1.6σ (1.4×10⁻⁸ ± 0.9) | ≈2.2σ (1.2×10⁻⁸ ± 0.5) | + +The bottom row is the physically meaningful bridge amplitude: +**~1.2–1.4×10⁻⁸, at ≈2σ per instrument**, mutually consistent between two +independent telescopes and matching the published luminous-red-galaxy +pair-bridge measurements (~1–2×10⁻⁸; de Graaff et al. 2019, Tanimura et +al. 2019). The campaign therefore reports a *reproduction* of the +literature amplitude under honest errors and physical nulls — not an +independent 5σ detection; upgrading it would need model-based two-halo +subtraction and a joint-map likelihood. The recurring moral of the series +holds to the last experiment: every strong claim, put under its own +strictest control, shrinks to its honest core — and the honest core here +still says the bridges are real. + +## Back to the series + +Back to the series: [The Geometry of the Cosmic Web: A Research Program](/mathematics/2026/07/03/geometry-of-cosmic-web-research-program/) · [Two Ways to See a Cosmic Filament](/mathematics/2026/07/04/cosmic-web-two-models/). + +
+ +
+
+

References

+
    +
  1. R. A. Sunyaev & Ya. B. Zel'dovich (1972). "The observation of relic radiation as a test of the nature of X-ray radiation from the clusters of galaxies." Comments Astrophys. Space Phys. 4, 173.
    2. +
  2. Planck Collaboration (2016). "Planck 2015 results. XXII. A map of the thermal Sunyaev–Zeldovich effect." A&A 594, A22.
    3. +
  3. W. Coulton et al. (2024). "Atacama Cosmology Telescope: high-resolution component-separated maps across one third of the sky." Phys. Rev. D 109, 063530. The ACT DR6 Compton-y map.
    4. +
  4. A. de Graaff et al. (2019). "Probing the missing baryons with the Sunyaev–Zel'dovich effect from filaments." A&A 624, A48.
    5. +
  5. H. Tanimura et al. (2019). "A search for warm/hot gas filaments between pairs of SDSS luminous red galaxies." MNRAS 483, 223–234.
    6. +
  6. Experiment reports E3, E3b, E3c, E3d, E3d-v2, E3e, in research/cosmic-web/docs/ of the repository.
    7. +
+
+
diff --git a/_config.yml b/_config.yml index 22a077a629c..bc4463f6fe2 100644 --- a/_config.yml +++ b/_config.yml @@ -66,6 +66,13 @@ exclude: - "node_modules" - "vendor" +# Appendix pages: rendered at pinned permalinks but never part of site.posts, +# so they stay out of pagination, archives, and feeds; reachable via the +# series navigation inside the posts. +collections: + appendices: + output: true + # Re-include precomputed data assets that the global "*.json" exclude swallows. include: - "public/data" diff --git a/_data/series.yml b/_data/series.yml index 8ddade330df..062e1fd772d 100644 --- a/_data/series.yml +++ b/_data/series.yml @@ -42,3 +42,42 @@ title: "The Sub-Riemannian Exponential Map of SE(2)" url: /mathematics/2026/05/05/geometry-of-seeing-A5-sr-exponential/ status: draft + +- id: geometry-of-cosmic-web + title: "Geometry of the Cosmic Web" + subtitle: > + A research program testing whether the orientation-lifted geometry of the + Geometry of Seeing series can find — and explain — cosmic filaments: a + scoping document, a benchmark whose corrected verdict favours the simple + model, and the constructive result the refutations left behind, a measured + transverse correction to Zel'dovich transport. + posts: + - part: 1 + title: "The Geometry of the Cosmic Web: A Research Program" + url: /mathematics/2026/07/03/geometry-of-cosmic-web-research-program/ + status: published + - part: 2 + title: "Two Ways to See a Cosmic Filament" + url: /mathematics/2026/07/04/cosmic-web-two-models/ + status: published + appendices: + - part: B1 + title: "How the Universe Moves Its Matter: Transport Models" + url: /mathematics/2026/07/06/cosmic-web-B1-transport-models/ + status: published + - part: B2 + title: "The Tidal Frame: How Collapse Chooses Directions" + url: /mathematics/2026/07/07/cosmic-web-B2-tidal-frame/ + status: published + - part: B3 + title: "How to Grade a Model Honestly" + url: /mathematics/2026/07/08/cosmic-web-B3-honest-benchmarks/ + status: published + - part: B4 + title: "The Transverse-Damping Model, in Full" + url: /mathematics/2026/07/09/cosmic-web-B4-transverse-damping/ + status: published + - part: B5 + title: "Reading the Sky's Hot Gas" + url: /mathematics/2026/07/10/cosmic-web-B5-reading-gas-maps/ + status: published diff --git a/_includes/posts_collate_permalink.html b/_includes/posts_collate_permalink.html index 815081d23fb..ea4404ce1dd 100644 --- a/_includes/posts_collate_permalink.html +++ b/_includes/posts_collate_permalink.html @@ -17,7 +17,8 @@ -->{% endcomment %} -{% for post in site.posts reversed %}{% if(post.group == page.group ) %} +{% assign visible_posts = site.posts | where_exp: "p", "p.listed != false" %} +{% for post in visible_posts reversed %}{% if(post.group == page.group ) %}

{{ post.title }}

{{ post.description }} diff --git a/_includes/tagged_posts_list.html b/_includes/tagged_posts_list.html index ae0e1fb3e9f..3015fdc021e 100644 --- a/_includes/tagged_posts_list.html +++ b/_includes/tagged_posts_list.html @@ -10,7 +10,8 @@

{{ page.title }}

    - {% for post in site.posts %} + {% assign visible_posts = site.posts | where_exp: "p", "p.listed != false" %} + {% for post in visible_posts %} {% if post.tags contains page.tag %}

  • diff --git a/_layouts/tag_index.html b/_layouts/tag_index.html index 19329ed84a6..cb53d1e0abb 100644 --- a/_layouts/tag_index.html +++ b/_layouts/tag_index.html @@ -10,7 +10,8 @@

    {{ page.title }}

      - {% for post in site.posts %} + {% assign visible_posts = site.posts | where_exp: "p", "p.listed != false" %} + {% for post in visible_posts %} {% for tag in post.tags %} {% if tag == page.tag %}
    • diff --git a/_posts/2026-04-25-geometry-of-seeing-visual-cortex-se2.md b/_posts/2026-04-25-geometry-of-seeing-visual-cortex-se2.md index d0d63f1f4f4..f9017a298cb 100644 --- a/_posts/2026-04-25-geometry-of-seeing-visual-cortex-se2.md +++ b/_posts/2026-04-25-geometry-of-seeing-visual-cortex-se2.md @@ -119,7 +119,8 @@ Mapped over a patch of cortex, the preferred orientations rotate continuously completing a full $\pi$-rotation over a distance of about 1 mm. The resulting pattern of orientation preferences is called an *orientation map*, and it exhibits a characteristic structure of *pinwheels*: point singularities -around which the preferred orientation rotates by $\pm\pi/2$. +of topological charge $\pm\tfrac12$ — encircle one and the preferred orientation +sweeps through a full half-turn, $\pm\pi$.
    @@ -441,7 +446,7 @@ $$[X_1, X_2] \;=\; X_3 \;=\; -\sin\theta\,\partial_x + \cos\theta\,\partial_y.$$ ### Hörmander condition ⇒ V1 is a contact manifold -Because $X_3 = [X_1, X_2]$ is *not* in $\mathrm{span}\{X_1, X_2\}$ but the +Because $[X_1, X_2] = -X_3$ is *not* in $\mathrm{span}\{X_1, X_2\}$ but the three together $\{X_1,\, X_2,\, [X_1, X_2]\}$ span the full tangent space $T_g\,\mathrm{SE}(2) \cong \mathbb{R}^3$ at every $g$, the 2-plane field @@ -452,13 +457,13 @@ The Hörmander condition (1967): the vector fields and all thei Lie brackets together span the full tangent space at every point. It is the key criterion for hypoellipticity and, via Chow–Rashevskii, for the existence of horizontal paths between any two points. -For SE(2): $\{X_1, X_2, [X_1,X_2]\} = \{X_1, X_2, X_3\}$ already spans -$\mathbb{R}^3$, so depth-1 brackets suffice. +For SE(2): $\{X_1, X_2, [X_1,X_2]\}$ spans the same space as $\{X_1, X_2, X_3\}$ — +all of $\mathbb{R}^3$ — so depth-1 brackets suffice. See Appendix A2 for the full statement. satisfies the Hörmander (bracket-generating) condition. By the -**Chow–Rashevskii theorem** (1938, 1938) any two configurations in +**Chow–Rashevskii theorem** (Rashevskii 1938, Chow 1939) any two configurations in $\mathrm{SE}(2)$ can therefore be joined by a *horizontal* path — a curve whose velocity lies in $\mathcal{H}$ at every point. Geometrically, $\mathcal{H}$ is a **contact structure** on $\mathrm{SE}(2)$, and Petitot's @@ -496,11 +501,12 @@ The spatial projection $(x(t), y(t))$ of the solution is the **perceptually comp contour** that the visual system infers between two oriented line elements $(x_0,\theta_0)$ and $(x_1,\theta_1)$. -The simplification $u_1 = 1$ (unit forward speed) reduces the problem to minimising -$$\int_0^L \kappa^2(s)\,ds$$ -where $L$ is arc length and $\kappa = u_2/u_1 = \dot\theta$ is the signed curvature. -This is the **Euler elastica functional**: the total squared bending energy of the -projected curve. +With the normalisation $u_1 = 1$ (unit forward speed), $s$ becomes arc length and +$\kappa = u_2/u_1 = \dot\theta$ is the signed curvature. The sub-Riemannian extremals +then project onto the critical curves of the **Euler elastica functional** +$$\int_0^L \kappa^2(s)\,ds,$$ +the total squared bending energy of the projected curve — the classical bridge, +made precise in Part 2, between geodesics on SE(2) and Euler's elastica.
@@ -527,9 +533,9 @@ projected curve. $k \in (0, 1)$ — inflectional, $\kappa(s) = 2k\,\mathrm{cn}(s\mid k^{2})$; $k = 1$ — the Euler / Cornu spiral, the separatrix $\kappa(s) = 2\,\mathrm{sech}\,s$; - $k > 1$ — non-inflectional, parametrised by - $m = 2 - k \in (0, 1)$ with $\kappa(s) = 2\,\mathrm{dn}(s\mid m)$ - (closed-loop curves with one-signed curvature). + $k > 1$ — non-inflectional, $\kappa(s) = 2\,\mathrm{dn}(s\mid m)$ + with its own modulus $m \in (0, 1)$ (closed-loop curves with one-signed + curvature); the slider just sweeps $m$ downward as $k$ runs past 1. Drag the slider to set the maximum $k$ rendered; the vertical bar on the right is the colour scale, with the red tick marking the separatrix $k = 1$. As $k \to 1$ the period $4K(k^{2})$ diverges and the inflectional @@ -583,8 +589,10 @@ $L_{\mathrm{SR}}$: equal or smaller $L_{\mathrm{SR}}$. At a Maxwell point, two distinct globally-optimal geodesics meet with *exactly* the same SR length. -For a sub-Riemannian manifold as symmetric as SE(2), the cut and Maxwell loci coincide -(this is part of what Sachkov and I proved in arXiv:0807.4731). +For a sub-Riemannian manifold as symmetric as SE(2), the cut and Maxwell loci are +tightly linked: characterising the Maxwell strata is the main result of +arXiv:0807.4731 (Sachkov and I), and pinning the cut locus down exactly is the +subject of arXiv:0903.0727. The first point on the cut locus along a given geodesic is the **cut time** $t_\mathrm{cut}$, and it equals the first time the exponential map is no longer injective. diff --git a/_posts/2026-04-28-geometry-of-seeing-elastica-jacobi.md b/_posts/2026-04-28-geometry-of-seeing-elastica-jacobi.md index 97c7869e09c..01a389537e1 100644 --- a/_posts/2026-04-28-geometry-of-seeing-elastica-jacobi.md +++ b/_posts/2026-04-28-geometry-of-seeing-elastica-jacobi.md @@ -85,9 +85,13 @@ The **Hamiltonian** for the PMP maximisation condition is $$H = h_1 u_1 + h_2 u_2 - \nu \sqrt{u_1^2 + u_2^2},$$ -where $\nu \in \{0, \tfrac{1}{2}\}$ is the abnormality constant. -For **normal extremals** ($\nu = \tfrac{1}{2}$), maximising over $u_1, u_2$ -gives the **normal Hamiltonian** +where $\nu \in \{0, 1\}$ is the abnormality constant ($\nu = 1$ for +**normal** extremals, $\nu = 0$ for abnormal; Appendix A3 carries out the +maximisation in full). +Minimising length is equivalent to minimising the energy +$\tfrac12\!\int(u_1^2 + u_2^2)\,dt$ over a fixed interval (Cauchy–Schwarz), +and for normal extremals the PMP maximisation then +delivers the **normal (sub-Riemannian) Hamiltonian** $$\mathcal{H}_n = \tfrac{1}{2}\bigl(h_1^2 + h_2^2\bigr).$$ @@ -191,15 +195,14 @@ delivered for the SR-on-SE(2) costate phase. The pendulum equation lives on the costate, but the figures below plot the **curvature $\kappa(s)$ of the projected plane curve in its own Euclidean -arc length $s$**. These two are tied together by a quick computation. +arc length $s$** — and $s$ is not the costate time $t$: the two are tied +together by $ds = |h_1|\,dt$. -In unit-speed parametrisation $h_1^{2}+h_2^{2}=1$, the projected speed is -$|h_1| = |\sin(\varphi/2)|$ and the heading derivative is $\dot\theta = h_2 = \cos(\varphi/2)$. -Reparametrising by Euclidean arc length $s$ with $ds = |h_1|\,dt$, and using -$\dot\varphi = \pm 2 h_3$, a short calculation gives an explicit closed-form -$\kappa(s)$ in each regime — the three Jacobi-elliptic curvature profiles -below. They satisfy the **elastica curvature ODE** (the Duffing form -equivalent to the pendulum) +Pushing the pendulum solution through that reparametrisation is the technical +heart of the reduction, and it is carried out in full in Appendix A3. The +outcome is what matters here: in each regime $\kappa(s)$ comes out in closed +form as one of the three Jacobi-elliptic profiles below, and each one solves +the **elastica curvature ODE** — the Duffing form equivalent to the pendulum, $$\kappa''(s) + \tfrac{1}{2}\kappa(s)^{3} - \mu\,\kappa(s) \;=\; 0,$$ @@ -374,8 +377,9 @@ The browser figures on this page use the same AGM algorithm implemented in For the Jacobi functions themselves: ```python -from elliptic import ellipj +from elliptic import ellipj, ellipticK +K = ellipticK(k**2) # quarter-period of sn, cn s = np.linspace(-2*K, 2*K, 800) sn, cn, dn = ellipj(s, k**2) kappa = 2 * k * cn # curvature of inflectional elastica @@ -417,9 +421,9 @@ In the `elliptic` package: ```python from elliptic import elliptic12 -phi = np.arcsin(k * sn) -E_vals, F_vals = elliptic12(phi, k**2) # E(φ|k²) and F(φ|k²) -x = 2 * (E_vals - F_vals / 2) # exact formula from Sachkov (2011) +am = np.arcsin(sn) # Jacobi amplitude am(s | k²) +F_vals, E_vals = elliptic12(am, k**2) # F(am | k²) = s and E(am | k²) +x = 2 * E_vals - F_vals # x(s) = 2 E(am(s)|k²) − s (Sachkov 2011) ``` The full closed-form expressions — due to Sachkov (2011) — express every @@ -435,8 +439,9 @@ A few landmarks worth noting: - **$k = 0.1$** (inflectional): nearly straight, very gentle curvature oscillation. The curve barely bends before straightening again. -- **$k \approx 0.71$** (inflectional): the "figure-eight" lemniscate — the curve - crosses itself once per period and the endpoints of one period coincide. +- **$k \approx 0.909$** (inflectional): the "figure-eight" lemniscate — at the + modulus where $2E(k^2) = K(k^2)$, the curve crosses itself once per period and + the endpoints of one period coincide. This is the **Maxwell stratum** for the symmetric geodesics (Part 3). - **$k \to 1^-$** (inflectional → Euler spiral): the period $4K(k^2)$ diverges and diff --git a/_posts/2026-07-03-geometry-of-cosmic-web-research-program.md b/_posts/2026-07-03-geometry-of-cosmic-web-research-program.md new file mode 100644 index 00000000000..cd25cef0d16 --- /dev/null +++ b/_posts/2026-07-03-geometry-of-cosmic-web-research-program.md @@ -0,0 +1,642 @@ +--- +layout: distill +title: "The Geometry of the Cosmic Web: A Research Program" +subtitle: > + Matter in the Universe collapses into a web of filaments. This post scopes a + falsifiable research program: lift the cosmic web onto the position–orientation + manifold $\mathbb{R}^3 \times S^2 \cong \mathrm{SE}(3)/\mathrm{SO}(2)$ — the same + machinery the visual cortex uses to complete contours — and test with public + simulation and survey data whether sub-Riemannian geodesics trace, and perhaps + shape, the filaments. +date: 2026-07-03 09:00:00 +categories: [mathematics] +tags: [sub-riemannian, SE3, cosmic-web, filaments, cosmology, optimal-control] +description: > + Scoping document for a research program connecting the sub-Riemannian geometry of + the "Geometry of Seeing" series to cosmic-web filaments: literature review, + hypotheses with falsifiable predictions, ordered experiments on public data, + and the analysis plan with defined metrics. +series: geometry-of-cosmic-web +series_title: "Geometry of the Cosmic Web" +series_part: 1 +comments: true +published: true +--- + +
+ +
+
Status — the program has been run to completion
+

This document is the plan, written before the experiments. The +plan was then executed in full, and every hypothesis below now carries a +verdict. The story of what happened — with interactive figures built from +the experiment data — is +Part 2: Two Ways +to See a Cosmic Filament. In brief:

+
    +
  • The detector claim (H1) died — and the way it died is +the best part. It first appeared to win with spectacular statistics; a +flaw in our own scoring was manufacturing the win, and the corrected race +reversed the verdict. The simpler standard method matches or beats the +lifted one nearly everywhere.
    • +
  • The physics claim (H4) died in the bulk — matter +builds filaments by falling across them, not by flowing along +them. The companion theory question (T1) resolved the same way: +filaments are pile-ups (shocks) of the matter flow, not shortest +paths.
    • +
  • The real-sky test (H3) succeeded — the webs both +methods draw on 274,000 real galaxies sit on measurably hot gas (up to +9σ under the strictest controls), so the extracted networks are +physically real.
    • +
  • One clean survivor: the lifted geometry reads the +web's directions better than the standard method, in +simulations and on the real sky alike.
    • +
  • An unplanned product: chasing "what correction does +the standard transport model actually need?" produced a one-rule model — +brake sideways at the web — that performs within a hair of its +theoretical best (see Part 2 and +Appendix B4).
    • +
+

Full experiment reports: research/cosmic-web/docs/ in the +repository, entry point SYNTHESIS.md.

+
+ +
+
What this document is
+A scoping document, not a result. It defines a research program: +the question, what the literature already says, the candidate edge, the caveats, +four hypotheses with falsifiable predictions and the metrics that will judge them, +and an ordered sequence of experiments on public data. Every experiment below can +be run on a single workstation with public datasets. The program follows the loop +literature → edge & caveats → hypothesis → test → analysis → next +hypothesis, and each hypothesis states in advance what result would kill it. +
+ +## The question + +Gravity organises matter into a **cosmic web**: sheets, filaments of width $$\sim 1{-}3\,h^{-1}\mathrm{Mpc}$$, and nodes, surrounding vast voids. Filaments are +not merely *places* — each carries a **local direction** (its tangent). The +[Geometry of Seeing]({% post_url 2026-04-25-geometry-of-seeing-visual-cortex-se2 %}) +series developed the mathematics of exactly this situation in 2D: the visual cortex +lifts an image from $$\mathbb{R}^2$$ to the position–orientation space $$\mathbb{R}^2 \times S^1 \cong \mathrm{SE}(2)$$, and completes contours along +sub-Riemannian geodesics[^subriemannian]. The 3D analogue is the homogeneous space[^homogeneous] + +$$ +\mathbb{R}^3 \times S^2 \;\cong\; \mathrm{SE}(3)/\mathrm{SO}(2), +$$ + +positions plus unit directions modulo roll about the tangent — the state space +used for fibre tracking in diffusion MRI and vessel tracking in retinal imaging +(Duits & Franken 2011; Portegies et al. 2015; Duits, Boscain, Rossi & +Sachkov 2014). + +The program asks two **separate** questions, in increasing order of ambition: + +1. **Methodological (C1).** Is the orientation-lifted manifold a *better + instrument* for extracting filament spines from noisy density fields than + density-only methods — especially at crossings and junctions, where + $$\mathrm{SE}(2)$$/$$\mathrm{SE}(3)$$ lifting is provably advantageous in imaging? +2. **Physical (C2).** Do the *dynamics* of structure formation themselves follow + sub-Riemannian geodesics of an effective metric on the lifted space — i.e., is + the intrinsic geometry not just a good detector but part of the mechanism? + +C1 is concrete, testable, and publishable on its own. C2 is speculative; the +scoping below keeps it honest by demanding a dynamical test in simulations, not +just skeleton overlap. Conflating the two is the main failure mode this document +is designed to avoid. + +## What the literature already says + +**Consensus physics.** In $$\Lambda$$CDM[^lcdm], small Gaussian density perturbations[^gaussian-perturbations] grow +by gravitational instability in an expanding FLRW[^flrw] background. Collapse is +**anisotropic**: the tidal tensor $$T_{ij} = \partial_i \partial_j \Phi$$ (Hessian of +the peculiar gravitational potential) has an eigenframe, and matter collapses +first along the eigenvector with the largest eigenvalue (forming sheets or +"pancakes"), then along the second (filaments), then the third (halos/nodes). The +**Zel'dovich approximation** (Zel'dovich 1970) captures this with the Lagrangian +map $$\mathbf{x}(\mathbf{q},t) = \mathbf{q} - D(t)\,\nabla_q \Phi(\mathbf{q})$$, +where $$D(t)$$ is the linear growth factor[^growth-factor]; the **adhesion model** (Gurbatov, +Saichev & Shandarin 1989) adds an infinitesimal viscosity (Burgers equation[^burgers]) +so matter *sticks* to sheets and filaments after shell-crossing. Bond, Kofman +& Pogosyan (1996) showed the filamentary pattern is already encoded in the +initial tidal field around proto-clusters — hence "cosmic web". + +In plain words: gravity does not crush a blob of matter evenly. The +surrounding matter stretches it more strongly along some directions than +others, and the blob gives way one axis at a time — first flattening into a +sheet, then draining into a filament, finally pooling into a node. The +animation below plays this sequence; the three arrows are the tidal +directions, ordered by how hard each squeezes. (The transport models are +developed from scratch in +[Appendix B1](/mathematics/2026/07/06/cosmic-web-B1-transport-models/); the +tidal frame in +[Appendix B2](/mathematics/2026/07/07/cosmic-web-B2-tidal-frame/).) + +[^subriemannian]: A geometry in which movement is allowed only along certain directions at each point, and path length is measured under that restriction; its shortest paths trade distance travelled against turning (see Glossary). + +[^homogeneous]: A space that looks the same from every point: a family of symmetries can carry any point to any other, so no location is special. + +[^lcdm]: The standard cosmological model: most matter is "cold dark matter" — slow-moving and invisible — and $$\Lambda$$ (Lambda) is the constant energy of empty space that accelerates the expansion (see Glossary). + +[^gaussian-perturbations]: Tiny random ripples in the early distribution of matter whose statistics follow the bell curve: fully described by the typical ripple strength at each size, with no preferred shapes or directions. + +[^flrw]: Friedmann–Lemaître–Robertson–Walker: the solution of Einstein's equations describing a universe that is on average the same everywhere and expands uniformly in time. + +[^growth-factor]: The overall factor by which small density ripples have grown by time $$t$$: multiply the initial ripple pattern by $$D(t)$$ to get its strength at that time. + +[^burgers]: The simplest equation of motion for a fluid with no pressure; adding a vanishingly small viscosity makes particles stop streaming through one another and instead pile up where they meet. + +
+ +
+
+ + cloud +
+
+
+ Animation. Anisotropic gravitational collapse in the + tidal eigenframe: the same cloud of matter collapses first along the + strongest tidal direction (λ₁ — becoming a sheet), then along + the second (λ₂ — a filament), and finally along the weakest + (λ₃ — a node). Filaments point along the λ₃ direction: the + axis gravity squeezes last. +
+
+ +
+ +**How filaments are found today.** Four families of methods, all operating on +scalar or tensor fields in $$\mathbb{R}^3$$: + +- **T-web** (Hahn et al. 2007; Forero-Romero et al. 2009): classify each point by + how many eigenvalues of $$T_{ij}$$ exceed a threshold $$\lambda_{\mathrm{th}}$$; + two ⇒ filament, with the axis along the eigenvector of the *smallest* + eigenvalue. +- **V-web** (Hoffman et al. 2012): same classification using the velocity shear + tensor $$\Sigma_{ij} = -\tfrac{1}{2H_0}(\partial_i v_j + \partial_j v_i)$$. +- **Multiscale filters**: MMF (Aragón-Calvo et al. 2007) and NEXUS/NEXUS+ + (Cautun, van de Weygaert & Jones 2013) — Hessian-based morphology filters + over a scale-space, a direct cousin of Frangi vesselness in medical imaging. +- **Topological skeletons**: DisPerSE (Sousbie 2011) extracts the filamentary + skeleton via discrete Morse theory and persistent homology[^persistent-homology]; T-ReX (Bonnaire et + al. 2020) uses regularised minimum spanning trees. + +Libeskind et al. (2018) compared twelve such web finders on the same simulation: +they disagree substantially on filament boundaries and junctions — evidence that +the *instrument* question (C1) is genuinely open. + +**Observational anchors.** Filaments are detected as mass and gas, not just as +galaxy overdensities: weak-lensing[^weak-lensing] detections of inter-cluster filaments (Epps +& Hudson 2017), stacked thermal Sunyaev–Zel'dovich (tSZ)[^tsz] signal from the warm–hot +intergalactic medium between luminous-red-galaxy pairs (de Graaff et al. 2019; +Tanimura et al. 2019), and 3D Lyman-$$\alpha$$ forest tomography[^lyman-tomography] of the web at $$z \sim 2.3$$ (CLAMATO; Lee et al. 2018). Galaxy spins align with filament axes in +a mass-dependent way (Tempel & Libeskind 2013; Codis et al. 2012). These give +us **independent channels** to validate any new skeleton: lensing mass, tSZ gas, +and spin alignment. (How gas maps are made, stacked, and defended against +false positives is +[Appendix B5](/mathematics/2026/07/10/cosmic-web-B5-reading-gas-maps/).) + +**The vision-side toolbox** (developed in the series and its appendices): build an +**orientation score** $$U(\mathbf{x},\mathbf{n})$$ by correlating the data with +rotated anisotropic wavelets; evolve it with **left-invariant (hypoelliptic) +diffusion**[^hypoelliptic] that smooths strongly along the direction $$\mathbf{n}$$ and weakly +across and in orientation, which enhances elongated coherent structures while +keeping crossings separated; extract curves as **sub-Riemannian geodesics** that +penalise bending. In 2D this reproduces the psychophysical association field +(Duits, Boscain, Rossi & Sachkov 2014); in 3D on $$\mathrm{SE}(3)/\mathrm{SO}(2)$$ it underlies crossing-preserving enhancement of +diffusion-MRI fibre fields (Duits & Franken 2011; Portegies et al. 2015). + +**The candidate edge**, in one sentence: *filament finders in cosmology are still +density/Hessian methods in $$\mathbb{R}^3$$; nobody appears to have run the +orientation-lifted $$\mathrm{SE}(3)$$ machinery — which demonstrably beats Hessian +methods at crossings in imaging — on cosmic-web fields, and the physics itself +(anisotropic tidal collapse) supplies a natural drift and anisotropy for the +lifted generator.* + +## From GR to an effective geometry — and its honest limits + +Cold dark matter is pressureless dust following geodesics of spacetime. In the +weak-field, sub-horizon limit the dynamics reduce to Vlasov–Poisson[^vlasov-poisson] in comoving +coordinates[^comoving]; anisotropy enters through the tidal eigenframe. Two geometric +observations motivate the lift: + +**Jacobi/Maupertuis metric.**[^jacobi] For a test particle with conserved energy $$E$$ in a +static potential $$\Phi$$, trajectories are geodesics of the conformally flat +Riemannian metric $$g^{\mathrm{J}} = 2m\,(E - \Phi)\, g_{\mathrm{Euclid}}$$. Paths +are "cheap" where $$\Phi$$ is deep — along potential valleys, i.e. filaments. +In plain words: matter should prefer routes through gravity's valleys the +way light bends toward denser glass — and the valleys of the cosmic +potential *are* the filaments. +**Caveat, stated up front:** in an expanding universe with a growing potential, +energy is *not* conserved along comoving trajectories, so the Jacobi construction +does not literally apply. Making the argument respectable in comoving coordinates +(where the Zel'dovich flow is potential, $$\mathbf{v} \propto \nabla_q \Phi$$) is +itself a theory work-item (T1 below), not an assumption we grant ourselves. + +**Orientation is physical here.** The tidal tensor gives every point a *frame* $$(e_1, e_2, e_3)$$ with $$\lambda_1 \ge \lambda_2 \ge \lambda_3$$, and filaments +extend along $$e_3$$. So the natural generator on $$\mathbb{R}^3 \times S^2$$ is not +isotropic: diffusion should be strong along the local tangent, weak across it and +in orientation, with coefficients tied to the tidal eigenvalues: + +$$ +\partial_t U += D_\parallel\, \mathcal{A}_3^2\, U ++ D_\perp \left( \mathcal{A}_1^2 + \mathcal{A}_2^2 \right) U ++ D_S\, \Delta_{S^2} U ++ \mu\, \mathcal{A}_3\, U, +$$ + +where $$\mathcal{A}_i$$ are the left-invariant vector fields[^left-invariant] on $$\mathrm{SE}(3)/\mathrm{SO}(2)$$ ($$\mathcal{A}_3$$ = transport along $$\mathbf{n}$$), $$\Delta_{S^2}$$ is the spherical Laplacian in the orientation variable, $$D_\parallel \gg D_\perp$$, and the drift $$\mu$$ and the ratios $$D_\parallel : D_\perp : D_S$$ are functions of $$(\lambda_1, \lambda_2, \lambda_3)$$ +to be calibrated (see H2/E1). This is the direct 3D analogue of the $$\mathrm{SE}(2)$$ hypoelliptic evolution in +[Part 1]({% post_url 2026-04-25-geometry-of-seeing-visual-cortex-se2 %}), with the +physics entering through the coefficients instead of being bolted on afterwards. + +In plain words: imagine the box of galaxies copied once for every possible +direction, and heat spreading through this stack of copies — flowing easily +*along* each copy's own direction, reluctantly sideways, and only slowly +leaking between copies of nearby directions. Structures aligned with a copy's +direction glow; everything else washes out. The equation above is that +heat-flow rule, with the local gravity field (the $$\lambda_i$$) deciding how +eager the flow is in each direction. + +## Hypotheses + +In plain words, the four bets, from safest to boldest: **H1** — the +orientation-aware eye traces filaments better than the standard one. +**H2** — telling that eye about the local gravity field improves it +further. **H3** — the webs it draws sit on real mass and gas, not just on +galaxy counts. **H4** — the boldest — matter itself *travels* along the +geometry's preferred paths, so the lift is part of the physics, not just +a detector. Each bet comes with a pre-registered way to lose, stated +below. + +Conventions used throughout: tidal eigenvalues ordered $$\lambda_1 \ge \lambda_2 \ge \lambda_3$$ with eigenvectors $$e_1, e_2, e_3$$; +filament axis along $$e_3$$. "Spine" = 1D curve set output by a filament finder. +All metrics are defined in the data-analysis plan below. + +**H1 (instrument).** *Orientation-lifted extraction recovers filament spines more +faithfully than density-only methods, with the largest gains at +crossings/junctions.* +Falsifiable prediction: on simulations with known structure (E0, E1), the $$\mathrm{SE}(3)$$ spines beat DisPerSE and NEXUS spines on spine-distance and +junction recovery at matched total spine length; if the gain at junctions is not +statistically significant, H1 fails. + +**H2 (physics-informed generator).** *Coupling the generator's coefficients to +the tidal eigenvalues improves extraction over a fixed-coefficient generator.* +Falsifiable prediction: the tidally-modulated diffusion (coefficients as +functions of $$\lambda_i$$) outperforms the best constant-coefficient run under the +same metrics with the same parameter budget (calibrated on one sub-box, tested on +held-out sub-boxes). If constant coefficients do as well, the "physics prior" +adds nothing — H2 fails even if H1 holds. + +**H3 (mass follows the geodesic spines).** *The sub-Riemannian spine network +traces real mass and gas at least as well as standard skeletons.* +Falsifiable prediction: stacking weak-lensing convergence and tSZ maps along $$\mathrm{SE}(3)$$ spines (E3) yields stack significance at least matching DisPerSE +spines on the same footprint, at matched spine length and after identical masking. +A materially lower stack SNR kills H3. + +**H4 (formation, the strong claim).** *Matter transport during web assembly +follows sub-Riemannian geodesics of the effective metric.* +Falsifiable prediction: in an N-body simulation[^nbody], lift particle trajectories $$(\mathbf{x}(t), \hat{\mathbf{v}}(t))$$ to $$\mathbb{R}^3 \times S^2$$ and compare +them, between fixed snapshots, to SR geodesics of the calibrated metric with the +same endpoints. H4 requires the geodesic prediction to beat the straight-line +(Zel'dovich ballistic) baseline on transport error by a pre-registered margin +(metric M4 in the data-analysis plan). If SR geodesics do not beat Zel'dovich, H4 is dead and C2 with it — +and the program remains a methods paper (C1). + +## Experiments, in order + +In plain words, the campaign: first settle whether the theory even +permits the boldest claim (T1); race the two methods on toy universes +where the answer is known (E0); repeat on real simulated gravity (E1); +watch matter actually move and ask whose paths it follows (E2); and only +then take the surviving method to the real sky and check its webs +against maps of mass and hot gas (E3). Each experiment gates the next; a +kill criterion at any stage stops the branch, and the analysis of each +stage sets the coefficients or priors of the following one. + +**T1 (theory, parallel track).** Derive — or refute — a Jacobi-type variational +principle for the Zel'dovich/adhesion flow in comoving coordinates, giving the +effective metric whose geodesics the flow follows. Candidate route: the adhesion +model is the zero-viscosity limit of Burgers flow, whose characteristics *are* +extremals of an action; recast that action on $$\mathbb{R}^3 \times S^2$$ and read +off the metric and the correct $$\lambda_i$$-dependence of $$D_\parallel, D_\perp, D_S, \mu$$. Also connects to optimal-transport +reconstruction of the early Universe (Brenier, Frisch et al. 2002), which is +Monge–Ampère[^optimal-transport] — i.e. already a geodesic statement in a Wasserstein geometry. +Deliverable: a note fixing the functional form of the coefficients used in E1–E3 +instead of leaving them free parameters. + +**E0 (synthetic ground truth; days, laptop).** +Generate Gaussian random fields with a $$\Lambda$$CDM-like power spectrum in a $$256^3$$ box, displace particles with the Zel'dovich map at several growth factors, +deposit density with cloud-in-cell. Ground-truth spines and junctions are known +from the deformation-tensor eigenstructure of the initial field. Build the +orientation score with 3D steerable ridge filters[^steerable] over $$\sim 3$$ scales and $$\sim 60{-}160$$ orientations (a $$256^3 \times 60$$ float32 score is $$\sim 4$$ GB — +workstation-feasible), run the lifted diffusion, trace SR geodesics, project to $$\mathbb{R}^3$$. Benchmark against DisPerSE on the same fields across noise levels +and sampling densities. **Tests H1. Kill criterion: no significant gain at any +noise level.** + +**E1 (real gravity; weeks, one GPU/big-RAM node).** +Public N-body data: a Quijote fiducial snapshot (Villaescusa-Navarro et al. 2020) +and/or IllustrisTNG-100-Dark (Nelson et al. 2019). Compute $$T_{ij}$$ by FFT of the +deposited density, smoothed at $$2\,h^{-1}$$Mpc; run the pipeline with and without +tidal modulation of the coefficients; calibrate on one octant, evaluate on the +others. Extraction quality is judged against the E0-style metrics (using +high-resolution DisPerSE-on-particles as reference where no analytic truth +exists) plus two physical alignments: spine tangent vs. $$e_3(T)$$, and spine +tangent vs. DM particle velocities. **Tests H1 on real gravity and H2. Kill +criterion for H2: tidal modulation ≤ constant coefficients on held-out volumes.** + +**E2 (dynamical test of formation; runs on E1's data).** +Between consecutive Quijote/TNG snapshots, select particles ending on filament +spines; compare their lifted trajectories to (a) SR geodesics of the calibrated +metric, (b) Zel'dovich straight-line transport, (c) geodesics of the isotropic +Jacobi metric with no orientation lift. **Tests H4 — the only experiment that can +support C2.** Pre-register the margin before running (metric M4). + +**E3 (observations; after E1 passes).** +SDSS DR17 spectroscopic sample (selection-corrected density field, redshift-space +distortions[^rsd] treated at least by anisotropic smoothing along the line of sight — +a known caveat — see the caveats section), spines extracted with coefficients frozen from E1. Stack the +public Planck 2018 lensing convergence map and Compton-$$y$$ map along spines vs. +(i) DisPerSE spines on the identical catalogue and (ii) randomised control +spines. Optional extension at $$z \sim 2.3$$ with CLAMATO tomography, where sparse +sampling should favour the orientation lift. Also re-measure the spin–filament +alignment trend with the new spines (a sharper mass transition supports H3). +**Tests H3.** + +## Data-analysis plan: the metrics, defined + +Every claim in the program is judged by one of the six metrics below, all +frozen before any experiment ran. (Why benchmarks need this discipline — +matched budgets, held-out scoring, and the one rule whose violation later +became the program's central lesson — is +[Appendix B3](/mathematics/2026/07/08/cosmic-web-B3-honest-benchmarks/).) + +- **M1 — spine distance.** Sample both skeletons at $$0.1\,h^{-1}$$Mpc; report the + two directed median point-to-curve distances + $$d_{A \to B} = \mathrm{median}_{x \in S_A} \min_{y \in S_B} \lVert x - y \rVert$$ + and the completeness/purity pair: fraction of truth within + $$r_0 = 0.5\,h^{-1}$$Mpc of the estimate, and vice versa, at **matched total + spine length** (prune both skeletons to equal length before comparing — + otherwise longer skeletons win purity-free). +- **M2 — junction recovery.** Precision/recall of ground-truth junction points + recovered within $$r_0$$; junctions are where H1 predicts the win. +- **M3 — alignment statistic.** For spine samples with tangent $$t_i$$: + $$A = \langle \lvert t_i \cdot e_3(x_i) \rvert \rangle$$. Under an isotropic null + $$\mathbb{E}[A] = 1/2$$; significance by permutation over randomly rotated + spines. Same statistic against normalised DM velocities. +- **M4 — transport error (E2).** For particle $$p$$ over snapshot interval + $$[t_1, t_2]$$, $$\varepsilon_p = \tfrac{1}{L_p} \int \lVert x_p(t) - \gamma_p(t) \rVert \, dt$$, + path-length-normalised, where $$\gamma_p$$ is the + candidate curve with matched endpoints. Compare distributions of + $$\varepsilon_p$$ (SR geodesic vs. Zel'dovich vs. isotropic Jacobi) with a paired + test; pre-registered success margin: median error reduction $$\ge 10\%$$ over + Zel'dovich. Below that, H4 is rejected regardless of p-values. +- **M5 — stack SNR (E3).** Mean excess convergence + $$\Delta\kappa$$ (or Compton-$$y$$) in tubes of radius $$1\,h^{-1}$$Mpc around + spines; $$\mathrm{SNR} = (\Delta\kappa - \langle \Delta\kappa_{\mathrm{ctrl}} \rangle)/\sigma_{\mathrm{ctrl}}$$ + over $$\ge 1000$$ control realisations + (randomly rotated/translated spines respecting the survey mask). +- **M6 — topology.** Betti curves[^betti] $$\beta_0, \beta_1$$ of the skeleton vs. + persistence threshold; compared to the reference skeleton's, on the same field. + +Calibration discipline: all free parameters ($$D_\parallel, D_\perp, D_S, \mu$$ and +their $$\lambda_i$$-dependence, wavelet scales, persistence thresholds) are set on +designated calibration volumes only; every reported number comes from held-out +volumes or sky areas. Parameter count is part of the model comparison (H2). + +## Caveats and failure modes, catalogued now + +1. **Analogy ≠ mechanism.** Success of $$\mathrm{SE}(3)$$ methods in imaging says + nothing about cosmology by itself; only E2 speaks to mechanism. The write-up + must keep C1 results from leaking into C2 claims. +2. **Flexible-parameter self-deception.** The lifted generator has more knobs + than DisPerSE. Held-out evaluation and parameter-count-aware comparison (H2) + are the guardrails. +3. **Jacobi-metric gap.** Until T1 lands, "geodesics of an effective metric" is a + motivated ansatz, not a derivation. If T1 refutes it, E2 becomes a pure + null-test and C2 should be dropped from the framing. +4. **Redshift-space distortions.** Observed density fields are anisotropic along + the line of sight (Kaiser squashing, Fingers-of-God); a spurious win in E3 + could come from the orientation machinery absorbing RSD anisotropy rather + than tracing mass. Mitigation: test on RSD-mocked simulation catalogues in E1 + before touching data. +5. **Static snapshot vs. dynamic web.** Filaments migrate; skeletons from a + single snapshot are time-slices of a flow. E2's snapshot-pair design addresses + this partially; a full time-dependent treatment is future work. +6. **Selection and masks.** Survey selection can imprint fake elongation. + Controls in M5 must respect the exact mask geometry. +7. **Compute ceiling.** $$\mathrm{SE}(3)$$ diffusion at $$512^3 \times 160$$ + orientations exceeds a workstation; the program is scoped at $$256^3 \times 60$$ + with fast separable kernel approximations (Portegies et al. 2015). Resolution + sensitivity must be reported. + +## Success criteria and outcomes + +- **Minimum publishable outcome (C1):** E0+E1 show significant junction-recovery + and spine-distance gains → methods paper: "orientation-score filament finding + for cosmic-web fields", regardless of E2/E3. +- **Strong outcome (C1+C2):** additionally, E2 beats the Zel'dovich baseline at + the pre-registered margin and T1 supplies the variational derivation → the + intrinsic-geometry claim has dynamical support and the observational program + (E3) becomes decisive. +- **Null outcome:** no E0/E1 gains → write the negative result with the + benchmark suite; the comparison framework itself (twelve finders vs. a lifted + one on common metrics) is a useful contribution. + +## Open questions / next tests + +- T1: does the adhesion action admit a clean $$\mathbb{R}^3 \times S^2$$ + reformulation, and what $$\lambda_i$$-dependence does it force on the + coefficients? +- Should the geodesic penalty include torsion as well as curvature (in 3D, twist + of the spine matters at junction handoffs)? +- Branching: handled by persistence pruning, or does the lifted space admit a + principled branching prior (junctions are crossings in $$\mathbb{R}^3$$ but + *separated* points in $$\mathbb{R}^3 \times S^2$$)? +- Curvature diagnostics: do Ollivier–Ricci/Forman curvatures[^network-curvature] of the spine graph + correlate with tSZ brightness along filaments? +- High-$$z$$: does the orientation lift stabilise CLAMATO skeletons at sparse + sampling, where density-only methods degrade fastest? + +## Glossary + +- **$$\Lambda$$CDM** — standard cosmological model: cold dark matter plus a + cosmological constant in an expanding FLRW spacetime. +- **Tidal tensor $$T_{ij}$$** — Hessian $$\partial_i \partial_j \Phi$$ of the peculiar + gravitational potential; its eigenframe ($$\lambda_1 \ge \lambda_2 \ge \lambda_3$$) + sets the anisotropy of collapse; filament axis $$\parallel e_3$$. +- **Zel'dovich approximation** — first-order Lagrangian perturbation theory: + ballistic comoving displacement $$\mathbf{x} = \mathbf{q} - D(t)\nabla_q\Phi(\mathbf{q})$$. +- **Adhesion model** — Zel'dovich flow regularised by infinitesimal Burgers + viscosity so matter sticks at shell-crossing, producing persistent + sheets/filaments. +- **T-web / V-web** — web classification by eigenvalue counts of the tidal / + velocity-shear tensor above a threshold. +- **MMF / NEXUS / DisPerSE / T-ReX** — multiscale Hessian filters (first two), + discrete-Morse/persistence skeleton, and regularised-MST filament finders. +- **WHIM** — warm–hot intergalactic medium, $$10^5$$–$$10^7$$ K gas in filaments. +- **tSZ / Compton-$$y$$** — thermal Sunyaev–Zel'dovich effect: CMB spectral + distortion proportional to line-of-sight electron pressure; $$y$$ is its + amplitude map. +- **RSD** — redshift-space distortions: peculiar velocities shift redshifts, + distorting the inferred radial positions of galaxies. +- **CLAMATO** — Lyman-$$\alpha$$ forest tomography survey giving a 3D absorption + map of the web at $$z \sim 2.3$$. +- **Orientation score $$U(\mathbf{x},\mathbf{n})$$** — lift of a scalar field to + position–orientation space by correlation with rotated anisotropic wavelets. +- **Hypoelliptic diffusion** — degenerate diffusion generating smoothing in + missing directions only through commutators (Hörmander condition); here: + strong along $$\mathbf{n}$$, weak across and in orientation. +- **Sub-Riemannian (SR) geodesic** — shortest path when motion is restricted to + a distribution of allowed directions; in the lift, curves that trade length + against turning. +- **Jacobi/Maupertuis metric** — conformal metric $$2m(E-\Phi)\,g_{\mathrm{Euclid}}$$ + whose geodesics are fixed-energy mechanical trajectories. +- **Spine** — the 1D curve network a filament finder outputs. +- **M1–M6** — the six evaluation metrics defined in the data-analysis plan. +- **C1/C2, H1–H4, E0–E3, T1** — the two claims, four hypotheses, four + experiments, and one theory work-item defined in the sections above. + +[^persistent-homology]: A topology tool that tracks how connected pieces, loops, and voids appear and merge as a detection threshold is swept; features that survive over a wide range of the sweep are treated as real, short-lived ones as noise. + +[^weak-lensing]: The gravity of matter lying between us and distant galaxies slightly bends their light and distorts their apparent shapes; averaging the shapes of many background galaxies maps the intervening mass, visible or dark. + +[^tsz]: Hot electrons in cosmic gas give a small energy kick to photons of the cosmic microwave background passing through them; the resulting distortion on the sky traces the pressure of hot gas (see Glossary). + +[^lyman-tomography]: Light from many background galaxies picks up absorption dips from the hydrogen gas it crosses; combining the dips along many neighbouring sightlines yields a 3D map of that gas. CLAMATO is the survey that produced such a map (see Glossary). + +[^hypoelliptic]: A diffusion that spreads directly only along a few allowed directions, yet ends up smoothing in every direction because combinations of the allowed moves can reach them all (see Glossary). + +[^vlasov-poisson]: The paired equations for a vast crowd of particles interacting only through the gravity of their combined mass: one equation moves the crowd, the other recomputes the gravity that the crowd itself generates. + +[^comoving]: Coordinates that stretch together with the expanding Universe, so the overall expansion is factored out and only motion relative to it remains. + +[^jacobi]: A rescaling of ordinary distance by how fast a particle of fixed energy would move at each point; after the rescaling, the particle's possible trajectories become the shortest paths of the new geometry (see Glossary). + +[^left-invariant]: Direction fields written in each point's own frame — "forward along my axis", "sideways", "turn" — the same recipe at every point, so the smoothing rule does not depend on where you stand. + +[^nbody]: A computer simulation that follows millions of mass points evolving under their mutual gravity — the standard tool for computing how cosmic structure grows once the density ripples are no longer small. + +[^optimal-transport]: Optimal transport asks for the cheapest way to rearrange one pile of mass into another; the Monge–Ampère equation is the condition the cheapest rearrangement must satisfy, and "Wasserstein geometry" measures the distance between two mass distributions by that cheapest cost. + +[^steerable]: Oriented template patterns designed so that the response at any angle can be computed exactly by combining a small fixed set of measured responses — every orientation for the price of a few. + +[^rsd]: A galaxy's distance is inferred from the stretching of its light, but the galaxy's own motion adds to that stretch, so the inferred 3D map is squashed or smeared along the line of sight (see Glossary). + +[^betti]: Counts of topological features: $$\beta_0$$ is the number of separate connected pieces, $$\beta_1$$ the number of independent loops. + +[^network-curvature]: Two recipes for assigning a curvature number to the nodes and edges of a network: roughly, positive where the network is densely interlinked, negative where it branches out like a tree. + +
+ +
+
+

References

+
    +
  1. Ya. B. Zel'dovich (1970). "Gravitational instability: an approximate theory for large density perturbations." Astron. Astrophys. 5, 84–89.
    2. +
  2. J. R. Bond, L. Kofman & D. Pogosyan (1996). "How filaments of galaxies are woven into the cosmic web." Nature 380, 603–606. arXiv:astro-ph/9512141.
    3. +
  3. S. N. Gurbatov, A. I. Saichev & S. F. Shandarin (1989). "The large-scale structure of the universe in the frame of the model equation of non-linear diffusion." MNRAS 236, 385–402.
    4. +
  4. O. Hahn, C. Porciani, C. M. Carollo & A. Dekel (2007). "Properties of dark matter haloes in clusters, filaments, sheets and voids." MNRAS 375, 489–499. arXiv:astro-ph/0610280.
    5. +
  5. J. E. Forero-Romero et al. (2009). "A dynamical classification of the cosmic web." MNRAS 396, 1815–1824. arXiv:0809.4135.
    6. +
  6. Y. Hoffman et al. (2012). "A kinematic classification of the cosmic web." MNRAS 425, 2049–2057. arXiv:1201.3367.
    7. +
  7. M. A. Aragón-Calvo et al. (2007). "The multiscale morphology filter: identifying and extracting spatial patterns in the galaxy distribution." A&A 474, 315–338. arXiv:0705.2072.
    8. +
  8. M. Cautun, R. van de Weygaert & B. J. T. Jones (2013). "NEXUS: tracing the cosmic web connection." MNRAS 429, 1286–1308. arXiv:1209.2043.
    9. +
  9. T. Sousbie (2011). "The persistent structure of the Universe — I. Theory and implementation." MNRAS 414, 350–383. arXiv:1009.4015.
    10. +
  10. T. Bonnaire et al. (2020). "T-ReX: a graph-based filament detection method." A&A 637, A18. arXiv:1912.00732.
    11. +
  11. N. I. Libeskind et al. (2018). "Tracing the cosmic web." MNRAS 473, 1195–1217. arXiv:1705.03021.
    12. +
  12. S. D. Epps & M. J. Hudson (2017). "The weak-lensing masses of filaments between luminous red galaxies." MNRAS 468, 2605–2613. arXiv:1702.08485.
    13. +
  13. A. de Graaff et al. (2019). "Probing the missing baryons with the Sunyaev–Zel'dovich effect from filaments." A&A 624, A48. arXiv:1709.10378.
    14. +
  14. H. Tanimura et al. (2019). "A search for warm/hot gas filaments between pairs of SDSS luminous red galaxies." MNRAS 483, 223–234. arXiv:1709.05024.
    15. +
  15. K.-G. Lee et al. (2018). "First data release of the COSMOS Lyα mapping and tomography observations (CLAMATO)." ApJS 237, 31. arXiv:1710.02894.
    16. +
  16. E. Tempel & N. I. Libeskind (2013). "Galaxy spin alignment in filaments and sheets: observational evidence." ApJL 775, L42. arXiv:1308.2816.
    17. +
  17. S. Codis et al. (2012). "Connecting the cosmic web to the spin of dark haloes." MNRAS 427, 3320–3336. arXiv:1201.5794.
    18. +
  18. Y. Brenier, U. Frisch et al. (2003). "Reconstruction of the early Universe as a convex optimization problem." MNRAS 346, 501–524. arXiv:astro-ph/0304214.
    19. +
  19. R. Duits & E. Franken (2011). "Left-invariant diffusions on the space of positions and orientations and their application to crossing-preserving smoothing of HARDI images." Int. J. Comput. Vis. 92, 231–264.
    20. +
  20. J. Portegies, G. Sanguinetti, S. Meesters & R. Duits (2015). "New approximation of a scale space kernel on SE(3) and applications in neuroimaging." SSVM 2015, LNCS 9087. arXiv:1506.02529.
    21. +
  21. R. Duits, U. Boscain, F. Rossi & Yu. Sachkov (2014). "Association fields via cuspless sub-Riemannian geodesics in SE(2)." J. Math. Imaging Vis. 49, 384–417. arXiv:1301.6976.
    22. +
  22. F. Villaescusa-Navarro et al. (2020). "The Quijote simulations." ApJS 250, 2. arXiv:1909.05273.
    23. +
  23. D. Nelson et al. (2019). "The IllustrisTNG simulations: public data release." Comput. Astrophys. Cosmol. 6, 2. arXiv:1812.05609.
    24. +
+
+
+ + + diff --git a/_posts/2026-07-04-cosmic-web-two-models.md b/_posts/2026-07-04-cosmic-web-two-models.md new file mode 100644 index 00000000000..a7a2bc42f07 --- /dev/null +++ b/_posts/2026-07-04-cosmic-web-two-models.md @@ -0,0 +1,1292 @@ +--- +layout: distill +title: "Two Ways to See a Cosmic Filament" +subtitle: > + Galaxies trace an invisible web of filaments across the Universe. We taught + a computer two ways of seeing it — one borrowed from how your own visual + cortex completes broken contours — and raced them on worlds where the answer + is known. The race had a surprise winner, a flaw in our own scoring that + briefly crowned the wrong one, and an ending nobody planned: a small new + physics rule that predicts where matter goes nearly as well as theoretically + possible. +date: 2026-07-04 09:00:00 +categories: [mathematics] +tags: [cosmic-web, sub-riemannian, SE3, filaments, data-analysis] +description: > + A benchmark study of two filament detectors — the standard curvature + (Hessian) detector and an orientation lift adapted from models of the visual + cortex — told for a general reader, with interactive figures built from the + actual experiment data: the three levels of reality the tests run on, where + each method works and fails, a scoring flaw and its correction, and the + transverse-damping transport rule the program distilled at the end. +series: geometry-of-cosmic-web +series_title: "Geometry of the Cosmic Web" +series_part: 2 +comments: true +published: true +--- + +
+ +
+
How to read this article
+This is the story of a research program: can the mathematics your visual +cortex uses to complete broken contours — developed in the +Geometry +of Seeing series — find the filaments of the cosmic web? The main +text tells that story in plain language. Every technical term gets a short +margin note the first time it appears, and the full machinery lives in five +appendices linked where relevant: the transport models of cosmology +(B1), +the tidal frame that gives the web its directions +(B2), +how to race two methods without rigging the race +(B3), +the transport rule this program produced +(B4), +and how to read hot-gas maps of the sky +(B5). +Every number comes from the experiment data in the repository +(research/cosmic-web/); nothing is illustrative. +
+ +## The problem + +You are handed a box of points — galaxies — and told that most of them +scatter around an invisible network of curves with branch points, while the +rest are clutter. Recover the network. This is the observational situation in +cosmology: gravity organises matter into **filaments** of width a few +megaparsecs[^megaparsec], and galaxy surveys sample this web sparsely and noisily. + +Your visual system solves a two-dimensional version of this instantly: a +dashed line reads as *one line*, because the cortex pools evidence from +collinear fragments. The mathematical mechanism, developed in the series, is +a **lift**: instead of working on the image plane $$\mathbb{R}^2$$, the cortex +represents position *and* orientation, $$\mathbb{R}^2 \times S^1$$. The +question here is whether the 3D version of that trick, + +$$ +\mathbb{R}^3 \times S^2 \;\cong\; \mathrm{SE}(3)/\mathrm{SO}(2), +$$ + +buys anything for cosmic filaments — and, more ambitiously, whether the +geometry is not just a detector but part of the *physics*. + +## Three levels of reality + +There is a problem with testing anything on the real sky: nobody has the +answer key. No catalogue says where the true filaments are — the true +network is exactly what everyone is trying to find. So this study, like most +of cosmology, works on a ladder of three worlds, each one a controlled model +of the one above it. + +At the top sits the **real sky**: a slice of the BOSS galaxy survey, every +dot a real galaxy, out to billions of light-years. You can already see the +clumps and strands by eye — but you cannot grade a method here, only check +that what it finds is physically real (we do that at the end, with maps of +hot gas). + +One level down is a **gravity simulation**: start from the smooth infant +Universe, let gravity act on a couple of million mass points, and the same +web pattern emerges on its own. Here we know every particle's full history — +where it started, where it ended — so questions about *motion* have exact +answers, even though the filaments themselves are still nobody-said-so. + +At the bottom is a **toy universe with the answers printed on it**: an +invented network of curves, with fake galaxies sprinkled along it and +clutter added. It is the least realistic world and the only one where +"completely right" is a checkable statement — so this is where the race +between the two methods is scored. + +Every claim in this article was tested at the bottom of this ladder first, +then walked upward as far as it survived. + +
+ +
+ Three panels: top, a fan-shaped slice of the BOSS galaxy survey with each dot a real galaxy; bottom left, the web-like density field of a 2.1-million-particle gravity simulation; bottom right, a synthetic Voronoi web with the exact true filaments drawn in red +
+ The three worlds the experiments run on. Top: one thin + slice of the real sky (BOSS survey; Earth sits at the bottom tip of the + fan) — real, but with no answer key. Bottom left: a gravity simulation — + the web emerges by itself, and every particle's history is known. + Bottom right: a synthetic toy where the true filament network (red) is + known exactly, so methods can be graded. Realism decreases downward; + control increases. +
+
+ +
+ +## The two models + +Both start identically: blur the galaxy points into a smooth density map, so +that instead of isolated dots there is a landscape with hills where galaxies +crowd and plains where they don't. They differ in how they ask "is there a +filament through this point?" + +**Model 1 — the mountain-ridge reading (the Hessian baseline).** Walking +along a mountain ridge, the ground falls away steeply to both sides and +stays level ahead. A filament is the same shape in the density landscape. +Model 1 measures, at every point, how the density curves in each direction — +the mathematical object holding those curvatures is the **Hessian +matrix**[^hessian] — and calls "filament" wherever the landscape drops +steeply in two directions and stays flat along the third[^hessian-score]. +This is the workhorse of the field (the standard MMF and NEXUS filament +finders are refinements of it). It has one structural limit: its direction +reading comes from a single quadratic form, which is like judging direction +through permanently blurred glasses — the response has angular +bandwidth 2[^angular-bandwidth] and *cannot be made sharper*, no matter how +clean the data. + +**Model 2 — the searchlight reading (the SE(3) orientation lift).** Instead +of one blurred direction estimate per point, measure every direction +separately: slide a long, thin, cigar-shaped filter over the data and record +how strongly it responds when aligned this way, that way, every way (we +sample 42 axes)[^lift-formula]. The result no longer lives in ordinary space +but on $$\mathbb{R}^3 \times S^2$$ — position *and* direction — and its +directional sharpness grows with the cigar's length: a knob the Hessian +simply does not have. This is the 3D version of what the visual cortex does +with contours. The theory also offers a second ingredient — a +**hypoelliptic diffusion**[^hypoelliptic] on the lifted space that lets +evidence flow along hypothesised curves (the contour-completion mechanism +of the visual cortex) — remember this one; its fate below is an honest +surprise. + +The figure below demonstrates the difference directly. A dashed curve of +points sits in uniform clutter. One oriented filter (the ellipse) is placed +on the curve and rotated through all directions; the panel on the right +records its response at each angle. Increasing the filter length sharpens +its directional selectivity; the grey lobe shows the sharpest response a +quadratic (Hessian-type) model can express regardless of the data. + +[^megaparsec]: One megaparsec (Mpc) is about 3.26 million light-years. Cosmologists often quote distances in h⁻¹Mpc, where $$h \approx 0.7$$ encodes the measured expansion rate of the Universe; 1 h⁻¹Mpc is roughly 1.4 Mpc. + +[^hessian]: The matrix of second derivatives of the density, recording how the density curves in every direction around a point; its eigenvalues are the curvatures along the three principal axes, and its eigenvectors are those axes. + +[^hessian-score]: Precisely: with Hessian eigenvalues $$\lambda_1 \ge \lambda_2 \ge \lambda_3$$, the ridge score is $$R_{\mathrm{hess}} = -\tfrac{1}{2}(\lambda_2+\lambda_3)_+$$ and the filament direction is the top eigenvector. + +[^lift-formula]: Precisely, in Fourier space: $$U(\mathbf{x},\mathbf{n}) = \mathcal{F}^{-1}[\,\sigma_\perp^2 \vert\mathbf{k}_\perp\vert^2 e^{-\frac{1}{2}(\sigma_\parallel^2 k_\parallel^2 + \sigma_\perp^2 \vert\mathbf{k}_\perp\vert^2)} \hat{f}(\mathbf{k})]$$ — the transverse Laplacian of an $$\mathbf{n}$$-elongated Gaussian correlated with the density: large exactly on ridges aligned with $$\mathbf{n}$$. Sharpness grows with the aspect ratio $$\sigma_\parallel/\sigma_\perp$$; the final score at a point is $$\max_{\mathbf{n}} U$$. + +[^angular-bandwidth]: A limit on how finely the response can vary as the probe direction rotates: bandwidth 2 means the response traces one broad bump around the circle of directions (like $$\cos^2$$) and can never form a narrower peak, however clean the data. + +[^hypoelliptic]: A smoothing process that acts directly only along a few allowed directions — here, along the current orientation — yet eventually reaches all directions through combinations of the allowed moves (see Glossary). + +
+ +
+
+ + +   filter length: + + 1.5× + +
+
+
+ Interactive. Left: a dashed curve of points (dark) in + uniform clutter (light), with one oriented ridge filter (blue ellipse) + rotating on the curve. Right: its response at every angle (blue lobe; + live) against the sharpest response any quadratic/Hessian model can + express (grey lobe — angular bandwidth 2, fixed by algebra, not by + data). At filter length 1–1.5× the two are similar; stretch to 4–6× and + the blue lobe narrows onto the true tangent while the grey one cannot. + This extra angular sharpness is the entire advantage — and, as the + benchmarks show, also the source of the lift's weaknesses at corners + and junctions. +
+
+ +
+ +**Fairness rules.** A race between methods is only as good as its rules. +Here, everything after the scoring step is *identical* for both models: +the same thresholding, the same skeletonisation[^skeletonisation], and — +crucially — both must draw the **same total length of curve**, so neither +can win just by drawing more. Each model gets the same small budget of +tuning attempts, locked in on one practice universe[^realization] before +the real scoring begins on 50 fresh ones. The race isolates exactly one +variable: blurred local curvature versus sharp oriented measurement. (Why +each of these rules exists, and what goes wrong without them, is +[Appendix B3](/mathematics/2026/07/08/cosmic-web-B3-honest-benchmarks/) — +one of them turns out to carry this whole article.) + +## The test bed: universes with exact answers + +The race is scored on the bottom rung of the ladder — the toy with the +answer key. The first experiment battery (E0) manufactures that truth: +seed points dropped in a box, their **Voronoi diagram**[^voronoi] computed, +and the *edges* of the Voronoi cells — the lines where three cell walls +meet — taken as the filament network. It is a classic cartoon of the cosmic +web, and geometrically honest about the thing filaments do that trips up +detectors: branch. One variant keeps the edges straight; a second bends +each into a smooth arc[^bezier]. Fake galaxies are then sprinkled along the +network with sideways scatter, extra clumps at the junctions, and 25% pure +clutter; both models get the identical blurred field. + +Explore the raw material below — the same box, at four sampling densities +from "starved" (2,500 galaxies) to "saturated" (80,000). Red is the exact +truth; blue is each model's recovered skeleton. + +[^skeletonisation]: Reducing a thick detected region to a centreline one grid cell wide — the "skeleton" of curves on which all scores are computed. + +[^realization]: One random draw of a synthetic universe; the "seed" is the number that initialises the random generator, so each seed labels one reproducible test universe. + +[^voronoi]: A division of space into cells, one per seed point, each cell containing everything closer to its own seed than to any other; the cells' walls and edges form a natural web-like network. + +[^bezier]: A smooth curve steered by a few control points — the standard way computer graphics draws curved strokes. + +
+ +
+
+ + variant: + + + + galaxies: + + +
+ 6-voxel slab of a Voronoi-web galaxy field with exact truth curves in red and the skeletons recovered by the SE(3) lift and the Hessian baseline in blue, at the selected variant and sampling density +
+ Interactive. One 6-voxel slab of a 128³ test box (1 voxel + = 1 h⁻¹Mpc). Left: input galaxy field with the exact truth (red). Middle: + SE(3)-lift skeleton (blue). Right: Hessian skeleton (blue). At 1.2–2.5k + galaxies both struggle in different ways — the lift draws smoother, + longer strands, the Hessian hugs density clumps; at 80k both are + near-perfect. Buttons switch the precomputed result images. +
+
+ +
+ +## The result — and the evaluation flaw that preceded it + +Fifty held-out random universes per variant, five sampling densities, three +scores: **completeness** (fraction of the true network within 2 voxels[^voxel] of the +estimate), **purity** (the converse), **junction F1** (branch-point recovery). + +Before the verdict, the most instructive result of this study. The first +version of this benchmark showed the lift beating the Hessian by **+7 +points of completeness** at sparse sampling, on 48 of 50 seeds, p ≈ 10⁻¹⁴[^pvalue]. +That result was wrong, and the mechanism matters beyond this application. + +The comparison requires both methods to output skeletons of **equal total +length**, because a longer skeleton covers more of the true network +regardless of quality. We enforced that constraint one step too early — on +an intermediate quantity from which each method then produced its final +skeleton — assuming the final lengths would match. They did not: at sparse +sampling the lift's skeletons came out about **19% longer** than the +Hessian's (1,468 vs 1,239 voxels). Most of its apparent advantage was extra +length, not extra accuracy. The flaw surfaced only when a later experiment +required enforcing the constraint on the final skeleton itself; re-running +the benchmark under the corrected procedure reversed the outcome. Every +figure below uses the corrected procedure. + +The corrected verdict: **the Hessian matches or beats the lift at every +sampling density**. At the ultra-sparse end (1,200 galaxies) the two are +statistically tied (Δ = +0.004, p = 0.36); everywhere else the Hessian wins +completeness by 4–7 points on 50 of 50 seeds, and junction F1 with it. + +
+
In plain words
+Imagine grading two students who each trace a river network on a map, where +the score is "how much of the true river did your tracing cover?" — and one +student was quietly allowed to draw a 19% longer line. On sparse data, more +line means more coverage, skill or not. Our scoring made exactly that +mistake, and the extra length masqueraded as a discovery with spectacular +statistics. Equalise the pencil, and the simpler method wins the race +almost everywhere. The full anatomy of this mistake — and the other ways a +method comparison can quietly rig itself — is +Appendix B3. +
+ +[^voxel]: One cell of the 3D grid a box is divided into — the three-dimensional analogue of a pixel. In these experiments one voxel corresponds to one h⁻¹Mpc. + +[^pvalue]: The probability of seeing a difference at least this large by pure chance if the two methods were in fact equally good; smaller means less likely to be luck. Throughout, p comes from the Wilcoxon signed-rank test, which uses only the per-universe paired differences (see Glossary). + +
+ +
+
+ metric: + + + variant: + + +
+
+
+ Interactive. Mean score vs. galaxies per box (log axis), + 50 held-out seeds, skeleton length now genuinely matched. Blue: SE(3) + lift; orange: Hessian. Hover for values. The Hessian curve sits on or + above the lift's at every density; the gap closes to a tie only at the + ultra-sparse end. Junction F1 (third toggle) is where both models are + weakest overall. +
+
+ +
+ +Averages can hide variation between realizations, so the next figure shows +every seed: each dot is one test universe, plotted by the *paired +difference* (lift − Hessian) on that realization. Above the zero line, the +lift won that universe. Under the corrected procedure the clouds sit at +zero for 1.2k galaxies and below zero everywhere else. Note that the flawed +procedure produced a +0.07 cloud that looked equally decisive in the other +direction — statistical strength does not certify that a comparison was +constructed correctly. + +
+ +
+
+ metric: + + + variant: + + +
+
+
+ Interactive. Per-seed paired differences (positive = + lift wins), 50 dots per column, with the per-column mean (bar) and win + count. At 1.2k galaxies the cloud straddles zero (statistical tie, + p ≈ 0.4); from 2.5k upward it sits entirely below zero — the Hessian + wins on 50/50 seeds, Wilcoxon p ≈ 10⁻¹⁵. Junction F1 favours the + Hessian at essentially all levels. +
+
+ +
+ +**Why the intuition failed.** The dashed-line argument — that integrating +along a hypothesised curve pools evidence sparse data cannot supply locally — +is real, and it is why the *unmatched* benchmark looked so good. But the +elongated window pays for that pooling: it rounds corners, overshoots +endpoints, displaces spines from winding crests, and blurs junctions. Once +skeleton length is genuinely equal, those costs eat the pooling gain almost +exactly; the residue is a tie in a narrow ultra-sparse band (800–1,200 +galaxies) — and pushing sparser still (400–600), the Hessian wins *again* +(−0.02, p ≈ 0.003): below a floor, the long window mostly integrates noise. +A useful way to say it: the lift buys *smoothness and connectivity*, the +Hessian buys *positional accuracy* — and on these benchmarks, positional +accuracy is what the scores reward at every density. + +## Where each model fails + +**Failure 1 — the lift at junctions.** Orientation-selective measurement is +weakest exactly where direction is ill-defined: at branch points the cigar +averages across the corner. With node clumps removed from the toy (junctions +implied only by filament continuity), the Hessian wins junction F1 by +0.04–0.05 (p ≈ 10⁻⁴). If your science is about *nodes*, the lift is the +wrong tool. + +**Failure 2 — the diffusion surprise.** The vision theory's second ingredient, +hypoelliptic diffusion (smooth strongly along $$\mathbf{n}$$, weakly across and +in orientation — the contour-completion flow), was swept over three bend +levels up to 50% sag, three sparsity levels, weak and strong settings, with +scoring binned by local curve curvature. Strong diffusion **loses +everywhere** (−0.06 to −0.12, p ≈ 2×10⁻⁶), including the most-curved third of +the network where it was most expected to help; weak diffusion is neutral to +harmful at ordinary sparsity and buys a whisper (+0.01, p = 0.03) only at the +ultra-sparse level. Essentially all of the lift's power is in the +sharp oriented measurement, not in evidence propagation. (The "your +numerics were too crude" objection was tested and closed: a finer, +provably convergent implementation[^trotter] reproduces the result. The +verdict is about the idea, not the code.) + +**Failure 3 — gravity-shaped webs (with one nuance).** Real filaments are +not tubes. On gravity-evolved boxes, scored by a method-neutral criterion — +*which model's spines capture more mass at equal length* — the toy advantage +does not transfer. On Zel'dovich fields[^zeldovich] (winding, ribbon-like structures) +the Hessian wins at every sampling density and every filter scale tried +(p ≤ 0.001). On full N-body fields[^nbody] the verdict softens but does not flip: +long cigars still lose badly (−0.05), while the *shortest* filter +(σ∥ = 3 vox) closes to a statistical tie at sparse sampling (−0.005, +p = 0.55) and a small deficit when dense. Toggle the figure below between +the two field types: the lift never actually leads on gravity-shaped mass, +and its best case is "as good as the simpler model". + +[^trotter]: An 8-step Trotter splitting at equal total diffusion, which converges to the true left-invariant semigroup on the lifted space; it reproduces the coarse result at both sparsity levels. + +[^zeldovich]: Density fields evolved with the Zel'dovich approximation: matter coasts along straight lines set by the initial gravity field. It captures the first stage of collapse, when matter flattens into sheet-like "pancakes" (see Glossary). + +[^nbody]: Fields from a simulation that follows many mass points evolving under their mutual gravity; "PM" (particle-mesh) means the gravity is computed on a grid at each step for speed. + +
+ +
+
+ field type: + + +
+
+
+ Interactive. Filament-band mass coverage (fraction of + particles in intermediate-density environments within 3 voxels of the + spine network) at matched spine length, on gravity-shaped fields. + σ∥ = 3, 4.5, 6 are the lift at three cigar lengths. The Hessian leads at + both sampling densities regardless of scale — the honest negative that + keeps the toy result in its lane. +
+
+ +
+ +**Failure 4 — the physics claim.** The ambitious version of the program said +the lifted geometry isn't just a detector: matter *transport* should follow +sub-Riemannian geodesics that are cheap along the filament axis. That +predicts trajectories' deviations from straight-line motion should point +*along* filaments. We ran N-body simulations and measured exactly that, for +200,000 tracked particles. + +
+ +
+
+
+ Interactive (hover). The dynamical test. For particles + binned by distance to the nearest filament spine: P2 = squared projection + of each trajectory's deviation-from-chord onto the filament axis + (isotropic null 1/3, dashed). Far from filaments P2 ≈ 0.21 — + deviations are strongly perpendicular: transverse pancake + infall, the mechanism that builds filaments, and the opposite of + along-axis geodesic transport. Only within ~4 voxels of a spine does the + signal turn weakly parallel (0.364). Bars show the mean deviation + magnitude — nearly all nonlinear transport happens at the web. +
+
+ +
+ +The verdict is blunt: **filaments are built by matter falling across them, +not flowing along them.** A filament is not a river channelling flow; it is +a wall being built by things crashing into it from both sides. The lifted +geometry survives only as a *static descriptor* — spine tangents from the +lift align with the tidal eigenframe[^tidal-eigenframe] (the local set of +axes gravity itself defines — [Appendix B2](/mathematics/2026/07/07/cosmic-web-B2-tidal-frame/)) +at 0.73–0.76 versus the Hessian's 0.67 (isotropic null 0.5), without ever +being shown the tidal field — but the geodesic *transport* story is refuted +in the bulk, with a weak, sign-correct residual for matter already captured +inside filament tubes. + +## A real-Universe coda + +One rung of the ladder remains: the real sky. There is still no answer key +there — but there is something almost as good. Filaments should contain hot +gas, and hot gas leaves a faint, measurable imprint on the relic light of +the Big Bang as it passes through. So if the webs our methods draw are real, +the sky should be slightly "hotter" along them than elsewhere. That imprint +is measured in **Compton-y maps** (how such maps are made and what can fake +a signal in them is +[Appendix B5](/mathematics/2026/07/10/cosmic-web-B5-reading-gas-maps/)). + +The final experiment left simulations behind: 274,000 real BOSS +CMASS galaxies[^boss] (z = 0.45–0.55)[^redshift] tiled into eighteen 512 h⁻¹Mpc cubes, spines +extracted by both methods at matched length, and the networks stacked +against Planck maps[^planck-act] with footprint-matched rotated controls. Three things +happened. First, a **Compton-y detection**[^compton-y] — rising to **~9σ** under the +strictest controls (nulls matched to the spine points' galactic-latitude +distribution): the extracted spine networks sit on measurably hot gas, so +the web the methods draw is physically real. Follow-up tests showed that +signal is carried mostly by the gas of the survey galaxies' own halos — +but a closing experiment — stacking 876,000 close galaxy *pairs* with an +estimator that cancels any symmetric halo by construction, then holding +it to jackknife errors[^jackknife] and physically-unconnected control pairs — found +the gas **between** the halos at an amplitude of ~1.2–1.4×10⁻⁸, +consistent between ACT and Planck and with published measurements, +at ≈2σ per instrument: honest evidence for the filament bridges, +reproducing the field's amplitude rather than claiming a new detection. +The figure below shows that trajectory of the claim explicitly — the same +measurement under progressively stricter controls. Second, the Hessian's spines carry more of the spine-stack signal +than the lift's on every statistic, consistent with everything above. Third — and this is the +lift's one clean win, replicated from simulation to sky — its spine +tangents align with the tidal eigenframe at 0.677 ± 0.016 vs the Hessian's +0.617 ± 0.016 across all eighteen tiles (isotropic null 0.5). The lifted +geometry reads the *anisotropy* of the real cosmic web better, even while +the simpler model finds its mass better. + +
+ +
+
+
+ The same measurement under stricter controls. The + inter-galaxy bridge signal (gas between close galaxy pairs), as a + significance, for both instruments: first with naive per-pair errors, + then with sky-patch jackknife errors (pairs overlap on the sky), then + after subtracting physically-unconnected control pairs (same geometry, + no possible bridge). Planck's large drop at the last step is measured + beam leakage; what survives on both instruments — about two sigma at + an amplitude near 10⁻⁸ — is the honest bridge evidence. +
+
+ +
+ +## The correction, measured + +The refutations left a constructive question: if matter does not travel +along filament-aligned geodesics, what correction *does* standard +transport need? Cosmology's workhorse shortcut — the **Zel'dovich +model** — predicts where matter ends up by sending every parcel along a +straight line set by the initial conditions, no further gravity computed. +It is startlingly good for something so simple, and it fails in a specific +way: it lets matter coast *through* the walls and filaments that real +gravity would have stopped it at. (The full family of these fast transport +models, from 1970 to the modern ones, is +[Appendix B1](/mathematics/2026/07/06/cosmic-web-B1-transport-models/).) +Our simulations use exactly the same initial conditions, so for every particle we can +subtract the model's prediction from the truth and examine the **residual +directly**: the adjustment term itself. + +The result reverses the original conjecture's orientation while +confirming its spirit. The residual is large near the web (about 4.4 +h⁻¹Mpc per particle) and it *is* organized by the local tidal frame — but +it points **across** the filament axis, not along it, at every distance. +Physically: straight-line transport overshoots *through* forming walls +and filaments; real gravity arrests that crossing. The correction the +standard model needs is transverse braking at the web, not longitudinal +flow along it. + +
+ +
+
+
+ Direction of the correction. For each tracked + particle, the residual (true minus Zel'dovich position, and true minus + Zel'dovich velocity) is projected onto the local filament axis; the + curve shows the mean squared projection by distance to the web. An + isotropic correction would sit on the dashed line (1/3). Everywhere + below it: the correction is preferentially perpendicular to + filaments, most strongly far from the web, approaching isotropy inside + the tubes where motion is virialized. +
+
+ +
+ +## From correction to model, to the bound + +A measured correction invites a model. The candidate is one rule added to +Zel'dovich's straight lines: *the first time a particle crashes into a +dense region, take away most of its sideways speed — the part carrying it +across the local filament — and let it keep the part moving along the +filament*. A brake that only acts sideways, and only at the web. Three +questions, answered in order: + +1. **Does the rule help?** Yes — against full simulation truth from + identical starting conditions, it beats both plain Zel'dovich and the + classical "stick at walls" model (adhesion, 1989), most clearly right + at the web. The one tuning knob was deliberately set to favour the + *competitor*, so the win is conservative. +2. **Is the remaining error the rule's fault, or its steering's?** The + rule needs to know each filament's direction, and must estimate it + from its own imperfect matter map. So run an **oracle test**: hand the + rule the *true* directions (read from the finished simulation — pure + cheating, impossible in practice) and see how good it could ever be. + With perfect steering the rule wins everywhere, in every distance + band. The physics of the brake is right; the practical cost is the + steering. +3. **How close can an honest version get?** After a short tuning campaign + (brake strength, how coarsely directions are estimated), the frozen + final recipe scores **4.52** on held-out worlds against Zel'dovich's + 4.98 — within 0.05 of the oracle's 4.47. Ninety percent of the error + that *could* be recovered, is. The exact recipe, every parameter + frozen, is [Appendix B4](/mathematics/2026/07/09/cosmic-web-B4-transverse-damping/). + +The ladder below shows where that leaves the model among its neighbours — +including MUSCLE (2016), the best published recipe in this class, which +our one-rule model statistically ties. The frozen recipe was then +stress-tested without any re-tuning on universes with different clumpiness, +different resolution, and a different cosmology — it kept beating plain +Zel'dovich in every condition, cutting the error by 3–17% — and its truth +reference was cross-checked against two independent professional +simulation codes. + +
+ +
+
+
+ The model ladder. Each row is one recipe for predicting + where matter ends up; the score is how far its predictions land from + the true simulated positions (median, in grid cells of about 1.4 + million light-years — lower is better; held-out test worlds). Grey: + the two classical recipes — straight-line motion, and sticking at + walls in all directions equally. Blue: this program's rule, in three + stages of refinement — a brake that acts only sideways, only + at the web. Green: the best published recipe of this kind, which the + final one-rule model statistically ties. The dashed red line is the + limit no version of the rule can beat: the score it gets when handed + the true filament directions instead of estimating them. +
+
+ +
+
+
+ Where the gains live. The same prediction error, split + by how close a particle ends up to the filament web. The sideways brake + earns its keep exactly where filaments form (left bars), which is where + the correction was measured in the first place; far from the web (right + bars) all three recipes agree, as they must — there is nothing there to + brake against. +
+
+ +
+ +The frozen recipe, its capabilities, limitations, failure modes, and an +integration guide are documented in a model card +([`research/cosmic-web/docs/MODEL-CARD.md`](https://github.com/moiseevigor/moiseevigor.github.io/blob/research/geometry-of-cosmic-web/research/cosmic-web/docs/MODEL-CARD.md)) — +the program's end product: not a verdict but a usable object. + +## So which model should you use? + +| Your situation | Use | Why | +|---|---|---| +| Finding filament spines, any sampling density | **Hessian** | matches or beats the lift at every level tested (50/50 seeds from 2.5k up); simpler and cheaper | +| Ultra-sparse tube-like data (a narrow 800–1,200-galaxy band) | either | statistical tie (p ≈ 0.4); sparser still, the Hessian wins again | +| Junctions / nodes are the science | **Hessian** | orientation selectivity fails where direction is ill-defined | +| Gravity-realistic ribbons, mass-tracing | **Hessian** | wins band mass coverage on ZA, N-body, and at 0.5 Mpc resolution | +| Purity- or junction-critical at moderate sparsity | **hybrid (sum of both scores)** | +0.05 purity and +0.04 junction F1 over the Hessian at 5k (p ≈ 10⁻⁴), at a small completeness cost | +| Describing web *anisotropy* (tangent statistics) | **SE(3) lift** | tidal-frame alignment 0.73–0.76 vs 0.67 — the one job it does better | +| Modelling filament *formation* | **neither as geodesics** | assembly is transverse infall; E2 refutes along-axis transport | + +Summary of the program: a theoretically motivated method, a benchmark that +initially appeared to confirm it, an evaluation flaw exposed when a later +experiment tightened the procedure, and a corrected comparison in which the +simple model wins nearly everywhere — with the lifted geometry retaining +value as an anisotropy descriptor, a hybrid component, and a physics probe. +Each negative result (the evaluation flaw, junctions, diffusion, +gravity-evolved fields, transport) determined the design of the next +experiment, and the dynamical pipeline independently reproduced known +Zel'dovich collapse behaviour, which supports trusting the negative +results. The general lesson: **a matched comparison must enforce the match +on the quantity that determines the score.** We held an intermediate +quantity equal and assumed the final one would follow; it deviated by 19%, +and that deviation read as a discovery until it was re-measured. + +Full protocols, per-experiment reports with all tables and p-values, and +one-command reproduction live in +[`research/cosmic-web/`](https://github.com/moiseevigor/moiseevigor.github.io/tree/research/geometry-of-cosmic-web/research/cosmic-web) +(experiments E0–E2, `docs/SYNTHESIS.md`). The research program itself is +scoped in the +[companion post]({% post_url 2026-07-03-geometry-of-cosmic-web-research-program %}). + +## Glossary + +- **Filament / cosmic web** — the network of matter overdensities (width + ~1–3 h⁻¹Mpc) connecting galaxy clusters; sheets, filaments, nodes, voids. +- **Completeness / purity** — fraction of truth within r₀ = 2 vox of the + estimate / fraction of the estimate within r₀ of truth, at matched + skeleton length. +- **Junction F1** — harmonic mean of precision and recall of branch-point + recovery within 3 vox. +- **Hessian ridgeness** — $$-(\lambda_2+\lambda_3)/2$$ of the smoothed + density's second-derivative matrix; angular bandwidth 2 in direction. +- **Orientation score $$U(\mathbf{x},\mathbf{n})$$** — response of an + $$\mathbf{n}$$-elongated anisotropic ridge filter; a function on + $$\mathbb{R}^3\times S^2$$. +- **Hypoelliptic diffusion** — degenerate smoothing on the lifted space, + strong along $$\mathbf{n}$$; the contour-completion flow. Rejected by every + calibration in these experiments. +- **Matched spine length** — both models' skeletons must have equal total + length before scoring, so completeness/purity trade on equal terms. + Enforced on the final drawn skeleton itself, not on an intermediate + quantity — the difference between the two is the "uneven race" this + article is built around. +- **Zel'dovich approximation / pancake infall** — ballistic displacement + model of structure formation; collapse proceeds sheet → filament → node, + with motion *transverse* to the forming structure. +- **Wilcoxon p** — signed-rank test on per-seed paired differences. +- **P2 statistic (E2)** — squared projection of a trajectory's mid-path + deviation-from-chord onto the local filament axis; isotropic null 1/3. + +[^tidal-eigenframe]: At each point the gravity of surrounding matter stretches and squeezes along three natural perpendicular axes; this local set of axes is the tidal eigenframe. Filaments tend to point along the axis of weakest squeezing. + +[^boss]: BOSS (the Baryon Oscillation Spectroscopic Survey, part of the Sloan Digital Sky Survey) mapped millions of galaxy positions; CMASS is its catalogue of massive galaxies selected to have roughly constant stellar mass. + +[^redshift]: z is redshift: the expansion of the Universe stretches light from distant galaxies toward longer wavelengths, and z measures that stretch. It doubles as a distance and look-back-time label — at z ≈ 0.5 the light left its galaxy about five billion years ago. + +[^planck-act]: Planck is a space telescope and ACT (the Atacama Cosmology Telescope) a ground-based one; both mapped the cosmic microwave background — the relic light of the early Universe — over large areas of sky, giving two independent maps to check the same signal. + +[^compton-y]: Hot electrons along the line of sight give a small energy kick to the relic microwave photons passing through them; the Compton-y map records the size of that kick across the sky, so a bright y signal means hot gas. + +[^jackknife]: Error bars estimated by removing one chunk of the data at a time and remeasuring; the spread across the remeasurements gives the uncertainty. + +
+ + + + + + + diff --git a/archives.html b/archives.html index 12f6c424e3e..6750dbb6c8a 100644 --- a/archives.html +++ b/archives.html @@ -6,7 +6,8 @@ permalink: /archives/ --- -{% for post in site.posts %} +{% assign visible_posts = site.posts | where_exp: "p", "p.listed != false" %} +{% for post in visible_posts %} {% capture this_year %}{{ post.date | date: "%Y" }}{% endcapture %} {% capture this_month %}{{ post.date | date: "%B" }}{% endcapture %} {% capture next_year %}{{ post.previous.date | date: "%Y" }}{% endcapture %} diff --git a/atom.xml b/atom.xml index 4c608a66074..4737ddbf997 100644 --- a/atom.xml +++ b/atom.xml @@ -11,7 +11,8 @@ layout: null {{ site.time | date_to_rfc822 }} {{ site.time | date_to_rfc822 }} Jekyll v{{ jekyll.version }} - {% for post in site.posts limit:100 %} + {% assign visible_posts = site.posts | where_exp: "p", "p.listed != false" %} + {% for post in visible_posts limit:100 %} {{ post.title | xml_escape }} {{ post.content | xml_escape }} diff --git a/index.html b/index.html index 0b19e73e0a6..e50ab67ca4a 100644 --- a/index.html +++ b/index.html @@ -6,7 +6,7 @@
- {% for post in paginator.posts %} + {% for post in paginator.posts %}{% unless post.listed == false %} {%- assign wc = post.content | number_of_words -%} {%- assign mins = wc | divided_by: 90 -%} {%- if mins == 0 %}{%- assign mins = 1 %}{%- endif -%} @@ -63,6 +63,7 @@

{% endif %} + {% endunless %} {% endfor %}

diff --git a/projects/geometry-of-seeing.md b/projects/geometry-of-seeing.md index 194a7697a38..3a6c773c156 100644 --- a/projects/geometry-of-seeing.md +++ b/projects/geometry-of-seeing.md @@ -165,11 +165,11 @@ elliptic cosine, with spatial period $T = 4K(k^2)$. - **Petitot's contact model** (Part 1): why V1 lifts the image to SE(2) and why modal completion is a geodesic problem. -- **Complete parametrisation** (Part 2): all three inflectional families, the - Euler separatrix, and the non-inflectional family, written in closed form using - $\mathrm{sn}, \mathrm{cn}, \mathrm{dn}$. +- **Complete parametrisation** (Part 2): all three families — the inflectional + family, the Euler separatrix, and the non-inflectional family — written in + closed form using $\mathrm{sn}, \mathrm{cn}, \mathrm{dn}$. - **Maxwell strata** (Part 3): discrete symmetry group $$\mathbb{Z}_2 \times \mathbb{Z}_2$$, - the first Maxwell time $$t_{\max}^1 = 2\pi/\sqrt{H}$$, loss of optimality. + the first Maxwell time $$t_{\max}^1 = 4K(k^2)/\omega_0$$, loss of optimality. - **The open problem** (Part 4): the cut time is bounded above by $$t_{\max}^1$$ but the exact value for $k \in (0,1)$ remains unproved. diff --git a/public/css/distill.css b/public/css/distill.css index 077386fd551..fc64744c1f1 100644 --- a/public/css/distill.css +++ b/public/css/distill.css @@ -110,7 +110,10 @@ .d-article { padding-bottom: 80px; } -.d-article { counter-reset: section figure; } +/* Reset counters on the real
only — a nested
+ * (used as a references wrapper on some posts) must NOT restart the section + * count, or its "References" heading renders as "1." instead of continuing. */ +article.d-article { counter-reset: section figure; } /* ── Layout layers ───────────────────────────────────────────────────────── */ @@ -460,6 +463,46 @@ figcaption strong { color: var(--text-light); } .d-references li { margin-bottom: 0.5em; } .d-references a { color: var(--accent); } +/* ── Footnotes / endnotes ────────────────────────────────────────────────── */ +/* + * kramdown appends
as a bare child of .d-article, + * OUTSIDE the .l-body wrappers the markdown uses — so with no rule here it + * spans the full viewport while every other block stays capped at the body + * column. Center + cap it on the same axis as .l-body, and give it the muted + * scholarly treatment of .d-references (its sibling apparatus). A "Notes" + * label orients the reader, who otherwise meets a numbered list with no title. + */ +.d-article .footnotes { + max-width: calc(var(--body-w) + 32px + var(--gutter-w)); + margin: 60px auto 0; + padding-top: 24px; + border-top: 1px solid var(--border); +} +.d-article .footnotes::before { + content: "Notes"; + display: block; max-width: var(--body-w); + font-family: var(--sans); font-size: 1rem; font-weight: 700; + letter-spacing: 0.08em; text-transform: uppercase; + color: var(--text-muted); margin-bottom: 12px; +} +.d-article .footnotes ol { + max-width: var(--body-w); + font-family: var(--sans); font-size: 0.875rem; line-height: 1.7; + color: var(--text-muted); padding-left: 1.4em; margin: 0; +} +.d-article .footnotes li { margin-bottom: 0.5em; } +.d-article .footnotes p { + font-family: var(--sans); font-size: 0.875rem; line-height: 1.7; + color: var(--text-muted); margin: 0; +} +.d-article .footnotes a { color: var(--accent); } +/* Back-reference arrow (↩) — quieten it so it doesn't compete with the note */ +.d-article .footnotes a.reversefootnote { + font-family: var(--mono); text-decoration: none; + color: var(--text-muted); opacity: 0.45; margin-left: 6px; +} +.d-article .footnotes a.reversefootnote:hover { opacity: 1; color: var(--accent); } + /* ── Series navigation ───────────────────────────────────────────────────── */ .d-series-nav { display: flex; justify-content: space-between; align-items: flex-start; @@ -492,6 +535,7 @@ figcaption strong { color: var(--text-light); } .d-header { padding: 40px 20px 36px; } .d-footer { padding: 20px; } .l-body, .l-middle, .l-page { padding: 0 20px; } + .d-article .footnotes { padding: 24px 20px 0; } } @media (max-width: 560px) { .d-title { font-size: 1.6rem; } diff --git a/public/data/cosmic-web/post_data.js b/public/data/cosmic-web/post_data.js new file mode 100644 index 00000000000..8e9dfe401de --- /dev/null +++ b/public/data/cosmic-web/post_data.js @@ -0,0 +1 @@ +window.CW_DATA = {"straight": {"levels": [1200, 2500, 5000, 20000, 80000], "agg": {"se3_lift": {"C": [0.8135, 0.8672, 0.8857, 0.8897, 0.8823], "P": [0.6851, 0.847, 0.9189, 0.9374, 0.9323], "jF1": [0.3832, 0.4645, 0.5124, 0.5297, 0.5235]}, "hessian": {"C": [0.8107, 0.912, 0.9382, 0.9431, 0.9391], "P": [0.6951, 0.7581, 0.9134, 0.9738, 0.9725], "jF1": [0.4546, 0.4695, 0.5407, 0.592, 0.591]}}, "deltas": {"dC": [[-0.0018, -0.026, 0.0055, -0.0197, 0.0009, -0.0099, -0.0347, 0.0495, 0.0113, 0.0334, 0.009, -0.0363, -0.0088, 0.0494, 0.0116, 0.0559, 0.0067, -0.0017, 0.0348, -0.0237, 0.0328, -0.0123, 0.0057, -0.0205, 0.0339, -0.0094, -0.0124, 0.0404, -0.0017, 0.0422, -0.0019, 0.0139, 0.0283, 0.0148, -0.0138, 0.0088, 0.0077, -0.026, 0.0035, -0.0335, -0.0283, 0.0011, -0.0559, -0.0313, 0.015, -0.0285, 0.0124, 0.0268, 0.0241, 0.0], [-0.0252, -0.1229, -0.0419, -0.0568, -0.0833, -0.0592, -0.0852, -0.0204, -0.0183, -0.0276, -0.0694, -0.0633, -0.0605, -0.0272, -0.0665, 0.012, -0.0452, -0.0666, -0.0394, -0.0494, -0.0128, 0.0094, -0.0346, -0.0258, -0.0351, -0.0579, -0.0368, -0.0234, -0.0194, -0.0347, -0.055, -0.0954, 0.0011, -0.0277, -0.0866, -0.0433, -0.0342, -0.0753, -0.0376, -0.0987, -0.0548, -0.0277, -0.0507, -0.0432, -0.0648, -0.0336, -0.0122, -0.0201, -0.0405, -0.0524], [-0.0586, -0.1005, -0.0632, -0.0583, -0.0956, -0.0523, -0.1162, -0.0507, -0.003, -0.0061, -0.053, -0.049, -0.1039, -0.0402, -0.063, -0.0245, -0.0404, -0.0389, -0.0465, -0.0415, -0.0388, -0.0118, -0.0082, -0.0848, -0.0478, -0.0803, -0.0159, -0.0527, -0.0143, -0.0164, -0.0644, -0.0988, -0.0757, -0.0278, -0.0873, -0.0546, -0.0375, -0.0754, -0.0428, -0.0851, -0.019, -0.0355, -0.0308, -0.1075, -0.0249, -0.0316, -0.0662, -0.0492, -0.0783, -0.057], [-0.0455, -0.0752, -0.0441, -0.0907, -0.0697, -0.0616, -0.101, -0.0415, -0.0114, -0.0017, -0.0545, -0.041, -0.0516, -0.037, -0.0732, -0.0188, -0.0752, -0.1135, -0.0616, -0.0271, -0.0466, -0.0348, -0.026, -0.0532, -0.028, -0.0401, -0.0423, -0.0519, -0.0323, -0.0303, -0.0868, -0.0813, -0.0506, -0.0442, -0.0566, -0.047, -0.0341, -0.0928, -0.0513, -0.0819, -0.0379, -0.0389, -0.0483, -0.0983, -0.0606, -0.049, -0.0698, -0.052, -0.0732, -0.038], [-0.0548, -0.0711, -0.065, -0.0599, -0.0864, -0.0555, -0.14, -0.0201, -0.0379, -0.0276, -0.0517, -0.0481, -0.0664, -0.0315, -0.0693, -0.0061, -0.0739, -0.0593, -0.0676, -0.042, -0.048, -0.0374, -0.0537, -0.0504, -0.0536, -0.0402, -0.0502, -0.042, -0.0361, -0.0499, -0.0571, -0.084, -0.0621, -0.0535, -0.0532, -0.0589, -0.0382, -0.0772, -0.0618, -0.0869, -0.0239, -0.0453, -0.0649, -0.0888, -0.0374, -0.0609, -0.079, -0.0465, -0.0831, -0.0817]], "djF1": [[-0.1194, -0.0665, -0.1135, -0.1335, -0.1029, 0.0753, -0.1063, -0.0828, -0.0091, 0.0004, -0.063, -0.0476, -0.1185, -0.1303, -0.0077, -0.0863, -0.0862, -0.0135, 0.0151, -0.0966, -0.1833, 0.0334, -0.0302, -0.096, 0.0117, -0.0417, 0.01, -0.116, -0.1038, -0.0518, 0.0241, -0.0239, -0.0988, -0.1171, -0.0278, -0.1455, -0.026, -0.0833, -0.0855, -0.1603, -0.1209, -0.029, -0.0565, -0.1441, 0.0085, -0.1512, -0.121, -0.1611, -0.1361, -0.0511], [0.0309, -0.0694, -0.0117, -0.0521, -0.008, 0.0216, -0.0303, 0.0006, 0.0035, -0.1036, -0.0054, -0.018, -0.0451, 0.1072, -0.0444, 0.0022, -0.0073, -0.0848, 0.0712, 0.0987, -0.0369, 0.0189, 0.0411, -0.0328, 0.082, 0.0325, -0.0418, -0.0649, 0.0411, -0.0448, 0.0395, -0.0379, -0.1207, -0.0441, 0.0093, -0.1217, 0.0451, -0.041, -0.0406, -0.0349, 0.0689, 0.0576, -0.0689, -0.0137, 0.0201, 0.0955, -0.0241, 0.0779, -0.0303, 0.0634], [-0.0133, -0.0515, 0.0188, -0.0577, 0.0404, 0.0433, -0.014, -0.0432, 0.0442, -0.0777, -0.1231, 0.0301, -0.1677, -0.0346, -0.0117, 0.0239, -0.0869, -0.0134, -0.021, -0.0621, -0.0907, -0.0731, 0.0041, -0.025, -0.043, -0.0876, 0.0439, 0.0049, 0.0458, -0.0569, -0.0363, 0.0018, -0.0472, 0.0129, 0.023, -0.0936, 0.0549, 0.0324, -0.0651, 0.0076, -0.0386, -0.0763, -0.0359, -0.1249, 0.0099, -0.0427, 0.0158, -0.0944, -0.027, -0.0352], [-0.0331, -0.0641, -0.0864, -0.0911, -0.0473, -0.037, -0.0818, -0.0839, -0.0328, 0.0194, -0.0234, -0.0293, -0.0752, 0.0363, -0.1463, 0.012, -0.1365, -0.1552, -0.0094, -0.0938, -0.0534, -0.0924, -0.0694, -0.0746, -0.0313, -0.0296, -0.0307, -0.1284, -0.1141, -0.1144, -0.1802, -0.0558, -0.0555, -0.0458, -0.0862, -0.0822, -0.1726, -0.0837, -0.0231, -0.1086, -0.0156, -0.0343, -0.0286, -0.149, -0.0596, -0.008, 0.007, -0.0315, 0.0386, -0.0424], [-0.0781, -0.1231, -0.0392, -0.0618, -0.0702, -0.111, -0.1362, -0.0388, -0.091, 0.0207, -0.0679, 0.0673, -0.1123, -0.1109, -0.1647, 0.0174, -0.1061, -0.1328, -0.0432, -0.0441, -0.0815, -0.065, -0.0167, -0.0714, -0.086, 0.0124, -0.0175, -0.0804, -0.0491, -0.0001, -0.0911, -0.122, -0.0772, -0.0166, -0.0197, -0.131, -0.0833, -0.1039, 0.0233, -0.1068, -0.0208, -0.1246, -0.0418, -0.1457, -0.0421, -0.1013, -0.0611, -0.0154, -0.0916, -0.1243]]}}, "curved": {"levels": [1200, 2500, 5000, 20000, 80000], "agg": {"se3_lift": {"C": [0.8134, 0.8688, 0.8915, 0.8921, 0.8836], "P": [0.6772, 0.8284, 0.9089, 0.9329, 0.9273], "jF1": [0.3759, 0.4585, 0.5077, 0.5203, 0.5192]}, "hessian": {"C": [0.8097, 0.9189, 0.9557, 0.956, 0.9496], "P": [0.7071, 0.7785, 0.8655, 0.9707, 0.9742], "jF1": [0.4352, 0.4838, 0.5104, 0.5932, 0.6003]}}, "deltas": {"dC": [[0.0372, -0.0758, 0.0102, -0.0061, -0.0385, -0.0128, 0.0199, 0.0053, 0.0029, 0.0233, 0.0305, 0.0461, 0.0209, 0.0227, 0.0126, -0.018, -0.0087, -0.0166, 0.0519, -0.0269, -0.0232, -0.0353, 0.0197, 0.0103, 0.049, 0.0154, -0.0252, 0.0269, 0.0146, 0.0363, 0.0007, 0.0197, 0.0115, 0.0144, -0.0509, -0.0379, 0.0154, -0.0401, 0.0011, -0.028, 0.0251, -0.0125, -0.0143, -0.0138, 0.0566, 0.0603, -0.0198, 0.0442, -0.0365, 0.0188], [-0.0715, -0.1575, -0.0666, -0.0678, -0.0621, -0.0563, -0.0962, -0.0003, -0.0735, -0.0392, -0.0202, -0.0483, -0.0196, -0.0613, -0.1028, -0.0086, -0.0649, -0.0698, -0.0291, -0.0281, -0.0339, -0.0161, -0.0623, -0.0464, -0.0391, -0.0353, -0.0346, -0.042, -0.0084, -0.0589, -0.039, -0.0505, -0.0644, -0.0494, -0.109, -0.0462, -0.0307, -0.0244, -0.0501, -0.0792, -0.0555, -0.0368, -0.0526, -0.0653, -0.0437, -0.0086, -0.0553, -0.0428, -0.0668, -0.0146], [-0.0721, -0.1273, -0.084, -0.0581, -0.0531, -0.0631, -0.1096, -0.0538, -0.0874, -0.0178, -0.0402, -0.0368, -0.0544, -0.0464, -0.095, -0.03, -0.09, -0.0712, -0.0612, -0.0385, -0.0699, -0.0101, -0.049, -0.0567, -0.0249, -0.0619, -0.0651, -0.0417, -0.046, -0.047, -0.0759, -0.0819, -0.056, -0.0417, -0.1443, -0.076, -0.0629, -0.1098, -0.1032, -0.0985, -0.0354, -0.0865, -0.0447, -0.1129, -0.0659, -0.0247, -0.0779, -0.0588, -0.0774, -0.0166], [-0.0868, -0.1407, -0.072, -0.0575, -0.0701, -0.0619, -0.0708, -0.0262, -0.0958, -0.062, -0.0526, -0.0313, -0.0702, -0.0414, -0.0905, -0.0283, -0.0854, -0.1122, -0.0516, -0.0387, -0.0824, -0.0167, -0.0417, -0.0795, -0.0093, -0.0326, -0.0488, -0.0544, -0.0378, -0.0589, -0.072, -0.1004, -0.06, -0.0535, -0.1367, -0.074, -0.0546, -0.1076, -0.1044, -0.0982, -0.0191, -0.0516, -0.0552, -0.0741, -0.0767, -0.0263, -0.0433, -0.0527, -0.0792, -0.045], [-0.0971, -0.1463, -0.0602, -0.0679, -0.0642, -0.0619, -0.1079, -0.0112, -0.0886, -0.0524, -0.058, -0.0366, -0.0784, -0.0473, -0.0798, -0.034, -0.0838, -0.1323, -0.0607, -0.0353, -0.0816, -0.0241, -0.0527, -0.0643, -0.0341, -0.0334, -0.0533, -0.0785, -0.0193, -0.0457, -0.042, -0.0961, -0.0707, -0.0421, -0.1039, -0.0475, -0.0543, -0.1038, -0.1016, -0.1056, -0.0132, -0.0509, -0.063, -0.1114, -0.0965, -0.0417, -0.0619, -0.08, -0.0885, -0.0325]], "djF1": [[0.0067, -0.1057, -0.0644, -0.0869, -0.1355, 0.0092, -0.0609, -0.1453, -0.015, -0.0598, 0.0249, -0.0642, -0.0878, 0.0044, -0.0953, -0.0171, 0.0138, -0.0799, -0.0195, -0.0478, -0.1254, -0.1347, -0.0095, -0.1444, -0.0614, 0.0106, -0.1222, -0.0998, 0.0331, -0.0078, -0.1193, 0.0074, -0.021, -0.16, -0.1882, -0.1447, -0.0346, -0.0648, -0.0109, -0.0381, -0.0432, -0.0832, -0.0097, -0.0896, -0.056, -0.0138, -0.0588, 0.0116, -0.0394, -0.1172], [0.0341, -0.034, -0.1243, -0.1466, -0.0757, 0.0026, -0.0904, -0.02, -0.0095, 0.0047, 0.0024, -0.0057, -0.0987, 0.0062, -0.0365, -0.0643, -0.0582, 0.0356, 0.0618, -0.1314, 0.0125, 0.0387, -0.0657, -0.0438, -0.0691, -0.0046, 0.0183, -0.0493, -0.0658, -0.1431, 0.0179, 0.0182, -0.0685, 0.0446, -0.0399, -0.0589, 0.0542, -0.0694, -0.0207, 0.0557, -0.0611, -0.0496, 0.0265, -0.0198, 0.0077, 0.0507, 0.0627, 0.0639, -0.1158, -0.0405], [0.0117, -0.0767, 0.0295, -0.049, -0.0079, -0.0191, -0.0696, 0.0007, 0.0476, -0.099, 0.0261, 0.0355, -0.0119, -0.0149, -0.045, 0.0014, -0.0596, 0.0268, 0.03, 0.0114, -0.0647, 0.0103, -0.043, 0.0132, 0.0133, 0.0273, 0.0223, 0.0503, 0.031, -0.0416, -0.0287, 0.127, 0.0179, -0.0394, -0.1072, -0.0328, 0.0915, -0.006, 0.05, -0.0233, 0.0227, -0.0829, 0.0304, -0.0254, -0.0547, -0.0028, 0.0752, -0.0192, 0.0405, 0.0442], [-0.0253, -0.0141, -0.0953, -0.0908, -0.08, -0.0641, -0.1243, 0.0419, -0.0581, -0.0838, 0.0058, -0.1319, -0.0697, -0.0522, -0.1421, -0.0247, -0.2102, -0.0128, -0.025, -0.0131, -0.1933, -0.0489, -0.1339, -0.0709, -0.1437, -0.0592, 0.0074, -0.0819, -0.117, -0.0922, -0.121, 0.0249, -0.119, -0.0631, -0.012, -0.1263, -0.0511, -0.1, -0.0494, -0.1373, -0.102, -0.0403, -0.1014, -0.1696, -0.026, -0.0099, -0.0298, -0.0811, -0.1161, -0.014], [-0.0444, -0.1956, -0.0579, -0.0492, -0.0522, -0.0805, -0.1034, -0.0755, -0.0609, -0.0451, -0.1305, -0.1192, -0.1095, -0.0635, -0.1099, -0.0367, -0.2345, -0.1729, -0.0523, -0.0188, -0.1402, -0.0454, -0.0855, -0.0632, -0.1088, -0.0255, -0.0418, -0.1173, -0.0637, -0.0258, -0.0324, -0.0922, -0.1046, -0.0317, -0.091, -0.022, -0.1155, -0.0829, -0.0146, -0.1082, -0.0941, -0.0814, -0.0978, -0.204, -0.0652, -0.0234, -0.037, -0.0523, -0.128, -0.0477]]}}, "e2": {"bins": ["0\u20132", "2\u20134", "4\u20138", "8\u201316", "16\u201364"], "P1": [0.559, 0.537, 0.544, 0.572, 0.564], "P2": [0.364, 0.349, 0.327, 0.233, 0.208], "dev": [1.72, 1.22, 0.39, 0.27, 0.25]}, "e1b_sigpar": {"levels": [5000, 20000], "methods": ["hessian", "lift_b0", "lift_s3", "lift_s45"], "band": {"hessian": [0.286, 0.3335], "lift_b0": [0.2684, 0.2908], "lift_s3": [0.2691, 0.3095], "lift_s45": [0.2663, 0.2974]}}, "e1b_pm": {"levels": [5000, 20000], "methods": ["hessian", "lift_b0", "lift_s3", "lift_s45"], "band": {"hessian": [0.2969, 0.3329], "lift_b0": [0.2412, 0.2833], "lift_s3": [0.2924, 0.3102], "lift_s45": [0.2703, 0.3047]}}}; \ No newline at end of file diff --git a/public/img/posts/cosmic-web/slice_curved_1200.png b/public/img/posts/cosmic-web/slice_curved_1200.png new file mode 100644 index 00000000000..d45ec26fe5b Binary files /dev/null and b/public/img/posts/cosmic-web/slice_curved_1200.png differ diff --git a/public/img/posts/cosmic-web/slice_curved_20000.png b/public/img/posts/cosmic-web/slice_curved_20000.png new file mode 100644 index 00000000000..d0afeb062de Binary files /dev/null and b/public/img/posts/cosmic-web/slice_curved_20000.png differ diff --git a/public/img/posts/cosmic-web/slice_curved_2500.png b/public/img/posts/cosmic-web/slice_curved_2500.png new file mode 100644 index 00000000000..9cca1d7837c Binary files /dev/null and b/public/img/posts/cosmic-web/slice_curved_2500.png differ diff --git a/public/img/posts/cosmic-web/slice_curved_5000.png b/public/img/posts/cosmic-web/slice_curved_5000.png new file mode 100644 index 00000000000..5fb0b5e5999 Binary files /dev/null and b/public/img/posts/cosmic-web/slice_curved_5000.png differ diff --git a/public/img/posts/cosmic-web/slice_curved_80000.png b/public/img/posts/cosmic-web/slice_curved_80000.png new file mode 100644 index 00000000000..bb435c57f4d Binary files /dev/null and b/public/img/posts/cosmic-web/slice_curved_80000.png differ diff --git a/public/img/posts/cosmic-web/slice_straight_1200.png b/public/img/posts/cosmic-web/slice_straight_1200.png new file mode 100644 index 00000000000..98200802792 Binary files /dev/null and b/public/img/posts/cosmic-web/slice_straight_1200.png differ diff --git a/public/img/posts/cosmic-web/slice_straight_20000.png b/public/img/posts/cosmic-web/slice_straight_20000.png new file mode 100644 index 00000000000..621c5489e81 Binary files /dev/null and b/public/img/posts/cosmic-web/slice_straight_20000.png differ diff --git a/public/img/posts/cosmic-web/slice_straight_2500.png b/public/img/posts/cosmic-web/slice_straight_2500.png new file mode 100644 index 00000000000..b95d248a8f4 Binary files /dev/null and b/public/img/posts/cosmic-web/slice_straight_2500.png differ diff --git a/public/img/posts/cosmic-web/slice_straight_5000.png b/public/img/posts/cosmic-web/slice_straight_5000.png new file mode 100644 index 00000000000..de341a09ba2 Binary files /dev/null and b/public/img/posts/cosmic-web/slice_straight_5000.png differ diff --git a/public/img/posts/cosmic-web/slice_straight_80000.png b/public/img/posts/cosmic-web/slice_straight_80000.png new file mode 100644 index 00000000000..35219c49d64 Binary files /dev/null and b/public/img/posts/cosmic-web/slice_straight_80000.png differ diff --git a/public/img/posts/cosmic-web/three_universes.png b/public/img/posts/cosmic-web/three_universes.png new file mode 100644 index 00000000000..92f3e0d7eb1 Binary files /dev/null and b/public/img/posts/cosmic-web/three_universes.png differ diff --git a/research/cosmic-web/Makefile b/research/cosmic-web/Makefile new file mode 100644 index 00000000000..ce54d165fca --- /dev/null +++ b/research/cosmic-web/Makefile @@ -0,0 +1,38 @@ +# Reproduction map: paper artefact -> command. Run `make all` (hours) or +# individual targets. Requires: python3.13 venv with requirements.txt. +PY := .venv/bin/python + +toy-benchmark: ## Sec 5.1-5.2 / corrected E0 tables + $(PY) scripts/run_e0.py --eval-seeds 50 + $(PY) scripts/run_e0.py --curved --eval-seeds 50 + $(PY) scripts/write_e0_report.py + +residual: ## Fig 1, Table 1 (E4) + $(PY) scripts/run_e4.py + +model-ladder: ## Fig 2, Table 2 (E5, E5b, E5c, E5d) + $(PY) scripts/run_e5.py && $(PY) scripts/run_e5b.py + $(PY) scripts/run_e5c.py && $(PY) scripts/run_e5d.py + +baselines: ## Table 3 (E9: 2LPT, MUSCLE) + $(PY) scripts/run_e9.py + +transfer: ## Fig 3, Table 4 (E6, E7, E7b) + $(PY) scripts/run_e6.py && $(PY) scripts/run_e7.py && $(PY) scripts/run_e7b.py + +field-level: ## Fig 4 (E8) + $(PY) scripts/run_e8.py + +truth-validation: ## Sec 2.1 (E10 convergence; E12 external, needs data/camels*) + $(PY) scripts/run_e10.py + $(PY) scripts/run_e12.py + E12_TAG=camels_astrid E12_ICS=camels $(PY) scripts/run_e12.py + +figures: ## regenerate docs/paper-figures/* + $(PY) scripts/make_paper_figures.py + +paper: figures ## compile paper/main.pdf (needs tectonic: brew install tectonic) + cd paper && tectonic main.tex + +all: toy-benchmark residual model-ladder baselines transfer field-level truth-validation figures paper +.PHONY: all toy-benchmark residual model-ladder baselines transfer field-level truth-validation figures paper diff --git a/research/cosmic-web/README.md b/research/cosmic-web/README.md new file mode 100644 index 00000000000..3e64a05c375 --- /dev/null +++ b/research/cosmic-web/README.md @@ -0,0 +1,36 @@ +# Geometry of the Cosmic Web — experiments + +Code for the research program scoped in the blog post +[`_posts/2026-07-03-geometry-of-cosmic-web-research-program.md`](../../_posts/2026-07-03-geometry-of-cosmic-web-research-program.md): +does lifting the cosmic web to the position–orientation manifold +R³×S² ≅ SE(3)/SO(2) improve filament extraction (claim C1), and do the +dynamics follow sub-Riemannian geodesics there (claim C2)? + +## Layout + +- `src/` — pipeline modules: + - `fields.py` — synthetic webs: Voronoi filament model (exact truth, + optionally Bézier-curved) and Zel'dovich boxes; CIC deposit; galaxy sampling. + - `lift.py` — orientation score on R³×S² (elongated anisotropic ridge + filters, Fourier-domain) and hypoelliptic diffusion (splitting scheme). + - `spines.py` — ridgeness fields (lifted + Hessian baseline), matched-length + skeleton extraction, junction detection. Both methods share identical + post-processing so the comparison isolates the lift. + - `metrics.py` — M1 spine distance/completeness/purity, M2 junction P/R/F1 + (defined in the scoping post, §5). +- `scripts/run_e0.py` — experiment E0a: calibrate on seed 1, evaluate on + held-out seeds, straight (`default`) or curved (`--curved`) filaments. + `--smoke` for a fast 64³ sanity run. +- `scripts/smoke_test.py` — assert-based self-checks. +- `docs/` — experiment reports with figures in `docs/figures/`. +- `artifacts/` — result JSONs (gitignored, regenerable). + +## Reproduce + +```bash +cd research/cosmic-web +python3.13 -m venv .venv && .venv/bin/pip install numpy scipy matplotlib scikit-image +.venv/bin/python scripts/smoke_test.py +.venv/bin/python scripts/run_e0.py # straight filaments, ~4 min +.venv/bin/python scripts/run_e0.py --curved # curved filaments, ~4 min +``` diff --git a/research/cosmic-web/docs/E0-report.md b/research/cosmic-web/docs/E0-report.md new file mode 100644 index 00000000000..9c9ec9ae2dc --- /dev/null +++ b/research/cosmic-web/docs/E0-report.md @@ -0,0 +1,106 @@ +# E0a — orientation-lifted vs Hessian filament extraction (Voronoi toy, exact truth) + +**Question (H1):** does lifting a galaxy density field to the position–orientation manifold R³×S² recover filament spines better than the quadratic (Hessian) orientation response, at identical post-processing and matched skeleton length? + +**Setup.** Voronoi filament model in a 128³ box (1 voxel = 1 h⁻¹Mpc at the nominal L=128 h⁻¹Mpc): filaments are Voronoi-cell edges (straight variant) or Bézier-bent edges (curved variant, sag ≈ 15% of length); galaxies = arclength-uniform points with transverse Gaussian scatter σ=0.8 vox + 15% node clumps + 25% uniform background; field = CIC deposit → log(1+δ) → Gaussian smooth σ=1. Exact truth: the generating curves and vertices. + +**Protocol.** Each method gets 3 candidate configs; the best on calibration seed 1 (mean C+P+jF1 over sampling levels) is frozen, then evaluated on held-out seeds. Both methods share hysteresis thresholding, matched-length skeletonization (mask volume bisection, tube area 9 vox²), speckle pruning ≥4 vox, and junction clustering. + + +## Metric definitions + +- **Completeness (M1)** = fraction of truth-curve sample points within r₀=2 vox of the estimated skeleton. +- **Purity (M1)** = fraction of estimated skeleton voxels within r₀=2 vox of a truth curve. +- **Junction F1 (M2)** = harmonic mean of precision/recall of greedy one-to-one matching of estimated junction centroids to true Voronoi vertices within r₀=3 vox. +- **Δ(lift−hess) C** = per-seed paired completeness difference, mean over held-out seeds; (k/n seeds +) counts seeds where the lift wins. +- All numbers: mean ± sd over held-out eval seeds; matched skeleton length ⇒ completeness/purity trade on equal footing. + + +## Straight filaments + +Chosen configs — lift: `{'sig_par': 6.0, 'sig_perp': 1.5, 'diff_iter': 2}`, hessian: `{'sigma': 2.0}`. + + +### Per-method summary + +| n_gal | method | completeness mean ± sd | median (p10–p90) | purity | junction F1 | +|---|---|---|---|---|---| +| 1200 | se3_lift | 0.813 ± 0.032 | 0.812 (0.780–0.852) | 0.685 ± 0.059 | 0.383 ± 0.056 | +| 1200 | hessian | 0.811 ± 0.029 | 0.812 (0.777–0.843) | 0.695 ± 0.034 | 0.455 ± 0.051 | +| 2500 | se3_lift | 0.867 ± 0.037 | 0.871 (0.823–0.904) | 0.847 ± 0.052 | 0.464 ± 0.065 | +| 2500 | hessian | 0.912 ± 0.023 | 0.908 (0.887–0.945) | 0.758 ± 0.054 | 0.469 ± 0.057 | +| 5000 | se3_lift | 0.886 ± 0.038 | 0.892 (0.837–0.929) | 0.919 ± 0.029 | 0.512 ± 0.060 | +| 5000 | hessian | 0.938 ± 0.018 | 0.939 (0.913–0.960) | 0.913 ± 0.049 | 0.541 ± 0.066 | +| 20000 | se3_lift | 0.890 ± 0.038 | 0.896 (0.838–0.930) | 0.937 ± 0.022 | 0.530 ± 0.076 | +| 20000 | hessian | 0.943 ± 0.025 | 0.949 (0.910–0.973) | 0.974 ± 0.012 | 0.592 ± 0.057 | +| 80000 | se3_lift | 0.882 ± 0.037 | 0.886 (0.829–0.930) | 0.932 ± 0.023 | 0.523 ± 0.073 | +| 80000 | hessian | 0.939 ± 0.024 | 0.943 (0.906–0.968) | 0.973 ± 0.012 | 0.591 ± 0.054 | + +### Paired differences (the H1 evidence) + +| n_gal | n seeds | ΔC mean | ΔC median (p10–p90) | lift wins | Wilcoxon p (ΔC) | ΔjF1 mean | Wilcoxon p (ΔjF1) | +|---|---|---|---|---|---|---|---| +| 1200 | 50 | **+0.003** | +0.002 (-0.029–+0.035) | 27/50 | 0.47 | -0.071 | 5.9e-10 | +| 2500 | 50 | **-0.045** | -0.041 (-0.083–-0.018) | 3/50 | 2.5e-14 | -0.005 | 0.5 | +| 5000 | 50 | **-0.053** | -0.050 (-0.096–-0.016) | 0/50 | 1.8e-15 | -0.028 | 0.00065 | +| 20000 | 50 | **-0.053** | -0.050 (-0.087–-0.028) | 0/50 | 1.8e-15 | -0.062 | 5e-11 | +| 80000 | 50 | **-0.057** | -0.054 (-0.083–-0.036) | 0/50 | 1.8e-15 | -0.068 | 5.6e-11 | + +![E0a straight: box plots with per-seed points of paired delta completeness and delta junction F1 vs galaxy count; positive favours the lift](figures/e0_delta_straight.png) + +![E0a straight: completeness, purity, junction F1 vs galaxy count, both methods, mean ± sd over held-out seeds](figures/e0_curves_straight.png) + +![E0a straight: 6-voxel slab of the galaxy field with truth (red) and extracted skeletons (blue) for both methods](figures/e0_slice_straight.png) + + +## Curved filaments + +Chosen configs — lift: `{'sig_par': 6.0, 'sig_perp': 1.5, 'diff_iter': 2}`, hessian: `{'sigma': 1.5}`. + + +### Per-method summary + +| n_gal | method | completeness mean ± sd | median (p10–p90) | purity | junction F1 | +|---|---|---|---|---|---| +| 1200 | se3_lift | 0.813 ± 0.030 | 0.815 (0.775–0.849) | 0.677 ± 0.050 | 0.376 ± 0.047 | +| 1200 | hessian | 0.810 ± 0.024 | 0.806 (0.784–0.841) | 0.707 ± 0.036 | 0.435 ± 0.039 | +| 2500 | se3_lift | 0.869 ± 0.032 | 0.874 (0.835–0.895) | 0.828 ± 0.055 | 0.459 ± 0.061 | +| 2500 | hessian | 0.919 ± 0.017 | 0.921 (0.899–0.936) | 0.778 ± 0.034 | 0.484 ± 0.042 | +| 5000 | se3_lift | 0.891 ± 0.036 | 0.899 (0.846–0.930) | 0.909 ± 0.029 | 0.508 ± 0.053 | +| 5000 | hessian | 0.956 ± 0.013 | 0.953 (0.944–0.975) | 0.866 ± 0.050 | 0.510 ± 0.052 | +| 20000 | se3_lift | 0.892 ± 0.043 | 0.902 (0.834–0.941) | 0.933 ± 0.026 | 0.520 ± 0.071 | +| 20000 | hessian | 0.956 ± 0.021 | 0.960 (0.932–0.982) | 0.971 ± 0.017 | 0.593 ± 0.058 | +| 80000 | se3_lift | 0.884 ± 0.046 | 0.888 (0.817–0.931) | 0.927 ± 0.028 | 0.519 ± 0.069 | +| 80000 | hessian | 0.950 ± 0.024 | 0.951 (0.919–0.977) | 0.974 ± 0.013 | 0.600 ± 0.053 | + +### Paired differences (the H1 evidence) + +| n_gal | n seeds | ΔC mean | ΔC median (p10–p90) | lift wins | Wilcoxon p (ΔC) | ΔjF1 mean | Wilcoxon p (ΔjF1) | +|---|---|---|---|---|---|---|---| +| 1200 | 50 | **+0.004** | +0.010 (-0.037–+0.044) | 30/50 | 0.36 | -0.059 | 8.9e-10 | +| 2500 | 50 | **-0.050** | -0.049 (-0.074–-0.016) | 0/50 | 1.8e-15 | -0.025 | 0.0054 | +| 5000 | 50 | **-0.064** | -0.062 (-0.104–-0.029) | 0/50 | 7.6e-10 | -0.003 | 0.75 | +| 20000 | 50 | **-0.064** | -0.059 (-0.101–-0.028) | 0/50 | 1.8e-15 | -0.073 | 3.1e-12 | +| 80000 | 50 | **-0.066** | -0.062 (-0.104–-0.033) | 0/50 | 1.8e-15 | -0.081 | 1.8e-15 | + +![E0a curved: box plots with per-seed points of paired delta completeness and delta junction F1 vs galaxy count; positive favours the lift](figures/e0_delta_curved.png) + +![E0a curved: completeness, purity, junction F1 vs galaxy count, both methods, mean ± sd over held-out seeds](figures/e0_curves_curved.png) + +![E0a curved: 6-voxel slab of the galaxy field with truth (red) and extracted skeletons (blue) for both methods](figures/e0_slice_curved.png) + + +## Verdict on H1 (toy-level) + +- **straight**: at the sparsest level (n_gal=1200) the lift wins completeness on 27/50 held-out seeds, mean Δ = +0.003 (Wilcoxon signed-rank p = 0.47); at the densest level Δ = -0.057 (p = 1.8e-15). +- **curved**: at the sparsest level (n_gal=1200) the lift wins completeness on 30/50 held-out seeds, mean Δ = +0.004 (Wilcoxon signed-rank p = 0.36); at the densest level Δ = -0.066 (p = 1.8e-15). + +**Not supported (CORRECTED result).** An earlier version of this report claimed a sparse-regime lift win (+0.07 completeness at n_gal=2500). That was an evaluation artifact: the original extractor targeted mask volume, and realized skeleton lengths differed systematically between methods (lift ~19% longer at sparse levels) — longer skeletons buy completeness mechanically. With skeleton length enforced (iterative correction to the target), the Hessian matches or beats the lift at every sampling level on both variants; the lift's best case is a statistical tie at the ultra-sparse level (n_gal=1200, p≈0.4). Junction F1 favours the Hessian at essentially all levels. The diffusion component remains rejected by calibration (weak diffusion shows a small positive only at n_gal=1200 in the E0b sweep, +0.01). Detection of the artifact: an instrument change made for E1 (connected-web extraction) retroactively changed E0 parent scores; the n_est fields in the archived E0 results confirmed the length mismatch. + + +## Open questions / next tests + +1. **Why doesn't diffusion help?** Candidates: bend level too mild (sag 15%); diffusion parameters not co-calibrated with filter scales; matched-length extraction absorbing its benefit. Next: sweep bend_frac ∈ {0.15, 0.3, 0.5} × diffusion strength, and score curvature-binned completeness (does diffusion win specifically on high-curvature arcs?). +2. **Junctions:** node clumps make junctions easy for both methods. Re-run with node_frac=0 — the lift should shine when junctions are only implied by filament continuity. +3. **Gravity realism (E1):** port the benchmark to a Zel'dovich / N-body field where filaments are tidal-sheared, not tubular — the regime the physics prior (tidal-modulated coefficients, H2) targets. +4. **Tier B:** the current extraction skeletonises the ridgeness; true SR geodesic tracing on the lifted graph is not yet exercised and is where curvature penalties enter explicitly. diff --git a/research/cosmic-web/docs/E0b-sweeps.md b/research/cosmic-web/docs/E0b-sweeps.md new file mode 100644 index 00000000000..77c7043b599 --- /dev/null +++ b/research/cosmic-web/docs/E0b-sweeps.md @@ -0,0 +1,37 @@ +# E0b — diffusion sweep (Q1) and clump-free junctions (Q2) + + +## Sweep A: does diffusion help anywhere? + +Completeness (mean over 20 seeds), lift only; paired Δ(diff − off) with Wilcoxon p. Curvature bins are terciles of discrete Menger curvature of truth points (bin 2 = most curved third). + +| n_gal | bend | diff | C overall | Δ vs off (p) | C bin0 | C bin2 | Δ bin2 vs off (p) | +|---|---|---|---|---|---|---|---| +| 1200 | 0.15 | off | 0.810 | — | 0.799 | 0.839 | — | +| 1200 | 0.15 | weak | 0.822 | +0.013 (0.027) | 0.815 | 0.833 | -0.006 (0.19) | +| 1200 | 0.15 | strong | 0.749 | -0.061 (1.9e-06) | 0.744 | 0.754 | -0.085 (1.9e-06) | +| 1200 | 0.3 | off | 0.802 | — | 0.787 | 0.827 | — | +| 1200 | 0.3 | weak | 0.803 | +0.001 (0.81) | 0.797 | 0.819 | -0.008 (0.11) | +| 1200 | 0.3 | strong | 0.736 | -0.066 (1.9e-06) | 0.726 | 0.737 | -0.090 (1.9e-06) | +| 1200 | 0.5 | off | 0.761 | — | 0.753 | 0.789 | — | +| 1200 | 0.5 | weak | 0.772 | +0.011 (0.0064) | 0.767 | 0.790 | +0.001 (0.45) | +| 1200 | 0.5 | strong | 0.703 | -0.058 (5.7e-06) | 0.699 | 0.706 | -0.083 (1.9e-06) | +| 2500 | 0.15 | off | 0.897 | — | 0.896 | 0.892 | — | +| 2500 | 0.15 | weak | 0.877 | -0.020 (4.8e-05) | 0.878 | 0.869 | -0.022 (0.00013) | +| 2500 | 0.15 | strong | 0.797 | -0.100 (1.9e-06) | 0.792 | 0.794 | -0.098 (1.9e-06) | +| 2500 | 0.3 | off | 0.886 | — | 0.887 | 0.882 | — | +| 2500 | 0.3 | weak | 0.871 | -0.015 (0.001) | 0.871 | 0.867 | -0.016 (0.002) | +| 2500 | 0.3 | strong | 0.786 | -0.100 (1.9e-06) | 0.781 | 0.781 | -0.101 (1.9e-06) | +| 2500 | 0.5 | off | 0.869 | — | 0.871 | 0.860 | — | +| 2500 | 0.5 | weak | 0.849 | -0.020 (0.0001) | 0.850 | 0.839 | -0.021 (4.8e-05) | +| 2500 | 0.5 | strong | 0.750 | -0.119 (1.9e-06) | 0.747 | 0.740 | -0.121 (1.9e-06) | + + +## Sweep B: junctions with node_frac = 0 + +| n_gal | method | completeness | junction F1 | jF1 Δ(lift−hess) (p) | +|---|---|---|---|---| +| 2500 | se3_lift | 0.895 | 0.383 | **-0.043** (0.0001) | +| 2500 | hessian | 0.932 | 0.426 | | +| 5000 | se3_lift | 0.883 | 0.428 | **-0.052** (0.00021) | +| 5000 | hessian | 0.948 | 0.481 | | diff --git a/research/cosmic-web/docs/E0c-hybrid-report.md b/research/cosmic-web/docs/E0c-hybrid-report.md new file mode 100644 index 00000000000..d2b22b3d72b --- /dev/null +++ b/research/cosmic-web/docs/E0c-hybrid-report.md @@ -0,0 +1,14 @@ +# E0c — junction/spine hybrids on the curved toy (exact truth) + +Setup: E0-curved protocol, 20 held-out seeds, sparse levels only (where the division of labour lives). Hybrids are sums/maxima of percentile-normalised parent scores; identical matched-length pipeline. + +| n_gal | method | completeness | purity | junction F1 | ΔC vs best parent (p) | ΔjF1 vs best parent (p) | +|---|---|---|---|---|---|---| +| 2500 | hessian | 0.916 ± 0.018 | 0.780 | 0.493 ± 0.049 | — | — | +| 2500 | se3_lift | 0.897 ± 0.024 | 0.792 | 0.437 ± 0.058 | — | — | +| 2500 | hyb_sum | 0.921 ± 0.022 | 0.829 | 0.483 ± 0.050 | +0.005 vs hessian (0.28) | -0.010 vs hessian (0.18) | +| 2500 | hyb_max | 0.900 ± 0.023 | 0.793 | 0.440 ± 0.067 | -0.016 vs hessian (0.0056) | -0.053 vs hessian (6.3e-05) | +| 5000 | hessian | 0.954 ± 0.010 | 0.872 | 0.514 ± 0.057 | — | — | +| 5000 | se3_lift | 0.911 ± 0.031 | 0.867 | 0.495 ± 0.067 | — | — | +| 5000 | hyb_sum | 0.935 ± 0.023 | 0.924 | 0.553 ± 0.057 | -0.019 vs hessian (9.5e-06) | +0.040 vs hessian (0.00013) | +| 5000 | hyb_max | 0.912 ± 0.027 | 0.866 | 0.490 ± 0.069 | -0.042 vs hessian (1.9e-06) | -0.024 vs hessian (0.027) | diff --git a/research/cosmic-web/docs/E0d-followups.md b/research/cosmic-web/docs/E0d-followups.md new file mode 100644 index 00000000000..9863cfbf701 --- /dev/null +++ b/research/cosmic-web/docs/E0d-followups.md @@ -0,0 +1,35 @@ +# E0d — post-correction follow-ups + + +## A. Splitting convergence (is the diffusion negative numerical?) + +Equal total diffusion; fine = 8 Trotter steps. Curved toy, 20 seeds, corrected extractor. + +| n_gal | diff | completeness | Δ vs off (p) | +|---|---|---|---| +| 1200 | off | 0.810 ± 0.024 | — | +| 1200 | coarse | 0.822 ± 0.026 | +0.0128 (0.027) | +| 1200 | fine | 0.819 ± 0.030 | +0.0089 (0.16) | +| 2500 | off | 0.897 ± 0.024 | — | +| 2500 | coarse | 0.877 ± 0.030 | -0.0196 (4.8e-05) | +| 2500 | fine | 0.880 ± 0.027 | -0.0169 (0.00021) | + +## B. Sparsity limit (does the lift ever win honestly?) + +| n_gal | ΔC (lift − hessian) | lift wins | Wilcoxon p | C hessian | C lift | +|---|---|---|---|---|---| +| 400 | -0.0165 | 8/30 | 0.0037 | 0.593 | 0.576 | +| 600 | -0.0214 | 8/30 | 0.002 | 0.693 | 0.671 | +| 800 | -0.0056 | 10/30 | 0.26 | 0.737 | 0.731 | +| 1200 | +0.0026 | 17/30 | 0.49 | 0.809 | 0.812 | + +## C. Hybrid on gravity fields (truncated ZA, 12 seeds) + +| n_gal | method | mass coverage | band coverage | Δband vs hessian (p) | +|---|---|---|---|---| +| 5000 | hessian | 0.2178 | 0.2860 ± 0.0149 | — | +| 5000 | se3_lift | 0.2029 | 0.2684 ± 0.0144 | -0.0176 (0.00049) | +| 5000 | hyb_sum | 0.2096 | 0.2775 ± 0.0122 | -0.0085 (0.042) | +| 20000 | hessian | 0.2497 | 0.3335 ± 0.0099 | — | +| 20000 | se3_lift | 0.2177 | 0.2908 ± 0.0137 | -0.0427 (0.00049) | +| 20000 | hyb_sum | 0.2374 | 0.3180 ± 0.0155 | -0.0155 (0.0068) | diff --git a/research/cosmic-web/docs/E1-report.md b/research/cosmic-web/docs/E1-report.md new file mode 100644 index 00000000000..2a0f6baa8e0 --- /dev/null +++ b/research/cosmic-web/docs/E1-report.md @@ -0,0 +1,54 @@ +# E1 — gravity-shaped fields (Zel'dovich, D=1.0): H1 transfer and first H2 test + +Setup: 12 Zel'dovich boxes (128³, L=128 h⁻¹Mpc, BBKS spectrum, σ₈=0.8·D, truncated ZA R_t=2), full 2.1M-particle density as clean reference, sparse galaxy subsamples as input. All skeletons at matched length 2400 vox. Completeness/purity vs the clean reference of EACH method (2×2, conservative). M3 alignment: mean |tangent·e3| of spine tangents vs clean-field tidal eigenframe (isotropic null 0.5). H2 weight: w = 1 + β(⟨n,e3_sparse⟩² − 1/3). + +| n_gal | method | C vs ref_hess | C vs ref_lift | P vs ref_hess | align e3 | +|---|---|---|---|---|---| +| 5000 | hessian | 0.396 ± 0.046 | 0.471 ± 0.047 | 0.379 | 0.665 | +| 5000 | lift_b0 | 0.332 ± 0.041 | 0.499 ± 0.055 | 0.351 | 0.734 | +| 5000 | lift_b1 | 0.292 ± 0.029 | 0.462 ± 0.048 | 0.358 | 0.719 | +| 5000 | lift_b2 | 0.300 ± 0.036 | 0.467 ± 0.050 | 0.356 | 0.722 | +| 20000 | hessian | 0.525 ± 0.026 | 0.571 ± 0.037 | 0.520 | 0.674 | +| 20000 | lift_b0 | 0.435 ± 0.039 | 0.629 ± 0.052 | 0.487 | 0.759 | +| 20000 | lift_b1 | 0.424 ± 0.037 | 0.640 ± 0.037 | 0.472 | 0.785 | +| 20000 | lift_b2 | 0.426 ± 0.037 | 0.638 ± 0.037 | 0.480 | 0.790 | + +## Paired tests + +- **H1 transfer, n_gal=5000** (lift β=0 vs hessian, judged on the hessian's own clean reference): ΔC = -0.064, wins 0/12, Wilcoxon p = 0.00049 +- **H2, n_gal=5000, β=1** (tidal weight vs β=0, same reference): ΔC = -0.040, wins 1/12, Wilcoxon p = 0.00098 +- **H2, n_gal=5000, β=2** (tidal weight vs β=0, same reference): ΔC = -0.032, wins 1/12, Wilcoxon p = 0.0015 +- **H1 transfer, n_gal=20000** (lift β=0 vs hessian, judged on the hessian's own clean reference): ΔC = -0.090, wins 0/12, Wilcoxon p = 0.00049 +- **H2, n_gal=20000, β=1** (tidal weight vs β=0, same reference): ΔC = -0.011, wins 5/12, Wilcoxon p = 0.3 +- **H2, n_gal=20000, β=2** (tidal weight vs β=0, same reference): ΔC = -0.009, wins 4/12, Wilcoxon p = 0.34 + +## Verdict + +1. **H1 transfer: inconclusive — the design caught its own confound.** The + 2×2 reference matrix shows strong method-affinity: each method scores + ~0.07–0.19 higher against the clean reference extracted by its own + family. Since the two clean references only overlap at C≈0.60, reference + choice dominates the method effect. Skeleton-vs-skeleton scoring cannot + settle H1 on fields without exact truth; a method-neutral criterion + (mass/particle coverage, or dynamical tests) is required. This supersedes + the naive reading "lift loses 0/12 on ref_hessian". +2. **H2 first pass: not supported.** Multiplicative tidal-alignment + weighting (β=1,2) is significantly negative at n_gal=5000 and null at + 20000 against ref_hessian (mildly positive against ref_lift — again + reference-dependent). The sparse-field tidal frame may simply be too + noisy at these densities (estimated from the same data it re-weights). +3. **M3 — the clean positive finding:** spine tangents from the + *unweighted* lift align with the clean-field tidal eigenvector e3 at + 0.734–0.759 vs the Hessian's 0.665–0.674 (isotropic null 0.5). The + orientation-lifted skeleton is more physically oriented despite receiving + no tidal information — supporting the claim that the lifted geometry + captures the anisotropy of gravitational collapse better than pointwise + curvature. + +## Next + +E2 (dynamical test, H4) needs beyond-Zel'dovich trajectories: under pure ZA +the ballistic baseline is exact by construction. Plan: minimal particle-mesh +N-body (leapfrog, FFT Poisson) to generate curved trajectories, then test +whether transport near filaments follows the lifted geometry's geodesics +better than the ballistic chord (metric M4). diff --git a/research/cosmic-web/docs/E10-report.md b/research/cosmic-web/docs/E10-report.md new file mode 100644 index 00000000000..1972764261b --- /dev/null +++ b/research/cosmic-web/docs/E10-report.md @@ -0,0 +1,16 @@ +# E10 — truth convergence (step doubling) + + +- truth self-convergence (90 vs 180 steps): median shift 0.008 vox +- vs truth(90 steps): ZA 4.457, damp 4.153, advantage -6.8% +- vs truth(180 steps): ZA 4.460, damp 4.155, advantage -6.8% + +Grid-axis convergence: covered by E7/E7b (256³ truths; advantage persists at −9% to −12%). External-simulation cross-check (Quijote snapshot) remains open pending download approval. + +## Verdict + +Truth is numerically converged in time (0.008 vox shift under step +doubling — three orders below the measured effects) and the model-vs-ZA +comparison is invariant to it. With the grid axis covered by E7/E7b, the +referee's truth-validation point is addressed up to the optional +external-simulation cross-check. diff --git a/research/cosmic-web/docs/E11-report.md b/research/cosmic-web/docs/E11-report.md new file mode 100644 index 00000000000..9735b9950ff --- /dev/null +++ b/research/cosmic-web/docs/E11-report.md @@ -0,0 +1,18 @@ +# E11 — hybrid: Lagrangian (MUSCLE) trigger + directional damping + +| model | transport all | web | r(k=0.53) | T(k=0.34) | +|---|---|---|---|---| +| muscle | 4.49 ± 0.12 | 5.57 ± 0.07 | 0.527 | 0.433 | +| damp | 4.52 ± 0.18 | 5.77 ± 0.16 | 0.388 | 0.410 | +| damp_msc | 4.67 ± 0.18 | 6.15 ± 0.17 | 0.374 | 0.343 | + +## Verdict + +Negative, and diagnostic: the Lagrangian-trigger hybrid (4.67 ± 0.18) is +worse than BOTH parents (MUSCLE 4.49, Eulerian-trigger damping 4.52), and +weakest at the web (6.15 vs 5.57/5.77). The evolving Eulerian density +carries trigger information that the precomputed Lagrangian collapse time +lacks; conversely MUSCLE's advantage must live in its +displacement-divergence construction, not its trigger. A working hybrid +would need to combine at the displacement/field level, not the trigger +level — recorded as future work, not pursued (diminishing returns). diff --git a/research/cosmic-web/docs/E12-camels_astrid-report.md b/research/cosmic-web/docs/E12-camels_astrid-report.md new file mode 100644 index 00000000000..62f0c0fd7dd --- /dev/null +++ b/research/cosmic-web/docs/E12-camels_astrid-report.md @@ -0,0 +1,30 @@ +# E12 (camels_astrid) — external validation, LCDM CV_0 + +Our PM evolved from CAMELS' own 2LPT ICs (z=127→0, Ωm=0.3, 256³, 25 Mpc/h box). Node-rounding sanity: fraction of IC displacements beyond 0.45 voxels = 0.037. + +- **Tier 1 (truth quality):** our PM vs Arepo, matched IDs: median offset 0.407 Mpc/h. + +- **Tier 2 (external stress test):** ZA 1.140 vs damping model 1.127 Mpc/h median vs Arepo truth (0.1 Mpc cells — 10× beyond tested resolution range). + + +| k (h/Mpc) | r our-PM | T our-PM | r ZA | r damp | +|---|---|---|---|---| +| 0.35 | 0.999 | 0.918 | 0.982 | 0.983 | +| 0.66 | 0.984 | 0.897 | 0.924 | 0.925 | +| 1.26 | 0.937 | 0.859 | 0.702 | 0.711 | +| 2.40 | 0.838 | 0.863 | 0.144 | 0.201 | +| 4.57 | 0.574 | 0.794 | 0.017 | 0.012 | +| 8.69 | 0.293 | 0.707 | -0.001 | -0.002 | + +## Verdict (two-code external validation) + +Same shared CV_0 ICs (per-ID prediction test: TNG-derived psi predicts +Astrid's z=15 positions to 0.068 vox median), truth from MP-Gadget +instead of Arepo. Results identical to the Arepo case to three decimals +(0.407 / 1.140 / 1.127 Mpc/h), explained by the decisive number: +**Arepo and MP-Gadget agree with each other to a median 0.029 Mpc/h per +matched particle** (p90 0.204). Production codes thus define a consensus +truth ~14x tighter than our PM's offset from it, and our PM's 0.41 Mpc/h +distance from that consensus is itself an order of magnitude below the +4-5 Mpc/h effects the program measures. Truth validation is closed at +two independent codes. diff --git a/research/cosmic-web/docs/E12-report.md b/research/cosmic-web/docs/E12-report.md new file mode 100644 index 00000000000..5fc65bb121e --- /dev/null +++ b/research/cosmic-web/docs/E12-report.md @@ -0,0 +1,25 @@ +# E12 — external validation vs CAMELS CV_0 (Arepo, LCDM) + +Our PM evolved from CAMELS' own 2LPT ICs (z=127→0, Ωm=0.3, 256³, 25 Mpc/h box). Node-rounding sanity: fraction of IC displacements beyond 0.45 voxels = 0.037. + +- **Tier 1 (truth quality):** our PM vs Arepo, matched IDs: median offset 0.407 Mpc/h. + +- **Tier 2 (external stress test):** ZA 1.140 vs damping model 1.127 Mpc/h median vs Arepo truth (0.1 Mpc cells — 10× beyond tested resolution range). + + +| k (h/Mpc) | r our-PM | T our-PM | r ZA | r damp | +|---|---|---|---|---| +| 0.35 | 0.999 | 0.918 | 0.982 | 0.983 | +| 0.66 | 0.984 | 0.897 | 0.924 | 0.925 | +| 1.26 | 0.937 | 0.859 | 0.702 | 0.711 | +| 2.40 | 0.837 | 0.864 | 0.144 | 0.202 | +| 4.57 | 0.574 | 0.795 | 0.017 | 0.012 | +| 8.69 | 0.295 | 0.709 | -0.001 | -0.002 | + +## Note on suite selection for the second-code check + +SIMBA_DM CV_0 was downloaded first but found to be byte-identical to +IllustrisTNG_DM CV_0 on the CAMELS server (range-request hash check): +the DM-only CV "twins" of those two suites are one shared simulation. +Astrid_DM CV_0 (MP-Gadget) is a genuinely distinct run (different bytes +and size) and is used as the independent second code in E12-astrid. diff --git a/research/cosmic-web/docs/E1b-256-report.md b/research/cosmic-web/docs/E1b-256-report.md new file mode 100644 index 00000000000..fcdcdf2a3a2 --- /dev/null +++ b/research/cosmic-web/docs/E1b-256-report.md @@ -0,0 +1,12 @@ +# E1b-256 — resolution check (0.5 h⁻¹Mpc voxels, filament cross-sections resolved) + +Setup: 6 truncated-ZA boxes, 256³ voxels over the same physical L=128 h⁻¹Mpc volume, physically matched scales (r₀=3 h⁻¹Mpc, spine budget 2400 h⁻¹Mpc). Band = 2<ρ<20. + +| n_gal | method | mass coverage | band coverage | Δband vs hessian (p) | +|---|---|---|---|---| +| 5000 | hessian | 0.2289 ± 0.0108 | 0.2825 ± 0.0123 | — | +| 5000 | lift_3mpc | 0.2158 ± 0.0028 | 0.2678 ± 0.0035 | -0.0146 (0.16) | +| 5000 | lift_6mpc | 0.1890 ± 0.0114 | 0.2331 ± 0.0112 | -0.0494 (0.031) | +| 20000 | hessian | 0.2627 ± 0.0122 | 0.3272 ± 0.0179 | — | +| 20000 | lift_3mpc | 0.2481 ± 0.0091 | 0.3109 ± 0.0113 | -0.0163 (0.062) | +| 20000 | lift_6mpc | 0.2207 ± 0.0070 | 0.2730 ± 0.0069 | -0.0542 (0.031) | diff --git a/research/cosmic-web/docs/E1b-pm-report.md b/research/cosmic-web/docs/E1b-pm-report.md new file mode 100644 index 00000000000..e8eec198d31 --- /dev/null +++ b/research/cosmic-web/docs/E1b-pm-report.md @@ -0,0 +1,14 @@ +# E1b-pm — method-neutral H1 on gravity fields (PM N-body): mass coverage at matched spine length + +Setup: 8 PM N-body boxes (128³, EdS, a: 0.1→1). Criterion needs no reference skeleton: fraction of all 128³ particles within r₀=3.0 vox of the spine set, spine length matched at 2400 vox. Note the spine budget covers only ~4% of the volume if spread randomly — differences measure how well spines sit on mass. + +| n_gal | method | mass coverage | band coverage (2<ρ<20) | Δband vs hessian (p) | Δband vs lift_b0 (p) | +|---|---|---|---|---|---| +| 5000 | hessian | 0.4840 ± 0.0224 | 0.2969 ± 0.0172 | — | +0.0557 (0.0078) | +| 5000 | lift_s3 | 0.4727 ± 0.0159 | 0.2924 ± 0.0117 | -0.0045 (0.55) | +0.0512 (0.0078) | +| 5000 | lift_s45 | 0.4448 ± 0.0206 | 0.2703 ± 0.0146 | -0.0267 (0.0078) | +0.0291 (0.0078) | +| 5000 | lift_b0 | 0.4105 ± 0.0242 | 0.2412 ± 0.0196 | -0.0557 (0.0078) | — | +| 20000 | hessian | 0.5228 ± 0.0161 | 0.3329 ± 0.0094 | — | +0.0496 (0.0078) | +| 20000 | lift_s3 | 0.4993 ± 0.0137 | 0.3102 ± 0.0067 | -0.0228 (0.0078) | +0.0268 (0.016) | +| 20000 | lift_s45 | 0.4915 ± 0.0173 | 0.3047 ± 0.0128 | -0.0282 (0.0078) | +0.0214 (0.0078) | +| 20000 | lift_b0 | 0.4662 ± 0.0151 | 0.2833 ± 0.0127 | -0.0496 (0.0078) | — | diff --git a/research/cosmic-web/docs/E1b-report.md b/research/cosmic-web/docs/E1b-report.md new file mode 100644 index 00000000000..1efdc9fc403 --- /dev/null +++ b/research/cosmic-web/docs/E1b-report.md @@ -0,0 +1,14 @@ +# E1b — method-neutral H1 on gravity fields: mass coverage at matched spine length + +Setup: 12 truncated-ZA boxes as in E1. Criterion needs no reference skeleton: fraction of all 128³ particles within r₀=3.0 vox of the spine set, spine length matched at 2400 vox. Note the spine budget covers only ~4% of the volume if spread randomly — differences measure how well spines sit on mass. + +| n_gal | method | mass coverage | band coverage (2<ρ<20) | Δband vs hessian (p) | Δband vs lift_b0 (p) | +|---|---|---|---|---|---| +| 5000 | hessian | 0.2178 ± 0.0143 | 0.2860 ± 0.0149 | — | +0.0176 (0.00049) | +| 5000 | lift_b0 | 0.2029 ± 0.0144 | 0.2684 ± 0.0144 | -0.0176 (0.00049) | — | +| 5000 | lift_b1_sparse | 0.1842 ± 0.0115 | 0.2425 ± 0.0133 | -0.0435 (0.00098) | -0.0259 (0.0015) | +| 5000 | lift_b1_clean | 0.1900 ± 0.0126 | 0.2504 ± 0.0141 | -0.0357 (0.00049) | -0.0180 (0.0034) | +| 20000 | hessian | 0.2497 ± 0.0106 | 0.3335 ± 0.0099 | — | +0.0427 (0.00049) | +| 20000 | lift_b0 | 0.2177 ± 0.0146 | 0.2908 ± 0.0137 | -0.0427 (0.00049) | — | +| 20000 | lift_b1_sparse | 0.2166 ± 0.0141 | 0.2880 ± 0.0148 | -0.0455 (0.00049) | -0.0027 (0.62) | +| 20000 | lift_b1_clean | 0.2126 ± 0.0144 | 0.2828 ± 0.0144 | -0.0507 (0.00049) | -0.0080 (0.092) | diff --git a/research/cosmic-web/docs/E1b-sigpar-report.md b/research/cosmic-web/docs/E1b-sigpar-report.md new file mode 100644 index 00000000000..34e95e54668 --- /dev/null +++ b/research/cosmic-web/docs/E1b-sigpar-report.md @@ -0,0 +1,14 @@ +# E1b — method-neutral H1 on gravity fields: mass coverage at matched spine length + +Setup: 12 truncated-ZA boxes as in E1. Criterion needs no reference skeleton: fraction of all 128³ particles within r₀=3.0 vox of the spine set, spine length matched at 2400 vox. Note the spine budget covers only ~4% of the volume if spread randomly — differences measure how well spines sit on mass. + +| n_gal | method | mass coverage | band coverage (2<ρ<20) | Δband vs hessian (p) | Δband vs lift_b0 (p) | +|---|---|---|---|---|---| +| 5000 | hessian | 0.2178 ± 0.0143 | 0.2860 ± 0.0149 | — | +0.0176 (0.00049) | +| 5000 | lift_s3 | 0.2037 ± 0.0126 | 0.2691 ± 0.0139 | -0.0169 (0.00098) | +0.0007 (0.52) | +| 5000 | lift_s45 | 0.2014 ± 0.0128 | 0.2663 ± 0.0148 | -0.0197 (0.0015) | -0.0021 (0.73) | +| 5000 | lift_b0 | 0.2029 ± 0.0144 | 0.2684 ± 0.0144 | -0.0176 (0.00049) | — | +| 20000 | hessian | 0.2497 ± 0.0106 | 0.3335 ± 0.0099 | — | +0.0427 (0.00049) | +| 20000 | lift_s3 | 0.2305 ± 0.0180 | 0.3095 ± 0.0191 | -0.0240 (0.00098) | +0.0188 (0.0015) | +| 20000 | lift_s45 | 0.2219 ± 0.0083 | 0.2974 ± 0.0082 | -0.0361 (0.00049) | +0.0066 (0.077) | +| 20000 | lift_b0 | 0.2177 ± 0.0146 | 0.2908 ± 0.0137 | -0.0427 (0.00049) | — | diff --git a/research/cosmic-web/docs/E2-report.md b/research/cosmic-web/docs/E2-report.md new file mode 100644 index 00000000000..3b87f627d3f --- /dev/null +++ b/research/cosmic-web/docs/E2-report.md @@ -0,0 +1,14 @@ +# E2 (H4-lite) — dynamics near filaments vs the lifted-geometry premise + +Setup: 4 PM N-body runs (128³ grid & particles, EdS, a: 0.1→1.0, 90 steps), 50k tracked particles per run over a ∈ [0.5, 1.0]. Spines: SE(3) lift on the final density; e3 = tidal minor eigenvector (filament axis). P1: mean |v̂·e3| at a=1 (null 0.5). P2: squared projection of the mid-path deviation from the straight chord onto e3 (null 1/3). dev = mean deviation magnitude (voxels = h⁻¹Mpc). + +| d to spine (vox) | n/seed | P1 ⟨\|v̂·e3\|⟩ | P2 ⟨(d̂ev·e3)²⟩ | ⟨\|dev\|⟩ vox | +|---|---|---|---|---| +| 0–2 | 20262 | 0.559 ± 0.013 | 0.364 ± 0.010 | 1.72 | +| 2–4 | 7244 | 0.537 ± 0.008 | 0.349 ± 0.004 | 1.22 | +| 4–8 | 6730 | 0.544 ± 0.011 | 0.327 ± 0.005 | 0.39 | +| 8–16 | 10315 | 0.572 ± 0.009 | 0.233 ± 0.005 | 0.27 | +| 16–64 | 5447 | 0.564 ± 0.020 | 0.208 ± 0.014 | 0.25 | + +Nulls: P1 = 0.5, P2 = 1/3 ≈ 0.333 (isotropic). Values are across-seed mean ± sd; per-seed n is large so per-seed standard errors are negligible — seed scatter is the relevant uncertainty. + diff --git a/research/cosmic-web/docs/E3-report.md b/research/cosmic-web/docs/E3-report.md new file mode 100644 index 00000000000..9a9673af933 --- /dev/null +++ b/research/cosmic-web/docs/E3-report.md @@ -0,0 +1,44 @@ +# E3 — BOSS CMASS × Planck lensing/tSZ (tiled shell) + +Setup: CMASS-North, z ∈ [0.45, 0.55] (291188 galaxies; 274075 used across 18 tiles of 512 Mpc/h at 4 Mpc/h voxels). Spines at matched length (2400 vox per tile), both methods. Stacks: mean map value at spine sky positions vs 200 footprint-constrained RA-rotated/DEC-reflected controls. First pass: no systematics weights, single shell, projected through the full shell depth. + +## M3 on real data (spine–tidal-frame alignment, null 0.5) + +- **hessian**: 0.617 ± 0.016 (mean ± sd over tiles) +- **se3_lift**: 0.677 ± 0.016 (mean ± sd over tiles) + +## Cross-signal stacks + +| spine set | map | stack | null mean ± std | SNR | +|---|---|---|---|---| +| hessian | kappa | 1.078e-03 | -9.332e-04 ± 5.979e-03 | **0.34** | +| hessian | y | 4.517e-08 | -2.951e-08 ± 1.549e-08 | **4.82** | +| se3_lift | kappa | -1.400e-03 | -6.835e-05 ± 5.940e-03 | **-0.22** | +| se3_lift | y | 3.343e-08 | -2.487e-08 ± 1.409e-08 | **4.14** | + +Reading: positive SNR means the spine network sits on more lensing convergence / hot gas than footprint-matched random placements. H3 asks whether the lift's spines carry at least the Hessian's signal at equal length. + + +## Verdict (first pass) + +1. **H3 — spine networks trace real gas: supported.** Both methods' spine + networks show a ~4–5σ Compton-y excess over footprint-matched controls; + Planck lensing is null (noise-dominated at these scales, as expected). +2. **H3 comparison — Hessian ≥ lift on real cross-signals** (4.8σ vs 4.1σ + at matched length), consistent with the corrected H1 and E1b: the + simpler model is also the better mass/gas tracer on real data. +3. **M3-real — the lift's descriptor advantage replicates on the real + Universe**: spine–tidal-frame alignment 0.677 ± 0.016 vs 0.617 ± 0.016 + across 18 independent tiles (null 0.5) — a clean, tile-replicated + separation matching the simulation finding. + +## Caveats (would tighten a second pass) + +- Controls rotate in RA / reflect in DEC, which changes galactic latitude; + a latitude-dependent y foreground could bias the SNR. Fixed-galactic- + latitude nulls are the right refinement. +- The y excess includes the halo gas of the CMASS tracers themselves, not + only intergalactic filament gas; masking tracer positions would isolate + the WHIM component. +- No FKP/systematics weights; single redshift shell; spines projected + through the full ~250 Mpc/h shell depth. diff --git a/research/cosmic-web/docs/E3b-report.md b/research/cosmic-web/docs/E3b-report.md new file mode 100644 index 00000000000..ebf3c95dc2e --- /dev/null +++ b/research/cosmic-web/docs/E3b-report.md @@ -0,0 +1,29 @@ +# E3b — second-pass tests of the Compton-y detection + +Nulls: 200 control sets drawn inside the survey footprint and rejection-matched to the spine points' galactic-latitude histogram (2° bins) — a latitude-dependent foreground cannot inflate these. Tracer mask: 7.0′ around every shell galaxy, applied identically to signal and nulls. + +| spine set | H3a: b-matched SNR | H3b: halo-masked SNR (kept) | n spine points | +|---|---|---|---| +| hessian | **8.99** | **1.95** (14%) | 41535 | +| se3_lift | **6.86** | **0.07** (17%) | 40426 | + +![Radial Compton-y excess profile around spine networks, signal minus latitude-matched null vs angular distance, both methods](figures/e3b_profile.png) + +Reading: H3a tests foreground robustness; H3b isolates gas away from the tracer halos (WHIM candidate); the profile shape (H3c) separates extended filament gas from point-source-like halo gas. + + +## Verdicts + +- **H3a (foreground robustness): supported, strengthened.** The stack rises + to 8.99σ / 6.86σ under latitude-matched nulls — stricter controls made + the detection cleaner, not weaker. +- **H3b (between-halos gas): not established at Planck depth.** After + masking the tracer halos the excess is 1.95σ (Hessian) / 0.07σ (lift): + the detected y is dominated by the tracers' own halo gas, with at most a + hint of a diffuse component. +- **H3c (profile): extended but non-discriminating.** Positive excess out + to ≥60′ with smooth decay; at a 10′ beam and with clustered tracers this + is consistent with summed halo contributions along the web and cannot by + itself isolate filament gas. +- The Hessian spines beat the lift's on every cross-signal statistic, + completing the pattern from the corrected H1, E1b, and E3. diff --git a/research/cosmic-web/docs/E3c-report.md b/research/cosmic-web/docs/E3c-report.md new file mode 100644 index 00000000000..db6e2baa365 --- /dev/null +++ b/research/cosmic-web/docs/E3c-report.md @@ -0,0 +1,27 @@ +# E3c — the WHIM question at ACT DR6 depth + +ACT DR6 + Planck ILC Compton-y map (1.6′ beam, CAR), BOSS CMASS spine networks restricted to the ACT footprint. Same battery as E3b: 200 latitude-matched nulls; tracer-halo masks at [3.0, 7.0] arcmin; radial profile bins [0, 2, 4, 8, 16, 32] arcmin. + +| spine set | n pts in ACT | H3a SNR | H3b 3′ mask (kept) | H3b 7′ mask (kept) | +|---|---|---|---|---| +| hessian | 18804 | **2.95** | **1.77** (65%) | **1.11** (14%) | +| se3_lift | 18917 | **1.62** | **-0.08** (67%) | **0.62** (18%) | + +![Radial Compton-y excess profile around spine networks at ACT resolution](figures/e3c_profile.png) + + +## Verdict (pre-registered gate: masked residual ≥ 3σ to proceed) + +- **H3b at ACT depth: not established.** Masked residuals: 1.77σ (3′ mask, + 65% of points kept) and 1.11σ (7′, 14% kept) for the Hessian spines; + ~0 for the lift's. Model A (halo-only gas) survives. Result: an upper + limit on the between-halo Compton-y along CMASS spines at these scales. +- **Instrument–analysis mismatch identified.** Only 45% of spine points + lie in the ACT footprint, and the unmasked stack drops to 2.95σ (vs 9σ + on Planck): the shell-projected signal is coherent on degree scales, + which the ACT DR6 ILC map suppresses while excelling at arcminutes. + Exploiting ACT depth requires a small-scale design (thin redshift + shells or individual-filament cross-sections), not 250 Mpc/h + projections — the concrete next-proposal insight. +- Per the program gate (below 3σ → stop and publish limits), this closes + the observational branch. diff --git a/research/cosmic-web/docs/E3d-report.md b/research/cosmic-web/docs/E3d-report.md new file mode 100644 index 00000000000..d10f90bbc5c --- /dev/null +++ b/research/cosmic-web/docs/E3d-report.md @@ -0,0 +1,23 @@ +# E3d — galaxy-pair bridge stack (small-scale WHIM design) + +Setup: CMASS-N z 0.45–0.55 pairs with transverse separation 6.0–14.0 Mpc/h and line-of-sight separation < 6.0 Mpc/h (570049 pairs). Estimator: y at the pair midpoint minus the mean of the four off-axis points at identical angular distance from each galaxy (±90° rotations) — a circularly symmetric halo contributes zero by construction, so any positive mean is gas that lives preferentially ON the inter-pair axis. Errors: bootstrap over pairs (500 resamples). + +| map | pairs used | bridge excess (y) | error | SNR | +|---|---|---|---|---| +| ACT | 237813 | 8.211e-09 | 9.448e-09 | **0.87** | +| PLANCK | 570049 | 3.389e-08 | 2.265e-09 | **14.96** | + +## Verdict and caveat + +- **Planck 15σ is a bridge + leakage upper bound, not a pure detection.** + At a 10′ beam, the second halo's smeared profile contributes more at the + midpoint than at the off-axis control (which sits farther from it), so + the estimator retains a positive halo term. The measured 3.4×10⁻⁸ bounds + the true bridge from above. +- **ACT is clean (leakage negligible at 1.6′) but underpowered:** + ±9.4×10⁻⁹ per-pair-set error cannot resolve a literature-level bridge of + ~1×10⁻⁸ (would be ~1σ — exactly what we see: 0.87σ). +- Both maps are mutually consistent with Model B at the ~10⁻⁸ level. + Next (E3d-v2): full CMASS z-range, disc-averaged sampling, multi-angle + ring controls — projected ACT error ~2–3×10⁻⁹, turning 10⁻⁸ into a + 3–5σ test. diff --git a/research/cosmic-web/docs/E3d2-report.md b/research/cosmic-web/docs/E3d2-report.md new file mode 100644 index 00000000000..cf5651dc82f --- /dev/null +++ b/research/cosmic-web/docs/E3d2-report.md @@ -0,0 +1,24 @@ +# E3d-v2 — powered pair-bridge measurement + +CMASS-N z 0.43–0.7 (876639 pairs, transverse 6.0–14.0 Mpc/h, LOS < 6.0). Estimator: 2.0′-disc-averaged y at the midpoint minus the mean over ring controls at ±60/±90/±120° around each galaxy. Bootstrap errors over pairs. + +| map | pairs used | bridge excess (y) | error | SNR | +|---|---|---|---|---| +| ACT | 361057 | 2.278e-08 | 4.348e-09 | **5.24** | +| PLANCK | 876639 | 2.966e-08 | 1.536e-09 | **19.31** | + +## Verdict — gate passed + +- **Model B established at ACT depth: 5.24σ** (gate: ≥3σ). At 1.6′ + resolution second-halo beam leakage is negligible, so the on-axis + excess is inter-halo gas: the filament bridge. Amplitude + 2.28×10⁻⁸, consistent with Planck's 2.97×10⁻⁸ (19σ, small residual + leakage expected at 10′) and with published LRG-pair bridge + measurements (~1–2×10⁻⁸ for narrower separation windows). +- Two instruments, one geometric estimator, agreeing amplitudes — + the between-halos question that E3b/E3c could only bound is now a + detection under the design matched to the instrument. +- Remaining caveats for a publication-grade number: model-based + two-halo subtraction (our ring estimator is geometric), dust/CIB + null tests (145 vs 90 GHz, NILC vs MILCA), and bootstrap over sky + patches rather than pairs (pairs overlap on the sky). diff --git a/research/cosmic-web/docs/E3e-report.md b/research/cosmic-web/docs/E3e-report.md new file mode 100644 index 00000000000..0127020b1d2 --- /dev/null +++ b/research/cosmic-web/docs/E3e-report.md @@ -0,0 +1,41 @@ +# E3e — validity battery for the bridge detection + +Jackknife errors over 50 RA patches (pairs overlap on the sky, so pair-bootstrap is too optimistic). Null pairs: same transverse window, line-of-sight separation 25–40 Mpc/h — same sky geometry, no physical bridge. + +| map | pair set | n | bridge (y) | jackknife err | SNR | +|---|---|---|---|---|---| +| ACT | connected | 361057 | 2.278e-08 | 6.911e-09 | **3.30** | +| ACT | null | 460389 | 8.449e-09 | 5.130e-09 | **1.65** | +| PLANCK | connected | 876639 | 2.967e-08 | 3.715e-09 | **7.99** | +| PLANCK | null | 1117971 | 1.801e-08 | 3.828e-09 | **4.70** | + +## ACT bridge vs transverse separation (connected pairs) + +| separation (Mpc/h) | n | bridge (y) | err | SNR | +|---|---|---|---|---| +| 6-8 | 82303 | 2.601e-08 | 1.227e-08 | 2.12 | +| 8-10 | 87752 | 2.753e-08 | 1.200e-08 | 2.29 | +| 10-12 | 92889 | 1.766e-08 | 1.017e-08 | 1.74 | +| 12-14 | 98109 | 1.732e-08 | 1.121e-08 | 1.54 | + +## Verdict — the claim after the battery + +1. **Honest errors**: jackknife over sky patches deflates pair-bootstrap + SNRs by ~1.6–2.4×: ACT connected 3.30σ (gate ≥3 still passed), + Planck 7.99σ. +2. **Null pairs**: zero on ACT (1.65σ) but **4.7σ nonzero on Planck** + (1.80×10⁻⁸) — direct quantitative confirmation of second-halo beam + leakage at 10′. +3. **Null-subtracted bridge** (the physically meaningful number): + ACT 1.4×10⁻⁸ ± 0.9 (≈1.6σ), Planck 1.2×10⁻⁸ ± 0.5 (≈2.2σ). + Mutually consistent and matching published LRG-pair bridge amplitudes + (~1×10⁻⁸). +4. **Final claim**: the pipeline reproduces the literature bridge + amplitude with honest errors and physical nulls, at ~2σ per + instrument. The earlier 5.24σ was on-axis excess including + correlated-structure/leakage components; a >3σ *bridge-specific* + claim from this data would need model-based halo subtraction and a + joint-map likelihood — the natural scope of a dedicated paper. +5. Separation bins are flat within errors (2.6→1.7×10⁻⁸ over 6–14 + Mpc/h) — no steep decline, consistent with a genuine extended + component plus leakage, discriminating power limited at n≈90k/bin. diff --git a/research/cosmic-web/docs/E4-report.md b/research/cosmic-web/docs/E4-report.md new file mode 100644 index 00000000000..86ae3a2f02d --- /dev/null +++ b/research/cosmic-web/docs/E4-report.md @@ -0,0 +1,31 @@ +# E4 (H4') — the geometry-adjustment term on top of flat transport + +Setup: 6 PM sims (128³, EdS), 50k tracked particles each. Baseline: the Zel'dovich prediction for the SAME particle from the same initial conditions. R = true − ZA displacement at a=1; Rv = true − ZA velocity. If the lifted geometry is a valid adjustment term, R and Rv should point preferentially along the filament axis e3 near the web (null 1/3) with amplitude growing toward spines. + +| d to spine (vox) | n/seed | ⟨(R̂·e3)²⟩ | ⟨(R̂v·e3)²⟩ | RMS R∥ (vox) | RMS R⊥/axis | anisotropy R∥/R⊥ | +|---|---|---|---|---|---|---| +| 0–2 | 20658 | 0.298 ± 0.002 | 0.312 ± 0.003 | 4.36 | 4.92 | **0.89** | +| 2–4 | 7269 | 0.297 ± 0.004 | 0.300 ± 0.004 | 3.77 | 4.19 | **0.90** | +| 4–8 | 6632 | 0.264 ± 0.004 | 0.280 ± 0.003 | 2.40 | 3.07 | **0.78** | +| 8–16 | 10320 | 0.218 ± 0.002 | 0.218 ± 0.003 | 1.86 | 2.91 | **0.64** | +| 16–64 | 5120 | 0.208 ± 0.003 | 0.203 ± 0.003 | 1.77 | 2.83 | **0.63** | + +Null for the direction statistics: 1/3 (isotropic). R∥/R⊥ = 1 means the correction is isotropic; > 1 means the residual transport is preferentially along the filament axis — the measured size of the geometry adjustment. + + +## Verdict on H4' (geometry as an adjustment term) + +Refuted in its stated form — and answered constructively. The correction +to flat (Zel'dovich) transport is large (RMS ≈ 4.4 vox near spines), and +it IS organized by the tidal frame — but its direction statistic sits +below the isotropic null everywhere (0.21–0.30 vs 1/3) and the per-axis +ratio R∥/R⊥ is 0.63–0.89: the residual points preferentially ACROSS the +filament axis at all distances. The dominant adjustment term to current +(ZA-type) models is transverse arrest at shell-crossing — precisely the +adhesion model's viscosity, made anisotropic by the tidal frame — not +longitudinal geodesic transport. An improved effective model would add a +tidal-frame-anisotropic damping of the perpendicular momentum components; +the lean-space geometry enters through e1/e2, not e3. Consistent with T1 +(shocks, not geodesics) and E2 (transverse bulk deviations; the weak +in-tube parallel signal was a chord effect and trends toward isotropy in +the ZA-residual frame). diff --git a/research/cosmic-web/docs/E5-report.md b/research/cosmic-web/docs/E5-report.md new file mode 100644 index 00000000000..ae842585e1c --- /dev/null +++ b/research/cosmic-web/docs/E5-report.md @@ -0,0 +1,33 @@ +# E5 (H-T2) — transverse-damping adjustment vs ZA and isotropic sticking + +Effective models evolved from PM initial conditions (rho_c = 5.0, calibrated on seed 1 by optimizing the ISOTROPIC competitor; 5 held-out seeds). Score: median per-particle distance to the PM-truth position at a=1 (voxels = h⁻¹Mpc), binned by distance to the web. + +| d to spine (vox) | ZA | isotropic stick | transverse damp | damp vs ZA | damp vs stick | +|---|---|---|---|---|---| +| 0–2 | 6.95 ± 0.24 | 7.50 ± 0.10 | 6.46 ± 0.07 | -7% | -14% | +| 2–4 | 5.43 ± 0.18 | 6.19 ± 0.15 | 5.67 ± 0.14 | +4% | -8% | +| 4–8 | 3.74 ± 0.15 | 3.88 ± 0.13 | 3.78 ± 0.13 | +1% | -3% | +| 8–64 | 3.38 ± 0.13 | 3.43 ± 0.13 | 3.37 ± 0.12 | -0% | -2% | +| **all** | 5.00 ± 0.22 | 5.34 ± 0.15 | 4.91 ± 0.14 | -2% | -8% | + +Reading: negative percentages mean the transverse-damping model is closer to the N-body truth. H-T2 asks for damp < ZA (the adjustment helps) and damp <= stick (keeping the along-filament component does not hurt, unlike full sticking). + + +## Verdict on H-T2 + +**Supported near the web, with an honest qualification in the outskirts.** +At 0–2 vox from spines the transverse-damping model beats plain ZA by 7% +and isotropic sticking by 14% (jackknife over 5 held-out seeds; the +crossing threshold was calibrated to favour the isotropic competitor). In +the 2–4 vox ring it slightly trails ZA (+4%) — plausibly premature damping +triggered by noisy tidal frames at moderate densities — while still +beating sticking by 8%. Far from the web all models agree. Net: best +overall model of the three (4.91 vs 5.00 vs 5.34). + +Constructive conclusion of the program: the "lean-space geometry +adjustment" conjecture, corrected by E4 into its transverse form, is a +WORKING model term — an anisotropic, tidal-frame-aware sticking condition +that improves the classical adhesion proxy. Natural refinements (partial +damping beta < 1, lambda-dependent thresholds, better frame estimation in +the outskirts) are tuning, deliberately left out to keep this a +hypothesis test rather than a fit. diff --git a/research/cosmic-web/docs/E5b-report.md b/research/cosmic-web/docs/E5b-report.md new file mode 100644 index 00000000000..675da77fab6 --- /dev/null +++ b/research/cosmic-web/docs/E5b-report.md @@ -0,0 +1,24 @@ +# E5b — oracle-frame test of the transverse-damping deficit + +Same protocol as E5 (rho_c = 5.0, 5 held-out seeds); 'oracle' uses the tidal frame of the true PM final field for damping decisions instead of the evolving ZA proxy. + +| d to spine (vox) | ZA | damp (proxy frames) | damp (oracle frames) | oracle vs ZA | +|---|---|---|---|---| +| 0–2 | 6.95 ± 0.24 | 6.51 ± 0.07 | 5.53 ± 0.08 | -20% | +| 2–4 | 5.43 ± 0.18 | 5.69 ± 0.14 | 5.20 ± 0.12 | -4% | +| 4–8 | 3.74 ± 0.15 | 3.79 ± 0.13 | 3.68 ± 0.12 | -1% | +| 8–64 | 3.38 ± 0.13 | 3.37 ± 0.12 | 3.33 ± 0.12 | -2% | +| **all** | 5.00 ± 0.22 | 4.93 ± 0.14 | 4.47 ± 0.12 | -10% | + +Reading: if the oracle closes the 2–4 vox gap to ZA, the E5 deficit was frame-estimation error and the damping physics is validated; if the gap persists, the crossing condition itself over-fires in the outskirts. + + +## Verdict + +H-E5b confirmed: with oracle frames the transverse-damping model beats ZA +in EVERY distance bin (−20% at the web, −4% at 2–4 vox, −10% overall). +The E5 outskirts deficit was frame-estimation error, not a failure of the +damping physics. Two consequences: (1) the anisotropic-adhesion term is +uniformly correct given good frames; (2) frame estimation is the binding +constraint — the oracle bound (4.47 vs proxy 4.93) prices ~0.46 vox of +recoverable error, more than any other refinement lever. diff --git a/research/cosmic-web/docs/E5c-report.md b/research/cosmic-web/docs/E5c-report.md new file mode 100644 index 00000000000..d99fb89c57b --- /dev/null +++ b/research/cosmic-web/docs/E5c-report.md @@ -0,0 +1,35 @@ +# E5c — refinement campaign (declared optimization) + +Config grid over frame source (ZA proxy vs model self-density), damping fraction β, crossing threshold ρ_c, and sequential (pancake-ordered) damping. Selection on seeds {2,3}; the top two configs validated on held-out seeds {4,5,6}. References: ZA and the E5b oracle bound (4.47 overall). + +## Selection scores (median voxel error, seeds 2–3) + +| config | overall error | +|---|---| +| C_self_b075_r5 | 4.662 | +| F_self_b075_r3_sq | 4.837 | +| A_proxy_b1_r5 | 4.926 | +| E_self_b1_r5_sq | 4.943 | +| B_self_b1_r5 | 5.024 | +| D_self_b1_r3 | 5.403 | + +## Held-out validation (seeds 4–6) + +| quantity | ZA | C_self_b075_r5 | F_self_b075_r3_sq | +|---|---|---|---| +| **all** | 4.98 ± 0.27 | 4.64 ± 0.18 | 4.83 ± 0.15 | +| 0–2 | — | 5.96 ± 0.13 | 6.26 ± 0.12 | +| 2–4 | — | 5.37 ± 0.23 | 5.73 ± 0.21 | +| 4–8 | — | 3.67 ± 0.14 | 3.83 ± 0.13 | +| 8–64 | — | 3.28 ± 0.11 | 3.26 ± 0.09 | + +## Verdict + +Held-out winner: **self-density frames + β = 0.75 partial transverse +damping** (C): 4.64 ± 0.18 vs ZA 4.98 ± 0.27 (−7% overall; −14% at the +web vs ZA), recovering ~63% of the oracle-bound gap (4.93 → 4.64 of the +0.46 available). Structural findings: (1) partial damping is what makes +self-density frames profitable — at β = 1 the model's own density +feedback over-triggers crossings and cancels the frame gain; (2) +sequential pancake-ordered damping (e1 then e2) underperforms simple +both-perpendicular damping at 1 Mpc resolution. diff --git a/research/cosmic-web/docs/E5d-report.md b/research/cosmic-web/docs/E5d-report.md new file mode 100644 index 00000000000..e3869ac31e2 --- /dev/null +++ b/research/cosmic-web/docs/E5d-report.md @@ -0,0 +1,6 @@ +# E5d — final knob round (beta x frame smoothing) + +Selection scores (seeds 2-3): b0.6_s2=4.532, b0.6_s4=4.556, b0.75_s2=4.640, b0.75_s4=4.671, b0.9_s2=4.831, b0.9_s4=4.874 + + +Held-out (seeds 4-6): ZA 4.98 +/- 0.27; **b0.6_s2** 4.52 +/- 0.18 (-9% vs ZA; oracle bound 4.47; E5c winner 4.64). diff --git a/research/cosmic-web/docs/E6-report.md b/research/cosmic-web/docs/E6-report.md new file mode 100644 index 00000000000..7a62a6ea263 --- /dev/null +++ b/research/cosmic-web/docs/E6-report.md @@ -0,0 +1,24 @@ +# E6 — transfer test of the frozen model (no re-calibration) + +Recipe frozen at β=0.6, frame smoothing 2.0 Mpc/h, ρ_c=5.0; 3 held-out seeds per condition. Voxel sizes differ across conditions, so compare the RELATIVE advantage over ZA, not absolute voxel errors. + +| condition | ZA all | model all | Δ all | ZA web | model web | Δ web | +|---|---|---|---|---|---|---| +| base | 4.98 | 4.52 | **-9%** | 6.93 | 5.78 | **-17%** | +| coarse | 2.09 | 2.04 | **-2%** | 3.37 | 3.08 | **-9%** | +| s8lo | 2.98 | 2.89 | **-3%** | 4.73 | 4.26 | **-10%** | +| s8hi | 7.20 | 6.23 | **-13%** | 9.36 | 7.33 | **-22%** | + +Success criterion: Δ ≤ 0 in every condition (advantage persists with frozen knobs). Degradation localizes which parameter is condition-dependent. + + +## Verdict + +H-transfer supported in all conditions with frozen knobs: the advantage +over ZA persists at 2 Mpc/h voxels (−2% overall, −9% web) and at both +clustering amplitudes, and it GROWS with sigma8 (−3% → −9% → −13% +overall; −10% → −17% → −22% at the web) — the monotone trend expected of +a physical shell-crossing correction, not a tuned artifact. The model +card's resolution/amplitude limitation is closed; remaining scope +limits are LCDM time-stepping (engineering) and multi-stream interiors +(out of scope by design). diff --git a/research/cosmic-web/docs/E7-report.md b/research/cosmic-web/docs/E7-report.md new file mode 100644 index 00000000000..623bc4bd976 --- /dev/null +++ b/research/cosmic-web/docs/E7-report.md @@ -0,0 +1,18 @@ +# E7 — final scope rows: LCDM dynamics and sub-Mpc resolution + +Frozen knobs throughout; errors in PHYSICAL Mpc/h; web = within 2 Mpc/h of the spine network. + +| run | voxel (Mpc/h) | ZA all | model all | Δ all | ZA web | model web | Δ web | +|---|---|---|---|---|---|---|---| +| LCDM Ωm=0.31 (3 seeds) | 1.0 | 4.93 | 4.45 | **-10%** | 6.90 | 5.73 | **-17%** | +| EdS 0.5 Mpc vox (1 seed) | 0.5 | 4.95 | 4.45 | **-10%** | 6.76 | 5.72 | **-15%** | + +## Verdict + +Both hypotheses supported with frozen knobs. H-LCDM: −10% overall / −17% +web under real-universe expansion (Ωm = 0.31), matching the EdS numbers — +cosmology-independent, as expected for a D-parametrized geometric term. +H-highres: −10% / −15% at 0.5 Mpc/h voxels (single-seed, indicative), with +absolute physical errors nearly identical to the 1 Mpc runs — the error +scale is set by the physics, not the grid. The model card's scope table +has no unverified rows remaining that are testable on this machine. diff --git a/research/cosmic-web/docs/E8-report.md b/research/cosmic-web/docs/E8-report.md new file mode 100644 index 00000000000..9bae47f00f9 --- /dev/null +++ b/research/cosmic-web/docs/E8-report.md @@ -0,0 +1,42 @@ +# E8 — field-level fidelity (the mock-catalogue criterion) + +Full-set density fields vs PM truth, 3 seeds, 1 Mpc/h voxels. r(k): phase/structure fidelity (1 = perfect); T(k): amplitude fidelity (1 = unbiased). + + +## Cross-correlation r(k) + +| k (h/Mpc) | ZA | isotropic stick | transverse damp | +|---|---|---|---| +| 0.06 | 0.997 | 0.991 | 0.997 | +| 0.09 | 0.993 | 0.970 | 0.989 | +| 0.14 | 0.975 | 0.890 | 0.962 | +| 0.22 | 0.924 | 0.730 | 0.901 | +| 0.34 | 0.710 | 0.374 | 0.716 | +| 0.53 | 0.288 | 0.089 | 0.388 | +| 0.82 | 0.057 | 0.008 | 0.095 | +| 1.26 | 0.005 | -0.001 | 0.009 | + +## Transfer T(k) + +| k (h/Mpc) | ZA | isotropic stick | transverse damp | +|---|---|---|---| +| 0.06 | 0.920 | 0.683 | 0.849 | +| 0.09 | 0.842 | 0.679 | 0.827 | +| 0.14 | 0.693 | 0.593 | 0.739 | +| 0.22 | 0.469 | 0.458 | 0.601 | +| 0.34 | 0.229 | 0.300 | 0.410 | +| 0.53 | 0.098 | 0.222 | 0.262 | +| 0.82 | 0.052 | 0.187 | 0.143 | +| 1.26 | 0.042 | 0.172 | 0.079 | + +## Verdict (H-field) + +Supported with quantified caveats. In the nonlinear regime the transverse +damping model beats ZA on both statistics (r at k=0.53: 0.388 vs 0.288; +T at k=0.34: 0.410 vs 0.229) — it extends the usable k-range of the mock — +while classical isotropic sticking degrades r(k) catastrophically at all +k > 0.1. Costs: ~7% amplitude deficit in the largest-scale bin +(0.849 vs 0.920; removable by the standard linear transfer rescaling) and +a small phase cost at intermediate k (r 0.962 vs 0.975 at k=0.14). E7b: +the 0.5 Mpc resolution row is now verified at n=3 (−12%, −9% on the two +added seeds). diff --git a/research/cosmic-web/docs/E9-report.md b/research/cosmic-web/docs/E9-report.md new file mode 100644 index 00000000000..a406bec7891 --- /dev/null +++ b/research/cosmic-web/docs/E9-report.md @@ -0,0 +1,46 @@ +# E9 — literature baselines: 2LPT and MUSCLE vs the frozen model + +Identical initial conditions, PM truth, 3 seeds. Transport errors in voxels (= h⁻¹Mpc); web = within 2 of the spine network. + +## Per-particle transport error (median) + +| model | all | web | +|---|---|---| +| ZA | 4.97 ± 0.27 | 6.92 ± 0.31 | +| 2LPT | 8.07 ± 0.39 | 8.21 ± 0.35 | +| MUSCLE | 4.49 ± 0.12 | 5.57 ± 0.07 | +| transverse damp | 4.52 ± 0.18 | 5.77 ± 0.16 | + +## Cross-correlation r(k) + +| k (h/Mpc) | ZA | 2LPT | MUSCLE | transverse damp | +|---|---|---|---|---| +| 0.14 | 0.975 | 0.933 | 0.956 | 0.962 | +| 0.34 | 0.710 | 0.691 | 0.771 | 0.716 | +| 0.53 | 0.288 | 0.430 | 0.527 | 0.388 | +| 0.82 | 0.057 | 0.144 | 0.228 | 0.095 | + +## Transfer T(k) + +| k (h/Mpc) | ZA | 2LPT | MUSCLE | transverse damp | +|---|---|---|---|---| +| 0.14 | 0.693 | 0.607 | 0.669 | 0.739 | +| 0.34 | 0.229 | 0.202 | 0.433 | 0.410 | +| 0.53 | 0.098 | 0.095 | 0.305 | 0.262 | +| 0.82 | 0.052 | 0.048 | 0.210 | 0.143 | + +## Verdict — the framing decision + +MUSCLE (Neyrinck 2016) statistically ties the frozen transverse-damping +model on transport (4.49 ± 0.12 vs 4.52 ± 0.18; slightly ahead at the +web) and wins small-scale phase fidelity, while the damping model leads +on mid-scale amplitude T(k). 2LPT degrades transport at these nonlinear +scales (8.07 — the known shell-crossing overshoot). Consequence for the +paper: the contribution is NOT "a better mock engine than the +literature"; it is the MECHANISM result — E4 measured that the correction +budget beyond ZA is transverse in the tidal frame, and a model built +from exactly that single directional ingredient reproduces MUSCLE-level +transport. MUSCLE-class shell-crossing corrections work, we can now say, +BECAUSE they implement transverse arrest; the two models' complementary +field-level strengths (phases vs amplitudes) suggest a hybrid as future +work. diff --git a/research/cosmic-web/docs/MODEL-CARD.md b/research/cosmic-web/docs/MODEL-CARD.md new file mode 100644 index 00000000000..23135d458cd --- /dev/null +++ b/research/cosmic-web/docs/MODEL-CARD.md @@ -0,0 +1,111 @@ +# Model card — anisotropic-adhesion effective transport model ("transverse damping") + +## What this model does + +Input: a linear initial density field (equivalently its Zel'dovich +displacement field ψ) on a periodic grid. Output: approximate final +comoving particle positions at a = 1, at a small fraction of an N-body +run's cost (~18 cheap steps vs 90 force-solving steps; no Poisson solves +except cheap density deposits). Downstream consumers: fast mock-catalogue +generation, initial-condition reconstruction experiments, and any +application currently using the Zel'dovich approximation or the classical +adhesion model as its transport engine. + +Recipe (frozen, E5d): evolve particles on straight Zel'dovich rays in +growth-factor steps ΔD = 0.05; at each step deposit the model's own +particles to a density grid; the first time a particle's local density +exceeds ρ_c = 5, remove **β = 60%** of its velocity components +perpendicular to the local filament axis e₃ (minor eigenvector of the +tidal tensor of the model's own density, smoothed at 2 h⁻¹Mpc, refreshed +every 3rd step); keep the along-filament component intact. + +## Capabilities + +| capability | evidence (median per-particle error vs 128³ PM N-body truth, held-out seeds) | +|---|---| +| Better global transport than Zel'dovich | 4.52 vs 4.98 vox (−9%), 3 held-out seeds × 50k particles | +| Better than classical isotropic adhesion proxy | isotropic sticking scores 5.34 (E5); this model 4.52 | +| Matches MUSCLE on transport; complementary field strengths | MUSCLE 4.49 ± 0.12 vs 4.52 ± 0.18 (tie); MUSCLE wins small-scale r(k), this model wins mid-scale T(k); 2LPT degrades to 8.07 (E9) | +| Near the frame-information optimum | oracle-frame bound 4.47; model reaches 4.52 (89% of the recoverable gap closed) | +| Largest gains where structure forms | at the web (0–2 vox from spines): −14% vs ZA already at the E5c stage; oracle shows −20% available | +| Preserves along-filament flow | by construction (e₃ component undamped); isotropic sticking destroys it | + +## Limitations + +| limitation | magnitude | +|---|---| +| One-shot damping (first crossing only) — no multi-stream hierarchy | untested beyond first crossing; likely underdamps cluster cores | +| Transfer verified with frozen knobs (E6, E7): voxels 0.5–2 h⁻¹Mpc, σ₈ ∈ [0.6, 1.0], EdS and flat ΛCDM (Ωm = 0.31); advantage grows with clustering and is cosmology/resolution-invariant in physical units | high-res verified at n = 3 seeds (−9% to −12%); field-level fidelity verified (E8: extends usable k-range, ~7% large-scale amplitude deficit removable by linear rescaling); multi-stream interiors remain out of scope | +| Sequential (pancake-ordered) damping variant underperforms here | −4% worse than both-perp at this resolution (E5c) — may differ at higher resolution | +| Gains are modest far from the web | −2% at 8–64 vox: the model is a web-region correction, not a global one | +| β, ρ_c calibrated on this box size / tracer density | re-calibrate for other setups; β optimum may sit slightly below 0.6 (flat minimum) | + +## Per-segment table (distance to the filament web, voxels = h⁻¹Mpc) + +| segment | n/seed | ZA err | this model (E5c stage / E5d winner overall) | oracle bound | +|---|---|---|---|---| +| 0–2 | ~21k | 6.95 | 5.96 | 5.53 | +| 2–4 | ~7k | 5.43 | 5.37 | 5.20 | +| 4–8 | ~7k | 3.74 | 3.67 | 3.68 | +| 8–64 | ~15k | 3.38 | 3.28 | 3.33 | +| **all** | 50k | **4.98** | **4.52** (E5d) | **4.47** | + +(Per-bin numbers from the E5c winner β=0.75/smooth-4; the E5d winner +β=0.6/smooth-2 improves the overall figure; jackknife errors ±0.1–0.3.) + +## Failure modes + +- **F1 — over-damping feedback at β = 1.** Symptom: model underperforms + even the ZA-proxy version. Where: everywhere, via density feedback + (damped particles pile up → more spurious crossings). Magnitude: +0.1 + vox global. Fix: β ≤ 0.75 (frozen recipe uses 0.6). +- **F2 — noisy frames in the outskirts.** Symptom: small deficit vs ZA at + 2–4 vox with proxy frames. Magnitude: +4% in that ring (E5). + Fix: self-density frames at 2 h⁻¹Mpc smoothing (in the recipe); oracle + test (E5b) confirms the physics is not at fault. +- **F3 — untreated post-crossing dynamics.** Symptom: residual error at + the web (5.5–6 vox) remains well above zero even at the oracle bound. + Where: virialized interiors. Impact: don't use this model for + intra-halo structure. Fix: out of scope; requires multi-stream + treatment. + +## Downstream integration guide + +1. Generate ψ from your linear density (`fields.zeldovich_box` internals). +2. Call `evolve_full(q_grid, psi_full, beta=0.6, smooth=2.0)` as in + `scripts/run_e5d.py` (ρ_c = 5, ΔD = 0.05, frames every 3rd step). +3. Re-calibrate (ρ_c, β) if your grid resolution or tracer density + differs; calibrate against a small PM truth run as in E5 (optimize the + *competitor* first to keep the comparison honest). +4. Do not interpret positions within ~2 vox of density peaks as resolved + halo structure (F3). + +## Files & artefacts + +| path | contents | +|---|---| +| `scripts/run_e5.py` | model v1 (proxy frames) + calibration protocol | +| `scripts/run_e5b.py` | oracle-frame bound experiment | +| `scripts/run_e5c.py` | config grid incl. sequential variant, `evolve_full` | +| `scripts/run_e5d.py` | final β × smoothing grid; frozen winner | +| `artifacts/e5*_results.json` | all runs, per-seed numbers | +| `docs/E4-report.md` … `docs/E5d-report.md` | the evidence chain | + +## Scope of the calibration/validation set + +| item | value | +|---|---| +| truth engine | PM N-body, 128³ grid & particles, EdS, a: 0.1→1, 90 KDK steps | +| box | 128 h⁻¹Mpc (1 vox = 1 h⁻¹Mpc), BBKS spectrum, σ₈ = 0.8 | +| seeds | selection {2,3}; validation {4,5,6}; ρ_c calibration seed 1 | +| tracked particles | 50,000/seed (250k held-out measurements) | + +## Reproduce + +```bash +cd research/cosmic-web +.venv/bin/python scripts/run_e5.py # v1 + rho_c calibration +.venv/bin/python scripts/run_e5b.py # oracle bound +.venv/bin/python scripts/run_e5c.py # config grid +.venv/bin/python scripts/run_e5d.py # final grid -> frozen recipe +``` diff --git a/research/cosmic-web/docs/PAPER-DRAFT.md b/research/cosmic-web/docs/PAPER-DRAFT.md new file mode 100644 index 00000000000..b8dcdb660bb --- /dev/null +++ b/research/cosmic-web/docs/PAPER-DRAFT.md @@ -0,0 +1,207 @@ +# The correction beyond Zel'dovich is transverse: a measured mechanism for shell-crossing models of cosmic-web transport + +*Draft — 2026-07-05. Source of every number: the experiment reports in `research/cosmic-web/docs/` (E0–E12, T1, MODEL-CARD).* + +## Abstract + +Lagrangian approximations to cosmic structure formation — the Zel'dovich approximation (ZA), 2LPT, adhesion, and shell-crossing prescriptions of the MUSCLE class — are the transport engines of fast mock catalogues and reconstruction, yet *why* shell-crossing corrections outperform perturbative ones has not been directly measured. Using particle-mesh (PM) N-body simulations evolved from the same initial conditions as the approximations they test, we measure the residual between true transport and each particle's own ZA prediction in the local tidal eigenframe. The residual is large near the web (RMS ≈ 4.4 h⁻¹Mpc within 2 h⁻¹Mpc of filament spines) and biased perpendicular to the filament axis at *all* distances (⟨(R̂·e₃)²⟩ = 0.21–0.30 versus the isotropic 1/3; 6 simulations × 50,000 particles). A single-ingredient model built from exactly this measurement — ZA rays plus partial (β = 0.6) transverse momentum damping at first shell-crossing — reduces held-out median transport error from 4.98 to 4.52 h⁻¹Mpc, within 0.05 of the frame-information oracle bound (4.47), and statistically ties MUSCLE (4.49 ± 0.12 vs 4.52 ± 0.18, 3 seeds) while 2LPT degrades to 8.07. With frozen parameters the advantage persists across σ₈ ∈ [0.6, 1.0] (growing with clustering, −3% → −13%), voxel scales 0.5–2 h⁻¹Mpc, and EdS versus flat ΛCDM. The exact variational structure of the ZA/adhesion flow — Hopf–Lax quadratic-cost optimal transport on a flat metric — identifies the measured term as the shock-formation correction: shell-crossing models work because they implement transverse arrest. + +## 1. Introduction + +Fast approximations to gravitational transport underpin much of large-scale-structure practice. The Zel'dovich approximation (ZA; Zel'dovich 1970) moves mass on straight ballistic trajectories set by the initial gravitational potential; second-order Lagrangian perturbation theory (2LPT; Bouchet et al. 1995, Scoccimarro 1998) adds the leading nonlinear correction; the adhesion model (Gurbatov, Saichev & Shandarin 1989) regularises the ZA's multi-streaming with an infinitesimal Burgers viscosity so that matter sticks at shell-crossing; spherical-collapse remappings and their multiscale extension MUSCLE (Neyrinck 2013, 2016) replace the ZA divergence with a collapse-aware prescription; ALPT (Kitaura & Heß 2013) splits scales between 2LPT and spherical collapse; COLA (Tassev, Zaldarriaga & Eisenstein 2013) trades exactness for a few corrective time steps. A consistent empirical pattern runs through this literature: at nonlinear scales, prescriptions that *arrest* particles at shell-crossing beat prescriptions that *perturb* their trajectories — MUSCLE-class models outperform 2LPT, which is known to overshoot in collapsed regions. What has been missing is a direct measurement of the correction these models are implicitly supplying: the residual between true gravitational transport and the ZA prediction, resolved in the frame that organises collapse. + +This paper supplies that measurement and traces its consequences. It is a mechanism paper, not a better-engine paper: the model we build is a probe of *why* shell-crossing corrections work, and its headline result is that a single directional ingredient reproduces MUSCLE-level transport accuracy. + +Our contributions, in order: + +1. **A direct measurement of the beyond-ZA correction (Section 3.1).** In PM N-body simulations evolved from the same initial conditions, the per-particle residual R = x_true − x_ZA is large near the web and preferentially *perpendicular* to the filament axis in the local tidal eigenframe at all distances: ⟨(R̂·e₃)²⟩ = 0.21–0.30 versus the isotropic 1/3, with per-axis RMS ratio R∥/R⊥ = 0.63–0.89 (6 simulations, 50,000 tracked particles each). +2. **A sufficiency test (Section 3.2).** A model containing only this ingredient — ZA rays plus partial transverse momentum damping at first shell-crossing — reaches held-out median transport error 4.52 ± 0.18 h⁻¹Mpc versus ZA's 4.98 ± 0.27, within 0.05 of an oracle bound (4.47 ± 0.12) obtained by giving the model the true final tidal frames. On identical initial conditions it statistically ties MUSCLE (4.49 ± 0.12) while 2LPT degrades to 8.07 ± 0.39. This explains the MUSCLE-class advantage: transverse arrest is the correction budget. +3. **Robustness with frozen parameters (Section 3.3).** The advantage transfers without re-calibration across σ₈ = 0.6–1.0 (growing monotonically with clustering), voxel scales 0.5–2 h⁻¹Mpc, and EdS versus flat ΛCDM (Ωm = 0.31); the PM truth is converged in time (0.008-voxel shift under step doubling); at the field level the model extends the usable k-range of ZA with a rescalable large-scale amplitude deficit. +4. **A theoretical identification (Section 4).** The exact variational principle of the ZA/adhesion flow is the Hopf–Lax formula — quadratic-cost optimal transport on a *flat* metric (Hopf 1950; Lax 1957; Brenier 1991; Frisch et al. 2002). Filaments are shocks of the transport map, and the measured transverse residual is the shock-formation term. +5. **Secondary results (Section 5):** an evaluation-methodology lesson (a proxy-matched comparison manufactured a p ≈ 10⁻¹⁴ false positive), the outcome of a filament-finder comparison including a descriptor that survives on real survey data, and an observational validation of the extracted web against Planck and ACT Compton-y maps. + +## 2. Methods + +### 2.1 Truth: PM N-body simulations and convergence + +Truth throughout is a particle-mesh (PM) N-body integrator: 128³ grid and 128³ particles in a periodic box of L = 128 h⁻¹Mpc (1 voxel = 1 h⁻¹Mpc), FFT Poisson solver, kick–drift–kick leapfrog with 90 steps from a = 0.1 to a = 1, Einstein–de Sitter (EdS) expansion in the primary runs and flat ΛCDM (Ωm = 0.31) in the transfer test. Initial conditions are Gaussian fields with a BBKS spectrum normalised to σ₈ = 0.8 (varied in Section 3.3), displaced by ZA to a = 0.1. Each approximation under test is evolved from the *same* initial field, so residuals are per-particle and exact. + +Time convergence of the truth (E10): doubling to 180 steps shifts the median final position by 0.008 voxels — three orders of magnitude below the effects measured — and leaves the model-versus-ZA comparison invariant (advantage −6.8% against both the 90-step and 180-step truths). Grid convergence is covered by the 256³ runs of Section 3.3. An external cross-check (E12) validates the integrator against a production code: evolved from the CAMELS CV_0 initial conditions (2LPT, Ωm = 0.3, 256³ particles, 25 h⁻¹Mpc box; Villaescusa-Navarro et al. 2021), our PM reproduces the official z = 0 particle positions to a median matched-ID offset of 0.41 h⁻¹Mpc against both Arepo and MP-Gadget truths (which agree with each other to 0.029 h⁻¹Mpc median — the code-consensus scale), with field cross-correlation r = 0.999, 0.984, 0.937 at k = 0.35, 0.66, 1.26 h Mpc⁻¹ — near-perfect agreement over the full range of scales used in this work, degrading only at k ≳ 2 h Mpc⁻¹ (r = 0.837 at k = 2.40) where PM force softening is expected to bite. + +### 2.2 The residual measurement (E4) + +For each tracked particle we compute R = x_true(a=1) − x_ZA(a=1) and its velocity analogue Rv, where x_ZA is the particle's own Zel'dovich prediction from the shared initial conditions. Residual directions are projected onto the local tidal eigenframe (e₁, e₂, e₃ with λ₁ ≥ λ₂ ≥ λ₃; filament axis along e₃), and statistics are binned by distance to the filament spine network extracted from the final density field. The direction statistic ⟨(R̂·e₃)²⟩ has isotropic null 1/3; values above it mean the correction points along filaments, below it across them. Sample: 6 PM simulations, 50,000 tracked particles each; quoted uncertainties are across-seed standard deviations. + +A precursor measurement (E2; 4 PM runs, 50,000 particles each, tracked over a ∈ [0.5, 1]) applied the same protocol to deviations of true trajectories from straight chords, independently of any model. + +### 2.3 The model: ZA rays with transverse damping at first crossing + +The frozen recipe (fixed by the declared optimisation campaign of E5c/E5d and used unchanged in every subsequent experiment): + +- Evolve particles on straight ZA rays in growth-factor steps ΔD = 0.05 (≈18 steps; no Poisson solves beyond cheap density deposits). +- At each step, deposit the model's own particles to a density grid. +- The first time a particle's local density exceeds ρ_c = 5, remove **β = 60%** of its velocity components perpendicular to the local filament axis e₃ — the minor eigenvector of the tidal tensor of the model's own density, smoothed at 2 h⁻¹Mpc and refreshed every third step. The along-axis component is never touched. + +The model has three fixed numbers (β, ρ_c, the smoothing scale). Two structural findings from the campaign: partial damping (β < 1) is required for self-estimated frames to pay — at β = 1 the model's own density feedback over-triggers crossings and cancels the frame gain — and pancake-ordered sequential damping (e₁ then e₂) underperforms simple both-perpendicular damping at this resolution (E5c). + +### 2.4 Baselines and the oracle bound + +Four baselines are run from identical initial conditions: plain ZA; an isotropic sticking model (the classical adhesion proxy: all velocity components damped at crossing); 2LPT; and MUSCLE (Neyrinck 2016). Additionally, an *oracle* variant of our model replaces the self-estimated tidal frames with frames from the true PM final field (E5b). The oracle is not a competitor but an information bound: it prices how much of the remaining error is frame estimation rather than physics. + +### 2.5 Evaluation protocol + +The primary score is the median per-particle distance between the model's a = 1 position and the PM-truth position, in voxels (= h⁻¹Mpc), overall and binned by distance to the web (spine network of the final field). Seed discipline: ρ_c was calibrated on seed 1 *by optimising the isotropic competitor* (a competitor-favouring choice); configuration selection used seeds {2, 3}; every reported validation number comes from held-out seeds ({4, 5, 6}, or the 5 held-out seeds of E5), with jackknife-over-seed errors. Field-level scores (Section 3.3) are the cross-correlation r(k) (phase fidelity) and transfer function T(k) (amplitude fidelity) against the truth density field. + +This protocol is strict because an earlier phase of the programme demonstrated how a plausible protocol can manufacture a discovery. In a filament-finder benchmark (Section 5.1), skeleton length — the quantity that mechanically buys completeness — was matched on a proxy (hysteresis-mask volume) rather than on the realised skeleton; the resulting +0.07 completeness "win" carried p ≈ 10⁻¹⁴ and was entirely an artifact of a 19% length mismatch. Every comparison in this paper therefore matches, calibrates, or holds out on the score-determining quantity itself: identical initial conditions, identical truth, frozen knobs, held-out seeds, and a competitor-favouring calibration where a choice had to be made. + +## 3. Results + +### 3.1 The correction beyond ZA is transverse in the tidal frame (E4) + +Table 1 gives the direction and amplitude of the residual R = x_true − x_ZA by distance to the web (6 simulations × 50,000 particles; mean ± across-seed sd; isotropic null 1/3). + +**Table 1 — the measured adjustment term (E4).** + +| d to spine (vox) | n/seed | ⟨(R̂·e₃)²⟩ | ⟨(R̂v·e₃)²⟩ | RMS R∥ (vox) | RMS R⊥ per axis (vox) | R∥/R⊥ | +|---|---|---|---|---|---|---| +| 0–2 | 20,658 | 0.298 ± 0.002 | 0.312 ± 0.003 | 4.36 | 4.92 | 0.89 | +| 2–4 | 7,269 | 0.297 ± 0.004 | 0.300 ± 0.004 | 3.77 | 4.19 | 0.90 | +| 4–8 | 6,632 | 0.264 ± 0.004 | 0.280 ± 0.003 | 2.40 | 3.07 | 0.78 | +| 8–16 | 10,320 | 0.218 ± 0.002 | 0.218 ± 0.003 | 1.86 | 2.91 | 0.64 | +| 16–64 | 5,120 | 0.208 ± 0.003 | 0.203 ± 0.003 | 1.77 | 2.83 | 0.63 | + +Three facts. (i) The correction is large where structure forms: RMS ≈ 4.4 voxels within 2 voxels of spines. (ii) It is organised by the tidal eigenframe — the statistics deviate strongly from isotropy. (iii) Its direction is *perpendicular* to the filament axis at every distance: ⟨(R̂·e₃)²⟩ = 0.21–0.30 < 1/3 and R∥/R⊥ = 0.63–0.89 everywhere, approaching (but never reaching) isotropy inside the tubes. Physically: ballistic ZA transport overshoots through forming walls and filaments, and the correction real gravity applies is transverse arrest at the web — not enhanced transport along it. + +The model-independent precursor (E2) agrees: deviations of true trajectories from straight chords are perpendicular to the filament axis in the bulk (⟨(d̂ev·e₃)²⟩ = 0.208–0.233 at 8–64 voxels, versus 1/3), turning weakly parallel (0.364 ± 0.010) only within ~4 voxels of spines, with velocity–axis alignment nearly flat (⟨|v̂·e₃|⟩ = 0.537–0.572, null 0.5; 4 simulations × 50,000 particles). Transverse pancake infall — the Zel'dovich collapse sequence — is what the instrument recovers, from scratch. + +### 3.2 One ingredient suffices: the damping model reaches the frame bound and ties MUSCLE (E5–E5d, E9) + +Table 2 tracks the model from first test to frozen recipe. All entries are median per-particle transport error in voxels against PM truth. + +**Table 2 — the model ladder.** + +| stage | configuration | held-out seeds | overall error | vs ZA | +|---|---|---|---|---| +| ZA baseline | — | 5 | 5.00 ± 0.22 | — | +| isotropic sticking (adhesion proxy) | full damping, all directions | 5 | 5.34 ± 0.15 | +7% | +| E5: first transverse model | β = 1, ZA-proxy frames | 5 | 4.91 ± 0.14 | −2% | +| E5c: refined | β = 0.75, self-density frames | 3 (seeds 4–6) | 4.64 ± 0.18 | −7% | +| E5d: **frozen recipe** | β = 0.6, 2 h⁻¹Mpc frames | 3 (seeds 4–6) | **4.52 ± 0.18** | **−9%** | +| E5b: oracle bound | true final-field frames | 5 | 4.47 ± 0.12 | −10% | + +with the ZA reference at 4.98 ± 0.27 on seeds 4–6. The largest gains sit exactly where the correction was measured: at 0–2 voxels from spines the first model already beats ZA by 7% and isotropic sticking by 14% (6.46 ± 0.07 vs 6.95 ± 0.24 vs 7.50 ± 0.10); the oracle shows −20% available at the web (5.53 ± 0.08). The frozen recipe lands 0.05 voxels above the oracle bound — 89% of the recoverable gap closed — and the oracle experiment (E5b) shows the model beats ZA in *every* distance bin once frames are exact, so the small 2–4-voxel deficit of the practical model (+4% vs ZA in E5) is frame-estimation error, not wrong physics. + +Table 3 places the frozen model against the literature baselines on identical initial conditions (E9; 3 seeds; jackknife errors). + +**Table 3 — literature baselines, identical ICs (E9).** + +| model | overall error (vox) | web (0–2 vox) error | +|---|---|---| +| ZA | 4.97 ± 0.27 | 6.92 ± 0.31 | +| 2LPT | 8.07 ± 0.39 | 8.21 ± 0.35 | +| MUSCLE | 4.49 ± 0.12 | 5.57 ± 0.07 | +| transverse damping (frozen) | 4.52 ± 0.18 | 5.77 ± 0.16 | + +2LPT *degrades* transport at these nonlinear scales — the known shell-crossing overshoot. MUSCLE statistically ties the frozen model overall and is slightly ahead at the web; at the field level (Section 3.3) MUSCLE wins small-scale phases while the damping model wins mid-scale amplitudes. This is the mechanism result: a model containing *only* the measured ingredient — transverse arrest at first crossing, in self-estimated tidal frames — reproduces MUSCLE-class transport accuracy. We conclude that shell-crossing corrections of the MUSCLE class work *because* they implement transverse arrest; the multiscale spherical-collapse machinery is one implementation of a correction whose measured form is directional damping across the web. + +### 3.3 Robustness: frozen knobs across clustering, resolution, cosmology; field-level fidelity (E6–E8, E10) + +Table 4 collects the transfer tests. All use the frozen recipe with no re-calibration; errors in the resolution rows are in physical h⁻¹Mpc. + +**Table 4 — transfer with frozen parameters.** + +| condition | seeds | ZA overall | model overall | Δ overall | ZA web | model web | Δ web | +|---|---|---|---|---|---|---|---| +| base (EdS, σ₈ = 0.8, 1 h⁻¹Mpc vox) | 3 | 4.98 | 4.52 | −9% | 6.93 | 5.78 | −17% | +| coarse (2 h⁻¹Mpc voxels) | 3 | 2.09 | 2.04 | −2% | 3.37 | 3.08 | −9% | +| σ₈ = 0.6 | 3 | 2.98 | 2.89 | −3% | 4.73 | 4.26 | −10% | +| σ₈ = 1.0 | 3 | 7.20 | 6.23 | −13% | 9.36 | 7.33 | −22% | +| flat ΛCDM, Ωm = 0.31 | 3 | 4.93 | 4.45 | −10% | 6.90 | 5.73 | −17% | +| 0.5 h⁻¹Mpc voxels (EdS) | 1 (+2 in E7b) | 4.95 | 4.45 | −10% | 6.76 | 5.72 | −15% | + +The advantage grows monotonically with clustering amplitude (−3% → −9% → −13% overall; −10% → −17% → −22% at the web over σ₈ = 0.6, 0.8, 1.0) — the behaviour expected of a physical shell-crossing correction, since higher σ₈ means more crossings, not of a tuned artifact. Under ΛCDM the numbers match EdS almost exactly, as expected for a growth-factor-parametrised geometric term. At 0.5 h⁻¹Mpc voxels the physical-unit errors are nearly identical to the 1 h⁻¹Mpc runs — the error scale is set by the physics, not the grid; the two additional seeds of E7b confirm the row at n = 3 (−12% and −9%). + +Field-level fidelity (E8; 3 seeds, 1 h⁻¹Mpc voxels): the model beats ZA on both cross-correlation and transfer amplitude in the nonlinear regime — r(k = 0.53 h Mpc⁻¹) = 0.388 vs 0.288, T(k = 0.34) = 0.410 vs 0.229 — extending the usable k-range of a ZA-based mock, while the classical isotropic sticking model destroys phase fidelity at all k > 0.1 h Mpc⁻¹ (r = 0.089 at k = 0.53). Costs, quantified: a ~7% amplitude deficit in the largest-scale bin (T = 0.849 vs ZA's 0.920 at k = 0.06 h Mpc⁻¹), removable by the standard linear transfer rescaling, and a small phase cost at intermediate k (r = 0.962 vs 0.975 at k = 0.14). Against the baselines (E9): MUSCLE leads r(k) at small scales (0.527 vs 0.388 at k = 0.53) while the damping model leads T(k) at mid scales (0.739 vs 0.669 at k = 0.14) — complementary strengths that suggest a hybrid. + +## 4. Theoretical interpretation: shocks of a flat optimal-transport map (T1) + +The measured transverse correction has an exact home in the variational structure of the models it corrects. Two short steps (full derivation in the programme's theory note, T1): + +**ZA is free motion in growth-factor time.** Writing the Zel'dovich map with the linear growth factor D as time, x(q, D) = q − D∇_q Φ₀(q), every trajectory is a straight line traversed at constant velocity: the potential exerts no force during the evolution — it is absorbed entirely into the initial velocities. The trajectories extremise the free action ∫|dx/dD|² dD, i.e. they are geodesics of the *flat* Euclidean metric. (This also refutes, constructively, any Jacobi-type "effective metric" picture in which paths are cheap along potential valleys: in the variables where the flow is simple there is no potential along the path left to conformalise.) + +**Adhesion is Hopf–Lax, i.e. quadratic-cost optimal transport.** The adhesion model regularises multi-streaming with Burgers dynamics ∂_D v + (v·∇)v = ν∆v, ν → 0⁺, v = ∇ψ. The Hopf–Cole transformation solves it exactly, and the vanishing-viscosity limit is the Hopf–Lax formula ψ(x, D) = min_q [ψ₀(q) + |x − q|²/2D] — a minimisation over straight-line transport paths with quadratic cost: precisely the Monge–Kantorovich optimal-transport structure (Hopf 1950; Lax 1957; Brenier 1991), used cosmologically in the MAK reconstruction (Frisch et al. 2002; Brenier et al. 2003). + +The geometry of cosmic-web transport is therefore flat metric, straight rays, plus a Legendre-type convexification of the initial potential. Filaments and nodes are the loci where the Hopf–Lax minimiser jumps between branches — the **shock set (caustics) of the transport map** — not geodesics of any curved or lifted metric. Structure lives in the singularities of the map, not in curved paths. + +This derivation predicts the measurements of Section 3.1 before they are made: matter arrives at a filament along straight inertial rays that terminate *on* the shock set — generically transverse to it (E2/E4: direction statistics below 1/3 everywhere) — and after absorption the surviving degree of freedom is the tangential pre-shock velocity, giving weak along-filament streaming inside the tube (E2: 0.364 ± 0.010 within ~4 voxels). The correction beyond ZA is, exactly, the shock-formation term: what happens to the transverse momentum at first crossing. Our model implements this term directly (β = 0.6 rather than the adhesion model's implicit full absorption), and Section 3.2 shows the single term carries the entire measurable correction budget at these scales. + +## 5. Secondary results + +### 5.1 The evaluation-methodology lesson: proxy matching and reference circularity + +The programme began as a filament-finder comparison and produced its most instructive result there. Comparing skeletons requires matched total spine length, because length mechanically buys completeness. The first benchmark enforced the match on a proxy — hysteresis-mask volume — assuming realised skeleton lengths would follow. They did not: the orientation-lift method's skeletons came out ~19% longer at sparse sampling (1,468 vs 1,239 voxels), and the resulting "+0.07 completeness, 48/50 seeds, p ≈ 10⁻¹⁴" advantage was pure length. Under the corrected extractor (iterative correction until the realised skeleton hits the target length) the sign flips: the Hessian baseline wins completeness at every sampling level from 2,500 galaxies up (Δ = −0.045 to −0.066 across both curvature variants, the lift winning at most 3/50 seeds, p ≤ 8×10⁻¹⁰ everywhere and ≈ 10⁻¹⁵ at most levels), with a statistical tie only at 1,200 (Δ = +0.003/+0.004, p = 0.47/0.36) — and pushing sparser still (400–600 galaxies) the Hessian wins again (Δ = −0.017/−0.021, p ≈ 0.003, 30 seeds). The artifact was detected only because a later instrument change retroactively shifted archived scores; the archived length fields confirmed the mismatch. The lesson generalises: *a matched comparison must enforce the match on the quantity that determines the score, and statistical strength (p ≈ 10⁻¹⁴) does not certify that the comparison was constructed correctly.* + +A second protocol failure was caught by design: scoring sparse-data skeletons against a clean-field *reference skeleton* is circular, because each method scores 0.07–0.19 higher against the reference extracted by its own family (E1, 2×2 reference matrix, 12 seeds; the two clean references agree only at completeness ≈ 0.60). The replacement criterion — mass coverage at matched length, which needs no reference — is what Sections 3 and 5.2 use. These two lessons motivated the strict protocol of Section 2.5. + +### 5.2 The filament-finder comparison: single-scale Hessian ≥ SE(3) orientation lift; the anisotropy descriptor survives + +Under honest matching, the simple single-scale Hessian ridge detector matches or beats the SE(3) orientation-lift detector essentially everywhere tested: exact-truth toys at every density (above), and gravity-shaped fields under the method-neutral mass criterion — on truncated-ZA boxes at all sampling levels (band coverage 0.2860 ± 0.0149 vs the lift's best 0.2691 ± 0.0139 at 5,000 galaxies, p ≈ 10⁻³, 12 seeds), on PM N-body fields (where the shortest-cigar lift closes to a statistical tie at sparse sampling: Δ = −0.0045, p = 0.55, 8 seeds), and at 256³ resolution (6 seeds). The lift's hypoelliptic-diffusion component is rejected outright: strong diffusion is harmful at every curvature and sparsity tested (−0.06 to −0.12, p ≈ 2×10⁻⁶), weak diffusion buys +0.01 (p = 0.03) only at ultra-sparse sampling, and an 8-step Trotter splitting at equal total diffusion reproduces the coarse result — the verdict is operator-level, not numerical (E0b, E0d). A hybrid summing the two percentile-normalised scores earns a niche: +0.05 purity and +0.040 junction F1 over the Hessian at 5,000 galaxies (p = 1.3×10⁻⁴, 20 seeds) at a −0.02 completeness cost (E0c). + +What survives for the lifted geometry is a *descriptor* role, consistent with Section 4's reading (the orientation manifold describes the shock set, not the transport): unweighted lift spines align with the clean-field tidal eigenvector e₃ at 0.734–0.759 versus the Hessian's 0.665–0.674 (isotropic null 0.5; E1, 12 seeds), and this replicates on the real Universe — 0.677 ± 0.016 vs 0.617 ± 0.016 across 18 independent BOSS CMASS tiles (E3). The lifted skeleton reads the anisotropy of the web better even where it loses on mass. + +### 5.3 Observational validation: the extracted web sits on real gas + +The web that the pipeline extracts from data is physically real. Stacking spine networks from 274,075 BOSS CMASS-North galaxies (z = 0.45–0.55, 18 tiles of 512 h⁻¹Mpc at 4 h⁻¹Mpc voxels) on the Planck Compton-y map yields an excess of 8.99σ (Hessian spines; 6.86σ lift) against 200 null sets matched to the spine points' galactic-latitude histogram — stricter controls *strengthened* the first-pass 4.8σ/4.1σ detection (E3, E3b; 41,535/40,426 spine points). Masking the tracer halos (7′) leaves 1.95σ/0.07σ: the spine-stack signal is dominantly the tracers' own halo gas, and an ACT DR6 test of the masked residual at arcminute resolution stayed below the pre-registered 3σ gate (1.77σ at 3′, 1.11σ at 7′; E3c). + +The between-halo component was instead measured with a pair-bridge estimator: y at the midpoint of close CMASS pairs (876,639 pairs, transverse separation 6–14 h⁻¹Mpc, line-of-sight < 6 h⁻¹Mpc) minus ring controls at ±60/±90/±120°, which cancels any circularly symmetric halo by construction (E3d-v2). Under the full validity battery (E3e) — jackknife errors over 50 RA patches (pair bootstrap is optimistic by 1.6–2.4×) and physically unconnected null pairs (same transverse window, line-of-sight 25–40 h⁻¹Mpc) — the on-axis excess is 3.30σ on ACT and 7.99σ on Planck, the null pairs expose a 4.7σ beam-leakage term on Planck (1.80×10⁻⁸) and none on ACT (1.65σ), and the null-subtracted bridge amplitude is 1.4×10⁻⁸ ± 0.9 (≈1.6σ, ACT) and 1.2×10⁻⁸ ± 0.5 (≈2.2σ, Planck) — mutually consistent and matching published LRG-pair bridge amplitudes (~1–2×10⁻⁸; de Graaff et al. 2019; Tanimura et al. 2019). We report this as a reproduction of the literature amplitude with honest errors and physical nulls at ~2σ per instrument, not as an independent detection. + +## 6. Limitations + +- **PM-only truth, externally checked at one configuration.** All transport truths come from our own particle-mesh integrator: time-converged (0.008-voxel shift under step doubling, E10), grid-checked at 256³, and validated against Arepo on the CAMELS CV_0 box (median matched-ID offset 0.41 h⁻¹Mpc; r ≥ 0.93 for k ≤ 1.3 h Mpc⁻¹, E12). Residual caveat: the external check covers one box size and cosmology; PM force softening still suppresses small-scale amplitudes (T ≈ 0.86–0.92 at the checked scales), a bias partially shared between truth and models. +- **EdS-primary.** The primary calibration and most experiments use Einstein–de Sitter expansion; ΛCDM is verified at one configuration (Ωm = 0.31, 3 seeds) with matching numbers, consistent with a growth-factor-parametrised term, but no broader cosmology scan was run. +- **One-shot damping.** The model damps at *first* shell-crossing only; multi-stream interiors are out of scope by design. The residual error at the web (5.5–6 voxels even at the oracle bound) is post-crossing physics that no first-crossing rule can represent; do not interpret model positions within ~2 voxels of density peaks as resolved halo structure. +- **BBKS spectrum.** Initial conditions use the BBKS transfer function rather than a Boltzmann-code spectrum; the mechanism measurement should be insensitive to this, but absolute numbers may shift. +- **Seed counts.** Held-out validation uses 3–5 seeds per condition (single seed for the first 0.5 h⁻¹Mpc row, extended to 3); errors are jackknife-over-seed. The consistency of the advantage across every condition mitigates, but does not replace, larger ensembles. +- **Calibration locality.** β and ρ_c were calibrated at this box size and tracer density; the transfer tests hold them frozen successfully across the scoped ranges (σ₈ 0.6–1.0, voxels 0.5–2 h⁻¹Mpc, EdS/ΛCDM), but new regimes warrant re-calibration against a small PM truth, optimising the competitor first. +- **Large-scale amplitude.** The field-level deficit at the largest scales (~7% in T(k) at k = 0.06 h Mpc⁻¹) requires the standard linear rescaling before use in mocks. + +## 7. Conclusions + +We measured the correction that ballistic (Zel'dovich) transport needs in order to match N-body gravity, resolved in the frame that organises gravitational collapse. The measurement is unambiguous: the correction is large near the web and points across the filament axis at every distance — transverse arrest, not enhanced along-filament transport. A model containing only that ingredient — straight ZA rays plus 60% transverse momentum damping at first shell-crossing in self-estimated tidal frames — closes 89% of the gap to its own frame-information bound, ties the best shell-crossing prescription in the literature (MUSCLE) on transport, beats the classical isotropic adhesion proxy everywhere, and transfers with frozen parameters across clustering amplitude, resolution, and cosmology, with the advantage growing where crossings multiply. The theory closes the loop: the exact variational structure of ZA/adhesion is flat quadratic-cost optimal transport, filaments are the shocks of that map, and the measured term is the shock-formation correction. + +The practical conclusion for the fast-mock literature is a mechanism, not an engine: MUSCLE-class shell-crossing corrections work because they implement transverse arrest. The two model families' complementary field-level strengths — MUSCLE's small-scale phases, transverse damping's mid-scale amplitudes — suggest a hybrid as the natural next engine, but not a naive one: triggering the directional damping on MUSCLE's precomputed Lagrangian collapse times underperforms both parents (4.67 ± 0.18 vs 4.49/4.52, weakest at the web; E11), so the evolving Eulerian density carries trigger information the Lagrangian collapse time lacks, and MUSCLE's own advantage must live in its displacement-divergence construction rather than its trigger. Hybridisation therefore has to happen at the displacement/field level; the natural next measurement is the second-crossing term that both model families currently omit. + +## Data availability + +All code, per-experiment reports with full tables and p-values, per-seed result files, and one-command reproduction scripts are in the repository under `research/cosmic-web/` (experiments E0–E12 and T1: `docs/`; runners: `scripts/`; per-seed numbers: `artifacts/`), including the model card with the frozen recipe and integration guide (`docs/MODEL-CARD.md`): https://github.com/moiseevigor/moiseevigor.github.io/tree/research/geometry-of-cosmic-web/research/cosmic-web + +## References + +- Aragón-Calvo, M. A., et al. (2007). "The multiscale morphology filter: identifying and extracting spatial patterns in the galaxy distribution." *A&A* 474, 315–338. +- Bond, J. R., Kofman, L., & Pogosyan, D. (1996). "How filaments of galaxies are woven into the cosmic web." *Nature* 380, 603–606. +- Bouchet, F. R., Colombi, S., Hivon, E., & Juszkiewicz, R. (1995). "Perturbative Lagrangian approach to gravitational instability." *A&A* 296, 575. +- Brenier, Y. (1991). "Polar factorization and monotone rearrangement of vector-valued functions." *Comm. Pure Appl. Math.* 44, 375–417. +- Brenier, Y., Frisch, U., et al. (2003). "Reconstruction of the early Universe as a convex optimization problem." *MNRAS* 346, 501–524. +- Cautun, M., van de Weygaert, R., & Jones, B. J. T. (2013). "NEXUS: tracing the cosmic web connection." *MNRAS* 429, 1286–1308. +- Codis, S., et al. (2012). "Connecting the cosmic web to the spin of dark haloes." *MNRAS* 427, 3320–3336. +- de Graaff, A., et al. (2019). "Probing the missing baryons with the Sunyaev–Zel'dovich effect from filaments." *A&A* 624, A48. +- Duits, R., & Franken, E. (2011). "Left-invariant diffusions on the space of positions and orientations and their application to crossing-preserving smoothing of HARDI images." *Int. J. Comput. Vis.* 92, 231–264. +- Duits, R., Boscain, U., Rossi, F., & Sachkov, Yu. (2014). "Association fields via cuspless sub-Riemannian geodesics in SE(2)." *J. Math. Imaging Vis.* 49, 384–417. +- Epps, S. D., & Hudson, M. J. (2017). "The weak-lensing masses of filaments between luminous red galaxies." *MNRAS* 468, 2605–2613. +- Frisch, U., Matarrese, S., Mohayaee, R., & Sobolevski, A. (2002). "A reconstruction of the initial conditions of the Universe by optimal mass transportation." *Nature* 417, 260–262. +- Gurbatov, S. N., Saichev, A. I., & Shandarin, S. F. (1989). "The large-scale structure of the universe in the frame of the model equation of non-linear diffusion." *MNRAS* 236, 385–402. +- Hahn, O., Porciani, C., Carollo, C. M., & Dekel, A. (2007). "Properties of dark matter haloes in clusters, filaments, sheets and voids." *MNRAS* 375, 489–499. +- Hopf, E. (1950). "The partial differential equation u_t + u u_x = μ u_xx." *Comm. Pure Appl. Math.* 3, 201–230. +- Kitaura, F.-S., & Heß, S. (2013). "Cosmological structure formation with augmented Lagrangian perturbation theory." *MNRAS* (Letters). +- Lax, P. D. (1957). "Hyperbolic systems of conservation laws II." *Comm. Pure Appl. Math.* 10, 537–566. +- Libeskind, N. I., et al. (2018). "Tracing the cosmic web." *MNRAS* 473, 1195–1217. +- Neyrinck, M. C. (2013). "Quantifying distortions of the Lagrangian dark-matter mesh in cosmology." *MNRAS*. +- Neyrinck, M. C. (2016). "Truthing the stretch: non-perturbative cosmological realizations with multiscale spherical collapse (MUSCLE)." *MNRAS*. +- Portegies, J., Sanguinetti, G., Meesters, S., & Duits, R. (2015). "New approximation of a scale space kernel on SE(3) and applications in neuroimaging." *SSVM 2015*, LNCS 9087. +- Scoccimarro, R. (1998). "Transients from initial conditions: a perturbative analysis." *MNRAS* 299, 1097–1118. +- Sousbie, T. (2011). "The persistent structure of the Universe — I. Theory and implementation." *MNRAS* 414, 350–383. +- Tanimura, H., et al. (2019). "A search for warm/hot gas filaments between pairs of SDSS luminous red galaxies." *MNRAS* 483, 223–234. +- Tassev, S., Zaldarriaga, M., & Eisenstein, D. J. (2013). "Solving large scale structure in ten easy steps with COLA." *JCAP*. +- Villaescusa-Navarro, F., et al. (2021). "The CAMELS project." *ApJ* 915, 71. +- Villaescusa-Navarro, F., et al. (2020). "The Quijote simulations." *ApJS* 250, 2. +- Zel'dovich, Ya. B. (1970). "Gravitational instability: an approximate theory for large density perturbations." *A&A* 5, 84–89. diff --git a/research/cosmic-web/docs/SYNTHESIS.md b/research/cosmic-web/docs/SYNTHESIS.md new file mode 100644 index 00000000000..ce6670475da --- /dev/null +++ b/research/cosmic-web/docs/SYNTHESIS.md @@ -0,0 +1,291 @@ +# Program synthesis — geometry of the cosmic web (E0–E2) + +One-line summary: **under correctly length-matched evaluation the simple +Hessian baseline matches or beats the orientation lift essentially +everywhere as a filament finder; the lift survives as an anisotropy +descriptor, a hybrid ingredient, and the program's physics probes produced +clean refutations of the geometric transport thesis.** + +> **Correction (2026-07-04).** The first E0 verdict ("lift wins sparse-regime +> completeness +0.07, p≈10⁻¹⁴") was an evaluation artifact. Length matching +> was enforced on a proxy (hysteresis-mask volume); realized skeleton lengths +> differed systematically (lift ~19% longer at sparse levels), and length +> buys completeness. The corrected extractor (iterative correction until the +> skeleton itself hits the target) flips the sign. Detected when an +> instrument change made for E1 retroactively shifted E0 parent scores; the +> archived n_est fields confirmed the mismatch. All numbers below are from +> the corrected reruns. + +All experiments: 128³ boxes (256³ resolution check), matched spine length, +calibrate/held-out discipline, paired Wilcoxon statistics. Chain of +hypotheses, in the order they were tested and what each result forced next: + +## H1 — the lift is a better filament finder + +- **Exact-truth toys (E0 corrected, 50 seeds × 2 curvature variants): + refuted.** The Hessian wins completeness at every level from n_gal=2500 up + (−0.045 to −0.066, 50/50 seeds, p ≈ 10⁻¹⁵) and junction F1 with it; at + ultra-sparse n_gal=1200 the methods are statistically tied (Δ ≈ +0.004, + p ≈ 0.4). The pre-correction "sparse-regime crossover" does not survive + honest length matching. +- **Gravity-shaped webs (E1, E1b): consistent with the corrected toys.** + Skeleton-vs-skeleton scoring first collapsed to reference circularity + (each method wins against its own clean reference — E1's 2×2 design + caught this). The method-neutral criterion (mass coverage at matched + length, E1b) gave the Hessian a significant win at *all* sampling + levels, in total mass AND in filament-band mass (2<ρ<20), on truncated-ZA + and PM N-body fields and at 256³ resolution; the short-cigar lift's best + case is a statistical tie at sparse sampling on N-body fields (p = 0.55). +- **Hybrid (E0c):** summing the two percentile-normalised scores ties the + Hessian's completeness at 2500 and beats it on purity (+0.05) and + junction F1 (+0.040, p = 1.3×10⁻⁴) at 5000, at a −0.02 completeness cost + — the two responses carry complementary information. + +## H2 — physics-informed (tidal) coefficients help + +**Dead.** Multiplicative tidal-alignment weighting hurts mass coverage even +with the *clean-field* tidal frame (E1b: b1_clean < b0), so this is not a +noise-in-the-estimate problem — the weighting itself is wrong. (E1's +sparse-frame version was already null-to-negative.) + +## H4 — transport follows sub-Riemannian geodesics of the lifted metric + +**Refuted in the bulk; a weak, sign-correct residual survives inside +filaments (E2, 4 PM N-body runs, 200k tracked particles).** Deviations of +true trajectories from ballistic chords are *perpendicular* to the filament +axis in the bulk (⟨(d̂ev·e3)²⟩ = 0.21–0.23 vs isotropic 1/3): transverse +pancake infall is what builds filaments, the opposite of "cheap along the +axis". Only within ~4 vox of spines does the deviation turn weakly parallel +(0.364 ± 0.010). Velocity–axis alignment is a flat ~0.56 (null 0.5). +Geodesic-shooting machinery (Tier B) is therefore unmotivated as a +*formation* model; at best it describes streaming *within* formed +filaments. + +## Sub-findings that survive + +1. **E1-M3:** unweighted lift spines align with the tidal eigenframe at + 0.73–0.76 vs Hessian 0.67 (null 0.5) — the lifted geometry is a better + *static descriptor* of web anisotropy even where it loses on mass. +2. **E0b (corrected rerun):** strong hypoelliptic diffusion (splitting + scheme) is harmful at every curvature × sparsity tested including the + most-curved tercile; weak diffusion is neutral-to-harmful except a small + positive (+0.01, p=0.03) at the ultra-sparse level. (Caveat: crude + numerical scheme, not the exact SE(3) kernel.) +3. **E0c hybrid:** the sum of normalised scores improves purity and + junction F1 over the Hessian at moderate sparsity — a practical recipe. +4. **E2's instrument recovered known Zel'dovich pancake physics** from + scratch — the perpendicular-infall signature — which validates the PM + + tracking pipeline. +5. **The length-matching lesson:** matched comparisons must enforce the + match on the score-buying quantity itself (realized skeleton length), + not a proxy (mask volume). The proxy manufactured a p≈10⁻¹⁴ phantom win. + +## Status of the program's claims + +- **C1 (methodological)** is refuted as a general claim: at honest matched + length the Hessian is at least as good everywhere tested, with the lift + tying only at ultra-sparse sampling. What survives of C1: the anisotropy + description (M3) and the hybrid's purity/junction gains. +- **C2 (physical)** is refuted as stated: the intrinsic-manifold geodesics + do not drive filament assembly; assembly is transverse collapse in the + tidal eigenframe, with the lifted geometry emerging as a *description* of + the end state (M3), not the mechanism. + +## What would continue the program (not executable locally) + +- **H3 / E3**: stacking real lensing/tSZ maps along spines from real survey + catalogues — requires downloading public datasets (SDSS catalogue, + Planck maps; several GB). Blocked on data access, not on method. +- Repeat E1b on full N-body (PM) fields instead of truncated ZA, and at + 256³ resolution where filament cross-sections are resolved. +- A proper SE(3) kernel (Portegies et al.) instead of the splitting scheme, + to close the E0b caveat. +- Junction-specialised hybrid (Hessian near direction-degenerate points, + lift along spines) — motivated by the consistent junction/spine split in + E0/E0b. + +## E0d addendum (post-correction follow-ups, 2026-07-04) + +1. **Splitting convergence:** 8-step Trotter splitting at equal total + diffusion reproduces the coarse-splitting result at both sparsity levels + (fine −0.017 vs coarse −0.020 at n_gal=2500; both ≈ +0.01 at 1200). The + diffusion verdict is operator-level, not numerical. Caveat closed. +2. **Sparsity limit:** no honest lift-win regime exists anywhere: at + n_gal=400–600 the Hessian wins again (−0.017/−0.021, p ≈ 0.003); the + 800–1200 tie is a narrow band, not a trend. The pooling advantage never + exceeds the positional-accuracy cost at any tested density. +3. **Hybrid on gravity:** hyb_sum halves the lift's band-coverage deficit + but still trails the Hessian (−0.009, p = 0.04 at 5k; −0.016 at 20k). + The hybrid's niche remains toy purity/junctions. + +With these, every locally executable hypothesis and caveat of the program +has a replicated verdict. Remaining: E3/H3 on real survey data (blocked on +data download approval). + +## E3 addendum (real data, 2026-07-04) + +BOSS CMASS-North (z 0.45–0.55, 274k galaxies in 18 tiles) × Planck: +**H3 supported** — spine networks carry a ~4–5σ Compton-y excess over +footprint-matched controls (lensing null, noise-dominated); the Hessian's +spines carry slightly more signal than the lift's at matched length +(4.8σ vs 4.1σ), consistent with the corrected H1/E1b. **M3-real:** the +lift's anisotropy-descriptor advantage replicates on the real Universe +(0.677 ± 0.016 vs 0.617 ± 0.016 over tiles, null 0.5). Caveats: control +placement does not preserve galactic latitude; tracer-halo gas not masked; +single shell, no systematics weights. With E3 executed, every hypothesis +of the program — H1–H4 — now has an empirical verdict. + +## E3b addendum (second pass, 2026-07-04) + +Latitude-matched nulls STRENGTHEN the y-detection (9.0σ Hessian / 6.9σ +lift) — H3a supported. Tracer-halo masking leaves 1.95σ / 0.07σ — the +signal is dominantly halo gas; a between-halos (WHIM) component is a hint, +not a result (H3b). The radial profile is extended to ≥60′ but cannot +discriminate filament gas from clustered halo gas at Planck resolution +(H3c). Program closed: pushing H3b further needs deeper tSZ data (e.g., +ACT/SPT), not more analysis of this set. + +## E3c addendum (ACT DR6, 2026-07-04) — program close + +The decisive WHIM test at ACT depth returns a null: masked residual 1.77σ +(3′) / 1.11σ (7′) — the halo-only model survives; between-halo gas along +CMASS spines is bounded, not detected. Diagnosis: shell-projection places +the signal at degree scales that ACT's map filters (unmasked stack 2.95σ +vs Planck's 9σ on the overlap); ACT-depth WHIM work needs a small-scale +analysis design. With the pre-registered gate failed, the program is now +closed end to end: every hypothesis (H1–H4, H3a–c) carries a replicated +verdict, and the two guaranteed outputs stand — the methods/benchmark +contribution and a measured 9σ web-gas detection with an honest +halo-vs-filament decomposition. + +## E3d addendum (2026-07-04) — terminal result: bridge gas detected + +E3c's null carried the diagnosis (wrong angular scales for ACT), and the +design it prescribed delivered: the galaxy-pair bridge stack (876k CMASS +pairs, disc-averaged sampling, six-angle ring controls that null any +circularly-symmetric halo by construction) detects inter-halo gas at +**5.24σ on ACT DR6** (2.28×10⁻⁸) with Planck concurring at 19σ +(2.97×10⁻⁸). Model A (halo-only baryons) is rejected; Model B (gas along +the bridges between halos) is established at both instruments. The +program therefore closes on a positive terminal measurement, reached by +following its own negative result's diagnosis. + +## E3e addendum (validity battery) — final calibration of the claim + +Jackknife errors and unconnected-pair nulls trim the E3d-v2 headline: +on-axis excess stands at 3.3σ (ACT, honest errors), but the null pairs +reveal Planck's 4.7σ fake bridge (beam leakage, as diagnosed) and the +null-subtracted bridge is ~1.2–1.4×10⁻⁸ at ≈2σ per instrument — +reproducing the published amplitude, short of an independent detection. +The program's recurring moral holds to the last experiment: every strong +claim, put under its own strictest control, shrinks to its honest core — +and the honest core here still says the bridges are real (two instruments, +right amplitude), just not yet ours to claim at 5σ. + +## T1 addendum (2026-07-04) — the theory box closes the table + +T1 executed (docs/T1-note.md): the Jacobi-type variational principle is +refuted constructively. In growth-factor time the Zel'dovich flow is free +motion (straight lines; the potential is absorbed into initial velocities, +leaving no force to conformalize), and the adhesion model's exact +variational principle is Hopf–Lax — quadratic-cost optimal transport on a +FLAT metric. Filaments are singularities (shocks/caustics) of the +transport map, not geodesics of a lifted metric; the derivation predicts +both E2 signatures (transverse arrival, weak in-tube streaming) and +explains why the lift works as a descriptor (M3) but not as a transport +model. With T1 closed, every item of the original program — T1, E0–E3, +H1–H4 — carries an executed verdict. The program is complete. + +## E4 addendum (H4', 2026-07-04) — the adjustment-term reframing, tested + +Reframed hypothesis (the program owner's original intent): the lifted +geometry is not the mechanism but an adjustment term on top of standard +models. E4 measured that term directly — the residual between true PM +transport and each particle's own Zel'dovich prediction. Result: the +adjustment is real and tidal-frame-anisotropic, but TRANSVERSE +(⟨(R̂·e₃)²⟩ = 0.21–0.30 < 1/3; R∥/R⊥ = 0.63–0.89 everywhere): the +correction current models need is anisotropic damping of motion across +walls/filaments (adhesion-like), not enhanced transport along them. +H4' refuted as stated; the constructive product is the measured form and +amplitude of the true adjustment term. Consistent with T1 and E2. + +## E5 addendum (H-T2, 2026-07-04) — the adjustment term becomes a model + +The E4-corrected form of the adjustment conjecture was built into an +effective transport model: ZA rays + transverse-only momentum damping in +the local tidal frame at shell-crossing. Against PM truth from identical +initial conditions (competitor-favouring calibration, 5 held-out seeds): +best model overall (4.91 vs ZA 5.00 vs isotropic stick 5.34 median voxel +error), with the clearest win exactly at the web (−7% vs ZA, −14% vs +stick at 0–2 vox) and a small deficit vs ZA in the 2–4 vox outskirts. +The program therefore ends constructively: from conjecture → refutation +of the strong form → measurement of the true correction → a working +anisotropic-adhesion term that improves the classical proxy. + +## E5c/E5d addendum (2026-07-05) — refinement converged at the frame bound + +The declared optimization campaign froze the model: ZA rays + 60% +transverse damping in self-density tidal frames (2 h⁻¹Mpc smoothing) at +first crossing (ρ_c = 5). Held-out: 4.52 vs ZA 4.98 (−9%), 0.05 vox from +the oracle frame-information bound (4.47) — 89% of the recoverable gap +closed. Structural findings: partial damping is required for self-density +frames to pay (full damping self-amplifies crossings); pancake-ordered +sequential damping underperforms at this resolution. Full specification, +capabilities, limitations and failure modes: docs/MODEL-CARD.md. The +refinement scope is complete; remaining error at the web is +post-crossing (multi-stream) physics, out of scope by design. + +## E6 addendum (2026-07-05) — transfer verified, trend physical + +The frozen model transfers without re-calibration to coarser voxels and +to weaker/stronger clustering, with the advantage growing monotonically +with sigma8 (−3% → −13% overall; up to −22% at the web at sigma8 = 1) — +the behaviour of a physical correction term. Model-card scope widened +accordingly. With this, every locally testable hypothesis — including the +card's own declared limitations — carries a verdict. + +## E7 addendum (2026-07-05) — scope table complete + +The last two unverified rows close: under flat ΛCDM (Ωm = 0.31) the frozen +model keeps −10%/−17% (overall/web), and at 0.5 Mpc/h voxels −10%/−15% +(one seed), with physical-unit errors nearly identical across all regimes +— the signature of a geometric, growth-parametrized correction. Every +scope row of the model card testable on this machine is now verified. +The program has no remaining executable hypothesis of any kind. + +## E9/E10 addendum (2026-07-05) — referee battery: the framing settles + +Literature baselines implemented and run on identical ICs: 2LPT degrades +transport at these scales (8.07; known shell-crossing overshoot); MUSCLE +ties the frozen model (4.49 vs 4.52) and wins small-scale phases, while +the damping model wins mid-scale amplitudes. The paper's contribution is +therefore the mechanism, not the engine: E4's measurement (the beyond-ZA +correction is transverse in the tidal frame) plus the demonstration that +this single directional ingredient reproduces MUSCLE-level accuracy — +explaining WHY shell-crossing corrections of the MUSCLE class work. E10: +truth converged in time (0.008 vox under step doubling); comparison +invariant. Referee points 1–3 are now executed; the optional external +cross-check (Quijote) awaits download approval. + +## E11/E12 addendum (2026-07-05) — hybrid refuted; truth externally validated + +E11: the naive hybrid (MUSCLE's Lagrangian collapse trigger + directional +damping) underperforms both parents (4.67 vs 4.49/4.52) — the evolving +Eulerian trigger carries real information; field-level combination is the +remaining (unpursued) hybrid route. E12: our PM truth, evolved from +CAMELS CV_0's own ICs, matches Arepo to 0.41 Mpc/h median per particle +(r ≥ 0.94 for k ≤ 1.3 h/Mpc) — the referee's external-validation item is +closed; at a 10× resolution extrapolation the frozen model still does not +harm (1.127 vs ZA 1.140). The three referee must-fix points (baselines, +truth validation, field-level metrics) are all executed; the paper draft +(docs/PAPER-DRAFT.md) carries the results. + +## E12-astrid addendum (2026-07-05) — two-code truth validation closed + +The second external code (MP-Gadget, Astrid_DM CV_0; shared ICs verified +by per-ID prediction at z=15 to 0.068 vox) reproduces the Arepo verdict +identically, and the codes agree with each other to 0.029 Mpc/h median +per particle. Hierarchy: code consensus 0.03 << our PM offset 0.41 << +measured effects 4-5 Mpc/h. The referee's truth item is closed at two +independent production codes. All three follow-up points (two-code +external check, paper audit, posts publication-ready) are complete. diff --git a/research/cosmic-web/docs/T1-note.md b/research/cosmic-web/docs/T1-note.md new file mode 100644 index 00000000000..2736b3acd47 --- /dev/null +++ b/research/cosmic-web/docs/T1-note.md @@ -0,0 +1,109 @@ +# T1 — the variational structure of the Zel'dovich/adhesion flow (refutation with derivation) + +**Question (from the scoping post):** derive — or refute — a Jacobi-type +variational principle for the Zel'dovich/adhesion flow in comoving +coordinates: an effective metric $g \propto (E-\Phi)\,g_{\mathrm{Euclid}}$ +(or a sub-Riemannian analogue on $\mathbb{R}^3 \times S^2$) whose geodesics +the flow follows, with "cheap along the filament axis" anisotropy. + +**Answer: refuted.** The flow has an exact variational principle, but it is +optimal transport with quadratic cost, not geodesic flow of a +potential-weighted metric. Three short steps. + +## 1. In growth-factor time, Zel'dovich dynamics is free motion + +Write the Zel'dovich map with the linear growth factor $D$ as the time +variable: + +$$ +\mathbf{x}(\mathbf{q}, D) = \mathbf{q} - D\,\nabla_q \Phi_0(\mathbf{q}), +\qquad +\frac{d\mathbf{x}}{dD} = -\nabla_q \Phi_0(\mathbf{q}) = \text{const along the trajectory}. +$$ + +Every trajectory is a **straight line traversed at constant velocity** in +$(\mathbf{x}, D)$. Such trajectories extremise the free action + +$$ +S[\mathbf{x}] = \int \left| \frac{d\mathbf{x}}{dD} \right|^2 dD , +$$ + +i.e. they are geodesics of the **flat Euclidean metric**. The gravitational +potential does not act as a force during the evolution at all — in $D$-time +it is entirely absorbed into the initial velocity field +$\mathbf{v}_0 = -\nabla\Phi_0$. This is the precise reason the Jacobi +construction cannot be repaired: the Jacobi metric +$2m(E-\Phi)g_{\mathrm{Euclid}}$ conformally reweights paths by the potential +*along* the path, but in the variables where the cosmic-web flow is simple +there is no potential along the path to reweight. Not only is energy not +conserved (the objection raised in the scoping post); there is no force +term left to build a conformal factor from. + +## 2. The adhesion model's variational principle is Hopf–Lax, i.e. optimal transport + +The adhesion model regularises multi-streaming with Burgers dynamics, +$\partial_D \mathbf{v} + (\mathbf{v}\cdot\nabla)\mathbf{v} = \nu \Delta \mathbf{v}$, +$\nu \to 0^+$, with $\mathbf{v} = \nabla \psi$. The Hopf–Cole +transformation solves it exactly, and the $\nu \to 0$ limit is the +**Hopf–Lax formula**: the velocity potential evolves as + +$$ +\psi(\mathbf{x}, D) = \min_{\mathbf{q}} \left[ \psi_0(\mathbf{q}) + +\frac{|\mathbf{x} - \mathbf{q}|^2}{2D} \right]. +$$ + +This *is* a variational principle — a minimisation over straight-line +transport paths with quadratic cost plus initial data — which is exactly the +Monge–Kantorovich optimal-transport structure with cost +$|\mathbf{x}-\mathbf{q}|^2$ (Brenier; used cosmologically in the MAK +reconstruction of Frisch et al.). The "geometry" that governs the cosmic +web is therefore: **flat metric, straight rays, plus a Legendre-type +convexification of the initial potential.** Filaments and nodes are the +places where the Hopf–Lax minimiser jumps between branches — the **shock +set (caustics) of an optimal-transport map** — not the geodesics of any +curved or lifted metric. Structure lives in the *singularities of the map*, +not in *curved paths*. + +## 3. Why this reproduces E2's measurements + +The derivation predicts the two dynamical signatures E2 measured: + +- Matter arrives at a filament along straight inertial rays that terminate + *on* the shock set — generically **transverse** to it. Hence the bulk + deviation-from-chord field points across, not along, filaments + (E2: $\langle(\hat{dev}\cdot e_3)^2\rangle = 0.21$–$0.23$ vs the + isotropic 1/3). +- After absorption into the shock, the surviving degree of freedom is the + tangential component of the pre-shock velocity: weak **along-filament + streaming inside the tube** (E2: 0.364 ± 0.010 within ~4 voxels of + spines). + +The orientation manifold $\mathbb{R}^3 \times S^2$ retains a legitimate +role, but as a *descriptor of the shock set* — the tangent structure of an +already-formed web (which is why the lift wins the alignment statistic M3 +on simulations and on the sky) — and not as the state space of transport. + +## Verdict + +- T1 as posed: **refuted, constructively.** No Jacobi-type + potential-weighted metric generates the flow; the correct variational + structure (Hopf–Lax / quadratic-cost optimal transport) was identified + instead, and it is flat. +- The refutation is consistent with, and explains, the empirical E2 + verdict obtained independently ten experiments earlier. +- Program table status: with T1 closed, every item in the original scoping + post — T1, E0–E3, H1–H4 — carries an executed verdict. + +## References + +- Ya. B. Zel'dovich (1970), *A&A* 5, 84 — the straight-line property is + implicit in the ballistic form of the map. +- S. N. Gurbatov, A. I. Saichev, S. F. Shandarin (1989), *MNRAS* 236, 385 — + adhesion model. +- E. Hopf (1950); P. D. Lax (1957) — the viscosity limit and the + variational formula. +- Y. Brenier (1991), *Comm. Pure Appl. Math.* 44, 375 — polar factorisation + and quadratic-cost optimal transport. +- U. Frisch, S. Matarrese, R. Mohayaee, A. Sobolevski (2002), *Nature* 417, + 260 — MAK reconstruction: optimal transport as the variational principle + of large-scale structure. diff --git a/research/cosmic-web/docs/figures/e0_curves_curved.png b/research/cosmic-web/docs/figures/e0_curves_curved.png new file mode 100644 index 00000000000..b7ace13dc00 Binary files /dev/null and b/research/cosmic-web/docs/figures/e0_curves_curved.png differ diff --git a/research/cosmic-web/docs/figures/e0_curves_straight.png b/research/cosmic-web/docs/figures/e0_curves_straight.png new file mode 100644 index 00000000000..c5e4f1000c1 Binary files /dev/null and b/research/cosmic-web/docs/figures/e0_curves_straight.png differ diff --git a/research/cosmic-web/docs/figures/e0_delta_curved.png b/research/cosmic-web/docs/figures/e0_delta_curved.png new file mode 100644 index 00000000000..9ab9a803e20 Binary files /dev/null and b/research/cosmic-web/docs/figures/e0_delta_curved.png differ diff --git a/research/cosmic-web/docs/figures/e0_delta_straight.png b/research/cosmic-web/docs/figures/e0_delta_straight.png new file mode 100644 index 00000000000..9b4e32bf71e Binary files /dev/null and b/research/cosmic-web/docs/figures/e0_delta_straight.png differ diff --git a/research/cosmic-web/docs/figures/e0_slice_curved.png b/research/cosmic-web/docs/figures/e0_slice_curved.png new file mode 100644 index 00000000000..50a22774d3b Binary files /dev/null and b/research/cosmic-web/docs/figures/e0_slice_curved.png differ diff --git a/research/cosmic-web/docs/figures/e0_slice_straight.png b/research/cosmic-web/docs/figures/e0_slice_straight.png new file mode 100644 index 00000000000..0aa72bf4919 Binary files /dev/null and b/research/cosmic-web/docs/figures/e0_slice_straight.png differ diff --git a/research/cosmic-web/docs/figures/e3b_profile.png b/research/cosmic-web/docs/figures/e3b_profile.png new file mode 100644 index 00000000000..6776ec40fa5 Binary files /dev/null and b/research/cosmic-web/docs/figures/e3b_profile.png differ diff --git a/research/cosmic-web/docs/figures/e3c_profile.png b/research/cosmic-web/docs/figures/e3c_profile.png new file mode 100644 index 00000000000..ec266da5bdc Binary files /dev/null and b/research/cosmic-web/docs/figures/e3c_profile.png differ diff --git a/research/cosmic-web/docs/paper-figures/fig1_residual_direction.pdf b/research/cosmic-web/docs/paper-figures/fig1_residual_direction.pdf new file mode 100644 index 00000000000..4105d014fda Binary files /dev/null and b/research/cosmic-web/docs/paper-figures/fig1_residual_direction.pdf differ diff --git a/research/cosmic-web/docs/paper-figures/fig1_residual_direction.png b/research/cosmic-web/docs/paper-figures/fig1_residual_direction.png new file mode 100644 index 00000000000..0c8cda349b7 Binary files /dev/null and b/research/cosmic-web/docs/paper-figures/fig1_residual_direction.png differ diff --git a/research/cosmic-web/docs/paper-figures/fig2_model_ladder.pdf b/research/cosmic-web/docs/paper-figures/fig2_model_ladder.pdf new file mode 100644 index 00000000000..bd675ee9005 Binary files /dev/null and b/research/cosmic-web/docs/paper-figures/fig2_model_ladder.pdf differ diff --git a/research/cosmic-web/docs/paper-figures/fig2_model_ladder.png b/research/cosmic-web/docs/paper-figures/fig2_model_ladder.png new file mode 100644 index 00000000000..442e0e16e24 Binary files /dev/null and b/research/cosmic-web/docs/paper-figures/fig2_model_ladder.png differ diff --git a/research/cosmic-web/docs/paper-figures/fig3_transfer.pdf b/research/cosmic-web/docs/paper-figures/fig3_transfer.pdf new file mode 100644 index 00000000000..fac3edb3d73 Binary files /dev/null and b/research/cosmic-web/docs/paper-figures/fig3_transfer.pdf differ diff --git a/research/cosmic-web/docs/paper-figures/fig3_transfer.png b/research/cosmic-web/docs/paper-figures/fig3_transfer.png new file mode 100644 index 00000000000..434e7137c02 Binary files /dev/null and b/research/cosmic-web/docs/paper-figures/fig3_transfer.png differ diff --git a/research/cosmic-web/docs/paper-figures/fig4_field_level.pdf b/research/cosmic-web/docs/paper-figures/fig4_field_level.pdf new file mode 100644 index 00000000000..9ee1f9142a8 Binary files /dev/null and b/research/cosmic-web/docs/paper-figures/fig4_field_level.pdf differ diff --git a/research/cosmic-web/docs/paper-figures/fig4_field_level.png b/research/cosmic-web/docs/paper-figures/fig4_field_level.png new file mode 100644 index 00000000000..99bca16c634 Binary files /dev/null and b/research/cosmic-web/docs/paper-figures/fig4_field_level.png differ diff --git a/research/cosmic-web/paper/README.md b/research/cosmic-web/paper/README.md new file mode 100644 index 00000000000..917a0437a0b --- /dev/null +++ b/research/cosmic-web/paper/README.md @@ -0,0 +1,2 @@ +Compile with `pdflatex main.tex` (run twice for cross-references and the table of citations); no BibTeX needed — the bibliography is inline. +Figures are loaded from `../docs/paper-figures/` (fig1–fig4 `.pdf`); copy them next to `main.tex` and drop the `../docs/paper-figures/` prefix before uploading to arXiv. diff --git a/research/cosmic-web/paper/main.tex b/research/cosmic-web/paper/main.tex new file mode 100644 index 00000000000..4d36b6c1177 --- /dev/null +++ b/research/cosmic-web/paper/main.tex @@ -0,0 +1,334 @@ +\documentclass[11pt]{article} + +\usepackage[margin=1in]{geometry} +\usepackage{amsmath} +\usepackage{amssymb} +\usepackage{graphicx} +\usepackage{booktabs} +\usepackage{hyperref} + +\title{The correction beyond Zel'dovich is transverse: a measured mechanism for shell-crossing models of cosmic-web transport} +\author{Igor Moiseev} +\date{\normalsize\itshape Draft --- 2026-07-05. Source of every number: the experiment reports in \normalfont\texttt{research/cosmic-web/docs/} \itshape (E0--E12, T1, MODEL-CARD).} + +\begin{document} + +\maketitle + +\begin{abstract} +Lagrangian approximations to cosmic structure formation --- the Zel'dovich approximation (ZA), 2LPT, adhesion, and shell-crossing prescriptions of the MUSCLE class --- are the transport engines of fast mock catalogues and reconstruction, yet \emph{why} shell-crossing corrections outperform perturbative ones has not been directly measured. Using particle-mesh (PM) $N$-body simulations evolved from the same initial conditions as the approximations they test, we measure the residual between true transport and each particle's own ZA prediction in the local tidal eigenframe. The residual is large near the web (RMS $\approx 4.4\,h^{-1}$Mpc within $2\,h^{-1}$Mpc of filament spines) and biased perpendicular to the filament axis at \emph{all} distances ($\langle(\hat{R}\cdot e_3)^2\rangle = 0.21$--$0.30$ versus the isotropic $1/3$; 6 simulations $\times$ 50,000 particles). A single-ingredient model built from exactly this measurement --- ZA rays plus partial ($\beta = 0.6$) transverse momentum damping at first shell-crossing --- reduces held-out median transport error from 4.98 to 4.52 $h^{-1}$Mpc, within 0.05 of the frame-information oracle bound (4.47), and statistically ties MUSCLE ($4.49 \pm 0.12$ vs $4.52 \pm 0.18$, 3 seeds) while 2LPT degrades to 8.07. With frozen parameters the advantage persists across $\sigma_8 \in [0.6, 1.0]$ (growing with clustering, $-3\% \rightarrow -13\%$), voxel scales 0.5--2 $h^{-1}$Mpc, and EdS versus flat $\Lambda$CDM. The exact variational structure of the ZA/adhesion flow --- Hopf--Lax quadratic-cost optimal transport on a flat metric --- identifies the measured term as the shock-formation correction: shell-crossing models work because they implement transverse arrest. +\end{abstract} + +\section{Introduction} +\label{sec:intro} + +Fast approximations to gravitational transport underpin much of large-scale-structure practice. The Zel'dovich approximation (ZA; \cite{zeldovich1970}) moves mass on straight ballistic trajectories set by the initial gravitational potential; second-order Lagrangian perturbation theory (2LPT; \cite{bouchet1995,scoccimarro1998}) adds the leading nonlinear correction; the adhesion model \cite{gurbatov1989} regularises the ZA's multi-streaming with an infinitesimal Burgers viscosity so that matter sticks at shell-crossing; spherical-collapse remappings and their multiscale extension MUSCLE \cite{neyrinck2013,neyrinck2016} replace the ZA divergence with a collapse-aware prescription; ALPT \cite{kitaura2013} splits scales between 2LPT and spherical collapse; COLA \cite{tassev2013} trades exactness for a few corrective time steps. A consistent empirical pattern runs through this literature: at nonlinear scales, prescriptions that \emph{arrest} particles at shell-crossing beat prescriptions that \emph{perturb} their trajectories --- MUSCLE-class models outperform 2LPT, which is known to overshoot in collapsed regions. What has been missing is a direct measurement of the correction these models are implicitly supplying: the residual between true gravitational transport and the ZA prediction, resolved in the frame that organises collapse. + +This paper supplies that measurement and traces its consequences. It is a mechanism paper, not a better-engine paper: the model we build is a probe of \emph{why} shell-crossing corrections work, and its headline result is that a single directional ingredient reproduces MUSCLE-level transport accuracy. + +Our contributions, in order: + +\begin{enumerate} +\item \textbf{A direct measurement of the beyond-ZA correction (Section~\ref{sec:residual}).} In PM $N$-body simulations evolved from the same initial conditions, the per-particle residual $R = x_{\mathrm{true}} - x_{\mathrm{ZA}}$ is large near the web and preferentially \emph{perpendicular} to the filament axis in the local tidal eigenframe at all distances: $\langle(\hat{R}\cdot e_3)^2\rangle = 0.21$--$0.30$ versus the isotropic $1/3$, with per-axis RMS ratio $R_\parallel/R_\perp = 0.63$--$0.89$ (6 simulations, 50,000 tracked particles each). +\item \textbf{A sufficiency test (Section~\ref{sec:sufficiency}).} A model containing only this ingredient --- ZA rays plus partial transverse momentum damping at first shell-crossing --- reaches held-out median transport error $4.52 \pm 0.18\,h^{-1}$Mpc versus ZA's $4.98 \pm 0.27$, within 0.05 of an oracle bound ($4.47 \pm 0.12$) obtained by giving the model the true final tidal frames. On identical initial conditions it statistically ties MUSCLE ($4.49 \pm 0.12$) while 2LPT degrades to $8.07 \pm 0.39$. This explains the MUSCLE-class advantage: transverse arrest is the correction budget. +\item \textbf{Robustness with frozen parameters (Section~\ref{sec:robustness}).} The advantage transfers without re-calibration across $\sigma_8 = 0.6$--$1.0$ (growing monotonically with clustering), voxel scales 0.5--2 $h^{-1}$Mpc, and EdS versus flat $\Lambda$CDM ($\Omega_m = 0.31$); the PM truth is converged in time (0.008-voxel shift under step doubling); at the field level the model extends the usable $k$-range of ZA with a rescalable large-scale amplitude deficit. +\item \textbf{A theoretical identification (Section~\ref{sec:theory}).} The exact variational principle of the ZA/adhesion flow is the Hopf--Lax formula --- quadratic-cost optimal transport on a \emph{flat} metric \cite{hopf1950,lax1957,brenier1991,frisch2002}. Filaments are shocks of the transport map, and the measured transverse residual is the shock-formation term. +\item \textbf{Secondary results (Section~\ref{sec:secondary}):} an evaluation-methodology lesson (a proxy-matched comparison manufactured a $p \approx 10^{-14}$ false positive), the outcome of a filament-finder comparison including a descriptor that survives on real survey data, and an observational validation of the extracted web against Planck and ACT Compton-$y$ maps. +\end{enumerate} + +\section{Methods} +\label{sec:methods} + +\subsection{Truth: PM \texorpdfstring{$N$}{N}-body simulations and convergence} +\label{sec:truth} + +Truth throughout is a particle-mesh (PM) $N$-body integrator: $128^3$ grid and $128^3$ particles in a periodic box of $L = 128\,h^{-1}$Mpc (1 voxel = 1 $h^{-1}$Mpc), FFT Poisson solver, kick--drift--kick leapfrog with 90 steps from $a = 0.1$ to $a = 1$, Einstein--de Sitter (EdS) expansion in the primary runs and flat $\Lambda$CDM ($\Omega_m = 0.31$) in the transfer test. Initial conditions are Gaussian fields with a BBKS spectrum normalised to $\sigma_8 = 0.8$ (varied in Section~\ref{sec:robustness}), displaced by ZA to $a = 0.1$. Each approximation under test is evolved from the \emph{same} initial field, so residuals are per-particle and exact. + +Time convergence of the truth (E10): doubling to 180 steps shifts the median final position by 0.008 voxels --- three orders of magnitude below the effects measured --- and leaves the model-versus-ZA comparison invariant (advantage $-6.8\%$ against both the 90-step and 180-step truths). Grid convergence is covered by the $256^3$ runs of Section~\ref{sec:robustness}. An external cross-check (E12) validates the integrator against a production code: evolved from the CAMELS CV\_0 initial conditions (2LPT, $\Omega_m = 0.3$, $256^3$ particles, 25 $h^{-1}$Mpc box; \cite{villaescusa2021}), our PM reproduces the official $z = 0$ particle positions to a median matched-ID offset of $0.41\,h^{-1}$Mpc against both Arepo and MP-Gadget truths (which agree with each other to $0.029\,h^{-1}$Mpc median --- the code-consensus scale), with field cross-correlation $r = 0.999$, 0.984, 0.937 at $k = 0.35$, 0.66, $1.26\,h\,\mathrm{Mpc}^{-1}$ --- near-perfect agreement over the full range of scales used in this work, degrading only at $k \gtrsim 2\,h\,\mathrm{Mpc}^{-1}$ ($r = 0.837$ at $k = 2.40$) where PM force softening is expected to bite. + +\subsection{The residual measurement (E4)} +\label{sec:residual-measurement} + +For each tracked particle we compute $R = x_{\mathrm{true}}(a{=}1) - x_{\mathrm{ZA}}(a{=}1)$ and its velocity analogue $R_v$, where $x_{\mathrm{ZA}}$ is the particle's own Zel'dovich prediction from the shared initial conditions. Residual directions are projected onto the local tidal eigenframe ($e_1$, $e_2$, $e_3$ with $\lambda_1 \geq \lambda_2 \geq \lambda_3$; filament axis along $e_3$), and statistics are binned by distance to the filament spine network extracted from the final density field. The direction statistic $\langle(\hat{R}\cdot e_3)^2\rangle$ has isotropic null $1/3$; values above it mean the correction points along filaments, below it across them. Sample: 6 PM simulations, 50,000 tracked particles each; quoted uncertainties are across-seed standard deviations. + +A precursor measurement (E2; 4 PM runs, 50,000 particles each, tracked over $a \in [0.5, 1]$) applied the same protocol to deviations of true trajectories from straight chords, independently of any model. + +\subsection{The model: ZA rays with transverse damping at first crossing} +\label{sec:model} + +The frozen recipe (fixed by the declared optimisation campaign of E5c/E5d and used unchanged in every subsequent experiment): + +\begin{itemize} +\item Evolve particles on straight ZA rays in growth-factor steps $\Delta D = 0.05$ ($\approx$18 steps; no Poisson solves beyond cheap density deposits). +\item At each step, deposit the model's own particles to a density grid. +\item The first time a particle's local density exceeds $\rho_c = 5$, remove {\boldmath$\beta = 60\%$} of its velocity components perpendicular to the local filament axis $e_3$ --- the minor eigenvector of the tidal tensor of the model's own density, smoothed at $2\,h^{-1}$Mpc and refreshed every third step. The along-axis component is never touched. +\end{itemize} + +The model has three fixed numbers ($\beta$, $\rho_c$, the smoothing scale). Two structural findings from the campaign: partial damping ($\beta < 1$) is required for self-estimated frames to pay --- at $\beta = 1$ the model's own density feedback over-triggers crossings and cancels the frame gain --- and pancake-ordered sequential damping ($e_1$ then $e_2$) underperforms simple both-perpendicular damping at this resolution (E5c). + +\subsection{Baselines and the oracle bound} +\label{sec:baselines} + +Four baselines are run from identical initial conditions: plain ZA; an isotropic sticking model (the classical adhesion proxy: all velocity components damped at crossing); 2LPT; and MUSCLE \cite{neyrinck2016}. Additionally, an \emph{oracle} variant of our model replaces the self-estimated tidal frames with frames from the true PM final field (E5b). The oracle is not a competitor but an information bound: it prices how much of the remaining error is frame estimation rather than physics. + +\subsection{Evaluation protocol} +\label{sec:protocol} + +The primary score is the median per-particle distance between the model's $a = 1$ position and the PM-truth position, in voxels (= $h^{-1}$Mpc), overall and binned by distance to the web (spine network of the final field). Seed discipline: $\rho_c$ was calibrated on seed 1 \emph{by optimising the isotropic competitor} (a competitor-favouring choice); configuration selection used seeds \{2, 3\}; every reported validation number comes from held-out seeds (\{4, 5, 6\}, or the 5 held-out seeds of E5), with jackknife-over-seed errors. Field-level scores (Section~\ref{sec:robustness}) are the cross-correlation $r(k)$ (phase fidelity) and transfer function $T(k)$ (amplitude fidelity) against the truth density field. + +This protocol is strict because an earlier phase of the programme demonstrated how a plausible protocol can manufacture a discovery. In a filament-finder benchmark (Section~\ref{sec:lesson}), skeleton length --- the quantity that mechanically buys completeness --- was matched on a proxy (hysteresis-mask volume) rather than on the realised skeleton; the resulting $+0.07$ completeness ``win'' carried $p \approx 10^{-14}$ and was entirely an artifact of a 19\% length mismatch. Every comparison in this paper therefore matches, calibrates, or holds out on the score-determining quantity itself: identical initial conditions, identical truth, frozen knobs, held-out seeds, and a competitor-favouring calibration where a choice had to be made. + +\section{Results} +\label{sec:results} + +\subsection{The correction beyond ZA is transverse in the tidal frame (E4)} +\label{sec:residual} + +Table~\ref{tab:residual} gives the direction and amplitude of the residual $R = x_{\mathrm{true}} - x_{\mathrm{ZA}}$ by distance to the web (6 simulations $\times$ 50,000 particles; mean $\pm$ across-seed sd; isotropic null $1/3$). + +\begin{table}[htbp] +\centering +\caption{The measured adjustment term (E4): direction and amplitude of the residual $R = x_{\mathrm{true}} - x_{\mathrm{ZA}}$ by distance to the filament spine (6 simulations $\times$ 50,000 particles; mean $\pm$ across-seed standard deviation; isotropic null $1/3$).} +\label{tab:residual} +\small +\begin{tabular}{lcccccc} +\toprule +$d$ to spine (vox) & $n$/seed & $\langle(\hat{R}\cdot e_3)^2\rangle$ & $\langle(\hat{R}_v\cdot e_3)^2\rangle$ & RMS $R_\parallel$ (vox) & RMS $R_\perp$ per axis (vox) & $R_\parallel/R_\perp$ \\ +\midrule +0--2 & 20,658 & $0.298 \pm 0.002$ & $0.312 \pm 0.003$ & 4.36 & 4.92 & 0.89 \\ +2--4 & 7,269 & $0.297 \pm 0.004$ & $0.300 \pm 0.004$ & 3.77 & 4.19 & 0.90 \\ +4--8 & 6,632 & $0.264 \pm 0.004$ & $0.280 \pm 0.003$ & 2.40 & 3.07 & 0.78 \\ +8--16 & 10,320 & $0.218 \pm 0.002$ & $0.218 \pm 0.003$ & 1.86 & 2.91 & 0.64 \\ +16--64 & 5,120 & $0.208 \pm 0.003$ & $0.203 \pm 0.003$ & 1.77 & 2.83 & 0.63 \\ +\bottomrule +\end{tabular} +\end{table} + +\begin{figure}[htbp] +\centering +\includegraphics[width=0.75\textwidth]{../docs/paper-figures/fig1_residual_direction.pdf} +\caption{Direction and amplitude of the beyond-ZA residual $R = x_{\mathrm{true}} - x_{\mathrm{ZA}}$ in the local tidal eigenframe, binned by distance to the filament spine (E4; 6 simulations $\times$ 50,000 particles). Lines (left axis): the direction statistic $\langle(\hat{R}\cdot e_3)^2\rangle$ for the position (circles) and velocity (squares) residuals; the dashed line marks the isotropic null $1/3$. Grey bars (right axis): RMS residual amplitude $|R|$ in $h^{-1}$Mpc. The correction is largest near the web (RMS $\approx 4.4\,h^{-1}$Mpc within $2\,h^{-1}$Mpc of spines) and points perpendicular to the filament axis at every distance ($\langle(\hat{R}\cdot e_3)^2\rangle = 0.21$--$0.30 < 1/3$).} +\label{fig:residual} +\end{figure} + +Three facts. (i) The correction is large where structure forms: RMS $\approx 4.4$ voxels within 2 voxels of spines. (ii) It is organised by the tidal eigenframe --- the statistics deviate strongly from isotropy. (iii) Its direction is \emph{perpendicular} to the filament axis at every distance: $\langle(\hat{R}\cdot e_3)^2\rangle = 0.21$--$0.30 < 1/3$ and $R_\parallel/R_\perp = 0.63$--$0.89$ everywhere, approaching (but never reaching) isotropy inside the tubes. Physically: ballistic ZA transport overshoots through forming walls and filaments, and the correction real gravity applies is transverse arrest at the web --- not enhanced transport along it. + +The model-independent precursor (E2) agrees: deviations of true trajectories from straight chords are perpendicular to the filament axis in the bulk ($\langle(\widehat{\mathrm{dev}}\cdot e_3)^2\rangle = 0.208$--$0.233$ at 8--64 voxels, versus $1/3$), turning weakly parallel ($0.364 \pm 0.010$) only within $\sim$4 voxels of spines, with velocity--axis alignment nearly flat ($\langle|\hat{v}\cdot e_3|\rangle = 0.537$--$0.572$, null 0.5; 4 simulations $\times$ 50,000 particles). Transverse pancake infall --- the Zel'dovich collapse sequence --- is what the instrument recovers, from scratch. + +\subsection{One ingredient suffices: the damping model reaches the frame bound and ties MUSCLE (E5--E5d, E9)} +\label{sec:sufficiency} + +Table~\ref{tab:ladder} tracks the model from first test to frozen recipe. All entries are median per-particle transport error in voxels against PM truth. + +\begin{table}[htbp] +\centering +\caption{The model ladder. All entries are median per-particle transport error in voxels against PM truth.} +\label{tab:ladder} +\small +\begin{tabular}{llccc} +\toprule +stage & configuration & held-out seeds & overall error & vs ZA \\ +\midrule +ZA baseline & --- & 5 & $5.00 \pm 0.22$ & --- \\ +isotropic sticking (adhesion proxy) & full damping, all directions & 5 & $5.34 \pm 0.15$ & $+7\%$ \\ +E5: first transverse model & $\beta = 1$, ZA-proxy frames & 5 & $4.91 \pm 0.14$ & $-2\%$ \\ +E5c: refined & $\beta = 0.75$, self-density frames & 3 (seeds 4--6) & $4.64 \pm 0.18$ & $-7\%$ \\ +E5d: \textbf{frozen recipe} & $\beta = 0.6$, $2\,h^{-1}$Mpc frames & 3 (seeds 4--6) & {\boldmath$4.52 \pm 0.18$} & {\boldmath$-9\%$} \\ +E5b: oracle bound & true final-field frames & 5 & $4.47 \pm 0.12$ & $-10\%$ \\ +\bottomrule +\end{tabular} +\end{table} + +\begin{figure}[htbp] +\centering +\includegraphics[width=0.85\textwidth]{../docs/paper-figures/fig2_model_ladder.pdf} +\caption{The model ladder on held-out seeds: median per-particle transport error against PM $N$-body truth, in $h^{-1}$Mpc. Adding the single measured ingredient --- transverse momentum damping at first shell-crossing --- carries ZA (4.98) through successive refinements to the frozen recipe ($\beta = 0.6$, $2\,h^{-1}$Mpc self-estimated frames; $4.52 \pm 0.18$), within 0.05 of the oracle bound obtained with true final-field frames (4.47, dashed line) and a statistical tie with MUSCLE \cite{neyrinck2016} ($4.49 \pm 0.12$). Isotropic sticking (5.34) and 2LPT (8.07) degrade transport relative to ZA.} +\label{fig:ladder} +\end{figure} + +with the ZA reference at $4.98 \pm 0.27$ on seeds 4--6. The largest gains sit exactly where the correction was measured: at 0--2 voxels from spines the first model already beats ZA by 7\% and isotropic sticking by 14\% ($6.46 \pm 0.07$ vs $6.95 \pm 0.24$ vs $7.50 \pm 0.10$); the oracle shows $-20\%$ available at the web ($5.53 \pm 0.08$). The frozen recipe lands 0.05 voxels above the oracle bound --- 89\% of the recoverable gap closed --- and the oracle experiment (E5b) shows the model beats ZA in \emph{every} distance bin once frames are exact, so the small 2--4-voxel deficit of the practical model ($+4\%$ vs ZA in E5) is frame-estimation error, not wrong physics. + +Table~\ref{tab:baselines} places the frozen model against the literature baselines on identical initial conditions (E9; 3 seeds; jackknife errors). + +\begin{table}[htbp] +\centering +\caption{Literature baselines, identical initial conditions (E9; 3 seeds; jackknife errors). Entries are median per-particle transport error in voxels against PM truth.} +\label{tab:baselines} +\begin{tabular}{lcc} +\toprule +model & overall error (vox) & web (0--2 vox) error \\ +\midrule +ZA & $4.97 \pm 0.27$ & $6.92 \pm 0.31$ \\ +2LPT & $8.07 \pm 0.39$ & $8.21 \pm 0.35$ \\ +MUSCLE & $4.49 \pm 0.12$ & $5.57 \pm 0.07$ \\ +transverse damping (frozen) & $4.52 \pm 0.18$ & $5.77 \pm 0.16$ \\ +\bottomrule +\end{tabular} +\end{table} + +2LPT \emph{degrades} transport at these nonlinear scales --- the known shell-crossing overshoot. MUSCLE statistically ties the frozen model overall and is slightly ahead at the web; at the field level (Section~\ref{sec:robustness}) MUSCLE wins small-scale phases while the damping model wins mid-scale amplitudes. This is the mechanism result: a model containing \emph{only} the measured ingredient --- transverse arrest at first crossing, in self-estimated tidal frames --- reproduces MUSCLE-class transport accuracy. We conclude that shell-crossing corrections of the MUSCLE class work \emph{because} they implement transverse arrest; the multiscale spherical-collapse machinery is one implementation of a correction whose measured form is directional damping across the web. + +\subsection{Robustness: frozen knobs across clustering, resolution, cosmology; field-level fidelity (E6--E8, E10)} +\label{sec:robustness} + +Table~\ref{tab:transfer} collects the transfer tests. All use the frozen recipe with no re-calibration; errors in the resolution rows are in physical $h^{-1}$Mpc. + +\begin{table}[htbp] +\centering +\caption{Transfer with frozen parameters. All conditions use the frozen recipe with no re-calibration; errors in the resolution rows are in physical $h^{-1}$Mpc.} +\label{tab:transfer} +\small +\begin{tabular}{lccccccc} +\toprule +condition & seeds & ZA overall & model overall & $\Delta$ overall & ZA web & model web & $\Delta$ web \\ +\midrule +base (EdS, $\sigma_8 = 0.8$, $1\,h^{-1}$Mpc vox) & 3 & 4.98 & 4.52 & $-9\%$ & 6.93 & 5.78 & $-17\%$ \\ +coarse ($2\,h^{-1}$Mpc voxels) & 3 & 2.09 & 2.04 & $-2\%$ & 3.37 & 3.08 & $-9\%$ \\ +$\sigma_8 = 0.6$ & 3 & 2.98 & 2.89 & $-3\%$ & 4.73 & 4.26 & $-10\%$ \\ +$\sigma_8 = 1.0$ & 3 & 7.20 & 6.23 & $-13\%$ & 9.36 & 7.33 & $-22\%$ \\ +flat $\Lambda$CDM, $\Omega_m = 0.31$ & 3 & 4.93 & 4.45 & $-10\%$ & 6.90 & 5.73 & $-17\%$ \\ +$0.5\,h^{-1}$Mpc voxels (EdS) & 1 (+2 in E7b) & 4.95 & 4.45 & $-10\%$ & 6.76 & 5.72 & $-15\%$ \\ +\bottomrule +\end{tabular} +\end{table} + +\begin{figure}[htbp] +\centering +\includegraphics[width=0.75\textwidth]{../docs/paper-figures/fig3_transfer.pdf} +\caption{Frozen-parameter transfer (no re-calibration): change in median transport error relative to ZA, in percent (negative is better), overall (blue) and within $2\,h^{-1}$Mpc of the web (orange). The advantage grows monotonically with clustering amplitude ($-3\% \rightarrow -9\% \rightarrow -13\%$ overall over $\sigma_8 = 0.6$, 0.8, 1.0), persists across voxel scales 0.5--2 $h^{-1}$Mpc, and is unchanged from EdS to flat $\Lambda$CDM ($\Omega_m = 0.31$).} +\label{fig:transfer} +\end{figure} + +The advantage grows monotonically with clustering amplitude ($-3\% \rightarrow -9\% \rightarrow -13\%$ overall; $-10\% \rightarrow -17\% \rightarrow -22\%$ at the web over $\sigma_8 = 0.6$, 0.8, 1.0) --- the behaviour expected of a physical shell-crossing correction, since higher $\sigma_8$ means more crossings, not of a tuned artifact. Under $\Lambda$CDM the numbers match EdS almost exactly, as expected for a growth-factor-parametrised geometric term. At $0.5\,h^{-1}$Mpc voxels the physical-unit errors are nearly identical to the $1\,h^{-1}$Mpc runs --- the error scale is set by the physics, not the grid; the two additional seeds of E7b confirm the row at $n = 3$ ($-12\%$ and $-9\%$). + +Field-level fidelity (E8; 3 seeds, $1\,h^{-1}$Mpc voxels): the model beats ZA on both cross-correlation and transfer amplitude in the nonlinear regime --- $r(k = 0.53\,h\,\mathrm{Mpc}^{-1}) = 0.388$ vs 0.288, $T(k = 0.34) = 0.410$ vs 0.229 --- extending the usable $k$-range of a ZA-based mock, while the classical isotropic sticking model destroys phase fidelity at all $k > 0.1\,h\,\mathrm{Mpc}^{-1}$ ($r = 0.089$ at $k = 0.53$). Costs, quantified: a $\sim$7\% amplitude deficit in the largest-scale bin ($T = 0.849$ vs ZA's 0.920 at $k = 0.06\,h\,\mathrm{Mpc}^{-1}$), removable by the standard linear transfer rescaling, and a small phase cost at intermediate $k$ ($r = 0.962$ vs 0.975 at $k = 0.14$). Against the baselines (E9): MUSCLE leads $r(k)$ at small scales (0.527 vs 0.388 at $k = 0.53$) while the damping model leads $T(k)$ at mid scales (0.739 vs 0.669 at $k = 0.14$) --- complementary strengths that suggest a hybrid. + +\begin{figure}[htbp] +\centering +\includegraphics[width=\textwidth]{../docs/paper-figures/fig4_field_level.pdf} +\caption{Field-level fidelity against PM $N$-body truth (E8/E9; 3 seeds, $1\,h^{-1}$Mpc voxels). Left: cross-correlation $r(k)$ (phase fidelity). Right: transfer function $T(k)$ (amplitude fidelity). The transverse-damping model extends the usable $k$-range of ZA ($r = 0.388$ vs 0.288 at $k = 0.53\,h\,\mathrm{Mpc}^{-1}$; $T = 0.410$ vs 0.229 at $k = 0.34$) at the cost of a $\sim$7\% amplitude deficit in the largest-scale bin ($T = 0.849$ vs 0.920 at $k = 0.06$), removable by the standard linear rescaling. MUSCLE leads small-scale phases (0.527 vs 0.388 at $k = 0.53$) while the damping model leads mid-scale amplitudes ($T = 0.739$ vs 0.669 at $k = 0.14$) --- complementary strengths.} +\label{fig:field} +\end{figure} + +\section{Theoretical interpretation: shocks of a flat optimal-transport map (T1)} +\label{sec:theory} + +The measured transverse correction has an exact home in the variational structure of the models it corrects. Two short steps (full derivation in the programme's theory note, T1): + +\textbf{ZA is free motion in growth-factor time.} Writing the Zel'dovich map with the linear growth factor $D$ as time, $x(q, D) = q - D\nabla_q \Phi_0(q)$, every trajectory is a straight line traversed at constant velocity: the potential exerts no force during the evolution --- it is absorbed entirely into the initial velocities. The trajectories extremise the free action $\int |dx/dD|^2\, dD$, i.e.\ they are geodesics of the \emph{flat} Euclidean metric. (This also refutes, constructively, any Jacobi-type ``effective metric'' picture in which paths are cheap along potential valleys: in the variables where the flow is simple there is no potential along the path left to conformalise.) + +\textbf{Adhesion is Hopf--Lax, i.e.\ quadratic-cost optimal transport.} The adhesion model regularises multi-streaming with Burgers dynamics $\partial_D v + (v\cdot\nabla)v = \nu\Delta v$, $\nu \to 0^+$, $v = \nabla\psi$. The Hopf--Cole transformation solves it exactly, and the vanishing-viscosity limit is the Hopf--Lax formula $\psi(x, D) = \min_q\, [\psi_0(q) + |x - q|^2/2D]$ --- a minimisation over straight-line transport paths with quadratic cost: precisely the Monge--Kantorovich optimal-transport structure \cite{hopf1950,lax1957,brenier1991}, used cosmologically in the MAK reconstruction \cite{frisch2002,brenier2003}. + +The geometry of cosmic-web transport is therefore flat metric, straight rays, plus a Legendre-type convexification of the initial potential. Filaments and nodes are the loci where the Hopf--Lax minimiser jumps between branches --- the \textbf{shock set (caustics) of the transport map} --- not geodesics of any curved or lifted metric. Structure lives in the singularities of the map, not in curved paths. + +This derivation predicts the measurements of Section~\ref{sec:residual} before they are made: matter arrives at a filament along straight inertial rays that terminate \emph{on} the shock set --- generically transverse to it (E2/E4: direction statistics below $1/3$ everywhere) --- and after absorption the surviving degree of freedom is the tangential pre-shock velocity, giving weak along-filament streaming inside the tube (E2: $0.364 \pm 0.010$ within $\sim$4 voxels). The correction beyond ZA is, exactly, the shock-formation term: what happens to the transverse momentum at first crossing. Our model implements this term directly ($\beta = 0.6$ rather than the adhesion model's implicit full absorption), and Section~\ref{sec:sufficiency} shows the single term carries the entire measurable correction budget at these scales. + +\section{Secondary results} +\label{sec:secondary} + +\subsection{The evaluation-methodology lesson: proxy matching and reference circularity} +\label{sec:lesson} + +The programme began as a filament-finder comparison and produced its most instructive result there. Comparing skeletons requires matched total spine length, because length mechanically buys completeness. The first benchmark enforced the match on a proxy --- hysteresis-mask volume --- assuming realised skeleton lengths would follow. They did not: the orientation-lift method's skeletons came out $\sim$19\% longer at sparse sampling (1,468 vs 1,239 voxels), and the resulting ``$+0.07$ completeness, 48/50 seeds, $p \approx 10^{-14}$'' advantage was pure length. Under the corrected extractor (iterative correction until the realised skeleton hits the target length) the sign flips: the Hessian baseline wins completeness at every sampling level from 2,500 galaxies up ($\Delta = -0.045$ to $-0.066$ across both curvature variants, the lift winning at most 3/50 seeds, $p \leq 8\times10^{-10}$ everywhere and $\approx 10^{-15}$ at most levels), with a statistical tie only at 1,200 ($\Delta = +0.003/+0.004$, $p = 0.47/0.36$) --- and pushing sparser still (400--600 galaxies) the Hessian wins again ($\Delta = -0.017/-0.021$, $p \approx 0.003$, 30 seeds). The artifact was detected only because a later instrument change retroactively shifted archived scores; the archived length fields confirmed the mismatch. The lesson generalises: \emph{a matched comparison must enforce the match on the quantity that determines the score, and statistical strength ($p \approx 10^{-14}$) does not certify that the comparison was constructed correctly.} + +A second protocol failure was caught by design: scoring sparse-data skeletons against a clean-field \emph{reference skeleton} is circular, because each method scores 0.07--0.19 higher against the reference extracted by its own family (E1, $2\times2$ reference matrix, 12 seeds; the two clean references agree only at completeness $\approx 0.60$). The replacement criterion --- mass coverage at matched length, which needs no reference --- is what Sections~\ref{sec:results} and~\ref{sec:finders} use. These two lessons motivated the strict protocol of Section~\ref{sec:protocol}. + +\subsection{The filament-finder comparison: single-scale Hessian \texorpdfstring{$\geq$}{>=} SE(3) orientation lift; the anisotropy descriptor survives} +\label{sec:finders} + +Under honest matching, the simple single-scale Hessian ridge detector matches or beats the SE(3) orientation-lift detector essentially everywhere tested: exact-truth toys at every density (above), and gravity-shaped fields under the method-neutral mass criterion --- on truncated-ZA boxes at all sampling levels (band coverage $0.2860 \pm 0.0149$ vs the lift's best $0.2691 \pm 0.0139$ at 5,000 galaxies, $p \approx 10^{-3}$, 12 seeds), on PM $N$-body fields (where the shortest-cigar lift closes to a statistical tie at sparse sampling: $\Delta = -0.0045$, $p = 0.55$, 8 seeds), and at $256^3$ resolution (6 seeds). The lift's hypoelliptic-diffusion component is rejected outright: strong diffusion is harmful at every curvature and sparsity tested ($-0.06$ to $-0.12$, $p \approx 2\times10^{-6}$), weak diffusion buys $+0.01$ ($p = 0.03$) only at ultra-sparse sampling, and an 8-step Trotter splitting at equal total diffusion reproduces the coarse result --- the verdict is operator-level, not numerical (E0b, E0d). A hybrid summing the two percentile-normalised scores earns a niche: $+0.05$ purity and $+0.040$ junction F1 over the Hessian at 5,000 galaxies ($p = 1.3\times10^{-4}$, 20 seeds) at a $-0.02$ completeness cost (E0c). + +What survives for the lifted geometry is a \emph{descriptor} role, consistent with Section~\ref{sec:theory}'s reading (the orientation manifold describes the shock set, not the transport): unweighted lift spines align with the clean-field tidal eigenvector $e_3$ at 0.734--0.759 versus the Hessian's 0.665--0.674 (isotropic null 0.5; E1, 12 seeds), and this replicates on the real Universe --- $0.677 \pm 0.016$ vs $0.617 \pm 0.016$ across 18 independent BOSS CMASS tiles (E3). The lifted skeleton reads the anisotropy of the web better even where it loses on mass. + +\subsection{Observational validation: the extracted web sits on real gas} +\label{sec:observational} + +The web that the pipeline extracts from data is physically real. Stacking spine networks from 274,075 BOSS CMASS-North galaxies ($z = 0.45$--$0.55$, 18 tiles of $512\,h^{-1}$Mpc at $4\,h^{-1}$Mpc voxels) on the Planck Compton-$y$ map yields an excess of $8.99\sigma$ (Hessian spines; $6.86\sigma$ lift) against 200 null sets matched to the spine points' galactic-latitude histogram --- stricter controls \emph{strengthened} the first-pass $4.8\sigma$/$4.1\sigma$ detection (E3, E3b; 41,535/40,426 spine points). Masking the tracer halos ($7'$) leaves $1.95\sigma$/$0.07\sigma$: the spine-stack signal is dominantly the tracers' own halo gas, and an ACT DR6 test of the masked residual at arcminute resolution stayed below the pre-registered $3\sigma$ gate ($1.77\sigma$ at $3'$, $1.11\sigma$ at $7'$; E3c). + +The between-halo component was instead measured with a pair-bridge estimator: $y$ at the midpoint of close CMASS pairs (876,639 pairs, transverse separation 6--14 $h^{-1}$Mpc, line-of-sight $< 6\,h^{-1}$Mpc) minus ring controls at $\pm 60^\circ/\pm 90^\circ/\pm 120^\circ$, which cancels any circularly symmetric halo by construction (E3d-v2). Under the full validity battery (E3e) --- jackknife errors over 50 RA patches (pair bootstrap is optimistic by 1.6--2.4$\times$) and physically unconnected null pairs (same transverse window, line-of-sight 25--40 $h^{-1}$Mpc) --- the on-axis excess is $3.30\sigma$ on ACT and $7.99\sigma$ on Planck, the null pairs expose a $4.7\sigma$ beam-leakage term on Planck ($1.80\times10^{-8}$) and none on ACT ($1.65\sigma$), and the null-subtracted bridge amplitude is $1.4\times10^{-8} \pm 0.9$ ($\approx 1.6\sigma$, ACT) and $1.2\times10^{-8} \pm 0.5$ ($\approx 2.2\sigma$, Planck) --- mutually consistent and matching published LRG-pair bridge amplitudes ($\sim$1--2$\times10^{-8}$; \cite{degraaff2019,tanimura2019}). We report this as a reproduction of the literature amplitude with honest errors and physical nulls at $\sim$2$\sigma$ per instrument, not as an independent detection. + +\section{Limitations} +\label{sec:limitations} + +\begin{itemize} +\item \textbf{PM-only truth, externally checked at one configuration.} All transport truths come from our own particle-mesh integrator: time-converged (0.008-voxel shift under step doubling, E10), grid-checked at $256^3$, and validated against Arepo on the CAMELS CV\_0 box (median matched-ID offset $0.41\,h^{-1}$Mpc; $r \geq 0.93$ for $k \leq 1.3\,h\,\mathrm{Mpc}^{-1}$, E12). Residual caveat: the external check covers one box size and cosmology; PM force softening still suppresses small-scale amplitudes ($T \approx 0.86$--$0.92$ at the checked scales), a bias partially shared between truth and models. +\item \textbf{EdS-primary.} The primary calibration and most experiments use Einstein--de Sitter expansion; $\Lambda$CDM is verified at one configuration ($\Omega_m = 0.31$, 3 seeds) with matching numbers, consistent with a growth-factor-parametrised term, but no broader cosmology scan was run. +\item \textbf{One-shot damping.} The model damps at \emph{first} shell-crossing only; multi-stream interiors are out of scope by design. The residual error at the web (5.5--6 voxels even at the oracle bound) is post-crossing physics that no first-crossing rule can represent; do not interpret model positions within $\sim$2 voxels of density peaks as resolved halo structure. +\item \textbf{BBKS spectrum.} Initial conditions use the BBKS transfer function rather than a Boltzmann-code spectrum; the mechanism measurement should be insensitive to this, but absolute numbers may shift. +\item \textbf{Seed counts.} Held-out validation uses 3--5 seeds per condition (single seed for the first $0.5\,h^{-1}$Mpc row, extended to 3); errors are jackknife-over-seed. The consistency of the advantage across every condition mitigates, but does not replace, larger ensembles. +\item \textbf{Calibration locality.} $\beta$ and $\rho_c$ were calibrated at this box size and tracer density; the transfer tests hold them frozen successfully across the scoped ranges ($\sigma_8$ 0.6--1.0, voxels 0.5--2 $h^{-1}$Mpc, EdS/$\Lambda$CDM), but new regimes warrant re-calibration against a small PM truth, optimising the competitor first. +\item \textbf{Large-scale amplitude.} The field-level deficit at the largest scales ($\sim$7\% in $T(k)$ at $k = 0.06\,h\,\mathrm{Mpc}^{-1}$) requires the standard linear rescaling before use in mocks. +\end{itemize} + +\section{Conclusions} +\label{sec:conclusions} + +We measured the correction that ballistic (Zel'dovich) transport needs in order to match $N$-body gravity, resolved in the frame that organises gravitational collapse. The measurement is unambiguous: the correction is large near the web and points across the filament axis at every distance --- transverse arrest, not enhanced along-filament transport. A model containing only that ingredient --- straight ZA rays plus 60\% transverse momentum damping at first shell-crossing in self-estimated tidal frames --- closes 89\% of the gap to its own frame-information bound, ties the best shell-crossing prescription in the literature (MUSCLE) on transport, beats the classical isotropic adhesion proxy everywhere, and transfers with frozen parameters across clustering amplitude, resolution, and cosmology, with the advantage growing where crossings multiply. The theory closes the loop: the exact variational structure of ZA/adhesion is flat quadratic-cost optimal transport, filaments are the shocks of that map, and the measured term is the shock-formation correction. + +The practical conclusion for the fast-mock literature is a mechanism, not an engine: MUSCLE-class shell-crossing corrections work because they implement transverse arrest. The two model families' complementary field-level strengths --- MUSCLE's small-scale phases, transverse damping's mid-scale amplitudes --- suggest a hybrid as the natural next engine, but not a naive one: triggering the directional damping on MUSCLE's precomputed Lagrangian collapse times underperforms both parents ($4.67 \pm 0.18$ vs 4.49/4.52, weakest at the web; E11), so the evolving Eulerian density carries trigger information the Lagrangian collapse time lacks, and MUSCLE's own advantage must live in its displacement-divergence construction rather than its trigger. Hybridisation therefore has to happen at the displacement/field level; the natural next measurement is the second-crossing term that both model families currently omit. + +\section*{Data availability} + +All code, per-experiment reports with full tables and $p$-values, per-seed result files, and one-command reproduction scripts are in the repository under \texttt{research/cosmic-web/} (experiments E0--E12 and T1: \texttt{docs/}; runners: \texttt{scripts/}; per-seed numbers: \texttt{artifacts/}), including the model card with the frozen recipe and integration guide (\texttt{docs/MODEL-CARD.md}): \url{https://github.com/moiseevigor/moiseevigor.github.io/tree/research/geometry-of-cosmic-web/research/cosmic-web} + +\begin{thebibliography}{99} + +\bibitem{aragon2007} Arag\'{o}n-Calvo, M. A., et al. (2007). ``The multiscale morphology filter: identifying and extracting spatial patterns in the galaxy distribution.'' \emph{A\&A} 474, 315--338. + +\bibitem{bond1996} Bond, J. R., Kofman, L., \& Pogosyan, D. (1996). ``How filaments of galaxies are woven into the cosmic web.'' \emph{Nature} 380, 603--606. + +\bibitem{bouchet1995} Bouchet, F. R., Colombi, S., Hivon, E., \& Juszkiewicz, R. (1995). ``Perturbative Lagrangian approach to gravitational instability.'' \emph{A\&A} 296, 575. + +\bibitem{brenier1991} Brenier, Y. (1991). ``Polar factorization and monotone rearrangement of vector-valued functions.'' \emph{Comm. Pure Appl. Math.} 44, 375--417. + +\bibitem{brenier2003} Brenier, Y., Frisch, U., et al. (2003). ``Reconstruction of the early Universe as a convex optimization problem.'' \emph{MNRAS} 346, 501--524. + +\bibitem{cautun2013} Cautun, M., van de Weygaert, R., \& Jones, B. J. T. (2013). ``NEXUS: tracing the cosmic web connection.'' \emph{MNRAS} 429, 1286--1308. + +\bibitem{codis2012} Codis, S., et al. (2012). ``Connecting the cosmic web to the spin of dark haloes.'' \emph{MNRAS} 427, 3320--3336. + +\bibitem{degraaff2019} de Graaff, A., et al. (2019). ``Probing the missing baryons with the Sunyaev--Zel'dovich effect from filaments.'' \emph{A\&A} 624, A48. + +\bibitem{duits2011} Duits, R., \& Franken, E. (2011). ``Left-invariant diffusions on the space of positions and orientations and their application to crossing-preserving smoothing of HARDI images.'' \emph{Int. J. Comput. Vis.} 92, 231--264. + +\bibitem{duits2014} Duits, R., Boscain, U., Rossi, F., \& Sachkov, Yu. (2014). ``Association fields via cuspless sub-Riemannian geodesics in SE(2).'' \emph{J. Math. Imaging Vis.} 49, 384--417. + +\bibitem{epps2017} Epps, S. D., \& Hudson, M. J. (2017). ``The weak-lensing masses of filaments between luminous red galaxies.'' \emph{MNRAS} 468, 2605--2613. + +\bibitem{frisch2002} Frisch, U., Matarrese, S., Mohayaee, R., \& Sobolevski, A. (2002). ``A reconstruction of the initial conditions of the Universe by optimal mass transportation.'' \emph{Nature} 417, 260--262. + +\bibitem{gurbatov1989} Gurbatov, S. N., Saichev, A. I., \& Shandarin, S. F. (1989). ``The large-scale structure of the universe in the frame of the model equation of non-linear diffusion.'' \emph{MNRAS} 236, 385--402. + +\bibitem{hahn2007} Hahn, O., Porciani, C., Carollo, C. M., \& Dekel, A. (2007). ``Properties of dark matter haloes in clusters, filaments, sheets and voids.'' \emph{MNRAS} 375, 489--499. + +\bibitem{hopf1950} Hopf, E. (1950). ``The partial differential equation $u_t + u u_x = \mu u_{xx}$.'' \emph{Comm. Pure Appl. Math.} 3, 201--230. + +\bibitem{kitaura2013} Kitaura, F.-S., \& He\ss{}, S. (2013). ``Cosmological structure formation with augmented Lagrangian perturbation theory.'' \emph{MNRAS} (Letters). + +\bibitem{lax1957} Lax, P. D. (1957). ``Hyperbolic systems of conservation laws II.'' \emph{Comm. Pure Appl. Math.} 10, 537--566. + +\bibitem{libeskind2018} Libeskind, N. I., et al. (2018). ``Tracing the cosmic web.'' \emph{MNRAS} 473, 1195--1217. + +\bibitem{neyrinck2013} Neyrinck, M. C. (2013). ``Quantifying distortions of the Lagrangian dark-matter mesh in cosmology.'' \emph{MNRAS}. + +\bibitem{neyrinck2016} Neyrinck, M. C. (2016). ``Truthing the stretch: non-perturbative cosmological realizations with multiscale spherical collapse (MUSCLE).'' \emph{MNRAS}. + +\bibitem{portegies2015} Portegies, J., Sanguinetti, G., Meesters, S., \& Duits, R. (2015). ``New approximation of a scale space kernel on SE(3) and applications in neuroimaging.'' \emph{SSVM 2015}, LNCS 9087. + +\bibitem{scoccimarro1998} Scoccimarro, R. (1998). ``Transients from initial conditions: a perturbative analysis.'' \emph{MNRAS} 299, 1097--1118. + +\bibitem{sousbie2011} Sousbie, T. (2011). ``The persistent structure of the Universe --- I. Theory and implementation.'' \emph{MNRAS} 414, 350--383. + +\bibitem{tanimura2019} Tanimura, H., et al. (2019). ``A search for warm/hot gas filaments between pairs of SDSS luminous red galaxies.'' \emph{MNRAS} 483, 223--234. + +\bibitem{tassev2013} Tassev, S., Zaldarriaga, M., \& Eisenstein, D. J. (2013). ``Solving large scale structure in ten easy steps with COLA.'' \emph{JCAP}. + +\bibitem{villaescusa2021} Villaescusa-Navarro, F., et al. (2021). ``The CAMELS project.'' \emph{ApJ} 915, 71. + +\bibitem{villaescusa2020} Villaescusa-Navarro, F., et al. (2020). ``The Quijote simulations.'' \emph{ApJS} 250, 2. + +\bibitem{zeldovich1970} Zel'dovich, Ya. B. (1970). ``Gravitational instability: an approximate theory for large density perturbations.'' \emph{A\&A} 5, 84--89. + +\end{thebibliography} + +\end{document} diff --git a/research/cosmic-web/requirements.txt b/research/cosmic-web/requirements.txt new file mode 100644 index 00000000000..5759fd9ed7d --- /dev/null +++ b/research/cosmic-web/requirements.txt @@ -0,0 +1,8 @@ +astropy==8.0.0 +h5py==3.16.0 +hdf5plugin==7.0.0 +healpy==1.19.0 +matplotlib==3.11.0 +numpy==2.5.0 +scikit-image==0.26.0 +scipy==1.18.0 diff --git a/research/cosmic-web/scripts/make_paper_figures.py b/research/cosmic-web/scripts/make_paper_figures.py new file mode 100644 index 00000000000..40e40e5736c --- /dev/null +++ b/research/cosmic-web/scripts/make_paper_figures.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Publication figures for PAPER-DRAFT.md -> docs/paper-figures/*.pdf(+png). + +Fig 1 E4: direction of the beyond-ZA residual vs distance to the web. +Fig 2 Model ladder: median transport error vs the oracle bound, + including literature baselines (2LPT, MUSCLE). +Fig 3 Frozen-knob transfer: advantage vs ZA across sigma8, resolution, + cosmology (overall and web bins). +Fig 4 Field-level fidelity: r(k) and T(k) for ZA / MUSCLE / model. + +All numbers read from artifacts/ where available; ladder/transfer values +are the frozen report numbers (sources in comments). +""" + +import json +from pathlib import Path + +import numpy as np +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "docs" / "paper-figures" +OUT.mkdir(exist_ok=True) + +plt.rcParams.update({"font.size": 9, "axes.titlesize": 9.5, + "figure.dpi": 150, "savefig.bbox": "tight"}) +BLUE, ORANGE, RED, GREY, GREEN = ("#2b6cb0", "#dd6b20", "#c53030", + "#666666", "#2f855a") + + +def save(fig, name): + fig.savefig(OUT / f"{name}.pdf") + fig.savefig(OUT / f"{name}.png", dpi=200) + plt.close(fig) + print("wrote", name) + + +# ---------------------------------------------------------------- fig 1: E4 +recs = json.loads((ROOT / "artifacts" / "e4_results.json").read_text()) +BINS = [(0, 2), (2, 4), (4, 8), (8, 16), (16, 64)] +labels = [f"{lo}–{hi}" for lo, hi in BINS] +pfR = [np.mean([r[f"pfR_{lo}_{hi}"] for r in recs]) for lo, hi in BINS] +pfV = [np.mean([r[f"pfV_{lo}_{hi}"] for r in recs]) for lo, hi in BINS] +rmsR = [np.mean([r[f"rmsR_{lo}_{hi}"] for r in recs]) for lo, hi in BINS] + +fig, ax = plt.subplots(figsize=(4.6, 3.2)) +ax2 = ax.twinx() +ax2.bar(labels, rmsR, color=GREY, alpha=0.25, width=0.55, zorder=1) +ax2.set_ylabel(r"RMS residual $|\mathbf{R}|$ ($h^{-1}$Mpc)", color=GREY) +ax2.tick_params(axis="y", colors=GREY) +ax.plot(labels, pfR, "o-", color=BLUE, label="position residual", zorder=3) +ax.plot(labels, pfV, "s-", color=ORANGE, label="velocity residual", zorder=3) +ax.axhline(1 / 3, ls="--", c="k", lw=1) +ax.text(4.1, 1 / 3 + 0.004, "isotropic (1/3)", ha="right", fontsize=8) +ax.set_xlabel(r"distance to filament spine ($h^{-1}$Mpc)") +ax.set_ylabel(r"$\langle(\hat{\mathbf{R}}\cdot e_3)^2\rangle$") +ax.set_ylim(0.15, 0.40) +ax.legend(frameon=False, loc="upper right", fontsize=8) +ax.set_zorder(2); ax.patch.set_visible(False) +ax.set_title("Direction of the beyond-Zel'dovich residual") +save(fig, "fig1_residual_direction") + +# ------------------------------------------------------------ fig 2: ladder +# Sources: E5/E5c/E5d/E9 reports; oracle bound E5b. +MODELS = [ + ("2LPT", 8.07, 0.39, GREY), + ("isotropic sticking", 5.34, 0.15, GREY), + ("Zel'dovich", 4.98, 0.27, GREY), + ("+ transverse damping (E5)", 4.91, 0.14, BLUE), + ("+ self frames, $\\beta$=0.75 (E5c)", 4.64, 0.18, BLUE), + ("frozen: $\\beta$=0.6, 2 Mpc frames (E5d)", 4.52, 0.14, BLUE), + ("MUSCLE (Neyrinck 2016)", 4.49, 0.12, GREEN), +] +BOUND = 4.47 +fig, ax = plt.subplots(figsize=(4.8, 3.2)) +ys = np.arange(len(MODELS))[::-1] +for y, (name, v, e, c) in zip(ys, MODELS): + ax.errorbar(v, y, xerr=e, fmt="o", color=c, capsize=3, ms=6) + ax.text(v + (0.14 if v < 6.5 else -0.14), y, f"{v:.2f}", + va="center", ha="left" if v < 6.5 else "right", fontsize=8) +ax.axvline(BOUND, ls="--", color=RED, lw=1.2) +ax.text(BOUND - 0.04, len(MODELS) - 0.6, "oracle bound 4.47", + rotation=90, va="top", ha="right", fontsize=8, color=RED) +ax.set_yticks(ys) +ax.set_yticklabels([m[0] for m in MODELS], fontsize=8.5) +ax.set_xlabel(r"median transport error vs $N$-body truth ($h^{-1}$Mpc)") +ax.set_xlim(4.2, 8.7) +ax.set_title("Model ladder (held-out seeds)") +save(fig, "fig2_model_ladder") + +# --------------------------------------------------------- fig 3: transfer +# Sources: E6-report, E7-report (frozen knobs). +CONDS = [(r"$\sigma_8$=0.6", -3, -10), (r"$\sigma_8$=0.8", -9, -17), + (r"$\sigma_8$=1.0", -13, -22), ("2 Mpc vox", -2, -9), + ("0.5 Mpc vox", -10, -15), (r"$\Lambda$CDM", -10, -17)] +fig, ax = plt.subplots(figsize=(4.8, 3.0)) +xs = np.arange(len(CONDS)) +w = 0.38 +ax.bar(xs - w / 2, [c[1] for c in CONDS], w, color=BLUE, label="overall") +ax.bar(xs + w / 2, [c[2] for c in CONDS], w, color=ORANGE, + label=r"web ($<2\,h^{-1}$Mpc)") +ax.axhline(0, color="k", lw=0.8) +ax.set_xticks(xs) +ax.set_xticklabels([c[0] for c in CONDS], fontsize=8) +ax.set_ylabel("error change vs Zel'dovich (%)") +ax.legend(frameon=False, fontsize=8) +ax.set_title("Frozen-parameter transfer (no re-calibration)") +save(fig, "fig3_transfer") + +# ------------------------------------------------------ fig 4: field level +e9 = json.loads((ROOT / "artifacts" / "e9_results.json").read_text()) +KBINS = np.geomspace(2 * np.pi / 128, np.pi / 2, 9) +kc = np.sqrt(KBINS[:-1] * KBINS[1:]) +fig, axes = plt.subplots(1, 2, figsize=(7.2, 3.0), sharex=True) +for stat, ax, ylab in [("r", axes[0], "cross-correlation $r(k)$"), + ("T", axes[1], "transfer $T(k)$")]: + for name, col, lab in [("za", GREY, "Zel'dovich"), + ("muscle", GREEN, "MUSCLE"), + ("damp", BLUE, "transverse damping")]: + vals = np.mean([r[f"{name}_{stat}"] for r in e9], axis=0) + ax.semilogx(kc, vals, "o-", ms=4, color=col, label=lab) + ax.set_xlabel(r"$k$ ($h\,$Mpc$^{-1}$)") + ax.set_ylabel(ylab) + if stat == "T": + ax.axhline(1, ls=":", c="k", lw=0.8) +axes[0].legend(frameon=False, fontsize=8) +fig.suptitle("Field-level fidelity vs $N$-body truth (3 seeds)", y=1.02) +save(fig, "fig4_field_level") diff --git a/research/cosmic-web/scripts/make_post_assets.py b/research/cosmic-web/scripts/make_post_assets.py new file mode 100644 index 00000000000..6c3dc8925fd --- /dev/null +++ b/research/cosmic-web/scripts/make_post_assets.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Assets for the introductory blog post. + +1. Slice-image series: seed-2 realization per variant x sampling level, + 3 panels (field+truth | lift | hessian) -> public/img/posts/cosmic-web/. +2. post_data.json: compact aggregates of E0/E0b/E1b/E2 results for the + post's inline interactive plots. +""" + +import json +import sys +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parents[1] +SITE = ROOT.parents[1] +IMG = SITE / "public" / "img" / "posts" / "cosmic-web" +IMG.mkdir(parents=True, exist_ok=True) +sys.path.insert(0, str(ROOT / "src")) + +import fields, lift, spines # noqa: E402 + +import matplotlib # noqa: E402 +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + +N, SEED = 128, 2 +LEVELS = [1200, 2500, 5000, 20000, 80000] +_AXES = lift.hemisphere_axes(42) + + +def make(seed, bend, n_gal): + rng = np.random.default_rng(seed) + segments, vertices = fields.voronoi_web(N, 32, rng) + polys = fields.curve_polylines(segments, bend, rng) + truth_pts, total = fields.sample_polylines(polys) + gal = fields.polyline_galaxies(polys, vertices, N, n_gal, 0.25, 0.8, rng) + field = fields.galaxy_field(gal, N, n_gal, rng) + n_target = int(total) + U = lift.orientation_score(field, _AXES, 6.0, 1.5) + sk_l = spines.extract_matched(spines.lifted_ridgeness(U), n_target) + sk_h = spines.extract_matched(spines.hessian_ridgeness(field, 1.5), n_target) + return field, truth_pts, sk_l, sk_h + + +def slice_fig(variant, bend, n_gal): + field, truth_pts, sk_l, sk_h = make(SEED, bend, n_gal) + z0, slab = N // 2, 6 + sl = slice(z0 - slab // 2, z0 + slab // 2) + tp = truth_pts[(truth_pts[:, 2] >= sl.start) & (truth_pts[:, 2] < sl.stop)] + fig, axs = plt.subplots(1, 3, figsize=(13.5, 4.8)) + for ax, (title, skel) in zip(axs, [ + (f"galaxy field (n={n_gal}) + truth", None), + ("SE(3) orientation lift", sk_l), ("Hessian baseline", sk_h)]): + ax.imshow(field[:, :, sl].mean(axis=2).T, origin="lower", + cmap="Greys", interpolation="nearest") + ax.plot(tp[:, 0], tp[:, 1], ".", ms=1.0, color="tab:red", + alpha=0.5 if skel is not None else 1.0) + if skel is not None: + p = spines.skeleton_points(skel) + p = p[(p[:, 2] >= sl.start) & (p[:, 2] < sl.stop)] + ax.plot(p[:, 0], p[:, 1], ".", ms=1.7, color="tab:blue") + ax.set_title(title, fontsize=11) + ax.set_xticks([]); ax.set_yticks([]) + fig.tight_layout(pad=0.6) + out = IMG / f"slice_{variant}_{n_gal}.png" + fig.savefig(out, dpi=115) + plt.close(fig) + print("wrote", out.name) + + +def bake_data(): + data = {} + for tag in ("straight", "curved"): + raw = json.loads((ROOT / "artifacts" / f"e0_results_{tag}.json").read_text()) + res = raw["results"] + levels = sorted({r["n_gal"] for r in res}) + agg = {} + for m in ("se3_lift", "hessian"): + agg[m] = { + "C": [round(float(np.mean([r["scores"][m]["M1"]["completeness"] + for r in res if r["n_gal"] == L])), 4) for L in levels], + "P": [round(float(np.mean([r["scores"][m]["M1"]["purity"] + for r in res if r["n_gal"] == L])), 4) for L in levels], + "jF1": [round(float(np.mean([r["scores"][m]["M2"]["f1"] + for r in res if r["n_gal"] == L])), 4) for L in levels], + } + deltas = {} + for key, sub in [("dC", "M1"), ("djF1", "M2")]: + kk = "completeness" if sub == "M1" else "f1" + deltas[key] = [[round( + r["scores"]["se3_lift"][sub][kk] - r["scores"]["hessian"][sub][kk], 4) + for r in res if r["n_gal"] == L] for L in levels] + data[tag] = {"levels": levels, "agg": agg, "deltas": deltas} + + e2 = json.loads((ROOT / "artifacts" / "e2_results.json").read_text()) + bins = ["0_2", "2_4", "4_8", "8_16", "16_64"] + data["e2"] = { + "bins": ["0–2", "2–4", "4–8", "8–16", "16–64"], + "P1": [round(float(np.mean([r[f"align_v_{b}"] for r in e2])), 3) for b in bins], + "P2": [round(float(np.mean([r[f"par_frac_{b}"] for r in e2])), 3) for b in bins], + "dev": [round(float(np.mean([r[f"dev_vox_{b}"] for r in e2])), 2) for b in bins], + } + + for tag in ("sigpar", "pm"): + f = ROOT / "artifacts" / f"e1b_{tag}_results.json" + if not f.exists(): + continue + recs = json.loads(f.read_text()) + methods = sorted({r["method"] for r in recs}) + levels = sorted({r["n_gal"] for r in recs}) + data[f"e1b_{tag}"] = {"levels": levels, "methods": methods, "band": { + m: [round(float(np.mean([r["band_cov"] for r in recs + if r["method"] == m and r["n_gal"] == L])), 4) for L in levels] + for m in methods}} + + out = SITE / "public" / "data" / "cosmic-web" / "post_data.js" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text("window.CW_DATA = " + json.dumps(data) + ";") + print("wrote", out.name, f"({out.stat().st_size/1024:.0f} KB)") + + +if __name__ == "__main__": + for variant, bend in [("straight", 0.0), ("curved", 0.15)]: + for n_gal in LEVELS: + slice_fig(variant, bend, n_gal) + bake_data() diff --git a/research/cosmic-web/scripts/make_three_universes.py b/research/cosmic-web/scripts/make_three_universes.py new file mode 100644 index 00000000000..39a8ce81ae2 --- /dev/null +++ b/research/cosmic-web/scripts/make_three_universes.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""'Three universes' figure for the reworked posts: the real sky (BOSS +wedge), a gravity simulation (PM), and the synthetic toy (Voronoi with +exact truth) — the modelling ladder from reality to controlled experiment. + +Output: public/img/posts/cosmic-web/three_universes.png (+ individual +panels for flexible layout). +""" + +import sys +from pathlib import Path + +import numpy as np +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from scipy.ndimage import gaussian_filter + +ROOT = Path(__file__).resolve().parents[1] +SITE = ROOT.parents[1] +OUT = SITE / "public" / "img" / "posts" / "cosmic-web" +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) + +import fields, pm # noqa: E402 +import run_e3 as e3 # noqa: E402 + +plt.rcParams.update({"font.size": 11}) + + +def real_panel(ax): + """BOSS CMASS wedge: a 4-degree declination slab, comoving coords.""" + e3.Z_RANGE = (0.43, 0.70) + ra, dec, z, xyz = e3.load_galaxies() + m = (dec > 25) & (dec < 28) + ra, xyz = ra[m], xyz[m] + chi = np.linalg.norm(xyz, axis=1) + sel = (ra > 130) & (ra < 230) + th = np.radians(ra[sel] - 180.0) # fan centred, opens upward + chi = chi[sel] + x = chi * np.sin(th) + y = chi * np.cos(th) + ax.scatter(x, y, s=1.3, c="#1a365d", alpha=0.45, lw=0) + ax.set_aspect("equal") + ax.set_title("The real sky", fontsize=13, pad=8) + ax.set_xlabel("BOSS survey: one thin slice of the sky, seen from Earth at the bottom tip —\n" + "every dot is a real galaxy, out to billions of light-years", + fontsize=9) + ax.set_xticks([]); ax.set_yticks([]) + for s in ax.spines.values(): + s.set_visible(False) + + +def sim_panel(ax): + """PM N-body slab (seed 4).""" + rng = np.random.default_rng(4) + x, *_ = pm.pm_sim(128, 128.0, rng, n_steps=90, track=1) + rho = fields.cic_deposit(x, 128) + sl = np.log10(1 + rho[:, :, 60:68].mean(axis=2)) + ax.imshow(sl.T, origin="lower", cmap="Blues", interpolation="bilinear") + ax.set_title("A gravity simulation", fontsize=13, pad=8) + ax.set_xlabel("2.1 million particles moved by gravity alone —\n" + "the same web pattern emerges by itself", + fontsize=9) + ax.set_xticks([]); ax.set_yticks([]) + for s in ax.spines.values(): + s.set_visible(False) + + +def toy_panel(ax): + """Voronoi toy with exact truth network overlaid.""" + rng = np.random.default_rng(2) + segments, vertices = fields.voronoi_web(128, 32, rng) + polys = fields.curve_polylines(segments, 0.15, rng) + truth, _ = fields.sample_polylines(polys) + gal = fields.polyline_galaxies(polys, vertices, 128, 5000, 0.25, 0.8, rng) + field = fields.galaxy_field(gal, 128, 5000, rng) + z0 = 64 + sl = field[:, :, z0 - 4:z0 + 4].mean(axis=2) + ax.imshow(sl.T, origin="lower", cmap="Blues", interpolation="bilinear") + tp = truth[(truth[:, 2] > z0 - 4) & (truth[:, 2] < z0 + 4)] + ax.plot(tp[:, 0], tp[:, 1], ".", ms=1.4, color="#c53030", alpha=0.8) + ax.set_title("A toy with the answers", fontsize=13, pad=8) + ax.set_xlabel("An invented web where the true filaments (red)\n" + "are known exactly — so methods can be graded", + fontsize=9) + ax.set_xticks([]); ax.set_yticks([]) + for s in ax.spines.values(): + s.set_visible(False) + + +def main(): + fig = plt.figure(figsize=(11.5, 9.0)) + gs = fig.add_gridspec(2, 2, height_ratios=[1.0, 1.35], + hspace=0.30, wspace=0.10) + real_panel(fig.add_subplot(gs[0, :])) + sim_panel(fig.add_subplot(gs[1, 0])) + toy_panel(fig.add_subplot(gs[1, 1])) + fig.suptitle("One question, three levels of reality — " + "each level a controlled model of the one above it", + fontsize=13.5, y=0.985) + fig.savefig(OUT / "three_universes.png", dpi=150, bbox_inches="tight", pad_inches=0.25) + print("wrote three_universes.png") + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e0.py b/research/cosmic-web/scripts/run_e0.py new file mode 100644 index 00000000000..6b5612ca983 --- /dev/null +++ b/research/cosmic-web/scripts/run_e0.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""E0a: accuracy benchmark on the Voronoi filament model (exact ground truth). + +Protocol +-------- +1. Calibration (seed 1 only): each method gets the SAME budget of 3 + parameter configs; the config with the best mean (C + P + jF1) across + sampling levels wins. +2. Evaluation (seeds 2-4, never used for calibration): both methods run + with their frozen configs through identical post-processing at matched + skeleton length; scored against exact truth (M1 spine metrics, M2 + junction recovery). + +Usage: run_e0.py [--smoke] +Results -> artifacts/e0_results.json, figures -> docs/figures/. +""" + +import json +import sys +import time +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +import fields, lift, spines, metrics # noqa: E402 + +SMOKE = "--smoke" in sys.argv +BEND = 0.15 if "--curved" in sys.argv else 0.0 +TAG = "curved" if BEND else "straight" +N_EVAL = (int(sys.argv[sys.argv.index("--eval-seeds") + 1]) + if "--eval-seeds" in sys.argv else 5) +N_WORKERS = 8 + +CFG = dict( + N=64 if SMOKE else 128, + n_seeds=16 if SMOKE else 32, + n_axes=42, + sigma_perp_gal=0.8, # transverse galaxy scatter (voxels) + bg_frac=0.25, + bend_frac=BEND, # 0 = straight Voronoi edges, >0 = Bezier bend + n_gal_levels=[8000] if SMOKE else [1200, 2500, 5000, 20000, 80000], + cal_seed=1, + eval_seeds=[2] if SMOKE else list(range(2, 2 + N_EVAL)), + r0_spine=2.0, + r0_junction=3.0, +) + +# same-size config grids: one knob family per method +LIFT_GRID = [ + dict(sig_par=6.0, sig_perp=1.5, diff_iter=2), + dict(sig_par=9.0, sig_perp=1.5, diff_iter=2), + dict(sig_par=6.0, sig_perp=1.5, diff_iter=0), # ablation: no diffusion +] +HESS_GRID = [dict(sigma=1.5), dict(sigma=2.0), dict(sigma=3.0)] + +_AXES = lift.hemisphere_axes(CFG["n_axes"]) +_W = lift.axis_adjacency(_AXES) + + +def make_realization(seed, n_gal): + rng = np.random.default_rng(seed) + segments, vertices = fields.voronoi_web(CFG["N"], CFG["n_seeds"], rng) + polys = fields.curve_polylines(segments, CFG["bend_frac"], rng) + truth_pts, total_len = fields.sample_polylines(polys) + gal = fields.polyline_galaxies(polys, vertices, CFG["N"], n_gal, + CFG["bg_frac"], CFG["sigma_perp_gal"], rng) + field = fields.galaxy_field(gal, CFG["N"], n_gal, rng) + return field, truth_pts, vertices, int(total_len) + + +def ridgeness(field, method, p): + if method == "hessian": + return spines.hessian_ridgeness(field, p["sigma"]) + U = lift.orientation_score(field, _AXES, p["sig_par"], p["sig_perp"]) + if p["diff_iter"]: + U = lift.hypoelliptic_diffuse(U, _AXES, _W, n_iter=p["diff_iter"]) + return spines.lifted_ridgeness(U) + + +def evaluate(field, method, p, truth_pts, truth_j, n_target): + t0 = time.time() + R = ridgeness(field, method, p) + skel = spines.extract_matched(R, n_target) + m1 = metrics.spine_metrics(truth_pts, spines.skeleton_points(skel), + CFG["r0_spine"]) + m2 = metrics.junction_metrics(truth_j, spines.junction_points(skel), + CFG["r0_junction"]) + return {"M1": m1, "M2": m2, "runtime_s": round(time.time() - t0, 1), + "skel": skel} + + +def combined(sc): + return sc["M1"]["completeness"] + sc["M1"]["purity"] + sc["M2"]["f1"] + + +def calibrate(): + """Pick each method's config on the calibration seed only.""" + data = {n: make_realization(CFG["cal_seed"], n) for n in CFG["n_gal_levels"]} + chosen = {} + for method, grid in [("se3_lift", LIFT_GRID), ("hessian", HESS_GRID)]: + scores = [] + for p in grid: + s = np.mean([combined(evaluate(f, method, p, tp, tj, nt)) + for f, tp, tj, nt in data.values()]) + scores.append(s) + print(f" cal {method} {p}: {s:.3f}") + chosen[method] = grid[int(np.argmax(scores))] + print(f" -> {method}: {chosen[method]}") + return chosen + + +def _init_worker(cfg): + CFG.clear() + CFG.update(cfg) + + +def eval_one(args): + seed, n_gal, chosen = args + field, truth_pts, truth_j, n_target = make_realization(seed, n_gal) + sc = {m: evaluate(field, m, chosen[m], truth_pts, truth_j, n_target) + for m in ("se3_lift", "hessian")} + for v in sc.values(): + v.pop("skel") + return {"seed": seed, "n_gal": n_gal, "scores": sc} + + +def main(): + from multiprocessing import Pool + + print("calibration (seed 1)") + chosen = calibrate() + + # one serial realization for the slice figure + seed0, ngal0 = CFG["eval_seeds"][0], CFG["n_gal_levels"][min(1, len(CFG["n_gal_levels"]) - 1)] + field, truth_pts, truth_j, n_target = make_realization(seed0, ngal0) + sc0 = {m: evaluate(field, m, chosen[m], truth_pts, truth_j, n_target) + for m in ("se3_lift", "hessian")} + save_slice_figure(field, truth_pts, sc0, seed0, ngal0) + + jobs = [(s, n, chosen) for s in CFG["eval_seeds"] + for n in CFG["n_gal_levels"]] + t0 = time.time() + with Pool(N_WORKERS, initializer=_init_worker, initargs=(dict(CFG),)) as pool: + results = pool.map(eval_one, jobs) + print(f"{len(jobs)} realizations on {N_WORKERS} workers " + f"in {time.time()-t0:.0f}s") + + out = ROOT / "artifacts" / f"e0_results_{TAG}.json" + out.write_text(json.dumps({ + "config": CFG, "chosen_params": chosen, "results": results}, indent=1)) + print(f"wrote {out}") + if not SMOKE: + save_curves_figure(results) + + +def save_slice_figure(field, truth_pts, sc, seed, n_gal, slab=6): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + N = CFG["N"] + z0 = N // 2 + sl = slice(z0 - slab // 2, z0 + slab // 2) + fig, axs = plt.subplots(1, 3, figsize=(15, 5.2)) + tp = truth_pts[(truth_pts[:, 2] >= sl.start) & (truth_pts[:, 2] < sl.stop)] + panels = [("galaxy field + truth", None), + ("SE(3) lift skeleton", sc["se3_lift"]["skel"]), + ("Hessian skeleton", sc["hessian"]["skel"])] + for ax, (title, skel) in zip(axs, panels): + ax.imshow(field[:, :, sl].mean(axis=2).T, origin="lower", + cmap="Greys", interpolation="nearest") + ax.plot(tp[:, 0], tp[:, 1], ".", ms=1.0, color="tab:red", + alpha=0.5 if skel is not None else 1.0) + if skel is not None: + p = spines.skeleton_points(skel) + p = p[(p[:, 2] >= sl.start) & (p[:, 2] < sl.stop)] + ax.plot(p[:, 0], p[:, 1], ".", ms=1.6, color="tab:blue") + ax.set_title(title) + ax.set_xlabel("x (voxels = $h^{-1}$Mpc)") + ax.set_ylabel("y (voxels)") + fig.suptitle(f"E0a-{TAG} slab (thickness {slab} vox), eval seed={seed}, " + f"n_gal={n_gal}; red = truth, blue = estimate") + fig.tight_layout() + fig.savefig(ROOT / "docs" / "figures" / f"e0_slice_{TAG}.png", dpi=130) + plt.close(fig) + + +def save_curves_figure(results): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + levels = sorted({r["n_gal"] for r in results}) + fig, axs = plt.subplots(1, 3, figsize=(14, 4.4)) + for name, color in [("se3_lift", "tab:blue"), ("hessian", "tab:orange")]: + for ax, key, sub in [(axs[0], "completeness", "M1"), + (axs[1], "purity", "M1"), (axs[2], "f1", "M2")]: + vals = [[r["scores"][name][sub][key] for r in results + if r["n_gal"] == L] for L in levels] + ax.errorbar(levels, [np.mean(v) for v in vals], + yerr=[np.std(v) for v in vals], marker="o", + color=color, label=name, capsize=3) + for ax, t in zip(axs, [f"M1 completeness (frac truth < " + f"{CFG['r0_spine']} vox of estimate)", + f"M1 purity (frac est < {CFG['r0_spine']} vox of truth)", + f"M2 junction F1 (r0 = {CFG['r0_junction']} vox)"]): + ax.set_xscale("log") + ax.set_xlabel("n_gal (galaxies per $128^3$ box)") + ax.set_ylim(0, 1.05) + ax.set_title(t, fontsize=10) + ax.grid(alpha=0.3) + ax.legend() + fig.suptitle(f"E0a-{TAG} held-out means ± sd (eval seeds " + f"{CFG['eval_seeds']}), matched skeleton length") + fig.tight_layout() + fig.savefig(ROOT / "docs" / "figures" / f"e0_curves_{TAG}.png", dpi=130) + plt.close(fig) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e0b_sweeps.py b/research/cosmic-web/scripts/run_e0b_sweeps.py new file mode 100644 index 00000000000..24a5b95caa0 --- /dev/null +++ b/research/cosmic-web/scripts/run_e0b_sweeps.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""E0b: two follow-up sweeps from the E0 open questions. + +Sweep A (Q1 — does hypoelliptic diffusion ever pay?): + bend_frac x diffusion-strength x sampling, lift method only, scoring + overall completeness AND curvature-binned completeness (terciles of + discrete Menger curvature over truth points). If diffusion helps, it + should show first in the high-curvature bin at extreme sparsity. + +Sweep B (Q2 — junctions without node clumps): + node_frac=0 so junctions are only implied by filament continuity; + lift vs hessian, junction F1. + +Results -> artifacts/e0b_sweeps.json, figures + docs/E0b-sweeps.md. +""" + +import json +import sys +import time +from multiprocessing import Pool +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +import fields, lift, spines, metrics # noqa: E402 + +N = 128 +N_SEEDS_WEB = 32 +R0 = 2.0 +R0_J = 3.0 +EVAL_SEEDS = list(range(2, 22)) # 20 held-out seeds +DIFF_CONFIGS = { + "off": dict(n_iter=0), + "weak": dict(n_iter=1, sig_par=4.0, sig_perp=0.5, alpha=0.35), + "strong": dict(n_iter=3, sig_par=6.0, sig_perp=0.5, alpha=0.5), +} +BENDS = [0.15, 0.30, 0.50] +LEVELS_A = [1200, 2500] +LEVELS_B = [2500, 5000] + +_AXES = lift.hemisphere_axes(42) +_W = lift.axis_adjacency(_AXES) + + +def menger_curvature(pts): + """Discrete curvature per point of an ordered polyline sample + (endpoints get their neighbour's value).""" + a, b, c = pts[:-2], pts[1:-1], pts[2:] + ab, bc, ca = (np.linalg.norm(b - a, axis=1), + np.linalg.norm(c - b, axis=1), + np.linalg.norm(a - c, axis=1)) + area2 = np.linalg.norm(np.cross(b - a, c - a), axis=1) + with np.errstate(divide="ignore", invalid="ignore"): + k = area2 / (ab * bc * ca) + k = np.nan_to_num(k) + return np.concatenate([[k[0]], k, [k[-1]]]) + + +def make_web(seed, bend, n_gal, node_frac=0.15): + rng = np.random.default_rng(seed) + segments, vertices = fields.voronoi_web(N, N_SEEDS_WEB, rng) + polys = fields.curve_polylines(segments, bend, rng) + tp, curv = [], [] + for p in polys: + d, _ = fields.sample_polylines([p], step=0.25) + tp.append(d) + curv.append(menger_curvature(d)) + truth_pts = np.vstack(tp) + curvature = np.concatenate(curv) + total_len = sum(np.linalg.norm(np.diff(p, axis=0), axis=1).sum() + for p in polys) + gal = fields.polyline_galaxies(polys, vertices, N, n_gal, 0.25, 0.8, rng, + node_frac=node_frac) + field = fields.galaxy_field(gal, N, n_gal, rng) + return field, truth_pts, curvature, vertices, int(total_len) + + +def lift_skeleton(field, diff_cfg, n_target): + U = lift.orientation_score(field, _AXES, 6.0, 1.5) + if diff_cfg["n_iter"]: + U = lift.hypoelliptic_diffuse(U, _AXES, _W, diff_cfg["sig_par"], + diff_cfg["sig_perp"], diff_cfg["alpha"], + diff_cfg["n_iter"]) + return spines.extract_matched(spines.lifted_ridgeness(U), n_target) + + +def job_a(args): + seed, bend, diff_name, n_gal = args + field, truth_pts, curv, _, n_target = make_web(seed, bend, n_gal) + skel = lift_skeleton(field, DIFF_CONFIGS[diff_name], n_target) + est = spines.skeleton_points(skel) + from scipy.spatial import cKDTree + d = cKDTree(est).query(truth_pts)[0] if len(est) else \ + np.full(len(truth_pts), np.inf) + hit = d < R0 + terc = np.percentile(curv, [33.3, 66.7]) + bins = np.digitize(curv, terc) + return {"sweep": "A", "seed": seed, "bend": bend, "diff": diff_name, + "n_gal": n_gal, + "completeness": float(hit.mean()), + "completeness_by_curv": [float(hit[bins == b].mean()) + for b in range(3)], + "curv_tercile_edges": [float(t) for t in terc]} + + +def job_b(args): + seed, method, n_gal = args + field, truth_pts, _, vertices, n_target = make_web( + seed, 0.15, n_gal, node_frac=0.0) + if method == "se3_lift": + skel = lift_skeleton(field, DIFF_CONFIGS["off"], n_target) + else: + skel = spines.extract_matched( + spines.hessian_ridgeness(field, 1.5), n_target) + m1 = metrics.spine_metrics(truth_pts, spines.skeleton_points(skel), R0) + m2 = metrics.junction_metrics(vertices, spines.junction_points(skel), R0_J) + return {"sweep": "B", "seed": seed, "method": method, "n_gal": n_gal, + "completeness": m1["completeness"], "jF1": m2["f1"], + "jprec": m2["precision"], "jrec": m2["recall"]} + + +def main(): + jobs_a = [(s, b, d, n) for s in EVAL_SEEDS for b in BENDS + for d in DIFF_CONFIGS for n in LEVELS_A] + jobs_b = [(s, m, n) for s in EVAL_SEEDS for m in ("se3_lift", "hessian") + for n in LEVELS_B] + t0 = time.time() + with Pool(8) as pool: + res_a = pool.map(job_a, jobs_a) + res_b = pool.map(job_b, jobs_b) + print(f"{len(jobs_a)}+{len(jobs_b)} evals in {time.time()-t0:.0f}s") + out = ROOT / "artifacts" / "e0b_sweeps.json" + out.write_text(json.dumps({"A": res_a, "B": res_b}, indent=1)) + print(f"wrote {out}") + summarize(res_a, res_b) + + +def summarize(res_a, res_b): + from scipy.stats import wilcoxon + lines = ["# E0b — diffusion sweep (Q1) and clump-free junctions (Q2)\n"] + + lines.append("\n## Sweep A: does diffusion help anywhere?\n") + lines.append("Completeness (mean over 20 seeds), lift only; paired " + "Δ(diff − off) with Wilcoxon p. Curvature bins are " + "terciles of discrete Menger curvature of truth points " + "(bin 2 = most curved third).\n") + lines.append("| n_gal | bend | diff | C overall | Δ vs off (p) " + "| C bin0 | C bin2 | Δ bin2 vs off (p) |") + lines.append("|---|---|---|---|---|---|---|---|") + for n_gal in LEVELS_A: + for bend in BENDS: + base = {r["seed"]: r for r in res_a + if r["bend"] == bend and r["n_gal"] == n_gal + and r["diff"] == "off"} + for dn in DIFF_CONFIGS: + rs = [r for r in res_a if r["bend"] == bend + and r["n_gal"] == n_gal and r["diff"] == dn] + c = np.array([r["completeness"] for r in rs]) + c2 = np.array([r["completeness_by_curv"][2] for r in rs]) + if dn == "off": + d_str = p_str = d2_str = "—" + else: + d = np.array([r["completeness"] - base[r["seed"]]["completeness"] + for r in rs]) + d2 = np.array([r["completeness_by_curv"][2] + - base[r["seed"]]["completeness_by_curv"][2] + for r in rs]) + d_str = f"{d.mean():+.3f} ({wilcoxon(d).pvalue:.2g})" + d2_str = f"{d2.mean():+.3f} ({wilcoxon(d2).pvalue:.2g})" + c0 = np.mean([r["completeness_by_curv"][0] for r in rs]) + lines.append(f"| {n_gal} | {bend} | {dn} | {c.mean():.3f} " + f"| {d_str} | {c0:.3f} | {c2.mean():.3f} " + f"| {d2_str} |") + lines.append("") + + lines.append("\n## Sweep B: junctions with node_frac = 0\n") + lines.append("| n_gal | method | completeness | junction F1 | jF1 " + "Δ(lift−hess) (p) |") + lines.append("|---|---|---|---|---|") + for n_gal in LEVELS_B: + lifts = {r["seed"]: r for r in res_b + if r["n_gal"] == n_gal and r["method"] == "se3_lift"} + hesss = {r["seed"]: r for r in res_b + if r["n_gal"] == n_gal and r["method"] == "hessian"} + d = np.array([lifts[s]["jF1"] - hesss[s]["jF1"] for s in lifts]) + pj = wilcoxon(d).pvalue if np.any(d) else 1.0 + for m, rs in [("se3_lift", lifts), ("hessian", hesss)]: + c = np.mean([r["completeness"] for r in rs.values()]) + f = np.mean([r["jF1"] for r in rs.values()]) + delta = f"**{d.mean():+.3f}** ({pj:.2g})" if m == "se3_lift" else "" + lines.append(f"| {n_gal} | {m} | {c:.3f} | {f:.3f} | {delta} |") + + out = ROOT / "docs" / "E0b-sweeps.md" + out.write_text("\n".join(lines) + "\n") + print(f"wrote {out}") + print("\n".join(lines[-12:])) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e0c_hybrid.py b/research/cosmic-web/scripts/run_e0c_hybrid.py new file mode 100644 index 00000000000..044c201f022 --- /dev/null +++ b/research/cosmic-web/scripts/run_e0c_hybrid.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""E0c: junction/spine hybrid — can combining the two scores beat both? + +Motivation: E0/E0b showed a consistent division of labour (lift wins spines +in sparse data, Hessian wins junctions). Test the simplest combinations of +the percentile-normalised ridgeness fields: + + hyb_sum = R_lift/q(R_lift) + R_hess/q(R_hess), q = 99.5th percentile + hyb_max = max of the two normalised fields + +on the curved Voronoi toy at sparse sampling, against both parents, same +matched-length pipeline, exact truth. Report -> docs/E0c-hybrid-report.md. +""" + +import json +import sys +import time +from multiprocessing import Pool +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +import fields, lift, metrics, spines # noqa: E402 + +N = 128 +SEEDS = list(range(2, 22)) +LEVELS = [2500, 5000] +_AXES = lift.hemisphere_axes(42) + + +def norm(R): + return R / max(np.percentile(R, 99.5), 1e-12) + + +def run_seed(seed): + recs = [] + for n_gal in LEVELS: + rng = np.random.default_rng(seed) + segments, vertices = fields.voronoi_web(N, 32, rng) + polys = fields.curve_polylines(segments, 0.15, rng) + truth_pts, total = fields.sample_polylines(polys) + gal = fields.polyline_galaxies(polys, vertices, N, n_gal, 0.25, 0.8, rng) + field = fields.galaxy_field(gal, N, n_gal, rng) + n_target = int(total) + + R_h = norm(spines.hessian_ridgeness(field, 1.5)) + R_l = norm(spines.lifted_ridgeness( + lift.orientation_score(field, _AXES, 6.0, 1.5))) + ests = {"hessian": R_h, "se3_lift": R_l, + "hyb_sum": R_h + R_l, "hyb_max": np.maximum(R_h, R_l)} + for mname, R in ests.items(): + skel = spines.extract_matched(R, n_target) + m1 = metrics.spine_metrics(truth_pts, + spines.skeleton_points(skel), 2.0) + m2 = metrics.junction_metrics(vertices, + spines.junction_points(skel), 3.0) + recs.append({"seed": seed, "n_gal": n_gal, "method": mname, + "C": m1["completeness"], "P": m1["purity"], + "jF1": m2["f1"]}) + return recs + + +def main(): + t0 = time.time() + with Pool(8) as pool: + recs = [r for rs in pool.map(run_seed, SEEDS) for r in rs] + print(f"{len(SEEDS)} seeds in {time.time()-t0:.0f}s") + (ROOT / "artifacts" / "e0c_results.json").write_text(json.dumps(recs, indent=1)) + report(recs) + + +def report(recs): + from scipy.stats import wilcoxon + methods = ["hessian", "se3_lift", "hyb_sum", "hyb_max"] + md = ["# E0c — junction/spine hybrids on the curved toy (exact truth)\n", + f"Setup: E0-curved protocol, {len(SEEDS)} held-out seeds, sparse " + "levels only (where the division of labour lives). Hybrids are " + "sums/maxima of percentile-normalised parent scores; identical " + "matched-length pipeline.\n"] + md.append("| n_gal | method | completeness | purity | junction F1 " + "| ΔC vs best parent (p) | ΔjF1 vs best parent (p) |") + md.append("|---|---|---|---|---|---|---|") + for n_gal in LEVELS: + by = {m: {r["seed"]: r for r in recs + if r["n_gal"] == n_gal and r["method"] == m} + for m in methods} + for m in methods: + C = np.array([by[m][s]["C"] for s in SEEDS]) + P = np.array([by[m][s]["P"] for s in SEEDS]) + F = np.array([by[m][s]["jF1"] for s in SEEDS]) + cells = [] + if m.startswith("hyb"): + pc_ref = ("hessian" if np.mean([by["hessian"][s]["C"] + for s in SEEDS]) >= np.mean([by["se3_lift"][s]["C"] + for s in SEEDS]) else "se3_lift") + dC = np.array([by[m][s]["C"] - by[pc_ref][s]["C"] + for s in SEEDS]) + pj_ref = ("hessian" if np.mean([by["hessian"][s]["jF1"] + for s in SEEDS]) >= np.mean([by["se3_lift"][s]["jF1"] + for s in SEEDS]) else "se3_lift") + dF = np.array([by[m][s]["jF1"] - by[pj_ref][s]["jF1"] + for s in SEEDS]) + cells = [f"{dC.mean():+.3f} vs {pc_ref} " + f"({wilcoxon(dC).pvalue:.2g})", + f"{dF.mean():+.3f} vs {pj_ref} " + f"({wilcoxon(dF).pvalue:.2g})"] + else: + cells = ["—", "—"] + md.append(f"| {n_gal} | {m} | {C.mean():.3f} ± {C.std():.3f} " + f"| {P.mean():.3f} | {F.mean():.3f} ± {F.std():.3f} " + f"| {cells[0]} | {cells[1]} |") + out = ROOT / "docs" / "E0c-hybrid-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + print("\n".join(md[3:])) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e0d_followups.py b/research/cosmic-web/scripts/run_e0d_followups.py new file mode 100644 index 00000000000..c98c2cd1fd9 --- /dev/null +++ b/research/cosmic-web/scripts/run_e0d_followups.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""E0d: three follow-ups forced by the correction pass. + +A. Splitting convergence (closes the E0b caveat properly): my hypoelliptic + diffusion used 1-3 coarse splitting steps. Trotter: many small + alternations converge to the true left-invariant semigroup. Compare, at + equal TOTAL diffusion (same accumulated parallel variance and angular + mixing): off | coarse (1 step) | fine (8 steps). If fine ~ coarse, the + diffusion negative is about the operator, not the numerics. + +B. Sparsity limit: corrected E0 showed a tie at n_gal=1200. Does the lift + ever WIN under honest matching as sampling gets extreme? + n_gal in {400, 600, 800, 1200}, 30 seeds, curved toy. + +C. Hybrid on gravity fields: does hyb_sum's purity/junction gain on toys + translate into mass/band coverage on truncated-ZA webs? + +Report -> docs/E0d-followups.md. +""" + +import json +import sys +import time +from multiprocessing import Pool +from pathlib import Path + +import numpy as np +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +import fields, lift, metrics, spines # noqa: E402 + +N = 128 +_AXES = lift.hemisphere_axes(42) +_W = lift.axis_adjacency(_AXES) + +DIFF = { + "off": None, + # total parallel variance 16 (sigma 4), angular mixing alpha_tot ~ 0.35 + "coarse": dict(n_iter=1, sig_par=4.0, sig_perp=0.5, alpha=0.35), + "fine": dict(n_iter=8, sig_par=4.0 / np.sqrt(8), sig_perp=0.5 / np.sqrt(8), + alpha=1 - (1 - 0.35) ** (1 / 8)), +} + + +def norm(R): + return R / max(np.percentile(R, 99.5), 1e-12) + + +def make_toy(seed, n_gal): + rng = np.random.default_rng(seed) + segments, vertices = fields.voronoi_web(N, 32, rng) + polys = fields.curve_polylines(segments, 0.15, rng) + truth_pts, total = fields.sample_polylines(polys) + gal = fields.polyline_galaxies(polys, vertices, N, n_gal, 0.25, 0.8, rng) + field = fields.galaxy_field(gal, N, n_gal, rng) + return field, truth_pts, int(total) + + +def job_a(args): + seed, n_gal, dname = args + field, truth_pts, n_target = make_toy(seed, n_gal) + U = lift.orientation_score(field, _AXES, 6.0, 1.5) + if DIFF[dname]: + p = DIFF[dname] + U = lift.hypoelliptic_diffuse(U, _AXES, _W, p["sig_par"], + p["sig_perp"], p["alpha"], p["n_iter"]) + skel = spines.extract_matched(spines.lifted_ridgeness(U), n_target) + m1 = metrics.spine_metrics(truth_pts, spines.skeleton_points(skel), 2.0) + return {"part": "A", "seed": seed, "n_gal": n_gal, "diff": dname, + "C": m1["completeness"], "P": m1["purity"]} + + +def job_b(args): + seed, n_gal, method = args + field, truth_pts, n_target = make_toy(seed, n_gal) + if method == "hessian": + R = spines.hessian_ridgeness(field, 1.5) + else: + R = spines.lifted_ridgeness( + lift.orientation_score(field, _AXES, 6.0, 1.5)) + skel = spines.extract_matched(R, n_target) + m1 = metrics.spine_metrics(truth_pts, spines.skeleton_points(skel), 2.0) + return {"part": "B", "seed": seed, "n_gal": n_gal, "method": method, + "C": m1["completeness"], "P": m1["purity"]} + + +def job_c(seed): + rng = np.random.default_rng(seed) + rho, pos = fields.zeldovich_box(N, 128.0, 1.0, rng, trunc=2.0) + idx = (np.floor(pos).astype(int)) % N + rp = rho[idx[:, 0], idx[:, 1], idx[:, 2]] + band = (rp > 2) & (rp < 20) + recs = [] + for n_gal in (5000, 20000): + gal_field = fields.galaxy_field(pos, N, n_gal, rng, adaptive=True) + R_h = norm(spines.hessian_ridgeness(gal_field, 1.5)) + R_l = norm(spines.lifted_ridgeness( + lift.orientation_score(gal_field, _AXES, 6.0, 1.5))) + for mname, R in [("hessian", R_h), ("se3_lift", R_l), + ("hyb_sum", R_h + R_l)]: + pts = spines.skeleton_points( + spines.extract_matched(R, 2400, lo_frac=1.0)) + tree = cKDTree(pts) + recs.append({"part": "C", "seed": seed, "n_gal": n_gal, + "method": mname, + "mass_cov": float((tree.query(pos)[0] < 3.0).mean()), + "band_cov": float((tree.query(pos[band])[0] < 3.0).mean())}) + return recs + + +def main(): + seeds_a = list(range(2, 22)) + seeds_b = list(range(2, 32)) + seeds_c = list(range(1, 13)) + jobs_a = [(s, n, d) for s in seeds_a for n in (1200, 2500) for d in DIFF] + jobs_b = [(s, n, m) for s in seeds_b for n in (400, 600, 800, 1200) + for m in ("hessian", "se3_lift")] + t0 = time.time() + with Pool(8) as pool: + res_a = pool.map(job_a, jobs_a) + res_b = pool.map(job_b, jobs_b) + res_c = [r for rs in pool.map(job_c, seeds_c) for r in rs] + print(f"{len(jobs_a)}+{len(jobs_b)}+{len(seeds_c)} jobs " + f"in {time.time()-t0:.0f}s") + (ROOT / "artifacts" / "e0d_results.json").write_text( + json.dumps({"A": res_a, "B": res_b, "C": res_c}, indent=1)) + report(res_a, res_b, res_c) + + +def report(res_a, res_b, res_c): + from scipy.stats import wilcoxon + md = ["# E0d — post-correction follow-ups\n"] + + md.append("\n## A. Splitting convergence (is the diffusion negative " + "numerical?)\n") + md.append("Equal total diffusion; fine = 8 Trotter steps. Curved toy, " + "20 seeds, corrected extractor.\n") + md.append("| n_gal | diff | completeness | Δ vs off (p) |") + md.append("|---|---|---|---|") + for n_gal in (1200, 2500): + base = {r["seed"]: r["C"] for r in res_a + if r["n_gal"] == n_gal and r["diff"] == "off"} + for d in DIFF: + rs = [r for r in res_a if r["n_gal"] == n_gal and r["diff"] == d] + C = np.array([r["C"] for r in rs]) + if d == "off": + cell = "—" + else: + dd = np.array([r["C"] - base[r["seed"]] for r in rs]) + cell = f"{dd.mean():+.4f} ({wilcoxon(dd).pvalue:.2g})" + md.append(f"| {n_gal} | {d} | {C.mean():.3f} ± {C.std():.3f} " + f"| {cell} |") + + md.append("\n## B. Sparsity limit (does the lift ever win honestly?)\n") + md.append("| n_gal | ΔC (lift − hessian) | lift wins | Wilcoxon p " + "| C hessian | C lift |") + md.append("|---|---|---|---|---|---|") + seeds_b = sorted({r["seed"] for r in res_b}) + for n_gal in (400, 600, 800, 1200): + by = {m: {r["seed"]: r["C"] for r in res_b + if r["n_gal"] == n_gal and r["method"] == m} + for m in ("hessian", "se3_lift")} + d = np.array([by["se3_lift"][s] - by["hessian"][s] for s in seeds_b]) + md.append(f"| {n_gal} | {d.mean():+.4f} | {int((d>0).sum())}" + f"/{len(d)} | {wilcoxon(d).pvalue:.2g} " + f"| {np.mean(list(by['hessian'].values())):.3f} " + f"| {np.mean(list(by['se3_lift'].values())):.3f} |") + + md.append("\n## C. Hybrid on gravity fields (truncated ZA, 12 seeds)\n") + md.append("| n_gal | method | mass coverage | band coverage " + "| Δband vs hessian (p) |") + md.append("|---|---|---|---|---|") + seeds_c = sorted({r["seed"] for r in res_c}) + for n_gal in (5000, 20000): + by = {m: {r["seed"]: r for r in res_c + if r["n_gal"] == n_gal and r["method"] == m} + for m in ("hessian", "se3_lift", "hyb_sum")} + for m in ("hessian", "se3_lift", "hyb_sum"): + mc = np.mean([by[m][s]["mass_cov"] for s in seeds_c]) + bc = np.array([by[m][s]["band_cov"] for s in seeds_c]) + if m == "hessian": + cell = "—" + else: + d = np.array([by[m][s]["band_cov"] - by["hessian"][s]["band_cov"] + for s in seeds_c]) + cell = f"{d.mean():+.4f} ({wilcoxon(d).pvalue:.2g})" + md.append(f"| {n_gal} | {m} | {mc:.4f} | {bc.mean():.4f} " + f"± {bc.std():.4f} | {cell} |") + + out = ROOT / "docs" / "E0d-followups.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + print("\n".join(md[2:])) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e1.py b/research/cosmic-web/scripts/run_e1.py new file mode 100644 index 00000000000..a941d5761c3 --- /dev/null +++ b/research/cosmic-web/scripts/run_e1.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""E1: gravity-shaped fields (Zel'dovich boxes) — H1 transfer + first H2 test. + +No exact truth exists here, so the reference skeleton is extracted from the +CLEAN full-particle density (2.1M particles, effectively noise-free) and +methods are scored on sparse galaxy subsamples against it. To avoid +favouring either method, references from BOTH methods are used (2x2): +beating the baseline even when judged against the baseline's own clean +reference is a conservative win. + +H2 (first pass): multiply the orientation score by a tidal-alignment weight +w(x,n) = 1 + beta * (^2 - 1/3), e3 = eigenvector of the smallest +eigenvalue of the tidal tensor T = Hess(phi) computed FROM THE SPARSE FIELD +itself (usable on real data). beta = 0 is the unmodulated control. + +M3: alignment of final spine tangents with the clean-field tidal e3 +(isotropic null: E|cos| = 0.5). + +Results -> artifacts/e1_results.json, report -> docs/E1-report.md. +""" + +import json +import sys +import time +from multiprocessing import Pool +from pathlib import Path + +import numpy as np +from scipy.ndimage import gaussian_filter +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +import fields, lift, spines, metrics # noqa: E402 + +N, L, D, TRUNC = 128, 128.0, 1.0, 2.0 +N_TARGET = 2400 # matched spine budget (voxels), all skeletons +R0 = 3.0 +SEEDS = list(range(1, 13)) +LEVELS = [5000, 20000] +BETAS = [0.0, 1.0, 2.0] + +_AXES = lift.hemisphere_axes(42) + + +def lifted_ridgeness_weighted(field, e3, beta, sig_par=6.0): + U = lift.orientation_score(field, _AXES, sig_par, 1.5) + if beta: + for a, n in enumerate(_AXES): + dot2 = (e3 @ n) ** 2 + U[a] *= np.maximum(1 + beta * (dot2 - 1 / 3), 0.05).astype(np.float32) + return spines.lifted_ridgeness(U) + + +def spine_tangents(pts, radius=3.0): + """Local PCA tangent per skeleton point.""" + tree = cKDTree(pts) + tans = np.zeros_like(pts) + for i, p in enumerate(pts): + nb = pts[tree.query_ball_point(p, radius)] + if len(nb) < 3: + continue + c = nb - nb.mean(0) + _, _, vt = np.linalg.svd(c, full_matrices=False) + tans[i] = vt[0] + return tans + + +def alignment(pts, tans, e3): + ok = np.linalg.norm(tans, axis=1) > 0 + idx = np.clip(np.round(pts[ok]).astype(int), 0, N - 1) + e = e3[idx[:, 0], idx[:, 1], idx[:, 2]] + return float(np.mean(np.abs(np.sum(tans[ok] * e, axis=1)))) + + +def run_seed(seed): + rng = np.random.default_rng(seed) + rho, pos = fields.zeldovich_box(N, L, D, rng, trunc=TRUNC) + clean = gaussian_filter(np.log1p(rho), 1.0, mode="wrap") + e3_clean = fields.tidal_frame(clean) + + refs = { + "ref_hessian": spines.skeleton_points(spines.extract_matched( + spines.hessian_ridgeness(clean, 1.5), N_TARGET, lo_frac=1.0)), + "ref_lift": spines.skeleton_points(spines.extract_matched( + lifted_ridgeness_weighted(clean, None, 0.0), N_TARGET, + lo_frac=1.0)), + } + + recs = [] + for n_gal in LEVELS: + gal_field = fields.galaxy_field(pos, N, n_gal, rng, adaptive=True) + e3_sparse = fields.tidal_frame(gal_field) + ests = {"hessian": spines.hessian_ridgeness(gal_field, 1.5)} + for beta in BETAS: + ests[f"lift_b{beta:g}"] = lifted_ridgeness_weighted( + gal_field, e3_sparse, beta) + for mname, R in ests.items(): + skel = spines.extract_matched(R, N_TARGET, lo_frac=1.0) + pts = spines.skeleton_points(skel) + rec = {"seed": seed, "n_gal": n_gal, "method": mname} + for rname, ref in refs.items(): + m1 = metrics.spine_metrics(ref, pts, R0) + rec[f"C_{rname}"] = m1["completeness"] + rec[f"P_{rname}"] = m1["purity"] + rec["align_e3"] = alignment(pts, spine_tangents(pts), e3_clean) + recs.append(rec) + return recs + + +def main(): + t0 = time.time() + with Pool(6) as pool: + all_recs = [r for rs in pool.map(run_seed, SEEDS) for r in rs] + print(f"{len(SEEDS)} seeds in {time.time()-t0:.0f}s") + out = ROOT / "artifacts" / "e1_results.json" + out.write_text(json.dumps(all_recs, indent=1)) + report(all_recs) + + +def report(recs): + from scipy.stats import wilcoxon + md = ["# E1 — gravity-shaped fields (Zel'dovich, D=1.0): H1 transfer " + "and first H2 test\n", + f"Setup: {len(SEEDS)} Zel'dovich boxes ({N}³, L={L:.0f} h⁻¹Mpc, " + "BBKS spectrum, σ₈=0.8·D, truncated ZA R_t=2), full 2.1M-particle density as clean " + "reference, sparse galaxy subsamples as input. All skeletons at " + f"matched length {N_TARGET} vox. Completeness/purity vs the clean " + "reference of EACH method (2×2, conservative). M3 alignment: mean " + "|tangent·e3| of spine tangents vs clean-field tidal eigenframe " + "(isotropic null 0.5). H2 weight: w = 1 + β(⟨n,e3_sparse⟩² − 1/3)." + "\n"] + md.append("| n_gal | method | C vs ref_hess | C vs ref_lift " + "| P vs ref_hess | align e3 |") + md.append("|---|---|---|---|---|---|") + methods = ["hessian"] + [f"lift_b{b:g}" for b in BETAS] + for n_gal in LEVELS: + for m in methods: + rs = [r for r in recs if r["n_gal"] == n_gal and r["method"] == m] + md.append( + f"| {n_gal} | {m} " + f"| {np.mean([r['C_ref_hessian'] for r in rs]):.3f} ± " + f"{np.std([r['C_ref_hessian'] for r in rs]):.3f} " + f"| {np.mean([r['C_ref_lift'] for r in rs]):.3f} ± " + f"{np.std([r['C_ref_lift'] for r in rs]):.3f} " + f"| {np.mean([r['P_ref_hessian'] for r in rs]):.3f} " + f"| {np.mean([r['align_e3'] for r in rs]):.3f} |") + + md.append("\n## Paired tests\n") + for n_gal in LEVELS: + by = {m: {r["seed"]: r for r in recs + if r["n_gal"] == n_gal and r["method"] == m} + for m in methods} + d_h1 = np.array([by["lift_b0"][s]["C_ref_hessian"] + - by["hessian"][s]["C_ref_hessian"] for s in SEEDS]) + md.append(f"- **H1 transfer, n_gal={n_gal}** (lift β=0 vs hessian, " + f"judged on the hessian's own clean reference): " + f"ΔC = {d_h1.mean():+.3f}, wins {int((d_h1>0).sum())}" + f"/{len(d_h1)}, Wilcoxon p = {wilcoxon(d_h1).pvalue:.2g}") + for b in BETAS[1:]: + d = np.array([by[f"lift_b{b:g}"][s]["C_ref_hessian"] + - by["lift_b0"][s]["C_ref_hessian"] for s in SEEDS]) + md.append(f"- **H2, n_gal={n_gal}, β={b:g}** (tidal weight vs " + f"β=0, same reference): ΔC = {d.mean():+.3f}, wins " + f"{int((d>0).sum())}/{len(d)}, " + f"Wilcoxon p = {wilcoxon(d).pvalue:.2g}") + + out = ROOT / "docs" / "E1-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + print("\n".join(md[-8:])) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e10.py b/research/cosmic-web/scripts/run_e10.py new file mode 100644 index 00000000000..f7b0459c796 --- /dev/null +++ b/research/cosmic-web/scripts/run_e10.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""E10 (referee point 2, local part): truth convergence. + +Step-doubling test on the PM truth: same ICs, 90 vs 180 KDK steps. +Reports (a) the truth's own convergence (median per-particle shift between +the two runs) and (b) the stability of the model-vs-ZA comparison when +evaluated against each truth. The grid axis is covered by E7/E7b (256^3 +runs, advantage persists). External-simulation cross-check (Quijote) +remains open pending data download approval. +Report -> docs/E10-report.md. +""" +import json, sys, time +from pathlib import Path +import numpy as np + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")); sys.path.insert(0, str(ROOT / "scripts")) +import fields, pm # noqa: E402 +from run_e5 import D_STEPS, N # noqa: E402 +from run_e8 import evolve # noqa: E402 + +SEED = 4 + +def main(): + t0 = time.time() + truths = {} + for steps in (90, 180): + rng = np.random.default_rng(SEED) + x, v, tidx, ta, tpos, q_tr, psi_tr = pm.pm_sim( + N, 128.0, rng, n_steps=steps, track=50000) + truths[steps] = tpos[-1].astype(float) + d = truths[180] - truths[90] + d -= N * np.round(d / N) + conv = float(np.median(np.linalg.norm(d, axis=1))) + + seedseq = np.random.default_rng(SEED).integers(0, 2 ** 32) + _, pos_q = fields.zeldovich_box(N, 128.0, 0.0, np.random.default_rng(seedseq)) + _, pos_d = fields.zeldovich_box(N, 128.0, 1.0, np.random.default_rng(seedseq)) + psi_full = pos_d - pos_q + psi_full -= N * np.round(psi_full / N) + xz = ((pos_q + D_STEPS[-1] * psi_full) % N)[tidx] + xm = evolve(pos_q, psi_full, "damp")[tidx] + + lines = [f"truth self-convergence (90 vs 180 steps): median shift {conv:.3f} vox"] + for steps, truth in truths.items(): + errs = {} + for name, xmod in (("za", xz), ("damp", xm)): + e = xmod - truth; e -= N * np.round(e / N) + errs[name] = float(np.median(np.linalg.norm(e, axis=1))) + lines.append(f"vs truth({steps} steps): ZA {errs['za']:.3f}, damp " + f"{errs['damp']:.3f}, advantage " + f"{100*(errs['damp']-errs['za'])/errs['za']:+.1f}%") + md = ["# E10 — truth convergence (step doubling)\n", ""] + \ + [f"- {l}" for l in lines] + \ + ["", "Grid-axis convergence: covered by E7/E7b (256³ truths; " + "advantage persists at −9% to −12%). External-simulation " + "cross-check (Quijote snapshot) remains open pending download " + "approval.", ""] + (ROOT / "docs" / "E10-report.md").write_text("\n".join(md)) + (ROOT / "artifacts" / "e10_results.json").write_text(json.dumps( + {"conv_vox": conv, "lines": lines}, indent=1)) + print("\n".join(lines)) + print(f"total {time.time()-t0:.0f}s") + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e11.py b/research/cosmic-web/scripts/run_e11.py new file mode 100644 index 00000000000..933a9a198e8 --- /dev/null +++ b/research/cosmic-web/scripts/run_e11.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""E11: the hybrid — MUSCLE's Lagrangian trigger + directional damping. + +E9 showed MUSCLE ties the frozen model with complementary strengths. The +principled hybrid keeps our directional (transverse-only) damping but +replaces the ad-hoc Eulerian trigger (rho > 5) with MUSCLE's Lagrangian +multiscale collapse condition: particle q crosses when D * delta_R(q) >= +3/2 at any smoothing scale R — precomputable per particle as D_cross(q). + +Models: MUSCLE | damp (frozen, Eulerian trigger) | damp_msc (hybrid). +Seeds {4,5,6}; transport + field metrics. Report -> docs/E11-report.md. +""" +import json, sys, time +from multiprocessing import Pool +from pathlib import Path +import numpy as np +from scipy.ndimage import gaussian_filter +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")); sys.path.insert(0, str(ROOT / "scripts")) +import fields, lift, pm, spines # noqa: E402 +from run_e5 import D_STEPS, N, gather, jk # noqa: E402 +from run_e8 import KBINS, evolve as evolve_damp, spectra # noqa: E402 + +SEEDS = [4, 5, 6] +BETA, SMOOTH = 0.6, 2.0 +SCALES = (0.0, 1.0, 2.0, 4.0, 8.0) + + +def d_cross_field(N_, L, seedseq): + """Per-Lagrangian-cell first-collapse growth factor (MUSCLE trigger).""" + dk, _, _ = fields._delta_k_of(N_, L, np.random.default_rng(seedseq)) + delta = np.fft.irfftn(dk, s=(N_, N_, N_)) + dcross = np.full(delta.shape, np.inf) + for R in SCALES: + dR = gaussian_filter(delta, R, mode="wrap") if R > 0 else delta + with np.errstate(divide="ignore"): + dc = np.where(dR > 0, 1.5 / dR, np.inf) + dcross = np.minimum(dcross, dc) + return dcross.ravel() + + +def evolve_hybrid(q_grid, psi_full, dcross): + x = (q_grid + D_STEPS[0] * psi_full) % N + v = psi_full.copy() + crossed = np.zeros(len(x), bool) + e3f = None + for i in range(1, len(D_STEPS)): + D = D_STEPS[i] + dD = D - D_STEPS[i - 1] + x = (x + v * dD) % N + rho = fields.cic_deposit(x, N) + if e3f is None or i % 3 == 0: + clean = gaussian_filter(np.log1p(rho), 1.0, mode="wrap") + e3f = fields.tidal_frame(clean, smooth=SMOOTH) + hit = (~crossed) & (dcross <= D) + if hit.any(): + idx = np.clip(np.round(x[hit]).astype(int), 0, N - 1) % N + e = e3f[idx[:, 0], idx[:, 1], idx[:, 2]] + par = e * np.sum(v[hit] * e, axis=1, keepdims=True) + v[hit] = par + (1 - BETA) * (v[hit] - par) + crossed |= hit + return x + + +def run_seed(seed): + rng = np.random.default_rng(seed) + x_pm, v_pm, tidx, ta, tpos, q_tr, psi_tr = pm.pm_sim( + N, 128.0, rng, n_steps=90, track=50000) + seedseq = np.random.default_rng(seed).integers(0, 2 ** 32) + _, pos_q = fields.zeldovich_box(N, 128.0, 0.0, np.random.default_rng(seedseq)) + _, pos_d = fields.zeldovich_box(N, 128.0, 1.0, np.random.default_rng(seedseq)) + psi_full = pos_d - pos_q + psi_full -= N * np.round(psi_full / N) + truth = tpos[-1].astype(float) + d_truth = fields.cic_deposit(x_pm, N) - 1.0 + clean = gaussian_filter(np.log1p(1 + d_truth), 1.0, mode="wrap") + sp = spines.skeleton_points(spines.extract_matched( + spines.lifted_ridgeness(lift.orientation_score(clean, lift.hemisphere_axes(42), 6.0, 1.5)), + 2400, lo_frac=1.0)) + web = cKDTree(sp).query(truth)[0] < 2.0 + + models = { + "muscle": fields.muscle_positions(N, 128.0, np.random.default_rng(seedseq)), + "damp": evolve_damp(pos_q, psi_full, "damp"), + "damp_msc": evolve_hybrid(pos_q, psi_full, + d_cross_field(N, 128.0, seedseq)), + } + rec = {"seed": seed} + for name, xf in models.items(): + err = xf[tidx] - truth + err -= N * np.round(err / N) + err = np.linalg.norm(err, axis=1) + rec[f"{name}_all"] = float(np.median(err)) + rec[f"{name}_web"] = float(np.median(err[web])) + r, t = spectra(fields.cic_deposit(xf, N) - 1.0, d_truth) + rec[f"{name}_r"], rec[f"{name}_T"] = r, t + return rec + + +def main(): + t0 = time.time() + with Pool(3) as pool: + recs = pool.map(run_seed, SEEDS) + print(f"{len(SEEDS)} seeds in {time.time()-t0:.0f}s") + (ROOT / "artifacts" / "e11_results.json").write_text(json.dumps(recs, indent=1)) + kc = np.sqrt(KBINS[:-1] * KBINS[1:]) + names = ["muscle", "damp", "damp_msc"] + md = ["# E11 — hybrid: Lagrangian (MUSCLE) trigger + directional damping\n"] + md.append("| model | transport all | web | r(k=0.53) | T(k=0.34) |") + md.append("|---|---|---|---|---|") + for n in names: + a, ae = jk([r[f"{n}_all"] for r in recs]) + w, we = jk([r[f"{n}_web"] for r in recs]) + rr = np.mean([r[f"{n}_r"][5] for r in recs]) + tt = np.mean([r[f"{n}_T"][4] for r in recs]) + md.append(f"| {n} | {a:.2f} ± {ae:.2f} | {w:.2f} ± {we:.2f} " + f"| {rr:.3f} | {tt:.3f} |") + (ROOT / "docs" / "E11-report.md").write_text("\n".join(md) + "\n") + print("\n".join(md[1:])) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e12.py b/research/cosmic-web/scripts/run_e12.py new file mode 100644 index 00000000000..622aa15c0b4 --- /dev/null +++ b/research/cosmic-web/scripts/run_e12.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""E12: external validation against CAMELS CV_0 (IllustrisTNG_DM, Arepo). + +Tier 1 (referee M1): evolve our PM code from CAMELS' own 2LPT ICs +(z=127 -> z=0, LCDM Om=0.3) and compare to the official z=0 snapshot: +field-level r(k)/T(k) and matched-ID per-particle displacement. +Tier 2: score ZA and the frozen damping model against the EXTERNAL truth. +Caveat by design: the 25 Mpc/h box probes 0.1 Mpc cells — a 10x +extrapolation beyond the model's tested resolution range; tier 2 is a +stress test, tier 1 is the referee item. Report -> docs/E12-report.md. +""" +import json, sys, time +from pathlib import Path +import numpy as np +import h5py, hdf5plugin # noqa: F401 +from scipy.ndimage import gaussian_filter + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")); sys.path.insert(0, str(ROOT / "scripts")) +import fields, pm # noqa: E402 +from run_e5 import D_STEPS, gather # noqa: E402 + +N = 256 +OM = 0.3 +A_I, A_F, STEPS = 1.0 / 128.0, 1.0, 180 +BETA, SMOOTH_MPC, RHO_C = 0.6, 2.0, 5.0 +VOX = 25.0 / 256.0 # Mpc/h per voxel +import os +TAG = os.environ.get("E12_TAG", "camels") +DATA = ROOT / "data" / TAG +ICS_DIR = ROOT / "data" / os.environ.get("E12_ICS", TAG) + + +def read_ics(): + xs, ids = [], [] + for i in range(8): + with h5py.File(ICS_DIR / "ICs" / f"ics.{i}.hdf5") as g: + xs.append(np.asarray(g["PartType1"]["Coordinates"])) + ids.append(np.asarray(g["PartType1"]["ParticleIDs"])) + x = np.concatenate(xs) / (25000.0 / N) # kpc -> voxels + ids = np.concatenate(ids) + q = np.round(x) # ICs sit near grid nodes + psi_ic = x - q + psi_ic -= N * np.round(psi_ic / N) + frac_far = float((np.abs(psi_ic) > 0.45).mean()) + D_i = pm.growth_factor([A_I], OM)[0] + return q % N, psi_ic / D_i, ids, frac_far, D_i + + +def read_truth(): + with h5py.File(DATA / "snapshot_090.hdf5") as f: + x = np.asarray(f["PartType1"]["Coordinates"]) / (25000.0 / N) + ids = np.asarray(f["PartType1"]["ParticleIDs"]) + return x % N, ids + + +def run_pm(q, psi, D_i): + Ha = lambda a: np.sqrt(OM * a ** -3 + 1 - OM) + eps = 1e-5 + Dv = pm.growth_factor([A_I, A_I + eps], OM) + dDda = (Dv[1] - Dv[0]) / eps + x = (q + D_i * psi) % N + p = A_I * dDda * A_I ** 2 * Ha(A_I) * psi + k1 = np.fft.fftfreq(N) * 2 * np.pi + kx, ky, kz = np.meshgrid(k1, k1, k1[: N // 2 + 1], indexing="ij") + k2 = kx ** 2 + ky ** 2 + kz ** 2 + k2[0, 0, 0] = 1.0 + a_grid = np.linspace(A_I, A_F, STEPS + 1) + F = pm._forces(x, N, A_I, kx, ky, kz, k2, OM) + for i in range(STEPS): + a, a_new = a_grid[i], a_grid[i + 1] + a_half = 0.5 * (a + a_new) + dtau = (a_new - a) / (a_half ** 2 * Ha(a_half)) + p += a * F * (dtau / 2) + x = (x + p / a_half * dtau) % N + F = pm._forces(x, N, a_new, kx, ky, kz, k2, OM) + p += F * (dtau / 2) * a_new + return x + + +def evolve_damp(q, psi): + x = (q + D_STEPS[0] * psi) % N + v = psi.copy() + crossed = np.zeros(len(x), bool) + e3f = None + for i in range(1, len(D_STEPS)): + dD = D_STEPS[i] - D_STEPS[i - 1] + x = (x + v * dD) % N + rho = fields.cic_deposit(x, N) + if e3f is None or i % 3 == 0: + clean = gaussian_filter(np.log1p(rho), 1.0, mode="wrap") + e3f = fields.tidal_frame(clean, smooth=SMOOTH_MPC / VOX) + hit = (~crossed) & (gather(rho, x) > RHO_C) + if hit.any(): + idx = np.clip(np.round(x[hit]).astype(int), 0, N - 1) % N + e = e3f[idx[:, 0], idx[:, 1], idx[:, 2]] + par = e * np.sum(v[hit] * e, axis=1, keepdims=True) + v[hit] = par + (1 - BETA) * (v[hit] - par) + crossed |= hit + return x + + +def spectra(da, db): + fa, fb = np.fft.rfftn(da), np.fft.rfftn(db) + k1 = np.fft.fftfreq(N) * 2 * np.pi / VOX # physical h/Mpc + kx, ky, kz = np.meshgrid(k1, k1, k1[: N // 2 + 1], indexing="ij") + k = np.sqrt(kx ** 2 + ky ** 2 + kz ** 2) + out = [] + for lo, hi in zip(KB[:-1], KB[1:]): + m = (k >= lo) & (k < hi) + paa, pbb = np.mean(np.abs(fa[m]) ** 2), np.mean(np.abs(fb[m]) ** 2) + pab = np.mean((fa[m] * np.conj(fb[m])).real) + out.append((float(pab / np.sqrt(paa * pbb)), + float(np.sqrt(paa / pbb)))) + return out + + +KB = np.geomspace(2 * np.pi / 25.0, 12.0, 7) + + +def main(): + t0 = time.time() + q, psi, ids_ic, frac_far, D_i = read_ics() + print(f"ICs: D(a_i)={D_i:.4f}, frac|psi_ic|>0.45vox={frac_far:.3f}") + xt, ids_t = read_truth() + order_t = np.argsort(ids_t) + order_i = np.argsort(ids_ic) + xt_m = xt[order_t] + + d_truth = fields.cic_deposit(xt, N) - 1.0 + results = {"frac_far": frac_far} + + print("our PM from CAMELS ICs...") + x_pm = run_pm(q, psi, D_i) + d = (x_pm[order_i] - xt_m) + d -= N * np.round(d / N) + results["pm_vs_arepo_median_mpc"] = float( + np.median(np.linalg.norm(d, axis=1)) * VOX) + results["pm_spectra"] = spectra(fields.cic_deposit(x_pm, N) - 1.0, d_truth) + print(f" matched-ID median offset: {results['pm_vs_arepo_median_mpc']:.3f} Mpc/h") + + for name, xm in [("za", (q + psi) % N), ("damp", evolve_damp(q, psi))]: + d = xm[order_i] - xt_m + d -= N * np.round(d / N) + results[f"{name}_median_mpc"] = float( + np.median(np.linalg.norm(d, axis=1)) * VOX) + results[f"{name}_spectra"] = spectra( + fields.cic_deposit(xm, N) - 1.0, d_truth) + print(f" {name}: median {results[f'{name}_median_mpc']:.3f} Mpc/h") + + (ROOT / "artifacts" / f"e12_{TAG}_results.json").write_text( + json.dumps(results, indent=1)) + kc = np.sqrt(KB[:-1] * KB[1:]) + md = [f"# E12 ({TAG}) — external validation, LCDM CV_0\n", + f"Our PM evolved from CAMELS' own 2LPT ICs (z=127→0, Ωm=0.3, " + f"256³, 25 Mpc/h box). Node-rounding sanity: fraction of IC " + f"displacements beyond 0.45 voxels = {frac_far:.3f}.\n", + f"- **Tier 1 (truth quality):** our PM vs Arepo, matched IDs: " + f"median offset {results['pm_vs_arepo_median_mpc']:.3f} Mpc/h.\n", + f"- **Tier 2 (external stress test):** ZA " + f"{results['za_median_mpc']:.3f} vs damping model " + f"{results['damp_median_mpc']:.3f} Mpc/h median vs Arepo truth " + f"(0.1 Mpc cells — 10× beyond tested resolution range).\n", + "\n| k (h/Mpc) | r our-PM | T our-PM | r ZA | r damp |", + "|---|---|---|---|---|"] + for i, k in enumerate(kc): + md.append(f"| {k:.2f} | {results['pm_spectra'][i][0]:.3f} " + f"| {results['pm_spectra'][i][1]:.3f} " + f"| {results['za_spectra'][i][0]:.3f} " + f"| {results['damp_spectra'][i][0]:.3f} |") + (ROOT / "docs" / f"E12-{TAG}-report.md").write_text("\n".join(md) + "\n") + print(f"total {time.time()-t0:.0f}s") + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e1b.py b/research/cosmic-web/scripts/run_e1b.py new file mode 100644 index 00000000000..979979ffa0d --- /dev/null +++ b/research/cosmic-web/scripts/run_e1b.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""E1b: method-NEUTRAL H1 test on gravity fields + H2 upper bound. + +E1's skeleton-vs-skeleton scoring collapsed to reference circularity. Here +the criterion needs no reference skeleton at all: at matched spine length, +which method's spine network captures more MASS (the actual particles)? + + M_mass(r0) = fraction of all 2.1M particles within r0 of the spine set. + +Configs: hessian | lift b0 | lift b1_sparse (tidal frame from the sparse +field, as in E1) | lift b1_clean (tidal frame from the clean field — an +upper bound: is tidal information useful in principle, or only too noisy +when estimated from sparse data?). + +Results -> artifacts/e1b_results.json, report -> docs/E1b-report.md. +""" + +import json +import sys +import time +from multiprocessing import Pool +from pathlib import Path + +import numpy as np +from scipy.ndimage import gaussian_filter +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +import fields, pm, spines # noqa: E402 +sys.path.insert(0, str(ROOT / "scripts")) +from run_e1 import lifted_ridgeness_weighted # noqa: E402 + +N, L, D, TRUNC = 128, 128.0, 1.0, 2.0 +N_TARGET = 2400 +R0 = 3.0 +PM = "--pm" in sys.argv # real N-body fields instead of truncated ZA +SEEDS = list(range(1, 9)) if PM else list(range(1, 13)) +LEVELS = [5000, 20000] +TAG = "pm" if PM else "sigpar" + + +def mass_coverage(spine_pts, pos, r0=R0): + if len(spine_pts) == 0: + return 0.0 + d = cKDTree(spine_pts).query(pos)[0] + return float((d < r0).mean()) + + +def band_mask(rho, pos, lo=2.0, hi=20.0): + """Particles in filament-band environments: cell density in (lo, hi), + excluding voids and cluster cores.""" + idx = (np.floor(pos).astype(int)) % N + rp = rho[idx[:, 0], idx[:, 1], idx[:, 2]] + return (rp > lo) & (rp < hi) + + +def run_seed(seed): + rng = np.random.default_rng(seed) + if PM: + pos, _, _, _, _, _, _ = pm.pm_sim(N, L, rng, n_steps=90, track=1) + rho = fields.cic_deposit(pos, N) + else: + rho, pos = fields.zeldovich_box(N, L, D, rng, trunc=TRUNC) + band = band_mask(rho, pos) + clean = gaussian_filter(np.log1p(rho), 1.0, mode="wrap") + e3_clean = fields.tidal_frame(clean) + + recs = [] + for n_gal in LEVELS: + gal_field = fields.galaxy_field(pos, N, n_gal, rng, adaptive=True) + e3_sparse = fields.tidal_frame(gal_field) + ests = { + "hessian": spines.hessian_ridgeness(gal_field, 1.5), + "lift_s3": lifted_ridgeness_weighted(gal_field, None, 0.0, 3.0), + "lift_s45": lifted_ridgeness_weighted(gal_field, None, 0.0, 4.5), + "lift_b0": lifted_ridgeness_weighted(gal_field, None, 0.0), + } + for mname, R in ests.items(): + skel = spines.extract_matched(R, N_TARGET, lo_frac=1.0) + pts = spines.skeleton_points(skel) + recs.append({"seed": seed, "n_gal": n_gal, "method": mname, + "n_spine": len(pts), + "mass_cov": mass_coverage(pts, pos), + "band_cov": mass_coverage(pts, pos[band])}) + return recs + + +def main(): + t0 = time.time() + with Pool(4 if PM else 6) as pool: + recs = [r for rs in pool.map(run_seed, SEEDS) for r in rs] + print(f"{len(SEEDS)} seeds in {time.time()-t0:.0f}s") + (ROOT / "artifacts" / f"e1b_{TAG}_results.json").write_text( + json.dumps(recs, indent=1)) + report(recs) + + +def report(recs): + from scipy.stats import wilcoxon + methods = ["hessian", "lift_s3", "lift_s45", "lift_b0"] + md = [f"# E1b-{TAG} — method-neutral H1 on gravity fields " + f"({'PM N-body' if PM else 'truncated ZA'}): mass coverage at matched spine length\n", + f"Setup: {len(SEEDS)} {'PM N-body' if PM else 'truncated-ZA'} boxes. Criterion " + f"needs no reference skeleton: fraction of all {N}³ particles " + f"within r₀={R0} vox of the spine set, spine length matched at " + f"{N_TARGET} vox. Note the spine budget covers only ~" + f"{N_TARGET * 4 * np.pi * R0 ** 2 / 3 / N ** 3 * 100:.0f}% of the " + "volume if spread randomly — differences measure how well spines " + "sit on mass.\n"] + md.append("| n_gal | method | mass coverage | band coverage (2<ρ<20) " + "| Δband vs hessian (p) | Δband vs lift_b0 (p) |") + md.append("|---|---|---|---|---|---|") + for n_gal in LEVELS: + by = {m: {r["seed"]: r for r in recs + if r["n_gal"] == n_gal and r["method"] == m} + for m in methods} + for m in methods: + c = np.array([by[m][s]["mass_cov"] for s in SEEDS]) + b = np.array([by[m][s]["band_cov"] for s in SEEDS]) + cells = [] + for refm in ("hessian", "lift_b0"): + if m == refm: + cells.append("—") + continue + d = np.array([by[m][s]["band_cov"] - by[refm][s]["band_cov"] + for s in SEEDS]) + cells.append(f"{d.mean():+.4f} ({wilcoxon(d).pvalue:.2g})") + md.append(f"| {n_gal} | {m} | {c.mean():.4f} ± {c.std():.4f} " + f"| {b.mean():.4f} ± {b.std():.4f} " + f"| {cells[0]} | {cells[1]} |") + out = ROOT / "docs" / f"E1b-{TAG}-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + print("\n".join(md[4:])) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e1b_256.py b/research/cosmic-web/scripts/run_e1b_256.py new file mode 100644 index 00000000000..1b0c5b4d9fd --- /dev/null +++ b/research/cosmic-web/scripts/run_e1b_256.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""E1b-256: resolution check of the gravity-field verdict. + +Same physical setup as E1b (L = 128 h^-1 Mpc box, same galaxy densities, +same physical filter scales and r0), but at 256^3 so the voxel is +0.5 h^-1 Mpc and filament cross-sections are resolved. If the Hessian's +mass-coverage win was an artefact of unresolved (1-voxel-wide) filaments, +it should shrink or flip here. + +Physical-scale mapping (voxels doubled): hessian sigma 3, lift cigars +(sig_par, sig_perp) = (12, 3) ["6 Mpc"] and (6, 3) ["3 Mpc"], spine budget +4800 vox, r0 = 6 vox. +""" + +import json +import sys +import time +from multiprocessing import Pool +from pathlib import Path + +import numpy as np +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +import fields, lift, spines # noqa: E402 + +N, L, D, TRUNC = 256, 128.0, 1.0, 2.0 +N_TARGET = 4800 +R0 = 6.0 +SEEDS = list(range(1, 7)) +LEVELS = [5000, 20000] + +_AXES = lift.hemisphere_axes(42) + + +def lift_ridge(field, sig_par, sig_perp): + U = lift.orientation_score(field, _AXES, sig_par, sig_perp) + return spines.lifted_ridgeness(U) + + +def coverage(spine_pts, pos, r0=R0): + if len(spine_pts) == 0: + return 0.0 + return float((cKDTree(spine_pts).query(pos)[0] < r0).mean()) + + +def run_seed(seed): + t0 = time.time() + rng = np.random.default_rng(seed) + rho, pos = fields.zeldovich_box(N, L, D, rng, trunc=TRUNC) + idx = (np.floor(pos).astype(int)) % N + rp = rho[idx[:, 0], idx[:, 1], idx[:, 2]] + band = (rp > 2) & (rp < 20) + + recs = [] + for n_gal in LEVELS: + gal_field = fields.galaxy_field(pos, N, n_gal, rng, adaptive=True) + ests = { + "hessian": spines.hessian_ridgeness(gal_field, 3.0), + "lift_3mpc": lift_ridge(gal_field, 6.0, 3.0), + "lift_6mpc": lift_ridge(gal_field, 12.0, 3.0), + } + for mname, R in ests.items(): + pts = spines.skeleton_points( + spines.extract_matched(R, N_TARGET, lo_frac=1.0)) + recs.append({"seed": seed, "n_gal": n_gal, "method": mname, + "mass_cov": coverage(pts, pos), + "band_cov": coverage(pts, pos[band])}) + print(f"seed {seed} done in {time.time()-t0:.0f}s") + return recs + + +def main(): + t0 = time.time() + with Pool(3) as pool: + recs = [r for rs in pool.map(run_seed, SEEDS) for r in rs] + print(f"{len(SEEDS)} seeds in {time.time()-t0:.0f}s") + (ROOT / "artifacts" / "e1b_256_results.json").write_text( + json.dumps(recs, indent=1)) + report(recs) + + +def report(recs): + from scipy.stats import wilcoxon + methods = ["hessian", "lift_3mpc", "lift_6mpc"] + md = ["# E1b-256 — resolution check (0.5 h⁻¹Mpc voxels, filament " + "cross-sections resolved)\n", + f"Setup: {len(SEEDS)} truncated-ZA boxes, 256³ voxels over the " + "same physical L=128 h⁻¹Mpc volume, physically matched scales " + "(r₀=3 h⁻¹Mpc, spine budget 2400 h⁻¹Mpc). Band = 2<ρ<20.\n"] + md.append("| n_gal | method | mass coverage | band coverage " + "| Δband vs hessian (p) |") + md.append("|---|---|---|---|---|") + for n_gal in LEVELS: + by = {m: {r["seed"]: r for r in recs + if r["n_gal"] == n_gal and r["method"] == m} + for m in methods} + for m in methods: + c = np.array([by[m][s]["mass_cov"] for s in SEEDS]) + b = np.array([by[m][s]["band_cov"] for s in SEEDS]) + if m == "hessian": + cell = "—" + else: + d = np.array([by[m][s]["band_cov"] - by["hessian"][s]["band_cov"] + for s in SEEDS]) + cell = f"{d.mean():+.4f} ({wilcoxon(d).pvalue:.2g})" + md.append(f"| {n_gal} | {m} | {c.mean():.4f} ± {c.std():.4f} " + f"| {b.mean():.4f} ± {b.std():.4f} | {cell} |") + out = ROOT / "docs" / "E1b-256-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + print("\n".join(md[3:])) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e2.py b/research/cosmic-web/scripts/run_e2.py new file mode 100644 index 00000000000..9b724005593 --- /dev/null +++ b/research/cosmic-web/scripts/run_e2.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""E2 (H4-lite): do trajectories near filaments follow the lifted geometry? + +The full H4 claim is that matter transport follows sub-Riemannian geodesics +of a metric that is cheap ALONG the local filament direction. Before any +geodesic machinery, two falsifiable dynamical prerequisites are measured on +PM N-body trajectories (method-neutral ground truth — no skeleton scoring): + +P1 (flow anisotropy): near filament spines, velocity direction aligns with + the tidal eigenvector e3 (the filament axis): <|v_hat . e3|> > 0.5 null, + decaying with distance from the spine. + +P2 (ballistic-correction direction): over a in [0.5, 1.0], the deviation of + the true path from its straight chord is preferentially ALONG e3. + Parallel fraction f = <(dev_hat . e3)^2> > 1/3 (isotropic null) near + spines. If deviations are instead perpendicular (pure infall), an + SR geodesic that bends along the filament CANNOT beat the chord, and H4 + fails at the premise level. + +Results -> artifacts/e2_results.json, report -> docs/E2-report.md. +""" + +import json +import sys +import time +from multiprocessing import Pool +from pathlib import Path + +import numpy as np +from scipy.ndimage import gaussian_filter +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +import fields, lift, pm, spines # noqa: E402 + +N, L = 128, 128.0 +N_TARGET = 2400 +SEEDS = [1, 2, 3, 4] +D_BINS = [(0, 2), (2, 4), (4, 8), (8, 16), (16, 64)] + +_AXES = lift.hemisphere_axes(42) + + +def lifted_spines(field): + U = lift.orientation_score(field, _AXES, 6.0, 1.5) + return spines.skeleton_points( + spines.extract_matched(spines.lifted_ridgeness(U), N_TARGET, + lo_frac=1.0)) + + +def e3_at(e3, pts): + idx = np.clip(np.round(pts).astype(int), 0, N - 1) % N + return e3[idx[:, 0], idx[:, 1], idx[:, 2]] + + +def run_seed(seed): + t0 = time.time() + rng = np.random.default_rng(seed) + x, v, tidx, ta, tpos, _, _ = pm.pm_sim(N, L, rng, n_steps=90, track=50000) + rho = fields.cic_deposit(x, N) + clean = gaussian_filter(np.log1p(rho), 1.0, mode="wrap") + e3 = fields.tidal_frame(clean) + sp = lifted_spines(clean) + tree = cKDTree(sp) + + # tracked-particle quantities + p_end = tpos[-1].astype(float) + d_spine = tree.query(p_end)[0] + + vv = v[tidx] + vhat = vv / np.maximum(np.linalg.norm(vv, axis=1, keepdims=True), 1e-12) + align_v = np.abs(np.sum(vhat * e3_at(e3, p_end), axis=1)) + + # chord deviation over the tracked window (periodic-aware) + p_start = tpos[0].astype(float) + disp = p_end - p_start + disp -= N * np.round(disp / N) + mid_true = tpos[len(ta) // 2].astype(float) + frac = (ta[len(ta) // 2] - ta[0]) / (ta[-1] - ta[0]) + mid_chord = p_start + frac * disp + dev = mid_true - mid_chord + dev -= N * np.round(dev / N) + dev_norm = np.linalg.norm(dev, axis=1) + ok = dev_norm > 0.1 # ignore near-static particles + devhat = dev[ok] / dev_norm[ok, None] + par_frac = np.sum(devhat * e3_at(e3, mid_true[ok]), axis=1) ** 2 + + rec = {"seed": seed, "runtime_s": round(time.time() - t0)} + for lo, hi in D_BINS: + m_all = (d_spine >= lo) & (d_spine < hi) + m_dev = m_all[ok] + rec[f"align_v_{lo}_{hi}"] = float(align_v[m_all].mean()) + rec[f"par_frac_{lo}_{hi}"] = float(par_frac[m_dev].mean()) + rec[f"dev_vox_{lo}_{hi}"] = float(dev_norm[m_all].mean()) + rec[f"n_{lo}_{hi}"] = int(m_all.sum()) + return rec + + +def main(): + t0 = time.time() + with Pool(4) as pool: + recs = pool.map(run_seed, SEEDS) + print(f"{len(SEEDS)} PM sims in {time.time()-t0:.0f}s") + (ROOT / "artifacts" / "e2_results.json").write_text(json.dumps(recs, indent=1)) + report(recs) + + +def report(recs): + md = ["# E2 (H4-lite) — dynamics near filaments vs the lifted-geometry " + "premise\n", + f"Setup: {len(SEEDS)} PM N-body runs ({N}³ grid & particles, EdS, " + "a: 0.1→1.0, 90 steps), 50k tracked particles per run over " + "a ∈ [0.5, 1.0]. Spines: SE(3) lift on the final density; e3 = " + "tidal minor eigenvector (filament axis). P1: mean |v̂·e3| at a=1 " + "(null 0.5). P2: squared projection of the mid-path deviation " + "from the straight chord onto e3 (null 1/3). dev = mean deviation " + "magnitude (voxels = h⁻¹Mpc).\n"] + md.append("| d to spine (vox) | n/seed | P1 ⟨\\|v̂·e3\\|⟩ | P2 ⟨(d̂ev·e3)²⟩ " + "| ⟨\\|dev\\|⟩ vox |") + md.append("|---|---|---|---|---|") + for lo, hi in D_BINS: + a = [r[f"align_v_{lo}_{hi}"] for r in recs] + p = [r[f"par_frac_{lo}_{hi}"] for r in recs] + d = [r[f"dev_vox_{lo}_{hi}"] for r in recs] + n = int(np.mean([r[f"n_{lo}_{hi}"] for r in recs])) + md.append(f"| {lo}–{hi} | {n} | {np.mean(a):.3f} ± {np.std(a):.3f} " + f"| {np.mean(p):.3f} ± {np.std(p):.3f} " + f"| {np.mean(d):.2f} |") + md.append("\nNulls: P1 = 0.5, P2 = 1/3 ≈ 0.333 (isotropic). Values are " + "across-seed mean ± sd; per-seed n is large so per-seed " + "standard errors are negligible — seed scatter is the " + "relevant uncertainty.\n") + out = ROOT / "docs" / "E2-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + print("\n".join(md[-10:])) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e3.py b/research/cosmic-web/scripts/run_e3.py new file mode 100644 index 00000000000..9faaa56087c --- /dev/null +++ b/research/cosmic-web/scripts/run_e3.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +"""E3 (first pass): real data — BOSS CMASS galaxies x Planck lensing / tSZ. + +Pipeline +-------- +1. CMASS-North galaxies, redshift shell z in [0.45, 0.55] -> comoving + Cartesian (flat LCDM, Om=0.31, units Mpc/h), voxelized into a 512 Mpc/h + cube at 4 Mpc/h voxels (128^3), CIC + log + adaptive smoothing. +2. Spines at matched length with BOTH methods (Hessian, SE(3) lift). +3. M3-real: alignment of spine tangents with the tidal eigenframe of the + observed field (isotropic null 0.5). +4. Mass/gas cross-signal: stack the Planck 2018 lensing convergence map and + the Planck MILCA Compton-y map at the sky positions of the spine voxels; + null distribution from RA-rotated controls constrained to the survey + footprint; SNR = (stack - mean_null) / std_null. + +First-pass honesty: no systematics weights, single redshift shell, +footprint approximated by galaxy-occupied HEALPix cells, projection through +a ~250 Mpc/h shell dilutes the signal. A detection is a bonus; the primary +readout is M3-real and the lift-vs-Hessian comparison at equal length. + +Report -> docs/E3-report.md. +""" + +import json +import sys +import tarfile +import time +from pathlib import Path + +import numpy as np +from scipy.integrate import quad +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +DATA = ROOT / "data" +sys.path.insert(0, str(ROOT / "src")) + +import fields, lift, spines # noqa: E402 + +OM, H0 = 0.31, 100.0 # Mpc/h units +Z_RANGE = (0.45, 0.55) +BOX = 512.0 # Mpc/h +NGRID = 128 # -> 4 Mpc/h voxels +N_TARGET = 2400 +N_CONTROLS = 200 +_AXES = lift.hemisphere_axes(42) + + +def comoving_distance(z): + integrand = lambda zz: 1.0 / np.sqrt(OM * (1 + zz) ** 3 + 1 - OM) + return 299792.458 / H0 * quad(integrand, 0, z)[0] + + +def load_galaxies(): + from astropy.io import fits + with fits.open(DATA / "galaxy_DR12v5_CMASS_North.fits.gz") as f: + d = f[1].data + ra, dec, z = d["RA"], d["DEC"], d["Z"] + m = (z > Z_RANGE[0]) & (z < Z_RANGE[1]) + ra, dec, z = ra[m], dec[m], z[m] + zg = np.linspace(Z_RANGE[0] - 0.01, Z_RANGE[1] + 0.01, 60) + chi_i = np.array([comoving_distance(zz) for zz in zg]) + chi = np.interp(z, zg, chi_i) + th, ph = np.radians(90 - dec), np.radians(ra) + xyz = np.stack([chi * np.sin(th) * np.cos(ph), + chi * np.sin(th) * np.sin(ph), + chi * np.cos(th)], axis=1) + return ra, dec, z, xyz + + +def tile_origins(xyz, min_gal=5000): + """Non-overlapping BOX^3 tiles covering the sample; keep tiles with at + least min_gal galaxies.""" + lo = xyz.min(axis=0) + idx = np.floor((xyz - lo) / BOX).astype(int) + origins = [] + for key in {tuple(i) for i in idx}: + origin = lo + np.array(key) * BOX + n = int(np.all((xyz >= origin) & (xyz < origin + BOX), axis=1).sum()) + if n >= min_gal: + origins.append((origin, n)) + return origins + + +def build_field(xyz, origin): + """BOX^3 cube at origin -> CIC density field.""" + inbox = np.all((xyz >= origin) & (xyz < origin + BOX), axis=1) + pos_vox = (xyz[inbox] - origin) / (BOX / NGRID) + n_gal = int(inbox.sum()) + field = fields.cic_deposit(pos_vox, NGRID) + from scipy.ndimage import gaussian_filter + sig = max(1.0, 0.35 * (NGRID ** 3 / n_gal) ** (1 / 3)) + field = gaussian_filter(np.log1p(field), sig, mode="nearest") + return field, n_gal + + +def spine_sets(field): + out = {} + out["hessian"] = spines.skeleton_points(spines.extract_matched( + spines.hessian_ridgeness(field, 1.5), N_TARGET, lo_frac=1.0)) + U = lift.orientation_score(field, _AXES, 6.0, 1.5) + out["se3_lift"] = spines.skeleton_points(spines.extract_matched( + spines.lifted_ridgeness(U), N_TARGET, lo_frac=1.0)) + return out + + +def spine_tangent_alignment(pts, field): + e3 = fields.tidal_frame(field) + tree = cKDTree(pts) + tans, keep = [], [] + for i, p in enumerate(pts): + nb = pts[tree.query_ball_point(p, 3.0)] + if len(nb) < 3: + continue + c = nb - nb.mean(0) + tans.append(np.linalg.svd(c, full_matrices=False)[2][0]) + keep.append(i) + tans = np.array(tans) + idx = np.clip(np.round(pts[keep]).astype(int), 0, field.shape[0] - 1) + e = e3[idx[:, 0], idx[:, 1], idx[:, 2]] + return float(np.mean(np.abs(np.sum(tans * e, axis=1)))) + + +def vox_to_radec(pts, origin): + xyz = pts * (BOX / NGRID) + origin + r = np.linalg.norm(xyz, axis=1) + dec = 90 - np.degrees(np.arccos(xyz[:, 2] / r)) + ra = np.degrees(np.arctan2(xyz[:, 1], xyz[:, 0])) % 360 + return ra, dec + + +def radec_to_gal_vec(ra, dec): + from astropy.coordinates import SkyCoord + import astropy.units as u + g = SkyCoord(ra=ra * u.deg, dec=dec * u.deg, frame="icrs").galactic + return np.radians(90 - g.b.deg), np.radians(g.l.deg) + + +def load_maps(): + import healpy as hp + maps = {} + # lensing: klm alms -> smoothed kappa map + tgz = DATA / "COM_Lensing_4096_R3.00.tgz" + lens_dir = DATA / "lensing" + if not lens_dir.exists(): + with tarfile.open(tgz) as t: + t.extractall(DATA) + klm_file = next(DATA.glob("**/dat_klm.fits")) + klm = hp.read_alm(str(klm_file)) + kappa = hp.alm2map(klm, nside=1024, lmax=2048) + maps["kappa"] = hp.smoothing(kappa, fwhm=np.radians(10 / 60), lmax=2048) + # Compton-y (MILCA full) from the YSZ component-map tarball + ycands = list(DATA.glob("**/*milca*ymap*.fits")) + \ + list(DATA.glob("**/*ymaps*milca*.fits")) + if not ycands: + ytgz = next(DATA.glob("COM_CompMap_YSZ_*.tgz")) + with tarfile.open(ytgz) as t: + t.extractall(DATA) + ycands = list(DATA.glob("**/*milca*.fits")) + maps["y"] = hp.read_map(str(ycands[0]), field=0) + return maps + + +def footprint_mask(ra, dec, nside=64): + import healpy as hp + th, ph = radec_to_gal_vec(ra, dec) + pix = hp.ang2pix(nside, th, ph) + mask = np.zeros(hp.nside2npix(nside), bool) + mask[np.unique(pix)] = True + return mask + + +def stack(map_, ra, dec, footprint, rng, n_controls=N_CONTROLS): + import healpy as hp + nside = hp.get_nside(map_) + + def sample(ra_, dec_): + th, ph = radec_to_gal_vec(ra_, dec_) + ok = footprint[hp.ang2pix(64, th, ph)] + if ok.mean() < 0.9: + return None + return float(np.mean(hp.get_interp_val(map_, th[ok], ph[ok]))) + + signal = sample(ra, dec) + nulls = [] + tries = 0 + while len(nulls) < n_controls and tries < n_controls * 10: + tries += 1 + v = sample((ra + rng.uniform(5, 355)) % 360, + dec if rng.random() < 0.5 else + dec.mean() * 2 - dec) # RA rotation +/- DEC reflection + if v is not None: + nulls.append(v) + nulls = np.array(nulls) + return {"stack": signal, "null_mean": float(nulls.mean()), + "null_std": float(nulls.std()), "n_controls": len(nulls), + "snr": float((signal - nulls.mean()) / nulls.std())} + + +def main(): + t0 = time.time() + print("loading galaxies...") + ra_g, dec_g, z_g, xyz = load_galaxies() + print(f" {len(ra_g)} CMASS-N galaxies in z {Z_RANGE}") + + tiles = tile_origins(xyz) + print(f" {len(tiles)} tiles with >= 5000 galaxies " + f"({sum(n for _, n in tiles)} galaxies used)") + + all_sky = {"hessian": [[], []], "se3_lift": [[], []]} + aligns = {"hessian": [], "se3_lift": []} + for i, (origin, n) in enumerate(tiles): + field, n_gal = build_field(xyz, origin) + sets = spine_sets(field) + for mname, pts in sets.items(): + aligns[mname].append(spine_tangent_alignment(pts, field)) + ra_s, dec_s = vox_to_radec(pts, origin) + all_sky[mname][0].append(ra_s) + all_sky[mname][1].append(dec_s) + print(f" tile {i+1}/{len(tiles)} ({n} gal) done") + + align = {m: float(np.mean(v)) for m, v in aligns.items()} + align_sd = {m: float(np.std(v)) for m, v in aligns.items()} + print(" alignment (mean over tiles):", align) + + print("loading Planck maps...") + maps = load_maps() + fp = footprint_mask(ra_g, dec_g) + rng = np.random.default_rng(1) + + results = {"n_gal_shell": len(ra_g), "n_tiles": len(tiles), + "n_gal_used": int(sum(n for _, n in tiles)), + "alignment_e3": align, "alignment_e3_sd": align_sd, + "stacks": {}} + for mname in all_sky: + ra_s = np.concatenate(all_sky[mname][0]) + dec_s = np.concatenate(all_sky[mname][1]) + for mapname in ("kappa", "y"): + key = f"{mname}_{mapname}" + results["stacks"][key] = stack(maps[mapname], ra_s, dec_s, fp, rng) + print(f" {key}: SNR = {results['stacks'][key]['snr']:.2f} " + f"({len(ra_s)} spine points)") + + (ROOT / "artifacts" / "e3_results.json").write_text( + json.dumps(results, indent=1)) + report(results) + print(f"total {time.time()-t0:.0f}s") + + +def report(res): + md = ["# E3 — BOSS CMASS × Planck lensing/tSZ (tiled shell)\n", + f"Setup: CMASS-North, z ∈ [{Z_RANGE[0]}, {Z_RANGE[1]}] " + f"({res['n_gal_shell']} galaxies; {res['n_gal_used']} used across " + f"{res['n_tiles']} tiles of {BOX:.0f} Mpc/h at " + f"{BOX/NGRID:.0f} Mpc/h voxels). " + f"Spines at matched length ({N_TARGET} vox per tile), both methods. " + "Stacks: mean map value at spine sky positions vs " + f"{N_CONTROLS} footprint-constrained RA-rotated/DEC-reflected " + "controls. First pass: no systematics weights, single shell, " + "projected through the full shell depth.\n"] + md.append("## M3 on real data (spine–tidal-frame alignment, null 0.5)\n") + for m, a in res["alignment_e3"].items(): + md.append(f"- **{m}**: {a:.3f} ± {res['alignment_e3_sd'][m]:.3f} " + f"(mean ± sd over tiles)") + md.append("\n## Cross-signal stacks\n") + md.append("| spine set | map | stack | null mean ± std | SNR |") + md.append("|---|---|---|---|---|") + for key, s in res["stacks"].items(): + mname, mapname = key.rsplit("_", 1) + md.append(f"| {mname} | {mapname} | {s['stack']:.3e} " + f"| {s['null_mean']:.3e} ± {s['null_std']:.3e} " + f"| **{s['snr']:.2f}** |") + md.append("\nReading: positive SNR means the spine network sits on " + "more lensing convergence / hot gas than footprint-matched " + "random placements. H3 asks whether the lift's spines carry " + "at least the Hessian's signal at equal length.\n") + out = ROOT / "docs" / "E3-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e3b.py b/research/cosmic-web/scripts/run_e3b.py new file mode 100644 index 00000000000..9a7cddfcb9f --- /dev/null +++ b/research/cosmic-web/scripts/run_e3b.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +"""E3b: second-pass tests of the E3 Compton-y detection. + +H3a (foreground robustness): re-test the stack against nulls that PRESERVE + the galactic-latitude distribution — control points are drawn uniformly + inside the survey footprint but rejection-matched to the spine points' + b-histogram (2 deg bins). A latitude-dependent y foreground can no + longer inflate the SNR. +H3b (WHIM component): mask all sky within theta_mask of ANY shell galaxy + (the tracer halos) and stack only surviving spine points; nulls masked + identically. A surviving excess is gas between halos. +H3c (extended profile): mean y vs angular distance from the spine network + (rings at random azimuth around each spine point), signal minus + b-matched null, per bin. Filament signature = extended profile. + +Uses the spine sets regenerated exactly as in run_e3 (same tiles, same +methods). Report -> docs/E3b-report.md, figure -> docs/figures/e3b_profile.png. +""" + +import json +import sys +import time +from pathlib import Path + +import numpy as np +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) + +import run_e3 as e3 # noqa: E402 (reuse loaders, tiling, spine sets) + +N_NULLS = 200 +THETA_MASK_ARCMIN = 7.0 +PROFILE_BINS_ARCMIN = [0, 5, 10, 20, 40, 80] +RNG = np.random.default_rng(11) + + +def to_gal(ra, dec): + from astropy.coordinates import SkyCoord + import astropy.units as u + g = SkyCoord(ra=ra * u.deg, dec=dec * u.deg, frame="icrs").galactic + return g.l.deg, g.b.deg + + +def gal_to_vec(l, b): + th, ph = np.radians(90 - b), np.radians(l) + return np.stack([np.sin(th) * np.cos(ph), np.sin(th) * np.sin(ph), + np.cos(th)], axis=1) + + +def footprint_pool(ra_g, dec_g, n_pool=400000, nside=64): + """Uniform random sky points inside the galaxy-occupied footprint.""" + import healpy as hp + lg, bg = to_gal(ra_g, dec_g) + occ = np.unique(hp.ang2pix(nside, np.radians(90 - bg), np.radians(lg))) + pix = occ[RNG.integers(0, len(occ), n_pool)] + th, ph = hp.pix2ang(nside, pix) + # jitter within ~pixel scale + scale = np.sqrt(hp.nside2pixarea(nside)) + th = np.clip(th + RNG.normal(0, scale / 2, n_pool), 1e-4, np.pi - 1e-4) + ph = (ph + RNG.normal(0, scale / 2, n_pool)) % (2 * np.pi) + keep = np.isin(hp.ang2pix(nside, th, ph), occ) + return 90 - np.degrees(th[keep]), np.degrees(ph[keep]) # b, l + + +def b_matched_controls(b_sig, pool_b, pool_l, n_nulls, bin_deg=2.0): + """Draw n_nulls control sets from the pool, matching the signal's + b-histogram.""" + edges = np.arange(pool_b.min() - bin_deg, pool_b.max() + 2 * bin_deg, bin_deg) + sig_counts, _ = np.histogram(b_sig, edges) + pool_bin = np.digitize(pool_b, edges) - 1 + by_bin = {i: np.where(pool_bin == i)[0] for i in range(len(edges) - 1) + if (pool_bin == i).any()} + controls = [] + for _ in range(n_nulls): + idx = [] + for i, c in enumerate(sig_counts): + if c == 0 or i not in by_bin: + continue + idx.append(RNG.choice(by_bin[i], size=c, replace=True)) + idx = np.concatenate(idx) + controls.append((pool_b[idx], pool_l[idx])) + return controls + + +def mean_y(ymap, b, l): + import healpy as hp + return float(np.mean(hp.get_interp_val( + ymap, np.radians(90 - b), np.radians(l)))) + + +def ring_points(b, l, theta_deg): + """One random-azimuth point at angular distance theta from each (b,l).""" + v = gal_to_vec(l, b) + # random orthogonal direction + r = RNG.normal(size=v.shape) + r -= v * np.sum(r * v, axis=1, keepdims=True) + r /= np.linalg.norm(r, axis=1, keepdims=True) + w = v * np.cos(np.radians(theta_deg)) + r * np.sin(np.radians(theta_deg)) + b2 = 90 - np.degrees(np.arccos(np.clip(w[:, 2], -1, 1))) + l2 = np.degrees(np.arctan2(w[:, 1], w[:, 0])) % 360 + return b2, l2 + + +def main(): + t0 = time.time() + print("galaxies + tiles + spines (as in E3)...") + ra_g, dec_g, z_g, xyz = e3.load_galaxies() + tiles = e3.tile_origins(xyz) + sky = {"hessian": [[], []], "se3_lift": [[], []]} + for origin, n in tiles: + field, _ = e3.build_field(xyz, origin) + for mname, pts in e3.spine_sets(field).items(): + ra_s, dec_s = e3.vox_to_radec(pts, origin) + sky[mname][0].append(ra_s) + sky[mname][1].append(dec_s) + print(f" {len(tiles)} tiles done") + + print("maps + pools...") + maps = e3.load_maps() + ymap = maps["y"] + pool_b, pool_l = footprint_pool(ra_g, dec_g) + lg_gal, bg_gal = to_gal(ra_g, dec_g) + gal_tree = cKDTree(gal_to_vec(lg_gal, bg_gal)) + chord_mask = 2 * np.sin(np.radians(THETA_MASK_ARCMIN / 60) / 2) + + results = {} + for mname in sky: + ra_s = np.concatenate(sky[mname][0]) + dec_s = np.concatenate(sky[mname][1]) + l_s, b_s = to_gal(ra_s, dec_s) + controls = b_matched_controls(b_s, pool_b, pool_l, N_NULLS) + + # H3a: b-matched stack + sig = mean_y(ymap, b_s, l_s) + nulls = np.array([mean_y(ymap, cb, cl) for cb, cl in controls]) + h3a = {"snr": float((sig - nulls.mean()) / nulls.std()), + "stack": sig, "null_mean": float(nulls.mean()), + "null_std": float(nulls.std())} + + # H3b: tracer-halo masking (same mask on signal and nulls) + def unmasked(b_, l_): + d, _ = gal_tree.query(gal_to_vec(l_, b_)) + return d > chord_mask + keep = unmasked(b_s, l_s) + sig_m = mean_y(ymap, b_s[keep], l_s[keep]) + nulls_m = [] + for cb, cl in controls: + k = unmasked(cb, cl) + nulls_m.append(mean_y(ymap, cb[k], cl[k])) + nulls_m = np.array(nulls_m) + h3b = {"snr": float((sig_m - nulls_m.mean()) / nulls_m.std()), + "kept_frac": float(keep.mean()), + "stack": sig_m, "null_mean": float(nulls_m.mean()), + "null_std": float(nulls_m.std())} + + # H3c: radial profile (signal - null, per ring) + prof = [] + for i in range(len(PROFILE_BINS_ARCMIN) - 1): + theta = 0.5 * (PROFILE_BINS_ARCMIN[i] + PROFILE_BINS_ARCMIN[i + 1]) / 60 + if theta == 0: + theta = PROFILE_BINS_ARCMIN[1] / 2 / 60 + bs, ls = (b_s, l_s) if i == 0 else ring_points(b_s, l_s, theta) + v_sig = mean_y(ymap, bs, ls) + v_nul = [] + for cb, cl in controls[:60]: + cbb, cll = (cb, cl) if i == 0 else ring_points(cb, cl, theta) + v_nul.append(mean_y(ymap, cbb, cll)) + v_nul = np.array(v_nul) + prof.append({"theta_arcmin": theta * 60, + "excess": float(v_sig - v_nul.mean()), + "err": float(v_nul.std()), + "snr": float((v_sig - v_nul.mean()) / v_nul.std())}) + results[mname] = {"H3a": h3a, "H3b": h3b, "profile": prof, + "n_points": len(b_s)} + print(f" {mname}: H3a SNR={h3a['snr']:.2f} " + f"H3b SNR={h3b['snr']:.2f} (kept {h3b['kept_frac']:.0%})") + + (ROOT / "artifacts" / "e3b_results.json").write_text( + json.dumps(results, indent=1)) + report(results) + profile_figure(results) + print(f"total {time.time()-t0:.0f}s") + + +def profile_figure(results): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + fig, ax = plt.subplots(figsize=(7.5, 4.6)) + for mname, color in [("hessian", "tab:orange"), ("se3_lift", "tab:blue")]: + p = results[mname]["profile"] + th = [q["theta_arcmin"] for q in p] + ax.errorbar(th, [q["excess"] for q in p], yerr=[q["err"] for q in p], + marker="o", color=color, label=mname, capsize=3) + ax.axhline(0, color="k", lw=0.8) + ax.set_xlabel("angular distance from spine network (arcmin)") + ax.set_ylabel("Compton-y excess over b-matched null") + ax.set_title("E3b: radial y profile around spine networks " + "(errors = null scatter)") + ax.legend() + fig.tight_layout() + fig.savefig(ROOT / "docs" / "figures" / "e3b_profile.png", dpi=130) + plt.close(fig) + + +def report(res): + md = ["# E3b — second-pass tests of the Compton-y detection\n", + f"Nulls: {N_NULLS} control sets drawn inside the survey footprint " + "and rejection-matched to the spine points' galactic-latitude " + "histogram (2° bins) — a latitude-dependent foreground cannot " + f"inflate these. Tracer mask: {THETA_MASK_ARCMIN}′ around every " + "shell galaxy, applied identically to signal and nulls.\n"] + md.append("| spine set | H3a: b-matched SNR | H3b: halo-masked SNR " + "(kept) | n spine points |") + md.append("|---|---|---|---|") + for m, r in res.items(): + md.append(f"| {m} | **{r['H3a']['snr']:.2f}** " + f"| **{r['H3b']['snr']:.2f}** ({r['H3b']['kept_frac']:.0%}) " + f"| {r['n_points']} |") + md.append("\n![Radial Compton-y excess profile around spine networks, " + "signal minus latitude-matched null vs angular distance, " + "both methods](figures/e3b_profile.png)\n") + md.append("Reading: H3a tests foreground robustness; H3b isolates gas " + "away from the tracer halos (WHIM candidate); the profile " + "shape (H3c) separates extended filament gas from " + "point-source-like halo gas.\n") + out = ROOT / "docs" / "E3b-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e3c.py b/research/cosmic-web/scripts/run_e3c.py new file mode 100644 index 00000000000..0c843d589f6 --- /dev/null +++ b/research/cosmic-web/scripts/run_e3c.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""E3c: the WHIM question at ACT depth. + +Repeat the E3b battery (latitude-matched nulls, tracer-halo masking, radial +profile) on the ACT DR6 + Planck ILC Compton-y map (CAR pixelization, 1.6' +beam) over the ACT x BOSS overlap. The decisive change vs Planck-only: +halo gas can be masked at ~3' instead of 7', and the profile resolves +scales an order of magnitude finer — if between-halo gas exists along the +spine networks at detectable levels, this is where it shows. + +Report -> docs/E3c-report.md, figure -> docs/figures/e3c_profile.png. +""" + +import json +import sys +import time +from pathlib import Path + +import numpy as np +from scipy.ndimage import map_coordinates +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) + +import run_e3 as e3 # noqa: E402 +from run_e3b import (b_matched_controls, footprint_pool, gal_to_vec, # noqa: E402 + ring_points, to_gal) + +N_NULLS = 200 +MASK_ARCMIN = [3.0, 7.0] +PROFILE_BINS_ARCMIN = [0, 2, 4, 8, 16, 32] +RNG = np.random.default_rng(13) + + +class CarMap: + """Minimal CAR-map sampler (astropy WCS + bilinear interpolation).""" + + def __init__(self, path): + from astropy.io import fits + from astropy.wcs import WCS + hdu = fits.open(path)[0] + self.data = np.asarray(hdu.data, dtype=np.float32) + if self.data.ndim > 2: + self.data = self.data[0] + self.wcs = WCS(hdu.header).celestial + self.ny, self.nx = self.data.shape + + def sample(self, ra, dec): + """Bilinear sample; NaN outside coverage (zeros or out of bounds).""" + x, y = self.wcs.world_to_pixel_values(ra, dec) + inside = (x > 0) & (x < self.nx - 1) & (y > 0) & (y < self.ny - 1) + vals = np.full(len(np.atleast_1d(x)), np.nan) + if inside.any(): + v = map_coordinates(self.data, + [np.atleast_1d(y)[inside], + np.atleast_1d(x)[inside]], order=1) + v[v == 0.0] = np.nan # unobserved CAR pixels are 0 + vals[inside] = v + return vals + + def covered(self, ra, dec): + return ~np.isnan(self.sample(ra, dec)) + + +def gal_to_radec(b, l): + from astropy.coordinates import SkyCoord + import astropy.units as u + c = SkyCoord(l=l * u.deg, b=b * u.deg, frame="galactic").icrs + return c.ra.deg, c.dec.deg + + +def mean_y_act(car, b, l): + ra, dec = gal_to_radec(b, l) + v = car.sample(ra, dec) + return float(np.nanmean(v)), float(np.mean(~np.isnan(v))) + + +def main(): + t0 = time.time() + print("galaxies + tiles + spines (as in E3)...") + ra_g, dec_g, z_g, xyz = e3.load_galaxies() + tiles = e3.tile_origins(xyz) + sky = {"hessian": [[], []], "se3_lift": [[], []]} + for origin, n in tiles: + field, _ = e3.build_field(xyz, origin) + for mname, pts in e3.spine_sets(field).items(): + ra_s, dec_s = e3.vox_to_radec(pts, origin) + sky[mname][0].append(ra_s) + sky[mname][1].append(dec_s) + print(f" {len(tiles)} tiles done") + + print("loading ACT DR6 y-map...") + car = CarMap(ROOT / "data" / "ilc_actplanck_ymap.fits") + print(f" map {car.ny} x {car.nx}") + + pool_b, pool_l = footprint_pool(ra_g, dec_g) + # restrict control pool to ACT coverage + pra, pdec = gal_to_radec(pool_b, pool_l) + pk = car.covered(pra, pdec) + pool_b, pool_l = pool_b[pk], pool_l[pk] + print(f" control pool in ACT coverage: {pk.mean():.0%}") + + lg_gal, bg_gal = to_gal(ra_g, dec_g) + gal_tree = cKDTree(gal_to_vec(lg_gal, bg_gal)) + + results = {} + for mname in sky: + ra_s = np.concatenate(sky[mname][0]) + dec_s = np.concatenate(sky[mname][1]) + l_s, b_s = to_gal(ra_s, dec_s) + k = car.covered(ra_s, dec_s) + l_s, b_s = l_s[k], b_s[k] + print(f"{mname}: {k.mean():.0%} of spine points in ACT coverage " + f"({k.sum()} used)") + controls = b_matched_controls(b_s, pool_b, pool_l, N_NULLS) + + sig, _ = mean_y_act(car, b_s, l_s) + nulls = np.array([mean_y_act(car, cb, cl)[0] for cb, cl in controls]) + rec = {"n_points": int(k.sum()), + "H3a": {"snr": float((sig - nulls.mean()) / nulls.std()), + "stack": sig, "null_mean": float(nulls.mean()), + "null_std": float(nulls.std())}} + + for mask_am in MASK_ARCMIN: + chord = 2 * np.sin(np.radians(mask_am / 60) / 2) + + def unmasked(b_, l_): + d, _ = gal_tree.query(gal_to_vec(l_, b_)) + return d > chord + keep = unmasked(b_s, l_s) + sig_m, _ = mean_y_act(car, b_s[keep], l_s[keep]) + nulls_m = [] + for cb, cl in controls: + km = unmasked(cb, cl) + nulls_m.append(mean_y_act(car, cb[km], cl[km])[0]) + nulls_m = np.array(nulls_m) + rec[f"H3b_{mask_am:g}am"] = { + "snr": float((sig_m - nulls_m.mean()) / nulls_m.std()), + "kept_frac": float(keep.mean())} + print(f" mask {mask_am:g}': SNR = {rec[f'H3b_{mask_am:g}am']['snr']:.2f} " + f"(kept {keep.mean():.0%})") + + prof = [] + for i in range(len(PROFILE_BINS_ARCMIN) - 1): + theta = 0.5 * (PROFILE_BINS_ARCMIN[i] + PROFILE_BINS_ARCMIN[i + 1]) / 60 + if i == 0: + theta = PROFILE_BINS_ARCMIN[1] / 2 / 60 + bs, ls = (b_s, l_s) if i == 0 else ring_points(b_s, l_s, theta) + v_sig, _ = mean_y_act(car, bs, ls) + v_nul = [] + for cb, cl in controls[:60]: + cbb, cll = (cb, cl) if i == 0 else ring_points(cb, cl, theta) + v_nul.append(mean_y_act(car, cbb, cll)[0]) + v_nul = np.array(v_nul) + prof.append({"theta_arcmin": theta * 60, + "excess": float(v_sig - v_nul.mean()), + "err": float(v_nul.std()), + "snr": float((v_sig - v_nul.mean()) / v_nul.std())}) + rec["profile"] = prof + results[mname] = rec + print(f" H3a SNR = {rec['H3a']['snr']:.2f}") + + (ROOT / "artifacts" / "e3c_results.json").write_text( + json.dumps(results, indent=1)) + report(results) + figure(results) + print(f"total {time.time()-t0:.0f}s") + + +def figure(results): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + fig, ax = plt.subplots(figsize=(7.5, 4.6)) + for mname, color in [("hessian", "tab:orange"), ("se3_lift", "tab:blue")]: + p = results[mname]["profile"] + ax.errorbar([q["theta_arcmin"] for q in p], + [q["excess"] for q in p], yerr=[q["err"] for q in p], + marker="o", color=color, label=mname, capsize=3) + ax.axhline(0, color="k", lw=0.8) + ax.set_xlabel("angular distance from spine network (arcmin)") + ax.set_ylabel("Compton-y excess over b-matched null") + ax.set_title("E3c: radial y profile at ACT DR6 resolution (1.6′ beam)") + ax.legend() + fig.tight_layout() + fig.savefig(ROOT / "docs" / "figures" / "e3c_profile.png", dpi=130) + plt.close(fig) + + +def report(res): + md = ["# E3c — the WHIM question at ACT DR6 depth\n", + "ACT DR6 + Planck ILC Compton-y map (1.6′ beam, CAR), BOSS CMASS " + "spine networks restricted to the ACT footprint. Same battery as " + f"E3b: {N_NULLS} latitude-matched nulls; tracer-halo masks at " + f"{MASK_ARCMIN} arcmin; radial profile bins " + f"{PROFILE_BINS_ARCMIN} arcmin.\n"] + md.append("| spine set | n pts in ACT | H3a SNR | H3b 3′ mask (kept) " + "| H3b 7′ mask (kept) |") + md.append("|---|---|---|---|---|") + for m, r in res.items(): + md.append( + f"| {m} | {r['n_points']} | **{r['H3a']['snr']:.2f}** " + f"| **{r['H3b_3am']['snr']:.2f}** ({r['H3b_3am']['kept_frac']:.0%}) " + f"| **{r['H3b_7am']['snr']:.2f}** ({r['H3b_7am']['kept_frac']:.0%}) |") + md.append("\n![Radial Compton-y excess profile around spine networks at " + "ACT resolution](figures/e3c_profile.png)\n") + out = ROOT / "docs" / "E3c-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e3d.py b/research/cosmic-web/scripts/run_e3d.py new file mode 100644 index 00000000000..7dc6ce43e83 --- /dev/null +++ b/research/cosmic-web/scripts/run_e3d.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""E3d: galaxy-pair bridge stack at ACT depth — the small-scale WHIM design. + +E3c showed our shell-projection signal lives at degree scales that the ACT +DR6 ILC map filters out. This is the design matched to ACT instead: + +1. Select close CMASS pairs: comoving transverse separation 6-14 Mpc/h, + line-of-sight separation < 6 Mpc/h (physically-connected candidates; + the de Graaff/Tanimura geometry). +2. For each pair, sample y at the sky MIDPOINT (the bridge), and subtract + the halo-symmetric estimate: the mean of y at the four points obtained + by rotating the midpoint by +/-90 deg around EACH galaxy (same angular + distance from that halo, off the bridge axis). A circularly symmetric + halo contributes identically on- and off-axis, so Model A (halo-only) + predicts zero by construction. +3. Errors: bootstrap over pairs; plus a rotation null (whole-pair-set + azimuthal scrambling) as a cross-check. +4. Run on BOTH maps (ACT DR6 ILC and Planck MILCA) for the same pairs. + +Report -> docs/E3d-report.md. +""" + +import json +import sys +import time +from pathlib import Path + +import numpy as np +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) + +import run_e3 as e3 # noqa: E402 +from run_e3c import CarMap # noqa: E402 + +SEP_T = (6.0, 14.0) # transverse comoving separation, Mpc/h +SEP_LOS = 6.0 # max line-of-sight separation, Mpc/h +N_BOOT = 500 +RNG = np.random.default_rng(17) + + +def find_pairs(xyz): + """Pairs by 3D distance then split into transverse/LOS components.""" + tree = cKDTree(xyz) + cand = tree.query_pairs(r=np.hypot(SEP_T[1], SEP_LOS), output_type="ndarray") + a, b = xyz[cand[:, 0]], xyz[cand[:, 1]] + mid = 0.5 * (a + b) + los_hat = mid / np.linalg.norm(mid, axis=1, keepdims=True) + d = b - a + d_los = np.abs(np.sum(d * los_hat, axis=1)) + d_t = np.sqrt(np.maximum(np.sum(d * d, axis=1) - d_los ** 2, 0)) + keep = (d_los < SEP_LOS) & (d_t > SEP_T[0]) & (d_t < SEP_T[1]) + return cand[keep], mid[keep] + + +def unit(v): + return v / np.linalg.norm(v, axis=1, keepdims=True) + + +def sky_vec(xyz): + return unit(xyz) + + +def rotate_about(p, axis, ang): + """Rodrigues rotation of unit vectors p about unit vectors axis.""" + c, s = np.cos(ang), np.sin(ang) + cross = np.cross(axis, p) + dot = np.sum(axis * p, axis=1, keepdims=True) + return c * p + s * cross + (1 - c) * dot * axis + + +def vec_to_radec(v): + dec = np.degrees(np.arcsin(np.clip(v[:, 2], -1, 1))) + ra = np.degrees(np.arctan2(v[:, 1], v[:, 0])) % 360 + return ra, dec + + +def bridge_estimator(car, g1, g2, mid): + """y(mid) - mean of off-axis rotations about each galaxy. + Returns per-pair estimate (NaN where any sample leaves coverage).""" + est_pts = [] + for g in (g1, g2): + for ang in (np.pi / 2, -np.pi / 2): + est_pts.append(rotate_about(mid, g, ang)) + ra_m, dec_m = vec_to_radec(mid) + y_mid = car.sample(ra_m, dec_m) + y_off = np.zeros_like(y_mid) + for p in est_pts: + ra_o, dec_o = vec_to_radec(unit(p)) + y_off += car.sample(ra_o, dec_o) + return y_mid - y_off / 4 + + +def run_map(name, car, g1v, g2v, midv): + est = bridge_estimator(car, g1v, g2v, midv) + ok = ~np.isnan(est) + est = est[ok] + n = len(est) + boot = np.array([np.mean(est[RNG.integers(0, n, n)]) for _ in range(N_BOOT)]) + mean = float(est.mean()) + err = float(boot.std()) + print(f" {name}: {n} pairs in coverage, bridge = {mean:.3e} " + f"± {err:.3e} (SNR {mean/err:.2f})") + return {"n_pairs": n, "bridge": mean, "err": err, + "snr": float(mean / err)} + + +def main(): + t0 = time.time() + print("galaxies + pairs...") + ra_g, dec_g, z_g, xyz = e3.load_galaxies() + pairs, mid_xyz = find_pairs(xyz) + print(f" {len(pairs)} pairs " + f"(transverse {SEP_T[0]}-{SEP_T[1]} Mpc/h, LOS < {SEP_LOS})") + + g1v = sky_vec(xyz[pairs[:, 0]]) + g2v = sky_vec(xyz[pairs[:, 1]]) + midv = sky_vec(mid_xyz) + + results = {"n_pairs_total": int(len(pairs))} + print("ACT DR6...") + act = CarMap(ROOT / "data" / "ilc_actplanck_ymap.fits") + results["act"] = run_map("ACT", act, g1v, g2v, midv) + del act + + print("Planck MILCA (healpy)...") + import healpy as hp + ymap = hp.read_map(str(next(ROOT.glob("data/**/*milca*.fits"))), field=0) + + class HP: + def sample(self, ra, dec): + from run_e3b import to_gal + l, b = to_gal(ra, dec) + return hp.get_interp_val(ymap, np.radians(90 - b), np.radians(l)) + results["planck"] = run_map("Planck", HP(), g1v, g2v, midv) + + (ROOT / "artifacts" / "e3d_results.json").write_text( + json.dumps(results, indent=1)) + report(results) + print(f"total {time.time()-t0:.0f}s") + + +def report(res): + md = ["# E3d — galaxy-pair bridge stack (small-scale WHIM design)\n", + f"Setup: CMASS-N z 0.45–0.55 pairs with transverse separation " + f"{SEP_T[0]}–{SEP_T[1]} Mpc/h and line-of-sight separation " + f"< {SEP_LOS} Mpc/h ({res['n_pairs_total']} pairs). Estimator: " + "y at the pair midpoint minus the mean of the four off-axis " + "points at identical angular distance from each galaxy " + "(±90° rotations) — a circularly symmetric halo contributes " + "zero by construction, so any positive mean is gas that lives " + "preferentially ON the inter-pair axis. Errors: bootstrap over " + f"pairs ({N_BOOT} resamples).\n"] + md.append("| map | pairs used | bridge excess (y) | error | SNR |") + md.append("|---|---|---|---|---|") + for m in ("act", "planck"): + r = res[m] + md.append(f"| {m.upper()} | {r['n_pairs']} | {r['bridge']:.3e} " + f"| {r['err']:.3e} | **{r['snr']:.2f}** |") + out = ROOT / "docs" / "E3d-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e3d2.py b/research/cosmic-web/scripts/run_e3d2.py new file mode 100644 index 00000000000..40359ba998b --- /dev/null +++ b/research/cosmic-web/scripts/run_e3d2.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""E3d-v2: the powered pair-bridge measurement. + +Three upgrades over E3d, each attacking a specific weakness: +1. Full CMASS-N redshift range (0.43-0.70) -> ~2.5x the pairs. +2. Disc-averaged sampling: each evaluation point becomes the mean of a + 7-point hexagonal patch (radius 2') -> beats down small-scale map + noise without washing out a >=6 Mpc/h-wide bridge. +3. Ring controls: off-axis points at +/-60, +/-90, +/-120 degrees around + EACH galaxy (12 controls/pair) -> lower estimator variance AND less + second-halo leakage than the +/-90-only version. + +Gate: ACT bridge SNR >= 3 -> Model B established at ACT depth; +2-3 sigma -> supported-but-not-decisive; < 2 -> bounded again. +Report -> docs/E3d2-report.md. +""" + +import json +import sys +import time +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) + +import run_e3 as e3 # noqa: E402 +import run_e3d as d1 # noqa: E402 +from run_e3c import CarMap # noqa: E402 + +Z_FULL = (0.43, 0.70) +DISC_ARCMIN = 2.0 +RING_ANGLES = np.radians([60, 90, 120, -60, -90, -120]) +N_BOOT = 500 +RNG = np.random.default_rng(23) + + +def disc_offsets(v, radius_rad): + """7-point hexagonal patch around each unit vector v.""" + out = [v] + r = RNG.normal(size=v.shape) # fixed random tangent frame per point + r -= v * np.sum(r * v, axis=1, keepdims=True) + r /= np.linalg.norm(r, axis=1, keepdims=True) + s = np.cross(v, r) + for k in range(6): + a = k * np.pi / 3 + w = v * np.cos(radius_rad) + (r * np.cos(a) + s * np.sin(a)) * np.sin(radius_rad) + out.append(w) + return out + + +def disc_sample(sampler, v): + vals = np.zeros(len(v)) + for w in disc_offsets(v, np.radians(DISC_ARCMIN / 60)): + ra, dec = d1.vec_to_radec(w) + vals += sampler(ra, dec) + return vals / 7 + + +def bridge_v2(sampler, g1, g2, mid): + y_mid = disc_sample(sampler, mid) + y_off = np.zeros_like(y_mid) + for g in (g1, g2): + for ang in RING_ANGLES: + y_off += disc_sample(sampler, d1.unit(d1.rotate_about(mid, g, ang))) + return y_mid - y_off / (2 * len(RING_ANGLES)) + + +def run(name, sampler, g1v, g2v, midv): + est = bridge_v2(sampler, g1v, g2v, midv) + ok = ~np.isnan(est) + est = est[ok] + n = len(est) + boot = np.array([np.mean(est[RNG.integers(0, n, n)]) for _ in range(N_BOOT)]) + mean, err = float(est.mean()), float(boot.std()) + print(f" {name}: {n} pairs, bridge = {mean:.3e} ± {err:.3e} " + f"(SNR {mean/err:.2f})") + return {"n_pairs": n, "bridge": mean, "err": err, "snr": float(mean / err)} + + +def main(): + t0 = time.time() + e3.Z_RANGE = Z_FULL # widen the shell before loading + print(f"galaxies (z {Z_FULL}) + pairs...") + ra_g, dec_g, z_g, xyz = e3.load_galaxies() + print(f" {len(ra_g)} galaxies") + pairs, mid_xyz = d1.find_pairs(xyz) + print(f" {len(pairs)} pairs") + + g1v = d1.sky_vec(xyz[pairs[:, 0]]) + g2v = d1.sky_vec(xyz[pairs[:, 1]]) + midv = d1.sky_vec(mid_xyz) + + results = {"n_pairs_total": int(len(pairs)), "z_range": Z_FULL, + "disc_arcmin": DISC_ARCMIN, + "ring_angles_deg": [60, 90, 120, -60, -90, -120]} + print("ACT DR6...") + act = CarMap(ROOT / "data" / "ilc_actplanck_ymap.fits") + results["act"] = run("ACT", act.sample, g1v, g2v, midv) + del act + + print("Planck MILCA...") + import healpy as hp + ymap = hp.read_map(str(next(ROOT.glob("data/**/*milca*.fits"))), field=0) + from run_e3b import to_gal + + def planck_sample(ra, dec): + l, b = to_gal(ra, dec) + return hp.get_interp_val(ymap, np.radians(90 - b), np.radians(l)) + results["planck"] = run("Planck", planck_sample, g1v, g2v, midv) + + (ROOT / "artifacts" / "e3d2_results.json").write_text( + json.dumps(results, indent=1)) + report(results) + print(f"total {time.time()-t0:.0f}s") + + +def report(res): + md = ["# E3d-v2 — powered pair-bridge measurement\n", + f"CMASS-N z {res['z_range'][0]}–{res['z_range'][1]} " + f"({res['n_pairs_total']} pairs, transverse {d1.SEP_T[0]}–" + f"{d1.SEP_T[1]} Mpc/h, LOS < {d1.SEP_LOS}). Estimator: " + f"{res['disc_arcmin']}′-disc-averaged y at the midpoint minus " + "the mean over ring controls at ±60/±90/±120° around each " + "galaxy. Bootstrap errors over pairs.\n"] + md.append("| map | pairs used | bridge excess (y) | error | SNR |") + md.append("|---|---|---|---|---|") + for m in ("act", "planck"): + r = res[m] + md.append(f"| {m.upper()} | {r['n_pairs']} | {r['bridge']:.3e} " + f"| {r['err']:.3e} | **{r['snr']:.2f}** |") + out = ROOT / "docs" / "E3d2-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e3e.py b/research/cosmic-web/scripts/run_e3e.py new file mode 100644 index 00000000000..6e46dbf70bb --- /dev/null +++ b/research/cosmic-web/scripts/run_e3e.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""E3e: validity battery for the 5.2-sigma bridge detection. + +A. Sky-patch jackknife errors: pairs overlap on the sky, so pair-bootstrap + underestimates the variance. Split pairs into ~50 RA patches and use + delete-one-patch jackknife. Gate: ACT SNR >= 3 with honest errors. +B. Unconnected-pairs null: same transverse window (6-14 Mpc/h) but + line-of-sight separation 25-40 Mpc/h — same sky geometry, no physical + bridge. A real signal vanishes; leakage/systematics would not. +C. Transverse-separation bins (connected pairs, ACT): a physical bridge + declines gently with separation; second-halo leakage falls steeply. + +Report -> docs/E3e-report.md. +""" + +import json +import sys +import time +from pathlib import Path + +import numpy as np +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) + +import run_e3 as e3 # noqa: E402 +import run_e3d as d1 # noqa: E402 +import run_e3d2 as d2 # noqa: E402 +from run_e3c import CarMap # noqa: E402 + +Z_FULL = (0.43, 0.70) +LOS_NULL = (25.0, 40.0) +N_PATCH = 50 + + +def find_pairs_window(xyz, sep_t, los_lo, los_hi): + tree = cKDTree(xyz) + cand = tree.query_pairs(r=np.hypot(sep_t[1], los_hi), output_type="ndarray") + a, b = xyz[cand[:, 0]], xyz[cand[:, 1]] + mid = 0.5 * (a + b) + los_hat = mid / np.linalg.norm(mid, axis=1, keepdims=True) + d = b - a + d_los = np.abs(np.sum(d * los_hat, axis=1)) + d_t = np.sqrt(np.maximum(np.sum(d * d, axis=1) - d_los ** 2, 0)) + keep = (d_los >= los_lo) & (d_los < los_hi) & \ + (d_t > sep_t[0]) & (d_t < sep_t[1]) + return cand[keep], mid[keep], d_t[keep] + + +def jackknife(est, ra_mid, n_patch=N_PATCH): + """Delete-one-RA-patch jackknife mean and error.""" + edges = np.quantile(ra_mid, np.linspace(0, 1, n_patch + 1)) + patch = np.clip(np.searchsorted(edges, ra_mid) - 1, 0, n_patch - 1) + total, n = est.sum(), len(est) + means = [] + for p in range(n_patch): + m = patch == p + if m.sum() == 0: + continue + means.append((total - est[m].sum()) / (n - m.sum())) + means = np.array(means) + k = len(means) + mean = float(est.mean()) + err = float(np.sqrt((k - 1) / k * np.sum((means - means.mean()) ** 2))) + return mean, err + + +def measure(sampler, g1v, g2v, midv, ra_mid, label): + est = d2.bridge_v2(sampler, g1v, g2v, midv) + ok = ~np.isnan(est) + mean, err = jackknife(est[ok], ra_mid[ok]) + print(f" {label}: {int(ok.sum())} pairs, bridge = {mean:.3e} " + f"± {err:.3e} (jackknife SNR {mean/err:.2f})") + return {"n_pairs": int(ok.sum()), "bridge": mean, "err": err, + "snr": float(mean / err)} + + +def main(): + t0 = time.time() + e3.Z_RANGE = Z_FULL + print("galaxies + pair sets...") + ra_g, dec_g, z_g, xyz = e3.load_galaxies() + pc, mid_c, dt_c = find_pairs_window(xyz, d1.SEP_T, 0.0, d1.SEP_LOS) + pu, mid_u, _ = find_pairs_window(xyz, d1.SEP_T, *LOS_NULL) + print(f" connected: {len(pc)} unconnected(null): {len(pu)}") + + sets = {} + for name, pairs, mid, dt in [("connected", pc, mid_c, dt_c), + ("null", pu, mid_u, None)]: + g1v = d1.sky_vec(xyz[pairs[:, 0]]) + g2v = d1.sky_vec(xyz[pairs[:, 1]]) + midv = d1.sky_vec(mid) + ra_mid, _ = d1.vec_to_radec(midv) + sets[name] = (g1v, g2v, midv, ra_mid, dt) + + results = {"n_connected": int(len(pc)), "n_null": int(len(pu)), + "los_null": LOS_NULL, "n_patch": N_PATCH} + print("ACT DR6...") + act = CarMap(ROOT / "data" / "ilc_actplanck_ymap.fits") + for name in ("connected", "null"): + g1v, g2v, midv, ra_mid, _ = sets[name] + results[f"act_{name}"] = measure(act.sample, g1v, g2v, midv, ra_mid, + f"ACT {name}") + # C: separation bins on ACT, connected pairs + g1v, g2v, midv, ra_mid, dt = sets["connected"] + est = d2.bridge_v2(act.sample, g1v, g2v, midv) + bins = [(6, 8), (8, 10), (10, 12), (12, 14)] + results["act_sep_bins"] = [] + for lo, hi in bins: + m = (dt >= lo) & (dt < hi) & ~np.isnan(est) + mean, err = jackknife(est[m], ra_mid[m]) + results["act_sep_bins"].append( + {"sep": f"{lo}-{hi}", "bridge": mean, "err": err, + "snr": float(mean / err), "n": int(m.sum())}) + print(f" ACT sep {lo}-{hi} Mpc/h: {mean:.3e} ± {err:.3e}") + del act + + print("Planck MILCA...") + import healpy as hp + ymap = hp.read_map(str(next(ROOT.glob("data/**/*milca*.fits"))), field=0) + from run_e3b import to_gal + + def planck_sample(ra, dec): + l, b = to_gal(ra, dec) + return hp.get_interp_val(ymap, np.radians(90 - b), np.radians(l)) + for name in ("connected", "null"): + g1v, g2v, midv, ra_mid, _ = sets[name] + results[f"planck_{name}"] = measure(planck_sample, g1v, g2v, midv, + ra_mid, f"Planck {name}") + + (ROOT / "artifacts" / "e3e_results.json").write_text( + json.dumps(results, indent=1)) + report(results) + print(f"total {time.time()-t0:.0f}s") + + +def report(res): + md = ["# E3e — validity battery for the bridge detection\n", + f"Jackknife errors over {res['n_patch']} RA patches (pairs overlap " + "on the sky, so pair-bootstrap is too optimistic). Null pairs: " + f"same transverse window, line-of-sight separation " + f"{res['los_null'][0]:.0f}–{res['los_null'][1]:.0f} Mpc/h — same " + "sky geometry, no physical bridge.\n"] + md.append("| map | pair set | n | bridge (y) | jackknife err | SNR |") + md.append("|---|---|---|---|---|---|") + for m in ("act", "planck"): + for s in ("connected", "null"): + r = res[f"{m}_{s}"] + md.append(f"| {m.upper()} | {s} | {r['n_pairs']} " + f"| {r['bridge']:.3e} | {r['err']:.3e} " + f"| **{r['snr']:.2f}** |") + md.append("\n## ACT bridge vs transverse separation (connected pairs)\n") + md.append("| separation (Mpc/h) | n | bridge (y) | err | SNR |") + md.append("|---|---|---|---|---|") + for b in res["act_sep_bins"]: + md.append(f"| {b['sep']} | {b['n']} | {b['bridge']:.3e} " + f"| {b['err']:.3e} | {b['snr']:.2f} |") + out = ROOT / "docs" / "E3e-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e4.py b/research/cosmic-web/scripts/run_e4.py new file mode 100644 index 00000000000..cfbf8badf63 --- /dev/null +++ b/research/cosmic-web/scripts/run_e4.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""E4 (H4'): is the CORRECTION to flat transport organized by the lifted +geometry? + +Reframing (user): the lifted geometry was never meant to replace the +standard description, but to be an adjustment term on top of it. T1 shows +the leading-order law is flat optimal transport (Zel'dovich + sticking); +the testable question is whether the RESIDUAL — true N-body transport minus +the ZA baseline from the same initial conditions — is anisotropic in the +tidal frame, preferentially along the filament axis e3, growing toward the +web. + +Measurements (per PM sim, tracked particles, at a = 1): + R = x_true - x_ZA (residual displacement, periodic-aware) + Rv = v_true - v_ZA (residual velocity; v_ZA = dD/dtau * psi) + M-E4a: <(R_hat . e3)^2> per distance-to-spine bin (null 1/3) + M-E4b: <(Rv_hat . e3)^2> per bin (null 1/3) + M-E4c: effect size — RMS parallel vs RMS transverse (per-axis) residual + near the web vs far: the measured 'geometry adjustment' amplitude. +Errors: jackknife over sims. Report -> docs/E4-report.md. +""" + +import json +import sys +import time +from multiprocessing import Pool +from pathlib import Path + +import numpy as np +from scipy.ndimage import gaussian_filter +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +import fields, lift, pm, spines # noqa: E402 + +N, L = 128, 128.0 +SEEDS = [1, 2, 3, 4, 5, 6] +D_BINS = [(0, 2), (2, 4), (4, 8), (8, 16), (16, 64)] +A_FINAL = 1.0 +_AXES = lift.hemisphere_axes(42) + + +def lifted_spines(field): + U = lift.orientation_score(field, _AXES, 6.0, 1.5) + return spines.skeleton_points(spines.extract_matched( + spines.lifted_ridgeness(U), 2400, lo_frac=1.0)) + + +def e3_at(e3, pts): + idx = np.clip(np.round(pts).astype(int), 0, N - 1) % N + return e3[idx[:, 0], idx[:, 1], idx[:, 2]] + + +def par_frac(vecs, e3v, floor=1e-3): + nrm = np.linalg.norm(vecs, axis=1) + ok = nrm > floor + h = vecs[ok] / nrm[ok, None] + return np.sum(h * e3v[ok], axis=1) ** 2, ok + + +def run_seed(seed): + rng = np.random.default_rng(seed) + x, v, tidx, ta, tpos, q_tr, psi_tr = pm.pm_sim(N, L, rng, n_steps=90, + track=50000) + a = ta[-1] + # ZA baseline for the SAME particles at final a (D = a in EdS): + x_za = (q_tr + a * psi_tr) % N + # ZA velocity in pm's convention (vel = p/a, ZA momentum p = a^{3/2} psi): + v_za = np.sqrt(a) * psi_tr + + x_true = tpos[-1].astype(float) + R = x_true - x_za + R -= N * np.round(R / N) + Rv = v[tidx] - v_za + + rho = fields.cic_deposit(x, N) + clean = gaussian_filter(np.log1p(rho), 1.0, mode="wrap") + e3 = fields.tidal_frame(clean) + d_spine = cKDTree(lifted_spines(clean)).query(x_true)[0] + e3v = e3_at(e3, x_true) + + pfR, okR = par_frac(R, e3v, floor=0.1) + pfV, okV = par_frac(Rv, e3v, floor=1e-4) + # per-axis RMS decomposition of R + par_comp = np.sum(R * e3v, axis=1) + perp2 = np.maximum(np.sum(R * R, axis=1) - par_comp ** 2, 0) + + rec = {"seed": seed, "a_final": float(a)} + for lo, hi in D_BINS: + m = (d_spine >= lo) & (d_spine < hi) + rec[f"pfR_{lo}_{hi}"] = float(pfR[m[okR]].mean()) + rec[f"pfV_{lo}_{hi}"] = float(pfV[m[okV]].mean()) + rec[f"rmsPar_{lo}_{hi}"] = float(np.sqrt(np.mean(par_comp[m] ** 2))) + rec[f"rmsPerp_{lo}_{hi}"] = float(np.sqrt(np.mean(perp2[m]) / 2)) + rec[f"rmsR_{lo}_{hi}"] = float(np.sqrt(np.mean(np.sum(R[m]**2, 1)))) + rec[f"n_{lo}_{hi}"] = int(m.sum()) + return rec + + +def jk(vals): + v = np.array(vals) + k = len(v) + means = np.array([np.delete(v, i).mean() for i in range(k)]) + return float(v.mean()), float(np.sqrt((k - 1) / k * + np.sum((means - means.mean()) ** 2))) + + +def main(): + t0 = time.time() + with Pool(6) as pool: + recs = pool.map(run_seed, SEEDS) + print(f"{len(SEEDS)} sims in {time.time()-t0:.0f}s") + (ROOT / "artifacts" / "e4_results.json").write_text(json.dumps(recs, indent=1)) + report(recs) + + +def report(recs): + md = ["# E4 (H4') — the geometry-adjustment term on top of flat transport\n", + f"Setup: {len(SEEDS)} PM sims (128³, EdS), 50k tracked particles " + "each. Baseline: the Zel'dovich prediction for the SAME particle " + "from the same initial conditions. R = true − ZA displacement at " + "a=1; Rv = true − ZA velocity. If the lifted geometry is a valid " + "adjustment term, R and Rv should point preferentially along the " + "filament axis e3 near the web (null 1/3) with amplitude growing " + "toward spines.\n"] + md.append("| d to spine (vox) | n/seed | ⟨(R̂·e3)²⟩ | ⟨(R̂v·e3)²⟩ " + "| RMS R∥ (vox) | RMS R⊥/axis | anisotropy R∥/R⊥ |") + md.append("|---|---|---|---|---|---|---|") + for lo, hi in D_BINS: + a, ae = jk([r[f"pfR_{lo}_{hi}"] for r in recs]) + b, be = jk([r[f"pfV_{lo}_{hi}"] for r in recs]) + rp, _ = jk([r[f"rmsPar_{lo}_{hi}"] for r in recs]) + rt, _ = jk([r[f"rmsPerp_{lo}_{hi}"] for r in recs]) + n = int(np.mean([r[f"n_{lo}_{hi}"] for r in recs])) + md.append(f"| {lo}–{hi} | {n} | {a:.3f} ± {ae:.3f} " + f"| {b:.3f} ± {be:.3f} | {rp:.2f} | {rt:.2f} " + f"| **{rp/rt:.2f}** |") + md.append("\nNull for the direction statistics: 1/3 (isotropic). " + "R∥/R⊥ = 1 means the correction is isotropic; > 1 means the " + "residual transport is preferentially along the filament " + "axis — the measured size of the geometry adjustment.\n") + out = ROOT / "docs" / "E4-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + print("\n".join(md[3:])) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e5.py b/research/cosmic-web/scripts/run_e5.py new file mode 100644 index 00000000000..6f0e052bb37 --- /dev/null +++ b/research/cosmic-web/scripts/run_e5.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""E5 (H-T2): does the tidal-frame TRANSVERSE damping term improve the model? + +E4 measured the correction that flat (Zel'dovich) transport needs: large, +tidal-frame-organized, and transverse. This experiment builds that +correction into an effective model and tests it head-to-head: + + M0 plain ZA x(D) = q + D psi, no interaction ever + M1 isotropic stick at first shell-crossing, all momentum -> 0 + M2 transverse damp at first shell-crossing, remove the e1/e2 + components of the velocity in the LOCAL tidal + frame; keep the e3 (along-filament) component + +All three are evolved from the same initial conditions as a PM N-body run +(the truth), and scored per particle: + M-E5a: median |x_model - x_PM| at a=1, binned by distance to the web + M-E5b: velocity-direction fidelity near the web + +Shell-crossing proxy: first time the local (ZA-evolved) density at the +particle exceeds rho_c; rho_c is calibrated on seed 1 by optimizing M1 +(the ESTABLISHED model), then frozen — so the tuning favours the +competitor, not our candidate. Report -> docs/E5-report.md. +""" + +import json +import sys +import time +from multiprocessing import Pool +from pathlib import Path + +import numpy as np +from scipy.ndimage import gaussian_filter, map_coordinates +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +import fields, lift, pm, spines # noqa: E402 + +N, L = 128, 128.0 +SEEDS = [1, 2, 3, 4, 5, 6] +D_STEPS = np.arange(0.10, 1.0001, 0.05) +FRAME_EVERY = 3 # recompute tidal frame every k steps +D_BINS = [(0, 2), (2, 4), (4, 8), (8, 64)] +RHO_GRID = [2.0, 3.0, 5.0] +_AXES = lift.hemisphere_axes(42) + + +def za_density(q_grid, psi_full, D): + return fields.cic_deposit((q_grid + D * psi_full) % N, N) + + +def gather(grid, pos): + return map_coordinates(grid, [pos[:, 0] % N, pos[:, 1] % N, + pos[:, 2] % N], order=1, mode="wrap") + + +def evolve(q_tr, psi_tr, q_grid, psi_full, rho_c, mode): + """Evolve tracked particles under one effective model.""" + x = (q_tr + D_STEPS[0] * psi_tr) % N + v = psi_tr.copy() # dx/dD + crossed = np.zeros(len(x), bool) + frame = None + for i in range(1, len(D_STEPS)): + D = D_STEPS[i] + dD = D_STEPS[i] - D_STEPS[i - 1] + x = (x + v * dD) % N + if mode == "za": + continue + rho = za_density(q_grid, psi_full, D) + if frame is None or i % FRAME_EVERY == 0: + clean = gaussian_filter(np.log1p(rho), 1.0, mode="wrap") + e3 = fields.tidal_frame(clean) + new = (~crossed) & (gather(rho, x) > rho_c) + if new.any(): + if mode == "stick": + v[new] = 0.0 + else: # transverse damp: keep e3 part + idx = np.clip(np.round(x[new]).astype(int), 0, N - 1) % N + e = e3[idx[:, 0], idx[:, 1], idx[:, 2]] + v[new] = e * np.sum(v[new] * e, axis=1, keepdims=True) + crossed |= new + return x + + +def run_seed(args): + seed, rho_c = args + rng = np.random.default_rng(seed) + x_pm, v_pm, tidx, ta, tpos, q_tr, psi_tr = pm.pm_sim( + N, L, rng, n_steps=90, track=50000) + # full-set ZA ingredients (regenerate deterministically, as pm does) + seedseq = np.random.default_rng(seed).integers(0, 2 ** 32) + _, pos_q = fields.zeldovich_box(N, L, 0.0, np.random.default_rng(seedseq)) + _, pos_d = fields.zeldovich_box(N, L, 1.0, np.random.default_rng(seedseq)) + psi_full = pos_d - pos_q + psi_full -= N * np.round(psi_full / N) + + truth = tpos[-1].astype(float) + clean = gaussian_filter(np.log1p(fields.cic_deposit(x_pm, N)), 1.0, + mode="wrap") + U = lift.orientation_score(clean, _AXES, 6.0, 1.5) + sp = spines.skeleton_points(spines.extract_matched( + spines.lifted_ridgeness(U), 2400, lo_frac=1.0)) + d_spine = cKDTree(sp).query(truth)[0] + vt = v_pm[tidx] + vt_hat = vt / np.maximum(np.linalg.norm(vt, axis=1, keepdims=True), 1e-12) + + rec = {"seed": seed, "rho_c": rho_c} + for mode in ("za", "stick", "damp"): + xm = evolve(q_tr, psi_tr, pos_q, psi_full, rho_c, mode) + err = xm - truth + err -= N * np.round(err / N) + err = np.linalg.norm(err, axis=1) + for lo, hi in D_BINS: + m = (d_spine >= lo) & (d_spine < hi) + rec[f"{mode}_err_{lo}_{hi}"] = float(np.median(err[m])) + rec[f"{mode}_err_all"] = float(np.median(err)) + return rec + + +def calibrate_rho(): + """Pick rho_c minimizing the ISOTROPIC-stick model's error on seed 1.""" + best, best_err = RHO_GRID[0], np.inf + for rho_c in RHO_GRID: + rec = run_seed((1, rho_c)) + e = rec["stick_err_all"] + print(f" cal rho_c={rho_c}: stick median err {e:.2f}") + if e < best_err: + best, best_err = rho_c, e + print(f" -> rho_c = {best}") + return best + + +def main(): + t0 = time.time() + print("calibrating rho_c on seed 1 (optimizing the competitor)...") + rho_c = calibrate_rho() + with Pool(5) as pool: + recs = pool.map(run_seed, [(s, rho_c) for s in SEEDS[1:]]) + print(f"{len(recs)} eval sims in {time.time()-t0:.0f}s") + (ROOT / "artifacts" / "e5_results.json").write_text(json.dumps( + {"rho_c": rho_c, "results": recs}, indent=1)) + report(recs, rho_c) + + +def jk(vals): + v = np.array(vals); k = len(v) + means = np.array([np.delete(v, i).mean() for i in range(k)]) + return float(v.mean()), float(np.sqrt((k - 1) / k * + np.sum((means - means.mean())**2))) + + +def report(recs, rho_c): + md = ["# E5 (H-T2) — transverse-damping adjustment vs ZA and isotropic " + "sticking\n", + f"Effective models evolved from PM initial conditions " + f"(rho_c = {rho_c}, calibrated on seed 1 by optimizing the " + f"ISOTROPIC competitor; {len(recs)} held-out seeds). Score: " + "median per-particle distance to the PM-truth position at a=1 " + "(voxels = h⁻¹Mpc), binned by distance to the web.\n"] + md.append("| d to spine (vox) | ZA | isotropic stick | transverse damp " + "| damp vs ZA | damp vs stick |") + md.append("|---|---|---|---|---|---|") + for key in [f"{lo}_{hi}" for lo, hi in D_BINS] + ["all"]: + za, zae = jk([r[f"za_err_{key}"] for r in recs]) + st, ste = jk([r[f"stick_err_{key}"] for r in recs]) + dm, dme = jk([r[f"damp_err_{key}"] for r in recs]) + lab = key.replace("_", "–") if key != "all" else "**all**" + md.append(f"| {lab} | {za:.2f} ± {zae:.2f} | {st:.2f} ± {ste:.2f} " + f"| {dm:.2f} ± {dme:.2f} | {100*(dm-za)/za:+.0f}% " + f"| {100*(dm-st)/st:+.0f}% |") + md.append("\nReading: negative percentages mean the transverse-damping " + "model is closer to the N-body truth. H-T2 asks for damp < " + "ZA (the adjustment helps) and damp <= stick (keeping the " + "along-filament component does not hurt, unlike full " + "sticking).\n") + out = ROOT / "docs" / "E5-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + print("\n".join(md[3:])) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e5b.py b/research/cosmic-web/scripts/run_e5b.py new file mode 100644 index 00000000000..57a9d7f1a3e --- /dev/null +++ b/research/cosmic-web/scripts/run_e5b.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""E5b (H-E5b): is the 2-4 vox deficit of the transverse-damping model an +estimation artifact or a physics failure? + +E5 attributed the model's small outskirts deficit (+4% vs ZA at 2-4 vox) +to noisy tidal frames from the evolving ZA proxy density. Test: rerun the +SAME model with oracle frames — the tidal eigenframe of the true PM final +field — for the damping decisions. If the deficit closes, the term's +physics is validated and the error is in frame estimation; if it persists, +the damping condition itself over-fires in the outskirts. + +Modes: za | damp (E5, proxy frames) | damp_oracle (PM-final frames). +rho_c frozen at the E5-calibrated value. Report -> docs/E5b-report.md. +""" + +import json +import sys +import time +from multiprocessing import Pool +from pathlib import Path + +import numpy as np +from scipy.ndimage import gaussian_filter +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) + +import fields, lift, pm, spines # noqa: E402 +from run_e5 import (D_BINS, D_STEPS, FRAME_EVERY, N, L, _AXES, # noqa: E402 + gather, jk, za_density) + +SEEDS = [2, 3, 4, 5, 6] +RHO_C = 5.0 + + +def evolve(q_tr, psi_tr, q_grid, psi_full, rho_c, mode, oracle_e3=None): + x = (q_tr + D_STEPS[0] * psi_tr) % N + v = psi_tr.copy() + crossed = np.zeros(len(x), bool) + e3 = oracle_e3 + for i in range(1, len(D_STEPS)): + D = D_STEPS[i] + dD = D_STEPS[i] - D_STEPS[i - 1] + x = (x + v * dD) % N + if mode == "za": + continue + rho = za_density(q_grid, psi_full, D) + if mode == "damp" and (e3 is None or i % FRAME_EVERY == 0): + clean = gaussian_filter(np.log1p(rho), 1.0, mode="wrap") + e3 = fields.tidal_frame(clean) + new = (~crossed) & (gather(rho, x) > rho_c) + if new.any(): + idx = np.clip(np.round(x[new]).astype(int), 0, N - 1) % N + e = e3[idx[:, 0], idx[:, 1], idx[:, 2]] + v[new] = e * np.sum(v[new] * e, axis=1, keepdims=True) + crossed |= new + return x + + +def run_seed(seed): + rng = np.random.default_rng(seed) + x_pm, v_pm, tidx, ta, tpos, q_tr, psi_tr = pm.pm_sim( + N, L, rng, n_steps=90, track=50000) + seedseq = np.random.default_rng(seed).integers(0, 2 ** 32) + _, pos_q = fields.zeldovich_box(N, L, 0.0, np.random.default_rng(seedseq)) + _, pos_d = fields.zeldovich_box(N, L, 1.0, np.random.default_rng(seedseq)) + psi_full = pos_d - pos_q + psi_full -= N * np.round(psi_full / N) + + truth = tpos[-1].astype(float) + clean_pm = gaussian_filter(np.log1p(fields.cic_deposit(x_pm, N)), 1.0, + mode="wrap") + e3_oracle = fields.tidal_frame(clean_pm) + U = lift.orientation_score(clean_pm, _AXES, 6.0, 1.5) + sp = spines.skeleton_points(spines.extract_matched( + spines.lifted_ridgeness(U), 2400, lo_frac=1.0)) + d_spine = cKDTree(sp).query(truth)[0] + + rec = {"seed": seed} + for mode in ("za", "damp", "damp_oracle"): + xm = evolve(q_tr, psi_tr, pos_q, psi_full, RHO_C, + "za" if mode == "za" else + ("damp" if mode == "damp" else "oracle"), + oracle_e3=e3_oracle if mode == "damp_oracle" else None) + err = xm - truth + err -= N * np.round(err / N) + err = np.linalg.norm(err, axis=1) + for lo, hi in D_BINS: + m = (d_spine >= lo) & (d_spine < hi) + rec[f"{mode}_err_{lo}_{hi}"] = float(np.median(err[m])) + rec[f"{mode}_err_all"] = float(np.median(err)) + return rec + + +def main(): + t0 = time.time() + with Pool(5) as pool: + recs = pool.map(run_seed, SEEDS) + print(f"{len(recs)} sims in {time.time()-t0:.0f}s") + (ROOT / "artifacts" / "e5b_results.json").write_text( + json.dumps(recs, indent=1)) + report(recs) + + +def report(recs): + md = ["# E5b — oracle-frame test of the transverse-damping deficit\n", + f"Same protocol as E5 (rho_c = {RHO_C}, {len(recs)} held-out " + "seeds); 'oracle' uses the tidal frame of the true PM final field " + "for damping decisions instead of the evolving ZA proxy.\n"] + md.append("| d to spine (vox) | ZA | damp (proxy frames) " + "| damp (oracle frames) | oracle vs ZA |") + md.append("|---|---|---|---|---|") + for key in [f"{lo}_{hi}" for lo, hi in D_BINS] + ["all"]: + za, zae = jk([r[f"za_err_{key}"] for r in recs]) + dm, dme = jk([r[f"damp_err_{key}"] for r in recs]) + do, doe = jk([r[f"damp_oracle_err_{key}"] for r in recs]) + lab = key.replace("_", "–") if key != "all" else "**all**" + md.append(f"| {lab} | {za:.2f} ± {zae:.2f} | {dm:.2f} ± {dme:.2f} " + f"| {do:.2f} ± {doe:.2f} | {100*(do-za)/za:+.0f}% |") + md.append("\nReading: if the oracle closes the 2–4 vox gap to ZA, the " + "E5 deficit was frame-estimation error and the damping " + "physics is validated; if the gap persists, the crossing " + "condition itself over-fires in the outskirts.\n") + out = ROOT / "docs" / "E5b-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + print("\n".join(md[3:])) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e5c.py b/research/cosmic-web/scripts/run_e5c.py new file mode 100644 index 00000000000..1bc3770cec2 --- /dev/null +++ b/research/cosmic-web/scripts/run_e5c.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +"""E5c: refinement campaign for the transverse-damping model (declared +optimization, per program owner's direction). + +E5b showed frame estimation is the binding constraint (oracle bound 4.47 +vs proxy 4.93 median voxel error). Levers, in evidence order: + +1. SELF-density frames: evolve the FULL particle set under the model and + compute crossing density + tidal frames from the model's own particles + (damped particles pile onto the web -> sharper late-time frames than + the pure-ZA proxy). +2. Partial damping beta (v_perp *= 1-beta). +3. Crossing threshold rho_c. +4. Sequential damping (pancake physics): first crossing removes only the + e1 component, second crossing removes e2. + +Protocol: 6 configs selected on seeds {2,3}; the best two validated on +held-out seeds {4,5,6} against ZA, the E5 proxy model, and the E5b oracle +bound. Report -> docs/E5c-report.md. +""" + +import json +import sys +import time +from multiprocessing import Pool +from pathlib import Path + +import numpy as np +from scipy.ndimage import gaussian_filter +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) + +import fields, lift, pm, spines # noqa: E402 +from run_e5 import D_BINS, D_STEPS, N, L, _AXES, gather, jk # noqa: E402 + +SEL_SEEDS = [2, 3] +VAL_SEEDS = [4, 5, 6] + +CONFIGS = { + "A_proxy_b1_r5": dict(frames="za", beta=1.0, rho_c=5.0, seq=False), + "B_self_b1_r5": dict(frames="self", beta=1.0, rho_c=5.0, seq=False), + "C_self_b075_r5": dict(frames="self", beta=0.75, rho_c=5.0, seq=False), + "D_self_b1_r3": dict(frames="self", beta=1.0, rho_c=3.0, seq=False), + "E_self_b1_r5_sq": dict(frames="self", beta=1.0, rho_c=5.0, seq=True), + "F_self_b075_r3_sq": dict(frames="self", beta=0.75, rho_c=3.0, seq=True), +} + + +def evolve_full(q_grid, psi_full, cfg): + """Evolve the FULL particle set under one model config; returns final + positions of all particles.""" + x = (q_grid + D_STEPS[0] * psi_full) % N + v = psi_full.copy() + n_cross = np.zeros(len(x), np.int8) + e_frames = None + for i in range(1, len(D_STEPS)): + dD = D_STEPS[i] - D_STEPS[i - 1] + x = (x + v * dD) % N + # density for crossing detection (and frames if self) + if cfg["frames"] == "self": + rho = fields.cic_deposit(x, N) + else: + rho = fields.cic_deposit((q_grid + D_STEPS[i] * psi_full) % N, N) + if e_frames is None or i % 3 == 0: + clean = gaussian_filter(np.log1p(rho), 1.0, mode="wrap") + evals_dirs = fields.tidal_frame_full(clean) \ + if cfg["seq"] else (None, fields.tidal_frame(clean)) + e1f, e3f = (evals_dirs[0], evals_dirs[1]) if cfg["seq"] \ + else (None, evals_dirs[1]) + max_cross = 2 if cfg["seq"] else 1 + hit = (n_cross < max_cross) & (gather(rho, x) > cfg["rho_c"]) + if hit.any(): + idx = np.clip(np.round(x[hit]).astype(int), 0, N - 1) % N + if not cfg["seq"]: + e = e3f[idx[:, 0], idx[:, 1], idx[:, 2]] + par = e * np.sum(v[hit] * e, axis=1, keepdims=True) + v[hit] = par + (1 - cfg["beta"]) * (v[hit] - par) + else: + first = hit & (n_cross == 0) + second = hit & (n_cross == 1) + for mask, ef in ((first, e1f), (second, None)): + if not mask.any(): + continue + jdx = np.clip(np.round(x[mask]).astype(int), 0, N - 1) % N + if ef is not None: # remove e1 component only + e = ef[jdx[:, 0], jdx[:, 1], jdx[:, 2]] + comp = np.sum(v[mask] * e, axis=1, keepdims=True) + v[mask] = v[mask] - cfg["beta"] * comp * e + else: # second crossing: keep only e3 + e = e3f[jdx[:, 0], jdx[:, 1], jdx[:, 2]] + par = e * np.sum(v[mask] * e, axis=1, keepdims=True) + v[mask] = par + (1 - cfg["beta"]) * (v[mask] - par) + n_cross[hit] += 1 + return x + + +def run_seed(args): + seed, config_names = args + rng = np.random.default_rng(seed) + x_pm, v_pm, tidx, ta, tpos, q_tr, psi_tr = pm.pm_sim( + N, L, rng, n_steps=90, track=50000) + seedseq = np.random.default_rng(seed).integers(0, 2 ** 32) + _, pos_q = fields.zeldovich_box(N, L, 0.0, np.random.default_rng(seedseq)) + _, pos_d = fields.zeldovich_box(N, L, 1.0, np.random.default_rng(seedseq)) + psi_full = pos_d - pos_q + psi_full -= N * np.round(psi_full / N) + + truth = tpos[-1].astype(float) + clean_pm = gaussian_filter(np.log1p(fields.cic_deposit(x_pm, N)), 1.0, + mode="wrap") + sp = spines.skeleton_points(spines.extract_matched( + spines.lifted_ridgeness(lift.orientation_score(clean_pm, _AXES, 6.0, 1.5)), + 2400, lo_frac=1.0)) + d_spine = cKDTree(sp).query(truth)[0] + + rec = {"seed": seed} + # ZA baseline (tracked subset analytic) + xz = (q_tr + D_STEPS[-1] * psi_tr) % N + err = xz - truth; err -= N * np.round(err / N) + err = np.linalg.norm(err, axis=1) + rec["za_all"] = float(np.median(err)) + for name in config_names: + xf = evolve_full(pos_q, psi_full, CONFIGS[name]) + xm = xf[tidx] + err = xm - truth; err -= N * np.round(err / N) + err = np.linalg.norm(err, axis=1) + rec[f"{name}_all"] = float(np.median(err)) + for lo, hi in D_BINS: + m = (d_spine >= lo) & (d_spine < hi) + rec[f"{name}_{lo}_{hi}"] = float(np.median(err[m])) + return rec + + +def main(): + t0 = time.time() + names = list(CONFIGS) + print("selection on seeds", SEL_SEEDS) + with Pool(2) as pool: + sel = pool.map(run_seed, [(s, names) for s in SEL_SEEDS]) + scores = {n: np.mean([r[f"{n}_all"] for r in sel]) for n in names} + for n, v in sorted(scores.items(), key=lambda kv: kv[1]): + print(f" {n}: {v:.3f}") + top2 = sorted(scores, key=scores.get)[:2] + print("validating", top2, "on seeds", VAL_SEEDS) + with Pool(3) as pool: + val = pool.map(run_seed, [(s, top2) for s in VAL_SEEDS]) + (ROOT / "artifacts" / "e5c_results.json").write_text(json.dumps( + {"selection": sel, "scores": scores, "top2": top2, + "validation": val}, indent=1)) + report(scores, top2, val) + print(f"total {time.time()-t0:.0f}s") + + +def report(scores, top2, val): + md = ["# E5c — refinement campaign (declared optimization)\n", + "Config grid over frame source (ZA proxy vs model self-density), " + "damping fraction β, crossing threshold ρ_c, and sequential " + "(pancake-ordered) damping. Selection on seeds {2,3}; the top two " + "configs validated on held-out seeds {4,5,6}. References: ZA and " + "the E5b oracle bound (4.47 overall).\n", + "## Selection scores (median voxel error, seeds 2–3)\n"] + md.append("| config | overall error |") + md.append("|---|---|") + for n, v in sorted(scores.items(), key=lambda kv: kv[1]): + md.append(f"| {n} | {v:.3f} |") + md.append("\n## Held-out validation (seeds 4–6)\n") + md.append("| quantity | ZA | " + " | ".join(top2) + " |") + md.append("|---|---|" + "---|" * len(top2)) + za, zae = jk([r["za_all"] for r in val]) + row = [f"| **all** | {za:.2f} ± {zae:.2f} "] + for n in top2: + m, e = jk([r[f"{n}_all"] for r in val]) + row.append(f"| {m:.2f} ± {e:.2f} ") + md.append("".join(row) + "|") + for lo, hi in D_BINS: + cells = [f"| {lo}–{hi} | — "] + for n in top2: + m, e = jk([r[f"{n}_{lo}_{hi}"] for r in val]) + cells.append(f"| {m:.2f} ± {e:.2f} ") + md.append("".join(cells) + "|") + out = ROOT / "docs" / "E5c-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e5d.py b/research/cosmic-web/scripts/run_e5d.py new file mode 100644 index 00000000000..bd87c04219d --- /dev/null +++ b/research/cosmic-web/scripts/run_e5d.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""E5d: final knob round around the E5c winner (declared optimization). + +Two remaining levers near the optimum: damping fraction beta in +{0.6, 0.75, 0.9} x tidal-frame smoothing scale in {2, 4} Mpc/h (default +was 4). Self-density frames throughout, rho_c = 5, both-perp damping. +Selection on seeds {2,3}, winner validated on {4,5,6}. +Report -> docs/E5d-report.md. +""" + +import json, sys, time +from multiprocessing import Pool +from pathlib import Path +import numpy as np +from scipy.ndimage import gaussian_filter +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) + +import fields, lift, pm, spines # noqa: E402 +from run_e5 import D_BINS, D_STEPS, N, L, _AXES, gather, jk # noqa: E402 + +SEL_SEEDS = [2, 3] +VAL_SEEDS = [4, 5, 6] +RHO_C = 5.0 +CONFIGS = {f"b{b:g}_s{s:g}": dict(beta=b, smooth=s) + for b in (0.6, 0.75, 0.9) for s in (2.0, 4.0)} + + +def evolve_full(q_grid, psi_full, beta, smooth): + x = (q_grid + D_STEPS[0] * psi_full) % N + v = psi_full.copy() + crossed = np.zeros(len(x), bool) + e3f = None + for i in range(1, len(D_STEPS)): + dD = D_STEPS[i] - D_STEPS[i - 1] + x = (x + v * dD) % N + rho = fields.cic_deposit(x, N) + if e3f is None or i % 3 == 0: + clean = gaussian_filter(np.log1p(rho), 1.0, mode="wrap") + e3f = fields.tidal_frame(clean, smooth=smooth) + hit = (~crossed) & (gather(rho, x) > RHO_C) + if hit.any(): + idx = np.clip(np.round(x[hit]).astype(int), 0, N - 1) % N + e = e3f[idx[:, 0], idx[:, 1], idx[:, 2]] + par = e * np.sum(v[hit] * e, axis=1, keepdims=True) + v[hit] = par + (1 - beta) * (v[hit] - par) + crossed |= hit + return x + + +def run_seed(args): + seed, names = args + rng = np.random.default_rng(seed) + x_pm, v_pm, tidx, ta, tpos, q_tr, psi_tr = pm.pm_sim( + N, L, rng, n_steps=90, track=50000) + seedseq = np.random.default_rng(seed).integers(0, 2 ** 32) + _, pos_q = fields.zeldovich_box(N, L, 0.0, np.random.default_rng(seedseq)) + _, pos_d = fields.zeldovich_box(N, L, 1.0, np.random.default_rng(seedseq)) + psi_full = pos_d - pos_q + psi_full -= N * np.round(psi_full / N) + truth = tpos[-1].astype(float) + rec = {"seed": seed} + xz = (q_tr + D_STEPS[-1] * psi_tr) % N + err = xz - truth; err -= N * np.round(err / N) + rec["za_all"] = float(np.median(np.linalg.norm(err, axis=1))) + for name in names: + cfg = CONFIGS[name] + xf = evolve_full(pos_q, psi_full, cfg["beta"], cfg["smooth"]) + err = xf[tidx] - truth; err -= N * np.round(err / N) + rec[f"{name}_all"] = float(np.median(np.linalg.norm(err, axis=1))) + return rec + + +def main(): + t0 = time.time() + names = list(CONFIGS) + with Pool(2) as pool: + sel = pool.map(run_seed, [(s, names) for s in SEL_SEEDS]) + scores = {n: float(np.mean([r[f"{n}_all"] for r in sel])) for n in names} + for n, v in sorted(scores.items(), key=lambda kv: kv[1]): + print(f" {n}: {v:.3f}") + best = min(scores, key=scores.get) + print("validating", best) + with Pool(3) as pool: + val = pool.map(run_seed, [(s, [best]) for s in VAL_SEEDS]) + za, zae = jk([r["za_all"] for r in val]) + bm, bme = jk([r[f"{best}_all"] for r in val]) + (ROOT / "artifacts" / "e5d_results.json").write_text(json.dumps( + {"scores": scores, "best": best, "validation": val}, indent=1)) + md = ["# E5d — final knob round (beta x frame smoothing)\n", + f"Selection scores (seeds 2-3): " + + ", ".join(f"{n}={v:.3f}" for n, v in + sorted(scores.items(), key=lambda kv: kv[1])) + "\n", + f"\nHeld-out (seeds 4-6): ZA {za:.2f} +/- {zae:.2f}; " + f"**{best}** {bm:.2f} +/- {bme:.2f} " + f"({100*(bm-za)/za:+.0f}% vs ZA; oracle bound 4.47; " + f"E5c winner 4.64).\n"] + (ROOT / "docs" / "E5d-report.md").write_text("\n".join(md)) + print(f"held-out: ZA {za:.2f}, {best} {bm:.2f}") + print(f"total {time.time()-t0:.0f}s") + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e6.py b/research/cosmic-web/scripts/run_e6.py new file mode 100644 index 00000000000..badf03384be --- /dev/null +++ b/research/cosmic-web/scripts/run_e6.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""E6 (H-transfer): does the frozen model transfer without re-calibration? + +The model card declares the recipe (beta=0.6, 2 Mpc/h frames, rho_c=5) +tested only at 1 Mpc/h voxels, sigma8=0.8. Three transfer conditions, all +knobs FROZEN: + + coarse : L=256 Mpc/h on the same 128^3 grid (2 Mpc/h voxels) + s8lo : sigma8 = 0.6 (weaker clustering) + s8hi : sigma8 = 1.0 (stronger clustering) + +Per condition, seeds {4,5,6} (held-out family): PM truth, ZA baseline, +frozen model. Metric: overall median per-particle error and the 0-2 vox +web bin. Success: the model's advantage over ZA persists (negative delta) +in every condition. Report -> docs/E6-report.md. +""" + +import json +import sys +import time +from multiprocessing import Pool +from pathlib import Path + +import numpy as np +from scipy.ndimage import gaussian_filter +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) + +import fields, lift, pm, spines # noqa: E402 +from run_e5 import D_STEPS, N, gather, jk # noqa: E402 + +SEEDS = [4, 5, 6] +BETA, SMOOTH, RHO_C = 0.6, 2.0, 5.0 +CONDITIONS = { + "base": dict(L=128.0, sigma8=0.8), # reference; matches E5d setup + "coarse": dict(L=256.0, sigma8=0.8), + "s8lo": dict(L=128.0, sigma8=0.6), + "s8hi": dict(L=128.0, sigma8=1.0), +} +_AXES = lift.hemisphere_axes(42) + + +def evolve_frozen(q_grid, psi_full): + x = (q_grid + D_STEPS[0] * psi_full) % N + v = psi_full.copy() + crossed = np.zeros(len(x), bool) + e3f = None + for i in range(1, len(D_STEPS)): + dD = D_STEPS[i] - D_STEPS[i - 1] + x = (x + v * dD) % N + rho = fields.cic_deposit(x, N) + if e3f is None or i % 3 == 0: + clean = gaussian_filter(np.log1p(rho), 1.0, mode="wrap") + e3f = fields.tidal_frame(clean, smooth=SMOOTH) + hit = (~crossed) & (gather(rho, x) > RHO_C) + if hit.any(): + idx = np.clip(np.round(x[hit]).astype(int), 0, N - 1) % N + e = e3f[idx[:, 0], idx[:, 1], idx[:, 2]] + par = e * np.sum(v[hit] * e, axis=1, keepdims=True) + v[hit] = par + (1 - BETA) * (v[hit] - par) + crossed |= hit + return x + + +def run_job(args): + cond, seed = args + cfg = CONDITIONS[cond] + rng = np.random.default_rng(seed) + x_pm, v_pm, tidx, ta, tpos, q_tr, psi_tr = pm.pm_sim( + N, cfg["L"], rng, n_steps=90, track=50000, sigma8=cfg["sigma8"]) + seedseq = np.random.default_rng(seed).integers(0, 2 ** 32) + _, pos_q = fields.zeldovich_box(N, cfg["L"], 0.0, + np.random.default_rng(seedseq), + sigma8=cfg["sigma8"]) + _, pos_d = fields.zeldovich_box(N, cfg["L"], 1.0, + np.random.default_rng(seedseq), + sigma8=cfg["sigma8"]) + psi_full = pos_d - pos_q + psi_full -= N * np.round(psi_full / N) + + truth = tpos[-1].astype(float) + clean = gaussian_filter(np.log1p(fields.cic_deposit(x_pm, N)), 1.0, + mode="wrap") + sp = spines.skeleton_points(spines.extract_matched( + spines.lifted_ridgeness(lift.orientation_score(clean, _AXES, 6.0, 1.5)), + 2400, lo_frac=1.0)) + d_spine = cKDTree(sp).query(truth)[0] + web = d_spine < 2.0 + + rec = {"cond": cond, "seed": seed} + xz = (q_tr + D_STEPS[-1] * psi_tr) % N + for name, xm in [("za", xz), ("model", evolve_frozen(pos_q, psi_full)[tidx])]: + err = xm - truth + err -= N * np.round(err / N) + err = np.linalg.norm(err, axis=1) + rec[f"{name}_all"] = float(np.median(err)) + rec[f"{name}_web"] = float(np.median(err[web])) + return rec + + +def main(): + t0 = time.time() + jobs = [(c, s) for c in CONDITIONS for s in SEEDS] + with Pool(6) as pool: + recs = pool.map(run_job, jobs) + print(f"{len(jobs)} jobs in {time.time()-t0:.0f}s") + (ROOT / "artifacts" / "e6_results.json").write_text(json.dumps(recs, indent=1)) + report(recs) + + +def report(recs): + md = ["# E6 — transfer test of the frozen model (no re-calibration)\n", + f"Recipe frozen at β={BETA}, frame smoothing {SMOOTH} Mpc/h, " + f"ρ_c={RHO_C}; {len(SEEDS)} held-out seeds per condition. " + "Voxel sizes differ across conditions, so compare the RELATIVE " + "advantage over ZA, not absolute voxel errors.\n"] + md.append("| condition | ZA all | model all | Δ all | ZA web | " + "model web | Δ web |") + md.append("|---|---|---|---|---|---|---|") + for cond in CONDITIONS: + rs = [r for r in recs if r["cond"] == cond] + za, _ = jk([r["za_all"] for r in rs]) + mo, _ = jk([r["model_all"] for r in rs]) + zw, _ = jk([r["za_web"] for r in rs]) + mw, _ = jk([r["model_web"] for r in rs]) + md.append(f"| {cond} | {za:.2f} | {mo:.2f} | " + f"**{100*(mo-za)/za:+.0f}%** | {zw:.2f} | {mw:.2f} | " + f"**{100*(mw-zw)/zw:+.0f}%** |") + md.append("\nSuccess criterion: Δ ≤ 0 in every condition (advantage " + "persists with frozen knobs). Degradation localizes which " + "parameter is condition-dependent.\n") + out = ROOT / "docs" / "E6-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + print("\n".join(md[3:])) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e7.py b/research/cosmic-web/scripts/run_e7.py new file mode 100644 index 00000000000..34d04b8b8cc --- /dev/null +++ b/research/cosmic-web/scripts/run_e7.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""E7: the model card's last two unverified scope rows. + +H-LCDM : the frozen model's advantage persists under real-universe + expansion (flat LCDM, Omega_m = 0.31) — 3 held-out seeds, 128^3. +H-highres: it persists at 0.5 Mpc/h voxels (N = 256, same physical box, + EdS) — one seed (smoke-level, indicative). + +All knobs frozen (beta = 0.6, frame smoothing 2 Mpc/h, rho_c = 5). The +effective models are parametrized by the growth factor D directly, so +cosmology enters only through the truth run and the a<->D mapping; the +web bin is defined in physical Mpc/h (< 2). Report -> docs/E7-report.md. +""" + +import json +import sys +import time +from multiprocessing import Pool +from pathlib import Path + +import numpy as np +from scipy.ndimage import gaussian_filter, map_coordinates +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +import fields, lift, pm, spines # noqa: E402 + +D_STEPS = np.arange(0.10, 1.0001, 0.05) +BETA, SMOOTH_MPC, RHO_C = 0.6, 2.0, 5.0 +_AXES = lift.hemisphere_axes(42) + +JOBS = [ + dict(name="lcdm_s4", N=128, L=128.0, omega_m=0.31, seed=4), + dict(name="lcdm_s5", N=128, L=128.0, omega_m=0.31, seed=5), + dict(name="lcdm_s6", N=128, L=128.0, omega_m=0.31, seed=6), + dict(name="hires_s4", N=256, L=128.0, omega_m=1.0, seed=4), +] + + +def gather_g(grid, pos, N): + return map_coordinates(grid, [pos[:, 0] % N, pos[:, 1] % N, + pos[:, 2] % N], order=1, mode="wrap") + + +def evolve_frozen(q_grid, psi_full, N, vox_mpc): + x = (q_grid + D_STEPS[0] * psi_full) % N + v = psi_full.copy() + crossed = np.zeros(len(x), bool) + e3f = None + for i in range(1, len(D_STEPS)): + dD = D_STEPS[i] - D_STEPS[i - 1] + x = (x + v * dD) % N + rho = fields.cic_deposit(x, N) + if e3f is None or i % 3 == 0: + clean = gaussian_filter(np.log1p(rho), 1.0, mode="wrap") + e3f = fields.tidal_frame(clean, smooth=SMOOTH_MPC / vox_mpc) + hit = (~crossed) & (gather_g(rho, x, N) > RHO_C) + if hit.any(): + idx = np.clip(np.round(x[hit]).astype(int), 0, N - 1) % N + e = e3f[idx[:, 0], idx[:, 1], idx[:, 2]] + par = e * np.sum(v[hit] * e, axis=1, keepdims=True) + v[hit] = par + (1 - BETA) * (v[hit] - par) + crossed |= hit + return x + + +def run_job(job): + t0 = time.time() + N, L = job["N"], job["L"] + vox = L / N + rng = np.random.default_rng(job["seed"]) + x_pm, v_pm, tidx, ta, tpos, q_tr, psi_tr = pm.pm_sim( + N, L, rng, n_steps=90, track=50000, omega_m=job["omega_m"]) + seedseq = np.random.default_rng(job["seed"]).integers(0, 2 ** 32) + _, pos_q = fields.zeldovich_box(N, L, 0.0, np.random.default_rng(seedseq)) + _, pos_d = fields.zeldovich_box(N, L, 1.0, np.random.default_rng(seedseq)) + psi_full = pos_d - pos_q + psi_full -= N * np.round(psi_full / N) + + truth = tpos[-1].astype(float) + clean = gaussian_filter(np.log1p(fields.cic_deposit(x_pm, N)), 1.0, + mode="wrap") + n_spine = int(2400 / vox) # constant physical spine budget + sp = spines.skeleton_points(spines.extract_matched( + spines.lifted_ridgeness(lift.orientation_score(clean, _AXES, + 6.0 / vox, 1.5 / vox)), + n_spine, lo_frac=1.0)) + d_spine = cKDTree(sp).query(truth)[0] * vox # physical Mpc/h + web = d_spine < 2.0 + + rec = {"name": job["name"], "vox": vox} + xz = (q_tr + D_STEPS[-1] * psi_tr) % N + for name, xm in [("za", xz), + ("model", evolve_frozen(pos_q, psi_full, N, vox)[tidx])]: + err = xm - truth + err -= N * np.round(err / N) + err = np.linalg.norm(err, axis=1) * vox # physical Mpc/h + rec[f"{name}_all"] = float(np.median(err)) + rec[f"{name}_web"] = float(np.median(err[web])) + rec["runtime_s"] = round(time.time() - t0) + print(f" {job['name']}: ZA {rec['za_all']:.2f} model " + f"{rec['model_all']:.2f} Mpc/h [{rec['runtime_s']}s]") + return rec + + +def main(): + t0 = time.time() + with Pool(4) as pool: + recs = pool.map(run_job, JOBS) + print(f"total {time.time()-t0:.0f}s") + (ROOT / "artifacts" / "e7_results.json").write_text(json.dumps(recs, indent=1)) + report(recs) + + +def report(recs): + md = ["# E7 — final scope rows: LCDM dynamics and sub-Mpc resolution\n", + "Frozen knobs throughout; errors in PHYSICAL Mpc/h; web = within " + "2 Mpc/h of the spine network.\n"] + md.append("| run | voxel (Mpc/h) | ZA all | model all | Δ all " + "| ZA web | model web | Δ web |") + md.append("|---|---|---|---|---|---|---|---|") + lcdm = [r for r in recs if r["name"].startswith("lcdm")] + za = np.mean([r["za_all"] for r in lcdm]) + mo = np.mean([r["model_all"] for r in lcdm]) + zw = np.mean([r["za_web"] for r in lcdm]) + mw = np.mean([r["model_web"] for r in lcdm]) + md.append(f"| LCDM Ωm=0.31 (3 seeds) | 1.0 | {za:.2f} | {mo:.2f} " + f"| **{100*(mo-za)/za:+.0f}%** | {zw:.2f} | {mw:.2f} " + f"| **{100*(mw-zw)/zw:+.0f}%** |") + hi = [r for r in recs if r["name"].startswith("hires")][0] + md.append(f"| EdS 0.5 Mpc vox (1 seed) | 0.5 | {hi['za_all']:.2f} " + f"| {hi['model_all']:.2f} " + f"| **{100*(hi['model_all']-hi['za_all'])/hi['za_all']:+.0f}%** " + f"| {hi['za_web']:.2f} | {hi['model_web']:.2f} " + f"| **{100*(hi['model_web']-hi['za_web'])/hi['za_web']:+.0f}%** |") + out = ROOT / "docs" / "E7-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + print("\n".join(md[2:])) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e7b.py b/research/cosmic-web/scripts/run_e7b.py new file mode 100644 index 00000000000..d4f85e8eaff --- /dev/null +++ b/research/cosmic-web/scripts/run_e7b.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +"""E7b: two more high-res seeds to convert the 0.5 Mpc row from n=1 to n=3.""" +import json, sys +from multiprocessing import Pool +from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "src")) +from run_e7 import run_job + +JOBS = [dict(name=f"hires_s{s}", N=256, L=128.0, omega_m=1.0, seed=s) + for s in (5, 6)] + +if __name__ == "__main__": + with Pool(2) as pool: + recs = pool.map(run_job, JOBS) + (ROOT / "artifacts" / "e7b_results.json").write_text(json.dumps(recs, indent=1)) + for r in recs: + print(r["name"], "ZA", r["za_all"], "model", r["model_all"]) diff --git a/research/cosmic-web/scripts/run_e8.py b/research/cosmic-web/scripts/run_e8.py new file mode 100644 index 00000000000..0d716f502f3 --- /dev/null +++ b/research/cosmic-web/scripts/run_e8.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""E8 (H-field): does the frozen model deliver better DENSITY FIELDS, +not just better particle positions? + +The model card's stated downstream use is mock-catalogue generation, which +consumes fields, not trajectories. Transport error and field fidelity can +dissociate. Per seed: evolve the FULL particle set under ZA, isotropic +sticking, and the frozen transverse-damping model; deposit densities; +compare each with the PM truth field via + r(k) = P_cross / sqrt(P_model P_truth) (phase fidelity) + T(k) = sqrt(P_model / P_truth) (amplitude fidelity) +in logarithmic k bins to half-Nyquist. Report -> docs/E8-report.md. +""" + +import json +import sys +import time +from multiprocessing import Pool +from pathlib import Path + +import numpy as np +from scipy.ndimage import gaussian_filter + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) + +import fields, pm # noqa: E402 +from run_e5 import D_STEPS, N, gather # noqa: E402 + +SEEDS = [4, 5, 6] +BETA, SMOOTH, RHO_C = 0.6, 2.0, 5.0 +KBINS = np.geomspace(2 * np.pi / 128, np.pi / 2, 9) # h/Mpc, to half-Nyquist + + +def evolve(q_grid, psi_full, mode): + x = (q_grid + D_STEPS[0] * psi_full) % N + v = psi_full.copy() + crossed = np.zeros(len(x), bool) + e3f = None + for i in range(1, len(D_STEPS)): + dD = D_STEPS[i] - D_STEPS[i - 1] + x = (x + v * dD) % N + if mode == "za": + continue + rho = fields.cic_deposit(x, N) + if mode == "damp" and (e3f is None or i % 3 == 0): + clean = gaussian_filter(np.log1p(rho), 1.0, mode="wrap") + e3f = fields.tidal_frame(clean, smooth=SMOOTH) + hit = (~crossed) & (gather(rho, x) > RHO_C) + if hit.any(): + if mode == "stick": + v[hit] = 0.0 + else: + idx = np.clip(np.round(x[hit]).astype(int), 0, N - 1) % N + e = e3f[idx[:, 0], idx[:, 1], idx[:, 2]] + par = e * np.sum(v[hit] * e, axis=1, keepdims=True) + v[hit] = par + (1 - BETA) * (v[hit] - par) + crossed |= hit + return x + + +def spectra(delta_a, delta_b): + """Binned cross r(k) and transfer T(k) of field a against truth b.""" + fa, fb = np.fft.rfftn(delta_a), np.fft.rfftn(delta_b) + k1 = np.fft.fftfreq(N, d=1.0) * 2 * np.pi # voxel = 1 Mpc/h here + kx, ky, kz = np.meshgrid(k1, k1, k1[: N // 2 + 1], indexing="ij") + k = np.sqrt(kx ** 2 + ky ** 2 + kz ** 2) + r_out, t_out = [], [] + for lo, hi in zip(KBINS[:-1], KBINS[1:]): + m = (k >= lo) & (k < hi) + paa = float(np.mean(np.abs(fa[m]) ** 2)) + pbb = float(np.mean(np.abs(fb[m]) ** 2)) + pab = float(np.mean((fa[m] * np.conj(fb[m])).real)) + r_out.append(pab / np.sqrt(paa * pbb)) + t_out.append(np.sqrt(paa / pbb)) + return r_out, t_out + + +def run_seed(seed): + rng = np.random.default_rng(seed) + x_pm, *_ = pm.pm_sim(N, 128.0, rng, n_steps=90, track=1) + seedseq = np.random.default_rng(seed).integers(0, 2 ** 32) + _, pos_q = fields.zeldovich_box(N, 128.0, 0.0, np.random.default_rng(seedseq)) + _, pos_d = fields.zeldovich_box(N, 128.0, 1.0, np.random.default_rng(seedseq)) + psi_full = pos_d - pos_q + psi_full -= N * np.round(psi_full / N) + + d_truth = fields.cic_deposit(x_pm, N) - 1.0 + rec = {"seed": seed} + for mode in ("za", "stick", "damp"): + d_m = fields.cic_deposit(evolve(pos_q, psi_full, mode), N) - 1.0 + r, t = spectra(d_m, d_truth) + rec[f"{mode}_r"] = r + rec[f"{mode}_T"] = t + return rec + + +def main(): + t0 = time.time() + with Pool(3) as pool: + recs = pool.map(run_seed, SEEDS) + print(f"{len(SEEDS)} seeds in {time.time()-t0:.0f}s") + (ROOT / "artifacts" / "e8_results.json").write_text(json.dumps(recs, indent=1)) + report(recs) + + +def report(recs): + kc = np.sqrt(KBINS[:-1] * KBINS[1:]) + md = ["# E8 — field-level fidelity (the mock-catalogue criterion)\n", + f"Full-set density fields vs PM truth, {len(SEEDS)} seeds, " + "1 Mpc/h voxels. r(k): phase/structure fidelity (1 = perfect); " + "T(k): amplitude fidelity (1 = unbiased).\n"] + for stat in ("r", "T"): + md.append(f"\n## {'Cross-correlation r(k)' if stat=='r' else 'Transfer T(k)'}\n") + md.append("| k (h/Mpc) | ZA | isotropic stick | transverse damp |") + md.append("|---|---|---|---|") + for i, k in enumerate(kc): + row = [f"| {k:.2f} "] + for mode in ("za", "stick", "damp"): + vals = [r[f"{mode}_{stat}"][i] for r in recs] + row.append(f"| {np.mean(vals):.3f} ") + md.append("".join(row) + "|") + out = ROOT / "docs" / "E8-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + print("\n".join(md[3:16])) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/run_e9.py b/research/cosmic-web/scripts/run_e9.py new file mode 100644 index 00000000000..f1ce39f5183 --- /dev/null +++ b/research/cosmic-web/scripts/run_e9.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""E9 (referee point 1): the literature baselines — 2LPT and MUSCLE. + +The decisive comparison the program lacked: the frozen transverse-damping +model against the field's actual improved-transport methods, on identical +initial conditions, both per-particle and field-level. + +Models: ZA | 2LPT (Bouchet/Scoccimarro EdS coefficients) | MUSCLE +(Neyrinck 2016 multiscale spherical collapse) | transverse damping +(frozen recipe). Truth: PM N-body. Seeds {4,5,6}. +Report -> docs/E9-report.md. +""" + +import json +import sys +import time +from multiprocessing import Pool +from pathlib import Path + +import numpy as np +from scipy.ndimage import gaussian_filter +from scipy.spatial import cKDTree + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) + +import fields, lift, pm, spines # noqa: E402 +from run_e5 import D_STEPS, N, gather, jk # noqa: E402 +from run_e8 import KBINS, evolve as evolve_damp, spectra # noqa: E402 + +SEEDS = [4, 5, 6] +KSHOW = [2, 4, 5, 6] # k-bin indices to tabulate + + +def run_seed(seed): + rng = np.random.default_rng(seed) + x_pm, v_pm, tidx, ta, tpos, q_tr, psi_tr = pm.pm_sim( + N, 128.0, rng, n_steps=90, track=50000) + seedseq = np.random.default_rng(seed).integers(0, 2 ** 32) + _, pos_q = fields.zeldovich_box(N, 128.0, 0.0, np.random.default_rng(seedseq)) + _, pos_d = fields.zeldovich_box(N, 128.0, 1.0, np.random.default_rng(seedseq)) + psi_full = pos_d - pos_q + psi_full -= N * np.round(psi_full / N) + + truth = tpos[-1].astype(float) + d_truth = fields.cic_deposit(x_pm, N) - 1.0 + clean = gaussian_filter(np.log1p(1 + d_truth), 1.0, mode="wrap") + sp = spines.skeleton_points(spines.extract_matched( + spines.lifted_ridgeness(lift.orientation_score(clean, lift.hemisphere_axes(42), 6.0, 1.5)), + 2400, lo_frac=1.0)) + d_spine = cKDTree(sp).query(truth)[0] + web = d_spine < 2.0 + + models = { + "za": (pos_q + D_STEPS[-1] * psi_full) % N, + "lpt2": fields.lpt2_positions(N, 128.0, np.random.default_rng(seedseq)), + "muscle": fields.muscle_positions(N, 128.0, np.random.default_rng(seedseq)), + "damp": evolve_damp(pos_q, psi_full, "damp"), + } + rec = {"seed": seed} + for name, xf in models.items(): + xm = xf[tidx] + err = xm - truth + err -= N * np.round(err / N) + err = np.linalg.norm(err, axis=1) + rec[f"{name}_all"] = float(np.median(err)) + rec[f"{name}_web"] = float(np.median(err[web])) + r, t = spectra(fields.cic_deposit(xf, N) - 1.0, d_truth) + rec[f"{name}_r"] = r + rec[f"{name}_T"] = t + return rec + + +def main(): + t0 = time.time() + with Pool(3) as pool: + recs = pool.map(run_seed, SEEDS) + print(f"{len(SEEDS)} seeds in {time.time()-t0:.0f}s") + (ROOT / "artifacts" / "e9_results.json").write_text(json.dumps(recs, indent=1)) + report(recs) + + +def report(recs): + kc = np.sqrt(KBINS[:-1] * KBINS[1:]) + names = ["za", "lpt2", "muscle", "damp"] + labels = {"za": "ZA", "lpt2": "2LPT", "muscle": "MUSCLE", + "damp": "transverse damp"} + md = ["# E9 — literature baselines: 2LPT and MUSCLE vs the frozen model\n", + f"Identical initial conditions, PM truth, {len(SEEDS)} seeds. " + "Transport errors in voxels (= h⁻¹Mpc); web = within 2 of the " + "spine network.\n", + "## Per-particle transport error (median)\n"] + md.append("| model | all | web |") + md.append("|---|---|---|") + for n in names: + a, ae = jk([r[f"{n}_all"] for r in recs]) + w, we = jk([r[f"{n}_web"] for r in recs]) + md.append(f"| {labels[n]} | {a:.2f} ± {ae:.2f} | {w:.2f} ± {we:.2f} |") + for stat, title in [("r", "Cross-correlation r(k)"), + ("T", "Transfer T(k)")]: + md.append(f"\n## {title}\n") + md.append("| k (h/Mpc) | " + " | ".join(labels[n] for n in names) + " |") + md.append("|---|" + "---|" * len(names)) + for i in KSHOW: + row = [f"| {kc[i]:.2f} "] + for n in names: + row.append(f"| {np.mean([r[f'{n}_{stat}'][i] for r in recs]):.3f} ") + md.append("".join(row) + "|") + out = ROOT / "docs" / "E9-report.md" + out.write_text("\n".join(md) + "\n") + print(f"wrote {out}") + print("\n".join(md[3:12])) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/scripts/smoke_test.py b/research/cosmic-web/scripts/smoke_test.py new file mode 100644 index 00000000000..3bd6d58da12 --- /dev/null +++ b/research/cosmic-web/scripts/smoke_test.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Minimal self-checks that fail loudly if core logic breaks.""" + +import sys +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +import fields, lift, spines, metrics # noqa: E402 + +rng = np.random.default_rng(0) + +# --- Voronoi truth exists and lies in the box +segs, verts = fields.voronoi_web(64, 16, rng) +assert len(segs) > 10 and len(verts) > 4, "voronoi web too sparse" +pts = fields.sample_segments(segs) +assert pts.min() >= 0 and pts.max() <= 64 + +# --- metrics are perfect on identical sets +m = metrics.spine_metrics(pts, pts, r0=2.0) +assert m["completeness"] == 1.0 and m["purity"] == 1.0 +j = metrics.junction_metrics(verts, verts, r0=3.0) +assert j["f1"] == 1.0 + +# --- orientation score peaks at the correct axis on a straight ridge +N = 48 +field = np.zeros((N, N, N)) +field[N // 2, N // 2, :] = 1.0 # bright line along z +from scipy.ndimage import gaussian_filter +field = gaussian_filter(field, 1.0, mode="wrap") +axes = lift.hemisphere_axes(42) +U = lift.orientation_score(field, axes, sig_par=6.0, sig_perp=1.5, scales=(1.0,)) +best_axis = axes[np.argmax(U[:, N // 2, N // 2, N // 2])] +assert abs(best_axis[2]) > 0.95, f"score not aligned with ridge: {best_axis}" + +# --- skeleton of a clean ridge recovers the line +R = spines.lifted_ridgeness(U) +skel = spines.extract_matched(R, n_target=N) +sk_pts = spines.skeleton_points(skel) +d = np.abs(sk_pts[:, :2] - N // 2).max() +assert d <= 2, f"skeleton strays {d} voxels from a clean straight ridge" + +print("all smoke checks passed") diff --git a/research/cosmic-web/scripts/write_e0_report.py b/research/cosmic-web/scripts/write_e0_report.py new file mode 100644 index 00000000000..6eb513725ee --- /dev/null +++ b/research/cosmic-web/scripts/write_e0_report.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +"""Generate docs/E0-report.md from artifacts/e0_results_{straight,curved}.json.""" + +import json +import sys +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parents[1] + + +def load(tag): + p = ROOT / "artifacts" / f"e0_results_{tag}.json" + return json.loads(p.read_text()) if p.exists() else None + + +def agg(results, name, sub, key, level): + return [r["scores"][name][sub][key] for r in results if r["n_gal"] == level] + + +def paired_delta(results, sub, key, level): + """Per-seed (se3_lift - hessian) at one level.""" + return [r["scores"]["se3_lift"][sub][key] - r["scores"]["hessian"][sub][key] + for r in results if r["n_gal"] == level] + + +def table(data): + res = data["results"] + levels = sorted({r["n_gal"] for r in res}) + lines = ["| n_gal | method | completeness mean ± sd | median (p10–p90) " + "| purity | junction F1 |", + "|---|---|---|---|---|---|"] + for L in levels: + for name in ("se3_lift", "hessian"): + c = np.array(agg(res, name, "M1", "completeness", L)) + p = agg(res, name, "M1", "purity", L) + f = agg(res, name, "M2", "f1", L) + lines.append( + f"| {L} | {name} | {c.mean():.3f} ± {c.std():.3f} " + f"| {np.median(c):.3f} ({np.percentile(c, 10):.3f}–" + f"{np.percentile(c, 90):.3f}) " + f"| {np.mean(p):.3f} ± {np.std(p):.3f} " + f"| {np.mean(f):.3f} ± {np.std(f):.3f} |") + return "\n".join(lines) + + +def delta_table(data): + """Paired per-seed differences (se3_lift - hessian), the H1 evidence.""" + from scipy.stats import wilcoxon + res = data["results"] + levels = sorted({r["n_gal"] for r in res}) + lines = ["| n_gal | n seeds | ΔC mean | ΔC median (p10–p90) | lift wins " + "| Wilcoxon p (ΔC) | ΔjF1 mean | Wilcoxon p (ΔjF1) |", + "|---|---|---|---|---|---|---|---|"] + for L in levels: + dC = np.array(paired_delta(res, "M1", "completeness", L)) + dF = np.array(paired_delta(res, "M2", "f1", L)) + pC = wilcoxon(dC).pvalue if np.any(dC) else 1.0 + pF = wilcoxon(dF).pvalue if np.any(dF) else 1.0 + lines.append( + f"| {L} | {len(dC)} | **{dC.mean():+.3f}** " + f"| {np.median(dC):+.3f} ({np.percentile(dC, 10):+.3f}–" + f"{np.percentile(dC, 90):+.3f}) " + f"| {int((dC > 0).sum())}/{len(dC)} | {pC:.2g} " + f"| {dF.mean():+.3f} | {pF:.2g} |") + return "\n".join(lines) + + +def delta_figure(data, tag): + """Box + jittered scatter of paired deltas per level: the key plot.""" + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + res = data["results"] + levels = sorted({r["n_gal"] for r in res}) + rng = np.random.default_rng(0) + fig, axs = plt.subplots(1, 2, figsize=(12, 4.6)) + for ax, sub, key, title in [ + (axs[0], "M1", "completeness", + "Δ completeness (SE(3) lift − Hessian), paired per seed"), + (axs[1], "M2", "f1", + "Δ junction F1 (SE(3) lift − Hessian), paired per seed")]: + ds = [paired_delta(res, sub, key, L) for L in levels] + ax.axhline(0, color="k", lw=0.8) + ax.boxplot(ds, positions=range(len(levels)), widths=0.5, + showfliers=False) + for i, d in enumerate(ds): + ax.plot(i + rng.uniform(-0.15, 0.15, len(d)), d, ".", + ms=3, color="tab:blue", alpha=0.4) + ax.set_xticks(range(len(levels)), [str(L) for L in levels]) + ax.set_xlabel("n_gal (galaxies per $128^3$ box)") + ax.set_ylabel("Δ (dimensionless, + favours lift)") + ax.set_title(title, fontsize=10) + ax.grid(alpha=0.3, axis="y") + fig.suptitle(f"E0a-{tag}: paired per-seed differences over " + f"{max(len(d) for d in ds)} held-out seeds " + "(box = quartiles, dots = individual seeds)") + fig.tight_layout() + fig.savefig(ROOT / "docs" / "figures" / f"e0_delta_{tag}.png", dpi=130) + plt.close(fig) + + +def spark(vals, lo=0.0, hi=1.0): + blocks = "▁▂▃▄▅▆▇█" + return "".join(blocks[int(np.clip((v - lo) / (hi - lo), 0, 0.999) * 8)] + for v in vals) + + +def main(): + straight, curved = load("straight"), load("curved") + assert straight, "run scripts/run_e0.py first" + + md = ["# E0a — orientation-lifted vs Hessian filament extraction " + "(Voronoi toy, exact truth)\n"] + md.append( + "**Question (H1):** does lifting a galaxy density field to the " + "position–orientation manifold R³×S² recover filament spines better " + "than the quadratic (Hessian) orientation response, at identical " + "post-processing and matched skeleton length?\n\n" + "**Setup.** Voronoi filament model in a 128³ box (1 voxel = 1 h⁻¹Mpc " + "at the nominal L=128 h⁻¹Mpc): filaments are Voronoi-cell edges " + "(straight variant) or Bézier-bent edges (curved variant, sag ≈ 15% " + "of length); galaxies = arclength-uniform points with transverse " + "Gaussian scatter σ=0.8 vox + 15% node clumps + 25% uniform " + "background; field = CIC deposit → log(1+δ) → Gaussian smooth σ=1. " + "Exact truth: the generating curves and vertices.\n\n" + "**Protocol.** Each method gets 3 candidate configs; the best on " + "calibration seed 1 (mean C+P+jF1 over sampling levels) is frozen, " + "then evaluated on held-out seeds. Both methods share hysteresis " + "thresholding, matched-length skeletonization (mask volume " + "bisection, tube area 9 vox²), speckle pruning ≥4 vox, and junction " + "clustering.\n") + + md.append("\n## Metric definitions\n\n" + "- **Completeness (M1)** = fraction of truth-curve sample " + "points within r₀=2 vox of the estimated skeleton.\n" + "- **Purity (M1)** = fraction of estimated skeleton voxels " + "within r₀=2 vox of a truth curve.\n" + "- **Junction F1 (M2)** = harmonic mean of precision/recall of " + "greedy one-to-one matching of estimated junction centroids to " + "true Voronoi vertices within r₀=3 vox.\n" + "- **Δ(lift−hess) C** = per-seed paired completeness " + "difference, mean over held-out seeds; (k/n seeds +) counts " + "seeds where the lift wins.\n" + "- All numbers: mean ± sd over held-out eval seeds; matched " + "skeleton length ⇒ completeness/purity trade on equal " + "footing.\n") + + for tag, data in [("straight", straight), ("curved", curved)]: + if data is None: + continue + md.append(f"\n## {tag.capitalize()} filaments\n") + md.append(f"Chosen configs — lift: `{data['chosen_params']['se3_lift']}`, " + f"hessian: `{data['chosen_params']['hessian']}`.\n") + md.append("\n### Per-method summary\n") + md.append(table(data)) + md.append("\n### Paired differences (the H1 evidence)\n") + md.append(delta_table(data)) + delta_figure(data, tag) + md.append(f"\n![E0a {tag}: box plots with per-seed points of paired " + f"delta completeness and delta junction F1 vs galaxy " + f"count; positive favours the lift]" + f"(figures/e0_delta_{tag}.png)\n") + md.append(f"![E0a {tag}: completeness, purity, junction F1 vs " + f"galaxy count, both methods, mean ± sd over held-out " + f"seeds](figures/e0_curves_{tag}.png)\n") + md.append(f"![E0a {tag}: 6-voxel slab of the galaxy field with truth " + f"(red) and extracted skeletons (blue) for both methods]" + f"(figures/e0_slice_{tag}.png)\n") + + # verdict, computed from the numbers + md.append("\n## Verdict on H1 (toy-level)\n") + res = straight["results"] + levels = sorted({r["n_gal"] for r in res}) + verdict = [] + from scipy.stats import wilcoxon + for tag, data in [("straight", straight), ("curved", curved)]: + if data is None: + continue + r = data["results"] + d = paired_delta(r, "M1", "completeness", levels[0]) + dd = paired_delta(r, "M1", "completeness", levels[-1]) + p_sparse = wilcoxon(d).pvalue if len(d) > 5 else float("nan") + p_dense = wilcoxon(dd).pvalue if len(dd) > 5 else float("nan") + verdict.append( + f"- **{tag}**: at the sparsest level (n_gal={levels[0]}) the " + f"lift wins completeness on {sum(x > 0 for x in d)}/{len(d)} " + f"held-out seeds, mean Δ = {np.mean(d):+.3f} (Wilcoxon " + f"signed-rank p = {p_sparse:.2g}); at the densest level " + f"Δ = {np.mean(dd):+.3f} (p = {p_dense:.2g}).") + md.append("\n".join(verdict)) + md.append( + "\n**Not supported (CORRECTED result).** An earlier version of this " + "report claimed a sparse-regime lift win (+0.07 completeness at " + "n_gal=2500). That was an evaluation artifact: the original " + "extractor targeted mask volume, and realized skeleton lengths " + "differed systematically between methods (lift ~19% longer at " + "sparse levels) — longer skeletons buy completeness mechanically. " + "With skeleton length enforced (iterative correction to the " + "target), the Hessian matches or beats the lift at every sampling " + "level on both variants; the lift's best case is a statistical tie " + "at the ultra-sparse level (n_gal=1200, p≈0.4). Junction F1 " + "favours the Hessian at essentially all levels. The diffusion " + "component remains rejected by calibration (weak diffusion shows a " + "small positive only at n_gal=1200 in the E0b sweep, +0.01). " + "Detection of the artifact: an instrument change made for E1 " + "(connected-web extraction) retroactively changed E0 parent " + "scores; the n_est fields in the archived E0 results confirmed the " + "length mismatch.\n") + + md.append( + "\n## Open questions / next tests\n\n" + "1. **Why doesn't diffusion help?** Candidates: bend level too mild " + "(sag 15%); diffusion parameters not co-calibrated with filter " + "scales; matched-length extraction absorbing its benefit. Next: " + "sweep bend_frac ∈ {0.15, 0.3, 0.5} × diffusion strength, and score " + "curvature-binned completeness (does diffusion win specifically on " + "high-curvature arcs?).\n" + "2. **Junctions:** node clumps make junctions easy for both " + "methods. Re-run with node_frac=0 — the lift should shine when " + "junctions are only implied by filament continuity.\n" + "3. **Gravity realism (E1):** port the benchmark to a Zel'dovich / " + "N-body field where filaments are tidal-sheared, not tubular — the " + "regime the physics prior (tidal-modulated coefficients, H2) " + "targets.\n" + "4. **Tier B:** the current extraction skeletonises the ridgeness; " + "true SR geodesic tracing on the lifted graph is not yet exercised " + "and is where curvature penalties enter explicitly.\n") + + out = ROOT / "docs" / "E0-report.md" + out.write_text("\n".join(md)) + print(f"wrote {out}") + + # compact terminal summary for chat + print("\nΔ completeness (lift − hessian), by sampling level:") + for tag, data in [("straight", straight), ("curved", curved)]: + if data is None: + continue + res = data["results"] + ds = [float(np.mean(paired_delta(res, "M1", "completeness", L))) + for L in levels] + print(f" {tag:9s} {spark([d + 0.5 for d in ds])} " + + " ".join(f"{d:+.3f}" for d in ds)) + + +if __name__ == "__main__": + main() diff --git a/research/cosmic-web/src/fields.py b/research/cosmic-web/src/fields.py new file mode 100644 index 00000000000..9aabac1292d --- /dev/null +++ b/research/cosmic-web/src/fields.py @@ -0,0 +1,340 @@ +"""Synthetic cosmic-web fields for E0. + +Two generators: +- Voronoi filament model: exact ground-truth spines/junctions (E0a, accuracy). +- Zel'dovich box: realistic gravity-shaped web, no exact truth (E0b, robustness). + +Units: box of size N voxels = L h^-1 Mpc, so 1 voxel = L/N h^-1 Mpc. +All positions are in voxel coordinates. +""" + +import numpy as np +from scipy.spatial import Voronoi + + +# ---------------------------------------------------------------- Voronoi model + +def voronoi_web(N, n_seeds, rng): + """Voronoi filament network in an N^3 box. + + Filaments of the cosmic-web Voronoi model are the *edges* of the Voronoi + polyhedra (segments shared by >= 3 cell faces); nodes are the vertices. + Returns (segments [n,2,3], vertices [m,3]) in voxel coords, clipped to + a margin so no infinite/boundary artefacts enter the truth. + """ + seeds = rng.uniform(0, N, size=(n_seeds, 3)) + # mirror-pad seeds across each face so interior cells are bounded + pads = [] + for dim in range(3): + lo = seeds.copy(); lo[:, dim] = -lo[:, dim] + hi = seeds.copy(); hi[:, dim] = 2 * N - hi[:, dim] + pads += [lo, hi] + vor = Voronoi(np.vstack([seeds] + pads)) + + # collect polygon edges of every finite ridge (face); an edge shared by + # >= 3 faces lies where three cells meet -> filament segment + edge_count = {} + for face in vor.ridge_vertices: + if -1 in face: + continue + m = len(face) + for i in range(m): + a, b = face[i], face[(i + 1) % m] + edge_count[(min(a, b), max(a, b))] = edge_count.get((min(a, b), max(a, b)), 0) + 1 + + margin = 0.02 * N + inside = lambda p: np.all(p > margin) and np.all(p < N - margin) + segments, vert_ids = [], set() + for (a, b), c in edge_count.items(): + if c < 3: + continue + pa, pb = vor.vertices[a], vor.vertices[b] + if inside(pa) and inside(pb): + segments.append((pa, pb)) + vert_ids.update((a, b)) + segments = np.array(segments) + vertices = vor.vertices[sorted(vert_ids)] + vertices = vertices[[inside(v) for v in vertices]] + return segments, vertices + + +def sample_segments(segments, step=0.25): + """Densely sample polyline points along segments (voxel coords).""" + pts = [] + for pa, pb in segments: + n = max(2, int(np.linalg.norm(pb - pa) / step)) + t = np.linspace(0, 1, n)[:, None] + pts.append(pa + t * (pb - pa)) + return np.vstack(pts) + + +def curve_polylines(segments, bend_frac, rng, n_pts=24): + """Bend each straight segment into a quadratic Bezier whose control point + is offset perpendicular to the chord by ~bend_frac * length. bend_frac=0 + returns the straight segments as 2-point polylines. Endpoints (junctions) + are preserved.""" + polys = [] + for pa, pb in segments: + if bend_frac == 0: + polys.append(np.array([pa, pb])) + continue + d = pb - pa + L = np.linalg.norm(d) + u = rng.normal(size=3) + u -= (u @ d) / L ** 2 * d + u /= np.linalg.norm(u) + ctrl = (pa + pb) / 2 + u * bend_frac * L * rng.uniform(0.5, 1.0) + t = np.linspace(0, 1, n_pts)[:, None] + polys.append((1 - t) ** 2 * pa + 2 * t * (1 - t) * ctrl + t ** 2 * pb) + return polys + + +def sample_polylines(polys, step=0.25): + """Arclength-uniform dense points along each polyline; returns + (points [n,3], total_length).""" + pts, total = [], 0.0 + for p in polys: + seg = np.diff(p, axis=0) + seglen = np.linalg.norm(seg, axis=1) + total += seglen.sum() + for (a, d), l in zip(zip(p[:-1], seg), seglen): + n = max(1, int(l / step)) + t = np.arange(n)[:, None] / n + pts.append(a + t * d) + return np.vstack(pts), total + + +def polyline_galaxies(polys, vertices, N, n_gal, bg_frac, sigma_perp, rng, + node_frac=0.15): + """Galaxies along (possibly curved) filaments + node clumps + background. + Arclength-uniform sampling, isotropic Gaussian scatter sigma_perp.""" + dense, _ = sample_polylines(polys, step=0.2) + n_bg = int(n_gal * bg_frac) + n_node = int(n_gal * node_frac) + n_fil = n_gal - n_bg - n_node + pos = dense[rng.integers(0, len(dense), n_fil)] + \ + rng.normal(0, sigma_perp, (n_fil, 3)) + node_pos = vertices[rng.integers(0, len(vertices), n_node)] + \ + rng.normal(0, 1.5 * sigma_perp, (n_node, 3)) + bg_pos = rng.uniform(0, N, (n_bg, 3)) + return np.vstack([pos, node_pos, bg_pos]) % N + + +def voronoi_galaxies(segments, vertices, N, n_gal, bg_frac, sigma_perp, rng, + node_frac=0.15): + """Galaxies: along filament edges (transverse Gaussian scatter sigma_perp + voxels), clumps at nodes, and a uniform background fraction.""" + lengths = np.linalg.norm(segments[:, 1] - segments[:, 0], axis=1) + n_bg = int(n_gal * bg_frac) + n_node = int(n_gal * node_frac) + n_fil = n_gal - n_bg - n_node + + seg_idx = rng.choice(len(segments), size=n_fil, p=lengths / lengths.sum()) + t = rng.uniform(0, 1, n_fil)[:, None] + pos = segments[seg_idx, 0] + t * (segments[seg_idx, 1] - segments[seg_idx, 0]) + pos = pos + rng.normal(0, sigma_perp, pos.shape) + + node_pos = vertices[rng.integers(0, len(vertices), n_node)] + \ + rng.normal(0, 1.5 * sigma_perp, (n_node, 3)) + bg_pos = rng.uniform(0, N, (n_bg, 3)) + return np.vstack([pos, node_pos, bg_pos]) % N + + +# ---------------------------------------------------------------- Zel'dovich + +def _bbks_pk(k, gamma=0.21): + """BBKS transfer-function power spectrum shape, n_s = 1 (arbitrary norm).""" + q = np.maximum(k, 1e-8) / gamma + T = np.log(1 + 2.34 * q) / (2.34 * q) * \ + (1 + 3.89 * q + (16.1 * q) ** 2 + (5.46 * q) ** 3 + (6.71 * q) ** 4) ** -0.25 + return k * T ** 2 + + +def zeldovich_box(N, L, D, rng, trunc=0.0, sigma8=0.8): + """Zel'dovich-evolved density: delta_lin normalised to sigma(R=8)=0.8, + particles displaced by D * psi, CIC-deposited. trunc > 0 (Mpc/h) applies + Gaussian truncation to the displacement field (truncated ZA; avoids + post-shell-crossing washout). Returns density/mean and particle + positions (voxel coords).""" + kf = 2 * np.pi / L + k1 = np.fft.fftfreq(N, d=1.0 / N) * kf + kx, ky, kz = np.meshgrid(k1, k1, k1[: N // 2 + 1], indexing="ij") + k2 = kx ** 2 + ky ** 2 + kz ** 2 + k = np.sqrt(k2) + + amp = np.sqrt(_bbks_pk(k)) + noise = rng.normal(size=k.shape) + 1j * rng.normal(size=k.shape) + dk = amp * noise + dk[0, 0, 0] = 0 + + # normalise: sigma(delta smoothed with Gaussian R = 8 Mpc/h) = 0.8 + dk_s = dk * np.exp(-0.5 * k2 * 8.0 ** 2) # k in h/Mpc + delta_s = np.fft.irfftn(dk_s, s=(N, N, N)) + dk *= sigma8 / delta_s.std() + + tk = np.exp(-0.5 * k2 * trunc ** 2) if trunc else 1.0 + with np.errstate(divide="ignore", invalid="ignore"): + psi_kx, psi_ky, psi_kz = (1j * kj / np.where(k2 == 0, 1, k2) * dk * tk + for kj in (kx, ky, kz)) + # displacement in Mpc/h -> voxels + to_vox = N / L + q = np.arange(N) + 0.5 + qx, qy, qz = np.meshgrid(q, q, q, indexing="ij") + pos = np.stack([ + qx + D * np.fft.irfftn(psi_kx, s=(N, N, N)) * to_vox, + qy + D * np.fft.irfftn(psi_ky, s=(N, N, N)) * to_vox, + qz + D * np.fft.irfftn(psi_kz, s=(N, N, N)) * to_vox, + ], axis=-1).reshape(-1, 3) % N + return cic_deposit(pos, N), pos + + +# ---------------------------------------------------------------- deposit + +def cic_deposit(pos, N): + """Cloud-in-cell mass deposit onto an N^3 periodic grid; returns rho/mean.""" + grid = np.zeros((N, N, N)) + p = pos % N + i0 = np.floor(p - 0.5).astype(int) + f = (p - 0.5) - i0 + for dx in (0, 1): + for dy in (0, 1): + for dz in (0, 1): + w = (np.abs(1 - dx - f[:, 0]) * np.abs(1 - dy - f[:, 1]) + * np.abs(1 - dz - f[:, 2])) + np.add.at(grid, ((i0[:, 0] + dx) % N, (i0[:, 1] + dy) % N, + (i0[:, 2] + dz) % N), w) + return grid / grid.mean() + + +def galaxy_field(pos, N, n_gal, rng, smooth_sigma=1.0, adaptive=False): + """Poisson-subsample positions to n_gal 'galaxies', CIC-deposit, and apply + log transform + Gaussian smoothing. adaptive=True sets the smoothing to + ~0.35x the mean galaxy separation (floor smooth_sigma) — necessary for + volume-filling samples where fixed light smoothing leaves shot noise.""" + from scipy.ndimage import gaussian_filter + if adaptive: + smooth_sigma = max(smooth_sigma, 0.35 * (N ** 3 / n_gal) ** (1 / 3)) + sel = rng.choice(len(pos), size=min(n_gal, len(pos)), replace=False) + field = cic_deposit(pos[sel], N) + return gaussian_filter(np.log1p(field), smooth_sigma, mode="wrap") + + +# ------------------------------------------------------------- tidal frame + +def tidal_frame(field, smooth=4.0): + """e3 (min-eigenvalue eigenvector of T = Hess phi) per voxel, from a + density field via FFT Poisson solve; T_ij(k) = k_i k_j delta_k / k^2.""" + delta = field / field.mean() - 1 if field.min() >= 0 else field + dk = np.fft.rfftn(delta) + N = field.shape[0] + k1 = np.fft.fftfreq(N) * 2 * np.pi + kx, ky, kz = np.meshgrid(k1, k1, k1[: N // 2 + 1], indexing="ij") + k2 = kx ** 2 + ky ** 2 + kz ** 2 + k2[0, 0, 0] = 1.0 + sm = np.exp(-0.5 * k2 * smooth ** 2) + ks = [kx, ky, kz] + T = np.empty((N, N, N, 3, 3), dtype=np.float32) + for i in range(3): + for j in range(i, 3): + tij = np.fft.irfftn(ks[i] * ks[j] / k2 * dk * sm, s=(N, N, N)) + T[..., i, j] = T[..., j, i] = tij + vals, vecs = np.linalg.eigh(T) # ascending eigenvalues + return vecs[..., :, 0] # e3: min-eigenvalue eigenvector + + +def tidal_frame_full(field, smooth=4.0): + """(e1, e3): max- and min-eigenvalue eigenvectors of the tidal tensor + per voxel — e1 is the first-collapse direction, e3 the filament axis.""" + delta = field / field.mean() - 1 if field.min() >= 0 else field + N = field.shape[0] + dk = np.fft.rfftn(delta) + k1 = np.fft.fftfreq(N) * 2 * np.pi + kx, ky, kz = np.meshgrid(k1, k1, k1[: N // 2 + 1], indexing="ij") + k2 = kx ** 2 + ky ** 2 + kz ** 2 + k2[0, 0, 0] = 1.0 + sm = np.exp(-0.5 * k2 * smooth ** 2) + ks = [kx, ky, kz] + T = np.empty((N, N, N, 3, 3), dtype=np.float32) + for i in range(3): + for j in range(i, 3): + tij = np.fft.irfftn(ks[i] * ks[j] / k2 * dk * sm, s=(N, N, N)) + T[..., i, j] = T[..., j, i] = tij + vals, vecs = np.linalg.eigh(T) + return vecs[..., :, 2], vecs[..., :, 0] + + +# ------------------------------------------------- literature baselines + +def _delta_k_of(N, L, rng, sigma8=0.8): + """The same normalized linear delta_k that zeldovich_box draws.""" + kf = 2 * np.pi / L + k1 = np.fft.fftfreq(N, d=1.0 / N) * kf + kx, ky, kz = np.meshgrid(k1, k1, k1[: N // 2 + 1], indexing="ij") + k2 = kx ** 2 + ky ** 2 + kz ** 2 + amp = np.sqrt(_bbks_pk(np.sqrt(k2))) + noise = rng.normal(size=k2.shape) + 1j * rng.normal(size=k2.shape) + dk = amp * noise + dk[0, 0, 0] = 0 + dk_s = dk * np.exp(-0.5 * k2 * 8.0 ** 2) + delta_s = np.fft.irfftn(dk_s, s=(N, N, N)) + dk *= sigma8 / delta_s.std() + return dk, (kx, ky, kz), k2 + + +def lpt2_positions(N, L, rng, D=1.0, sigma8=0.8): + """Second-order LPT positions at growth factor D (EdS coefficient). + x = q - D grad(phi1) + (3/7) D^2 grad(phi2), + lap(phi1) = delta, lap(phi2) = sum_{i= 3/2) + at any smoothing scale R; collapsed points get theta = -3. + Psi = grad(invlap(theta)); x = q + Psi.""" + from scipy.ndimage import gaussian_filter + dk, ks, k2 = _delta_k_of(N, L, rng, sigma8) + k2s = np.where(k2 == 0, 1, k2) + to_vox = N / L + delta = np.fft.irfftn(dk, s=(N, N, N)) + collapsed = np.zeros(delta.shape, bool) + for R in scales_mpc: + dR = gaussian_filter(delta, R * to_vox, mode="wrap") if R > 0 else delta + collapsed |= (D * dR >= 1.5) + arg = np.maximum(1.0 - (2.0 / 3.0) * D * delta, 0.0) + theta = 3.0 * (np.sqrt(arg) - 1.0) + theta[collapsed] = -3.0 + th_k = np.fft.rfftn(theta) + q = np.arange(N) + 0.5 + qx, qy, qz = np.meshgrid(q, q, q, indexing="ij") + grid = [qx, qy, qz] + pos = [] + for i, kk in enumerate(ks): + psi_i = np.fft.irfftn(-1j * kk * th_k / k2s, s=(N, N, N)) + pos.append(grid[i] + psi_i * to_vox) + return np.stack(pos, axis=-1).reshape(-1, 3) % N diff --git a/research/cosmic-web/src/lift.py b/research/cosmic-web/src/lift.py new file mode 100644 index 00000000000..fff768836a1 --- /dev/null +++ b/research/cosmic-web/src/lift.py @@ -0,0 +1,80 @@ +"""Orientation score on R^3 x S^2: lift a scalar field to position-orientation +space with elongated anisotropic ridge filters (Fourier-domain). + +The score for axis n is U_n = [ |k_perp|^2 * exp(-1/2 (s_par^2 k_par^2 + +s_perp^2 |k_perp|^2)) ] applied in Fourier space: minus the transverse +Laplacian of an n-elongated Gaussian smoothing — positive on bright ridges +aligned with n. Angular selectivity grows with s_par / s_perp; a Hessian +(quadratic) score cannot exceed angular bandwidth 2, this can. +""" + +import numpy as np + + +def hemisphere_axes(n_axes=42): + """Quasi-uniform axes on the upper hemisphere (golden spiral). + Orientations are axial (n ~ -n), so a hemisphere suffices.""" + i = np.arange(n_axes) + z = (i + 0.5) / n_axes + phi = i * np.pi * (3 - np.sqrt(5)) + r = np.sqrt(1 - z ** 2) + return np.stack([r * np.cos(phi), r * np.sin(phi), z], axis=1) + + +def _kgrid(N): + k1 = np.fft.fftfreq(N) * 2 * np.pi # voxel units + kx, ky, kz = np.meshgrid(k1, k1, k1[: N // 2 + 1], indexing="ij") + return kx, ky, kz + + +def orientation_score(field, axes, sig_par=6.0, sig_perp=1.5, scales=(1.0, 1.6)): + """U[a, x, y, z] float32, max over scales (scale-normalised by sig_perp^2).""" + N = field.shape[0] + F = np.fft.rfftn(field - field.mean()) + kx, ky, kz = _kgrid(N) + k2 = kx ** 2 + ky ** 2 + kz ** 2 + U = np.zeros((len(axes),) + field.shape, dtype=np.float32) + for s in scales: + sp, st = sig_par * s, sig_perp * s + for a, n in enumerate(axes): + kpar = kx * n[0] + ky * n[1] + kz * n[2] + kperp2 = k2 - kpar ** 2 + filt = st ** 2 * kperp2 * np.exp(-0.5 * ((sp * kpar) ** 2 + st ** 2 * kperp2)) + resp = np.fft.irfftn(F * filt, s=field.shape).astype(np.float32) + np.maximum(U[a], resp, out=U[a]) + return U + + +def axis_adjacency(axes, k_nn=5): + """Row-normalised adjacency among axes by axial angle (|dot|), for + orientation diffusion. Returns (n_axes, n_axes) float32.""" + dots = np.abs(axes @ axes.T) + np.fill_diagonal(dots, -1) + W = np.zeros_like(dots, dtype=np.float32) + idx = np.argsort(-dots, axis=1)[:, :k_nn] + for i, row in enumerate(idx): + W[i, row] = dots[i, row] + W /= W.sum(axis=1, keepdims=True) + return W + + +def hypoelliptic_diffuse(U, axes, W, sig_par=4.0, sig_perp=0.5, alpha=0.35, + n_iter=2): + """Splitting scheme for left-invariant diffusion on R^3 x S^2: + per-channel anisotropic Gaussian along its own axis (strong parallel, + weak transverse), then orientation smoothing U <- (1-a)U + a W U.""" + N = U.shape[1] + kx, ky, kz = _kgrid(N) + k2 = kx ** 2 + ky ** 2 + kz ** 2 + kernels = [] + for n in axes: + kpar = kx * n[0] + ky * n[1] + kz * n[2] + kernels.append(np.exp(-0.5 * ((sig_par * kpar) ** 2 + + sig_perp ** 2 * (k2 - kpar ** 2)) + ).astype(np.float32)) + for _ in range(n_iter): + for a in range(len(axes)): + U[a] = np.fft.irfftn(np.fft.rfftn(U[a]) * kernels[a], + s=U.shape[1:]).astype(np.float32) + U[:] = (1 - alpha) * U + alpha * np.tensordot(W, U, axes=(1, 0)) + return U diff --git a/research/cosmic-web/src/metrics.py b/research/cosmic-web/src/metrics.py new file mode 100644 index 00000000000..d62486f42fd --- /dev/null +++ b/research/cosmic-web/src/metrics.py @@ -0,0 +1,49 @@ +"""Evaluation metrics M1/M2 (defined in the scoping post, section 5). + +M1 spine distance: directed median point-to-curve distances between truth +and estimate point sets, plus completeness (fraction of truth points within +r0 of the estimate) and purity (fraction of estimate points within r0 of +truth), at matched skeleton length. + +M2 junction recovery: greedy one-to-one matching of junction points within +r0 -> precision, recall, F1. + +All distances in voxels; the caller reports the voxel size. +""" + +import numpy as np +from scipy.spatial import cKDTree + + +def spine_metrics(truth_pts, est_pts, r0=2.0): + if len(est_pts) == 0: + return {"median_d_truth_to_est": np.inf, "median_d_est_to_truth": np.inf, + "completeness": 0.0, "purity": 0.0, "r0": r0, "n_est": 0} + t_tree, e_tree = cKDTree(truth_pts), cKDTree(est_pts) + d_te = e_tree.query(truth_pts)[0] # truth -> estimate + d_et = t_tree.query(est_pts)[0] # estimate -> truth + return { + "median_d_truth_to_est": float(np.median(d_te)), + "median_d_est_to_truth": float(np.median(d_et)), + "completeness": float((d_te < r0).mean()), + "purity": float((d_et < r0).mean()), + "r0": r0, + "n_est": len(est_pts), + } + + +def junction_metrics(truth_j, est_j, r0=3.0): + if len(est_j) == 0 or len(truth_j) == 0: + return {"precision": 0.0, "recall": 0.0, "f1": 0.0, + "n_true": len(truth_j), "n_est": len(est_j), "r0": r0} + d = np.linalg.norm(truth_j[:, None] - est_j[None], axis=2) + matched_t, matched_e = set(), set() + for ti, ei in sorted(zip(*np.where(d < r0)), key=lambda p: d[p]): + if ti not in matched_t and ei not in matched_e: + matched_t.add(ti); matched_e.add(ei) + tp = len(matched_t) + prec = tp / len(est_j) + rec = tp / len(truth_j) + f1 = 2 * prec * rec / (prec + rec) if prec + rec else 0.0 + return {"precision": prec, "recall": rec, "f1": f1, + "n_true": len(truth_j), "n_est": len(est_j), "r0": r0} diff --git a/research/cosmic-web/src/pm.py b/research/cosmic-web/src/pm.py new file mode 100644 index 00000000000..ec136c381b1 --- /dev/null +++ b/research/cosmic-web/src/pm.py @@ -0,0 +1,113 @@ +"""Minimal particle-mesh N-body (Einstein-de Sitter), enough to give +beyond-Zel'dovich curved trajectories for the E2 dynamical test. + +Conformal-time formulation, code units (grid cells, H0 = 1, Omega = 1): + da/dtau = sqrt(a) (EdS; tau = 2 sqrt(a)) + dx/dtau = p / a + dp/dtau = -a grad(phi), lap(phi) = (3 / 2a) * delta +Initial conditions: Zel'dovich positions and growing-mode momenta +p = a^{3/2} psi at a_init, from the same BBKS linear field as fields.py. +Kick-drift-kick leapfrog, CIC deposit/gather, spectral Poisson solve. + +ponytail: EdS only (D = a), 128^3-scale boxes, no adaptive steps — this is +an instrument for trajectory *shape*, not precision cosmology. +""" + +import numpy as np + +import fields + + +def _cic_gather(grid, pos): + """Trilinear (CIC) interpolation of a grid at particle positions.""" + N = grid.shape[0] + p = pos % N + i0 = np.floor(p - 0.5).astype(int) + f = (p - 0.5) - i0 + out = np.zeros(len(p)) + for dx in (0, 1): + wx = np.abs(1 - dx - f[:, 0]) + for dy in (0, 1): + wy = np.abs(1 - dy - f[:, 1]) + for dz in (0, 1): + w = wx * wy * np.abs(1 - dz - f[:, 2]) + out += w * grid[(i0[:, 0] + dx) % N, (i0[:, 1] + dy) % N, + (i0[:, 2] + dz) % N] + return out + + +def _forces(pos, N, a, kx, ky, kz, k2, omega_m=1.0): + delta = fields.cic_deposit(pos, N) - 1.0 + phi_k = np.fft.rfftn(delta) * (-1.5 * omega_m / a) / k2 + phi_k[0, 0, 0] = 0 + return np.stack([ + _cic_gather(np.fft.irfftn(-1j * kj * phi_k, s=(N, N, N)), pos) + for kj in (kx, ky, kz)], axis=1) + + +def growth_factor(a_grid, omega_m): + """Linear growth D(a) for flat LCDM, normalized D(1) = 1.""" + from scipy.integrate import quad + ol = 1.0 - omega_m + H = lambda a: np.sqrt(omega_m * a ** -3 + ol) + D_un = lambda a: H(a) * quad(lambda x: (x * H(x)) ** -3, 1e-6, a)[0] + D1 = D_un(1.0) + return np.array([D_un(a) / D1 for a in a_grid]) + + +def pm_sim(N, L, rng_master, a_init=0.1, a_final=1.0, n_steps=90, trunc=0.0, + track=50000, track_from=0.5, sigma8=0.8, omega_m=1.0): + """Run the PM simulation. + + ICs: the same linear field the Zel'dovich generator draws (obtained by + running it at D=0 and D=1 and differencing the positions — avoids + refactoring its internals). Returns (pos_final [Np,3] voxels, + vel_final = p/a [Np,3], track_idx, track_a, track_pos [n_a, track, 3]); + tracked-subset positions stored every step once a >= track_from. + """ + seedseq = rng_master.integers(0, 2 ** 32) + r1 = np.random.default_rng(seedseq) + r2 = np.random.default_rng(seedseq) + _, pos_q = fields.zeldovich_box(N, L, D=0.0, rng=r1, trunc=trunc, sigma8=sigma8) + _, pos_d = fields.zeldovich_box(N, L, D=1.0, rng=r2, trunc=trunc, sigma8=sigma8) + psi = pos_d - pos_q + psi -= N * np.round(psi / N) # unwrap periodicity + + ol = 1.0 - omega_m + Ha = lambda a: np.sqrt(omega_m * a ** -3 + ol) # H(a)/H0 + if omega_m == 1.0: + D_i, dDda_i = a_init, 1.0 + else: + eps = 1e-4 + D_i, D_ip = growth_factor([a_init, a_init + eps], omega_m) + dDda_i = (D_ip - D_i) / eps + x = (pos_q + D_i * psi) % N + # p = a * dx/dtau = a * (dD/da) * (a^2 H) * psi (EdS: a^{3/2} psi) + p = a_init * dDda_i * a_init ** 2 * Ha(a_init) * psi + + k1 = np.fft.fftfreq(N) * 2 * np.pi + kx, ky, kz = np.meshgrid(k1, k1, k1[: N // 2 + 1], indexing="ij") + k2 = kx ** 2 + ky ** 2 + kz ** 2 + k2[0, 0, 0] = 1.0 + + a_grid = np.linspace(a_init, a_final, n_steps + 1) + track_idx = np.random.default_rng(0).choice(len(x), track, replace=False) + track_a, track_pos = [], [] + q_track = pos_q[track_idx].copy() # Lagrangian positions + psi_track = psi[track_idx].copy() # ZA displacement field at D=1 + + F = _forces(x, N, a_init, kx, ky, kz, k2, omega_m) + for i in range(n_steps): + a, a_new = a_grid[i], a_grid[i + 1] + a_half = 0.5 * (a + a_new) + dtau = (a_new - a) / (a_half ** 2 * Ha(a_half)) + p += a * F * (dtau / 2) # kick (F = -grad phi) + x = (x + p / a_half * dtau) % N # drift + F = _forces(x, N, a_new, kx, ky, kz, k2, omega_m) + p += F * (dtau / 2) * a_new + if a_new >= track_from: + track_a.append(a_new) + track_pos.append(x[track_idx].astype(np.float32)) + vel = p / a_new + return (x, vel, track_idx, np.array(track_a), np.array(track_pos), + q_track, psi_track) diff --git a/research/cosmic-web/src/spines.py b/research/cosmic-web/src/spines.py new file mode 100644 index 00000000000..e047e1c3ecc --- /dev/null +++ b/research/cosmic-web/src/spines.py @@ -0,0 +1,107 @@ +"""Spine extraction and the Hessian baseline. + +Both methods produce a scalar 'ridgeness' field and then share the SAME +post-processing (hysteresis threshold -> 3D skeletonization -> junctions), +so the comparison isolates exactly one variable: the orientation lift + +hypoelliptic diffusion vs. the quadratic (Hessian) orientation response. + +Baseline note: max_n of the quadratic score -(tr H - n^T H n)/2 over the +sphere is attained at the largest-eigenvalue eigenvector, giving ridgeness +-(l2 + l3)/2 — i.e. the Hessian baseline IS the angular-bandwidth-2 version +of the lifted score. That makes it the textbook-fair ablation. +""" + +import numpy as np +from scipy.ndimage import gaussian_filter, label, center_of_mass +from skimage.filters import apply_hysteresis_threshold +from skimage.morphology import skeletonize + + +# ---------------------------------------------------------------- ridgeness + +def hessian_ridgeness(field, sigma=2.0): + """-(l2 + l3)/2 of the Hessian of the sigma-smoothed field (bright ridges), + multi-scale-free single-sigma baseline; scale-normalised by sigma^2.""" + f = gaussian_filter(field, sigma, mode="wrap") + grads = np.gradient(f) + H = np.empty(field.shape + (3, 3)) + for i in range(3): + gi = np.gradient(grads[i]) + for j in range(3): + H[..., i, j] = gi[j] + lam = np.linalg.eigvalsh(H) # ascending: l3 <= l2 <= l1 + return np.clip(-(lam[..., 0] + lam[..., 1]) / 2, 0, None) * sigma ** 2 + + +def lifted_ridgeness(U): + """max over orientation channels of the (diffused) orientation score.""" + return np.clip(U.max(axis=0), 0, None) + + +# ---------------------------------------------------------------- extraction + +def _mask(R, hi_val, lo_frac=0.5): + return apply_hysteresis_threshold(R, lo_frac * hi_val, hi_val) + + +def extract_matched(R, n_target, tube_area=9.0, lo_frac=0.5, min_component=4): + """Matched-length skeleton. Bisect the hi threshold VALUE so the mask + volume ~ n_target * tube_area voxels (mask volume is strictly monotone + in the threshold, unlike skeleton length), skeletonize, then correct + tube_area from the measured skeleton length and repeat (<=3 rounds) so + the SKELETON length itself lands near n_target. Components smaller than + min_component voxels are dropped. Same procedure for every method keeps + the comparison fair. lo_frac=1.0 disables hysteresis (plain threshold) + — required on fully-connected webs where hysteresis floods.""" + + def _bisect_mask(v_target): + lo_v, hi_v = float(np.percentile(R, 50)), float(R.max()) + best_mid, best_err = lo_v, np.inf + for _ in range(24): + mid = 0.5 * (lo_v + hi_v) + v = int(_mask(R, mid, lo_frac).sum()) + err = abs(v - v_target) + if err < best_err: + best_mid, best_err = mid, err + if err <= 0.05 * v_target: + break + if v > v_target: + lo_v = mid + else: + hi_v = mid + return best_mid + + def _skel_at(v_target): + skel = skeletonize(_mask(R, _bisect_mask(v_target), lo_frac)) + lab, nlab = label(skel, structure=np.ones((3, 3, 3))) + if nlab: + sizes = np.bincount(lab.ravel()) + skel &= (sizes[lab] >= min_component) & (lab > 0) + return skel + + v_target = n_target * tube_area + skel = _skel_at(v_target) + for _ in range(3): + n = int(skel.sum()) + if n == 0 or abs(n - n_target) <= 0.15 * n_target: + break + v_target *= n_target / n + skel = _skel_at(v_target) + return skel + + +def skeleton_points(skel): + return np.argwhere(skel).astype(float) + + +def junction_points(skel, min_neighbors=3): + """Skeleton voxels with >= min_neighbors skeleton neighbours (26-conn), + clustered to centroids.""" + from scipy.ndimage import convolve + kern = np.ones((3, 3, 3)); kern[1, 1, 1] = 0 + nn = convolve(skel.astype(np.int8), kern, mode="wrap") + jmask = skel & (nn >= min_neighbors) + lab, nlab = label(jmask, structure=np.ones((3, 3, 3))) + if nlab == 0: + return np.empty((0, 3)) + return np.array(center_of_mass(jmask, lab, range(1, nlab + 1)))