From 6814a6549419bed612647d42e2a7d57f56d5a14e Mon Sep 17 00:00:00 2001 From: Nicolas Maman Date: Sun, 4 Oct 2026 15:11:01 -0300 Subject: [PATCH 1/2] Three buffer overruns found by an audit of every fixed-size scratch block - compound: a baked tree carried one traversal stack in the compound's block (2 KB), but tree queries take a worker's share of a tree's stack (12 KB a worker), so a query on any worker past the first wrote 12 KB a worker beyond it into the heap -- reached from the parallel continuous pass, bullets swept against a compound. Baked trees point at the module's own per-worker stacks now (query and distance), made with the module's scratch; the block holds none. dynamic_tree exports the share sizes (query_stack_bytes, query_dist_bytes). - manifold: the hull clip polygons held 64 vertices, but an incident face of up to 64 sides clipped against a reference face of up to 64 can reach 128 (two 40-sided cylinders cap to cap at a twist give an 80-gon); the last worker's spilled past the allocation. They hold 128 now. - mesh: the ray and shape casts pushed the far child at every level unguarded, so a tree leaning right deeper than the 256-entry stack overran it; every mesh traversal's push is guarded now (a pathological tree drops the far subtree rather than write past the stack). Tests: test_compound_workers casts as workers 1, 7, 31 and 63 (main crashes); test_wide_faces collides 40-sided cylinders as the last worker; test_deep_tree_cast casts through a 300-deep right spine as the last worker. test_compound's permanent allocations grow by the two stacks. --- aephysics/compound/module.ae | 21 +++++++++++++--- aephysics/dynamic_tree/module.ae | 7 +++++- aephysics/manifold/module.ae | 11 ++++++--- aephysics/mesh/module.ae | 40 +++++++++++++++++++++--------- tests/test_compound.ae | 36 +++++++++++++++++++++++++-- tests/test_manifold.ae | 24 ++++++++++++++++++ tests/test_mesh.ae | 42 ++++++++++++++++++++++++++++++++ 7 files changed, 159 insertions(+), 22 deletions(-) diff --git a/aephysics/compound/module.ae b/aephysics/compound/module.ae index 68d1a50..b365d15 100644 --- a/aephysics/compound/module.ae +++ b/aephysics/compound/module.ae @@ -362,6 +362,7 @@ create_compound(def: *CompoundDef) -> *CompoundData { convex_count = capsule_count + hull_count + sphere_count shape_count = convex_count + mesh_count if shape_count == 0 || shape_count >= MAX_CHILD_SHAPES { return null } + tree_stacks_ready() tree = dynamic_tree.tree_create(shape_count) child_index = 0 @@ -481,8 +482,9 @@ create_compound(def: *CompoundDef) -> *CompoundData { byte_count = byte_count + align8(tree.node_end * sizeof(DynamicTreeNode)) proxy_offset = byte_count byte_count = byte_count + align8(tree.proxy_count * sizeof(TreeProxy)) - stack_offset = byte_count - byte_count = byte_count + align8(dynamic_tree.TREE_STACK_SIZE * 4) + // No stack in the block: queries take a worker's share of a stack, so the + // baked tree points at the module's per-worker stacks (tree_stacks_ready). + stack_offset = 0 material_offset = byte_count byte_count = byte_count + align8(table.count * sizeof(SurfaceMaterial)) material_map_offset = byte_count @@ -540,7 +542,7 @@ create_compound(def: *CompoundDef) -> *CompoundData { proxy_count: tree.proxy_count, proxy_capacity: tree.proxy_count, proxy_free_list: NULL_INDEX, swap_nodes: null, leaf_indices: null, leaf_nodes: null, leaf_centers: null, rebuild_capacity: 0, dfs_ordered: tree.dfs_ordered, - stack: block + stack_offset, stack_dist: null, copy_stack: null } + stack: g_tree_stack, stack_dist: g_tree_dists, copy_stack: null } dynamic_tree.tree_destroy(&tree) memcpy(block + material_offset, table.materials, table.count * sizeof(SurfaceMaterial)) @@ -617,6 +619,18 @@ overlap_visitor(proxy_id: int, user_data: long, context: ptr) -> bool { return true } +// Every baked tree's traversal stacks, one share per worker (the world's +// trees carry their own): a query on worker k uses the k-th share, which the +// one stack the block used to hold could not give a worker past the first. +var g_tree_stack: ptr = null +var g_tree_dists: ptr = null +tree_stacks_ready() { + if g_tree_stack == null { + g_tree_stack = basics.alloc(native.MAX_WORKERS * dynamic_tree.query_stack_bytes()) + g_tree_dists = basics.alloc(native.MAX_WORKERS * dynamic_tree.query_dist_bytes()) + } +} + var g_local: ptr = null // Vec3[MAX_SHAPE_CAST_POINTS]: the proxy in the compound's frame local_points() -> ptr { return native.scratch(g_local, distance.MAX_SHAPE_CAST_POINTS * sizeof(Vec3)) } var g_child: ptr = null // Vec3[MAX_SHAPE_CAST_POINTS]: the proxy in a child's frame @@ -794,6 +808,7 @@ planes_ready() { reserve_scratch() { scratch_ready() planes_ready() + tree_stacks_ready() } collide_mover_and_compound(planes: PlaneResult[], capacity: int, c: *CompoundData, mover: Capsule) -> int { diff --git a/aephysics/dynamic_tree/module.ae b/aephysics/dynamic_tree/module.ae index 9c7b245..5f88bb9 100644 --- a/aephysics/dynamic_tree/module.ae +++ b/aephysics/dynamic_tree/module.ae @@ -26,7 +26,7 @@ import aephysics.basics exports ( DynamicTreeNode, TreeProxy, TreeStats, DynamicTree, RayCastInput, BoxCastInput, - NULL_INDEX, default_category_bits, default_mask_bits, TREE_STACK_SIZE, + NULL_INDEX, default_category_bits, default_mask_bits, TREE_STACK_SIZE, query_stack_bytes, query_dist_bytes, LEAF_NODE, MOVED_NODE, NODE_INDEX_MASK, EMPTY_NODE, ROOT_NODE, is_leaf, is_node_moved, is_empty_node, left_child, proxy_id_of, node_height, tree_create, tree_destroy, tree_create_proxy, tree_destroy_proxy, tree_move_proxy, @@ -291,6 +291,11 @@ tree_create(proxy_capacity: int) -> DynamicTree { // The calling worker's traversal stacks: queries run from any worker // (the continuous pass, the sensors), so each has a block of its own. +// One worker's share of a tree's traversal stacks: a tree that keeps them +// outside itself (a compound's baked tree) allocates MAX_WORKERS of these. +query_stack_bytes() -> int { return TREE_STACK_SIZE * REBUILD_ITEM_INTS * 4 } +query_dist_bytes() -> int { return TREE_STACK_SIZE * 8 } + query_stack(t: *DynamicTree) -> ptr { return native.scratch(t.stack, TREE_STACK_SIZE * REBUILD_ITEM_INTS * 4) } query_dists(t: *DynamicTree) -> ptr { return native.scratch(t.stack_dist, TREE_STACK_SIZE * 8) } diff --git a/aephysics/manifold/module.ae b/aephysics/manifold/module.ae index cd55a6a..ffeb80a 100644 --- a/aephysics/manifold/module.ae +++ b/aephysics/manifold/module.ae @@ -214,9 +214,12 @@ set_manifold_point(m: *LocalManifold, i: int, p: LocalManifoldPoint) { // The clip polygons and the unreduced points, allocated once. One collision // at a time until the parallel layer gives each worker its own. var g_clip1: ptr = null -clip1() -> ptr { return native.scratch(g_clip1, MAX_CLIP_POINTS * sizeof(ClipVertex)) } +// A polygon clipped against a face gains up to a vertex per side: an incident +// face of up to 64 sides against a reference face of up to 64 reaches 128. +const CLIP_CAPACITY = 2 * MAX_CLIP_POINTS +clip1() -> ptr { return native.scratch(g_clip1, CLIP_CAPACITY * sizeof(ClipVertex)) } var g_clip2: ptr = null -clip2() -> ptr { return native.scratch(g_clip2, MAX_CLIP_POINTS * sizeof(ClipVertex)) } +clip2() -> ptr { return native.scratch(g_clip2, CLIP_CAPACITY * sizeof(ClipVertex)) } var g_clip_points: ptr = null clip_points() -> ptr { return native.scratch(g_clip_points, MAX_CLIP_POINTS * sizeof(LocalManifoldPoint)) } // A's edges, gathered once before the edge-pair loop reads them a @@ -227,8 +230,8 @@ b_frame_scratch() -> ptr { return edge_axes() + AXIS_FIELDS * AXIS_SLOTS * 4 } scratch_ready() { if g_clip1 == null { - g_clip1 = basics.alloc(native.MAX_WORKERS * (MAX_CLIP_POINTS * sizeof(ClipVertex))) - g_clip2 = basics.alloc(native.MAX_WORKERS * (MAX_CLIP_POINTS * sizeof(ClipVertex))) + g_clip1 = basics.alloc(native.MAX_WORKERS * (CLIP_CAPACITY * sizeof(ClipVertex))) + g_clip2 = basics.alloc(native.MAX_WORKERS * (CLIP_CAPACITY * sizeof(ClipVertex))) g_clip_points = basics.alloc(native.MAX_WORKERS * (MAX_CLIP_POINTS * sizeof(LocalManifoldPoint))) g_edge_axes = basics.alloc(native.MAX_WORKERS * EDGE_SCRATCH_BYTES) } diff --git a/aephysics/mesh/module.ae b/aephysics/mesh/module.ae index 627f224..1f85a7e 100644 --- a/aephysics/mesh/module.ae +++ b/aephysics/mesh/module.ae @@ -1282,8 +1282,10 @@ overlap_mesh(sh: Mesh, shape_transform: Transform, proxy: ShapeProxy) -> bool { i = i + 1 } } else { - stack[count] = index + n.count_or_offset - count = count + 1 + if count < MESH_STACK_SIZE { + stack[count] = index + n.count_or_offset + count = count + 1 + } index = index + 1 continue } @@ -1341,14 +1343,19 @@ ray_cast_mesh(sh: Mesh, origin: Vec3, translation: Vec3, max_fraction: float) -> } } else { // Front to back along the split axis. + // The far child waits on the stack; a tree deeper than the stack + // (a pathological one) drops it rather than write past the stack. + far = index + 1 if axis_of(ray_delta, n.axis) > 0.0 { - stack[count] = index + n.count_or_offset + far = index + n.count_or_offset index = index + 1 } else { - stack[count] = index + 1 index = index + n.count_or_offset } - count = count + 1 + if count < MESH_STACK_SIZE { + stack[count] = far + count = count + 1 + } continue } } @@ -1422,14 +1429,19 @@ shape_cast_mesh(sh: Mesh, proxy: ShapeProxy, translation: Vec3, max_fraction: fl i = i + 1 } } else { + // The far child waits on the stack (dropped, not written past the + // stack, by a tree deeper than it). + far = index + 1 if axis_of(inv_delta, n.axis) > 0.0 { - stack[count] = index + n.count_or_offset + far = index + n.count_or_offset index = index + 1 } else { - stack[count] = index + 1 index = index + n.count_or_offset } - count = count + 1 + if count < MESH_STACK_SIZE { + stack[count] = far + count = count + 1 + } continue } } @@ -1542,8 +1554,10 @@ collide_mover_and_mesh(planes: PlaneResult[], capacity: int, sh: Mesh, mover: Ca i = i + 1 } } else { - stack[count] = index + n.count_or_offset - count = count + 1 + if count < MESH_STACK_SIZE { + stack[count] = index + n.count_or_offset + count = count + 1 + } index = index + 1 continue } @@ -1596,8 +1610,10 @@ query_mesh(sh: Mesh, bounds: AABB, visitor: fn(Vec3, Vec3, Vec3, int, ptr) -> bo i = i + 1 } } else { - stack[count] = index + n.count_or_offset - count = count + 1 + if count < MESH_STACK_SIZE { + stack[count] = index + n.count_or_offset + count = count + 1 + } index = index + 1 continue } diff --git a/tests/test_compound.ae b/tests/test_compound.ae index e5418d9..5ea5f17 100644 --- a/tests/test_compound.ae +++ b/tests/test_compound.ae @@ -23,6 +23,7 @@ import aephysics.material import aephysics.sphere import aephysics.capsule import aephysics.compound +import aephysics.native import aephysics.shape extern calloc(count: int, size: int) -> ptr @@ -592,6 +593,36 @@ test_many_mesh_materials() { mesh.destroy_mesh(md) } +// A compound's baked tree queried from every worker: each takes its own +// share of the per-worker stack (the block held one stack, and a worker +// past the first wrote 12 KB a worker beyond it). +test_compound_workers() { + d = make_defs(0, 0, 0, 40) + spheres = d.spheres as CompoundSphereDef[] + i = 0 + while i < 40 { + spheres[i] = compound.compound_sphere_def(manifold.sphere(math.vec3(1.5 * (i as float), 0.0, 0.0), 0.5), make_material(0.5, i)) + i = i + 1 + } + c = compound.create_compound(&d.def) + origin = math.vec3(80.0, 0.0, 0.0) + translation = math.vec3(0.0 - 100.0, 0.0, 0.0) + expected = compound.ray_cast_compound(c, origin, translation, 1.0) + agree = true + workers = [ 1, 7, 31, 63 ] + k = 0 + while k < 4 { + native.set_worker_index(workers[k]) + out = compound.ray_cast_compound(c, origin, translation, 1.0) + if out.hit != expected.hit || out.child_index != expected.child_index || out.fraction != expected.fraction { agree = false } + k = k + 1 + } + native.set_worker_index(0) + ensure("workers: the nearest sphere from worker 0 (${expected.child_index})", expected.hit && expected.child_index == 39) + ensure("workers: every worker casts the same", agree) + compound.destroy_compound(c) + free_defs(d) +} test_overlap() { mat = material.default_surface_material() point_block = calloc(2, sizeof(Vec3)) @@ -919,13 +950,14 @@ main() { test_children_and_bounds() test_casts() test_many_mesh_materials() + test_compound_workers() test_overlap() test_query_and_mover() test_through_shape() test_field() test_brute_force_rays() - // The scratch stays allocated: compound's three blocks, mesh's three, sphere's two, shape's one. - ensure("every other counted allocation was freed (${basics.alloc_count() - before})", basics.alloc_count() == before + 9) + // The scratch stays allocated: compound's five blocks (its baked trees' per-worker stacks among them), mesh's three, sphere's two, shape's one. + ensure("every other counted allocation was freed (${basics.alloc_count() - before})", basics.alloc_count() == before + 11) println("compound: ${checks} checks") if failures == 0 { diff --git a/tests/test_manifold.ae b/tests/test_manifold.ae index 102355f..55b76f0 100644 --- a/tests/test_manifold.ae +++ b/tests/test_manifold.ae @@ -13,6 +13,7 @@ import aephysics.math import aephysics.basics import aephysics.distance import aephysics.hull +import aephysics.native import aephysics.manifold extern calloc(count: int, size: int) -> ptr @@ -758,6 +759,28 @@ test_many_edges() { hull.destroy_hull(b) } +// Two 40-sided cylinders cap to cap, the top turned half a side: the caps +// overlap in an 80-gon, past the 64 vertices the clip buffers held (the +// last worker's spilled past the allocation). Collided as that worker. +test_wide_faces() { + a = hull.create_cylinder(1.0, 1.0, 0.0, 40) + b = hull.create_cylinder(1.0, 1.0, 0.0, 40) + ensure("wide faces: 40-sided cylinders", a != null && b != null && a.vertex_count == 80) + low = hull.compute_hull_aabb(a, math.transform_identity()) + height = low.upper.y - low.lower.y + xf = Transform { p: math.vec3(0.0, height - 0.005, 0.0), q: exact_quat(math.vec3_axis_y(), math.PI / 40.0) } + points = new_manifold() + m = manifold.local_manifold(points) + cache = manifold.empty_sat_cache() + native.set_worker_index(native.MAX_WORKERS - 1) + manifold.collide_hulls(&m, 8, a, b, xf, &cache) + native.set_worker_index(0) + ensure("wide faces: four points (${m.point_count})", m.point_count == 4) + ensure("wide faces: normal up (${m.normal.y})", m.normal.y > 0.999) + free(points) + hull.destroy_hull(a) + hull.destroy_hull(b) +} // --- hull against capsule and sphere ----------------------------------------------------- // A thin capsule stabbed through the +x +y edge of a box: the isolated @@ -968,6 +991,7 @@ main() { test_ridge_crossing() test_edge_axis_oracle() test_many_edges() + test_wide_faces() test_hull_capsule_edge_deep() test_sphere_hull_seam() test_rounded_pairs() diff --git a/tests/test_mesh.ae b/tests/test_mesh.ae index 205f891..279e02f 100644 --- a/tests/test_mesh.ae +++ b/tests/test_mesh.ae @@ -15,6 +15,7 @@ import aephysics.distance import aephysics.manifold import aephysics.triangle_manifold import aephysics.mesh +import aephysics.native extern calloc(count: int, size: int) -> ptr extern exit(code: int) @@ -568,9 +569,50 @@ test_brute_force_casts() { mesh.destroy_mesh(wave) } +// A mesh whose tree leans right all the way down: 300 thin triangles at +// x = -1e5 / 2^k, each median split peeling one off. A ray against the split +// axis pushes a far child at every level, past the 256-entry stack (the last +// worker's spilled past the allocation); now the stack holds or drops. +test_deep_tree_cast() { + count = 300 + vertices_block = calloc(3 * count, sizeof(Vec3)) + vertices = vertices_block as Vec3[] + indices_block = calloc(3 * count, 4) + indices = indices_block as int[] + x = 0.0 - 100000.0 + k = 0 + while k < count { + vertices[3 * k] = math.vec3(x, 0.0, 0.0) + vertices[3 * k + 1] = math.vec3(x, 1.0, 0.0) + vertices[3 * k + 2] = math.vec3(x, 0.0, 1.0) + indices[3 * k] = 3 * k + indices[3 * k + 1] = 3 * k + 1 + indices[3 * k + 2] = 3 * k + 2 + x = 0.5 * x + k = k + 1 + } + mdef = mesh.mesh_def() + mdef.vertices = vertices_block + mdef.indices = indices_block + mdef.vertex_count = 3 * count + mdef.triangle_count = count + mdef.use_median_split = true + md = mesh.create_mesh(&mdef, null, 0) + ensure("deep tree: built", md != null) + if md != null { + native.set_worker_index(native.MAX_WORKERS - 1) + out = mesh.ray_cast_mesh(mesh.mesh(md, math.vec3_one()), math.vec3(1.0, 0.2, 0.2), math.vec3(0.0 - 2.0, 0.0, 0.0), 1.0) + native.set_worker_index(0) + ensure("deep tree: the ray hits the nearest sheet (${out.hit}, ${out.fraction})", out.hit && out.fraction > 0.49 && out.fraction <= 0.5 + 0.000001) + mesh.destroy_mesh(md) + } + free(vertices_block) + free(indices_block) +} main() { before = basics.alloc_count() test_valley() + test_deep_tree_cast() test_creators() test_ray_casts() test_query_and_overlap() From af60bb99b831add3c24d773c21b81f167b81d815 Mon Sep 17 00:00:00 2001 From: Nicolas Maman Date: Sun, 4 Oct 2026 15:12:10 -0300 Subject: [PATCH 2/2] test_broad_phase: the compound's scratch reserved before the allocation count The baked trees' per-worker stacks are the compound module's scratch now, made on first use; test_compound_pairs made them inside the counted region. --- tests/test_broad_phase.ae | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_broad_phase.ae b/tests/test_broad_phase.ae index 3b14c23..eee6b47 100644 --- a/tests/test_broad_phase.ae +++ b/tests/test_broad_phase.ae @@ -368,6 +368,8 @@ test_compound_pairs() { } main() { + // The compound's scratch (its baked trees' per-worker stacks) is the module's, reserved before the count. + compound.reserve_scratch() before = basics.alloc_count() test_keys() test_grid()