Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
3fe1d7e
Apply corridor lengths control by using least neighbors count selecti…
dmigwi Jul 28, 2026
6dd4eae
Truncate the regularly repeated long prompts and descriptions in the …
dmigwi Jul 28, 2026
f20afb7
Add the visited cells marker
dmigwi Jul 29, 2026
56e6678
Document the relationship between leastNeighborsBias value and path l…
dmigwi Jul 29, 2026
1d56901
Create an adjacency list as a representation of the traversal history
dmigwi Jul 29, 2026
46fe1cb
introduce a warning if duplicate tool calls are made by the agent
dmigwi Jul 29, 2026
d2251c4
handle model hallucination as a malformed-response error
dmigwi Jul 29, 2026
9bd4852
Fix the maze shared data that created a data race
dmigwi Jul 30, 2026
cb77328
Set strict to be true in the typescript compilation parameters
dmigwi Jul 30, 2026
5565a04
Log agent's win or loss in the level
dmigwi Jul 30, 2026
db5c761
Append application version to the downloaded logs file
dmigwi Jul 30, 2026
414915d
Add traversal speed interpratation of the BatchEfficiencyRank classif…
dmigwi Jul 30, 2026
18db7ff
Fix the random refreshes from the live server instance
dmigwi Jul 31, 2026
c6f6abe
Add status invariant check before persistence or rendering
dmigwi Jul 31, 2026
a4046fa
rename and add comments to maze config constants
dmigwi Jul 31, 2026
0054315
enforce type predicate on the various status assertions
dmigwi Jul 31, 2026
50c0847
Add junction branching tests
dmigwi Jul 31, 2026
3f28c92
Add benchmark tests for both environments
dmigwi Aug 1, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 79 additions & 0 deletions .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -130,3 +130,82 @@ jobs:
language: JavaScript
label: code-coverage/frontend
fail-on-error: false

benchmark:
name: Maze Benchmarks
runs-on: ubuntu-latest
timeout-minutes: 10
needs:
- test
# Pull requests run these too, scoped to the files that can move the numbers. The parity gate is
# only useful before a merge, not after it, and the whole sweep costs about eight seconds.
if: >-
github.event_name == 'workflow_dispatch' ||
github.event_name == 'pull_request' ||
(github.event_name == 'push' && github.ref == 'refs/heads/master')

steps:
- name: Check out code
uses: actions/checkout@v7
with:
# Full history so a pull request can be diffed against its base commit below.
fetch-depth: 0

# The workflow has no path filters, because every other job here should run on any change. The
# scoping therefore happens per job rather than at the trigger.
- name: Decide whether generation changed
id: scope
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
if [ "${{ github.event_name }}" != "pull_request" ]; then
echo "run=true" >> "$GITHUB_OUTPUT"
exit 0
fi

if git diff --name-only "$BASE_SHA" HEAD | grep -qE '^(maze/|frontend/bench/|frontend/app/(maze|traversal|config)\.ts|scripts/bench-report\.mjs|vitest\.bench\.config\.ts)'; then
echo "run=true" >> "$GITHUB_OUTPUT"
else
echo "run=false" >> "$GITHUB_OUTPUT"
fi

- name: Set up Go
if: steps.scope.outputs.run == 'true'
uses: actions/setup-go@v6
with:
go-version-file: go.mod
cache: true

- name: Set up pnpm
if: steps.scope.outputs.run == 'true'
uses: pnpm/action-setup@v6
with:
version: 11.7.0
run_install: false

- name: Set up Node.js
if: steps.scope.outputs.run == 'true'
uses: actions/setup-node@v6
with:
node-version: 22
cache: pnpm

- name: Install frontend dependencies
if: steps.scope.outputs.run == 'true'
run: pnpm install --frozen-lockfile

- name: Run maze branching benchmarks
if: steps.scope.outputs.run == 'true'
run: make ci-bench

# Uploaded even when the parity gate fails, since the report is what identifies which cases
# diverged. The console log is not a usable history: comparing a metric against an earlier
# commit otherwise means opening two runs and reading them side by side.
- name: Upload benchmark report
if: always() && steps.scope.outputs.run == 'true'
uses: actions/upload-artifact@v5
with:
name: maze-benchmark-report-${{ github.sha }}
path: bench-report.json
if-no-files-found: error
retention-days: 90
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,6 @@ public/css/*.min.css
_DS*
.DS_Store
.vscode
.claude
.claude
# Benchmark report emitted by "make ci-bench"
bench-report.json
12 changes: 11 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
.PHONY: help ci lint govulncheck deps frontend-install frontend-deps frontend-typecheck frontend-lint frontend-test frontend-quality frontend-build frontend-local test coverage clean-coverage
.PHONY: help ci ci-bench lint govulncheck deps frontend-install frontend-deps frontend-typecheck frontend-lint frontend-test frontend-quality frontend-build frontend-local frontend-bench test go-bench coverage clean-coverage

COVERAGE_FILE := coverage.out
GOCACHE := $(CURDIR)/.gocache
Expand All @@ -11,6 +11,7 @@ help:
@printf '%s\n' \
'Available targets:' \
' make ci Run the local equivalent of the CI pipeline.' \
' make ci-bench Run Go and frontend benchmark suites.' \
' make lint Run golangci-lint.' \
' make govulncheck Run govulncheck against this module.' \
' make frontend-install Install the pinned frontend toolchain.' \
Expand All @@ -23,6 +24,9 @@ help:

ci: lint frontend-lint govulncheck test

ci-bench: frontend-deps
node ./scripts/bench-report.mjs

lint:
golangci-lint run

Expand Down Expand Up @@ -55,6 +59,9 @@ frontend-lint:
frontend-test:
CI=true $(PNPM) --config.confirmModulesPurge=false run test:frontend

frontend-bench: frontend-deps
node ./scripts/bench-report.mjs --frontend-only

frontend-quality: frontend-deps
CI=true $(PNPM) --config.confirmModulesPurge=false run quality:frontend

Expand All @@ -67,6 +74,9 @@ test: deps frontend-deps frontend-typecheck frontend-build frontend-test
go test -race -covermode=atomic -coverprofile=$(COVERAGE_FILE) ./...
rm -f $(COVERAGE_FILE)

go-bench:
node ./scripts/bench-report.mjs --go-only

coverage:
go tool cover -func=$(COVERAGE_FILE)

Expand Down
18 changes: 9 additions & 9 deletions frontend/app/__snapshots__/maze.test.ts.snap
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
exports[`maze > generates a deterministic maze layout for a fixed random source 1`] = `
{
"finalPosition": {
"x": 9,
"x": 5,
"y": 1,
},
"maze": [
Expand All @@ -14,7 +14,7 @@ exports[`maze > generates a deterministic maze layout for a fixed random source
"---",
"-",
"---",
"-",
"|",
"---",
"-",
"---",
Expand All @@ -27,7 +27,7 @@ exports[`maze > generates a deterministic maze layout for a fixed random source
" ",
" ",
" ",
" ",
"|",
" ",
" ",
" ",
Expand All @@ -40,10 +40,10 @@ exports[`maze > generates a deterministic maze layout for a fixed random source
" ",
"-",
"---",
"-",
"---",
"-",
"---",
"|",
" ",
"|",
" ",
"|",
],
[
Expand All @@ -55,7 +55,7 @@ exports[`maze > generates a deterministic maze layout for a fixed random source
" ",
" ",
" ",
" ",
"|",
" ",
"|",
],
Expand All @@ -68,7 +68,7 @@ exports[`maze > generates a deterministic maze layout for a fixed random source
"---",
"-",
"---",
"-",
"|",
" ",
"|",
],
Expand Down
1 change: 1 addition & 0 deletions frontend/app/agent/config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,4 +78,5 @@ describe("agent config", () => {
}),
).toBe(CONFIG.agentConfig.invalidEndpointMessage)
})

})
76 changes: 62 additions & 14 deletions frontend/app/agent/context.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
AGENT_CONTEXT_TOOLS,
buildAgentMessages,
buildAgentToolHandlers,
buildDuplicateToolCallMessage,
describeAgentRankIdentity,
} from "./context"
import {
Expand Down Expand Up @@ -33,7 +34,8 @@ function createAgent(overrides: Partial<AgentApiConfig> = {}): AgentApiConfig {
endpoint: new URL("https://agents.example/chat"),
enabled: true,
gameLevel: 4,
requestsCount: 2,
turnCount: 2,
decayUnitsCharged: 2,
...overrides,
}
}
Expand All @@ -44,17 +46,17 @@ const expectedAgentPrompt = [
"currentCell is your current position; destinationCell is the target.",
"The maze is randomly generated at each level with exactly one path to the destination.",
"traversalHistory entries matching your playerName record your past moves in chronological order.",
"Each entry includes openMoves — the open exits from that cell are fixed since creation — helping you reconstruct the maze's path flow; so entries recorded by other players are just as trustworthy as your own.",
"openMoves count reveals the physical maze structure at that cell: one open move is a dead end (unless that is your start or destination cell); two is a corridor; three or more is a junction.",
"Each entry's openMoves maps every open exit from that cell directly to the neighboring cell it leads to and whether that neighbor is already visited — exits from a cell are fixed since creation, so this helps you reconstruct the maze's path flow without computing adjacency yourself; entries recorded by other players are just as trustworthy as your own.",
"openMoves key count reveals the physical maze structure at that cell: one open exit is a dead end (unless that is your start or destination cell); two is a corridor; three or more is a junction.",
"traversalHistory only records the first visit to each cell; cells revisited during backtracking are not duplicated, so apparent gaps are expected.",
"Revisiting a cell already in traversalHistory is not a mistake — once the current path is confirmed as leading to a dead end, backtracking through those cells is usually the only way to reach unexplored territory or the destination.",
"By design, the maze never guarantees a direct route from start to destination; the only valid path may require moving away from the target before turning towards it.",
"Tool results reflect the maze state at the time of each call — a repeat call may return updated or identical data depending on what has changed.",
"get_last_replay_result reflects the most recent replay across all agents; lastPlayerName identifies whose outcome it is.",
"lastMoveStatus being null means no moves have been made yet; invalid-move means the last prediction hit a wall; malformed-response means the previous response was not valid JSON and a penalty of 2 decay units was charged; applied means it succeeded. A turn with any valid moves costs a constant 1 decay units regardless of how many moves it applied; invalid moves (any moves after the last valid applied move) add a further penalty of 2 decay units on top — the maximum possible in a turn is 3 decay units.",
"lastMoveStatus being null means no moves have been made yet; invalid-move means the last prediction hit a wall; malformed-response means the previous response was not valid JSON, requested a tool that does not exist, or ignored a duplicate tool call warning — in all cases a penalty of 2 decay units was charged; applied means it succeeded. A turn with any valid moves costs a constant 1 decay units regardless of how many moves it applied; invalid moves (any moves after the last valid applied move) add a further penalty of 2 decay units on top — the maximum possible in a turn is 3 decay units.",
"get_prediction_rules provides the required response format and move count guidance.",
"Moves replay in submitted order until the destination is reached or the first invalid move (a wall collision or out-of-bounds step) is hit.",
"Longer, well-reasoned predictions are strictly cheaper per move than single-stepping — a trailblazer can set a new scores retention record, a navigator's odds of finishing drop sharply, and a backtracker is almost certain to fail unless it corrects course.",
"Because the charge above is per turn rather than per move, a longer prediction whose moves all land, covers more new cells for the same decay — that ratio is your traversal speed, and it is the rank you carry: a trailblazer can set a new scores retention record, a navigator's odds of finishing drop sharply, and a backtracker is almost certain to fail unless it corrects course.",
"lastMoveStatus reached-target or status won means the game is complete — stop predicting.",
].join(" ")

Expand Down Expand Up @@ -87,6 +89,7 @@ function createState(overrides: Partial<State> = {}): State {
["|", " ", " ", " ", "|"],
["|", "---", "|", "---", "|"],
],
startPosition: { x: 1, y: 1 },
playerPosition: { x: 1, y: 1 },
traversalHistory: [selfVisit(0, 0, ["MoveRight"]), agentVisit(0, 1, "Blue", ["MoveRight"])],
finalPosition: { x: 3, y: 1 },
Expand All @@ -95,13 +98,12 @@ function createState(overrides: Partial<State> = {}): State {
lastRoundScore: 0,
lastAttemptRetentionUnits: null,
bestWinRetentionUnits: null,
lastWinRequestCount: null,
bestWinRequestCount: null,
lastWinTraversalSpeedUnits: null,
bestWinTraversalSpeedUnits: null,
winSummary: "",
canResume: false,
wallWeight: 1,
scoreDecayUnits: 0,
agentRequestCount: 0,
turnCount: 0,
cumulativeRoundCount: 0,
clock: null,
...overrides,
Expand Down Expand Up @@ -140,12 +142,24 @@ describe("agent context", () => {
destinationCell: { row: 0, col: 1 },
})
expect(toolHandlers.get_traversal_history({})).toEqual({
traversalHistory: [selfVisit(0, 0, ["MoveRight"]), agentVisit(0, 1, "Blue", ["MoveRight"])],
traversalHistory: [
{
playerName: "Self",
cell: { row: 0, col: 0 },
openMoves: { MoveRight: { row: 0, col: 1, visited: true } },
},
{
playerName: "Blue",
cell: { row: 0, col: 1 },
openMoves: { MoveRight: { row: 0, col: 2, visited: false } },
},
],
})
expect(toolHandlers.get_prediction_rules({})).toEqual({
suggestedMovesPerTurn: 4,
uniqueCellsVisited: 1,
requestsMade: 2,
decayUnitsCharged: 2,
turnsTaken: 2,
batchEfficiencyRank: "backtracker",
expectedResponseSchema,
})
Expand All @@ -161,13 +175,14 @@ describe("agent context", () => {
})

it("defaults a fresh agent's prediction rules to a trailblazer level regardless of raw counts", () => {
const freshAgent = createAgent({ requestsCount: undefined })
const freshAgent = createAgent({ turnCount: undefined, decayUnitsCharged: undefined })
const toolHandlers = buildAgentToolHandlers(createState(), null, freshAgent)

expect(toolHandlers.get_prediction_rules({})).toEqual({
suggestedMovesPerTurn: 4,
uniqueCellsVisited: 1,
requestsMade: 0,
decayUnitsCharged: 0,
turnsTaken: 0,
batchEfficiencyRank: "trailblazer",
expectedResponseSchema,
})
Expand All @@ -181,13 +196,46 @@ describe("agent context", () => {
},
{
role: "user",
content: `It is Blue's turn to predict Tapoo maze moves. Use the available tools to inspect the current maze state.`,
content: `It is Blue's turn to predict next moves. Use the available tools to see the maze state.`,
},
])
})

})

describe("buildDuplicateToolCallMessage", () => {
it("names a single duplicate call as an explicit warning tied to malformed-response", () => {
const message = buildDuplicateToolCallMessage([
{ id: "call_2", function: { name: "get_game_status", arguments: {} } },
])

expect(message).toEqual({
role: "user",
content:
"Warning: get_game_status (call_2) won't yield any new information. " +
"You may still call any tools you haven't used yet, or respond now with only the moves JSON. " +
"Requesting these tool call(s) once again will be treated as a malformed-response.",
})
})

it("lists multiple duplicate calls together, in order", () => {
const message = buildDuplicateToolCallMessage([
{ id: "call_2", function: { name: "get_game_status", arguments: {} } },
{ id: "call_3", function: { name: "get_maze_positions", arguments: {} } },
])

expect(message.content).toContain(
"get_game_status (call_2), get_maze_positions (call_3) won't yield any new information.",
)
})

it("falls back to placeholders when a call is missing its name or id", () => {
const message = buildDuplicateToolCallMessage([{ function: { arguments: {} } }])

expect(message.content).toContain("unknown (no id)")
})
})

describe("describeAgentRankIdentity", () => {
it("tells a trailblazer to maintain the rank rather than climb toward it", () => {
const description = describeAgentRankIdentity("Blue", "trailblazer")
Expand Down
Loading
Loading