Merge pull request #6 from NetX-lab/codex/benchmark-simulator-cleanup

Release NetOpsBench 0.2 scalable benchmark runtime

Author: yyyyyt123

.dockerignore

@@ -0,0 +1,27 @@
+.git
+.github
+.env
+.env.*
+!.env.example
+.netopsbench*
+.mypy_cache
+.pytest_cache
+.ruff_cache
+.venv
+venv
+build
+dist
+*.egg-info
+**/__pycache__
+**/*.py[cod]
+**/.DS_Store
+**/*.log
+
+docs
+examples
+lab-topology
+scenario_results
+scenarios
+scripts
+tests
+native/client-agent/target

.env.example

@@ -32,7 +32,6 @@ NETOPSBENCH_INFLUXDB_BUCKET=netopsbench
 # NETOPSBENCH_LOG_LEVEL=INFO                   # DEBUG | INFO | WARNING | ERROR
 # NETOPSBENCH_NO_SUDO=0                        # Set to 1 in CI/test environments without sudo
 # NETOPSBENCH_PYTHON=/path/to/python3          # Interpreter used by repository shell wrappers
-# NETOPSBENCH_TRAFFIC_PARALLELISM=32           # Parallel docker exec workers for benchmark traffic start/stop
 
 # --- Optional semantic fault-type judge ---
 # NETOPSBENCH_FAULT_TYPE_JUDGE_ENABLED=0

.github/workflows/client-agent-image.yml

@@ -0,0 +1,81 @@
+name: client-agent-image
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "v*"
+    paths:
+      - ".github/workflows/client-agent-image.yml"
+      - ".dockerignore"
+      - "containers/client/**"
+      - "native/client-agent/**"
+  workflow_dispatch:
+
+concurrency:
+  group: client-agent-image-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Set immutable image tags
+        id: tags
+        shell: bash
+        run: |
+          tags="yyyyyt123/netopsbench-client:sha-${GITHUB_SHA}"
+          if [[ "${GITHUB_REF}" == "refs/heads/main" ]]; then
+            tags="${tags}"$'\n'"yyyyyt123/netopsbench-client:latest"
+          fi
+          if [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then
+            tags="${tags}"$'\n'"yyyyyt123/netopsbench-client:${GITHUB_REF_NAME}"
+          fi
+          {
+            echo "value<<EOF"
+            echo "${tags}"
+            echo "EOF"
+          } >> "${GITHUB_OUTPUT}"
+
+      - name: Build and publish linux/amd64 image
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: containers/client/Dockerfile
+          platforms: linux/amd64
+          push: true
+          provenance: false
+          tags: ${{ steps.tags.outputs.value }}
+          labels: |
+            org.opencontainers.image.source=https://github.com/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
+
+      - name: Publish manifest digest
+        run: echo "digest=${{ steps.build.outputs.digest }}" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Verify anonymous digest pull
+        env:
+          IMAGE_DIGEST: ${{ steps.build.outputs.digest }}
+        run: |
+          docker logout
+          for attempt in $(seq 1 12); do
+            if docker manifest inspect "yyyyyt123/netopsbench-client@${IMAGE_DIGEST}" >/dev/null; then
+              exit 0
+            fi
+            sleep 5
+          done
+          exit 1

.github/workflows/docs-pages.yml

@@ -4,6 +4,13 @@ on:
   push:
     branches:
       - main
+    paths:
+      - "docs/**"
+      - ".github/workflows/docs-pages.yml"
+  pull_request:
+    paths:
+      - "docs/**"
+      - ".github/workflows/docs-pages.yml"
   workflow_dispatch:
 
 permissions:
@@ -12,7 +19,7 @@ permissions:
   id-token: write
 
 concurrency:
-  group: docs-pages
+  group: docs-pages-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -32,6 +39,7 @@ jobs:
           cache-dependency-path: docs/package-lock.json
 
       - name: Configure Pages
+        if: github.event_name != 'pull_request'
         uses: actions/configure-pages@v5
         with:
           enablement: true
@@ -45,11 +53,13 @@ jobs:
         run: npm run build
 
       - name: Upload Pages artifact
+        if: github.event_name != 'pull_request'
         uses: actions/upload-pages-artifact@v3
         with:
           path: docs/out
 
   deploy:
+    if: github.event_name != 'pull_request'
     needs: build
     runs-on: ubuntu-latest
     environment:

.github/workflows/test.yml

@@ -9,7 +9,45 @@ on:
   pull_request:
 
 jobs:
-  lint:
+  rust:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: native/client-agent
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install pinned Rust
+        run: |
+          rustup toolchain install 1.97.1 --profile minimal --component clippy,rustfmt
+          rustup default 1.97.1
+
+      - name: Format
+        run: cargo fmt --check
+
+      - name: Clippy
+        run: cargo clippy --locked --all-targets -- -D warnings
+
+      - name: Test
+        run: cargo test --locked
+
+  client-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build client image
+        run: docker build -f containers/client/Dockerfile -t netopsbench-client:test .
+
+      - name: Verify native-only runtime
+        run: |
+          docker run --rm netopsbench-client:test netopsbench-client-agent --version
+          docker run --rm netopsbench-client:test sh -lc \
+            'command -v ping && command -v traceroute && ! command -v python3 && ! command -v iperf3'
+
+  wheel:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -20,6 +58,46 @@ jobs:
         with:
           python-version: "3.12"
 
+      - name: Build wheel
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip wheel --no-deps --wheel-dir dist .
+
+      - name: Verify package contents and imports
+        run: |
+          python - <<'PY'
+          import pathlib
+          import zipfile
+
+          wheel = next(pathlib.Path("dist").glob("netopsbench-*.whl"))
+          with zipfile.ZipFile(wheel) as archive:
+              names = set(archive.namelist())
+          forbidden = {
+              "netopsbench/platform/simulator/contracts.py",
+              "netopsbench/platform/simulator/engine.py",
+              "netopsbench/platform/simulator/payloads.py",
+          }
+          unexpected = sorted(forbidden & names)
+          if unexpected:
+              raise SystemExit(f"wheel contains removed modules: {unexpected}")
+          PY
+          python -m venv /tmp/netopsbench-wheel-smoke
+          /tmp/netopsbench-wheel-smoke/bin/pip install dist/netopsbench-*.whl
+          cd /tmp
+          /tmp/netopsbench-wheel-smoke/bin/python -c "from netopsbench.sdk import DiagnosticSession, IncidentEngine, NetOpsBench"
+          /tmp/netopsbench-wheel-smoke/bin/netopsbench --help >/dev/null
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
       - name: Install lint dependencies
         run: |
           python -m pip install --upgrade pip

.gitignore

@@ -1,4 +1,5 @@
 __pycache__/
+native/client-agent/target/
 *.py[cod]
 *$py.class
 .venv/

containers/client/Dockerfile

@@ -0,0 +1,27 @@
+# syntax=docker/dockerfile:1
+FROM rust:1.97.1-alpine3.24@sha256:3c38f3f82c2f3d73da3b38e18d279393a04cb43ddded0e35088a8c3324d40900 AS builder
+
+RUN apk add --no-cache musl-dev
+WORKDIR /src/native/client-agent
+COPY native/client-agent/Cargo.toml native/client-agent/Cargo.lock native/client-agent/rust-toolchain.toml ./
+COPY native/client-agent/src ./src
+RUN cargo build --locked --release --target x86_64-unknown-linux-musl
+
+FROM alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b
+
+RUN apk add --no-cache \
+      bash \
+      ca-certificates \
+      curl \
+      ethtool \
+      iproute2 \
+      iputils \
+      procps \
+      tcpdump \
+      traceroute
+
+COPY --from=builder \
+  /src/native/client-agent/target/x86_64-unknown-linux-musl/release/netopsbench-client-agent \
+  /usr/local/bin/netopsbench-client-agent
+
+CMD ["sleep", "infinity"]

docs/STRUCTURE.md

@@ -7,7 +7,6 @@ This documentation site follows a Harbor-Lite layout:
 - `src/lib/` - shared site logic such as source loading, layout options, and base-path helpers
 - `content/docs/` - the single source of truth for published documentation content
 - `public/assets/` - static assets served by the site at runtime
-- `_legacy/` - archived historical markdown files kept for reference only
 
 ## Rules
 
@@ -21,7 +20,6 @@ This documentation site follows a Harbor-Lite layout:
 ## Asset guidance
 
 - Use `public/assets/` for images that must be publicly addressable from pages or MDX.
-- Treat `assets/` as temporary or source-material storage only until it is fully audited.
 - If an asset is referenced via `/assets/...`, it should exist under `public/assets/`.
 
 ## Validation

docs/content/docs/build-your-agent/custom-agents.mdx

@@ -47,7 +47,6 @@ with NetOpsBench(workspace=".") as bench:
 | `symptoms` | Episode observations, including Pingmesh summaries when available. |
 | `tools` | Optional runtime or MCP-backed tool access. |
 | `metadata` | Runtime metadata such as worker-specific environment values. |
-| `ground_truth` | Optional ground truth. Benchmark agents should not use it for diagnosis. |
 
 The reference LLM agent builds a compact prompt from topology size, symptom keys, episode data, Pingmesh windows, and high-signal anomalies. A custom agent may instead use rules, symbolic logic, retrieval, additional tool calls, or another model wrapper.
 
@@ -68,7 +67,7 @@ The reference LLM agent builds a compact prompt from topology size, symptom keys
 
 Return structured fields rather than only natural language. Free-form explanations are useful for review, but scoring depends on verdict, fault type, and location fields.
 
-For reproducibility, NetOpsBench saves a per-case runtime trace beside the raw scenario result. The bundled reference agent captures private LLM and tool events by attaching `context.trace.langchain_callback()` to its LangChain-compatible runtime. Custom non-LangChain agents can use advanced manual recorder methods such as `context.trace.record_llm_request(...)` and `context.trace.record_llm_response(...)` when they need private model calls in the trace. The harness writes ATIF v1.7 `trajectory.atif.json` artifacts for Harbor-style inspection while keeping ground truth out of the agent trajectory; scoring details are linked separately through `traces/results.jsonl`. Use `netopsbench trace view` to sync trace-enabled runs into the local Harbor viewer cache, or `netopsbench trace view <run_id>` to ensure a specific saved run is available in the viewer.
+For reproducibility, NetOpsBench saves a per-attempt runtime trace beside the raw scenario result. The bundled reference agent captures private LLM and tool events by attaching `context.trace.langchain_callback()` to its LangChain-compatible runtime. Custom non-LangChain agents can use advanced manual recorder methods such as `context.trace.record_llm_request(...)` and `context.trace.record_llm_response(...)` when they need private model calls in the trace. The harness writes uniquely named ATIF v1.7 `trajectory-<trace-id>.atif.json` artifacts for Harbor-style inspection while keeping ground truth out of the agent trajectory; scoring details are linked separately through `traces/results.jsonl`. Use `netopsbench trace view` to sync trace-enabled runs into the local Harbor viewer cache, or `netopsbench trace view <run_id>` to ensure a specific saved run is available in the viewer.
 
 Trace storage preserves visible agent-environment interactions with secret redaction and per-field size limits. NetOpsBench does not monkeypatch arbitrary LLM SDKs, so fully private model prompts and responses are captured only when the agent uses a supported framework callback or the manual recorder methods.
 
@@ -90,18 +89,17 @@ NetOpsBench scores the returned result along these dimensions:
 
 | Dimension | What must be correct |
 |---|---|
-| Detection | Faulted cases should return `fault_detected`; healthy negative samples should return `network_healthy`. |
+| Detection | Faulted cases should return `fault_detected`; healthy cases should return `network_healthy`. |
 | Device localization | `findings.location.device` should match the ground-truth device or an accepted equivalent location. |
 | Interface localization | `findings.location.interface` should match when interface-level ground truth exists. |
 | Fault type | `findings.fault_type` should match the canonical fault label, or a semantically equivalent label when the optional judge is enabled. |
 | Efficiency | Reports may include diagnosis time, tool calls, and token usage from metadata. |
 
-For interface-applicable faults, localization score is split between device and interface correctness. Healthy negative samples receive credit only for the correct `network_healthy` verdict.
+For interface-applicable faults, localization score is split between device and interface correctness. Healthy cases receive credit only for the correct `network_healthy` verdict.
 
 ## Practical checklist
 
 - Use exact verdict strings: `fault_detected`, `network_healthy`, `inconclusive`.
 - Include device and interface when the evidence supports localization.
 - Prefer canonical fault names from the built-in fault catalog.
-- Treat `ground_truth` as evaluator data, not as diagnosis input.
 - Return `success=False` and an `inconclusive` verdict when the agent runtime fails.