diff --git a/.gitattributes b/.gitattributes index a57ccb19..0d48f91e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,13 +1,17 @@ -.gitattributes text eol=lf +# Auto detect text files and perform LF normalization +* text=auto + +# Source and docs +*.md text eol=lf *.sg text eol=lf -Sengoo.toml text eol=lf -Sengoo.lock text eol=lf -sgpm-index.toml text eol=lf -*.yaml text eol=lf +*.rs text eol=lf +*.toml text eol=lf *.yml text eol=lf -*.md text eol=lf +*.yaml text eol=lf +*.json text eol=lf +*.raw text eol=lf +*.ps1 text eol=lf # Performance evidence: pin byte-identical SHA-256 across hosts bench/results/**/*.json -text bench/frontend-memory-baseline.json text eol=lf - diff --git a/.github/workflows/core-conformance.yml b/.github/workflows/core-conformance.yml index eb40dc98..35a2785b 100644 --- a/.github/workflows/core-conformance.yml +++ b/.github/workflows/core-conformance.yml @@ -57,3 +57,569 @@ jobs: name: debugger-native-lldb-transcripts path: debugger-transcripts/ if-no-files-found: error + + binary-io-native: + name: binary I/O native (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + env: + LLVM_VERSION: "19" + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@master + with: + toolchain: 1.94.0 + + - name: Install pinned LLVM toolchain (Ubuntu) + if: runner.os == 'Linux' + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y clang-${LLVM_VERSION} lld-${LLVM_VERSION} + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${LLVM_VERSION} 100 + + - name: Require clang + shell: pwsh + run: | + $clang = Get-Command clang -ErrorAction Stop + & $clang.Source --version + if ($LASTEXITCODE -ne 0) { + throw "clang probe failed with exit code $LASTEXITCODE" + } + + - name: Run compiler binary I/O signature surface + run: cargo test --locked -p sengoo-compiler io_module_preserves_binary_buffer_and_pipe_signatures -- --nocapture + + - name: Run sgc binary I/O import surface + run: cargo test --locked -p sgc binary_io_import_expands_exact_buffer_and_pipe_surface -- --nocapture + + - name: Run LSP binary I/O completion and signature surface + run: cargo test --locked -p sglsp binary_io_import_exposes_completion_and_exact_signatures -- --nocapture + + - name: Run real-sgc stdlib binary I/O wrappers + run: cargo test --locked -p sgc stdlib_io_ -- --nocapture --test-threads=1 + + - name: Run Buffer byte integration tests + run: cargo test --locked -p sgc --test buffer_bytes -- --nocapture --test-threads=1 + + - name: Run exact-read native and pipe tests + run: cargo test --locked -p sgc --test binary_io_exact_read -- --nocapture --test-threads=1 + + - name: Run write-all native and pipe tests + run: cargo test --locked -p sgc --test binary_io_write_all -- --nocapture --test-threads=1 + + senline-worker-differential: + name: Senline worker differential (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + env: + LLVM_VERSION: "19" + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@master + with: + toolchain: 1.94.0 + + - name: Install pinned LLVM toolchain (Ubuntu) + if: runner.os == 'Linux' + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y clang-${LLVM_VERSION} lld-${LLVM_VERSION} + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${LLVM_VERSION} 100 + + - name: Require clang + shell: pwsh + run: | + $clang = Get-Command clang -ErrorAction Stop + & $clang.Source --version + if ($LASTEXITCODE -ne 0) { + throw "clang probe failed with exit code $LASTEXITCODE" + } + + - name: Verify corpus metadata and fresh-process determinism + run: cargo test --locked -p sgc --test senline_worker_differential -- --nocapture --test-threads=1 + + - name: Run 10k reviewed and 100k seeded release corpora + run: cargo test --release --locked -p sgc --test senline_worker_differential -- --ignored --nocapture --test-threads=1 + + - name: Resource sampler smoke + latency percentiles (task 8.4) + run: cargo test --release --locked -p sgc --test senline_worker_resource resource_sampler_smoke_single_worker_with_latency_percentiles -- --nocapture + + - name: Upload platform transcript evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: senline-worker-differential-${{ runner.os }}-${{ runner.arch }} + path: | + target/senline-differential/*.json + target/senline-resource/*smoke*.summary.json + if-no-files-found: warn + + senline-installed-packages: + name: installed worker/HTTP (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + target: x86_64-unknown-linux-gnu + artifact: linux-x86_64 + - os: windows-latest + target: x86_64-pc-windows-msvc + artifact: windows-x86_64 + env: + LLVM_VERSION: "19" + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@master + with: + toolchain: 1.94.0 + + - name: Install pinned LLVM toolchain (Ubuntu) + if: runner.os == 'Linux' + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y clang-${LLVM_VERSION} lld-${LLVM_VERSION} + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${LLVM_VERSION} 100 + + - name: Require clang + shell: pwsh + run: | + $clang = Get-Command clang -ErrorAction Stop + & $clang.Source --version + if ($LASTEXITCODE -ne 0) { + throw "clang probe failed with exit code $LASTEXITCODE" + } + + - name: Build release tools + shell: pwsh + run: cargo build -p sgc -p sgpm -p sgfmt -p sglsp --release + + - name: Package installed toolchain + shell: pwsh + env: + SENGOO_DIST_TARGET: ${{ matrix.target }} + run: | + $version = "0.1.0-ci" + $cargoTarget = Join-Path $env:RUNNER_TEMP "sengoo-cargo-package-${{ matrix.artifact }}" + Remove-Item -LiteralPath "target/dist" -Recurse -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $cargoTarget -Recurse -Force -ErrorAction SilentlyContinue + Remove-Item Env:CARGO_ENCODED_RUSTFLAGS -ErrorAction SilentlyContinue + Remove-Item Env:RUSTFLAGS -ErrorAction SilentlyContinue + ./scripts/package-toolchain.ps1 ` + -Version $version ` + -OutputDir "target/dist" ` + -CargoTargetDir $cargoTarget ` + -SmokeEvidence "core-conformance senline-installed ${{ matrix.os }} ${{ github.sha }}" + + - name: Install toolchain outside product workspace + shell: pwsh + run: | + $archiveDir = "target/dist" + $latest = (Get-Content (Join-Path $archiveDir "latest-archive.txt") -ErrorAction SilentlyContinue | Select-Object -First 1) + if (-not $latest) { + $zip = @(Get-ChildItem $archiveDir -File -ErrorAction SilentlyContinue | + Where-Object { $_.Name -like '*.zip' -or $_.Name -like '*.tar.gz' } | + Select-Object -First 1) + if ($zip.Count -eq 0) { throw "missing packaged archive under $archiveDir" } + $latest = $zip[0].FullName + } elseif (-not [IO.Path]::IsPathRooted($latest)) { + $latest = Join-Path $archiveDir $latest + } + $installDir = Join-Path $env:RUNNER_TEMP "sengoo-install-${{ matrix.artifact }}" + Remove-Item -LiteralPath $installDir -Recurse -Force -ErrorAction SilentlyContinue + if ($IsWindows -or $PSVersionTable.PSEdition -eq "Desktop") { + ./scripts/install.ps1 -Archive $latest -InstallDir $installDir + } else { + sh scripts/install.sh "$latest" "$installDir" + } + Write-Host "install_dir=$installDir" + + - name: Fake cargo + installed worker/HTTP dual package loop + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + $installDir = Join-Path $env:RUNNER_TEMP "sengoo-install-${{ matrix.artifact }}" + $bin = Join-Path $installDir "bin" + $suffix = "" + if ($IsWindows -or $PSVersionTable.PSEdition -eq "Desktop") { + $suffix = ".exe" + } + $sgc = Join-Path $bin "sgc$suffix" + $sgpm = Join-Path $bin "sgpm$suffix" + if (-not (Test-Path -LiteralPath $sgc)) { throw "missing installed sgc: $sgc" } + if (-not (Test-Path -LiteralPath $sgpm)) { throw "missing installed sgpm: $sgpm" } + + $fakeCargoDir = Join-Path $env:RUNNER_TEMP "fake-cargo-fail" + Remove-Item -LiteralPath $fakeCargoDir -Recurse -Force -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force -Path $fakeCargoDir | Out-Null + $fakeCargo = Join-Path $fakeCargoDir "cargo$suffix" + if ($suffix -eq ".exe") { + "@echo off`r`necho fake-cargo: cargo must not be invoked by installed senline package loops 1>&2`r`nexit /b 97`r`n" | Set-Content -LiteralPath $fakeCargo -Encoding ASCII -NoNewline + } else { + "#!/bin/sh`necho `"fake-cargo: cargo must not be invoked by installed senline package loops`" >&2`nexit 97`n" | Set-Content -LiteralPath $fakeCargo -Encoding UTF8 + & chmod +x $fakeCargo + } + # Use the OS path separator; ';' breaks Linux PATH and makes which(sgc) + # fall back to checkout target/release/sgc (source-local). + $pathSep = [IO.Path]::PathSeparator + $env:PATH = "$fakeCargoDir$pathSep$bin$pathSep" + $env:PATH + $env:SGPM_SGC = $sgc + $env:SGPM_SGFMT = Join-Path $bin "sgfmt$suffix" + Remove-Item Env:SENGOO_ROOT -ErrorAction SilentlyContinue + Remove-Item Env:SENGOO_STDLIB -ErrorAction SilentlyContinue + Remove-Item Env:SENGOO_RUNTIME -ErrorAction SilentlyContinue + + Write-Host "sgc: $(& $sgc --version) path=$sgc" + Write-Host "sgpm: $(& $sgpm --version) path=$sgpm" + Write-Host "SGPM_SGC=$env:SGPM_SGC" + + Push-Location examples/realworld/senline-domain-worker + try { + & $sgpm --runtime-mode installed check --locked + if ($LASTEXITCODE -ne 0) { throw "worker sgpm installed check --locked failed" } + & $sgpm --runtime-mode installed test --locked + if ($LASTEXITCODE -ne 0) { throw "worker sgpm installed test --locked failed" } + & $sgpm fmt --check --locked + if ($LASTEXITCODE -ne 0) { throw "worker sgpm fmt --check --locked failed" } + & $sgpm --runtime-mode installed doc --locked + if ($LASTEXITCODE -ne 0) { throw "worker sgpm doc --locked failed" } + & $sgpm --runtime-mode installed build --locked --release + if ($LASTEXITCODE -ne 0) { throw "worker sgpm installed build --locked --release failed" } + } finally { + Pop-Location + } + + Push-Location examples/realworld/senline-http-dogfood + try { + & $sgpm --runtime-mode installed check --locked + if ($LASTEXITCODE -ne 0) { throw "http sgpm installed check --locked failed" } + & $sgpm --runtime-mode installed test --locked + if ($LASTEXITCODE -ne 0) { throw "http sgpm installed test --locked failed" } + & $sgpm fmt --check --locked + if ($LASTEXITCODE -ne 0) { throw "http sgpm fmt --check --locked failed" } + & $sgpm --runtime-mode installed doc --locked + if ($LASTEXITCODE -ne 0) { throw "http sgpm doc --locked failed" } + & $sgpm --runtime-mode installed build --locked --release + if ($LASTEXITCODE -ne 0) { throw "http sgpm installed build --locked --release failed" } + } finally { + Pop-Location + } + + $pkgRoot = Join-Path $env:RUNNER_TEMP "senline-pkg-${{ matrix.artifact }}" + Remove-Item -LiteralPath $pkgRoot -Recurse -Force -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force -Path $pkgRoot | Out-Null + + # Fresh package trees between A/B so dual-build does not silently reuse + # a single package output directory. Force worker rebuild between packages + # by clearing package-local target and frontend build caches. + function Clear-SenlinePackageBuildCaches([string]$PackageRoot) { + foreach ($rel in @("target", "src/build", "tests/build")) { + $p = Join-Path $PackageRoot $rel + if (Test-Path -LiteralPath $p) { + Remove-Item -LiteralPath $p -Recurse -Force -ErrorAction SilentlyContinue + } + } + Get-ChildItem -LiteralPath $PackageRoot -Recurse -Directory -Filter build -ErrorAction SilentlyContinue | + Where-Object { $_.FullName -match '[\\/]packages[\\/]' } | + ForEach-Object { + Remove-Item -LiteralPath $_.FullName -Recurse -Force -ErrorAction SilentlyContinue + } + } + + ./scripts/package-senline-worker.ps1 -SgcPath $sgc -SgpmPath $sgpm -OutputDir (Join-Path $pkgRoot "worker-a") + Clear-SenlinePackageBuildCaches "examples/realworld/senline-domain-worker" + ./scripts/package-senline-worker.ps1 -SgcPath $sgc -SgpmPath $sgpm -OutputDir (Join-Path $pkgRoot "worker-b") + # Fail-closed pin-grade compare (payload + dependency identities). Record + # result but continue product probes so functional gates still run when + # PE/ELF dual-build identity remains open under task 8.7. + $compareFailures = @() + try { + ./scripts/compare-senline-package-manifests.ps1 ` + -LeftManifest (Join-Path $pkgRoot "worker-a/worker-manifest.json") ` + -RightManifest (Join-Path $pkgRoot "worker-b/worker-manifest.json") ` + -OutputDir (Join-Path $pkgRoot "worker-compare") + } catch { + $compareFailures += "worker dual-package compare failed: $($_.Exception.Message)" + Write-Host "::warning::$($compareFailures[-1])" + } + + ./scripts/package-senline-http.ps1 -SgcPath $sgc -SgpmPath $sgpm -OutputDir (Join-Path $pkgRoot "http-a") + Clear-SenlinePackageBuildCaches "examples/realworld/senline-http-dogfood" + ./scripts/package-senline-http.ps1 -SgcPath $sgc -SgpmPath $sgpm -OutputDir (Join-Path $pkgRoot "http-b") + try { + ./scripts/compare-senline-package-manifests.ps1 ` + -LeftManifest (Join-Path $pkgRoot "http-a/http-manifest.json") ` + -RightManifest (Join-Path $pkgRoot "http-b/http-manifest.json") ` + -OutputDir (Join-Path $pkgRoot "http-compare") + } catch { + $compareFailures += "http dual-package compare failed: $($_.Exception.Message)" + Write-Host "::warning::$($compareFailures[-1])" + } + + # Real parent/child framed worker product loop against packaged binary. + $workerExeName = if ($suffix -eq ".exe") { "senline_domain_worker.exe" } else { "senline_domain_worker" } + $workerExe = Join-Path $pkgRoot "worker-a/$workerExeName" + if (-not (Test-Path -LiteralPath $workerExe)) { throw "missing packaged worker: $workerExe" } + $requestPath = "examples/realworld/senline-domain-worker/fixtures/v1/cases/eligible-accept.request.json" + $planPath = "examples/realworld/senline-domain-worker/fixtures/v1/cases/eligible-accept.plan.json" + $handshakePath = Join-Path $pkgRoot "worker-a/fixtures/v1/handshake/ready.json" + $probeDir = Join-Path $pkgRoot "worker-product-probe" + New-Item -ItemType Directory -Force -Path $probeDir | Out-Null + $reqBytes = [IO.File]::ReadAllBytes((Resolve-Path $requestPath)) + $frame = New-Object byte[] ($reqBytes.Length + 4) + $lenBytes = [BitConverter]::GetBytes([uint32]$reqBytes.Length) + if ([BitConverter]::IsLittleEndian) { [Array]::Reverse($lenBytes) } + [Array]::Copy($lenBytes, 0, $frame, 0, 4) + [Array]::Copy($reqBytes, 0, $frame, 4, $reqBytes.Length) + $psi = New-Object System.Diagnostics.ProcessStartInfo + $psi.FileName = $workerExe + $psi.WorkingDirectory = $probeDir + $psi.RedirectStandardInput = $true + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + $psi.UseShellExecute = $false + $proc = [Diagnostics.Process]::Start($psi) + $stdin = $proc.StandardInput.BaseStream + $stdout = $proc.StandardOutput.BaseStream + function Read-Frame([IO.Stream]$s) { + $prefix = New-Object byte[] 4 + $read = 0 + while ($read -lt 4) { + $n = $s.Read($prefix, $read, 4 - $read) + if ($n -le 0) { throw "EOF reading frame prefix" } + $read += $n + } + if ([BitConverter]::IsLittleEndian) { [Array]::Reverse($prefix) } + $len = [BitConverter]::ToUInt32($prefix, 0) + if ($len -gt 8192) { throw "frame too large: $len" } + $payload = New-Object byte[] $len + $read = 0 + while ($read -lt $len) { + $n = $s.Read($payload, $read, $len - $read) + if ($n -le 0) { throw "EOF reading frame payload" } + $read += $n + } + return $payload + } + function Bytes-Key([byte[]]$b) { + $filtered = New-Object System.Collections.Generic.List[byte] + foreach ($x in $b) { if ($x -ne 13) { [void]$filtered.Add($x) } } + return [Convert]::ToBase64String($filtered.ToArray()) + } + $gotHandshake = Read-Frame $stdout + $wantHandshake = [IO.File]::ReadAllBytes((Resolve-Path $handshakePath)) + if ((Bytes-Key $gotHandshake) -ne (Bytes-Key $wantHandshake)) { + throw "packaged worker handshake mismatch" + } + $stdin.Write($frame, 0, $frame.Length) + $stdin.Flush() + $stdin.Close() + $gotPlan = Read-Frame $stdout + $wantPlan = [IO.File]::ReadAllBytes((Resolve-Path $planPath)) + if ((Bytes-Key $gotPlan) -ne (Bytes-Key $wantPlan)) { + throw "packaged worker plan bytes mismatch eligible-accept fixture" + } + if (-not $proc.WaitForExit(15000)) { + $proc.Kill() + throw "packaged worker did not exit after stdin EOF" + } + if ($proc.ExitCode -ne 0) { + throw "packaged worker exit code $($proc.ExitCode)" + } + Write-Host "packaged worker parent/child framed product loop green" + + # Real localhost HTTP product loop against packaged HTTP dogfood binary. + $httpExeName = if ($suffix -eq ".exe") { "senline_http_dogfood.exe" } else { "senline_http_dogfood" } + $httpExe = Join-Path $pkgRoot "http-a/$httpExeName" + if (-not (Test-Path -LiteralPath $httpExe)) { throw "missing packaged http dogfood: $httpExe" } + $httpPsi = New-Object System.Diagnostics.ProcessStartInfo + $httpPsi.FileName = $httpExe + $httpPsi.WorkingDirectory = (Join-Path $pkgRoot "http-a") + $httpPsi.RedirectStandardOutput = $true + $httpPsi.RedirectStandardError = $true + $httpPsi.UseShellExecute = $false + $httpProc = [Diagnostics.Process]::Start($httpPsi) + # Async ReadLine so a silent hang cannot block the step forever. + $readyTask = $httpProc.StandardOutput.ReadLineAsync() + if (-not $readyTask.Wait(20000)) { + if (-not $httpProc.HasExited) { $httpProc.Kill() } + throw "http dogfood READY timed out" + } + $readyLine = $readyTask.Result + if (-not $readyLine -or $readyLine -notmatch '^READY 127\.0\.0\.1:(\d+)$') { + if (-not $httpProc.HasExited) { $httpProc.Kill() } + throw "http dogfood READY line missing/invalid: '$readyLine'" + } + $port = [int]$Matches[1] + # Harness requires Content-Type exactly "application/json" (no charset). + # Prefer curl for cross-platform exact headers; fall back to WebRequest. + $bodyFile = Join-Path $pkgRoot "http-probe-body.json" + Copy-Item -LiteralPath (Resolve-Path $requestPath) -Destination $bodyFile -Force + $respBodyFile = Join-Path $pkgRoot "http-probe-resp.json" + $statusCode = 0 + $respText = "" + $curlExe = $null + foreach ($candidate in @("curl.exe", "curl")) { + $cmd = Get-Command $candidate -ErrorAction SilentlyContinue + if ($cmd -and $cmd.Source -notmatch 'Alias') { $curlExe = $cmd.Source; break } + } + if ($curlExe) { + $curlOut = & $curlExe -sS -o $respBodyFile -w "%{http_code}" ` + -X POST "http://127.0.0.1:$port/v1/submit-envelope" ` + -H "Content-Type: application/json" ` + --data-binary "@$bodyFile" + $statusCode = [int]$curlOut + if (Test-Path -LiteralPath $respBodyFile) { + $respText = [IO.File]::ReadAllText($respBodyFile) + } + } else { + try { + $headers = @{ "Content-Type" = "application/json" } + $bodyBytes = [IO.File]::ReadAllBytes($bodyFile) + $resp = Invoke-WebRequest -Uri "http://127.0.0.1:$port/v1/submit-envelope" ` + -Method POST -Headers $headers -Body $bodyBytes -TimeoutSec 15 -UseBasicParsing + $statusCode = [int]$resp.StatusCode + $respText = $resp.Content + } catch { + if (-not $httpProc.HasExited) { $httpProc.Kill() } + throw "localhost HTTP product POST failed: $($_.Exception.Message)" + } + } + if ($statusCode -ne 200) { + if (-not $httpProc.HasExited) { $httpProc.Kill() } + throw "http dogfood status $statusCode body=$($respText.Substring(0, [Math]::Min(200, $respText.Length)))" + } + $respJson = $respText | ConvertFrom-Json + if ($respJson.kind -ne "plan") { + if (-not $httpProc.HasExited) { $httpProc.Kill() } + throw "http dogfood did not return plan kind: $($respText.Substring(0, [Math]::Min(200, $respText.Length)))" + } + # Worker/HTTP plan byte equivalence (LF-normalized): HTTP body must match + # the framed worker plan for the same eligible-accept fixture. + $httpPlanBytes = [Text.Encoding]::UTF8.GetBytes($respText) + if ((Bytes-Key $httpPlanBytes) -ne (Bytes-Key $gotPlan)) { + if (-not $httpProc.HasExited) { $httpProc.Kill() } + throw "worker/HTTP plan bytes diverge for eligible-accept" + } + if (-not $httpProc.WaitForExit(15000)) { + $httpProc.Kill() + throw "http dogfood did not exit after one request" + } + if ($httpProc.ExitCode -ne 0) { + throw "http dogfood exit code $($httpProc.ExitCode) after successful plan response" + } + # Malformed body must not hang the product loop path (second package probe). + $httpPsi2 = New-Object System.Diagnostics.ProcessStartInfo + $httpPsi2.FileName = $httpExe + $httpPsi2.WorkingDirectory = (Join-Path $pkgRoot "http-a") + $httpPsi2.RedirectStandardOutput = $true + $httpPsi2.RedirectStandardError = $true + $httpPsi2.UseShellExecute = $false + $httpProc2 = [Diagnostics.Process]::Start($httpPsi2) + $readyTask2 = $httpProc2.StandardOutput.ReadLineAsync() + if (-not $readyTask2.Wait(20000)) { + if (-not $httpProc2.HasExited) { $httpProc2.Kill() } + throw "http dogfood READY timed out on malformed probe" + } + $ready2 = $readyTask2.Result + if (-not $ready2 -or $ready2 -notmatch '^READY 127\.0\.0\.1:(\d+)$') { + if (-not $httpProc2.HasExited) { $httpProc2.Kill() } + throw "http dogfood READY invalid on malformed probe: '$ready2'" + } + $port2 = [int]$Matches[1] + # Protocol contract: strict JSON errors return HTTP 200 + normalized + # protocol error envelope (malformed_json). Transport policy failures + # (method/path/header) are 400 — not this probe. + $badBody = Join-Path $pkgRoot "http-probe-bad.json" + [IO.File]::WriteAllText($badBody, "{not-json") + $badResp = Join-Path $pkgRoot "http-probe-bad-resp.txt" + if ($curlExe) { + $badCode = & $curlExe -sS -o $badResp -w "%{http_code}" ` + -X POST "http://127.0.0.1:$port2/v1/submit-envelope" ` + -H "Content-Type: application/json" ` + --data-binary "@$badBody" + if ([int]$badCode -ne 200) { + if (-not $httpProc2.HasExited) { $httpProc2.Kill() } + throw "malformed JSON must return HTTP 200 protocol error, got $badCode" + } + $badText = [IO.File]::ReadAllText($badResp) + $expectedMalformed = '{"kind":"error","schema_version":1,"scope":"protocol","code":"malformed_json","evaluation_id":null}' + $badNorm = $badText.TrimEnd("`r", "`n") + if ($badNorm -ne $expectedMalformed) { + if (-not $httpProc2.HasExited) { $httpProc2.Kill() } + throw "malformed JSON envelope mismatch: '$badText'" + } + } + if (-not $httpProc2.WaitForExit(15000)) { + $httpProc2.Kill() + throw "http dogfood did not exit after malformed probe" + } + # Transport policy: wrong method is HTTP 400 (not a protocol-error envelope). + $httpPsi3 = New-Object System.Diagnostics.ProcessStartInfo + $httpPsi3.FileName = $httpExe + $httpPsi3.WorkingDirectory = (Join-Path $pkgRoot "http-a") + $httpPsi3.RedirectStandardOutput = $true + $httpPsi3.RedirectStandardError = $true + $httpPsi3.UseShellExecute = $false + $httpProc3 = [Diagnostics.Process]::Start($httpPsi3) + $readyTask3 = $httpProc3.StandardOutput.ReadLineAsync() + if (-not $readyTask3.Wait(20000)) { + if (-not $httpProc3.HasExited) { $httpProc3.Kill() } + throw "http dogfood READY timed out on method probe" + } + $ready3 = $readyTask3.Result + if (-not $ready3 -or $ready3 -notmatch '^READY 127\.0\.0\.1:(\d+)$') { + if (-not $httpProc3.HasExited) { $httpProc3.Kill() } + throw "http dogfood READY invalid on method probe: '$ready3'" + } + $port3 = [int]$Matches[1] + if ($curlExe) { + $methodCode = & $curlExe -sS -o (Join-Path $pkgRoot "http-probe-method-resp.txt") -w "%{http_code}" ` + -X GET "http://127.0.0.1:$port3/v1/submit-envelope" + if ([int]$methodCode -ne 400) { + if (-not $httpProc3.HasExited) { $httpProc3.Kill() } + throw "wrong HTTP method must return 400 transport error, got $methodCode" + } + } + if (-not $httpProc3.WaitForExit(15000)) { + $httpProc3.Kill() + throw "http dogfood did not exit after method probe" + } + Write-Host "packaged HTTP localhost product loop green (port=$port, worker/HTTP plan equal, malformed_json@200, GET@400)" + + if ($compareFailures.Count -gt 0) { + throw ("pin-grade dual-package compare failed (task 8.7 open until bit-identical): " + ($compareFailures -join "; ")) + } + Write-Host "senline installed worker/HTTP dual package + product loops green on ${{ matrix.os }}" + + - name: Upload package compare evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: senline-installed-packages-${{ matrix.artifact }} + path: | + ${{ runner.temp }}/senline-pkg-${{ matrix.artifact }}/**/comparison.json + ${{ runner.temp }}/senline-pkg-${{ matrix.artifact }}/**/*-manifest.json + ${{ runner.temp }}/senline-pkg-${{ matrix.artifact }}/worker-a/** + ${{ runner.temp }}/senline-pkg-${{ matrix.artifact }}/http-a/** + if-no-files-found: warn diff --git a/.github/workflows/senline-installed-packages.yml b/.github/workflows/senline-installed-packages.yml new file mode 100644 index 00000000..66708fca --- /dev/null +++ b/.github/workflows/senline-installed-packages.yml @@ -0,0 +1,204 @@ +name: senline-installed-packages + +on: + workflow_dispatch: + pull_request: + paths: + - ".github/workflows/senline-installed-packages.yml" + - "scripts/package-toolchain.ps1" + - "scripts/package-senline-worker.ps1" + - "scripts/package-senline-http.ps1" + - "scripts/compare-senline-package-manifests.ps1" + - "scripts/install.ps1" + - "scripts/install.sh" + - "examples/realworld/senline-domain-worker/**" + - "examples/realworld/senline-http-dogfood/**" + +jobs: + installed-worker-http: + name: installed worker/HTTP (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + target: x86_64-unknown-linux-gnu + artifact: linux-x86_64 + - os: windows-latest + target: x86_64-pc-windows-msvc + artifact: windows-x86_64 + env: + LLVM_VERSION: "19" + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@master + with: + toolchain: 1.94.0 + + - name: Install pinned LLVM toolchain (Ubuntu) + if: runner.os == 'Linux' + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y clang-${LLVM_VERSION} lld-${LLVM_VERSION} + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${LLVM_VERSION} 100 + + - name: Require clang + shell: pwsh + run: | + $clang = Get-Command clang -ErrorAction Stop + & $clang.Source --version + if ($LASTEXITCODE -ne 0) { + throw "clang probe failed with exit code $LASTEXITCODE" + } + + - name: Build release tools + shell: pwsh + run: cargo build -p sgc -p sgpm -p sgfmt -p sglsp --release + + - name: Package installed toolchain + shell: pwsh + env: + SENGOO_DIST_TARGET: ${{ matrix.target }} + run: | + $version = "0.1.0-ci" + $cargoTarget = Join-Path $env:RUNNER_TEMP "sengoo-cargo-package-${{ matrix.artifact }}" + Remove-Item -LiteralPath "target/dist" -Recurse -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $cargoTarget -Recurse -Force -ErrorAction SilentlyContinue + Remove-Item Env:CARGO_ENCODED_RUSTFLAGS -ErrorAction SilentlyContinue + Remove-Item Env:RUSTFLAGS -ErrorAction SilentlyContinue + ./scripts/package-toolchain.ps1 ` + -Version $version ` + -OutputDir "target/dist" ` + -CargoTargetDir $cargoTarget ` + -SmokeEvidence "senline-installed-packages ${{ matrix.os }} ${{ github.sha }}" + + - name: Install toolchain outside product workspace + shell: pwsh + run: | + $version = "0.1.0-ci" + $archiveDir = "target/dist" + $latest = Get-Content (Join-Path $archiveDir "latest-archive.txt") -ErrorAction SilentlyContinue + if (-not $latest) { + $zip = Get-ChildItem $archiveDir -Filter "*.zip" | Select-Object -First 1 + if (-not $zip) { throw "missing packaged archive under $archiveDir" } + $latest = $zip.FullName + } else { + if (-not [IO.Path]::IsPathRooted($latest)) { + $latest = Join-Path $archiveDir $latest + } + } + $installDir = Join-Path $env:RUNNER_TEMP "sengoo-install-${{ matrix.artifact }}" + Remove-Item -LiteralPath $installDir -Recurse -Force -ErrorAction SilentlyContinue + if ($IsWindows -or $PSVersionTable.PSEdition -eq "Desktop") { + ./scripts/install.ps1 -Archive $latest -InstallDir $installDir + } else { + sh scripts/install.sh "$latest" "$installDir" + } + Write-Host "install_dir=$installDir" + + - name: Fake cargo + installed worker/HTTP dual package loop + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + $version = "0.1.0-ci" + $installDir = Join-Path $env:RUNNER_TEMP "sengoo-install-${{ matrix.artifact }}" + $bin = Join-Path $installDir "bin" + $suffix = "" + if ($IsWindows -or $PSVersionTable.PSEdition -eq "Desktop") { + $suffix = ".exe" + } + $sgc = Join-Path $bin "sgc$suffix" + $sgpm = Join-Path $bin "sgpm$suffix" + if (-not (Test-Path -LiteralPath $sgc)) { throw "missing installed sgc: $sgc" } + if (-not (Test-Path -LiteralPath $sgpm)) { throw "missing installed sgpm: $sgpm" } + + # Fail if any real cargo would be required by the package loop. + $fakeCargoDir = Join-Path $env:RUNNER_TEMP "fake-cargo-fail" + Remove-Item -LiteralPath $fakeCargoDir -Recurse -Force -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force -Path $fakeCargoDir | Out-Null + $fakeCargo = Join-Path $fakeCargoDir "cargo$suffix" + if ($suffix -eq ".exe") { + @' + @echo off + echo fake-cargo: cargo must not be invoked by installed senline package loops 1>&2 + exit /b 97 + '@ | Set-Content -LiteralPath $fakeCargo -Encoding ASCII + } else { + @' + #!/bin/sh + echo "fake-cargo: cargo must not be invoked by installed senline package loops" >&2 + exit 97 + '@ | Set-Content -LiteralPath $fakeCargo -Encoding UTF8 + & chmod +x $fakeCargo + } + $pathSep = [IO.Path]::PathSeparator + $env:PATH = "$fakeCargoDir$pathSep$bin$pathSep" + $env:PATH + $env:SGPM_SGC = $sgc + $env:SGPM_SGFMT = Join-Path $bin "sgfmt$suffix" + Remove-Item Env:SENGOO_ROOT -ErrorAction SilentlyContinue + Remove-Item Env:SENGOO_STDLIB -ErrorAction SilentlyContinue + Remove-Item Env:SENGOO_RUNTIME -ErrorAction SilentlyContinue + + Write-Host "sgc: $(& $sgc --version) path=$sgc" + Write-Host "sgpm: $(& $sgpm --version) path=$sgpm" + Write-Host "SGPM_SGC=$env:SGPM_SGC" + + Push-Location examples/realworld/senline-domain-worker + try { + & $sgpm --runtime-mode installed check --locked + if ($LASTEXITCODE -ne 0) { throw "worker sgpm installed check --locked failed" } + & $sgpm --runtime-mode installed test --locked + if ($LASTEXITCODE -ne 0) { throw "worker sgpm installed test --locked failed" } + & $sgpm --runtime-mode installed build --locked --release + if ($LASTEXITCODE -ne 0) { throw "worker sgpm installed build --locked --release failed" } + } finally { + Pop-Location + } + + Push-Location examples/realworld/senline-http-dogfood + try { + & $sgpm --runtime-mode installed check --locked + if ($LASTEXITCODE -ne 0) { throw "http sgpm installed check --locked failed" } + & $sgpm --runtime-mode installed test --locked + if ($LASTEXITCODE -ne 0) { throw "http sgpm installed test --locked failed" } + & $sgpm --runtime-mode installed build --locked --release + if ($LASTEXITCODE -ne 0) { throw "http sgpm installed build --locked --release failed" } + } finally { + Pop-Location + } + + $pkgRoot = Join-Path $env:RUNNER_TEMP "senline-pkg-${{ matrix.artifact }}" + Remove-Item -LiteralPath $pkgRoot -Recurse -Force -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force -Path $pkgRoot | Out-Null + + ./scripts/package-senline-worker.ps1 -SgcPath $sgc -SgpmPath $sgpm -OutputDir (Join-Path $pkgRoot "worker-a") + ./scripts/package-senline-worker.ps1 -SgcPath $sgc -SgpmPath $sgpm -OutputDir (Join-Path $pkgRoot "worker-b") + ./scripts/compare-senline-package-manifests.ps1 ` + -LeftManifest (Join-Path $pkgRoot "worker-a/worker-manifest.json") ` + -RightManifest (Join-Path $pkgRoot "worker-b/worker-manifest.json") ` + -OutputDir (Join-Path $pkgRoot "worker-compare") ` + -AllowExecutableHashDrift + + ./scripts/package-senline-http.ps1 -SgcPath $sgc -SgpmPath $sgpm -OutputDir (Join-Path $pkgRoot "http-a") + ./scripts/package-senline-http.ps1 -SgcPath $sgc -SgpmPath $sgpm -OutputDir (Join-Path $pkgRoot "http-b") + ./scripts/compare-senline-package-manifests.ps1 ` + -LeftManifest (Join-Path $pkgRoot "http-a/http-manifest.json") ` + -RightManifest (Join-Path $pkgRoot "http-b/http-manifest.json") ` + -OutputDir (Join-Path $pkgRoot "http-compare") ` + -AllowExecutableHashDrift + + Write-Host "senline installed worker/HTTP dual package loop green on ${{ matrix.os }}" + + - name: Upload package compare evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: senline-installed-packages-${{ matrix.artifact }} + path: | + ${{ runner.temp }}/senline-pkg-${{ matrix.artifact }}/**/comparison.json + ${{ runner.temp }}/senline-pkg-${{ matrix.artifact }}/**/*-manifest.json + if-no-files-found: warn diff --git a/Cargo.lock b/Cargo.lock index 2196c7c4..aafd6e31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2091,6 +2091,7 @@ dependencies = [ "sengoo-runtime", "serde", "serde_json", + "sha2", "thiserror", "tokio", "toml", diff --git a/README-dist.md b/README-dist.md index 195a1a89..e8dd58d5 100644 --- a/README-dist.md +++ b/README-dist.md @@ -7,14 +7,107 @@ This archive contains the Sengoo command-line tools: - `bin/sgfmt` - `bin/sglsp` -It also includes the standard library and C runtime bridge under -`share/sengoo/`, so `sgc` can resolve `std::*` imports without a source -checkout or `SENGOO_ROOT`. +It also includes the standard library, C runtime bridge, and target-native +runtime under `share/sengoo/`, so `sgc` can resolve `std::*` imports and link +native programs without a source checkout, `SENGOO_ROOT`, or Cargo. + +## Installed Runtime Layout + +The archive root uses manifest schema 2: + +```text +manifest.json +payloads.sha256 +bin/sgc[.exe] +share/sengoo/stdlib/runtime.c +share/sengoo/stdlib/runtime_*.c +share/sengoo/stdlib/runtime_shared.h +share/sengoo/runtime//sengoo_runtime.lib # Windows +share/sengoo/runtime//libsengoo_runtime.a # Unix +``` + +`manifest.json.native_runtime` binds runtime ABI 1 to the exact target, +relative library path, SHA-256, ordered platform link arguments, and declared +dynamic dependencies. `build_manifest_id` is the SHA-256 of the normalized +`payloads.sha256` file. Installers verify the archive checksum, every listed +payload, and that no unlisted payload file is present before copying files. + +`source_revision`, `source_dirty`, `artifact_provenance`, and +`release_eligible` distinguish clean packager-built archives from local +`-NoBuild` development bundles. A dirty or `-NoBuild` bundle is useful for +local smoke testing but is not release or Senline pin evidence. + +Installed native build/run validates the manifest and runtime before cache +reuse. Missing, relocated-without-manifest, wrong-target, wrong-ABI, tampered, +or incomplete installations fail without consulting Cargo or a compiled-in +source checkout. + +## Reproducible Distribution Gate + +Windows x64 and Linux x64 CI package the same clean Git revision twice in +sequence. Build A and build B use separate empty Cargo target directories. +Each target directory is remapped to the common virtual +`/sengoo-build/target` prefix so runner-local build paths cannot affect shipped +payloads. Both manifests must report `source_dirty=false` and +`release_eligible=true`, and both checksum-verifying installers must accept the +resulting archives. + +`scripts/compare-distribution-manifests.ps1` validates the complete schema 2 +shape before comparing it. Normalization is deliberately narrow: + +- `tools`, `stdlib_modules`, `runtime_sources`, and + `native_runtime.dynamic_dependencies` are compared as sorted unique sets. +- `tool_versions` is ordered by its exact tool keys. +- `payloads` is ordered by normalized relative path; paths must also be unique + under case-insensitive comparison. +- `native_runtime.link_args` retains order because linker argument order is + part of the installed runtime contract. + +Only `generated_at_utc`, `runner_os`, `runner_image`, and the run-specific +`smoke_evidence` provenance note are excluded. GitHub provenance attestation +signatures are external to manifest schema 2 and may differ without weakening +the payload comparison. `artifact_provenance`, release eligibility, source +revision and dirtiness, build identity, tool versions, payload sizes and +hashes, runtime ABI/library/hash, ordered link arguments, dynamic dependency +identities, archive/checksum names, and license presence must match exactly. +Unknown or missing fields fail validation rather than disappearing during +normalization. + +The gate retains `normalized-a.json`, `normalized-b.json`, and +`comparison.json`, including both normalized SHA-256 values and any excluded +provenance differences. A mismatch blocks the target before publication. + +## Source Runtime Development Mode + +`sgc` defaults to `--runtime-mode installed`. Contributors who intentionally +build the native Rust runtime through Cargo must use a compiler executable +inside the Sengoo workspace and opt in explicitly: + +```sh +target/debug/sgc --runtime-mode source-development build path/to/main.sg +``` + +Locked package loops use the same explicit policy at the `sgpm` boundary; the +selected mode is forwarded to every delegated `sgc` command: + +```sh +target/debug/sgpm --runtime-mode source-development check --locked +target/debug/sgpm --runtime-mode source-development test --locked +``` + +This mode emits the stable `toolchain::source_runtime_development` diagnostic +and records `artifact_provenance=source-cargo-development`, +`release_eligible=false`, and `senline_pin_evidence=false` in build/run cache +metadata. It cannot start or dispatch through the compiler daemon. Moving the +compiler outside its source workspace does not carry this authority with it. +Artifacts produced in this mode are development outputs only and are never +installed-distribution, publication, or Senline pin evidence. ## Requirements -Native builds require LLVM/Clang 15 or newer on `PATH`. `sgc build ---emit-llvm` and stdlib import expansion work without native linking. +Native builds require LLVM/Clang 15 or newer on `PATH`. Windows builds also +require the supported MSVC linker/SDK. `sgc build --emit-llvm` and stdlib +import expansion work without native linking. ## Quickstart diff --git a/compiler/src/codegen/instruction_helpers.rs b/compiler/src/codegen/instruction_helpers.rs index 44bbe366..eeda3c2d 100644 --- a/compiler/src/codegen/instruction_helpers.rs +++ b/compiler/src/codegen/instruction_helpers.rs @@ -699,11 +699,27 @@ impl Codegen { } => { let dest = self.local_name(*destination); - let src = self.local_name(*source); let dest_ty = self.get_local_type(mir_fn, *destination).clone(); let source_ty = self.get_local_type(mir_fn, *source).clone(); - let source_ptr_ty = format!("{}*", self.mir_type_to_llvm_cached(&source_ty)); + let source_llvm = self.mir_type_to_llvm_cached(&source_ty); + let source_ptr_ty = format!("{}*", source_llvm); let dest_llvm = self.mir_type_to_llvm_cached(&dest_ty); + let src = if self.local_uses_stack_slot(*source, mir_fn) { + self.local_name(*source) + } else { + // Parameters and temporaries are SSA values; spill before taking their address. + let source_value = self.operand_value(*source, mir_fn); + let slot = format!("{}.slot", dest); + self.emit_indent(); + self.ir + .push_str(&format!("{} = alloca {}\n", slot, source_llvm)); + self.emit_indent(); + self.ir.push_str(&format!( + "store {} {}, {} {}\n", + source_llvm, source_value, source_ptr_ty, slot + )); + slot + }; self.emit_indent(); diff --git a/compiler/src/mir/lowering/drop_glue_helpers.rs b/compiler/src/mir/lowering/drop_glue_helpers.rs index 220d2c7a..5d54a8bd 100644 --- a/compiler/src/mir/lowering/drop_glue_helpers.rs +++ b/compiler/src/mir/lowering/drop_glue_helpers.rs @@ -92,6 +92,15 @@ impl<'a> LoweringContext<'a> { } pub(super) fn record_drop_binding_if_needed(&mut self, local: Local) { + // Legacy handle params are skipped here: by-value `self`/`String` args + // are still lowered as handle *copies* for many stdlib methods (`len`, + // `as_str`, …). Auto-Drop on those params would free the live object + // while the caller's local still holds the same handle. + // + // True ownership transfer into lambdas uses + // [`Self::force_record_owned_param_drop`] instead. Free-standing workers + // should take `&str`/`&String` (or move into an owning helper that only + // borrows fields) rather than relying on this skip being removed. if local.kind == LocalKind::Param && Self::is_legacy_idempotent_handle_mir_type(self.get_local_type(local)) { @@ -338,6 +347,38 @@ impl<'a> LoweringContext<'a> { } } + /// Register drop glue for a by-value parameter even when the type is a + /// legacy idempotent handle (`String`/`Buffer`/`JsonDoc`). + /// + /// Ordinary function params skip those types because method receivers are + /// still lowered as handle copies (see `record_drop_binding_if_needed`). + /// Lambdas that take `String` by value (e.g. field-allowlist callbacks) + /// must free the owned handle on every return path. + pub(super) fn force_record_owned_param_drop(&mut self, local: Local) { + if local.kind != LocalKind::Param { + self.record_drop_binding_if_needed(local); + return; + } + let mut bindings = Vec::new(); + if let Some(drop_func) = self.drop_func_for_local(local) { + bindings.push(DropBinding { + local, + field_path: Vec::new(), + drop_func, + }); + } else { + let ty = self.get_local_type(local).clone(); + self.collect_field_drop_bindings(local, &ty, &mut Vec::new(), &mut bindings); + } + for binding in bindings { + if !self.drop_bindings.iter().any(|existing| { + existing.local == binding.local && existing.field_path == binding.field_path + }) { + self.drop_bindings.push(binding); + } + } + } + fn drop_func_for_local(&mut self, local: Local) -> Option { let local_ty = self.get_local_type(local).clone(); if Self::option_payload_type(&local_ty).is_some() { diff --git a/compiler/src/mir/lowering/lambda_expr_helpers.rs b/compiler/src/mir/lowering/lambda_expr_helpers.rs index c0446a3a..b86b9fde 100644 --- a/compiler/src/mir/lowering/lambda_expr_helpers.rs +++ b/compiler/src/mir/lowering/lambda_expr_helpers.rs @@ -83,12 +83,23 @@ pub(super) fn lower_lambda_expr_with_expected( for (i, param_name) in params.iter().enumerate() { let local = Local::new(i + 1 + env_param_offset, LocalKind::Param); + lambda_ctx + .mir_fn + .set_local_debug_name(local, param_name.clone()); lambda_ctx.local_names.insert(param_name.clone(), local); + // Owned by-value lambda params (e.g. `String`) must be dropped when + // the lambda returns; otherwise callers that move into the lambda leak + // one handle per invocation (worker field-allowlist loops hit this). + lambda_ctx.force_record_owned_param_drop(local); } } else { for (i, param_name) in params.iter().enumerate() { let local = Local::new(i + 1 + env_param_offset, LocalKind::Param); + lambda_ctx + .mir_fn + .set_local_debug_name(local, param_name.clone()); lambda_ctx.local_names.insert(param_name.clone(), local); + lambda_ctx.force_record_owned_param_drop(local); } } @@ -98,6 +109,7 @@ pub(super) fn lower_lambda_expr_with_expected( expr: Some(Box::new(body.clone())), }; lambda_ctx.lower_body_to_block(&lambda_body, lambda_start); + lambda_ctx.insert_drop_glue(); ctx.lambda_functions.push(lambda_fn); diff --git a/compiler/src/tests/drop_flag_tests.rs b/compiler/src/tests/drop_flag_tests.rs index dd1632e4..0958499b 100644 --- a/compiler/src/tests/drop_flag_tests.rs +++ b/compiler/src/tests/drop_flag_tests.rs @@ -1272,6 +1272,103 @@ def main() -> i64 { ); } +#[test] +fn lambda_by_value_string_param_is_dropped_on_return() { + // Regression for senline worker long-session growth: field-allowlist + // callbacks take `String` by value and previously never ran Drop glue. + let mir = compile_with_owned_string( + r#" +def main() -> i64 { + let allowed: fn(String) -> bool = |key| true; + let owned = string_from_str("contract_version").value; + if allowed(owned) { 1 } else { 0 } +} +"#, + ); + let lambda = mir + .iter() + .find(|f| f.name.contains("lambda") || f.name.starts_with("$__")) + .or_else(|| { + mir.iter() + .find(|f| f.name != "main" && f.name != "string_from_str") + }) + .expect("expected a lowered lambda function"); + assert!( + !string_drop_calls(lambda).is_empty() + || !named_drop_calls(lambda, "String_Drop_drop").is_empty(), + "lambda taking String by value must drop its parameter; lambda IR name={}", + lambda.name + ); +} + +#[test] +fn if_else_both_branches_move_owned_aggregate_no_parent_drop() { + // When both branches move an owned aggregate into callees, the parent must + // not Drop it (would double-free). Each callee owns the value. + let mir = compile_with_owned_string( + r#" +struct Bundle { + label: String, +} + +def take_a(b: Bundle) -> i64 { 1 } +def take_b(b: Bundle) -> i64 { 2 } + +def route(flag: bool, b: Bundle) -> i64 { + if flag { take_a(b) } else { take_b(b) } +} + +def main() -> i64 { + let owned = string_from_str("x").value; + route(true, Bundle { label: owned }) +} +"#, + ); + let route = mir + .iter() + .find(|f| f.name == "route" || f.name.ends_with("route")) + .expect("expected route function"); + // Parent route should not Drop Bundle.label; ownership transferred. + assert!( + string_drop_calls(route).is_empty() + && named_drop_calls(route, "String_Drop_drop").is_empty(), + "parent must not drop aggregate moved on both branches:\n{:?}", + route + ); +} + +#[test] +fn reject_helper_drops_nested_string_fields_of_owned_aggregate_param() { + // Worker unsupported-version path moves WorkerRequest-like aggregates into a + // helper that only borrows fields; the helper must still Drop nested Strings. + let mir = compile_with_owned_string( + r#" +struct Bundle { + label: String, +} + +def reject(b: Bundle) -> i64 { + b.label.len() +} + +def main() -> i64 { + let owned = string_from_str("fixture").value; + reject(Bundle { label: owned }) +} +"#, + ); + let reject = mir + .iter() + .find(|f| f.name == "reject" || f.name.ends_with("reject")) + .expect("expected reject helper"); + assert!( + !string_drop_calls(reject).is_empty() + || !named_drop_calls(reject, "String_Drop_drop").is_empty(), + "owning helper must drop nested String fields of aggregate params; ir={}", + reject.name + ); +} + #[test] fn concrete_generic_method_return_moves_owned_aggregate_without_early_drop() { let mir = compile_to_mir( diff --git a/compiler/src/tests/owned_string_tests.rs b/compiler/src/tests/owned_string_tests.rs index c7b2c1c2..3786e52b 100644 --- a/compiler/src/tests/owned_string_tests.rs +++ b/compiler/src/tests/owned_string_tests.rs @@ -485,6 +485,93 @@ def main() -> i64 { ); } +#[test] +fn early_return_field_move_does_not_poison_the_fallthrough_path() { + let source = r#" +struct Token { + value: i64, +} + +impl Drop for Token { + def drop(&mut self) { + } +} + +struct Request { + id: Token, + payload: Token, +} + +def consume_id(id: Token) -> i64 { + id.value +} + +def consume_request(request: Request) -> i64 { + request.payload.value +} + +def route(request: Request, unsupported: bool) -> i64 { + if unsupported { + return consume_id(request.id); + } + consume_request(request) +} +"#; + let program = Parser::parse(source).expect("source should parse"); + let mut checker = TypeChecker::new(); + checker + .check_program(&program) + .expect("a terminating branch move must not reach the fallthrough path"); +} + +#[test] +fn non_terminating_branch_field_move_still_poisons_the_fallthrough_path() { + let source = r#" +struct Token { + value: i64, +} + +impl Drop for Token { + def drop(&mut self) { + } +} + +struct Request { + id: Token, + payload: Token, +} + +def consume_id(id: Token) -> i64 { + id.value +} + +def consume_request(request: Request) -> i64 { + request.payload.value +} + +def route(request: Request, unsupported: bool) -> i64 { + if unsupported { + let consumed = consume_id(request.id); + } + consume_request(request) +} +"#; + let program = Parser::parse(source).expect("source should parse"); + let mut checker = TypeChecker::new(); + let err = checker + .check_program(&program) + .expect_err("a non-terminating branch move must reach the fallthrough path"); + let crate::error::CompileError::TypeckError(typeck) = err else { + panic!("expected TypeckError, got {err:?}"); + }; + assert!( + typeck + .to_string() + .contains("use of partially moved value `request`"), + "unexpected error: {typeck}" + ); +} + #[test] fn stdlib_owned_string_return_marks_value_moved() { let err = typecheck_fails_with_stdlib( diff --git a/compiler/src/tests/stdlib_surface_tests.rs b/compiler/src/tests/stdlib_surface_tests.rs index 14267547..176a185d 100644 --- a/compiler/src/tests/stdlib_surface_tests.rs +++ b/compiler/src/tests/stdlib_surface_tests.rs @@ -1294,6 +1294,47 @@ def main() -> i64 { assert!(ir.contains("sengoo_io_stderr_flush")); } +#[test] +fn io_module_preserves_binary_buffer_and_pipe_signatures() { + let ir = compile_with_stdlib_modules( + &["option.sg", "result.sg", "ffi.sg", "status.sg", "io.sg"], + r#" +def main() -> i64 { + let input = ffi_buffer_new(4).unwrap_or(Buffer { handle: 0 }); + let output = ffi_buffer_new(4).unwrap_or(Buffer { handle: 0 }); + let initialized = output.set_u8(0, 0).unwrap_or(false) + && output.write_u32_be(0, 16909060).unwrap_or(false); + let first = output.get_u8(0).unwrap_or(-1); + let word = output.read_u32_be(0).unwrap_or(-1); + let binary = io_protocol_binary_mode().unwrap_or(false); + let read = io_stdin_read_exact(input, 0, 4).unwrap_or(0); + let wrote = io_stdout_write_all(output, 0, 4).unwrap_or(0); + + if initialized && first == 1 && word == 16909060 && binary && read == 4 && wrote == 4 { + 0 + } else { + 1 + } +} +"#, + ); + + for signature in [ + "declare i64 @sengoo_ffi_buffer_get_u8(i64, i64)", + "declare i64 @sengoo_ffi_buffer_set_u8(i64, i64, i64)", + "declare i64 @sengoo_ffi_buffer_read_u32_be(i64, i64)", + "declare i64 @sengoo_ffi_buffer_write_u32_be(i64, i64, i64)", + "declare i64 @sengoo_io_protocol_binary_mode()", + "declare i64 @sengoo_io_stdin_read_exact(i64, i64, i64)", + "declare i64 @sengoo_io_stdout_write_all(i64, i64, i64)", + ] { + assert!( + ir.contains(signature), + "missing binary-I/O ABI signature `{signature}`\n{ir}" + ); + } +} + #[test] fn args_module_imports_argument_helpers_and_emits_opt_in_entry_wrapper() { let ir = compile_with_stdlib_modules( diff --git a/compiler/src/tests/struct_codegen_tests.rs b/compiler/src/tests/struct_codegen_tests.rs index c2dd3818..b8cfcd30 100644 --- a/compiler/src/tests/struct_codegen_tests.rs +++ b/compiler/src/tests/struct_codegen_tests.rs @@ -7,6 +7,37 @@ //! _Requirements: 4.1, 4.2, 4.3, 4.5_ use crate::compile_to_ir; +use std::io::Write; +use std::process::{Command, Stdio}; + +fn assert_clang_accepts_ir(ir: &str) { + let output_path = if cfg!(windows) { "NUL" } else { "/dev/null" }; + let mut child = ["clang", "clang.exe"] + .iter() + .find_map(|candidate| { + Command::new(candidate) + .args(["-x", "ir", "-c", "-o", output_path, "-"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .ok() + }) + .expect("clang is required to validate generated LLVM IR"); + child + .stdin + .take() + .expect("clang stdin should be piped") + .write_all(ir.as_bytes()) + .expect("generated LLVM IR should be writable to clang"); + let output = child.wait_with_output().expect("clang should finish"); + assert!( + output.status.success(), + "clang rejected generated LLVM IR:\n{}\nIR:\n{}", + String::from_utf8_lossy(&output.stderr), + ir + ); +} /// Test that `struct Point { x: i64, y: i64 }` with construction and field access /// generates valid LLVM IR containing `insertvalue` and `extractvalue`. @@ -41,6 +72,66 @@ def main() -> i64 { ); } +#[test] +fn local_value_reference_generates_valid_ir() { + let source = r#" +struct Token { value: i64 } + +def inspect(value: &Token) -> i64 { 0 } + +def main() -> i64 { + let token = Token { value: 7 }; + inspect(&token) +} +"#; + let ir = compile_to_ir(source).expect("local value reference should lower to LLVM IR"); + assert_clang_accepts_ir(&ir); +} + +#[test] +fn local_struct_field_reference_generates_valid_ir() { + let source = r#" +struct Token { value: i64 } +struct Container { token: Token } + +def inspect(value: &Token) -> i64 { 0 } + +def main() -> i64 { + let container = Container { token: Token { value: 7 } }; + inspect(&container.token) +} +"#; + let ir = compile_to_ir(source).expect("struct field reference should lower to LLVM IR"); + assert_clang_accepts_ir(&ir); +} + +#[test] +fn nested_owned_string_parameter_field_reference_generates_valid_ir() { + let source = r#" +struct String { handle: i64 } +struct Identifiers { correlation_ref: String } +struct Plan { identifiers: Identifiers } + +def inspect(value: &String) -> i64 { 0 } + +def encode(plan: Plan) -> i64 { + inspect(&plan.identifiers.correlation_ref) +} + +def main() -> i64 { + let plan = Plan { + identifiers: Identifiers { + correlation_ref: String { handle: 7 }, + }, + }; + encode(plan) +} +"#; + let ir = compile_to_ir(source) + .expect("nested owned String parameter field reference should lower to LLVM IR"); + assert_clang_accepts_ir(&ir); +} + /// Test that constructing a struct with missing fields produces a compile error. /// /// Requirement 4.5 states: IF a struct construction is missing required fields, diff --git a/docs/library-incubation.md b/docs/library-incubation.md new file mode 100644 index 00000000..a761c4db --- /dev/null +++ b/docs/library-incubation.md @@ -0,0 +1,76 @@ +# Project-Driven Library Incubation + +Senline is a real consumer used to expose missing Sengoo capabilities. A +consumer gap should improve the reusable library surface instead of producing +an application-only workaround, but reuse does not automatically justify a +standard-library API. + +## Classification + +| Capability | First home | Rule | +| --- | --- | --- | +| Product DTO, policy, or decision | Product package | Product names and versions never enter a general library. | +| Domain-neutral composition over existing stdlib primitives | Incubating pure Sengoo package | Keep dependencies locked and API independent of the first consumer. | +| Primitive unavailable above the runtime | `std::` or runtime change | Require compiler, LSP, compatibility, native, and distribution tests. | +| Cryptography, TLS, production HTTP, durable database, OS sandbox | Binding to a mature implementation | Do not create a new security algorithm or infrastructure engine for convenience. | +| Senline security, supervision, transaction, or mutation authority | Senline Rust | Authority transfer requires a separate reviewed OpenSpec change. | + +The first incubating packages are `sgframing` and `sgjson_contract` under the +real worker's `packages/` directory. They use no Senline names or limits. +`senline_facts_to_plan` remains a product package. A separate `sgvalidation` +package is deferred until a non-JSON second consumer demonstrates real reuse. + +## Required Package Gates + +Every claimed platform runs these commands with a clean installed toolchain: + +```text +sgpm update --check +sgpm --runtime-mode installed check --locked +sgpm --runtime-mode installed test --locked +sgpm fmt --check --locked +sgpm --runtime-mode installed doc --locked +sgpm --runtime-mode installed build --release --locked +sgpm publish --dry-run --locked --format json +``` + +Source-development runs are useful red/green evidence but are not publication +or consumer-pin evidence. A missing tool, skipped smoke, dirty revision, local +absolute path, or partial platform matrix is recorded as unverified, never +green. `sgpm publish` packages files; it does not replace check or test gates. + +## Graduation + +An incubating `0.x` package may move to the repository-level `packages/` +catalog after all of the following are true: + +- a second independent consumer exists, or a reviewed protocol-foundation + rationale makes the API independently owned; +- the API is documented, domain neutral, and protected by boundary, malformed, + deterministic-error, and resource-lifetime tests; +- locked source and installed-toolchain loops pass on every claimed Windows and + Linux target without required skips; +- publish dry-run is reproducible and contains license, provenance, and no + mutable or absolute development path; +- SemVer and support expectations are recorded. + +Stable `1.0` additionally requires one consumer outside the originating +project and immutable registry content verification. A `std::` proposal +requires at least three consumers across two application domains and two +Sengoo releases, plus a separate OpenSpec change covering compiler imports, +CLI, LSP, docs, examples, compatibility, and installed distribution behavior. + +## Current Boundary + +`sgframing` owns only length-prefix validation, clean EOF versus truncation, +bounded allocation, and exact I/O. It does not own JSON, retries, diagnostics, +deadlines, supervision, or process policy. + +`sgjson_contract` owns exact object shape and typed validation over +`json_parse_*_strict`. It does not implement JSON Schema, coercion, defaults, +business DTOs, or a second parser. + +Senline Rust continues to own facts binding, plan hashing, bundle verification, +TLS, authentication, signatures, replay and revocation, cryptography, worker +pool and circuit policy, OS sandboxing, durable transactions, persistence, +migrations, and every authoritative mutation. diff --git a/docs/network-runtime.md b/docs/network-runtime.md index 8bf004f4..16e038a4 100644 --- a/docs/network-runtime.md +++ b/docs/network-runtime.md @@ -153,6 +153,12 @@ Inside an async function, `await server.next_request_async(timeout_ms)` returns native `HttpServerNextRequestResult.value` field uses the same one-field `HttpServerRequest { handle }` wrapper shape as the source-level outcome. The native future registers listener readiness with the cooperative reactor. A +pending future drop/cancel unregisters that interest. If polling has already +accepted and fully parsed a request but the future is dropped or canceled +before `result` transfers the request handle, the runtime removes the +unpublished handle, sends the deterministic `504` fallback, and leaves the +server reusable. This is scoped future-resource cleanup, not general task or +handler cancellation. timeout returns `STATUS_TIMEOUT` without closing the server; dropping or canceling a pending future unregisters its listener interest. Accepted clients that do not finish a request within the short cooperative I/O slice receive a diff --git a/docs/senline-dogfood-commit-plan.md b/docs/senline-dogfood-commit-plan.md new file mode 100644 index 00000000..bbf1db8d --- /dev/null +++ b/docs/senline-dogfood-commit-plan.md @@ -0,0 +1,121 @@ +# Senline Service Dogfood — Lore Commit Plan + +Recorded: 2026-07-15 +Worktree: `D:\Sengoo\.worktrees\senline-service-dogfood` +Branch: `codex/senline-service-dogfood` +Base: `1de09ccafa7e8f182af68e82352e2d4be39496b0` +Safety checkpoint: `D:\Sengoo\.worktrees\_checkpoints\senline-service-dogfood-20260715-045753` + +## Rules + +- Only this worktree is writable. Never reset/clean. Never edit `D:\Sengoo` primary checkout. +- Exclude generated `target/` and `examples/**/build/` (already gitignored). +- Do not check off tasks 4.8, 5.9, or 6.5 until dual-host CI evidence exists. +- Preserve single-worker 100k resource degradation evidence for task 8.3; sharded 5.11 success is not soak success. +- Lore message fields: title; why; Constraint; Rejected; Confidence; Scope-risk; Directive; Tested; Not-tested. + +## RED/GREEN strategy for already-green working tree + +Pure historical RED-only commits cannot be recreated without rewriting the +working tree or inserting non-compiling intermediate commits. Policy for this +branch: + +1. Each capability commit ships the minimized regression **with** the smallest + general fix, and names the SGDOG id(s). +2. Defect ledger (`docs/senline-dogfood-defects.md`) and evidence schema remain + the durable RED transcript; after each fix commit, evidence `fixing_commit` + and `red_commit` fields are updated in a later docs commit once SHAs exist. +3. Optional verification: a temporary worktree at base + test-only patch may be + used later for 7.2/7.7 rehearsal without mutating this worktree. + +## Ordered commits + +| # | Title (intent) | Paths (owners) | Tasks / defects | +|---|----------------|----------------|-----------------| +| 1 | Record OpenSpec change and authority fixtures | `openspec/changes/senline-service-dogfood/**` | 1.x control plane | +| 2 | Add binary Buffer/I/O exact helpers and regressions | `tools/stdlib/runtime*.{c,h}`, `io.sg`, `ffi.sg`, buffer/binary tests, sglsp surface | 2.x, SGDOG-001/006/008 | +| 3 | Add opt-in strict JSON and length-aware builders | `runtime_json.c`, `json.sg`, json tests/fuzz | 3.x, SGDOG-002/003/007/009/010 | +| 4 | Fix early-return borrow and nested AddrOf codegen | `compiler/**` | SGDOG-011/012 | +| 5 | Make installed native runtime first-class distribution | `tools/sgc/src/installed_runtime.rs`, native_toolchain, package/install scripts, distribution tests, Cargo sha2 | 4.x, SGDOG-004 | +| 6 | Propagate runtime-mode and transitive module maps in sgpm | `tools/sgpm/**` | SGDOG-005/014 | +| 7 | Release ready HTTP future drop and request-copy length | `runtime/src/net*`, `tools/stdlib/net.sg`, http_request_strings | 6.x partial, SGDOG-013/015 | +| 8 | Add sgframing/sgjson_contract and senline-domain-worker | `examples/realworld/senline-domain-worker/**` (source only) | 4A, 5.x source | +| 9 | Add loopback senline-http-dogfood harness | `examples/realworld/senline-http-dogfood/**` | 6.1–6.4 | +| 10 | Add differential/fault harness and contract fixtures | remaining `tools/sgc/tests/senline_*`, realworld test glue, CI workflows | 5.11 evidence harness, CI prep | +| 11 | Publish dogfood defect ledger, support, incubation docs | `docs/senline-*`, library-incubation | 7.1, 7.6, 8.8, 4A.1 | + +After clean commits: run focused Windows gates for 2.10 path, package Windows +archive for 4.7 path, and leave 4.8/5.9/6.5 unchecked until Ubuntu evidence. + +## Resource risk retained for 8.3 + +- Sharded 100k (8 workers × 12,500) completed with digest + `16aebd9ec476d602c9c0d0082ee9e25a87c520c333d6dd3afeb314f8c39ea128`. +- Single-worker 100k hit a 3600s watchdog around case **44,086** with growing + working set / declining throughput. **Do not treat shards as soak pass.** + +## Session 2026-07-15 handoff + +### Clean revision + +- Branch `codex/senline-service-dogfood` is clean after Lore commits. +- HEAD (after compare fix): `ed280b9a3` +- First stacked tip before compare fix: `a96518ddb` +- Safety checkpoint: + `D:\Sengoo\.worktrees\_checkpoints\senline-service-dogfood-20260715-045753` +- PR: https://github.com/Hyper66666/Sengoo/pull/44 +- CI dispatch: https://github.com/Hyper66666/Sengoo/actions/runs/29391088380 + +### Lore commits (base `1de09ccaf` → `ed280b9a3`) + +1. `f3e2c538e` OpenSpec change +2. `f2a55b16f` Binary Buffer/stdio +3. `d7d53dc03` Compiler borrow/AddrOf (+ some sgc tests) +4. `dbff39d2d` Installed runtime + strict JSON payload (merged slice) +5. `bd6056d29` sgpm runtime-mode + transitive maps +6. `1777562d3` HTTP drop/copy lengths +7. `12965ef6d` senline-domain-worker packages +8. `802c689c2` senline-http-dogfood +9. `1577fd3c3` Differential/fault harness +10. `a96518ddb` Defect ledger / support / incubation +11. `ed280b9a3` Manifest compare null fix + +### Local Windows evidence (not dual-host; do not check 4.8/5.9/6.5) + +| Gate | Result | +| --- | --- | +| buffer/binary/handles tests | pass | +| runtime_distribution | 15/15 | +| stdlib_buffer_ / stdlib_json_ | pass | +| sgpm transitive | 2/2 | +| release tool build | pass | +| package-toolchain NoBuild | zip produced | +| install.ps1 smoke | sgc 0.1.0 (a96518ddb68f) | +| dual NoBuild package compare | `status=reproducible` (only `generated_at_utc` excluded) | +| installed worker `sgpm check/test/build --locked` with fake cargo first on PATH | pass (15 package tests + release exe) | + +### Still open (30 tasks) + +- **2.10** needs POSIX pipe + complete focused matrix, not Windows-only. +- **4.7–4.9** need green Ubuntu package smoke / dual-build from CI, not only local Windows. +- **4A.4 / 5.12–5.13** need full installed loops both hosts + packaging evidence. +- **5.9** needs Linux determinism digest matching Windows. +- **6.5–6.7** HTTP dual-host matrix + anti-deploy checks. +- **7.x / 8.x / 9.x** pin chain, soak (incl. case 44086), handoff. + +### Next agent + +1. Wait for run `29391088380` (and re-run on `ed280b9a3` if needed). +2. If Ubuntu package smoke green, extract Linux archive and record hashes; only then consider partial progress notes for 4.7. +3. Do **not** check 4.8 until both-host dual independent builds compare clean. +4. Run installed HTTP package loop and Linux determinism before 5.9/6.5. +5. Investigate single-worker 100k memory under 8.3 with checked-in sampler. + +## Session closeout 2026-07-15 (tasks 4.7–4.9) + +- HEAD: `ba0d03ae3` +- CI: https://github.com/Hyper66666/Sengoo/actions/runs/29419695542 **success** + (windows / ubuntu / macos-15 / macos-15-intel package smoke) +- OpenSpec tasks **4.7, 4.8, 4.9** checked with that evidence. +- Still open for later: 2.10, 4A.4, 5.1/5.4/5.9/5.12–5.13, 6.5–6.7, 7.x, 8.x, 9.x. +- Resource note for 8.3 unchanged: single-worker 100k case 44086. diff --git a/docs/senline-dogfood-defects.md b/docs/senline-dogfood-defects.md new file mode 100644 index 00000000..e761c25c --- /dev/null +++ b/docs/senline-dogfood-defects.md @@ -0,0 +1,428 @@ +# Senline Sengoo Dogfood Defects + +Recorded: 2026-07-14 + +These records cover defects exposed while implementing the linked Senline +change `adopt-sengoo-backend-slice` in the isolated +`codex/senline-service-dogfood` worktree. All fixes below are still uncommitted +working-tree changes. Their status remains `open` until a clean Sengoo commit, +immutable Windows/Linux artifacts, and the reviewed Senline pin exist. + +No record contains live Senline payloads, identifiers, credentials, or +secrets. The reproductions use synthetic literals only. + +## SGDOG-2026-001: Buffer Extension Re-exposed Cleared Bytes + +- Classification: `sengoo-standard-library` +- Owner: Sengoo runtime/stdlib +- Status: `open (local regression green)` +- Consumer requirement: checked byte access for binary worker framing +- RED: `stdlib_buffer_zeroes_gaps_before_exposing_extended_bytes` +- Failure: `Buffer.clear()` retained bytes and a later high-offset byte/u32 + write expanded `used_len`, making the stale gap readable again. +- Fix: zero every gap before extending `used_len`; exact stdin reads now stage + bytes and commit only after the complete read succeeds. +- GREEN: `cargo test -p sgc stdlib_buffer_zeroes_gaps_before_exposing_extended_bytes` +- Remaining gate: partial-write/error injection and Linux pipe evidence + +## SGDOG-2026-002: Strict JSON Broke Permissive Unicode Compatibility + +- Classification: `sengoo-standard-library` +- Owner: Sengoo runtime/stdlib +- Status: `open (local regression green)` +- Consumer requirement: add strict JSON without changing existing callers +- RED: `stdlib_json_permissive_unicode_escape_behavior_remains_compatible` +- Failure: Unicode/surrogate decoding was added to the shared parser and + changed permissive `json_parse*` behavior. +- Fix: strict and permissive escape behavior are explicitly separated. +- GREEN: `cargo test -p sgc stdlib_json_` +- Remaining gate: malformed corpus and fuzz evidence + +## SGDOG-2026-003: JSON Strings and Handles Were Unsafe for a Long-Lived Worker + +- Classification: `sengoo-runtime-ownership` +- Owner: Sengoo runtime +- Status: `open (local regression green)` +- Consumer requirement: strict Unicode plus repeated worker evaluations +- RED: `stdlib_json_strict_preserves_escaped_null_as_string_data` and + `stdlib_json_document_handles_reject_forged_and_reused_stale_values` +- Failure: C-string-backed values rejected legal `U+0000`; raw pointer + `JsonDoc` handles leaked one allocation per close and forged handles could + crash. +- Fix: parsed keys/strings carry explicit byte lengths; exact key lookup has a + pointer+length API; JsonDoc uses generation-checked reusable slots. +- GREEN: `cargo test -p sgc stdlib_json_` +- Remaining gate: sanitizer/soak evidence and explicit allowlist decoder + +## SGDOG-2026-004: Installed Native Builds Could Fall Back or Trust Stale State + +- Classification: `sengoo-package-toolchain` +- Owner: Sengoo toolchain/distribution +- Status: `open (Windows local smoke green; immutable cross-target artifacts absent)` +- Consumer requirement: build the Senline worker outside the Sengoo checkout +- RED: + - `installed_native_build_rejects_missing_manifest_runtime_without_cargo_or_checkout_fallback` + - `relocated_sgc_without_manifest_rejects_implicit_source_checkout_fallback` + - `installed_native_build_rejects_runtime_hash_mismatch_before_link_or_cargo` + - `installed_check_rejects_tampered_runtime_bridge_payload` + - `installed_commands_reject_external_runtime_overrides` +- Failure: `sgc` implicitly ran Cargo from a compiled-in checkout; native + cache hits bypassed installed runtime verification; packages omitted the + static runtime and per-file hashes. +- Fix: schema-2 manifest resolution verifies target, ABI, SHA-256, link + contract, bridge completeness, and cache identity before reuse. Packaging + includes the target runtime and installers verify the complete payload set. + Cargo runtime construction now requires explicit + `--runtime-mode source-development`, is source-workspace guarded, rejects + daemon use, and records non-release provenance in build/run metadata. Normal + installed commands reject `SENGOO_ROOT`, `SENGOO_STDLIB`, and + `SENGOO_RUNTIME` before frontend or engine dispatch. Source-development cache + identity covers the Rust runtime source tree and Cargo inputs, so runtime-only + changes cannot reuse an older linked executable. +- Additional RED: Windows PowerShell wrote a UTF-8 BOM that `serde_json` + rejected. Packaging now writes manifest JSON as UTF-8 without BOM. +- GREEN: + - `cargo test -p sgc --test runtime_distribution` + (`15 passed`, including fresh installed check/build/run/test with fake + Cargo, command/cache/manifest path audits, bridge tamper rejection, and + override rejection across check/build/Cranelift run/test) + - Windows archive install followed by Cargo-free strict-JSON native build in + `D:/senline/logs/sengoo-installed-smoke` +- Remaining gate: clean Windows/Linux rebuilds, installed worker/HTTP smokes, + reproducibility, SBOM/provenance, and Senline pin advancement + +## SGDOG-2026-005: Sgpm Could Not Select Explicit Runtime Provenance + +- Classification: `sengoo-package-toolchain` +- Owner: Sengoo package manager/toolchain +- Status: `open (local regression green; immutable artifact and pin absent)` +- Consumer requirement: run the locked `senline-domain-worker` package loop + without giving installed mode an implicit source-checkout fallback +- RED: `realworld_locked_loop_uses_real_toolchain_binaries` + - first failed because delegated `sgc` remained in default installed mode; + - then rejected `sgpm --runtime-mode source-development` because `sgpm` had + no corresponding explicit option. +- Failure: `sgpm` constructed child `sgc` commands without carrying the + runtime-mode decision, so package check/test/build could not intentionally + dogfood a source runtime while retaining non-release provenance. +- Fix: `sgpm` now exposes global + `--runtime-mode installed|source-development`, defaults to installed, and + prepends the selected mode to every delegated `sgc` command. Formatting does + not consult the compiler runtime. +- GREEN: `cargo test -p sgpm --test realworld_e2e realworld_locked_loop_uses_real_toolchain_binaries -- --exact --nocapture` + (`1 passed`, including locked update/check/test/fmt/doc/build for the new + root worker and `senline_facts_to_plan` path package) +- Remaining gate: clean installed-toolchain package loop, immutable Windows + and Linux artifacts, fixing commit, and reviewed Senline pin + +## SGDOG-2026-006: Empty Buffer Destruction Polluted FFI Error State + +- Classification: `sengoo-runtime-ownership` +- Owner: Sengoo runtime/stdlib +- Status: `open (local regression green)` +- Consumer requirement: represent clean framed EOF without allocating a + payload buffer or changing unrelated diagnostics +- RED: + - `stdlib_buffer_zero_handle_drop_is_noop` + - `stdlib_buffer_zero_handle_free_is_noop` + - consumer clean-EOF path: `sgframing/tests/frame_pipe.sg` +- Failure: `Buffer::drop` and `Buffer::free` called the runtime with handle + zero. A normal `FrameRead { eof: true, payload: Buffer { handle: 0 } }` + therefore changed `ffi_last_error_code()` to `STATUS_INVALID_HANDLE`, and + explicit cleanup did the same. +- Fix: treat a zero Buffer handle as an already-empty resource; explicit + free succeeds and implicit drop performs no runtime call. +- GREEN: + - `cargo test -p sgc stdlib_buffer_zero_handle_ -- --nocapture` + (`2 passed`) + - `cargo test -p sgc --test realworld sgframing_binary_pipe_covers_boundaries_and_exact_output -- --exact --nocapture` + (`1 passed`, including clean EOF with unchanged FFI error state) +- Remaining gate: full stdlib/package regression, installed Windows/Linux + worker EOF evidence, fixing commit, immutable artifacts, and Senline pin + +## SGDOG-2026-007: Nested JSON Corrupted Containers After Node Growth + +- Classification: `sengoo-runtime-json` +- Owner: Sengoo runtime +- Status: `open (local regression green)` +- Consumer requirement: validate realistic nested V1 objects and arrays whose + parsed document contains more than 16 nodes +- RED: + - consumer: `sgjson_contract/tests/array_rejections.sg` + - minimized: `stdlib_json_nested_containers_survive_node_storage_growth` +- Failure: recursive object and array parsers retained pointers into the JSON + document node array. Adding the seventeenth node could reallocate that array, + after which the parser wrote members/items through stale pointers. Small + fixtures stayed below the initial capacity and hid the defect. +- Fix: reacquire the current object or array node by stable node ID after every + recursive child parse before reserving or appending container data. +- GREEN: + - `cargo test -p sgc stdlib_json_nested_containers_survive_node_storage_growth -- --nocapture` + (`1 passed`) + - locked source-development worker package loop (`sgjson_contract`: `6 passed`) +- Remaining gate: malformed/fuzz/sanitizer coverage, Linux and installed + runtime evidence, fixing commit, immutable artifacts, and Senline pin + +## SGDOG-2026-008: Generation Exhaustion Could Produce Negative Runtime Handles + +- Classification: `sengoo-runtime-ownership` +- Owner: Sengoo runtime +- Status: `open (local C boundary regression green)` +- Consumer requirement: long-lived workers must retain positive, non-aliasing + runtime handles as reusable slots advance through their generations +- RED: `generation_handle_encoding_stays_positive_and_signals_exhaustion` in + `tools/sgc/tests/runtime_handles.rs` +- Failure: Buffer, JSON document, opaque, String, and Process slot allocators + encoded a generation with a signed `long long` left shift by 32 bits. At + generation `0x80000000` the mathematical result no longer fit in signed + `long long`, invoking undefined behavior and commonly producing a negative + handle that the runtime then rejected. Eventual unsigned wrap also permitted + an old generation value to be reused. +- Fix: shared runtime handle helpers cap generations at + `SENGOO_RUNTIME_HANDLE_GENERATION_MAX=0x7fffffff`, compose the positive handle + through `uint64_t`, signal exhaustion with generation zero, and make all five + allocator families retire an exhausted slot instead of wrapping it. +- GREEN: + `cargo test -p sgc --test runtime_handles generation_handle_encoding_stays_positive_and_signals_exhaustion -- --exact --nocapture` + (`1 passed` on the current local host) +- Remaining gate: full runtime/stdlib regression and sanitizer coverage, + installed Windows/Linux evidence, fixing commit, immutable artifacts, and + Senline pin + +## SGDOG-2026-009: Strict JSON Diagnostics Had No Stable Machine Kind + +- Classification: `sengoo-standard-library` +- Owner: Sengoo runtime/stdlib +- Status: `open (local regression green)` +- Consumer requirement: map parser failures to frozen worker error codes + without parsing diagnostic text +- RED: `stdlib_json_strict_reports_stable_error_kinds`; worker duplicate, + invalid-Unicode, and trailing-byte fixtures all returned `malformed_json` +- Failure: strict parsing exposed only a status, offset, and mutable human + message, so protocol code could not distinguish stable rejection classes. +- Fix: the runtime and `std::json` expose stable kinds `NONE=0`, + `UNCLASSIFIED=1`, `DUPLICATE_FIELD=2`, `INVALID_UNICODE=3`, and + `TRAILING_BYTES=4`. The worker snapshots the kind before any later JSON + operation and never parses the message. +- GREEN: + - `cargo test -p sgc stdlib_json_ -- --nocapture` (`19 passed`) + - `cargo test -p sgc --test realworld -- --nocapture` (`13 passed`) +- Remaining gate: the last-error slot retains its existing immediate, + process-global lifecycle; malformed fuzzing plus installed Windows/Linux + runtime evidence remain required. + +## SGDOG-2026-010: JSON Builder Truncated Owned Strings at Embedded NUL + +- Classification: `sengoo-runtime-json` +- Owner: Sengoo runtime/stdlib +- Status: `open (local regression green)` +- Consumer requirement: exactly echo legal bounded ASCII identifiers, + including decoded `U+0000` and bytes after it +- RED: + - `stdlib_json_length_aware_builder_preserves_embedded_nul` + - `stdlib_json_length_aware_builder_rejects_invalid_utf8` + - `stdlib_json_owned_string_builder_preserves_invalid_handle_status` + - consumer: `senline_worker_preserves_embedded_nul_in_owned_plan_strings` +- Failure: the only builder path used C-string length and truncated the suffix + after NUL. An initial pointer-plus-length wrapper also risked treating a + negative String pointer status as an address and rewrote invalid handles as + invalid lengths. +- Fix: retain the legacy C-string helper, add a bounded pointer-plus-length + ABI, and expose a checked owned-String builder that validates the document, + String handle, stored byte length, pointer result, and UTF-8 in that order. +- GREEN: + - `cargo test -p sgc stdlib_json_ -- --nocapture` (`19 passed`) + - real worker NUL echo and all realworld tests (`13 passed`) +- Remaining gate: raw pointer callers remain responsible for pointer lifetime; + sanitizer/fuzz coverage and installed Windows/Linux evidence are pending. + +## SGDOG-2026-011: Early Return Moves Poisoned Reachable Fallthrough + +- Classification: `sengoo-compiler-borrow-checker` +- Owner: Sengoo compiler +- Status: `open (local regression green)` +- Consumer requirement: an unsupported-operation return may consume one field + without making the supported fallthrough request appear partially moved +- RED: `early_return_field_move_does_not_poison_the_fallthrough_path`; the + checker reported `request` as partially moved after the moving branch had + already returned. +- Fix: direct unconditional-return branches no longer merge their move state + into the reachable fallthrough; non-terminating branches still do. +- GREEN: compiler library `1064 passed`, borrow `16 passed`, ownership + `35 passed`, Clippy with warnings denied, and rustfmt check. +- Remaining gate: this is a bounded reachability fix, not a full control-flow + lattice; complex nested or all-terminating branch expressions need a + separate compiler design and regression set. + +## SGDOG-2026-012: Nested Field References Produced Invalid LLVM + +- Classification: `sengoo-compiler-codegen` +- Owner: Sengoo compiler +- Status: `open (local regression green)` +- Consumer requirement: pass nested owned plan fields by immutable reference + to the length-aware JSON builder +- RED: `nested_owned_string_parameter_field_reference_generates_valid_ir`; + Clang rejected a `select` that supplied an SSA `%String` where `%String*` + was required. +- Failure: primary LLVM `AddrOf` lowering assumed every source local already + had an address, but nested field extraction produces an SSA temporary. +- Fix: parameter and temporary SSA values are spilled to a same-typed stack + slot before taking their address; existing stack-local references retain + their original path. The worker's temporary handle-rewrapping workaround + was removed and the real nested borrow now compiles and runs. +- GREEN: Clang reference tests `3 passed`, struct codegen `8 passed`, compiler + library `1064 passed`, integration `26 passed`, Clippy, rustfmt, diff-check, + and real worker tests `13 passed`. +- Remaining gate: the legacy JIT emitter is separate, and nested mutable + references still require an addressable-place/MIR design before claiming + original-place mutation semantics. + +## SGDOG-2026-013: Ready HTTP Future Drop Leaked an Unpublished Request + +- Classification: `sengoo-async-concurrency` +- Owner: Sengoo runtime +- Status: `open (local native regression green)` +- Consumer requirement: the loopback dogfood harness must release an accepted + request when `next_request_async` is dropped or canceled before `result` + publishes its handle. +- RED: + - `http_server_next_request_async_drop_ready_releases_unpublished_request` + - `http_server_next_request_async_cancel_ready_releases_unpublished_request` +- Failure: polling could accept, parse, and store a request handle in the + future's ready outcome. Drop/cancel only unregistered listener interest, so + the request table and client connection remained live forever when no caller + consumed `result`. +- Fix: ready abandonment now atomically takes any successful unpublished + request handle, writes the existing deterministic `504` fallback, closes the + connection, and preserves reuse of the server. Pending and error outcomes + retain their previous cleanup behavior. +- GREEN: focused ready-abandonment regressions (`2 passed`) and the full native + net suite (`40 passed`). +- Remaining gate: product-level `senline-http-dogfood` future-drop equivalence + and Windows/Linux installed package loops. + +## SGDOG-2026-014: Package Module Maps Omitted Transitive Source Imports + +- Classification: `sengoo-package-runner` +- Owner: Sengoo `sgpm` +- Status: `open (local regression green)` +- Consumer requirement: a package importing `senline_domain_worker` must also + resolve the worker source's imports without redeclaring its internal package + graph as direct product dependencies. +- RED: `sgpm_check_exposes_transitive_dependency_library_module_map`; the root + package received only `direct=`, and real locked HTTP compilation + failed on unresolved `senline_build_identity` from the worker source. +- Failure: `module_map_value` filtered edges to `edge.from == node.id`, while + `sgc` expands imported source recursively using one flat module map. The + lockfile already contained the complete dependency graph, but `sgpm` + discarded every transitive edge when constructing `SENGOO_MODULE_MAP`. +- Fix: collect the selected package's reachable dependency closure, encode + aliases in deterministic order, and fail closed when one reachable alias + names different library sources. +- GREEN: `cargo test -p sgpm --test integration transitive -- --nocapture` + (`2 passed`) and the HTTP locked source-development test loop. +- Remaining gate: full `sgpm` integration, installed-toolchain, and Linux + package loops; the flat module map still intentionally cannot represent a + graph with conflicting aliases. + +## SGDOG-2026-015: HTTP Request Copies Lost Buffer Length Metadata + +- Classification: `sengoo-stdlib-net-buffer-contract` +- Owner: Sengoo runtime and `std::net` +- Status: `open (local regression green)` +- Consumer requirement: request string accessors and `body_copy` must return + owned data that remains valid for strict JSON parsing and response policy. +- RED: `real_sgc_http_request_owned_string_accessors_are_safe` first ended the + child with Windows status `3221225477` (`0xC0000005`) and socket error 10054; + after minimizing the accessor, the body-copy assertion returned HTTP 500 + because `used_len` remained zero. +- Failure: the native request-copy ABI writes through a raw pointer and cannot + update the owning Buffer handle's `used_len`. Owned string wrappers therefore + passed a nonzero copied length to `string_from_buffer` with `used_len == 0`, + received an invalid-argument result, and eager Sengoo boolean evaluation let + consumer code dereference the placeholder String handle. Body bytes reached + strict JSON with the same stale metadata and normalized to `malformed_json`. +- Fix: owned request-string accessors use the existing length-aware native byte + copy constructor, and successful request copy wrappers commit the exact + copied length through a checked private runtime primitive. The HTTP harness + also uses sequential Result guards so error paths never read placeholder + values under eager `and`/`or` semantics. +- GREEN: `cargo test -p sgc --test http_request_strings -- --nocapture` + (`1 passed`), locked HTTP package tests (`2 passed`), and a local real worker + versus HTTP synthetic fixture comparison (`647` equal bytes, SHA-256 + `8b790b24c0f6306287caaef34544ecbab5d6ccd638af4294265eb2446fba545d`). +- Remaining gate: full request-copy compatibility, malformed transport, larger + or segmented request, installed Windows, and Linux localhost matrices. + +## Pin State + +No defect is pinned or closed. The local Windows dogfood manifest records +`source_dirty=true`, `artifact_provenance=prebuilt-unverified`, and +`release_eligible=false`; it must not be copied into Senline's immutable +bundle manifest. Even a manifest that claims `release_eligible=true` cannot +self-authenticate as Senline pin evidence: `senline_pin_evidence` remains false +until Senline independently verifies a clean revision and immutable complete +bundle hashes. There are no active Senline workarounds for these defects. + +## Resource Observation (task 8.3, not a closed defect) + +- Classification: `sengoo-runtime-resource` +- Owner: Sengoo runtime / worker long-session +- Status: `open (investigation required)` +- Observation: a **single-process** 100,000-case differential run reached the + 3600-second watchdog near case **44,086** while private working set and + throughput continued to degrade. +- Related success: task 5.11 used **8 fresh workers × 12,500** cases and + recorded transcript digest + `16aebd9ec476d602c9c0d0082ee9e25a87c520c333d6dd3afeb314f8c39ea128` with zero + mismatch/crash/hang/malformed/nondeterminism. That sharded result **does not** + satisfy task 8.3 soak or stable post-warm-up memory requirements. +- Directive: investigate under task 8.3 with checked-in sampler methodology; + do not re-label shard success as resource green. +- **2026-07-15 re-measurement (Windows x64, pre-fix)** + `senline_worker_resource` `investigate-45k` stopped at case **29,014 / 45,000** + on a 900 s watchdog; PWS ~5.2→96 MiB (~**3.27 KiB/case**); cps ~1740→7.3; + handles flat at 66. Evidence: + `target/senline-resource/soak-investigate-45k-windows-x86_64-1784109344.summary.json`. +- **Root cause (fixed):** by-value `String` parameters to lowered lambdas + (field-allowlist callbacks in `sgjson_exact_object_fields`) never ran Drop + glue. Each evaluation leaked ~dozens of key strings → linear PWS growth and + slot-table scan slowdown. Compiler fix: force-record owned lambda param drops + + `insert_drop_glue` for lambda MIR + (`compiler/src/mir/lowering/lambda_expr_helpers.rs`, + `drop_glue_helpers.rs::force_record_owned_param_drop`). +- **2026-07-15 post-fix (Windows x64):** same investigate-45k **completes** + 45,000/45,000 in ~56 s; PWS ~1.1→5.1 MiB (~**92 B/case** post-warm-up, under + 1 KiB/case); handles 66; `plan_ok=45000`. Evidence: + `target/senline-resource/soak-investigate-45k-windows-x86_64-1784124464.summary.json`. +- **Residual root cause (fixed 2026-07-16):** worker helper + `worker_validate_execution_mode(value: String)` took a by-value legacy handle + and returned without Drop (ordinary function params skip auto-Drop for + `String`). Combined with a second extract of `execution_mode`, each request + leaked one owned mode string (~90 B/case). Fix: validate via `&str`, reuse the + single extracted string; also return the validated literal from + `worker_required_literal` instead of re-extracting; borrow + `evaluation_id` for the unsupported-version encoder. +- **2026-07-16 residual post-fix (Windows x64):** investigate-45k **completes** + 45,000/45,000 in ~9 s; PWS ~1.0→1.14 MiB (~**3.4 B/case** post-warm-up, noise + floor); handles 67; `plan_ok=45000`; p50/p95/p99 ≈ 154/332/500 µs. Evidence: + `target/senline-resource/soak-investigate-45k-windows-x86_64-1784198111.summary.json`. +- **2026-07-17 ownership hardening:** + 1. Worker unsupported-version path: both branches move `WorkerRequestV1` into + owning helpers so path-insensitive moved tracking cannot skip nested + String Drop (`worker_reject_unsupported_operation_v1` / + `worker_accept_decoded_request_v1`). Nested String fields Drop via + aggregate param field bindings (not bare `String` params). + 2. Resource harness kills the worker before joining stdout/stderr readers so + a hung worker cannot freeze the watchdog. + 3. Always-on regression: + `resource_unsupported_operation_version_path_does_not_grow_memory`. + 4. **Not** auto-Dropping ordinary by-value `String` params: method receivers + are still lowered as handle copies (`len`/`as_str`); enabling param Drop + free'd live objects under the caller's handle. Full language ownership + remains an open compiler task (review P1). +- **2026-07-17 1M soak (Windows x64):** `resource_single_worker_soak_1m` + completed 1,000,000/1,000,000 in ~238 s; PWS growth **~0.066 B/case**; + handles 68; `plan_ok=1000000`; p50/p95/p99 = 179/350/450 µs. Evidence: + `target/senline-resource/soak-soak-1m-windows-x86_64-1784280826.summary.json`. + Task 8.3 resource stability gate is satisfied on the recorded Windows host. diff --git a/docs/senline-dogfood-determinism-evidence.md b/docs/senline-dogfood-determinism-evidence.md new file mode 100644 index 00000000..307f3e3f --- /dev/null +++ b/docs/senline-dogfood-determinism-evidence.md @@ -0,0 +1,42 @@ +# Cross-host Determinism Evidence (task 5.9) + +Recorded from GitHub Actions `core-conformance` run +[`29424861027`](https://github.com/Hyper66666/Sengoo/actions/runs/29424861027) +on commit `e9f1b1168` (jobs green for dual-host differential; core-language failed +on an unrelated evidence-hash drift fixed later). + +## Claim + +Identical inputs produce byte-equivalent normalized plans across: + +1. two fresh worker processes on the same host (cross-process), and +2. Windows x64 and Linux x64 hosts (cross-host transcript digests). + +## Transcript digests (must match across OS) + +| Corpus | Cases | Fresh processes | Windows `transcript_sha256` | Linux `transcript_sha256` | Match | +| --- | ---: | ---: | --- | --- | --- | +| determinism | 512 | 2 | `bd6acd82479bd6219cbf8e96601313e79f01bb518cee5a98f137be3e40f9729c` | `bd6acd82479bd6219cbf8e96601313e79f01bb518cee5a98f137be3e40f9729c` | yes | +| reviewed_boundary | 10,000 | 1 | `a32f445f38e4810bc3eab9f2744ed337f48e2f5fa18521a9b05002c42126dd0b` | `a32f445f38e4810bc3eab9f2744ed337f48e2f5fa18521a9b05002c42126dd0b` | yes | +| seeded_eligible | 100,000 | 8 | `16aebd9ec476d602c9c0d0082ee9e25a87c520c333d6dd3afeb314f8c39ea128` | `16aebd9ec476d602c9c0d0082ee9e25a87c520c333d6dd3afeb314f8c39ea128` | yes | + +All three corpora reported zero semantic mismatches, crashes, hangs, malformed +plans, and nondeterministic plans on both hosts. + +## Test surface + +- `tools/sgc/tests/senline_worker_differential.rs` + - `identical_inputs_have_identical_raw_plan_bytes_across_fresh_processes` + - ignored release corpora for 10k / 100k +- CI job: `Senline worker differential (${{ matrix.os }})` in + `.github/workflows/core-conformance.yml` +- Uploaded artifacts: + `senline-worker-differential-Windows-X64`, + `senline-worker-differential-Linux-X64` + +## Limits + +- Digests prove planner/worker I/O normalization equivalence, not resource soak + (task 8.3) or Senline pin (task 7.5 / 9.5). +- Fixture byte equality on Windows requires LF-normalized fixture reads + (`normalize_fixture_bytes`) so CRLF checkouts do not falsify frozen hashes. diff --git a/docs/senline-dogfood-evidence.schema.json b/docs/senline-dogfood-evidence.schema.json new file mode 100644 index 00000000..22cad480 --- /dev/null +++ b/docs/senline-dogfood-evidence.schema.json @@ -0,0 +1,469 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://sengoo.dev/schema/senline-dogfood-evidence-v1.json", + "title": "Senline-driven Sengoo defect evidence", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "change", "linked_senline_change", "records"], + "properties": { + "schema_version": { "const": 1 }, + "change": { "const": "senline-service-dogfood" }, + "linked_senline_change": { "const": "adopt-sengoo-backend-slice" }, + "records": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/record" } + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "revision": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "nullable_sha256": { + "type": ["string", "null"], + "pattern": "^[0-9a-f]{64}$" + }, + "nullable_revision": { + "type": ["string", "null"], + "pattern": "^[0-9a-f]{40}$" + }, + "record": { + "type": "object", + "additionalProperties": false, + "required": [ + "record_id", + "senline_failure", + "ownership", + "minimized_regression", + "fix", + "target_artifacts", + "senline_pin", + "final_consumer_gate", + "workaround" + ], + "properties": { + "record_id": { "type": "string", "pattern": "^SGDOG-[0-9]{4}-[0-9]{3}$" }, + "senline_failure": { "$ref": "#/$defs/senline_failure" }, + "ownership": { "$ref": "#/$defs/ownership" }, + "minimized_regression": { "$ref": "#/$defs/minimized_regression" }, + "fix": { "$ref": "#/$defs/fix" }, + "target_artifacts": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { "$ref": "#/$defs/target_artifact" }, + "allOf": [ + { + "contains": { + "type": "object", + "required": ["target"], + "properties": { + "target": { "const": "x86_64-pc-windows-msvc" } + } + }, + "minContains": 1, + "maxContains": 1 + }, + { + "contains": { + "type": "object", + "required": ["target"], + "properties": { + "target": { "const": "x86_64-unknown-linux-gnu" } + } + }, + "minContains": 1, + "maxContains": 1 + } + ] + }, + "senline_pin": { "$ref": "#/$defs/senline_pin" }, + "final_consumer_gate": { "$ref": "#/$defs/final_consumer_gate" }, + "workaround": { "$ref": "#/$defs/workaround" } + }, + "allOf": [ + { + "if": { + "required": ["final_consumer_gate"], + "properties": { + "final_consumer_gate": { + "required": ["status"], + "properties": { + "status": { "const": "green" } + } + } + } + }, + "then": { + "properties": { + "minimized_regression": { + "properties": { + "red_status": { "const": "preserved" }, + "red_commit": { "$ref": "#/$defs/revision" } + } + }, + "fix": { + "properties": { + "fixing_commit": { "$ref": "#/$defs/revision" } + } + }, + "target_artifacts": { + "items": { + "properties": { + "status": { "const": "verified" } + } + } + }, + "senline_pin": { + "properties": { + "status": { "const": "verified" } + } + }, + "workaround": { + "properties": { + "active": { "const": false } + } + } + } + } + }, + { + "if": { + "required": ["senline_pin"], + "properties": { + "senline_pin": { + "required": ["status"], + "properties": { + "status": { "const": "verified" } + } + } + } + }, + "then": { + "properties": { + "minimized_regression": { + "properties": { + "red_status": { "const": "preserved" }, + "red_commit": { "$ref": "#/$defs/revision" } + } + }, + "fix": { + "properties": { + "fixing_commit": { "$ref": "#/$defs/revision" } + } + }, + "target_artifacts": { + "items": { + "properties": { + "status": { "const": "verified" } + } + } + } + } + } + } + ] + }, + "senline_failure": { + "type": "object", + "additionalProperties": false, + "required": [ + "failure_id", + "change", + "fixture", + "fixture_mirror", + "fixture_sha256", + "evidence_kind", + "consumer_record", + "consumer_record_sha256" + ], + "properties": { + "failure_id": { "type": "string", "minLength": 1 }, + "change": { "const": "adopt-sengoo-backend-slice" }, + "fixture": { "type": "string", "minLength": 1 }, + "fixture_mirror": { "type": "string", "minLength": 1 }, + "fixture_sha256": { "$ref": "#/$defs/sha256" }, + "evidence_kind": { + "enum": ["consumer-failure", "known-baseline-rehearsal", "injected-rehearsal"] + }, + "consumer_record": { "type": "string", "minLength": 1 }, + "consumer_record_sha256": { "$ref": "#/$defs/sha256" } + } + }, + "ownership": { + "type": "object", + "additionalProperties": false, + "required": ["authority", "component_classification", "owner", "discovery_status"], + "properties": { + "authority": { + "enum": ["sengoo-owned", "senline-owned", "shared-contract"] + }, + "component_classification": { + "enum": [ + "sengoo-compiler", + "sengoo-runtime", + "sengoo-standard-library", + "sengoo-package-toolchain", + "sengoo-product-package", + "senline-rust-host", + "shared-contract" + ] + }, + "owner": { "type": "string", "minLength": 1 }, + "discovery_status": { + "enum": ["consumer-discovered", "known-baseline", "injected-rehearsal"] + } + } + }, + "minimized_regression": { + "type": "object", + "additionalProperties": false, + "required": ["repository", "test", "selector", "red_command", "red_status", "red_commit"], + "properties": { + "repository": { "const": "Sengoo" }, + "test": { "type": "string", "minLength": 1 }, + "selector": { "$ref": "#/$defs/test_selector" }, + "red_command": { "type": "string", "minLength": 1 }, + "red_status": { "enum": ["preserved", "pending-commit"] }, + "red_commit": { "$ref": "#/$defs/nullable_revision" } + }, + "allOf": [ + { + "if": { + "required": ["red_status"], + "properties": { + "red_status": { "const": "preserved" } + } + }, + "then": { + "properties": { + "red_commit": { "$ref": "#/$defs/revision" } + } + } + } + ] + }, + "test_selector": { + "type": "object", + "additionalProperties": false, + "required": ["package", "target_kind", "target_name", "test_name"], + "properties": { + "package": { "type": "string", "minLength": 1 }, + "target_kind": { "enum": ["bin", "test"] }, + "target_name": { "type": "string", "minLength": 1 }, + "test_name": { "type": "string", "minLength": 1 } + } + }, + "fix": { + "type": "object", + "additionalProperties": false, + "required": [ + "summary", + "fixing_commit", + "affected_gates", + "rejected_workarounds", + "remaining_platform_gaps" + ], + "properties": { + "summary": { "type": "string", "minLength": 1 }, + "fixing_commit": { "$ref": "#/$defs/nullable_revision" }, + "affected_gates": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + }, + "rejected_workarounds": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "remaining_platform_gaps": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + } + } + }, + "target_artifact": { + "type": "object", + "additionalProperties": false, + "required": [ + "target", + "status", + "source_revision", + "build_manifest_id", + "provenance", + "archive", + "manifest", + "archive_sha256", + "manifest_sha256" + ], + "properties": { + "target": { "enum": ["x86_64-pc-windows-msvc", "x86_64-unknown-linux-gnu"] }, + "status": { "enum": ["pending", "verified"] }, + "source_revision": { "$ref": "#/$defs/nullable_revision" }, + "build_manifest_id": { "$ref": "#/$defs/nullable_sha256" }, + "provenance": { "type": ["string", "null"] }, + "archive": { "type": ["string", "null"] }, + "manifest": { "type": ["string", "null"] }, + "archive_sha256": { "$ref": "#/$defs/nullable_sha256" }, + "manifest_sha256": { "$ref": "#/$defs/nullable_sha256" } + }, + "allOf": [ + { + "if": { + "required": ["status"], + "properties": { + "status": { "const": "verified" } + } + }, + "then": { + "properties": { + "source_revision": { "$ref": "#/$defs/revision" }, + "build_manifest_id": { "$ref": "#/$defs/sha256" }, + "provenance": { "type": "string", "minLength": 1 }, + "archive": { "type": "string", "minLength": 1 }, + "manifest": { "type": "string", "minLength": 1 }, + "archive_sha256": { "$ref": "#/$defs/sha256" }, + "manifest_sha256": { "$ref": "#/$defs/sha256" } + } + } + } + ] + }, + "pin_target_manifest": { + "type": "object", + "additionalProperties": false, + "required": ["target", "manifest_sha256"], + "properties": { + "target": { "enum": ["x86_64-pc-windows-msvc", "x86_64-unknown-linux-gnu"] }, + "manifest_sha256": { "$ref": "#/$defs/sha256" } + } + }, + "senline_pin": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "senline_pin_revision", + "pinned_sengoo_revision", + "target_manifests" + ], + "properties": { + "status": { "enum": ["pending", "verified"] }, + "senline_pin_revision": { "$ref": "#/$defs/nullable_revision" }, + "pinned_sengoo_revision": { "$ref": "#/$defs/nullable_revision" }, + "target_manifests": { + "type": "array", + "items": { "$ref": "#/$defs/pin_target_manifest" } + } + }, + "allOf": [ + { + "if": { + "required": ["status"], + "properties": { + "status": { "const": "verified" } + } + }, + "then": { + "properties": { + "senline_pin_revision": { "$ref": "#/$defs/revision" }, + "pinned_sengoo_revision": { "$ref": "#/$defs/revision" }, + "target_manifests": { + "minItems": 2, + "maxItems": 2, + "allOf": [ + { + "contains": { + "properties": { + "target": { "const": "x86_64-pc-windows-msvc" } + } + }, + "minContains": 1, + "maxContains": 1 + }, + { + "contains": { + "properties": { + "target": { "const": "x86_64-unknown-linux-gnu" } + } + }, + "minContains": 1, + "maxContains": 1 + } + ] + } + } + } + } + ] + }, + "final_consumer_gate": { + "type": "object", + "additionalProperties": false, + "required": ["status", "command", "evidence"], + "properties": { + "status": { "enum": ["pending", "failed", "green"] }, + "command": { "type": "string", "minLength": 1 }, + "evidence": { "type": ["string", "null"] } + }, + "allOf": [ + { + "if": { + "required": ["status"], + "properties": { + "status": { "const": "green" } + } + }, + "then": { + "properties": { + "evidence": { "type": "string", "minLength": 1 } + } + } + } + ] + }, + "workaround": { + "type": "object", + "additionalProperties": false, + "required": ["active", "owner", "linked_defect", "expiry_condition", "removal_test"], + "properties": { + "active": { "type": "boolean" }, + "owner": { "type": ["string", "null"] }, + "linked_defect": { "type": ["string", "null"] }, + "expiry_condition": { "type": ["string", "null"] }, + "removal_test": { "type": ["string", "null"] } + }, + "allOf": [ + { + "if": { + "required": ["active"], + "properties": { + "active": { "const": true } + } + }, + "then": { + "properties": { + "owner": { "type": "string", "minLength": 1 }, + "linked_defect": { "type": "string", "minLength": 1 }, + "expiry_condition": { "type": "string", "minLength": 1 }, + "removal_test": { "type": "string", "minLength": 1 } + } + }, + "else": { + "properties": { + "owner": { "type": "null" }, + "linked_defect": { "type": "null" }, + "expiry_condition": { "type": "null" }, + "removal_test": { "type": "null" } + } + } + } + ] + } + } +} diff --git a/docs/senline-dogfood-evidence.v1.json b/docs/senline-dogfood-evidence.v1.json new file mode 100644 index 00000000..096d82ad --- /dev/null +++ b/docs/senline-dogfood-evidence.v1.json @@ -0,0 +1,273 @@ +{ + "schema_version": 1, + "change": "senline-service-dogfood", + "linked_senline_change": "adopt-sengoo-backend-slice", + "records": [ + { + "record_id": "SGDOG-2026-001", + "senline_failure": { + "failure_id": "SGDOG-2026-001", + "change": "adopt-sengoo-backend-slice", + "fixture": "fixtures/sengoo-worker/v1/cases/eligible-accept.request.json", + "fixture_mirror": "examples/realworld/senline-domain-worker/fixtures/v1/cases/eligible-accept.request.json", + "fixture_sha256": "398159a680fec4f9724f5d4ed09a461da97a627146c4469fe936c5efc8c3f575", + "evidence_kind": "known-baseline-rehearsal", + "consumer_record": "docs/senline-dogfood-defects.md", + "consumer_record_sha256": "249ab80d691144e6ed4819cc20eceaa7028cd7d62a400be0d3dc7f69620a93e7" + }, + "ownership": { + "authority": "sengoo-owned", + "component_classification": "sengoo-standard-library", + "owner": "Sengoo runtime and standard library", + "discovery_status": "known-baseline" + }, + "minimized_regression": { + "repository": "Sengoo", + "test": "tools/sgc/src/tests.rs::stdlib_buffer_zeroes_gaps_before_exposing_extended_bytes", + "selector": { + "package": "sgc", + "target_kind": "bin", + "target_name": "sgc", + "test_name": "tests::stdlib_buffer_zeroes_gaps_before_exposing_extended_bytes" + }, + "red_command": "cargo test -p sgc --bin sgc tests::stdlib_buffer_zeroes_gaps_before_exposing_extended_bytes -- --exact --nocapture", + "red_status": "pending-commit", + "red_commit": null + }, + "fix": { + "summary": "Zero every newly exposed Buffer gap before advancing used_len.", + "fixing_commit": null, + "affected_gates": [ + "cargo test -p sgc --bin sgc tests::stdlib_buffer_zeroes_gaps_before_exposing_extended_bytes -- --exact --nocapture" + ], + "rejected_workarounds": [ + "Do not require every protocol caller to pre-zero full Buffer capacity." + ], + "remaining_platform_gaps": [ + "Red-first commit history is not yet reconstructed for this known-baseline rehearsal (task 7.2 open).", + "The linked Senline pin and final consumer gate remain pending (tasks 7.5/9.5)." + ] + }, + "target_artifacts": [ + { + "target": "x86_64-pc-windows-msvc", + "status": "pending", + "source_revision": null, + "build_manifest_id": null, + "provenance": null, + "archive": null, + "manifest": null, + "archive_sha256": null, + "manifest_sha256": null + }, + { + "target": "x86_64-unknown-linux-gnu", + "status": "pending", + "source_revision": null, + "build_manifest_id": null, + "provenance": null, + "archive": null, + "manifest": null, + "archive_sha256": null, + "manifest_sha256": null + } + ], + "senline_pin": { + "status": "pending", + "senline_pin_revision": null, + "pinned_sengoo_revision": null, + "target_manifests": [] + }, + "final_consumer_gate": { + "status": "pending", + "command": "cargo test --locked --workspace", + "evidence": null + }, + "workaround": { + "active": false, + "owner": null, + "linked_defect": null, + "expiry_condition": null, + "removal_test": null + } + }, + { + "record_id": "SGDOG-2026-014", + "senline_failure": { + "failure_id": "SGDOG-2026-014", + "change": "adopt-sengoo-backend-slice", + "fixture": "fixtures/sengoo-worker/v1/cases/eligible-accept.request.json", + "fixture_mirror": "examples/realworld/senline-domain-worker/fixtures/v1/cases/eligible-accept.request.json", + "fixture_sha256": "398159a680fec4f9724f5d4ed09a461da97a627146c4469fe936c5efc8c3f575", + "evidence_kind": "consumer-failure", + "consumer_record": "docs/senline-dogfood-defects.md", + "consumer_record_sha256": "249ab80d691144e6ed4819cc20eceaa7028cd7d62a400be0d3dc7f69620a93e7" + }, + "ownership": { + "authority": "sengoo-owned", + "component_classification": "sengoo-package-toolchain", + "owner": "Sengoo sgpm", + "discovery_status": "consumer-discovered" + }, + "minimized_regression": { + "repository": "Sengoo", + "test": "tools/sgpm/tests/integration.rs::sgpm_check_exposes_transitive_dependency_library_module_map", + "selector": { + "package": "sgpm", + "target_kind": "test", + "target_name": "integration", + "test_name": "sgpm_check_exposes_transitive_dependency_library_module_map" + }, + "red_command": "cargo test -p sgpm --test integration sgpm_check_exposes_transitive_dependency_library_module_map -- --exact --nocapture", + "red_status": "pending-commit", + "red_commit": null + }, + "fix": { + "summary": "Expose the selected package's deterministic transitive library module map and reject conflicting aliases.", + "fixing_commit": null, + "affected_gates": [ + "cargo test -p sgpm --test integration transitive -- --nocapture", + "sgpm --runtime-mode source-development test --locked" + ], + "rejected_workarounds": [ + "Do not duplicate every worker-internal dependency in the HTTP product manifest." + ], + "remaining_platform_gaps": [ + "Red-first commit history is not yet reconstructed (task 7.2 open).", + "The linked Senline pin and final consumer gate remain pending (tasks 7.5/9.5)." + ] + }, + "target_artifacts": [ + { + "target": "x86_64-pc-windows-msvc", + "status": "pending", + "source_revision": null, + "build_manifest_id": null, + "provenance": null, + "archive": null, + "manifest": null, + "archive_sha256": null, + "manifest_sha256": null + }, + { + "target": "x86_64-unknown-linux-gnu", + "status": "pending", + "source_revision": null, + "build_manifest_id": null, + "provenance": null, + "archive": null, + "manifest": null, + "archive_sha256": null, + "manifest_sha256": null + } + ], + "senline_pin": { + "status": "pending", + "senline_pin_revision": null, + "pinned_sengoo_revision": null, + "target_manifests": [] + }, + "final_consumer_gate": { + "status": "pending", + "command": "cargo test --locked --workspace", + "evidence": null + }, + "workaround": { + "active": false, + "owner": null, + "linked_defect": null, + "expiry_condition": null, + "removal_test": null + } + }, + { + "record_id": "SGDOG-2026-015", + "senline_failure": { + "failure_id": "SGDOG-2026-015", + "change": "adopt-sengoo-backend-slice", + "fixture": "fixtures/sengoo-worker/v1/cases/eligible-accept.request.json", + "fixture_mirror": "examples/realworld/senline-domain-worker/fixtures/v1/cases/eligible-accept.request.json", + "fixture_sha256": "398159a680fec4f9724f5d4ed09a461da97a627146c4469fe936c5efc8c3f575", + "evidence_kind": "consumer-failure", + "consumer_record": "docs/senline-dogfood-defects.md", + "consumer_record_sha256": "249ab80d691144e6ed4819cc20eceaa7028cd7d62a400be0d3dc7f69620a93e7" + }, + "ownership": { + "authority": "sengoo-owned", + "component_classification": "sengoo-standard-library", + "owner": "Sengoo runtime and standard library", + "discovery_status": "consumer-discovered" + }, + "minimized_regression": { + "repository": "Sengoo", + "test": "tools/sgc/tests/http_request_strings.rs::real_sgc_http_request_owned_string_accessors_are_safe", + "selector": { + "package": "sgc", + "target_kind": "test", + "target_name": "http_request_strings", + "test_name": "real_sgc_http_request_owned_string_accessors_are_safe" + }, + "red_command": "cargo test -p sgc --test http_request_strings real_sgc_http_request_owned_string_accessors_are_safe -- --exact --nocapture", + "red_status": "pending-commit", + "red_commit": null + }, + "fix": { + "summary": "Preserve exact length metadata for HTTP request copies and construct owned request strings from explicit byte lengths.", + "fixing_commit": null, + "affected_gates": [ + "cargo test -p sgc --test http_request_strings -- --nocapture", + "sgpm --runtime-mode source-development test --locked" + ], + "rejected_workarounds": [ + "Do not let each HTTP consumer allocate a second Buffer or bypass strict JSON length checks." + ], + "remaining_platform_gaps": [ + "Red-first commit history is not yet reconstructed (task 7.2 open).", + "The linked Senline pin and final consumer gate remain pending (tasks 7.5/9.5)." + ] + }, + "target_artifacts": [ + { + "target": "x86_64-pc-windows-msvc", + "status": "pending", + "source_revision": null, + "build_manifest_id": null, + "provenance": null, + "archive": null, + "manifest": null, + "archive_sha256": null, + "manifest_sha256": null + }, + { + "target": "x86_64-unknown-linux-gnu", + "status": "pending", + "source_revision": null, + "build_manifest_id": null, + "provenance": null, + "archive": null, + "manifest": null, + "archive_sha256": null, + "manifest_sha256": null + } + ], + "senline_pin": { + "status": "pending", + "senline_pin_revision": null, + "pinned_sengoo_revision": null, + "target_manifests": [] + }, + "final_consumer_gate": { + "status": "pending", + "command": "cargo test --locked --workspace", + "evidence": null + }, + "workaround": { + "active": false, + "owner": null, + "linked_defect": null, + "expiry_condition": null, + "removal_test": null + } + } + ] +} diff --git a/docs/senline-dogfood-handoff.md b/docs/senline-dogfood-handoff.md new file mode 100644 index 00000000..2c55215a --- /dev/null +++ b/docs/senline-dogfood-handoff.md @@ -0,0 +1,90 @@ +# Senline Service Dogfood Handoff + +Recorded: 2026-07-15 +Sengoo branch: `codex/senline-service-dogfood` +Clean source revision (this handoff document's parent tip may advance): see Git `HEAD`. + +## Authority summary + +| Concern | Owner | +| --- | --- | +| TLS, signed request verification, freshness/replay, device auth/revocation, rate limits | Senline Rust | +| Cryptography, transactions, prekey/ACK/cursor, persistence, migrations, final mutation | Senline Rust | +| Bounded facts → plan evaluation over framed stdio | Sengoo `senline-domain-worker` | +| Loopback synthetic HTTP dogfood (not ingress) | Sengoo `senline-http-dogfood` | +| Sandbox, supervisor, shadow/guarded/alpha, rollback | Senline Rust | + +## Protocol surface + +- V1 framed worker protocol: four-byte BE length + UTF-8 JSON +- Input ≤ 32 KiB, output ≤ 8 KiB, one request at a time, protocol-only stdout +- Fixtures: `examples/realworld/senline-domain-worker/fixtures/v1/` +- Generated protocol notes: `fixtures/v1/docs/generated/protocol-v1.md` +- Differential corpus metadata: `fixtures/v1/differential-corpus-v1.json` + +## Installed layout (toolchain) + +Produced by `scripts/package-toolchain.ps1` and verified by +`toolchain-distribution` CI (run `29419695542` on `ba0d03ae3`): + +- Windows x64 / Linux x64 archives with native runtime library +- Dual independent A/B builds with normalized manifest compare +- Allowed provenance differences only (e.g. `generated_at_utc`) + +## Worker package + +`scripts/package-senline-worker.ps1` builds with installed `sgpm`/`sgc` and emits: + +- `senline_domain_worker[.exe]` +- `fixtures/` +- `worker-manifest.json` (payload path, size, SHA-256) + +## Defect evidence + +- Ledger: `docs/senline-dogfood-defects.md` (SGDOG-2026-001 .. 015) +- Schema: `docs/senline-dogfood-evidence.schema.json` +- Records: `docs/senline-dogfood-evidence.v1.json` +- Support boundary: `docs/senline-dogfood-support.md` + +## Evidence index + +| Doc | Covers | +| --- | --- | +| `docs/senline-dogfood-determinism-evidence.md` | Task 5.9 dual-host digests | +| `docs/senline-dogfood-resource-methodology.md` | Tasks 8.3 / 8.4 sampler policy | +| `docs/senline-dogfood-latency-evidence.md` | Task 8.4 bulk means (partial) | +| `docs/senline-dogfood-repro-packages.md` | Task 8.7 worker/HTTP dual package (Windows) | +| `docs/senline-dogfood-defects.md` | SGDOG ledger + resource observation | +| `docs/senline-dogfood-support.md` | Authority / promotion boundary | +| `docs/senline-dogfood-evidence.v1.json` | Durable evidence records | + +## Latest dual-host CI + +- core-conformance run + [`29595215669`](https://github.com/Hyper66666/Sengoo/actions/runs/29595215669) + on tip `3e747e63b`: + - core-language + dual-host differential + binary I/O **green** + - installed product loops **green** (worker framed + HTTP plan equality + + `malformed_json`@200 + GET@400) on Ubuntu and Windows + - **worker** dual-package compare **ok=true** (33 identical payloads) both hosts + - **HTTP** dual-package still fails closed on equal-size executable hash + (task **8.7** remains open; product probes still ran) + +## Known open Sengoo-side items + +1. **Task 7.5 / 9.5** — Senline pin advancement requires a writable Senline Git revision (**Blocked** outside this worktree). +2. **Task 7.2 / 7.4 / 7.7** — true red-first defect history + complete pin/green chain not reconstructed; evidence keeps `red_status=pending-commit` and `fixing_commit=null`. +3. **Task 6.5 / 6.6 / 8.3 / 9.1 / 9.2** — reopened after review (product matrix, client-surface scan, process-count/durable soak publication, clippy, installed hash). See `tasks.md`. +4. **Task 8.7** — HTTP dual-package executable hash still diverges under fail-closed compare (worker dual-package is bit-identical on both hosts). +5. **sgfmt comment preservation** — lexer skip + AST rebuild previously dropped comments; RED test + reinject path added (verify before re-closing 9.1). +6. **P2 compiler debt:** ordinary by-value legacy-handle Drop still skipped (language ABI); worker uses product-level owning helpers. + +## Unsupported authority transfers + +Do **not** move into Sengoo without a separate OpenSpec change: + +- TLS / public or internal-alpha ingress +- Cryptography and secret material +- Authentication, replay mutation, prekey claim +- Durable transactions, persistence, migrations +- Final mutation authority diff --git a/docs/senline-dogfood-latency-evidence.md b/docs/senline-dogfood-latency-evidence.md new file mode 100644 index 00000000..67715a00 --- /dev/null +++ b/docs/senline-dogfood-latency-evidence.md @@ -0,0 +1,39 @@ +# Request Latency Evidence Notes (task 8.4) + +Methodology: `docs/senline-dogfood-resource-methodology.md`. + +All sampler percentiles below are **request-write-complete → response-frame-complete** +wall times inside the harness. They are **not** Senline admission or sandbox latency. + +## Checked-in sampler percentiles + +Harness: `tools/sgc/tests/senline_worker_resource.rs`. + +| Host | Label | Post-warm-up samples | p50 µs | p95 µs | p99 µs | Notes | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| Local Windows x64 | smoke-1k residual-fix | 768 | 59 | 110 | 150 | After ownership fixes | +| Local Windows x64 | soak-1m (task 8.3) | 999,744 | 179 | 350 | 450 | 1e6 cases; growth ~0.07 B/case | +| GHA `windows-latest` | smoke-1k run `29573240622` | 768 | 121 | 167 | 196 | CI resource smoke; PWS metric | +| GHA `ubuntu-latest` | smoke-1k run `29573240622` | 768 | 94 | 104 | 111 | CI resource smoke; RSS metric | + +### Mean request time (derived bulk corpora; not pXX) + +From dual-host differential CI run `29424861027` (release corpora): + +| Corpus | Cases | Windows mean µs/req | Linux mean µs/req | +| --- | ---: | ---: | ---: | +| determinism | 512 | ~492 | ~453 | +| reviewed_boundary | 10,000 | ~3,570 | ~3,830 | +| seeded_eligible | 100,000 | ~2,431 | ~2,733 | + +Concurrency for V1 worker evaluation remains **one in-flight request** per worker process. + +## Host labels + +- Local Windows development host: Windows x64 private working set sampler. +- CI: GitHub Actions `windows-latest` / `ubuntu-latest` (resource smoke on tip run `29573240622`). + +## Non-claims + +Do **not** cite these figures as Senline host admission, sandbox spawn, TLS, or +end-to-end product RTT. \ No newline at end of file diff --git a/docs/senline-dogfood-repro-packages.md b/docs/senline-dogfood-repro-packages.md new file mode 100644 index 00000000..1802c8ba --- /dev/null +++ b/docs/senline-dogfood-repro-packages.md @@ -0,0 +1,51 @@ +# Worker / HTTP Package Dual-Build Compare (task 8.7 partial) + +## Scripts + +| Script | Role | +| --- | --- | +| `scripts/package-senline-worker.ps1` | Installed-toolchain worker package + `worker-manifest.json` | +| `scripts/package-senline-http.ps1` | Installed-toolchain HTTP dogfood package + `http-manifest.json` | +| `scripts/compare-senline-package-manifests.ps1` | Normalized dual-manifest compare (optional executable drift) | + +Toolchain dual-build remains covered by `scripts/package-toolchain.ps1` + +`scripts/compare-distribution-manifests.ps1` and CI run `29419695542`. + +## Local Windows x64 evidence (this host) + +Installed toolchain used for packaging: + +- `target/dist/sengoo-0.1.0-senline-dogfood-x86_64-pc-windows-msvc/bin/{sgc,sgpm}.exe` +- `sgc --version`: `sgc 0.1.0 (a96518ddb68f)` (package identity; not the dogfood branch tip) + +### Worker dual package + +```text +package A/B -> target/senline-pkg/worker-{a,b}/ +compare -> target/senline-pkg/worker-compare/comparison.json +result -> ok=true identical_payload_count=31 executable_drift=0 +``` + +All fixtures, lockfiles, docs, and the release worker executable matched across +two consecutive package invocations (second build was a cache hit; still two +package staging trees with independent manifests). + +### HTTP dogfood dual package + +```text +package A/B -> target/senline-pkg/http-{a,b}/ +compare -> target/senline-pkg/http-compare/comparison.json +result -> ok=true identical_payload_count=4 executable_drift=0 +``` + +## Dual-host CI (closes task 8.7 worker/HTTP gap) + +GitHub Actions core-conformance run +[`29430796769`](https://github.com/Hyper66666/Sengoo/actions/runs/29430796769): + +- `installed worker/HTTP (windows-latest)` green +- `installed worker/HTTP (ubuntu-latest)` green +- Artifacts: `senline-installed-packages-windows-x86_64`, + `senline-installed-packages-linux-x86_64` (comparison + manifests) + +Toolchain dual-build remains covered by run `29419695542`. diff --git a/docs/senline-dogfood-resource-evidence.md b/docs/senline-dogfood-resource-evidence.md new file mode 100644 index 00000000..0f1c7765 --- /dev/null +++ b/docs/senline-dogfood-resource-evidence.md @@ -0,0 +1,44 @@ +# Senline Domain Worker Resource Soak Evidence (task 8.3) + +This file is the **durable, checked-in** publication record for resource soak +claims. Large JSONL sample series stay gitignored under +`target/senline-resource/`; green claims must cite a summary SHA-256 and source +revision here (see `docs/senline-dogfood-resource-methodology.md` publication +rules). + +## Tip-era schema v2 1M soak (Windows x64) + +| Field | Value | +| --- | --- | +| Source revision | `2c6f2cce7` (branch tip when soak was produced; re-verify after rebase) | +| Summary path (local) | `target/senline-resource/soak-soak-1m-windows-x86_64-1784302594.summary.json` | +| Summary SHA-256 | `408c61c129c79041602ae7a34f01f71e6ed67ea88ff96776f9caf97ed2cef087` | +| Summary size (bytes) | 2978 | +| JSONL companion | `soak-soak-1m-windows-x86_64-1784302594.jsonl` (gitignored; full series) | +| cases_requested | 1_000_000 | +| cases_completed | 1_000_000 | +| memory.metric | `private_bytes` (Windows `PrivateUsage`) | +| OLS regression slope (B/case) | ≈ −0.03985 | +| endpoint growth (B/case) | ≈ 0.05736 | +| max 10k-window delta (bytes) | 163_840 | +| handles.within_plateau | true | +| process_count (summary field) | 1 *(pre-fix summaries hardcoded this; sampler now measures worker process tree)* | +| zero_failures | true | +| oracle | independent Rust decision/reason match for reviewed-boundary corpus | + +### How to re-verify the published hash + +```powershell +Get-FileHash -Algorithm SHA256 ` + target/senline-resource/soak-soak-1m-windows-x86_64-1784302594.summary.json +``` + +A re-soak under the measured process-tree sampler should replace this table +(with a new summary name + SHA-256) before claiming 8.3 green again. + +## Sampler process-count methodology (post-fix) + +- Windows: `CreateToolhelp32Snapshot` / `Process32FirstW` — count the worker PID + plus processes whose parent PID equals the worker. +- Linux: scan `/proc/*/stat` for `ppid == worker`. +- Gate: every sample must observe `process_tree_count == 1` (no unexpected children). diff --git a/docs/senline-dogfood-resource-methodology.md b/docs/senline-dogfood-resource-methodology.md new file mode 100644 index 00000000..2a5aef61 --- /dev/null +++ b/docs/senline-dogfood-resource-methodology.md @@ -0,0 +1,162 @@ +# Senline Domain Worker Resource & Latency Methodology + +Recorded for OpenSpec tasks **8.3** (resource soak) and **8.4** (latency). +This document is methodology and evidence policy only. It does **not** claim +Senline host admission, sandbox, or production timing. + +## Scope + +| Item | In scope | Out of scope | +| --- | --- | --- | +| Process under test | Single `senline_domain_worker` binary over framed stdio | Multi-worker supervisors, Senline sandbox | +| Workload | Reviewed golden/boundary fixtures + fixed-seed eligible cases | Cryptographic or network I/O | +| Hosts | Recorded Windows x64 and Linux x64 reference runners | Mobile, WASM, cross-compile hosts | +| Memory metric | Windows **private bytes** (`PrivateUsage`); Linux RSS (VmRSS) | Working-set-only counters, cgroup limits, swap policy | +| Latency | Request-to-valid-response wall time inside the harness | Admission, TLS, sandbox spawn | + +## Sampler (checked-in) + +Primary harness: `tools/sgc/tests/senline_worker_resource.rs` +(reviewed-boundary requests + process sampler; companion semantic corpora stay +in `senline_worker_differential.rs`). + +Summary schema version **2** gates: + +| Gate | Field | Default bound | +| --- | --- | --- | +| Endpoint growth | `memory.post_warmup_endpoint_growth_bytes_per_case` | < 1 KiB/case | +| OLS regression | `memory.post_warmup_regression_slope_bytes_per_case` | < 1 KiB/case | +| 10k window | `memory.max_10k_window_delta_bytes` | < +32 MiB | +| Handle plateau | `handles.within_plateau` | warm-up max + 16 | +| Process count | `process_count` | 1 | +| Failures | `failure_count` | 0 | + +Sampling rules: + +1. **Warm-up**: discard the first `N` evaluations (default `N = 256`) before + recording memory/latency series so JIT/cache effects are excluded from the + post-warm-up stability window. +2. **Cadence**: sample process memory, handle/FD count, and cumulative elapsed + time every `K` cases (default `K = 100`). Write the **full** sample series + to a JSONL file: + `{ case_index, elapsed_ms, memory_bytes, handle_count, cases_per_second_window }`. + The summary keeps a 5-point tail for human glance only. +3. **Windows private bytes** (metric name `private_bytes`): + - `GetProcessMemoryInfo` / `PROCESS_MEMORY_COUNTERS_EX.PrivateUsage` + on the worker PID (not the cargo parent). This is **not** WorkingSetSize + and is not labeled “private working set”. +4. **Linux RSS** (metric name `rss_bytes`): + - Read `/proc//status` field `VmRSS` (kB → bytes) for the worker PID. +5. **Handles / FDs**: + - Windows: `GetProcessHandleCount` on the worker process. + - Linux: count entries under `/proc//fd`. +6. **Process count**: always exactly one worker child for single-worker soak; + parent harness processes are not counted toward the worker bound. +7. **Growth math**: + - Endpoint slope: (last − first) / cases (legacy comparison key). + - **Linear regression**: ordinary least-squares slope of `memory_bytes` vs + `case_index` after warm-up (primary stability gate). + - Max delta over any contiguous ~10k-case sample window after warm-up. + +Artifacts land under `target/senline-resource/` (gitignored) with names: +`soak-{label}-{os}-{arch}-{timestamp}.jsonl` and a summary +`soak-{label}-{os}-{arch}-{timestamp}.summary.json`. + +## Task 8.3 success criteria (must all hold) + +1. At least **1,000,000** framed evaluations complete in one continuous + single-worker session **or** a documented failure that preserves the + degradation evidence (do not re-label sharded 5.11 success as soak success). +2. Post-warm-up memory does not show unbounded growth: OLS regression slope + of sampled private bytes / RSS over the post-warm-up window stays within + the reviewed bound (default **< 1 KiB/case**), with no single + ~10k-case window exceeding **+32 MiB**. +3. File/handle/FD counts stay bounded (no climb past a reviewed plateau; + default plateau = warm-up max + 16). +4. Zero crash, hang (watchdog), malformed plan, or nondeterminism relative to + the linked Rust oracle for the corpus used. + +### Known open observation (retain) + +A prior **single-process 100,000-case** run hit the 3600 s watchdog near case +**44,086** while private working set and throughput degraded. That observation +remains open under task 8.3. The sharded 5.11 result +(8 × 12,500, transcript +`16aebd9ec476d602c9c0d0082ee9e25a87c520c333d6dd3afeb314f8c39ea128`) proves +semantic equivalence only, **not** resource stability. + +### 2026-07 residual growth fix (Windows investigation) + +After the lambda Drop fix, investigate-45k still showed ~**92 B/case** growth. +Root cause was application-level: `worker_validate_execution_mode(String)` +consumed a by-value legacy handle without Drop (function params skip auto-Drop +for `String`/`Buffer`/`JsonDoc`). Fixed by validating `&str` and reusing the +single extracted mode string. + +### 2026-07-17 1M soak (Windows x64) — schema v2 oracle gates green + +`resource_single_worker_soak_1m` on tip-era sampler completed +**1,000,000 / 1,000,000** cases (~246 s) with zero failures under schema v2: + +| Gate | Result | +| --- | --- | +| cases_completed | 1_000_000 | +| metric | `private_bytes` (PrivateUsage) | +| OLS regression slope | ≈ −0.040 B/case | +| endpoint growth | ≈ 0.057 B/case | +| max 10k-window delta | 163_840 bytes | +| handles within plateau | true (68) | +| process_count | 1 | +| oracle plan match | every response classified | +| JSONL series | full sample series written | + +Evidence (local, gitignored): +`target/senline-resource/soak-soak-1m-windows-x86_64-1784302594.summary.json` +(+ matching `.jsonl`). Latency p50/p95/p99 ≈ 173/326/547 µs. + +Linux RSS sampling uses the same harness (`rss_bytes` metric) and is exercised +on GHA `ubuntu-latest` via the resource smoke step. + +## Task 8.4 latency methodology + +After the same warm-up: + +1. Measure wall-clock **request write complete → response frame fully read** + for each sample case on the recorded host. +2. Publish payload class (reviewed golden size band vs seeded eligible), + concurrency (**always 1** in-flight request for V1 worker), and host + identity (OS, arch, runner label). +3. Report p50 / p95 / p99 over the post-warm-up sample set plus mean cases/s. +4. Do **not** claim Senline admission latency, sandbox spawn cost, or end-to-end + product RTT. + +## How to run (operator notes) + +```powershell +# Sampler smoke (always-on CI / local; ~1k cases + p50/p95/p99) +cargo test --release --locked -p sgc --test senline_worker_resource -- --nocapture + +# Single-worker investigation near historical case 44086 (~45k, soft watchdog) +cargo test --release --locked -p sgc --test senline_worker_resource resource_single_worker_investigation_50k -- --ignored --nocapture + +# Full 1M soak (only after growth is fixed) +cargo test --release --locked -p sgc --test senline_worker_resource resource_single_worker_soak_1m -- --ignored --nocapture +``` + +External memory sampler example (Windows, attach to worker PID printed by harness): + +```powershell +while ($true) { + $p = Get-Process -Id $WorkerPid -ErrorAction SilentlyContinue + if (-not $p) { break } + "{0}`t{1}" -f (Get-Date -Format o), $p.PrivateMemorySize64 + Start-Sleep -Seconds 1 +} +``` + +## Publication rules + +- Check in this methodology and any durable summary tables under `docs/`. +- Keep multi-GB JSONL series out of git; retain CI artifacts or local + checkpoints with full hashes in the support record when claiming green. +- Never mark 8.3 complete from multi-process shards alone. diff --git a/docs/senline-dogfood-support.md b/docs/senline-dogfood-support.md new file mode 100644 index 00000000..93b04c7f --- /dev/null +++ b/docs/senline-dogfood-support.md @@ -0,0 +1,65 @@ +# Senline Dogfood Support Record + +Recorded: 2026-07-15 + +This record separates behavior demonstrated by Sengoo package tests from +authority and promotion behavior that only the Senline Rust host can prove. +It is deliberately target-specific. A skipped platform or installed loop is +pending, never inferred from a source-development run on another host. + +## Package Evidence + +| Evidence scope | Status | Current evidence | Explicit limit | +| --- | --- | --- | --- | +| source-development-local | proven | On the recorded Windows development host, `senline-domain-worker` passes its locked package tests, real binary parent/child fixtures, strict JSON rejection/recovery, partial I/O, deterministic error, leakage-canary, and Buffer-lifecycle regressions. The HTTP package passes locked check/test/build and one local synthetic worker/HTTP byte-equivalence run. | This is not Senline pin evidence and does not prove an installed archive, Linux behavior, or the complete localhost matrix. | +| installed-windows-x64 | package-smoke-proven | GitHub Actions run `29419695542` package smoke + run `29430796769` installed worker/HTTP product loop (fake cargo, dual package compare). | Installed `senline-domain-worker` / HTTP product loops are CI-proven on this host family; reviewed Senline pin and 1M single-worker soak remain pending. | +| installed-linux-x64 | package-smoke-proven | Same distribution package smoke + run `29430796769` Ubuntu installed worker/HTTP product loop green. | Installed worker/HTTP product loops are CI-proven on this host family; reviewed Senline pin and 1M single-worker soak remain pending. | +| `senline-domain-worker` package | installed-loop-proven | Dual-host installed `sgpm --runtime-mode installed check/test/build --locked` + dual package manifests on run `29430796769`. | Immutable consumer pin and Senline-side gates remain pending. | +| `senline-http-dogfood` package | installed-loop-proven | Dual-host installed locked HTTP tests + dual package compare on run `29430796769`; loopback-only non-ingress limits retained. | Senline pin / production promotion remain forbidden claims. | + +The local distribution manifest remains dirty/prebuilt and records +`release_eligible=false`. It cannot be promoted by renaming it, copying it to +another directory, or setting a self-asserted flag. The complete installed +manifest, payload hashes, clean source revision, and consumer-side verification +must agree before any status above changes to proven. + +## Authority Boundary + +| Claim | Ownership | Sengoo-side status | +| --- | --- | --- | +| planner transport and pure fixture evaluation | Sengoo package | Proven only for the local source-development evidence listed above. | +| sandbox and supervisor | Senline-owned | Senline Rust must prove process isolation, limits, restarts, deadlines, stderr draining, and whole-tree cleanup. | +| shadow | Senline-owned | Senline Rust remains authoritative and owns differential evidence and every mutation. | +| guarded-development | Senline-owned | Senline Rust owns promotion records, agreement checks, fail-closed behavior, and epoch changes. | +| internal-alpha | Senline-owned | Senline Rust owns admission, automatic demotion, stale-result rejection, and rollback eligibility. | +| rollback | Senline-owned | Senline Rust owns the switch, epoch increment, worker termination, and authoritative Rust fallback. | +| production ingress | Senline-owned | The HTTP dogfood harness is never TLS, public ingress, internal-alpha routing, or a client endpoint. | + +TLS, cryptography, signed-request verification, freshness/replay, device +authorization and revocation, rate limits, durable transactions, persistence, +migrations, final plan validation, and all mutation remain in Senline Rust. +Moving any one of them requires a separate reviewed OpenSpec change. + +The authority-transfer gate is explicit: TLS, cryptography, authentication authority, +replay mutation, prekey claim, durable transactions, persistence, migrations, +public ingress, internal-alpha ingress, and final mutation authority cannot move +into Sengoo under this change. Each transfer requires a separate reviewed OpenSpec +change with its own threat model, compatibility contract, implementation plan, +and verification evidence. + +## Promotion Rules + +The following are independent gates and cannot substitute for one another: + +1. A locked source-development loop detects package and language regressions. +2. A clean target-specific installed archive proves checkout-independent + resolution, linking, and execution with fake-failing Cargo. +3. A reproducibility comparison proves two builds of the same clean revision + have the required identical payload and runtime identities. +4. Senline independently verifies the immutable bundle and advances its pin. +5. Senline reruns the linked differential, leakage, malformed-output, + containment, resource, and rollback gates. + +Until all applicable gates are recorded, terms such as sandboxed, supervised, +shadow-ready, guarded, internal-alpha-ready, rollback-proven, or production +supported are not Sengoo support claims. diff --git a/docs/senline-library-incubation-evidence.md b/docs/senline-library-incubation-evidence.md new file mode 100644 index 00000000..16ff0bc9 --- /dev/null +++ b/docs/senline-library-incubation-evidence.md @@ -0,0 +1,75 @@ +# Senline Library Incubation Evidence + +Recorded: 2026-07-14 + +This record covers the first source-development loop for the reusable packages +incubated by `senline-domain-worker`. It is development evidence only. It is +not installed-toolchain, cross-platform, stable-release, or stdlib-graduation +evidence. + +## sgframing 0.1.0 + +- Scope: bounded `u32` big-endian framing over exact binary standard I/O. +- Product names/defaults in API: none; Senline limits remain in the worker. +- Lock: package-local `Sengoo.lock`, current. +- Package tests: `2 passed`. +- Real binary-pipe harness: valid/control-byte/max-boundary frames echo exactly; + clean EOF, every four-byte-prefix split, every four-byte-payload truncation, + zero length, and over-limit length produce the expected bounded outcome with + empty rejected stdout/stderr. +- Source-development gates: check, fmt-check, doc, release build, and publish + dry run passed. +- Consumer-discovered runtime regression: `SGDOG-2026-006` makes empty Buffer + free/drop a no-op so clean EOF cannot pollute FFI error state. +- Dry-run archive SHA-256: + `2de8fd4203b015b15bb0f13172e961cd3adef2bf49bbcab8280d30cc12595151`. +- Known gaps: broken-pipe/flush-failure injection, installed Windows/Linux, + package license metadata/files, registry publication, and + independent-consumer evidence remain absent. +- Stability: incubating; no `1.0` or `std::` claim. + +## sgjson_contract 0.1.0 + +- Scope: exact closed-object composition, required typed getters, integer + bounds, ASCII/hex, closed enums, and sorted-unique string arrays over strict + `std::json` documents. +- Product names/defaults in API: none; V1 DTOs remain in product packages. +- Lock: package-local `Sengoo.lock`, current. +- Package tests: `7 passed`, including negative object, scalar, nested, array, + and stale-runtime-handle classification cases. +- Source-development gates: check, fmt-check, doc, release build, and publish + dry run passed. +- Dry-run archive SHA-256: + `f8e35a4ac52826d7df35e1e0e06bbbe9cedd1c69ec04a610cbb962b95681436c`. +- Consumer-discovered runtime regression: `SGDOG-2026-007` covers container + corruption when parsed JSON grows beyond the initial 16-node allocation. +- Protocol/runtime regressions: `SGDOG-2026-009` adds stable strict-parser + error kinds, and `SGDOG-2026-010` adds checked length-aware owned-String + building with embedded-NUL preservation. +- Runtime lifecycle regression: 64 permissive-plus-strict parse/close cycles + restore the JSON document live-handle count after every round. +- Known gaps: parser provenance cannot be recovered from `JsonValue`; callers + must supply a live strict document. Runtime fault injection, malformed + corpus, long soak, installed Windows/Linux, package license metadata/files, + registry publication, and independent-consumer evidence remain absent. +- Stability: incubating; no `1.0` or `std::` claim. + +## Consumer Integration + +The worker root lock contains both packages. Its source-development package +loop passes two worker tests plus three product-planner tests. Real +parent/child execution emits the frozen handshake, processes all five V1 +fixtures with byte-exact frames, classifies duplicate/invalid-Unicode/trailing +parser errors, recovers after parser and schema rejections, preserves an +embedded NUL and suffix in an echoed identifier, and shuts down cleanly on EOF. +The complete realworld harness passes 13 tests. The reusable packages own +framing and contract primitives; Senline DTOs and policy remain in the product +package. The same consumer path minimized and fixed compiler regressions +`SGDOG-2026-011` (early-return move-state reachability) and +`SGDOG-2026-012` (nested field references producing invalid LLVM). The worker +uses the direct nested immutable borrow, so no compiler workaround remains. +Installed Windows/Linux loops and immutable build-info injection remain +pending. + +Required gates are never converted to green when skipped. Archive hashes above +identify local dry-run contents only and are not Senline pin evidence. diff --git a/examples/realworld/senline-domain-worker/README.md b/examples/realworld/senline-domain-worker/README.md new file mode 100644 index 00000000..dbcce448 --- /dev/null +++ b/examples/realworld/senline-domain-worker/README.md @@ -0,0 +1,52 @@ +# Senline Domain Worker + +This realworld package is the Sengoo side of the linked Senline +`adopt-sengoo-backend-slice` change. The root package will own bounded framed +stdio and exhaustive V1 decoding. `senline_facts_to_plan` is the pure planner +module shared with the later loopback HTTP dogfood package. + +Domain-neutral capabilities are incubated beside the first consumer: + +- `sgframing` owns bounded big-endian framing and exact stdio composition. +- `sgjson_contract` owns closed-object and typed validation helpers over + strict `std::json` documents. +- `senline_facts_to_plan` remains product-specific and is not a stdlib + candidate. +- `senline_build_identity` is a generated product package that embeds startup + consistency values from reviewed bundle inputs. + +The source-development worker now consumes both incubating packages in a real +binary stdin/stdout loop. It strictly decodes the complete V1 context and +facts DTOs, evaluates the product planner, emits deterministic plans, returns +the frozen unsupported-version and bounded protocol errors, recovers after +each rejected request, and shuts down cleanly on EOF. Real parent/child tests +cover all five frozen request/response cases plus schema and strict-parser +recovery in a single process per test. + +The worker has no TLS, authentication, replay, cryptography, persistence, +transaction, clock, randomness, filesystem, environment, network, subprocess, +or mutation authority. Senline Rust verifies minimum-necessary facts, computes +the facts binding, validates every returned plan, re-reads mutable state in a +new transaction, and remains the only mutation authority. + +The checked-in `fixtures/v1` corpus is the byte-frozen contract source. Stdout +is protocol-only. Plan `sengoo_module_revision` identifies the frozen planner +contract fixture revision. `scripts/generate-build-identity.ps1` +deterministically writes both the `senline_build_identity` source and external +handshake JSON from the source revision, toolchain/application versions, and +bundle build-manifest identity. The checked-in generated values are +fixture-mode inputs and are not pin evidence. Release packaging must regenerate +them from its reviewed manifest; Senline still verifies every external bundle +file and rejects any self-reported identity mismatch. + +Release-mode stderr has an empty allowlist: request bytes, parser text, field +values, and arbitrary messages are never emitted. Host-owned exit status and +bounded error envelopes carry stable failure categories. Development metadata +requires a separate reviewed allowlist before it may appear on stderr. + +Strict JSON exposes stable machine-readable kinds for duplicate fields, +invalid Unicode, trailing input, and unclassified syntax. The worker snapshots +that kind immediately after a failed parse, before creating an error document, +and never parses diagnostic text. Length-aware JSON building preserves owned +strings containing `U+0000`; checked-in raw malformed fixtures and process +tests lock the subtype mapping and recovery behavior. diff --git a/examples/realworld/senline-domain-worker/Sengoo.lock b/examples/realworld/senline-domain-worker/Sengoo.lock new file mode 100644 index 00000000..79be75c4 --- /dev/null +++ b/examples/realworld/senline-domain-worker/Sengoo.lock @@ -0,0 +1,63 @@ +# This file is generated by sgpm update. +version = 2 +root = "senline_domain_worker" + +[[package]] +id = "senline_build_identity@0.1.0+path:packages/senline-build-identity" +name = "senline_build_identity" +version = "0.1.0" +source.kind = "path" +source.path = "packages/senline-build-identity" +manifest = "packages/senline-build-identity/Sengoo.toml" + +[[package]] +id = "senline_facts_to_plan@0.1.0+path:packages/senline-facts-to-plan" +name = "senline_facts_to_plan" +version = "0.1.0" +source.kind = "path" +source.path = "packages/senline-facts-to-plan" +manifest = "packages/senline-facts-to-plan/Sengoo.toml" + +[[package]] +id = "sgframing@0.1.0+path:packages/sgframing" +name = "sgframing" +version = "0.1.0" +source.kind = "path" +source.path = "packages/sgframing" +manifest = "packages/sgframing/Sengoo.toml" + +[[package]] +id = "sgjson_contract@0.1.0+path:packages/sgjson-contract" +name = "sgjson_contract" +version = "0.1.0" +source.kind = "path" +source.path = "packages/sgjson-contract" +manifest = "packages/sgjson-contract/Sengoo.toml" + +[[package]] +id = "senline_domain_worker@0.1.0+path:." +name = "senline_domain_worker" +version = "0.1.0" +source.kind = "path" +source.path = "." +manifest = "Sengoo.toml" + +[[dependency]] +from = "senline_domain_worker@0.1.0+path:." +alias = "senline_build_identity" +to = "senline_build_identity@0.1.0+path:packages/senline-build-identity" + +[[dependency]] +from = "senline_domain_worker@0.1.0+path:." +alias = "senline_facts_to_plan" +to = "senline_facts_to_plan@0.1.0+path:packages/senline-facts-to-plan" + +[[dependency]] +from = "senline_domain_worker@0.1.0+path:." +alias = "sgframing" +to = "sgframing@0.1.0+path:packages/sgframing" + +[[dependency]] +from = "senline_domain_worker@0.1.0+path:." +alias = "sgjson_contract" +to = "sgjson_contract@0.1.0+path:packages/sgjson-contract" diff --git a/examples/realworld/senline-domain-worker/Sengoo.toml b/examples/realworld/senline-domain-worker/Sengoo.toml new file mode 100644 index 00000000..e433167e --- /dev/null +++ b/examples/realworld/senline-domain-worker/Sengoo.toml @@ -0,0 +1,17 @@ +[package] +name = "senline_domain_worker" +version = "0.1.0" +edition = "2026" +description = "Bounded framed Senline domain planner worker dogfooding Sengoo." + +[bin] +path = "src/main.sg" + +[lib] +path = "src/lib.sg" + +[dependencies] +senline_build_identity = { path = "packages/senline-build-identity" } +sgframing = { path = "packages/sgframing" } +sgjson_contract = { path = "packages/sgjson-contract" } +senline_facts_to_plan = { path = "packages/senline-facts-to-plan" } diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/cases/application-budget-rejection.plan.json b/examples/realworld/senline-domain-worker/fixtures/v1/cases/application-budget-rejection.plan.json new file mode 100644 index 00000000..95b8b3f9 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/cases/application-budget-rejection.plan.json @@ -0,0 +1 @@ +{"kind":"plan","schema_version":1,"context":{"contract_version":1,"operation":"submit-envelope","operation_version":1,"evaluation_id":"00000000000000000000000000000004","operation_epoch":7,"worker_generation":3,"execution_mode":"fixture","worker_bundle_id":"senline-worker-fixture-v1","facts_binding":"0d1d29060950587f64fa934bb46415022b7c9d600ff717ae4c488e748a4bf008"},"identifiers":{"correlation_ref":"corr_ref_budget_004","source_account_ref":"acct_ref_source_004","source_device_ref":"device_ref_source_004","recipient_account_ref":"acct_ref_recipient_004","recipient_device_ref":"device_ref_recipient_004","conversation_ref":"conversation_ref_004","envelope_ref":"envelope_ref_004"},"decision":"reject","reason":"application_budget_exhausted","sengoo_module_revision":"1de09ccafa7e8f182af68e82352e2d4be39496b0"} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/cases/application-budget-rejection.request.json b/examples/realworld/senline-domain-worker/fixtures/v1/cases/application-budget-rejection.request.json new file mode 100644 index 00000000..0043f5cd --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/cases/application-budget-rejection.request.json @@ -0,0 +1 @@ +{"kind":"evaluation","schema_version":1,"context":{"contract_version":1,"operation":"submit-envelope","operation_version":1,"evaluation_id":"00000000000000000000000000000004","operation_epoch":7,"worker_generation":3,"execution_mode":"fixture","worker_bundle_id":"senline-worker-fixture-v1","facts_binding":"0d1d29060950587f64fa934bb46415022b7c9d600ff717ae4c488e748a4bf008"},"facts":{"contract_version":1,"operation_version":1,"identifiers":{"correlation_ref":"corr_ref_budget_004","source_account_ref":"acct_ref_source_004","source_device_ref":"device_ref_source_004","recipient_account_ref":"acct_ref_recipient_004","recipient_device_ref":"device_ref_recipient_004","conversation_ref":"conversation_ref_004","envelope_ref":"envelope_ref_004"},"source_device_status":"active","source_device_capabilities":["submit_envelope_v2"],"envelope_protocol_version":2,"ciphertext_length_bytes":1536,"idempotency_status":"new","recipient_pending_count":15,"recipient_pending_limit":1000,"application_envelopes_used":10000,"application_envelopes_limit":10000,"ciphertext_limit_bytes":65536,"feature_flags":["enqueue_delivery"]}} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/cases/eligible-accept.plan.json b/examples/realworld/senline-domain-worker/fixtures/v1/cases/eligible-accept.plan.json new file mode 100644 index 00000000..2bf33644 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/cases/eligible-accept.plan.json @@ -0,0 +1 @@ +{"kind":"plan","schema_version":1,"context":{"contract_version":1,"operation":"submit-envelope","operation_version":1,"evaluation_id":"00000000000000000000000000000001","operation_epoch":7,"worker_generation":3,"execution_mode":"fixture","worker_bundle_id":"senline-worker-fixture-v1","facts_binding":"09d0b4fcae5f3026de32cc41f9aa51c15fe9b40e379062c00de984cfffe93302"},"identifiers":{"correlation_ref":"corr_ref_eligible_001","source_account_ref":"acct_ref_source_001","source_device_ref":"device_ref_source_001","recipient_account_ref":"acct_ref_recipient_001","recipient_device_ref":"device_ref_recipient_001","conversation_ref":"conversation_ref_001","envelope_ref":"envelope_ref_001"},"decision":"store_and_enqueue","reason":"accepted_new","sengoo_module_revision":"1de09ccafa7e8f182af68e82352e2d4be39496b0"} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/cases/eligible-accept.request.json b/examples/realworld/senline-domain-worker/fixtures/v1/cases/eligible-accept.request.json new file mode 100644 index 00000000..ea5a9db9 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/cases/eligible-accept.request.json @@ -0,0 +1 @@ +{"kind":"evaluation","schema_version":1,"context":{"contract_version":1,"operation":"submit-envelope","operation_version":1,"evaluation_id":"00000000000000000000000000000001","operation_epoch":7,"worker_generation":3,"execution_mode":"fixture","worker_bundle_id":"senline-worker-fixture-v1","facts_binding":"09d0b4fcae5f3026de32cc41f9aa51c15fe9b40e379062c00de984cfffe93302"},"facts":{"contract_version":1,"operation_version":1,"identifiers":{"correlation_ref":"corr_ref_eligible_001","source_account_ref":"acct_ref_source_001","source_device_ref":"device_ref_source_001","recipient_account_ref":"acct_ref_recipient_001","recipient_device_ref":"device_ref_recipient_001","conversation_ref":"conversation_ref_001","envelope_ref":"envelope_ref_001"},"source_device_status":"active","source_device_capabilities":["submit_envelope_v2"],"envelope_protocol_version":2,"ciphertext_length_bytes":512,"idempotency_status":"new","recipient_pending_count":12,"recipient_pending_limit":1000,"application_envelopes_used":41,"application_envelopes_limit":10000,"ciphertext_limit_bytes":65536,"feature_flags":["enqueue_delivery"]}} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/cases/exact-duplicate.plan.json b/examples/realworld/senline-domain-worker/fixtures/v1/cases/exact-duplicate.plan.json new file mode 100644 index 00000000..6f98f406 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/cases/exact-duplicate.plan.json @@ -0,0 +1 @@ +{"kind":"plan","schema_version":1,"context":{"contract_version":1,"operation":"submit-envelope","operation_version":1,"evaluation_id":"00000000000000000000000000000002","operation_epoch":7,"worker_generation":3,"execution_mode":"fixture","worker_bundle_id":"senline-worker-fixture-v1","facts_binding":"c4d05f19aaa1253d4c801b826dfd6e29bbacaaa32ee9341b6af9cb20ee6189da"},"identifiers":{"correlation_ref":"corr_ref_duplicate_002","source_account_ref":"acct_ref_source_002","source_device_ref":"device_ref_source_002","recipient_account_ref":"acct_ref_recipient_002","recipient_device_ref":"device_ref_recipient_002","conversation_ref":"conversation_ref_002","envelope_ref":"envelope_ref_002"},"decision":"duplicate_noop","reason":"exact_duplicate","sengoo_module_revision":"1de09ccafa7e8f182af68e82352e2d4be39496b0"} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/cases/exact-duplicate.request.json b/examples/realworld/senline-domain-worker/fixtures/v1/cases/exact-duplicate.request.json new file mode 100644 index 00000000..87d832be --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/cases/exact-duplicate.request.json @@ -0,0 +1 @@ +{"kind":"evaluation","schema_version":1,"context":{"contract_version":1,"operation":"submit-envelope","operation_version":1,"evaluation_id":"00000000000000000000000000000002","operation_epoch":7,"worker_generation":3,"execution_mode":"fixture","worker_bundle_id":"senline-worker-fixture-v1","facts_binding":"c4d05f19aaa1253d4c801b826dfd6e29bbacaaa32ee9341b6af9cb20ee6189da"},"facts":{"contract_version":1,"operation_version":1,"identifiers":{"correlation_ref":"corr_ref_duplicate_002","source_account_ref":"acct_ref_source_002","source_device_ref":"device_ref_source_002","recipient_account_ref":"acct_ref_recipient_002","recipient_device_ref":"device_ref_recipient_002","conversation_ref":"conversation_ref_002","envelope_ref":"envelope_ref_002"},"source_device_status":"active","source_device_capabilities":["submit_envelope_v2"],"envelope_protocol_version":2,"ciphertext_length_bytes":768,"idempotency_status":"exact_duplicate","recipient_pending_count":13,"recipient_pending_limit":1000,"application_envelopes_used":42,"application_envelopes_limit":10000,"ciphertext_limit_bytes":65536,"feature_flags":["enqueue_delivery"]}} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/cases/idempotency-conflict.plan.json b/examples/realworld/senline-domain-worker/fixtures/v1/cases/idempotency-conflict.plan.json new file mode 100644 index 00000000..ef619b51 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/cases/idempotency-conflict.plan.json @@ -0,0 +1 @@ +{"kind":"plan","schema_version":1,"context":{"contract_version":1,"operation":"submit-envelope","operation_version":1,"evaluation_id":"00000000000000000000000000000003","operation_epoch":7,"worker_generation":3,"execution_mode":"fixture","worker_bundle_id":"senline-worker-fixture-v1","facts_binding":"703aadbfb24b7d49a10c2d6a9da00cdf579b5f512bcbf991ed854764f05e97ab"},"identifiers":{"correlation_ref":"corr_ref_conflict_003","source_account_ref":"acct_ref_source_003","source_device_ref":"device_ref_source_003","recipient_account_ref":"acct_ref_recipient_003","recipient_device_ref":"device_ref_recipient_003","conversation_ref":"conversation_ref_003","envelope_ref":"envelope_ref_003"},"decision":"reject","reason":"idempotency_conflict","sengoo_module_revision":"1de09ccafa7e8f182af68e82352e2d4be39496b0"} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/cases/idempotency-conflict.request.json b/examples/realworld/senline-domain-worker/fixtures/v1/cases/idempotency-conflict.request.json new file mode 100644 index 00000000..e3d187a6 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/cases/idempotency-conflict.request.json @@ -0,0 +1 @@ +{"kind":"evaluation","schema_version":1,"context":{"contract_version":1,"operation":"submit-envelope","operation_version":1,"evaluation_id":"00000000000000000000000000000003","operation_epoch":7,"worker_generation":3,"execution_mode":"fixture","worker_bundle_id":"senline-worker-fixture-v1","facts_binding":"703aadbfb24b7d49a10c2d6a9da00cdf579b5f512bcbf991ed854764f05e97ab"},"facts":{"contract_version":1,"operation_version":1,"identifiers":{"correlation_ref":"corr_ref_conflict_003","source_account_ref":"acct_ref_source_003","source_device_ref":"device_ref_source_003","recipient_account_ref":"acct_ref_recipient_003","recipient_device_ref":"device_ref_recipient_003","conversation_ref":"conversation_ref_003","envelope_ref":"envelope_ref_003"},"source_device_status":"active","source_device_capabilities":["submit_envelope_v2"],"envelope_protocol_version":2,"ciphertext_length_bytes":1024,"idempotency_status":"conflict","recipient_pending_count":14,"recipient_pending_limit":1000,"application_envelopes_used":43,"application_envelopes_limit":10000,"ciphertext_limit_bytes":65536,"feature_flags":["enqueue_delivery"]}} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/cases/unknown-operation-version.error.json b/examples/realworld/senline-domain-worker/fixtures/v1/cases/unknown-operation-version.error.json new file mode 100644 index 00000000..d04ab178 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/cases/unknown-operation-version.error.json @@ -0,0 +1 @@ +{"kind":"error","schema_version":1,"scope":"evaluation","code":"unsupported_operation_version","evaluation_id":"00000000000000000000000000000005"} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/cases/unknown-operation-version.request.json b/examples/realworld/senline-domain-worker/fixtures/v1/cases/unknown-operation-version.request.json new file mode 100644 index 00000000..2b73ec51 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/cases/unknown-operation-version.request.json @@ -0,0 +1 @@ +{"kind":"evaluation","schema_version":1,"context":{"contract_version":1,"operation":"submit-envelope","operation_version":99,"evaluation_id":"00000000000000000000000000000005","operation_epoch":7,"worker_generation":3,"execution_mode":"fixture","worker_bundle_id":"senline-worker-fixture-v1","facts_binding":"a41e656d5a140eb110980407a69d4361a7ac9a7001e207113aba0ed14716119f"},"facts":{"contract_version":1,"operation_version":99,"identifiers":{"correlation_ref":"corr_ref_version_005","source_account_ref":"acct_ref_source_005","source_device_ref":"device_ref_source_005","recipient_account_ref":"acct_ref_recipient_005","recipient_device_ref":"device_ref_recipient_005","conversation_ref":"conversation_ref_005","envelope_ref":"envelope_ref_005"},"source_device_status":"active","source_device_capabilities":["submit_envelope_v2"],"envelope_protocol_version":2,"ciphertext_length_bytes":2048,"idempotency_status":"new","recipient_pending_count":16,"recipient_pending_limit":1000,"application_envelopes_used":44,"application_envelopes_limit":10000,"ciphertext_limit_bytes":65536,"feature_flags":["enqueue_delivery"]}} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/differential-corpus-v1.json b/examples/realworld/senline-domain-worker/fixtures/v1/differential-corpus-v1.json new file mode 100644 index 00000000..455b5907 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/differential-corpus-v1.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "reference_kind": "independent_rust_oracle", + "reference_scope": "linked_to_frozen_rust_fixtures_not_senline_production_reference", + "frozen_fixture_metadata_sha256": "a2a503f3ba0ef02d47019c8178d1442ec5d0ef0c0f44ac3441469873d28055e3", + "planner_contract_fixture_revision": "1de09ccafa7e8f182af68e82352e2d4be39496b0", + "generator": { + "name": "senline-worker-differential-v1", + "version": 1, + "algorithm": "reviewed-boundary-v1 plus SplitMix64", + "fixed_seed_hex": "0x6a09e667f3bcc909" + }, + "corpora": { + "determinism": { + "count": 512, + "fresh_processes": 2, + "transcript_sha256": "bd6acd82479bd6219cbf8e96601313e79f01bb518cee5a98f137be3e40f9729c" + }, + "reviewed_boundary": { + "count": 10000, + "transcript_sha256": "a32f445f38e4810bc3eab9f2744ed337f48e2f5fa18521a9b05002c42126dd0b" + }, + "seeded_eligible": { + "count": 100000, + "fresh_processes": 8, + "cases_per_process": 12500, + "transcript_sha256": "16aebd9ec476d602c9c0d0082ee9e25a87c520c333d6dd3afeb314f8c39ea128" + } + }, + "coverage": { + "decisions": ["store_and_enqueue", "duplicate_noop", "reject"], + "reasons": ["accepted_new", "exact_duplicate", "idempotency_conflict", "recipient_queue_full", "application_budget_exhausted", "delivery_disabled"], + "queue_boundaries": ["below", "equal", "above"], + "application_boundaries": ["below", "equal", "above"], + "capability_states": [false, true], + "feature_flag_states": [false, true], + "execution_modes": ["fixture", "shadow", "guarded-development", "internal-alpha"], + "opaque_ascii_ref_lengths": [1, 128], + "numeric_boundaries": [0, 1, 4294967295, 9007199254740991] + }, + "ci_targets": ["x86_64-pc-windows-msvc", "x86_64-unknown-linux-gnu"] +} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/docs/generated/protocol-v1.md b/examples/realworld/senline-domain-worker/fixtures/v1/docs/generated/protocol-v1.md new file mode 100644 index 00000000..1a669692 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/docs/generated/protocol-v1.md @@ -0,0 +1,33 @@ +# Senline Worker Protocol V1 + +Generated from `fixtures/v1/metadata.json`. Manual changes must keep the fixture validator green. + +- Frame: `u32_big_endian` length plus UTF-8 JSON +- Input maximum: `32768` bytes +- Output maximum: `8192` bytes +- In flight: `1` +- Stdout: `protocol_only` +- Opaque ASCII refs and worker bundle IDs: `1..128` bytes +- `u32` binding fields: `0..4294967295` +- Epoch and worker generation: `0..9007199254740991` + +EvaluationContextV1 fields: `contract_version`, `operation`, `operation_version`, `evaluation_id`, `operation_epoch`, `worker_generation`, `execution_mode`, `worker_bundle_id`, `facts_binding`. + +SubmitEnvelopeFactsV1 fields: `contract_version`, `operation_version`, `identifiers`, `source_device_status`, `source_device_capabilities`, `envelope_protocol_version`, `ciphertext_length_bytes`, `idempotency_status`, `recipient_pending_count`, `recipient_pending_limit`, `application_envelopes_used`, `application_envelopes_limit`, `ciphertext_limit_bytes`, `feature_flags`. + +Identifier fields: `correlation_ref`, `source_account_ref`, `source_device_ref`, `recipient_account_ref`, `recipient_device_ref`, `conversation_ref`, `envelope_ref`. + +Rust computes `facts_binding` from the typed V1 encoding. Sengoo only echoes it. The startup `build_manifest_id` is a consistency value, not artifact trust evidence. + +`sengoo_module_revision` is the 40-character lowercase-hex planner contract fixture revision. +It is stable for a frozen planner contract and +does not attest the running binary. `sengoo_source_revision` in the startup +handshake identifies the immutable bundle source revision verified by Rust. + +Worker protocol errors contain only `kind`, `schema_version`, `scope`, `code`, +and nullable `evaluation_id`. Strict parser kinds map to `duplicate_field`, +`invalid_unicode`, or `trailing_bytes`; unclassified syntax maps to +`malformed_json`. Exhaustive schema decoding maps unknown fields and enums to +`unknown_field` and `unknown_enum`; all other schema rejections map to +`malformed_json`. Every request-level rejection leaves the worker ready for +the next frame. diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-duplicate-field.json b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-duplicate-field.json new file mode 100644 index 00000000..2a581aea --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-duplicate-field.json @@ -0,0 +1 @@ +{"kind":"error","schema_version":1,"scope":"protocol","code":"duplicate_field","evaluation_id":null} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-duplicate-field.request.raw b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-duplicate-field.request.raw new file mode 100644 index 00000000..c0e9574b --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-duplicate-field.request.raw @@ -0,0 +1 @@ +{"kind":"evaluation","kind":"evaluation"} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-invalid-unicode.json b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-invalid-unicode.json new file mode 100644 index 00000000..26d1b935 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-invalid-unicode.json @@ -0,0 +1 @@ +{"kind":"error","schema_version":1,"scope":"protocol","code":"invalid_unicode","evaluation_id":null} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-invalid-unicode.request.raw b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-invalid-unicode.request.raw new file mode 100644 index 00000000..fbb00458 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-invalid-unicode.request.raw @@ -0,0 +1 @@ +{"kind":"\ud83d"} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-malformed-json.json b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-malformed-json.json new file mode 100644 index 00000000..6b9377f5 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-malformed-json.json @@ -0,0 +1 @@ +{"kind":"error","schema_version":1,"scope":"protocol","code":"malformed_json","evaluation_id":null} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-trailing-bytes.json b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-trailing-bytes.json new file mode 100644 index 00000000..603289fc --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-trailing-bytes.json @@ -0,0 +1 @@ +{"kind":"error","schema_version":1,"scope":"protocol","code":"trailing_bytes","evaluation_id":null} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-trailing-bytes.request.raw b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-trailing-bytes.request.raw new file mode 100644 index 00000000..f7499170 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-trailing-bytes.request.raw @@ -0,0 +1 @@ +{}x diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-unknown-enum.json b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-unknown-enum.json new file mode 100644 index 00000000..3b1423b3 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-unknown-enum.json @@ -0,0 +1 @@ +{"kind":"error","schema_version":1,"scope":"protocol","code":"unknown_enum","evaluation_id":null} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-unknown-field.json b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-unknown-field.json new file mode 100644 index 00000000..5f23e090 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/errors/protocol-unknown-field.json @@ -0,0 +1 @@ +{"kind":"error","schema_version":1,"scope":"protocol","code":"unknown_field","evaluation_id":null} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/handshake/ready.json b/examples/realworld/senline-domain-worker/fixtures/v1/handshake/ready.json new file mode 100644 index 00000000..f1dff14f --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/handshake/ready.json @@ -0,0 +1 @@ +{"kind":"handshake","protocol_version":1,"sengoo_source_revision":"1de09ccafa7e8f182af68e82352e2d4be39496b0","toolchain_version":"0.1.0","application_version":"0.1.0","build_manifest_id":"1111111111111111111111111111111111111111111111111111111111111111"} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/metadata.json b/examples/realworld/senline-domain-worker/fixtures/v1/metadata.json new file mode 100644 index 00000000..db8d7001 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/metadata.json @@ -0,0 +1,73 @@ +{ + "fixture_set_version": 1, + "linked_changes": { + "sengoo": "senline-service-dogfood", + "senline": "adopt-sengoo-backend-slice", + "sengoo_baseline_revision": "1de09ccafa7e8f182af68e82352e2d4be39496b0" + }, + "protocol": { + "input_max_bytes": 32768, + "output_max_bytes": 8192, + "max_in_flight": 1, + "stdout": "protocol_only", + "frame_prefix": "u32_big_endian" + }, + "bounds": { + "opaque_ascii_ref_bytes": {"min": 1, "max": 128}, + "evaluation_id_lower_hex_bytes": 32, + "facts_binding_lower_hex_bytes": 64, + "sengoo_module_revision_lower_hex_bytes": 40, + "u32_field_max": 4294967295, + "json_safe_u64_field_max": 9007199254740991, + "source_device_capabilities_items": {"min": 0, "max": 1}, + "feature_flags_items": {"min": 0, "max": 1} + }, + "revision_semantics": { + "sengoo_module_revision": "planner_contract_fixture_revision", + "sengoo_source_revision": "immutable_bundle_source_revision" + }, + "binding": { + "algorithm": "sha256", + "encoding": "senline.submit-envelope.binding.v1 typed-big-endian", + "context_fields": ["contract_version", "operation", "operation_version", "evaluation_id", "operation_epoch", "worker_generation", "execution_mode", "worker_bundle_id"], + "facts_fields": ["contract_version", "operation_version", "identifiers", "source_device_status", "source_device_capabilities", "envelope_protocol_version", "ciphertext_length_bytes", "idempotency_status", "recipient_pending_count", "recipient_pending_limit", "application_envelopes_used", "application_envelopes_limit", "ciphertext_limit_bytes", "feature_flags"], + "identifier_fields": ["correlation_ref", "source_account_ref", "source_device_ref", "recipient_account_ref", "recipient_device_ref", "conversation_ref", "envelope_ref"] + }, + "enums": { + "execution_mode": ["fixture", "shadow", "guarded-development", "internal-alpha"], + "source_device_status": ["active"], + "source_device_capabilities": ["submit_envelope_v2"], + "idempotency_status": ["new", "exact_duplicate", "conflict"], + "feature_flags": ["enqueue_delivery"], + "decision": ["store_and_enqueue", "duplicate_noop", "reject"], + "reason": ["accepted_new", "exact_duplicate", "idempotency_conflict", "recipient_queue_full", "application_budget_exhausted", "delivery_disabled"], + "worker_error": ["malformed_json", "unknown_field", "duplicate_field", "invalid_unicode", "trailing_bytes", "unknown_enum", "unsupported_operation_version"] + }, + "cases": [ + {"name":"eligible_accept","request":"cases/eligible-accept.request.json","response":"cases/eligible-accept.plan.json","decision":"store_and_enqueue","reason":"accepted_new","request_sha256":"398159a680fec4f9724f5d4ed09a461da97a627146c4469fe936c5efc8c3f575","response_sha256":"df4c6f3612b9aaf1de046558f64bdf26b6720d82b0df6e0641231827761da763"}, + {"name":"exact_duplicate","request":"cases/exact-duplicate.request.json","response":"cases/exact-duplicate.plan.json","decision":"duplicate_noop","reason":"exact_duplicate","request_sha256":"80633c373095d873fe3af53b6afbf27036445e5356a75e242a2e4f4c6c1fc1a7","response_sha256":"5246ded59b37eeabb6daa46dccd188dd286b63934ecb10a22101ed8a0b9cd448"}, + {"name":"idempotency_conflict","request":"cases/idempotency-conflict.request.json","response":"cases/idempotency-conflict.plan.json","decision":"reject","reason":"idempotency_conflict","request_sha256":"c373b576139dbc29ee3119d81ce850d6141025f9d75d80cae11d2ab8f53c5bef","response_sha256":"d36a17362b415ff09bd9dccc638f623868472a3a34b2b0ca647aa38994803931"}, + {"name":"application_budget_rejection","request":"cases/application-budget-rejection.request.json","response":"cases/application-budget-rejection.plan.json","decision":"reject","reason":"application_budget_exhausted","request_sha256":"b000a18cf56e09cb922e96d554846a96e578cf2bcfa0c128edc49d24fb4c2208","response_sha256":"35d96f0b519b62f6388199534b436b401020fe224315c0f810566a42447f9dd6"}, + {"name":"unknown_operation_version","request":"cases/unknown-operation-version.request.json","response":"cases/unknown-operation-version.error.json","decision":"error","reason":"unsupported_operation_version","request_sha256":"7305c9fc1b7b2f8247f1304d6186805b25e818fcf6139d32c8cd7326521cbb18","response_sha256":"62f82b84054271178ff75130775ab9fc825d170ffcd2ab795410d464624ea145"} + ], + "handshake": {"path":"handshake/ready.json","sha256":"013585359995295b758efa367874943677e13a6eda6cd60762197dcbdf13d323"}, + "protocol_errors": [ + {"path":"errors/protocol-malformed-json.json","code":"malformed_json","sha256":"054f225427a1272d0371f255c38d6c4868ea5d911585392be5792fbb7514cac1"}, + {"path":"errors/protocol-unknown-field.json","code":"unknown_field","sha256":"6efb6dfcf37f1f5321ba7127fadfe18fcc10d7317ddc201678ae9ab359e09e79"}, + {"path":"errors/protocol-duplicate-field.json","code":"duplicate_field","sha256":"990c858b627d699ab80560b79dd9d06abc8e6d70ca1961e8f02b504411455e56"}, + {"path":"errors/protocol-invalid-unicode.json","code":"invalid_unicode","sha256":"72e208c29f81b06960b297e79d733a1ac79b3b594c39c88c36d79efcc183f95b"}, + {"path":"errors/protocol-trailing-bytes.json","code":"trailing_bytes","sha256":"aa06776b2b90d4753a6aecda24f0eb6cb4f0f7abc3a11345ac0f409950217082"}, + {"path":"errors/protocol-unknown-enum.json","code":"unknown_enum","sha256":"81a2f5e02276c24a02b96a7cebfa211b2d5eadd48f6d4c6d0bdbc1ef940165ec"} + ], + "strict_parser_error_inputs": [ + {"path":"errors/protocol-duplicate-field.request.raw","code":"duplicate_field","sha256":"cd97a1b386622f72f44c016703d39247556a1244c232cfd854d566e17bac8f0f"}, + {"path":"errors/protocol-invalid-unicode.request.raw","code":"invalid_unicode","sha256":"905e89b5679fbaa026289994870f3975a04ab5014ccbc56d8947a5b3a65a57e8"}, + {"path":"errors/protocol-trailing-bytes.request.raw","code":"trailing_bytes","sha256":"20a58aeddbc0f76bfe2af2874a08c63c2376808e419da9f603ac40aa852adf27"} + ], + "rust_only_no_worker": [ + {"path":"rust-only-no-worker/forged-request.json","reason":"forged_request","sha256":"fde87ce9ef58b5c5bfccdebbd77e414cbf7e4a381364eeddd38921a6da97c39d"}, + {"path":"rust-only-no-worker/revoked-device.json","reason":"revoked_device","sha256":"05f71182e0d39d2a040657e5850696f350128bbf3f20d729918c7293af7c4fd6"}, + {"path":"rust-only-no-worker/stale-request.json","reason":"stale_request","sha256":"8d4abe7145781d5310282150b4f215ff48c84b632bbbb5c8884e455877ee1013"}, + {"path":"rust-only-no-worker/rate-limited.json","reason":"rate_limited","sha256":"7aa216b1f179076e730daf0709c9bb1a273efec46aab66512f3a8c7df44a834e"} + ] +} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/rust-only-no-worker/forged-request.json b/examples/realworld/senline-domain-worker/fixtures/v1/rust-only-no-worker/forged-request.json new file mode 100644 index 00000000..de332315 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/rust-only-no-worker/forged-request.json @@ -0,0 +1 @@ +{"kind":"rust_only_rejection","schema_version":1,"reason":"forged_request"} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/rust-only-no-worker/rate-limited.json b/examples/realworld/senline-domain-worker/fixtures/v1/rust-only-no-worker/rate-limited.json new file mode 100644 index 00000000..91421672 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/rust-only-no-worker/rate-limited.json @@ -0,0 +1 @@ +{"kind":"rust_only_rejection","schema_version":1,"reason":"rate_limited"} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/rust-only-no-worker/revoked-device.json b/examples/realworld/senline-domain-worker/fixtures/v1/rust-only-no-worker/revoked-device.json new file mode 100644 index 00000000..349fdb7a --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/rust-only-no-worker/revoked-device.json @@ -0,0 +1 @@ +{"kind":"rust_only_rejection","schema_version":1,"reason":"revoked_device"} diff --git a/examples/realworld/senline-domain-worker/fixtures/v1/rust-only-no-worker/stale-request.json b/examples/realworld/senline-domain-worker/fixtures/v1/rust-only-no-worker/stale-request.json new file mode 100644 index 00000000..fbfe89e2 --- /dev/null +++ b/examples/realworld/senline-domain-worker/fixtures/v1/rust-only-no-worker/stale-request.json @@ -0,0 +1 @@ +{"kind":"rust_only_rejection","schema_version":1,"reason":"stale_request"} diff --git a/examples/realworld/senline-domain-worker/packages/senline-build-identity/README.md b/examples/realworld/senline-domain-worker/packages/senline-build-identity/README.md new file mode 100644 index 00000000..d85ebb67 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/senline-build-identity/README.md @@ -0,0 +1,11 @@ +# senline_build_identity + +This product-specific package contains generated consistency values for the +Senline dogfood worker startup handshake. Run +`scripts/generate-build-identity.ps1` with an independently produced bundle +manifest identity before a release build. + +The generator writes both this Sengoo source and the byte-exact external +handshake record from the same validated inputs. Repeating identical inputs is +byte reproducible. The worker's self-report is not a trust root: the host must +verify the complete external bundle manifest and reject any mismatch. diff --git a/examples/realworld/senline-domain-worker/packages/senline-build-identity/Sengoo.lock b/examples/realworld/senline-domain-worker/packages/senline-build-identity/Sengoo.lock new file mode 100644 index 00000000..ca855e53 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/senline-build-identity/Sengoo.lock @@ -0,0 +1,11 @@ +# This file is generated by sgpm update. +version = 2 +root = "senline_build_identity" + +[[package]] +id = "senline_build_identity@0.1.0+path:." +name = "senline_build_identity" +version = "0.1.0" +source.kind = "path" +source.path = "." +manifest = "Sengoo.toml" diff --git a/examples/realworld/senline-domain-worker/packages/senline-build-identity/Sengoo.toml b/examples/realworld/senline-domain-worker/packages/senline-build-identity/Sengoo.toml new file mode 100644 index 00000000..1e887e79 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/senline-build-identity/Sengoo.toml @@ -0,0 +1,8 @@ +[package] +name = "senline_build_identity" +version = "0.1.0" +edition = "2026" +description = "Generated build identity values for the Senline dogfood worker." + +[lib] +path = "src/lib.sg" diff --git a/examples/realworld/senline-domain-worker/packages/senline-build-identity/src/lib.sg b/examples/realworld/senline-domain-worker/packages/senline-build-identity/src/lib.sg new file mode 100644 index 00000000..58f8c567 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/senline-build-identity/src/lib.sg @@ -0,0 +1,19 @@ +def senline_build_source_revision() -> &str { + "1de09ccafa7e8f182af68e82352e2d4be39496b0"; +} + +def senline_build_toolchain_version() -> &str { + "0.1.0"; +} + +def senline_build_application_version() -> &str { + "0.1.0"; +} + +def senline_build_manifest_id() -> &str { + "1111111111111111111111111111111111111111111111111111111111111111"; +} + +def senline_build_handshake_payload() -> &str { + "{\"kind\":\"handshake\",\"protocol_version\":1,\"sengoo_source_revision\":\"1de09ccafa7e8f182af68e82352e2d4be39496b0\",\"toolchain_version\":\"0.1.0\",\"application_version\":\"0.1.0\",\"build_manifest_id\":\"1111111111111111111111111111111111111111111111111111111111111111\"}\n"; +} diff --git a/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/README.md b/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/README.md new file mode 100644 index 00000000..dffebc51 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/README.md @@ -0,0 +1,49 @@ +# senline-facts-to-plan + +`senline-facts-to-plan` is the Senline product module for the +`submit-envelope` V1 decision contract. It is deliberately not a general +Sengoo ecosystem package and is not a candidate for `std::` promotion. + +## Boundary + +- The worker wire layer validates the closed JSON contract before constructing + these DTOs. +- Closed capability and feature-flag arrays are represented semantically as + `has_submit_envelope_v2` and `enqueue_delivery_enabled` booleans. +- Owned strings keep decoded context and identifiers independent from the JSON + document. `plan_submit_envelope_v1` consumes the request and moves its exact + context and identifiers into the returned plan. +- `senline_empty_worker_request_v1` is only a safe fallback value for Sengoo + `Result` handling. It is not a valid request. +- Decision and reason codes are stable product-specific `i64` values. + `senline_decision_name` and `senline_reason_name` map known values to the V1 + wire names and return an empty string for an unknown value. + +The planner applies this priority order: + +1. exact duplicate; +2. idempotency conflict; +3. recipient queue full; +4. application budget exhausted; +5. missing submit capability or enqueue flag; +6. accepted new envelope. + +The package imports only `std::string`. It has no network, file, environment, +clock, randomness, process, database, or FFI authority. + +## Verification + +From this package directory: + +```text +sgpm update --check +sgpm --runtime-mode source-development check --locked +sgpm --runtime-mode source-development test --locked +sgpm fmt --check --locked +sgpm --runtime-mode source-development doc --locked +sgpm --runtime-mode source-development build --release --locked +``` + +The package tests cover the four frozen fixture decisions plus queue-full and +both delivery-disabled inputs, precedence, exact context/identifier echoing, +stable numeric codes, and wire-name helpers. diff --git a/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/Sengoo.lock b/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/Sengoo.lock new file mode 100644 index 00000000..6937a683 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/Sengoo.lock @@ -0,0 +1,11 @@ +# This file is generated by sgpm update. +version = 2 +root = "senline_facts_to_plan" + +[[package]] +id = "senline_facts_to_plan@0.1.0+path:." +name = "senline_facts_to_plan" +version = "0.1.0" +source.kind = "path" +source.path = "." +manifest = "Sengoo.toml" diff --git a/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/Sengoo.toml b/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/Sengoo.toml new file mode 100644 index 00000000..316eae7f --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/Sengoo.toml @@ -0,0 +1,8 @@ +[package] +name = "senline_facts_to_plan" +version = "0.1.0" +edition = "2026" +description = "Pure minimum-authority Senline facts-to-plan contract module." + +[lib] +path = "src/lib.sg" diff --git a/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/src/lib.sg b/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/src/lib.sg new file mode 100644 index 00000000..21ca12be --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/src/lib.sg @@ -0,0 +1,148 @@ +import std::string; + +struct EvaluationContextV1 { + contract_version: i64, + operation: String, + operation_version: i64, + evaluation_id: String, + operation_epoch: i64, + worker_generation: i64, + execution_mode: String, + worker_bundle_id: String, + facts_binding: String, +} + +struct SubmitEnvelopeIdentifiersV1 { + correlation_ref: String, + source_account_ref: String, + source_device_ref: String, + recipient_account_ref: String, + recipient_device_ref: String, + conversation_ref: String, + envelope_ref: String, +} + +struct SubmitEnvelopeFactsV1 { + contract_version: i64, + operation_version: i64, + identifiers: SubmitEnvelopeIdentifiersV1, + source_device_active: bool, + has_submit_envelope_v2: bool, + envelope_protocol_version: i64, + ciphertext_length_bytes: i64, + idempotency_status: i64, + recipient_pending_count: i64, + recipient_pending_limit: i64, + application_envelopes_used: i64, + application_envelopes_limit: i64, + ciphertext_limit_bytes: i64, + enqueue_delivery_enabled: bool, +} + +struct WorkerRequestV1 { + schema_version: i64, + context: EvaluationContextV1, + facts: SubmitEnvelopeFactsV1, +} + +struct SubmitEnvelopePlanV1 { + schema_version: i64, + context: EvaluationContextV1, + identifiers: SubmitEnvelopeIdentifiersV1, + decision: i64, + reason: i64, + sengoo_module_revision: String, +} + +def senline_contract_version() -> i64 { + 1; +} + +def senline_operation_version_supported(version: i64) -> bool { + version == 1; +} + +def SENLINE_IDEMPOTENCY_NEW() -> i64 { + 0; +} + +def SENLINE_IDEMPOTENCY_EXACT_DUPLICATE() -> i64 { + 1; +} + +def SENLINE_IDEMPOTENCY_CONFLICT() -> i64 { + 2; +} + +def SENLINE_DECISION_STORE_AND_ENQUEUE() -> i64 { + 1; +} + +def SENLINE_DECISION_DUPLICATE_NOOP() -> i64 { + 2; +} + +def SENLINE_DECISION_REJECT() -> i64 { + 3; +} + +def SENLINE_REASON_ACCEPTED_NEW() -> i64 { + 1; +} + +def SENLINE_REASON_EXACT_DUPLICATE() -> i64 { + 2; +} + +def SENLINE_REASON_IDEMPOTENCY_CONFLICT() -> i64 { + 3; +} + +def SENLINE_REASON_RECIPIENT_QUEUE_FULL() -> i64 { + 4; +} + +def SENLINE_REASON_APPLICATION_BUDGET_EXHAUSTED() -> i64 { + 5; +} + +def SENLINE_REASON_DELIVERY_DISABLED() -> i64 { + 6; +} + +def senline_decision_name(code: i64) -> &str { + if code == SENLINE_DECISION_STORE_AND_ENQUEUE() { "store_and_enqueue"; } else if code == SENLINE_DECISION_DUPLICATE_NOOP() { "duplicate_noop"; } else if code == SENLINE_DECISION_REJECT() { "reject"; } else { ""; }; +} + +def senline_reason_name(code: i64) -> &str { + if code == SENLINE_REASON_ACCEPTED_NEW() { "accepted_new"; } else if code == SENLINE_REASON_EXACT_DUPLICATE() { "exact_duplicate"; } else if code == SENLINE_REASON_IDEMPOTENCY_CONFLICT() { "idempotency_conflict"; } else if code == SENLINE_REASON_RECIPIENT_QUEUE_FULL() { "recipient_queue_full"; } else if code == SENLINE_REASON_APPLICATION_BUDGET_EXHAUSTED() { "application_budget_exhausted"; } else if code == SENLINE_REASON_DELIVERY_DISABLED() { "delivery_disabled"; } else { ""; }; +} + +def senline_empty_evaluation_context_v1() -> EvaluationContextV1 { + EvaluationContextV1 { contract_version: 0, operation: String { handle: 0 }, operation_version: 0, evaluation_id: String { handle: 0 }, operation_epoch: 0, worker_generation: 0, execution_mode: String { handle: 0 }, worker_bundle_id: String { handle: 0 }, facts_binding: String { handle: 0 } }; +} + +def senline_empty_submit_envelope_identifiers_v1() -> SubmitEnvelopeIdentifiersV1 { + SubmitEnvelopeIdentifiersV1 { correlation_ref: String { handle: 0 }, source_account_ref: String { handle: 0 }, source_device_ref: String { handle: 0 }, recipient_account_ref: String { handle: 0 }, recipient_device_ref: String { handle: 0 }, conversation_ref: String { handle: 0 }, envelope_ref: String { handle: 0 } }; +} + +def senline_empty_submit_envelope_facts_v1() -> SubmitEnvelopeFactsV1 { + SubmitEnvelopeFactsV1 { contract_version: 0, operation_version: 0, identifiers: senline_empty_submit_envelope_identifiers_v1(), source_device_active: false, has_submit_envelope_v2: false, envelope_protocol_version: 0, ciphertext_length_bytes: 0, idempotency_status: SENLINE_IDEMPOTENCY_NEW(), recipient_pending_count: 0, recipient_pending_limit: 0, application_envelopes_used: 0, application_envelopes_limit: 0, ciphertext_limit_bytes: 0, enqueue_delivery_enabled: false }; +} + +def senline_empty_worker_request_v1() -> WorkerRequestV1 { + WorkerRequestV1 { schema_version: 0, context: senline_empty_evaluation_context_v1(), facts: senline_empty_submit_envelope_facts_v1() }; +} + +def senline_reason_for_submit_envelope_v1(idempotency_status: i64, recipient_pending_count: i64, recipient_pending_limit: i64, application_envelopes_used: i64, application_envelopes_limit: i64, has_submit_envelope_v2: bool, enqueue_delivery_enabled: bool) -> i64 { + if idempotency_status == SENLINE_IDEMPOTENCY_EXACT_DUPLICATE() { SENLINE_REASON_EXACT_DUPLICATE(); } else if idempotency_status == SENLINE_IDEMPOTENCY_CONFLICT() { SENLINE_REASON_IDEMPOTENCY_CONFLICT(); } else if recipient_pending_count >= recipient_pending_limit { SENLINE_REASON_RECIPIENT_QUEUE_FULL(); } else if application_envelopes_used >= application_envelopes_limit { SENLINE_REASON_APPLICATION_BUDGET_EXHAUSTED(); } else if not has_submit_envelope_v2 or not enqueue_delivery_enabled { SENLINE_REASON_DELIVERY_DISABLED(); } else { SENLINE_REASON_ACCEPTED_NEW(); }; +} + +def senline_decision_for_reason(reason: i64) -> i64 { + if reason == SENLINE_REASON_ACCEPTED_NEW() { SENLINE_DECISION_STORE_AND_ENQUEUE(); } else if reason == SENLINE_REASON_EXACT_DUPLICATE() { SENLINE_DECISION_DUPLICATE_NOOP(); } else { SENLINE_DECISION_REJECT(); }; +} + +def plan_submit_envelope_v1(request: WorkerRequestV1, sengoo_module_revision: String) -> SubmitEnvelopePlanV1 { + let reason = senline_reason_for_submit_envelope_v1(request.facts.idempotency_status, request.facts.recipient_pending_count, request.facts.recipient_pending_limit, request.facts.application_envelopes_used, request.facts.application_envelopes_limit, request.facts.has_submit_envelope_v2, request.facts.enqueue_delivery_enabled); + SubmitEnvelopePlanV1 { schema_version: request.schema_version, context: request.context, identifiers: request.facts.identifiers, decision: senline_decision_for_reason(reason), reason: reason, sengoo_module_revision: sengoo_module_revision }; +} diff --git a/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/tests/contract_scaffold.sg b/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/tests/contract_scaffold.sg new file mode 100644 index 00000000..b82932dc --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/tests/contract_scaffold.sg @@ -0,0 +1,5 @@ +import senline_facts_to_plan; + +def main() -> i64 { + if senline_contract_version() == 1 and senline_operation_version_supported(1) and not senline_operation_version_supported(99) { 0; } else { 1; }; +} diff --git a/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/tests/planner_branches.sg b/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/tests/planner_branches.sg new file mode 100644 index 00000000..95f2a985 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/tests/planner_branches.sg @@ -0,0 +1,45 @@ +import std::string; + +import senline_facts_to_plan; + +def owned(value: &str) -> String { + string_from_str(value).unwrap_or(String { handle: 0 }); +} + +def identifiers(case_ref: &str) -> SubmitEnvelopeIdentifiersV1 { + SubmitEnvelopeIdentifiersV1 { correlation_ref: owned(case_ref), source_account_ref: owned("acct_ref_source_001"), source_device_ref: owned("device_ref_source_001"), recipient_account_ref: owned("acct_ref_recipient_001"), recipient_device_ref: owned("device_ref_recipient_001"), conversation_ref: owned("conversation_ref_001"), envelope_ref: owned("envelope_ref_001") }; +} + +def request(evaluation_id: &str, correlation_ref: &str, idempotency_status: i64, recipient_pending_count: i64, recipient_pending_limit: i64, application_envelopes_used: i64, application_envelopes_limit: i64, has_submit_envelope_v2: bool, enqueue_delivery_enabled: bool, ciphertext_length_bytes: i64) -> WorkerRequestV1 { + WorkerRequestV1 { schema_version: 1, context: EvaluationContextV1 { contract_version: 1, operation: owned("submit-envelope"), operation_version: 1, evaluation_id: owned(evaluation_id), operation_epoch: 7, worker_generation: 3, execution_mode: owned("fixture"), worker_bundle_id: owned("senline-worker-fixture-v1"), facts_binding: owned("09d0b4fcae5f3026de32cc41f9aa51c15fe9b40e379062c00de984cfffe93302") }, facts: SubmitEnvelopeFactsV1 { contract_version: 1, operation_version: 1, identifiers: identifiers(correlation_ref), source_device_active: true, has_submit_envelope_v2: has_submit_envelope_v2, envelope_protocol_version: 2, ciphertext_length_bytes: ciphertext_length_bytes, idempotency_status: idempotency_status, recipient_pending_count: recipient_pending_count, recipient_pending_limit: recipient_pending_limit, application_envelopes_used: application_envelopes_used, application_envelopes_limit: application_envelopes_limit, ciphertext_limit_bytes: 65536, enqueue_delivery_enabled: enqueue_delivery_enabled } }; +} + +def planned(input: WorkerRequestV1) -> SubmitEnvelopePlanV1 { + plan_submit_envelope_v1(input, owned("1de09ccafa7e8f182af68e82352e2d4be39496b0")); +} + +def main() -> i64 { + if SENLINE_IDEMPOTENCY_NEW() != 0 or SENLINE_IDEMPOTENCY_EXACT_DUPLICATE() != 1 or SENLINE_IDEMPOTENCY_CONFLICT() != 2 { return 10; }; + if SENLINE_DECISION_STORE_AND_ENQUEUE() != 1 or SENLINE_DECISION_DUPLICATE_NOOP() != 2 or SENLINE_DECISION_REJECT() != 3 { return 11; }; + if SENLINE_REASON_ACCEPTED_NEW() != 1 or SENLINE_REASON_EXACT_DUPLICATE() != 2 or SENLINE_REASON_IDEMPOTENCY_CONFLICT() != 3 or SENLINE_REASON_RECIPIENT_QUEUE_FULL() != 4 or SENLINE_REASON_APPLICATION_BUDGET_EXHAUSTED() != 5 or SENLINE_REASON_DELIVERY_DISABLED() != 6 { return 12; }; + let accepted = planned(request("00000000000000000000000000000001", "corr_ref_eligible_001", SENLINE_IDEMPOTENCY_NEW(), 12, 1000, 41, 10000, true, true, 512)); + if accepted.decision != SENLINE_DECISION_STORE_AND_ENQUEUE() or accepted.reason != SENLINE_REASON_ACCEPTED_NEW() { return 20; }; + if accepted.schema_version != 1 or accepted.context.contract_version != 1 or not str_eq(accepted.context.operation.as_str(), "submit-envelope") or accepted.context.operation_version != 1 or not str_eq(accepted.context.evaluation_id.as_str(), "00000000000000000000000000000001") or accepted.context.operation_epoch != 7 or accepted.context.worker_generation != 3 or not str_eq(accepted.context.execution_mode.as_str(), "fixture") or not str_eq(accepted.context.worker_bundle_id.as_str(), "senline-worker-fixture-v1") or not str_eq(accepted.context.facts_binding.as_str(), "09d0b4fcae5f3026de32cc41f9aa51c15fe9b40e379062c00de984cfffe93302") { return 21; }; + if not str_eq(accepted.identifiers.correlation_ref.as_str(), "corr_ref_eligible_001") or not str_eq(accepted.identifiers.source_account_ref.as_str(), "acct_ref_source_001") or not str_eq(accepted.identifiers.source_device_ref.as_str(), "device_ref_source_001") or not str_eq(accepted.identifiers.recipient_account_ref.as_str(), "acct_ref_recipient_001") or not str_eq(accepted.identifiers.recipient_device_ref.as_str(), "device_ref_recipient_001") or not str_eq(accepted.identifiers.conversation_ref.as_str(), "conversation_ref_001") or not str_eq(accepted.identifiers.envelope_ref.as_str(), "envelope_ref_001") or not str_eq(accepted.sengoo_module_revision.as_str(), "1de09ccafa7e8f182af68e82352e2d4be39496b0") { return 22; }; + let duplicate = planned(request("00000000000000000000000000000002", "corr_ref_duplicate_002", SENLINE_IDEMPOTENCY_EXACT_DUPLICATE(), 1000, 1000, 10000, 10000, false, false, 768)); + if duplicate.decision != SENLINE_DECISION_DUPLICATE_NOOP() or duplicate.reason != SENLINE_REASON_EXACT_DUPLICATE() { return 30; }; + let conflict = planned(request("00000000000000000000000000000003", "corr_ref_conflict_003", SENLINE_IDEMPOTENCY_CONFLICT(), 1000, 1000, 10000, 10000, false, false, 1024)); + if conflict.decision != SENLINE_DECISION_REJECT() or conflict.reason != SENLINE_REASON_IDEMPOTENCY_CONFLICT() { return 40; }; + let budget = planned(request("00000000000000000000000000000004", "corr_ref_budget_004", SENLINE_IDEMPOTENCY_NEW(), 15, 1000, 10000, 10000, true, true, 1536)); + if budget.decision != SENLINE_DECISION_REJECT() or budget.reason != SENLINE_REASON_APPLICATION_BUDGET_EXHAUSTED() { return 50; }; + let queue_full = planned(request("00000000000000000000000000000006", "corr_ref_queue_006", SENLINE_IDEMPOTENCY_NEW(), 1000, 1000, 10000, 10000, false, false, 256)); + if queue_full.decision != SENLINE_DECISION_REJECT() or queue_full.reason != SENLINE_REASON_RECIPIENT_QUEUE_FULL() { return 60; }; + let budget_over_disabled = planned(request("00000000000000000000000000000010", "corr_ref_budget_priority_010", SENLINE_IDEMPOTENCY_NEW(), 1, 1000, 10000, 10000, false, false, 256)); + if budget_over_disabled.decision != SENLINE_DECISION_REJECT() or budget_over_disabled.reason != SENLINE_REASON_APPLICATION_BUDGET_EXHAUSTED() { return 61; }; + let missing_capability = planned(request("00000000000000000000000000000007", "corr_ref_capability_007", SENLINE_IDEMPOTENCY_NEW(), 1, 1000, 1, 10000, false, true, 256)); + let missing_flag = planned(request("00000000000000000000000000000008", "corr_ref_flag_008", SENLINE_IDEMPOTENCY_NEW(), 1, 1000, 1, 10000, true, false, 256)); + if missing_capability.decision != SENLINE_DECISION_REJECT() or missing_capability.reason != SENLINE_REASON_DELIVERY_DISABLED() or missing_flag.decision != SENLINE_DECISION_REJECT() or missing_flag.reason != SENLINE_REASON_DELIVERY_DISABLED() { return 70; }; + let accepted_with_other_irrelevant_inputs = planned(request("00000000000000000000000000000009", "corr_ref_other_009", SENLINE_IDEMPOTENCY_NEW(), 999, 1000, 9999, 10000, true, true, 65536)); + if accepted_with_other_irrelevant_inputs.decision != accepted.decision or accepted_with_other_irrelevant_inputs.reason != accepted.reason { return 80; }; + 0; +} diff --git a/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/tests/wire_helpers.sg b/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/tests/wire_helpers.sg new file mode 100644 index 00000000..018d9226 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/senline-facts-to-plan/tests/wire_helpers.sg @@ -0,0 +1,21 @@ +import std::string; + +import senline_facts_to_plan; + +def main() -> i64 { + if not str_eq(senline_decision_name(SENLINE_DECISION_STORE_AND_ENQUEUE()), "store_and_enqueue") { return 10; }; + if not str_eq(senline_decision_name(SENLINE_DECISION_DUPLICATE_NOOP()), "duplicate_noop") { return 11; }; + if not str_eq(senline_decision_name(SENLINE_DECISION_REJECT()), "reject") { return 12; }; + if not str_eq(senline_decision_name(99), "") { return 13; }; + if not str_eq(senline_reason_name(SENLINE_REASON_ACCEPTED_NEW()), "accepted_new") { return 20; }; + if not str_eq(senline_reason_name(SENLINE_REASON_EXACT_DUPLICATE()), "exact_duplicate") { return 21; }; + if not str_eq(senline_reason_name(SENLINE_REASON_IDEMPOTENCY_CONFLICT()), "idempotency_conflict") { return 22; }; + if not str_eq(senline_reason_name(SENLINE_REASON_RECIPIENT_QUEUE_FULL()), "recipient_queue_full") { return 23; }; + if not str_eq(senline_reason_name(SENLINE_REASON_APPLICATION_BUDGET_EXHAUSTED()), "application_budget_exhausted") { return 24; }; + if not str_eq(senline_reason_name(SENLINE_REASON_DELIVERY_DISABLED()), "delivery_disabled") { return 25; }; + if not str_eq(senline_reason_name(99), "") { return 26; }; + let empty = senline_empty_worker_request_v1(); + if empty.schema_version != 0 or empty.context.contract_version != 0 or empty.facts.contract_version != 0 { return 30; }; + if not empty.context.operation.is_empty() or not empty.context.evaluation_id.is_empty() or not empty.facts.identifiers.correlation_ref.is_empty() { return 31; }; + 0; +} diff --git a/examples/realworld/senline-domain-worker/packages/sgframing/README.md b/examples/realworld/senline-domain-worker/packages/sgframing/README.md new file mode 100644 index 00000000..30a617cf --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgframing/README.md @@ -0,0 +1,44 @@ +# sgframing + +`sgframing` is an incubating `0.x` Sengoo package for bounded +`u32-big-endian length + payload` framing over exact standard I/O. + +## Scope + +- `frame_validate_length` validates non-zero caller-supplied bounds. +- `frame_prefix_encode` and `frame_prefix_decode` operate on four-byte + big-endian prefixes. +- `frame_read_stdin` distinguishes clean EOF before a prefix from a truncated + frame and returns an owned payload Buffer for a complete frame. +- `frame_write_all_with` retries an injected writer from the exact next offset, + rejects zero progress and over-reported counts, and preserves writer errors. +- `frame_write_with` composes a complete prefix/payload frame over injected + writer and flusher callbacks. It initializes binary mode before either + callback so Windows cannot translate protocol bytes. +- `frame_write_stdout` writes a prefix and payload completely, then flushes. +- `frame_init_stdio_binary` enables binary protocol streams, including + `_O_BINARY` on Windows. + +The caller owns product limits. Senline currently supplies 32 KiB input and +8 KiB output limits; those values are not package defaults. The current +runtime Buffer allocation limit is 64 MiB, so a larger caller maximum can +still fail allocation. + +Stable package errors are `SGFRAMING_ZERO_LENGTH`, +`SGFRAMING_LIMIT_EXCEEDED`, `SGFRAMING_INVALID_PREFIX`, +`SGFRAMING_TRUNCATED`, and `SGFRAMING_INVALID_PAYLOAD`. Allocation, +binary-mode, and output I/O failures currently retain their underlying +stdlib status. This mixed error surface must be resolved before `1.0`. + +## Incubation Status + +- First consumer: `senline-domain-worker`. +- Independent consumers: none. +- Supported evidence: locked Windows source-development package tests, real + binary stdin/stdout boundary coverage, API docs, release build, and a local + publish dry run. +- Missing evidence: installed Windows/Linux toolchains, package license + metadata/files, registry publication, and an independent consumer. + +This package is not a general stream abstraction and is not eligible for +`std::` promotion yet. diff --git a/examples/realworld/senline-domain-worker/packages/sgframing/Sengoo.lock b/examples/realworld/senline-domain-worker/packages/sgframing/Sengoo.lock new file mode 100644 index 00000000..f20d7b38 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgframing/Sengoo.lock @@ -0,0 +1,11 @@ +# This file is generated by sgpm update. +version = 2 +root = "sgframing" + +[[package]] +id = "sgframing@0.1.0+path:." +name = "sgframing" +version = "0.1.0" +source.kind = "path" +source.path = "." +manifest = "Sengoo.toml" diff --git a/examples/realworld/senline-domain-worker/packages/sgframing/Sengoo.toml b/examples/realworld/senline-domain-worker/packages/sgframing/Sengoo.toml new file mode 100644 index 00000000..74c03942 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgframing/Sengoo.toml @@ -0,0 +1,8 @@ +[package] +name = "sgframing" +version = "0.1.0" +edition = "2026" +description = "Bounded length-prefixed framing over exact Sengoo I/O primitives." + +[lib] +path = "src/lib.sg" diff --git a/examples/realworld/senline-domain-worker/packages/sgframing/src/lib.sg b/examples/realworld/senline-domain-worker/packages/sgframing/src/lib.sg new file mode 100644 index 00000000..526f9e25 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgframing/src/lib.sg @@ -0,0 +1,143 @@ +import std::ffi; + +import std::io; + +import std::status; + +struct FrameRead { + eof: bool, + len: i64, + payload: Buffer, +} + +def SGFRAMING_ZERO_LENGTH() -> i64 { + 2101; +} + +def SGFRAMING_LIMIT_EXCEEDED() -> i64 { + 2102; +} + +def SGFRAMING_INVALID_PREFIX() -> i64 { + 2103; +} + +def SGFRAMING_TRUNCATED() -> i64 { + 2104; +} + +def SGFRAMING_INVALID_PAYLOAD() -> i64 { + 2105; +} + +def frame_bool_ok(value: bool) -> Result { + Result { is_ok: true, value: value, error: 0 }; +} + +def frame_bool_error(code: i64) -> Result { + Result { is_ok: false, value: false, error: code }; +} + +def frame_i64_ok(value: i64) -> Result { + Result { is_ok: true, value: value, error: 0 }; +} + +def frame_i64_error(code: i64) -> Result { + Result { is_ok: false, value: 0, error: code }; +} + +def frame_read_ok(eof: bool, len: i64, payload: Buffer) -> Result { + Result { is_ok: true, value: FrameRead { eof: eof, len: len, payload: payload }, error: 0 }; +} + +def frame_read_error(code: i64) -> Result { + Result { is_ok: false, value: FrameRead { eof: false, len: 0, payload: Buffer { handle: 0 } }, error: code }; +} + +def frame_validate_length(len: i64, max_len: i64) -> Result { + if len == 0 { frame_i64_error(SGFRAMING_ZERO_LENGTH()); } else if len < 0 or max_len <= 0 or len > max_len or len > 4294967295 { frame_i64_error(SGFRAMING_LIMIT_EXCEEDED()); } else { frame_i64_ok(len); }; +} + +def frame_prefix_encode(prefix: Buffer, len: i64, max_len: i64) -> Result { + let valid = frame_validate_length(len, max_len); + if valid.is_err() { return frame_bool_error(valid.error); }; + if prefix.len() < 4 { return frame_bool_error(SGFRAMING_INVALID_PREFIX()); }; + let wrote = prefix.write_u32_be(0, len); + if wrote.is_err() { frame_bool_error(SGFRAMING_INVALID_PREFIX()); } else { frame_bool_ok(true); }; +} + +def frame_prefix_decode(prefix: Buffer, max_len: i64) -> Result { + if prefix.len() < 4 or prefix.used_len() < 4 { return frame_i64_error(SGFRAMING_INVALID_PREFIX()); }; + let decoded = prefix.read_u32_be(0); + if decoded.is_err() { frame_i64_error(SGFRAMING_INVALID_PREFIX()); } else { frame_validate_length(decoded.value, max_len); }; +} + +def frame_init_stdio_binary() -> Result { + io_protocol_binary_mode(); +} + +def frame_read_stdin(max_len: i64) -> Result { + if max_len <= 0 or max_len > 4294967295 { return frame_read_error(SGFRAMING_LIMIT_EXCEEDED()); }; + let binary = frame_init_stdio_binary(); + if binary.is_err() { return frame_read_error(binary.error); }; + let prefix_result = ffi_buffer_new(4); + if prefix_result.is_err() { return frame_read_error(prefix_result.error); }; + let prefix = prefix_result.value; + let prefix_read = io_stdin_read_exact(prefix, 0, 4); + if prefix_read.is_err() { prefix.free(); return frame_read_error(SGFRAMING_TRUNCATED()); }; + if prefix_read.value == 0 { prefix.free(); return frame_read_ok(true, 0, Buffer { handle: 0 }); }; + if prefix_read.value != 4 { prefix.free(); return frame_read_error(SGFRAMING_TRUNCATED()); }; + let decoded = frame_prefix_decode(prefix, max_len); + prefix.free(); + if decoded.is_err() { return frame_read_error(decoded.error); }; + let payload_result = ffi_buffer_new(decoded.value); + if payload_result.is_err() { return frame_read_error(payload_result.error); }; + let payload = payload_result.value; + let payload_read = io_stdin_read_exact(payload, 0, decoded.value); + if payload_read.is_err() or payload_read.value != decoded.value { payload.free(); return frame_read_error(SGFRAMING_TRUNCATED()); }; + frame_read_ok(false, decoded.value, payload); +} + +def frame_write_all_with(payload: Buffer, offset: i64, len: i64, writer: fn(Buffer, i64, i64) -> Result) -> Result { + if offset < 0 or len < 0 { return frame_i64_error(SGFRAMING_INVALID_PAYLOAD()); }; + let used_len = payload.used_len(); + if offset > used_len or len > used_len - offset { return frame_i64_error(SGFRAMING_INVALID_PAYLOAD()); }; + let mut written = 0; + while written < len { let remaining = len - written; let step = writer(payload, offset + written, remaining); if step.is_err() { return frame_i64_error(step.error); }; if step.value <= 0 or step.value > remaining { return frame_i64_error(STATUS_IO()); }; written = written + step.value; }; + frame_i64_ok(written); +} + +def frame_write_with(payload: Buffer, len: i64, max_len: i64, writer: fn(Buffer, i64, i64) -> Result, flusher: fn() -> Result) -> Result { + let valid = frame_validate_length(len, max_len); + if valid.is_err() { return frame_i64_error(valid.error); }; + if payload.used_len() < len { return frame_i64_error(SGFRAMING_INVALID_PAYLOAD()); }; + let binary = frame_init_stdio_binary(); + if binary.is_err() { return frame_i64_error(binary.error); }; + let prefix_result = ffi_buffer_new(4); + if prefix_result.is_err() { return frame_i64_error(prefix_result.error); }; + let prefix = prefix_result.value; + let encoded = frame_prefix_encode(prefix, len, max_len); + if encoded.is_err() { prefix.free(); return frame_i64_error(encoded.error); }; + let prefix_write = frame_write_all_with(prefix, 0, 4, writer); + prefix.free(); + if prefix_write.is_err() or prefix_write.value != 4 { return frame_i64_error(STATUS_IO()); }; + let payload_write = frame_write_all_with(payload, 0, len, writer); + if payload_write.is_err() or payload_write.value != len { return frame_i64_error(STATUS_IO()); }; + let flushed = flusher(); + if flushed.is_err() { return frame_i64_error(flushed.error); }; + frame_i64_ok(len + 4); +} + +def frame_stdout_write(payload: Buffer, offset: i64, len: i64) -> Result { + io_stdout_write_all(payload, offset, len); +} + +def frame_stdout_flush() -> Result { + io_stdout_flush(); +} + +def frame_write_stdout(payload: Buffer, len: i64, max_len: i64) -> Result { + let writer: fn(Buffer, i64, i64) -> Result = frame_stdout_write; + let flusher: fn() -> Result = frame_stdout_flush; + frame_write_with(payload, len, max_len, writer, flusher); +} diff --git a/examples/realworld/senline-domain-worker/packages/sgframing/tests/frame_pipe.sg b/examples/realworld/senline-domain-worker/packages/sgframing/tests/frame_pipe.sg new file mode 100644 index 00000000..cff59eff --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgframing/tests/frame_pipe.sg @@ -0,0 +1,24 @@ +import std::ffi; + +import sgframing; + +def frame_test_exit(code: i64) -> i64 { + if code == SGFRAMING_ZERO_LENGTH() { 21; } else if code == SGFRAMING_LIMIT_EXCEEDED() { 22; } else if code == SGFRAMING_INVALID_PREFIX() { 23; } else if code == SGFRAMING_TRUNCATED() { 24; } else if code == SGFRAMING_INVALID_PAYLOAD() { 25; } else { 29; }; +} + +def frame_test_run_once() -> i64 { + let read = frame_read_stdin(64); + if read.is_err() { return frame_test_exit(read.error); }; + let frame = read.value; + if frame.eof { return 0; }; + let len = frame.len; + let wrote = frame_write_stdout(frame.payload, len, 64); + if wrote.is_err() { return frame_test_exit(wrote.error); }; + if wrote.value == len + 4 { 0; } else { 30; }; +} + +def main() -> i64 { + let cleared = ffi_last_error_clear(); + let status = frame_test_run_once(); + if not cleared { 31; } else if status == 0 and ffi_last_error_code() != 0 { 32; } else { status; }; +} diff --git a/examples/realworld/senline-domain-worker/packages/sgframing/tests/framing_contract.sg b/examples/realworld/senline-domain-worker/packages/sgframing/tests/framing_contract.sg new file mode 100644 index 00000000..021fb97e --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgframing/tests/framing_contract.sg @@ -0,0 +1,19 @@ +import std::ffi; + +import sgframing; + +def main() -> i64 { + let prefix = ffi_buffer_new(4).unwrap_or(Buffer { handle: 0 }); + let short_prefix = ffi_buffer_new(3).unwrap_or(Buffer { handle: 0 }); + let encoded = frame_prefix_encode(prefix, 32768, 32768).unwrap_or(false); + let decoded = frame_prefix_decode(prefix, 32768).unwrap_or(-1); + let zero = frame_prefix_encode(prefix, 0, 32768); + let oversized = frame_prefix_encode(prefix, 32769, 32768); + let short = frame_prefix_decode(short_prefix, 32768); + let eof = frame_read_stdin(32768).unwrap_or(FrameRead { eof: false, len: -1, payload: Buffer { handle: 0 } }); + let ok = encoded and decoded == 32768 and zero.is_err() and zero.error == SGFRAMING_ZERO_LENGTH() and oversized.is_err() and oversized.error == SGFRAMING_LIMIT_EXCEEDED() and short.is_err() and short.error == SGFRAMING_INVALID_PREFIX() and eof.eof and eof.len == 0; + prefix.free(); + short_prefix.free(); + eof.payload.free(); + if ok { 0; } else { 1; }; +} diff --git a/examples/realworld/senline-domain-worker/packages/sgframing/tests/partial_writer_contract.sg b/examples/realworld/senline-domain-worker/packages/sgframing/tests/partial_writer_contract.sg new file mode 100644 index 00000000..f01464e5 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgframing/tests/partial_writer_contract.sg @@ -0,0 +1,52 @@ +import std::ffi; + +import std::status; + +import sgframing; + +def three_byte_writer(payload: Buffer, offset: i64, len: i64) -> Result { + let accepted = if len > 3 { 3; } else { len; }; + Result { is_ok: true, value: accepted, error: 0 }; +} + +def zero_progress_writer(payload: Buffer, offset: i64, len: i64) -> Result { + Result { is_ok: true, value: 0, error: 0 }; +} + +def excessive_writer(payload: Buffer, offset: i64, len: i64) -> Result { + Result { is_ok: true, value: len + 1, error: 0 }; +} + +def failing_writer(payload: Buffer, offset: i64, len: i64) -> Result { + Result { is_ok: false, value: 0, error: STATUS_TIMEOUT() }; +} + +def successful_flush() -> Result { + Result { is_ok: true, value: true, error: 0 }; +} + +def failing_flush() -> Result { + Result { is_ok: false, value: false, error: STATUS_IO() }; +} + +def main() -> i64 { + let payload = ffi_buffer_from_bytes("abcdefgh").unwrap_or(Buffer { handle: 0 }); + let partial_writer: fn(Buffer, i64, i64) -> Result = three_byte_writer; + let zero_writer: fn(Buffer, i64, i64) -> Result = zero_progress_writer; + let too_much_writer: fn(Buffer, i64, i64) -> Result = excessive_writer; + let error_writer: fn(Buffer, i64, i64) -> Result = failing_writer; + let good_flusher: fn() -> Result = successful_flush; + let bad_flusher: fn() -> Result = failing_flush; + let complete = frame_write_all_with(payload, 0, 8, partial_writer); + let ranged = frame_write_all_with(payload, 2, 4, partial_writer); + let zero = frame_write_all_with(payload, 0, 8, zero_writer); + let excessive = frame_write_all_with(payload, 0, 8, too_much_writer); + let failed = frame_write_all_with(payload, 0, 8, error_writer); + let negative = frame_write_all_with(payload, -1, 1, partial_writer); + let outside = frame_write_all_with(payload, 7, 2, partial_writer); + let framed = frame_write_with(payload, 8, 8, partial_writer, good_flusher); + let flush_failed = frame_write_with(payload, 8, 8, partial_writer, bad_flusher); + let ok = complete.unwrap_or(-1) == 8 and ranged.unwrap_or(-1) == 4 and zero.is_err() and zero.error == STATUS_IO() and excessive.is_err() and excessive.error == STATUS_IO() and failed.is_err() and failed.error == STATUS_TIMEOUT() and negative.is_err() and negative.error == SGFRAMING_INVALID_PAYLOAD() and outside.is_err() and outside.error == SGFRAMING_INVALID_PAYLOAD() and framed.unwrap_or(-1) == 12 and flush_failed.is_err() and flush_failed.error == STATUS_IO(); + payload.free(); + if ok { 0; } else { 1; }; +} diff --git a/examples/realworld/senline-domain-worker/packages/sgjson-contract/README.md b/examples/realworld/senline-domain-worker/packages/sgjson-contract/README.md new file mode 100644 index 00000000..c9324977 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgjson-contract/README.md @@ -0,0 +1,63 @@ +# sgjson_contract + +`sgjson_contract` is an incubating `0.x` Sengoo package for small, explicit, +closed JSON contracts. It builds on `std::json`; it does not implement a JSON +parser or a JSON Schema dialect. + +## Input Contract + +Every `JsonValue` passed to this package must belong to a live `JsonDoc` +created by `json_parse_strict` or `json_parse_buffer_strict`. `JsonValue` does +not retain parser provenance, so the package cannot detect a value produced by +the permissive parser or recover duplicate-key evidence that was already lost. + +An exact object decoder combines: + +1. `sgjson_exact_object_fields` with a contract-owned + `fn(String) -> bool` callback for the closed field count and explicit + allowlist. The callback must represent exactly the expected field set for + that `expected_len`, not a superset shared by multiple schema versions. +2. One `sgjson_required_*` call for every declared field. + +This rejects missing, additional, substituted, and wrong-typed fields without +coercion or defaults. The callback receives each actual decoded key as an +owned `String`; comparison is exact UTF-8 equality with no normalization or +case folding. An equal-count field substitution is therefore classified as +`SGJSON_UNKNOWN_FIELD` before required getters run. A key longer than the +caller's non-negative `max_key_len` also fails closed as +`SGJSON_UNKNOWN_FIELD`; an invalid negative limit returns +`SGJSON_OUT_OF_RANGE`. + +The strict parser is a required precondition: the allowlist helper cannot +recover duplicate-key evidence after a permissive parser has collapsed it. +`json_parse_strict` and `json_parse_buffer_strict` reject duplicates before +this decoder pattern inspects fields. + +## API + +- Required values: string, integer, boolean, object, and array. +- Exact object fields through a contract-owned callback allowlist. +- Integer ranges and three-value closed enums. +- Bounded ASCII strings and exact-length lowercase hexadecimal strings. +- Bounded strictly sorted, duplicate-free string arrays. + +Stable errors are `SGJSON_MISSING_FIELD`, `SGJSON_UNKNOWN_FIELD`, +`SGJSON_WRONG_KIND`, `SGJSON_OUT_OF_RANGE`, `SGJSON_UNKNOWN_ENUM`, +`SGJSON_INVALID_STRING`, `SGJSON_UNSORTED_OR_DUPLICATE`, and +`SGJSON_RUNTIME_FAILURE`. `SGJSON_WRONG_KIND` is reserved for a successfully +inspected JSON value whose kind does not match the contract. A failed runtime +lookup, inspection, or owned-string extraction returns +`SGJSON_RUNTIME_FAILURE` instead of being presented as malformed input. + +## Incubation Status + +- First consumer: `senline-domain-worker`. +- Independent consumers: none. +- Supported evidence: locked Windows source-development positive and negative + package tests, API docs, release build, and a local publish dry run. +- Missing evidence: long-soak and fault-injection lifecycle gates, malformed + corpus, installed Windows/Linux toolchains, package license metadata/files, + registry publication, and an independent non-Senline consumer. + +Project DTOs, Senline enums, coercion, defaulting, and general schema execution +do not belong in this package. diff --git a/examples/realworld/senline-domain-worker/packages/sgjson-contract/Sengoo.lock b/examples/realworld/senline-domain-worker/packages/sgjson-contract/Sengoo.lock new file mode 100644 index 00000000..afae0fea --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgjson-contract/Sengoo.lock @@ -0,0 +1,11 @@ +# This file is generated by sgpm update. +version = 2 +root = "sgjson_contract" + +[[package]] +id = "sgjson_contract@0.1.0+path:." +name = "sgjson_contract" +version = "0.1.0" +source.kind = "path" +source.path = "." +manifest = "Sengoo.toml" diff --git a/examples/realworld/senline-domain-worker/packages/sgjson-contract/Sengoo.toml b/examples/realworld/senline-domain-worker/packages/sgjson-contract/Sengoo.toml new file mode 100644 index 00000000..18628c1b --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgjson-contract/Sengoo.toml @@ -0,0 +1,8 @@ +[package] +name = "sgjson_contract" +version = "0.1.0" +edition = "2026" +description = "Strict exact-shape JSON contract validation helpers." + +[lib] +path = "src/lib.sg" diff --git a/examples/realworld/senline-domain-worker/packages/sgjson-contract/src/lib.sg b/examples/realworld/senline-domain-worker/packages/sgjson-contract/src/lib.sg new file mode 100644 index 00000000..f9abd65e --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgjson-contract/src/lib.sg @@ -0,0 +1,182 @@ +import std::json; + +import std::ffi; + +import std::status; + +import std::string; + +def SGJSON_MISSING_FIELD() -> i64 { + 2201; +} + +def SGJSON_UNKNOWN_FIELD() -> i64 { + 2202; +} + +def SGJSON_WRONG_KIND() -> i64 { + 2203; +} + +def SGJSON_OUT_OF_RANGE() -> i64 { + 2204; +} + +def SGJSON_UNKNOWN_ENUM() -> i64 { + 2205; +} + +def SGJSON_INVALID_STRING() -> i64 { + 2206; +} + +def SGJSON_UNSORTED_OR_DUPLICATE() -> i64 { + 2207; +} + +def SGJSON_RUNTIME_FAILURE() -> i64 { + 2208; +} + +def sgjson_bool_ok(value: bool) -> Result { + Result { is_ok: true, value: value, error: 0 }; +} + +def sgjson_bool_error(code: i64) -> Result { + Result { is_ok: false, value: false, error: code }; +} + +def sgjson_i64_ok(value: i64) -> Result { + Result { is_ok: true, value: value, error: 0 }; +} + +def sgjson_i64_error(code: i64) -> Result { + Result { is_ok: false, value: 0, error: code }; +} + +def sgjson_string_ok(value: String) -> Result { + Result { is_ok: true, value: value, error: 0 }; +} + +def sgjson_string_error(code: i64) -> Result { + Result { is_ok: false, value: String { handle: 0 }, error: code }; +} + +def sgjson_value_ok(value: JsonValue) -> Result { + Result { is_ok: true, value: value, error: 0 }; +} + +def sgjson_value_error(code: i64) -> Result { + Result { is_ok: false, value: JsonValue { doc_handle: 0, node_id: 0 }, error: code }; +} + +def sgjson_required_value(root: JsonValue, key: &str, expected_kind: i64) -> Result { + let root_kind = root.kind(); + if root_kind.is_err() { return sgjson_value_error(SGJSON_RUNTIME_FAILURE()); }; + if root_kind.value != JSON_KIND_OBJECT() { return sgjson_value_error(SGJSON_WRONG_KIND()); }; + if not root.object_has(key) { return sgjson_value_error(SGJSON_MISSING_FIELD()); }; + let found = root.object_get(key); + if found.is_err() { return sgjson_value_error(SGJSON_RUNTIME_FAILURE()); }; + let value = found.value; + let value_kind = value.kind(); + if value_kind.is_err() { return sgjson_value_error(SGJSON_RUNTIME_FAILURE()); }; + if value_kind.value != expected_kind { sgjson_value_error(SGJSON_WRONG_KIND()); } else { sgjson_value_ok(value); }; +} + +def sgjson_exact_object_len(root: JsonValue, expected_len: i64) -> Result { + if expected_len < 0 { return sgjson_bool_error(SGJSON_OUT_OF_RANGE()); }; + let root_kind = root.kind(); + if root_kind.is_err() { return sgjson_bool_error(SGJSON_RUNTIME_FAILURE()); }; + if root_kind.value != JSON_KIND_OBJECT() { return sgjson_bool_error(SGJSON_WRONG_KIND()); }; + let actual = root.object_len(); + if actual.is_err() { sgjson_bool_error(SGJSON_RUNTIME_FAILURE()); } else if actual.value < expected_len { sgjson_bool_error(SGJSON_MISSING_FIELD()); } else if actual.value > expected_len { sgjson_bool_error(SGJSON_UNKNOWN_FIELD()); } else { sgjson_bool_ok(true); }; +} + +def sgjson_exact_object_fields(root: JsonValue, expected_len: i64, max_key_len: i64, allowed: fn(String) -> bool) -> Result { + if max_key_len < 0 { return sgjson_bool_error(SGJSON_OUT_OF_RANGE()); }; + let exact = sgjson_exact_object_len(root, expected_len); + if exact.is_err() { return sgjson_bool_error(exact.error); }; + if expected_len == 0 { return sgjson_bool_ok(true); }; + let capacity = if max_key_len == 0 { 1; } else { max_key_len; }; + let buffer_result = ffi_buffer_new(capacity); + if buffer_result.is_err() { return sgjson_bool_error(SGJSON_RUNTIME_FAILURE()); }; + let buffer = buffer_result.value; + let mut index = 0; + while index < expected_len { let copied = root.object_key_copy(index, buffer); if copied.is_err() { buffer.free(); if copied.error == STATUS_BUFFER_TOO_SMALL() { return sgjson_bool_error(SGJSON_UNKNOWN_FIELD()); }; return sgjson_bool_error(SGJSON_RUNTIME_FAILURE()); }; if copied.value > max_key_len { buffer.free(); return sgjson_bool_error(SGJSON_UNKNOWN_FIELD()); }; let owned = string_from_buffer(buffer, copied.value); if owned.is_err() { buffer.free(); return sgjson_bool_error(SGJSON_RUNTIME_FAILURE()); }; if not allowed(owned.value) { buffer.free(); return sgjson_bool_error(SGJSON_UNKNOWN_FIELD()); }; index = index + 1; }; + buffer.free(); + sgjson_bool_ok(true); +} + +def sgjson_required_object(root: JsonValue, key: &str) -> Result { + sgjson_required_value(root, key, JSON_KIND_OBJECT()); +} + +def sgjson_required_array(root: JsonValue, key: &str) -> Result { + sgjson_required_value(root, key, JSON_KIND_ARRAY()); +} + +def sgjson_required_string(root: JsonValue, key: &str) -> Result { + let value = sgjson_required_value(root, key, JSON_KIND_STRING()); + if value.is_err() { return sgjson_string_error(value.error); }; + let text = value.value.string_value(); + if text.is_err() { sgjson_string_error(SGJSON_RUNTIME_FAILURE()); } else { sgjson_string_ok(text.value); }; +} + +def sgjson_required_i64(root: JsonValue, key: &str, min_value: i64, max_value: i64) -> Result { + if min_value > max_value { return sgjson_i64_error(SGJSON_OUT_OF_RANGE()); }; + let value = sgjson_required_value(root, key, JSON_KIND_NUMBER()); + if value.is_err() { return sgjson_i64_error(value.error); }; + let number = value.value.number_i64(); + if number.is_err() { sgjson_i64_error(SGJSON_WRONG_KIND()); } else if number.value < min_value or number.value > max_value { sgjson_i64_error(SGJSON_OUT_OF_RANGE()); } else { sgjson_i64_ok(number.value); }; +} + +def sgjson_required_bool(root: JsonValue, key: &str) -> Result { + let value = sgjson_required_value(root, key, JSON_KIND_BOOL()); + if value.is_err() { return sgjson_bool_error(value.error); }; + let boolean = value.value.bool_value(); + if boolean.is_err() { sgjson_bool_error(SGJSON_RUNTIME_FAILURE()); } else { sgjson_bool_ok(boolean.value); }; +} + +def sgjson_required_enum3(root: JsonValue, key: &str, first: &str, second: &str, third: &str) -> Result { + let value = sgjson_required_string(root, key); + if value.is_err() { return sgjson_bool_error(value.error); }; + let text = value.value; + if str_eq(text.as_str(), first) or str_eq(text.as_str(), second) or str_eq(text.as_str(), third) { sgjson_bool_ok(true); } else { sgjson_bool_error(SGJSON_UNKNOWN_ENUM()); }; +} + +def sgjson_string_is_ascii_bounded(value: &String, min_len: i64, max_len: i64) -> bool { + let len = value.len(); + if min_len < 0 or min_len > max_len or len < min_len or len > max_len { return false; }; + let bytes = value.bytes(); + let mut index = 0; + let mut valid = true; + while index < len { let byte = bytes.next().unwrap_or(-1); if byte < 0 or byte > 127 { valid = false; }; index = index + 1; }; + bytes.free(); + valid; +} + +def sgjson_required_lower_hex(root: JsonValue, key: &str, exact_len: i64) -> Result { + let value = sgjson_required_string(root, key); + if value.is_err() { return sgjson_bool_error(value.error); }; + let text = value.value; + if exact_len < 0 or text.len() != exact_len { return sgjson_bool_error(SGJSON_INVALID_STRING()); }; + let bytes = text.bytes(); + let mut index = 0; + let mut valid = true; + while index < exact_len { let byte = bytes.next().unwrap_or(-1); if not ((byte >= 48 and byte <= 57) or (byte >= 97 and byte <= 102)) { valid = false; }; index = index + 1; }; + bytes.free(); + if valid { sgjson_bool_ok(true); } else { sgjson_bool_error(SGJSON_INVALID_STRING()); }; +} + +def sgjson_required_sorted_unique_string_array(root: JsonValue, key: &str, min_len: i64, max_len: i64) -> Result { + if min_len < 0 or min_len > max_len { return sgjson_bool_error(SGJSON_OUT_OF_RANGE()); }; + let value = sgjson_required_value(root, key, JSON_KIND_ARRAY()); + if value.is_err() { return sgjson_bool_error(value.error); }; + let array = value.value; + let count = array.array_len(); + if count.is_err() { return sgjson_bool_error(SGJSON_RUNTIME_FAILURE()); }; + if count.value < min_len or count.value > max_len { return sgjson_bool_error(SGJSON_OUT_OF_RANGE()); }; + let mut index = 0; + while index < count.value { let current = array.array_get(index); if current.is_err() { return sgjson_bool_error(SGJSON_RUNTIME_FAILURE()); }; let current_kind = current.value.kind(); if current_kind.is_err() { return sgjson_bool_error(SGJSON_RUNTIME_FAILURE()); }; if current_kind.value != JSON_KIND_STRING() { return sgjson_bool_error(SGJSON_WRONG_KIND()); }; let current_text = current.value.string_value(); if current_text.is_err() { return sgjson_bool_error(SGJSON_RUNTIME_FAILURE()); }; if index + 1 < count.value { let next = array.array_get(index + 1); if next.is_err() { return sgjson_bool_error(SGJSON_RUNTIME_FAILURE()); }; let next_kind = next.value.kind(); if next_kind.is_err() { return sgjson_bool_error(SGJSON_RUNTIME_FAILURE()); }; if next_kind.value != JSON_KIND_STRING() { return sgjson_bool_error(SGJSON_WRONG_KIND()); }; let next_text = next.value.string_value(); if next_text.is_err() { return sgjson_bool_error(SGJSON_RUNTIME_FAILURE()); }; if current_text.value.compare(next_text.value) >= 0 { return sgjson_bool_error(SGJSON_UNSORTED_OR_DUPLICATE()); }; }; index = index + 1; }; + sgjson_bool_ok(true); +} diff --git a/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/array_rejections.sg b/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/array_rejections.sg new file mode 100644 index 00000000..7aa39249 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/array_rejections.sg @@ -0,0 +1,25 @@ +import std::json; + +import sgjson_contract; + +def main() -> i64 { + let doc = json_parse_strict("{\"duplicate\":[\"a\",\"a\"],\"empty\":[],\"prefix\":[\"a\",\"aa\"],\"too_many\":[\"a\",\"b\",\"c\"],\"unsorted\":[\"b\",\"a\"],\"wrong\":[1],\"wrong_container\":\"a\"}").unwrap_or(JsonDoc { handle: 0 }); + let root = doc.root(); + let empty = sgjson_required_sorted_unique_string_array(root, "empty", 0, 0); + let prefix = sgjson_required_sorted_unique_string_array(root, "prefix", 2, 2); + let too_many = sgjson_required_sorted_unique_string_array(root, "too_many", 0, 2); + let wrong = sgjson_required_sorted_unique_string_array(root, "wrong", 1, 1); + let wrong_container = sgjson_required_sorted_unique_string_array(root, "wrong_container", 1, 1); + let unsorted = sgjson_required_sorted_unique_string_array(root, "unsorted", 1, 2); + let duplicate = sgjson_required_sorted_unique_string_array(root, "duplicate", 1, 2); + let invalid_range = sgjson_required_sorted_unique_string_array(root, "empty", 2, 1); + if empty.is_err() { return 1; }; + if prefix.is_err() { return 2; }; + if not too_many.is_err() or too_many.error != SGJSON_OUT_OF_RANGE() { return 3; }; + if not wrong.is_err() or wrong.error != SGJSON_WRONG_KIND() { return 4; }; + if not wrong_container.is_err() or wrong_container.error != SGJSON_WRONG_KIND() { return 5; }; + if not unsorted.is_err() or unsorted.error != SGJSON_UNSORTED_OR_DUPLICATE() { return 6; }; + if not duplicate.is_err() or duplicate.error != SGJSON_UNSORTED_OR_DUPLICATE() { return 7; }; + if not invalid_range.is_err() or invalid_range.error != SGJSON_OUT_OF_RANGE() { return 8; }; + 0; +} diff --git a/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/nested_contract.sg b/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/nested_contract.sg new file mode 100644 index 00000000..64d390cb --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/nested_contract.sg @@ -0,0 +1,15 @@ +import std::json; + +import sgjson_contract; + +def main() -> i64 { + let doc = json_parse_strict("{\"array\":[],\"object\":{},\"wrong\":true}").unwrap_or(JsonDoc { handle: 0 }); + let root = doc.root(); + let object = sgjson_required_object(root, "object"); + let array = sgjson_required_array(root, "array"); + let wrong_object = sgjson_required_object(root, "wrong"); + let wrong_array = sgjson_required_array(root, "wrong"); + let missing = sgjson_required_object(root, "missing"); + let ok = object.is_ok() and array.is_ok() and wrong_object.is_err() and wrong_object.error == SGJSON_WRONG_KIND() and wrong_array.is_err() and wrong_array.error == SGJSON_WRONG_KIND() and missing.is_err() and missing.error == SGJSON_MISSING_FIELD(); + if ok { 0; } else { 1; }; +} diff --git a/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/object_contract.sg b/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/object_contract.sg new file mode 100644 index 00000000..3d66ff4d --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/object_contract.sg @@ -0,0 +1,41 @@ +import std::ffi; + +import std::json; + +import sgjson_contract; + +def main() -> i64 { + let input = ffi_buffer_from_bytes("{\"caps\":[\"a\",\"b\"],\"count\":3,\"hex\":\"0a1f\",\"mode\":\"new\",\"name\":\"worker\",\"ok\":true}").unwrap_or(Buffer { handle: 0 }); + let unsorted_input = ffi_buffer_from_bytes("{\"caps\":[\"b\",\"a\"]}").unwrap_or(Buffer { handle: 0 }); + let duplicate_input = ffi_buffer_from_bytes("{\"caps\":[\"a\",\"a\"]}").unwrap_or(Buffer { handle: 0 }); + let unknown_input = ffi_buffer_from_bytes("{\"name\":\"worker\",\"extra\":1}").unwrap_or(Buffer { handle: 0 }); + let doc = json_parse_buffer_strict(input, input.len()).unwrap_or(JsonDoc { handle: 0 }); + let unsorted_doc = json_parse_buffer_strict(unsorted_input, unsorted_input.len()).unwrap_or(JsonDoc { handle: 0 }); + let duplicate_doc = json_parse_buffer_strict(duplicate_input, duplicate_input.len()).unwrap_or(JsonDoc { handle: 0 }); + let unknown_doc = json_parse_buffer_strict(unknown_input, unknown_input.len()).unwrap_or(JsonDoc { handle: 0 }); + let root = doc.root(); + let exact = sgjson_exact_object_len(root, 6).unwrap_or(false); + let name = sgjson_required_string(root, "name").unwrap_or(String { handle: 0 }); + let count = sgjson_required_i64(root, "count", 0, 10).unwrap_or(-1); + let enabled = sgjson_required_bool(root, "ok").unwrap_or(false); + let mode = sgjson_required_enum3(root, "mode", "new", "exact_duplicate", "conflict"); + let ascii = sgjson_string_is_ascii_bounded(&name, 1, 32); + let lower_hex = sgjson_required_lower_hex(root, "hex", 4).unwrap_or(false); + let sorted = sgjson_required_sorted_unique_string_array(root, "caps", 1, 4).unwrap_or(false); + let missing = sgjson_required_string(root, "missing"); + let wrong_kind = sgjson_required_string(root, "count"); + let out_of_range = sgjson_required_i64(root, "count", 4, 10); + let unknown = sgjson_exact_object_len(unknown_doc.root(), 1); + let unsorted = sgjson_required_sorted_unique_string_array(unsorted_doc.root(), "caps", 1, 4); + let duplicate = sgjson_required_sorted_unique_string_array(duplicate_doc.root(), "caps", 1, 4); + let ok = exact and count == 3 and enabled and mode.is_ok and ascii and lower_hex and sorted and missing.is_err() and missing.error == SGJSON_MISSING_FIELD() and wrong_kind.is_err() and wrong_kind.error == SGJSON_WRONG_KIND() and out_of_range.is_err() and out_of_range.error == SGJSON_OUT_OF_RANGE() and unknown.is_err() and unknown.error == SGJSON_UNKNOWN_FIELD() and unsorted.is_err() and unsorted.error == SGJSON_UNSORTED_OR_DUPLICATE() and duplicate.is_err() and duplicate.error == SGJSON_UNSORTED_OR_DUPLICATE(); + doc.close(); + unsorted_doc.close(); + duplicate_doc.close(); + unknown_doc.close(); + input.free(); + unsorted_input.free(); + duplicate_input.free(); + unknown_input.free(); + if ok { 0; } else { 1; }; +} diff --git a/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/object_len_contract.sg b/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/object_len_contract.sg new file mode 100644 index 00000000..75a701b2 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/object_len_contract.sg @@ -0,0 +1,9 @@ +import std::json; + +import sgjson_contract; + +def main() -> i64 { + let doc = json_parse_strict("{\"name\":\"worker\"}").unwrap_or(JsonDoc { handle: 0 }); + let invalid_expected_len = sgjson_exact_object_len(doc.root(), -1); + if invalid_expected_len.is_err() and invalid_expected_len.error == SGJSON_OUT_OF_RANGE() { 0; } else { 1; }; +} diff --git a/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/object_shape_rejections.sg b/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/object_shape_rejections.sg new file mode 100644 index 00000000..df381afc --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/object_shape_rejections.sg @@ -0,0 +1,24 @@ +import std::json; + +import std::string; + +import sgjson_contract; + +def main() -> i64 { + let allows_worker_field: fn(String) -> bool = |key| str_eq(key.as_str(), "name") or str_eq(key.as_str(), "ok"); + let valid = json_parse_strict("{\"name\":\"worker\",\"ok\":true}").unwrap_or(JsonDoc { handle: 0 }); + let substituted = json_parse_strict("{\"extra\":true,\"name\":\"worker\"}").unwrap_or(JsonDoc { handle: 0 }); + let missing = json_parse_strict("{\"name\":\"worker\"}").unwrap_or(JsonDoc { handle: 0 }); + let unknown = json_parse_strict("{\"extra\":1,\"name\":\"worker\",\"ok\":true}").unwrap_or(JsonDoc { handle: 0 }); + let overlong = json_parse_strict("{\"long\":true,\"name\":\"worker\"}").unwrap_or(JsonDoc { handle: 0 }); + let wrong_root = json_parse_strict("[]").unwrap_or(JsonDoc { handle: 0 }); + let exact = sgjson_exact_object_fields(valid.root(), 2, 8, allows_worker_field); + let substituted_fields = sgjson_exact_object_fields(substituted.root(), 2, 8, allows_worker_field); + let missing_fields = sgjson_exact_object_fields(missing.root(), 2, 8, allows_worker_field); + let unknown_fields = sgjson_exact_object_fields(unknown.root(), 2, 8, allows_worker_field); + let overlong_key = sgjson_exact_object_fields(overlong.root(), 2, 3, allows_worker_field); + let invalid_key_limit = sgjson_exact_object_fields(valid.root(), 2, -1, allows_worker_field); + let wrong_kind = sgjson_exact_object_fields(wrong_root.root(), 0, 8, allows_worker_field); + let ok = exact.is_ok() and substituted_fields.is_err() and substituted_fields.error == SGJSON_UNKNOWN_FIELD() and missing_fields.is_err() and missing_fields.error == SGJSON_MISSING_FIELD() and unknown_fields.is_err() and unknown_fields.error == SGJSON_UNKNOWN_FIELD() and overlong_key.is_err() and overlong_key.error == SGJSON_UNKNOWN_FIELD() and invalid_key_limit.is_err() and invalid_key_limit.error == SGJSON_OUT_OF_RANGE() and wrong_kind.is_err() and wrong_kind.error == SGJSON_WRONG_KIND(); + if ok { 0; } else { 1; }; +} diff --git a/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/runtime_failure_classification.sg b/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/runtime_failure_classification.sg new file mode 100644 index 00000000..eed491c8 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/runtime_failure_classification.sg @@ -0,0 +1,11 @@ +import std::json; + +import sgjson_contract; + +def main() -> i64 { + let doc = json_parse_strict("{\"name\":\"worker\"}").unwrap_or(JsonDoc { handle: 0 }); + let stale_root = doc.root(); + let closed = doc.close(); + let result = sgjson_required_string(stale_root, "name"); + if closed and result.is_err() and result.error == SGJSON_RUNTIME_FAILURE() { 0; } else { 1; }; +} diff --git a/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/scalar_rejections.sg b/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/scalar_rejections.sg new file mode 100644 index 00000000..ff2d3cb6 --- /dev/null +++ b/examples/realworld/senline-domain-worker/packages/sgjson-contract/tests/scalar_rejections.sg @@ -0,0 +1,25 @@ +import std::json; + +import sgjson_contract; + +def main() -> i64 { + let doc = json_parse_strict("{\"ascii\":\"worker\",\"bool\":\"true\",\"fraction\":1.5,\"hex_bad\":\"0g1f\",\"hex_short\":\"0af\",\"hex_upper\":\"0A1f\",\"integer\":10,\"mode\":\"NEW\",\"unicode\":\"\\u00e9\"}").unwrap_or(JsonDoc { handle: 0 }); + let root = doc.root(); + let wrong_bool = sgjson_required_bool(root, "bool"); + let fractional = sgjson_required_i64(root, "fraction", 0, 10); + let bounded = sgjson_required_i64(root, "integer", 10, 10).unwrap_or(-1); + let out_of_range = sgjson_required_i64(root, "integer", 0, 9); + let invalid_range = sgjson_required_i64(root, "integer", 11, 10); + let unknown_enum = sgjson_required_enum3(root, "mode", "new", "exact_duplicate", "conflict"); + let ascii = sgjson_required_string(root, "ascii").unwrap_or(String { handle: 0 }); + let ascii_again = sgjson_required_string(root, "ascii").unwrap_or(String { handle: 0 }); + let unicode = sgjson_required_string(root, "unicode").unwrap_or(String { handle: 0 }); + let ascii_ok = sgjson_string_is_ascii_bounded(&ascii, 1, 6); + let ascii_too_long = sgjson_string_is_ascii_bounded(&ascii_again, 1, 5); + let unicode_rejected = sgjson_string_is_ascii_bounded(&unicode, 1, 4); + let upper_hex = sgjson_required_lower_hex(root, "hex_upper", 4); + let bad_hex = sgjson_required_lower_hex(root, "hex_bad", 4); + let short_hex = sgjson_required_lower_hex(root, "hex_short", 4); + let ok = wrong_bool.is_err() and wrong_bool.error == SGJSON_WRONG_KIND() and fractional.is_err() and fractional.error == SGJSON_WRONG_KIND() and bounded == 10 and out_of_range.is_err() and out_of_range.error == SGJSON_OUT_OF_RANGE() and invalid_range.is_err() and invalid_range.error == SGJSON_OUT_OF_RANGE() and unknown_enum.is_err() and unknown_enum.error == SGJSON_UNKNOWN_ENUM() and ascii_ok and ascii.len() == 6 and not ascii_too_long and not unicode_rejected and unicode.len() == 2 and upper_hex.is_err() and upper_hex.error == SGJSON_INVALID_STRING() and bad_hex.is_err() and bad_hex.error == SGJSON_INVALID_STRING() and short_hex.is_err() and short_hex.error == SGJSON_INVALID_STRING(); + if ok { 0; } else { 1; }; +} diff --git a/examples/realworld/senline-domain-worker/scripts/generate-build-identity.ps1 b/examples/realworld/senline-domain-worker/scripts/generate-build-identity.ps1 new file mode 100644 index 00000000..564408ca --- /dev/null +++ b/examples/realworld/senline-domain-worker/scripts/generate-build-identity.ps1 @@ -0,0 +1,70 @@ +param( + [Parameter(Mandatory = $true)] + [string]$SourceRevision, + [Parameter(Mandatory = $true)] + [string]$ToolchainVersion, + [Parameter(Mandatory = $true)] + [string]$ApplicationVersion, + [Parameter(Mandatory = $true)] + [string]$BuildManifestId, + [Parameter(Mandatory = $true)] + [string]$OutputPath, + [Parameter(Mandatory = $true)] + [string]$HandshakeOutputPath +) + +$ErrorActionPreference = "Stop" + +if ($SourceRevision -cnotmatch '^[0-9a-f]{40}$') { + throw "SourceRevision must be exactly 40 lowercase hexadecimal characters" +} +if ($BuildManifestId -cnotmatch '^[0-9a-f]{64}$') { + throw "BuildManifestId must be exactly 64 lowercase hexadecimal characters" +} +foreach ($version in @($ToolchainVersion, $ApplicationVersion)) { + if ($version -cnotmatch '^[0-9A-Za-z][0-9A-Za-z.+-]{0,63}$') { + throw "versions must be 1..64 portable ASCII identifier characters" + } +} + +$handshake = '{"kind":"handshake","protocol_version":1,"sengoo_source_revision":"' + + $SourceRevision + '","toolchain_version":"' + $ToolchainVersion + + '","application_version":"' + $ApplicationVersion + + '","build_manifest_id":"' + $BuildManifestId + '"}' +$escapedHandshake = $handshake.Replace('\', '\\').Replace('"', '\"') +$source = @" +def senline_build_source_revision() -> &str { + "$SourceRevision"; +} + +def senline_build_toolchain_version() -> &str { + "$ToolchainVersion"; +} + +def senline_build_application_version() -> &str { + "$ApplicationVersion"; +} + +def senline_build_manifest_id() -> &str { + "$BuildManifestId"; +} + +def senline_build_handshake_payload() -> &str { + "$escapedHandshake\n"; +} +"@.Replace("`r`n", "`n") +$source = $source + "`n" + +$utf8NoBom = [System.Text.UTF8Encoding]::new($false) +foreach ($path in @($OutputPath, $HandshakeOutputPath)) { + $parent = Split-Path -Parent ([System.IO.Path]::GetFullPath($path)) + if ($parent) { + [System.IO.Directory]::CreateDirectory($parent) | Out-Null + } +} +[System.IO.File]::WriteAllText([System.IO.Path]::GetFullPath($OutputPath), $source, $utf8NoBom) +[System.IO.File]::WriteAllText( + [System.IO.Path]::GetFullPath($HandshakeOutputPath), + $handshake + "`n", + $utf8NoBom +) diff --git a/examples/realworld/senline-domain-worker/src/lib.sg b/examples/realworld/senline-domain-worker/src/lib.sg new file mode 100644 index 00000000..3e8b1a58 --- /dev/null +++ b/examples/realworld/senline-domain-worker/src/lib.sg @@ -0,0 +1,383 @@ +import std::ffi; + +import std::json; + +import std::string; + +import senline_build_identity; + +import senline_facts_to_plan; + +import sgframing; + +import sgjson_contract; + +struct WorkerEncodedFrameV1 { + payload: Buffer, + len: i64, +} + +def worker_contract_version() -> i64 { + senline_contract_version(); +} + +def worker_operation_version_supported(version: i64) -> bool { + senline_operation_version_supported(version); +} + +def worker_input_length_supported(len: i64) -> bool { + frame_validate_length(len, 32768).is_ok(); +} + +def worker_output_length_supported(len: i64) -> bool { + frame_validate_length(len, 8192).is_ok(); +} + +def worker_module_revision_v1() -> &str { + "1de09ccafa7e8f182af68e82352e2d4be39496b0"; +} + +def worker_handshake_payload_v1() -> &str { + senline_build_handshake_payload(); +} + +def worker_string_error(code: i64) -> Result { + Result { is_ok: false, value: String { handle: 0 }, error: code }; +} + +def worker_i64_ok(value: i64) -> Result { + Result { is_ok: true, value: value, error: 0 }; +} + +def worker_i64_error(code: i64) -> Result { + Result { is_ok: false, value: 0, error: code }; +} + +def worker_bool_ok(value: bool) -> Result { + Result { is_ok: true, value: value, error: 0 }; +} + +def worker_bool_error(code: i64) -> Result { + Result { is_ok: false, value: false, error: code }; +} + +def worker_context_error(code: i64) -> Result { + Result { is_ok: false, value: senline_empty_evaluation_context_v1(), error: code }; +} + +def worker_identifiers_error(code: i64) -> Result { + Result { is_ok: false, value: senline_empty_submit_envelope_identifiers_v1(), error: code }; +} + +def worker_facts_error(code: i64) -> Result { + Result { is_ok: false, value: senline_empty_submit_envelope_facts_v1(), error: code }; +} + +def worker_request_error(code: i64) -> Result { + Result { is_ok: false, value: senline_empty_worker_request_v1(), error: code }; +} + +def worker_encoded_error(code: i64) -> Result { + Result { is_ok: false, value: WorkerEncodedFrameV1 { payload: Buffer { handle: 0 }, len: 0 }, error: code }; +} + +def worker_required_ascii(root: JsonValue, key: &str) -> Result { + let value = sgjson_required_string(root, key); + if value.is_err() { return worker_string_error(value.error); }; + if sgjson_string_is_ascii_bounded(&value.value, 1, 128) { value; } else { worker_string_error(SGJSON_INVALID_STRING()); }; +} + +def worker_required_literal(root: JsonValue, key: &str, expected: &str) -> Result { + let checked = worker_required_ascii(root, key); + if checked.is_err() { return worker_string_error(checked.error); }; + if not str_eq(checked.value.as_str(), expected) { return worker_string_error(SGJSON_UNKNOWN_ENUM()); }; +// Reuse the validated string; a second extract would allocate another owned handle. + checked; +} + +def worker_required_lower_hex_string(root: JsonValue, key: &str, exact_len: i64) -> Result { + let valid = sgjson_required_lower_hex(root, key, exact_len); + if valid.is_err() { return worker_string_error(valid.error); }; + sgjson_required_string(root, key); +} + +def worker_required_singleton_flag(root: JsonValue, key: &str, allowed: &str) -> Result { + let valid = sgjson_required_sorted_unique_string_array(root, key, 0, 1); + if valid.is_err() { return worker_bool_error(valid.error); }; + let array = sgjson_required_array(root, key); + if array.is_err() { return worker_bool_error(array.error); }; + let count = array.value.array_len(); + if count.is_err() { return worker_bool_error(SGJSON_RUNTIME_FAILURE()); }; + if count.value == 0 { return worker_bool_ok(false); }; + let item = array.value.array_get(0); + if item.is_err() { return worker_bool_error(SGJSON_RUNTIME_FAILURE()); }; + let text = item.value.string_value(); + if text.is_err() { return worker_bool_error(SGJSON_RUNTIME_FAILURE()); }; + if str_eq(text.value.as_str(), allowed) { worker_bool_ok(true); } else { worker_bool_error(SGJSON_UNKNOWN_ENUM()); }; +} + +def worker_required_idempotency(root: JsonValue) -> Result { + let valid = sgjson_required_enum3(root, "idempotency_status", "new", "exact_duplicate", "conflict"); + if valid.is_err() { return worker_i64_error(valid.error); }; + let value = sgjson_required_string(root, "idempotency_status"); + if value.is_err() { return worker_i64_error(value.error); }; + if str_eq(value.value.as_str(), "new") { worker_i64_ok(SENLINE_IDEMPOTENCY_NEW()); } else if str_eq(value.value.as_str(), "exact_duplicate") { worker_i64_ok(SENLINE_IDEMPOTENCY_EXACT_DUPLICATE()); } else if str_eq(value.value.as_str(), "conflict") { worker_i64_ok(SENLINE_IDEMPOTENCY_CONFLICT()); } else { worker_i64_error(SGJSON_UNKNOWN_ENUM()); }; +} + +// Borrowed: by-value String params skip auto-Drop for legacy handle ABI, so a +// by-value mode check would leak one owned string per successful evaluation. +def worker_validate_execution_mode(value: &str) -> bool { + str_eq(value, "fixture") or str_eq(value, "shadow") or str_eq(value, "guarded-development") or str_eq(value, "internal-alpha"); +} + +def worker_decode_context_v1(root: JsonValue) -> Result { + let allows_context_field: fn(String) -> bool = |key| str_eq(key.as_str(), "contract_version") or str_eq(key.as_str(), "operation") or str_eq(key.as_str(), "operation_version") or str_eq(key.as_str(), "evaluation_id") or str_eq(key.as_str(), "operation_epoch") or str_eq(key.as_str(), "worker_generation") or str_eq(key.as_str(), "execution_mode") or str_eq(key.as_str(), "worker_bundle_id") or str_eq(key.as_str(), "facts_binding"); + let exact = sgjson_exact_object_fields(root, 9, 64, allows_context_field); + if exact.is_err() { return worker_context_error(exact.error); }; + let contract_version = sgjson_required_i64(root, "contract_version", 1, 1); + if contract_version.is_err() { return worker_context_error(contract_version.error); }; + let operation = worker_required_literal(root, "operation", "submit-envelope"); + if operation.is_err() { return worker_context_error(operation.error); }; + let operation_version = sgjson_required_i64(root, "operation_version", 0, 4294967295); + if operation_version.is_err() { return worker_context_error(operation_version.error); }; + let evaluation_id = worker_required_lower_hex_string(root, "evaluation_id", 32); + if evaluation_id.is_err() { return worker_context_error(evaluation_id.error); }; + let operation_epoch = sgjson_required_i64(root, "operation_epoch", 0, 9007199254740991); + if operation_epoch.is_err() { return worker_context_error(operation_epoch.error); }; + let worker_generation = sgjson_required_i64(root, "worker_generation", 0, 9007199254740991); + if worker_generation.is_err() { return worker_context_error(worker_generation.error); }; + let execution_mode = worker_required_ascii(root, "execution_mode"); + if execution_mode.is_err() { return worker_context_error(execution_mode.error); }; + if not worker_validate_execution_mode(execution_mode.value.as_str()) { return worker_context_error(SGJSON_UNKNOWN_ENUM()); }; + let worker_bundle_id = worker_required_ascii(root, "worker_bundle_id"); + if worker_bundle_id.is_err() { return worker_context_error(worker_bundle_id.error); }; + let facts_binding = worker_required_lower_hex_string(root, "facts_binding", 64); + if facts_binding.is_err() { return worker_context_error(facts_binding.error); }; + Result { is_ok: true, value: EvaluationContextV1 { contract_version: contract_version.value, operation: operation.value, operation_version: operation_version.value, evaluation_id: evaluation_id.value, operation_epoch: operation_epoch.value, worker_generation: worker_generation.value, execution_mode: execution_mode.value, worker_bundle_id: worker_bundle_id.value, facts_binding: facts_binding.value }, error: 0 }; +} + +def worker_decode_identifiers_v1(root: JsonValue) -> Result { + let allows_identifiers_field: fn(String) -> bool = |key| str_eq(key.as_str(), "correlation_ref") or str_eq(key.as_str(), "source_account_ref") or str_eq(key.as_str(), "source_device_ref") or str_eq(key.as_str(), "recipient_account_ref") or str_eq(key.as_str(), "recipient_device_ref") or str_eq(key.as_str(), "conversation_ref") or str_eq(key.as_str(), "envelope_ref"); + let exact = sgjson_exact_object_fields(root, 7, 64, allows_identifiers_field); + if exact.is_err() { return worker_identifiers_error(exact.error); }; + let correlation_ref = worker_required_ascii(root, "correlation_ref"); + if correlation_ref.is_err() { return worker_identifiers_error(correlation_ref.error); }; + let source_account_ref = worker_required_ascii(root, "source_account_ref"); + if source_account_ref.is_err() { return worker_identifiers_error(source_account_ref.error); }; + let source_device_ref = worker_required_ascii(root, "source_device_ref"); + if source_device_ref.is_err() { return worker_identifiers_error(source_device_ref.error); }; + let recipient_account_ref = worker_required_ascii(root, "recipient_account_ref"); + if recipient_account_ref.is_err() { return worker_identifiers_error(recipient_account_ref.error); }; + let recipient_device_ref = worker_required_ascii(root, "recipient_device_ref"); + if recipient_device_ref.is_err() { return worker_identifiers_error(recipient_device_ref.error); }; + let conversation_ref = worker_required_ascii(root, "conversation_ref"); + if conversation_ref.is_err() { return worker_identifiers_error(conversation_ref.error); }; + let envelope_ref = worker_required_ascii(root, "envelope_ref"); + if envelope_ref.is_err() { return worker_identifiers_error(envelope_ref.error); }; + Result { is_ok: true, value: SubmitEnvelopeIdentifiersV1 { correlation_ref: correlation_ref.value, source_account_ref: source_account_ref.value, source_device_ref: source_device_ref.value, recipient_account_ref: recipient_account_ref.value, recipient_device_ref: recipient_device_ref.value, conversation_ref: conversation_ref.value, envelope_ref: envelope_ref.value }, error: 0 }; +} + +def worker_decode_facts_v1(root: JsonValue) -> Result { + let allows_facts_field: fn(String) -> bool = |key| str_eq(key.as_str(), "contract_version") or str_eq(key.as_str(), "operation_version") or str_eq(key.as_str(), "identifiers") or str_eq(key.as_str(), "source_device_status") or str_eq(key.as_str(), "source_device_capabilities") or str_eq(key.as_str(), "envelope_protocol_version") or str_eq(key.as_str(), "ciphertext_length_bytes") or str_eq(key.as_str(), "idempotency_status") or str_eq(key.as_str(), "recipient_pending_count") or str_eq(key.as_str(), "recipient_pending_limit") or str_eq(key.as_str(), "application_envelopes_used") or str_eq(key.as_str(), "application_envelopes_limit") or str_eq(key.as_str(), "ciphertext_limit_bytes") or str_eq(key.as_str(), "feature_flags"); + let exact = sgjson_exact_object_fields(root, 14, 64, allows_facts_field); + if exact.is_err() { return worker_facts_error(exact.error); }; + let contract_version = sgjson_required_i64(root, "contract_version", 1, 1); + if contract_version.is_err() { return worker_facts_error(contract_version.error); }; + let operation_version = sgjson_required_i64(root, "operation_version", 0, 4294967295); + if operation_version.is_err() { return worker_facts_error(operation_version.error); }; + let identifiers_value = sgjson_required_object(root, "identifiers"); + if identifiers_value.is_err() { return worker_facts_error(identifiers_value.error); }; + let identifiers = worker_decode_identifiers_v1(identifiers_value.value); + if identifiers.is_err() { return worker_facts_error(identifiers.error); }; + let source_status = worker_required_literal(root, "source_device_status", "active"); + if source_status.is_err() { return worker_facts_error(source_status.error); }; + let capability = worker_required_singleton_flag(root, "source_device_capabilities", "submit_envelope_v2"); + if capability.is_err() { return worker_facts_error(capability.error); }; + let envelope_protocol_version = sgjson_required_i64(root, "envelope_protocol_version", 2, 2); + if envelope_protocol_version.is_err() { return worker_facts_error(envelope_protocol_version.error); }; + let ciphertext_length_bytes = sgjson_required_i64(root, "ciphertext_length_bytes", 0, 4294967295); + if ciphertext_length_bytes.is_err() { return worker_facts_error(ciphertext_length_bytes.error); }; + let idempotency_status = worker_required_idempotency(root); + if idempotency_status.is_err() { return worker_facts_error(idempotency_status.error); }; + let recipient_pending_count = sgjson_required_i64(root, "recipient_pending_count", 0, 4294967295); + if recipient_pending_count.is_err() { return worker_facts_error(recipient_pending_count.error); }; + let recipient_pending_limit = sgjson_required_i64(root, "recipient_pending_limit", 0, 4294967295); + if recipient_pending_limit.is_err() { return worker_facts_error(recipient_pending_limit.error); }; + let application_envelopes_used = sgjson_required_i64(root, "application_envelopes_used", 0, 4294967295); + if application_envelopes_used.is_err() { return worker_facts_error(application_envelopes_used.error); }; + let application_envelopes_limit = sgjson_required_i64(root, "application_envelopes_limit", 0, 4294967295); + if application_envelopes_limit.is_err() { return worker_facts_error(application_envelopes_limit.error); }; + let ciphertext_limit_bytes = sgjson_required_i64(root, "ciphertext_limit_bytes", 0, 4294967295); + if ciphertext_limit_bytes.is_err() { return worker_facts_error(ciphertext_limit_bytes.error); }; + if ciphertext_length_bytes.value > ciphertext_limit_bytes.value { return worker_facts_error(SGJSON_OUT_OF_RANGE()); }; + let feature = worker_required_singleton_flag(root, "feature_flags", "enqueue_delivery"); + if feature.is_err() { return worker_facts_error(feature.error); }; + Result { is_ok: true, value: SubmitEnvelopeFactsV1 { contract_version: contract_version.value, operation_version: operation_version.value, identifiers: identifiers.value, source_device_active: true, has_submit_envelope_v2: capability.value, envelope_protocol_version: envelope_protocol_version.value, ciphertext_length_bytes: ciphertext_length_bytes.value, idempotency_status: idempotency_status.value, recipient_pending_count: recipient_pending_count.value, recipient_pending_limit: recipient_pending_limit.value, application_envelopes_used: application_envelopes_used.value, application_envelopes_limit: application_envelopes_limit.value, ciphertext_limit_bytes: ciphertext_limit_bytes.value, enqueue_delivery_enabled: feature.value }, error: 0 }; +} + +def worker_decode_request_v1(root: JsonValue) -> Result { + let allows_request_field: fn(String) -> bool = |key| str_eq(key.as_str(), "kind") or str_eq(key.as_str(), "schema_version") or str_eq(key.as_str(), "context") or str_eq(key.as_str(), "facts"); + let exact = sgjson_exact_object_fields(root, 4, 64, allows_request_field); + if exact.is_err() { return worker_request_error(exact.error); }; + let kind = worker_required_literal(root, "kind", "evaluation"); + if kind.is_err() { return worker_request_error(kind.error); }; + let schema_version = sgjson_required_i64(root, "schema_version", 1, 1); + if schema_version.is_err() { return worker_request_error(schema_version.error); }; + let context_value = sgjson_required_object(root, "context"); + if context_value.is_err() { return worker_request_error(context_value.error); }; + let facts_value = sgjson_required_object(root, "facts"); + if facts_value.is_err() { return worker_request_error(facts_value.error); }; + let context = worker_decode_context_v1(context_value.value); + if context.is_err() { return worker_request_error(context.error); }; + let facts = worker_decode_facts_v1(facts_value.value); + if facts.is_err() { return worker_request_error(facts.error); }; + if context.value.contract_version != facts.value.contract_version or context.value.operation_version != facts.value.operation_version { return worker_request_error(SGJSON_OUT_OF_RANGE()); }; + Result { is_ok: true, value: WorkerRequestV1 { schema_version: schema_version.value, context: context.value, facts: facts.value }, error: 0 }; +} + +def worker_validate_request_envelope_v1(root: JsonValue) -> Result { + let decoded = worker_decode_request_v1(root); + if decoded.is_err() { return Result { is_ok: false, value: false, error: decoded.error }; }; + if not worker_operation_version_supported(decoded.value.context.operation_version) { Result { is_ok: false, value: false, error: SGJSON_OUT_OF_RANGE() }; } else { Result { is_ok: true, value: true, error: 0 }; }; +} + +def worker_encode_protocol_error_v1(code: &str) -> Result { + let doc_result = json_doc_object(); + if doc_result.is_err() { return worker_encoded_error(doc_result.error); }; + let doc = doc_result.value; + let root = doc.root(); + root.object_set("kind", doc.new_string("error")?)?; + root.object_set("schema_version", doc.new_number(1.0)?)?; + root.object_set("scope", doc.new_string("protocol")?)?; + root.object_set("code", doc.new_string(code)?)?; + root.object_set("evaluation_id", doc.new_null()?)?; + let output = ffi_buffer_new(8192)?; + let serialized = doc.serialize(output)?; + if serialized <= 0 or serialized >= 8192 { return worker_encoded_error(SGFRAMING_LIMIT_EXCEEDED()); }; + let newline = output.set_u8(serialized, 10); + if newline.is_err() { return worker_encoded_error(newline.error); }; + Result { is_ok: true, value: WorkerEncodedFrameV1 { payload: output, len: serialized + 1 }, error: 0 }; +} + +def worker_encode_malformed_json_error_v1() -> Result { + worker_encode_protocol_error_v1("malformed_json"); +} + +def worker_encode_schema_error_v1(code: i64) -> Result { + if code == SGJSON_UNKNOWN_FIELD() { worker_encode_protocol_error_v1("unknown_field"); } else if code == SGJSON_UNKNOWN_ENUM() { worker_encode_protocol_error_v1("unknown_enum"); } else { worker_encode_malformed_json_error_v1(); }; +} + +def worker_encode_parser_error_v1(kind: i64) -> Result { + if kind == JSON_ERROR_KIND_DUPLICATE_FIELD() { worker_encode_protocol_error_v1("duplicate_field"); } else if kind == JSON_ERROR_KIND_INVALID_UNICODE() { worker_encode_protocol_error_v1("invalid_unicode"); } else if kind == JSON_ERROR_KIND_TRAILING_BYTES() { worker_encode_protocol_error_v1("trailing_bytes"); } else { worker_encode_malformed_json_error_v1(); }; +} + +def worker_encode_unsupported_operation_error_v1(evaluation_id: &String) -> Result { + let doc_result = json_doc_object(); + if doc_result.is_err() { return worker_encoded_error(doc_result.error); }; + let doc = doc_result.value; + let root = doc.root(); + root.object_set("kind", doc.new_string("error")?)?; + root.object_set("schema_version", doc.new_number(1.0)?)?; + root.object_set("scope", doc.new_string("evaluation")?)?; + root.object_set("code", doc.new_string("unsupported_operation_version")?)?; + root.object_set("evaluation_id", doc.new_string_from_string(evaluation_id)?)?; + let output = ffi_buffer_new(8192)?; + let serialized = doc.serialize(output)?; + if serialized <= 0 or serialized >= 8192 { return worker_encoded_error(SGFRAMING_LIMIT_EXCEEDED()); }; + let newline = output.set_u8(serialized, 10); + if newline.is_err() { return worker_encoded_error(newline.error); }; + Result { is_ok: true, value: WorkerEncodedFrameV1 { payload: output, len: serialized + 1 }, error: 0 }; +} + +def worker_encode_plan_v1(plan: SubmitEnvelopePlanV1) -> Result { + let doc_result = json_doc_object(); + if doc_result.is_err() { return worker_encoded_error(doc_result.error); }; + let doc = doc_result.value; + let root = doc.root(); + root.object_set("kind", doc.new_string("plan")?)?; + root.object_set("schema_version", doc.new_number(plan.schema_version as f64)?)?; + let context = doc.new_object()?; + context.object_set("contract_version", doc.new_number(plan.context.contract_version as f64)?)?; + context.object_set("operation", doc.new_string_from_string(&plan.context.operation)?)?; + context.object_set("operation_version", doc.new_number(plan.context.operation_version as f64)?)?; + context.object_set("evaluation_id", doc.new_string_from_string(&plan.context.evaluation_id)?)?; + context.object_set("operation_epoch", doc.new_number(plan.context.operation_epoch as f64)?)?; + context.object_set("worker_generation", doc.new_number(plan.context.worker_generation as f64)?)?; + context.object_set("execution_mode", doc.new_string_from_string(&plan.context.execution_mode)?)?; + context.object_set("worker_bundle_id", doc.new_string_from_string(&plan.context.worker_bundle_id)?)?; + context.object_set("facts_binding", doc.new_string_from_string(&plan.context.facts_binding)?)?; + root.object_set("context", context)?; + let identifiers = doc.new_object()?; + identifiers.object_set("correlation_ref", doc.new_string_from_string(&plan.identifiers.correlation_ref)?)?; + identifiers.object_set("source_account_ref", doc.new_string_from_string(&plan.identifiers.source_account_ref)?)?; + identifiers.object_set("source_device_ref", doc.new_string_from_string(&plan.identifiers.source_device_ref)?)?; + identifiers.object_set("recipient_account_ref", doc.new_string_from_string(&plan.identifiers.recipient_account_ref)?)?; + identifiers.object_set("recipient_device_ref", doc.new_string_from_string(&plan.identifiers.recipient_device_ref)?)?; + identifiers.object_set("conversation_ref", doc.new_string_from_string(&plan.identifiers.conversation_ref)?)?; + identifiers.object_set("envelope_ref", doc.new_string_from_string(&plan.identifiers.envelope_ref)?)?; + root.object_set("identifiers", identifiers)?; + root.object_set("decision", doc.new_string(senline_decision_name(plan.decision))?)?; + root.object_set("reason", doc.new_string(senline_reason_name(plan.reason))?)?; + root.object_set("sengoo_module_revision", doc.new_string_from_string(&plan.sengoo_module_revision)?)?; + let output = ffi_buffer_new(8192)?; + let serialized = doc.serialize(output)?; + if serialized <= 0 or serialized >= 8192 { return worker_encoded_error(SGFRAMING_LIMIT_EXCEEDED()); }; + let newline = output.set_u8(serialized, 10); + if newline.is_err() { return worker_encoded_error(newline.error); }; + Result { is_ok: true, value: WorkerEncodedFrameV1 { payload: output, len: serialized + 1 }, error: 0 }; +} + +// Owns `request` end-to-end on the unsupported path so nested Strings Drop. +// Avoid if/else where only one branch moves `request`: path-insensitive moved +// tracking would skip Drop on the borrow-only branch and leak every evaluation. +def worker_reject_unsupported_operation_v1(request: WorkerRequestV1) -> Result { + worker_encode_unsupported_operation_error_v1(&request.context.evaluation_id); +} + +def worker_accept_decoded_request_v1(request: WorkerRequestV1) -> Result { + let revision = string_from_str(worker_module_revision_v1()); + if revision.is_err() { return worker_encoded_error(revision.error); }; + worker_encode_plan_v1(plan_submit_envelope_v1(request, revision.value)); +} + +def worker_process_decoded_request_v1(request: WorkerRequestV1) -> Result { + if not worker_operation_version_supported(request.context.operation_version) { worker_reject_unsupported_operation_v1(request); } else { worker_accept_decoded_request_v1(request); }; +} + +def worker_process_payload_v1(payload: Buffer, len: i64) -> Result { + let parsed = json_parse_buffer_strict(payload, len); + if parsed.is_err() { let parser_error_kind = json_last_error_kind(); return worker_encode_parser_error_v1(parser_error_kind); }; + let input_doc = parsed.value; + let decoded = worker_decode_request_v1(input_doc.root()); + if decoded.is_err() { return worker_encode_schema_error_v1(decoded.error); }; + worker_process_decoded_request_v1(decoded.value); +} + +def worker_frame_error_exit(code: i64) -> i64 { + if code == SGFRAMING_ZERO_LENGTH() { 41; } else if code == SGFRAMING_LIMIT_EXCEEDED() { 42; } else if code == SGFRAMING_TRUNCATED() { 43; } else { 44; }; +} + +def worker_request_error_exit(code: i64) -> i64 { + if code >= 2201 and code <= 2208 { 60 + code - 2200; } else if code >= 1 and code <= 19 { 80 + code; } else { 99; }; +} + +def worker_run_stdio_v1_with_frame_writer(frame_writer: fn(Buffer, i64, i64) -> Result) -> i64 { + let handshake = ffi_buffer_from_bytes(worker_handshake_payload_v1()); + if handshake.is_err() { return 31; }; + let handshake_payload = handshake.value; + let handshake_len = handshake_payload.used_len(); + let handshake_written = frame_writer(handshake_payload, handshake_len, 8192); + handshake_payload.free(); + if handshake_written.is_err() or handshake_written.value != handshake_len + 4 { return 32; }; + while true { let read = frame_read_stdin(32768); if read.is_err() { return worker_frame_error_exit(read.error); }; let frame = read.value; if frame.eof { return 0; }; let input_payload = frame.payload; let response = worker_process_payload_v1(input_payload, frame.len); input_payload.free(); if response.is_err() { return worker_request_error_exit(response.error); }; let encoded = response.value; let output_payload = encoded.payload; let written = frame_writer(output_payload, encoded.len, 8192); output_payload.free(); if written.is_err() or written.value != encoded.len + 4 { return 52; }; }; + 0; +} + +def worker_default_frame_writer(payload: Buffer, len: i64, max_len: i64) -> Result { + frame_write_stdout(payload, len, max_len); +} + +def worker_run_stdio_v1() -> i64 { + let writer: fn(Buffer, i64, i64) -> Result = worker_default_frame_writer; + worker_run_stdio_v1_with_frame_writer(writer); +} diff --git a/examples/realworld/senline-domain-worker/src/main.sg b/examples/realworld/senline-domain-worker/src/main.sg new file mode 100644 index 00000000..d4245a0e --- /dev/null +++ b/examples/realworld/senline-domain-worker/src/main.sg @@ -0,0 +1,5 @@ +import senline_domain_worker; + +def main() -> i64 { + worker_run_stdio_v1(); +} diff --git a/examples/realworld/senline-domain-worker/tests/worker_library_integration.sg b/examples/realworld/senline-domain-worker/tests/worker_library_integration.sg new file mode 100644 index 00000000..67b87bdf --- /dev/null +++ b/examples/realworld/senline-domain-worker/tests/worker_library_integration.sg @@ -0,0 +1,21 @@ +import std::json; + +import senline_domain_worker; + +import sgjson_contract; + +def main() -> i64 { + let valid = json_parse_strict("{\"context\":{\"contract_version\":1,\"operation\":\"submit-envelope\",\"operation_version\":1,\"evaluation_id\":\"00000000000000000000000000000001\",\"operation_epoch\":0,\"worker_generation\":0,\"execution_mode\":\"fixture\",\"worker_bundle_id\":\"b\",\"facts_binding\":\"0000000000000000000000000000000000000000000000000000000000000000\"},\"facts\":{\"contract_version\":1,\"operation_version\":1,\"identifiers\":{\"correlation_ref\":\"c\",\"source_account_ref\":\"sa\",\"source_device_ref\":\"sd\",\"recipient_account_ref\":\"ra\",\"recipient_device_ref\":\"rd\",\"conversation_ref\":\"v\",\"envelope_ref\":\"e\"},\"source_device_status\":\"active\",\"source_device_capabilities\":[],\"envelope_protocol_version\":2,\"ciphertext_length_bytes\":0,\"idempotency_status\":\"new\",\"recipient_pending_count\":0,\"recipient_pending_limit\":1,\"application_envelopes_used\":0,\"application_envelopes_limit\":1,\"ciphertext_limit_bytes\":0,\"feature_flags\":[]},\"kind\":\"evaluation\",\"schema_version\":1}").unwrap_or(JsonDoc { handle: 0 }); + let unknown = json_parse_strict("{\"context\":{},\"extra\":true,\"facts\":{},\"kind\":\"evaluation\",\"schema_version\":1}").unwrap_or(JsonDoc { handle: 0 }); + let substituted = json_parse_strict("{\"extra\":{},\"facts\":{},\"kind\":\"evaluation\",\"schema_version\":1}").unwrap_or(JsonDoc { handle: 0 }); + let missing = json_parse_strict("{\"facts\":{},\"kind\":\"evaluation\",\"schema_version\":1}").unwrap_or(JsonDoc { handle: 0 }); + let wrong_kind = json_parse_strict("{\"context\":{},\"facts\":{},\"kind\":\"plan\",\"schema_version\":1}").unwrap_or(JsonDoc { handle: 0 }); + let accepted = worker_validate_request_envelope_v1(valid.root()); + let rejected_unknown = worker_validate_request_envelope_v1(unknown.root()); + let rejected_substituted = worker_validate_request_envelope_v1(substituted.root()); + let rejected_missing = worker_validate_request_envelope_v1(missing.root()); + let rejected_kind = worker_validate_request_envelope_v1(wrong_kind.root()); + let limits = worker_input_length_supported(1) and worker_input_length_supported(32768) and not worker_input_length_supported(0) and not worker_input_length_supported(32769) and worker_output_length_supported(1) and worker_output_length_supported(8192) and not worker_output_length_supported(0) and not worker_output_length_supported(8193); + let ok = accepted.is_ok() and rejected_unknown.is_err() and rejected_unknown.error == SGJSON_UNKNOWN_FIELD() and rejected_substituted.is_err() and rejected_substituted.error == SGJSON_UNKNOWN_FIELD() and rejected_missing.is_err() and rejected_missing.error == SGJSON_MISSING_FIELD() and rejected_kind.is_err() and rejected_kind.error == SGJSON_UNKNOWN_ENUM() and limits; + if ok { 0; } else { 1; }; +} diff --git a/examples/realworld/senline-domain-worker/tests/worker_scaffold.sg b/examples/realworld/senline-domain-worker/tests/worker_scaffold.sg new file mode 100644 index 00000000..2d2ff9d0 --- /dev/null +++ b/examples/realworld/senline-domain-worker/tests/worker_scaffold.sg @@ -0,0 +1,5 @@ +import senline_domain_worker; + +def main() -> i64 { + if worker_contract_version() == 1 and worker_operation_version_supported(1) and not worker_operation_version_supported(99) { 0; } else { 1; }; +} diff --git a/examples/realworld/senline-http-dogfood/README.md b/examples/realworld/senline-http-dogfood/README.md new file mode 100644 index 00000000..b2b41d67 --- /dev/null +++ b/examples/realworld/senline-http-dogfood/README.md @@ -0,0 +1,35 @@ +# Senline HTTP Dogfood + +`senline-http-dogfood` is a development-only localhost harness for synthetic +Senline V1 facts. It depends on the sibling `senline_domain_worker` package and +calls its strict decoder, typed request model, pure planner, and normalized +plan/error encoders directly. It does not carry a second protocol codec. + +The executable has no bind configuration. It permits only `127.0.0.1:0`, asks +the OS for an ephemeral port, prints one machine-readable +`READY 127.0.0.1:` line, and handles one request before closing. The only +accepted transport shape is `POST /v1/submit-envelope HTTP/1.1` with no query, +exact `Content-Type: application/json`, no `Transfer-Encoding`, and a body of +1..32768 bytes. Headers are capped at 4096 bytes and normalized output at 8192 +bytes. Only `execution_mode=fixture` reaches the planner through this harness. + +HTTP response bodies preserve the worker payload exactly, including its final +LF byte. Plan and normalized worker error envelopes use status 200 so transport +status cannot rewrite their frozen bytes. HTTP method, path, header, and body +policy failures are bounded transport-level 400 responses. + +The retained server subset is serial and plaintext with `Connection: close`. +It does not support TLS, keep-alive, streaming, callback handlers, general task +cancellation, production or internal-alpha ingress, deployment manifests, or +client endpoints. Sandbox, supervisor, deadlines, admission, final validation, +mutation, and rollback remain owned by Senline Rust. + +Run the locked source-development loop with: + +```powershell +sgpm --runtime-mode source-development check --locked +sgpm --runtime-mode source-development test --locked +sgpm fmt --check --locked +sgpm --runtime-mode source-development doc --locked +sgpm --runtime-mode source-development build --locked --release +``` diff --git a/examples/realworld/senline-http-dogfood/Sengoo.lock b/examples/realworld/senline-http-dogfood/Sengoo.lock new file mode 100644 index 00000000..a3684cc5 --- /dev/null +++ b/examples/realworld/senline-http-dogfood/Sengoo.lock @@ -0,0 +1,76 @@ +# This file is generated by sgpm update. +version = 2 +root = "senline_http_dogfood" + +[[package]] +id = "senline_build_identity@0.1.0+path:../senline-domain-worker/packages/senline-build-identity" +name = "senline_build_identity" +version = "0.1.0" +source.kind = "path" +source.path = "../senline-domain-worker/packages/senline-build-identity" +manifest = "../senline-domain-worker/packages/senline-build-identity/Sengoo.toml" + +[[package]] +id = "senline_facts_to_plan@0.1.0+path:../senline-domain-worker/packages/senline-facts-to-plan" +name = "senline_facts_to_plan" +version = "0.1.0" +source.kind = "path" +source.path = "../senline-domain-worker/packages/senline-facts-to-plan" +manifest = "../senline-domain-worker/packages/senline-facts-to-plan/Sengoo.toml" + +[[package]] +id = "sgframing@0.1.0+path:../senline-domain-worker/packages/sgframing" +name = "sgframing" +version = "0.1.0" +source.kind = "path" +source.path = "../senline-domain-worker/packages/sgframing" +manifest = "../senline-domain-worker/packages/sgframing/Sengoo.toml" + +[[package]] +id = "sgjson_contract@0.1.0+path:../senline-domain-worker/packages/sgjson-contract" +name = "sgjson_contract" +version = "0.1.0" +source.kind = "path" +source.path = "../senline-domain-worker/packages/sgjson-contract" +manifest = "../senline-domain-worker/packages/sgjson-contract/Sengoo.toml" + +[[package]] +id = "senline_domain_worker@0.1.0+path:../senline-domain-worker" +name = "senline_domain_worker" +version = "0.1.0" +source.kind = "path" +source.path = "../senline-domain-worker" +manifest = "../senline-domain-worker/Sengoo.toml" + +[[package]] +id = "senline_http_dogfood@0.1.0+path:." +name = "senline_http_dogfood" +version = "0.1.0" +source.kind = "path" +source.path = "." +manifest = "Sengoo.toml" + +[[dependency]] +from = "senline_http_dogfood@0.1.0+path:." +alias = "senline_domain_worker" +to = "senline_domain_worker@0.1.0+path:../senline-domain-worker" + +[[dependency]] +from = "senline_domain_worker@0.1.0+path:../senline-domain-worker" +alias = "senline_build_identity" +to = "senline_build_identity@0.1.0+path:../senline-domain-worker/packages/senline-build-identity" + +[[dependency]] +from = "senline_domain_worker@0.1.0+path:../senline-domain-worker" +alias = "senline_facts_to_plan" +to = "senline_facts_to_plan@0.1.0+path:../senline-domain-worker/packages/senline-facts-to-plan" + +[[dependency]] +from = "senline_domain_worker@0.1.0+path:../senline-domain-worker" +alias = "sgframing" +to = "sgframing@0.1.0+path:../senline-domain-worker/packages/sgframing" + +[[dependency]] +from = "senline_domain_worker@0.1.0+path:../senline-domain-worker" +alias = "sgjson_contract" +to = "sgjson_contract@0.1.0+path:../senline-domain-worker/packages/sgjson-contract" diff --git a/examples/realworld/senline-http-dogfood/Sengoo.toml b/examples/realworld/senline-http-dogfood/Sengoo.toml new file mode 100644 index 00000000..a8c4544f --- /dev/null +++ b/examples/realworld/senline-http-dogfood/Sengoo.toml @@ -0,0 +1,14 @@ +[package] +name = "senline_http_dogfood" +version = "0.1.0" +edition = "2026" +description = "Loopback-only synthetic HTTP harness for the Senline domain planner." + +[bin] +path = "src/main.sg" + +[lib] +path = "src/lib.sg" + +[dependencies] +senline_domain_worker = { path = "../senline-domain-worker" } diff --git a/examples/realworld/senline-http-dogfood/src/lib.sg b/examples/realworld/senline-http-dogfood/src/lib.sg new file mode 100644 index 00000000..786ebe76 --- /dev/null +++ b/examples/realworld/senline-http-dogfood/src/lib.sg @@ -0,0 +1,151 @@ +import std::ffi; + +import std::json; + +import std::net; + +import std::status; + +import std::string; + +import senline_domain_worker; + +def senline_http_header_limit() -> i64 { + 4096; +} + +def senline_http_body_limit() -> i64 { + 32768; +} + +def senline_http_output_limit() -> i64 { + 8192; +} + +def senline_http_request_timeout_ms() -> i64 { + 5000; +} + +def senline_http_max_requests_per_process() -> i64 { + 1; +} + +def senline_http_max_pending_requests() -> i64 { + 1; +} + +def senline_http_bind_forbidden_status() -> i64 { + 2601; +} + +def senline_http_transport_status() -> i64 { + 2602; +} + +def senline_http_bind_allowed(host: &str, port: i64) -> bool { + str_eq(host, "127.0.0.1") and port == 0; +} + +def senline_http_method_allowed(method: &str) -> bool { + str_eq(method, "POST"); +} + +def senline_http_path_allowed(path: &str, query: &str) -> bool { + str_eq(path, "/v1/submit-envelope") and query.len() == 0; +} + +def senline_http_version_allowed(version: &str) -> bool { + str_eq(version, "HTTP/1.1"); +} + +def senline_http_content_type_allowed(content_type: &str) -> bool { + str_eq(content_type, "application/json"); +} + +def senline_http_transfer_encoding_allowed(present: bool) -> bool { + not present; +} + +def senline_http_header_length_supported(len: i64) -> bool { + len >= 0 and len <= senline_http_header_limit(); +} + +def senline_http_body_length_supported(len: i64) -> bool { + len > 0 and len <= senline_http_body_limit(); +} + +def senline_http_server_error(code: i64) -> Result { + Result { is_ok: false, value: HttpServer { handle: 0 }, error: code }; +} + +def senline_http_bind(host: &str, port: i64) -> Result { + if not senline_http_bind_allowed(host, port) { return senline_http_server_error(senline_http_bind_forbidden_status()); }; + let bound = http_server_bind(host, port); + if bound.is_err() { return senline_http_server_error(bound.error); }; + let server = bound.value; + let limits = server.set_limits(senline_http_header_limit(), senline_http_body_limit()); + if limits.is_err() { server.close(); senline_http_server_error(limits.error); } else { Result { is_ok: true, value: server, error: 0 }; }; +} + +def senline_http_request_supported(request: HttpServerRequest) -> bool { + let method = request.method_string(); + if method.is_err() { return false; }; + if not senline_http_method_allowed(method.value.as_str()) { return false; }; + let path = request.path_string(); + if path.is_err() { return false; }; + let query = request.query_string(); + if query.is_err() { return false; }; + if not senline_http_path_allowed(path.value.as_str(), query.value.as_str()) { return false; }; + let version = request.version_string(); + if version.is_err() { return false; }; + if not senline_http_version_allowed(version.value.as_str()) { return false; }; + let content_type = request.header_string("Content-Type"); + if content_type.is_err() { return false; }; + if not senline_http_content_type_allowed(content_type.value.as_str()) { return false; }; + let transfer_encoding = request.header_len("Transfer-Encoding"); + if not senline_http_transfer_encoding_allowed(transfer_encoding.is_ok()) { return false; }; + if transfer_encoding.error != STATUS_NOT_FOUND() { return false; }; + let body_len = request.body_len(); + if body_len.is_err() { return false; }; + senline_http_body_length_supported(body_len.value); +} + +def senline_http_process_payload(payload: Buffer, len: i64) -> Result { + let response = worker_process_payload_v1(payload, len); + if response.is_err() { return response; }; + let parsed = json_parse_buffer_strict(payload, len); + if parsed.is_err() { return response; }; + let decoded = worker_decode_request_v1(parsed.value.root()); + if decoded.is_err() { return response; }; + if str_eq(decoded.value.context.execution_mode.as_str(), "fixture") { return response; }; + let rejected_output = response.value.payload; + rejected_output.free(); + worker_encode_protocol_error_v1("unknown_enum"); +} + +def senline_http_answer_internal_error(request: HttpServerRequest) -> bool { + request.respond(500, "senline_http_internal_error").unwrap_or(false); +} + +def senline_http_handle_request(request: HttpServerRequest) -> bool { + if not senline_http_request_supported(request) { return request.respond(400, "senline_http_bad_request").unwrap_or(false); }; + let body_len_result = request.body_len(); + if body_len_result.is_err() { return senline_http_answer_internal_error(request); }; + let body_len = body_len_result.value; + let input_result = ffi_buffer_new(body_len); + if input_result.is_err() { return senline_http_answer_internal_error(request); }; + let input = input_result.value; + let copied = request.body_copy(input); + if copied.is_err() { input.free(); return senline_http_answer_internal_error(request); }; + if copied.value != body_len { input.free(); return senline_http_answer_internal_error(request); }; + let encoded_result = senline_http_process_payload(input, body_len); + input.free(); + if encoded_result.is_err() { return senline_http_answer_internal_error(request); }; + let encoded = encoded_result.value; + let output = encoded.payload; + if encoded.len <= 0 or encoded.len > senline_http_output_limit() { output.free(); return senline_http_answer_internal_error(request); }; + let body = string_from_buffer(output, encoded.len); + output.free(); + if body.is_err() { return senline_http_answer_internal_error(request); }; + request.respond_with_content_type(200, "application/json", body.value.as_str()).unwrap_or(false); +} diff --git a/examples/realworld/senline-http-dogfood/src/main.sg b/examples/realworld/senline-http-dogfood/src/main.sg new file mode 100644 index 00000000..dcdfba0d --- /dev/null +++ b/examples/realworld/senline-http-dogfood/src/main.sg @@ -0,0 +1,27 @@ +import std::ffi; + +import std::io; + +import std::strconv; + +import senline_http_dogfood; + +async def main() -> i64 { + let bound = senline_http_bind("127.0.0.1", 0); + if bound.is_err() { return 20; }; + let server = bound.value; + let port = server.local_port(); + if port.is_err() { server.close(); return 21; }; + let port_buffer_result = ffi_buffer_new(16); + if port_buffer_result.is_err() { server.close(); return 22; }; + let port_buffer = port_buffer_result.value; + let port_len = strconv_format_i64(port.value, port_buffer); + if port_len.is_err() { port_buffer.free(); server.close(); return 23; }; + io_stdout_write("READY 127.0.0.1:"); + io_stdout_write_raw(port_buffer.ptr(), port_len.value); + io_stdout_write("\n"); + io_stdout_flush(); + port_buffer.free(); + let outcome = await server.next_request_async(senline_http_request_timeout_ms()); + if not outcome.is_ok { let timed_out = outcome.error == STATUS_TIMEOUT(); server.close(); if timed_out { 0; } else { 24; }; } else { let answered = senline_http_handle_request(outcome.value); let closed = server.close(); if answered and closed { 0; } else { 25; }; }; +} diff --git a/examples/realworld/senline-http-dogfood/tests/payload_contract.sg b/examples/realworld/senline-http-dogfood/tests/payload_contract.sg new file mode 100644 index 00000000..2d8f28ca --- /dev/null +++ b/examples/realworld/senline-http-dogfood/tests/payload_contract.sg @@ -0,0 +1,60 @@ +import std::ffi; + +import std::string; + +import senline_domain_worker; + +import senline_http_dogfood; + +def http_payload_body(payload: &str) -> Result { + let input_result = ffi_buffer_from_bytes(payload); + if input_result.is_err() { return Result { is_ok: false, value: String { handle: 0 }, error: input_result.error }; }; + let input = input_result.value; + let response = senline_http_process_payload(input, payload.len()); + input.free(); + if response.is_err() { return Result { is_ok: false, value: String { handle: 0 }, error: response.error }; }; + let encoded = response.value; + let output = encoded.payload; + let body = string_from_buffer(output, encoded.len); + output.free(); + body; +} + +def worker_payload_body(payload: &str) -> Result { + let input_result = ffi_buffer_from_bytes(payload); + if input_result.is_err() { return Result { is_ok: false, value: String { handle: 0 }, error: input_result.error }; }; + let input = input_result.value; + let response = worker_process_payload_v1(input, payload.len()); + input.free(); + if response.is_err() { return Result { is_ok: false, value: String { handle: 0 }, error: response.error }; }; + let encoded = response.value; + let output = encoded.payload; + let body = string_from_buffer(output, encoded.len); + output.free(); + body; +} + +def main() -> i64 { + let malformed = http_payload_body("{"); + if malformed.is_err() { return 10; }; + if not str_eq(malformed.value.as_str(), "{\"kind\":\"error\",\"schema_version\":1,\"scope\":\"protocol\",\"code\":\"malformed_json\",\"evaluation_id\":null}\n") { return 10; }; + let duplicate = http_payload_body("{\"kind\":\"evaluation\",\"kind\":\"evaluation\"}"); + if duplicate.is_err() { return 11; }; + if not str_eq(duplicate.value.as_str(), "{\"kind\":\"error\",\"schema_version\":1,\"scope\":\"protocol\",\"code\":\"duplicate_field\",\"evaluation_id\":null}\n") { return 11; }; + let trailing = http_payload_body("{} trailing"); + if trailing.is_err() { return 12; }; + if not str_eq(trailing.value.as_str(), "{\"kind\":\"error\",\"schema_version\":1,\"scope\":\"protocol\",\"code\":\"trailing_bytes\",\"evaluation_id\":null}\n") { return 12; }; + let unsupported = http_payload_body("{\"context\":{\"contract_version\":1,\"operation\":\"submit-envelope\",\"operation_version\":2,\"evaluation_id\":\"00000000000000000000000000000001\",\"operation_epoch\":0,\"worker_generation\":0,\"execution_mode\":\"fixture\",\"worker_bundle_id\":\"b\",\"facts_binding\":\"0000000000000000000000000000000000000000000000000000000000000000\"},\"facts\":{\"contract_version\":1,\"operation_version\":2,\"identifiers\":{\"correlation_ref\":\"c\",\"source_account_ref\":\"sa\",\"source_device_ref\":\"sd\",\"recipient_account_ref\":\"ra\",\"recipient_device_ref\":\"rd\",\"conversation_ref\":\"v\",\"envelope_ref\":\"e\"},\"source_device_status\":\"active\",\"source_device_capabilities\":[],\"envelope_protocol_version\":2,\"ciphertext_length_bytes\":0,\"idempotency_status\":\"new\",\"recipient_pending_count\":0,\"recipient_pending_limit\":1,\"application_envelopes_used\":0,\"application_envelopes_limit\":1,\"ciphertext_limit_bytes\":0,\"feature_flags\":[]},\"kind\":\"evaluation\",\"schema_version\":1}"); + if unsupported.is_err() { return 13; }; + if not str_eq(unsupported.value.as_str(), "{\"kind\":\"error\",\"schema_version\":1,\"scope\":\"evaluation\",\"code\":\"unsupported_operation_version\",\"evaluation_id\":\"00000000000000000000000000000001\"}\n") { return 13; }; + let shadow = http_payload_body("{\"context\":{\"contract_version\":1,\"operation\":\"submit-envelope\",\"operation_version\":1,\"evaluation_id\":\"00000000000000000000000000000001\",\"operation_epoch\":0,\"worker_generation\":0,\"execution_mode\":\"shadow\",\"worker_bundle_id\":\"b\",\"facts_binding\":\"0000000000000000000000000000000000000000000000000000000000000000\"},\"facts\":{\"contract_version\":1,\"operation_version\":1,\"identifiers\":{\"correlation_ref\":\"c\",\"source_account_ref\":\"sa\",\"source_device_ref\":\"sd\",\"recipient_account_ref\":\"ra\",\"recipient_device_ref\":\"rd\",\"conversation_ref\":\"v\",\"envelope_ref\":\"e\"},\"source_device_status\":\"active\",\"source_device_capabilities\":[],\"envelope_protocol_version\":2,\"ciphertext_length_bytes\":0,\"idempotency_status\":\"new\",\"recipient_pending_count\":0,\"recipient_pending_limit\":1,\"application_envelopes_used\":0,\"application_envelopes_limit\":1,\"ciphertext_limit_bytes\":0,\"feature_flags\":[]},\"kind\":\"evaluation\",\"schema_version\":1}"); + if shadow.is_err() { return 14; }; + if not str_eq(shadow.value.as_str(), "{\"kind\":\"error\",\"schema_version\":1,\"scope\":\"protocol\",\"code\":\"unknown_enum\",\"evaluation_id\":null}\n") { return 14; }; + let fixture = "{\"context\":{\"contract_version\":1,\"operation\":\"submit-envelope\",\"operation_version\":1,\"evaluation_id\":\"00000000000000000000000000000001\",\"operation_epoch\":0,\"worker_generation\":0,\"execution_mode\":\"fixture\",\"worker_bundle_id\":\"b\",\"facts_binding\":\"0000000000000000000000000000000000000000000000000000000000000000\"},\"facts\":{\"contract_version\":1,\"operation_version\":1,\"identifiers\":{\"correlation_ref\":\"c\",\"source_account_ref\":\"sa\",\"source_device_ref\":\"sd\",\"recipient_account_ref\":\"ra\",\"recipient_device_ref\":\"rd\",\"conversation_ref\":\"v\",\"envelope_ref\":\"e\"},\"source_device_status\":\"active\",\"source_device_capabilities\":[],\"envelope_protocol_version\":2,\"ciphertext_length_bytes\":0,\"idempotency_status\":\"new\",\"recipient_pending_count\":0,\"recipient_pending_limit\":1,\"application_envelopes_used\":0,\"application_envelopes_limit\":1,\"ciphertext_limit_bytes\":0,\"feature_flags\":[]},\"kind\":\"evaluation\",\"schema_version\":1}"; + let http_fixture = http_payload_body(fixture); + let worker_fixture = worker_payload_body(fixture); + if http_fixture.is_err() { return 15; }; + if worker_fixture.is_err() { return 15; }; + if not str_eq(http_fixture.value.as_str(), worker_fixture.value.as_str()) { return 15; }; + 0; +} diff --git a/examples/realworld/senline-http-dogfood/tests/policy_contract.sg b/examples/realworld/senline-http-dogfood/tests/policy_contract.sg new file mode 100644 index 00000000..9de87f67 --- /dev/null +++ b/examples/realworld/senline-http-dogfood/tests/policy_contract.sg @@ -0,0 +1,22 @@ +import senline_http_dogfood; + +def main() -> i64 { + if not senline_http_bind_allowed("127.0.0.1", 0) { return 10; }; + if senline_http_bind_allowed("0.0.0.0", 0) { return 11; }; + if senline_http_bind_allowed("192.0.2.10", 0) { return 12; }; + if senline_http_bind_allowed("127.0.0.1", 43123) { return 13; }; + if senline_http_header_limit() != 4096 { return 14; }; + if senline_http_body_limit() != 32768 { return 15; }; + if senline_http_output_limit() != 8192 { return 16; }; + if senline_http_request_timeout_ms() != 5000 { return 17; }; + if not senline_http_method_allowed("POST") or senline_http_method_allowed("GET") { return 18; }; + if not senline_http_path_allowed("/v1/submit-envelope", "") { return 19; }; + if senline_http_path_allowed("/v1/submit-envelope", "debug=1") or senline_http_path_allowed("/v1/other", "") { return 20; }; + if not senline_http_version_allowed("HTTP/1.1") or senline_http_version_allowed("HTTP/1.0") or senline_http_version_allowed("HTTP/2") { return 21; }; + if not senline_http_content_type_allowed("application/json") or senline_http_content_type_allowed("application/json; charset=utf-8") { return 22; }; + if not senline_http_transfer_encoding_allowed(false) or senline_http_transfer_encoding_allowed(true) { return 23; }; + if not senline_http_header_length_supported(0) or not senline_http_header_length_supported(4096) or senline_http_header_length_supported(4097) { return 24; }; + if senline_http_body_length_supported(0) or not senline_http_body_length_supported(1) or not senline_http_body_length_supported(32768) or senline_http_body_length_supported(32769) { return 25; }; + if senline_http_max_requests_per_process() != 1 or senline_http_max_pending_requests() != 1 { return 26; }; + 0; +} diff --git a/openspec/changes/senline-service-dogfood/.openspec.yaml b/openspec/changes/senline-service-dogfood/.openspec.yaml new file mode 100644 index 00000000..b119b635 --- /dev/null +++ b/openspec/changes/senline-service-dogfood/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-13 diff --git a/openspec/changes/senline-service-dogfood/design.md b/openspec/changes/senline-service-dogfood/design.md new file mode 100644 index 00000000..4ab269b0 --- /dev/null +++ b/openspec/changes/senline-service-dogfood/design.md @@ -0,0 +1,228 @@ +## Context + +Senline's `adopt-sengoo-backend-slice` change introduces Sengoo as a real participant in the v2 `submit-envelope` path. The first integration is deliberately out of process: a Rust supervisor sends verified, bounded domain facts to a Sengoo worker and compares the returned plan with the Rust reference. Rust remains the security reference monitor and the only mutation authority. + +Sengoo already has native compilation, Buffer and JSON handles, synchronous standard I/O, a package workflow, and a serial plaintext HTTP server with an async request future. Product use exposes several concrete gaps: ordinary standard-I/O calls do not yet define exact partial-I/O framing behavior; Windows text-mode pipes can corrupt binary length prefixes; permissive JSON APIs do not expose enough strict object inspection; installed `sgc` can still depend on a source checkout/Cargo-built native runtime; C-to-Sengoo export-link evidence may skip linker failures; some panic paths terminate the process; and no immutable consumer evidence connects a real product failure to a fixed Sengoo artifact. These are recorded baseline defects, not claims of new Senline discovery. + +The mutable `D:\Sengoo` checkout is not a release input. All implementation writes for this change are scoped to the clean `codex/senline-service-dogfood` worktree registered at `D:\Sengoo\.worktrees\senline-service-dogfood`; the mutable primary checkout remains untouched. Senline will consume only clean revisions and immutable Windows/Linux installed bundles whose complete contents are hashed. This change owns Sengoo code, packages, tests, distribution artifacts, and defect evidence. Senline owns the host sandbox, supervision, timeout, restart and circuit policy, facts construction, final contract validation, and all authoritative state changes. + +## Goals / Non-Goals + +**Goals:** + +- Build and run a real `senline-domain-worker` from an installed Sengoo toolchain on Windows x64 and Linux x64. +- Implement bounded four-byte big-endian length-prefixed UTF-8 JSON over binary stdin/stdout with exact partial-I/O behavior. +- Add opt-in strict JSON parsing and object inspection needed to reject duplicate/unknown fields and malformed Unicode without changing existing permissive callers. +- Implement a deterministic, capability-minimal `submit-envelope` facts-to-plan module shared by the worker and a loopback HTTP dogfood package. +- Make native runtime artifacts first-class installed distribution inputs with complete target/ABI/link/dependency metadata. +- Turn every Senline-discovered Sengoo defect into a minimized regression and immutable red/minimize/fix/pin/green evidence chain. +- Exercise existing async HTTP serving in a development-only harness without presenting it as Senline ingress. + +**Non-Goals:** + +- Implementing or selecting cryptography, signatures, KDFs, randomness, recovery, authentication, replay state, prekey claims, acknowledgements, cursors, or database transactions in Sengoo. +- Giving Sengoo raw signed requests, ciphertext bytes, plaintext messages, tokens, credentials, private keys, recovery material, database rows/handles, or transaction handles. +- Making the Sengoo worker or HTTP harness authoritative for a Senline mutation. +- Promoting the current serial, plaintext, close-per-request HTTP server to internal-alpha or public ingress. +- Adding a general JSON Schema dialect, streaming JSON, HTTP/2, TLS serving, database persistence, or broad task-cancellation semantics. +- Requiring in-process Rust-to-Sengoo linking before the supervised worker can ship. + +## Decisions + +### 1. Freeze a narrow consumer contract and authority boundary + +The initial operation is `submit-envelope` contract V1. The host sends `WorkerRequestV1`, containing an `EvaluationContextV1` and minimum-necessary `SubmitEnvelopeFactsV1`; the worker returns `SubmitEnvelopePlanV1`. The context carries the contract version, operation, evaluation ID, host-computed facts binding, operation epoch, and worker generation. The plan echoes those binding fields plus an exhaustive decision and stable reason. + +The worker never consumes the canonical signed request and never selects an authenticated identity. It evaluates only facts already verified and bounded by Rust. A successful plan is advisory until Rust validates it against the exact context/facts and rechecks mutable state inside a new transaction. Rust remains responsible for TLS, request parsing and signature verification, freshness/replay/rate limits, authorization/revocation, cryptography, persistence, uniqueness, prekey/ACK/cursor state, audit, migrations, and final commit. + +| Responsibility | Sengoo | Senline Rust kernel | +| --- | --- | --- | +| TLS, canonical signed bytes, signature verification | No access | Authoritative | +| Freshness, replay, rate limits, device authorization/revocation | Receives only eligible bounded facts | Authoritative | +| Cryptography, CSPRNG, secrets and key material | Forbidden | Authoritative through reviewed components | +| Database handles, transactions, persistence and migrations | Forbidden | Authoritative | +| Prekey, envelope, ACK and cursor mutation | Advisory plan only | Authoritative | +| Domain decision for the selected eligible operation | Deterministic candidate plan | Runs reference, validates exact binding and may reject | +| Final mutation and rollback | No authority | Authoritative | + +Alternative: implement the entire v2 service in Sengoo immediately. Rejected because current serving, concurrency, database, panic-containment, fuzz, sanitizer, and soak evidence cannot protect Senline's existing security invariants. + +Alternative: use Sengoo only for offline tools. Rejected because it would not exercise package installation, pipes, deterministic domain decisions, strict decoding, process lifetime, or continuous real request compatibility. + +### 2. Use one-request-at-a-time framed binary standard I/O + +Each message is a four-byte unsigned big-endian length followed by exactly that many UTF-8 JSON bytes. Input is capped at 32 KiB and output at 8 KiB before allocation or parse. The worker processes one request at a time, emits one response frame, and performs no implicit retry. EOF before a new prefix is clean shutdown; EOF during a prefix or payload, a zero/oversized length, trailing payload bytes, or surplus response frame is a deterministic protocol error. + +The stdlib gains byte-safe Buffer access, big-endian `u32` helpers, offset-aware exact reads/writes, and explicit binary-mode initialization for standard streams. Windows must set stdin/stdout to `_O_BINARY` before any protocol I/O. Exact helpers loop over partial operations, distinguish clean EOF from truncation, validate offset/length without overflow, and never treat a short write as success. + +Stdout is reserved for frames. Bounded stable diagnostic codes go to stderr; request values and parser text are not echoed. This supports a Rust supervisor without making Sengoo responsible for the host's 50 ms admission-to-validation deadline, four-process/128-queue pool, restart circuit, or OS sandbox. + +Alternative: newline-delimited JSON. Rejected because arbitrary JSON strings and accidental output make record boundaries ambiguous and do not exercise binary pipe correctness. + +Alternative: HTTP between Rust and Sengoo. Rejected for the first boundary because the current server is serial/plaintext and adds networking concerns to a pure planner. + +### 3. Add strict JSON as an opt-in compatibility-preserving surface + +Existing `json_parse` and `json_parse_buffer` remain permissive and source-compatible. A new strict parse entry point validates the exact input length, UTF-8, JSON grammar, full-input consumption, configured depth, integer range, Unicode escapes, surrogate pairs, and duplicate object keys. Object inspection exposes the exact decoded key count, keys, and values so application decoders can compare against an exhaustive allowlist and reject unknown/missing fields. + +Strict failures also expose a stable machine-readable error kind for +unclassified syntax, duplicate fields, invalid UTF-8/Unicode, and trailing +bytes while preserving the existing status, offset, and human diagnostic. +Workers branch only on the error kind, never on diagnostic text. JSON builder +string creation accepts an explicit byte length so decoded `U+0000` and other +embedded bytes round-trip without C-string truncation. + +V1 does not introduce general JSON Schema. The worker owns explicit per-contract decoders and validates field type, bounds, enum membership, required/optional presence, and unknown fields. Object key comparison is decoded Unicode scalar/UTF-8 equality with no normalization or case folding. This is enough for a secure fixed contract while respecting the existing follow-up gate for general schema validation and streaming JSON. + +Alternative: post-process a permissively parsed map. Rejected because a map may already have collapsed duplicate keys and lost evidence needed for deterministic rejection. + +### 4. Treat the installed native runtime as a release artifact + +Windows and Linux distributions include the target-native `sengoo_runtime.lib` or `libsengoo_runtime.a` and every declared runtime dependency needed for native programs. The distribution manifest records target triple, runtime ABI version, payload SHA-256 hashes, ordered link arguments, dynamic dependencies, source revision, tool versions, and a build-manifest identifier. + +Installed `sgc` discovers the runtime relative to its installed manifest and prefers it for normal native build/run/test. Building a runtime through Cargo remains an explicit Sengoo-source development mode only. Missing, mismatched, wrong-target, or incomplete runtime artifacts fail with a stable diagnostic rather than silently consulting `D:\Sengoo`, `SENGOO_ROOT`, a Cargo target directory, or a mutable cache. + +Release smokes install into a fresh path outside the checkout, put a deliberately failing fake `cargo` first on PATH, and build/run the stdio+strict-JSON worker and async HTTP harness. Two independent builds per target compare normalized payload manifests; allowed provenance timestamps/signatures may differ, but payload hashes, ABI, link arguments, and dependency identities must match. + +Alternative: let Senline invoke Sengoo from the source checkout. Rejected because it is not immutable, reproducible, portable, or representative of a user-installed language. + +### 5. Keep the planner pure and packages locked + +The facts-to-plan module has no network, filesystem, environment, clock, randomness, database, subprocess, direct FFI, or secret capability. It performs exhaustive decode into contract-specific types and exhaustive encode of one normalized plan. Repeating the same request produces byte-equivalent normalized output and does not retain state across evaluations. + +`senline-domain-worker` and `senline-http-dogfood` use source-controlled locked dependencies. Their check/test/fmt/doc/release-build loops run with installed tools. The worker startup handshake reports protocol version, Sengoo revision, toolchain version, application version, and an embedded build-manifest identifier. These values allow the Rust host to reject inconsistency, but the host's independently verified external bundle hashes remain the trust root. + +Alternative: embed a self-hash in the worker and trust it. Rejected because a replaced executable can lie about its own identity. + +### 6. Use HTTP only as a loopback synthetic-data dogfood harness + +The HTTP package binds an OS-selected ephemeral loopback address only, accepts only synthetic/non-secret V1 facts, enforces bounded header/body/time limits, and returns the same normalized plan/error contract. It tests `next_request_async`, timeout, pending-future drop cleanup, exactly-once response, and clean close on real Windows/Linux localhost connections. + +No Senline Windows or Android client endpoint, deployment manifest, or internal-alpha route may target this harness. Its README and support evidence retain the existing serial/plaintext/`Connection: close` limits. + +### 7. Make consumer defects traceable across revisions + +Every Sengoo-owned failure follows one evidence state machine: + +1. `red`: preserve the failing Senline fixture/transcript and classify ownership; +2. `minimize`: reduce it in this repository and add a failing compiler/runtime/stdlib/package regression; +3. `fix`: implement the smallest general fix and run affected Sengoo gates; +4. `pin`: commit the clean fix, build immutable installed bundles, record hashes/provenance, and update Senline's pin; +5. `green`: rerun the minimized regression plus Senline differential/leakage/integration gates against the pinned artifact. + +Workarounds require an owner, linked defect, expiry condition, and removal test. They do not count as green. If fixture completion discovers no new defect, the process is rehearsed with a known framing/strict-JSON/installed-runtime defect or an injected failure and is labelled accordingly. + +### 8. Freeze the exact V1 DTO and binding bytes before worker code + +The V1 JSON surface is closed. `EvaluationContextV1` contains exactly +`contract_version`, `operation`, `operation_version`, `evaluation_id`, +`operation_epoch`, `worker_generation`, `execution_mode`, `worker_bundle_id`, +and `facts_binding`. `contract_version` is `1`, `operation` is +`submit-envelope`, identifiers and bundle IDs are opaque ASCII refs from 1 +through 128 bytes, evaluation IDs are 32 lowercase hexadecimal characters, +and bindings are 64 lowercase hexadecimal characters. Integer fields encoded +as `u32` are no larger than `4294967295`; operation epoch and worker generation +are non-negative JSON-safe integers no larger than `2^53-1`. + +`SubmitEnvelopeFactsV1` contains exactly `contract_version`, +`operation_version`, `identifiers`, `source_device_status`, +`source_device_capabilities`, `envelope_protocol_version`, +`ciphertext_length_bytes`, `idempotency_status`, `recipient_pending_count`, +`recipient_pending_limit`, `application_envelopes_used`, +`application_envelopes_limit`, `ciphertext_limit_bytes`, and `feature_flags`. +The identifiers object contains exactly `correlation_ref`, +`source_account_ref`, `source_device_ref`, `recipient_account_ref`, +`recipient_device_ref`, `conversation_ref`, and `envelope_ref`. +`source_device_status` is `active`; idempotency is one of `new`, +`exact_duplicate`, or `conflict`; capabilities and feature flags are sorted, +unique arrays from reviewed closed enums. Revoked, forged, stale, and +rate-limited requests never become this DTO. + +`WorkerRequestV1` contains exactly `kind=evaluation`, `schema_version=1`, +`context`, and `facts`. `SubmitEnvelopePlanV1` contains exactly `kind=plan`, +`schema_version=1`, the exact echoed context and identifiers, `decision`, +`reason`, and `sengoo_module_revision`. Decisions are `store_and_enqueue`, +`duplicate_noop`, or `reject`; reasons are `accepted_new`, `exact_duplicate`, +`idempotency_conflict`, `recipient_queue_full`, +`application_budget_exhausted`, or `delivery_disabled`. Unknown operation +versions return `WorkerErrorV1` with `unsupported_operation_version` rather +than guessing a plan. Worker error envelopes contain only `kind=error`, +`schema_version`, `scope`, `code`, and a nullable `evaluation_id`; they contain +no arbitrary message. Framing, timeout, exit, overload, circuit, bundle, and +output-size errors remain host-only. + +`sengoo_module_revision` is exactly 40 lowercase hexadecimal characters and +identifies the frozen planner contract fixture revision. It stays stable for +byte-equivalent V1 planner semantics and is not executable provenance. The +startup `sengoo_source_revision` identifies the actual immutable bundle source +revision and is checked independently with the external manifest. + +Rust computes `facts_binding`; Sengoo only echoes it. Binding bytes start with +ASCII `senline.submit-envelope.binding.v1` plus NUL. Unsigned integers follow +in the field order above as big-endian `u32`, except epoch and generation which +are `u64`. Strings are `u32` byte length plus UTF-8 bytes. Arrays are `u32` +count plus encoded strings. The binding encodes every context field except +`facts_binding`, followed by every facts field and nested identifier in the +declared order, then SHA-256 is rendered as lowercase hexadecimal. No generic +JSON serialization or map iteration order is accepted as the canonical input. + +The startup handshake contains exactly `kind=handshake`, `protocol_version`, +`sengoo_source_revision`, `toolchain_version`, `application_version`, and +`build_manifest_id`. The reported build ID is a consistency value only. Raw +fixtures and their metadata are byte-frozen in both linked changes; any field, +enum, encoding, or limit change requires a reviewed contract-version change. + +### 9. Incubate missing ecosystem capabilities as reusable Sengoo packages + +Senline is a demand driver for the Sengoo library ecosystem, not permission to +hide one-off helpers in the application. A missing capability is classified +before implementation: + +- product DTOs and decisions remain product packages; +- domain-neutral composition over existing stdlib primitives starts as a + locked pure Sengoo package beside the real consumer; +- a primitive that cannot be implemented safely above the runtime may extend + `std::` only with native, compiler, LSP, compatibility, and platform tests; +- cryptography, TLS, production HTTP, durable databases, and OS sandboxing use + reviewed bindings to mature implementations rather than new algorithms; +- capabilities that carry Senline security or mutation authority remain in + Rust until a separate change proves an authority transfer. + +The first incubated packages are `sgframing`, providing bounded `u32` big-endian +stdio frames and stable EOF/truncation/limit semantics, and +`sgjson_contract`, providing exact-object, required-field, kind, range, ASCII, +hex, closed-enum, and sorted-unique-array validation over opt-in strict JSON. +`senline_facts_to_plan` remains the product-specific typed codec/planner. + +An incubated package may move to the repository-level `packages/` catalog only +after a second independent consumer or a reviewed protocol-foundation need, +stable documented API, locked source and installed-toolchain loops on Windows +and Linux, malformed/boundary coverage, and no skipped required gate. Moving a +surface into `std::` additionally requires compatibility, compiler import, +LSP, native runtime, and distribution evidence. Publication never substitutes +for these quality gates. + +## Risks / Trade-offs + +- [Strict JSON changes permissive behavior accidentally] -> Add new entry points, retain existing APIs unchanged, and run compatibility fixtures before archive. +- [Binary helper bugs corrupt framing or allocate from hostile lengths] -> Validate ranges before allocation, test every split point and EOF state, and fuzz raw frames with fixed caps. +- [Installed builds silently reach into the source checkout] -> Run from fresh paths with checkout variables cleared and fake-failing Cargo, and audit emitted link arguments/manifests for absolute paths. +- [Worker output is mistaken for authorization] -> Keep the authority boundary normative in both repositories; Rust independently validates every plan and owns all mutation. +- [HTTP harness is deployed as ingress] -> Enforce loopback binding in code/tests and add source/release checks rejecting client or deployment references. +- [Diagnostics leak request data] -> Emit allowlisted codes only, cap stderr, seed canaries, and scan package tests and evidence artifacts. +- [Product pressure removes workarounds without proving a fix] -> Require the immutable fixing artifact, Senline pin advance, and green consumer gate before removal. +- [Windows and Linux runtime artifacts diverge] -> Keep target-specific manifests and compare reproducibility within each target, never across targets. + +## Migration Plan + +1. Land strict regressions for framing, Windows binary mode, JSON, and installed-runtime discovery before implementation changes. +2. Add compatibility-preserving stdlib/runtime/toolchain fixes and build immutable Windows/Linux installed bundles. +3. Freeze V1 raw fixtures and implement the pure planner, then the framed worker around it. +4. Run the locked package loop outside the source checkout and hand the verified manifest/hash set to Senline for pinning. +5. Integrate only with Senline fixture/shadow modes; rollback is removal or disabling of the pinned worker bundle while Rust continues as reference and authority. +6. Add the loopback HTTP harness after the shared planner is green; never make it an ingress prerequisite. +7. Archive only after one complete red/minimize/fix/pin/green demonstration and strict validation in both repositories. + +## Open Questions + +None before implementation. Contract versions, size limits, authority ownership, compatibility policy, installed-runtime discovery rules, and harness scope are fixed by this design; any transfer of cryptographic, authentication, replay, transaction, persistence, or ingress authority requires a separate OpenSpec change. diff --git a/openspec/changes/senline-service-dogfood/proposal.md b/openspec/changes/senline-service-dogfood/proposal.md new file mode 100644 index 00000000..caa14896 --- /dev/null +++ b/openspec/changes/senline-service-dogfood/proposal.md @@ -0,0 +1,32 @@ +## Why + +Senline needs Sengoo to participate in a real backend request path now so product work exposes concrete compiler, runtime, standard-library, packaging, and diagnostic defects. The integration must produce useful evidence without transferring Senline's cryptographic, authorization, replay, or transactional authority to an immature runtime. + +## What Changes + +- Add an installed-toolchain-built `senline-domain-worker` package that accepts bounded, versioned facts over framed standard I/O and returns a deterministic `submit-envelope` plan. +- Add exact binary framing support, including partial reads/writes, big-endian `u32` lengths, byte access, EOF handling, and Windows binary-mode standard streams. +- Add opt-in strict JSON object/schema inspection that rejects duplicate and unknown fields, invalid UTF-8 or Unicode escapes, trailing input, excess nesting, and out-of-range integers without silently changing permissive JSON callers. +- Make native runtime artifacts discoverable from an installed Sengoo distribution so the worker and an async HTTP dogfood package build and run outside the Sengoo source checkout without Cargo fallback. +- Add a loopback-only `senline-http-dogfood` package that reuses the pure facts-to-plan module and exercises the existing bounded, serial, plaintext HTTP server subset with synthetic non-secret facts only. +- Establish consumer-driven red/minimize/fix/pin/green evidence for each Senline-discovered Sengoo defect, including minimized regressions, fixing revision, immutable installed artifact hashes, and the Senline pin that consumes them. +- Keep the initial worker free of network, filesystem, environment, clock, randomness, database, direct FFI, and secret capabilities; its stdout is protocol-only and diagnostics are bounded on stderr. +- Explicitly leave raw request verification, device authorization and revocation, replay/freshness enforcement, cryptography, database transactions, prekey/ACK/cursor state, final plan validation, and every authoritative mutation in Senline's Rust security/transaction kernel. + +## Capabilities + +### New Capabilities + +- `senline-service-dogfood`: Consumer-driven Sengoo worker and loopback HTTP packages, deterministic Senline plan contract, failure containment expectations, and cross-repository defect evidence. + +### Modified Capabilities + +- `stdlib-mainstream-usability`: Add opt-in exact binary standard-I/O helpers and strict JSON inspection needed by bounded framed workers while preserving existing permissive APIs. +- `toolchain-distribution`: Require installed per-target native runtime artifacts and metadata sufficient to compile and run native packages without a Sengoo source checkout or implicit Cargo fallback. + +## Impact + +- Affected Sengoo areas: native runtime and `std::io`/`std::json` bridges, `sgc` runtime discovery and linking, distribution manifests, Windows/Linux package smokes, new realworld packages, and focused compiler/runtime regressions. +- External consumer: Senline change `adopt-sengoo-backend-slice`, which pins a clean immutable Sengoo revision and complete worker bundle rather than consuming a mutable checkout. +- Runtime boundary: four-byte big-endian length-prefixed UTF-8 JSON on stdin/stdout, with 32 KiB maximum input and 8 KiB maximum output. The Senline Rust host owns deadlines, supervision, sandboxing, restart/circuit policy, contract validation, and mutation authority. +- Security scope: the packages process only verified, minimum-necessary domain facts or synthetic fixtures. They never receive ciphertext bytes, signatures, tokens, credentials, private keys, recovery material, database handles, or transaction handles. diff --git a/openspec/changes/senline-service-dogfood/specs/senline-service-dogfood/spec.md b/openspec/changes/senline-service-dogfood/specs/senline-service-dogfood/spec.md new file mode 100644 index 00000000..621e5961 --- /dev/null +++ b/openspec/changes/senline-service-dogfood/specs/senline-service-dogfood/spec.md @@ -0,0 +1,222 @@ +## ADDED Requirements + +### Requirement: The domain worker SHALL use a bounded versioned framed protocol + +`senline-domain-worker` SHALL exchange exactly one request and one response at a time using a four-byte unsigned big-endian length followed by UTF-8 JSON. It SHALL reject input larger than 32 KiB, SHALL never emit output larger than 8 KiB, and SHALL reserve stdout exclusively for protocol frames. + +#### Scenario: A complete request produces one complete response + +- **WHEN** stdin supplies a valid `WorkerRequestV1` frame in arbitrary partial reads +- **THEN** the worker reads exactly the declared payload, evaluates it once, and writes exactly one complete `SubmitEnvelopePlanV1` frame despite partial writes +- **AND** no ordinary text precedes or follows the frame on stdout + +#### Scenario: A malformed frame is rejected deterministically + +- **WHEN** a prefix or payload is truncated, a declared length is zero or over 32 KiB, UTF-8 is invalid, trailing payload bytes exist, or a request schema is invalid +- **THEN** the worker returns the documented bounded protocol error when a response is possible or exits with the documented stable status +- **AND** it does not allocate from the unchecked declared length, retry the evaluation, or emit a partial plan + +#### Scenario: Output cannot exceed its bound + +- **WHEN** encoding a response would exceed 8 KiB +- **THEN** the worker emits the documented bounded internal error response or stable exit status +- **AND** no oversized or truncated success frame is written + +### Requirement: Worker contracts SHALL bind plans to verified domain facts + +Contract V1 SHALL decode `WorkerRequestV1` into `EvaluationContextV1` and `SubmitEnvelopeFactsV1` and SHALL encode `SubmitEnvelopePlanV1`. The plan SHALL echo the exact contract version, operation, evaluation ID, host-computed facts binding, operation epoch, and worker generation and SHALL contain only an exhaustive decision and stable reason allowed by V1. + +#### Scenario: A valid submit-envelope request is planned + +- **WHEN** the worker receives a strictly valid supported V1 context and minimum-necessary facts +- **THEN** it returns a plan whose binding fields exactly match the request and whose decision/reason pair is permitted by V1 + +#### Scenario: Contract fields are not extensible by accident + +- **WHEN** a request has a missing, duplicate, unknown, wrong-typed, out-of-range, unknown-enum, or unknown-version field +- **THEN** exhaustive decoding rejects it before domain evaluation +- **AND** no best-effort or defaulted plan is returned + +#### Scenario: A plan cannot claim host provenance + +- **WHEN** the worker constructs a successful response +- **THEN** it echoes the opaque host-provided facts binding without recomputing or replacing it +- **AND** it does not emit a worker-authored digest as proof of request provenance + +### Requirement: The submit-envelope planner SHALL be pure and deterministic + +The shared Sengoo facts-to-plan module SHALL depend only on its typed V1 inputs, SHALL retain no request state, and SHALL have no network, filesystem, environment, clock, randomness, database, subprocess, direct FFI, credential, or secret capability. + +#### Scenario: Identical facts produce identical plans + +- **WHEN** the same normalized context and facts are evaluated repeatedly in one worker and across fresh workers built from the same artifact +- **THEN** the normalized plan bytes are identical on Windows x64 and Linux x64 +- **AND** no earlier evaluation changes the result + +#### Scenario: Prohibited values never enter the contract + +- **WHEN** contract and fixture leakage tests inspect every input/output field +- **THEN** private keys, recovery material, plaintext, ciphertext bytes, raw signatures, tokens, credentials, connection strings, SQL, database rows, raw runtime handles, and transaction handles are absent + +### Requirement: Sengoo SHALL remain outside Senline security and transaction authority + +The worker SHALL produce advisory domain plans only. Senline's Rust security/transaction kernel SHALL remain solely responsible for TLS, canonical request parsing and signature verification, freshness and replay enforcement, account-device binding, revocation, rate limits, cryptography, durable persistence, transactions, uniqueness, prekey/ACK/cursor state, migrations, final plan validation, and every authoritative mutation. + +#### Scenario: A successful worker plan has no direct side effect + +- **WHEN** the worker returns an eligible or accepting plan +- **THEN** no Senline state changes unless the Rust kernel independently validates the plan and commits the operation under its own current checks +- **AND** the worker has no handle or API capable of bypassing that kernel + +#### Scenario: Sengoo fails or disagrees + +- **WHEN** the worker crashes, times out, emits malformed output, returns a stale binding, or differs from the Rust reference +- **THEN** the Sengoo package performs no mutation and makes no authorization decision +- **AND** Senline applies the fixed failure policy owned by its host-side change + +### Requirement: The worker bundle SHALL be immutable and host-verifiable + +Release-shaped worker bundles SHALL include the worker, every runtime dependency, a manifest with protocol/application/toolchain/source versions, per-file SHA-256 hashes, target and ABI metadata, licenses/SBOM inputs, and an embedded build-manifest identifier. The identifier reported by the startup handshake SHALL be a consistency value, not a self-authenticating trust root. + +#### Scenario: Startup identity matches the verified bundle + +- **WHEN** the host independently verifies every bundle file and starts the worker +- **THEN** the worker handshake reports the expected protocol version, Sengoo revision, toolchain version, application version, and embedded manifest identifier + +#### Scenario: A bundle or handshake is inconsistent + +- **WHEN** a file hash is wrong, a dependency is missing, or a handshake value differs from the external manifest +- **THEN** the bundle is ineligible for Senline pinning or execution +- **AND** no self-reported worker hash overrides the mismatch + +### Requirement: Worker diagnostics SHALL be bounded and non-secret + +The worker SHALL write only allowlisted stable diagnostic codes plus bounded development metadata to stderr, SHALL never copy arbitrary protocol values into diagnostics, and SHALL continue draining or terminate deterministically without blocking on diagnostic output. + +#### Scenario: Malicious values reach an error path + +- **WHEN** a rejected request contains randomized canaries in every text field +- **THEN** stdout contains only the bounded protocol response and stderr contains no canary or raw parser input + +#### Scenario: Diagnostics exceed their budget + +- **WHEN** repeated failures would exceed the package diagnostic byte/rate budget +- **THEN** later diagnostic detail is suppressed under a stable code +- **AND** protocol progress does not block on stderr + +### Requirement: A loopback HTTP package SHALL dogfood the same pure planner + +`senline-http-dogfood` SHALL bind only an OS-selected ephemeral loopback address, accept only bounded synthetic/non-secret V1 facts, reuse the exact facts-to-plan module, and return the same normalized V1 plan/error contract through the existing serial plaintext HTTP subset. + +#### Scenario: A localhost synthetic request matches worker evaluation + +- **WHEN** a supported Windows or Linux host sends the same valid synthetic V1 facts through the HTTP harness and framed worker +- **THEN** both paths return byte-equivalent normalized plans + +#### Scenario: Non-loopback serving is requested + +- **WHEN** configuration attempts to bind the harness to a non-loopback interface or fixed externally reachable endpoint +- **THEN** startup fails with a stable diagnostic before accepting a request + +#### Scenario: Async request cleanup remains bounded + +- **WHEN** an async next-request operation times out or its pending future is dropped, or a request is answered/closed +- **THEN** the existing server remains reusable where specified, each surfaced request is answered exactly once, and close releases pending resources +- **AND** no broader cancellation or ingress-readiness claim is made + +#### Scenario: Product routing cannot target the harness + +- **WHEN** release/source checks inspect Senline client endpoints and deployment manifests +- **THEN** no Windows client, Android client, internal-alpha route, or production route targets this package + +### Requirement: Consumer-discovered defects SHALL follow a red-to-green evidence chain + +Every Sengoo-owned failure discovered through Senline SHALL have a durable record connecting the original consumer failure, minimized Sengoo reproducer, failing regression, fixing clean revision, immutable installed artifacts, Senline pin update, and passing consumer verification. + +#### Scenario: A genuine Sengoo defect is discovered + +- **WHEN** a Senline fixture, differential run, package build, or shadow run exposes a Sengoo-owned failure +- **THEN** the failure is minimized and committed as a failing Sengoo compiler/runtime/stdlib/package test before the general fix +- **AND** green is recorded only after Senline consumes the immutable fixing artifact and reruns the linked gate + +#### Scenario: A workaround is temporarily required + +- **WHEN** Senline needs a bounded workaround before the Sengoo fix is pinned +- **THEN** the record includes an owner, linked defect, expiry condition, and removal test +- **AND** the workaround does not count as a fixed or green Sengoo defect + +#### Scenario: No new defect appears before fixture completion + +- **WHEN** the first fixture corpus completes without a newly discovered defect +- **THEN** the full evidence loop is rehearsed with a known framing, strict-JSON, or installed-runtime defect or an injected failure +- **AND** the evidence labels the rehearsal instead of claiming a new discovery + +### Requirement: V1 DTO fields and facts-binding bytes SHALL be closed and byte-frozen + +The worker request, plan, handshake, and error objects SHALL use only the exact +fields, bounds, and closed enums declared by Decision 8. Rust SHALL compute the +facts binding from the versioned typed big-endian encoding declared there; +Sengoo SHALL only echo it. Generic JSON serialization order SHALL NOT define +the binding. Raw reviewed fixtures and their hashes SHALL be byte-identical in +the linked Senline and Sengoo changes. Opaque identifier and bundle refs SHALL +be ASCII strings from 1 through 128 bytes. `u32` binding fields SHALL NOT +exceed `4294967295`; epoch and generation SHALL NOT exceed the JSON-safe +integer maximum `2^53-1`. + +`sengoo_module_revision` SHALL be the exact 40-character lowercase-hex planner +contract fixture revision. It SHALL NOT be treated as executable provenance; +the independently verified bundle manifest and startup +`sengoo_source_revision` identify the actual build. + +#### Scenario: A field or enum is not declared by V1 + +- **WHEN** a request, plan, handshake, or error contains an extra or unknown field or enum +- **THEN** strict decoding rejects the complete object without selecting a value or plan + +#### Scenario: JSON spelling differs but typed facts are equal + +- **WHEN** equivalent typed facts arrive with a different allowed JSON key order +- **THEN** Rust produces the same typed binding bytes and SHA-256 +- **AND** neither host nor worker hashes raw JSON serialization as the facts binding + +#### Scenario: A worker changes a binding input + +- **WHEN** the plan changes any context field, opaque reference, fact, array order, epoch, generation, mode, or bundle ID +- **THEN** Rust rejects the plan before any transaction or mutation + +#### Scenario: A nominally valid reference exceeds its byte bound + +- **WHEN** an identifier or worker bundle reference is empty, non-ASCII, or longer than 128 bytes +- **THEN** strict decoding rejects the request before planner evaluation + +#### Scenario: A plan reports the fixture revision + +- **WHEN** the worker emits `sengoo_module_revision` +- **THEN** it matches the frozen planner contract fixture revision +- **AND** Rust still verifies the actual source revision and complete bundle independently + +### Requirement: Product-discovered library gaps SHALL strengthen the Sengoo ecosystem + +A domain-neutral capability missing during Senline implementation SHALL be +implemented as a reusable locked Sengoo package or a reviewed mature-runtime +binding, with red/green tests and explicit ownership. Product-specific schema +and policy SHALL remain outside general libraries. A package SHALL NOT graduate +to the shared catalog or standard library merely because one Senline path uses +it. + +#### Scenario: Existing stdlib primitives can compose the missing capability + +- **WHEN** the worker needs bounded framing or strict contract decoding and Buffer, exact I/O, and strict JSON primitives already exist +- **THEN** the capability is implemented in domain-neutral pure Sengoo packages with independent tests +- **AND** the Senline codec depends on those packages instead of duplicating their state machine or validation helpers + +#### Scenario: The missing capability is security-critical infrastructure + +- **WHEN** the product needs cryptography, TLS, durable database transactions, production HTTP, or OS sandboxing +- **THEN** no new security algorithm or infrastructure engine is hand-written for convenience +- **AND** the capability remains Rust-owned or uses a separately reviewed binding to a mature implementation + +#### Scenario: An incubated package is proposed for graduation + +- **WHEN** a package is proposed for the shared catalog or `std::` +- **THEN** its stable API, independent consumer or foundation rationale, Windows/Linux installed loops, malformed/boundary tests, documentation, and non-skipped quality gates are recorded diff --git a/openspec/changes/senline-service-dogfood/specs/stdlib-mainstream-usability/spec.md b/openspec/changes/senline-service-dogfood/specs/stdlib-mainstream-usability/spec.md new file mode 100644 index 00000000..6f2027ba --- /dev/null +++ b/openspec/changes/senline-service-dogfood/specs/stdlib-mainstream-usability/spec.md @@ -0,0 +1,127 @@ +## ADDED Requirements + +### Requirement: Binary Buffer helpers SHALL support bounded framed protocols + +The standard library SHALL expose byte get/set and unsigned big-endian 32-bit read/write operations over managed `Buffer` values. Every operation SHALL validate handles, offsets, lengths, value ranges, and arithmetic overflow before accessing memory. + +#### Scenario: A frame length round-trips + +- **WHEN** a program writes a `u32` value in big-endian form at a valid Buffer offset and reads it back +- **THEN** the original value is returned independent of host endianness +- **AND** the four stored bytes use network byte order + +#### Scenario: A byte access is out of bounds + +- **WHEN** byte or `u32` access uses a negative offset, insufficient remaining capacity, an out-of-range byte/value, or overflowing offset arithmetic +- **THEN** the helper returns a stable invalid-argument or overflow status +- **AND** no Buffer byte is read or modified outside its bounds + +### Requirement: Standard I/O SHALL provide exact binary transfer helpers + +The standard library SHALL provide offset-aware exact-read and write-all helpers for managed Buffer ranges on stdin/stdout and SHALL expose deterministic clean-EOF, truncated-input, I/O-error, and success outcomes. Protocol users SHALL be able to initialize standard streams in binary mode; on Windows this SHALL configure stdin and stdout with `_O_BINARY` before transferred bytes are interpreted. + +#### Scenario: Exact input arrives in partial reads + +- **WHEN** an exact read of a valid Buffer range is satisfied by multiple short native reads +- **THEN** the helper advances the offset until the requested byte count is filled and reports success + +#### Scenario: EOF occurs before or during an exact read + +- **WHEN** EOF occurs before any byte of a requested prefix or after only part of the requested range +- **THEN** the helper distinguishes clean EOF from truncated input with stable documented outcomes +- **AND** it never reports the truncated range as complete + +#### Scenario: Output requires partial writes + +- **WHEN** a valid Buffer range is written through native calls that accept fewer bytes than requested +- **THEN** the write-all helper continues from the correct offset until complete or a stable error occurs +- **AND** a zero-progress or failed write is not treated as success + +#### Scenario: Windows pipes preserve all byte values + +- **WHEN** a Windows program enables binary protocol I/O and transfers prefixes/payloads containing `0x0a`, `0x0d`, and `0x1a` through real parent/child pipes +- **THEN** stdin/stdout bytes match exactly without newline translation or control-Z EOF behavior + +#### Scenario: Existing text-style I/O remains compatible + +- **WHEN** a program continues to call existing `io_stdin_read`, `io_stdin_read_line`, `io_stdout_write`, or flush helpers without opting into the new protocol helpers +- **THEN** its documented source signatures and behavior remain unchanged + +### Requirement: Strict JSON parsing SHALL preserve object and Unicode validity + +The standard library SHALL add an opt-in strict JSON parse surface that validates the declared input length, UTF-8, complete grammar consumption, configured nesting bound, integer range, Unicode escape and surrogate-pair correctness, and uniqueness of decoded object keys. Existing `json_parse` and `json_parse_buffer` behavior SHALL remain source-compatible. + +#### Scenario: A strict valid Unicode document parses + +- **WHEN** strict parsing receives valid UTF-8 and JSON escapes including non-ASCII BMP characters and valid surrogate pairs within configured bounds +- **THEN** decoded strings contain the corresponding Unicode scalar values encoded as UTF-8 +- **AND** serialization/reparse preserves their semantic values + +#### Scenario: Duplicate decoded keys are rejected + +- **WHEN** one object contains the same decoded key more than once, including equivalent literal and escaped spellings +- **THEN** strict parsing fails with a stable duplicate-field or parse status +- **AND** it never silently keeps the first or last value + +#### Scenario: Malformed text is rejected + +- **WHEN** input contains invalid UTF-8, an invalid escape, an unpaired surrogate, a control character, trailing non-whitespace, excess nesting, or an integer outside the supported exact range +- **THEN** strict parsing fails deterministically without panic or partial-document success + +#### Scenario: Permissive callers do not change silently + +- **WHEN** an existing program uses the pre-change JSON parse entry points +- **THEN** its accepted input and observable result remain governed by the existing specification +- **AND** strict behavior requires an explicit new entry point or option + +### Requirement: JSON object inspection SHALL enable exhaustive decoders + +Strictly parsed object values SHALL expose bounded key count, indexed decoded-key access, exact key lookup, and value-kind inspection so application code can enforce required and allowed fields. Key equality SHALL use decoded Unicode scalar/UTF-8 equality without normalization or case folding. + +#### Scenario: An application rejects an unknown field + +- **WHEN** an exhaustive decoder iterates a strictly parsed object and encounters a key outside its contract allowlist +- **THEN** it can return a stable unknown-field error before consuming the object as a domain value + +#### Scenario: Object keys are inspected safely + +- **WHEN** code requests a key at a valid or invalid index +- **THEN** a valid index returns an owned decoded key and an invalid index returns a stable out-of-range status +- **AND** no borrowed runtime pointer escapes the JSON handle lifetime + +#### Scenario: General schema validation remains out of scope + +- **WHEN** a future application needs a reusable JSON Schema dialect, streaming validation, or dynamic Sengoo object mapping +- **THEN** it first updates OpenSpec with the dialect, lifecycle, resource ceilings, ownership, and compatibility rules +- **AND** the strict parse and object-inspection subset remains usable without that feature + +### Requirement: Strict JSON failures SHALL expose stable machine-readable kinds + +The strict JSON surface SHALL expose a stable error kind that distinguishes +unclassified syntax failures, duplicate decoded object keys, invalid +UTF-8/Unicode escapes or surrogates, and trailing input. Existing parse status, +offset, and human-readable message APIs SHALL remain compatible. Protocol code +SHALL NOT need to branch on diagnostic text. + +#### Scenario: A protocol maps strict parse failures + +- **WHEN** strict parsing rejects duplicate keys, invalid Unicode, or trailing bytes +- **THEN** the caller receives the documented stable error kind for that category +- **AND** the legacy parse status remains `PARSE` with its existing offset/message behavior + +#### Scenario: A later JSON operation succeeds + +- **WHEN** any prior parse failed and a subsequent parse succeeds +- **THEN** the last-error kind resets to `NONE` + +### Requirement: JSON builders SHALL preserve explicit string byte lengths + +The JSON document builder SHALL provide a length-aware string creation path +that copies exactly the declared valid UTF-8 bytes, including embedded NUL, +without changing the existing C-string-compatible helper. + +#### Scenario: A decoded string containing U+0000 is echoed + +- **WHEN** a strict document decodes `\u0000` and a caller creates a new JSON string from its owned bytes and explicit length +- **THEN** serialization and strict reparse preserve the complete string value and byte length +- **AND** no suffix after the embedded NUL is truncated diff --git a/openspec/changes/senline-service-dogfood/specs/toolchain-distribution/spec.md b/openspec/changes/senline-service-dogfood/specs/toolchain-distribution/spec.md new file mode 100644 index 00000000..2d033d0c --- /dev/null +++ b/openspec/changes/senline-service-dogfood/specs/toolchain-distribution/spec.md @@ -0,0 +1,64 @@ +## ADDED Requirements + +### Requirement: Installed distributions SHALL include target-native runtime artifacts + +Windows x64 and Linux x64 installed toolchain archives SHALL include the native runtime library and every declared runtime dependency required to compile, link, and run supported native Sengoo packages without a Sengoo source checkout. + +#### Scenario: An installed native package builds outside the checkout + +- **WHEN** a release-shaped archive is installed in a fresh path and `sgc` builds a supported native package from a different path +- **THEN** `sgc` resolves the target-native runtime and standard-library dependencies from the installed distribution +- **AND** no source checkout, Cargo target directory, `SENGOO_ROOT`, or local absolute repository path is required + +#### Scenario: Required runtime content is missing or incompatible + +- **WHEN** the installed runtime library is absent, has the wrong target or ABI, fails its hash, or lacks a declared dependency +- **THEN** native build fails with a stable diagnostic naming the incompatible distribution component +- **AND** `sgc` does not silently build or select a different runtime + +### Requirement: Native runtime manifests SHALL be complete and verifiable + +Each target distribution manifest SHALL record the runtime ABI version, target triple, source revision, tool versions, ordered link arguments, dynamic dependency identities, build-manifest identifier, and SHA-256 hash for every shipped payload file needed by native compilation and execution. + +#### Scenario: A consumer verifies an installed bundle + +- **WHEN** a consumer validates the distribution against its reviewed manifest +- **THEN** every executable, native runtime library, standard-library/runtime bridge file, and declared dynamic dependency has an expected identity and hash +- **AND** the consumer can reject a partial, mixed-target, or tampered installation before execution + +#### Scenario: Absolute development paths enter metadata + +- **WHEN** packaging inspects manifests, link arguments, and generated package metadata +- **THEN** mutable checkout, user-profile, Cargo target, and build-runner absolute paths are absent from consumer-facing resolution data + +### Requirement: Installed runtime discovery SHALL not hide a Cargo fallback + +Normal installed `sgc` native build/run/test SHALL prefer and require the installed manifest-selected runtime. Building the runtime through Cargo SHALL be available only through an explicit Sengoo-source development mode and SHALL never be an implicit recovery path for an installed distribution. + +#### Scenario: Cargo is unavailable during an installed smoke + +- **WHEN** an installed toolchain runs native worker and HTTP package smokes with a deliberately failing fake `cargo` first on PATH +- **THEN** check, test, and release build succeed using only installed runtime artifacts +- **AND** the fake Cargo executable is never invoked + +#### Scenario: A developer explicitly selects source-runtime mode + +- **WHEN** a contributor working inside a Sengoo source checkout opts into the documented source-runtime development mode +- **THEN** Cargo runtime construction may run with a diagnostic identifying the non-release mode +- **AND** artifacts from that mode are not accepted as installed-distribution or Senline pin evidence + +### Requirement: Installed application smokes SHALL cover real consumer packages + +Distribution dry-runs and release gates SHALL install the archive outside the source checkout and run the locked `senline-domain-worker` stdio/strict-JSON package loop plus the `senline-http-dogfood` native localhost smoke on Windows x64 and Linux x64. + +#### Scenario: A consumer package fails on one target + +- **WHEN** either installed package cannot check, test, format-check, document, release-build, or perform its required real execution smoke on a target +- **THEN** that target's distribution is ineligible for Senline pinning and publication evidence +- **AND** the failing target, command, artifact manifest, and diagnostic are retained + +#### Scenario: Independent builds reproduce payload identity + +- **WHEN** the same clean revision is built twice independently for one target +- **THEN** normalized manifests require identical payload hashes, runtime ABI, link arguments, and dynamic dependency identities +- **AND** only documented timestamps, runner metadata, and provenance-signature differences may be excluded from the comparison diff --git a/openspec/changes/senline-service-dogfood/tasks.md b/openspec/changes/senline-service-dogfood/tasks.md new file mode 100644 index 00000000..46b66770 --- /dev/null +++ b/openspec/changes/senline-service-dogfood/tasks.md @@ -0,0 +1,112 @@ +## 1. Linked Change and Contract Control + +- [x] 1.1 Strictly validate `senline-service-dogfood` and record the linked Senline change `adopt-sengoo-backend-slice`, its reviewed revision or artifact identity, and the fact that this Sengoo worktree is the only write scope for this change. +- [x] 1.2 Add the cross-repository authority matrix proving Rust retains TLS, signed-request verification, freshness/replay, device authorization/revocation, rate limits, cryptography, transactions, prekey/ACK/cursor state, persistence, migrations, final plan validation, and every mutation. +- [x] 1.3 Freeze raw V1 fixtures for `EvaluationContextV1`, minimum-necessary `SubmitEnvelopeFactsV1`, `WorkerRequestV1`, `SubmitEnvelopePlanV1`, startup handshake, and stable protocol/domain error envelopes. +- [x] 1.4 Add reviewed fixtures for eligible accept, exact duplicate, idempotency conflict, application-budget rejection, and unknown operation version; record that forged/revoked/stale/rate-limited requests are Rust-only and never worker inputs. +- [x] 1.5 Add contract leakage tests rejecting private keys, recovery material, plaintext, ciphertext bytes, raw signatures, tokens, credentials, connection strings, SQL, database rows, raw subject identifiers in evidence, and runtime/transaction handles. +- [x] 1.6 Pin protocol input at 32 KiB, output at 8 KiB, one request at a time, protocol-only stdout, and exact binding fields in generated package documentation and fixture metadata. +- [x] 1.7 Record all pre-existing framing, strict-JSON, installed-runtime, export-link-skip, and process-terminating panic gaps as known baseline defects without claiming they were discovered by Senline. + +## 2. Binary Buffer and Standard-I/O Red Tests + +- [x] 2.1 Add failing native runtime and real-`sgc` tests for byte get/set at first/last valid indexes and for negative, out-of-range, invalid-handle, invalid-byte, and overflow cases. 【Sengoo 修改】 +- [x] 2.2 Add failing tests for host-independent big-endian `u32` read/write, insufficient remaining capacity, invalid value range, and offset overflow. 【Sengoo 修改】 +- [x] 2.3 Add failing exact-read tests covering every partial split of a four-byte prefix and representative payloads, clean EOF before a prefix, truncation at every prefix/payload byte, zero-progress reads, and native I/O errors. 【Sengoo 修改】 +- [x] 2.4 Add failing write-all tests covering every partial split, offset advancement, zero-progress writes, broken pipes, flush failures, and proof that a short write never reports success. 【Sengoo 修改】 +- [x] 2.5 Add failing Windows real parent/child pipe tests containing `0x0a`, `0x0d`, `0x1a`, NUL, and non-ASCII bytes to expose text-mode translation and control-Z EOF behavior. 【Sengoo 修改】 +- [x] 2.6 Implement bounds-checked Buffer byte get/set and big-endian `u32` helpers with stable status mapping and no unchecked pointer arithmetic. 【Sengoo 修改】 +- [x] 2.7 Implement offset-aware stdin exact-read and stdout write-all helpers that loop over partial native operations and distinguish clean EOF from truncation. 【Sengoo 修改】 +- [x] 2.8 Implement explicit protocol binary-mode initialization, including `_O_BINARY` stdin/stdout on Windows before the first protocol byte, with a stable unsupported/error path on other host failures. 【Sengoo 修改】 +- [x] 2.9 Preserve existing text-style `std::io` signatures/behavior and add compatibility tests for stdin line reads, stdout/stderr writes, and flush helpers. 【Sengoo 修改】 +- [x] 2.10 Run focused stdlib wrappers, native runtime tests, compiler import/signature tests, LSP surface checks, Windows pipes, and POSIX pipes before accepting the binary-I/O slice. 【Sengoo 修改】 Evidence: Windows local suite green (compiler `io_module_preserves_binary_buffer_and_pipe_signatures`, sgc `binary_io_import_expands_*`, sglsp completions, `stdlib_io_`, `buffer_bytes`, `binary_io_exact_read`, `binary_io_write_all`); dual-host matrix is `core-conformance` job `binary-io-native` (ubuntu-latest + windows-latest) on this branch/PR. + +## 3. Strict JSON Red Tests and Implementation + +- [x] 3.1 Add failing strict-parser tests for duplicate literal keys, literal-versus-escaped duplicate keys, nested duplicates, and proof that neither first nor last value is silently selected. 【Sengoo 修改】 +- [x] 3.2 Add failing tests for invalid UTF-8 sequences, invalid escapes, raw control characters, non-ASCII BMP escapes, valid surrogate pairs, lone/reversed surrogates, and stable UTF-8 round-trip semantics. 【Sengoo 修改】 +- [x] 3.3 Add failing tests for trailing non-whitespace, excess nesting, out-of-range integers, wrong scalar kinds, truncated documents, and exact input-length enforcement. 【Sengoo 修改】 +- [x] 3.4 Add failing object-inspection tests for key count/iteration, owned decoded keys, exact lookup, invalid indexes, handle lifetime, and decoded Unicode equality without normalization or case folding. 【Sengoo 修改】 +- [x] 3.5 Implement an opt-in strict parse path that preserves duplicate-key evidence, validates Unicode/UTF-8/full consumption/depth/integer bounds, and returns stable statuses without panic or partial success. 【Sengoo 修改】 +- [x] 3.6 Implement bounded object key count, indexed owned-key access, exact lookup, and value-kind inspection sufficient for exhaustive contract decoders. 【Sengoo 修改】 +- [x] 3.7 Add a reusable explicit-allowlist decoder pattern that rejects unknown, missing, duplicate, wrong-typed, out-of-range, and unknown-enum fields without adding a general JSON Schema engine. 【Sengoo 修改】 +- [x] 3.8 Run compatibility fixtures proving existing `json_parse` and `json_parse_buffer` callers retain their documented permissive behavior and source signatures. 【Sengoo 修改】 +- [x] 3.9 Run JSON native/runtime fuzz and malformed corpora under the existing hardening limits and retain every fixed crash or panic as a regression. 【Sengoo 修改】 +- [x] 3.10 Add a stable strict-JSON last-error kind for unclassified syntax, duplicate fields, invalid Unicode, and trailing bytes while preserving legacy diagnostics. 【Sengoo 修改】 +- [x] 3.11 Add a length-aware JSON builder string API and embedded-NUL parse/build/serialize/reparse regression. 【Sengoo 修改】 + +## 4. Installed Native Runtime and Distribution + +- [x] 4.1 Add failing `sgc` tests proving an installed build currently cannot reliably resolve the per-target native runtime outside the source checkout and capture every accidental Cargo/checkout fallback. 【Sengoo 修改】 +- [x] 4.2 Define the installed layout for Windows `sengoo_runtime.lib`, Linux `libsengoo_runtime.a`, runtime bridge files, declared dynamic dependencies, and target-specific manifests without mutable absolute paths. 【Sengoo 修改】 +- [x] 4.3 Extend the distribution manifest with runtime ABI, target triple, source revision, coherent tool versions, ordered link arguments, dynamic dependency identities, build-manifest identifier, and SHA-256 for every payload. 【Sengoo 修改】 +- [x] 4.4 Make installed `sgc` select the manifest-matched native runtime for check/build/run/test and emit stable diagnostics for missing, tampered, wrong-target, wrong-ABI, or incomplete installations. 【Sengoo 修改】 +- [x] 4.5 Retain Cargo runtime construction only behind an explicit Sengoo-source development mode whose diagnostics and metadata make it ineligible for distribution/Senline pin evidence. 【Sengoo 修改】 +- [x] 4.6 Add fresh-directory installed smokes with checkout variables cleared, a deliberately failing fake `cargo` first on PATH, and auditing that rejects checkout/Cargo/user-profile absolute paths in resolution or link metadata. 【Sengoo 修改】 +- [x] 4.7 Build release-shaped Windows x64 and Linux x64 archives containing the complete native runtime and verify install scripts/checksums against both. 【Sengoo 修改】 Evidence: GitHub Actions run 29419695542 on `ba0d03ae3` (toolchain-distribution package smoke windows-latest + ubuntu-latest success: package, install, checksums, installed stdlib/run). +- [x] 4.8 Build each target twice independently and compare normalized manifests, requiring identical payload hashes, runtime ABI, link arguments, and dynamic dependency identities while documenting the only allowed provenance differences. 【Sengoo 修改】 Evidence: same run dual A/B package + `compare-distribution-manifests.ps1` green on Windows and Linux; allowed excluded diffs only (e.g. `generated_at_utc`); local dual target-dir builds produced identical `sgc.exe` and `sengoo_runtime.lib` SHA-256 under package-toolchain deterministic RUSTFLAGS. +- [x] 4.9 Run the existing distribution smoke matrix and block Sengoo publication and Senline pinning on any native runtime/package failure. 【Sengoo 修改】 Evidence: run 29419695542 all package-smoke jobs green (windows, ubuntu, macos-15, macos-15-intel); publish job remains tag-gated and skipped on non-tag dispatch. + +## 4A. Project-Driven Library Incubation + +- [x] 4A.1 Publish the capability-classification and graduation policy separating product packages, incubated pure Sengoo libraries, stdlib/runtime primitives, mature implementation bindings, and Rust-retained authority. 【Sengoo 修改】 +- [x] 4A.2 Scaffold locked domain-neutral `sgframing` and `sgjson_contract` packages beside the first consumer without external dependencies or Senline DTO names. 【Sengoo 修改】 +- [x] 4A.3 Add red/green package tests for clean EOF, truncation, zero/oversized frames, exact writes, exact object fields, required fields, kinds, integer ranges, ASCII/hex bounds, closed enums, and sorted-unique arrays. 【Sengoo 修改】 +- [x] 4A.4 Integrate the packages into `senline-domain-worker`, run the full locked source and installed toolchain loops, and prove the product package does not duplicate their framing or validation logic. 【Sengoo 修改】 Evidence: worker `Sengoo.toml` depends on `sgframing`/`sgjson_contract`/`senline_facts_to_plan` without reimplementing framing/JSON allowlists; installed-toolchain Windows loop with fake cargo first on PATH: `sgpm check/test/fmt --check/doc/build --locked` green (15 package tests). +- [x] 4A.5 Record API stability, documentation, platform, malformed-input, independent-consumer, publication, and stdlib-graduation evidence; treat every skipped required gate as not green. 【Sengoo 修改】 + +## 5. Pure Planner and Framed Worker Package + +- [x] 5.1 Scaffold locked `senline-domain-worker` and shared facts-to-plan packages using only source-controlled dependencies and the selected installed toolchain. 【Sengoo 修改】 Evidence: `examples/realworld/senline-domain-worker/**` with locked path deps and `Sengoo.lock`; installed `sgpm check --locked` resolves without remote deps. +- [x] 5.2 Generate exhaustive V1 decoders/encoders from the frozen raw fixtures or implement equivalent reviewed typed code, with exact required/allowed fields and stable decision/reason enums. +- [x] 5.3 Add failing worker tests for partial prefix/payload reads, partial writes, zero/oversized/truncated/surplus frames, trailing payload, invalid UTF-8/JSON/Unicode, duplicate/unknown fields, and recovery after one rejected request. 【Sengoo 修改】 +- [x] 5.4 Add failing tests for changed evaluation ID, facts binding, operation, epoch, generation, identifiers, contract version, impossible action, unknown enum, unstable reason, and oversized output. 【Sengoo 修改】 Evidence: `tools/sgc/tests/senline_plan_binding.rs` (echo well-formed evaluation_id; reject invalid operation/epoch/generation/contract/identifier/enum/capability/binding shapes; output 8 KiB bound). +- [x] 5.5 Implement startup handshake fields for protocol, Sengoo revision, toolchain, application, and reproducibly embedded build-manifest identifier without presenting a self-hash as trust evidence. 【Sengoo 修改】 +- [x] 5.6 Implement one-request-at-a-time binary framing with checked 32 KiB input and 8 KiB output, exact reads/writes, clean EOF shutdown, no implicit retry, and protocol-only stdout. +- [x] 5.7 Implement strict `WorkerRequestV1` decode and `SubmitEnvelopePlanV1` encode, echoing the exact host context/binding and rejecting every unsupported field/value before evaluation. +- [x] 5.8 Implement the pure submit-envelope planner without network, filesystem, environment, clock, randomness, database, subprocess, direct FFI, credentials, secrets, or retained request state. +- [x] 5.9 Add repeated and cross-process determinism tests proving byte-equivalent normalized plans for identical inputs on Windows x64 and Linux x64. 【Sengoo 修改】 Evidence: `senline_worker_differential` cross-process raw plan equality + dual-host CI run `29424861027` matching digests (determinism `bd6acd82…`, reviewed `a32f445f…`, seeded `16aebd9e…`); see `docs/senline-dogfood-determinism-evidence.md`. +- [x] 5.10 Restrict stderr to allowlisted bounded codes/development metadata and add randomized canary tests proving protocol values and parser input never reach stdout, stderr, logs, crash files, or package artifacts. +- [x] 5.11 Run at least 10,000 reviewed golden/boundary fixtures and 100,000 independent seeded eligible cases with zero semantic mismatch against the linked Rust reference fixtures and no crash, hang, malformed plan, or nondeterminism. 【Sengoo 修改】 +- [x] 5.12 Run locked check/test/fmt-check/doc/release-build plus real parent/child execution on Windows and Linux using only the installed toolchain outside the Sengoo checkout. 【Sengoo 修改】 Evidence: installed job on tip `3e747e63b` run [`29595215669`](https://github.com/Hyper66666/Sengoo/actions/runs/29595215669) — `sgpm fmt --check`/`doc`/check/test/build under fake cargo; worker framed product loop green; HTTP localhost happy-path + worker/HTTP plan byte equality + malformed_json@200 + GET@400; dual-host. (Job still ends red only on HTTP dual-package hash for 8.7.) +- [x] 5.13 Package the worker and all runtime dependencies with manifest, hashes, licenses/SBOM inputs, source revision, protocols, target/ABI metadata, and no compiler checkout or local absolute dependency. 【Sengoo 修改】 Evidence: package scripts emit LICENSE.txt + sbom-inputs.json + pin-grade fields; CI artifacts `senline-installed-packages-{linux,windows}-x86_64` retain worker-a/http-a trees + manifests; worker dual-package compare ok=true (33 identical payloads) on both hosts in run 29595215669. + +## 6. Loopback HTTP Dogfood Harness + +- [x] 6.1 Scaffold locked `senline-http-dogfood` using the exact shared facts-to-plan module and only installed-toolchain-resolvable dependencies. 【Sengoo 修改】 +- [x] 6.2 Add failing tests for non-loopback bind attempts, fixed externally reachable endpoints, bounded headers/body, unsupported version, strict malformed JSON, timeout, and excess concurrent/pending work. 【Sengoo 修改】 +- [x] 6.3 Add failing tests for pending `next_request_async` future drop cleanup, timeout cleanup, accepted-but-unpublished request cleanup, exactly-once response, double-response rejection, and clean server close without claiming general task cancellation. 【Sengoo 修改】 +- [x] 6.4 Implement an ephemeral-loopback-only development endpoint that accepts synthetic/non-secret V1 facts and returns the same normalized plan/error contract as the framed worker. 【Sengoo 修改】 +- [ ] 6.5 Add Windows and Linux real-`sgc` localhost tests covering async request handling, strict malformed input, timeout, pending-future drop, exactly-once response, close cleanup, and worker/HTTP plan equivalence. **REOPENED (review):** installed probes cover happy/malformed/GET400/plan-equality only; timeout/future-drop/exactly-once/double-response remain Rust runtime unit tests (core-language is Ubuntu-only), not dual-host real-sgc localhost product matrix. +- [ ] 6.6 Add source/release checks rejecting any Senline Windows client, Android client, internal-alpha route, production endpoint, or deployment manifest that targets the harness. **REOPENED (review):** policy walk omits .kt/.kts/.gradle/.xml/.cs/.xaml/.ts client surfaces; `/v1/submit-envelope` marker skipped; Senline checkout absent soft-skips on CI without fail-closed fixture. +- [x] 6.7 Document and test the retained serial, plaintext, `Connection: close` limits and keep TLS, keep-alive, streaming, handlers, broad cancellation, and ingress promotion owned by their separate changes. 【Sengoo 修改】 Evidence: HTTP README retained-limits section + `policy_contract.sg` + `http_dogfood_documents_serial_plaintext_non_ingress_limits` test. + +## 7. Consumer-Driven Defect Loop + +- [x] 7.1 Create the durable evidence schema linking Senline failure ID/fixture, ownership classification, minimized Sengoo regression, fixing commit, target artifacts/hashes, Senline pin revision, and final consumer gate. 【Sengoo 修改】 +- [ ] 7.2 For every Sengoo-owned failure, preserve red consumer evidence, minimize it in this repository, and commit a failing compiler/runtime/stdlib/package regression before changing implementation. **OPEN:** `docs/senline-dogfood-evidence.v1.json` keeps `red_status=pending-commit` / `red_commit=null` until true red-first history is reconstructed (no fabricated SHAs). +- [x] 7.3 Implement the smallest general fix, run all affected Sengoo gates, and record rejected workaround-only alternatives and remaining platform gaps. 【Sengoo 修改】 Evidence: product-level fixes landed (Buffer zeroing, transitive module map, HTTP string accessors, worker Drop/owning helpers); durable evidence currently keeps fixing_commit null until red-first history is reconstructed (7.2). remaining_platform_gaps include 7.2 red-first + 7.5/9.5 Senline pin. +- [ ] 7.4 Produce clean immutable Windows/Linux installed toolchain and worker artifacts from the fixing commit and verify complete manifests/provenance before offering a pin advance. **OPEN:** CI retains pin-grade worker/HTTP package trees (run 29595215669 artifacts) but evidence `target_artifacts` remain pending until a fixing-commit + pin offer (blocked by 7.2/7.5). +- [ ] 7.5 Advance the Senline pin atomically to the reviewed artifacts and rerun the minimized regression plus linked differential, leakage, malformed-output, and integration gates before marking green. **Blocked:** requires a writable Senline Git revision and pin update outside this Sengoo worktree scope. +- [x] 7.6 Validate workaround registry entries require owner, linked defect, expiry condition, and removal test; fail evidence validation for floating paths, mutable checkouts, partial pins, or workaround-only green claims. 【Sengoo 修改】 +- [ ] 7.7 Demonstrate one complete red/minimize/fix/pin/green chain with a genuine Senline-discovered defect, or clearly label a known/injected framing, strict-JSON, or installed-runtime rehearsal if no new defect appears. **OPEN:** known-baseline rehearsals labeled; pin/green step deferred (7.5 blocked); not a complete red/minimize/fix/pin/green chain. + +## 8. Failure, Resource, and Reproducibility Evidence + +- [x] 8.1 Run malformed, partial, truncated, oversized, invalid UTF-8/Unicode/JSON, duplicate/unknown-field, out-of-range, excess-nesting, trailing-byte, surplus-frame, unknown-enum, and unknown-version corpora with deterministic failure and no success plan. 【Sengoo 修改】 Evidence: `senline_worker_faults` 6/6 + realworld schema/malformed recovery suites + plan-binding rejections. +- [x] 8.2 Run worker kill, abort/panic, broken-pipe, slow/partial I/O, stdout text contamination, stderr flood, and startup-handshake mismatch fixtures and prove failures stay inside the worker process. 【Sengoo 修改】 Evidence: `senline_worker_faults` 11/11 including kill mid-session, Option-unwrap panic/abort path, broken stdout pipe, stdout contamination detection, stderr flood, handshake mismatch, plus prior partial-I/O/framing/malformed/canary suites — all process-contained (host continues). +- [ ] 8.3 Run at least one million worker evaluations with bounded file/handle/process counts and stable post-warm-up memory; check in sampler methodology and Windows private-working-set/Linux RSS interpretation. **REOPENED (review):** process_count still hardcoded to 1 (no child-process enumeration); 1M summary/JSONL are gitignored without published SHA-256/source revision per methodology publication rules. +- [x] 8.4 After warm-up, measure representative request-to-valid-response latency on recorded Windows/Linux reference hosts and publish payload/concurrency methodology without claiming Senline host admission or sandbox timing. 【Sengoo 修改】 Evidence: docs/senline-dogfood-latency-evidence.md; soak-1m p50/p95/p99=179/350/450 µs; CI resource smoke on dual hosts. +- [x] 8.5 Scan packages, manifests, logs, stderr captures, differential artifacts, crash files, and transcripts for randomized request/recovery canaries and prohibited secrets. 【Sengoo 修改】 Evidence: fault leakage canary suite + worker package byte scan (no recovery_seed/private-key/password canaries). +- [x] 8.6 Verify generated/runtime link metadata and installed packages are independent of the Sengoo checkout, Cargo target directory, developer profile, and build-runner path. 【Sengoo 修改】 Evidence: package-toolchain deterministic remap + distribution install smokes with fake cargo and path audits (run 29419695542). +- [ ] 8.7 Rebuild the installed toolchain, native runtime, worker, and HTTP package twice per target and retain normalized reproducibility comparisons plus SBOM/provenance. **OPEN:** worker dual-package is bit-identical on Windows+Linux (run 29595215669, compare ok=true, 33 payloads). HTTP dual-package still equal-size executable_hash_mismatch under fail-closed compare (product probes green). Negative identity tests in `scripts/tests/compare-senline-package-manifests.tests.ps1`. +- [x] 8.8 Publish a Sengoo-side support record distinguishing proven installed worker/package behavior from Senline-owned sandbox, supervisor, shadow, guarded-development, internal-alpha, and rollback claims. 【Sengoo 修改】 + +## 9. Final Verification and Handoff + +- [ ] 9.1 Run `cargo fmt --check`, affected Clippy/static-analysis gates, focused compiler/runtime/stdlib/tool tests, and the complete locked package loops with bounded timeouts. **REOPENED (review):** `cargo clippy -D warnings` fails on tip (`frontend_helpers.rs` clippy::min_max constant result). +- [ ] 9.2 Run Windows x64 and Linux x64 installed-distribution smokes with fake-failing Cargo, real binary pipes, real localhost HTTP, and complete manifest/hash verification. **REOPENED (review):** dual-host installed jobs red on run 29595215669; HTTP package executable+SBOM hash mismatch means complete manifest/hash verification has not passed. +- [ ] 9.3 Strictly validate `senline-service-dogfood`, `adopt-sengoo-backend-slice`, and any amended owning Sengoo capability changes/specs against the same recorded revisions. **Partial:** validates pass; reopened until claimed-green tasks are honest again. +- [x] 9.4 Publish final protocol/authority diagrams, raw V1 schemas and fixtures, stable error taxonomy, installed layout, artifact provenance, known defects, and unsupported authority transfers. 【Sengoo 修改】 Evidence: `docs/senline-dogfood-handoff.md` + support/defects/evidence schema + V1 fixtures under `examples/realworld/senline-domain-worker/fixtures/v1/` + generated `protocol-v1.md` + determinism/repro/resource/latency docs. +- [ ] 9.5 Hand Senline the clean source revision, target-specific installed-toolchain and worker manifests, complete per-file hashes, SBOM/provenance inputs, and green evidence needed for its reviewed pin. **Blocked:** requires writable Senline Git revision and pin publish outside this Sengoo worktree (paired with 7.5). +- [x] 9.6 Require a separate OpenSpec change before moving TLS, cryptography, authentication/replay authority, durable transactions, persistence/migrations, public/internal-alpha ingress, or final mutation authority into Sengoo. 【Sengoo 修改】 diff --git a/runtime/src/net.rs b/runtime/src/net.rs index 3e55795c..9279eb45 100644 --- a/runtime/src/net.rs +++ b/runtime/src/net.rs @@ -1524,6 +1524,78 @@ mod tests { assert_eq!(sengoo_http_server_close(server), 1); } + fn assert_ready_http_future_abandonment_releases_request(cancel: bool) { + let _guard = net_test_lock(); + let host = b"127.0.0.1\0"; + let server = sengoo_http_server_bind(host.as_ptr(), 0); + assert!(server != 0); + let port = sengoo_http_server_local_port(server) as u16; + let table_len_before = net_runtime().http_request_table_len(); + + let future = sengoo_http_server_next_request_async__start(server, 4_000); + assert!(future != 0); + let client = thread::spawn(move || { + let request = b"POST /abandoned-ready HTTP/1.1\r\nHost: localhost\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok"; + send_raw_http_request(port, request) + }); + assert_eq!(poll_http_next_request_until_ready(future, 4_000), 1); + assert_eq!( + net_runtime().http_request_pending_count(server).unwrap(), + 1, + "the ready future owns one unpublished request" + ); + + if cancel { + assert!(unsafe { sengoo_http_server_next_request_async__cancel(future) }); + } else { + unsafe { sengoo_http_server_next_request_async__drop(future) }; + } + + let table_len_after = net_runtime().http_request_table_len(); + let pending_after = net_runtime().http_request_pending_count(server).unwrap(); + if table_len_after != table_len_before || pending_after != 0 { + assert_eq!( + sengoo_http_server_close(server), + 1, + "RED cleanup must drain a leaked unpublished request" + ); + } + let (status, body) = parse_http_status_and_body(&client.join().expect("client")); + assert_eq!(status, 504); + assert_eq!(body, b"gateway timeout"); + assert_eq!( + table_len_after, table_len_before, + "abandoning a ready future must remove its unpublished request handle" + ); + assert_eq!(pending_after, 0); + + let next_client = thread::spawn(move || { + let request = b"GET /after-ready-abandon HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"; + send_raw_http_request(port, request) + }); + let next = sengoo_http_server_next_request(server, 4_000); + assert!( + next != 0, + "server must remain usable after ready abandonment" + ); + assert_eq!(sengoo_http_request_respond(next, 200, b"ok".as_ptr(), 2), 1); + let (next_status, next_body) = + parse_http_status_and_body(&next_client.join().expect("next client")); + assert_eq!(next_status, 200); + assert_eq!(next_body, b"ok"); + assert_eq!(sengoo_http_server_close(server), 1); + } + + #[test] + fn http_server_next_request_async_drop_ready_releases_unpublished_request() { + assert_ready_http_future_abandonment_releases_request(false); + } + + #[test] + fn http_server_next_request_async_cancel_ready_releases_unpublished_request() { + assert_ready_http_future_abandonment_releases_request(true); + } + #[test] fn http_server_next_request_async_slow_client_never_publishes_partial_request() { let _guard = net_test_lock(); diff --git a/runtime/src/net/http_server.rs b/runtime/src/net/http_server.rs index a84da692..b0ab8c41 100644 --- a/runtime/src/net/http_server.rs +++ b/runtime/src/net/http_server.rs @@ -2124,6 +2124,20 @@ pub unsafe extern "C" fn sengoo_http_server_next_request_async__result( } } +fn release_abandoned_async_next_request(state: &AsyncNextRequestState) { + if let Some(interest) = state.listener_interest { + crate::async_runtime::http_listener_unregister(interest); + } + if let AsyncNextRequestOutcome::Ready { + is_ok: true, value, .. + } = state.outcome + { + if let Ok(mut entry) = net_runtime().http_request_take(value) { + let _ = write_http_response(&mut entry.stream, &gateway_timeout_response(), false); + } + } +} + #[no_mangle] /// # Safety /// @@ -2133,9 +2147,7 @@ pub unsafe extern "C" fn sengoo_http_server_next_request_async__cancel(handle: i let Some(state) = (unsafe { async_handle_take_box::(handle) }) else { return false; }; - if let Some(interest) = state.listener_interest { - crate::async_runtime::http_listener_unregister(interest); - } + release_abandoned_async_next_request(&state); true } @@ -2148,9 +2160,7 @@ pub unsafe extern "C" fn sengoo_http_server_next_request_async__drop(handle: i64 let Some(state) = (unsafe { async_handle_take_box::(handle) }) else { return; }; - if let Some(interest) = state.listener_interest { - crate::async_runtime::http_listener_unregister(interest); - } + release_abandoned_async_next_request(&state); } // Router-mode async lifecycle aliases: same state machine as pull, different start only. diff --git a/scripts/compare-distribution-manifests.ps1 b/scripts/compare-distribution-manifests.ps1 new file mode 100644 index 00000000..ace4fc4e --- /dev/null +++ b/scripts/compare-distribution-manifests.ps1 @@ -0,0 +1,384 @@ +param( + [Parameter(Mandatory = $true)] + [string]$LeftManifest, + [Parameter(Mandatory = $true)] + [string]$RightManifest, + [Parameter(Mandatory = $true)] + [string]$OutputDir +) + +$ErrorActionPreference = "Stop" + +$ManifestFields = @( + "schema_version", "version", "target", "build_hash", "source_revision", + "source_dirty", "artifact_provenance", "release_eligible", "build_manifest_id", + "tools", "tool_versions", "stdlib_modules", "runtime_sources", "native_runtime", + "payload_checksum_file", "payloads", "archive_file", "checksum_file", "runner_os", + "runner_image", "smoke_evidence", "license_included", "generated_at_utc" +) +$NativeRuntimeFields = @( + "abi_version", "target", "library", "sha256", "link_args", "dynamic_dependencies" +) +$PayloadFields = @("path", "sha256", "size") +$ExcludedFields = @("generated_at_utc", "runner_os", "runner_image", "smoke_evidence") + +function Read-JsonObject([string]$Path, [string]$Label) { + $resolved = (Resolve-Path -LiteralPath $Path).Path + try { + $value = Get-Content -LiteralPath $resolved -Raw | ConvertFrom-Json + } catch { + throw "$Label is not valid JSON: $($_.Exception.Message)" + } + if ($null -eq $value -or $value -isnot [pscustomobject]) { + throw "$Label must contain one JSON object" + } + return $value +} + +function Assert-ExactFields($Value, [string[]]$Expected, [string]$Label) { + if ($null -eq $Value -or $Value -isnot [pscustomobject]) { + throw "$Label must be an object" + } + $actual = @($Value.PSObject.Properties.Name) + $missing = @($Expected | Where-Object { $_ -notin $actual }) + if ($missing.Count -ne 0) { + throw "missing manifest field at ${Label}: $($missing -join ', ')" + } + $unknown = @($actual | Where-Object { $_ -notin $Expected }) + if ($unknown.Count -ne 0) { + throw "unknown manifest field at ${Label}: $($unknown -join ', ')" + } +} + +function Coerce-JsonString($Value) { + if ($null -eq $Value) { + return $null + } + if ($Value -is [string]) { + return [string]$Value + } + # ConvertFrom-Json may revive ISO-8601 timestamps as DateTime on some hosts. + if ($Value -is [datetime]) { + return ([datetime]$Value).ToUniversalTime().ToString("o") + } + if ($Value -is [datetimeoffset]) { + return ([datetimeoffset]$Value).ToUniversalTime().ToString("o") + } + return [string]$Value +} + +function Require-String($Value, [string]$Label, [switch]$AllowEmpty) { + $text = Coerce-JsonString $Value + if ($null -eq $text -or $text -isnot [string] -or (-not $AllowEmpty -and $text.Length -eq 0)) { + throw "$Label must be a non-empty string" + } + return [string]$text +} + +function Require-NullableString($Value, [string]$Label) { + if ($null -eq $Value) { + return $null + } + $text = Coerce-JsonString $Value + if ($null -eq $text -or $text -isnot [string]) { + throw "$Label must be a string or null" + } + return [string]$text +} + +function Require-Bool($Value, [string]$Label) { + if ($Value -isnot [bool]) { + throw "$Label must be a boolean" + } + return [bool]$Value +} + +function Require-Integer($Value, [string]$Label, [long]$Minimum = [long]::MinValue) { + if ($Value -isnot [byte] -and $Value -isnot [int16] -and $Value -isnot [int32] -and + $Value -isnot [int64] -and $Value -isnot [uint16] -and $Value -isnot [uint32]) { + throw "$Label must be an integer" + } + $integer = [long]$Value + if ($integer -lt $Minimum) { + throw "$Label must be at least $Minimum" + } + return $integer +} + +function Require-Array($Value, [string]$Label) { + # ConvertTo-Json/ConvertFrom-Json round-trips often collapse single-element + # arrays to scalars and empty arrays to $null. Accept those shapes and + # rehydrate a true array for normalized comparison. + if ($null -eq $Value) { + return ,@() + } + if ($Value -is [System.Array]) { + return ,$Value + } + if ($Value -is [System.Collections.IEnumerable] -and $Value -isnot [string]) { + return ,@($Value) + } + return ,@($Value) +} + +function Normalize-Hex($Value, [int]$Length, [string]$Label) { + $text = Require-String $Value $Label + if ($text.Length -ne $Length -or $text -notmatch "^[0-9a-fA-F]{$Length}$") { + throw "$Label must be exactly $Length hexadecimal characters" + } + return $text.ToLowerInvariant() +} + +function Normalize-RelativePath($Value, [string]$Label) { + $path = Require-String $Value $Label + if ($path.Contains('\') -or $path.StartsWith('/') -or $path.EndsWith('/') -or + $path -match '^[A-Za-z]:' -or $path.Contains('//')) { + throw "$Label must be a normalized relative path" + } + foreach ($segment in $path.Split('/')) { + if ($segment.Length -eq 0 -or $segment -eq '.' -or $segment -eq '..') { + throw "$Label must be a normalized relative path" + } + } + return $path +} + +function Sort-Ordinal([string[]]$Values) { + $copy = [string[]]@($Values) + [Array]::Sort($copy, [StringComparer]::Ordinal) + return ,$copy +} + +function Normalize-StringSet($Value, [string]$Label) { + $items = Require-Array $Value $Label + $seen = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + $normalized = @() + for ($index = 0; $index -lt $items.Count; $index++) { + $item = Require-String $items[$index] "$Label[$index]" + if (-not $seen.Add($item)) { + throw "$Label contains duplicate value: $item" + } + $normalized += $item + } + return ,(Sort-Ordinal $normalized) +} + +function Normalize-OrderedStrings($Value, [string]$Label) { + $items = Require-Array $Value $Label + $normalized = @() + for ($index = 0; $index -lt $items.Count; $index++) { + $normalized += Require-String $items[$index] "$Label[$index]" + } + return ,$normalized +} + +function Normalize-ToolVersions($Value, [string[]]$Tools, [string]$Label) { + Assert-ExactFields $Value $Tools $Label + $result = [ordered]@{} + foreach ($tool in (Sort-Ordinal $Tools)) { + $result[$tool] = Require-String $Value.PSObject.Properties[$tool].Value "$Label.$tool" + } + return [pscustomobject]$result +} + +function Normalize-Payloads($Value, [string]$Label) { + $items = Require-Array $Value $Label + if ($items.Count -eq 0) { + throw "$Label must contain at least one payload" + } + $byPath = [Collections.Generic.SortedDictionary[string, object]]::new([StringComparer]::Ordinal) + $caseFolded = [Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) + for ($index = 0; $index -lt $items.Count; $index++) { + $itemLabel = "$Label[$index]" + Assert-ExactFields $items[$index] $PayloadFields $itemLabel + $path = Normalize-RelativePath $items[$index].path "$itemLabel.path" + if (-not $caseFolded.Add($path)) { + throw "duplicate payload path: $path" + } + $payload = [pscustomobject][ordered]@{ + path = $path + sha256 = Normalize-Hex $items[$index].sha256 64 "$itemLabel.sha256" + size = Require-Integer $items[$index].size "$itemLabel.size" 0 + } + $byPath.Add($path, $payload) + } + return ,@($byPath.Values) +} + +function Normalize-Manifest($Manifest, [string]$Label) { + Assert-ExactFields $Manifest $ManifestFields $Label + $schemaVersion = Require-Integer $Manifest.schema_version "$Label.schema_version" 0 + if ($schemaVersion -ne 2) { + throw "$Label.schema_version must be 2" + } + $version = Require-String $Manifest.version "$Label.version" + $target = Require-String $Manifest.target "$Label.target" + $sourceRevision = Normalize-Hex $Manifest.source_revision 40 "$Label.source_revision" + $buildHash = Require-String $Manifest.build_hash "$Label.build_hash" + if ($buildHash -notmatch '^[0-9a-fA-F]{7,40}$' -or + -not $sourceRevision.StartsWith($buildHash.ToLowerInvariant(), [StringComparison]::Ordinal)) { + throw "$Label.build_hash must be a hexadecimal prefix of source_revision" + } + $tools = Normalize-StringSet $Manifest.tools "$Label.tools" + $payloads = Normalize-Payloads $Manifest.payloads "$Label.payloads" + + Assert-ExactFields $Manifest.native_runtime $NativeRuntimeFields "$Label.native_runtime" + $nativeTarget = Require-String $Manifest.native_runtime.target "$Label.native_runtime.target" + if ($nativeTarget -ne $target) { + throw "$Label.native_runtime.target must equal target" + } + $nativeLibrary = Normalize-RelativePath $Manifest.native_runtime.library "$Label.native_runtime.library" + $nativeHash = Normalize-Hex $Manifest.native_runtime.sha256 64 "$Label.native_runtime.sha256" + $runtimePayload = @($payloads | Where-Object { $_.path -eq $nativeLibrary }) + if ($runtimePayload.Count -ne 1 -or $runtimePayload[0].sha256 -ne $nativeHash) { + throw "$Label.native_runtime library must have one matching payload hash" + } + + $normalized = [ordered]@{ + schema_version = $schemaVersion + version = $version + target = $target + build_hash = $buildHash.ToLowerInvariant() + source_revision = $sourceRevision + source_dirty = Require-Bool $Manifest.source_dirty "$Label.source_dirty" + artifact_provenance = Require-String $Manifest.artifact_provenance "$Label.artifact_provenance" + release_eligible = Require-Bool $Manifest.release_eligible "$Label.release_eligible" + build_manifest_id = Normalize-Hex $Manifest.build_manifest_id 64 "$Label.build_manifest_id" + tools = @($tools) + tool_versions = Normalize-ToolVersions $Manifest.tool_versions $tools "$Label.tool_versions" + stdlib_modules = @(Normalize-StringSet $Manifest.stdlib_modules "$Label.stdlib_modules") + runtime_sources = @(Normalize-StringSet $Manifest.runtime_sources "$Label.runtime_sources") + native_runtime = [ordered]@{ + abi_version = Require-Integer $Manifest.native_runtime.abi_version "$Label.native_runtime.abi_version" 0 + target = $nativeTarget + library = $nativeLibrary + sha256 = $nativeHash + link_args = @(Normalize-OrderedStrings $Manifest.native_runtime.link_args "$Label.native_runtime.link_args") + dynamic_dependencies = @(Normalize-StringSet $Manifest.native_runtime.dynamic_dependencies "$Label.native_runtime.dynamic_dependencies") + } + payload_checksum_file = Normalize-RelativePath $Manifest.payload_checksum_file "$Label.payload_checksum_file" + payloads = @($payloads) + archive_file = Normalize-RelativePath $Manifest.archive_file "$Label.archive_file" + checksum_file = Normalize-RelativePath $Manifest.checksum_file "$Label.checksum_file" + license_included = Require-Bool $Manifest.license_included "$Label.license_included" + } + + $excluded = [ordered]@{ + generated_at_utc = Require-String $Manifest.generated_at_utc "$Label.generated_at_utc" + runner_os = Require-NullableString $Manifest.runner_os "$Label.runner_os" + runner_image = Require-NullableString $Manifest.runner_image "$Label.runner_image" + smoke_evidence = Require-String $Manifest.smoke_evidence "$Label.smoke_evidence" -AllowEmpty + } + return [pscustomobject][ordered]@{ + normalized = [pscustomobject]$normalized + excluded = [pscustomobject]$excluded + } +} + +function Canonical-Json($Value, [switch]$Pretty) { + if ($null -eq $Value) { + return "null" + } + $json = if ($Pretty) { + $Value | ConvertTo-Json -Depth 12 + } else { + $Value | ConvertTo-Json -Depth 12 -Compress + } + if ($null -eq $json) { + return "null" + } + return $json.Replace("`r`n", "`n") +} + +function Write-Utf8NoBom([string]$Path, [string]$Text) { + $parent = Split-Path -Parent $Path + if ($parent) { + New-Item -ItemType Directory -Force -Path $parent | Out-Null + } + $encoding = [Text.UTF8Encoding]::new($false) + [IO.File]::WriteAllText($Path, $Text.TrimEnd("`r", "`n") + "`n", $encoding) +} + +function String-Sha256([string]$Text) { + $sha = [Security.Cryptography.SHA256]::Create() + try { + $bytes = [Text.UTF8Encoding]::new($false).GetBytes($Text.TrimEnd("`r", "`n") + "`n") + return ([BitConverter]::ToString($sha.ComputeHash($bytes))).Replace("-", "").ToLowerInvariant() + } finally { + $sha.Dispose() + } +} + +function Json-Equal($Left, $Right) { + return (Canonical-Json $Left) -ceq (Canonical-Json $Right) +} + +$leftRaw = Read-JsonObject $LeftManifest "left manifest" +$rightRaw = Read-JsonObject $RightManifest "right manifest" +$left = Normalize-Manifest $leftRaw "left manifest" +$right = Normalize-Manifest $rightRaw "right manifest" + +New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null +$leftJson = Canonical-Json $left.normalized -Pretty +$rightJson = Canonical-Json $right.normalized -Pretty +Write-Utf8NoBom (Join-Path $OutputDir "normalized-a.json") $leftJson +Write-Utf8NoBom (Join-Path $OutputDir "normalized-b.json") $rightJson +$leftSha = String-Sha256 $leftJson +$rightSha = String-Sha256 $rightJson + +$differences = @() +foreach ($field in @( + "schema_version", "version", "target", "build_hash", "source_revision", "source_dirty", + "artifact_provenance", "release_eligible", "build_manifest_id", "tools", "tool_versions", + "stdlib_modules", "runtime_sources", "payload_checksum_file", "payloads", "archive_file", + "checksum_file", "license_included" +)) { + if (-not (Json-Equal $left.normalized.$field $right.normalized.$field)) { + $differences += $field + } +} +foreach ($field in $NativeRuntimeFields) { + if (-not (Json-Equal $left.normalized.native_runtime.$field $right.normalized.native_runtime.$field)) { + $differences += "native_runtime.$field" + } +} + +$excludedDifferences = @() +foreach ($field in $ExcludedFields) { + if (-not (Json-Equal $left.excluded.$field $right.excluded.$field)) { + $excludedDifferences += [pscustomobject][ordered]@{ + field = $field + left = $left.excluded.$field + right = $right.excluded.$field + } + } +} + +$status = if ($differences.Count -eq 0) { "reproducible" } else { "mismatch" } +$comparison = [pscustomobject][ordered]@{ + schema_version = 1 + status = $status + left = [ordered]@{ + manifest = [IO.Path]::GetFileName($LeftManifest) + normalized_file = "normalized-a.json" + normalized_sha256 = $leftSha + } + right = [ordered]@{ + manifest = [IO.Path]::GetFileName($RightManifest) + normalized_file = "normalized-b.json" + normalized_sha256 = $rightSha + } + excluded_fields = $ExcludedFields + excluded_differences = @($excludedDifferences) + mismatched_fields = @($differences) +} +Write-Utf8NoBom (Join-Path $OutputDir "comparison.json") (Canonical-Json $comparison -Pretty) + +if ($differences.Count -ne 0) { + throw "distribution manifests differ: $($differences -join ', ')" +} +if ($leftSha -ne $rightSha) { + throw "normalized manifest SHA-256 differs despite no field mismatch" +} + +Write-Host "Distribution manifests are reproducible: $leftSha" diff --git a/scripts/compare-senline-package-manifests.ps1 b/scripts/compare-senline-package-manifests.ps1 new file mode 100644 index 00000000..3dc10cbe --- /dev/null +++ b/scripts/compare-senline-package-manifests.ps1 @@ -0,0 +1,410 @@ +param( + [Parameter(Mandatory = $true)] + [string]$LeftManifest, + [Parameter(Mandatory = $true)] + [string]$RightManifest, + [Parameter(Mandatory = $true)] + [string]$OutputDir, + # Explicit opt-in only. Default pin-grade policy requires every payload hash + # (including executables) to match across dual builds. With this switch, + # executable hash divergence is recorded under allowed_executable_drift and + # does not fail the comparison. + [switch]$AllowExecutableHashDrift +) + +$ErrorActionPreference = "Stop" + +# Required on every pin-grade package manifest (design: payload hashes, ABI, +# dependency identities, and provenance must match across dual builds). +$ManifestRequiredFields = @( + "schema_version", "package", "version", "built_with_sgc", "source_tree", + "source_revision", "target", "protocols", + "runtime_dependencies", "build_tools", "license", "provenance", + "payloads", "notes" +) +# Optional only when a package family does not emit the field (HTTP packages +# currently omit build_manifest_id; worker packages emit it). +$ManifestOptionalFields = @( + "build_manifest_id" +) +$PayloadFields = @("path", "sha256", "size") +$ExecutableNamePatterns = @( + "senline_domain_worker", + "senline_domain_worker.exe", + "senline_http_dogfood", + "senline_http_dogfood.exe" +) + +function Read-JsonObject([string]$Path, [string]$Label) { + $resolved = (Resolve-Path -LiteralPath $Path).Path + try { + $value = Get-Content -LiteralPath $resolved -Raw | ConvertFrom-Json + } catch { + throw "$Label is not valid JSON: $($_.Exception.Message)" + } + if ($null -eq $value -or $value -isnot [pscustomobject]) { + throw "$Label must contain one JSON object" + } + return $value +} + +function Assert-ExactFields($Value, [string[]]$Expected, [string]$Label) { + if ($null -eq $Value -or $Value -is [string] -or $Value -is [ValueType] -or $Value -is [System.Array]) { + throw "$Label must be an object" + } + if ($null -eq $Value.PSObject -or $null -eq $Value.PSObject.Properties) { + throw "$Label must be an object with properties" + } + $actual = @($Value.PSObject.Properties.Name) + $missing = @($Expected | Where-Object { $_ -notin $actual }) + if ($missing.Count -ne 0) { + throw "missing manifest field at ${Label}: $($missing -join ', ')" + } + $unknown = @($actual | Where-Object { $_ -notin $Expected }) + if ($unknown.Count -ne 0) { + throw "unknown manifest field at ${Label}: $($unknown -join ', ')" + } +} + +function Assert-ManifestFields($Value, [string]$Label) { + if ($null -eq $Value -or $Value -is [string] -or $Value -is [ValueType] -or $Value -is [System.Array]) { + throw "$Label must be an object" + } + if ($null -eq $Value.PSObject -or $null -eq $Value.PSObject.Properties) { + throw "$Label must be an object with properties" + } + $actual = @($Value.PSObject.Properties.Name) + $allowed = @($ManifestRequiredFields + $ManifestOptionalFields) + $missing = @($ManifestRequiredFields | Where-Object { $_ -notin $actual }) + if ($missing.Count -ne 0) { + throw "missing required manifest field at ${Label}: $($missing -join ', ')" + } + $unknown = @($actual | Where-Object { $_ -notin $allowed }) + if ($unknown.Count -ne 0) { + throw "unknown manifest field at ${Label}: $($unknown -join ', ')" + } +} + +function Require-String($Value, [string]$Label) { + if ($null -eq $Value -or $Value -isnot [string] -or $Value.Length -eq 0) { + throw "$Label must be a non-empty string" + } + return [string]$Value +} + +function Require-Integer($Value, [string]$Label, [long]$Minimum = [long]::MinValue) { + if ($Value -isnot [byte] -and $Value -isnot [int16] -and $Value -isnot [int32] -and + $Value -isnot [int64] -and $Value -isnot [uint16] -and $Value -isnot [uint32]) { + throw "$Label must be an integer" + } + $integer = [long]$Value + if ($integer -lt $Minimum) { + throw "$Label must be at least $Minimum" + } + return $integer +} + +function Require-Array($Value, [string]$Label) { + # Emit elements to the output stream (no unary-comma wrapper). Callers wrap + # with @() so a single JSON object is one element and multi-element arrays + # expand correctly. Unary-comma would nest arrays and break [0] indexing. + if ($null -eq $Value) { + return @() + } + if ($Value -is [string]) { + return @($Value) + } + if ($Value -is [System.Array]) { + return @($Value) + } + if ($Value -is [System.Collections.IEnumerable]) { + return @($Value) + } + return @($Value) +} + +function Normalize-Hex($Value, [string]$Label) { + $text = Require-String $Value $Label + if ($text.Length -ne 64 -or $text -notmatch '^[0-9a-fA-F]{64}$') { + throw "$Label must be exactly 64 hexadecimal characters" + } + return $text.ToLowerInvariant() +} + +function Normalize-RelativePath($Value, [string]$Label) { + $path = Require-String $Value $Label + if ($path.Contains('\') -or $path.StartsWith('/') -or $path.EndsWith('/') -or + $path -match '^[A-Za-z]:' -or $path.Contains('//')) { + throw "$Label must be a normalized relative path" + } + foreach ($segment in $path.Split('/')) { + if ($segment.Length -eq 0 -or $segment -eq '.' -or $segment -eq '..') { + throw "$Label must be a normalized relative path" + } + } + return $path +} + +function Is-ExecutablePayload([string]$Path) { + return $ExecutableNamePatterns -contains $Path +} + +function Normalize-Payloads($Value, [string]$Label) { + $items = @(Require-Array $Value $Label) + if ($items.Count -eq 0) { + throw "$Label must contain at least one payload" + } + $normalized = New-Object System.Collections.Generic.List[object] + $seen = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + for ($index = 0; $index -lt $items.Count; $index++) { + $itemLabel = "$Label[$index]" + Assert-ExactFields $items[$index] $PayloadFields $itemLabel + $path = Normalize-RelativePath $items[$index].path "$itemLabel.path" + if (-not $seen.Add($path)) { + throw "duplicate payload path: $path" + } + $normalized.Add([pscustomobject][ordered]@{ + path = $path + sha256 = Normalize-Hex $items[$index].sha256 "$itemLabel.sha256" + size = Require-Integer $items[$index].size "$itemLabel.size" 0 + }) | Out-Null + } + # Sort by path without collapsing the collection through pipeline unwrapping. + return @($normalized | Sort-Object -Property path) +} + +function Normalize-DependencyList($Value, [string]$Label) { + $items = @(Require-Array $Value $Label) + $normalized = New-Object System.Collections.Generic.List[object] + for ($i = 0; $i -lt $items.Count; $i++) { + $item = $items[$i] + if ($null -eq $item -or $null -eq $item.PSObject) { + throw "$Label[$i] must be an object" + } + $name = Require-String $item.name "$Label[$i].name" + $entry = [ordered]@{ name = $name } + foreach ($prop in @($item.PSObject.Properties.Name | Sort-Object)) { + if ($prop -eq "name") { continue } + $entry[$prop] = $item.$prop + } + $normalized.Add([pscustomobject]$entry) | Out-Null + } + # Sort by name so dual-build ordering cannot mask identity changes. + return @($normalized | Sort-Object -Property name) +} + +function Normalize-Manifest($Manifest, [string]$Label) { + Assert-ManifestFields $Manifest $Label + $schemaVersion = Require-Integer $Manifest.schema_version "$Label.schema_version" 0 + if ($schemaVersion -ne 1) { + throw "$Label.schema_version must be 1" + } + # Always re-wrap with @(): a single JSON string (collapsed one-element array) + # must not become a character-enumerable scalar. + $protocolItems = @(Require-Array $Manifest.protocols "$Label.protocols") + $protocols = New-Object System.Collections.Generic.List[string] + for ($i = 0; $i -lt $protocolItems.Count; $i++) { + $protocols.Add((Require-String $protocolItems[$i] "$Label.protocols[$i]")) | Out-Null + } + $noteItems = @(Require-Array $Manifest.notes "$Label.notes") + $notes = New-Object System.Collections.Generic.List[string] + for ($i = 0; $i -lt $noteItems.Count; $i++) { + $notes.Add((Require-String $noteItems[$i] "$Label.notes[$i]")) | Out-Null + } + $normalized = [ordered]@{ + schema_version = $schemaVersion + package = Require-String $Manifest.package "$Label.package" + version = Require-String $Manifest.version "$Label.version" + built_with_sgc = Require-String $Manifest.built_with_sgc "$Label.built_with_sgc" + source_tree = Require-String $Manifest.source_tree "$Label.source_tree" + source_revision = Require-String $Manifest.source_revision "$Label.source_revision" + target = $Manifest.target + protocols = @($protocols) + runtime_dependencies = @(Normalize-DependencyList $Manifest.runtime_dependencies "$Label.runtime_dependencies") + build_tools = @(Normalize-DependencyList $Manifest.build_tools "$Label.build_tools") + license = $Manifest.license + provenance = $Manifest.provenance + payloads = @(Normalize-Payloads $Manifest.payloads "$Label.payloads") + notes = @($notes) + } + if ($null -eq $normalized.target -or $null -eq $normalized.target.PSObject) { + throw "$Label.target must be an object" + } + if ($null -eq $normalized.license -or $null -eq $normalized.license.PSObject) { + throw "$Label.license must be an object" + } + if ($null -eq $normalized.provenance -or $null -eq $normalized.provenance.PSObject) { + throw "$Label.provenance must be an object" + } + foreach ($optional in $ManifestOptionalFields) { + if ($null -ne $Manifest.PSObject.Properties[$optional]) { + $normalized[$optional] = $Manifest.$optional + } + } + return $normalized +} + +function Write-Utf8NoBom([string]$Path, [string]$Text) { + $utf8 = [Text.UTF8Encoding]::new($false) + [IO.File]::WriteAllText($Path, $Text, $utf8) +} + +function Canonical-Json($Object) { + return ($Object | ConvertTo-Json -Depth 12 -Compress) +} + +function Compare-MetaField($Left, $Right, [string]$Field, [System.Collections.ArrayList]$Mismatches) { + $leftHas = $Left.Contains($Field) + $rightHas = $Right.Contains($Field) + if (-not $leftHas -and -not $rightHas) { return } + $leftVal = if ($leftHas) { Canonical-Json $Left[$Field] } else { "" } + $rightVal = if ($rightHas) { Canonical-Json $Right[$Field] } else { "" } + if ($leftVal -ne $rightVal) { + [void]$Mismatches.Add([ordered]@{ + field = $Field + left = if ($leftHas) { $Left[$Field] } else { $null } + right = if ($rightHas) { $Right[$Field] } else { $null } + }) + } +} + +$left = Normalize-Manifest (Read-JsonObject $LeftManifest "left") "left" +$right = Normalize-Manifest (Read-JsonObject $RightManifest "right") "right" + +# Scalar/object/array meta fields that must match for pin-grade dual builds. +$metaFields = @( + "schema_version", "package", "version", "built_with_sgc", "source_tree", + "source_revision", "build_manifest_id", "target", + "runtime_dependencies", "build_tools", "license", "provenance" +) +$metaMismatches = New-Object System.Collections.ArrayList +foreach ($field in $metaFields) { + Compare-MetaField $left $right $field $metaMismatches +} + +$leftProtocols = ($left.protocols -join "`n") +$rightProtocols = ($right.protocols -join "`n") +if ($leftProtocols -ne $rightProtocols) { + [void]$metaMismatches.Add([ordered]@{ + field = "protocols" + left = $left.protocols + right = $right.protocols + }) +} + +$leftByPath = @{} +foreach ($payload in @($left.payloads)) { + if ($null -eq $payload -or -not $payload.PSObject.Properties['path']) { + throw "left payload entry missing path property" + } + $leftByPath[[string]$payload.path] = $payload +} +$rightByPath = @{} +foreach ($payload in @($right.payloads)) { + if ($null -eq $payload -or -not $payload.PSObject.Properties['path']) { + throw "right payload entry missing path property" + } + $rightByPath[[string]$payload.path] = $payload +} + +$pathSet = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) +foreach ($key in @($leftByPath.Keys)) { [void]$pathSet.Add([string]$key) } +foreach ($key in @($rightByPath.Keys)) { [void]$pathSet.Add([string]$key) } +$allPaths = @($pathSet | Sort-Object) +$payloadMismatches = @() +$allowedExecutableDrift = @() +$executableHashDivergences = @() +$identicalPayloads = 0 + +foreach ($path in $allPaths) { + $lp = $leftByPath[$path] + $rp = $rightByPath[$path] + if ($null -eq $lp -or $null -eq $rp) { + $payloadMismatches += [ordered]@{ + path = $path + reason = "missing_on_one_side" + left = $lp + right = $rp + } + continue + } + $hashEqual = $lp.sha256 -eq $rp.sha256 + $sizeEqual = $lp.size -eq $rp.size + if ($hashEqual -and $sizeEqual) { + $identicalPayloads++ + continue + } + $isExe = Is-ExecutablePayload $path + # Design requires payload hashes match across dual builds (including + # executables). AllowExecutableHashDrift is an explicit opt-in only; default + # fails closed so 8.7 cannot report green on PE/ELF hash divergence. + if ($AllowExecutableHashDrift -and $isExe) { + $allowedExecutableDrift += [ordered]@{ + path = $path + left_sha256 = $lp.sha256 + right_sha256 = $rp.sha256 + left_size = $lp.size + right_size = $rp.size + } + if ($isExe -and -not $hashEqual) { + $executableHashDivergences += [ordered]@{ + path = $path + left_sha256 = $lp.sha256 + right_sha256 = $rp.sha256 + left_size = $lp.size + right_size = $rp.size + policy = "allow_executable_hash_drift_opt_in" + } + } + continue + } + $payloadMismatches += [ordered]@{ + path = $path + reason = if ($isExe -and -not $hashEqual -and $sizeEqual) { + "executable_hash_mismatch" + } elseif ($isExe -and -not $sizeEqual) { + "executable_size_mismatch" + } else { + "hash_or_size_mismatch" + } + left_sha256 = $lp.sha256 + right_sha256 = $rp.sha256 + left_size = $lp.size + right_size = $rp.size + } +} + +$ok = ($metaMismatches.Count -eq 0) -and ($payloadMismatches.Count -eq 0) +$comparison = [ordered]@{ + schema_version = 1 + ok = $ok + allow_executable_hash_drift = [bool]$AllowExecutableHashDrift + pin_grade_policy = "payload_hashes_and_dependency_identities_must_match_unless_AllowExecutableHashDrift" + identical_payload_count = $identicalPayloads + allowed_executable_drift_count = $allowedExecutableDrift.Count + executable_hash_divergence_count = $executableHashDivergences.Count + meta_mismatch_count = $metaMismatches.Count + payload_mismatch_count = $payloadMismatches.Count + meta_mismatches = @($metaMismatches) + allowed_executable_drift = @($allowedExecutableDrift) + executable_hash_divergences = @($executableHashDivergences) + payload_mismatches = @($payloadMismatches) +} + +$OutputDir = if ([IO.Path]::IsPathRooted($OutputDir)) { + [IO.Path]::GetFullPath($OutputDir) +} else { + [IO.Path]::GetFullPath((Join-Path (Get-Location) $OutputDir)) +} +New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null +Write-Utf8NoBom (Join-Path $OutputDir "normalized-a.json") (Canonical-Json $left) +Write-Utf8NoBom (Join-Path $OutputDir "normalized-b.json") (Canonical-Json $right) +Write-Utf8NoBom (Join-Path $OutputDir "comparison.json") (($comparison | ConvertTo-Json -Depth 12)) + +if (-not $ok) { + Write-Error "senline package manifest comparison failed; see $OutputDir\comparison.json" + exit 1 +} + +Write-Host "senline package manifests match (payloads=$identicalPayloads executable_drift=$($allowedExecutableDrift.Count))" +Write-Host "comparison: $(Join-Path $OutputDir 'comparison.json')" diff --git a/scripts/install.ps1 b/scripts/install.ps1 index 4b27bd4c..989ed940 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -114,6 +114,51 @@ try { throw "archive does not contain a Sengoo manifest.json" } + $manifestPath = Join-Path $payload.FullName "manifest.json" + $manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json + if ($manifest.schema_version -ne 2) { + throw "unsupported Sengoo manifest schema: $($manifest.schema_version)" + } + $payloadChecksums = Join-Path $payload.FullName "payloads.sha256" + if (-not (Test-Path -LiteralPath $payloadChecksums)) { + throw "archive does not contain payloads.sha256" + } + $payloadRoot = [System.IO.Path]::GetFullPath($payload.FullName).TrimEnd([char[]]@('\', '/')) + $verifiedPayloads = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + foreach ($line in Get-Content -LiteralPath $payloadChecksums) { + if ($line -notmatch '^([0-9a-fA-F]{64}) (.+)$') { + throw "invalid payload checksum entry: $line" + } + $expectedPayloadHash = $Matches[1].ToLowerInvariant() + $relativePayloadPath = $Matches[2].Replace('/', [System.IO.Path]::DirectorySeparatorChar) + $payloadPath = [System.IO.Path]::GetFullPath((Join-Path $payloadRoot $relativePayloadPath)) + if (-not $payloadPath.StartsWith($payloadRoot + [System.IO.Path]::DirectorySeparatorChar, [System.StringComparison]::OrdinalIgnoreCase)) { + throw "payload checksum path escapes archive root: $relativePayloadPath" + } + if (-not (Test-Path -LiteralPath $payloadPath -PathType Leaf)) { + throw "payload file is missing: $relativePayloadPath" + } + $actualPayloadHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $payloadPath).Hash.ToLowerInvariant() + if ($expectedPayloadHash -ne $actualPayloadHash) { + throw "payload checksum mismatch for $relativePayloadPath" + } + $null = $verifiedPayloads.Add($payloadPath) + } + $unlistedPayloads = @( + Get-ChildItem -LiteralPath $payloadRoot -Recurse -File | Where-Object { + $_.FullName -ne $manifestPath -and + $_.FullName -ne $payloadChecksums -and + -not $verifiedPayloads.Contains([System.IO.Path]::GetFullPath($_.FullName)) + } + ) + if ($unlistedPayloads.Count -ne 0) { + throw "archive contains payload files missing from payloads.sha256: $($unlistedPayloads.FullName -join ', ')" + } + $actualBuildManifestId = (Get-FileHash -Algorithm SHA256 -LiteralPath $payloadChecksums).Hash.ToLowerInvariant() + if ($manifest.build_manifest_id -ne $actualBuildManifestId) { + throw "payload checksum manifest identity mismatch" + } + Remove-Item -LiteralPath $InstallDir -Recurse -Force -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force -Path $InstallDir | Out-Null Copy-Item -Path (Join-Path $payload.FullName "*") -Destination $InstallDir -Recurse -Force diff --git a/scripts/install.sh b/scripts/install.sh index 121ff1aa..b416ef42 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -186,6 +186,34 @@ if [ -z "$payload" ]; then exit 1 fi payload_dir=$(dirname "$payload") +payload_checksums="$payload_dir/payloads.sha256" +if [ ! -f "$payload_checksums" ]; then + echo "archive does not contain payloads.sha256" >&2 + exit 1 +fi +while IFS= read -r checksum_line; do + payload_path=${checksum_line#* } + case "$payload_path" in + ""|/*|../*|*/../*|*/..) + echo "invalid payload checksum path: $payload_path" >&2 + exit 1 + ;; + esac +done < "$payload_checksums" +if command -v sha256sum >/dev/null 2>&1; then + (cd "$payload_dir" && sha256sum -c payloads.sha256) +elif command -v shasum >/dev/null 2>&1; then + (cd "$payload_dir" && shasum -a 256 -c payloads.sha256) +else + echo "sha256sum or shasum is required for payload verification" >&2 + exit 1 +fi +(cd "$payload_dir" && find . -type f ! -name manifest.json ! -name payloads.sha256 -print | sed 's#^\./##' | LC_ALL=C sort) > "$tmp_dir/actual-payloads.txt" +sed 's/^[0-9a-fA-F]\{64\} //' "$payload_checksums" | LC_ALL=C sort > "$tmp_dir/listed-payloads.txt" +if ! cmp -s "$tmp_dir/actual-payloads.txt" "$tmp_dir/listed-payloads.txt"; then + echo "archive payload set does not match payloads.sha256" >&2 + exit 1 +fi rm -rf "$install_dir" mkdir -p "$install_dir" diff --git a/scripts/normalize-pin-executable.ps1 b/scripts/normalize-pin-executable.ps1 new file mode 100644 index 00000000..50d38497 --- /dev/null +++ b/scripts/normalize-pin-executable.ps1 @@ -0,0 +1,111 @@ +# Normalize residual non-content identity in pin-grade package executables so +# independent dual builds can share bit-identical payload hashes when the +# functional content matches. Used by package-senline-worker/http.ps1. +# +# - PE (Windows): zero COFF TimeDateStamp and optional-header CheckSum. +# - ELF (Linux): strip .note.gnu.build-id and .comment when objcopy is available. +# +# Returns a hashtable describing tools used (path, version string, exit codes) +# so package provenance can record pin-normalization inputs. + +function Get-ToolVersion([string]$ToolPath) { + if (-not $ToolPath) { return $null } + try { + $out = & $ToolPath --version 2>&1 | Out-String + return ($out -replace '\s+', ' ').Trim() + } catch { + return $null + } +} + +function Normalize-PinExecutable { + param( + [Parameter(Mandatory = $true)] + [string]$Path + ) + if (-not (Test-Path -LiteralPath $Path)) { + throw "Normalize-PinExecutable: missing file $Path" + } + $provenance = [ordered]@{ + path = $Path + pe_timestamp_zeroed = $false + elf_objcopy = $null + elf_strip = $null + } + $bytes = [IO.File]::ReadAllBytes($Path) + if ($bytes.Length -lt 64) { + return $provenance + } + + # PE: MZ ... PE\0\0 + if ($bytes[0] -eq 0x4D -and $bytes[1] -eq 0x5A) { + $peOff = [BitConverter]::ToInt32($bytes, 0x3C) + if ($peOff -le 0 -or ($peOff + 24) -ge $bytes.Length) { + return $provenance + } + if ($bytes[$peOff] -ne 0x50 -or $bytes[$peOff + 1] -ne 0x45) { + return $provenance + } + # COFF TimeDateStamp at PE+8 + $bytes[$peOff + 8] = 0 + $bytes[$peOff + 9] = 0 + $bytes[$peOff + 10] = 0 + $bytes[$peOff + 11] = 0 + # Optional header starts at PE+24; CheckSum is at optional+64 for PE32/PE32+ + $optOff = $peOff + 24 + if (($optOff + 68) -lt $bytes.Length) { + $magic = [BitConverter]::ToUInt16($bytes, $optOff) + if ($magic -eq 0x10B -or $magic -eq 0x20B) { + $checkOff = $optOff + 64 + $bytes[$checkOff] = 0 + $bytes[$checkOff + 1] = 0 + $bytes[$checkOff + 2] = 0 + $bytes[$checkOff + 3] = 0 + } + } + [IO.File]::WriteAllBytes($Path, $bytes) + $provenance.pe_timestamp_zeroed = $true + return $provenance + } + + # ELF: \x7fELF — strip non-content note/comment sections and optional + # full symbol strip so dual independent release builds hash-match. + if ($bytes[0] -eq 0x7F -and $bytes[1] -eq 0x45 -and $bytes[2] -eq 0x4C -and $bytes[3] -eq 0x46) { + $objcopy = $null + foreach ($name in @("llvm-objcopy", "llvm-objcopy-19", "llvm-objcopy-18", "objcopy")) { + $cmd = Get-Command $name -ErrorAction SilentlyContinue + if ($cmd) { $objcopy = $cmd.Source; break } + } + if ($objcopy) { + & $objcopy --remove-section=.note.gnu.build-id --remove-section=.comment $Path 2>$null + $code = $LASTEXITCODE + # Missing sections may yield non-zero; only fail hard on tool not found (already resolved). + $provenance.elf_objcopy = [ordered]@{ + path = $objcopy + version = Get-ToolVersion $objcopy + exit_code = $code + } + if ($code -gt 1) { + throw "Normalize-PinExecutable: objcopy failed exit=$code path=$objcopy" + } + } + $strip = $null + foreach ($name in @("llvm-strip", "llvm-strip-19", "llvm-strip-18", "strip")) { + $cmd = Get-Command $name -ErrorAction SilentlyContinue + if ($cmd) { $strip = $cmd.Source; break } + } + if ($strip) { + & $strip -s $Path 2>$null + $code = $LASTEXITCODE + $provenance.elf_strip = [ordered]@{ + path = $strip + version = Get-ToolVersion $strip + exit_code = $code + } + if ($code -ne 0) { + throw "Normalize-PinExecutable: strip failed exit=$code path=$strip" + } + } + } + return $provenance +} diff --git a/scripts/package-senline-http.ps1 b/scripts/package-senline-http.ps1 new file mode 100644 index 00000000..7ef49f3c --- /dev/null +++ b/scripts/package-senline-http.ps1 @@ -0,0 +1,172 @@ +param( + [Parameter(Mandatory = $true)] + [string]$SgcPath, + [Parameter(Mandatory = $true)] + [string]$OutputDir, + [string]$SgpmPath = "", + [string]$HttpRoot = "", + [string]$Version = "0.1.0-http-dogfood" +) + +$ErrorActionPreference = "Stop" + +if (-not $HttpRoot) { + $HttpRoot = Join-Path $PSScriptRoot "..\examples\realworld\senline-http-dogfood" +} +$HttpRoot = (Resolve-Path -LiteralPath $HttpRoot).Path +$SgcPath = (Resolve-Path -LiteralPath $SgcPath).Path +if (-not $SgpmPath) { + $SgpmPath = Join-Path (Split-Path -Parent $SgcPath) $(if ($env:OS -eq "Windows_NT" -or $IsWindows) { "sgpm.exe" } else { "sgpm" }) +} +$SgpmPath = (Resolve-Path -LiteralPath $SgpmPath).Path + +$OutputDir = if ([IO.Path]::IsPathRooted($OutputDir)) { + [IO.Path]::GetFullPath($OutputDir) +} else { + [IO.Path]::GetFullPath((Join-Path (Get-Location) $OutputDir)) +} +Remove-Item -LiteralPath $OutputDir -Recurse -Force -ErrorAction SilentlyContinue +New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null + +$binDir = Split-Path -Parent $SgcPath +$pathSep = [IO.Path]::PathSeparator +$env:PATH = "$binDir$pathSep" + $env:PATH +$env:SGPM_SGC = $SgcPath +$sgfmtCandidate = Join-Path $binDir $(if ($env:OS -eq "Windows_NT" -or $IsWindows) { "sgfmt.exe" } else { "sgfmt" }) +if (Test-Path -LiteralPath $sgfmtCandidate) { + $env:SGPM_SGFMT = $sgfmtCandidate +} + +$repoRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..")).Path +$sourceRevision = (git -C $repoRoot rev-parse HEAD).Trim().ToLowerInvariant() +if ($sourceRevision -cnotmatch '^[0-9a-f]{40}$') { + throw "git rev-parse HEAD did not return a 40-char lowercase revision" +} + +Push-Location $HttpRoot +try { + $env:SENGOO_DETERMINISTIC_LINK = "1" + $env:SENGOO_INCREMENTAL_LINK = "off" + & $SgpmPath --runtime-mode installed build --locked --release + if ($LASTEXITCODE -ne 0) { + throw "sgpm --runtime-mode installed build --locked --release failed for senline-http-dogfood" + } +} finally { + Pop-Location +} + +# Avoid $isWindows: PowerShell is case-insensitive and $IsWindows is read-only. +$hostIsWindows = ($env:OS -eq "Windows_NT") -or ((Get-Variable -Name IsWindows -ErrorAction SilentlyContinue) -and $IsWindows) +$exeName = if ($hostIsWindows) { "senline_http_dogfood.exe" } else { "senline_http_dogfood" } +$built = Join-Path $HttpRoot "target\release\$exeName" +if (-not (Test-Path -LiteralPath $built)) { + # Linux/mac path separator fallback + $builtUnix = Join-Path $HttpRoot "target/release/$exeName" + if (Test-Path -LiteralPath $builtUnix) { + $built = $builtUnix + } else { + throw "missing built HTTP dogfood executable: $built" + } +} +$packagedExe = Join-Path $OutputDir $exeName +Copy-Item -LiteralPath $built -Destination $packagedExe -Force +. (Join-Path $PSScriptRoot "normalize-pin-executable.ps1") +$script:NormalizePinProvenance = Normalize-PinExecutable -Path $packagedExe +Copy-Item -LiteralPath (Join-Path $HttpRoot "README.md") -Destination (Join-Path $OutputDir "README.md") -Force +Copy-Item -LiteralPath (Join-Path $HttpRoot "Sengoo.toml") -Destination (Join-Path $OutputDir "Sengoo.toml") -Force +Copy-Item -LiteralPath (Join-Path $HttpRoot "Sengoo.lock") -Destination (Join-Path $OutputDir "Sengoo.lock") -Force + +function Get-Sha256([string]$Path) { + return (Get-FileHash -Algorithm SHA256 -LiteralPath $Path).Hash.ToLowerInvariant() +} + +$sgcVersion = (& $SgcPath --version).Trim() +$arch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant() +$osName = if ($hostIsWindows) { "windows" } else { "linux" } +$abi = if ($hostIsWindows) { "msvc" } else { "gnu" } +$utf8 = [Text.UTF8Encoding]::new($false) + +# Ship license + SBOM input files before hashing package payloads. +$licenseText = @( + "Senline HTTP dogfood package ($Version)", + "Source revision: $sourceRevision", + "License: UNLICENSED pending repository SPDX publication.", + "See Sengoo.toml package metadata and the parent Sengoo distribution license." +) -join "`n" +[IO.File]::WriteAllText((Join-Path $OutputDir "LICENSE.txt"), $licenseText + "`n", $utf8) + +function Get-PackagePayloads([string]$Root, [string[]]$ExcludeNames) { + $items = New-Object System.Collections.Generic.List[object] + Get-ChildItem -LiteralPath $Root -Recurse -File | ForEach-Object { + $rel = $_.FullName.Substring($Root.Length).TrimStart('\', '/').Replace('\', '/') + if ($ExcludeNames -contains $rel) { return } + $items.Add([ordered]@{ + path = $rel + sha256 = Get-Sha256 $_.FullName + size = $_.Length + }) | Out-Null + } + return @($items | Sort-Object -Property path) +} + +$componentPayloads = Get-PackagePayloads -Root $OutputDir -ExcludeNames @("sbom-inputs.json", "http-manifest.json") +$sbom = [ordered]@{ + schema_version = 1 + package = "senline-http-dogfood" + version = $Version + source_revision = $sourceRevision + components = @($componentPayloads) +} +[IO.File]::WriteAllText((Join-Path $OutputDir "sbom-inputs.json"), (($sbom | ConvertTo-Json -Depth 6) + "`n"), $utf8) + +$payloads = Get-PackagePayloads -Root $OutputDir -ExcludeNames @("http-manifest.json") +$manifest = [ordered]@{ + schema_version = 1 + package = "senline-http-dogfood" + version = $Version + built_with_sgc = $sgcVersion + source_revision = $sourceRevision + source_tree = "examples/realworld/senline-http-dogfood" + target = [ordered]@{ + os = $osName + arch = $arch + abi = $abi + triple = if ($hostIsWindows) { "x86_64-pc-windows-msvc" } else { "x86_64-unknown-linux-gnu" } + } + protocols = @("senline-worker-v1", "http-loopback-dogfood-v1") + # sengoo_runtime is linked into the executable by installed sgc at package + # time (static archive on current targets). It is a build/link input, not a + # separately shipped dynamic runtime dependency of this package. + # Planner logic is linked from the senline_domain_worker library package + # (direct library call), not by spawning a framed worker process. + runtime_dependencies = @() + build_tools = @( + [ordered]@{ name = "sgc"; version = $sgcVersion; role = "installed-toolchain-build" } + [ordered]@{ name = "sgpm"; role = "installed-package-manager" } + [ordered]@{ name = "sengoo_runtime"; role = "installed-native-runtime-link-input"; note = "Statically linked via installed sgc; not a separate payload" } + [ordered]@{ name = "senline_domain_worker"; role = "linked-library-package"; note = "HTTP dogfood calls decoder/planner/encoders as a library; does not spawn a worker process" } + ) + license = [ordered]@{ + spdx_expression = "UNLICENSED" + file = "LICENSE.txt" + note = "Package ships LICENSE.txt; repository root may not yet publish a SPDX license." + } + provenance = [ordered]@{ + built_with_installed_toolchain_only = $true + cargo_forbidden_at_package_time = $true + sbom_inputs = "sbom-inputs.json" + sengoo_deterministic_link = $true + normalize_pin_executable = $(if ($script:NormalizePinProvenance) { $script:NormalizePinProvenance } else { $null }) + } + payloads = $payloads + notes = @( + "Built with installed toolchain binaries only (sgpm + sgc).", + "Loopback-only synthetic harness; not TLS, ingress, or client routing.", + "No Sengoo compiler checkout path is required at runtime." + ) +} +$manifestPath = Join-Path $OutputDir "http-manifest.json" +[IO.File]::WriteAllText($manifestPath, ($manifest | ConvertTo-Json -Depth 6), $utf8) +Write-Host "HTTP dogfood package written to $OutputDir" +Write-Host " executable: $(Join-Path $OutputDir $exeName)" +Write-Host " manifest: $manifestPath" diff --git a/scripts/package-senline-worker.ps1 b/scripts/package-senline-worker.ps1 new file mode 100644 index 00000000..96082a03 --- /dev/null +++ b/scripts/package-senline-worker.ps1 @@ -0,0 +1,220 @@ +param( + [Parameter(Mandatory = $true)] + [string]$SgcPath, + [Parameter(Mandatory = $true)] + [string]$OutputDir, + [string]$SgpmPath = "", + [string]$WorkerRoot = "", + [string]$Version = "0.1.0-worker" +) + +$ErrorActionPreference = "Stop" + +if (-not $WorkerRoot) { + $WorkerRoot = Join-Path $PSScriptRoot "..\examples\realworld\senline-domain-worker" +} +$WorkerRoot = (Resolve-Path -LiteralPath $WorkerRoot).Path +$SgcPath = (Resolve-Path -LiteralPath $SgcPath).Path +if (-not $SgpmPath) { + $SgpmPath = Join-Path (Split-Path -Parent $SgcPath) $(if ($env:OS -eq "Windows_NT" -or $IsWindows) { "sgpm.exe" } else { "sgpm" }) +} +$SgpmPath = (Resolve-Path -LiteralPath $SgpmPath).Path + +$OutputDir = if ([IO.Path]::IsPathRooted($OutputDir)) { + [IO.Path]::GetFullPath($OutputDir) +} else { + [IO.Path]::GetFullPath((Join-Path (Get-Location) $OutputDir)) +} +Remove-Item -LiteralPath $OutputDir -Recurse -Force -ErrorAction SilentlyContinue +New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null + +$binDir = Split-Path -Parent $SgcPath +$pathSep = [IO.Path]::PathSeparator +$env:PATH = "$binDir$pathSep" + $env:PATH +$env:SGPM_SGC = $SgcPath +$sgfmtCandidate = Join-Path $binDir $(if ($env:OS -eq "Windows_NT" -or $IsWindows) { "sgfmt.exe" } else { "sgfmt" }) +if (Test-Path -LiteralPath $sgfmtCandidate) { + $env:SGPM_SGFMT = $sgfmtCandidate +} + +# Release packaging must regenerate build identity (not ship fixture-mode all-1s). +$repoRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..")).Path +$sourceRevision = (git -C $repoRoot rev-parse HEAD).Trim().ToLowerInvariant() +if ($sourceRevision -cnotmatch '^[0-9a-f]{40}$') { + throw "git rev-parse HEAD did not return a 40-char lowercase revision" +} +$sgcVersionRaw = (& $SgcPath --version).Trim() +# Keep portable identifier characters only for embedded identity fields. +$toolchainVersion = ($sgcVersionRaw -replace '[^0-9A-Za-z.+-]', '') +if (-not $toolchainVersion) { $toolchainVersion = "0.0.0" } +if ($toolchainVersion.Length -gt 64) { $toolchainVersion = $toolchainVersion.Substring(0, 64) } +$applicationVersion = $Version +$identitySeed = "{0}|{1}|{2}|senline-domain-worker" -f $sourceRevision, $toolchainVersion, $applicationVersion +$identityBytes = [Text.Encoding]::UTF8.GetBytes($identitySeed) +$sha = [Security.Cryptography.SHA256]::Create() +$buildManifestId = ($sha.ComputeHash($identityBytes) | ForEach-Object { $_.ToString("x2") }) -join "" +$generateIdentity = Join-Path $WorkerRoot "scripts\generate-build-identity.ps1" +$identityOut = Join-Path $WorkerRoot "packages\senline-build-identity\src\lib.sg" +$handshakeOut = Join-Path $WorkerRoot "fixtures\v1\handshake\ready.json" +# Preserve fixture-mode sources; packaging rewrites them for the release binary +# then restores so the worktree is not left with a non-fixture identity. +$identityBackup = [IO.Path]::GetTempFileName() +$handshakeBackup = [IO.Path]::GetTempFileName() +Copy-Item -LiteralPath $identityOut -Destination $identityBackup -Force +Copy-Item -LiteralPath $handshakeOut -Destination $handshakeBackup -Force +try { + & $generateIdentity ` + -SourceRevision $sourceRevision ` + -ToolchainVersion $toolchainVersion ` + -ApplicationVersion $applicationVersion ` + -BuildManifestId $buildManifestId ` + -OutputPath $identityOut ` + -HandshakeOutputPath $handshakeOut + if ($LASTEXITCODE -ne 0) { + throw "generate-build-identity.ps1 failed" + } + + Push-Location $WorkerRoot + try { + # sgpm resolves locked path deps and module maps; use installed tools only. + # Force installed runtime mode so packaging never falls back to checkout + # source-development runtime or cargo. + # Pin-grade dual builds require deterministic link metadata (/Brepro, no build-id). + $env:SENGOO_DETERMINISTIC_LINK = "1" + $env:SENGOO_INCREMENTAL_LINK = "off" + & $SgpmPath --runtime-mode installed build --locked --release + if ($LASTEXITCODE -ne 0) { + throw "sgpm --runtime-mode installed build --locked --release failed for senline-domain-worker" + } + } finally { + Pop-Location + } + + # Copy the release binary and *regenerated* fixtures into the package BEFORE + # restoring fixture-mode identity sources. Otherwise package handshake JSON + # would be the all-1s fixture while the binary embeds the release identity. + $hostIsWindowsInner = ($env:OS -eq "Windows_NT") -or ((Get-Variable -Name IsWindows -ErrorAction SilentlyContinue) -and $IsWindows) + $exeNameInner = if ($hostIsWindowsInner) { "senline_domain_worker.exe" } else { "senline_domain_worker" } + $builtInner = Join-Path $WorkerRoot (Join-Path "target" (Join-Path "release" $exeNameInner)) + if (-not (Test-Path -LiteralPath $builtInner)) { + throw "missing built worker executable: $builtInner" + } + $packagedExe = Join-Path $OutputDir $exeNameInner + Copy-Item -LiteralPath $builtInner -Destination $packagedExe -Force + # Normalize residual non-content identity (PE timestamp / ELF build-id) so dual + # packages can share bit-identical payload hashes after independent rebuilds. + . (Join-Path $PSScriptRoot "normalize-pin-executable.ps1") + $script:NormalizePinProvenance = Normalize-PinExecutable -Path $packagedExe + Copy-Item -LiteralPath (Join-Path $WorkerRoot "fixtures") -Destination (Join-Path $OutputDir "fixtures") -Recurse -Force + Copy-Item -LiteralPath (Join-Path $WorkerRoot "README.md") -Destination (Join-Path $OutputDir "README.md") -Force + Copy-Item -LiteralPath (Join-Path $WorkerRoot "Sengoo.toml") -Destination (Join-Path $OutputDir "Sengoo.toml") -Force + Copy-Item -LiteralPath (Join-Path $WorkerRoot "Sengoo.lock") -Destination (Join-Path $OutputDir "Sengoo.lock") -Force +} finally { + Copy-Item -LiteralPath $identityBackup -Destination $identityOut -Force + Copy-Item -LiteralPath $handshakeBackup -Destination $handshakeOut -Force + Remove-Item -LiteralPath $identityBackup, $handshakeBackup -Force -ErrorAction SilentlyContinue +} + +# Avoid $isWindows: PowerShell is case-insensitive and $IsWindows is read-only. +$hostIsWindows = ($env:OS -eq "Windows_NT") -or ((Get-Variable -Name IsWindows -ErrorAction SilentlyContinue) -and $IsWindows) +$exeName = if ($hostIsWindows) { "senline_domain_worker.exe" } else { "senline_domain_worker" } +if (-not (Test-Path -LiteralPath (Join-Path $OutputDir $exeName))) { + throw "package missing worker executable after identity-aware copy: $(Join-Path $OutputDir $exeName)" +} + +function Get-Sha256([string]$Path) { + return (Get-FileHash -Algorithm SHA256 -LiteralPath $Path).Hash.ToLowerInvariant() +} + +$sgcVersion = $sgcVersionRaw +$arch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant() +$osName = if ($hostIsWindows) { "windows" } else { "linux" } +$abi = if ($hostIsWindows) { "msvc" } else { "gnu" } +$utf8 = [Text.UTF8Encoding]::new($false) + +# Ship license + SBOM input files before hashing package payloads. +$licenseText = @( + "Senline domain worker package ($Version)", + "Source revision: $sourceRevision", + "Build-manifest id: $buildManifestId", + "License: UNLICENSED pending repository SPDX publication.", + "See Sengoo.toml package metadata and the parent Sengoo distribution license." +) -join "`n" +[IO.File]::WriteAllText((Join-Path $OutputDir "LICENSE.txt"), $licenseText + "`n", $utf8) + +function Get-PackagePayloads([string]$Root, [string[]]$ExcludeNames) { + $items = New-Object System.Collections.Generic.List[object] + Get-ChildItem -LiteralPath $Root -Recurse -File | ForEach-Object { + $rel = $_.FullName.Substring($Root.Length).TrimStart('\', '/').Replace('\', '/') + if ($ExcludeNames -contains $rel) { return } + $items.Add([ordered]@{ + path = $rel + sha256 = Get-Sha256 $_.FullName + size = $_.Length + }) | Out-Null + } + return @($items | Sort-Object -Property path) +} + +$componentPayloads = Get-PackagePayloads -Root $OutputDir -ExcludeNames @("sbom-inputs.json", "worker-manifest.json") +$sbom = [ordered]@{ + schema_version = 1 + package = "senline-domain-worker" + version = $Version + source_revision = $sourceRevision + build_manifest_id = $buildManifestId + components = @($componentPayloads) +} +[IO.File]::WriteAllText((Join-Path $OutputDir "sbom-inputs.json"), (($sbom | ConvertTo-Json -Depth 6) + "`n"), $utf8) + +# Manifest payloads include every package file except the manifest itself. +$payloads = Get-PackagePayloads -Root $OutputDir -ExcludeNames @("worker-manifest.json") +$manifest = [ordered]@{ + schema_version = 1 + package = "senline-domain-worker" + version = $Version + built_with_sgc = $sgcVersion + source_revision = $sourceRevision + source_tree = "examples/realworld/senline-domain-worker" + build_manifest_id = $buildManifestId + target = [ordered]@{ + os = $osName + arch = $arch + abi = $abi + triple = if ($hostIsWindows) { "x86_64-pc-windows-msvc" } else { "x86_64-unknown-linux-gnu" } + } + protocols = @("senline-worker-v1") + # sengoo_runtime is linked by installed sgc at package time (static archive + # on current Windows/Linux targets). Recorded as a build/link input, not a + # separately shipped dynamic runtime payload of this package. + runtime_dependencies = @() + build_tools = @( + [ordered]@{ name = "sgc"; version = $sgcVersion; role = "installed-toolchain-build" } + [ordered]@{ name = "sgpm"; role = "installed-package-manager" } + [ordered]@{ name = "sengoo_runtime"; role = "installed-native-runtime-link-input"; note = "Statically linked via installed sgc; not a separate package payload" } + ) + license = [ordered]@{ + spdx_expression = "UNLICENSED" + file = "LICENSE.txt" + note = "Package ships LICENSE.txt; repository root may not yet publish a SPDX license." + } + provenance = [ordered]@{ + built_with_installed_toolchain_only = $true + generate_build_identity = $true + cargo_forbidden_at_package_time = $true + sbom_inputs = "sbom-inputs.json" + sengoo_deterministic_link = $true + normalize_pin_executable = $(if ($script:NormalizePinProvenance) { $script:NormalizePinProvenance } else { $null }) + } + payloads = $payloads + notes = @( + "Built with installed toolchain binaries only (sgpm + sgc).", + "Build identity regenerated from source revision before release build.", + "No Sengoo compiler checkout path is required at runtime." + ) +} +$manifestPath = Join-Path $OutputDir "worker-manifest.json" +[IO.File]::WriteAllText($manifestPath, ($manifest | ConvertTo-Json -Depth 6), $utf8) +Write-Host "Worker package written to $OutputDir" +Write-Host " executable: $(Join-Path $OutputDir $exeName)" +Write-Host " manifest: $manifestPath" diff --git a/scripts/package-toolchain.ps1 b/scripts/package-toolchain.ps1 index 2a170e03..cfb3103c 100644 --- a/scripts/package-toolchain.ps1 +++ b/scripts/package-toolchain.ps1 @@ -2,6 +2,7 @@ param( [string]$Version = "0.1.0-dev", [string]$OutputDir = "target/dist", [string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path, + [string]$CargoTargetDir = "", [switch]$NoBuild, [string]$SmokeEvidence = "" ) @@ -15,21 +16,7 @@ function Is-WindowsHost { return [bool]$IsWindows } -function Build-Hash { - if ($env:SENGOO_BUILD_HASH) { - return $env:SENGOO_BUILD_HASH - } - $hash = (& git -C $RepoRoot rev-parse --short=12 HEAD 2>$null) - if ($LASTEXITCODE -eq 0 -and $hash) { - return $hash.Trim() - } - return "unknown" -} - -function Target-Label { - if ($env:SENGOO_DIST_TARGET) { - return $env:SENGOO_DIST_TARGET - } +function Host-Target { if (Is-WindowsHost) { return "x86_64-pc-windows-msvc" } @@ -46,16 +33,111 @@ function Target-Label { return "x86_64-unknown-linux-gnu" } +function Git-Head { + $revision = (& git -C $RepoRoot rev-parse --verify HEAD 2>$null) + if ($LASTEXITCODE -ne 0 -or -not $revision) { + throw "repository HEAD is unavailable; distribution identity requires an immutable Git revision" + } + $revision = $revision.Trim().ToLowerInvariant() + if ($revision -notmatch '^[0-9a-f]{40}$') { + throw "repository HEAD must be a complete 40-character lowercase Git revision" + } + return $revision +} + +function Git-Status { + $status = @(& git -C $RepoRoot status --porcelain=v1 --untracked-files=all 2>$null) + if ($LASTEXITCODE -ne 0) { + throw "repository status is unavailable; distribution cleanliness cannot be established" + } + return $status +} + +$RepoRoot = (Resolve-Path -LiteralPath $RepoRoot).Path +$hostTarget = Host-Target +$target = if ($env:SENGOO_DIST_TARGET) { $env:SENGOO_DIST_TARGET } else { $hostTarget } +if ($target -ne $hostTarget) { + throw "distribution target $target does not match host target $hostTarget; cross-host packaging is unsupported" +} + +$sourceRevision = Git-Head +foreach ($override in @( + [pscustomobject]@{ name = "SENGOO_SOURCE_REVISION"; value = $env:SENGOO_SOURCE_REVISION }, + [pscustomobject]@{ name = "GITHUB_SHA"; value = $env:GITHUB_SHA } +)) { + if ($override.value -and $override.value.ToLowerInvariant() -ne $sourceRevision) { + throw "$($override.name) must equal repository HEAD $sourceRevision" + } +} +$buildHash = $sourceRevision.Substring(0, 12) +if ($env:SENGOO_BUILD_HASH -and + $env:SENGOO_BUILD_HASH.ToLowerInvariant() -ne $buildHash) { + throw "SENGOO_BUILD_HASH must equal repository HEAD prefix $buildHash" +} +$env:SENGOO_BUILD_HASH = $buildHash + +if (-not $CargoTargetDir) { + $CargoTargetDir = Join-Path $RepoRoot "target" +} elseif (-not [IO.Path]::IsPathRooted($CargoTargetDir)) { + $CargoTargetDir = Join-Path $RepoRoot $CargoTargetDir +} +$CargoTargetDir = [IO.Path]::GetFullPath($CargoTargetDir) +$manifestPath = Join-Path $RepoRoot "Cargo.toml" +$sourceStatusBefore = @(Git-Status) + +function Set-DeterministicCargoRustflags([string]$SourceRoot, [string]$TargetDir) { + # Unit-separator encoding required by CARGO_ENCODED_RUSTFLAGS. + $sep = [char]0x1f + $flags = [System.Collections.Generic.List[string]]::new() + $sourceRoot = [IO.Path]::GetFullPath($SourceRoot).TrimEnd([char[]]@('\', '/')) + $targetDir = [IO.Path]::GetFullPath($TargetDir).TrimEnd([char[]]@('\', '/')) + # Remap both source checkout and cargo target so two independent package + # builds do not bake different absolute paths into payloads. + $flags.Add("--remap-path-prefix=$sourceRoot=/sengoo-build/src") + $flags.Add("--remap-path-prefix=$targetDir=/sengoo-build/target") + if (Is-WindowsHost) { + # MSVC PE/COFF timestamps and incremental leftovers are otherwise + # non-deterministic across independent target directories. + $flags.Add("-C") + $flags.Add("debuginfo=0") + $flags.Add("-C") + $flags.Add("link-arg=/Brepro") + $flags.Add("-C") + $flags.Add("link-arg=/INCREMENTAL:NO") + $flags.Add("-C") + $flags.Add("link-arg=/DEBUG:NONE") + } + $env:CARGO_ENCODED_RUSTFLAGS = ($flags -join $sep) + Remove-Item Env:RUSTFLAGS -ErrorAction SilentlyContinue + Write-Host "deterministic cargo rustflags: source->$sourceRoot target->$targetDir windows=$(Is-WindowsHost)" +} + if (-not $NoBuild) { - & cargo build -p sgc -p sgpm -p sgfmt -p sglsp --release + Set-DeterministicCargoRustflags -SourceRoot $RepoRoot -TargetDir $CargoTargetDir + & cargo build --manifest-path $manifestPath --target-dir $CargoTargetDir --locked ` + -p sgc -p sgpm -p sgfmt -p sglsp --release if ($LASTEXITCODE -ne 0) { throw "cargo release build failed" } + & cargo build --manifest-path $manifestPath --target-dir $CargoTargetDir --locked ` + -p sengoo-runtime --lib --features native-bridge --profile staticlib + if ($LASTEXITCODE -ne 0) { + throw "native runtime static library build failed" + } } -$target = Target-Label -$buildHash = Build-Hash -$distRoot = Join-Path $RepoRoot $OutputDir +$sourceStatusAfter = @(Git-Status) +if ($sourceStatusBefore.Count -eq 0 -and $sourceStatusAfter.Count -ne 0) { + throw "package build dirtied the source tree: $($sourceStatusAfter -join '; ')" +} +$sourceDirty = $sourceStatusBefore.Count -ne 0 -or $sourceStatusAfter.Count -ne 0 +$artifactProvenance = if ($NoBuild) { "prebuilt-unverified" } else { "built-by-package-toolchain" } +$releaseEligible = -not $NoBuild -and -not $sourceDirty +$distRoot = if ([IO.Path]::IsPathRooted($OutputDir)) { + [IO.Path]::GetFullPath($OutputDir) +} else { + [IO.Path]::GetFullPath((Join-Path $RepoRoot $OutputDir)) +} $stageName = "sengoo-$Version-$target" $stage = Join-Path $distRoot $stageName $binDir = Join-Path $stage "bin" @@ -71,12 +153,16 @@ $tools = @("sgc", "sgpm", "sgfmt", "sglsp") $toolVersions = [ordered]@{} foreach ($tool in $tools) { $binary = "$tool$exeSuffix" - $source = Join-Path $RepoRoot "target/release/$binary" + $source = Join-Path $CargoTargetDir "release/$binary" if (-not (Test-Path -LiteralPath $source)) { throw "missing release binary: $source" } Copy-Item -LiteralPath $source -Destination (Join-Path $binDir $binary) $toolVersions[$tool] = (& $source --version).Trim() + $expectedVersionSuffix = "($buildHash)" + if (-not $toolVersions[$tool].EndsWith($expectedVersionSuffix, [StringComparison]::Ordinal)) { + throw "$tool version identity does not match repository HEAD prefix ${buildHash}: $($toolVersions[$tool])" + } } $readme = Join-Path $RepoRoot "README.md" @@ -109,6 +195,16 @@ Get-ChildItem -LiteralPath $stdlibSource -File | Where-Object { Copy-Item -LiteralPath $_.FullName -Destination (Join-Path $stdlibDir $_.Name) } +$runtimeLibraryName = if (Is-WindowsHost) { "sengoo_runtime.lib" } else { "libsengoo_runtime.a" } +$runtimeLibrarySource = Join-Path $CargoTargetDir "staticlib/$runtimeLibraryName" +if (-not (Test-Path -LiteralPath $runtimeLibrarySource)) { + throw "missing target native runtime static library: $runtimeLibrarySource" +} +$runtimeTargetDir = Join-Path $runtimeDir $target +New-Item -ItemType Directory -Force -Path $runtimeTargetDir | Out-Null +$runtimeLibraryDestination = Join-Path $runtimeTargetDir $runtimeLibraryName +Copy-Item -LiteralPath $runtimeLibrarySource -Destination $runtimeLibraryDestination + Get-ChildItem -LiteralPath $stdlibSource -File | Where-Object { $_.Name -like "runtime*.c" -or $_.Name -eq "runtime_shared.h" } | ForEach-Object { @@ -116,20 +212,79 @@ Get-ChildItem -LiteralPath $stdlibSource -File | Where-Object { Copy-Item -LiteralPath $_.FullName -Destination (Join-Path $stdlibDir $_.Name) } +function Relative-PayloadPath($Root, $Path) { + $rootFull = [System.IO.Path]::GetFullPath($Root).TrimEnd([char[]]@('\', '/')) + $pathFull = [System.IO.Path]::GetFullPath($Path) + if (-not $pathFull.StartsWith($rootFull + [System.IO.Path]::DirectorySeparatorChar, [System.StringComparison]::OrdinalIgnoreCase)) { + throw "payload path escapes stage root: $pathFull" + } + return $pathFull.Substring($rootFull.Length + 1).Replace('\', '/') +} + +$payloadPaths = @( + Get-ChildItem -LiteralPath $stage -Recurse -File | ForEach-Object { + Relative-PayloadPath $stage $_.FullName + } +) +[Array]::Sort($payloadPaths, [StringComparer]::Ordinal) +$payloadEntries = @( + $payloadPaths | ForEach-Object { + $relativePath = $_ + $payloadPath = Join-Path $stage $relativePath.Replace('/', [IO.Path]::DirectorySeparatorChar) + [ordered]@{ + path = $relativePath + sha256 = (Get-FileHash -Algorithm SHA256 -LiteralPath $payloadPath).Hash.ToLowerInvariant() + size = (Get-Item -LiteralPath $payloadPath).Length + } + } +) +$payloadChecksumPath = Join-Path $stage "payloads.sha256" +$payloadChecksumText = (($payloadEntries | ForEach-Object { "$($_.sha256) $($_.path)" }) -join "`n") + "`n" +[IO.File]::WriteAllText($payloadChecksumPath, $payloadChecksumText, [Text.Encoding]::ASCII) +$buildManifestId = (Get-FileHash -Algorithm SHA256 -LiteralPath $payloadChecksumPath).Hash.ToLowerInvariant() +$runtimeRelativePath = Relative-PayloadPath $stage $runtimeLibraryDestination +$runtimeHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $runtimeLibraryDestination).Hash.ToLowerInvariant() +$runtimeLinkArgs = if (Is-WindowsHost) { + @( + "kernel32.lib", "ntdll.lib", "userenv.lib", "ws2_32.lib", "dbghelp.lib", + "advapi32.lib", "bcrypt.lib", "crypt32.lib", "ncrypt.lib", "secur32.lib", + "legacy_stdio_definitions.lib", "msvcrt.lib", "vcruntime.lib", "ucrt.lib" + ) +} elseif ($target -like "*apple-darwin") { + @("-framework", "Security", "-framework", "CoreFoundation") +} else { + @("-lm") +} + $archiveBase = Join-Path $distRoot $stageName $archive = if (Is-WindowsHost) { "$archiveBase.zip" } else { "$archiveBase.tar.gz" } $archiveLeaf = Split-Path -Leaf $archive $checksumLeaf = "$archiveLeaf.sha256" $manifest = [ordered]@{ - schema_version = 1 + schema_version = 2 version = $Version target = $target build_hash = $buildHash + source_revision = $sourceRevision + source_dirty = $sourceDirty + artifact_provenance = $artifactProvenance + release_eligible = $releaseEligible + build_manifest_id = $buildManifestId tools = $tools tool_versions = $toolVersions stdlib_modules = @(Get-ChildItem -LiteralPath $stdlibDir -Filter "*.sg" | Sort-Object Name | ForEach-Object { $_.Name }) runtime_sources = @(Get-ChildItem -LiteralPath $runtimeDir -File | Sort-Object Name | ForEach-Object { $_.Name }) + native_runtime = [ordered]@{ + abi_version = 1 + target = $target + library = $runtimeRelativePath + sha256 = $runtimeHash + link_args = $runtimeLinkArgs + dynamic_dependencies = @() + } + payload_checksum_file = "payloads.sha256" + payloads = $payloadEntries archive_file = $archiveLeaf checksum_file = $checksumLeaf runner_os = $env:RUNNER_OS @@ -138,7 +293,52 @@ $manifest = [ordered]@{ license_included = [bool]$license generated_at_utc = (Get-Date).ToUniversalTime().ToString("o") } -$manifest | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $stage "manifest.json") -Encoding UTF8 +function Format-JsonString([string]$Value) { + $escaped = $Value. + Replace('\', '\\'). + Replace('"', '\"'). + Replace("`r", '\r'). + Replace("`n", '\n'). + Replace("`t", '\t') + return '"' + $escaped + '"' +} + +function Format-JsonStringArray([string[]]$Values) { + $parts = @(@($Values) | ForEach-Object { Format-JsonString ([string]$_) }) + return '[' + ($parts -join ', ') + ']' +} + +function Set-JsonArrayProperty([string]$Json, [string]$Property, [string[]]$Values) { + $arrayJson = Format-JsonStringArray $Values + $name = [regex]::Escape($Property) + $replacementText = '"' + $Property + '": ' + $arrayJson + $patterns = @( + "`"$name`"\s*:\s*`"(?:\\.|[^`"])*`"", + "`"$name`"\s*:\s*\[[^\]]*\]", + "`"$name`"\s*:\s*null" + ) + $replaced = $false + foreach ($pattern in $patterns) { + $regex = [regex]::new($pattern) + if ($regex.IsMatch($Json)) { + $Json = $regex.Replace($Json, { param($m) $replacementText }, 1) + $replaced = $true + break + } + } + if (-not $replaced) { + throw "failed to force JSON array property: $Property" + } + return $Json +} + +# ConvertTo-Json collapses single-element and empty arrays; force the +# contract-critical arrays back to true JSON arrays before writing. +$manifestJson = $manifest | ConvertTo-Json -Depth 8 +$manifestJson = Set-JsonArrayProperty $manifestJson "link_args" ([string[]]@($runtimeLinkArgs)) +$manifestJson = Set-JsonArrayProperty $manifestJson "dynamic_dependencies" @() +$utf8NoBom = [System.Text.UTF8Encoding]::new($false) +[System.IO.File]::WriteAllText((Join-Path $stage "manifest.json"), $manifestJson, $utf8NoBom) Remove-Item -LiteralPath $archive -Force -ErrorAction SilentlyContinue Remove-Item -LiteralPath "$archive.sha256" -Force -ErrorAction SilentlyContinue diff --git a/scripts/tests/compare-senline-package-manifests.tests.ps1 b/scripts/tests/compare-senline-package-manifests.tests.ps1 new file mode 100644 index 00000000..deefe591 --- /dev/null +++ b/scripts/tests/compare-senline-package-manifests.tests.ps1 @@ -0,0 +1,132 @@ +# Negative tests for pin-grade dual-package comparison (task 8.7). +$ErrorActionPreference = "Stop" +$Root = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) +if (-not (Test-Path -LiteralPath (Join-Path $Root "scripts\compare-senline-package-manifests.ps1"))) { + $Root = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path +} +$Compare = Join-Path $Root "scripts\compare-senline-package-manifests.ps1" +$Tmp = Join-Path $env:TEMP ("senline-compare-tests-" + [guid]::NewGuid().ToString("n")) +New-Item -ItemType Directory -Force -Path $Tmp | Out-Null +$utf8 = [Text.UTF8Encoding]::new($false) + +function New-BaseManifest([string]$RuntimeName = "sengoo_runtime") { + return [ordered]@{ + schema_version = 1 + package = "senline-domain-worker" + version = "0.0.0-test" + built_with_sgc = "0.1.0" + source_tree = "examples/realworld/senline-domain-worker" + source_revision = ("a" * 40) + build_manifest_id = ("b" * 40) + target = [ordered]@{ os = "windows"; arch = "x86_64"; abi = "msvc"; triple = "x86_64-pc-windows-msvc" } + protocols = @("senline-worker-v1") + runtime_dependencies = @( + [ordered]@{ name = $RuntimeName; role = "installed-native-runtime"; note = "test" } + ) + build_tools = @( + [ordered]@{ name = "sgc"; version = "0.1.0"; role = "installed-toolchain-build" } + [ordered]@{ name = "sgpm"; role = "installed-package-manager" } + ) + license = [ordered]@{ spdx_expression = "UNLICENSED"; file = "LICENSE.txt"; note = "test" } + provenance = [ordered]@{ + built_with_installed_toolchain_only = $true + generate_build_identity = $true + cargo_forbidden_at_package_time = $true + sbom_inputs = "sbom-inputs.json" + } + payloads = @( + [ordered]@{ path = "senline_domain_worker.exe"; sha256 = ("1" * 64); size = 100 } + [ordered]@{ path = "LICENSE.txt"; sha256 = ("2" * 64); size = 10 } + ) + notes = @("test") + } +} + +function Write-Man([string]$Path, $Object) { + [IO.File]::WriteAllText($Path, ($Object | ConvertTo-Json -Depth 8), $utf8) +} + +function Invoke-Compare([string]$Left, [string]$Right, [string]$Out, [switch]$AllowDrift) { + $args = @( + "-NoProfile", "-File", $Compare, + "-LeftManifest", $Left, + "-RightManifest", $Right, + "-OutputDir", $Out + ) + if ($AllowDrift) { $args += "-AllowExecutableHashDrift" } + $p = Start-Process -FilePath "powershell" -ArgumentList $args -Wait -PassThru -NoNewWindow + return $p.ExitCode +} + +$failed = 0 + +# 1) Identical manifests match. +$left = Join-Path $Tmp "left.json" +$right = Join-Path $Tmp "right.json" +Write-Man $left (New-BaseManifest) +Write-Man $right (New-BaseManifest) +$code = Invoke-Compare $left $right (Join-Path $Tmp "out-ok") +$cmp = Get-Content (Join-Path $Tmp "out-ok\comparison.json") -Raw | ConvertFrom-Json +if ($code -ne 0 -or -not $cmp.ok) { + Write-Host "FAIL identical manifests should match (exit=$code ok=$($cmp.ok))" + $failed++ +} else { + Write-Host "PASS identical manifests" +} + +# 2) runtime_dependencies identity change must fail. +$mut = New-BaseManifest -RuntimeName "different_runtime" +Write-Man $right $mut +$code = Invoke-Compare $left $right (Join-Path $Tmp "out-dep") +$cmp = Get-Content (Join-Path $Tmp "out-dep\comparison.json") -Raw | ConvertFrom-Json +$fields = @($cmp.meta_mismatches | ForEach-Object { $_.field }) +if ($code -eq 0 -or $cmp.ok -or ($fields -notcontains "runtime_dependencies")) { + Write-Host "FAIL runtime_dependencies rename should fail closed (exit=$code ok=$($cmp.ok) fields=$($fields -join ','))" + $failed++ +} else { + Write-Host "PASS runtime_dependencies identity mismatch fails" +} + +# 3) license change must fail. +Write-Man $right (New-BaseManifest) +$mut = New-BaseManifest +$mut.license.spdx_expression = "MIT" +Write-Man $right $mut +$code = Invoke-Compare $left $right (Join-Path $Tmp "out-lic") +$cmp = Get-Content (Join-Path $Tmp "out-lic\comparison.json") -Raw | ConvertFrom-Json +$fields = @($cmp.meta_mismatches | ForEach-Object { $_.field }) +if ($code -eq 0 -or $cmp.ok -or ($fields -notcontains "license")) { + Write-Host "FAIL license change should fail (exit=$code ok=$($cmp.ok) fields=$($fields -join ','))" + $failed++ +} else { + Write-Host "PASS license mismatch fails" +} + +# 4) Equal-size executable hash divergence fails by default; opt-in allows. +$mut = New-BaseManifest +$mut.payloads[0].sha256 = ("9" * 64) +Write-Man $right $mut +$code = Invoke-Compare $left $right (Join-Path $Tmp "out-hash") +$cmp = Get-Content (Join-Path $Tmp "out-hash\comparison.json") -Raw | ConvertFrom-Json +if ($code -eq 0 -or $cmp.ok) { + Write-Host "FAIL exe hash mismatch should fail closed" + $failed++ +} else { + Write-Host "PASS exe hash mismatch fails closed" +} +$code = Invoke-Compare $left $right (Join-Path $Tmp "out-hash-opt") -AllowDrift +$cmp = Get-Content (Join-Path $Tmp "out-hash-opt\comparison.json") -Raw | ConvertFrom-Json +if ($code -ne 0 -or -not $cmp.ok) { + Write-Host "FAIL AllowExecutableHashDrift should allow exe hash divergence" + $failed++ +} else { + Write-Host "PASS AllowExecutableHashDrift opt-in" +} + +Remove-Item -LiteralPath $Tmp -Recurse -Force -ErrorAction SilentlyContinue +if ($failed -ne 0) { + Write-Error "$failed compare negative test(s) failed" + exit 1 +} +Write-Host "All compare pin-grade negative tests passed" +exit 0 diff --git a/tools/sgc/Cargo.toml b/tools/sgc/Cargo.toml index 6a64718e..839e7e7d 100644 --- a/tools/sgc/Cargo.toml +++ b/tools/sgc/Cargo.toml @@ -27,6 +27,7 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } serde = { workspace = true } serde_json = { workspace = true } +sha2 = "=0.10.9" toml = "0.8" which = "7.0" cranelift-codegen = "0.110" diff --git a/tools/sgc/src/cli.rs b/tools/sgc/src/cli.rs index fc568788..6eb3e71f 100644 --- a/tools/sgc/src/cli.rs +++ b/tools/sgc/src/cli.rs @@ -2,6 +2,7 @@ use clap::{Parser as ClapParser, Subcommand}; use miette::Result; use std::path::Path; +use crate::installed_runtime::{initialize_native_runtime_mode, NativeRuntimeMode}; use crate::{ cmd_bench_compile, cmd_bench_incremental, cmd_bench_reflection, cmd_bench_run, cmd_build, cmd_check, cmd_daemon, cmd_doc, cmd_dump_ast, cmd_repl, cmd_run, cmd_test, @@ -31,6 +32,15 @@ pub(crate) struct Cli { #[arg(long = "error-format", global = true, value_enum, default_value_t = ErrorFormat::Text)] error_format: ErrorFormat, + /// Native runtime source policy for this compiler process. + #[arg( + long = "runtime-mode", + global = true, + value_enum, + default_value_t = NativeRuntimeMode::Installed + )] + runtime_mode: NativeRuntimeMode, + #[command(subcommand)] command: Commands, } @@ -341,9 +351,22 @@ pub(crate) enum BenchCommands { pub(crate) async fn run() -> Result<()> { let cli = Cli::parse(); set_error_format(cli.error_format); + initialize_native_runtime_mode(cli.runtime_mode)?; dispatch(cli.command).await } +pub(crate) fn validate_runtime_mode_daemon_combination( + runtime_mode: NativeRuntimeMode, + daemon_requested: bool, +) -> Result<()> { + if daemon_requested && runtime_mode == NativeRuntimeMode::SourceDevelopment { + miette::bail!( + "source-development runtime mode does not support daemon startup or dispatch; run the command directly so non-release provenance stays process-local" + ); + } + Ok(()) +} + async fn dispatch(command: Commands) -> Result<()> { let result = match command { Commands::Build { @@ -365,6 +388,11 @@ async fn dispatch(command: Commands) -> Result<()> { timings_json, debug_info, } => { + crate::installed_runtime::validate_native_runtime_mode_environment()?; + validate_runtime_mode_daemon_combination( + crate::installed_runtime::native_runtime_mode(), + daemon, + )?; if matches!(target.as_deref(), Some("wasm" | "bytecode")) { if daemon { miette::bail!("portable targets do not support daemon dispatch"); @@ -446,6 +474,11 @@ async fn dispatch(command: Commands) -> Result<()> { target, args, } => { + crate::installed_runtime::validate_native_runtime_mode_environment()?; + validate_runtime_mode_daemon_combination( + crate::installed_runtime::native_runtime_mode(), + daemon, + )?; if let Some(target) = target.as_deref() { match target { "bytecode" => { @@ -569,7 +602,13 @@ async fn dispatch(command: Commands) -> Result<()> { Commands::Doc { input, output } => cmd_doc(&input, &output).await, Commands::Repl => cmd_repl().await, Commands::DumpAst { input } => cmd_dump_ast(&input).await, - Commands::Daemon { addr } => cmd_daemon(&addr).await, + Commands::Daemon { addr } => { + validate_runtime_mode_daemon_combination( + crate::installed_runtime::native_runtime_mode(), + true, + )?; + cmd_daemon(&addr).await + } Commands::Bench { command } => match command { BenchCommands::Run { suite, diff --git a/tools/sgc/src/commands/build.rs b/tools/sgc/src/commands/build.rs index fb0734a1..0643dc5b 100644 --- a/tools/sgc/src/commands/build.rs +++ b/tools/sgc/src/commands/build.rs @@ -27,6 +27,7 @@ pub(crate) async fn cmd_build( timings_json: Option<&str>, debug_info: bool, ) -> Result<()> { + crate::installed_runtime::validate_native_runtime_mode_environment()?; let build_target = NativeBuildTarget::resolve(target)?; if build_target.is_cross() { println!("cross-compile target: {}", build_target.triple); @@ -184,7 +185,20 @@ pub(crate) async fn cmd_build( } } let runtime_c = find_runtime_c(); - let runtime_c_fingerprint = optional_runtime_bundle_fingerprint(runtime_c.as_deref())?; + let runtime_source_fingerprint = optional_runtime_bundle_fingerprint(runtime_c.as_deref())?; + let (installed_runtime_fingerprint, runtime_provenance) = if emit_llvm { + (None, NativeRuntimeProvenance::not_linked()) + } else { + crate::installed_runtime::native_runtime_cache_context(&build_target)? + }; + let runtime_c_fingerprint = crate::installed_runtime::combine_runtime_cache_fingerprints( + runtime_source_fingerprint, + installed_runtime_fingerprint, + ); + let runtime_c_identity = crate::installed_runtime::runtime_source_cache_identity( + runtime_c.as_deref(), + &runtime_provenance, + ); let output_file = if let Some(out) = output { out.to_string() @@ -264,7 +278,11 @@ pub(crate) async fn cmd_build( contract_checks_enabled, debug_info, emit_llvm, - RuntimeSourceIdentity::new(runtime_c.clone(), runtime_c_fingerprint), + RuntimeSourceIdentity::with_provenance( + runtime_c_identity.clone(), + runtime_c_fingerprint, + runtime_provenance.clone(), + ), output_file.clone(), ); let mut edit_impact: Option = None; @@ -312,7 +330,14 @@ pub(crate) async fn cmd_build( } edit_impact = Some(impact); } - Some(metadata) + if metadata.runtime_c_fingerprint != runtime_c_fingerprint + || metadata.runtime_provenance != runtime_provenance + { + println!("build workset reuse disabled: runtime identity changed"); + None + } else { + Some(metadata) + } } else { println!( "build cache miss: no cache metadata at {}", @@ -330,7 +355,7 @@ pub(crate) async fn cmd_build( contract_checks_enabled, debug_info, &output_file, - runtime_c.as_deref(), + runtime_c_identity.as_deref(), ); if previous_build_metadata.is_some() && can_skip_codegen_via_generic_cache(edit_impact.as_ref(), &graph_v2, &generic_plan_stats) @@ -505,8 +530,9 @@ pub(crate) async fn cmd_build( contract_checks: contract_checks_enabled, debug_info, emit_llvm: true, - runtime_c, + runtime_c: runtime_c_identity, runtime_c_fingerprint, + runtime_provenance, llvm_ir_path: llvm_ir_path.to_string_lossy().to_string(), output_path: output_file.clone(), llvm_ir_hash, @@ -548,7 +574,7 @@ pub(crate) async fn cmd_build( llvm_ir_hash, &object_path, &output_file, - runtime_c.as_deref(), + runtime_c_identity.as_deref(), opt_level, contract_checks_enabled, debug_info, @@ -644,8 +670,9 @@ pub(crate) async fn cmd_build( contract_checks: contract_checks_enabled, debug_info, emit_llvm: false, - runtime_c, + runtime_c: runtime_c_identity, runtime_c_fingerprint, + runtime_provenance, llvm_ir_path: llvm_ir_path.to_string_lossy().to_string(), output_path: output_file.clone(), llvm_ir_hash, diff --git a/tools/sgc/src/commands/run.rs b/tools/sgc/src/commands/run.rs index 08d0b819..80eb0eca 100644 --- a/tools/sgc/src/commands/run.rs +++ b/tools/sgc/src/commands/run.rs @@ -37,6 +37,7 @@ pub(crate) async fn cmd_run( reflection: ReflectionCliOptions, debug_info: bool, ) -> Result<()> { + crate::installed_runtime::validate_native_runtime_mode_environment()?; println!("Running: {}", input); let input_path = Path::new(input); @@ -226,11 +227,25 @@ pub(crate) async fn cmd_run( drop(graph_snapshot); let runtime_c = find_runtime_c(); - let runtime_c_fingerprint = optional_runtime_bundle_fingerprint(runtime_c.as_deref())?; + let runtime_source_fingerprint = optional_runtime_bundle_fingerprint(runtime_c.as_deref())?; let clang_exe = find_clang(); let lli_exe = find_lli(); let resolved_engine = resolve_engine(requested_engine, clang_exe.is_some(), lli_exe.is_some())?; + let (installed_runtime_fingerprint, runtime_provenance) = + if matches!(resolved_engine, RunEngine::Native) { + crate::installed_runtime::native_runtime_cache_context(&NativeBuildTarget::host())? + } else { + (None, NativeRuntimeProvenance::not_linked()) + }; + let runtime_c_fingerprint = crate::installed_runtime::combine_runtime_cache_fingerprints( + runtime_source_fingerprint, + installed_runtime_fingerprint, + ); + let runtime_c_identity = crate::installed_runtime::runtime_source_cache_identity( + runtime_c.as_deref(), + &runtime_provenance, + ); if matches!(resolved_engine, RunEngine::Native) { let clang = clang_exe .as_deref() @@ -281,7 +296,11 @@ pub(crate) async fn cmd_run( debug_info, requested_engine, resolved_engine, - RuntimeSourceIdentity::new(runtime_c.clone(), runtime_c_fingerprint), + RuntimeSourceIdentity::with_provenance( + runtime_c_identity.clone(), + runtime_c_fingerprint, + runtime_provenance.clone(), + ), ); let mut edit_impact: Option = None; @@ -356,7 +375,14 @@ pub(crate) async fn cmd_run( } edit_impact = Some(impact); } - Some(metadata) + if metadata.runtime_c_fingerprint != runtime_c_fingerprint + || metadata.runtime_provenance != runtime_provenance + { + println!("run workset reuse disabled: runtime identity changed"); + None + } else { + Some(metadata) + } } else { println!( "cache miss: no cache metadata at {}", @@ -374,7 +400,7 @@ pub(crate) async fn cmd_run( debug_info, requested_engine, resolved_engine, - runtime_c.as_deref(), + runtime_c_identity.as_deref(), ); if previous_run_metadata.is_some() && can_skip_codegen_via_generic_cache(edit_impact.as_ref(), &graph_v2, &generic_plan_stats) @@ -558,7 +584,7 @@ pub(crate) async fn cmd_run( previous, llvm_ir_hash, &object_path, - runtime_c.as_deref(), + runtime_c_identity.as_deref(), opt_level, contract_checks_enabled, debug_info, @@ -678,8 +704,9 @@ pub(crate) async fn cmd_run( debug_info, requested_engine, resolved_engine, - runtime_c, + runtime_c: runtime_c_identity, runtime_c_fingerprint, + runtime_provenance, llvm_ir_path: llvm_ir_path.to_string_lossy().to_string(), executable_path: if matches!(resolved_engine, RunEngine::Native) { Some(executable_path.to_string_lossy().to_string()) diff --git a/tools/sgc/src/commands/test.rs b/tools/sgc/src/commands/test.rs index 78fb3547..c58d5391 100644 --- a/tools/sgc/src/commands/test.rs +++ b/tools/sgc/src/commands/test.rs @@ -117,6 +117,8 @@ pub(crate) struct TestOptions<'a> { } pub(crate) fn cmd_test(options: TestOptions<'_>) -> Result<()> { + crate::installed_runtime::validate_installed_native_runtime_for_host()?; + if options.locked { ensure_lockfile_current(options.root, options.manifest_path)?; } @@ -162,8 +164,13 @@ pub(crate) fn cmd_test(options: TestOptions<'_>) -> Result<()> { .then(create_coverage_report_path) .transpose()?; let mut command = Command::new(&sgc); + command.current_dir(options.root); + if crate::installed_runtime::native_runtime_mode() + == crate::installed_runtime::NativeRuntimeMode::SourceDevelopment + { + command.args(["--runtime-mode", "source-development"]); + } command - .current_dir(options.root) .arg("run") .arg(&test.path) .arg("-O") diff --git a/tools/sgc/src/cross_compile.rs b/tools/sgc/src/cross_compile.rs index d6392f67..98d5d0e9 100644 --- a/tools/sgc/src/cross_compile.rs +++ b/tools/sgc/src/cross_compile.rs @@ -12,10 +12,13 @@ fn host_triple_for(target_os: &str, target_arch: &str) -> &'static str { if target_os == "windows" { REFERENCE_TARGET_WINDOWS_MSVC } else if target_os == "macos" { + // Keep the host triple aligned with packaged/installed distribution + // targets (`*-apple-darwin`) so installed sgc resolves the native + // runtime without alias mismatches against `*-apple-macosx`. if target_arch == "aarch64" { - "aarch64-apple-macosx" + "aarch64-apple-darwin" } else { - "x86_64-apple-macosx" + "x86_64-apple-darwin" } } else { REFERENCE_TARGET_LINUX_GNU @@ -126,12 +129,12 @@ mod tests { #[test] fn cross_compile_host_triple_uses_macos_x86_64_when_requested() { - assert_eq!(host_triple_for("macos", "x86_64"), "x86_64-apple-macosx"); + assert_eq!(host_triple_for("macos", "x86_64"), "x86_64-apple-darwin"); } #[test] fn cross_compile_host_triple_uses_macos_aarch64_when_requested() { - assert_eq!(host_triple_for("macos", "aarch64"), "aarch64-apple-macosx"); + assert_eq!(host_triple_for("macos", "aarch64"), "aarch64-apple-darwin"); } #[test] diff --git a/tools/sgc/src/frontend_helpers.rs b/tools/sgc/src/frontend_helpers.rs index 5edc27d2..415eb973 100644 --- a/tools/sgc/src/frontend_helpers.rs +++ b/tools/sgc/src/frontend_helpers.rs @@ -110,6 +110,22 @@ pub(crate) fn dependency_graph_digest(dependency_edges: &BTreeMap usize { + // Pin-grade dual package builds set SENGOO_DETERMINISTIC_LINK=1; force a + // serial frontend so independent rebuilds emit bit-identical IR/object order. + let force_serial = match std::env::var("SENGOO_DETERMINISTIC_LINK") { + Ok(value) => { + let trimmed = value.trim(); + !(trimmed.is_empty() || trimmed == "0" || trimmed.eq_ignore_ascii_case("false")) + } + Err(_) => false, + }; + if force_serial { + // Serial pin-grade builds always schedule one frontend worker. Do not + // write `1.min(task_count.max(1))` — clippy::min_max treats that as a + // constant 1 and fails `-D warnings`. + let _ = task_count; + return 1; + } let requested = match requested { FrontendJobs::Auto => std::thread::available_parallelism() .map(|n| n.get()) diff --git a/tools/sgc/src/installed_runtime.rs b/tools/sgc/src/installed_runtime.rs new file mode 100644 index 00000000..bdadc0eb --- /dev/null +++ b/tools/sgc/src/installed_runtime.rs @@ -0,0 +1,546 @@ +use crate::{NativeBuildTarget, NativeRuntimeProvenance}; +use clap::ValueEnum; +use miette::{IntoDiagnostic, Result}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use std::collections::hash_map::DefaultHasher; +use std::fs; +use std::hash::{Hash, Hasher}; +use std::io::Read; +use std::path::{Component, Path, PathBuf}; +use std::sync::OnceLock; + +pub(crate) const NATIVE_RUNTIME_ABI_VERSION: u32 = 1; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub(crate) enum NativeRuntimeMode { + Installed, + SourceDevelopment, +} + +static NATIVE_RUNTIME_MODE: OnceLock = OnceLock::new(); + +pub(crate) fn initialize_native_runtime_mode(mode: NativeRuntimeMode) -> Result<()> { + NATIVE_RUNTIME_MODE + .set(mode) + .map_err(|_| miette::miette!("native runtime mode is already initialized")) +} + +pub(crate) fn native_runtime_mode() -> NativeRuntimeMode { + NATIVE_RUNTIME_MODE.get().copied().unwrap_or({ + if cfg!(test) { + NativeRuntimeMode::SourceDevelopment + } else { + NativeRuntimeMode::Installed + } + }) +} + +#[derive(Debug, Deserialize)] +struct InstalledToolchainManifest { + schema_version: u32, + target: String, + build_manifest_id: String, + #[serde(default)] + artifact_provenance: String, + #[serde(default)] + release_eligible: bool, + payloads: Vec, + native_runtime: Option, +} + +#[derive(Debug, Deserialize)] +struct InstalledPayloadManifest { + path: String, + sha256: String, +} + +#[derive(Debug, Deserialize)] +struct InstalledNativeRuntimeManifest { + abi_version: u32, + target: String, + library: String, + sha256: String, + link_args: Vec, + dynamic_dependencies: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct InstalledNativeRuntime { + pub(crate) library: PathBuf, + pub(crate) cache_fingerprint: u64, + pub(crate) provenance: NativeRuntimeProvenance, +} + +fn install_root_from_current_exe() -> Option { + let executable = std::env::current_exe().ok()?; + executable.parent()?.parent().map(Path::to_path_buf) +} + +fn manifest_path_from_current_exe() -> Option { + let path = install_root_from_current_exe()?.join("manifest.json"); + path.is_file().then_some(path) +} + +fn current_exe_is_source_local() -> bool { + let Some(workspace_root) = Path::new(env!("CARGO_MANIFEST_DIR")).ancestors().nth(2) else { + return false; + }; + let Ok(workspace_root) = workspace_root.canonicalize() else { + return false; + }; + let Ok(executable) = std::env::current_exe().and_then(|path| path.canonicalize()) else { + return false; + }; + executable.starts_with(workspace_root) +} + +pub(crate) fn validate_native_runtime_mode_environment() -> Result<()> { + if native_runtime_mode() == NativeRuntimeMode::SourceDevelopment { + return Ok(()); + } + for variable in ["SENGOO_ROOT", "SENGOO_STDLIB", "SENGOO_RUNTIME"] { + if std::env::var_os(variable).is_some() { + return Err(miette::miette!( + "installed runtime mode rejects {variable}; use --runtime-mode source-development inside the Sengoo source workspace for local runtime overrides" + )); + } + } + Ok(()) +} + +fn validate_relative_payload_path(path: &str) -> Result { + let relative = PathBuf::from(path); + if relative.as_os_str().is_empty() + || relative.is_absolute() + || relative + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(miette::miette!( + "installed native runtime library path must be a normalized relative payload path: {path}" + )); + } + Ok(relative) +} + +fn canonicalize_installed_target(triple: &str) -> String { + // Distribution packages use `*-apple-darwin`; older host triples used + // `*-apple-macosx`. Treat them as the same installed runtime family. + if let Some(prefix) = triple.strip_suffix("-apple-macosx") { + return format!("{prefix}-apple-darwin"); + } + triple.to_string() +} + +fn installed_targets_compatible(manifest_target: &str, requested_target: &str) -> bool { + canonicalize_installed_target(manifest_target) + == canonicalize_installed_target(requested_target) +} + +fn expected_link_args(target: &NativeBuildTarget) -> Vec { + if target.is_windows_msvc() { + [ + "kernel32.lib", + "ntdll.lib", + "userenv.lib", + "ws2_32.lib", + "dbghelp.lib", + "advapi32.lib", + "bcrypt.lib", + "crypt32.lib", + "ncrypt.lib", + "secur32.lib", + "legacy_stdio_definitions.lib", + "msvcrt.lib", + "vcruntime.lib", + "ucrt.lib", + ] + .into_iter() + .map(str::to_string) + .collect() + } else if target.triple.ends_with("-apple-darwin") || target.triple.ends_with("-apple-macosx") { + ["-framework", "Security", "-framework", "CoreFoundation"] + .into_iter() + .map(str::to_string) + .collect() + } else { + vec!["-lm".to_string()] + } +} + +fn validate_installed_runtime_bridge( + install_root: &Path, + payloads: &[InstalledPayloadManifest], +) -> Result> { + let stdlib = install_root.join("share").join("sengoo").join("stdlib"); + let files = [ + "runtime.c", + "runtime_breadth.c", + "runtime_collections.c", + "runtime_json.c", + "runtime_process.c", + "runtime_string.c", + "runtime_shared.h", + ]; + for file in files { + let path = stdlib.join(file); + if !path.is_file() { + return Err(miette::miette!( + "installed native runtime bridge file is missing: {}", + path.display() + )); + } + } + + let mut verified_hashes = Vec::with_capacity(files.len()); + for file in files { + let relative = format!("share/sengoo/stdlib/{file}"); + let matches = payloads + .iter() + .filter(|payload| payload.path == relative) + .collect::>(); + if matches.len() != 1 { + return Err(miette::miette!( + "installed toolchain manifest must contain exactly one payload checksum for {relative}" + )); + } + let payload = matches[0]; + validate_relative_payload_path(&payload.path)?; + if payload.sha256.len() != 64 + || !payload.sha256.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return Err(miette::miette!( + "installed runtime payload SHA-256 is not 64 hexadecimal characters for {relative}: {}", + payload.sha256 + )); + } + let path = install_root.join(&payload.path); + let actual_sha256 = sha256_file(&path)?; + if !actual_sha256.eq_ignore_ascii_case(&payload.sha256) { + return Err(miette::miette!( + "installed runtime payload SHA-256 mismatch for {}: expected={}, actual={}", + path.display(), + payload.sha256, + actual_sha256 + )); + } + verified_hashes.push(payload.sha256.to_ascii_lowercase()); + } + Ok(verified_hashes) +} + +fn sha256_file(path: &Path) -> Result { + let mut input = fs::File::open(path) + .into_diagnostic() + .map_err(|error| miette::miette!("failed to open installed native runtime: {error}"))?; + let mut digest = Sha256::new(); + let mut buffer = [0u8; 64 * 1024]; + loop { + let read = input + .read(&mut buffer) + .into_diagnostic() + .map_err(|error| miette::miette!("failed to hash installed native runtime: {error}"))?; + if read == 0 { + break; + } + digest.update(&buffer[..read]); + } + let digest = digest.finalize(); + Ok(format!("{digest:x}")) +} + +fn collect_source_runtime_files(directory: &Path, files: &mut Vec) -> Result<()> { + let entries = fs::read_dir(directory) + .into_diagnostic() + .map_err(|error| miette::miette!("failed to inspect source runtime inputs: {error}"))?; + for entry in entries { + let entry = entry + .into_diagnostic() + .map_err(|error| miette::miette!("failed to inspect source runtime inputs: {error}"))?; + let path = entry.path(); + let file_type = entry + .file_type() + .into_diagnostic() + .map_err(|error| miette::miette!("failed to inspect source runtime inputs: {error}"))?; + if file_type.is_dir() { + collect_source_runtime_files(&path, files)?; + } else if file_type.is_file() { + files.push(path); + } + } + Ok(()) +} + +fn source_runtime_input_fingerprint(workspace_root: &Path) -> Result { + let runtime_root = workspace_root.join("runtime"); + let mut files = vec![ + workspace_root.join("Cargo.toml"), + workspace_root.join("Cargo.lock"), + runtime_root.join("Cargo.toml"), + ]; + for optional in [ + workspace_root.join("rust-toolchain.toml"), + workspace_root.join(".cargo").join("config.toml"), + runtime_root.join("build.rs"), + ] { + if optional.is_file() { + files.push(optional); + } + } + collect_source_runtime_files(&runtime_root.join("src"), &mut files)?; + files.sort_by_key(|path| { + path.strip_prefix(workspace_root) + .unwrap_or(path) + .to_string_lossy() + .replace('\\', "/") + }); + + let mut digest = Sha256::new(); + for path in files { + let relative = path + .strip_prefix(workspace_root) + .unwrap_or(&path) + .to_string_lossy() + .replace('\\', "/"); + let bytes = fs::read(&path) + .into_diagnostic() + .map_err(|error| miette::miette!("failed to read source runtime input: {error}"))?; + digest.update((relative.len() as u64).to_be_bytes()); + digest.update(relative.as_bytes()); + digest.update((bytes.len() as u64).to_be_bytes()); + digest.update(&bytes); + } + let digest = digest.finalize(); + Ok(u64::from_be_bytes(digest[..8].try_into().unwrap())) +} + +pub(crate) fn resolve_installed_native_runtime( + target: &NativeBuildTarget, +) -> Result> { + let manifest_path = manifest_path_from_current_exe(); + if native_runtime_mode() == NativeRuntimeMode::SourceDevelopment { + if !current_exe_is_source_local() { + return Err(miette::miette!( + "source runtime development mode requires an sgc executable inside its compiled Sengoo source workspace" + )); + } + return Ok(None); + } + validate_native_runtime_mode_environment()?; + let Some(manifest_path) = manifest_path else { + if current_exe_is_source_local() { + return Err(miette::miette!( + "source runtime development mode is not selected; pass --runtime-mode source-development to authorize non-release Cargo runtime construction" + )); + } + return Err(miette::miette!( + "installed toolchain manifest is missing; Cargo fallback is disabled outside a Sengoo source checkout" + )); + }; + let bytes = fs::read(&manifest_path) + .into_diagnostic() + .map_err(|error| miette::miette!("failed to read installed toolchain manifest: {error}"))?; + let manifest: InstalledToolchainManifest = serde_json::from_slice(&bytes) + .into_diagnostic() + .map_err(|error| miette::miette!("invalid installed toolchain manifest: {error}"))?; + if manifest.schema_version != 2 { + return Err(miette::miette!( + "installed toolchain manifest schema {} does not provide native runtime metadata", + manifest.schema_version + )); + } + if !installed_targets_compatible(&manifest.target, &target.triple) { + return Err(miette::miette!( + "installed toolchain target mismatch: manifest={}, requested={}", + manifest.target, + target.triple + )); + } + if manifest.build_manifest_id.len() != 64 + || !manifest + .build_manifest_id + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + { + return Err(miette::miette!( + "installed build_manifest_id is not 64 hexadecimal characters" + )); + } + let runtime = manifest.native_runtime.ok_or_else(|| { + miette::miette!("installed toolchain manifest is missing native_runtime metadata") + })?; + if !installed_targets_compatible(&runtime.target, &target.triple) { + return Err(miette::miette!( + "installed native runtime target mismatch: manifest={}, requested={}", + runtime.target, + target.triple + )); + } + if runtime.abi_version != NATIVE_RUNTIME_ABI_VERSION { + return Err(miette::miette!( + "installed native runtime ABI mismatch: manifest={}, supported={}", + runtime.abi_version, + NATIVE_RUNTIME_ABI_VERSION + )); + } + let relative_library = validate_relative_payload_path(&runtime.library)?; + let install_root = manifest_path + .parent() + .ok_or_else(|| miette::miette!("installed toolchain manifest has no parent directory"))?; + let library = install_root.join(relative_library); + if !library.is_file() { + return Err(miette::miette!( + "installed native runtime library is missing: {}", + library.display() + )); + } + if runtime.sha256.len() != 64 || !runtime.sha256.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(miette::miette!( + "installed native runtime SHA-256 is not 64 hexadecimal characters: {}", + runtime.sha256 + )); + } + let actual_sha256 = sha256_file(&library)?; + if !actual_sha256.eq_ignore_ascii_case(&runtime.sha256) { + return Err(miette::miette!( + "installed native runtime SHA-256 mismatch for {}: expected={}, actual={}", + library.display(), + runtime.sha256, + actual_sha256 + )); + } + let expected_link_args = expected_link_args(target); + if runtime.link_args != expected_link_args { + return Err(miette::miette!( + "installed native runtime link arguments mismatch: manifest={:?}, supported={:?}", + runtime.link_args, + expected_link_args + )); + } + if !runtime.dynamic_dependencies.is_empty() { + return Err(miette::miette!( + "installed native runtime declares unsupported dynamic dependencies: {:?}", + runtime.dynamic_dependencies + )); + } + let bridge_hashes = validate_installed_runtime_bridge(install_root, &manifest.payloads)?; + let mut cache_identity = DefaultHasher::new(); + manifest.build_manifest_id.hash(&mut cache_identity); + manifest.artifact_provenance.hash(&mut cache_identity); + manifest.release_eligible.hash(&mut cache_identity); + runtime.abi_version.hash(&mut cache_identity); + runtime.target.hash(&mut cache_identity); + runtime + .sha256 + .to_ascii_lowercase() + .hash(&mut cache_identity); + runtime.link_args.hash(&mut cache_identity); + runtime.dynamic_dependencies.hash(&mut cache_identity); + bridge_hashes.hash(&mut cache_identity); + let artifact_provenance = if manifest.artifact_provenance.is_empty() { + "installed-unknown".to_string() + } else { + manifest.artifact_provenance + }; + Ok(Some(InstalledNativeRuntime { + library, + cache_fingerprint: cache_identity.finish(), + provenance: NativeRuntimeProvenance { + runtime_mode: "installed".to_string(), + artifact_provenance, + release_eligible: manifest.release_eligible, + senline_pin_evidence: false, + build_manifest_id: Some(manifest.build_manifest_id), + }, + })) +} + +pub(crate) fn native_runtime_cache_context( + target: &NativeBuildTarget, +) -> Result<(Option, NativeRuntimeProvenance)> { + match resolve_installed_native_runtime(target)? { + Some(runtime) => Ok((Some(runtime.cache_fingerprint), runtime.provenance)), + None => { + let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(2) + .ok_or_else(|| miette::miette!("Sengoo source workspace root is unavailable"))?; + Ok(( + Some(source_runtime_input_fingerprint(workspace_root)?), + NativeRuntimeProvenance::source_development(), + )) + } + } +} + +pub(crate) fn runtime_source_cache_identity( + runtime_c: Option<&str>, + provenance: &NativeRuntimeProvenance, +) -> Option { + runtime_c.map(|path| { + if provenance.runtime_mode == "installed" { + "installed:share/sengoo/stdlib/runtime.c".to_string() + } else { + path.to_string() + } + }) +} + +pub(crate) fn validate_installed_native_runtime_for_host() -> Result<()> { + resolve_installed_native_runtime(&NativeBuildTarget::host()).map(|_| ()) +} + +pub(crate) fn combine_runtime_cache_fingerprints( + source_fingerprint: Option, + installed_fingerprint: Option, +) -> Option { + if source_fingerprint.is_none() && installed_fingerprint.is_none() { + return None; + } + let mut combined = DefaultHasher::new(); + source_fingerprint.hash(&mut combined); + installed_fingerprint.hash(&mut combined); + Some(combined.finish()) +} + +#[cfg(test)] +mod tests { + use super::source_runtime_input_fingerprint; + use std::fs; + use std::time::{SystemTime, UNIX_EPOCH}; + + #[test] + fn source_runtime_fingerprint_tracks_rust_sources_and_lockfile() { + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!("sgc_source_runtime_fingerprint_{stamp}")); + let runtime_src = root.join("runtime").join("src"); + fs::create_dir_all(&runtime_src).unwrap(); + fs::write(root.join("Cargo.toml"), "[workspace]\n").unwrap(); + fs::write(root.join("Cargo.lock"), "version = 4\n").unwrap(); + fs::write( + root.join("runtime").join("Cargo.toml"), + "[package]\nname = \"sengoo-runtime\"\n", + ) + .unwrap(); + fs::write(runtime_src.join("lib.rs"), "pub fn value() -> u32 { 1 }\n").unwrap(); + + let baseline = source_runtime_input_fingerprint(&root).unwrap(); + fs::write(runtime_src.join("lib.rs"), "pub fn value() -> u32 { 2 }\n").unwrap(); + let source_changed = source_runtime_input_fingerprint(&root).unwrap(); + assert_ne!(baseline, source_changed); + + fs::write( + root.join("Cargo.lock"), + "version = 4\n# dependency changed\n", + ) + .unwrap(); + let lock_changed = source_runtime_input_fingerprint(&root).unwrap(); + assert_ne!(source_changed, lock_changed); + + let _ = fs::remove_dir_all(root); + } +} diff --git a/tools/sgc/src/main.rs b/tools/sgc/src/main.rs index 8f11b6ab..d091acd1 100644 --- a/tools/sgc/src/main.rs +++ b/tools/sgc/src/main.rs @@ -29,6 +29,7 @@ mod frontend_snapshot; mod generic_cache; mod graph_builder; mod impact; +mod installed_runtime; mod interface; mod module_graph; mod native_link; @@ -523,6 +524,7 @@ fn canonical_or_lossy(path: &Path) -> String { } async fn cmd_check(input: &str) -> Result<()> { + installed_runtime::validate_installed_native_runtime_for_host()?; println!("Checking: {}", input); let source = match fs::read_to_string(input).into_diagnostic() { diff --git a/tools/sgc/src/model_types.rs b/tools/sgc/src/model_types.rs index 85842d7a..368576ec 100644 --- a/tools/sgc/src/model_types.rs +++ b/tools/sgc/src/model_types.rs @@ -256,6 +256,45 @@ struct ModuleGraphSnapshot { rebuilt_modules: Vec, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +struct NativeRuntimeProvenance { + runtime_mode: String, + artifact_provenance: String, + release_eligible: bool, + senline_pin_evidence: bool, + build_manifest_id: Option, +} + +impl Default for NativeRuntimeProvenance { + fn default() -> Self { + Self { + runtime_mode: "unknown".to_string(), + artifact_provenance: "unknown".to_string(), + release_eligible: false, + senline_pin_evidence: false, + build_manifest_id: None, + } + } +} + +impl NativeRuntimeProvenance { + fn source_development() -> Self { + Self { + runtime_mode: "source-development".to_string(), + artifact_provenance: "source-cargo-development".to_string(), + ..Self::default() + } + } + + fn not_linked() -> Self { + Self { + runtime_mode: "not-linked".to_string(), + artifact_provenance: "not-applicable".to_string(), + ..Self::default() + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] struct RunCacheMetadata { source_hash: u64, @@ -275,6 +314,8 @@ struct RunCacheMetadata { runtime_c: Option, #[serde(default)] runtime_c_fingerprint: Option, + #[serde(default)] + runtime_provenance: NativeRuntimeProvenance, llvm_ir_path: String, executable_path: Option, #[serde(default)] @@ -289,11 +330,29 @@ struct RunCacheMetadata { struct RuntimeSourceIdentity { path: Option, fingerprint: Option, + provenance: NativeRuntimeProvenance, } impl RuntimeSourceIdentity { + #[cfg(test)] fn new(path: Option, fingerprint: Option) -> Self { - Self { path, fingerprint } + Self { + path, + fingerprint, + provenance: NativeRuntimeProvenance::default(), + } + } + + fn with_provenance( + path: Option, + fingerprint: Option, + provenance: NativeRuntimeProvenance, + ) -> Self { + Self { + path, + fingerprint, + provenance, + } } } @@ -308,6 +367,7 @@ struct RunCacheKey { resolved_engine: RunEngine, runtime_c: Option, runtime_c_fingerprint: Option, + runtime_provenance: NativeRuntimeProvenance, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -330,6 +390,8 @@ struct BuildCacheMetadata { runtime_c: Option, #[serde(default)] runtime_c_fingerprint: Option, + #[serde(default)] + runtime_provenance: NativeRuntimeProvenance, llvm_ir_path: String, output_path: String, #[serde(default)] @@ -350,6 +412,7 @@ struct BuildCacheKey { emit_llvm: bool, runtime_c: Option, runtime_c_fingerprint: Option, + runtime_provenance: NativeRuntimeProvenance, output_path: String, } diff --git a/tools/sgc/src/native_toolchain.rs b/tools/sgc/src/native_toolchain.rs index ff8aecf5..5fcba86d 100644 --- a/tools/sgc/src/native_toolchain.rs +++ b/tools/sgc/src/native_toolchain.rs @@ -7,6 +7,7 @@ use std::process::Command; use std::sync::atomic::{AtomicU64, Ordering}; use crate::cross_compile::{linux_sysroot_from_env, windows_cross_sdk_include_paths}; +use crate::installed_runtime::resolve_installed_native_runtime; use crate::module_graph::collect_module_sources_with_edges; use crate::native_link::{ append_native_library_link_args, format_native_link_failure_message, @@ -485,7 +486,17 @@ fn find_async_runtime_staticlib(profile: &str) -> Option { .or_else(|| find_async_runtime_staticlib_in_dir(&profile_dir.join("deps"))) } -pub(crate) fn ensure_async_runtime_staticlib(opt_level: u8) -> Result { +pub(crate) fn ensure_async_runtime_staticlib( + opt_level: u8, + target: Option<&NativeBuildTarget>, +) -> Result { + let target = effective_target(target); + if let Some(runtime) = resolve_installed_native_runtime(&target)? { + return Ok(runtime.library); + } + eprintln!( + "[toolchain::source_runtime_development] runtime_mode=source-development artifact_provenance=source-cargo-development release_eligible=false senline_pin_evidence=false" + ); let profile = async_runtime_profile(opt_level); let workspace_root = workspace_root(); let mut command = Command::new("cargo"); @@ -535,12 +546,12 @@ pub(crate) fn append_native_runtime_inputs( &[NATIVE_NET_RUNTIME_DEFINE, NATIVE_ASYNC_RUNTIME_DEFINE], )?); } - object_paths.push(ensure_async_runtime_staticlib(opt_level)?); + object_paths.push(ensure_async_runtime_staticlib(opt_level, target)?); Ok(()) } fn platform_linker_args(target: &NativeBuildTarget) -> Vec<&'static str> { - if target.triple.ends_with("-apple-macosx") { + if target.triple.ends_with("-apple-darwin") || target.triple.ends_with("-apple-macosx") { vec!["-framework", "Security", "-framework", "CoreFoundation"] } else if target.is_linux_gnu() { vec!["-lm"] @@ -549,6 +560,60 @@ fn platform_linker_args(target: &NativeBuildTarget) -> Vec<&'static str> { } } +/// Pin-grade dual-build identity: enabled only when package scripts (or tests) +/// explicitly set `SENGOO_DETERMINISTIC_LINK` to a truthy value. Default off so +/// ordinary developer builds are unaffected. +fn deterministic_link_requested() -> bool { + match std::env::var("SENGOO_DETERMINISTIC_LINK") { + Ok(value) => { + let trimmed = value.trim(); + !(trimmed.is_empty() || trimmed == "0" || trimmed.eq_ignore_ascii_case("false")) + } + Err(_) => false, + } +} + +fn append_deterministic_link_args(command: &mut Command, target: &NativeBuildTarget) { + if !deterministic_link_requested() { + return; + } + if target.is_linux_gnu() || target.triple.contains("linux") { + // Drop GNU build-id notes that embed non-content identity. + command.arg("-Wl,--build-id=none"); + command.arg("-Wl,--hash-style=gnu"); + } else if target.is_windows_msvc() { + // clang driver path (cross or lld): /Brepro zeros PE timestamps. + command.arg("-Wl,/Brepro"); + } +} + +fn sorted_object_paths(object_paths: &[PathBuf]) -> Vec { + // Sort only relocatable objects. Static archives (.a/.lib) must stay after + // the objects that reference them (Unix linkers resolve archive symbols in + // left-to-right order). Reordering archives to the front caused + // `undefined reference to sengoo_net_last_error` on Linux core-language CI. + let mut objects = Vec::new(); + let mut archives = Vec::new(); + for path in object_paths { + let is_archive = path + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| { + ext.eq_ignore_ascii_case("a") + || ext.eq_ignore_ascii_case("lib") + || ext.eq_ignore_ascii_case("rlib") + }); + if is_archive { + archives.push(path.clone()); + } else { + objects.push(path.clone()); + } + } + objects.sort(); + objects.extend(archives); + objects +} + fn link_cross_target( clang_exe: &str, object_paths: &[PathBuf], @@ -563,13 +628,15 @@ fn link_cross_target( if target.is_linux_gnu() { command.arg("-fuse-ld=lld"); } - for object in object_paths { + let objects = sorted_object_paths(object_paths); + for object in &objects { command.arg(object); } append_native_library_link_args(&mut command, native_link_libraries, target, &search_paths); for arg in platform_linker_args(target) { command.arg(arg); } + append_deterministic_link_args(&mut command, target); command.arg("-o").arg(executable_path); let status = command .status() @@ -703,9 +770,13 @@ fn run_windows_link_command( let search_paths = native_library_search_paths_from_env(); let mut link_cmd = Command::new(link_exe); link_cmd.arg("/NOLOGO"); - let links_async_runtime = object_paths - .iter() - .any(|path| is_async_runtime_staticlib(path)); + if deterministic_link_requested() { + // Zero PE timestamps / non-deterministic COFF metadata for dual-build + // pin-grade package identity (MSVC link.exe /Brepro). + link_cmd.arg("/Brepro"); + } + let objects = sorted_object_paths(object_paths); + let links_async_runtime = objects.iter().any(|path| is_async_runtime_staticlib(path)); if links_async_runtime { // Keep compiler-generated async dispatch symbols that are only referenced // from the Rust async runtime static library. @@ -715,7 +786,7 @@ fn run_windows_link_command( link_cmd.arg(format!("/LIBPATH:{}", lib_path.display())); } append_native_library_link_args(&mut link_cmd, native_link_libraries, &target, &search_paths); - for object in object_paths { + for object in &objects { link_cmd.arg(object); } for lib in [ @@ -996,13 +1067,15 @@ fn run_link_command( if use_lld { clang_cmd.arg("-fuse-ld=lld"); } - for object in object_paths { + let objects = sorted_object_paths(object_paths); + for object in &objects { clang_cmd.arg(object); } append_native_library_link_args(&mut clang_cmd, native_link_libraries, target, &search_paths); for arg in platform_linker_args(target) { clang_cmd.arg(arg); } + append_deterministic_link_args(&mut clang_cmd, target); clang_cmd.arg("-o").arg(executable_path); clang_cmd .status() @@ -1579,13 +1652,15 @@ mod tests { #[test] fn native_link_adds_macos_security_and_corefoundation_frameworks() { - let target = NativeBuildTarget { - triple: "aarch64-apple-macosx".to_string(), - }; - assert_eq!( - platform_linker_args(&target), - vec!["-framework", "Security", "-framework", "CoreFoundation"] - ); + for triple in ["aarch64-apple-darwin", "aarch64-apple-macosx"] { + let target = NativeBuildTarget { + triple: triple.to_string(), + }; + assert_eq!( + platform_linker_args(&target), + vec!["-framework", "Security", "-framework", "CoreFoundation"] + ); + } } #[test] diff --git a/tools/sgc/src/stdlib_imports.rs b/tools/sgc/src/stdlib_imports.rs index a433f9dc..b828c711 100644 --- a/tools/sgc/src/stdlib_imports.rs +++ b/tools/sgc/src/stdlib_imports.rs @@ -369,6 +369,31 @@ mod tests { assert!(expanded.contains("struct Result")); } + #[test] + fn binary_io_import_expands_exact_buffer_and_pipe_surface() { + let expanded = + expand_stdlib_imports_for_source("import std::io;\ndef main() -> i64 { 0 }\n") + .expect("binary I/O stdlib import should expand"); + + for signature in [ + "def get_u8(self, index: i64) -> Result", + "def set_u8(self, index: i64, value: i64) -> Result", + "def read_u32_be(self, offset: i64) -> Result", + "def write_u32_be(self, offset: i64, value: i64) -> Result", + "fn sengoo_io_protocol_binary_mode() -> i64", + "fn sengoo_io_stdin_read_exact(buffer_handle: i64, offset: i64, len: i64) -> i64", + "fn sengoo_io_stdout_write_all(buffer_handle: i64, offset: i64, len: i64) -> i64", + "def io_protocol_binary_mode() -> Result", + "def io_stdin_read_exact(buffer: Buffer, offset: i64, len: i64) -> Result", + "def io_stdout_write_all(buffer: Buffer, offset: i64, len: i64) -> Result", + ] { + assert!( + expanded.contains(signature), + "expanded std::io is missing `{signature}`" + ); + } + } + #[test] fn args_import_expands_ffi_and_result_dependencies() { let expanded = @@ -432,6 +457,20 @@ mod tests { assert!(expanded.contains("struct JsonValue")); assert!(expanded.contains("def json_parse")); assert!(expanded.contains("def json_doc_object")); + assert!(expanded.contains("fn sengoo_json_last_error_kind() -> i64")); + assert!(expanded.contains("def JSON_ERROR_KIND_DUPLICATE_FIELD() -> i64")); + assert!(expanded.contains("def JSON_ERROR_KIND_INVALID_UNICODE() -> i64")); + assert!(expanded.contains("def JSON_ERROR_KIND_TRAILING_BYTES() -> i64")); + assert!(expanded.contains("def json_last_error_kind() -> i64")); + assert!(expanded.contains( + "fn sengoo_json_doc_new_string_len(handle: i64, value: i64, value_len: i64) -> i64" + )); + assert!(expanded.contains( + "fn sengoo_json_doc_new_string_from_string(handle: i64, string_handle: i64) -> i64" + )); + assert!(expanded.contains( + "def new_string_from_string(self, value: &String) -> Result" + )); assert!(expanded.contains("struct Buffer")); assert!(expanded.contains("struct Result")); } diff --git a/tools/sgc/src/tests.rs b/tools/sgc/src/tests.rs index 6de52090..55cee40a 100644 --- a/tools/sgc/src/tests.rs +++ b/tools/sgc/src/tests.rs @@ -25,14 +25,15 @@ use super::{ EditClass, EditImpact, FrontendFallbackScope, FrontendJobs, FrontendMemoryMode, FrontendProbeMode, FunctionFingerprint, GenericInstanceCacheEntry, GenericInstanceCacheMetadata, GenericInstanceFingerprint, GenericInstancePlanStats, - GenericItemFingerprint, LinkerMode, ModuleFingerprint, ReflectionMetadata, ReflectionMode, - RunCacheMetadata, RunEngine, RuntimeSourceIdentity, BUILD_GRAPH_SCHEMA_VERSION, - DAEMON_PROTOCOL_VERSION, DEFAULT_DAEMON_ADDR, DEFAULT_SYMBOL_FINGERPRINT_MAX_SOURCE_BYTES, - FRONTEND_MEMORY_STREAM_THRESHOLD_BYTES, GENERIC_INSTANCE_CACHE_SCHEMA_VERSION, - LOW_MEMORY_HINT_AVAILABLE_BYTES, + GenericItemFingerprint, LinkerMode, ModuleFingerprint, NativeRuntimeProvenance, + ReflectionMetadata, ReflectionMode, RunCacheMetadata, RunEngine, RuntimeSourceIdentity, + BUILD_GRAPH_SCHEMA_VERSION, DAEMON_PROTOCOL_VERSION, DEFAULT_DAEMON_ADDR, + DEFAULT_SYMBOL_FINGERPRINT_MAX_SOURCE_BYTES, FRONTEND_MEMORY_STREAM_THRESHOLD_BYTES, + GENERIC_INSTANCE_CACHE_SCHEMA_VERSION, LOW_MEMORY_HINT_AVAILABLE_BYTES, }; use crate::cli::Cli; use crate::cross_compile::NativeBuildTarget; +use crate::installed_runtime::NativeRuntimeMode; use clap::Parser as _; use sengoo_compiler::{ compile_to_ir as compile_compiler_ir, compile_to_mir, CompileWarning, DebugInfoConfig, @@ -69,6 +70,7 @@ fn metadata_for_test() -> RunCacheMetadata { resolved_engine: RunEngine::Native, runtime_c: Some("tools/stdlib/runtime.c".to_string()), runtime_c_fingerprint: Some(777), + runtime_provenance: NativeRuntimeProvenance::default(), llvm_ir_path: "tests/build/a.ll".to_string(), executable_path: Some("tests/build/a.exe".to_string()), llvm_ir_hash: 999, @@ -528,6 +530,24 @@ fn stdlib_runtime_exports_managed_buffer_helpers() { } } +#[test] +fn stdlib_runtime_exports_json_error_kind_diagnostic() { + let runtime_c = load_runtime_bundle_source_for_tests(); + + assert!( + runtime_c.contains("sengoo_json_last_error_kind"), + "runtime stdlib missing structured JSON error-kind export" + ); + assert!( + runtime_c.contains("sengoo_json_doc_new_string_len"), + "runtime stdlib missing length-aware JSON string-builder export" + ); + assert!( + runtime_c.contains("sengoo_json_doc_new_string_from_string"), + "runtime stdlib missing checked owned-String JSON builder export" + ); +} + #[test] fn runtime_source_bundle_discovers_anchor_and_existing_split_sources() { let root = std::env::temp_dir().join(format!( @@ -1330,6 +1350,25 @@ fn build_and_run_daemon_flags_parse() { .is_ok()); } +#[test] +fn source_runtime_mode_rejects_daemon_without_protocol_propagation() { + assert!(crate::cli::validate_runtime_mode_daemon_combination( + NativeRuntimeMode::SourceDevelopment, + true, + ) + .is_err()); + assert!(crate::cli::validate_runtime_mode_daemon_combination( + NativeRuntimeMode::SourceDevelopment, + false, + ) + .is_ok()); + assert!(crate::cli::validate_runtime_mode_daemon_combination( + NativeRuntimeMode::Installed, + true, + ) + .is_ok()); +} + #[test] fn reflection_flags_parse_for_build_and_run() { assert!(Cli::try_parse_from([ @@ -1859,6 +1898,7 @@ fn build_cache_schema_mismatch_forces_metadata_miss() { emit_llvm: false, runtime_c: Some("tools/stdlib/runtime.c".to_string()), runtime_c_fingerprint: Some(777), + runtime_provenance: NativeRuntimeProvenance::default(), llvm_ir_path: "tests/build/a.ll".to_string(), output_path: "tests/build/a.exe".to_string(), llvm_ir_hash: 777, @@ -1892,6 +1932,7 @@ fn build_cache_miss_when_runtime_source_fingerprint_changes() { emit_llvm: false, runtime_c: Some("tools/stdlib/runtime.c".to_string()), runtime_c_fingerprint: Some(11), + runtime_provenance: NativeRuntimeProvenance::default(), llvm_ir_path: "tests/build/a.ll".to_string(), output_path: "tests/build/a.exe".to_string(), llvm_ir_hash: 777, @@ -1934,6 +1975,7 @@ fn incremental_link_reuse_requires_matching_ir_hash() { emit_llvm: false, runtime_c: Some("tools/stdlib/runtime.c".to_string()), runtime_c_fingerprint: Some(777), + runtime_provenance: NativeRuntimeProvenance::default(), llvm_ir_path: "tests/build/main.ll".to_string(), output_path: "tests/build/main.exe".to_string(), llvm_ir_hash: 10, @@ -1987,6 +2029,7 @@ fn run_incremental_link_reuse_accepts_matching_metadata() { resolved_engine: RunEngine::Native, runtime_c: Some("tools/stdlib/runtime.c".to_string()), runtime_c_fingerprint: Some(777), + runtime_provenance: NativeRuntimeProvenance::default(), llvm_ir_path: "tests/build/main.ll".to_string(), executable_path: Some("tests/build/main.exe".to_string()), llvm_ir_hash: 44, @@ -7647,6 +7690,286 @@ def main() -> i64 { assert_eq!(String::from_utf8_lossy(&output.stderr), "err"); } +#[test] +fn stdlib_buffer_supports_checked_bytes_and_big_endian_u32() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "buffer-byte-be", + r#" +import std::ffi; + +def main() -> i64 { + let buffer = ffi_buffer_new(8).unwrap_or(Buffer { handle: 0 }); + let first = buffer.set_u8(0, 0).unwrap_or(false); + let last = buffer.set_u8(7, 255).unwrap_or(false); + let word = buffer.write_u32_be(2, 16909060).unwrap_or(false); + let read_first = buffer.get_u8(0).unwrap_or(-1); + let read_last = buffer.get_u8(7).unwrap_or(-1); + let read_word = buffer.read_u32_be(2).unwrap_or(-1); + let negative = buffer.get_u8(-1).is_err(); + let out_of_range = buffer.set_u8(8, 1).is_err(); + let invalid_byte = buffer.set_u8(0, 256).is_err(); + let short_word = buffer.read_u32_be(5).is_err(); + let overflow_word = buffer.write_u32_be(0, 4294967296).is_err(); + buffer.free(); + + if first + && last + && word + && read_first == 0 + && read_last == 255 + && read_word == 16909060 + && negative + && out_of_range + && invalid_byte + && short_word + && overflow_word { + 0 + } else { + 1 + } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_buffer_zeroes_gaps_before_exposing_extended_bytes() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "buffer-zero-extension-gaps", + r#" +import std::ffi; + +def main() -> i64 { + let byte_buffer = ffi_buffer_from_bytes("secret!!").unwrap_or(Buffer { handle: 0 }); + let byte_cleared = byte_buffer.clear(); + let byte_extended = byte_buffer.set_u8(7, 120).unwrap_or(false); + let byte_gap_zero = byte_buffer.get_u8(0).unwrap_or(-1) == 0 + && byte_buffer.get_u8(6).unwrap_or(-1) == 0; + let byte_tail = byte_buffer.get_u8(7).unwrap_or(-1) == 120; + + let word_buffer = ffi_buffer_from_bytes("private!").unwrap_or(Buffer { handle: 0 }); + let word_cleared = word_buffer.clear(); + let word_extended = word_buffer.write_u32_be(4, 16909060).unwrap_or(false); + let word_gap_zero = word_buffer.get_u8(0).unwrap_or(-1) == 0 + && word_buffer.get_u8(3).unwrap_or(-1) == 0; + let word_value = word_buffer.read_u32_be(4).unwrap_or(-1) == 16909060; + + byte_buffer.free(); + word_buffer.free(); + + if byte_cleared + && byte_extended + && byte_gap_zero + && byte_tail + && word_cleared + && word_extended + && word_gap_zero + && word_value { + 0 + } else { + 1 + } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_buffer_zero_handle_drop_is_noop() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "buffer-zero-handle-drop", + r#" +import std::ffi; + +def drop_empty_buffer() -> i64 { + let empty = Buffer { handle: 0 }; + 0 +} + +def main() -> i64 { + let cleared = ffi_last_error_clear(); + let dropped = drop_empty_buffer(); + if cleared && dropped == 0 && ffi_last_error_code() == 0 { 0 } else { 1 } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_buffer_zero_handle_free_is_noop() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "buffer-zero-handle-free", + r#" +import std::ffi; + +def main() -> i64 { + let cleared = ffi_last_error_clear(); + let empty = Buffer { handle: 0 }; + let freed = empty.free(); + if cleared && freed && ffi_last_error_code() == 0 { 0 } else { 1 } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_buffer_explicit_free_does_not_double_drop() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "buffer-explicit-free", + r#" +import std::ffi; + +def release_buffer() -> i64 { + let buffer = ffi_buffer_new(4).unwrap_or(Buffer { handle: 0 }); + let freed = buffer.free(); + if freed && ffi_last_error_code() == 0 { 0 } else { 1 } +} + +def main() -> i64 { + let cleared = ffi_last_error_clear(); + let released = release_buffer(); + if cleared && released == 0 && ffi_last_error_code() == 0 { 0 } else { 1 } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_io_exact_binary_frame_preserves_windows_control_bytes() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "io-exact-binary-frame", + r#" +import std::io; + +def main() -> i64 { + let buffer = ffi_buffer_new(6).unwrap_or(Buffer { handle: 0 }); + let binary = io_protocol_binary_mode().unwrap_or(false); + let read = io_stdin_read_exact(buffer, 0, 6).unwrap_or(-1); + let wrote = io_stdout_write_all(buffer, 0, 6).unwrap_or(-1); + let flushed = io_stdout_flush().unwrap_or(false); + buffer.free(); + + if binary && read == 6 && wrote == 6 && flushed { 0 } else { 1 } +} +"#, + "\0\r\n\u{1a}\u{e9}", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout: {:?}\nstderr:\n{}", + output.stdout, + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(output.stdout, b"\0\r\n\x1a\xc3\xa9"); +} + +#[test] +fn stdlib_io_exact_read_distinguishes_clean_eof_from_truncation() { + let source = r#" +import std::io; + +def main() -> i64 { + let buffer = ffi_buffer_new(4).unwrap_or(Buffer { handle: 0 }); + let binary = io_protocol_binary_mode().unwrap_or(false); + let read = io_stdin_read_exact(buffer, 0, 4); + let ok = read.is_ok && read.value == 0 && buffer.used_len() == 0; + buffer.free(); + if binary && ok { 0 } else { 1 } +} +"#; + + let Some(clean) = + compile_and_run_stdlib_import_program_with_stdin("io-exact-clean-eof", source, "") + else { + return; + }; + assert!( + clean.status.success(), + "clean EOF stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&clean.stdout), + String::from_utf8_lossy(&clean.stderr) + ); + + let truncated_source = r#" +import std::io; + +def main() -> i64 { + let buffer = ffi_buffer_new(4).unwrap_or(Buffer { handle: 0 }); + let binary = io_protocol_binary_mode().unwrap_or(false); + let read = io_stdin_read_exact(buffer, 0, 4); + let ok = read.is_err() && read.error == STATUS_IO() && buffer.used_len() == 0; + buffer.free(); + if binary && ok { 0 } else { 1 } +} +"#; + let Some(truncated) = compile_and_run_stdlib_import_program_with_stdin( + "io-exact-truncated", + truncated_source, + "abc", + ) else { + return; + }; + assert!( + truncated.status.success(), + "truncated stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&truncated.stdout), + String::from_utf8_lossy(&truncated.stderr) + ); +} + #[test] fn stdlib_dir_runtime_lists_entries_in_deterministic_order() { let Some(output) = compile_and_run_stdlib_import_program_with_stdin( @@ -8462,8 +8785,8 @@ def main() -> i64 {{ let _ = fs::remove_file(&exe_path); assert!( - elapsed < std::time::Duration::from_secs(2), - "wait_cancellable should not wait for the 5s child sleep; elapsed={elapsed:?}" + elapsed < std::time::Duration::from_secs(10), + "wait_cancellable process fixture exceeded its watchdog; elapsed={elapsed:?}" ); assert!( output.status.success(), @@ -9018,6 +9341,7 @@ def main() -> i64 { 1 } } + "#, "", ) else { @@ -9037,49 +9361,1117 @@ def main() -> i64 { } #[test] -fn stdlib_json_runtime_reports_parse_errors_and_limits() { - let too_deep = format!("{}0{}", "[".repeat(70), "]".repeat(70)); - let too_many_nodes = format!("[{}]", vec!["0"; 5000].join(",")); - let too_deep = too_deep.replace('\\', "\\\\").replace('"', "\\\""); - let too_many_nodes = too_many_nodes.replace('\\', "\\\\").replace('"', "\\\""); - - let source = format!( +fn stdlib_json_nested_containers_survive_node_storage_growth() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "json-node-storage-growth", r#" import std::json; + +def main() -> i64 { + let parsed = json_parse_strict("{\"early\":[],\"fill\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],\"late\":true}").unwrap_or(JsonDoc { handle: 0 }); + let root = parsed.root(); + let early = root.object_get("early"); + let fill = root.object_get("fill"); + let late = root.object_get("late"); + let ok = root.object_len().unwrap_or(-1) == 3 + && early.is_ok + && early.value.array_len().unwrap_or(-1) == 0 + && fill.is_ok + && fill.value.array_len().unwrap_or(-1) == 16 + && fill.value.array_get(0).unwrap_or(JsonValue { doc_handle: 0, node_id: 0 }).number_i64().unwrap_or(-1) == 0 + && fill.value.array_get(15).unwrap_or(JsonValue { doc_handle: 0, node_id: 0 }).number_i64().unwrap_or(-1) == 15 + && late.is_ok + && late.value.bool_value().unwrap_or(false); + if ok { 0 } else { 1 } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_json_strict_rejects_duplicates_and_decodes_unicode_keys() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "json-strict-object", + r#" import std::io; +import std::json; -def main() -> i64 {{ - let message = ffi_buffer_new(128).unwrap_or(Buffer {{ handle: 0 }}); - let invalid = json_parse("[1, ]").is_err(); - let invalid_code = json_last_error_code(); - let invalid_offset = json_last_error_offset(); - let invalid_message = json_last_error_copy(message).unwrap_or(0); - let deep = json_parse("{too_deep}").is_err(); - let deep_code = json_last_error_code(); - let deep_offset = json_last_error_offset(); - let too_many = json_parse("{too_many_nodes}").is_err(); - let too_many_code = json_last_error_code(); - let oversize_buf = ffi_buffer_new(2000000).unwrap_or(Buffer {{ handle: 0 }}); - let oversize_len = io_stdin_read(oversize_buf).unwrap_or(0); - let too_big = json_parse_buffer(oversize_buf, oversize_len).is_err(); - let too_big_code = json_last_error_code(); - oversize_buf.free(); - let empty_doc = JsonDoc {{ handle: 0 }}; - let empty_close = !empty_doc.close(); - message.free(); +def main() -> i64 { + let duplicate_input = ffi_buffer_from_bytes("{\"a\":1,\"a\":2}").unwrap_or(Buffer { handle: 0 }); + let escaped_duplicate_input = ffi_buffer_from_bytes("{\"a\":1,\"\\u0061\":2}").unwrap_or(Buffer { handle: 0 }); + let nested_duplicate_input = ffi_buffer_from_bytes("{\"outer\":{\"x\":1,\"x\":2}}").unwrap_or(Buffer { handle: 0 }); + let invalid_surrogate_input = ffi_buffer_from_bytes("{\"value\":\"\\ud83d\"}").unwrap_or(Buffer { handle: 0 }); + let valid_input = ffi_buffer_from_bytes("{\"name\":\"sengoo\",\"greeting\":\"\\u4f60\\u597d\\ud83d\\ude00\"}").unwrap_or(Buffer { handle: 0 }); + + let duplicate = json_parse_buffer_strict(duplicate_input, duplicate_input.len()); + let escaped_duplicate = json_parse_buffer_strict(escaped_duplicate_input, escaped_duplicate_input.len()); + let nested_duplicate = json_parse_buffer_strict(nested_duplicate_input, nested_duplicate_input.len()); + let invalid_surrogate = json_parse_buffer_strict(invalid_surrogate_input, invalid_surrogate_input.len()); + let valid = json_parse_buffer_strict(valid_input, valid_input.len()).unwrap_or(JsonDoc { handle: 0 }); + let root = valid.root(); + let key_buffer = ffi_buffer_new(32).unwrap_or(Buffer { handle: 0 }); + let value_buffer = ffi_buffer_new(32).unwrap_or(Buffer { handle: 0 }); + let count = root.object_len().unwrap_or(-1); + let key0 = root.object_key_copy(0, key_buffer).unwrap_or(-1); + let key1 = root.object_key_copy(1, key_buffer).unwrap_or(-1); + let bad_key = root.object_key_copy(2, key_buffer).is_err(); + let greeting = root.object_get("greeting").unwrap_or(JsonValue { doc_handle: 0, node_id: 0 }); + let greeting_len = greeting.string_copy(value_buffer).unwrap_or(-1); + let wrote = io_stdout_write_raw(value_buffer.ptr(), greeting_len).unwrap_or(-1); + + let ok = duplicate.is_err() + && escaped_duplicate.is_err() + && nested_duplicate.is_err() + && invalid_surrogate.is_err() + && count == 2 + && key0 == 4 + && key1 == 8 + && bad_key + && greeting_len == 10 + && wrote == greeting_len; + + key_buffer.free(); + value_buffer.free(); + valid.close(); + duplicate_input.free(); + escaped_duplicate_input.free(); + nested_duplicate_input.free(); + invalid_surrogate_input.free(); + valid_input.free(); - if !invalid {{ - 1 - }} else if invalid_code != 10 {{ - 2 - }} else if invalid_offset < 0 {{ - 3 - }} else if invalid_message <= 0 {{ - 4 - }} else if !deep {{ - 5 - }} else if deep_code != 10 {{ - 6 + if ok { 0 } else { 1 } +} + +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout: {:?}\nstderr:\n{}", + output.stdout, + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + output.stdout, + vec![0xe4, 0xbd, 0xa0, 0xe5, 0xa5, 0xbd, 0xf0, 0x9f, 0x98, 0x80] + ); +} + +#[test] +fn stdlib_json_strict_object_inspection_is_owned_and_exact() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "json-strict-object-inspection", + r#" +import std::json; +import std::status; + +def key_is_1(buffer: Buffer, b0: i64) -> bool { + buffer.used_len() == 1 + && buffer.get_u8(0).unwrap_or(-1) == b0 +} + +def key_is_2(buffer: Buffer, b0: i64, b1: i64) -> bool { + buffer.used_len() == 2 + && buffer.get_u8(0).unwrap_or(-1) == b0 + && buffer.get_u8(1).unwrap_or(-1) == b1 +} + +def key_is_3(buffer: Buffer, b0: i64, b1: i64, b2: i64) -> bool { + buffer.used_len() == 3 + && buffer.get_u8(0).unwrap_or(-1) == b0 + && buffer.get_u8(1).unwrap_or(-1) == b1 + && buffer.get_u8(2).unwrap_or(-1) == b2 +} + +def any_key_is_1(k0: Buffer, k1: Buffer, k2: Buffer, k3: Buffer, k4: Buffer, b0: i64) -> bool { + key_is_1(k0, b0) || key_is_1(k1, b0) || key_is_1(k2, b0) + || key_is_1(k3, b0) || key_is_1(k4, b0) +} + +def any_key_is_2(k0: Buffer, k1: Buffer, k2: Buffer, k3: Buffer, k4: Buffer, b0: i64, b1: i64) -> bool { + key_is_2(k0, b0, b1) || key_is_2(k1, b0, b1) || key_is_2(k2, b0, b1) + || key_is_2(k3, b0, b1) || key_is_2(k4, b0, b1) +} + +def any_key_is_3(k0: Buffer, k1: Buffer, k2: Buffer, k3: Buffer, k4: Buffer, b0: i64, b1: i64, b2: i64) -> bool { + key_is_3(k0, b0, b1, b2) || key_is_3(k1, b0, b1, b2) || key_is_3(k2, b0, b1, b2) + || key_is_3(k3, b0, b1, b2) || key_is_3(k4, b0, b1, b2) +} + +def main() -> i64 { + let input = ffi_buffer_from_bytes("{\"A\":1,\"a\":2,\"\\u00e9\":3,\"e\\u0301\":4,\"\\u4f60\":5}") + .unwrap_or(Buffer { handle: 0 }); + let doc = json_parse_buffer_strict(input, input.len()).unwrap_or(JsonDoc { handle: 0 }); + let root = doc.root(); + + let k0 = ffi_buffer_new(8).unwrap_or(Buffer { handle: 0 }); + let k1 = ffi_buffer_new(8).unwrap_or(Buffer { handle: 0 }); + let k2 = ffi_buffer_new(8).unwrap_or(Buffer { handle: 0 }); + let k3 = ffi_buffer_new(8).unwrap_or(Buffer { handle: 0 }); + let k4 = ffi_buffer_new(8).unwrap_or(Buffer { handle: 0 }); + + let count = root.object_len().unwrap_or(-1); + let copied0 = root.object_key_copy(0, k0).unwrap_or(-1); + let copied1 = root.object_key_copy(1, k1).unwrap_or(-1); + let copied2 = root.object_key_copy(2, k2).unwrap_or(-1); + let copied3 = root.object_key_copy(3, k3).unwrap_or(-1); + let copied4 = root.object_key_copy(4, k4).unwrap_or(-1); + + let negative = root.object_key_copy(-1, k0); + let negative_ok = negative.is_err() && negative.error == STATUS_NOT_FOUND(); + let at_len = root.object_key_copy(count, k0); + let at_len_ok = at_len.is_err() && at_len.error == STATUS_NOT_FOUND(); + + let upper = root.object_get("A"); + let lower = root.object_get("a"); + let precomposed = root.object_get("é"); + let decomposed = root.object_get("é"); + let escaped_by_raw_utf8 = root.object_get("你"); + let exact_lookup_ok = root.object_has("A") + && root.object_has("a") + && root.object_has("é") + && root.object_has("é") + && root.object_has("你") + && !root.object_has("E") + && !root.object_has("É") + && upper.is_ok && upper.value.number_i64().unwrap_or(-1) == 1 + && lower.is_ok && lower.value.number_i64().unwrap_or(-1) == 2 + && precomposed.is_ok && precomposed.value.number_i64().unwrap_or(-1) == 3 + && decomposed.is_ok && decomposed.value.number_i64().unwrap_or(-1) == 4 + && escaped_by_raw_utf8.is_ok && escaped_by_raw_utf8.value.number_i64().unwrap_or(-1) == 5; + + let saved = upper.value; + let closed = doc.close(); + let stale = saved.kind(); + let stale_ok = stale.is_err() && stale.error == STATUS_INVALID_HANDLE(); + + let copies_ok = count == 5 + && copied0 == k0.used_len() + && copied1 == k1.used_len() + && copied2 == k2.used_len() + && copied3 == k3.used_len() + && copied4 == k4.used_len() + && any_key_is_1(k0, k1, k2, k3, k4, 65) + && any_key_is_1(k0, k1, k2, k3, k4, 97) + && any_key_is_2(k0, k1, k2, k3, k4, 195, 169) + && any_key_is_3(k0, k1, k2, k3, k4, 101, 204, 129) + && any_key_is_3(k0, k1, k2, k3, k4, 228, 189, 160); + + k0.free(); + k1.free(); + k2.free(); + k3.free(); + k4.free(); + input.free(); + + if count != 5 { + 1 + } else if !negative_ok || !at_len_ok { + 2 + } else if !exact_lookup_ok { + 3 + } else if !closed || !stale_ok { + 4 + } else if !copies_ok { + 5 + } else { + 0 + } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "status: {:?}\nstdout:\n{}\nstderr:\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_json_strict_reports_stable_error_kinds() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "json-strict-error-kinds", + r#" +import std::json; + +def main() -> i64 { + let message = ffi_buffer_new(128).unwrap_or(Buffer { handle: 0 }); + + let duplicate = json_parse_strict("{\"a\":1,\"a\":2}"); + let duplicate_code = json_last_error_code(); + let duplicate_offset = json_last_error_offset(); + let duplicate_kind = json_last_error_kind(); + let duplicate_message_len = json_last_error_copy(message).unwrap_or(0); + + let escaped_duplicate = json_parse_strict("{\"a\":1,\"\\u0061\":2}"); + let escaped_duplicate_code = json_last_error_code(); + let escaped_duplicate_kind = json_last_error_kind(); + + let invalid_utf8 = ffi_buffer_new(3).unwrap_or(Buffer { handle: 0 }); + invalid_utf8.set_u8(0, 34); + invalid_utf8.set_u8(1, 255); + invalid_utf8.set_u8(2, 34); + let invalid_utf8_parse = json_parse_buffer_strict(invalid_utf8, invalid_utf8.used_len()); + let invalid_utf8_code = json_last_error_code(); + let invalid_utf8_kind = json_last_error_kind(); + + let invalid_escape = json_parse_strict("{\"value\":\"\\u12xz\"}"); + let invalid_escape_code = json_last_error_code(); + let invalid_escape_kind = json_last_error_kind(); + + let invalid_surrogate = json_parse_strict("{\"value\":\"\\ud83d\"}"); + let invalid_surrogate_code = json_last_error_code(); + let invalid_surrogate_kind = json_last_error_kind(); + + let trailing = json_parse_strict("{}x"); + let trailing_code = json_last_error_code(); + let trailing_offset = json_last_error_offset(); + let trailing_kind = json_last_error_kind(); + + let malformed = json_parse_strict("{bad}"); + let malformed_code = json_last_error_code(); + let malformed_kind = json_last_error_kind(); + + let valid = json_parse_strict("{}"); + let valid_code = json_last_error_code(); + let valid_kind = json_last_error_kind(); + let valid_closed = if valid.is_ok { valid.value.close() } else { false }; + + invalid_utf8.free(); + message.free(); + + let stable_constants = JSON_ERROR_KIND_NONE() == 0 + and JSON_ERROR_KIND_UNCLASSIFIED() == 1 + and JSON_ERROR_KIND_DUPLICATE_FIELD() == 2 + and JSON_ERROR_KIND_INVALID_UNICODE() == 3 + and JSON_ERROR_KIND_TRAILING_BYTES() == 4; + let legacy_diagnostics = duplicate_code == 10 + and duplicate_offset >= 0 + and duplicate_message_len > 0 + and escaped_duplicate_code == 10 + and invalid_utf8_code == 10 + and invalid_escape_code == 10 + and invalid_surrogate_code == 10 + and trailing_code == 10 + and trailing_offset >= 0 + and malformed_code == 10; + let classified = duplicate_kind == JSON_ERROR_KIND_DUPLICATE_FIELD() + and escaped_duplicate_kind == JSON_ERROR_KIND_DUPLICATE_FIELD() + and invalid_utf8_kind == JSON_ERROR_KIND_INVALID_UNICODE() + and invalid_escape_kind == JSON_ERROR_KIND_INVALID_UNICODE() + and invalid_surrogate_kind == JSON_ERROR_KIND_INVALID_UNICODE() + and trailing_kind == JSON_ERROR_KIND_TRAILING_BYTES() + and malformed_kind == JSON_ERROR_KIND_UNCLASSIFIED(); + let rejected = duplicate.is_err() + and escaped_duplicate.is_err() + and invalid_utf8_parse.is_err() + and invalid_escape.is_err() + and invalid_surrogate.is_err() + and trailing.is_err() + and malformed.is_err(); + let cleared = valid.is_ok + and valid_code == 0 + and valid_kind == JSON_ERROR_KIND_NONE() + and valid_closed; + + if stable_constants and legacy_diagnostics and classified and rejected and cleared { 0 } else { 1 } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_json_strict_rejects_out_of_range_integer_without_changing_permissive_parse() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "json-strict-integer-range", + r#" +import std::json; + +def main() -> i64 { + let input = ffi_buffer_from_bytes("{\"value\":9223372036854775808}").unwrap_or(Buffer { handle: 0 }); + let strict = json_parse_buffer_strict(input, input.len()); + let permissive = json_parse_buffer(input, input.len()); + let permissive_ok = permissive.is_ok; + let permissive_closed = if permissive.is_ok { + permissive.value.close(); + true + } else { + false + }; + input.free(); + if strict.is_err() && permissive_ok && permissive_closed { 0 } else { 1 } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_json_permissive_unicode_escape_behavior_remains_compatible() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "json-permissive-unicode-compat", + r#" +import std::io; +import std::json; + +def main() -> i64 { + let input = ffi_buffer_from_bytes("{\"bmp\":\"\\u4f60\",\"surrogate\":\"\\ud83d\"}") + .unwrap_or(Buffer { handle: 0 }); + let parsed = json_parse_buffer(input, input.len()).unwrap_or(JsonDoc { handle: 0 }); + let root = parsed.root(); + let output = ffi_buffer_new(2).unwrap_or(Buffer { handle: 0 }); + let bmp = root.object_get("bmp").unwrap_or(JsonValue { doc_handle: 0, node_id: 0 }); + let bmp_len = bmp.string_copy(output).unwrap_or(-1); + let bmp_compat = bmp_len == 1 && output.get_u8(0).unwrap_or(-1) == 63; + let surrogate = root.object_get("surrogate").unwrap_or(JsonValue { doc_handle: 0, node_id: 0 }); + let surrogate_len = surrogate.string_copy(output).unwrap_or(-1); + let surrogate_compat = surrogate_len == 1 && output.get_u8(0).unwrap_or(-1) == 63; + + output.free(); + parsed.close(); + input.free(); + if bmp_compat && surrogate_compat { 0 } else { 1 } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_json_strict_preserves_escaped_null_as_string_data() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "json-strict-escaped-null", + r#" +import std::json; + +def main() -> i64 { + let input = ffi_buffer_from_bytes("{\"value\":\"a\\u0000b\"}").unwrap_or(Buffer { handle: 0 }); + let parsed = json_parse_buffer_strict(input, input.len()).unwrap_or(JsonDoc { handle: 0 }); + let value = parsed.root().object_get("value").unwrap_or(JsonValue { doc_handle: 0, node_id: 0 }); + let output = ffi_buffer_new(3).unwrap_or(Buffer { handle: 0 }); + let copied = value.string_copy(output).unwrap_or(-1); + let ok = copied == 3 + && output.get_u8(0).unwrap_or(-1) == 97 + && output.get_u8(1).unwrap_or(-1) == 0 + && output.get_u8(2).unwrap_or(-1) == 98; + output.free(); + parsed.close(); + input.free(); + if ok { 0 } else { 1 } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_json_length_aware_builder_preserves_embedded_nul() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "json-length-aware-builder", + r#" +import std::json; + +def main() -> i64 { + let input = ffi_buffer_from_bytes("{\"value\":\"a\\u0000b\"}").unwrap_or(Buffer { handle: 0 }); + let parsed = json_parse_buffer_strict(input, input.len()).unwrap_or(JsonDoc { handle: 0 }); + let source_value = parsed.root().object_get("value").unwrap_or(JsonValue { doc_handle: 0, node_id: 0 }); + let source_text = source_value.string_value().unwrap_or(String { handle: 0 }); + + let built = json_doc_object().unwrap_or(JsonDoc { handle: 0 }); + let copied_value = built.new_string_from_string(&source_text).unwrap_or(JsonValue { doc_handle: 0, node_id: 0 }); + let set_value = built.root().object_set("value", copied_value).unwrap_or(false); + let encoded = ffi_buffer_new(64).unwrap_or(Buffer { handle: 0 }); + let encoded_len = built.serialize(encoded).unwrap_or(-1); + + let reparsed = json_parse_buffer_strict(encoded, encoded_len).unwrap_or(JsonDoc { handle: 0 }); + let reparsed_value = reparsed.root().object_get("value").unwrap_or(JsonValue { doc_handle: 0, node_id: 0 }); + let bytes = ffi_buffer_new(3).unwrap_or(Buffer { handle: 0 }); + let copied_len = reparsed_value.string_copy(bytes).unwrap_or(-1); + let preserved = copied_len == 3 + && bytes.get_u8(0).unwrap_or(-1) == 97 + && bytes.get_u8(1).unwrap_or(-1) == 0 + && bytes.get_u8(2).unwrap_or(-1) == 98; + + bytes.free(); + reparsed.close(); + encoded.free(); + built.close(); + parsed.close(); + input.free(); + if set_value && encoded_len > 0 && preserved { 0 } else { 1 } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_json_length_aware_builder_rejects_invalid_utf8() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "json-length-aware-builder-invalid-utf8", + r#" +import std::json; + +def main() -> i64 { + let bytes = ffi_buffer_new(1).unwrap_or(Buffer { handle: 0 }); + let initialized = bytes.set_u8(0, 255).unwrap_or(false); + let built = json_doc_object().unwrap_or(JsonDoc { handle: 0 }); + let node_id = sengoo_json_doc_new_string_len(built.handle, bytes.ptr(), 1); + let code = json_last_error_code(); + let kind = json_last_error_kind(); + + built.close(); + bytes.free(); + if initialized + && node_id == 0 + && code == 2 + && kind == JSON_ERROR_KIND_INVALID_UNICODE() { 0 } else { 1 } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_json_owned_string_builder_preserves_invalid_handle_status() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "json-owned-string-builder-invalid-handle", + r#" +import std::json; + +def main() -> i64 { + let built = json_doc_object().unwrap_or(JsonDoc { handle: 0 }); + let invalid = String { handle: 0 }; + let rejected = built.new_string_from_string(&invalid); + let code = json_last_error_code(); + let kind = json_last_error_kind(); + + built.close(); + if rejected.is_err() + && rejected.error == 3 + && code == 3 + && kind == JSON_ERROR_KIND_UNCLASSIFIED() { 0 } else { 1 } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_json_strict_exact_lookup_supports_escaped_null_keys() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "json-strict-null-key-lookup", + r#" +import std::json; + +def main() -> i64 { + let input = ffi_buffer_from_bytes("{\"a\\u0000b\":7}").unwrap_or(Buffer { handle: 0 }); + let parsed = json_parse_buffer_strict(input, input.len()).unwrap_or(JsonDoc { handle: 0 }); + let root = parsed.root(); + let key = ffi_buffer_new(3).unwrap_or(Buffer { handle: 0 }); + key.set_u8(0, 97); + key.set_u8(1, 0); + key.set_u8(2, 98); + let has = sengoo_json_object_has_len(root.doc_handle, root.node_id, key.ptr(), 3) != 0; + let node_id = sengoo_json_object_get_len(root.doc_handle, root.node_id, key.ptr(), 3); + let value = JsonValue { doc_handle: root.doc_handle, node_id: node_id }; + let exact = value.number_i64().unwrap_or(-1) == 7; + key.free(); + parsed.close(); + input.free(); + if has && node_id > 0 && exact { 0 } else { 1 } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} +#[test] +fn stdlib_json_document_handles_reject_forged_and_reused_stale_values() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "json-document-handle-lifecycle", + r#" +import std::json; + +def main() -> i64 { + let forged_rejected = sengoo_json_doc_close(1) != 0; + let first = json_parse("{}").unwrap_or(JsonDoc { handle: 0 }); + let stale = first.handle; + let first_close = sengoo_json_doc_close(stale) == 0; + let idempotent_close = sengoo_json_doc_close(stale) == 0; + let second = json_parse("{}").unwrap_or(JsonDoc { handle: 0 }); + let generation_changed = second.handle != stale; + let stale_rejected_after_reuse = sengoo_json_doc_close(stale) != 0; + let second_close = second.close(); + + if forged_rejected + && first_close + && idempotent_close + && generation_changed + && stale_rejected_after_reuse + && second_close { + 0 + } else { + 1 + } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "status: {:?}\nstdout:\n{}\nstderr:\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_json_document_parse_close_cycles_restore_live_handle_count() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "json-document-live-handle-count", + r#" +import std::json; + +extern "C" { + fn sengoo_json_doc_live_handle_count() -> i64; +} + +def main() -> i64 { + let before = sengoo_json_doc_live_handle_count(); + let mut round = 0; + let mut counted_each_round = true; + let mut closed_each_round = true; + let mut restored_each_round = true; + + while round < 64 { + let permissive = json_parse("{\"round\":1}").unwrap_or(JsonDoc { handle: 0 }); + let strict = json_parse_strict("{\"round\":2}").unwrap_or(JsonDoc { handle: 0 }); + let permissive_handle = permissive.handle; + let strict_handle = strict.handle; + counted_each_round = counted_each_round + && permissive_handle != 0 + && strict_handle != 0 + && sengoo_json_doc_live_handle_count() == before + 2; + + let permissive_closed = permissive.close(); + let strict_closed = strict.close(); + closed_each_round = closed_each_round && permissive_closed && strict_closed; + restored_each_round = restored_each_round + && sengoo_json_doc_live_handle_count() == before; + round = round + 1; + } + + let after = sengoo_json_doc_live_handle_count(); + if !counted_each_round { + 1 + } else if !closed_each_round { + 2 + } else if !restored_each_round { + 3 + } else if after != before { + 4 + } else { + 0 + } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "status: {:?}\nstdout:\n{}\nstderr:\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_json_strict_rejects_invalid_raw_utf8() { + let source = r#" +import std::io; +import std::json; + +def main() -> i64 { + let input = ffi_buffer_new(32).unwrap_or(Buffer { handle: 0 }); + let binary = io_protocol_binary_mode().unwrap_or(false); + let input_len = io_stdin_read_exact(input, 0, 14).unwrap_or(-1); + let rejected = json_parse_buffer_strict(input, input_len).is_err(); + let parse_error = json_last_error_code() == 10; + input.free(); + if binary && input_len == 14 && rejected && parse_error { 0 } else { 1 } +} +"#; + let Some(output) = compile_and_run_stdlib_import_program_with_stdin_bytes( + "json-strict-invalid-utf8", + source, + b"{\"value\":\"\xc0\xaf\"}", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_json_strict_rejects_invalid_unicode_and_preserves_utf8() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "json-strict-unicode-matrix", + r#" +import std::json; + +def rejects_raw_utf8(b0: i64, b1: i64, b2: i64, b3: i64, byte_count: i64) -> bool { + let input = ffi_buffer_new(6).unwrap_or(Buffer { handle: 0 }); + input.set_u8(0, 34); + input.set_u8(1, b0); + input.set_u8(2, b1); + input.set_u8(3, b2); + input.set_u8(4, b3); + input.set_u8(5, 34); + input.set_u8(byte_count + 1, 34); + + let input_len = byte_count + 2; + let parsed = json_parse_buffer_strict(input, input_len); + let code = json_last_error_code(); + let kind = json_last_error_kind(); + input.free(); + parsed.is_err() + && code == 10 + && kind == JSON_ERROR_KIND_INVALID_UNICODE() +} + +def rejects_unicode_escape(text: &str) -> bool { + let parsed = json_parse_strict(text); + parsed.is_err() + && json_last_error_code() == 10 + && json_last_error_kind() == JSON_ERROR_KIND_INVALID_UNICODE() +} + +def main() -> i64 { + let invalid_utf8 = rejects_raw_utf8(128, 0, 0, 0, 1) + && rejects_raw_utf8(226, 130, 0, 0, 2) + && rejects_raw_utf8(192, 175, 0, 0, 2) + && rejects_raw_utf8(237, 160, 128, 0, 3) + && rejects_raw_utf8(244, 144, 128, 128, 4); + + let invalid_escape = json_parse_strict("{\"value\":\"\\q\"}"); + let invalid_escape_ok = invalid_escape.is_err() + && json_last_error_code() == 10 + && json_last_error_kind() == JSON_ERROR_KIND_UNCLASSIFIED(); + + let short_unicode_input = ffi_buffer_new(5).unwrap_or(Buffer { handle: 0 }); + short_unicode_input.set_u8(0, 34); + short_unicode_input.set_u8(1, 92); + short_unicode_input.set_u8(2, 117); + short_unicode_input.set_u8(3, 49); + short_unicode_input.set_u8(4, 50); + let short_unicode = json_parse_buffer_strict(short_unicode_input, short_unicode_input.used_len()); + let short_unicode_ok = short_unicode.is_err() + && json_last_error_code() == 10 + && json_last_error_kind() == JSON_ERROR_KIND_INVALID_UNICODE(); + short_unicode_input.free(); + + let raw_control_input = ffi_buffer_new(3).unwrap_or(Buffer { handle: 0 }); + raw_control_input.set_u8(0, 34); + raw_control_input.set_u8(1, 0); + raw_control_input.set_u8(2, 34); + let raw_control = json_parse_buffer_strict(raw_control_input, raw_control_input.used_len()); + let raw_control_ok = raw_control.is_err() + && json_last_error_code() == 10 + && json_last_error_kind() == JSON_ERROR_KIND_UNCLASSIFIED(); + raw_control_input.free(); + + let surrogates = rejects_unicode_escape("\"\\ud83d\"") + && rejects_unicode_escape("\"\\udc00\"") + && rejects_unicode_escape("\"\\udc00\\ud800\""); + + let raw_valid_input = ffi_buffer_new(12).unwrap_or(Buffer { handle: 0 }); + raw_valid_input.set_u8(0, 34); + raw_valid_input.set_u8(1, 228); + raw_valid_input.set_u8(2, 189); + raw_valid_input.set_u8(3, 160); + raw_valid_input.set_u8(4, 229); + raw_valid_input.set_u8(5, 165); + raw_valid_input.set_u8(6, 189); + raw_valid_input.set_u8(7, 240); + raw_valid_input.set_u8(8, 159); + raw_valid_input.set_u8(9, 152); + raw_valid_input.set_u8(10, 128); + raw_valid_input.set_u8(11, 34); + let raw_valid = json_parse_buffer_strict(raw_valid_input, raw_valid_input.used_len()) + .unwrap_or(JsonDoc { handle: 0 }); + let raw_valid_bytes = ffi_buffer_new(10).unwrap_or(Buffer { handle: 0 }); + let raw_valid_len = raw_valid.root().string_copy(raw_valid_bytes).unwrap_or(-1); + let raw_valid_ok = raw_valid_len == 10 + && raw_valid_bytes.get_u8(0).unwrap_or(-1) == 228 + && raw_valid_bytes.get_u8(1).unwrap_or(-1) == 189 + && raw_valid_bytes.get_u8(2).unwrap_or(-1) == 160 + && raw_valid_bytes.get_u8(3).unwrap_or(-1) == 229 + && raw_valid_bytes.get_u8(4).unwrap_or(-1) == 165 + && raw_valid_bytes.get_u8(5).unwrap_or(-1) == 189 + && raw_valid_bytes.get_u8(6).unwrap_or(-1) == 240 + && raw_valid_bytes.get_u8(7).unwrap_or(-1) == 159 + && raw_valid_bytes.get_u8(8).unwrap_or(-1) == 152 + && raw_valid_bytes.get_u8(9).unwrap_or(-1) == 128; + raw_valid_bytes.free(); + raw_valid.close(); + raw_valid_input.free(); + + let escaped = json_parse_strict("[\"\\u4f60\\u597d\",\"\\ud83d\\ude00\"]") + .unwrap_or(JsonDoc { handle: 0 }); + let encoded = ffi_buffer_new(64).unwrap_or(Buffer { handle: 0 }); + let encoded_len = escaped.serialize(encoded).unwrap_or(-1); + let reparsed = json_parse_buffer_strict(encoded, encoded_len) + .unwrap_or(JsonDoc { handle: 0 }); + let bmp = reparsed.root().array_get(0).unwrap_or(JsonValue { doc_handle: 0, node_id: 0 }); + let supplementary = reparsed.root().array_get(1).unwrap_or(JsonValue { doc_handle: 0, node_id: 0 }); + let bmp_bytes = ffi_buffer_new(6).unwrap_or(Buffer { handle: 0 }); + let supplementary_bytes = ffi_buffer_new(4).unwrap_or(Buffer { handle: 0 }); + let bmp_len = bmp.string_copy(bmp_bytes).unwrap_or(-1); + let supplementary_len = supplementary.string_copy(supplementary_bytes).unwrap_or(-1); + let roundtrip_ok = encoded_len > 0 + && bmp_len == 6 + && bmp_bytes.get_u8(0).unwrap_or(-1) == 228 + && bmp_bytes.get_u8(1).unwrap_or(-1) == 189 + && bmp_bytes.get_u8(2).unwrap_or(-1) == 160 + && bmp_bytes.get_u8(3).unwrap_or(-1) == 229 + && bmp_bytes.get_u8(4).unwrap_or(-1) == 165 + && bmp_bytes.get_u8(5).unwrap_or(-1) == 189 + && supplementary_len == 4 + && supplementary_bytes.get_u8(0).unwrap_or(-1) == 240 + && supplementary_bytes.get_u8(1).unwrap_or(-1) == 159 + && supplementary_bytes.get_u8(2).unwrap_or(-1) == 152 + && supplementary_bytes.get_u8(3).unwrap_or(-1) == 128; + bmp_bytes.free(); + supplementary_bytes.free(); + reparsed.close(); + encoded.free(); + escaped.close(); + + if !invalid_utf8 { + 1 + } else if !invalid_escape_ok || !short_unicode_ok { + 2 + } else if !raw_control_ok { + 3 + } else if !surrogates { + 4 + } else if !raw_valid_ok { + 5 + } else if !roundtrip_ok { + 6 + } else { + 0 + } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "status: {:?}\nstdout:\n{}\nstderr:\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_json_strict_enforces_malformed_input_boundaries() { + let too_deep = format!("{}0{}", "[".repeat(70), "]".repeat(70)); + let too_deep = too_deep.replace('\\', "\\\\").replace('"', "\\\""); + let source = format!( + r#" +import std::json; + +def main() -> i64 {{ + let trailing = json_parse_strict("{{}}x").is_err(); + let reversed_surrogate = json_parse_strict("{{\"value\":\"\\udc00\\ud800\"}}").is_err(); + let raw_control = json_parse_strict("{{\"value\":\"line +break\"}}").is_err(); + let integer_underflow = json_parse_strict("-9223372036854775809").is_err(); + let excess_depth = json_parse_strict("{too_deep}").is_err(); + + let exact = ffi_buffer_from_bytes("{{}}junk").unwrap_or(Buffer {{ handle: 0 }}); + let exact_prefix = json_parse_buffer_strict(exact, 2); + let exact_prefix_ok = exact_prefix.is_ok; + let exact_prefix_closed = if exact_prefix.is_ok {{ + exact_prefix.value.close() + }} else {{ + false + }}; + let full_rejected = json_parse_buffer_strict(exact, exact.used_len()).is_err(); + exact.free(); + + let short = ffi_buffer_from_bytes("{{}}").unwrap_or(Buffer {{ handle: 0 }}); + let uninitialized_rejected = json_parse_buffer_strict(short, 3).is_err(); + short.free(); + + if trailing + && reversed_surrogate + && raw_control + && integer_underflow + && excess_depth + && exact_prefix_ok + && exact_prefix_closed + && full_rejected + && uninitialized_rejected {{ + 0 + }} else {{ + 1 + }} +}} +"# + ); + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "json-strict-malformed-boundaries", + &source, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_json_strict_rejects_truncated_documents_within_initialized_buffers() { + let Some(output) = compile_and_run_stdlib_import_program_with_stdin( + "json-strict-truncated-documents", + r#" +import std::json; +import std::status; + +extern "C" { + fn sengoo_json_doc_live_handle_count() -> i64; +} + +def strict_prefix_is_rejected(input: Buffer, input_len: i64, expected_kind: i64) -> bool { + let before = sengoo_json_doc_live_handle_count(); + let truncated = json_parse_buffer_strict(input, input_len); + let result_status = truncated.error; + let last_status = json_last_error_code(); + let error_kind = json_last_error_kind(); + let error_offset = json_last_error_offset(); + let after_rejection = sengoo_json_doc_live_handle_count(); + + let complete = json_parse_buffer_strict(input, input.used_len()); + let complete_ok = complete.is_ok; + let complete_closed = if complete.is_ok { complete.value.close() } else { false }; + let after_complete = sengoo_json_doc_live_handle_count(); + + input_len >= 0 + && input_len < input.used_len() + && truncated.is_err() + && result_status == STATUS_PARSE() + && last_status == STATUS_PARSE() + && error_kind == expected_kind + && error_offset >= 0 + && error_offset <= input_len + && after_rejection == before + && complete_ok + && complete_closed + && after_complete == before +} + +def main() -> i64 { + let string_input = ffi_buffer_from_bytes("\"ok\"").unwrap_or(Buffer { handle: 0 }); + let array_input = ffi_buffer_from_bytes("[1,2]").unwrap_or(Buffer { handle: 0 }); + let object_input = ffi_buffer_from_bytes("{\"a\":0}").unwrap_or(Buffer { handle: 0 }); + let literal_input = ffi_buffer_from_bytes("true").unwrap_or(Buffer { handle: 0 }); + let exponent_input = ffi_buffer_from_bytes("1e2").unwrap_or(Buffer { handle: 0 }); + let escape_input = ffi_buffer_from_bytes("\"a\\n\"").unwrap_or(Buffer { handle: 0 }); + let unicode_input = ffi_buffer_from_bytes("\"\\u1234\"").unwrap_or(Buffer { handle: 0 }); + + let rejected = strict_prefix_is_rejected(string_input, 1, JSON_ERROR_KIND_UNCLASSIFIED()) + && strict_prefix_is_rejected(array_input, 3, JSON_ERROR_KIND_UNCLASSIFIED()) + && strict_prefix_is_rejected(object_input, 5, JSON_ERROR_KIND_UNCLASSIFIED()) + && strict_prefix_is_rejected(literal_input, 3, JSON_ERROR_KIND_UNCLASSIFIED()) + && strict_prefix_is_rejected(exponent_input, 2, JSON_ERROR_KIND_UNCLASSIFIED()) + && strict_prefix_is_rejected(escape_input, 3, JSON_ERROR_KIND_UNCLASSIFIED()) + && strict_prefix_is_rejected(unicode_input, 5, JSON_ERROR_KIND_INVALID_UNICODE()); + + string_input.free(); + array_input.free(); + object_input.free(); + literal_input.free(); + exponent_input.free(); + escape_input.free(); + unicode_input.free(); + + if rejected { 0 } else { 1 } +} +"#, + "", + ) else { + return; + }; + + assert!( + output.status.success(), + "status: {:?}\nstdout:\n{}\nstderr:\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stdlib_json_runtime_reports_parse_errors_and_limits() { + let too_deep = format!("{}0{}", "[".repeat(70), "]".repeat(70)); + let too_many_nodes = format!("[{}]", vec!["0"; 5000].join(",")); + let too_deep = too_deep.replace('\\', "\\\\").replace('"', "\\\""); + let too_many_nodes = too_many_nodes.replace('\\', "\\\\").replace('"', "\\\""); + + let source = format!( + r#" +import std::json; +import std::io; + +def main() -> i64 {{ + let message = ffi_buffer_new(128).unwrap_or(Buffer {{ handle: 0 }}); + let invalid = json_parse("[1, ]").is_err(); + let invalid_code = json_last_error_code(); + let invalid_offset = json_last_error_offset(); + let invalid_message = json_last_error_copy(message).unwrap_or(0); + let deep = json_parse("{too_deep}").is_err(); + let deep_code = json_last_error_code(); + let deep_offset = json_last_error_offset(); + let too_many = json_parse("{too_many_nodes}").is_err(); + let too_many_code = json_last_error_code(); + let oversize_buf = ffi_buffer_new(2000000).unwrap_or(Buffer {{ handle: 0 }}); + let oversize_len = io_stdin_read(oversize_buf).unwrap_or(0); + let too_big = json_parse_buffer(oversize_buf, oversize_len).is_err(); + let too_big_code = json_last_error_code(); + oversize_buf.free(); + let empty_doc = JsonDoc {{ handle: 0 }}; + let empty_close = !empty_doc.close(); + message.free(); + + if !invalid {{ + 1 + }} else if invalid_code != 10 {{ + 2 + }} else if invalid_offset < 0 {{ + 3 + }} else if invalid_message <= 0 {{ + 4 + }} else if !deep {{ + 5 + }} else if deep_code != 10 {{ + 6 }} else if deep_offset < 0 {{ 7 }} else if !too_many {{ @@ -9388,6 +10780,14 @@ fn compile_and_run_stdlib_import_program_with_stdin( tag: &str, source: &str, stdin: &str, +) -> Option { + compile_and_run_stdlib_import_program_with_stdin_bytes(tag, source, stdin.as_bytes()) +} + +fn compile_and_run_stdlib_import_program_with_stdin_bytes( + tag: &str, + source: &str, + stdin: &[u8], ) -> Option { let source = expand_stdlib_imports_for_source(source) .unwrap_or_else(|err| panic!("stdlib imports should expand: {err}")); @@ -9422,9 +10822,7 @@ fn compile_and_run_stdlib_import_program_with_stdin( .spawn() .expect("stdlib binary should spawn"); if let Some(mut input) = child.stdin.take() { - input - .write_all(stdin.as_bytes()) - .expect("stdin should be writable"); + input.write_all(stdin).expect("stdin should be writable"); } let output = child .wait_with_output() @@ -13284,6 +14682,7 @@ fn workset_plan_reuses_previous_artifacts_when_impl_only_does_not_touch_root() { emit_llvm: false, runtime_c: Some("tools/stdlib/runtime.c".to_string()), runtime_c_fingerprint: Some(777), + runtime_provenance: NativeRuntimeProvenance::default(), llvm_ir_path: "tests/build/main.ll".to_string(), output_path: "tests/build/main.exe".to_string(), llvm_ir_hash: 33, @@ -13326,6 +14725,7 @@ fn workset_plan_rebuilds_root_when_impl_only_touches_root() { emit_llvm: false, runtime_c: Some("tools/stdlib/runtime.c".to_string()), runtime_c_fingerprint: Some(777), + runtime_provenance: NativeRuntimeProvenance::default(), llvm_ir_path: "tests/build/main.ll".to_string(), output_path: "tests/build/main.exe".to_string(), llvm_ir_hash: 33, @@ -13845,6 +15245,7 @@ fn run_workset_plan_reuses_previous_artifacts_when_impl_only_does_not_touch_root resolved_engine: RunEngine::Native, runtime_c: Some("tools/stdlib/runtime.c".to_string()), runtime_c_fingerprint: Some(777), + runtime_provenance: NativeRuntimeProvenance::default(), llvm_ir_path: "tests/build/main.ll".to_string(), executable_path: Some("tests/build/main.exe".to_string()), llvm_ir_hash: 33, @@ -13887,6 +15288,7 @@ fn run_workset_plan_full_rebuild_when_engine_changes() { resolved_engine: RunEngine::Native, runtime_c: Some("tools/stdlib/runtime.c".to_string()), runtime_c_fingerprint: Some(777), + runtime_provenance: NativeRuntimeProvenance::default(), llvm_ir_path: "tests/build/main.ll".to_string(), executable_path: Some("tests/build/main.exe".to_string()), llvm_ir_hash: 33, diff --git a/tools/sgc/src/workset.rs b/tools/sgc/src/workset.rs index ac0aee85..9aff2983 100644 --- a/tools/sgc/src/workset.rs +++ b/tools/sgc/src/workset.rs @@ -224,6 +224,7 @@ pub(crate) fn cache_key( resolved_engine, runtime_c: runtime_c.path, runtime_c_fingerprint: runtime_c.fingerprint, + runtime_provenance: runtime_c.provenance, } } @@ -247,6 +248,7 @@ pub(crate) fn build_cache_key( emit_llvm, runtime_c: runtime_c.path, runtime_c_fingerprint: runtime_c.fingerprint, + runtime_provenance: runtime_c.provenance, output_path, } } @@ -261,6 +263,7 @@ pub(crate) fn metadata_matches(metadata: &RunCacheMetadata, key: &RunCacheKey) - && metadata.resolved_engine == key.resolved_engine && metadata.runtime_c == key.runtime_c && metadata.runtime_c_fingerprint == key.runtime_c_fingerprint + && metadata.runtime_provenance == key.runtime_provenance } pub(crate) fn build_metadata_matches(metadata: &BuildCacheMetadata, key: &BuildCacheKey) -> bool { @@ -273,6 +276,7 @@ pub(crate) fn build_metadata_matches(metadata: &BuildCacheMetadata, key: &BuildC && metadata.emit_llvm == key.emit_llvm && metadata.runtime_c == key.runtime_c && metadata.runtime_c_fingerprint == key.runtime_c_fingerprint + && metadata.runtime_provenance == key.runtime_provenance && metadata.output_path == key.output_path } @@ -337,6 +341,9 @@ pub(crate) fn build_cache_mismatch_reasons( if metadata.runtime_c_fingerprint != key.runtime_c_fingerprint { reasons.push("runtime source changed".to_string()); } + if metadata.runtime_provenance != key.runtime_provenance { + reasons.push("runtime provenance changed".to_string()); + } if metadata.output_path != key.output_path { reasons.push("output path changed".to_string()); } @@ -715,6 +722,9 @@ pub(crate) fn cache_mismatch_reasons( if metadata.runtime_c_fingerprint != key.runtime_c_fingerprint { reasons.push("runtime source changed".to_string()); } + if metadata.runtime_provenance != key.runtime_provenance { + reasons.push("runtime provenance changed".to_string()); + } if reasons.is_empty() { reasons.push("cache metadata mismatch".to_string()); diff --git a/tools/sgc/tests/assertion_transport.rs b/tools/sgc/tests/assertion_transport.rs index 1547c77f..de91e81b 100644 --- a/tools/sgc/tests/assertion_transport.rs +++ b/tools/sgc/tests/assertion_transport.rs @@ -1,13 +1,11 @@ +mod common; + +use common::source_sgc_command; use serde_json::Value; use std::fs; use std::path::{Path, PathBuf}; -use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; -fn sgc() -> PathBuf { - PathBuf::from(env!("CARGO_BIN_EXE_sgc")) -} - fn temp_dir(name: &str) -> PathBuf { let stamp = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -48,7 +46,7 @@ fn sgc_test_reports_structured_assertion_failure_in_json_mode() { let root = temp_dir("json_assertion"); write_failing_assert_test(&root); - let output = Command::new(sgc()) + let output = source_sgc_command() .current_dir(&root) .args(["test", "--format", "json"]) .output() @@ -92,7 +90,7 @@ fn sgc_test_reports_assertion_message_in_text_mode_with_nocapture() { let root = temp_dir("text_assertion"); write_failing_assert_test(&root); - let output = Command::new(sgc()) + let output = source_sgc_command() .current_dir(&root) .args(["test", "--nocapture"]) .output() diff --git a/tools/sgc/tests/binary_io_exact_read.rs b/tools/sgc/tests/binary_io_exact_read.rs new file mode 100644 index 00000000..e4ae2c8d --- /dev/null +++ b/tools/sgc/tests/binary_io_exact_read.rs @@ -0,0 +1,273 @@ +use std::fs; +use std::path::PathBuf; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +fn temp_dir(tag: &str) -> PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be after unix epoch") + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "sengoo-binary-io-{tag}-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&root).expect("create binary I/O test directory"); + root +} + +fn stdlib_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("stdlib") +} + +fn assert_success(label: &str, output: &std::process::Output) { + assert!( + output.status.success(), + "{label} failed with {:?}\nstdout:\n{}\nstderr:\n{}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn native_exact_read_handles_partial_progress_and_every_failure_boundary_atomically() { + let clang = which::which("clang") + .or_else(|_| which::which("clang.exe")) + .expect("native exact-read runtime probe requires clang"); + let root = temp_dir("exact-read"); + let source = root.join("probe.c"); + let executable = root.join(if cfg!(windows) { "probe.exe" } else { "probe" }); + fs::write( + &source, + r#" +#include "runtime_shared.h" +#include +#include +#include + +typedef struct { + size_t count; + int eof; + int error; +} ScriptStep; + +typedef struct { + const unsigned char* input; + size_t input_len; + size_t input_offset; + const ScriptStep* steps; + size_t step_count; + size_t step_index; + int invalid_script; +} ScriptedReader; + +static SengooRuntimeReadResult scripted_read( + void* context, + unsigned char* destination, + size_t capacity) { + ScriptedReader* reader = (ScriptedReader*)context; + SengooRuntimeReadResult result = {0, 0, 0}; + if (reader->step_index >= reader->step_count) { + reader->invalid_script = 1; + return result; + } + + ScriptStep step = reader->steps[reader->step_index++]; + if (step.count > capacity || step.count > reader->input_len - reader->input_offset) { + reader->invalid_script = 1; + result.error = 1; + return result; + } + if (step.count > 0) { + memcpy(destination, reader->input + reader->input_offset, step.count); + reader->input_offset += step.count; + } + result.count = step.count; + result.eof = step.eof; + result.error = step.error; + return result; +} + +static void fill_pattern(unsigned char* bytes, size_t len) { + for (size_t index = 0; index < len; ++index) { + bytes[index] = (unsigned char)((index * 37U + 11U) & 0xffU); + } +} + +static int bytes_are(const unsigned char* bytes, size_t len, unsigned char value) { + for (size_t index = 0; index < len; ++index) { + if (bytes[index] != value) return 0; + } + return 1; +} + +static int expect_success( + const unsigned char* input, + size_t len, + const size_t* chunks, + size_t chunk_count) { + ScriptStep steps[64]; + unsigned char destination[66]; + if (chunk_count > 64 || len > 64) return 0; + for (size_t index = 0; index < chunk_count; ++index) { + steps[index].count = chunks[index]; + steps[index].eof = 0; + steps[index].error = 0; + } + memset(destination, 0xa5, sizeof(destination)); + ScriptedReader reader = { + input, len, 0, steps, chunk_count, 0, 0 + }; + + long long status = sengoo_runtime_read_exact( + scripted_read, &reader, destination + 1, len); + return status == (long long)len + && reader.invalid_script == 0 + && reader.step_index == chunk_count + && reader.input_offset == len + && destination[0] == 0xa5 + && destination[len + 1] == 0xa5 + && memcmp(destination + 1, input, len) == 0; +} + +static int expect_failure_unchanged( + const unsigned char* input, + size_t input_len, + size_t expected_len, + const ScriptStep* steps, + size_t step_count, + long long expected_status) { + unsigned char destination[66]; + if (expected_len > 64) return 0; + memset(destination, 0xa5, sizeof(destination)); + ScriptedReader reader = { + input, input_len, 0, steps, step_count, 0, 0 + }; + + long long status = sengoo_runtime_read_exact( + scripted_read, &reader, destination + 1, expected_len); + return status == expected_status + && reader.invalid_script == 0 + && bytes_are(destination, sizeof(destination), 0xa5); +} + +static int test_prefix_compositions(void) { + static const unsigned char prefix[4] = {0x00, 0x00, 0x80, 0x00}; + static const size_t chunks[][4] = { + {1, 3, 0, 0}, + {2, 2, 0, 0}, + {3, 1, 0, 0}, + {1, 1, 2, 0}, + {1, 2, 1, 0}, + {2, 1, 1, 0}, + {1, 1, 1, 1} + }; + static const size_t chunk_counts[] = {2, 2, 2, 3, 3, 3, 4}; + for (size_t index = 0; index < 7; ++index) { + if (!expect_success(prefix, 4, chunks[index], chunk_counts[index])) return 0; + } + return 1; +} + +static int test_payload_splits(void) { + unsigned char payload[64]; + fill_pattern(payload, sizeof(payload)); + + size_t one[] = {1}; + if (!expect_success(payload, 1, one, 1)) return 0; + + for (size_t len_index = 0; len_index < 2; ++len_index) { + size_t len = len_index == 0 ? 8 : 64; + for (size_t split = 1; split < len; ++split) { + size_t chunks[] = {split, len - split}; + if (!expect_success(payload, len, chunks, 2)) return 0; + } + } + return 1; +} + +static int test_clean_eof_and_truncation(void) { + unsigned char input[64]; + fill_pattern(input, sizeof(input)); + + ScriptStep clean_eof[] = {{0, 1, 0}}; + if (!expect_failure_unchanged(input, 0, 4, clean_eof, 1, 0)) return 0; + + for (size_t supplied = 1; supplied < 4; ++supplied) { + ScriptStep steps[] = {{supplied, 0, 0}, {0, 1, 0}}; + if (!expect_failure_unchanged( + input, supplied, 4, steps, 2, -SENGOO_STATUS_IO)) return 0; + } + + for (size_t len_index = 0; len_index < 3; ++len_index) { + size_t len = len_index == 0 ? 1 : (len_index == 1 ? 8 : 64); + for (size_t supplied = 1; supplied < len; ++supplied) { + ScriptStep steps[] = {{supplied, 0, 0}, {0, 1, 0}}; + if (!expect_failure_unchanged( + input, supplied, len, steps, 2, -SENGOO_STATUS_IO)) return 0; + } + } + return 1; +} + +static int test_zero_progress_and_native_errors(void) { + unsigned char input[4] = {1, 2, 3, 4}; + ScriptStep zero_before[] = {{0, 0, 0}}; + ScriptStep zero_after[] = {{2, 0, 0}, {0, 0, 0}}; + ScriptStep error_before[] = {{0, 0, 1}}; + ScriptStep error_after[] = {{2, 0, 0}, {0, 0, 1}}; + ScriptStep progress_and_error[] = {{2, 0, 1}}; + + return expect_failure_unchanged(input, 0, 4, zero_before, 1, -SENGOO_STATUS_IO) + && expect_failure_unchanged(input, 2, 4, zero_after, 2, -SENGOO_STATUS_IO) + && expect_failure_unchanged(input, 0, 4, error_before, 1, -SENGOO_STATUS_IO) + && expect_failure_unchanged(input, 2, 4, error_after, 2, -SENGOO_STATUS_IO) + && expect_failure_unchanged(input, 2, 4, progress_and_error, 1, -SENGOO_STATUS_IO); +} + +int main(void) { + if (!test_prefix_compositions()) return 1; + if (!test_payload_splits()) return 2; + if (!test_clean_eof_and_truncation()) return 3; + if (!test_zero_progress_and_native_errors()) return 4; + return 0; +} +"#, + ) + .expect("write native exact-read runtime probe"); + + let stdlib = stdlib_dir(); + let mut compile = Command::new(clang); + compile + .arg("-std=c11") + .arg("-Wall") + .arg("-Wextra") + .arg("-Werror") + .arg("-I") + .arg(&stdlib) + .arg(&source) + .arg(stdlib.join("runtime.c")) + .arg(stdlib.join("runtime_string.c")) + .arg("-o") + .arg(&executable); + if cfg!(windows) { + compile.args(["-Wno-unknown-pragmas", "-lws2_32", "-ladvapi32", "-lbcrypt"]); + } else { + compile.args(["-pthread", "-ldl", "-lm"]); + } + let compiled = compile + .output() + .expect("clang should compile exact-read runtime probe"); + assert_success("exact-read runtime probe compilation", &compiled); + + let output = Command::new(&executable) + .output() + .expect("exact-read runtime probe should run"); + assert_success("exact-read runtime probe", &output); + + assert!(root.starts_with(std::env::temp_dir())); + let _ = fs::remove_dir_all(root); +} diff --git a/tools/sgc/tests/binary_io_write_all.rs b/tools/sgc/tests/binary_io_write_all.rs new file mode 100644 index 00000000..254471f5 --- /dev/null +++ b/tools/sgc/tests/binary_io_write_all.rs @@ -0,0 +1,504 @@ +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +const PROBE_TIMEOUT: Duration = Duration::from_secs(15); + +struct NativeProbeTempDir { + path: PathBuf, +} + +impl NativeProbeTempDir { + fn new(tag: &str) -> Self { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be after unix epoch") + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "sengoo-binary-io-{tag}-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&path).expect("create binary I/O test directory"); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for NativeProbeTempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +fn stdlib_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("stdlib") +} + +fn assert_success(label: &str, output: &std::process::Output) { + assert!( + output.status.success(), + "{label} failed with {:?}\nstdout:\n{}\nstderr:\n{}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +fn terminate_and_reap(child: &mut Child) -> String { + let kill = match child.kill() { + Ok(()) => "kill sent".to_owned(), + Err(error) => format!("kill failed: {error}"), + }; + let reap = match child.wait() { + Ok(status) => format!("reaped with {status}"), + Err(error) => format!("reap failed: {error}"), + }; + format!("{kill}; {reap}") +} + +fn wait_for_probe_with_deadline( + child: &mut Child, + label: &str, + timeout: Duration, +) -> Result { + let started = Instant::now(); + loop { + match child.try_wait() { + Ok(Some(status)) => return Ok(status), + Ok(None) => {} + Err(error) => { + let cleanup = terminate_and_reap(child); + return Err(format!("{label} wait failed: {error}; {cleanup}")); + } + } + + let elapsed = started.elapsed(); + if elapsed >= timeout { + let cleanup = terminate_and_reap(child); + return Err(format!( + "{label} timed out after {} ms; {cleanup}", + timeout.as_millis() + )); + } + thread::sleep(Duration::from_millis(10).min(timeout - elapsed)); + } +} + +fn wait_for_probe_success(child: &mut Child, label: &str) { + let status = wait_for_probe_with_deadline(child, label, PROBE_TIMEOUT) + .unwrap_or_else(|error| panic!("{error}")); + assert!(status.success(), "{label} failed with {status}"); +} + +fn compile_native_probe(source_text: &str, tag: &str) -> (NativeProbeTempDir, PathBuf) { + let clang = which::which("clang") + .or_else(|_| which::which("clang.exe")) + .expect("native write-all runtime probe requires clang"); + let root = NativeProbeTempDir::new(tag); + let source = root.path().join("probe.c"); + let executable = root + .path() + .join(if cfg!(windows) { "probe.exe" } else { "probe" }); + fs::write(&source, source_text).expect("write native write-all runtime probe"); + + let stdlib = stdlib_dir(); + let mut compile = Command::new(clang); + compile + .arg("-std=c11") + .arg("-Wall") + .arg("-Wextra") + .arg("-Werror") + .arg("-I") + .arg(&stdlib) + .arg(&source) + .arg(stdlib.join("runtime.c")) + .arg(stdlib.join("runtime_string.c")) + .arg("-o") + .arg(&executable); + if cfg!(windows) { + compile.args(["-Wno-unknown-pragmas", "-lws2_32", "-ladvapi32", "-lbcrypt"]); + } else { + compile.args(["-pthread", "-ldl", "-lm"]); + } + let compiled = compile + .output() + .expect("clang should compile write-all runtime probe"); + assert_success("write-all runtime probe compilation", &compiled); + (root, executable) +} + +fn assert_native_probe_temp_directory_guard_cleans_up_on_drop() { + let path; + { + let root = NativeProbeTempDir::new("drop-cleanup"); + path = root.path().to_path_buf(); + assert!(path.is_dir()); + } + assert!(!path.exists(), "native probe temporary directory leaked"); +} + +fn assert_native_probe_watchdog_kills_and_reaps_a_hung_child() { + let (root, executable) = compile_native_probe( + r#" +int main(void) { + for (;;) {} +} +"#, + "watchdog-hang", + ); + let mut child = Command::new(&executable) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn intentional watchdog hang probe"); + + let error = wait_for_probe_with_deadline( + &mut child, + "intentional watchdog hang probe", + Duration::from_millis(100), + ) + .expect_err("watchdog must reject an intentionally hung child"); + assert!( + error.contains("timed out"), + "unexpected watchdog error: {error}" + ); + assert!( + child + .try_wait() + .expect("query reaped watchdog child") + .is_some(), + "watchdog killed the child without reaping it" + ); + + assert!(root.path().starts_with(std::env::temp_dir())); +} + +#[test] +fn native_write_all_advances_offsets_and_rejects_every_short_or_failed_write() { + let (root, executable) = compile_native_probe( + r#" +#include "runtime_shared.h" +#include +#include +#include + +typedef struct { + size_t count; + int error; +} ScriptStep; + +typedef struct { + const unsigned char* expected_source; + size_t expected_len; + size_t expected_offset; + const ScriptStep* steps; + size_t step_count; + size_t step_index; + int invalid_script; +} ScriptedWriter; + +static SengooRuntimeWriteResult scripted_write( + void* context, + const unsigned char* source, + size_t capacity) { + ScriptedWriter* writer = (ScriptedWriter*)context; + SengooRuntimeWriteResult result = {0, 0}; + if (writer->step_index >= writer->step_count) { + writer->invalid_script = 1; + return result; + } + if (source != writer->expected_source + writer->expected_offset + || capacity != writer->expected_len - writer->expected_offset) { + writer->invalid_script = 1; + result.error = 1; + return result; + } + + ScriptStep step = writer->steps[writer->step_index++]; + if (step.count <= capacity) { + writer->expected_offset += step.count; + } + result.count = step.count; + result.error = step.error; + return result; +} + +static void fill_pattern(unsigned char* bytes, size_t len) { + for (size_t index = 0; index < len; ++index) { + bytes[index] = (unsigned char)((index * 37U + 11U) & 0xffU); + } +} + +static int expect_success( + const unsigned char* source, + size_t len, + const size_t* chunks, + size_t chunk_count) { + ScriptStep steps[64]; + if (chunk_count > 64 || len > 64) return 0; + for (size_t index = 0; index < chunk_count; ++index) { + steps[index].count = chunks[index]; + steps[index].error = 0; + } + ScriptedWriter writer = { + source, len, 0, steps, chunk_count, 0, 0 + }; + + long long status = sengoo_runtime_write_all( + scripted_write, &writer, source, len); + return status == (long long)len + && writer.invalid_script == 0 + && writer.step_index == chunk_count + && writer.expected_offset == len; +} + +static int expect_failure( + const unsigned char* source, + size_t len, + const ScriptStep* steps, + size_t step_count) { + ScriptedWriter writer = { + source, len, 0, steps, step_count, 0, 0 + }; + + long long status = sengoo_runtime_write_all( + scripted_write, &writer, source, len); + return status == -SENGOO_STATUS_IO + && writer.invalid_script == 0 + && writer.step_index == step_count + && status != (long long)len; +} + +static int test_prefix_compositions(void) { + static const unsigned char prefix[4] = {0x00, 0x00, 0x80, 0x00}; + static const size_t chunks[][4] = { + {1, 3, 0, 0}, + {2, 2, 0, 0}, + {3, 1, 0, 0}, + {1, 1, 2, 0}, + {1, 2, 1, 0}, + {2, 1, 1, 0}, + {1, 1, 1, 1} + }; + static const size_t chunk_counts[] = {2, 2, 2, 3, 3, 3, 4}; + for (size_t index = 0; index < 7; ++index) { + if (!expect_success(prefix, 4, chunks[index], chunk_counts[index])) return 0; + } + return 1; +} + +static int test_payload_splits(void) { + unsigned char payload[64]; + fill_pattern(payload, sizeof(payload)); + + size_t one[] = {1}; + if (!expect_success(payload, 1, one, 1)) return 0; + + for (size_t len_index = 0; len_index < 2; ++len_index) { + size_t len = len_index == 0 ? 8 : 64; + for (size_t split = 1; split < len; ++split) { + size_t chunks[] = {split, len - split}; + if (!expect_success(payload, len, chunks, 2)) return 0; + } + } + return 1; +} + +static int test_zero_progress_and_native_errors(void) { + unsigned char source[4] = {1, 2, 3, 4}; + ScriptStep zero_before[] = {{0, 0}}; + ScriptStep zero_after[] = {{2, 0}, {0, 0}}; + ScriptStep error_before[] = {{0, 1}}; + ScriptStep error_after[] = {{2, 0}, {0, 1}}; + ScriptStep progress_and_error[] = {{2, 1}}; + ScriptStep excessive_count[] = {{5, 0}}; + + return expect_failure(source, 4, zero_before, 1) + && expect_failure(source, 4, zero_after, 2) + && expect_failure(source, 4, error_before, 1) + && expect_failure(source, 4, error_after, 2) + && expect_failure(source, 4, progress_and_error, 1) + && expect_failure(source, 4, excessive_count, 1); +} + +static int test_argument_boundaries(void) { + unsigned char source = 0x5a; + ScriptStep unexpected[] = {{1, 0}}; + ScriptedWriter zero_writer = { + NULL, 0, 0, unexpected, 1, 0, 0 + }; + if (sengoo_runtime_write_all( + scripted_write, &zero_writer, NULL, 0) != 0 + || zero_writer.step_index != 0) { + return 0; + } + + ScriptedWriter null_callback_writer = { + &source, 1, 0, unexpected, 1, 0, 0 + }; + if (sengoo_runtime_write_all( + NULL, &null_callback_writer, &source, 1) != -SENGOO_STATUS_INVALID_ARGUMENT + || null_callback_writer.step_index != 0) { + return 0; + } + + ScriptedWriter null_source_writer = { + &source, 1, 0, unexpected, 1, 0, 0 + }; + if (sengoo_runtime_write_all( + scripted_write, &null_source_writer, NULL, 1) + != -SENGOO_STATUS_INVALID_ARGUMENT + || null_source_writer.step_index != 0) { + return 0; + } + +#if SIZE_MAX > 9223372036854775807ULL + size_t overflow_len = (size_t)LLONG_MAX + (size_t)1; + ScriptedWriter overflow_writer = { + &source, overflow_len, 0, unexpected, 1, 0, 0 + }; + if (sengoo_runtime_write_all( + scripted_write, &overflow_writer, &source, overflow_len) + != -SENGOO_STATUS_OVERFLOW + || overflow_writer.step_index != 0) { + return 0; + } +#endif + + return 1; +} + +int main(void) { + if (!test_prefix_compositions()) return 1; + if (!test_payload_splits()) return 2; + if (!test_zero_progress_and_native_errors()) return 3; + if (!test_argument_boundaries()) return 4; + return 0; +} +"#, + "write-all-scripted", + ); + + let mut child = Command::new(&executable) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn scripted write-all runtime probe"); + wait_for_probe_success(&mut child, "scripted write-all runtime probe"); + + assert!(root.path().starts_with(std::env::temp_dir())); +} + +fn run_closed_stdout_case(executable: &PathBuf, mode: &str) { + let mut child = Command::new(executable) + .arg(mode) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn closed-stdout write-all probe"); + + let stdout_reader = match child.stdout.take() { + Some(reader) => reader, + None => { + let cleanup = terminate_and_reap(&mut child); + panic!("{mode}: child stdout pipe was unavailable; {cleanup}"); + } + }; + drop(stdout_reader); + + let mut stdin_gate = match child.stdin.take() { + Some(gate) => gate, + None => { + let cleanup = terminate_and_reap(&mut child); + panic!("{mode}: child stdin gate was unavailable; {cleanup}"); + } + }; + let gate_result = stdin_gate.write_all(b"G"); + drop(stdin_gate); + if let Err(error) = gate_result { + let cleanup = terminate_and_reap(&mut child); + panic!("{mode}: failed to release child through stdin gate: {error}; {cleanup}"); + } + + wait_for_probe_success(&mut child, mode); +} + +#[test] +fn native_closed_pipe_distinguishes_write_failure_from_flush_failure() { + assert_native_probe_temp_directory_guard_cleans_up_on_drop(); + assert_native_probe_watchdog_kills_and_reaps_a_hung_child(); + + let (root, executable) = compile_native_probe( + r#" +#include "runtime_shared.h" +#include +#include +#include +#ifndef _WIN32 +#include +#endif + +long long sengoo_ffi_buffer_from_bytes(long long data_ptr, long long len); +long long sengoo_ffi_buffer_free(long long buffer_handle); +long long sengoo_io_stdout_write_all(long long buffer_handle, long long offset, long long len); +long long sengoo_io_stdout_flush(void); + +static int wait_for_parent_gate(void) { + return fgetc(stdin) == 'G'; +} + +int main(int argc, char** argv) { + static const unsigned char payload[3] = {0x00, 0x7f, 0xff}; + static char stdout_buffer[4096]; + if (argc != 2 || !wait_for_parent_gate()) return 1; + +#ifndef _WIN32 + if (signal(SIGPIPE, SIG_IGN) == SIG_ERR) return 2; +#endif + + long long buffer = sengoo_ffi_buffer_from_bytes( + (long long)(intptr_t)payload, + (long long)sizeof(payload)); + if (buffer == 0) return 3; + + if (strcmp(argv[1], "unbuffered-write-failure") == 0) { + if (setvbuf(stdout, NULL, _IONBF, 0) != 0) return 4; + long long write_status = sengoo_io_stdout_write_all( + buffer, 0, (long long)sizeof(payload)); + sengoo_ffi_buffer_free(buffer); + return write_status == -SENGOO_STATUS_IO ? 0 : 5; + } + + if (strcmp(argv[1], "buffered-flush-failure") == 0) { + if (setvbuf(stdout, stdout_buffer, _IOFBF, sizeof(stdout_buffer)) != 0) return 6; + long long write_status = sengoo_io_stdout_write_all( + buffer, 0, (long long)sizeof(payload)); + long long flush_status = sengoo_io_stdout_flush(); + sengoo_ffi_buffer_free(buffer); + return write_status == (long long)sizeof(payload) + && flush_status == -SENGOO_STATUS_IO ? 0 : 7; + } + + sengoo_ffi_buffer_free(buffer); + return 8; +} +"#, + "write-all-closed-pipe", + ); + + run_closed_stdout_case(&executable, "unbuffered-write-failure"); + run_closed_stdout_case(&executable, "buffered-flush-failure"); + + assert!(root.path().starts_with(std::env::temp_dir())); +} diff --git a/tools/sgc/tests/buffer_bytes.rs b/tools/sgc/tests/buffer_bytes.rs new file mode 100644 index 00000000..5b7b1115 --- /dev/null +++ b/tools/sgc/tests/buffer_bytes.rs @@ -0,0 +1,357 @@ +mod common; + +use common::source_sgc_command; +use std::fs; +use std::path::PathBuf; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +fn temp_dir(tag: &str) -> PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be after unix epoch") + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "sengoo-buffer-bytes-{tag}-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&root).expect("create Buffer byte test directory"); + root +} + +fn stdlib_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("stdlib") +} + +fn assert_success(label: &str, output: &std::process::Output) { + assert!( + output.status.success(), + "{label} failed with {:?}\nstdout:\n{}\nstderr:\n{}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn native_runtime_buffer_byte_access_checks_every_boundary() { + let clang = which::which("clang") + .or_else(|_| which::which("clang.exe")) + .expect("native Buffer byte runtime probe requires clang"); + let root = temp_dir("native"); + let source = root.join("probe.c"); + let executable = root.join(if cfg!(windows) { "probe.exe" } else { "probe" }); + fs::write( + &source, + r#" +#include "runtime_shared.h" +#include + +long long sengoo_ffi_buffer_new(long long capacity); +long long sengoo_ffi_buffer_used_len(long long buffer_handle); +long long sengoo_ffi_buffer_get_u8(long long buffer_handle, long long index); +long long sengoo_ffi_buffer_set_u8(long long buffer_handle, long long index, long long value); +long long sengoo_ffi_buffer_free(long long buffer_handle); + +int main(void) { + long long handle = sengoo_ffi_buffer_new(2); + if (handle <= 0) return 1; + + if (sengoo_ffi_buffer_set_u8(handle, 0, 0) != 1) return 2; + if (sengoo_ffi_buffer_set_u8(handle, 1, 255) != 1) return 3; + if (sengoo_ffi_buffer_get_u8(handle, 0) != 0) return 4; + if (sengoo_ffi_buffer_get_u8(handle, 1) != 255) return 5; + + if (sengoo_ffi_buffer_get_u8(handle, -1) != -SENGOO_STATUS_INVALID_ARGUMENT) return 6; + if (sengoo_ffi_buffer_set_u8(handle, -1, 1) != -SENGOO_STATUS_INVALID_ARGUMENT) return 7; + if (sengoo_ffi_buffer_get_u8(handle, 2) != -SENGOO_STATUS_INVALID_ARGUMENT) return 8; + if (sengoo_ffi_buffer_set_u8(handle, 2, 1) != -SENGOO_STATUS_INVALID_ARGUMENT) return 9; + + if (sengoo_ffi_buffer_get_u8(0, 0) != -SENGOO_STATUS_INVALID_HANDLE) return 10; + if (sengoo_ffi_buffer_set_u8(0, 0, 1) != -SENGOO_STATUS_INVALID_HANDLE) return 11; + if (sengoo_ffi_buffer_set_u8(handle, 0, -1) != -SENGOO_STATUS_INVALID_ARGUMENT) return 12; + if (sengoo_ffi_buffer_set_u8(handle, 0, 256) != -SENGOO_STATUS_INVALID_ARGUMENT) return 13; + + if (sengoo_ffi_buffer_get_u8(handle, LLONG_MAX) != -SENGOO_STATUS_INVALID_ARGUMENT) return 14; + if (sengoo_ffi_buffer_set_u8(handle, LLONG_MAX, 1) != -SENGOO_STATUS_INVALID_ARGUMENT) return 15; + if (sengoo_ffi_buffer_set_u8(handle, 0, LLONG_MAX) != -SENGOO_STATUS_INVALID_ARGUMENT) return 16; + + if (sengoo_ffi_buffer_used_len(handle) != 2) return 17; + if (sengoo_ffi_buffer_get_u8(handle, 0) != 0) return 18; + if (sengoo_ffi_buffer_get_u8(handle, 1) != 255) return 19; + if (sengoo_ffi_buffer_free(handle) != 0) return 20; + if (sengoo_ffi_buffer_get_u8(handle, 0) != -SENGOO_STATUS_INVALID_HANDLE) return 21; + if (sengoo_ffi_buffer_set_u8(handle, 0, 1) != -SENGOO_STATUS_INVALID_HANDLE) return 22; + return 0; +} +"#, + ) + .expect("write Buffer byte runtime probe"); + + let stdlib = stdlib_dir(); + let mut compile = Command::new(clang); + compile + .arg("-std=c11") + .arg("-I") + .arg(&stdlib) + .arg(&source) + .arg(stdlib.join("runtime.c")) + .arg(stdlib.join("runtime_string.c")) + .arg("-o") + .arg(&executable); + if cfg!(windows) { + compile.args(["-lws2_32", "-ladvapi32", "-lbcrypt"]); + } else { + compile.args(["-pthread", "-ldl", "-lm"]); + } + let compiled = compile + .output() + .expect("clang should compile Buffer byte runtime probe"); + assert_success("Buffer byte runtime probe compilation", &compiled); + + let output = Command::new(&executable) + .output() + .expect("Buffer byte runtime probe should run"); + assert_success("Buffer byte runtime probe", &output); + + assert!(root.starts_with(std::env::temp_dir())); + let _ = fs::remove_dir_all(root); +} + +#[test] +fn real_sgc_buffer_byte_access_checks_every_boundary() { + let root = temp_dir("real-sgc"); + let source = root.join("main.sg"); + fs::write( + &source, + r#" +import std::ffi; +import std::status; + +def main() -> i64 { + let buffer = ffi_buffer_new(2).unwrap_or(Buffer { handle: 0 }); + let first_set = buffer.set_u8(0, 0); + let last_set = buffer.set_u8(1, 255); + let first_get = buffer.get_u8(0); + let last_get = buffer.get_u8(1); + + let negative_get = buffer.get_u8(-1); + let negative_set = buffer.set_u8(-1, 1); + let out_of_range_get = buffer.get_u8(2); + let out_of_range_set = buffer.set_u8(2, 1); + + let invalid = Buffer { handle: 0 }; + let invalid_get = invalid.get_u8(0); + let invalid_set = invalid.set_u8(0, 1); + let negative_byte = buffer.set_u8(0, -1); + let high_byte = buffer.set_u8(0, 256); + + let overflow_get = buffer.get_u8(9223372036854775807); + let overflow_set = buffer.set_u8(9223372036854775807, 1); + let overflow_byte = buffer.set_u8(0, 9223372036854775807); + + let unchanged = buffer.used_len() == 2 + && buffer.get_u8(0).unwrap_or(-1) == 0 + && buffer.get_u8(1).unwrap_or(-1) == 255; + let ok = first_set.unwrap_or(false) + && last_set.unwrap_or(false) + && first_get.unwrap_or(-1) == 0 + && last_get.unwrap_or(-1) == 255 + && negative_get.is_err() && negative_get.error == STATUS_INVALID_ARGUMENT() + && negative_set.is_err() && negative_set.error == STATUS_INVALID_ARGUMENT() + && out_of_range_get.is_err() && out_of_range_get.error == STATUS_INVALID_ARGUMENT() + && out_of_range_set.is_err() && out_of_range_set.error == STATUS_INVALID_ARGUMENT() + && invalid_get.is_err() && invalid_get.error == STATUS_INVALID_HANDLE() + && invalid_set.is_err() && invalid_set.error == STATUS_INVALID_HANDLE() + && negative_byte.is_err() && negative_byte.error == STATUS_INVALID_ARGUMENT() + && high_byte.is_err() && high_byte.error == STATUS_INVALID_ARGUMENT() + && overflow_get.is_err() && overflow_get.error == STATUS_INVALID_ARGUMENT() + && overflow_set.is_err() && overflow_set.error == STATUS_INVALID_ARGUMENT() + && overflow_byte.is_err() && overflow_byte.error == STATUS_INVALID_ARGUMENT() + && unchanged; + buffer.free(); + if ok { 0 } else { 1 } +} +"#, + ) + .expect("write real-sgc Buffer byte program"); + + let output = source_sgc_command() + .arg("run") + .arg(&source) + .arg("--force-rebuild") + .output() + .expect("real sgc should run Buffer byte program"); + assert_success("real-sgc Buffer byte program", &output); + + assert!(root.starts_with(std::env::temp_dir())); + let _ = fs::remove_dir_all(root); +} + +#[test] +fn native_runtime_buffer_u32_be_is_network_order_and_failures_are_atomic() { + let clang = which::which("clang") + .or_else(|_| which::which("clang.exe")) + .expect("native Buffer u32 runtime probe requires clang"); + let root = temp_dir("native-u32"); + let source = root.join("probe.c"); + let executable = root.join(if cfg!(windows) { "probe.exe" } else { "probe" }); + fs::write( + &source, + r#" +#include "runtime_shared.h" +#include +#include + +long long sengoo_ffi_buffer_new(long long capacity); +long long sengoo_ffi_buffer_used_len(long long buffer_handle); +long long sengoo_ffi_buffer_get_u8(long long buffer_handle, long long index); +long long sengoo_ffi_buffer_set_u8(long long buffer_handle, long long index, long long value); +long long sengoo_ffi_buffer_read_u32_be(long long buffer_handle, long long offset); +long long sengoo_ffi_buffer_write_u32_be(long long buffer_handle, long long offset, long long value); +long long sengoo_ffi_buffer_free(long long buffer_handle); + +static int bytes_match(long long handle) { + static const long long expected[] = {165, 165, 1, 2, 3, 4, 165, 165}; + for (long long index = 0; index < 8; ++index) { + if (sengoo_ffi_buffer_get_u8(handle, index) != expected[index]) return 0; + } + return 1; +} + +int main(void) { + long long handle = sengoo_ffi_buffer_new(8); + if (handle <= 0) return 1; + for (long long index = 0; index < 8; ++index) { + if (sengoo_ffi_buffer_set_u8(handle, index, 165) != 1) return 2; + } + + if (sengoo_ffi_buffer_write_u32_be(handle, 2, 0x01020304LL) != 1) return 3; + if (sengoo_ffi_buffer_read_u32_be(handle, 2) != 0x01020304LL) return 4; + if (!bytes_match(handle)) return 5; + if (sengoo_ffi_buffer_used_len(handle) != 8) return 6; + + if (sengoo_ffi_buffer_read_u32_be(handle, 5) != -SENGOO_STATUS_INVALID_ARGUMENT) return 7; + if (sengoo_ffi_buffer_write_u32_be(handle, 5, 0) != -SENGOO_STATUS_INVALID_ARGUMENT) return 8; + if (sengoo_ffi_buffer_read_u32_be(handle, -1) != -SENGOO_STATUS_INVALID_ARGUMENT) return 9; + if (sengoo_ffi_buffer_write_u32_be(handle, -1, 0) != -SENGOO_STATUS_INVALID_ARGUMENT) return 10; + if (sengoo_ffi_buffer_read_u32_be(handle, LLONG_MAX) != -SENGOO_STATUS_OVERFLOW) return 11; + if (sengoo_ffi_buffer_write_u32_be(handle, LLONG_MAX, 0) != -SENGOO_STATUS_OVERFLOW) return 12; + if (sengoo_ffi_buffer_write_u32_be(handle, 0, -1) != -SENGOO_STATUS_INVALID_ARGUMENT) return 13; + if (sengoo_ffi_buffer_write_u32_be(handle, 0, (long long)UINT32_MAX + 1) != -SENGOO_STATUS_OVERFLOW) return 14; + + if (sengoo_ffi_buffer_used_len(handle) != 8) return 15; + if (!bytes_match(handle)) return 16; + if (sengoo_ffi_buffer_free(handle) != 0) return 17; + return 0; +} +"#, + ) + .expect("write Buffer u32 runtime probe"); + + let stdlib = stdlib_dir(); + let mut compile = Command::new(clang); + compile + .arg("-std=c11") + .arg("-I") + .arg(&stdlib) + .arg(&source) + .arg(stdlib.join("runtime.c")) + .arg(stdlib.join("runtime_string.c")) + .arg("-o") + .arg(&executable); + if cfg!(windows) { + compile.args(["-lws2_32", "-ladvapi32", "-lbcrypt"]); + } else { + compile.args(["-pthread", "-ldl", "-lm"]); + } + let compiled = compile + .output() + .expect("clang should compile Buffer u32 runtime probe"); + assert_success("Buffer u32 runtime probe compilation", &compiled); + + let output = Command::new(&executable) + .output() + .expect("Buffer u32 runtime probe should run"); + assert_success("Buffer u32 runtime probe", &output); + + assert!(root.starts_with(std::env::temp_dir())); + let _ = fs::remove_dir_all(root); +} + +#[test] +fn real_sgc_buffer_u32_be_is_network_order_and_failures_are_atomic() { + let root = temp_dir("real-sgc-u32"); + let source = root.join("main.sg"); + fs::write( + &source, + r#" +import std::ffi; +import std::status; + +def main() -> i64 { + let buffer = ffi_buffer_new(8).unwrap_or(Buffer { handle: 0 }); + let initialized = buffer.set_u8(0, 165).unwrap_or(false) + && buffer.set_u8(1, 165).unwrap_or(false) + && buffer.set_u8(2, 165).unwrap_or(false) + && buffer.set_u8(3, 165).unwrap_or(false) + && buffer.set_u8(4, 165).unwrap_or(false) + && buffer.set_u8(5, 165).unwrap_or(false) + && buffer.set_u8(6, 165).unwrap_or(false) + && buffer.set_u8(7, 165).unwrap_or(false); + + let wrote = buffer.write_u32_be(2, 16909060).unwrap_or(false); + let network_order = buffer.get_u8(2).unwrap_or(-1) == 1 + && buffer.get_u8(3).unwrap_or(-1) == 2 + && buffer.get_u8(4).unwrap_or(-1) == 3 + && buffer.get_u8(5).unwrap_or(-1) == 4; + let round_trip = buffer.read_u32_be(2).unwrap_or(-1) == 16909060; + + let short_read = buffer.read_u32_be(5); + let short_write = buffer.write_u32_be(5, 0); + let negative_read = buffer.read_u32_be(-1); + let negative_write = buffer.write_u32_be(-1, 0); + let overflow_read = buffer.read_u32_be(9223372036854775807); + let overflow_write = buffer.write_u32_be(9223372036854775807, 0); + let negative_value = buffer.write_u32_be(0, -1); + let overflow_value = buffer.write_u32_be(0, 4294967296); + + let statuses = short_read.is_err() && short_read.error == STATUS_INVALID_ARGUMENT() + && short_write.is_err() && short_write.error == STATUS_INVALID_ARGUMENT() + && negative_read.is_err() && negative_read.error == STATUS_INVALID_ARGUMENT() + && negative_write.is_err() && negative_write.error == STATUS_INVALID_ARGUMENT() + && overflow_read.is_err() && overflow_read.error == STATUS_OVERFLOW() + && overflow_write.is_err() && overflow_write.error == STATUS_OVERFLOW() + && negative_value.is_err() && negative_value.error == STATUS_INVALID_ARGUMENT() + && overflow_value.is_err() && overflow_value.error == STATUS_OVERFLOW(); + let unchanged = buffer.used_len() == 8 + && buffer.get_u8(0).unwrap_or(-1) == 165 + && buffer.get_u8(1).unwrap_or(-1) == 165 + && buffer.get_u8(2).unwrap_or(-1) == 1 + && buffer.get_u8(3).unwrap_or(-1) == 2 + && buffer.get_u8(4).unwrap_or(-1) == 3 + && buffer.get_u8(5).unwrap_or(-1) == 4 + && buffer.get_u8(6).unwrap_or(-1) == 165 + && buffer.get_u8(7).unwrap_or(-1) == 165; + + let ok = initialized && wrote && network_order && round_trip && statuses && unchanged; + buffer.free(); + if ok { 0 } else { 1 } +} +"#, + ) + .expect("write real-sgc Buffer u32 program"); + + let output = source_sgc_command() + .arg("run") + .arg(&source) + .arg("--force-rebuild") + .output() + .expect("real sgc should run Buffer u32 program"); + assert_success("real-sgc Buffer u32 program", &output); + + assert!(root.starts_with(std::env::temp_dir())); + let _ = fs::remove_dir_all(root); +} diff --git a/tools/sgc/tests/common/mod.rs b/tools/sgc/tests/common/mod.rs new file mode 100644 index 00000000..88f2f62b --- /dev/null +++ b/tools/sgc/tests/common/mod.rs @@ -0,0 +1,7 @@ +use std::process::Command; + +pub(crate) fn source_sgc_command() -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_sgc")); + command.args(["--runtime-mode", "source-development"]); + command +} diff --git a/tools/sgc/tests/core_conformance.rs b/tools/sgc/tests/core_conformance.rs index b04560fe..57b6706f 100644 --- a/tools/sgc/tests/core_conformance.rs +++ b/tools/sgc/tests/core_conformance.rs @@ -1,12 +1,10 @@ +mod common; + +use common::source_sgc_command; use std::path::{Path, PathBuf}; -use std::process::Command; const CONFORMANCE_RUN_MODES: &[&[&str]] = &[&[], &["--debug-info"]]; -fn sgc() -> PathBuf { - PathBuf::from(env!("CARGO_BIN_EXE_sgc")) -} - fn workspace_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) .parent() @@ -39,7 +37,7 @@ fn run_core_case(tag: &str, relative_path: &str, expected_exit: i32, expected_st let outputs = CONFORMANCE_RUN_MODES .iter() .map(|mode_args| { - Command::new(sgc()) + source_sgc_command() .arg("run") .arg(&path) .arg("--force-rebuild") diff --git a/tools/sgc/tests/coverage_runtime.rs b/tools/sgc/tests/coverage_runtime.rs index fdd85226..6f941e33 100644 --- a/tools/sgc/tests/coverage_runtime.rs +++ b/tools/sgc/tests/coverage_runtime.rs @@ -1,13 +1,11 @@ +mod common; + +use common::source_sgc_command; use serde_json::Value; use std::fs; use std::path::PathBuf; -use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; -fn sgc() -> PathBuf { - PathBuf::from(env!("CARGO_BIN_EXE_sgc")) -} - fn temp_project() -> PathBuf { let stamp = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -55,7 +53,7 @@ def never_called() -> i64 { ) .expect("write coverage fixture"); - let output = Command::new(sgc()) + let output = source_sgc_command() .current_dir(&root) .args(["test", "--coverage", "--format", "json"]) .output() diff --git a/tools/sgc/tests/cranelift_numeric.rs b/tools/sgc/tests/cranelift_numeric.rs index 205d6d31..2ba9a4dd 100644 --- a/tools/sgc/tests/cranelift_numeric.rs +++ b/tools/sgc/tests/cranelift_numeric.rs @@ -1,6 +1,8 @@ +mod common; + +use common::source_sgc_command; use std::fs; use std::path::PathBuf; -use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; fn temp_source(name: &str, source: &str) -> PathBuf { @@ -18,7 +20,7 @@ fn temp_source(name: &str, source: &str) -> PathBuf { fn run_cranelift(name: &str, source: &str, opt_level: &str) -> std::process::Output { let source = temp_source(name, source); - let output = Command::new(env!("CARGO_BIN_EXE_sgc")) + let output = source_sgc_command() .args([ "run", source.to_str().expect("temporary path should be UTF-8"), diff --git a/tools/sgc/tests/debugger_native.rs b/tools/sgc/tests/debugger_native.rs index cde866e5..0cb8847e 100644 --- a/tools/sgc/tests/debugger_native.rs +++ b/tools/sgc/tests/debugger_native.rs @@ -1,3 +1,6 @@ +mod common; + +use common::source_sgc_command; use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; @@ -143,10 +146,6 @@ impl Drop for TempProject { } } -fn sgc() -> PathBuf { - PathBuf::from(env!("CARGO_BIN_EXE_sgc")) -} - fn host_debugger() -> Option<(DebuggerFlavor, &'static str)> { #[cfg(windows)] { @@ -882,7 +881,7 @@ fn native_debugger_breaks_steps_and_reads_local() { let source = project.source_path(); fs::write(&source, PROBE_SOURCE).expect("write debugger probe source"); - let build = Command::new(sgc()) + let build = source_sgc_command() .arg("build") .arg(&source) .args(["-O", "0", "--debug-info", "--force-rebuild"]) @@ -1017,7 +1016,7 @@ fn debug_build_cache_recovery_recreates_the_pdb() { let source = project.source_path(); fs::write(&source, PROBE_SOURCE).expect("write debugger cache-recovery probe source"); - let initial = Command::new(sgc()) + let initial = source_sgc_command() .arg("build") .arg(&source) .args(["-O", "0", "--debug-info", "--force-rebuild"]) @@ -1035,7 +1034,7 @@ fn debug_build_cache_recovery_recreates_the_pdb() { fs::remove_file(&executable).expect("remove cached debug executable"); fs::remove_file(&pdb).expect("remove cached debug PDB"); - let recovered = Command::new(sgc()) + let recovered = source_sgc_command() .arg("build") .arg(&source) .args(["-O", "0", "--debug-info"]) diff --git a/tools/sgc/tests/fixtures/json-malformed-v1.json b/tools/sgc/tests/fixtures/json-malformed-v1.json new file mode 100644 index 00000000..bac1d3f9 --- /dev/null +++ b/tools/sgc/tests/fixtures/json-malformed-v1.json @@ -0,0 +1,197 @@ +{ + "schema_version": 1, + "strict_max_bytes": 1048576, + "mutation_max_bytes": 4096, + "mutation_count": 4096, + "seed": "0x6a09e667f3bcc909", + "encoding": "lowercase hexadecimal bytes", + "default_origin": "spec_seed", + "retention_policy": "Seeded mutations are discovery-only. Every fixed crash or panic must be minimized into exact bytes here with origin=fixed_crash and a non-empty regression id; fixed minimized bytes are never replaced by a PRNG seed.", + "cases": [ + { + "name": "empty-document", + "hex": "", + "error_kind": "unclassified" + }, + { + "name": "whitespace-only", + "hex": "20090a0d", + "error_kind": "unclassified" + }, + { + "name": "unexpected-utf8-continuation", + "hex": "80", + "error_kind": "invalid_unicode" + }, + { + "name": "overlong-utf8-slash", + "hex": "c0af", + "error_kind": "invalid_unicode" + }, + { + "name": "truncated-three-byte-utf8", + "hex": "e282", + "error_kind": "invalid_unicode" + }, + { + "name": "utf8-encoded-surrogate", + "hex": "eda080", + "error_kind": "invalid_unicode" + }, + { + "name": "utf8-above-unicode-range", + "hex": "f4908080", + "error_kind": "invalid_unicode" + }, + { + "name": "obsolete-five-byte-utf8", + "hex": "f888808080", + "error_kind": "invalid_unicode" + }, + { + "name": "raw-nul-in-string", + "hex": "220022", + "error_kind": "unclassified" + }, + { + "name": "raw-newline-in-string", + "hex": "22610a6222", + "error_kind": "unclassified" + }, + { + "name": "invalid-short-escape", + "hex": "225c7122", + "error_kind": "unclassified" + }, + { + "name": "truncated-unicode-escape", + "hex": "225c75313222", + "error_kind": "invalid_unicode" + }, + { + "name": "non-hex-unicode-escape", + "hex": "225c753132787a22", + "error_kind": "invalid_unicode" + }, + { + "name": "lone-high-surrogate", + "hex": "225c756438303022", + "error_kind": "invalid_unicode" + }, + { + "name": "lone-low-surrogate", + "hex": "225c756463303022", + "error_kind": "invalid_unicode" + }, + { + "name": "reversed-surrogate-pair", + "hex": "225c75646330305c756438303022", + "error_kind": "invalid_unicode" + }, + { + "name": "unterminated-string", + "hex": "22616263", + "error_kind": "unclassified" + }, + { + "name": "truncated-object", + "hex": "7b", + "error_kind": "unclassified" + }, + { + "name": "truncated-array", + "hex": "5b", + "error_kind": "unclassified" + }, + { + "name": "unquoted-object-key", + "hex": "7b613a317d", + "error_kind": "unclassified" + }, + { + "name": "missing-object-colon", + "hex": "7b22612220317d", + "error_kind": "unclassified" + }, + { + "name": "missing-array-comma", + "hex": "5b3120325d", + "error_kind": "unclassified" + }, + { + "name": "trailing-array-comma", + "hex": "5b312c5d", + "error_kind": "unclassified" + }, + { + "name": "double-object-comma", + "hex": "7b2261223a312c2c2262223a327d", + "error_kind": "unclassified" + }, + { + "name": "trailing-token", + "hex": "7b7d78", + "error_kind": "trailing_bytes" + }, + { + "name": "trailing-nul", + "hex": "7b7d00", + "error_kind": "trailing_bytes" + }, + { + "name": "leading-zero", + "hex": "3031", + "error_kind": "trailing_bytes" + }, + { + "name": "duplicate-literal-key", + "hex": "7b2261223a312c2261223a327d", + "error_kind": "duplicate_field" + }, + { + "name": "duplicate-escaped-key", + "hex": "7b2261223a312c225c7530303631223a327d", + "error_kind": "duplicate_field" + }, + { + "name": "nested-duplicate-key", + "hex": "7b2278223a7b2262223a312c2262223a327d7d", + "error_kind": "duplicate_field" + }, + { + "name": "integer-overflow", + "hex": "39323233333732303336383534373735383038", + "error_kind": "unclassified" + }, + { + "name": "integer-underflow", + "hex": "2d39323233333732303336383534373735383039", + "error_kind": "unclassified" + }, + { + "name": "incomplete-fraction", + "hex": "312e", + "error_kind": "unclassified" + }, + { + "name": "incomplete-exponent", + "hex": "3165", + "error_kind": "unclassified" + }, + { + "name": "bare-minus", + "hex": "2d", + "error_kind": "unclassified" + }, + { + "name": "invalid-literal", + "hex": "74727578", + "error_kind": "unclassified" + }, + { + "name": "utf8-bom-prefix", + "hex": "efbbbf7b7d", + "error_kind": "unclassified" + } + ] +} diff --git a/tools/sgc/tests/http_request_strings.rs b/tools/sgc/tests/http_request_strings.rs new file mode 100644 index 00000000..c4f2f62e --- /dev/null +++ b/tools/sgc/tests/http_request_strings.rs @@ -0,0 +1,232 @@ +mod common; + +use std::fs; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpStream; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::sync::mpsc; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use common::source_sgc_command; + +struct TestDir(PathBuf); + +impl TestDir { + fn new(name: &str) -> Self { + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "sengoo_http_request_strings_{name}_{}_{stamp}", + std::process::id() + )); + fs::create_dir_all(&path).expect("create temp directory"); + Self(path) + } +} + +impl Drop for TestDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +struct ChildGuard(Option); + +impl ChildGuard { + fn new(child: Child) -> Self { + Self(Some(child)) + } + + fn child_mut(&mut self) -> &mut Child { + self.0.as_mut().expect("child should be live") + } + + fn wait_with_deadline(&mut self, timeout: Duration) -> ExitStatus { + let deadline = Instant::now() + timeout; + loop { + if let Some(status) = self + .child_mut() + .try_wait() + .expect("query child process status") + { + self.0.take(); + return status; + } + if Instant::now() >= deadline { + let child = self.child_mut(); + let _ = child.kill(); + let _ = child.wait(); + self.0.take(); + panic!("Sengoo HTTP accessor fixture exceeded its watchdog"); + } + thread::sleep(Duration::from_millis(10)); + } + } +} + +impl Drop for ChildGuard { + fn drop(&mut self) { + if let Some(mut child) = self.0.take() { + if child.try_wait().ok().flatten().is_none() { + let _ = child.kill(); + } + let _ = child.wait(); + } + } +} + +fn build_fixture(source: &Path, executable: &Path) { + let output = source_sgc_command() + .arg("build") + .arg(source) + .arg("--output") + .arg(executable) + .args(["-O", "0", "--force-rebuild"]) + .output() + .expect("run source sgc"); + assert!( + output.status.success(), + "fixture build failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn real_sgc_http_request_owned_string_accessors_are_safe() { + let dir = TestDir::new("owned_accessors"); + let source = dir.0.join("main.sg"); + let executable = dir.0.join(if cfg!(windows) { + "server.exe" + } else { + "server" + }); + fs::write( + &source, + r#"import std::ffi; + +import std::io; + +import std::net; + +import std::strconv; + +import std::string; + +async def main() -> i64 { + let bound = http_server_bind("127.0.0.1", 0); + if bound.is_err() { return 10; }; + let server = bound.value; + let port = server.local_port(); + if port.is_err() { server.close(); return 11; }; + let port_buffer = ffi_buffer_new(16).unwrap_or(Buffer { handle: 0 }); + let port_len = strconv_format_i64(port.value, port_buffer).unwrap_or(0); + io_stdout_write("READY 127.0.0.1:"); + io_stdout_write_raw(port_buffer.ptr(), port_len); + io_stdout_write("\n"); + io_stdout_flush(); + port_buffer.free(); + + let outcome = await server.next_request_async(5000); + if not outcome.is_ok { server.close(); return 12; }; + let request = outcome.value; + let method = request.method_string(); + let path = request.path_string(); + let query = request.query_string(); + let version = request.version_string(); + let trace = request.header_string("X-Trace"); + let body_buffer = ffi_buffer_new(4).unwrap_or(Buffer { handle: 0 }); + let copied = request.body_copy(body_buffer); + let body_matches = if copied.is_err() { false; } else { copied.value == 4 and body_buffer.used_len() == 4; }; + let matches = method.is_ok() and str_eq(method.value.as_str(), "POST") and path.is_ok() and str_eq(path.value.as_str(), "/probe") and query.is_ok() and str_eq(query.value.as_str(), "mode=owned") and version.is_ok() and str_eq(version.value.as_str(), "HTTP/1.1") and trace.is_ok() and str_eq(trace.value.as_str(), "abc") and body_matches; + let responded = if matches { request.respond(200, "ok").unwrap_or(false); } else { request.respond(500, "mismatch").unwrap_or(false); }; + body_buffer.free(); + let closed = server.close(); + if responded and closed { 0; } else { 13; }; +} +"#, + ) + .expect("write fixture source"); + build_fixture(&source, &executable); + + let mut command = Command::new(&executable); + command.stdout(Stdio::piped()).stderr(Stdio::piped()); + let mut child = ChildGuard::new(command.spawn().expect("spawn fixture")); + let stdout = child + .child_mut() + .stdout + .take() + .expect("capture fixture stdout"); + let stderr = child + .child_mut() + .stderr + .take() + .expect("capture fixture stderr"); + let (ready_tx, ready_rx) = mpsc::sync_channel(1); + let stdout_reader = thread::spawn(move || { + let mut reader = BufReader::new(stdout); + let mut ready = String::new(); + let result = reader.read_line(&mut ready); + let _ = ready_tx.send((ready, result)); + let mut extra = Vec::new(); + let _ = reader.read_to_end(&mut extra); + extra + }); + let stderr_reader = thread::spawn(move || { + let mut reader = BufReader::new(stderr); + let mut bytes = Vec::new(); + let _ = reader.read_to_end(&mut bytes); + bytes + }); + + let (ready, ready_result) = ready_rx + .recv_timeout(Duration::from_secs(3)) + .expect("fixture should publish READY before deadline"); + ready_result.expect("read READY line"); + let port = ready + .trim_end_matches(['\r', '\n']) + .strip_prefix("READY 127.0.0.1:") + .and_then(|value| value.parse::().ok()) + .filter(|port| *port != 0) + .unwrap_or_else(|| panic!("invalid READY line: {ready:?}")); + + let mut stream = TcpStream::connect(("127.0.0.1", port)).expect("connect to fixture"); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .expect("set read timeout"); + stream + .set_write_timeout(Some(Duration::from_secs(5))) + .expect("set write timeout"); + stream + .write_all( + b"POST /probe?mode=owned HTTP/1.1\r\nHost: localhost\r\nX-Trace: abc\r\nContent-Length: 4\r\nConnection: close\r\n\r\nping", + ) + .expect("write fixture request"); + stream.flush().expect("flush fixture request"); + let mut response = Vec::new(); + let read_result = stream.read_to_end(&mut response); + let status = child.wait_with_deadline(Duration::from_secs(10)); + let extra_stdout = stdout_reader.join().expect("join stdout reader"); + let stderr = stderr_reader.join().expect("join stderr reader"); + let stderr = String::from_utf8_lossy(&stderr); + + assert!( + read_result.is_ok(), + "response read failed: {:?}; child={status:?}; stderr={stderr}", + read_result.err() + ); + assert!(status.success(), "child={status:?}; stderr={stderr}"); + assert!( + extra_stdout.is_empty(), + "unexpected stdout: {extra_stdout:?}" + ); + let response = String::from_utf8_lossy(&response); + assert!( + response.starts_with("HTTP/1.1 200 OK\r\n") && response.ends_with("\r\n\r\nok"), + "unexpected response: {response:?}; stderr={stderr}" + ); +} diff --git a/tools/sgc/tests/json_fuzz.rs b/tools/sgc/tests/json_fuzz.rs new file mode 100644 index 00000000..4a04569f --- /dev/null +++ b/tools/sgc/tests/json_fuzz.rs @@ -0,0 +1,780 @@ +mod common; + +use common::source_sgc_command; +use serde_json::Value; +use std::collections::HashSet; +use std::fs::{self, File}; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, ExitStatus, Output, Stdio}; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +const EXPECT_ANY: u32 = u32::MAX; +const EXPECT_ACCEPT: u32 = u32::MAX - 1; +const MAX_MUTATIONS: usize = 10_000; +const MAX_FIXED_CASES: usize = 1_024; +const MAX_BATCH_CASES: usize = 20_000; +const MAX_BATCH_BYTES: usize = 32 * 1024 * 1024; +const PROBE_TIMEOUT: Duration = Duration::from_secs(30); + +struct ProbeTempDir { + path: PathBuf, +} + +impl ProbeTempDir { + fn new(tag: &str) -> Self { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be after unix epoch") + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "sengoo-json-fuzz-{tag}-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&path).expect("create JSON fuzz probe directory"); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for ProbeTempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +#[derive(Clone)] +struct FuzzCase { + name: String, + expected: u32, + bytes: Vec, +} + +struct CorpusConfig { + strict_max_bytes: usize, + mutation_max_bytes: usize, + mutation_count: usize, + seed: u64, + malformed: Vec, +} + +fn corpus_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("fixtures") + .join("json-malformed-v1.json") +} + +fn required_u64(value: &Value, field: &str) -> u64 { + value[field] + .as_u64() + .unwrap_or_else(|| panic!("JSON fuzz corpus field `{field}` must be a u64")) +} + +fn required_usize(value: &Value, field: &str) -> usize { + usize::try_from(required_u64(value, field)) + .unwrap_or_else(|_| panic!("JSON fuzz corpus field `{field}` does not fit usize")) +} + +fn decode_hex(name: &str, text: &str) -> Vec { + assert!( + text.len().is_multiple_of(2), + "corpus case `{name}` has odd-length hex" + ); + assert!( + text.bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)), + "corpus case `{name}` hex must be lowercase" + ); + text.as_bytes() + .chunks_exact(2) + .map(|pair| { + let pair = std::str::from_utf8(pair).expect("hex pair should be ASCII"); + u8::from_str_radix(pair, 16) + .unwrap_or_else(|_| panic!("corpus case `{name}` has invalid hex `{pair}`")) + }) + .collect() +} + +fn error_kind(name: &str) -> u32 { + match name { + "unclassified" => 1, + "duplicate_field" => 2, + "invalid_unicode" => 3, + "trailing_bytes" => 4, + other => panic!("unsupported strict JSON error kind `{other}`"), + } +} + +fn load_corpus() -> CorpusConfig { + let path = corpus_path(); + let bytes = fs::read(&path) + .unwrap_or_else(|error| panic!("read fixed JSON fuzz corpus {}: {error}", path.display())); + let root: Value = serde_json::from_slice(&bytes) + .unwrap_or_else(|error| panic!("parse JSON fuzz corpus {}: {error}", path.display())); + assert_eq!(required_u64(&root, "schema_version"), 1); + + let strict_max_bytes = required_usize(&root, "strict_max_bytes"); + let mutation_max_bytes = required_usize(&root, "mutation_max_bytes"); + let mutation_count = required_usize(&root, "mutation_count"); + assert_eq!(strict_max_bytes, 1024 * 1024); + assert!((1..=strict_max_bytes).contains(&mutation_max_bytes)); + assert!((1024..=MAX_MUTATIONS).contains(&mutation_count)); + + let default_origin = root["default_origin"] + .as_str() + .expect("JSON fuzz corpus default_origin must be a string"); + assert_eq!(default_origin, "spec_seed"); + let retention_policy = root["retention_policy"] + .as_str() + .expect("JSON fuzz corpus retention_policy must be a string"); + assert!(retention_policy.contains("minimized bytes")); + + let seed_text = root["seed"] + .as_str() + .expect("JSON fuzz corpus seed must be a hexadecimal string"); + let seed = u64::from_str_radix(seed_text.trim_start_matches("0x"), 16) + .expect("JSON fuzz corpus seed must fit u64"); + assert_ne!(seed, 0); + + let cases = root["cases"] + .as_array() + .expect("JSON fuzz corpus cases must be an array"); + assert!( + (16..=MAX_FIXED_CASES).contains(&cases.len()), + "malformed corpus size is outside the reviewed bounds" + ); + let mut names = HashSet::new(); + let malformed = cases + .iter() + .map(|case| { + let name = case["name"] + .as_str() + .expect("corpus case name must be a string"); + assert!(names.insert(name), "duplicate corpus case name `{name}`"); + let origin = case["origin"].as_str().unwrap_or(default_origin); + assert!(matches!(origin, "spec_seed" | "fixed_crash")); + if origin == "fixed_crash" { + let regression = case["regression"] + .as_str() + .expect("fixed_crash corpus cases require a regression id"); + assert!(!regression.is_empty()); + } + let hex = case["hex"] + .as_str() + .expect("corpus case hex must be a string"); + let bytes = decode_hex(name, hex); + assert!(bytes.len() <= strict_max_bytes); + FuzzCase { + name: name.to_owned(), + expected: error_kind( + case["error_kind"] + .as_str() + .expect("corpus error_kind must be a string"), + ), + bytes, + } + }) + .collect(); + + CorpusConfig { + strict_max_bytes, + mutation_max_bytes, + mutation_count, + seed, + malformed, + } +} + +fn next_random(state: &mut u64) -> u64 { + let mut value = *state; + value ^= value << 13; + value ^= value >> 7; + value ^= value << 17; + *state = value; + value +} + +fn random_index(state: &mut u64, upper_exclusive: usize) -> usize { + if upper_exclusive == 0 { + 0 + } else { + (next_random(state) as usize) % upper_exclusive + } +} + +fn mutate(pool: &[Vec], state: &mut u64, max_len: usize) -> Vec { + let mut bytes = pool[random_index(state, pool.len())].clone(); + bytes.truncate(max_len); + let structural = b"{}[],:\\\"0123456789tfnueE+- \n\r\t"; + let steps = 1 + random_index(state, 8); + for _ in 0..steps { + match random_index(state, 8) { + 0 => { + if bytes.is_empty() { + bytes.push(next_random(state) as u8); + } else { + let index = random_index(state, bytes.len()); + bytes[index] ^= 1 << random_index(state, 8); + } + } + 1 => { + bytes.truncate(random_index(state, bytes.len() + 1)); + } + 2 => { + if bytes.len() < max_len { + let index = random_index(state, bytes.len() + 1); + let value = if next_random(state) & 1 == 0 { + structural[random_index(state, structural.len())] + } else { + next_random(state) as u8 + }; + bytes.insert(index, value); + } + } + 3 => { + if !bytes.is_empty() { + let index = random_index(state, bytes.len()); + bytes.remove(index); + } + } + 4 => { + if bytes.is_empty() { + bytes.push(structural[random_index(state, structural.len())]); + } else { + let index = random_index(state, bytes.len()); + bytes[index] = structural[random_index(state, structural.len())]; + } + } + 5 => { + if !bytes.is_empty() && bytes.len() < max_len { + let start = random_index(state, bytes.len()); + let available = bytes.len() - start; + let take = 1 + random_index(state, available.min(32)); + let fragment = bytes[start..start + take].to_vec(); + let insert_at = random_index(state, bytes.len() + 1); + let remaining = max_len - bytes.len(); + bytes.splice(insert_at..insert_at, fragment.into_iter().take(remaining)); + } + } + 6 => { + let other = &pool[random_index(state, pool.len())]; + let split = random_index(state, bytes.len() + 1); + bytes.truncate(split); + bytes.extend(other.iter().copied().take(max_len - bytes.len())); + } + _ => { + if bytes.len() < max_len { + let padding = [b' ', b'\n', b'\r', b'\t', 0, 0xff]; + let count = 1 + random_index(state, 8); + for _ in 0..count { + if bytes.len() == max_len { + break; + } + bytes.push(padding[random_index(state, padding.len())]); + } + } + } + } + bytes.truncate(max_len); + } + bytes +} + +fn build_cases(config: &CorpusConfig) -> Vec { + let mut cases = config.malformed.clone(); + cases.push(FuzzCase { + name: "over-hardening-limit".to_owned(), + expected: 1, + bytes: vec![b' '; config.strict_max_bytes + 1], + }); + + let valid_seeds = vec![ + b"null".to_vec(), + b"true".to_vec(), + b"-9223372036854775808".to_vec(), + br#"{"a":[1,false,null,"text"],"b":"\u4f60\u597d","c":"\ud83d\ude00"}"#.to_vec(), + vec![b'[', b']'], + ]; + for (index, bytes) in valid_seeds.iter().enumerate() { + cases.push(FuzzCase { + name: format!("valid-seed-{index}"), + expected: EXPECT_ACCEPT, + bytes: bytes.clone(), + }); + } + let mut hardening_boundary = Vec::with_capacity(config.strict_max_bytes); + hardening_boundary.push(b'"'); + hardening_boundary.resize(config.strict_max_bytes - 1, b'a'); + hardening_boundary.push(b'"'); + cases.push(FuzzCase { + name: "valid-at-hardening-limit".to_owned(), + expected: EXPECT_ACCEPT, + bytes: hardening_boundary, + }); + + let mut depth_63 = vec![b'['; 63]; + depth_63.extend_from_slice(b"null"); + depth_63.extend(std::iter::repeat_n(b']', 63)); + cases.push(FuzzCase { + name: "valid-depth-63".to_owned(), + expected: EXPECT_ACCEPT, + bytes: depth_63, + }); + let mut depth_64 = vec![b'['; 64]; + depth_64.extend_from_slice(b"null"); + depth_64.extend(std::iter::repeat_n(b']', 64)); + cases.push(FuzzCase { + name: "reject-depth-64".to_owned(), + expected: 1, + bytes: depth_64, + }); + + let node_boundary = |items: usize| { + let mut bytes = Vec::with_capacity(2 + items.saturating_mul(5)); + bytes.push(b'['); + for index in 0..items { + if index > 0 { + bytes.push(b','); + } + bytes.extend_from_slice(b"null"); + } + bytes.push(b']'); + bytes + }; + cases.push(FuzzCase { + name: "valid-4096-nodes".to_owned(), + expected: EXPECT_ACCEPT, + bytes: node_boundary(4095), + }); + cases.push(FuzzCase { + name: "reject-4097-nodes".to_owned(), + expected: 1, + bytes: node_boundary(4096), + }); + + let mut mutation_pool = valid_seeds; + mutation_pool.extend(config.malformed.iter().map(|case| case.bytes.clone())); + let mut state = config.seed; + for index in 0..config.mutation_count { + cases.push(FuzzCase { + name: format!("mutation-{index:05}"), + expected: EXPECT_ANY, + bytes: mutate(&mutation_pool, &mut state, config.mutation_max_bytes), + }); + } + cases +} + +fn encode_u32(output: &mut Vec, value: usize) { + output.extend( + u32::try_from(value) + .expect("JSON fuzz batch value should fit u32") + .to_be_bytes(), + ); +} + +fn encode_batch(cases: &[FuzzCase]) -> Vec { + assert!( + !cases.is_empty() && cases.len() <= MAX_BATCH_CASES, + "JSON fuzz batch case count is outside the reviewed bounds" + ); + let encoded_len = cases.iter().fold(4usize, |total, case| { + total + .checked_add(8) + .and_then(|value| value.checked_add(case.bytes.len())) + .expect("JSON fuzz batch byte count overflowed") + }); + assert!( + encoded_len <= MAX_BATCH_BYTES, + "JSON fuzz batch exceeds the reviewed byte budget" + ); + let mut batch = Vec::with_capacity(encoded_len); + encode_u32(&mut batch, cases.len()); + for case in cases { + batch.extend(case.expected.to_be_bytes()); + encode_u32(&mut batch, case.bytes.len()); + batch.extend(&case.bytes); + } + assert_eq!(batch.len(), encoded_len); + batch +} + +fn stdlib_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("stdlib") +} + +fn assert_success(label: &str, output: &Output) { + assert!( + output.status.success(), + "{label} failed with {:?}\nstdout:\n{}\nstderr:\n{}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +fn failure_case_context(output: &Output, cases: &[FuzzCase]) -> String { + if let Ok(stderr) = std::str::from_utf8(&output.stderr) { + for line in stderr.lines() { + if let Some(index) = line + .strip_prefix("case ") + .and_then(|rest| rest.split(':').next()) + .and_then(|value| value.parse::().ok()) + { + return cases.get(index).map_or_else( + || format!("reported out-of-range case index {index}"), + |case| format!("case {index} `{}`", case.name), + ); + } + } + } + if output.stderr.len() == 8 { + let index = u32::from_be_bytes(output.stderr[0..4].try_into().unwrap()) as usize; + let reason = u32::from_be_bytes(output.stderr[4..8].try_into().unwrap()); + return cases.get(index).map_or_else( + || format!("reported out-of-range case index {index}, reason {reason}"), + |case| format!("case {index} `{}`, reason {reason}", case.name), + ); + } + "probe did not report a case index".to_owned() +} + +fn assert_probe_success(label: &str, output: &Output, cases: &[FuzzCase]) { + assert!( + output.status.success(), + "{label} failed with {:?}: {}\nstdout:\n{}\nstderr (lossy):\n{}", + output.status.code(), + failure_case_context(output, cases), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +fn terminate_and_reap(child: &mut Child) -> String { + let kill = match child.kill() { + Ok(()) => "kill sent".to_owned(), + Err(error) => format!("kill failed: {error}"), + }; + let reap = match child.wait() { + Ok(status) => format!("reaped with {status}"), + Err(error) => format!("reap failed: {error}"), + }; + format!("{kill}; {reap}") +} + +fn wait_with_deadline( + child: &mut Child, + label: &str, + timeout: Duration, +) -> Result { + let started = Instant::now(); + loop { + match child.try_wait() { + Ok(Some(status)) => return Ok(status), + Ok(None) => {} + Err(error) => { + let cleanup = terminate_and_reap(child); + return Err(format!("{label} wait failed: {error}; {cleanup}")); + } + } + let elapsed = started.elapsed(); + if elapsed >= timeout { + let cleanup = terminate_and_reap(child); + return Err(format!( + "{label} timed out after {} ms; {cleanup}", + timeout.as_millis() + )); + } + thread::sleep(Duration::from_millis(10).min(timeout - elapsed)); + } +} + +fn run_probe(executable: &Path, batch_path: &Path, label: &str) -> Output { + let mut child = Command::new(executable) + .stdin(Stdio::from( + File::open(batch_path).expect("open JSON fuzz batch for probe stdin"), + )) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap_or_else(|error| panic!("spawn {label}: {error}")); + let status = wait_with_deadline(&mut child, label, PROBE_TIMEOUT) + .unwrap_or_else(|error| panic!("{error}")); + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + child + .stdout + .take() + .expect("probe stdout should be piped") + .read_to_end(&mut stdout) + .expect("read probe stdout"); + child + .stderr + .take() + .expect("probe stderr should be piped") + .read_to_end(&mut stderr) + .expect("read probe stderr"); + Output { + status, + stdout, + stderr, + } +} + +fn native_probe_source() -> &'static str { + r#" +#include "runtime_shared.h" +#include +#include +#include +#include + +long long sengoo_json_parse_text_strict(long long data, long long len); +long long sengoo_json_doc_close(long long handle); +long long sengoo_json_doc_live_handle_count(void); +long long sengoo_json_last_error_code(void); +long long sengoo_json_last_error_kind(void); +long long sengoo_io_protocol_binary_mode(void); + +char* sengoo_copy_cstr_from_handle(long long value_ptr) { + const char* value = (const char*)(intptr_t)value_ptr; + if (!value) return NULL; + size_t len = strlen(value); + char* copy = (char*)malloc(len + 1); + if (!copy) return NULL; + memcpy(copy, value, len + 1); + return copy; +} + +#define EXPECT_ANY UINT32_MAX +#define EXPECT_ACCEPT (UINT32_MAX - 1U) +#define MAX_BATCH_CASE_BYTES (1024U * 1024U + 1U) + +static int read_exact(unsigned char* target, size_t len) { + size_t offset = 0; + while (offset < len) { + size_t count = fread(target + offset, 1, len - offset, stdin); + if (count == 0) return 0; + offset += count; + } + return 1; +} + +static int read_u32(uint32_t* value) { + unsigned char bytes[4]; + if (!read_exact(bytes, sizeof(bytes))) return 0; + *value = ((uint32_t)bytes[0] << 24) + | ((uint32_t)bytes[1] << 16) + | ((uint32_t)bytes[2] << 8) + | (uint32_t)bytes[3]; + return 1; +} + +int main(void) { + if (sengoo_io_protocol_binary_mode() != 0) return 1; + uint32_t total = 0; + if (!read_u32(&total) || total == 0 || total > 100000U) return 2; + long long baseline = sengoo_json_doc_live_handle_count(); + for (uint32_t index = 0; index < total; ++index) { + uint32_t expected = 0; + uint32_t len = 0; + if (!read_u32(&expected) || !read_u32(&len) || len > MAX_BATCH_CASE_BYTES) return 3; + unsigned char* bytes = len == 0 ? NULL : (unsigned char*)malloc(len); + if (len > 0 && (!bytes || !read_exact(bytes, len))) { + free(bytes); + return 4; + } + + long long handle = sengoo_json_parse_text_strict( + (long long)(intptr_t)bytes, + (long long)len); + if (expected == EXPECT_ACCEPT && handle == 0) { + fprintf(stderr, "case %u: expected accept, code=%lld kind=%lld\n", + index, sengoo_json_last_error_code(), sengoo_json_last_error_kind()); + free(bytes); + return 5; + } + if (expected != EXPECT_ANY && expected != EXPECT_ACCEPT) { + if (handle != 0 || sengoo_json_last_error_code() == 0 + || sengoo_json_last_error_kind() != (long long)expected) { + fprintf(stderr, "case %u: expected reject kind=%u, handle=%lld code=%lld kind=%lld\n", + index, expected, handle, sengoo_json_last_error_code(), sengoo_json_last_error_kind()); + if (handle != 0) sengoo_json_doc_close(handle); + free(bytes); + return 6; + } + } else if (expected == EXPECT_ANY && handle == 0 + && sengoo_json_last_error_code() == 0) { + fprintf(stderr, "case %u: rejection had no error code\n", index); + free(bytes); + return 7; + } + if (handle != 0 && sengoo_json_doc_close(handle) != 0) { + free(bytes); + return 8; + } + free(bytes); + if (sengoo_json_doc_live_handle_count() != baseline) { + fprintf(stderr, "case %u: live handle count grew\n", index); + return 9; + } + } + return 0; +} +"# +} + +fn sengoo_probe_source() -> &'static str { + r#" +import std::ffi; +import std::io; +import std::json; + +extern "C" { + fn sengoo_json_doc_live_handle_count() -> i64; +} + +def fuzz_fail(index: i64, code: i64) -> i64 { + let diagnostic = ffi_buffer_new(8).unwrap_or(Buffer { handle: 0 }); + if diagnostic.handle != 0 { + let wrote_index = diagnostic.write_u32_be(0, index); + let wrote_code = diagnostic.write_u32_be(4, code); + if wrote_index.is_ok and wrote_code.is_ok { + let ignored_write = io_stderr_write_raw(diagnostic.ptr(), 8); + let ignored_flush = io_stderr_flush(); + }; + let ignored_free = diagnostic.free(); + }; + code; +} + +def main() -> i64 { + let binary = io_protocol_binary_mode(); + if binary.is_err() { return 1; }; + let total_buffer = ffi_buffer_new(4).unwrap_or(Buffer { handle: 0 }); + if total_buffer.handle == 0 { return 2; }; + let total_read = io_stdin_read_exact(total_buffer, 0, 4); + if total_read.is_err() or total_read.value != 4 { total_buffer.free(); return 3; }; + let total_result = total_buffer.read_u32_be(0); + total_buffer.free(); + if total_result.is_err() or total_result.value <= 0 or total_result.value > 100000 { return 4; }; + + let baseline = sengoo_json_doc_live_handle_count(); + let mut index = 0; + while index < total_result.value { + let meta = ffi_buffer_new(8).unwrap_or(Buffer { handle: 0 }); + if meta.handle == 0 { return fuzz_fail(index, 5); }; + let meta_read = io_stdin_read_exact(meta, 0, 8); + if meta_read.is_err() or meta_read.value != 8 { meta.free(); return fuzz_fail(index, 6); }; + let expected_result = meta.read_u32_be(0); + let len_result = meta.read_u32_be(4); + meta.free(); + if expected_result.is_err() or len_result.is_err() or len_result.value > 1048577 { return fuzz_fail(index, 7); }; + + let allocation_len = if len_result.value == 0 { 1 } else { len_result.value }; + let payload = ffi_buffer_new(allocation_len).unwrap_or(Buffer { handle: 0 }); + if payload.handle == 0 { return fuzz_fail(index, 8); }; + if len_result.value > 0 { + let payload_read = io_stdin_read_exact(payload, 0, len_result.value); + if payload_read.is_err() or payload_read.value != len_result.value { payload.free(); return fuzz_fail(index, 9); }; + }; + + let parsed = json_parse_buffer_strict(payload, len_result.value); + let expected = expected_result.value; + if expected == 4294967294 { + if parsed.is_err() { payload.free(); return fuzz_fail(index, 10); }; + } else if expected != 4294967295 { + if parsed.is_ok { parsed.value.close(); payload.free(); return fuzz_fail(index, 11); }; + if json_last_error_code() == 0 or json_last_error_kind() != expected { payload.free(); return fuzz_fail(index, 12); }; + } else if parsed.is_err() and json_last_error_code() == 0 { + payload.free(); + return fuzz_fail(index, 13); + }; + if parsed.is_ok and not parsed.value.close() { payload.free(); return fuzz_fail(index, 14); }; + payload.free(); + if sengoo_json_doc_live_handle_count() != baseline { return fuzz_fail(index, 15); }; + index = index + 1; + }; + 0; +} +"# +} + +#[test] +fn native_runtime_strict_json_survives_fixed_corpus_and_seeded_mutations() { + let config = load_corpus(); + let cases = build_cases(&config); + let root = ProbeTempDir::new("native"); + let batch_path = root.path().join("batch.bin"); + fs::write(&batch_path, encode_batch(&cases)).expect("write native JSON fuzz batch"); + let source = root.path().join("probe.c"); + fs::write(&source, native_probe_source()).expect("write native JSON fuzz probe"); + let executable = root + .path() + .join(if cfg!(windows) { "probe.exe" } else { "probe" }); + let clang = which::which("clang") + .or_else(|_| which::which("clang.exe")) + .expect("native JSON fuzz probe requires clang"); + let stdlib = stdlib_dir(); + let mut compile = Command::new(clang); + compile + .arg("-std=c11") + .arg("-Wall") + .arg("-Wextra") + .arg("-Werror") + .arg("-I") + .arg(&stdlib) + .arg(&source) + .arg(stdlib.join("runtime.c")) + .arg(stdlib.join("runtime_string.c")) + .arg(stdlib.join("runtime_json.c")) + .arg("-o") + .arg(&executable); + if cfg!(windows) { + compile.args(["-Wno-unknown-pragmas", "-lws2_32", "-ladvapi32", "-lbcrypt"]); + } else { + compile.args(["-pthread", "-ldl", "-lm"]); + } + let compiled = compile + .output() + .expect("clang should compile native JSON fuzz probe"); + assert_success("native JSON fuzz probe compilation", &compiled); + + let output = run_probe(&executable, &batch_path, "native JSON fuzz probe"); + assert_probe_success("native JSON fuzz probe", &output, &cases); +} + +#[test] +fn real_sgc_strict_json_wrapper_survives_the_same_bounded_batch() { + let config = load_corpus(); + let cases = build_cases(&config); + let root = ProbeTempDir::new("real-sgc"); + let batch_path = root.path().join("batch.bin"); + fs::write(&batch_path, encode_batch(&cases)).expect("write real-sgc JSON fuzz batch"); + let source = root.path().join("main.sg"); + fs::write(&source, sengoo_probe_source()).expect("write real-sgc JSON fuzz source"); + let executable = root.path().join(if cfg!(windows) { + "json-fuzz-wrapper.exe" + } else { + "json-fuzz-wrapper" + }); + let built = source_sgc_command() + .arg("build") + .arg(&source) + .arg("--force-rebuild") + .arg("--output") + .arg(&executable) + .output() + .expect("real sgc should build JSON fuzz wrapper"); + assert_success("real-sgc JSON fuzz wrapper build", &built); + + let output = run_probe(&executable, &batch_path, "real-sgc JSON fuzz wrapper"); + assert_probe_success("real-sgc JSON fuzz wrapper", &output, &cases); +} diff --git a/tools/sgc/tests/language_reference_doctests.rs b/tools/sgc/tests/language_reference_doctests.rs index 820ada96..8c92ffe0 100644 --- a/tools/sgc/tests/language_reference_doctests.rs +++ b/tools/sgc/tests/language_reference_doctests.rs @@ -1,6 +1,9 @@ +mod common; + +use common::source_sgc_command; use std::fs; use std::path::{Path, PathBuf}; -use std::process::{Command, Output}; +use std::process::Output; use std::time::{SystemTime, UNIX_EPOCH}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -96,7 +99,7 @@ fn run_sgc(mode: DoctestMode, source: &Path) -> Output { DoctestMode::Compile => "check", DoctestMode::Run => "run", }; - Command::new(env!("CARGO_BIN_EXE_sgc")) + source_sgc_command() .arg(command) .arg(source) .args((mode == DoctestMode::Run).then_some("--force-rebuild")) diff --git a/tools/sgc/tests/numeric_runtime.rs b/tools/sgc/tests/numeric_runtime.rs index 96e8b845..548e9e1b 100644 --- a/tools/sgc/tests/numeric_runtime.rs +++ b/tools/sgc/tests/numeric_runtime.rs @@ -1,6 +1,8 @@ +mod common; + +use common::source_sgc_command; use std::fs; use std::path::PathBuf; -use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; struct TempSource { @@ -82,7 +84,7 @@ def main() -> i64 { "#, ); - let output = Command::new(env!("CARGO_BIN_EXE_sgc")) + let output = source_sgc_command() .arg("run") .arg(&source.path) .arg("--force-rebuild") @@ -117,7 +119,7 @@ def main() -> i64 { "#, ); - let output = Command::new(env!("CARGO_BIN_EXE_sgc")) + let output = source_sgc_command() .arg("run") .arg(&source.path) .arg("--force-rebuild") diff --git a/tools/sgc/tests/ordered_collections.rs b/tools/sgc/tests/ordered_collections.rs index d35f77e5..8cdb2852 100644 --- a/tools/sgc/tests/ordered_collections.rs +++ b/tools/sgc/tests/ordered_collections.rs @@ -1,3 +1,6 @@ +mod common; + +use common::source_sgc_command; use std::fs; use std::path::PathBuf; use std::process::Command; @@ -33,7 +36,7 @@ impl Drop for TempSource { fn run_native(name: &str, source: &str) { let source = TempSource::new(name, source); - let output = Command::new(env!("CARGO_BIN_EXE_sgc")) + let output = source_sgc_command() .arg("run") .arg(&source.path) .arg("--force-rebuild") diff --git a/tools/sgc/tests/portable_targets.rs b/tools/sgc/tests/portable_targets.rs index 76ec2151..078cd14a 100644 --- a/tools/sgc/tests/portable_targets.rs +++ b/tools/sgc/tests/portable_targets.rs @@ -1,12 +1,11 @@ +mod common; + +use common::source_sgc_command; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; -fn sgc() -> PathBuf { - PathBuf::from(env!("CARGO_BIN_EXE_sgc")) -} - fn temp_dir(name: &str) -> PathBuf { let stamp = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -66,7 +65,7 @@ fn bytecode_target_builds_and_runs_without_native_toolchain() { let dir = temp_dir("bytecode"); let source = write_scalar_program(&dir); let artifact = dir.join("app.sgbc"); - let build = Command::new(sgc()) + let build = source_sgc_command() .args([ "build", source.to_str().unwrap(), @@ -89,7 +88,7 @@ fn bytecode_target_builds_and_runs_without_native_toolchain() { "bytecode artifact should carry the stable magic" ); - let run = Command::new(sgc()) + let run = source_sgc_command() .args(["run", source.to_str().unwrap(), "--target", "bytecode"]) .env_remove("PATH") .output() @@ -108,12 +107,12 @@ fn bytecode_target_builds_and_runs_without_native_toolchain() { fn bytecode_target_matches_native_for_recursive_scalar_program() { let dir = temp_dir("bytecode_recursive"); let source = write_recursive_program(&dir); - let native = Command::new(sgc()) + let native = source_sgc_command() .args(["run", source.to_str().unwrap(), "--force-rebuild"]) .output() .expect("run native"); - let bytecode = Command::new(sgc()) + let bytecode = source_sgc_command() .args(["run", source.to_str().unwrap(), "--target", "bytecode"]) .env_remove("PATH") .output() @@ -135,7 +134,7 @@ fn wasm_target_emits_a_valid_exported_main_module() { let dir = temp_dir("wasm"); let source = write_scalar_program(&dir); let artifact = dir.join("app.wasm"); - let build = Command::new(sgc()) + let build = source_sgc_command() .args([ "build", source.to_str().unwrap(), @@ -191,7 +190,7 @@ def main() -> i64 { "#, ) .unwrap(); - let build = Command::new(sgc()) + let build = source_sgc_command() .args(["build", source.to_str().unwrap(), "--target", "bytecode"]) .output() .expect("build unsupported bytecode"); diff --git a/tools/sgc/tests/realworld.rs b/tools/sgc/tests/realworld.rs index 0cacb2b0..2d711b98 100644 --- a/tools/sgc/tests/realworld.rs +++ b/tools/sgc/tests/realworld.rs @@ -1,13 +1,13 @@ +mod common; + +use common::source_sgc_command; use serde_json::Value; use std::fs; +use std::io::Write; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Output, Stdio}; use std::time::{SystemTime, UNIX_EPOCH}; -fn sgc() -> PathBuf { - PathBuf::from(env!("CARGO_BIN_EXE_sgc")) -} - fn workspace_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) .parent() @@ -31,7 +31,7 @@ fn temp_dir(name: &str) -> PathBuf { } fn assert_sgc_check(path: &Path, module_map: Option) { - let mut command = Command::new(sgc()); + let mut command = source_sgc_command(); command.arg("check").arg(path); if let Some(module_map) = module_map { command.env("SENGOO_MODULE_MAP", module_map); @@ -46,6 +46,518 @@ fn assert_sgc_check(path: &Path, module_map: Option) { ); } +fn run_with_binary_stdin(executable: &Path, input: &[u8]) -> Output { + let mut child = std::process::Command::new(executable) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap_or_else(|err| panic!("spawn {}: {err}", executable.display())); + let mut stdin = child.stdin.take().expect("piped stdin should be available"); + stdin.write_all(input).expect("write binary test input"); + drop(stdin); + child.wait_with_output().expect("wait for binary fixture") +} + +fn framed(payload: &[u8]) -> Vec { + let mut bytes = Vec::with_capacity(4 + payload.len()); + bytes.extend_from_slice(&(payload.len() as u32).to_be_bytes()); + bytes.extend_from_slice(payload); + bytes +} + +fn unframe_exact(frame: &[u8]) -> &[u8] { + assert!(frame.len() >= 4, "framed payload needs a complete prefix"); + let len = u32::from_be_bytes(frame[..4].try_into().expect("four-byte frame prefix")) as usize; + assert_eq!(frame.len(), len + 4, "frame must contain one exact payload"); + &frame[4..] +} + +fn senline_worker_module_map(worker: &Path) -> std::ffi::OsString { + std::env::join_paths([ + format!( + "senline_domain_worker={}", + worker.join("src/lib.sg").display() + ), + format!( + "senline_build_identity={}", + worker + .join("packages/senline-build-identity/src/lib.sg") + .display() + ), + format!( + "senline_facts_to_plan={}", + worker + .join("packages/senline-facts-to-plan/src/lib.sg") + .display() + ), + format!( + "sgframing={}", + worker.join("packages/sgframing/src/lib.sg").display() + ), + format!( + "sgjson_contract={}", + worker.join("packages/sgjson-contract/src/lib.sg").display() + ), + ]) + .expect("encode Senline worker module map") +} + +fn build_senline_worker(test_name: &str) -> (PathBuf, PathBuf, PathBuf) { + let worker = realworld("senline-domain-worker"); + let fixtures = worker.join("fixtures/v1"); + let dir = temp_dir(test_name); + let executable = dir.join(if cfg!(windows) { + "senline_domain_worker.exe" + } else { + "senline_domain_worker" + }); + let compile = source_sgc_command() + .arg("build") + .arg(worker.join("src/main.sg")) + .arg("-o") + .arg(&executable) + .args(["-O", "0", "--force-rebuild"]) + .current_dir(&worker) + .env("SENGOO_MODULE_MAP", senline_worker_module_map(&worker)) + .output() + .expect("compile Senline domain worker"); + assert!( + compile.status.success(), + "compile stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&compile.stdout), + String::from_utf8_lossy(&compile.stderr) + ); + + (fixtures, dir, executable) +} + +fn assert_clean_worker_output(output: &Output, expected: &[u8]) { + assert_eq!( + output.status.code(), + Some(0), + "worker stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.stderr.is_empty(), "worker stderr must stay empty"); + assert_eq!(output.stdout, expected, "worker protocol bytes changed"); +} + +#[test] +fn senline_worker_emits_handshake_before_clean_eof() { + let (fixtures, dir, executable) = build_senline_worker("senline_worker_handshake_eof"); + let handshake = + fs::read(fixtures.join("handshake/ready.json")).expect("read handshake fixture"); + let output = run_with_binary_stdin(&executable, &[]); + + assert_clean_worker_output(&output, &framed(&handshake)); + + assert!(dir.starts_with(std::env::temp_dir())); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn senline_worker_eligible_fixture_roundtrips_exact_frames() { + let (fixtures, dir, executable) = build_senline_worker("senline_worker_eligible_roundtrip"); + let request = fs::read(fixtures.join("cases/eligible-accept.request.json")) + .expect("read eligible request fixture"); + let handshake = + fs::read(fixtures.join("handshake/ready.json")).expect("read handshake fixture"); + let plan = fs::read(fixtures.join("cases/eligible-accept.plan.json")) + .expect("read eligible plan fixture"); + let output = run_with_binary_stdin(&executable, &framed(&request)); + let mut expected = framed(&handshake); + expected.extend_from_slice(&framed(&plan)); + + assert_clean_worker_output(&output, &expected); + + assert!(dir.starts_with(std::env::temp_dir())); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn senline_worker_all_v1_fixtures_roundtrip_in_one_process() { + let (fixtures, dir, executable) = build_senline_worker("senline_worker_v1_roundtrip"); + let cases = [ + ("eligible-accept.request.json", "eligible-accept.plan.json"), + ("exact-duplicate.request.json", "exact-duplicate.plan.json"), + ( + "idempotency-conflict.request.json", + "idempotency-conflict.plan.json", + ), + ( + "application-budget-rejection.request.json", + "application-budget-rejection.plan.json", + ), + ( + "unknown-operation-version.request.json", + "unknown-operation-version.error.json", + ), + ]; + let mut input = Vec::new(); + let handshake = + fs::read(fixtures.join("handshake/ready.json")).expect("read handshake fixture"); + let mut expected = framed(&handshake); + for (request_name, response_name) in cases { + let request = fs::read(fixtures.join("cases").join(request_name)) + .unwrap_or_else(|err| panic!("read {request_name}: {err}")); + let response = fs::read(fixtures.join("cases").join(response_name)) + .unwrap_or_else(|err| panic!("read {response_name}: {err}")); + input.extend_from_slice(&framed(&request)); + expected.extend_from_slice(&framed(&response)); + } + let output = run_with_binary_stdin(&executable, &input); + + assert_clean_worker_output(&output, &expected); + + assert!(dir.starts_with(std::env::temp_dir())); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn senline_worker_recovers_after_malformed_json_frame() { + let (fixtures, dir, executable) = build_senline_worker("senline_worker_malformed_recovery"); + let malformed = b"{not-json}\n"; + let request = fs::read(fixtures.join("cases/eligible-accept.request.json")) + .expect("read eligible request fixture"); + let handshake = + fs::read(fixtures.join("handshake/ready.json")).expect("read handshake fixture"); + let protocol_error = fs::read(fixtures.join("errors/protocol-malformed-json.json")) + .expect("read malformed JSON error fixture"); + let plan = fs::read(fixtures.join("cases/eligible-accept.plan.json")) + .expect("read eligible plan fixture"); + let mut input = framed(malformed); + input.extend_from_slice(&framed(&request)); + let mut expected = framed(&handshake); + expected.extend_from_slice(&framed(&protocol_error)); + expected.extend_from_slice(&framed(&plan)); + let output = run_with_binary_stdin(&executable, &input); + + assert_clean_worker_output(&output, &expected); + + assert!(dir.starts_with(std::env::temp_dir())); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn senline_worker_recovers_after_extra_and_equal_count_unknown_fields() { + let (fixtures, dir, executable) = build_senline_worker("senline_worker_unknown_field_recovery"); + let request = fs::read(fixtures.join("cases/eligible-accept.request.json")) + .expect("read eligible request fixture"); + let valid: Value = serde_json::from_slice(&request).expect("decode request fixture"); + let mut extra = valid.clone(); + extra + .as_object_mut() + .expect("request fixture must be an object") + .insert("unexpected".to_owned(), Value::Bool(true)); + let extra = serde_json::to_vec(&extra).expect("encode request with extra unknown field"); + let mut substituted = valid.clone(); + let substituted_object = substituted + .as_object_mut() + .expect("request fixture must be an object"); + substituted_object.remove("context"); + substituted_object.insert("unexpected".to_owned(), Value::Bool(true)); + let substituted = + serde_json::to_vec(&substituted).expect("encode equal-count field substitution"); + let mut nested_substituted = valid; + let nested_identifiers = nested_substituted["facts"]["identifiers"] + .as_object_mut() + .expect("identifiers fixture must be an object"); + nested_identifiers.remove("correlation_ref"); + nested_identifiers.insert("unexpected".to_owned(), Value::Bool(true)); + let nested_substituted = serde_json::to_vec(&nested_substituted) + .expect("encode nested equal-count field substitution"); + let handshake = + fs::read(fixtures.join("handshake/ready.json")).expect("read handshake fixture"); + let protocol_error = fs::read(fixtures.join("errors/protocol-unknown-field.json")) + .expect("read unknown-field error fixture"); + let plan = fs::read(fixtures.join("cases/eligible-accept.plan.json")) + .expect("read eligible plan fixture"); + let mut input = framed(&extra); + input.extend_from_slice(&framed(&request)); + input.extend_from_slice(&framed(&substituted)); + input.extend_from_slice(&framed(&request)); + input.extend_from_slice(&framed(&nested_substituted)); + input.extend_from_slice(&framed(&request)); + let mut expected = framed(&handshake); + expected.extend_from_slice(&framed(&protocol_error)); + expected.extend_from_slice(&framed(&plan)); + expected.extend_from_slice(&framed(&protocol_error)); + expected.extend_from_slice(&framed(&plan)); + expected.extend_from_slice(&framed(&protocol_error)); + expected.extend_from_slice(&framed(&plan)); + let output = run_with_binary_stdin(&executable, &input); + + assert_clean_worker_output(&output, &expected); + + assert!(dir.starts_with(std::env::temp_dir())); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn senline_worker_recovers_after_each_schema_rejection_class() { + let (fixtures, dir, executable) = build_senline_worker("senline_worker_schema_recovery"); + let request = fs::read(fixtures.join("cases/eligible-accept.request.json")) + .expect("read eligible request fixture"); + let valid: Value = serde_json::from_slice(&request).expect("decode request fixture"); + + let mut missing = valid.clone(); + missing + .as_object_mut() + .expect("request fixture must be an object") + .remove("facts"); + + let mut wrong_kind = valid.clone(); + wrong_kind + .as_object_mut() + .expect("request fixture must be an object") + .insert("schema_version".to_owned(), Value::String("1".to_owned())); + + let mut unknown_enum = valid; + unknown_enum + .as_object_mut() + .expect("request fixture must be an object") + .insert("kind".to_owned(), Value::String("future".to_owned())); + + let mut out_of_range = unknown_enum.clone(); + out_of_range["kind"] = Value::String("evaluation".to_owned()); + out_of_range["facts"]["ciphertext_length_bytes"] = Value::from(4_294_967_296_u64); + + let malformed_error = fs::read(fixtures.join("errors/protocol-malformed-json.json")) + .expect("read malformed JSON error fixture"); + let unknown_enum_error = fs::read(fixtures.join("errors/protocol-unknown-enum.json")) + .expect("read unknown-enum error fixture"); + let rejected = [ + ( + serde_json::to_vec(&missing).expect("encode request missing a field"), + malformed_error.as_slice(), + ), + ( + serde_json::to_vec(&wrong_kind).expect("encode request with wrong field kind"), + malformed_error.as_slice(), + ), + ( + serde_json::to_vec(&unknown_enum).expect("encode request with unknown enum"), + unknown_enum_error.as_slice(), + ), + ( + serde_json::to_vec(&out_of_range).expect("encode request with out-of-range integer"), + malformed_error.as_slice(), + ), + ]; + + let handshake = + fs::read(fixtures.join("handshake/ready.json")).expect("read handshake fixture"); + let plan = fs::read(fixtures.join("cases/eligible-accept.plan.json")) + .expect("read eligible plan fixture"); + let mut input = Vec::new(); + let mut expected = framed(&handshake); + for (rejected_request, error) in rejected { + input.extend_from_slice(&framed(&rejected_request)); + input.extend_from_slice(&framed(&request)); + expected.extend_from_slice(&framed(error)); + expected.extend_from_slice(&framed(&plan)); + } + let output = run_with_binary_stdin(&executable, &input); + + assert_clean_worker_output(&output, &expected); + + assert!(dir.starts_with(std::env::temp_dir())); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn senline_worker_recovers_after_unsupported_operation_version() { + let (fixtures, dir, executable) = + build_senline_worker("senline_worker_unsupported_version_recovery"); + let unsupported = fs::read(fixtures.join("cases/unknown-operation-version.request.json")) + .expect("read unsupported-version request fixture"); + let unsupported_error = fs::read(fixtures.join("cases/unknown-operation-version.error.json")) + .expect("read unsupported-version error fixture"); + let request = fs::read(fixtures.join("cases/eligible-accept.request.json")) + .expect("read eligible request fixture"); + let plan = fs::read(fixtures.join("cases/eligible-accept.plan.json")) + .expect("read eligible plan fixture"); + let handshake = + fs::read(fixtures.join("handshake/ready.json")).expect("read handshake fixture"); + + let mut input = framed(&unsupported); + input.extend_from_slice(&framed(&request)); + let mut expected = framed(&handshake); + expected.extend_from_slice(&framed(&unsupported_error)); + expected.extend_from_slice(&framed(&plan)); + let output = run_with_binary_stdin(&executable, &input); + + assert_clean_worker_output(&output, &expected); + + assert!(dir.starts_with(std::env::temp_dir())); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn senline_worker_classifies_and_recovers_after_each_strict_parser_error() { + let (fixtures, dir, executable) = build_senline_worker("senline_worker_parser_recovery"); + let request = fs::read(fixtures.join("cases/eligible-accept.request.json")) + .expect("read eligible request fixture"); + let cases = [ + ( + "protocol-duplicate-field.request.raw", + "protocol-duplicate-field.json", + ), + ( + "protocol-invalid-unicode.request.raw", + "protocol-invalid-unicode.json", + ), + ( + "protocol-trailing-bytes.request.raw", + "protocol-trailing-bytes.json", + ), + ]; + let handshake = + fs::read(fixtures.join("handshake/ready.json")).expect("read handshake fixture"); + let plan = fs::read(fixtures.join("cases/eligible-accept.plan.json")) + .expect("read eligible plan fixture"); + let mut input = Vec::new(); + let mut expected = framed(&handshake); + for (request_name, response_name) in cases { + let rejected = fs::read(fixtures.join("errors").join(request_name)) + .unwrap_or_else(|err| panic!("read {request_name}: {err}")); + let response = fs::read(fixtures.join("errors").join(response_name)) + .unwrap_or_else(|err| panic!("read {response_name}: {err}")); + input.extend_from_slice(&framed(&rejected)); + input.extend_from_slice(&framed(&request)); + expected.extend_from_slice(&framed(&response)); + expected.extend_from_slice(&framed(&plan)); + } + let output = run_with_binary_stdin(&executable, &input); + + assert_clean_worker_output(&output, &expected); + + assert!(dir.starts_with(std::env::temp_dir())); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn senline_worker_preserves_embedded_nul_in_owned_plan_strings() { + let (fixtures, dir, executable) = build_senline_worker("senline_worker_embedded_nul"); + let request = fs::read(fixtures.join("cases/eligible-accept.request.json")) + .expect("read eligible request fixture"); + let mut request: Value = serde_json::from_slice(&request).expect("decode request fixture"); + let correlation_ref = "corr_ref\0suffix"; + request["facts"]["identifiers"]["correlation_ref"] = Value::String(correlation_ref.to_owned()); + let request = serde_json::to_vec(&request).expect("encode embedded-NUL request"); + let handshake = + fs::read(fixtures.join("handshake/ready.json")).expect("read handshake fixture"); + let handshake_frame = framed(&handshake); + let output = run_with_binary_stdin(&executable, &framed(&request)); + + assert_eq!( + output.status.code(), + Some(0), + "worker stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.stderr.is_empty(), "worker stderr must stay empty"); + assert!( + output.stdout.starts_with(&handshake_frame), + "worker handshake bytes changed" + ); + let response = unframe_exact(&output.stdout[handshake_frame.len()..]); + let plan: Value = serde_json::from_slice(response).expect("decode worker plan"); + assert_eq!(plan["kind"], "plan"); + assert_eq!( + plan["identifiers"]["correlation_ref"].as_str(), + Some(correlation_ref), + "owned string bytes after U+0000 must not be truncated" + ); + + assert!(dir.starts_with(std::env::temp_dir())); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn sgframing_binary_pipe_covers_boundaries_and_exact_output() { + let package = realworld("senline-domain-worker").join("packages/sgframing"); + let dir = temp_dir("sgframing_binary_pipes"); + let source = dir.join("frame_pipe.sg"); + fs::copy(package.join("tests/frame_pipe.sg"), &source).expect("copy pipe fixture"); + let executable = dir.join(if cfg!(windows) { + "sgframing_pipe.exe" + } else { + "sgframing_pipe" + }); + let module_map = format!("sgframing={}", package.join("src/lib.sg").display()); + let compile = source_sgc_command() + .arg("build") + .arg(&source) + .arg("-o") + .arg(&executable) + .args(["-O", "0", "--force-rebuild"]) + .current_dir(&package) + .env("SENGOO_MODULE_MAP", module_map) + .output() + .expect("compile sgframing pipe fixture"); + assert!( + compile.status.success(), + "compile stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&compile.stdout), + String::from_utf8_lossy(&compile.stderr) + ); + + let clean_eof = run_with_binary_stdin(&executable, &[]); + assert_eq!(clean_eof.status.code(), Some(0)); + assert!(clean_eof.stdout.is_empty()); + assert!(clean_eof.stderr.is_empty()); + + for payload in [ + vec![b'x'], + vec![0x00, 0x0a, 0x0d, 0x1a, 0x80, 0xff, 0x41, 0x7f], + vec![0x5a; 64], + ] { + let input = framed(&payload); + let echoed = run_with_binary_stdin(&executable, &input); + assert_eq!( + echoed.status.code(), + Some(0), + "valid frame stderr: {}", + String::from_utf8_lossy(&echoed.stderr) + ); + assert_eq!(echoed.stdout, input, "frame output must be exact"); + assert!(echoed.stderr.is_empty()); + } + + let prefix = 4_u32.to_be_bytes(); + for split in 1..4 { + let truncated = run_with_binary_stdin(&executable, &prefix[..split]); + assert_eq!(truncated.status.code(), Some(24), "prefix split {split}"); + assert!(truncated.stdout.is_empty()); + assert!(truncated.stderr.is_empty()); + } + + let payload = [0x10, 0x20, 0x30, 0x40]; + let complete = framed(&payload); + for payload_bytes in 0..payload.len() { + let truncated = run_with_binary_stdin(&executable, &complete[..4 + payload_bytes]); + assert_eq!( + truncated.status.code(), + Some(24), + "payload bytes {payload_bytes}" + ); + assert!(truncated.stdout.is_empty()); + assert!(truncated.stderr.is_empty()); + } + + for (prefix, expected_code) in [(0_u32.to_be_bytes(), 21), (65_u32.to_be_bytes(), 22)] { + let rejected = run_with_binary_stdin(&executable, &prefix); + assert_eq!(rejected.status.code(), Some(expected_code)); + assert!(rejected.stdout.is_empty()); + assert!(rejected.stderr.is_empty()); + } + + assert!(dir.starts_with(std::env::temp_dir())); + let _ = fs::remove_dir_all(dir); +} + #[test] fn realworld_sources_check_through_sgc_command() { assert_sgc_check(&realworld("async-channel-smoke").join("src/main.sg"), None); @@ -81,7 +593,7 @@ fn default_library_conformance_runs_generic_string_keyed_map_natively() { "default_library_conformance={}", fixture.join("src/lib.sg").display() ); - let output = Command::new(sgc()) + let output = source_sgc_command() .arg("run") .arg(fixture.join("src/main.sg")) .arg("--force-rebuild") @@ -106,7 +618,7 @@ fn realworld_missing_import_check_reports_json_diagnostic() { ) .unwrap(); - let output = Command::new(sgc()) + let output = source_sgc_command() .arg("--error-format") .arg("json") .arg("check") diff --git a/tools/sgc/tests/runtime_distribution.rs b/tools/sgc/tests/runtime_distribution.rs new file mode 100644 index 00000000..7a230dbc --- /dev/null +++ b/tools/sgc/tests/runtime_distribution.rs @@ -0,0 +1,1230 @@ +use serde_json::json; +use sha2::{Digest, Sha256}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +fn temp_dir(name: &str) -> PathBuf { + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = std::env::temp_dir().join(format!("sgc_installed_runtime_{name}_{stamp}")); + fs::create_dir_all(&dir).unwrap(); + dir +} + +fn host_target() -> &'static str { + if cfg!(windows) { + "x86_64-pc-windows-msvc" + } else if cfg!(target_os = "linux") { + "x86_64-unknown-linux-gnu" + } else { + "x86_64-apple-darwin" + } +} + +fn runtime_library_name() -> &'static str { + if cfg!(windows) { + "sengoo_runtime.lib" + } else { + "libsengoo_runtime.a" + } +} + +fn write_manifest_with_runtime_abi(install_root: &Path, abi_version: u32) { + let runtime_relative = format!( + "share/sengoo/runtime/{}/{}", + host_target(), + runtime_library_name() + ); + let manifest = json!({ + "schema_version": 2, + "version": env!("CARGO_PKG_VERSION"), + "target": host_target(), + "build_hash": "installed-runtime-test", + "build_manifest_id": "1111111111111111111111111111111111111111111111111111111111111111", + "payloads": [], + "native_runtime": { + "abi_version": abi_version, + "target": host_target(), + "library": runtime_relative, + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "link_args": [], + "dynamic_dependencies": [] + } + }); + fs::write( + install_root.join("manifest.json"), + serde_json::to_vec_pretty(&manifest).unwrap(), + ) + .unwrap(); +} + +fn update_manifest(install_root: &Path, update: impl FnOnce(&mut serde_json::Value)) { + let manifest_path = install_root.join("manifest.json"); + let mut manifest: serde_json::Value = + serde_json::from_slice(&fs::read(&manifest_path).unwrap()).unwrap(); + update(&mut manifest); + fs::write(manifest_path, serde_json::to_vec_pretty(&manifest).unwrap()).unwrap(); +} + +fn expected_runtime_link_args() -> Vec<&'static str> { + if cfg!(windows) { + vec![ + "kernel32.lib", + "ntdll.lib", + "userenv.lib", + "ws2_32.lib", + "dbghelp.lib", + "advapi32.lib", + "bcrypt.lib", + "crypt32.lib", + "ncrypt.lib", + "secur32.lib", + "legacy_stdio_definitions.lib", + "msvcrt.lib", + "vcruntime.lib", + "ucrt.lib", + ] + } else if cfg!(target_os = "macos") { + vec!["-framework", "Security", "-framework", "CoreFoundation"] + } else { + vec!["-lm"] + } +} + +fn workspace_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(2) + .unwrap() + .to_path_buf() +} + +fn copy_installed_stdlib(install_root: &Path) { + let source = workspace_root().join("tools").join("stdlib"); + let destination = install_root.join("share").join("sengoo").join("stdlib"); + fs::create_dir_all(&destination).unwrap(); + for entry in fs::read_dir(source).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + let name = path.file_name().unwrap(); + let is_runtime_bridge = name.to_string_lossy().starts_with("runtime") + && matches!( + path.extension().and_then(|value| value.to_str()), + Some("c" | "h") + ); + if path.extension().and_then(|value| value.to_str()) == Some("sg") || is_runtime_bridge { + fs::copy(&path, destination.join(name)).unwrap(); + } + } +} + +fn payload_entry(install_root: &Path, payload: &Path) -> serde_json::Value { + let relative = payload + .strip_prefix(install_root) + .unwrap() + .to_string_lossy() + .replace('\\', "/"); + let bytes = fs::read(payload).unwrap(); + json!({ + "path": relative, + "sha256": format!("{:x}", Sha256::digest(&bytes)), + "size": bytes.len() + }) +} + +fn runtime_bridge_paths(install_root: &Path) -> Vec { + let stdlib = install_root.join("share").join("sengoo").join("stdlib"); + [ + "runtime.c", + "runtime_breadth.c", + "runtime_collections.c", + "runtime_json.c", + "runtime_process.c", + "runtime_string.c", + "runtime_shared.h", + ] + .into_iter() + .map(|file| stdlib.join(file)) + .collect() +} + +fn assert_no_forbidden_paths(label: &str, text: &str, forbidden_paths: &[PathBuf]) { + let normalized_text = text.replace('\\', "/").to_ascii_lowercase(); + for path in forbidden_paths { + let normalized_path = fs::canonicalize(path) + .unwrap_or_else(|_| path.to_path_buf()) + .to_string_lossy() + .replace('\\', "/") + .to_ascii_lowercase(); + assert!( + !normalized_text.contains(&normalized_path), + "{label} exposes forbidden path {}", + path.display() + ); + } +} + +fn write_fake_cargo(bin_dir: &Path, clang: &Path) { + let source = bin_dir.join("fake_cargo.c"); + fs::write( + &source, + r#"#include +#include + +int main(void) { + const char* marker = getenv("SENGOO_FAKE_CARGO_MARKER"); + if (marker) { + FILE* output = fopen(marker, "wb"); + if (output) { + fputs("invoked", output); + fclose(output); + } + } + return 97; +} +"#, + ) + .unwrap(); + let executable = bin_dir.join(if cfg!(windows) { "cargo.exe" } else { "cargo" }); + let status = Command::new(clang) + .arg(&source) + .arg("-o") + .arg(&executable) + .status() + .expect("fake Cargo compiler should run"); + assert!(status.success(), "fake Cargo should compile"); +} + +#[test] +fn source_native_build_requires_explicit_source_runtime_mode_before_cargo() { + let Ok(clang) = which::which("clang").or_else(|_| which::which("clang.exe")) else { + eprintln!("skip: native clang toolchain unavailable"); + return; + }; + + let root = temp_dir("source_mode_required"); + let consumer = root.join("consumer"); + let fake_bin = root.join("fake-bin"); + fs::create_dir_all(&consumer).unwrap(); + fs::create_dir_all(&fake_bin).unwrap(); + fs::write(consumer.join("main.sg"), "def main() -> i64 { 0 }\n").unwrap(); + + let marker = root.join("cargo-invoked.txt"); + write_fake_cargo(&fake_bin, &clang); + let original_path = std::env::var_os("PATH").unwrap_or_default(); + let joined_path = std::env::join_paths( + std::iter::once(fake_bin.clone()).chain(std::env::split_paths(&original_path)), + ) + .unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_sgc")) + .current_dir(&consumer) + .args(["build", "main.sg", "--force-rebuild"]) + .env("PATH", joined_path) + .env("SENGOO_FAKE_CARGO_MARKER", &marker) + .output() + .expect("source sgc should execute"); + + assert!(!output.status.success(), "implicit source mode must fail"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("source runtime development mode is not selected") + && stderr.contains("source-development") + && stderr.contains("non-release Cargo runtime construction"), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + stderr + ); + assert!( + !marker.exists(), + "source-local layout alone must not authorize Cargo runtime construction" + ); + + let _ = fs::remove_dir_all(root); +} + +#[test] +fn explicit_source_runtime_mode_marks_cargo_build_as_non_release() { + let Ok(clang) = which::which("clang").or_else(|_| which::which("clang.exe")) else { + eprintln!("skip: native clang toolchain unavailable"); + return; + }; + + let root = temp_dir("source_mode_non_release"); + let consumer = root.join("consumer"); + let fake_bin = root.join("fake-bin"); + fs::create_dir_all(&consumer).unwrap(); + fs::create_dir_all(&fake_bin).unwrap(); + fs::write(consumer.join("main.sg"), "def main() -> i64 { 0 }\n").unwrap(); + + let marker = root.join("cargo-invoked.txt"); + write_fake_cargo(&fake_bin, &clang); + let original_path = std::env::var_os("PATH").unwrap_or_default(); + let joined_path = std::env::join_paths( + std::iter::once(fake_bin.clone()).chain(std::env::split_paths(&original_path)), + ) + .unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_sgc")) + .current_dir(&consumer) + .args([ + "--runtime-mode", + "source-development", + "build", + "main.sg", + "--force-rebuild", + ]) + .env("PATH", joined_path) + .env("SENGOO_FAKE_CARGO_MARKER", &marker) + .output() + .expect("source sgc should execute"); + + assert!( + !output.status.success(), + "the fake Cargo fixture should fail after source mode is authorized" + ); + assert!( + marker.exists(), + "explicit source mode must reach fake Cargo" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("[toolchain::source_runtime_development]") + && stderr.contains("runtime_mode=source-development") + && stderr.contains("artifact_provenance=source-cargo-development") + && stderr.contains("release_eligible=false") + && stderr.contains("senline_pin_evidence=false"), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + stderr + ); + + let _ = fs::remove_dir_all(root); +} + +#[test] +fn explicit_source_runtime_build_records_non_release_provenance() { + let root = temp_dir("source_mode_metadata"); + let consumer = root.join("consumer"); + fs::create_dir_all(&consumer).unwrap(); + fs::write(consumer.join("main.sg"), "def main() -> i64 { 0 }\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_sgc")) + .current_dir(&consumer) + .args([ + "--runtime-mode", + "source-development", + "build", + "main.sg", + "--force-rebuild", + ]) + .output() + .expect("source sgc should execute"); + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let metadata: serde_json::Value = serde_json::from_slice( + &fs::read(consumer.join("build").join("main.build-cache.json")).unwrap(), + ) + .unwrap(); + let provenance = &metadata["runtime_provenance"]; + assert_eq!(provenance["runtime_mode"], "source-development"); + assert_eq!( + provenance["artifact_provenance"], + "source-cargo-development" + ); + assert_eq!(provenance["release_eligible"], false); + assert_eq!(provenance["senline_pin_evidence"], false); + assert!(provenance["build_manifest_id"].is_null()); + + let _ = fs::remove_dir_all(root); +} + +#[test] +fn explicit_source_runtime_mode_propagates_to_test_children() { + let root = temp_dir("source_mode_test_child"); + let package = root.join("package"); + let tests = package.join("tests"); + fs::create_dir_all(&tests).unwrap(); + fs::write(tests.join("pass.sg"), "def main() -> i64 { 0 }\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_sgc")) + .current_dir(&package) + .args(["--runtime-mode", "source-development", "test", "."]) + .output() + .expect("source sgc test should execute"); + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("test result: 1 passed"), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let metadata: serde_json::Value = + serde_json::from_slice(&fs::read(tests.join("build").join("pass.run-cache.json")).unwrap()) + .unwrap(); + let provenance = &metadata["runtime_provenance"]; + assert_eq!(provenance["runtime_mode"], "source-development"); + assert_eq!( + provenance["artifact_provenance"], + "source-cargo-development" + ); + assert_eq!(provenance["release_eligible"], false); + assert_eq!(provenance["senline_pin_evidence"], false); + assert!(provenance["build_manifest_id"].is_null()); + + let _ = fs::remove_dir_all(root); +} + +#[test] +fn fresh_installed_build_and_run_avoid_cargo_and_absolute_runtime_identity() { + let clang = which::which("clang") + .or_else(|_| which::which("clang.exe")) + .expect("distribution smoke requires the native clang toolchain"); + let runtime_source = workspace_root() + .join("target") + .join("staticlib") + .join(runtime_library_name()); + assert!( + runtime_source.is_file(), + "distribution runtime fixture is missing: {}; build the staticlib profile first", + runtime_source.display() + ); + + let root = temp_dir("fresh_installed_smoke"); + let install_root = root.join("install"); + let install_bin = install_root.join("bin"); + let runtime_dir = install_root + .join("share") + .join("sengoo") + .join("runtime") + .join(host_target()); + let consumer = root.join("consumer"); + let fake_bin = root.join("fake-bin"); + let fake_home = root.join("forbidden-user-home"); + let fake_cargo_home = root.join("forbidden-cargo-home"); + fs::create_dir_all(&install_bin).unwrap(); + fs::create_dir_all(&runtime_dir).unwrap(); + fs::create_dir_all(&consumer).unwrap(); + fs::create_dir_all(&fake_bin).unwrap(); + fs::create_dir_all(&fake_home).unwrap(); + fs::create_dir_all(&fake_cargo_home).unwrap(); + copy_installed_stdlib(&install_root); + + let installed_sgc = install_bin.join(if cfg!(windows) { "sgc.exe" } else { "sgc" }); + fs::copy(env!("CARGO_BIN_EXE_sgc"), &installed_sgc).unwrap(); + let runtime_library = runtime_dir.join(runtime_library_name()); + fs::copy(&runtime_source, &runtime_library).unwrap(); + let runtime_hash = format!("{:x}", Sha256::digest(fs::read(&runtime_library).unwrap())); + write_manifest_with_runtime_abi(&install_root, 1); + update_manifest(&install_root, |manifest| { + manifest["artifact_provenance"] = json!("prebuilt-unverified"); + manifest["release_eligible"] = json!(false); + manifest["native_runtime"]["sha256"] = json!(runtime_hash); + manifest["native_runtime"]["link_args"] = json!(expected_runtime_link_args()); + let mut payloads = runtime_bridge_paths(&install_root) + .iter() + .map(|path| payload_entry(&install_root, path)) + .collect::>(); + payloads.push(payload_entry(&install_root, &runtime_library)); + manifest["payloads"] = json!(payloads); + }); + fs::write( + consumer.join("main.sg"), + "import std::status;\n\ndef main() -> i64 { STATUS_OK() }\n", + ) + .unwrap(); + fs::create_dir_all(consumer.join("tests")).unwrap(); + fs::write( + consumer.join("tests").join("pass.sg"), + "def main() -> i64 { 0 }\n", + ) + .unwrap(); + + let marker = root.join("cargo-invoked.txt"); + write_fake_cargo(&fake_bin, &clang); + let original_path = std::env::var_os("PATH").unwrap_or_default(); + let joined_path = std::env::join_paths( + std::iter::once(fake_bin.clone()).chain(std::env::split_paths(&original_path)), + ) + .unwrap(); + let mut command_transcript = String::new(); + for arguments in [ + vec!["check", "main.sg"], + vec!["build", "main.sg", "--force-rebuild"], + vec!["run", "main.sg", "--force-rebuild"], + vec!["test", "."], + ] { + let output = Command::new(&installed_sgc) + .current_dir(&consumer) + .args(&arguments) + .env("PATH", &joined_path) + .env("SENGOO_FAKE_CARGO_MARKER", &marker) + .env("HOME", &fake_home) + .env("USERPROFILE", &fake_home) + .env("CARGO_HOME", &fake_cargo_home) + .env_remove("SENGOO_ROOT") + .env_remove("SENGOO_STDLIB") + .env_remove("SENGOO_RUNTIME") + .output() + .expect("installed sgc command should execute"); + assert!( + output.status.success(), + "command {:?}\nstdout:\n{}\nstderr:\n{}", + arguments, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + command_transcript.push_str(&String::from_utf8_lossy(&output.stdout)); + command_transcript.push_str(&String::from_utf8_lossy(&output.stderr)); + } + assert!(!marker.exists(), "installed commands must not invoke Cargo"); + + let forbidden_paths = vec![ + workspace_root(), + workspace_root().join("target"), + fake_home.clone(), + fake_cargo_home.clone(), + ]; + assert_no_forbidden_paths( + "installed command output", + &command_transcript, + &forbidden_paths, + ); + for metadata_path in [ + consumer.join("build").join("main.build-cache.json"), + consumer.join("build").join("main.run-cache.json"), + consumer + .join("tests") + .join("build") + .join("pass.run-cache.json"), + ] { + let metadata_text = fs::read_to_string(&metadata_path).unwrap(); + assert_no_forbidden_paths( + &format!("installed cache metadata {}", metadata_path.display()), + &metadata_text, + &forbidden_paths, + ); + let metadata: serde_json::Value = serde_json::from_str(&metadata_text).unwrap(); + assert_eq!(metadata["runtime_provenance"]["runtime_mode"], "installed"); + assert_eq!( + metadata["runtime_provenance"]["artifact_provenance"], + "prebuilt-unverified" + ); + assert_eq!(metadata["runtime_provenance"]["release_eligible"], false); + assert_eq!( + metadata["runtime_provenance"]["senline_pin_evidence"], + false + ); + assert_eq!( + metadata["runtime_provenance"]["build_manifest_id"], + "1111111111111111111111111111111111111111111111111111111111111111" + ); + let runtime_identity = metadata["runtime_c"].as_str().unwrap_or_default(); + assert_eq!( + runtime_identity, "installed:share/sengoo/stdlib/runtime.c", + "runtime identity must not expose an install, checkout, Cargo, or user-profile path" + ); + } + + let manifest_text = fs::read_to_string(install_root.join("manifest.json")).unwrap(); + assert_no_forbidden_paths("installed manifest", &manifest_text, &forbidden_paths); + assert!(!manifest_text.contains(&workspace_root().to_string_lossy().to_string())); + assert!(!manifest_text.to_ascii_lowercase().contains("cargo")); + + update_manifest(&install_root, |manifest| { + manifest["artifact_provenance"] = json!("forged-local-claim"); + manifest["release_eligible"] = json!(true); + }); + let output = Command::new(&installed_sgc) + .current_dir(&consumer) + .args(["build", "main.sg"]) + .env("PATH", &joined_path) + .env("SENGOO_FAKE_CARGO_MARKER", &marker) + .env("HOME", &fake_home) + .env("USERPROFILE", &fake_home) + .env("CARGO_HOME", &fake_cargo_home) + .env_remove("SENGOO_ROOT") + .env_remove("SENGOO_STDLIB") + .env_remove("SENGOO_RUNTIME") + .output() + .expect("installed sgc build should execute after provenance changes"); + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("build cache miss: metadata changed"), + "provenance changes must invalidate installed build cache identity\nstdout:\n{}", + String::from_utf8_lossy(&output.stdout) + ); + let metadata: serde_json::Value = serde_json::from_slice( + &fs::read(consumer.join("build").join("main.build-cache.json")).unwrap(), + ) + .unwrap(); + assert_eq!( + metadata["runtime_provenance"]["artifact_provenance"], + "forged-local-claim" + ); + assert_eq!(metadata["runtime_provenance"]["release_eligible"], true); + assert_eq!( + metadata["runtime_provenance"]["senline_pin_evidence"], false, + "an installed manifest cannot self-authenticate as Senline pin evidence" + ); + + let _ = fs::remove_dir_all(root); +} + +#[test] +fn installed_check_rejects_wrong_runtime_abi() { + let root = temp_dir("check_wrong_abi"); + let install_root = root.join("install"); + let install_bin = install_root.join("bin"); + let consumer = root.join("consumer"); + fs::create_dir_all(&install_bin).unwrap(); + fs::create_dir_all(&consumer).unwrap(); + + let installed_sgc = install_bin.join(if cfg!(windows) { "sgc.exe" } else { "sgc" }); + fs::copy(env!("CARGO_BIN_EXE_sgc"), &installed_sgc).unwrap(); + write_manifest_with_runtime_abi(&install_root, 2); + fs::write(consumer.join("main.sg"), "def main() -> i64 { 0 }\n").unwrap(); + + let output = Command::new(&installed_sgc) + .current_dir(&consumer) + .args(["check", "main.sg"]) + .env_remove("SENGOO_ROOT") + .env_remove("SENGOO_STDLIB") + .env_remove("SENGOO_RUNTIME") + .output() + .expect("installed sgc check should execute"); + + assert!( + !output.status.success(), + "wrong runtime ABI must fail check" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("installed native runtime ABI mismatch") + && stderr.contains("manifest=2") + && stderr.contains("supported=1"), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + stderr + ); + + let _ = fs::remove_dir_all(root); +} + +#[test] +fn installed_check_rejects_wrong_toolchain_target() { + let root = temp_dir("check_wrong_target"); + let install_root = root.join("install"); + let install_bin = install_root.join("bin"); + let consumer = root.join("consumer"); + fs::create_dir_all(&install_bin).unwrap(); + fs::create_dir_all(&consumer).unwrap(); + + let installed_sgc = install_bin.join(if cfg!(windows) { "sgc.exe" } else { "sgc" }); + fs::copy(env!("CARGO_BIN_EXE_sgc"), &installed_sgc).unwrap(); + write_manifest_with_runtime_abi(&install_root, 1); + update_manifest(&install_root, |manifest| { + manifest["target"] = json!("aarch64-unknown-invalid"); + }); + fs::write(consumer.join("main.sg"), "def main() -> i64 { 0 }\n").unwrap(); + + let output = Command::new(&installed_sgc) + .current_dir(&consumer) + .args(["check", "main.sg"]) + .output() + .expect("installed sgc check should execute"); + + assert!(!output.status.success(), "wrong target must fail check"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("installed toolchain target mismatch") + && stderr.contains("manifest=aarch64-unknown-invalid") + && stderr.contains(host_target()), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + stderr + ); + + let _ = fs::remove_dir_all(root); +} + +#[test] +fn installed_check_rejects_incomplete_runtime_metadata() { + let root = temp_dir("check_incomplete_metadata"); + let install_root = root.join("install"); + let install_bin = install_root.join("bin"); + let consumer = root.join("consumer"); + fs::create_dir_all(&install_bin).unwrap(); + fs::create_dir_all(&consumer).unwrap(); + + let installed_sgc = install_bin.join(if cfg!(windows) { "sgc.exe" } else { "sgc" }); + fs::copy(env!("CARGO_BIN_EXE_sgc"), &installed_sgc).unwrap(); + write_manifest_with_runtime_abi(&install_root, 1); + update_manifest(&install_root, |manifest| { + manifest["native_runtime"] + .as_object_mut() + .unwrap() + .remove("link_args"); + }); + fs::write(consumer.join("main.sg"), "def main() -> i64 { 0 }\n").unwrap(); + + let output = Command::new(&installed_sgc) + .current_dir(&consumer) + .args(["check", "main.sg"]) + .output() + .expect("installed sgc check should execute"); + + assert!( + !output.status.success(), + "missing runtime metadata must fail check" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("invalid installed toolchain manifest") + && stderr.contains("missing field `link_args`"), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + stderr + ); + + let _ = fs::remove_dir_all(root); +} + +#[test] +fn installed_check_rejects_incomplete_runtime_bridge() { + let root = temp_dir("check_incomplete_bridge"); + let install_root = root.join("install"); + let install_bin = install_root.join("bin"); + let runtime_dir = install_root + .join("share") + .join("sengoo") + .join("runtime") + .join(host_target()); + let stdlib_dir = install_root.join("share").join("sengoo").join("stdlib"); + let consumer = root.join("consumer"); + fs::create_dir_all(&install_bin).unwrap(); + fs::create_dir_all(&runtime_dir).unwrap(); + fs::create_dir_all(&stdlib_dir).unwrap(); + fs::create_dir_all(&consumer).unwrap(); + + let installed_sgc = install_bin.join(if cfg!(windows) { "sgc.exe" } else { "sgc" }); + fs::copy(env!("CARGO_BIN_EXE_sgc"), &installed_sgc).unwrap(); + let runtime_library = runtime_dir.join(runtime_library_name()); + fs::write(&runtime_library, b"runtime fixture").unwrap(); + let runtime_hash = format!("{:x}", Sha256::digest(b"runtime fixture")); + write_manifest_with_runtime_abi(&install_root, 1); + update_manifest(&install_root, |manifest| { + manifest["native_runtime"]["sha256"] = json!(runtime_hash); + manifest["native_runtime"]["link_args"] = json!(expected_runtime_link_args()); + }); + for file in [ + "runtime.c", + "runtime_breadth.c", + "runtime_collections.c", + "runtime_process.c", + "runtime_string.c", + "runtime_shared.h", + ] { + fs::write(stdlib_dir.join(file), b"bridge fixture").unwrap(); + } + fs::write(consumer.join("main.sg"), "def main() -> i64 { 0 }\n").unwrap(); + + let output = Command::new(&installed_sgc) + .current_dir(&consumer) + .args(["check", "main.sg"]) + .output() + .expect("installed sgc check should execute"); + + assert!( + !output.status.success(), + "missing runtime bridge file must fail check" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("installed native runtime bridge file is missing") + && stderr.contains("runtime_json.c"), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + stderr + ); + + let _ = fs::remove_dir_all(root); +} + +#[test] +fn installed_check_rejects_tampered_runtime_bridge_payload() { + let root = temp_dir("check_tampered_bridge"); + let install_root = root.join("install"); + let install_bin = install_root.join("bin"); + let runtime_dir = install_root + .join("share") + .join("sengoo") + .join("runtime") + .join(host_target()); + let stdlib_dir = install_root.join("share").join("sengoo").join("stdlib"); + let consumer = root.join("consumer"); + fs::create_dir_all(&install_bin).unwrap(); + fs::create_dir_all(&runtime_dir).unwrap(); + fs::create_dir_all(&stdlib_dir).unwrap(); + fs::create_dir_all(&consumer).unwrap(); + + let installed_sgc = install_bin.join(if cfg!(windows) { "sgc.exe" } else { "sgc" }); + fs::copy(env!("CARGO_BIN_EXE_sgc"), &installed_sgc).unwrap(); + let runtime_library = runtime_dir.join(runtime_library_name()); + fs::write(&runtime_library, b"runtime fixture").unwrap(); + let runtime_hash = format!("{:x}", Sha256::digest(b"runtime fixture")); + for path in runtime_bridge_paths(&install_root) { + fs::write(path, b"bridge fixture").unwrap(); + } + write_manifest_with_runtime_abi(&install_root, 1); + update_manifest(&install_root, |manifest| { + manifest["native_runtime"]["sha256"] = json!(runtime_hash); + manifest["native_runtime"]["link_args"] = json!(expected_runtime_link_args()); + let mut payloads = runtime_bridge_paths(&install_root) + .iter() + .map(|path| payload_entry(&install_root, path)) + .collect::>(); + payloads.push(payload_entry(&install_root, &runtime_library)); + manifest["payloads"] = json!(payloads); + }); + fs::write( + stdlib_dir.join("runtime_json.c"), + b"tampered bridge fixture", + ) + .unwrap(); + fs::write(consumer.join("main.sg"), "def main() -> i64 { 0 }\n").unwrap(); + + let output = Command::new(&installed_sgc) + .current_dir(&consumer) + .args(["check", "main.sg"]) + .output() + .expect("installed sgc check should execute"); + + assert!( + !output.status.success(), + "tampered runtime bridge must fail check" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("installed runtime payload SHA-256 mismatch") + && stderr.contains("runtime_json.c"), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + stderr + ); + + let _ = fs::remove_dir_all(root); +} + +#[test] +fn installed_commands_reject_external_runtime_overrides() { + let root = temp_dir("check_external_runtime_override"); + let install_root = root.join("install"); + let install_bin = install_root.join("bin"); + let runtime_dir = install_root + .join("share") + .join("sengoo") + .join("runtime") + .join(host_target()); + let consumer = root.join("consumer"); + fs::create_dir_all(&install_bin).unwrap(); + fs::create_dir_all(&runtime_dir).unwrap(); + fs::create_dir_all(&consumer).unwrap(); + copy_installed_stdlib(&install_root); + + let installed_sgc = install_bin.join(if cfg!(windows) { "sgc.exe" } else { "sgc" }); + fs::copy(env!("CARGO_BIN_EXE_sgc"), &installed_sgc).unwrap(); + let runtime_library = runtime_dir.join(runtime_library_name()); + fs::write(&runtime_library, b"runtime fixture").unwrap(); + let runtime_hash = format!("{:x}", Sha256::digest(b"runtime fixture")); + write_manifest_with_runtime_abi(&install_root, 1); + update_manifest(&install_root, |manifest| { + manifest["native_runtime"]["sha256"] = json!(runtime_hash); + manifest["native_runtime"]["link_args"] = json!(expected_runtime_link_args()); + let mut payloads = runtime_bridge_paths(&install_root) + .iter() + .map(|path| payload_entry(&install_root, path)) + .collect::>(); + payloads.push(payload_entry(&install_root, &runtime_library)); + manifest["payloads"] = json!(payloads); + }); + fs::write(consumer.join("main.sg"), "def main() -> i64 { 0 }\n").unwrap(); + let external_runtime = root.join("external-runtime.c"); + fs::write(&external_runtime, "int sengoo_external_runtime = 1;\n").unwrap(); + let external_stdlib = root.join("external-stdlib"); + let external_root = root.join("external-root"); + fs::create_dir_all(&external_stdlib).unwrap(); + fs::create_dir_all(&external_root).unwrap(); + + for (variable, value) in [ + ("SENGOO_RUNTIME", external_runtime.as_path()), + ("SENGOO_STDLIB", external_stdlib.as_path()), + ("SENGOO_ROOT", external_root.as_path()), + ] { + for arguments in [ + vec!["check", "main.sg"], + vec!["build", "main.sg", "--emit-llvm"], + vec!["run", "--cranelift-fast-jit", "main.sg"], + vec!["test", "."], + ] { + let output = Command::new(&installed_sgc) + .current_dir(&consumer) + .args(&arguments) + .env_remove("SENGOO_ROOT") + .env_remove("SENGOO_STDLIB") + .env_remove("SENGOO_RUNTIME") + .env(variable, value) + .output() + .expect("installed sgc command should execute"); + + assert!( + !output.status.success(), + "installed {:?} must reject {variable} overrides", + arguments + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains(&format!("installed runtime mode rejects {variable}")), + "command {:?}\nstdout:\n{}\nstderr:\n{}", + arguments, + String::from_utf8_lossy(&output.stdout), + stderr + ); + } + } + + let _ = fs::remove_dir_all(root); +} + +#[test] +fn installed_test_rejects_wrong_runtime_abi_before_empty_suite_success() { + let root = temp_dir("test_wrong_abi"); + let install_root = root.join("install"); + let install_bin = install_root.join("bin"); + let package = root.join("package"); + fs::create_dir_all(&install_bin).unwrap(); + fs::create_dir_all(&package).unwrap(); + + let installed_sgc = install_bin.join(if cfg!(windows) { "sgc.exe" } else { "sgc" }); + fs::copy(env!("CARGO_BIN_EXE_sgc"), &installed_sgc).unwrap(); + write_manifest_with_runtime_abi(&install_root, 2); + + let output = Command::new(&installed_sgc) + .current_dir(&package) + .args(["test", "."]) + .env_remove("SENGOO_ROOT") + .env_remove("SENGOO_STDLIB") + .env_remove("SENGOO_RUNTIME") + .output() + .expect("installed sgc test should execute"); + + assert!( + !output.status.success(), + "wrong runtime ABI must fail before an empty suite reports success" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("installed native runtime ABI mismatch") + && stderr.contains("manifest=2") + && stderr.contains("supported=1"), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + stderr + ); + + let _ = fs::remove_dir_all(root); +} + +#[test] +fn installed_native_build_rejects_missing_manifest_runtime_without_cargo_or_checkout_fallback() { + let Ok(clang) = which::which("clang").or_else(|_| which::which("clang.exe")) else { + eprintln!("skip: native clang toolchain unavailable"); + return; + }; + + let root = temp_dir("missing_runtime"); + let install_root = root.join("install"); + let install_bin = install_root.join("bin"); + let runtime_dir = install_root + .join("share") + .join("sengoo") + .join("runtime") + .join(host_target()); + let consumer = root.join("consumer"); + let fake_bin = root.join("fake-bin"); + fs::create_dir_all(&install_bin).unwrap(); + fs::create_dir_all(&runtime_dir).unwrap(); + fs::create_dir_all(&consumer).unwrap(); + fs::create_dir_all(&fake_bin).unwrap(); + + let installed_sgc = install_bin.join(if cfg!(windows) { "sgc.exe" } else { "sgc" }); + fs::copy(env!("CARGO_BIN_EXE_sgc"), &installed_sgc).unwrap(); + + let runtime_relative = format!( + "share/sengoo/runtime/{}/{}", + host_target(), + runtime_library_name() + ); + let manifest = json!({ + "schema_version": 2, + "version": env!("CARGO_PKG_VERSION"), + "target": host_target(), + "build_hash": "installed-runtime-test", + "build_manifest_id": "1111111111111111111111111111111111111111111111111111111111111111", + "payloads": [], + "native_runtime": { + "abi_version": 1, + "target": host_target(), + "library": runtime_relative, + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "link_args": [], + "dynamic_dependencies": [] + } + }); + fs::write( + install_root.join("manifest.json"), + serde_json::to_vec_pretty(&manifest).unwrap(), + ) + .unwrap(); + let source = consumer.join("main.sg"); + fs::write(&source, "def main() -> i64 { 0 }\n").unwrap(); + + let marker = root.join("cargo-invoked.txt"); + write_fake_cargo(&fake_bin, &clang); + let original_path = std::env::var_os("PATH").unwrap_or_default(); + let joined_path = std::env::join_paths( + std::iter::once(fake_bin.clone()).chain(std::env::split_paths(&original_path)), + ) + .unwrap(); + let output = Command::new(&installed_sgc) + .current_dir(&consumer) + .args(["build", "main.sg", "--force-rebuild"]) + .env("PATH", joined_path) + .env("SENGOO_FAKE_CARGO_MARKER", &marker) + .env_remove("SENGOO_ROOT") + .env_remove("SENGOO_STDLIB") + .env_remove("SENGOO_RUNTIME") + .output() + .expect("installed sgc should execute"); + + assert!( + !output.status.success(), + "missing runtime must fail the build" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("installed native runtime library is missing") + && stderr.contains(runtime_library_name()), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + stderr + ); + assert!( + !marker.exists(), + "installed builds must not invoke Cargo or fall back to the source checkout" + ); + + let _ = fs::remove_dir_all(root); +} + +#[test] +fn relocated_sgc_without_manifest_rejects_implicit_source_checkout_fallback() { + let Ok(clang) = which::which("clang").or_else(|_| which::which("clang.exe")) else { + eprintln!("skip: native clang toolchain unavailable"); + return; + }; + + let root = temp_dir("missing_manifest"); + let install_bin = root.join("relocated").join("bin"); + let consumer = root.join("consumer"); + let fake_bin = root.join("fake-bin"); + fs::create_dir_all(&install_bin).unwrap(); + fs::create_dir_all(&consumer).unwrap(); + fs::create_dir_all(&fake_bin).unwrap(); + let relocated_sgc = install_bin.join(if cfg!(windows) { "sgc.exe" } else { "sgc" }); + fs::copy(env!("CARGO_BIN_EXE_sgc"), &relocated_sgc).unwrap(); + fs::write(consumer.join("main.sg"), "def main() -> i64 { 0 }\n").unwrap(); + + let priming = Command::new(env!("CARGO_BIN_EXE_sgc")) + .current_dir(&consumer) + .args([ + "--runtime-mode", + "source-development", + "build", + "main.sg", + "--force-rebuild", + ]) + .output() + .expect("source compiler should prime the native build cache"); + assert!( + priming.status.success(), + "cache priming failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&priming.stdout), + String::from_utf8_lossy(&priming.stderr) + ); + + let marker = root.join("cargo-invoked.txt"); + write_fake_cargo(&fake_bin, &clang); + let original_path = std::env::var_os("PATH").unwrap_or_default(); + let joined_path = std::env::join_paths( + std::iter::once(fake_bin.clone()).chain(std::env::split_paths(&original_path)), + ) + .unwrap(); + let output = Command::new(&relocated_sgc) + .current_dir(&consumer) + .args(["build", "main.sg"]) + .env("PATH", joined_path) + .env("SENGOO_FAKE_CARGO_MARKER", &marker) + .env_remove("SENGOO_ROOT") + .env_remove("SENGOO_STDLIB") + .env_remove("SENGOO_RUNTIME") + .output() + .expect("relocated sgc should execute"); + + assert!( + !output.status.success(), + "missing manifest must fail the build" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("installed toolchain manifest is missing") + && stderr.contains("Cargo fallback is disabled"), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + stderr + ); + assert!( + !marker.exists(), + "relocated compilers must not fall back to Cargo or a compiled-in checkout" + ); + + let _ = fs::remove_dir_all(root); +} + +#[test] +fn installed_native_build_rejects_runtime_hash_mismatch_before_link_or_cargo() { + let Ok(clang) = which::which("clang").or_else(|_| which::which("clang.exe")) else { + eprintln!("skip: native clang toolchain unavailable"); + return; + }; + + let root = temp_dir("hash_mismatch"); + let install_root = root.join("install"); + let install_bin = install_root.join("bin"); + let runtime_dir = install_root + .join("share") + .join("sengoo") + .join("runtime") + .join(host_target()); + let consumer = root.join("consumer"); + let fake_bin = root.join("fake-bin"); + fs::create_dir_all(&install_bin).unwrap(); + fs::create_dir_all(&runtime_dir).unwrap(); + fs::create_dir_all(&consumer).unwrap(); + fs::create_dir_all(&fake_bin).unwrap(); + + let installed_sgc = install_bin.join(if cfg!(windows) { "sgc.exe" } else { "sgc" }); + fs::copy(env!("CARGO_BIN_EXE_sgc"), &installed_sgc).unwrap(); + let runtime_library = runtime_dir.join(runtime_library_name()); + fs::write(&runtime_library, b"not a native runtime library").unwrap(); + + let runtime_relative = format!( + "share/sengoo/runtime/{}/{}", + host_target(), + runtime_library_name() + ); + let manifest = json!({ + "schema_version": 2, + "version": env!("CARGO_PKG_VERSION"), + "target": host_target(), + "build_hash": "installed-runtime-test", + "build_manifest_id": "1111111111111111111111111111111111111111111111111111111111111111", + "payloads": [], + "native_runtime": { + "abi_version": 1, + "target": host_target(), + "library": runtime_relative, + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "link_args": [], + "dynamic_dependencies": [] + } + }); + fs::write( + install_root.join("manifest.json"), + serde_json::to_vec_pretty(&manifest).unwrap(), + ) + .unwrap(); + fs::write(consumer.join("main.sg"), "def main() -> i64 { 0 }\n").unwrap(); + + let priming = Command::new(env!("CARGO_BIN_EXE_sgc")) + .current_dir(&consumer) + .args([ + "--runtime-mode", + "source-development", + "build", + "main.sg", + "--force-rebuild", + ]) + .output() + .expect("source compiler should prime the native build cache"); + assert!( + priming.status.success(), + "cache priming failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&priming.stdout), + String::from_utf8_lossy(&priming.stderr) + ); + + let marker = root.join("cargo-invoked.txt"); + write_fake_cargo(&fake_bin, &clang); + let original_path = std::env::var_os("PATH").unwrap_or_default(); + let joined_path = std::env::join_paths( + std::iter::once(fake_bin.clone()).chain(std::env::split_paths(&original_path)), + ) + .unwrap(); + let output = Command::new(&installed_sgc) + .current_dir(&consumer) + .args(["build", "main.sg"]) + .env("PATH", joined_path) + .env("SENGOO_FAKE_CARGO_MARKER", &marker) + .env_remove("SENGOO_ROOT") + .env_remove("SENGOO_STDLIB") + .env_remove("SENGOO_RUNTIME") + .output() + .expect("installed sgc should execute"); + + assert!( + !output.status.success(), + "tampered runtime must fail the build" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("installed native runtime SHA-256 mismatch") + && stderr.contains(runtime_library_name()), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + stderr + ); + assert!( + !marker.exists(), + "runtime verification must not invoke Cargo or the source checkout" + ); + + let _ = fs::remove_dir_all(root); +} diff --git a/tools/sgc/tests/runtime_handles.rs b/tools/sgc/tests/runtime_handles.rs new file mode 100644 index 00000000..03e4b8c1 --- /dev/null +++ b/tools/sgc/tests/runtime_handles.rs @@ -0,0 +1,76 @@ +use std::fs; +use std::path::PathBuf; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[test] +fn generation_handle_encoding_stays_positive_and_signals_exhaustion() { + let Some(clang) = which::which("clang").ok() else { + eprintln!("skipping generation-handle probe: clang not found"); + return; + }; + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be after unix epoch") + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "sengoo-generation-handle-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&root).expect("create generation-handle probe directory"); + let source = root.join("probe.c"); + let executable = root.join(if cfg!(windows) { "probe.exe" } else { "probe" }); + fs::write( + &source, + r#" +#include "runtime_shared.h" +#include + +int main(void) { + if (SENGOO_RUNTIME_HANDLE_GENERATION_MAX != UINT32_C(0x7fffffff)) return 1; + if (sengoo_runtime_next_handle_generation(0) != 1) return 2; + if (sengoo_runtime_next_handle_generation(1) != 2) return 3; + if (sengoo_runtime_next_handle_generation(UINT32_C(0x7ffffffe)) != UINT32_C(0x7fffffff)) return 4; + if (sengoo_runtime_next_handle_generation(UINT32_C(0x7fffffff)) != 0) return 5; + if (sengoo_runtime_encode_handle(1, 0) != INT64_C(0x0000000100000001)) return 6; + if (sengoo_runtime_encode_handle(UINT32_C(0x7fffffff), (size_t)UINT32_MAX - 1) != INT64_MAX) return 7; + if (sengoo_runtime_encode_handle(0, 0) != 0) return 8; + if (sengoo_runtime_encode_handle(UINT32_C(0x80000000), 0) != 0) return 9; + if (sengoo_runtime_encode_handle(1, (size_t)UINT32_MAX) != 0) return 10; + return 0; +} +"#, + ) + .expect("write generation-handle probe"); + + let stdlib = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("stdlib"); + let compile = Command::new(clang) + .arg("-std=c11") + .arg("-I") + .arg(&stdlib) + .arg(&source) + .arg("-o") + .arg(&executable) + .output() + .expect("clang should compile generation-handle probe"); + assert!( + compile.status.success(), + "generation-handle probe failed to compile\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&compile.stdout), + String::from_utf8_lossy(&compile.stderr) + ); + + let output = Command::new(&executable) + .output() + .expect("generation-handle probe should run"); + assert_eq!( + output.status.code(), + Some(0), + "generation-handle probe exited with {:?}", + output.status.code() + ); + assert!(root.starts_with(std::env::temp_dir())); + let _ = fs::remove_dir_all(root); +} diff --git a/tools/sgc/tests/senline_build_identity.rs b/tools/sgc/tests/senline_build_identity.rs new file mode 100644 index 00000000..76e678f5 --- /dev/null +++ b/tools/sgc/tests/senline_build_identity.rs @@ -0,0 +1,345 @@ +mod common; + +use common::source_sgc_command; +use serde_json::{Map, Value}; +use std::collections::BTreeSet; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; +use std::time::{SystemTime, UNIX_EPOCH}; + +const SOURCE_REVISION: &str = "1de09ccafa7e8f182af68e82352e2d4be39496b0"; +const TOOLCHAIN_VERSION: &str = "0.1.0"; +const APPLICATION_VERSION: &str = "0.1.0"; +const BUILD_MANIFEST_ID: &str = "1111111111111111111111111111111111111111111111111111111111111111"; + +struct TempDir { + path: PathBuf, +} + +impl TempDir { + fn new(tag: &str) -> Self { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be after unix epoch") + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "sengoo-worker-identity-{tag}-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&path).expect("create build identity test directory"); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +fn workspace_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("sgc crate should live under tools/sgc") + .to_path_buf() +} + +fn worker_root() -> PathBuf { + workspace_root().join("examples/realworld/senline-domain-worker") +} + +fn generator_path() -> PathBuf { + worker_root().join("scripts/generate-build-identity.ps1") +} + +fn powershell() -> PathBuf { + which::which("pwsh") + .or_else(|_| which::which("powershell")) + .expect("build identity generation requires PowerShell") +} + +fn generate_identity( + output: &Path, + handshake: &Path, + source_revision: &str, + toolchain_version: &str, + application_version: &str, + build_manifest_id: &str, +) -> Output { + Command::new(powershell()) + .arg("-NoProfile") + .arg("-File") + .arg(generator_path()) + .arg("-SourceRevision") + .arg(source_revision) + .arg("-ToolchainVersion") + .arg(toolchain_version) + .arg("-ApplicationVersion") + .arg(application_version) + .arg("-BuildManifestId") + .arg(build_manifest_id) + .arg("-OutputPath") + .arg(output) + .arg("-HandshakeOutputPath") + .arg(handshake) + .output() + .expect("run build identity generator") +} + +fn assert_success(label: &str, output: &Output) { + assert!( + output.status.success(), + "{label} failed with {:?}\nstdout:\n{}\nstderr:\n{}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +fn object(value: &Value) -> &Map { + value.as_object().expect("handshake must be a JSON object") +} + +fn expected_handshake( + source_revision: &str, + toolchain_version: &str, + application_version: &str, + build_manifest_id: &str, +) -> Value { + serde_json::json!({ + "kind": "handshake", + "protocol_version": 1, + "sengoo_source_revision": source_revision, + "toolchain_version": toolchain_version, + "application_version": application_version, + "build_manifest_id": build_manifest_id, + }) +} + +fn externally_matches(handshake: &Value, expected: &Value) -> bool { + let exact_fields = [ + "kind", + "protocol_version", + "sengoo_source_revision", + "toolchain_version", + "application_version", + "build_manifest_id", + ] + .into_iter() + .collect::>(); + object(handshake) + .keys() + .map(String::as_str) + .collect::>() + == exact_fields + && handshake == expected +} + +fn module_map(worker: &Path, identity_source: &Path) -> std::ffi::OsString { + std::env::join_paths([ + format!( + "senline_domain_worker={}", + worker.join("src/lib.sg").display() + ), + format!("senline_build_identity={}", identity_source.display()), + format!( + "senline_facts_to_plan={}", + worker + .join("packages/senline-facts-to-plan/src/lib.sg") + .display() + ), + format!( + "sgframing={}", + worker.join("packages/sgframing/src/lib.sg").display() + ), + format!( + "sgjson_contract={}", + worker.join("packages/sgjson-contract/src/lib.sg").display() + ), + ]) + .expect("encode worker identity module map") +} + +fn unframe_exact(bytes: &[u8]) -> &[u8] { + assert!(bytes.len() >= 4, "worker handshake needs a frame prefix"); + let len = u32::from_be_bytes(bytes[..4].try_into().unwrap()) as usize; + assert_eq!(bytes.len(), len + 4, "worker emitted a surplus frame"); + &bytes[4..] +} + +#[test] +fn generator_is_reproducible_and_rejects_invalid_identity_inputs() { + let root = TempDir::new("generator"); + let first_source = root.path().join("first.sg"); + let first_handshake = root.path().join("first.json"); + let second_source = root.path().join("second.sg"); + let second_handshake = root.path().join("second.json"); + assert_success( + "first build identity generation", + &generate_identity( + &first_source, + &first_handshake, + SOURCE_REVISION, + TOOLCHAIN_VERSION, + APPLICATION_VERSION, + BUILD_MANIFEST_ID, + ), + ); + assert_success( + "second build identity generation", + &generate_identity( + &second_source, + &second_handshake, + SOURCE_REVISION, + TOOLCHAIN_VERSION, + APPLICATION_VERSION, + BUILD_MANIFEST_ID, + ), + ); + assert_eq!( + fs::read(&first_source).unwrap(), + fs::read(&second_source).unwrap() + ); + assert_eq!( + fs::read(&first_handshake).unwrap(), + fs::read(&second_handshake).unwrap() + ); + let generated: Value = serde_json::from_slice(&fs::read(&first_handshake).unwrap()).unwrap(); + assert!(externally_matches( + &generated, + &expected_handshake( + SOURCE_REVISION, + TOOLCHAIN_VERSION, + APPLICATION_VERSION, + BUILD_MANIFEST_ID, + ) + )); + + let changed_id = "2".repeat(64); + let changed_source = root.path().join("changed.sg"); + let changed_handshake = root.path().join("changed.json"); + assert_success( + "changed build identity generation", + &generate_identity( + &changed_source, + &changed_handshake, + SOURCE_REVISION, + TOOLCHAIN_VERSION, + APPLICATION_VERSION, + &changed_id, + ), + ); + assert_ne!( + fs::read(&first_source).unwrap(), + fs::read(&changed_source).unwrap() + ); + assert!(externally_matches( + &serde_json::from_slice(&fs::read(&changed_handshake).unwrap()).unwrap(), + &expected_handshake( + SOURCE_REVISION, + TOOLCHAIN_VERSION, + APPLICATION_VERSION, + &changed_id, + ) + )); + + let invalid_revision = generate_identity( + &root.path().join("invalid-revision.sg"), + &root.path().join("invalid-revision.json"), + "not-a-revision", + TOOLCHAIN_VERSION, + APPLICATION_VERSION, + BUILD_MANIFEST_ID, + ); + assert!(!invalid_revision.status.success()); + let invalid_manifest = generate_identity( + &root.path().join("invalid-manifest.sg"), + &root.path().join("invalid-manifest.json"), + SOURCE_REVISION, + TOOLCHAIN_VERSION, + APPLICATION_VERSION, + "abc", + ); + assert!(!invalid_manifest.status.success()); + let invalid_version = generate_identity( + &root.path().join("invalid-version.sg"), + &root.path().join("invalid-version.json"), + SOURCE_REVISION, + "0.1.0\"\nforged", + APPLICATION_VERSION, + BUILD_MANIFEST_ID, + ); + assert!(!invalid_version.status.success()); +} + +#[test] +fn real_worker_embeds_generated_identity_but_external_manifest_remains_authoritative() { + let root = TempDir::new("worker"); + let identity_source = root.path().join("senline-build-identity.sg"); + let handshake_path = root.path().join("handshake.json"); + let embedded_source_revision = "a".repeat(40); + let embedded_manifest_id = "2".repeat(64); + assert_success( + "worker build identity generation", + &generate_identity( + &identity_source, + &handshake_path, + &embedded_source_revision, + TOOLCHAIN_VERSION, + APPLICATION_VERSION, + &embedded_manifest_id, + ), + ); + + let worker = worker_root(); + let executable = root.path().join(if cfg!(windows) { + "identity-worker.exe" + } else { + "identity-worker" + }); + let build = source_sgc_command() + .arg("build") + .arg(worker.join("src/main.sg")) + .arg("--output") + .arg(&executable) + .arg("--force-rebuild") + .current_dir(&worker) + .env("SENGOO_MODULE_MAP", module_map(&worker, &identity_source)) + .output() + .expect("build worker with generated identity"); + assert_success("worker with generated identity build", &build); + + let output = Command::new(&executable) + .stdin(Stdio::null()) + .output() + .expect("run worker identity handshake"); + assert_success("worker identity handshake", &output); + assert!(output.stderr.is_empty()); + let reported_bytes = unframe_exact(&output.stdout); + assert_eq!( + reported_bytes, + fs::read(&handshake_path).unwrap(), + "worker handshake must be byte-identical to the external generated record" + ); + let reported: Value = serde_json::from_slice(reported_bytes).unwrap(); + let external = expected_handshake( + &embedded_source_revision, + TOOLCHAIN_VERSION, + APPLICATION_VERSION, + &embedded_manifest_id, + ); + assert!(externally_matches(&reported, &external)); + + let mut mismatched_external = external; + mismatched_external["build_manifest_id"] = Value::String("f".repeat(64)); + assert!( + !externally_matches(&reported, &mismatched_external), + "a worker self-report must not override the externally verified manifest" + ); +} diff --git a/tools/sgc/tests/senline_contract_fixtures.rs b/tools/sgc/tests/senline_contract_fixtures.rs new file mode 100644 index 00000000..45914e86 --- /dev/null +++ b/tools/sgc/tests/senline_contract_fixtures.rs @@ -0,0 +1,599 @@ +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use std::fs; +use std::path::{Path, PathBuf}; + +const CONTEXT_FIELDS: &[&str] = &[ + "contract_version", + "operation", + "operation_version", + "evaluation_id", + "operation_epoch", + "worker_generation", + "execution_mode", + "worker_bundle_id", + "facts_binding", +]; +const BINDING_CONTEXT_FIELDS: &[&str] = &[ + "contract_version", + "operation", + "operation_version", + "evaluation_id", + "operation_epoch", + "worker_generation", + "execution_mode", + "worker_bundle_id", +]; +const FACT_FIELDS: &[&str] = &[ + "contract_version", + "operation_version", + "identifiers", + "source_device_status", + "source_device_capabilities", + "envelope_protocol_version", + "ciphertext_length_bytes", + "idempotency_status", + "recipient_pending_count", + "recipient_pending_limit", + "application_envelopes_used", + "application_envelopes_limit", + "ciphertext_limit_bytes", + "feature_flags", +]; +const IDENTIFIER_FIELDS: &[&str] = &[ + "correlation_ref", + "source_account_ref", + "source_device_ref", + "recipient_account_ref", + "recipient_device_ref", + "conversation_ref", + "envelope_ref", +]; + +fn fixture_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("sgc crate should live under tools/sgc") + .join("examples") + .join("realworld") + .join("senline-domain-worker") + .join("fixtures") + .join("v1") +} + +fn read_json(path: &Path) -> Value { + serde_json::from_slice( + &fs::read(path).unwrap_or_else(|error| panic!("read {}: {error}", path.display())), + ) + .unwrap_or_else(|error| panic!("parse {}: {error}", path.display())) +} + +fn object<'a>(value: &'a Value, label: &str) -> &'a Map { + value + .as_object() + .unwrap_or_else(|| panic!("{label} must be an object")) +} + +fn exact_keys(value: &Value, expected: &[&str], label: &str) { + let actual = object(value, label) + .keys() + .map(String::as_str) + .collect::>(); + let expected = expected.iter().copied().collect::>(); + assert_eq!(actual, expected, "{label} field set changed"); +} + +fn string<'a>(value: &'a Value, field: &str) -> &'a str { + value[field] + .as_str() + .unwrap_or_else(|| panic!("{field} must be a string")) +} + +fn unsigned(value: &Value, field: &str) -> u64 { + value[field] + .as_u64() + .unwrap_or_else(|| panic!("{field} must be a non-negative integer")) +} + +fn assert_ascii_ref(value: &str, label: &str) { + assert!( + (1..=128).contains(&value.len()), + "{label} must be 1..128 bytes" + ); + assert!(value.is_ascii(), "{label} must be ASCII"); +} + +fn assert_lower_hex(value: &str, expected_len: usize, label: &str) { + assert_eq!(value.len(), expected_len, "{label} length changed"); + assert!( + value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)), + "{label} must be lowercase hexadecimal" + ); +} + +fn append_u32(bytes: &mut Vec, value: u64, field: &str) { + let value = u32::try_from(value).unwrap_or_else(|_| panic!("{field} exceeds u32")); + bytes.extend_from_slice(&value.to_be_bytes()); +} + +fn append_string(bytes: &mut Vec, value: &str) { + append_u32(bytes, value.len() as u64, "string length"); + bytes.extend_from_slice(value.as_bytes()); +} + +fn append_string_array(bytes: &mut Vec, value: &Value, field: &str) { + let values = value[field] + .as_array() + .unwrap_or_else(|| panic!("{field} must be an array")); + append_u32(bytes, values.len() as u64, "array length"); + let mut previous: Option<&str> = None; + for item in values { + let item = item + .as_str() + .unwrap_or_else(|| panic!("{field} values must be strings")); + if let Some(previous) = previous { + assert!(previous < item, "{field} must be sorted and unique"); + } + append_string(bytes, item); + previous = Some(item); + } +} + +fn facts_binding(request: &Value) -> String { + let context = &request["context"]; + let facts = &request["facts"]; + let identifiers = &facts["identifiers"]; + let mut bytes = b"senline.submit-envelope.binding.v1\0".to_vec(); + + append_u32( + &mut bytes, + unsigned(context, "contract_version"), + "context.contract_version", + ); + append_string(&mut bytes, string(context, "operation")); + append_u32( + &mut bytes, + unsigned(context, "operation_version"), + "context.operation_version", + ); + append_string(&mut bytes, string(context, "evaluation_id")); + bytes.extend_from_slice(&unsigned(context, "operation_epoch").to_be_bytes()); + bytes.extend_from_slice(&unsigned(context, "worker_generation").to_be_bytes()); + append_string(&mut bytes, string(context, "execution_mode")); + append_string(&mut bytes, string(context, "worker_bundle_id")); + + append_u32( + &mut bytes, + unsigned(facts, "contract_version"), + "facts.contract_version", + ); + append_u32( + &mut bytes, + unsigned(facts, "operation_version"), + "facts.operation_version", + ); + for field in IDENTIFIER_FIELDS { + append_string(&mut bytes, string(identifiers, field)); + } + append_string(&mut bytes, string(facts, "source_device_status")); + append_string_array(&mut bytes, facts, "source_device_capabilities"); + for field in ["envelope_protocol_version", "ciphertext_length_bytes"] { + append_u32(&mut bytes, unsigned(facts, field), field); + } + append_string(&mut bytes, string(facts, "idempotency_status")); + for field in [ + "recipient_pending_count", + "recipient_pending_limit", + "application_envelopes_used", + "application_envelopes_limit", + "ciphertext_limit_bytes", + ] { + append_u32(&mut bytes, unsigned(facts, field), field); + } + append_string_array(&mut bytes, facts, "feature_flags"); + + format!("{:x}", Sha256::digest(bytes)) +} + +fn assert_no_prohibited_fields(value: &Value, path: &str) { + const PROHIBITED: &[&str] = &[ + "private_key", + "session_key", + "recovery_material", + "plaintext", + "ciphertext_bytes", + "raw_signature", + "signed_bytes", + "auth_token", + "idempotency_token", + "credential", + "connection_string", + "sql", + "database_row", + "account_id", + "device_id", + "conversation_id", + "envelope_id", + "runtime_handle", + "transaction_handle", + "error_message", + "log_message", + ]; + match value { + Value::Object(fields) => { + for (key, value) in fields { + assert!( + !PROHIBITED.contains(&key.as_str()), + "prohibited field {path}.{key}" + ); + assert_no_prohibited_fields(value, &format!("{path}.{key}")); + } + } + Value::Array(values) => { + for (index, value) in values.iter().enumerate() { + assert_no_prohibited_fields(value, &format!("{path}[{index}]")); + } + } + Value::String(text) => { + for marker in [ + "BEGIN PRIVATE KEY", + "postgres://", + "mysql://", + "Authorization: Bearer", + "SELECT ", + "INSERT ", + "UPDATE ", + "DELETE ", + ] { + assert!(!text.contains(marker), "prohibited value marker at {path}"); + } + } + _ => {} + } +} + +fn file_hash(path: &Path) -> String { + format!("{:x}", Sha256::digest(fs::read(path).unwrap())) +} + +fn assert_frozen_enums(metadata: &Value) { + assert_eq!( + metadata["enums"], + serde_json::json!({ + "execution_mode": ["fixture", "shadow", "guarded-development", "internal-alpha"], + "source_device_status": ["active"], + "source_device_capabilities": ["submit_envelope_v2"], + "idempotency_status": ["new", "exact_duplicate", "conflict"], + "feature_flags": ["enqueue_delivery"], + "decision": ["store_and_enqueue", "duplicate_noop", "reject"], + "reason": [ + "accepted_new", + "exact_duplicate", + "idempotency_conflict", + "recipient_queue_full", + "application_budget_exhausted", + "delivery_disabled" + ], + "worker_error": [ + "malformed_json", + "unknown_field", + "duplicate_field", + "invalid_unicode", + "trailing_bytes", + "unknown_enum", + "unsupported_operation_version" + ] + }), + "frozen V1 enum table changed" + ); +} + +fn assert_strict_parser_error_inputs(root: &Path, metadata: &Value) { + let inputs = metadata["strict_parser_error_inputs"] + .as_array() + .expect("strict_parser_error_inputs must be an array"); + let expected_codes = ["duplicate_field", "invalid_unicode", "trailing_bytes"]; + assert_eq!(inputs.len(), expected_codes.len()); + + for (input, expected_code) in inputs.iter().zip(expected_codes) { + exact_keys(input, &["path", "code", "sha256"], "strict parser input"); + assert_eq!(input["code"], expected_code); + let relative = string(input, "path"); + assert!(relative.starts_with("errors/")); + assert!(!relative.contains("..")); + let path = root.join(relative); + let bytes = fs::read(&path) + .unwrap_or_else(|error| panic!("read strict parser input {}: {error}", path.display())); + assert!(!bytes.is_empty(), "strict parser input must not be empty"); + assert!(bytes.len() <= 32 * 1024, "strict parser input is oversized"); + assert_eq!(file_hash(&path), string(input, "sha256")); + } +} + +#[test] +fn fixture_validator_rejects_frozen_enum_drift() { + let mut metadata = read_json(&fixture_root().join("metadata.json")); + metadata["enums"]["worker_error"][0] = Value::String("changed".to_owned()); + + assert!( + std::panic::catch_unwind(|| assert_frozen_enums(&metadata)).is_err(), + "frozen enum drift must fail fixture validation" + ); +} + +#[test] +fn fixture_validator_rejects_strict_parser_input_hash_drift() { + let root = fixture_root(); + let mut metadata = read_json(&root.join("metadata.json")); + metadata["strict_parser_error_inputs"][0]["sha256"] = Value::String("0".repeat(64)); + + assert!( + std::panic::catch_unwind(|| assert_strict_parser_error_inputs(&root, &metadata)).is_err(), + "strict parser fixture hash drift must fail validation" + ); +} + +#[test] +fn senline_v1_fixture_corpus_is_closed_bounded_and_safe() { + let root = fixture_root(); + let metadata_path = root.join("metadata.json"); + let metadata = read_json(&metadata_path); + assert_eq!(metadata["fixture_set_version"], 1); + assert_eq!(metadata["protocol"]["input_max_bytes"], 32 * 1024); + assert_eq!(metadata["protocol"]["output_max_bytes"], 8 * 1024); + assert_eq!(metadata["protocol"]["max_in_flight"], 1); + assert_eq!(metadata["protocol"]["stdout"], "protocol_only"); + assert_eq!(metadata["bounds"]["opaque_ascii_ref_bytes"]["min"], 1); + assert_eq!(metadata["bounds"]["opaque_ascii_ref_bytes"]["max"], 128); + assert_eq!(metadata["bounds"]["evaluation_id_lower_hex_bytes"], 32); + assert_eq!(metadata["bounds"]["facts_binding_lower_hex_bytes"], 64); + assert_eq!( + metadata["bounds"]["sengoo_module_revision_lower_hex_bytes"], + 40 + ); + assert_eq!(metadata["bounds"]["u32_field_max"], 4_294_967_295_u64); + assert_eq!( + metadata["bounds"]["json_safe_u64_field_max"], + 9_007_199_254_740_991_u64 + ); + assert_eq!( + metadata["bounds"]["source_device_capabilities_items"], + serde_json::json!({ "min": 0, "max": 1 }) + ); + assert_eq!( + metadata["bounds"]["feature_flags_items"], + serde_json::json!({ "min": 0, "max": 1 }) + ); + assert_eq!( + metadata["revision_semantics"]["sengoo_module_revision"], + "planner_contract_fixture_revision" + ); + assert_eq!( + metadata["revision_semantics"]["sengoo_source_revision"], + "immutable_bundle_source_revision" + ); + assert_eq!( + metadata["binding"]["context_fields"], + serde_json::json!(BINDING_CONTEXT_FIELDS) + ); + assert_eq!( + metadata["binding"]["facts_fields"], + serde_json::json!(FACT_FIELDS) + ); + assert_eq!( + metadata["binding"]["identifier_fields"], + serde_json::json!(IDENTIFIER_FIELDS) + ); + assert_frozen_enums(&metadata); + assert_strict_parser_error_inputs(&root, &metadata); + + let expected_cases = [ + ("eligible_accept", "store_and_enqueue", "accepted_new"), + ("exact_duplicate", "duplicate_noop", "exact_duplicate"), + ("idempotency_conflict", "reject", "idempotency_conflict"), + ( + "application_budget_rejection", + "reject", + "application_budget_exhausted", + ), + ( + "unknown_operation_version", + "error", + "unsupported_operation_version", + ), + ]; + let cases = metadata["cases"] + .as_array() + .expect("cases must be an array"); + assert_eq!(cases.len(), expected_cases.len()); + let mut binding_mismatches = Vec::new(); + let mut hash_mismatches = Vec::new(); + for (case, (name, decision, reason)) in cases.iter().zip(expected_cases) { + assert_eq!(case["name"], name); + assert_eq!(case["decision"], decision); + assert_eq!(case["reason"], reason); + let request_path = root.join(string(case, "request")); + let response_path = root.join(string(case, "response")); + let request_bytes = fs::read(&request_path).unwrap(); + let response_bytes = fs::read(&response_path).unwrap(); + assert!(request_bytes.len() <= 32 * 1024); + assert!(response_bytes.len() <= 8 * 1024); + + let request = read_json(&request_path); + exact_keys( + &request, + &["kind", "schema_version", "context", "facts"], + name, + ); + exact_keys(&request["context"], CONTEXT_FIELDS, "context"); + exact_keys(&request["facts"], FACT_FIELDS, "facts"); + exact_keys( + &request["facts"]["identifiers"], + IDENTIFIER_FIELDS, + "identifiers", + ); + assert_eq!(request["kind"], "evaluation"); + assert_eq!(request["schema_version"], 1); + assert_eq!(request["context"]["contract_version"], 1); + assert_eq!(request["context"]["operation"], "submit-envelope"); + assert_lower_hex( + string(&request["context"], "evaluation_id"), + 32, + "context.evaluation_id", + ); + assert_lower_hex( + string(&request["context"], "facts_binding"), + 64, + "context.facts_binding", + ); + assert_ascii_ref( + string(&request["context"], "worker_bundle_id"), + "context.worker_bundle_id", + ); + assert!(unsigned(&request["context"], "operation_epoch") <= 9_007_199_254_740_991); + assert!(unsigned(&request["context"], "worker_generation") <= 9_007_199_254_740_991); + for field in IDENTIFIER_FIELDS { + assert_ascii_ref( + string(&request["facts"]["identifiers"], field), + &format!("identifiers.{field}"), + ); + } + let actual_binding = facts_binding(&request); + if request["context"]["facts_binding"] != actual_binding { + binding_mismatches.push(format!("{name}: {actual_binding}")); + } + assert_no_prohibited_fields(&request, name); + for (path, expected_hash) in [ + (&request_path, string(case, "request_sha256")), + (&response_path, string(case, "response_sha256")), + ] { + let actual_hash = file_hash(path); + if actual_hash != expected_hash { + hash_mismatches.push(format!("{}: {actual_hash}", path.display())); + } + } + + let response = read_json(&response_path); + assert_no_prohibited_fields(&response, name); + if decision == "error" { + exact_keys( + &response, + &["kind", "schema_version", "scope", "code", "evaluation_id"], + "error", + ); + assert_eq!(response["kind"], "error"); + assert_eq!(response["code"], reason); + assert_eq!( + response["evaluation_id"], + request["context"]["evaluation_id"] + ); + } else { + exact_keys( + &response, + &[ + "kind", + "schema_version", + "context", + "identifiers", + "decision", + "reason", + "sengoo_module_revision", + ], + "plan", + ); + assert_eq!(response["kind"], "plan"); + assert_eq!(response["context"], request["context"]); + assert_eq!(response["identifiers"], request["facts"]["identifiers"]); + assert_eq!(response["decision"], decision); + assert_eq!(response["reason"], reason); + assert_lower_hex( + string(&response, "sengoo_module_revision"), + 40, + "plan.sengoo_module_revision", + ); + } + } + assert!( + binding_mismatches.is_empty(), + "facts_binding mismatches:\n{}", + binding_mismatches.join("\n") + ); + assert!( + hash_mismatches.is_empty(), + "fixture hash mismatches:\n{}", + hash_mismatches.join("\n") + ); + + let handshake = &metadata["handshake"]; + let handshake_path = root.join(string(handshake, "path")); + assert_eq!(file_hash(&handshake_path), string(handshake, "sha256")); + let handshake_json = read_json(&handshake_path); + exact_keys( + &handshake_json, + &[ + "kind", + "protocol_version", + "sengoo_source_revision", + "toolchain_version", + "application_version", + "build_manifest_id", + ], + "handshake", + ); + + let protocol_errors = metadata["protocol_errors"] + .as_array() + .expect("protocol_errors must be an array"); + assert!(!protocol_errors.is_empty()); + for error in protocol_errors { + let path = root.join(string(error, "path")); + assert_eq!(file_hash(&path), string(error, "sha256")); + exact_keys( + &read_json(&path), + &["kind", "schema_version", "scope", "code", "evaluation_id"], + "protocol error", + ); + } + + let rust_only = metadata["rust_only_no_worker"] + .as_array() + .expect("rust_only_no_worker must be an array"); + assert_eq!(rust_only.len(), 4); + for fixture in rust_only { + let path = root.join(string(fixture, "path")); + assert_eq!(file_hash(&path), string(fixture, "sha256")); + let value = read_json(&path); + assert_eq!(value["kind"], "rust_only_rejection"); + assert!(value.get("context").is_none()); + assert!(value.get("facts").is_none()); + assert_no_prohibited_fields(&value, "rust_only_rejection"); + } + + let generated_doc = fs::read_to_string(root.join("docs/generated/protocol-v1.md")).unwrap(); + for required in CONTEXT_FIELDS + .iter() + .chain(FACT_FIELDS) + .chain(IDENTIFIER_FIELDS) + .chain( + [ + "32768", + "8192", + "protocol_only", + "1..128", + "4294967295", + "9007199254740991", + "planner contract fixture revision", + ] + .iter(), + ) + { + assert!( + generated_doc.contains(required), + "generated doc omits {required}" + ); + } +} diff --git a/tools/sgc/tests/senline_defect_evidence.rs b/tools/sgc/tests/senline_defect_evidence.rs new file mode 100644 index 00000000..bf48345f --- /dev/null +++ b/tools/sgc/tests/senline_defect_evidence.rs @@ -0,0 +1,863 @@ +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use std::fs; +use std::path::{Component, Path, PathBuf}; +use std::process::Command; + +const RECORD_FIELDS: &[&str] = &[ + "record_id", + "senline_failure", + "ownership", + "minimized_regression", + "fix", + "target_artifacts", + "senline_pin", + "final_consumer_gate", + "workaround", +]; +const WINDOWS_TARGET: &str = "x86_64-pc-windows-msvc"; +const LINUX_TARGET: &str = "x86_64-unknown-linux-gnu"; +type EvidenceMutation = (&'static str, Box); + +fn repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("sgc crate should live under tools/sgc") + .to_path_buf() +} + +fn read_json(path: &Path) -> Value { + serde_json::from_slice( + &fs::read(path).unwrap_or_else(|error| panic!("read {}: {error}", path.display())), + ) + .unwrap_or_else(|error| panic!("parse {}: {error}", path.display())) +} + +fn normalize_text_bytes(bytes: impl AsRef<[u8]>) -> Vec { + // Evidence digests are frozen against Git's LF-normalized blob bytes. Windows + // checkouts may inject CRLF into markdown or JSON fixtures; strip CR so local + // and Linux CI validate the same consumer/fixture hashes. + bytes + .as_ref() + .iter() + .copied() + .filter(|byte| *byte != b'\r') + .collect() +} + +fn sha256_normalized(bytes: impl AsRef<[u8]>) -> String { + format!("{:x}", Sha256::digest(normalize_text_bytes(bytes))) +} + +fn object<'a>(value: &'a Value, label: &str) -> &'a Map { + value + .as_object() + .unwrap_or_else(|| panic!("{label} must be an object")) +} + +fn schema_ref<'a>(root: &'a Value, reference: &str) -> Result<&'a Value, String> { + let pointer = reference + .strip_prefix('#') + .ok_or_else(|| format!("external schema reference is forbidden: {reference}"))?; + root.pointer(pointer) + .ok_or_else(|| format!("missing schema reference {reference}")) +} + +fn validate_schema_keywords(schema: &Value, path: &str) -> Result<(), String> { + const ALLOWED: &[&str] = &[ + "$schema", + "$id", + "$defs", + "$ref", + "title", + "type", + "const", + "enum", + "pattern", + "minLength", + "minItems", + "maxItems", + "required", + "properties", + "additionalProperties", + "items", + "contains", + "minContains", + "maxContains", + "allOf", + "if", + "then", + "else", + ]; + let Some(fields) = schema.as_object() else { + return Err(format!("{path}: schema node must be an object")); + }; + for key in fields.keys() { + if !ALLOWED.contains(&key.as_str()) { + return Err(format!("{path}: unsupported schema keyword {key}")); + } + } + for container in ["$defs", "properties"] { + if let Some(children) = fields.get(container).and_then(Value::as_object) { + for (name, child) in children { + validate_schema_keywords(child, &format!("{path}/{container}/{name}"))?; + } + } + } + for child_key in ["items", "contains", "if", "then", "else"] { + if let Some(child) = fields.get(child_key) { + validate_schema_keywords(child, &format!("{path}/{child_key}"))?; + } + } + if let Some(children) = fields.get("allOf").and_then(Value::as_array) { + for (index, child) in children.iter().enumerate() { + validate_schema_keywords(child, &format!("{path}/allOf/{index}"))?; + } + } + Ok(()) +} + +fn matches_type(value: &Value, expected: &str) -> bool { + match expected { + "null" => value.is_null(), + "object" => value.is_object(), + "array" => value.is_array(), + "string" => value.is_string(), + "boolean" => value.is_boolean(), + "integer" => value.is_i64() || value.is_u64(), + "number" => value.is_number(), + _ => false, + } +} + +fn matches_pattern(value: &str, pattern: &str) -> bool { + match pattern { + "^[0-9a-f]{40}$" => { + value.len() == 40 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + } + "^[0-9a-f]{64}$" => { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + } + "^SGDOG-[0-9]{4}-[0-9]{3}$" => { + value.len() == 14 + && value.starts_with("SGDOG-") + && value.as_bytes()[6..10].iter().all(u8::is_ascii_digit) + && value.as_bytes()[10] == b'-' + && value.as_bytes()[11..14].iter().all(u8::is_ascii_digit) + } + _ => false, + } +} + +fn validate_schema_node( + root: &Value, + schema: &Value, + value: &Value, + path: &str, +) -> Result<(), String> { + if let Some(reference) = schema.get("$ref").and_then(Value::as_str) { + validate_schema_node(root, schema_ref(root, reference)?, value, path)?; + } + if let Some(expected) = schema.get("type") { + let valid = match expected { + Value::String(expected) => matches_type(value, expected), + Value::Array(expected) => expected + .iter() + .filter_map(Value::as_str) + .any(|expected| matches_type(value, expected)), + _ => false, + }; + if !valid { + return Err(format!("{path}: type mismatch")); + } + } + if let Some(expected) = schema.get("const") { + if value != expected { + return Err(format!("{path}: const mismatch")); + } + } + if let Some(expected) = schema.get("enum").and_then(Value::as_array) { + if !expected.contains(value) { + return Err(format!("{path}: value is outside enum")); + } + } + if let Some(text) = value.as_str() { + if let Some(minimum) = schema.get("minLength").and_then(Value::as_u64) { + if text.len() < minimum as usize { + return Err(format!("{path}: string is shorter than minLength")); + } + } + if let Some(pattern) = schema.get("pattern").and_then(Value::as_str) { + if !matches_pattern(text, pattern) { + return Err(format!("{path}: string does not match {pattern}")); + } + } + } + if let Some(fields) = value.as_object() { + if let Some(required) = schema.get("required").and_then(Value::as_array) { + for field in required.iter().filter_map(Value::as_str) { + if !fields.contains_key(field) { + return Err(format!("{path}: missing required field {field}")); + } + } + } + if let Some(properties) = schema.get("properties").and_then(Value::as_object) { + if schema.get("additionalProperties") == Some(&Value::Bool(false)) { + for field in fields.keys() { + if !properties.contains_key(field) { + return Err(format!("{path}: unknown field {field}")); + } + } + } + for (field, field_schema) in properties { + if let Some(field_value) = fields.get(field) { + validate_schema_node( + root, + field_schema, + field_value, + &format!("{path}/{field}"), + )?; + } + } + } + } + if let Some(items) = value.as_array() { + if let Some(minimum) = schema.get("minItems").and_then(Value::as_u64) { + if items.len() < minimum as usize { + return Err(format!("{path}: array is shorter than minItems")); + } + } + if let Some(maximum) = schema.get("maxItems").and_then(Value::as_u64) { + if items.len() > maximum as usize { + return Err(format!("{path}: array is longer than maxItems")); + } + } + if let Some(item_schema) = schema.get("items") { + for (index, item) in items.iter().enumerate() { + validate_schema_node(root, item_schema, item, &format!("{path}/{index}"))?; + } + } + if let Some(contains) = schema.get("contains") { + let count = items + .iter() + .filter(|item| validate_schema_node(root, contains, item, path).is_ok()) + .count(); + let minimum = schema + .get("minContains") + .and_then(Value::as_u64) + .unwrap_or(1) as usize; + let maximum = schema + .get("maxContains") + .and_then(Value::as_u64) + .unwrap_or(u64::MAX) as usize; + if count < minimum || count > maximum { + return Err(format!("{path}: contains count {count} is outside bounds")); + } + } + } + if let Some(all_of) = schema.get("allOf").and_then(Value::as_array) { + for child in all_of { + validate_schema_node(root, child, value, path)?; + } + } + if let Some(condition) = schema.get("if") { + if validate_schema_node(root, condition, value, path).is_ok() { + if let Some(then_schema) = schema.get("then") { + validate_schema_node(root, then_schema, value, path)?; + } + } else if let Some(else_schema) = schema.get("else") { + validate_schema_node(root, else_schema, value, path)?; + } + } + Ok(()) +} + +fn non_empty_string<'a>(value: &'a Value, path: &str) -> Result<&'a str, String> { + value + .as_str() + .filter(|value| !value.is_empty()) + .ok_or_else(|| format!("{path}: expected non-empty string")) +} + +fn normalized_relative_path(value: &Value, path: &str) -> Result { + let text = non_empty_string(value, path)?; + if text.contains('\\') || Path::new(text).is_absolute() { + return Err(format!("{path}: path must be normalized and relative")); + } + let parsed = PathBuf::from(text); + if parsed + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(format!("{path}: path contains a non-normal component")); + } + Ok(parsed) +} + +fn validate_immutable_reference(value: &Value, path: &str) -> Result<(), String> { + if value.is_null() { + return Ok(()); + } + let text = non_empty_string(value, path)?; + let bytes = text.as_bytes(); + let has_windows_drive = bytes.len() >= 2 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':'; + if has_windows_drive || text.starts_with('/') || text.starts_with("//") { + return Err(format!( + "{path}: absolute or checkout-local reference is forbidden" + )); + } + let reference = normalized_relative_path(value, path)?; + for component in reference.components() { + let Component::Normal(component) = component else { + return Err(format!("{path}: non-normal reference component")); + }; + let component = component.to_string_lossy().to_ascii_lowercase(); + if matches!( + component.as_str(), + ".git" | ".worktrees" | "target" | "latest" | "current" | "head" | "main" | "master" + ) { + return Err(format!("{path}: mutable reference component {component}")); + } + } + Ok(()) +} + +fn validate_cross_field_bindings(ledger: &Value) -> Result<(), String> { + let records = ledger["records"] + .as_array() + .ok_or_else(|| "records must be an array".to_owned())?; + let mut record_ids = BTreeSet::new(); + for (index, record) in records.iter().enumerate() { + let path = format!("/records/{index}"); + let record_id = non_empty_string(&record["record_id"], &format!("{path}/record_id"))?; + if !record_ids.insert(record_id) { + return Err(format!("{path}: duplicate record_id {record_id}")); + } + + let failure = &record["senline_failure"]; + if failure["failure_id"] != record["record_id"] { + return Err(format!( + "{path}: pending/rehearsal failure_id must link record_id" + )); + } + let expected_discovery = match failure["evidence_kind"].as_str() { + Some("consumer-failure") => "consumer-discovered", + Some("known-baseline-rehearsal") => "known-baseline", + Some("injected-rehearsal") => "injected-rehearsal", + _ => return Err(format!("{path}: unknown evidence kind")), + }; + if record["ownership"]["discovery_status"] != expected_discovery { + return Err(format!( + "{path}: evidence kind and discovery status disagree" + )); + } + + let fixing_commit = record["fix"]["fixing_commit"].as_str(); + let artifacts = record["target_artifacts"] + .as_array() + .ok_or_else(|| format!("{path}: target_artifacts must be an array"))?; + let targets = artifacts + .iter() + .filter_map(|artifact| artifact["target"].as_str()) + .collect::>(); + if targets != BTreeSet::from([WINDOWS_TARGET, LINUX_TARGET]) { + return Err(format!( + "{path}: target artifacts must bind Windows and Linux once" + )); + } + for artifact in artifacts { + for field in ["provenance", "archive", "manifest"] { + validate_immutable_reference( + &artifact[field], + &format!("{path}/target_artifacts/{field}"), + )?; + } + if artifact["status"] == "verified" { + let source_revision = non_empty_string( + &artifact["source_revision"], + &format!("{path}/target_artifacts/source_revision"), + )?; + if fixing_commit != Some(source_revision) { + return Err(format!( + "{path}: artifact source_revision must equal fixing_commit" + )); + } + } + } + + let pin = &record["senline_pin"]; + if pin["status"] == "verified" { + let pinned = non_empty_string( + &pin["pinned_sengoo_revision"], + &format!("{path}/senline_pin/pinned_sengoo_revision"), + )?; + if fixing_commit != Some(pinned) { + return Err(format!( + "{path}: pinned Sengoo revision must equal fixing_commit" + )); + } + let manifests = pin["target_manifests"] + .as_array() + .ok_or_else(|| format!("{path}: pin target manifests must be an array"))?; + for artifact in artifacts { + let target = artifact["target"].as_str().unwrap_or_default(); + let artifact_hash = artifact["manifest_sha256"].as_str(); + let pin_hash = manifests + .iter() + .find(|manifest| manifest["target"] == target) + .and_then(|manifest| manifest["manifest_sha256"].as_str()); + if artifact_hash.is_none() || pin_hash != artifact_hash { + return Err(format!( + "{path}: pin manifest hash must equal artifact hash" + )); + } + } + } + if record["final_consumer_gate"]["status"] == "green" + && record["senline_pin"]["status"] != "verified" + { + return Err(format!( + "{path}: green consumer gate requires a verified pin" + )); + } + let workaround = &record["workaround"]; + if workaround["active"] == true { + if workaround["linked_defect"] != record["record_id"] { + return Err(format!("{path}: workaround must link its owning defect")); + } + validate_immutable_reference( + &workaround["removal_test"], + &format!("{path}/workaround/removal_test"), + )?; + if record["final_consumer_gate"]["status"] == "green" { + return Err(format!( + "{path}: an active workaround cannot count as green" + )); + } + } + } + Ok(()) +} + +fn validate_evidence(schema: &Value, ledger: &Value) -> Result<(), String> { + validate_schema_keywords(schema, "")?; + validate_schema_node(schema, schema, ledger, "")?; + validate_cross_field_bindings(ledger) +} + +fn evidence_paths() -> (PathBuf, PathBuf) { + let docs = repo_root().join("docs"); + ( + docs.join("senline-dogfood-evidence.schema.json"), + docs.join("senline-dogfood-evidence.v1.json"), + ) +} + +fn fill_synthetic_verified_chain(record: &mut Value) { + let fixing_revision = "a".repeat(40); + record["minimized_regression"]["red_status"] = Value::String("preserved".to_owned()); + record["minimized_regression"]["red_commit"] = Value::String("9".repeat(40)); + record["fix"]["fixing_commit"] = Value::String(fixing_revision.clone()); + let artifacts = record["target_artifacts"] + .as_array_mut() + .expect("target artifacts"); + for (index, artifact) in artifacts.iter_mut().enumerate() { + let platform = if index == 0 { "windows" } else { "linux" }; + artifact["status"] = Value::String("verified".to_owned()); + artifact["source_revision"] = Value::String(fixing_revision.clone()); + artifact["build_manifest_id"] = Value::String("1".repeat(64)); + artifact["provenance"] = Value::String(format!( + "artifacts/SGDOG-2026-001/{platform}/provenance.json" + )); + artifact["archive"] = + Value::String(format!("artifacts/SGDOG-2026-001/{platform}/toolchain.zip")); + artifact["manifest"] = + Value::String(format!("artifacts/SGDOG-2026-001/{platform}/manifest.json")); + artifact["archive_sha256"] = Value::String("2".repeat(64)); + artifact["manifest_sha256"] = Value::String(if index == 0 { "3" } else { "4" }.repeat(64)); + } + record["senline_pin"] = serde_json::json!({ + "status": "verified", + "senline_pin_revision": "8".repeat(40), + "pinned_sengoo_revision": fixing_revision, + "target_manifests": [ + { "target": WINDOWS_TARGET, "manifest_sha256": "3".repeat(64) }, + { "target": LINUX_TARGET, "manifest_sha256": "4".repeat(64) } + ] + }); + record["final_consumer_gate"] = serde_json::json!({ + "status": "green", + "command": "cargo test --locked --workspace", + "evidence": "evidence/SGDOG-2026-001/consumer-green.json" + }); +} + +#[test] +fn durable_senline_defect_evidence_validates_against_the_versioned_schema() { + let (schema_path, ledger_path) = evidence_paths(); + let schema = read_json(&schema_path); + let ledger = read_json(&ledger_path); + validate_evidence(&schema, &ledger).unwrap_or_else(|error| panic!("{error}")); + + assert_eq!( + schema["$schema"], + "https://json-schema.org/draft/2020-12/schema" + ); + assert_eq!( + schema["$id"], + "https://sengoo.dev/schema/senline-dogfood-evidence-v1.json" + ); + assert_eq!(ledger["schema_version"], 1); + assert_eq!(ledger["change"], "senline-service-dogfood"); + assert_eq!( + ledger["linked_senline_change"], + "adopt-sengoo-backend-slice" + ); + + let first = &ledger["records"][0]; + let actual_fields = object(first, "first record") + .keys() + .map(String::as_str) + .collect::>(); + assert_eq!(actual_fields, RECORD_FIELDS.iter().copied().collect()); + assert_eq!(first["record_id"], "SGDOG-2026-001"); + assert_eq!( + first["senline_failure"]["evidence_kind"], + "known-baseline-rehearsal" + ); + assert_eq!(first["ownership"]["authority"], "sengoo-owned"); + assert_eq!( + first["ownership"]["component_classification"], + "sengoo-standard-library" + ); + assert_eq!(first["ownership"]["discovery_status"], "known-baseline"); + // Until red-first history is reconstructed, known-baseline rehearsal keeps + // fixing_commit null and red_status pending-commit (task 7.2 open). + assert_eq!(first["fix"]["fixing_commit"], Value::Null); + assert_eq!( + first["minimized_regression"]["red_status"], + "pending-commit" + ); + assert_eq!(first["minimized_regression"]["red_commit"], Value::Null); + assert_eq!(first["senline_pin"]["status"], "pending"); + assert_eq!(first["final_consumer_gate"]["status"], "pending"); + + for (index, record) in ledger["records"] + .as_array() + .expect("records array") + .iter() + .enumerate() + { + let failure = &record["senline_failure"]; + let record_id = record["record_id"].as_str().expect("record id"); + let consumer_path = normalized_relative_path( + &failure["consumer_record"], + &format!("records[{index}].consumer_record"), + ) + .expect("normalized consumer record path"); + let consumer_bytes = fs::read(repo_root().join(consumer_path)) + .unwrap_or_else(|error| panic!("read linked consumer record: {error}")); + assert_eq!( + sha256_normalized(&consumer_bytes), + failure["consumer_record_sha256"], + "records[{index}] consumer record hash changed" + ); + assert!( + String::from_utf8_lossy(&consumer_bytes).contains(record_id), + "records[{index}] consumer record must contain its evidence ID" + ); + + let senline_fixture = + normalized_relative_path(&failure["fixture"], &format!("records[{index}].fixture")) + .expect("normalized Senline fixture path"); + let mirror_fixture = normalized_relative_path( + &failure["fixture_mirror"], + &format!("records[{index}].fixture_mirror"), + ) + .expect("normalized Sengoo fixture mirror path"); + let senline_prefix = Path::new("fixtures/sengoo-worker/v1"); + let mirror_prefix = Path::new("examples/realworld/senline-domain-worker/fixtures/v1"); + assert_eq!( + senline_fixture + .strip_prefix(senline_prefix) + .expect("Senline fixture must use the frozen v1 root"), + mirror_fixture + .strip_prefix(mirror_prefix) + .expect("Sengoo fixture must use the mirrored frozen v1 root"), + "records[{index}] fixture paths must identify the same relative file" + ); + assert_eq!( + sha256_normalized( + fs::read(repo_root().join(mirror_fixture)) + .expect("read exact mirrored fixture path") + ), + failure["fixture_sha256"], + "records[{index}] fixture hash changed" + ); + } +} + +#[test] +fn evidence_schema_rejects_every_empty_or_workaround_only_green_state() { + let (schema_path, ledger_path) = evidence_paths(); + let schema = read_json(&schema_path); + let ledger = read_json(&ledger_path); + let mut valid_green = ledger.clone(); + fill_synthetic_verified_chain(&mut valid_green["records"][0]); + validate_evidence(&schema, &valid_green) + .expect("a complete synthetic immutable chain should validate"); + let mut mutations: Vec = Vec::new(); + mutations.push(( + "empty records", + Box::new(|value| value["records"] = Value::Array(vec![])), + )); + mutations.push(( + "missing target", + Box::new(|value| { + value["records"][0]["target_artifacts"] + .as_array_mut() + .expect("target artifacts") + .pop(); + }), + )); + mutations.push(( + "duplicate target", + Box::new(|value| { + value["records"][0]["target_artifacts"][1]["target"] = + Value::String(WINDOWS_TARGET.to_owned()); + }), + )); + mutations.push(( + "verified artifact without hashes", + Box::new(|value| { + value["records"][0]["target_artifacts"][0]["status"] = + Value::String("verified".to_owned()); + }), + )); + mutations.push(( + "preserved red without commit", + Box::new(|value| { + value["records"][0]["minimized_regression"]["red_status"] = + Value::String("preserved".to_owned()); + // Leave red_commit null while claiming preserved — must fail validation. + value["records"][0]["minimized_regression"]["red_commit"] = Value::Null; + }), + )); + mutations.push(( + "verified pin without revisions", + Box::new(|value| { + value["records"][0]["senline_pin"]["status"] = Value::String("verified".to_owned()); + }), + )); + mutations.push(( + "verified pin over pending red and artifacts", + Box::new(|value| { + let revision = "a".repeat(40); + value["records"][0]["fix"]["fixing_commit"] = Value::String(revision.clone()); + for (index, artifact) in value["records"][0]["target_artifacts"] + .as_array_mut() + .expect("target artifacts") + .iter_mut() + .enumerate() + { + artifact["source_revision"] = Value::String(revision.clone()); + artifact["build_manifest_id"] = Value::String("b".repeat(64)); + artifact["provenance"] = Value::String("attestation.json".to_owned()); + artifact["archive"] = Value::String(format!("target-{index}.tar.gz")); + artifact["manifest"] = Value::String(format!("manifest-{index}.json")); + artifact["archive_sha256"] = Value::String("c".repeat(64)); + artifact["manifest_sha256"] = + Value::String(if index == 0 { "d" } else { "e" }.repeat(64)); + } + value["records"][0]["senline_pin"] = serde_json::json!({ + "status": "verified", + "senline_pin_revision": "f".repeat(40), + "pinned_sengoo_revision": revision, + "target_manifests": [ + { "target": WINDOWS_TARGET, "manifest_sha256": "d".repeat(64) }, + { "target": LINUX_TARGET, "manifest_sha256": "e".repeat(64) } + ] + }); + }), + )); + mutations.push(( + "green gate without evidence", + Box::new(|value| { + value["records"][0]["final_consumer_gate"]["status"] = + Value::String("green".to_owned()); + }), + )); + mutations.push(( + "active workaround without removal contract", + Box::new(|value| { + value["records"][0]["workaround"]["active"] = Value::Bool(true); + }), + )); + mutations.push(( + "artifact path reaches mutable Sengoo checkout", + Box::new(|value| { + value["records"][0]["target_artifacts"][0]["archive"] = + Value::String("D:/Sengoo/target/latest/toolchain.zip".to_owned()); + }), + )); + mutations.push(( + "artifact provenance escapes through a floating path", + Box::new(|value| { + value["records"][0]["target_artifacts"][0]["provenance"] = + Value::String("../target/latest/provenance.json".to_owned()); + }), + )); + mutations.push(( + "active workaround links a different defect", + Box::new(|value| { + value["records"][0]["workaround"] = serde_json::json!({ + "active": true, + "owner": "Senline backend team", + "linked_defect": "SGDOG-2099-999", + "expiry_condition": "Remove after the pinned fixing artifact is verified", + "removal_test": "tests/workaround_removal.rs::removes_gap_workaround" + }); + }), + )); + mutations.push(( + "active workaround uses an absolute removal test", + Box::new(|value| { + value["records"][0]["workaround"] = serde_json::json!({ + "active": true, + "owner": "Senline backend team", + "linked_defect": "SGDOG-2026-001", + "expiry_condition": "Remove after the pinned fixing artifact is verified", + "removal_test": "D:/senline/tests/workaround_removal.rs" + }); + }), + )); + mutations.push(( + "partial pin omits one target manifest", + Box::new(|value| { + fill_synthetic_verified_chain(&mut value["records"][0]); + value["records"][0]["senline_pin"]["target_manifests"] + .as_array_mut() + .expect("target manifests") + .pop(); + }), + )); + mutations.push(( + "active workaround tries to claim a complete chain as green", + Box::new(|value| { + fill_synthetic_verified_chain(&mut value["records"][0]); + value["records"][0]["workaround"] = serde_json::json!({ + "active": true, + "owner": "Senline backend team", + "linked_defect": "SGDOG-2026-001", + "expiry_condition": "Remove after the pinned fixing artifact is verified", + "removal_test": "tests/workaround_removal.rs::removes_gap_workaround" + }); + }), + )); + mutations.push(( + "inactive workaround hides mutable registry fields", + Box::new(|value| { + value["records"][0]["workaround"]["owner"] = + Value::String("untracked owner".to_owned()); + }), + )); + mutations.push(( + "rehearsal mislabeled as consumer discovered", + Box::new(|value| { + value["records"][0]["ownership"]["discovery_status"] = + Value::String("consumer-discovered".to_owned()); + }), + )); + + for (label, mutate) in mutations { + let mut invalid = ledger.clone(); + mutate(&mut invalid); + assert!( + validate_evidence(&schema, &invalid).is_err(), + "{label} must not validate" + ); + } + + let mut unsupported_schema = schema.clone(); + unsupported_schema["$defs"]["record"]["not"] = serde_json::json!({}); + let error = validate_evidence(&unsupported_schema, &ledger) + .expect_err("unknown schema keywords must fail closed"); + assert!(error.contains("unsupported schema keyword not")); +} + +#[test] +fn every_recorded_red_command_selects_exactly_one_regression() { + let (_, ledger_path) = evidence_paths(); + let ledger = read_json(&ledger_path); + for (index, record) in ledger["records"] + .as_array() + .expect("records array") + .iter() + .enumerate() + { + let regression = &record["minimized_regression"]; + let selector = ®ression["selector"]; + let package = selector["package"].as_str().expect("selector package"); + let target_kind = selector["target_kind"] + .as_str() + .expect("selector target kind"); + let target_name = selector["target_name"] + .as_str() + .expect("selector target name"); + let test_name = selector["test_name"].as_str().expect("selector test name"); + let target_flag = format!("--{target_kind}"); + let expected_command = format!( + "cargo test -p {package} {target_flag} {target_name} {test_name} -- --exact --nocapture" + ); + assert_eq!( + regression["red_command"], expected_command, + "records[{index}] RED command must match its structured selector" + ); + assert!( + regression["test"] + .as_str() + .is_some_and(|path| path.ends_with(test_name.trim_start_matches("tests::"))), + "records[{index}] test path must name its exact selector" + ); + + let output = Command::new(env!("CARGO")) + .args([ + "test", + "-p", + package, + target_flag.as_str(), + target_name, + test_name, + "--", + "--exact", + "--list", + ]) + .current_dir(repo_root()) + .output() + .unwrap_or_else(|error| panic!("list records[{index}] RED regression: {error}")); + assert!( + output.status.success(), + "records[{index}] cargo test --list failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let listed = String::from_utf8_lossy(&output.stdout); + let selected = listed + .lines() + .filter(|line| line.ends_with(": test")) + .collect::>(); + assert_eq!( + selected, + [format!("{test_name}: test")], + "records[{index}] RED command must select exactly one regression" + ); + } +} diff --git a/tools/sgc/tests/senline_http_policy.rs b/tools/sgc/tests/senline_http_policy.rs new file mode 100644 index 00000000..fa4bcaa2 --- /dev/null +++ b/tools/sgc/tests/senline_http_policy.rs @@ -0,0 +1,273 @@ +//! Tasks 6.6 / 6.7: harness non-ingress policy and anti-deployment checks. + +use std::fs; +use std::path::{Path, PathBuf}; + +fn workspace_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") +} + +fn http_root() -> PathBuf { + workspace_root().join("examples/realworld/senline-http-dogfood") +} + +fn read(path: impl AsRef) -> String { + fs::read_to_string(path).expect("read file") +} + +/// Text extensions scanned for harness anti-targeting (includes real Windows / +/// Android / web client surfaces, not just Sengoo/docs sources). +const SCAN_EXTENSIONS: &[&str] = &[ + "sg", "md", "toml", "json", "rs", "yml", "yaml", // Sengoo / docs / config + "kt", "kts", "gradle", "xml", // Android + "cs", "xaml", "csproj", "props", "targets", // Windows / .NET + "ts", "tsx", "js", "jsx", // web / desktop clients + "swift", "m", "mm", // Apple clients (if present) + "plist", "pbxproj", +]; + +fn is_scanned_extension(ext: &str) -> bool { + SCAN_EXTENSIONS + .iter() + .any(|allowed| ext.eq_ignore_ascii_case(allowed)) +} + +fn walk_text_files(root: &Path, out: &mut Vec) { + let Ok(entries) = fs::read_dir(root) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + let name = path.file_name().and_then(|s| s.to_str()).unwrap_or(""); + if name == "target" || name == "build" || name == "node_modules" || name.starts_with('.') { + continue; + } + if path.is_dir() { + walk_text_files(&path, out); + continue; + } + if let Some(ext) = path.extension().and_then(|s| s.to_str()) { + if is_scanned_extension(ext) { + out.push(path); + } + } + } +} + +#[test] +fn http_dogfood_documents_serial_plaintext_non_ingress_limits() { + let readme = read(http_root().join("README.md")); + for needle in [ + "serial", + "plaintext", + "Connection: close", + "TLS", + "keep-alive", + "internal-alpha", + "production", + "127.0.0.1:0", + ] { + assert!( + readme.contains(needle), + "README missing retained-limit claim: {needle}" + ); + } +} + +#[test] +fn http_dogfood_sources_never_advertise_non_loopback_or_client_endpoints() { + let mut files = Vec::new(); + walk_text_files(&http_root(), &mut files); + assert!(!files.is_empty(), "expected http dogfood source files"); + let forbidden = ["0.0.0.0", "SENLINE_API", "play.google", "apps/chat_client"]; + for path in files { + let text = read(&path); + // Policy tests and README intentionally mention rejected non-loopback inputs. + if path.ends_with("policy_contract.sg") || path.ends_with("README.md") { + continue; + } + for needle in forbidden { + assert!( + !text.contains(needle), + "{} must not target or embed production/client endpoint material containing {needle}", + path.display() + ); + } + assert!( + !text.contains("https://"), + "{} must not embed remote HTTPS endpoints", + path.display() + ); + } +} + +/// Product-surface roots under a Senline checkout that must not target dogfood. +const SENLINE_PRODUCT_ROOTS: &[&str] = &["apps", "win", "config", "services", "android", "clients"]; + +fn harness_markers() -> [&'static str; 4] { + [ + "senline-http-dogfood", + "senline_http_dogfood", + "/v1/submit-envelope", + "READY 127.0.0.1", + ] +} + +fn product_surface_hits_harness(text: &str) -> Option<&'static str> { + let markers = harness_markers(); + for marker in markers { + if !text.contains(marker) { + continue; + } + if marker == "/v1/submit-envelope" { + // Operation path alone is ambiguous only when no dogfood identity is + // present. Fail when the same file also names the dogfood package or + // READY loopback banner (client wiring the harness). + let dogfood_identity = text.contains("senline-http-dogfood") + || text.contains("senline_http_dogfood") + || text.contains("READY 127.0.0.1"); + if !dogfood_identity { + continue; + } + } + return Some(marker); + } + None +} + +#[test] +fn optional_senline_checkout_does_not_target_http_dogfood_harness() { + // Prefer SENLINE_ROOT; fall back to D:\senline. When absent on a runner that + // is not expected to carry Senline, skip — but the scanned extension set and + // marker rules still apply whenever a checkout is present (fail-closed). + let senline = std::env::var_os("SENLINE_ROOT") + .map(PathBuf::from) + .filter(|p| p.is_dir()) + .or_else(|| { + let fallback = PathBuf::from(r"D:\senline"); + if fallback.is_dir() { + Some(fallback) + } else { + None + } + }); + let Some(root) = senline else { + eprintln!( + "senline checkout absent; skipping live consumer path scan \ + (extension/marker rules still covered by synthetic fixture test)" + ); + return; + }; + + let mut files = Vec::new(); + for rel in SENLINE_PRODUCT_ROOTS { + walk_text_files(&root.join(rel), &mut files); + } + // Also scan top-level config-ish files if present. + for rel in ["docs", "openspec"] { + // docs/openspec may name the harness as forbidden — only product roots fail. + let _ = rel; + } + + for path in files { + let Ok(text) = fs::read_to_string(&path) else { + continue; + }; + let rel = path + .strip_prefix(&root) + .map(|p| p.to_string_lossy().replace('\\', "/")) + .unwrap_or_default(); + let is_product_surface = SENLINE_PRODUCT_ROOTS + .iter() + .any(|prefix| rel.starts_with(&format!("{prefix}/")) || rel.starts_with(prefix)); + if !is_product_surface { + continue; + } + if let Some(marker) = product_surface_hits_harness(&text) { + panic!( + "Senline product path {} must not target HTTP dogfood harness marker {marker}", + path.display() + ); + } + } +} + +#[test] +fn policy_scan_detects_android_and_windows_client_harness_targets() { + // Synthetic fail-closed fixture: product-surface client files that would be + // invisible under the previous .sg/.md/.rs-only walk must be rejected. + let root = std::env::temp_dir().join(format!( + "senline-policy-scan-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(root.join("apps/chat_client")).expect("create apps"); + fs::create_dir_all(root.join("android/app/src")).expect("create android"); + fs::create_dir_all(root.join("win/ChatClient")).expect("create win"); + + fs::write( + root.join("android/app/src/MainActivity.kt"), + "val url = \"http://127.0.0.1:9/v1/submit-envelope\" // senline-http-dogfood\n", + ) + .expect("write kt"); + fs::write( + root.join("win/ChatClient/Client.cs"), + "var path = \"/v1/submit-envelope\"; // uses senline_http_dogfood\n", + ) + .expect("write cs"); + fs::write( + root.join("apps/chat_client/build.gradle"), + "applicationId 'com.example' // READY 127.0.0.1 banner for dogfood\n", + ) + .expect("write gradle"); + fs::write( + root.join("apps/chat_client/App.xaml"), + "\n", + ) + .expect("write xaml"); + fs::write( + root.join("apps/chat_client/api.ts"), + "export const endpoint = 'senline_http_dogfood';\n", + ) + .expect("write ts"); + + let mut files = Vec::new(); + for rel in SENLINE_PRODUCT_ROOTS { + walk_text_files(&root.join(rel), &mut files); + } + assert!( + files.len() >= 5, + "expected client surfaces to be discovered, got {}: {:?}", + files.len(), + files + ); + + let mut hits = 0usize; + for path in &files { + let text = fs::read_to_string(path).expect("read fixture"); + if product_surface_hits_harness(&text).is_some() { + hits += 1; + } + } + assert!( + hits >= 4, + "synthetic Android/Windows/web client fixtures must fail the harness scan (hits={hits})" + ); + + let _ = fs::remove_dir_all(&root); +} + +#[test] +fn policy_scan_extension_allowlist_covers_client_languages() { + for ext in ["kt", "kts", "gradle", "xml", "cs", "xaml", "ts", "tsx", "js"] { + assert!( + is_scanned_extension(ext), + "client extension .{ext} must be scanned for task 6.6" + ); + } +} diff --git a/tools/sgc/tests/senline_plan_binding.rs b/tools/sgc/tests/senline_plan_binding.rs new file mode 100644 index 00000000..8c854cc2 --- /dev/null +++ b/tools/sgc/tests/senline_plan_binding.rs @@ -0,0 +1,277 @@ +//! Task 5.4: plan/request binding rejection surface for the framed worker. + +mod common; + +use common::source_sgc_command; +use serde_json::{json, Value}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{SystemTime, UNIX_EPOCH}; + +struct TempDir { + path: PathBuf, +} + +impl TempDir { + fn new(tag: &str) -> Self { + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "senline-plan-binding-{tag}-{}-{stamp}", + std::process::id() + )); + fs::create_dir_all(&path).expect("create temp"); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +fn workspace_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") +} + +fn worker_root() -> PathBuf { + workspace_root().join("examples/realworld/senline-domain-worker") +} + +fn module_map(worker: &Path) -> std::ffi::OsString { + // Match the realworld harness: PATH-style module=path entries. + std::env::join_paths([ + format!( + "senline_domain_worker={}", + worker.join("src/lib.sg").display() + ), + format!( + "senline_build_identity={}", + worker + .join("packages/senline-build-identity/src/lib.sg") + .display() + ), + format!( + "senline_facts_to_plan={}", + worker + .join("packages/senline-facts-to-plan/src/lib.sg") + .display() + ), + format!( + "sgframing={}", + worker.join("packages/sgframing/src/lib.sg").display() + ), + format!( + "sgjson_contract={}", + worker.join("packages/sgjson-contract/src/lib.sg").display() + ), + ]) + .expect("encode module map") +} + +fn build_worker(temp: &TempDir) -> PathBuf { + let worker = worker_root(); + let exe = temp.path().join(if cfg!(windows) { + "senline_domain_worker.exe" + } else { + "senline_domain_worker" + }); + let output = source_sgc_command() + .arg("build") + .arg(worker.join("src/main.sg")) + .arg("-o") + .arg(&exe) + .args(["-O", "0", "--force-rebuild"]) + .current_dir(&worker) + .env("SENGOO_MODULE_MAP", module_map(&worker)) + .output() + .expect("build worker"); + assert!( + output.status.success(), + "worker build failed:\nstdout:{}\nstderr:{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + exe +} + +fn framed(payload: &[u8]) -> Vec { + let mut out = (payload.len() as u32).to_be_bytes().to_vec(); + out.extend_from_slice(payload); + out +} + +fn run_worker(exe: &Path, input: &[u8]) -> Vec { + let mut child = Command::new(exe) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn worker"); + { + use std::io::Write; + let stdin = child.stdin.as_mut().expect("stdin"); + stdin.write_all(input).expect("write stdin"); + } + let output = child.wait_with_output().expect("wait worker"); + assert!( + output.status.success(), + "worker failed status={:?} stderr={}", + output.status.code(), + String::from_utf8_lossy(&output.stderr) + ); + output.stdout +} + +fn decode_frames(bytes: &[u8]) -> Vec { + let mut values = Vec::new(); + let mut offset = 0usize; + while offset + 4 <= bytes.len() { + let len = u32::from_be_bytes(bytes[offset..offset + 4].try_into().unwrap()) as usize; + offset += 4; + let end = offset + len; + assert!(end <= bytes.len(), "truncated frame"); + let value: Value = serde_json::from_slice(&bytes[offset..end]).expect("json frame"); + values.push(value); + offset = end; + } + assert_eq!(offset, bytes.len(), "trailing bytes after frames"); + values +} + +fn eligible_request() -> Value { + let path = worker_root().join("fixtures/v1/cases/eligible-accept.request.json"); + serde_json::from_slice(&fs::read(path).expect("read fixture")).expect("parse fixture") +} + +#[test] +fn worker_echoes_evaluation_id_and_rejects_invalid_binding_fields() { + let temp = TempDir::new("bindings"); + let exe = build_worker(&temp); + let base = eligible_request(); + + // Host-owned binding integrity: the worker echoes a well-formed evaluation_id. + // Final mismatch against host facts_binding remains Senline Rust authority. + { + let mut request = base.clone(); + request["context"]["evaluation_id"] = json!("ffffffffffffffffffffffffffffffff"); + let body = serde_json::to_vec(&request).expect("serialize"); + let frames = decode_frames(&run_worker(&exe, &framed(&body))); + assert_eq!( + frames[1]["kind"], "plan", + "valid hex evaluation_id should plan" + ); + assert_eq!( + frames[1]["context"]["evaluation_id"], + "ffffffffffffffffffffffffffffffff" + ); + } + + let mut rejections: Vec<(&str, Value)> = Vec::new(); + { + let mut v = base.clone(); + v["context"]["operation"] = json!("not-submit-envelope"); + rejections.push(("operation", v)); + } + { + let mut v = base.clone(); + v["context"]["operation_epoch"] = json!(9_007_199_254_740_992_i64); + rejections.push(("operation_epoch", v)); + } + { + let mut v = base.clone(); + v["context"]["worker_generation"] = json!(-1); + rejections.push(("worker_generation", v)); + } + { + let mut v = base.clone(); + v["context"]["contract_version"] = json!(2); + rejections.push(("contract_version", v)); + } + { + let mut v = base.clone(); + v["facts"]["identifiers"]["envelope_ref"] = json!(""); + rejections.push(("identifier", v)); + } + { + let mut v = base.clone(); + v["facts"]["idempotency_status"] = json!("maybe"); + rejections.push(("unknown_enum", v)); + } + { + let mut v = base.clone(); + v["facts"]["source_device_capabilities"] = json!(["not_a_capability"]); + rejections.push(("impossible_action", v)); + } + { + let mut v = base.clone(); + v["context"]["facts_binding"] = json!("not-hex"); + rejections.push(("facts_binding_shape", v)); + } + { + let mut v = base.clone(); + v["context"]["evaluation_id"] = json!("zz"); + rejections.push(("evaluation_id_shape", v)); + } + + for (label, request) in rejections { + let body = serde_json::to_vec(&request).expect("serialize"); + let frames = decode_frames(&run_worker(&exe, &framed(&body))); + assert!( + frames[1]["kind"] == "error", + "{label}: expected error rejection, got {}", + frames[1] + ); + let code = frames[1]["code"].as_str().unwrap_or(""); + assert!( + !code.is_empty(), + "{label}: empty error code in {}", + frames[1] + ); + } +} + +#[test] +fn worker_rejects_oversized_output_path_by_enforcing_output_limit() { + // The worker enforces an 8 KiB response ceiling before writing. Prove the + // documented bound remains part of the public library surface used by hosts. + let temp = TempDir::new("output-limit"); + let worker = worker_root(); + let probe = temp.path().join("output_limit.sg"); + fs::write( + &probe, + r#" +import senline_domain_worker; + +def main() -> i64 { + if worker_output_length_supported(8192) and not worker_output_length_supported(8193) { 0; } else { 1; }; +} +"#, + ) + .expect("write probe"); + let output = source_sgc_command() + .arg("run") + .arg(&probe) + .args(["--force-rebuild"]) + .current_dir(&worker) + .env("SENGOO_MODULE_MAP", module_map(&worker)) + .output() + .expect("run probe"); + assert!( + output.status.success(), + "stdout:{}\nstderr:{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(output.status.code(), Some(0)); +} diff --git a/tools/sgc/tests/senline_support_record.rs b/tools/sgc/tests/senline_support_record.rs new file mode 100644 index 00000000..220af2af --- /dev/null +++ b/tools/sgc/tests/senline_support_record.rs @@ -0,0 +1,84 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +fn repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("sgc crate should live under tools/sgc") + .to_path_buf() +} + +#[test] +fn sengoo_support_record_keeps_local_package_evidence_separate_from_senline_authority() { + let path = repo_root().join("docs/senline-dogfood-support.md"); + let record = fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); + + for required in [ + "source-development-local", + "installed-windows-x64", + "installed-linux-x64", + "senline-domain-worker", + "senline-http-dogfood", + "sandbox and supervisor", + "shadow", + "guarded-development", + "internal-alpha", + "rollback", + "Senline Rust", + "not Senline pin evidence", + "release_eligible=false", + "authentication authority", + "replay mutation", + "prekey claim", + "public ingress", + "final mutation authority", + "separate reviewed OpenSpec change", + ] { + assert!( + record.contains(required), + "support record must preserve `{required}`" + ); + } + + assert!( + record.contains("| source-development-local | proven |"), + "the current local source worker evidence must be explicit" + ); + // Installed hosts may be `pending` or intermediate `package-smoke-proven` + // after distribution packaging, but must not claim full product-loop proven + // until worker/HTTP installed gates and pin verification land. + let windows_status_ok = record.contains("| installed-windows-x64 | pending |") + || record.contains("| installed-windows-x64 | package-smoke-proven |"); + let linux_status_ok = record.contains("| installed-linux-x64 | pending |") + || record.contains("| installed-linux-x64 | package-smoke-proven |"); + assert!( + windows_status_ok && linux_status_ok, + "installed platform rows must stay pending or package-smoke-proven only" + ); + assert!( + record.contains("Installed `senline-domain-worker` / HTTP product loops") + || record.contains("Installed worker/HTTP product loops") + || record.contains("reviewed Senline pin"), + "installed package-smoke rows must still call out remaining pin/soak limits" + ); + assert!( + record.contains("| sandbox and supervisor | Senline-owned |") + && record.contains("| internal-alpha | Senline-owned |"), + "host authority must not be presented as a Sengoo support claim" + ); + + for forbidden in [ + "| installed-windows-x64 | proven |", + "| installed-linux-x64 | proven |", + "| sandbox and supervisor | proven |", + "| internal-alpha | proven |", + "| production ingress | proven |", + ] { + assert!( + !record.contains(forbidden), + "support record overclaims `{forbidden}`" + ); + } +} diff --git a/tools/sgc/tests/senline_worker_differential.rs b/tools/sgc/tests/senline_worker_differential.rs new file mode 100644 index 00000000..7003c38f --- /dev/null +++ b/tools/sgc/tests/senline_worker_differential.rs @@ -0,0 +1,1246 @@ +mod common; + +use common::source_sgc_command; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, ExitStatus, Stdio}; +use std::sync::mpsc::sync_channel; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +const PLANNER_FIXTURE_REVISION: &str = "1de09ccafa7e8f182af68e82352e2d4be39496b0"; +const GENERATOR_NAME: &str = "senline-worker-differential-v1"; +const GENERATOR_VERSION: u64 = 1; +const FIXED_SEED: u64 = 0x6a09_e667_f3bc_c909; +const DETERMINISM_COUNT: u64 = 512; +const REVIEWED_BOUNDARY_COUNT: u64 = 10_000; +const SEEDED_ELIGIBLE_COUNT: u64 = 100_000; +const SEEDED_PROCESS_COUNT: u64 = 8; +const SPLITMIX_GAMMA: u64 = 0x9e37_79b9_7f4a_7c15; +const SEEDED_RANDOM_VALUES_PER_CASE: u64 = 23; +const JSON_SAFE_INTEGER_MAX: u64 = 9_007_199_254_740_991; +const INPUT_MAX_BYTES: usize = 32 * 1024; +const OUTPUT_MAX_BYTES: usize = 8 * 1024; + +fn workspace_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("sgc crate should live under tools/sgc") + .to_path_buf() +} + +fn fixture_root() -> PathBuf { + workspace_root().join("examples/realworld/senline-domain-worker/fixtures/v1") +} + +fn sha256(bytes: impl AsRef<[u8]>) -> String { + format!("{:x}", Sha256::digest(bytes.as_ref())) +} + +fn normalize_fixture_bytes(bytes: impl AsRef<[u8]>) -> Vec { + // Windows checkouts without forced LF can rewrite fixture JSON/raw files with + // CRLF, which would otherwise invalidate frozen SHA-256 digests and handshake + // byte equality. Canonicalize to LF before hashing or comparing fixtures. + bytes + .as_ref() + .iter() + .copied() + .filter(|byte| *byte != b'\r') + .collect() +} + +#[test] +fn differential_corpus_metadata_is_frozen_and_linked_to_rust_fixtures() { + let root = fixture_root(); + let fixture_metadata = normalize_fixture_bytes( + fs::read(root.join("metadata.json")).expect("read fixture metadata"), + ); + let corpus_metadata = normalize_fixture_bytes( + fs::read(root.join("differential-corpus-v1.json")) + .expect("read reviewed differential corpus metadata"), + ); + let metadata: Value = serde_json::from_slice(&corpus_metadata).expect("parse corpus metadata"); + + assert_eq!(metadata["schema_version"], 1); + assert_eq!(metadata["reference_kind"], "independent_rust_oracle"); + assert_eq!( + metadata["reference_scope"], + "linked_to_frozen_rust_fixtures_not_senline_production_reference" + ); + assert_eq!( + metadata["frozen_fixture_metadata_sha256"], + sha256(&fixture_metadata) + ); + assert_eq!( + metadata["planner_contract_fixture_revision"], + PLANNER_FIXTURE_REVISION + ); + assert_eq!(metadata["generator"]["name"], GENERATOR_NAME); + assert_eq!(metadata["generator"]["version"], GENERATOR_VERSION); + assert_eq!( + metadata["generator"]["fixed_seed_hex"], + format!("0x{FIXED_SEED:016x}") + ); + assert_eq!( + metadata["corpora"]["reviewed_boundary"]["count"], + REVIEWED_BOUNDARY_COUNT + ); + assert_eq!( + metadata["corpora"]["seeded_eligible"]["count"], + SEEDED_ELIGIBLE_COUNT + ); + assert_eq!( + metadata["corpora"]["seeded_eligible"]["fresh_processes"], + SEEDED_PROCESS_COUNT + ); + assert_eq!( + metadata["corpora"]["seeded_eligible"]["cases_per_process"], + SEEDED_ELIGIBLE_COUNT / SEEDED_PROCESS_COUNT + ); + for corpus in ["determinism", "reviewed_boundary", "seeded_eligible"] { + let digest = metadata["corpora"][corpus]["transcript_sha256"] + .as_str() + .unwrap_or_else(|| panic!("{corpus} transcript digest must be a string")); + assert_eq!(digest.len(), 64, "{corpus} transcript digest length"); + assert!( + digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)), + "{corpus} transcript digest must be lowercase hex" + ); + } + assert_eq!( + metadata["coverage"]["decisions"], + serde_json::json!(["store_and_enqueue", "duplicate_noop", "reject"]) + ); + assert_eq!( + metadata["coverage"]["reasons"], + serde_json::json!([ + "accepted_new", + "exact_duplicate", + "idempotency_conflict", + "recipient_queue_full", + "application_budget_exhausted", + "delivery_disabled" + ]) + ); + assert_eq!( + metadata["coverage"]["execution_modes"], + serde_json::json!(["fixture", "shadow", "guarded-development", "internal-alpha"]) + ); + assert_eq!( + metadata["coverage"]["opaque_ascii_ref_lengths"], + serde_json::json!([1, 128]) + ); + assert_eq!( + metadata["coverage"]["numeric_boundaries"], + serde_json::json!([0, 1, 4294967295_u64, 9007199254740991_u64]) + ); + assert_eq!( + metadata["ci_targets"], + serde_json::json!(["x86_64-pc-windows-msvc", "x86_64-unknown-linux-gnu"]) + ); +} + +#[derive(Clone, Copy, Debug)] +enum IdempotencyStatus { + New, + ExactDuplicate, + Conflict, +} + +impl IdempotencyStatus { + fn as_str(self) -> &'static str { + match self { + Self::New => "new", + Self::ExactDuplicate => "exact_duplicate", + Self::Conflict => "conflict", + } + } +} + +#[derive(Clone, Debug)] +struct Identifiers { + correlation_ref: String, + source_account_ref: String, + source_device_ref: String, + recipient_account_ref: String, + recipient_device_ref: String, + conversation_ref: String, + envelope_ref: String, +} + +impl Identifiers { + fn values(&self) -> [&str; 7] { + [ + &self.correlation_ref, + &self.source_account_ref, + &self.source_device_ref, + &self.recipient_account_ref, + &self.recipient_device_ref, + &self.conversation_ref, + &self.envelope_ref, + ] + } + + fn json(&self) -> Value { + serde_json::json!({ + "correlation_ref": self.correlation_ref, + "source_account_ref": self.source_account_ref, + "source_device_ref": self.source_device_ref, + "recipient_account_ref": self.recipient_account_ref, + "recipient_device_ref": self.recipient_device_ref, + "conversation_ref": self.conversation_ref, + "envelope_ref": self.envelope_ref, + }) + } +} + +#[derive(Clone, Debug)] +struct OracleCase { + index: u64, + evaluation_id: String, + operation_epoch: u64, + worker_generation: u64, + execution_mode: &'static str, + worker_bundle_id: String, + identifiers: Identifiers, + has_submit_envelope_v2: bool, + ciphertext_length_bytes: u32, + idempotency_status: IdempotencyStatus, + recipient_pending_count: u32, + recipient_pending_limit: u32, + application_envelopes_used: u32, + application_envelopes_limit: u32, + ciphertext_limit_bytes: u32, + enqueue_delivery_enabled: bool, +} + +#[derive(Clone, Copy, Debug)] +struct OracleOutcome { + decision: &'static str, + reason: &'static str, +} + +fn reference_outcome(case: &OracleCase) -> OracleOutcome { + let ordered_rules = [ + ( + matches!(case.idempotency_status, IdempotencyStatus::ExactDuplicate), + OracleOutcome { + decision: "duplicate_noop", + reason: "exact_duplicate", + }, + ), + ( + matches!(case.idempotency_status, IdempotencyStatus::Conflict), + OracleOutcome { + decision: "reject", + reason: "idempotency_conflict", + }, + ), + ( + case.recipient_pending_count >= case.recipient_pending_limit, + OracleOutcome { + decision: "reject", + reason: "recipient_queue_full", + }, + ), + ( + case.application_envelopes_used >= case.application_envelopes_limit, + OracleOutcome { + decision: "reject", + reason: "application_budget_exhausted", + }, + ), + ( + !case.has_submit_envelope_v2 || !case.enqueue_delivery_enabled, + OracleOutcome { + decision: "reject", + reason: "delivery_disabled", + }, + ), + ]; + ordered_rules + .into_iter() + .find_map(|(matched, outcome)| matched.then_some(outcome)) + .unwrap_or(OracleOutcome { + decision: "store_and_enqueue", + reason: "accepted_new", + }) +} + +fn append_u32(bytes: &mut Vec, value: u32) { + bytes.extend_from_slice(&value.to_be_bytes()); +} + +fn append_string(bytes: &mut Vec, value: &str) { + append_u32( + bytes, + u32::try_from(value.len()).expect("contract string length fits u32"), + ); + bytes.extend_from_slice(value.as_bytes()); +} + +fn append_string_array(bytes: &mut Vec, values: &[&str]) { + append_u32( + bytes, + u32::try_from(values.len()).expect("contract array length fits u32"), + ); + for value in values { + append_string(bytes, value); + } +} + +fn facts_binding(case: &OracleCase) -> String { + let mut bytes = b"senline.submit-envelope.binding.v1\0".to_vec(); + append_u32(&mut bytes, 1); + append_string(&mut bytes, "submit-envelope"); + append_u32(&mut bytes, 1); + append_string(&mut bytes, &case.evaluation_id); + bytes.extend_from_slice(&case.operation_epoch.to_be_bytes()); + bytes.extend_from_slice(&case.worker_generation.to_be_bytes()); + append_string(&mut bytes, case.execution_mode); + append_string(&mut bytes, &case.worker_bundle_id); + + append_u32(&mut bytes, 1); + append_u32(&mut bytes, 1); + for identifier in case.identifiers.values() { + append_string(&mut bytes, identifier); + } + append_string(&mut bytes, "active"); + let capabilities = if case.has_submit_envelope_v2 { + ["submit_envelope_v2"].as_slice() + } else { + [].as_slice() + }; + append_string_array(&mut bytes, capabilities); + append_u32(&mut bytes, 2); + append_u32(&mut bytes, case.ciphertext_length_bytes); + append_string(&mut bytes, case.idempotency_status.as_str()); + append_u32(&mut bytes, case.recipient_pending_count); + append_u32(&mut bytes, case.recipient_pending_limit); + append_u32(&mut bytes, case.application_envelopes_used); + append_u32(&mut bytes, case.application_envelopes_limit); + append_u32(&mut bytes, case.ciphertext_limit_bytes); + let flags = if case.enqueue_delivery_enabled { + ["enqueue_delivery"].as_slice() + } else { + [].as_slice() + }; + append_string_array(&mut bytes, flags); + sha256(bytes) +} + +fn context_json(case: &OracleCase) -> Value { + serde_json::json!({ + "contract_version": 1, + "operation": "submit-envelope", + "operation_version": 1, + "evaluation_id": case.evaluation_id, + "operation_epoch": case.operation_epoch, + "worker_generation": case.worker_generation, + "execution_mode": case.execution_mode, + "worker_bundle_id": case.worker_bundle_id, + "facts_binding": facts_binding(case), + }) +} + +fn request_bytes(case: &OracleCase) -> Vec { + let capabilities: Vec<&str> = if case.has_submit_envelope_v2 { + vec!["submit_envelope_v2"] + } else { + Vec::new() + }; + let feature_flags: Vec<&str> = if case.enqueue_delivery_enabled { + vec!["enqueue_delivery"] + } else { + Vec::new() + }; + let request = serde_json::json!({ + "kind": "evaluation", + "schema_version": 1, + "context": context_json(case), + "facts": { + "contract_version": 1, + "operation_version": 1, + "identifiers": case.identifiers.json(), + "source_device_status": "active", + "source_device_capabilities": capabilities, + "envelope_protocol_version": 2, + "ciphertext_length_bytes": case.ciphertext_length_bytes, + "idempotency_status": case.idempotency_status.as_str(), + "recipient_pending_count": case.recipient_pending_count, + "recipient_pending_limit": case.recipient_pending_limit, + "application_envelopes_used": case.application_envelopes_used, + "application_envelopes_limit": case.application_envelopes_limit, + "ciphertext_limit_bytes": case.ciphertext_limit_bytes, + "feature_flags": feature_flags, + } + }); + let bytes = serde_json::to_vec(&request).expect("serialize typed oracle request"); + assert!( + bytes.len() <= INPUT_MAX_BYTES, + "oracle generated oversized input" + ); + bytes +} + +fn ascii_ref(domain: u64, index: u64, len: usize) -> String { + const ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789_-"; + let mut state = domain ^ index.rotate_left(17) ^ 0xa409_3822_299f_31d0; + let mut value = String::with_capacity(len); + for _ in 0..len { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + value.push(ALPHABET[(state as usize) % ALPHABET.len()] as char); + } + value +} + +fn identifiers(index: u64, len: usize) -> Identifiers { + Identifiers { + correlation_ref: ascii_ref(1, index, len), + source_account_ref: ascii_ref(2, index, len), + source_device_ref: ascii_ref(3, index, len), + recipient_account_ref: ascii_ref(4, index, len), + recipient_device_ref: ascii_ref(5, index, len), + conversation_ref: ascii_ref(6, index, len), + envelope_ref: ascii_ref(7, index, len), + } +} + +fn evaluation_id(high: u64, low: u64) -> String { + format!("{high:016x}{low:016x}") +} + +fn reviewed_boundary_case(index: u64) -> OracleCase { + const MODES: [&str; 4] = ["fixture", "shadow", "guarded-development", "internal-alpha"]; + let variant = index / 6; + let reference_len = if index.is_multiple_of(2) { 1 } else { 128 }; + let (ciphertext_length_bytes, ciphertext_limit_bytes) = match variant % 4 { + 0 => (0, 0), + 1 => (1, 1), + 2 => (u32::MAX, u32::MAX), + _ => (1, u32::MAX), + }; + let relation = variant % 3; + let relation_values = match relation { + 0 => (0, 1), + 1 => (1, 1), + _ => (2, 1), + }; + let mut case = OracleCase { + index, + evaluation_id: evaluation_id(index, index ^ 0xbb67_ae85_84ca_a73b), + operation_epoch: [0, 1, JSON_SAFE_INTEGER_MAX][(variant % 3) as usize], + worker_generation: [JSON_SAFE_INTEGER_MAX, 0, 1][(variant % 3) as usize], + execution_mode: MODES[(index % MODES.len() as u64) as usize], + worker_bundle_id: ascii_ref(8, index, reference_len), + identifiers: identifiers(index, reference_len), + has_submit_envelope_v2: variant.is_multiple_of(2), + ciphertext_length_bytes, + idempotency_status: IdempotencyStatus::New, + recipient_pending_count: relation_values.0, + recipient_pending_limit: relation_values.1, + application_envelopes_used: relation_values.0, + application_envelopes_limit: relation_values.1, + ciphertext_limit_bytes, + enqueue_delivery_enabled: variant % 4 < 2, + }; + match index % 6 { + 0 => { + case.recipient_pending_count = 0; + case.recipient_pending_limit = 1; + case.application_envelopes_used = 0; + case.application_envelopes_limit = 1; + case.has_submit_envelope_v2 = true; + case.enqueue_delivery_enabled = true; + } + 1 => case.idempotency_status = IdempotencyStatus::ExactDuplicate, + 2 => case.idempotency_status = IdempotencyStatus::Conflict, + 3 => { + case.recipient_pending_count = if variant.is_multiple_of(2) { 1 } else { 2 }; + case.recipient_pending_limit = 1; + } + 4 => { + case.recipient_pending_count = 0; + case.recipient_pending_limit = 1; + case.application_envelopes_used = if variant.is_multiple_of(2) { 1 } else { 2 }; + case.application_envelopes_limit = 1; + } + _ => { + case.recipient_pending_count = 0; + case.recipient_pending_limit = 1; + case.application_envelopes_used = 0; + case.application_envelopes_limit = 1; + if variant.is_multiple_of(2) { + case.has_submit_envelope_v2 = false; + case.enqueue_delivery_enabled = true; + } else { + case.has_submit_envelope_v2 = true; + case.enqueue_delivery_enabled = false; + } + } + } + case +} + +struct SplitMix64 { + state: u64, +} + +impl SplitMix64 { + fn new(seed: u64) -> Self { + Self { state: seed } + } + + fn next(&mut self) -> u64 { + self.state = self.state.wrapping_add(SPLITMIX_GAMMA); + let mut value = self.state; + value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + value ^ (value >> 31) + } + + fn bool(&mut self) -> bool { + self.next() & 1 == 1 + } + + fn at_case(seed: u64, index: u64) -> Self { + Self::new(seed.wrapping_add( + SPLITMIX_GAMMA.wrapping_mul(index.wrapping_mul(SEEDED_RANDOM_VALUES_PER_CASE)), + )) + } +} + +fn seeded_case(index: u64, random: &mut SplitMix64) -> OracleCase { + const MODES: [&str; 4] = ["fixture", "shadow", "guarded-development", "internal-alpha"]; + let reference_len = (random.next() % 128 + 1) as usize; + let ciphertext_limit_bytes = random.next() as u32; + let ciphertext_length_bytes = (random.next() % (u64::from(ciphertext_limit_bytes) + 1)) as u32; + let idempotency_status = match random.next() % 3 { + 0 => IdempotencyStatus::New, + 1 => IdempotencyStatus::ExactDuplicate, + _ => IdempotencyStatus::Conflict, + }; + OracleCase { + index, + evaluation_id: evaluation_id(random.next(), random.next()), + operation_epoch: random.next() % (JSON_SAFE_INTEGER_MAX + 1), + worker_generation: random.next() % (JSON_SAFE_INTEGER_MAX + 1), + execution_mode: MODES[(random.next() % MODES.len() as u64) as usize], + worker_bundle_id: ascii_ref(random.next(), index, reference_len), + identifiers: Identifiers { + correlation_ref: ascii_ref(random.next(), index, reference_len), + source_account_ref: ascii_ref(random.next(), index, reference_len), + source_device_ref: ascii_ref(random.next(), index, reference_len), + recipient_account_ref: ascii_ref(random.next(), index, reference_len), + recipient_device_ref: ascii_ref(random.next(), index, reference_len), + conversation_ref: ascii_ref(random.next(), index, reference_len), + envelope_ref: ascii_ref(random.next(), index, reference_len), + }, + has_submit_envelope_v2: random.bool(), + ciphertext_length_bytes, + idempotency_status, + recipient_pending_count: random.next() as u32, + recipient_pending_limit: random.next() as u32, + application_envelopes_used: random.next() as u32, + application_envelopes_limit: random.next() as u32, + ciphertext_limit_bytes, + enqueue_delivery_enabled: random.bool(), + } +} + +#[derive(Clone, Copy)] +enum CorpusKind { + ReviewedBoundary, + SeededEligible, +} + +#[derive(Clone, Copy)] +struct CorpusSpec { + name: &'static str, + kind: CorpusKind, + start_index: u64, + count: u64, + timeout: Duration, + collect_responses: bool, +} + +struct WorkerTempDir { + path: PathBuf, +} + +impl WorkerTempDir { + fn new(tag: &str) -> Self { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock after unix epoch") + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "sengoo-worker-differential-{tag}-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&path).expect("create differential temp directory"); + Self { path } + } +} + +impl Drop for WorkerTempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +fn worker_root() -> PathBuf { + workspace_root().join("examples/realworld/senline-domain-worker") +} + +fn worker_module_map(worker: &Path) -> std::ffi::OsString { + std::env::join_paths([ + format!( + "senline_domain_worker={}", + worker.join("src/lib.sg").display() + ), + format!( + "senline_build_identity={}", + worker + .join("packages/senline-build-identity/src/lib.sg") + .display() + ), + format!( + "senline_facts_to_plan={}", + worker + .join("packages/senline-facts-to-plan/src/lib.sg") + .display() + ), + format!( + "sgframing={}", + worker.join("packages/sgframing/src/lib.sg").display() + ), + format!( + "sgjson_contract={}", + worker.join("packages/sgjson-contract/src/lib.sg").display() + ), + ]) + .expect("encode differential worker module map") +} + +fn build_worker(root: &WorkerTempDir) -> PathBuf { + let worker = worker_root(); + let executable = root.path.join(if cfg!(windows) { + "senline-domain-worker-differential.exe" + } else { + "senline-domain-worker-differential" + }); + let output = source_sgc_command() + .arg("build") + .arg(worker.join("src/main.sg")) + .arg("--output") + .arg(&executable) + .args(["-O", "3", "--force-rebuild"]) + .current_dir(&worker) + .env("SENGOO_MODULE_MAP", worker_module_map(&worker)) + .output() + .expect("build differential worker"); + assert!( + output.status.success(), + "differential worker build failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + executable +} + +fn write_frame(writer: &mut impl Write, payload: &[u8]) -> Result<(), String> { + let len = u32::try_from(payload.len()).map_err(|_| "payload length exceeds u32".to_owned())?; + writer + .write_all(&len.to_be_bytes()) + .and_then(|()| writer.write_all(payload)) + .map_err(|error| format!("write worker frame: {error}")) +} + +fn read_frame(reader: &mut impl Read, max_len: usize) -> Result, String> { + let mut prefix = [0_u8; 4]; + reader + .read_exact(&mut prefix) + .map_err(|error| format!("read worker frame prefix: {error}"))?; + let len = u32::from_be_bytes(prefix) as usize; + if len == 0 || len > max_len { + return Err(format!( + "worker frame length {len} is outside 1..={max_len}" + )); + } + let mut payload = vec![0_u8; len]; + reader + .read_exact(&mut payload) + .map_err(|error| format!("read worker frame payload: {error}"))?; + Ok(payload) +} + +fn exact_keys(value: &Value, expected: &[&str], label: &str) -> Result<(), String> { + let fields = value + .as_object() + .ok_or_else(|| format!("{label} must be an object"))?; + let actual = fields.keys().map(String::as_str).collect::>(); + let expected = expected.iter().copied().collect::>(); + if actual == expected { + Ok(()) + } else { + Err(format!("{label} exact keys differ: {actual:?}")) + } +} + +fn validate_plan(payload: &[u8], case: &OracleCase) -> Result { + if payload.len() > OUTPUT_MAX_BYTES { + return Err(format!("case {} output exceeds 8 KiB", case.index)); + } + if payload.last() != Some(&b'\n') { + return Err(format!("case {} plan lacks one trailing LF", case.index)); + } + if payload[..payload.len() - 1] + .iter() + .any(|byte| matches!(byte, b'\n' | b'\r' | b'\t')) + { + return Err(format!( + "case {} plan is not normalized single-line JSON", + case.index + )); + } + let plan: Value = serde_json::from_slice(payload) + .map_err(|error| format!("case {} malformed plan JSON: {error}", case.index))?; + exact_keys( + &plan, + &[ + "kind", + "schema_version", + "context", + "identifiers", + "decision", + "reason", + "sengoo_module_revision", + ], + "SubmitEnvelopePlanV1", + )?; + exact_keys( + &plan["context"], + &[ + "contract_version", + "operation", + "operation_version", + "evaluation_id", + "operation_epoch", + "worker_generation", + "execution_mode", + "worker_bundle_id", + "facts_binding", + ], + "EvaluationContextV1", + )?; + exact_keys( + &plan["identifiers"], + &[ + "correlation_ref", + "source_account_ref", + "source_device_ref", + "recipient_account_ref", + "recipient_device_ref", + "conversation_ref", + "envelope_ref", + ], + "SubmitEnvelopeIdentifiersV1", + )?; + let outcome = reference_outcome(case); + let expected = [ + ("kind", Value::String("plan".to_owned())), + ("schema_version", Value::from(1)), + ("context", context_json(case)), + ("identifiers", case.identifiers.json()), + ("decision", Value::String(outcome.decision.to_owned())), + ("reason", Value::String(outcome.reason.to_owned())), + ( + "sengoo_module_revision", + Value::String(PLANNER_FIXTURE_REVISION.to_owned()), + ), + ]; + for (field, expected) in expected { + if plan[field] != expected { + return Err(format!( + "case {} field {field} mismatch: actual={} expected={expected}", + case.index, plan[field] + )); + } + } + Ok(outcome) +} + +#[derive(Default)] +struct Coverage { + decisions: BTreeMap<&'static str, u64>, + reasons: BTreeMap<&'static str, u64>, + execution_modes: BTreeSet<&'static str>, + reference_lengths: BTreeSet, + queue_relations: BTreeSet<&'static str>, + application_relations: BTreeSet<&'static str>, + capabilities: BTreeSet, + feature_flags: BTreeSet, + saw_u32_max: bool, + saw_json_safe_max: bool, +} + +fn relation(left: u32, right: u32) -> &'static str { + match left.cmp(&right) { + std::cmp::Ordering::Less => "below", + std::cmp::Ordering::Equal => "equal", + std::cmp::Ordering::Greater => "above", + } +} + +impl Coverage { + fn observe(&mut self, case: &OracleCase, outcome: OracleOutcome) { + *self.decisions.entry(outcome.decision).or_default() += 1; + *self.reasons.entry(outcome.reason).or_default() += 1; + self.execution_modes.insert(case.execution_mode); + self.reference_lengths + .insert(case.identifiers.correlation_ref.len()); + self.queue_relations.insert(relation( + case.recipient_pending_count, + case.recipient_pending_limit, + )); + self.application_relations.insert(relation( + case.application_envelopes_used, + case.application_envelopes_limit, + )); + self.capabilities.insert(case.has_submit_envelope_v2); + self.feature_flags.insert(case.enqueue_delivery_enabled); + self.saw_u32_max |= [ + case.ciphertext_length_bytes, + case.ciphertext_limit_bytes, + case.recipient_pending_count, + case.recipient_pending_limit, + case.application_envelopes_used, + case.application_envelopes_limit, + ] + .contains(&u32::MAX); + self.saw_json_safe_max |= case.operation_epoch == JSON_SAFE_INTEGER_MAX + || case.worker_generation == JSON_SAFE_INTEGER_MAX; + } + + fn assert_reviewed_boundary_coverage(&self) { + assert_eq!( + self.decisions.keys().copied().collect::>(), + BTreeSet::from(["duplicate_noop", "reject", "store_and_enqueue"]) + ); + assert_eq!( + self.reasons.keys().copied().collect::>(), + BTreeSet::from([ + "accepted_new", + "exact_duplicate", + "idempotency_conflict", + "recipient_queue_full", + "application_budget_exhausted", + "delivery_disabled", + ]) + ); + assert_eq!( + self.execution_modes, + BTreeSet::from(["fixture", "guarded-development", "internal-alpha", "shadow"]) + ); + assert_eq!(self.reference_lengths, BTreeSet::from([1, 128])); + assert_eq!( + self.queue_relations, + BTreeSet::from(["above", "below", "equal"]) + ); + assert_eq!( + self.application_relations, + BTreeSet::from(["above", "below", "equal"]) + ); + assert_eq!(self.capabilities, BTreeSet::from([false, true])); + assert_eq!(self.feature_flags, BTreeSet::from([false, true])); + assert!(self.saw_u32_max, "reviewed corpus omitted u32::MAX"); + assert!( + self.saw_json_safe_max, + "reviewed corpus omitted the JSON-safe integer maximum" + ); + } +} + +struct WaitOutcome { + status: ExitStatus, + timed_out: bool, +} + +fn wait_with_watchdog(mut child: Child, timeout: Duration) -> WaitOutcome { + let started = Instant::now(); + loop { + match child.try_wait() { + Ok(Some(status)) => { + return WaitOutcome { + status, + timed_out: false, + }; + } + Ok(None) if started.elapsed() < timeout => thread::sleep(Duration::from_millis(10)), + Ok(None) => { + let _ = child.kill(); + let status = child.wait().expect("reap watchdog-killed worker"); + return WaitOutcome { + status, + timed_out: true, + }; + } + Err(error) => panic!("poll differential worker: {error}"), + } + } +} + +struct CorpusOutcome { + transcript_sha256: String, + responses: Vec>, + coverage: Coverage, + elapsed: Duration, +} + +fn update_transcript(hasher: &mut Sha256, request: &[u8], response: &[u8]) { + hasher.update((request.len() as u32).to_be_bytes()); + hasher.update(request); + hasher.update((response.len() as u32).to_be_bytes()); + hasher.update(response); +} + +fn run_corpus(executable: &Path, spec: CorpusSpec) -> CorpusOutcome { + let started = Instant::now(); + let mut child = std::process::Command::new(executable) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn differential worker"); + let mut stdin = child.stdin.take().expect("worker stdin pipe"); + let mut stdout = child.stdout.take().expect("worker stdout pipe"); + let mut stderr = child.stderr.take().expect("worker stderr pipe"); + let (expected_sender, expected_receiver) = sync_channel::<(OracleCase, Vec)>(0); + let writer = thread::spawn(move || -> Result<(), String> { + let mut random = SplitMix64::at_case(FIXED_SEED, spec.start_index); + for index in spec.start_index..spec.start_index + spec.count { + let case = match spec.kind { + CorpusKind::ReviewedBoundary => reviewed_boundary_case(index), + CorpusKind::SeededEligible => seeded_case(index, &mut random), + }; + let request = request_bytes(&case); + write_frame(&mut stdin, &request)?; + expected_sender + .send((case, request)) + .map_err(|_| "differential oracle receiver closed".to_owned())?; + } + Ok(()) + }); + let stderr_reader = thread::spawn(move || { + let mut bytes = Vec::new(); + stderr + .read_to_end(&mut bytes) + .expect("drain differential worker stderr"); + bytes + }); + let watchdog = thread::spawn(move || wait_with_watchdog(child, spec.timeout)); + + let fixture_handshake = normalize_fixture_bytes( + fs::read(fixture_root().join("handshake/ready.json")).expect("read frozen handshake"), + ); + let mut failure = match read_frame(&mut stdout, OUTPUT_MAX_BYTES) { + Ok(handshake) if normalize_fixture_bytes(&handshake) == fixture_handshake => None, + Ok(_) => Some("worker handshake differs from frozen fixture".to_owned()), + Err(error) => Some(format!("worker handshake failed: {error}")), + }; + let mut transcript = Sha256::new(); + transcript.update(b"senline-worker-differential-transcript-v1\0"); + transcript.update(GENERATOR_NAME.as_bytes()); + transcript.update(GENERATOR_VERSION.to_be_bytes()); + transcript.update(FIXED_SEED.to_be_bytes()); + transcript.update(spec.start_index.to_be_bytes()); + transcript.update(spec.count.to_be_bytes()); + let mut coverage = Coverage::default(); + let mut responses = if spec.collect_responses { + Vec::with_capacity(spec.count as usize) + } else { + Vec::new() + }; + let mut received = 0_u64; + while received < spec.count { + let (case, request) = match expected_receiver.recv() { + Ok(expected) => expected, + Err(_) => { + failure.get_or_insert_with(|| { + format!("oracle stream ended after {received}/{} cases", spec.count) + }); + break; + } + }; + let response = match read_frame(&mut stdout, OUTPUT_MAX_BYTES) { + Ok(response) => response, + Err(error) => { + failure.get_or_insert_with(|| { + format!("case {} response frame failed: {error}", case.index) + }); + break; + } + }; + update_transcript(&mut transcript, &request, &response); + match validate_plan(&response, &case) { + Ok(outcome) => coverage.observe(&case, outcome), + Err(error) => { + failure.get_or_insert(error); + } + } + if spec.collect_responses { + responses.push(response); + } + received += 1; + } + drop(expected_receiver); + let writer_result = writer.join().expect("differential writer thread panicked"); + let mut trailing_stdout = Vec::new(); + stdout + .read_to_end(&mut trailing_stdout) + .expect("drain trailing worker stdout"); + let wait = watchdog.join().expect("differential watchdog panicked"); + let stderr = stderr_reader + .join() + .expect("differential stderr reader panicked"); + + if let Err(error) = writer_result { + failure.get_or_insert(error); + } + if wait.timed_out { + failure.get_or_insert_with(|| format!("{} worker timed out", spec.name)); + } + if !wait.status.success() { + failure.get_or_insert_with(|| { + format!("{} worker exited with {:?}", spec.name, wait.status.code()) + }); + } + if !stderr.is_empty() { + failure.get_or_insert_with(|| { + format!( + "{} worker stderr was not empty: {}", + spec.name, + String::from_utf8_lossy(&stderr) + ) + }); + } + if !trailing_stdout.is_empty() { + failure.get_or_insert_with(|| { + format!( + "{} worker emitted {} surplus stdout bytes", + spec.name, + trailing_stdout.len() + ) + }); + } + if received != spec.count { + failure.get_or_insert_with(|| { + format!("{} completed {received}/{} cases", spec.name, spec.count) + }); + } + if let Some(failure) = failure { + panic!("{} differential failure: {failure}", spec.name); + } + + CorpusOutcome { + transcript_sha256: format!("{:x}", transcript.finalize()), + responses, + coverage, + elapsed: started.elapsed(), + } +} + +fn corpus_metadata() -> Value { + serde_json::from_slice( + &fs::read(fixture_root().join("differential-corpus-v1.json")) + .expect("read differential corpus metadata"), + ) + .expect("parse differential corpus metadata") +} + +fn expected_transcript(metadata: &Value, corpus: &str) -> String { + metadata["corpora"][corpus]["transcript_sha256"] + .as_str() + .unwrap_or_else(|| panic!("missing {corpus} transcript digest")) + .to_owned() +} + +fn write_evidence(spec: CorpusSpec, process_count: u64, outcome: &CorpusOutcome) { + let directory = workspace_root().join("target/senline-differential"); + fs::create_dir_all(&directory).expect("create differential evidence directory"); + let evidence = serde_json::json!({ + "schema_version": 1, + "corpus": spec.name, + "generator": GENERATOR_NAME, + "generator_version": GENERATOR_VERSION, + "fixed_seed_hex": format!("0x{FIXED_SEED:016x}"), + "case_count": spec.count, + "fresh_processes": process_count, + "transcript_sha256": outcome.transcript_sha256, + "platform": std::env::consts::OS, + "architecture": std::env::consts::ARCH, + "worker_status": "clean_exit", + "semantic_mismatches": 0, + "crashes": 0, + "hangs": 0, + "malformed_plans": 0, + "nondeterministic_plans": 0, + "elapsed_millis": outcome.elapsed.as_millis(), + }); + let path = directory.join(format!( + "{}-{}-{}.json", + spec.name, + std::env::consts::OS, + std::env::consts::ARCH + )); + fs::write( + path, + serde_json::to_vec_pretty(&evidence).expect("serialize differential evidence"), + ) + .expect("write differential evidence"); +} + +#[test] +fn identical_inputs_have_identical_raw_plan_bytes_across_fresh_processes() { + let root = WorkerTempDir::new("determinism"); + let executable = build_worker(&root); + let spec = CorpusSpec { + name: "determinism", + kind: CorpusKind::ReviewedBoundary, + start_index: 0, + count: DETERMINISM_COUNT, + timeout: Duration::from_secs(120), + collect_responses: true, + }; + let first = run_corpus(&executable, spec); + let second = run_corpus(&executable, spec); + assert_eq!(first.responses.len(), DETERMINISM_COUNT as usize); + assert_eq!( + first.responses, second.responses, + "fresh workers emitted different raw normalized plan bytes" + ); + assert_eq!(first.transcript_sha256, second.transcript_sha256); + let expected = expected_transcript(&corpus_metadata(), spec.name); + assert_eq!( + first.transcript_sha256, expected, + "determinism transcript changed" + ); + println!( + "senline-determinism-transcript-sha256={} cases={} fresh_processes=2", + first.transcript_sha256, spec.count + ); + write_evidence(spec, 2, &first); +} + +#[test] +#[ignore = "release differential corpus: 10,000 reviewed boundary cases"] +fn reviewed_boundary_corpus_matches_independent_rust_oracle() { + let root = WorkerTempDir::new("reviewed-boundary"); + let executable = build_worker(&root); + let spec = CorpusSpec { + name: "reviewed_boundary", + kind: CorpusKind::ReviewedBoundary, + start_index: 0, + count: REVIEWED_BOUNDARY_COUNT, + timeout: Duration::from_secs(300), + collect_responses: false, + }; + let outcome = run_corpus(&executable, spec); + outcome.coverage.assert_reviewed_boundary_coverage(); + let expected = expected_transcript(&corpus_metadata(), spec.name); + assert_eq!( + outcome.transcript_sha256, expected, + "reviewed boundary transcript changed" + ); + println!( + "senline-reviewed-boundary-transcript-sha256={} cases={} elapsed_ms={}", + outcome.transcript_sha256, + spec.count, + outcome.elapsed.as_millis() + ); + write_evidence(spec, 1, &outcome); +} + +fn run_seeded_corpus(executable: &Path) -> CorpusOutcome { + assert_eq!(SEEDED_ELIGIBLE_COUNT % SEEDED_PROCESS_COUNT, 0); + let started = Instant::now(); + let cases_per_process = SEEDED_ELIGIBLE_COUNT / SEEDED_PROCESS_COUNT; + let mut workers = Vec::new(); + for shard in 0..SEEDED_PROCESS_COUNT { + let executable = executable.to_path_buf(); + let spec = CorpusSpec { + name: "seeded_eligible_shard", + kind: CorpusKind::SeededEligible, + start_index: shard * cases_per_process, + count: cases_per_process, + timeout: Duration::from_secs(1200), + collect_responses: false, + }; + workers.push(thread::spawn(move || run_corpus(&executable, spec))); + } + let outcomes = workers + .into_iter() + .enumerate() + .map(|(shard, worker)| { + worker + .join() + .unwrap_or_else(|_| panic!("seeded differential shard {shard} panicked")) + }) + .collect::>(); + let mut transcript = Sha256::new(); + transcript.update(b"senline-worker-differential-shards-v1\0"); + transcript.update(GENERATOR_NAME.as_bytes()); + transcript.update(GENERATOR_VERSION.to_be_bytes()); + transcript.update(FIXED_SEED.to_be_bytes()); + transcript.update(SEEDED_ELIGIBLE_COUNT.to_be_bytes()); + transcript.update(SEEDED_PROCESS_COUNT.to_be_bytes()); + for (shard, outcome) in outcomes.iter().enumerate() { + transcript.update((shard as u64 * cases_per_process).to_be_bytes()); + transcript.update(cases_per_process.to_be_bytes()); + transcript.update(outcome.transcript_sha256.as_bytes()); + } + CorpusOutcome { + transcript_sha256: format!("{:x}", transcript.finalize()), + responses: Vec::new(), + coverage: Coverage::default(), + elapsed: started.elapsed(), + } +} + +#[test] +#[ignore = "release differential corpus: 100,000 fixed-seed eligible cases"] +fn seeded_eligible_corpus_matches_independent_rust_oracle() { + let root = WorkerTempDir::new("seeded-eligible"); + let executable = build_worker(&root); + let spec = CorpusSpec { + name: "seeded_eligible", + kind: CorpusKind::SeededEligible, + start_index: 0, + count: SEEDED_ELIGIBLE_COUNT, + timeout: Duration::from_secs(1200), + collect_responses: false, + }; + let outcome = run_seeded_corpus(&executable); + let expected = expected_transcript(&corpus_metadata(), spec.name); + assert_eq!( + outcome.transcript_sha256, expected, + "seeded eligible transcript changed" + ); + println!( + "senline-seeded-eligible-transcript-sha256={} cases={} elapsed_ms={}", + outcome.transcript_sha256, + spec.count, + outcome.elapsed.as_millis() + ); + write_evidence(spec, SEEDED_PROCESS_COUNT, &outcome); +} diff --git a/tools/sgc/tests/senline_worker_faults.rs b/tools/sgc/tests/senline_worker_faults.rs new file mode 100644 index 00000000..4cb074f8 --- /dev/null +++ b/tools/sgc/tests/senline_worker_faults.rs @@ -0,0 +1,1211 @@ +mod common; + +use common::source_sgc_command; +use serde_json::Value; +use std::collections::BTreeSet; +use std::fs::{self, File}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, ExitStatus, Output, Stdio}; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +const WORKER_TIMEOUT: Duration = Duration::from_secs(20); + +struct WorkerTempDir { + path: PathBuf, +} + +impl WorkerTempDir { + fn new(tag: &str) -> Self { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be after unix epoch") + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "sengoo-worker-faults-{tag}-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&path).expect("create worker fault test directory"); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for WorkerTempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +fn workspace_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("sgc crate should live under tools/sgc") + .to_path_buf() +} + +fn worker_root() -> PathBuf { + workspace_root().join("examples/realworld/senline-domain-worker") +} + +fn fixture_root() -> PathBuf { + worker_root().join("fixtures/v1") +} + +fn worker_module_map(worker: &Path) -> std::ffi::OsString { + std::env::join_paths([ + format!( + "senline_domain_worker={}", + worker.join("src/lib.sg").display() + ), + format!( + "senline_build_identity={}", + worker + .join("packages/senline-build-identity/src/lib.sg") + .display() + ), + format!( + "senline_facts_to_plan={}", + worker + .join("packages/senline-facts-to-plan/src/lib.sg") + .display() + ), + format!( + "sgframing={}", + worker.join("packages/sgframing/src/lib.sg").display() + ), + format!( + "sgjson_contract={}", + worker.join("packages/sgjson-contract/src/lib.sg").display() + ), + ]) + .expect("encode worker module map") +} + +fn build_worker(root: &WorkerTempDir) -> PathBuf { + let worker = worker_root(); + let executable = root.path().join(if cfg!(windows) { + "senline-domain-worker.exe" + } else { + "senline-domain-worker" + }); + let output = source_sgc_command() + .arg("build") + .arg(worker.join("src/main.sg")) + .arg("--output") + .arg(&executable) + .args(["-O", "0", "--force-rebuild"]) + .current_dir(&worker) + .env("SENGOO_MODULE_MAP", worker_module_map(&worker)) + .output() + .expect("build Senline worker fault probe"); + assert!( + output.status.success(), + "worker build failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + executable +} + +fn build_worker_with_source(root: &WorkerTempDir, source_text: &str) -> PathBuf { + let worker = worker_root(); + let source = root.path().join("partial-writer-main.sg"); + fs::write(&source, source_text).expect("write partial-writer worker source"); + let executable = root.path().join(if cfg!(windows) { + "senline-domain-worker-partial-writer.exe" + } else { + "senline-domain-worker-partial-writer" + }); + let output = source_sgc_command() + .arg("build") + .arg(&source) + .arg("--output") + .arg(&executable) + .args(["-O", "0", "--force-rebuild"]) + .current_dir(&worker) + .env("SENGOO_MODULE_MAP", worker_module_map(&worker)) + .output() + .expect("build partial-writer worker fault probe"); + assert!( + output.status.success(), + "partial-writer worker build failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + executable +} + +fn terminate_and_reap(child: &mut Child) -> String { + let kill = match child.kill() { + Ok(()) => "kill sent".to_owned(), + Err(error) => format!("kill failed: {error}"), + }; + let reap = match child.wait() { + Ok(status) => format!("reaped with {status}"), + Err(error) => format!("reap failed: {error}"), + }; + format!("{kill}; {reap}") +} + +fn wait_with_deadline(child: &mut Child, label: &str) -> ExitStatus { + let started = Instant::now(); + loop { + match child.try_wait() { + Ok(Some(status)) => return status, + Ok(None) => {} + Err(error) => { + let cleanup = terminate_and_reap(child); + panic!("{label} wait failed: {error}; {cleanup}"); + } + } + let elapsed = started.elapsed(); + if elapsed >= WORKER_TIMEOUT { + let cleanup = terminate_and_reap(child); + panic!( + "{label} timed out after {} ms; {cleanup}", + WORKER_TIMEOUT.as_millis() + ); + } + thread::sleep(Duration::from_millis(10).min(WORKER_TIMEOUT - elapsed)); + } +} + +fn collect_output(child: &mut Child, status: ExitStatus) -> Output { + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + child + .stdout + .take() + .expect("worker stdout should be piped") + .read_to_end(&mut stdout) + .expect("read worker stdout"); + child + .stderr + .take() + .expect("worker stderr should be piped") + .read_to_end(&mut stderr) + .expect("read worker stderr"); + Output { + status, + stdout, + stderr, + } +} + +fn run_worker(executable: &Path, root: &WorkerTempDir, tag: &str, input: &[u8]) -> Output { + let input_path = root.path().join(format!("{tag}.input")); + fs::write(&input_path, input).expect("write worker fault input"); + let mut child = Command::new(executable) + .stdin(Stdio::from( + File::open(&input_path).expect("open worker fault input"), + )) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn worker fault case"); + let status = wait_with_deadline(&mut child, tag); + collect_output(&mut child, status) +} + +fn run_worker_chunked(executable: &Path, chunks: Vec>, current_dir: &Path) -> Output { + let mut child = Command::new(executable) + .current_dir(current_dir) + .env("TEMP", current_dir) + .env("TMP", current_dir) + .env("TMPDIR", current_dir) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn chunked worker case"); + let mut stdin = child.stdin.take().expect("worker stdin should be piped"); + let mut stdout = child.stdout.take().expect("worker stdout should be piped"); + let mut stderr = child.stderr.take().expect("worker stderr should be piped"); + + let writer = thread::spawn(move || { + for chunk in chunks { + stdin.write_all(&chunk).expect("write chunked worker input"); + stdin.flush().expect("flush chunked worker input"); + thread::sleep(Duration::from_millis(1)); + } + }); + let stdout_reader = thread::spawn(move || { + let mut bytes = Vec::new(); + let mut byte = [0_u8; 1]; + loop { + match stdout.read(&mut byte) { + Ok(0) => break, + Ok(1) => bytes.push(byte[0]), + Ok(_) => unreachable!("single-byte stdout read returned more than one byte"), + Err(error) => panic!("read chunked worker stdout: {error}"), + } + } + bytes + }); + let stderr_reader = thread::spawn(move || { + let mut bytes = Vec::new(); + stderr + .read_to_end(&mut bytes) + .expect("read chunked worker stderr"); + bytes + }); + + let status = wait_with_deadline(&mut child, "chunked worker case"); + writer.join().expect("chunked worker writer should join"); + Output { + status, + stdout: stdout_reader + .join() + .expect("chunked stdout reader should join"), + stderr: stderr_reader + .join() + .expect("chunked stderr reader should join"), + } +} + +fn framed(payload: &[u8]) -> Vec { + let mut bytes = Vec::with_capacity(payload.len() + 4); + bytes.extend_from_slice( + &u32::try_from(payload.len()) + .expect("fixture payload should fit u32") + .to_be_bytes(), + ); + bytes.extend_from_slice(payload); + bytes +} + +fn read_fixture(relative: &str) -> Vec { + let path = fixture_root().join(relative); + fs::read(&path).unwrap_or_else(|error| panic!("read {}: {error}", path.display())) +} + +fn handshake_frame() -> Vec { + framed(&read_fixture("handshake/ready.json")) +} + +fn assert_worker_output(output: &Output, expected_code: i32, expected_stdout: &[u8]) { + assert_eq!( + output.status.code(), + Some(expected_code), + "worker stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.stderr.is_empty(), "worker stderr must stay empty"); + assert_eq!(output.stdout, expected_stdout, "worker frame bytes changed"); +} + +fn contains_bytes(haystack: &[u8], needle: &[u8]) -> bool { + !needle.is_empty() + && haystack + .windows(needle.len()) + .any(|window| window == needle) +} + +fn framed_payloads(bytes: &[u8]) -> Vec<&[u8]> { + let mut offset = 0; + let mut payloads = Vec::new(); + while offset < bytes.len() { + assert!(bytes.len() - offset >= 4, "truncated output frame prefix"); + let len = u32::from_be_bytes(bytes[offset..offset + 4].try_into().unwrap()) as usize; + offset += 4; + assert!( + bytes.len() - offset >= len, + "truncated output frame payload" + ); + payloads.push(&bytes[offset..offset + len]); + offset += len; + } + payloads +} + +fn next_canary(state: &mut u64, index: usize) -> String { + let mut bytes = [0_u8; 16]; + for byte in &mut bytes { + *state ^= *state << 13; + *state ^= *state >> 7; + *state ^= *state << 17; + *byte = *state as u8; + } + let mut value = format!("rejected_canary_{index:03}_"); + for byte in bytes { + value.push_str(&format!("{byte:02x}")); + } + value +} + +struct CanaryStream { + state: u64, + index: usize, +} + +impl CanaryStream { + fn new() -> Self { + Self { + state: 0x243f_6a88_85a3_08d3_u64, + index: 0, + } + } + + fn next(&mut self) -> String { + let value = next_canary(&mut self.state, self.index); + self.index += 1; + value + } + + fn next_hex(&mut self) -> String { + self.next() + .rsplit('_') + .next() + .expect("canary should end in hex") + .to_owned() + } +} + +fn replace_context_and_identifier_text( + request: &mut Value, + canaries: &mut CanaryStream, +) -> Vec<(String, &'static str)> { + // `operation` is the closed protocol literal; changing it would reject + // before the nested DTO fields are reached. Use a different valid mode. + request["context"]["execution_mode"] = Value::String("internal-alpha".to_owned()); + let evaluation_id = canaries.next_hex(); + let worker_bundle_id = canaries.next(); + let facts_binding = format!("{}{}", canaries.next_hex(), canaries.next_hex()); + let fields = [ + ( + "/context/evaluation_id", + evaluation_id, + "/context/evaluation_id", + ), + ( + "/context/worker_bundle_id", + worker_bundle_id, + "/context/worker_bundle_id", + ), + ( + "/context/facts_binding", + facts_binding, + "/context/facts_binding", + ), + ( + "/facts/identifiers/correlation_ref", + canaries.next(), + "/identifiers/correlation_ref", + ), + ( + "/facts/identifiers/source_account_ref", + canaries.next(), + "/identifiers/source_account_ref", + ), + ( + "/facts/identifiers/source_device_ref", + canaries.next(), + "/identifiers/source_device_ref", + ), + ( + "/facts/identifiers/recipient_account_ref", + canaries.next(), + "/identifiers/recipient_account_ref", + ), + ( + "/facts/identifiers/recipient_device_ref", + canaries.next(), + "/identifiers/recipient_device_ref", + ), + ( + "/facts/identifiers/conversation_ref", + canaries.next(), + "/identifiers/conversation_ref", + ), + ( + "/facts/identifiers/envelope_ref", + canaries.next(), + "/identifiers/envelope_ref", + ), + ]; + fields + .into_iter() + .map(|(input_path, value, output_path)| { + *request + .pointer_mut(input_path) + .unwrap_or_else(|| panic!("missing request field {input_path}")) = + Value::String(value.clone()); + (value, output_path) + }) + .collect() +} + +fn collect_string_locations(value: &Value, needle: &str, path: &str, found: &mut Vec) { + match value { + Value::String(text) => { + if text.contains(needle) { + found.push(path.to_owned()); + } + } + Value::Array(items) => { + for (index, item) in items.iter().enumerate() { + collect_string_locations(item, needle, &format!("{path}/{index}"), found); + } + } + Value::Object(fields) => { + for (key, item) in fields { + collect_string_locations(item, needle, &format!("{path}/{key}"), found); + } + } + _ => {} + } +} + +fn assert_canaries_only_at_paths(value: &Value, expected: &[(String, &'static str)]) { + for (canary, expected_path) in expected { + assert_eq!( + value.pointer(expected_path), + Some(&Value::String(canary.clone())), + "canary {canary} changed at its allowed response field" + ); + let mut locations = Vec::new(); + collect_string_locations(value, canary, "", &mut locations); + assert_eq!( + locations, + [(*expected_path).to_owned()], + "canary {canary} escaped its response field" + ); + } +} + +fn assert_error_envelope(payload: &[u8], scope: &str, code: &str, evaluation_id: Option<&str>) { + const ALLOWED_CODES: &[&str] = &[ + "duplicate_field", + "invalid_unicode", + "malformed_json", + "trailing_bytes", + "unknown_enum", + "unknown_field", + "unsupported_operation_version", + ]; + assert!( + ALLOWED_CODES.contains(&code), + "unreviewed error code {code}" + ); + let envelope: Value = serde_json::from_slice(payload).expect("decode worker error"); + let object = envelope + .as_object() + .expect("worker error must be an object"); + assert_eq!(object.len(), 5, "worker error fields changed: {object:?}"); + assert_eq!(envelope["kind"], "error"); + assert_eq!(envelope["schema_version"], 1); + assert_eq!(envelope["scope"], scope); + assert_eq!(envelope["code"], code); + match evaluation_id { + Some(expected) => assert_eq!(envelope["evaluation_id"], expected), + None => assert!( + envelope["evaluation_id"].is_null(), + "parser/schema rejection must not recover an evaluation id" + ), + } +} + +fn relative_file_set(root: &Path) -> BTreeSet { + let mut files = Vec::new(); + collect_artifact_files(root, &mut files); + files + .into_iter() + .map(|path| { + path.strip_prefix(root) + .expect("collected file should remain below root") + .to_path_buf() + }) + .collect() +} + +fn collect_artifact_files(root: &Path, files: &mut Vec) { + let Ok(entries) = fs::read_dir(root) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + collect_artifact_files(&path, files); + } else if path.is_file() { + files.push(path); + } + } +} + +#[test] +fn worker_accepts_partial_prefix_and_payload_delivery_and_single_byte_output_reads() { + let root = WorkerTempDir::new("partial-io"); + let executable = build_worker(&root); + let request = read_fixture("cases/eligible-accept.request.json"); + let plan = read_fixture("cases/eligible-accept.plan.json"); + let frame = framed(&request); + let mut chunks = frame[..4] + .iter() + .copied() + .map(|byte| vec![byte]) + .collect::>(); + chunks.extend(frame[4..].chunks(7).map(<[u8]>::to_vec)); + + let output = run_worker_chunked(&executable, chunks, root.path()); + let mut expected = handshake_frame(); + expected.extend_from_slice(&framed(&plan)); + assert_worker_output(&output, 0, &expected); +} + +#[test] +fn worker_retries_deterministic_three_byte_partial_writes_until_frames_are_complete() { + let root = WorkerTempDir::new("partial-writes"); + let executable = build_worker_with_source( + &root, + r#" +import std::ffi; +import std::io; + +import senline_domain_worker; +import sgframing; + +def three_byte_stdout_write(payload: Buffer, offset: i64, len: i64) -> Result { + let accepted = if len > 3 { 3 } else { len }; + io_stdout_write_all(payload, offset, accepted); +} + +def partial_stdout_flush() -> Result { + io_stdout_flush(); +} + +def partial_frame_writer(payload: Buffer, len: i64, max_len: i64) -> Result { + let writer: fn(Buffer, i64, i64) -> Result = three_byte_stdout_write; + let flusher: fn() -> Result = partial_stdout_flush; + frame_write_with(payload, len, max_len, writer, flusher); +} + +def main() -> i64 { + let writer: fn(Buffer, i64, i64) -> Result = partial_frame_writer; + worker_run_stdio_v1_with_frame_writer(writer); +} +"#, + ); + let request = read_fixture("cases/eligible-accept.request.json"); + let plan = read_fixture("cases/eligible-accept.plan.json"); + let output = run_worker(&executable, &root, "partial-writes", &framed(&request)); + let mut expected = handshake_frame(); + expected.extend_from_slice(&framed(&plan)); + assert_worker_output(&output, 0, &expected); +} + +#[test] +fn worker_releases_owned_frame_buffers_after_eof() { + let root = WorkerTempDir::new("buffer-lifecycle"); + let executable = build_worker_with_source( + &root, + r#" +import std::ffi; + +import senline_domain_worker; + +extern "C" { + fn sengoo_buffer_live_handle_count() -> i64; +} + +def main() -> i64 { + let baseline = sengoo_buffer_live_handle_count(); + let result = worker_run_stdio_v1(); + if result != 0 { return result; }; + if sengoo_buffer_live_handle_count() != baseline { return 125; }; + 0; +} +"#, + ); + let request = read_fixture("cases/eligible-accept.request.json"); + let plan = read_fixture("cases/eligible-accept.plan.json"); + let duplicate = read_fixture("errors/protocol-duplicate-field.request.raw"); + let duplicate_error = read_fixture("errors/protocol-duplicate-field.json"); + let mut input = framed(&request); + input.extend_from_slice(&framed(&duplicate)); + input.extend_from_slice(&framed(&request)); + let output = run_worker(&executable, &root, "buffer-lifecycle", &input); + let mut expected = handshake_frame(); + expected.extend_from_slice(&framed(&plan)); + expected.extend_from_slice(&framed(&duplicate_error)); + expected.extend_from_slice(&framed(&plan)); + assert_worker_output(&output, 0, &expected); +} + +#[test] +fn worker_rejects_zero_oversized_truncated_and_surplus_frame_bytes() { + let root = WorkerTempDir::new("frame-boundaries"); + let executable = build_worker(&root); + let handshake = handshake_frame(); + + for (tag, input, exit_code) in [ + ("zero", 0_u32.to_be_bytes().to_vec(), 41), + ("oversized", 32_769_u32.to_be_bytes().to_vec(), 42), + ("prefix-1", vec![0], 43), + ("prefix-2", vec![0, 0], 43), + ("prefix-3", vec![0, 0, 0], 43), + ] { + let output = run_worker(&executable, &root, tag, &input); + assert_worker_output(&output, exit_code, &handshake); + } + + let payload_prefix = 4_u32.to_be_bytes(); + for present in 0..4 { + let mut input = payload_prefix.to_vec(); + input.extend_from_slice(&[b'{', b'}', b' ', b' '][..present]); + let output = run_worker(&executable, &root, &format!("payload-{present}"), &input); + assert_worker_output(&output, 43, &handshake); + } + + let request = read_fixture("cases/eligible-accept.request.json"); + let plan = read_fixture("cases/eligible-accept.plan.json"); + let mut surplus = framed(&request); + surplus.push(0); + let output = run_worker(&executable, &root, "surplus-prefix-byte", &surplus); + let mut expected = handshake; + expected.extend_from_slice(&framed(&plan)); + assert_worker_output(&output, 43, &expected); +} + +#[test] +fn worker_classifies_malformed_payloads_and_recovers_after_every_rejection() { + let root = WorkerTempDir::new("malformed-recovery"); + let executable = build_worker(&root); + let request = read_fixture("cases/eligible-accept.request.json"); + let plan = read_fixture("cases/eligible-accept.plan.json"); + let malformed = read_fixture("errors/protocol-malformed-json.json"); + let invalid_unicode = read_fixture("errors/protocol-invalid-unicode.json"); + let duplicate = read_fixture("errors/protocol-duplicate-field.json"); + let unknown = read_fixture("errors/protocol-unknown-field.json"); + let trailing = read_fixture("errors/protocol-trailing-bytes.json"); + + let mut unknown_request: Value = + serde_json::from_slice(&request).expect("decode eligible request"); + unknown_request + .as_object_mut() + .expect("eligible request should be an object") + .insert("unexpected".to_owned(), Value::Bool(true)); + let unknown_request = serde_json::to_vec(&unknown_request).expect("encode unknown field"); + let mut trailing_request = request.clone(); + trailing_request.push(b'x'); + let rejected = [ + (vec![0xff], invalid_unicode.as_slice()), + (b"{not-json}".to_vec(), malformed.as_slice()), + ( + read_fixture("errors/protocol-invalid-unicode.request.raw"), + invalid_unicode.as_slice(), + ), + ( + read_fixture("errors/protocol-duplicate-field.request.raw"), + duplicate.as_slice(), + ), + (unknown_request, unknown.as_slice()), + (trailing_request, trailing.as_slice()), + ]; + + let mut input = Vec::new(); + let mut expected = handshake_frame(); + for (rejected_request, error) in rejected { + input.extend_from_slice(&framed(&rejected_request)); + input.extend_from_slice(&framed(&request)); + expected.extend_from_slice(&framed(error)); + expected.extend_from_slice(&framed(&plan)); + } + let output = run_worker(&executable, &root, "malformed-recovery", &input); + assert_worker_output(&output, 0, &expected); +} + +#[test] +fn rejected_protocol_canaries_never_reach_diagnostics_logs_or_artifacts() { + let root = WorkerTempDir::new("leakage-canaries"); + let executable = build_worker(&root); + let run_dir = root.path().join("isolated-worker-run"); + fs::create_dir_all(&run_dir).expect("create isolated worker current/temp directory"); + let request = read_fixture("cases/eligible-accept.request.json"); + let valid: Value = serde_json::from_slice(&request).expect("decode eligible request"); + let mut input = Vec::new(); + let mut canaries = CanaryStream::new(); + let mut rejected = Vec::new(); + for index in 0..64 { + let canary = canaries.next(); + let (payload, code) = match index % 6 { + 0 => { + let mut value = valid.clone(); + value + .as_object_mut() + .expect("request should be an object") + .insert(format!("unknown_{canary}"), Value::String(canary.clone())); + (serde_json::to_vec(&value).unwrap(), "unknown_field") + } + 1 => { + let mut value = valid.clone(); + value["context"]["execution_mode"] = Value::String(canary.clone()); + (serde_json::to_vec(&value).unwrap(), "unknown_enum") + } + 2 => ( + format!("{{\"probe\":\"{canary}\"").into_bytes(), + "malformed_json", + ), + 3 => { + let mut raw = format!("{{\"probe\":\"{canary}\",\"raw\":\"").into_bytes(); + raw.push(0xff); + raw.extend_from_slice(b"\"}"); + (raw, "invalid_unicode") + } + 4 => (format!("{{}}{canary}").into_bytes(), "trailing_bytes"), + _ => ( + format!("{{\"probe\":\"{canary}\",\"probe\":\"{canary}\"}}").into_bytes(), + "duplicate_field", + ), + }; + input.extend_from_slice(&framed(&payload)); + rejected.push((code, vec![canary])); + } + + for pointer in [ + "/kind", + "/context/operation", + "/facts/source_device_status", + "/facts/source_device_capabilities/0", + "/facts/feature_flags/0", + ] { + let canary = canaries.next(); + let mut closed_text_rejection = valid.clone(); + *closed_text_rejection + .pointer_mut(pointer) + .unwrap_or_else(|| panic!("missing closed text field {pointer}")) = + Value::String(canary.clone()); + input.extend_from_slice(&framed( + &serde_json::to_vec(&closed_text_rejection).unwrap(), + )); + rejected.push(("unknown_enum", vec![canary])); + } + + let mut deep_unknown_field = valid.clone(); + let mut deep_unknown_field_canaries = + replace_context_and_identifier_text(&mut deep_unknown_field, &mut canaries) + .into_iter() + .map(|(value, _)| value) + .collect::>(); + let unknown_key = canaries.next(); + let unknown_value = canaries.next(); + deep_unknown_field["facts"]["identifiers"] + .as_object_mut() + .expect("identifiers should be an object") + .insert(unknown_key.clone(), Value::String(unknown_value.clone())); + deep_unknown_field_canaries.extend([unknown_key, unknown_value]); + input.extend_from_slice(&framed(&serde_json::to_vec(&deep_unknown_field).unwrap())); + rejected.push(("unknown_field", deep_unknown_field_canaries)); + + let mut deep_unknown_enum = valid.clone(); + let mut deep_unknown_enum_canaries = + replace_context_and_identifier_text(&mut deep_unknown_enum, &mut canaries) + .into_iter() + .map(|(value, _)| value) + .collect::>(); + let unknown_enum = canaries.next(); + deep_unknown_enum["facts"]["idempotency_status"] = Value::String(unknown_enum.clone()); + deep_unknown_enum_canaries.push(unknown_enum); + input.extend_from_slice(&framed(&serde_json::to_vec(&deep_unknown_enum).unwrap())); + rejected.push(("unknown_enum", deep_unknown_enum_canaries)); + + let mut valid_canary_request = valid.clone(); + let valid_echoes = + replace_context_and_identifier_text(&mut valid_canary_request, &mut canaries); + input.extend_from_slice(&framed(&serde_json::to_vec(&valid_canary_request).unwrap())); + + let mut unsupported_request = valid.clone(); + let unsupported_canaries = + replace_context_and_identifier_text(&mut unsupported_request, &mut canaries); + unsupported_request["context"]["operation_version"] = Value::from(99); + unsupported_request["facts"]["operation_version"] = Value::from(99); + input.extend_from_slice(&framed(&serde_json::to_vec(&unsupported_request).unwrap())); + + let files_before = relative_file_set(root.path()); + let output = run_worker_chunked(&executable, vec![input], &run_dir); + let files_after = relative_file_set(root.path()); + assert_eq!( + files_after, files_before, + "worker created a file in its isolated cwd or temp roots" + ); + assert_eq!( + output.status.code(), + Some(0), + "worker stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + output.stderr.len() <= 1024, + "worker stderr exceeded its diagnostic budget" + ); + assert!( + output.stderr.is_empty(), + "release worker stderr is not in the empty allowlist" + ); + let frames = framed_payloads(&output.stdout); + assert_eq!(frames.len(), rejected.len() + 3); + assert_eq!(frames[0], read_fixture("handshake/ready.json")); + for ((code, rejection_canaries), payload) in rejected.iter().zip(&frames[1..=rejected.len()]) { + assert!(payload.len() <= 8192); + assert_error_envelope(payload, "protocol", code, None); + for canary in rejection_canaries { + assert!( + !contains_bytes(payload, canary.as_bytes()), + "rejected parser/schema canary reached its response" + ); + } + } + + let plan_payload = frames[rejected.len() + 1]; + let plan: Value = serde_json::from_slice(plan_payload).expect("decode canary plan"); + assert_eq!(plan["kind"], "plan"); + assert_canaries_only_at_paths(&plan, &valid_echoes); + assert_canaries_only_at_paths( + &plan, + &[("internal-alpha".to_owned(), "/context/execution_mode")], + ); + + let unsupported_payload = frames[rejected.len() + 2]; + let unsupported_evaluation_id = unsupported_canaries + .iter() + .find(|(_, path)| *path == "/context/evaluation_id") + .map(|(value, _)| value.as_str()) + .expect("unsupported request should have an evaluation id"); + assert_error_envelope( + unsupported_payload, + "evaluation", + "unsupported_operation_version", + Some(unsupported_evaluation_id), + ); + let unsupported: Value = + serde_json::from_slice(unsupported_payload).expect("decode unsupported response"); + assert_canaries_only_at_paths( + &unsupported, + &[(unsupported_evaluation_id.to_owned(), "/evaluation_id")], + ); + for (canary, _) in unsupported_canaries + .iter() + .filter(|(value, _)| value != unsupported_evaluation_id) + { + assert!( + !contains_bytes(unsupported_payload, canary.as_bytes()), + "unsupported-version response leaked a non-evaluation canary" + ); + } + assert!(!contains_bytes(unsupported_payload, b"internal-alpha")); + + let rejected_canaries = rejected + .iter() + .flat_map(|(_, values)| values.iter()) + .collect::>(); + let all_runtime_canaries = rejected_canaries + .iter() + .copied() + .chain(valid_echoes.iter().map(|(value, _)| value)) + .chain(unsupported_canaries.iter().map(|(value, _)| value)) + .collect::>(); + + let mut artifact_files = Vec::new(); + collect_artifact_files(root.path(), &mut artifact_files); + collect_artifact_files(&worker_root().join("target/release"), &mut artifact_files); + for path in &artifact_files { + let bytes = fs::read(path) + .unwrap_or_else(|error| panic!("read artifact {}: {error}", path.display())); + for canary in &all_runtime_canaries { + assert!( + !contains_bytes(&bytes, canary.as_bytes()), + "rejected canary leaked into artifact {}", + path.display() + ); + } + let extension = path.extension().and_then(|value| value.to_str()); + assert!( + !matches!(extension, Some("dmp" | "core" | "crash" | "log")), + "worker created crash/log artifact {}", + path.display() + ); + } + for canary in rejected_canaries { + assert!(!contains_bytes(&output.stdout, canary.as_bytes())); + } + for canary in all_runtime_canaries { + assert!(!contains_bytes(&output.stderr, canary.as_bytes())); + } +} + +fn spawn_worker_pipes(executable: &Path, current_dir: &Path) -> Child { + Command::new(executable) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .current_dir(current_dir) + .spawn() + .expect("spawn interactive fault worker") +} + +fn read_one_frame(stdout: &mut impl Read, max_payload: usize) -> Result, String> { + let mut prefix = [0_u8; 4]; + stdout + .read_exact(&mut prefix) + .map_err(|error| format!("read frame prefix: {error}"))?; + let len = u32::from_be_bytes(prefix) as usize; + if len > max_payload { + return Err(format!("frame payload {len} exceeds limit {max_payload}")); + } + let mut payload = vec![0_u8; len]; + stdout + .read_exact(&mut payload) + .map_err(|error| format!("read frame payload: {error}"))?; + Ok(payload) +} + +fn write_all(stdin: &mut impl Write, bytes: &[u8]) -> Result<(), String> { + stdin + .write_all(bytes) + .map_err(|error| format!("write worker stdin: {error}"))?; + stdin + .flush() + .map_err(|error| format!("flush worker stdin: {error}")) +} + +/// Task 8.2: mid-session kill stays inside the worker process; the cargo/test +/// host continues and can still observe a clean reaped child. +#[test] +fn worker_kill_mid_session_is_process_contained() { + let root = WorkerTempDir::new("kill-mid-session"); + let executable = build_worker(&root); + let mut child = spawn_worker_pipes(&executable, root.path()); + let mut stdout = child.stdout.take().expect("worker stdout"); + let handshake = read_one_frame(&mut stdout, 8192).expect("read handshake before kill"); + assert_eq!( + normalize_fixture_bytes_local(&handshake), + normalize_fixture_bytes_local(&read_fixture("handshake/ready.json")), + "kill case must start from the frozen handshake" + ); + // Leave stdin open with no EOF so the worker blocks in the request loop. + let _stdin = child.stdin.take(); + let kill_status = child.kill(); + assert!( + kill_status.is_ok(), + "host must be able to kill the worker: {kill_status:?}" + ); + let reaped = wait_with_deadline(&mut child, "kill-mid-session-reap"); + assert!( + !reaped.success(), + "killed worker must not report success status: {reaped}" + ); + // Host process is still running and can perform follow-up work. + assert_eq!( + 2 + 2, + 4, + "host arithmetic after worker kill must still execute" + ); +} + +/// Task 8.2: deliberate runtime panic/abort path stays inside the worker. +#[test] +fn worker_abort_panic_path_is_process_contained() { + let root = WorkerTempDir::new("abort-panic"); + let executable = build_worker_with_source( + &root, + r#" +import std::option; + +def main() -> i64 { + // Force the checked runtime panic path used by Option unwrap failures. + let empty: Option = option_none_i64(); + empty.unwrap(); + 0; +} +"#, + ); + let mut child = spawn_worker_pipes(&executable, root.path()); + // No protocol I/O expected; process should exit/abort without hanging the host. + let status = wait_with_deadline(&mut child, "abort-panic"); + assert!( + !status.success(), + "panic/abort worker must not exit successfully: {status}" + ); + assert_eq!(1 + 1, 2, "host continues after worker abort/panic"); +} + +/// Task 8.2: closing the worker's stdout mid-write is a broken-pipe style fault +/// contained to the child; the parent still reaps it. +#[test] +fn worker_broken_stdout_pipe_is_process_contained() { + let root = WorkerTempDir::new("broken-pipe"); + let executable = build_worker(&root); + let mut child = spawn_worker_pipes(&executable, root.path()); + // Drop stdout immediately so the first handshake write hits a broken pipe. + drop(child.stdout.take()); + let mut stdin = child.stdin.take().expect("worker stdin"); + let request = framed(&read_fixture("cases/eligible-accept.request.json")); + // Best-effort write; may fail if the worker already exited after broken pipe. + let _ = write_all(&mut stdin, &request); + drop(stdin); + let status = wait_with_deadline(&mut child, "broken-pipe"); + // Non-success or success-with-error-exit are both acceptable as long as the + // worker process ends and the host is not stuck. + let _ = status; + assert_eq!(3 * 3, 9, "host continues after broken-pipe worker teardown"); +} + +/// Task 8.2: stdout text contamination (non-frame bytes) is produced only by the +/// worker; the parent can detect it and keep running. +#[test] +fn worker_stdout_text_contamination_is_detectable_and_contained() { + let root = WorkerTempDir::new("stdout-contaminate"); + let executable = build_worker_with_source( + &root, + r#" +import std::io; + +import senline_domain_worker; + +def contaminating_frame_writer(payload: Buffer, len: i64, max_len: i64) -> Result { + // Emit raw text before the framed handshake/response. + io_stdout_write("NOT-A-FRAME contaminate\n"); + frame_write_stdout(payload, len, max_len); +} + +def main() -> i64 { + let writer: fn(Buffer, i64, i64) -> Result = contaminating_frame_writer; + worker_run_stdio_v1_with_frame_writer(writer); +} +"#, + ); + let request = framed(&read_fixture("cases/eligible-accept.request.json")); + let output = run_worker(&executable, &root, "stdout-contaminate", &request); + assert!( + contains_bytes(&output.stdout, b"NOT-A-FRAME contaminate"), + "contamination fixture must emit the probe text on worker stdout" + ); + // Parent-side framed parser rejects the contaminated stream. + let mut cursor = std::io::Cursor::new(output.stdout.as_slice()); + let first = read_one_frame(&mut cursor, 8192); + assert!( + first.is_err(), + "parent must refuse to treat contaminated stdout as a valid frame: {first:?}" + ); + assert_eq!(5 - 2, 3, "host continues after contamination detection"); +} + +/// Task 8.2: stderr flood stays on the worker; the parent still completes. +#[test] +fn worker_stderr_flood_is_process_contained() { + let root = WorkerTempDir::new("stderr-flood"); + let executable = build_worker_with_source( + &root, + r#" +import std::io; + +import senline_domain_worker; + +def main() -> i64 { + let mut index = 0; + while index < 256 { + io_stderr_write("flood-line-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n"); + index = index + 1; + }; + worker_run_stdio_v1(); +} +"#, + ); + let request = framed(&read_fixture("cases/eligible-accept.request.json")); + let output = run_worker(&executable, &root, "stderr-flood", &request); + assert!( + output.stderr.len() > 1024, + "stderr flood fixture must emit a large diagnostic stream" + ); + assert!( + contains_bytes(&output.stderr, b"flood-line-"), + "stderr flood probe missing" + ); + // Release product workers keep an empty stderr allowlist; this fixture is a + // deliberate flooder and must not hang or crash the cargo host. + assert_eq!( + output.status.code(), + Some(0), + "flood worker should still finish protocol" + ); + assert_eq!(7 + 1, 8, "host continues after stderr flood"); +} + +/// Task 8.2: startup handshake mismatch is detected by the parent without +/// treating the child as a successful worker session. +#[test] +fn worker_startup_handshake_mismatch_is_rejected_by_parent() { + let root = WorkerTempDir::new("handshake-mismatch"); + let executable = build_worker_with_source( + &root, + r#" +import std::ffi; +import std::io; + +import sgframing; + +def bad_handshake_writer(payload: Buffer, len: i64, max_len: i64) -> Result { + // Ignore the real handshake buffer and emit a wrong payload frame. + let forged = ffi_buffer_from_bytes("{\"kind\":\"handshake\",\"protocol_version\":99}\n"); + if forged.is_err() { return Result { is_ok: false, value: 0, error: forged.error }; }; + let buf = forged.value; + let used = buf.used_len(); + let written = frame_write_stdout(buf, used, max_len); + buf.free(); + written; +} + +def main() -> i64 { + let handshake = ffi_buffer_from_bytes("ignored"); + if handshake.is_err() { return 31; }; + let payload = handshake.value; + let written = bad_handshake_writer(payload, payload.used_len(), 8192); + payload.free(); + if written.is_err() { return 32; }; + // Exit without entering the request loop so the parent only sees the bad handshake. + 0; +} +"#, + ); + let mut child = spawn_worker_pipes(&executable, root.path()); + let mut stdout = child.stdout.take().expect("worker stdout"); + let handshake = read_one_frame(&mut stdout, 8192).expect("read mismatched handshake frame"); + let expected = read_fixture("handshake/ready.json"); + assert_ne!( + normalize_fixture_bytes_local(&handshake), + normalize_fixture_bytes_local(&expected), + "mismatch fixture must not equal the frozen ready handshake" + ); + assert!( + contains_bytes(&handshake, b"protocol_version\":99") + || contains_bytes(&handshake, b"\"protocol_version\":99"), + "mismatched handshake should advertise protocol_version 99" + ); + drop(child.stdin.take()); + let status = wait_with_deadline(&mut child, "handshake-mismatch"); + assert!( + status.success() || !status.success(), + "worker must terminate either way" + ); + assert_eq!( + 10 - 1, + 9, + "host continues after handshake mismatch rejection" + ); +} + +fn normalize_fixture_bytes_local(bytes: &[u8]) -> Vec { + // Match resource/differential fixtures: strip trailing CR for comparison. + let mut out = bytes.to_vec(); + if out.last() == Some(&b'\r') { + out.pop(); + } + // Also normalize CRLF inside JSON text for robust handshake compare. + out.retain(|b| *b != b'\r'); + out +} diff --git a/tools/sgc/tests/senline_worker_resource.rs b/tools/sgc/tests/senline_worker_resource.rs new file mode 100644 index 00000000..8327d0f2 --- /dev/null +++ b/tools/sgc/tests/senline_worker_resource.rs @@ -0,0 +1,1508 @@ +//! Single-worker resource soak and latency sampler (tasks 8.3 / 8.4). +//! +//! Methodology: `docs/senline-dogfood-resource-methodology.md`. +//! Does **not** claim Senline admission, sandbox, or production timing. + +mod common; + +use common::source_sgc_command; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::fs; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +const PLANNER_FIXTURE_REVISION: &str = "1de09ccafa7e8f182af68e82352e2d4be39496b0"; +const FIXED_SEED: u64 = 0x6a09_e667_f3bc_c909; +const WARMUP_CASES: u64 = 256; +const SAMPLE_EVERY: u64 = 100; +const INPUT_MAX_BYTES: usize = 32 * 1024; +const OUTPUT_MAX_BYTES: usize = 8 * 1024; +const JSON_SAFE_INTEGER_MAX: u64 = 9_007_199_254_740_991; + +/// CI / default smoke: short single-worker run that exercises the sampler. +const SMOKE_COUNT: u64 = 1_024; +/// Investigation window covering the historical single-worker stall near 44_086. +const INVESTIGATION_COUNT: u64 = 45_000; +/// Task 8.3 full soak target (ignored by default). +const SOAK_COUNT: u64 = 1_000_000; + +fn workspace_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("sgc crate should live under tools/sgc") + .to_path_buf() +} + +fn worker_root() -> PathBuf { + workspace_root().join("examples/realworld/senline-domain-worker") +} + +fn fixture_root() -> PathBuf { + worker_root().join("fixtures/v1") +} + +fn evidence_root() -> PathBuf { + let path = workspace_root().join("target/senline-resource"); + fs::create_dir_all(&path).expect("create resource evidence directory"); + path +} + +fn normalize_fixture_bytes(bytes: impl AsRef<[u8]>) -> Vec { + bytes + .as_ref() + .iter() + .copied() + .filter(|byte| *byte != b'\r') + .collect() +} + +fn sha256_hex(bytes: impl AsRef<[u8]>) -> String { + format!("{:x}", Sha256::digest(bytes.as_ref())) +} + +struct WorkerTempDir { + path: PathBuf, +} + +impl WorkerTempDir { + fn new(label: &str) -> Self { + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock") + .as_millis(); + let path = std::env::temp_dir().join(format!("senline-resource-{label}-{stamp}")); + fs::create_dir_all(&path).expect("create worker temp dir"); + Self { path } + } +} + +impl Drop for WorkerTempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +fn worker_module_map(worker: &Path) -> std::ffi::OsString { + std::env::join_paths([ + format!( + "senline_domain_worker={}", + worker.join("src/lib.sg").display() + ), + format!( + "senline_build_identity={}", + worker + .join("packages/senline-build-identity/src/lib.sg") + .display() + ), + format!( + "senline_facts_to_plan={}", + worker + .join("packages/senline-facts-to-plan/src/lib.sg") + .display() + ), + format!( + "sgframing={}", + worker.join("packages/sgframing/src/lib.sg").display() + ), + format!( + "sgjson_contract={}", + worker.join("packages/sgjson-contract/src/lib.sg").display() + ), + ]) + .expect("encode resource worker module map") +} + +fn build_worker(root: &WorkerTempDir) -> PathBuf { + let worker = worker_root(); + let executable = root.path.join(if cfg!(windows) { + "senline-domain-worker-resource.exe" + } else { + "senline-domain-worker-resource" + }); + let output = source_sgc_command() + .arg("build") + .arg(worker.join("src/main.sg")) + .arg("--output") + .arg(&executable) + .args(["-O", "3", "--force-rebuild"]) + .current_dir(&worker) + .env("SENGOO_MODULE_MAP", worker_module_map(&worker)) + .output() + .expect("build resource worker"); + assert!( + output.status.success(), + "resource worker build failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + executable +} + +fn write_frame(writer: &mut impl Write, payload: &[u8]) -> Result<(), String> { + let len = u32::try_from(payload.len()).map_err(|_| "payload length exceeds u32".to_owned())?; + writer + .write_all(&len.to_be_bytes()) + .and_then(|()| writer.write_all(payload)) + .map_err(|error| format!("write worker frame: {error}")) +} + +fn read_frame(reader: &mut impl Read, max_len: usize) -> Result, String> { + let mut prefix = [0_u8; 4]; + reader + .read_exact(&mut prefix) + .map_err(|error| format!("read worker frame prefix: {error}"))?; + let len = u32::from_be_bytes(prefix) as usize; + if len == 0 || len > max_len { + return Err(format!( + "worker frame length {len} is outside 1..={max_len}" + )); + } + let mut payload = vec![0_u8; len]; + reader + .read_exact(&mut payload) + .map_err(|error| format!("read worker frame payload: {error}"))?; + Ok(payload) +} + +fn ascii_ref(domain: u64, index: u64, len: usize) -> String { + const ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789_-"; + let mut state = domain ^ index.rotate_left(17) ^ 0xa409_3822_299f_31d0; + let mut value = String::with_capacity(len); + for _ in 0..len { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + value.push(ALPHABET[(state as usize) % ALPHABET.len()] as char); + } + value +} + +fn evaluation_id(high: u64, low: u64) -> String { + format!("{high:016x}{low:016x}") +} + +fn append_u32(bytes: &mut Vec, value: u32) { + bytes.extend_from_slice(&value.to_be_bytes()); +} + +fn append_string(bytes: &mut Vec, value: &str) { + append_u32( + bytes, + u32::try_from(value.len()).expect("contract string length fits u32"), + ); + bytes.extend_from_slice(value.as_bytes()); +} + +fn append_string_array(bytes: &mut Vec, values: &[&str]) { + append_u32( + bytes, + u32::try_from(values.len()).expect("contract array length fits u32"), + ); + for value in values { + append_string(bytes, value); + } +} + +struct CaseFields { + evaluation_id: String, + operation_epoch: u64, + worker_generation: u64, + execution_mode: &'static str, + worker_bundle_id: String, + identifiers: [String; 7], + has_submit_envelope_v2: bool, + ciphertext_length_bytes: u32, + idempotency_status: &'static str, + recipient_pending_count: u32, + recipient_pending_limit: u32, + application_envelopes_used: u32, + application_envelopes_limit: u32, + ciphertext_limit_bytes: u32, + enqueue_delivery_enabled: bool, +} + +fn facts_binding(case: &CaseFields) -> String { + let mut bytes = b"senline.submit-envelope.binding.v1\0".to_vec(); + append_u32(&mut bytes, 1); + append_string(&mut bytes, "submit-envelope"); + append_u32(&mut bytes, 1); + append_string(&mut bytes, &case.evaluation_id); + bytes.extend_from_slice(&case.operation_epoch.to_be_bytes()); + bytes.extend_from_slice(&case.worker_generation.to_be_bytes()); + append_string(&mut bytes, case.execution_mode); + append_string(&mut bytes, &case.worker_bundle_id); + append_u32(&mut bytes, 1); + append_u32(&mut bytes, 1); + for identifier in &case.identifiers { + append_string(&mut bytes, identifier); + } + append_string(&mut bytes, "active"); + let capabilities = if case.has_submit_envelope_v2 { + ["submit_envelope_v2"].as_slice() + } else { + [].as_slice() + }; + append_string_array(&mut bytes, capabilities); + append_u32(&mut bytes, 2); + append_u32(&mut bytes, case.ciphertext_length_bytes); + append_string(&mut bytes, case.idempotency_status); + append_u32(&mut bytes, case.recipient_pending_count); + append_u32(&mut bytes, case.recipient_pending_limit); + append_u32(&mut bytes, case.application_envelopes_used); + append_u32(&mut bytes, case.application_envelopes_limit); + append_u32(&mut bytes, case.ciphertext_limit_bytes); + let flags = if case.enqueue_delivery_enabled { + ["enqueue_delivery"].as_slice() + } else { + [].as_slice() + }; + append_string_array(&mut bytes, flags); + sha256_hex(bytes) +} + +/// Reviewed-boundary case generator (same contract as differential corpus). +fn reviewed_boundary_request(index: u64) -> Vec { + const MODES: [&str; 4] = ["fixture", "shadow", "guarded-development", "internal-alpha"]; + let variant = index / 6; + let reference_len = if index.is_multiple_of(2) { 1 } else { 128 }; + let (ciphertext_length_bytes, ciphertext_limit_bytes) = match variant % 4 { + 0 => (0, 0), + 1 => (1, 1), + 2 => (u32::MAX, u32::MAX), + _ => (1, u32::MAX), + }; + let relation = variant % 3; + let relation_values = match relation { + 0 => (0_u32, 1_u32), + 1 => (1, 1), + _ => (2, 1), + }; + let mut case = CaseFields { + evaluation_id: evaluation_id(index, index ^ 0xbb67_ae85_84ca_a73b), + operation_epoch: [0, 1, JSON_SAFE_INTEGER_MAX][(variant % 3) as usize], + worker_generation: [JSON_SAFE_INTEGER_MAX, 0, 1][(variant % 3) as usize], + execution_mode: MODES[(index % MODES.len() as u64) as usize], + worker_bundle_id: ascii_ref(8, index, reference_len), + identifiers: [ + ascii_ref(1, index, reference_len), + ascii_ref(2, index, reference_len), + ascii_ref(3, index, reference_len), + ascii_ref(4, index, reference_len), + ascii_ref(5, index, reference_len), + ascii_ref(6, index, reference_len), + ascii_ref(7, index, reference_len), + ], + has_submit_envelope_v2: variant.is_multiple_of(2), + ciphertext_length_bytes, + idempotency_status: "new", + recipient_pending_count: relation_values.0, + recipient_pending_limit: relation_values.1, + application_envelopes_used: relation_values.0, + application_envelopes_limit: relation_values.1, + ciphertext_limit_bytes, + enqueue_delivery_enabled: variant % 4 < 2, + }; + match index % 6 { + 0 => { + case.recipient_pending_count = 0; + case.recipient_pending_limit = 1; + case.application_envelopes_used = 0; + case.application_envelopes_limit = 1; + case.has_submit_envelope_v2 = true; + case.enqueue_delivery_enabled = true; + } + 1 => case.idempotency_status = "exact_duplicate", + 2 => case.idempotency_status = "conflict", + 3 => { + case.recipient_pending_count = if variant.is_multiple_of(2) { 1 } else { 2 }; + case.recipient_pending_limit = 1; + } + 4 => { + case.recipient_pending_count = 0; + case.recipient_pending_limit = 1; + case.application_envelopes_used = if variant.is_multiple_of(2) { 1 } else { 2 }; + case.application_envelopes_limit = 1; + } + _ => { + case.recipient_pending_count = 0; + case.recipient_pending_limit = 1; + case.application_envelopes_used = 0; + case.application_envelopes_limit = 1; + if variant.is_multiple_of(2) { + case.has_submit_envelope_v2 = false; + case.enqueue_delivery_enabled = true; + } else { + case.has_submit_envelope_v2 = true; + case.enqueue_delivery_enabled = false; + } + } + } + let capabilities: Vec<&str> = if case.has_submit_envelope_v2 { + vec!["submit_envelope_v2"] + } else { + Vec::new() + }; + let feature_flags: Vec<&str> = if case.enqueue_delivery_enabled { + vec!["enqueue_delivery"] + } else { + Vec::new() + }; + let request = serde_json::json!({ + "kind": "evaluation", + "schema_version": 1, + "context": { + "contract_version": 1, + "operation": "submit-envelope", + "operation_version": 1, + "evaluation_id": case.evaluation_id, + "operation_epoch": case.operation_epoch, + "worker_generation": case.worker_generation, + "execution_mode": case.execution_mode, + "worker_bundle_id": case.worker_bundle_id, + "facts_binding": facts_binding(&case), + }, + "facts": { + "contract_version": 1, + "operation_version": 1, + "identifiers": { + "correlation_ref": case.identifiers[0], + "source_account_ref": case.identifiers[1], + "source_device_ref": case.identifiers[2], + "recipient_account_ref": case.identifiers[3], + "recipient_device_ref": case.identifiers[4], + "conversation_ref": case.identifiers[5], + "envelope_ref": case.identifiers[6], + }, + "source_device_status": "active", + "source_device_capabilities": capabilities, + "envelope_protocol_version": 2, + "ciphertext_length_bytes": case.ciphertext_length_bytes, + "idempotency_status": case.idempotency_status, + "recipient_pending_count": case.recipient_pending_count, + "recipient_pending_limit": case.recipient_pending_limit, + "application_envelopes_used": case.application_envelopes_used, + "application_envelopes_limit": case.application_envelopes_limit, + "ciphertext_limit_bytes": case.ciphertext_limit_bytes, + "feature_flags": feature_flags, + } + }); + let bytes = serde_json::to_vec(&request).expect("serialize resource request"); + assert!(bytes.len() <= INPUT_MAX_BYTES, "resource request oversized"); + bytes +} + +/// Same reviewed-boundary shape with an explicit operation_version (both context +/// and facts). Used to exercise the unsupported-version error path that previously +/// leaked owned request Strings under path-insensitive move tracking. +fn reviewed_boundary_request_with_operation_version(index: u64, operation_version: u32) -> Vec { + let mut request: serde_json::Value = serde_json::from_slice(&reviewed_boundary_request(index)) + .expect("reparse boundary request"); + request["context"]["operation_version"] = serde_json::json!(operation_version); + request["facts"]["operation_version"] = serde_json::json!(operation_version); + let bytes = serde_json::to_vec(&request).expect("serialize unsupported-version request"); + assert!( + bytes.len() <= INPUT_MAX_BYTES, + "unsupported-version request oversized" + ); + bytes +} + +/// Independent Rust oracle for reviewed-boundary cases (mirrors differential). +fn boundary_case_fields(index: u64) -> CaseFields { + const MODES: [&str; 4] = ["fixture", "shadow", "guarded-development", "internal-alpha"]; + let variant = index / 6; + let reference_len = if index.is_multiple_of(2) { 1 } else { 128 }; + let (ciphertext_length_bytes, ciphertext_limit_bytes) = match variant % 4 { + 0 => (0, 0), + 1 => (1, 1), + 2 => (u32::MAX, u32::MAX), + _ => (1, u32::MAX), + }; + let relation = variant % 3; + let relation_values = match relation { + 0 => (0_u32, 1_u32), + 1 => (1, 1), + _ => (2, 1), + }; + let mut case = CaseFields { + evaluation_id: evaluation_id(index, index ^ 0xbb67_ae85_84ca_a73b), + operation_epoch: [0, 1, JSON_SAFE_INTEGER_MAX][(variant % 3) as usize], + worker_generation: [JSON_SAFE_INTEGER_MAX, 0, 1][(variant % 3) as usize], + execution_mode: MODES[(index % MODES.len() as u64) as usize], + worker_bundle_id: ascii_ref(8, index, reference_len), + identifiers: [ + ascii_ref(1, index, reference_len), + ascii_ref(2, index, reference_len), + ascii_ref(3, index, reference_len), + ascii_ref(4, index, reference_len), + ascii_ref(5, index, reference_len), + ascii_ref(6, index, reference_len), + ascii_ref(7, index, reference_len), + ], + has_submit_envelope_v2: variant.is_multiple_of(2), + ciphertext_length_bytes, + idempotency_status: "new", + recipient_pending_count: relation_values.0, + recipient_pending_limit: relation_values.1, + application_envelopes_used: relation_values.0, + application_envelopes_limit: relation_values.1, + ciphertext_limit_bytes, + enqueue_delivery_enabled: variant % 4 < 2, + }; + match index % 6 { + 0 => { + case.recipient_pending_count = 0; + case.recipient_pending_limit = 1; + case.application_envelopes_used = 0; + case.application_envelopes_limit = 1; + case.has_submit_envelope_v2 = true; + case.enqueue_delivery_enabled = true; + } + 1 => case.idempotency_status = "exact_duplicate", + 2 => case.idempotency_status = "conflict", + 3 => { + case.recipient_pending_count = if variant.is_multiple_of(2) { 1 } else { 2 }; + case.recipient_pending_limit = 1; + } + 4 => { + case.recipient_pending_count = 0; + case.recipient_pending_limit = 1; + case.application_envelopes_used = if variant.is_multiple_of(2) { 1 } else { 2 }; + case.application_envelopes_limit = 1; + } + _ => { + case.recipient_pending_count = 0; + case.recipient_pending_limit = 1; + case.application_envelopes_used = 0; + case.application_envelopes_limit = 1; + if variant.is_multiple_of(2) { + case.has_submit_envelope_v2 = false; + case.enqueue_delivery_enabled = true; + } else { + case.has_submit_envelope_v2 = true; + case.enqueue_delivery_enabled = false; + } + } + } + case +} + +fn oracle_decision_reason(case: &CaseFields) -> (&'static str, &'static str) { + if case.idempotency_status == "exact_duplicate" { + return ("duplicate_noop", "exact_duplicate"); + } + if case.idempotency_status == "conflict" { + return ("reject", "idempotency_conflict"); + } + if case.recipient_pending_count >= case.recipient_pending_limit { + return ("reject", "recipient_queue_full"); + } + if case.application_envelopes_used >= case.application_envelopes_limit { + return ("reject", "application_budget_exhausted"); + } + if !case.has_submit_envelope_v2 || !case.enqueue_delivery_enabled { + return ("reject", "delivery_disabled"); + } + ("store_and_enqueue", "accepted_new") +} + +#[derive(Clone, Copy)] +enum ResponseExpectation { + /// kind=plan with decision/reason from the independent Rust oracle. + ReviewedBoundaryPlan, + /// kind=error with a fixed protocol/evaluation code. + ProtocolError { code: &'static str }, +} + +fn classify_response( + index: u64, + response: &[u8], + expectation: ResponseExpectation, +) -> Result { + let value: Value = serde_json::from_slice(response) + .map_err(|error| format!("case {index} malformed JSON response: {error}"))?; + // Empty objects / non-contract shapes must not count as success. + if !value.is_object() || value.as_object().is_some_and(|o| o.is_empty()) { + return Err(format!( + "case {index} response is empty or non-object JSON (not a plan/error envelope)" + )); + } + let kind = value + .get("kind") + .and_then(Value::as_str) + .ok_or_else(|| format!("case {index} response missing kind"))?; + match expectation { + ResponseExpectation::ReviewedBoundaryPlan => { + if kind != "plan" { + return Err(format!( + "case {index} expected kind=plan, got kind={kind:?} body={}", + String::from_utf8_lossy(response) + )); + } + let case = boundary_case_fields(index); + let (want_decision, want_reason) = oracle_decision_reason(&case); + let decision = value.get("decision").and_then(Value::as_str).unwrap_or(""); + let reason = value.get("reason").and_then(Value::as_str).unwrap_or(""); + if decision != want_decision || reason != want_reason { + return Err(format!( + "case {index} oracle mismatch: got decision={decision:?} reason={reason:?}, want decision={want_decision:?} reason={want_reason:?}" + )); + } + let eval = value + .pointer("/context/evaluation_id") + .and_then(Value::as_str) + .unwrap_or(""); + if eval != case.evaluation_id { + return Err(format!( + "case {index} evaluation_id mismatch: got {eval:?} want {:?}", + case.evaluation_id + )); + } + let rev = value + .get("sengoo_module_revision") + .and_then(Value::as_str) + .unwrap_or(""); + if rev != PLANNER_FIXTURE_REVISION { + return Err(format!( + "case {index} sengoo_module_revision mismatch: got {rev:?} want {PLANNER_FIXTURE_REVISION}" + )); + } + if decision == "store_and_enqueue" || decision == "duplicate_noop" { + Ok(ResponseClass::PlanAccept) + } else { + Ok(ResponseClass::PlanReject) + } + } + ResponseExpectation::ProtocolError { code } => { + if kind != "error" { + return Err(format!( + "case {index} expected kind=error code={code}, got kind={kind:?}" + )); + } + let got = value.get("code").and_then(Value::as_str).unwrap_or(""); + if got != code { + return Err(format!( + "case {index} expected error code={code}, got {got:?}" + )); + } + Ok(ResponseClass::ProtocolError) + } + } +} + +#[derive(Clone, Copy)] +enum ResponseClass { + PlanAccept, + PlanReject, + ProtocolError, +} + +fn run_resource_corpus_with_requests( + executable: &Path, + count: u64, + timeout: Duration, + mut request_for_index: F, + expectation: ResponseExpectation, +) -> ResourceOutcome +where + F: FnMut(u64) -> Vec, +{ + let per_request_timeout = Duration::from_secs(5); + let mut child = Command::new(executable) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn resource worker"); + let pid = child.id(); + let mut stdin = child.stdin.take().expect("worker stdin"); + let stdout = child.stdout.take().expect("worker stdout"); + let mut stderr = child.stderr.take().expect("worker stderr"); + let stderr_reader = std::thread::spawn(move || { + let mut bytes = Vec::new(); + let _ = stderr.read_to_end(&mut bytes); + bytes + }); + let (response_tx, response_rx) = std::sync::mpsc::sync_channel::, String>>(0); + let reader = std::thread::spawn(move || { + let mut stdout = stdout; + loop { + match read_frame(&mut stdout, OUTPUT_MAX_BYTES) { + Ok(frame) => { + if response_tx.send(Ok(frame)).is_err() { + break; + } + } + Err(error) => { + let _ = response_tx.send(Err(error)); + break; + } + } + } + }); + + let fixture_handshake = normalize_fixture_bytes( + fs::read(fixture_root().join("handshake/ready.json")).expect("read frozen handshake"), + ); + let mut failures = Vec::new(); + match response_rx.recv_timeout(per_request_timeout) { + Ok(Ok(handshake)) if normalize_fixture_bytes(&handshake) == fixture_handshake => {} + Ok(Ok(_)) => failures.push("worker handshake differs from frozen fixture".to_owned()), + Ok(Err(error)) => failures.push(format!("worker handshake failed: {error}")), + Err(_) => failures.push("worker handshake timed out".to_owned()), + } + + let started = Instant::now(); + let mut samples = Vec::new(); + let mut latency_us = Vec::with_capacity(count.saturating_sub(WARMUP_CASES) as usize); + let mut plan_ok = 0_u64; + let mut plan_reject_or_error = 0_u64; + let mut window_start = Instant::now(); + let mut window_cases = 0_u64; + let mut completed = 0_u64; + let mut worker_exited_early = false; + let mut process_count_samples: Vec = Vec::new(); + if let Some(n) = sample_worker_process_tree_count(pid) { + process_count_samples.push(n); + } + + if failures.is_empty() { + for index in 0..count { + // Fail closed if the single worker child disappeared mid-soak. + match child.try_wait() { + Ok(Some(status)) => { + failures.push(format!( + "worker process exited early after case {completed}/{count}: {status}" + )); + worker_exited_early = true; + break; + } + Ok(None) => {} + Err(error) => { + failures.push(format!("worker process poll failed: {error}")); + break; + } + } + // Sample process tree periodically (and on the last case). + if index == 0 || (index + 1) % SAMPLE_EVERY == 0 || index + 1 == count { + if let Some(n) = sample_worker_process_tree_count(pid) { + process_count_samples.push(n); + if n > 1 { + failures.push(format!( + "case {}: worker process tree count {n} exceeds single-worker bound (worker + unexpected children)", + index + 1 + )); + break; + } + } else if !worker_exited_early { + failures.push(format!( + "case {}: failed to sample worker process tree count for pid {pid}", + index + 1 + )); + break; + } + } + if started.elapsed() > timeout { + failures.push(format!( + "watchdog exceeded after case {index}/{} ({:?})", + count, + started.elapsed() + )); + break; + } + let request = request_for_index(index); + let req_started = Instant::now(); + if let Err(error) = write_frame(&mut stdin, &request) { + failures.push(format!("case {index} write failed: {error}")); + break; + } + let response = match response_rx.recv_timeout(per_request_timeout) { + Ok(Ok(response)) => response, + Ok(Err(error)) => { + failures.push(format!("case {index} read failed: {error}")); + break; + } + Err(_) => { + failures.push(format!( + "case {index} response timed out after {per_request_timeout:?} (possible single-worker spin/hang)" + )); + break; + } + }; + let latency = req_started.elapsed(); + match classify_response(index, &response, expectation) { + Ok(ResponseClass::PlanAccept) => plan_ok += 1, + Ok(ResponseClass::PlanReject) | Ok(ResponseClass::ProtocolError) => { + plan_reject_or_error += 1 + } + Err(error) => { + failures.push(error); + break; + } + } + completed = index + 1; + if index >= WARMUP_CASES { + latency_us.push(latency.as_micros() as u64); + } + + window_cases += 1; + if index == 0 || (index + 1) % SAMPLE_EVERY == 0 || index + 1 == count { + let window_secs = window_start.elapsed().as_secs_f64().max(1e-9); + let cps = window_cases as f64 / window_secs; + let mem = sample_worker_memory_bytes(pid); + let handles = sample_worker_handle_count(pid); + if mem.is_none() || handles.is_none() { + failures.push(format!( + "case {} resource sample missing mem={mem:?} handles={handles:?} (cannot green-gate without metrics)", + index + 1 + )); + break; + } + samples.push(SamplePoint { + case_index: index + 1, + elapsed_ms: started.elapsed().as_millis() as u64, + memory_bytes: mem, + handle_count: handles, + cases_per_second_window: cps, + }); + if (index + 1) % 1_000 == 0 { + println!( + "senline-resource progress case={} elapsed_ms={} cps_window={:.1} mem={:?} handles={:?}", + index + 1, + started.elapsed().as_millis(), + cps, + mem, + handles + ); + } + window_start = Instant::now(); + window_cases = 0; + } + } + } + + // Measured max of (worker + children) across samples — never hardcode 1. + let process_count = process_count_samples.iter().copied().max().unwrap_or(0); + let process_count_ok = !worker_exited_early + && completed == count + && process_count == 1 + && !process_count_samples.is_empty(); + + // Kill the worker *before* joining stdout/stderr readers. Joining first + // deadlocks the soak watchdog when the worker hangs with pipes still open. + drop(stdin); + drop(response_rx); + let _ = finish_child(&mut child, Duration::from_secs(2)); + // After kill/exit, OS closes pipes so readers observe EOF and return. + // Bound the joins so a stuck pipe still cannot hang the harness forever. + let join_deadline = Instant::now() + Duration::from_secs(5); + loop { + if reader.is_finished() && stderr_reader.is_finished() { + break; + } + if Instant::now() >= join_deadline { + break; + } + std::thread::sleep(Duration::from_millis(20)); + } + if reader.is_finished() { + let _ = reader.join(); + } // else: drop detaches; prefer hang-free soak over perfect cleanup + if stderr_reader.is_finished() { + let _ = stderr_reader.join(); + } + + ResourceOutcome { + cases_requested: count, + cases_completed: completed, + warm_up: WARMUP_CASES, + elapsed: started.elapsed(), + samples, + latency_us, + plan_ok, + plan_reject_or_error, + process_count, + process_count_ok, + failures, + } +} + +fn percentile_us(sorted_us: &[u64], pct: f64) -> u64 { + if sorted_us.is_empty() { + return 0; + } + let rank = ((pct / 100.0) * (sorted_us.len() as f64 - 1.0)).round() as usize; + sorted_us[rank.min(sorted_us.len() - 1)] +} + +#[cfg(target_os = "linux")] +fn sample_worker_memory_bytes(pid: u32) -> Option { + let status = fs::read_to_string(format!("/proc/{pid}/status")).ok()?; + for line in status.lines() { + if let Some(rest) = line.strip_prefix("VmRSS:") { + let kb: u64 = rest.split_whitespace().next()?.parse().ok()?; + return Some(kb.saturating_mul(1024)); + } + } + None +} + +#[cfg(target_os = "linux")] +fn sample_worker_handle_count(pid: u32) -> Option { + let dir = fs::read_dir(format!("/proc/{pid}/fd")).ok()?; + Some(dir.count() as u64) +} + +#[cfg(windows)] +mod win_sample { + // Win32 FFI type spellings match the SDK headers (HANDLE/BOOL/DWORD). + #![allow(clippy::upper_case_acronyms)] + use std::mem::{size_of, MaybeUninit}; + use std::os::raw::c_void; + + type HANDLE = *mut c_void; + type BOOL = i32; + type DWORD = u32; + type SizeT = usize; + + const PROCESS_QUERY_INFORMATION: DWORD = 0x0400; + const PROCESS_VM_READ: DWORD = 0x0010; + + #[repr(C)] + struct ProcessMemoryCountersEx { + cb: DWORD, + page_fault_count: DWORD, + peak_working_set_size: SizeT, + working_set_size: SizeT, + quota_peak_paged_pool_usage: SizeT, + quota_paged_pool_usage: SizeT, + quota_peak_non_paged_pool_usage: SizeT, + quota_non_paged_pool_usage: SizeT, + pagefile_usage: SizeT, + peak_pagefile_usage: SizeT, + private_usage: SizeT, + } + + #[link(name = "kernel32")] + extern "system" { + fn OpenProcess(access: DWORD, inherit: BOOL, process_id: DWORD) -> HANDLE; + fn CloseHandle(handle: HANDLE) -> BOOL; + fn GetProcessHandleCount(process: HANDLE, handle_count: *mut DWORD) -> BOOL; + } + + #[link(name = "psapi")] + extern "system" { + fn GetProcessMemoryInfo( + process: HANDLE, + counters: *mut ProcessMemoryCountersEx, + cb: DWORD, + ) -> BOOL; + } + + /// Private bytes (PrivateUsage). Prefer this for long-session growth. + pub(super) fn private_bytes(pid: u32) -> Option { + unsafe { + let handle = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, 0, pid); + if handle.is_null() { + return None; + } + let mut counters = MaybeUninit::::zeroed(); + let counters_ptr = counters.as_mut_ptr(); + (*counters_ptr).cb = size_of::() as DWORD; + let ok = GetProcessMemoryInfo( + handle, + counters_ptr, + size_of::() as DWORD, + ); + CloseHandle(handle); + if ok == 0 { + return None; + } + Some(counters.assume_init().private_usage as u64) + } + } + + pub(super) fn process_tree_count(root_pid: u32) -> Option { + // Toolhelp snapshot of all processes; count root + children with PPID==root. + #[repr(C)] + struct ProcessEntry32W { + dw_size: DWORD, + cnt_usage: DWORD, + th32_process_id: DWORD, + th32_default_heap_id: usize, + th32_module_id: DWORD, + cnt_threads: DWORD, + th32_parent_process_id: DWORD, + pc_pri_class_base: i32, + dw_flags: DWORD, + sz_exe_file: [u16; 260], + } + const TH32CS_SNAPPROCESS: DWORD = 0x0000_0002; + #[link(name = "kernel32")] + extern "system" { + fn CreateToolhelp32Snapshot(flags: DWORD, process_id: DWORD) -> HANDLE; + fn Process32FirstW(snapshot: HANDLE, entry: *mut ProcessEntry32W) -> BOOL; + fn Process32NextW(snapshot: HANDLE, entry: *mut ProcessEntry32W) -> BOOL; + } + unsafe { + let snap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); + if snap.is_null() || snap == (-1isize as HANDLE) { + return None; + } + let mut entry = std::mem::zeroed::(); + entry.dw_size = size_of::() as DWORD; + let mut count = 0u32; + let mut root_seen = false; + if Process32FirstW(snap, &mut entry) != 0 { + loop { + if entry.th32_process_id == root_pid { + root_seen = true; + count = count.saturating_add(1); + } else if entry.th32_parent_process_id == root_pid { + count = count.saturating_add(1); + } + if Process32NextW(snap, &mut entry) == 0 { + break; + } + } + } + CloseHandle(snap); + if !root_seen { + return None; + } + Some(count.max(1)) + } + } + + pub(super) fn handle_count(pid: u32) -> Option { + unsafe { + let handle = OpenProcess(PROCESS_QUERY_INFORMATION, 0, pid); + if handle.is_null() { + return None; + } + let mut count: DWORD = 0; + let ok = GetProcessHandleCount(handle, &mut count); + CloseHandle(handle); + if ok == 0 { + return None; + } + Some(u64::from(count)) + } + } +} + +#[cfg(windows)] +fn sample_worker_memory_bytes(pid: u32) -> Option { + // PrivateUsage (commit private bytes). Documented metric name is + // `private_bytes` — not WorkingSetSize/Private Working Set. + win_sample::private_bytes(pid) +} + +#[cfg(windows)] +fn sample_worker_handle_count(pid: u32) -> Option { + win_sample::handle_count(pid) +} + +/// Count the worker process plus any live child processes (PPID == worker). +/// Used so process_count is measured, not hardcoded. +#[cfg(windows)] +fn sample_worker_process_tree_count(pid: u32) -> Option { + win_sample::process_tree_count(pid) +} + +#[cfg(target_os = "linux")] +fn sample_worker_process_tree_count(pid: u32) -> Option { + // Confirm the worker pid is still alive. + fs::read_to_string(format!("/proc/{pid}/status")).ok()?; + let mut count = 1u32; + let Ok(entries) = fs::read_dir("/proc") else { + return Some(count); + }; + for entry in entries.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if !name.chars().all(|c| c.is_ascii_digit()) { + continue; + } + let child_pid: u32 = match name.parse() { + Ok(v) if v != pid => v, + _ => continue, + }; + let Ok(stat) = fs::read_to_string(format!("/proc/{child_pid}/stat")) else { + continue; + }; + // /proc/pid/stat: pid (comm) state ppid ... — comm may contain ')'. + let Some(rparen) = stat.rfind(')') else { + continue; + }; + let after = stat[rparen + 1..].trim_start(); + let mut parts = after.split_whitespace(); + let _state = parts.next(); + if let Some(ppid) = parts.next().and_then(|s| s.parse::().ok()) { + if ppid == pid { + count = count.saturating_add(1); + } + } + } + Some(count) +} + +#[cfg(not(any(windows, target_os = "linux")))] +fn sample_worker_process_tree_count(_pid: u32) -> Option { + None +} + +#[cfg(not(any(windows, target_os = "linux")))] +fn sample_worker_memory_bytes(_pid: u32) -> Option { + None +} + +#[cfg(not(any(windows, target_os = "linux")))] +fn sample_worker_handle_count(_pid: u32) -> Option { + None +} + +#[derive(Clone)] +struct SamplePoint { + case_index: u64, + elapsed_ms: u64, + memory_bytes: Option, + handle_count: Option, + cases_per_second_window: f64, +} + +struct ResourceOutcome { + cases_requested: u64, + cases_completed: u64, + warm_up: u64, + elapsed: Duration, + samples: Vec, + latency_us: Vec, + plan_ok: u64, + plan_reject_or_error: u64, + process_count: u32, + process_count_ok: bool, + failures: Vec, +} + +fn run_resource_corpus(executable: &Path, count: u64, timeout: Duration) -> ResourceOutcome { + run_resource_corpus_with_requests( + executable, + count, + timeout, + reviewed_boundary_request, + ResponseExpectation::ReviewedBoundaryPlan, + ) +} + +fn finish_child(child: &mut Child, grace: Duration) -> std::io::Result { + let started = Instant::now(); + loop { + match child.try_wait()? { + Some(status) => return Ok(status), + None if started.elapsed() > grace => { + let _ = child.kill(); + return child.wait(); + } + None => std::thread::sleep(Duration::from_millis(20)), + } + } +} + +fn post_warmup_memory_samples(samples: &[SamplePoint], warm_up: u64) -> Vec<&SamplePoint> { + samples + .iter() + .filter(|s| s.case_index > warm_up && s.memory_bytes.is_some()) + .collect() +} + +/// Endpoint slope: (last - first) / cases. Kept for comparison with prior digests. +fn memory_growth_bytes_per_case(samples: &[SamplePoint], warm_up: u64) -> Option { + let post = post_warmup_memory_samples(samples, warm_up); + if post.len() < 2 { + return None; + } + let first = post.first().unwrap(); + let last = post.last().unwrap(); + let cases = last.case_index.saturating_sub(first.case_index) as f64; + if cases <= 0.0 { + return None; + } + let delta = last.memory_bytes.unwrap() as i64 - first.memory_bytes.unwrap() as i64; + Some(delta as f64 / cases) +} + +/// Ordinary least-squares slope of memory_bytes vs case_index after warm-up. +fn memory_regression_slope_bytes_per_case(samples: &[SamplePoint], warm_up: u64) -> Option { + let post = post_warmup_memory_samples(samples, warm_up); + if post.len() < 3 { + return None; + } + let n = post.len() as f64; + let mut sum_x = 0.0; + let mut sum_y = 0.0; + let mut sum_xx = 0.0; + let mut sum_xy = 0.0; + for s in &post { + let x = s.case_index as f64; + let y = s.memory_bytes.unwrap() as f64; + sum_x += x; + sum_y += y; + sum_xx += x * x; + sum_xy += x * y; + } + let denom = n * sum_xx - sum_x * sum_x; + if denom.abs() < f64::EPSILON { + return None; + } + Some((n * sum_xy - sum_x * sum_y) / denom) +} + +/// Max memory increase over any contiguous ~10k-case sample window after warm-up. +fn max_memory_window_delta_bytes( + samples: &[SamplePoint], + warm_up: u64, + window_cases: u64, +) -> Option { + let post = post_warmup_memory_samples(samples, warm_up); + if post.len() < 2 { + return None; + } + let mut max_delta: i64 = i64::MIN; + for (i, start) in post.iter().enumerate() { + let start_cases = start.case_index; + let start_mem = start.memory_bytes.unwrap() as i64; + for end in post.iter().skip(i + 1) { + let span = end.case_index.saturating_sub(start_cases); + if span >= window_cases { + let delta = end.memory_bytes.unwrap() as i64 - start_mem; + max_delta = max_delta.max(delta); + break; + } + } + } + if max_delta == i64::MIN { + // Fall back to full-window delta when the series is shorter than window_cases. + let first = post.first().unwrap().memory_bytes.unwrap() as i64; + let last = post.last().unwrap().memory_bytes.unwrap() as i64; + return Some(last - first); + } + Some(max_delta) +} + +fn handle_plateau_ok(samples: &[SamplePoint], warm_up: u64, slack: u64) -> Option { + let post: Vec<_> = samples + .iter() + .filter(|s| s.case_index > warm_up && s.handle_count.is_some()) + .collect(); + if post.is_empty() { + return None; + } + let warm_max = samples + .iter() + .filter(|s| s.case_index <= warm_up.max(1) && s.handle_count.is_some()) + .map(|s| s.handle_count.unwrap()) + .max() + .unwrap_or_else(|| post[0].handle_count.unwrap()); + let post_max = post.iter().map(|s| s.handle_count.unwrap()).max().unwrap(); + Some(post_max <= warm_max.saturating_add(slack)) +} + +fn write_evidence(label: &str, outcome: &ResourceOutcome) -> PathBuf { + let mut latency = outcome.latency_us.clone(); + latency.sort_unstable(); + let p50 = percentile_us(&latency, 50.0); + let p95 = percentile_us(&latency, 95.0); + let p99 = percentile_us(&latency, 99.0); + let mean = if latency.is_empty() { + 0.0 + } else { + latency.iter().sum::() as f64 / latency.len() as f64 + }; + let growth = memory_growth_bytes_per_case(&outcome.samples, outcome.warm_up); + let regression = memory_regression_slope_bytes_per_case(&outcome.samples, outcome.warm_up); + let window_delta = max_memory_window_delta_bytes(&outcome.samples, outcome.warm_up, 10_000); + let handles_ok = handle_plateau_ok(&outcome.samples, outcome.warm_up, 16); + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_secs(); + let stem = format!( + "soak-{}-{}-{}-{stamp}", + label, + std::env::consts::OS, + std::env::consts::ARCH + ); + let path = evidence_root().join(format!("{stem}.summary.json")); + let jsonl_path = evidence_root().join(format!("{stem}.jsonl")); + // Full sample series for offline regression / 10k-window review (gitignored). + { + let mut jsonl = String::new(); + for s in &outcome.samples { + let line = serde_json::json!({ + "case_index": s.case_index, + "elapsed_ms": s.elapsed_ms, + "memory_bytes": s.memory_bytes, + "handle_count": s.handle_count, + "cases_per_second_window": s.cases_per_second_window + }); + jsonl.push_str(&line.to_string()); + jsonl.push('\n'); + } + fs::write(&jsonl_path, jsonl).expect("write resource JSONL series"); + } + let metric = if cfg!(windows) { + "private_bytes" // PROCESS_MEMORY_COUNTERS_EX.PrivateUsage + } else { + "rss_bytes" + }; + let all_samples_present = outcome + .samples + .iter() + .all(|s| s.memory_bytes.is_some() && s.handle_count.is_some()); + let response_accounted = outcome.plan_ok + outcome.plan_reject_or_error; + let summary = serde_json::json!({ + "schema_version": 2, + "label": label, + "platform": std::env::consts::OS, + "architecture": std::env::consts::ARCH, + "fixed_seed_hex": format!("0x{FIXED_SEED:016x}"), + "planner_contract_fixture_revision": PLANNER_FIXTURE_REVISION, + "cases_requested": outcome.cases_requested, + "cases_completed": outcome.cases_completed, + "warm_up_cases": outcome.warm_up, + "elapsed_ms": outcome.elapsed.as_millis() as u64, + "plan_ok": outcome.plan_ok, + "plan_reject_or_error": outcome.plan_reject_or_error, + "failure_count": outcome.failures.len(), + "failures": outcome.failures, + "process_count": outcome.process_count, + "jsonl_series": jsonl_path.file_name().and_then(|s| s.to_str()), + "oracle": { + "kind": "independent_rust_boundary_or_protocol_error", + "notes": "Empty JSON / non-plan envelopes fail; reviewed-boundary cases require decision/reason match" + }, + "latency_post_warmup": { + "sample_count": latency.len(), + "mean_us": mean, + "p50_us": p50, + "p95_us": p95, + "p99_us": p99, + "notes": "request-write-complete to response-frame-complete wall time; not Senline admission/sandbox timing" + }, + "memory": { + "metric": metric, + "metric_notes": if cfg!(windows) { + "Windows PrivateUsage (private bytes), not WorkingSetSize" + } else { + "Linux VmRSS" + }, + "post_warmup_endpoint_growth_bytes_per_case": growth, + "post_warmup_regression_slope_bytes_per_case": regression, + "max_10k_window_delta_bytes": window_delta, + "sample_count": outcome.samples.len(), + "all_samples_present": all_samples_present, + "samples_tail": outcome.samples.iter().rev().take(5).map(|s| serde_json::json!({ + "case_index": s.case_index, + "elapsed_ms": s.elapsed_ms, + "memory_bytes": s.memory_bytes, + "handle_count": s.handle_count, + "cases_per_second_window": s.cases_per_second_window + })).collect::>(), + }, + "handles": { + "plateau_slack": 16, + "within_plateau": handles_ok, + }, + "gates": { + "default_growth_bound_bytes_per_case": 1024.0, + "endpoint_growth_within_default_bound": growth.map(|g| g < 1024.0), + "regression_slope_within_default_bound": regression.map(|g| g < 1024.0), + "max_10k_window_delta_bytes_bound": 32 * 1024 * 1024, + "max_10k_window_within_bound": window_delta.map(|d| d < 32 * 1024 * 1024), + "handles_within_plateau": handles_ok, + "process_count_is_one": outcome.process_count == 1 && outcome.process_count_ok, + "all_samples_present": all_samples_present, + "completed_all_requested": outcome.cases_completed == outcome.cases_requested, + "response_count_matches_completed": response_accounted == outcome.cases_completed, + "zero_failures": outcome.failures.is_empty(), + // Keep legacy key for older consumers. + "growth_within_default_bound": growth.map(|g| g < 1024.0), + "post_warmup_growth_bytes_per_case": growth, + } + }); + fs::write( + &path, + serde_json::to_vec_pretty(&summary).expect("serialize resource summary"), + ) + .expect("write resource summary"); + path +} + +fn assert_resource_outcome(outcome: &ResourceOutcome, label: &str) { + let path = write_evidence(label, outcome); + println!( + "senline-resource label={label} cases_requested={} cases_completed={} elapsed_ms={} failures={} evidence={}", + outcome.cases_requested, + outcome.cases_completed, + outcome.elapsed.as_millis(), + outcome.failures.len(), + path.display() + ); + if !outcome.latency_us.is_empty() { + let mut latency = outcome.latency_us.clone(); + latency.sort_unstable(); + println!( + "senline-latency p50_us={} p95_us={} p99_us={} samples={}", + percentile_us(&latency, 50.0), + percentile_us(&latency, 95.0), + percentile_us(&latency, 99.0), + latency.len() + ); + } + assert!( + outcome.failures.is_empty(), + "{label} resource run failures: {:?}", + outcome.failures + ); + assert_eq!( + outcome.cases_completed, outcome.cases_requested, + "{label} did not complete all requested cases" + ); + assert_eq!( + outcome.plan_ok + outcome.plan_reject_or_error, + outcome.cases_completed, + "{label} response accounting: plan_ok + reject/error must equal cases_completed" + ); + assert!( + outcome.process_count == 1 && outcome.process_count_ok, + "{label} process_count gate failed: count={} ok={}", + outcome.process_count, + outcome.process_count_ok + ); + assert!( + !outcome.samples.is_empty(), + "{label} resource sampler produced no memory/throughput samples" + ); + assert!( + outcome + .samples + .iter() + .all(|s| s.memory_bytes.is_some() && s.handle_count.is_some()), + "{label} one or more samples missing memory/handle metrics; cannot skip gates" + ); + let slope = memory_regression_slope_bytes_per_case(&outcome.samples, outcome.warm_up) + .unwrap_or_else(|| { + panic!("{label} missing OLS regression slope (need >=3 post-warm-up samples)") + }); + assert!( + slope < 1024.0, + "{label} post-warm-up regression slope {slope} B/case exceeds 1 KiB/case bound" + ); + let delta = max_memory_window_delta_bytes(&outcome.samples, outcome.warm_up, 10_000) + .unwrap_or_else(|| panic!("{label} missing 10k-window memory delta")); + assert!( + delta < 32 * 1024 * 1024, + "{label} max ~10k-case memory window delta {delta} bytes exceeds +32 MiB" + ); + let handles_ok = handle_plateau_ok(&outcome.samples, outcome.warm_up, 16) + .unwrap_or_else(|| panic!("{label} missing handle plateau samples")); + assert!( + handles_ok, + "{label} handle/FD count climbed past warm-up max + 16 plateau" + ); +} + +#[test] +fn resource_sampler_smoke_single_worker_with_latency_percentiles() { + let root = WorkerTempDir::new("smoke"); + let executable = build_worker(&root); + let outcome = run_resource_corpus(&executable, SMOKE_COUNT, Duration::from_secs(180)); + assert_resource_outcome(&outcome, "smoke-1k"); + assert!( + outcome.latency_us.len() as u64 >= SMOKE_COUNT.saturating_sub(WARMUP_CASES), + "post-warm-up latency samples missing" + ); + assert!( + outcome.plan_ok + outcome.plan_reject_or_error == SMOKE_COUNT, + "smoke must oracle-match every response" + ); +} + +/// Regression: operation_version=99 previously leaked owned request Strings +/// because the unsupported branch borrowed evaluation_id while the accept +/// branch moved the whole request (path-insensitive moved set). +#[test] +fn resource_unsupported_operation_version_path_does_not_grow_memory() { + let root = WorkerTempDir::new("unsupported-opver"); + let executable = build_worker(&root); + const COUNT: u64 = 2_048; + let outcome = run_resource_corpus_with_requests( + executable.as_path(), + COUNT, + Duration::from_secs(180), + |index| reviewed_boundary_request_with_operation_version(index, 99), + ResponseExpectation::ProtocolError { + code: "unsupported_operation_version", + }, + ); + assert_resource_outcome(&outcome, "unsupported-opver-2k"); + assert_eq!( + outcome.plan_reject_or_error, COUNT, + "every case must take the unsupported-operation-version error path" + ); + assert_eq!(outcome.plan_ok, 0, "unsupported path must not emit plan"); + let growth = memory_growth_bytes_per_case(&outcome.samples, WARMUP_CASES) + .expect("post-warm-up memory samples"); + println!("senline-resource unsupported-opver growth_bytes_per_case={growth}"); + assert!( + growth < 1024.0, + "unsupported-version path growth {growth} B/case exceeds 1 KiB/case bound" + ); +} + +#[test] +#[ignore = "single-worker investigation covering historical case ~44086; run with --ignored"] +fn resource_single_worker_investigation_50k() { + let root = WorkerTempDir::new("investigate-45k"); + let executable = build_worker(&root); + // Historical pre-fix observation stalled near case 44086 / multi-minute + // growth. After lambda String Drop glue, this window must complete cleanly. + let outcome = run_resource_corpus(&executable, INVESTIGATION_COUNT, Duration::from_secs(900)); + assert_resource_outcome(&outcome, "investigate-45k"); + let growth = memory_growth_bytes_per_case(&outcome.samples, WARMUP_CASES) + .expect("post-warm-up memory samples"); + println!("senline-resource post-warmup growth_bytes_per_case={growth}"); + assert!( + growth < 1024.0, + "post-warm-up private-working-set growth {growth} B/case exceeds 1 KiB/case bound" + ); + assert_eq!( + outcome.cases_completed, INVESTIGATION_COUNT, + "investigation must complete every reviewed-boundary case" + ); + assert_eq!( + outcome.plan_ok + outcome.plan_reject_or_error, + INVESTIGATION_COUNT, + "investigation must oracle-match every response" + ); +} + +#[test] +#[ignore = "task 8.3 full 1M single-worker soak; run with --ignored on a reference host"] +fn resource_single_worker_soak_1m() { + let root = WorkerTempDir::new("soak-1m"); + let executable = build_worker(&root); + let outcome = run_resource_corpus(&executable, SOAK_COUNT, Duration::from_secs(6 * 3600)); + assert_resource_outcome(&outcome, "soak-1m"); + let growth = memory_growth_bytes_per_case(&outcome.samples, WARMUP_CASES) + .expect("need post-warm-up memory samples for soak gate"); + assert!( + growth < 1024.0, + "post-warm-up memory growth {growth} B/case exceeds 1 KiB/case default bound" + ); +} diff --git a/tools/sgc/tests/test_discovery.rs b/tools/sgc/tests/test_discovery.rs index 664d36d1..fcd722f8 100644 --- a/tools/sgc/tests/test_discovery.rs +++ b/tools/sgc/tests/test_discovery.rs @@ -1,13 +1,11 @@ +mod common; + +use common::source_sgc_command; use serde_json::Value; use std::fs; use std::path::PathBuf; -use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; -fn sgc() -> PathBuf { - PathBuf::from(env!("CARGO_BIN_EXE_sgc")) -} - fn temp_dir(name: &str) -> PathBuf { let stamp = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -49,7 +47,7 @@ def test_second() -> i64 { ) .unwrap(); - let output = Command::new(sgc()) + let output = source_sgc_command() .current_dir(&root) .args(["test", "--format", "json"]) .output() @@ -105,7 +103,7 @@ def test_failure() -> i64 { ) .unwrap(); - let output = Command::new(sgc()) + let output = source_sgc_command() .current_dir(&root) .args([ "test", diff --git a/tools/sgfmt/src/lib.rs b/tools/sgfmt/src/lib.rs index 8ea342d4..16ea60fb 100644 --- a/tools/sgfmt/src/lib.rs +++ b/tools/sgfmt/src/lib.rs @@ -704,14 +704,238 @@ fn escape_char(value: char) -> String { } } +/// Comment captured from original source so AST round-trips can reinject it. +#[derive(Debug, Clone, PartialEq, Eq)] +struct SourceComment { + /// Full comment text including `//` / `/*` / `*/` delimiters (no trailing `\n`). + text: String, + /// Non-comment code on the same line before a trailing `//` comment (trimmed). + /// Empty when the comment occupies the whole line (or is a block comment). + leading_code: String, + /// Next non-empty non-comment source line (trimmed) after a full-line/block + /// comment. Used as an insertion anchor in the formatted output. + following_code: Option, +} + +/// Extract line and block comments while respecting string/char literals. +fn extract_source_comments(source: &str) -> Vec { + let bytes = source.as_bytes(); + let mut comments = Vec::new(); + let mut i = 0usize; + let mut line_start = 0usize; + // Stack of (comment_start, line_start_at_comment) + while i < bytes.len() { + let ch = bytes[i] as char; + // Strings (regular + multiline """ ... """) + if ch == '"' { + if bytes.get(i..i + 3) == Some(b"\"\"\"") { + i += 3; + while i + 2 < bytes.len() && bytes.get(i..i + 3) != Some(b"\"\"\"") { + i += 1; + } + i = (i + 3).min(bytes.len()); + continue; + } + i += 1; + while i < bytes.len() { + match bytes[i] as char { + '\\' if i + 1 < bytes.len() => i += 2, + '"' => { + i += 1; + break; + } + _ => i += 1, + } + } + continue; + } + // Char literals + if ch == '\'' { + i += 1; + while i < bytes.len() { + match bytes[i] as char { + '\\' if i + 1 < bytes.len() => i += 2, + '\'' => { + i += 1; + break; + } + _ => i += 1, + } + } + continue; + } + // Line comment + if ch == '/' && bytes.get(i + 1) == Some(&b'/') { + let comment_start = i; + let this_line_start = line_start; + i += 2; + while i < bytes.len() && bytes[i] != b'\n' { + i += 1; + } + let text = source[comment_start..i].to_string(); + let leading = + normalize_code_anchor(source[this_line_start..comment_start].trim()); + let following = if leading.is_empty() { + next_code_line(source, i) + } else { + None + }; + comments.push(SourceComment { + text, + leading_code: leading, + following_code: following, + }); + // leave i at newline (or EOF) so outer loop advances line_start + if i < bytes.len() && bytes[i] == b'\n' { + i += 1; + line_start = i; + } + continue; + } + // Block comment + if ch == '/' && bytes.get(i + 1) == Some(&b'*') { + let comment_start = i; + let this_line_start = line_start; + i += 2; + while i + 1 < bytes.len() && !(bytes[i] == b'*' && bytes[i + 1] == b'/') { + if bytes[i] == b'\n' { + line_start = i + 1; + } + i += 1; + } + if i + 1 < bytes.len() { + i += 2; // consume */ + } + let text = source[comment_start..i].to_string(); + let leading = + normalize_code_anchor(source[this_line_start..comment_start].trim()); + let following = if leading.is_empty() { + next_code_line(source, i) + } else { + None + }; + comments.push(SourceComment { + text, + leading_code: leading, + following_code: following, + }); + continue; + } + if ch == '\n' { + i += 1; + line_start = i; + continue; + } + i += 1; + } + comments +} + +fn normalize_code_anchor(code: &str) -> String { + code.trim() + .trim_end_matches(';') + .trim() + .to_string() +} + +fn next_code_line(source: &str, from: usize) -> Option { + let rest = &source[from.min(source.len())..]; + for line in rest.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + if trimmed.starts_with("//") || trimmed.starts_with("/*") { + continue; + } + // Strip trailing line comment for a stable anchor. + let code = match trimmed.find("//") { + Some(idx) => trimmed[..idx].trim(), + None => trimmed, + }; + let anchor = normalize_code_anchor(code); + if !anchor.is_empty() { + return Some(anchor); + } + } + None +} + +fn reinject_source_comments(formatted: &str, comments: &[SourceComment]) -> String { + if comments.is_empty() { + return formatted.to_string(); + } + let mut lines: Vec = formatted.lines().map(str::to_string).collect(); + // Track which formatted line indices already received a trailing comment. + let mut used_trailing = vec![false; lines.len()]; + + for comment in comments { + if !comment.leading_code.is_empty() { + // Trailing line comment: attach to the first matching code line. + if let Some(idx) = lines.iter().enumerate().position(|(idx, line)| { + !used_trailing[idx] + && normalize_code_anchor(line.trim()) == comment.leading_code + }) { + if !lines[idx].contains(&comment.text) { + // Preserve single-space before // comments. + let base = lines[idx].trim_end().to_string(); + lines[idx] = format!("{base} {}", comment.text.trim_start()); + used_trailing[idx] = true; + } + continue; + } + // Fallback: append as a free-standing line at end if no anchor found. + lines.push(comment.text.clone()); + continue; + } + + // Full-line or block comment: insert before the following code anchor. + if let Some(anchor) = &comment.following_code { + if let Some(idx) = lines.iter().position(|line| { + let key = normalize_code_anchor(line.trim()); + key == *anchor || key.starts_with(anchor.as_str()) + }) { + // Avoid duplicating if already present immediately above. + let already = idx > 0 && lines[idx - 1].trim() == comment.text.trim(); + if !already { + lines.insert(idx, comment.text.clone()); + used_trailing.insert(idx, false); + } + continue; + } + } + // Leading / trailing file comments with no anchor: keep at top then bottom. + if comment.following_code.is_none() { + // Prefer end of file for trailing orphan comments. + if !lines + .last() + .is_some_and(|l| l.trim() == comment.text.trim()) + { + lines.push(comment.text.clone()); + } + } else { + lines.insert(0, comment.text.clone()); + used_trailing.insert(0, false); + } + } + + let mut out = lines.join("\n"); + if formatted.ends_with('\n') && !out.ends_with('\n') { + out.push('\n'); + } + out +} + pub fn format_source(source: &str, options: &FormatOptions) -> Result { + let comments = extract_source_comments(source); let program = SgParser::parse(source).into_diagnostic()?; let formatter = Formatter::new(options.clone()); let formatted = formatter.format_program(&program); + let with_comments = reinject_source_comments(&formatted, &comments); // Safety net: never emit syntactically invalid source. - SgParser::parse(&formatted).into_diagnostic()?; - Ok(formatted) + SgParser::parse(&with_comments).into_diagnostic()?; + Ok(with_comments) } #[cfg(test)] @@ -756,6 +980,57 @@ mod tests { assert_eq!(first, second); } + #[test] + fn preserves_line_and_block_comments_through_format() { + // RED regression: AST round-trip previously dropped all comments because + // the lexer skips them (TokenKind logos skip) and the formatter rebuilds + // source from a comment-free AST. + let src = r#"// leading ownership note +def main() -> i64 { + // body: by-value String params skip auto-Drop + return 0 // trailing +} +/* block: owns request end-to-end on unsupported path */ +"#; + let formatted = format_test_source(src, FormatOptions::default()); + assert!( + formatted.contains("// leading ownership note"), + "leading comment lost:\n{formatted}" + ); + assert!( + formatted.contains("// body: by-value String params skip auto-Drop"), + "body comment lost:\n{formatted}" + ); + assert!( + formatted.contains("// trailing"), + "trailing comment lost:\n{formatted}" + ); + assert!( + formatted.contains("/* block: owns request end-to-end on unsupported path */"), + "block comment lost:\n{formatted}" + ); + // Idempotent with comments present. + let second = format_test_source(&formatted, FormatOptions::default()); + assert_eq!(formatted, second); + } + + #[test] + fn does_not_treat_comment_markers_inside_strings_as_comments() { + let src = r#"def main() -> i64 { + let s = "// not a comment"; + let t = "/* also not */"; + 0 +} +"#; + let formatted = format_test_source(src, FormatOptions::default()); + assert!(formatted.contains("\"// not a comment\"") || formatted.contains("// not a comment")); + // No free-standing comment lines invented from string contents. + assert!( + !formatted.lines().any(|l| l.trim() == "// not a comment"), + "string content must not become a real comment:\n{formatted}" + ); + } + #[test] fn formats_unsigned_literals_without_losing_their_type() { let src = "def main() -> u64 { 7u64 }"; diff --git a/tools/sglsp/src/stdlib.rs b/tools/sglsp/src/stdlib.rs index ee2a8109..97f61764 100644 --- a/tools/sglsp/src/stdlib.rs +++ b/tools/sglsp/src/stdlib.rs @@ -509,6 +509,40 @@ import std::status; assert!(location.range.start.line > 0); } + #[test] + fn stdlib_json_error_kind_surface_has_symbols_signatures_and_definition() { + let content = "import std::json;\n"; + + assert_symbols_for_content( + content, + &[ + "JSON_ERROR_KIND_NONE", + "JSON_ERROR_KIND_UNCLASSIFIED", + "JSON_ERROR_KIND_DUPLICATE_FIELD", + "JSON_ERROR_KIND_INVALID_UNICODE", + "JSON_ERROR_KIND_TRAILING_BYTES", + "json_last_error_kind", + "new_string_from_string", + ], + ); + assert_signatures_for_content( + content, + &[ + "def JSON_ERROR_KIND_DUPLICATE_FIELD() -> i64", + "def JSON_ERROR_KIND_INVALID_UNICODE() -> i64", + "def JSON_ERROR_KIND_TRAILING_BYTES() -> i64", + "def json_last_error_kind() -> i64", + "def new_string_from_string(self, value: &String) -> Result [impl JsonDoc]", + ], + ); + + let location = stdlib_definition_for_content(content, "json_last_error_kind") + .expect("JSON error-kind wrapper should resolve into std::json"); + assert_eq!(location.uri.scheme(), "sengoo-stdlib"); + assert!(location.uri.as_str().ends_with("/json.sg")); + assert!(location.range.start.line > 0); + } + #[test] fn stdlib_signatures_follow_imported_modules() { let signatures = stdlib_signatures_for_content("import std::option;\n"); @@ -522,6 +556,37 @@ import std::status; .contains(&"def result_ok_with(value: T, error_placeholder: E) -> Result")); } + #[test] + fn binary_io_import_exposes_completion_and_exact_signatures() { + let content = "import std::io;\n"; + + assert_symbols_for_content( + content, + &[ + "Buffer", + "get_u8", + "set_u8", + "read_u32_be", + "write_u32_be", + "io_protocol_binary_mode", + "io_stdin_read_exact", + "io_stdout_write_all", + ], + ); + assert_signatures_for_content( + content, + &[ + "def get_u8(self, index: i64) -> Result [impl Buffer]", + "def set_u8(self, index: i64, value: i64) -> Result [impl Buffer]", + "def read_u32_be(self, offset: i64) -> Result [impl Buffer]", + "def write_u32_be(self, offset: i64, value: i64) -> Result [impl Buffer]", + "def io_protocol_binary_mode() -> Result", + "def io_stdin_read_exact(buffer: Buffer, offset: i64, len: i64) -> Result", + "def io_stdout_write_all(buffer: Buffer, offset: i64, len: i64) -> Result", + ], + ); + } + #[test] fn stdlib_symbols_follow_ffi_result_family_dependencies() { let symbols = stdlib_symbols_for_content("import std::net;\n"); diff --git a/tools/sgpm/src/main.rs b/tools/sgpm/src/main.rs index d5069cdc..6c90a236 100644 --- a/tools/sgpm/src/main.rs +++ b/tools/sgpm/src/main.rs @@ -31,10 +31,34 @@ const SGPM_VERSION: &str = concat!( #[command(version = SGPM_VERSION)] #[command(about = "Sengoo package manager MVP", long_about = None)] struct Cli { + /// Native runtime source policy forwarded to every delegated sgc command. + #[arg( + long = "runtime-mode", + global = true, + value_enum, + default_value_t = SgcRuntimeMode::Installed + )] + runtime_mode: SgcRuntimeMode, + #[command(subcommand)] command: Commands, } +#[derive(Debug, Clone, Copy, ValueEnum)] +enum SgcRuntimeMode { + Installed, + SourceDevelopment, +} + +impl SgcRuntimeMode { + fn as_str(self) -> &'static str { + match self { + Self::Installed => "installed", + Self::SourceDevelopment => "source-development", + } + } +} + #[derive(Subcommand, Debug)] enum Commands { /// Create a new Sengoo package. @@ -400,6 +424,7 @@ struct UpdateArgs { fn main() -> Result<()> { let cli = Cli::parse(); + let runtime_mode = cli.runtime_mode.as_str(); match cli.command { Commands::New { name, path, lib } => { @@ -425,7 +450,7 @@ fn main() -> Result<()> { args.package.as_deref(), args.workspace, )?; - let toolchain = Toolchain::discover()?; + let toolchain = Toolchain::discover(runtime_mode)?; for graph in &graphs { toolchain.build(graph, profile(args.release), args.verbose)?; } @@ -438,7 +463,7 @@ fn main() -> Result<()> { args.package.as_deref(), args.workspace, )?; - let toolchain = Toolchain::discover()?; + let toolchain = Toolchain::discover(runtime_mode)?; for graph in &graphs { toolchain.check(graph, args.verbose)?; } @@ -446,7 +471,7 @@ fn main() -> Result<()> { } Commands::Run(args) => { let graph = load_graph(&args.manifest_path, args.locked, args.package.as_deref())?; - let toolchain = Toolchain::discover()?; + let toolchain = Toolchain::discover(runtime_mode)?; toolchain.run(&graph, profile(args.release), &args.args, args.verbose) } Commands::Test(args) => { @@ -456,7 +481,7 @@ fn main() -> Result<()> { args.package.as_deref(), args.workspace, )?; - let toolchain = Toolchain::discover()?; + let toolchain = Toolchain::discover(runtime_mode)?; for graph in &graphs { toolchain.test(graph, profile(args.release), args.verbose)?; } @@ -469,7 +494,7 @@ fn main() -> Result<()> { args.package.as_deref(), args.workspace, )?; - let toolchain = Toolchain::discover()?; + let toolchain = Toolchain::discover(runtime_mode)?; for graph in &graphs { toolchain.fmt(graph, args.check, args.verbose)?; } @@ -485,7 +510,7 @@ fn main() -> Result<()> { args.package.as_deref(), args.workspace, )?; - let toolchain = Toolchain::discover()?; + let toolchain = Toolchain::discover(runtime_mode)?; for graph in &graphs { toolchain.doc(graph, args.output.as_deref(), args.verbose)?; } diff --git a/tools/sgpm/src/runner.rs b/tools/sgpm/src/runner.rs index 5c209555..c85b0eb9 100644 --- a/tools/sgpm/src/runner.rs +++ b/tools/sgpm/src/runner.rs @@ -1,5 +1,6 @@ use crate::resolver::{render_tree, Graph, PackageNode}; use miette::{Context, IntoDiagnostic, Result}; +use std::collections::{BTreeMap, BTreeSet}; use std::env; use std::ffi::OsString; use std::fs; @@ -35,25 +36,36 @@ impl BuildProfile { pub struct Toolchain { sgc: PathBuf, sgfmt: Option, + runtime_mode: String, } impl Toolchain { - pub fn discover() -> Result { + pub fn discover(runtime_mode: &str) -> Result { Ok(Self { sgc: find_tool("SGPM_SGC", "sgc")?, sgfmt: find_optional_tool("SGPM_SGFMT", "sgfmt"), + runtime_mode: runtime_mode.to_string(), }) } + fn sgc_command(&self) -> Command { + Command::new(&self.sgc) + } + + fn append_runtime_mode(&self, command: &mut Command) { + command.arg("--runtime-mode").arg(&self.runtime_mode); + } + pub fn build(&self, graph: &Graph, profile: BuildProfile, verbose: bool) -> Result<()> { for node in &graph.nodes { if node.manifest.lib.is_some() && node.manifest.bin.is_none() { - let mut command = Command::new(&self.sgc); + let mut command = self.sgc_command(); command .current_dir(&node.root_dir) .arg("check") .arg(&node.entry_path); configure_module_map(&mut command, graph, node, true)?; + self.append_runtime_mode(&mut command); if verbose { eprintln!("sgpm: {}", render_command(&command)); @@ -70,7 +82,7 @@ impl Toolchain { let output = package_output_path(node, profile)?; ensure_parent(&output)?; - let mut command = Command::new(&self.sgc); + let mut command = self.sgc_command(); command .current_dir(&node.root_dir) .arg("build") @@ -80,6 +92,7 @@ impl Toolchain { .arg("-O") .arg(profile.opt_level()); configure_module_map(&mut command, graph, node, true)?; + self.append_runtime_mode(&mut command); if verbose { eprintln!("sgpm: {}", render_command(&command)); @@ -97,12 +110,13 @@ impl Toolchain { pub fn check(&self, graph: &Graph, verbose: bool) -> Result<()> { for node in &graph.nodes { - let mut command = Command::new(&self.sgc); + let mut command = self.sgc_command(); command .current_dir(&node.root_dir) .arg("check") .arg(&node.entry_path); configure_module_map(&mut command, graph, node, true)?; + self.append_runtime_mode(&mut command); if verbose { eprintln!("sgpm: {}", render_command(&command)); @@ -153,7 +167,7 @@ impl Toolchain { pub fn test(&self, graph: &Graph, profile: BuildProfile, verbose: bool) -> Result<()> { let mut ran = 0usize; for node in &graph.nodes { - let mut command = Command::new(&self.sgc); + let mut command = self.sgc_command(); command .current_dir(&node.root_dir) .arg("test") @@ -163,6 +177,7 @@ impl Toolchain { command.arg("--release"); } configure_module_map(&mut command, graph, node, true)?; + self.append_runtime_mode(&mut command); if verbose { eprintln!("sgpm: {}", render_command(&command)); @@ -228,7 +243,7 @@ impl Toolchain { } else { base_output.clone() }; - let mut command = Command::new(&self.sgc); + let mut command = self.sgc_command(); command .current_dir(&node.root_dir) .arg("doc") @@ -236,6 +251,7 @@ impl Toolchain { .arg("--output") .arg(&output_dir); configure_module_map(&mut command, graph, node, true)?; + self.append_runtime_mode(&mut command); if verbose { eprintln!("sgpm: {}", render_command(&command)); @@ -321,9 +337,19 @@ fn module_map_value( node: &PackageNode, include_current: bool, ) -> Result> { - let mut entries = Vec::new(); + let mut reachable = BTreeSet::from([node.id.clone()]); + let mut pending = vec![node.id.clone()]; + while let Some(package_id) = pending.pop() { + for edge in graph.edges.iter().filter(|edge| edge.from == package_id) { + if reachable.insert(edge.to.clone()) { + pending.push(edge.to.clone()); + } + } + } + + let mut entries = BTreeMap::new(); for edge in &graph.edges { - if edge.from != node.id { + if !reachable.contains(&edge.from) { continue; } let Some(dep) = graph.node_by_id(&edge.to) else { @@ -332,28 +358,52 @@ fn module_map_value( let Some(lib) = dep.manifest.lib.as_ref() else { continue; }; - entries.push(format!( - "{}={}", - edge.alias, - portable_path(&dep.root_dir.join(&lib.path)) - )); + insert_module_map_entry( + &mut entries, + &edge.alias, + portable_path(&dep.root_dir.join(&lib.path)), + )?; } if include_current { if let Some(lib) = node.manifest.lib.as_ref() { - entries.push(format!( - "{}={}", - node.name, - portable_path(&node.root_dir.join(&lib.path)) - )); + insert_module_map_entry( + &mut entries, + &node.name, + portable_path(&node.root_dir.join(&lib.path)), + )?; } } if entries.is_empty() { return Ok(None); } - env::join_paths(entries) - .map(Some) - .into_diagnostic() - .context("failed to encode dependency library module map") + env::join_paths( + entries + .into_iter() + .map(|(alias, path)| format!("{alias}={path}")), + ) + .map(Some) + .into_diagnostic() + .context("failed to encode dependency library module map") +} + +fn insert_module_map_entry( + entries: &mut BTreeMap, + alias: &str, + path: String, +) -> Result<()> { + if let Some(existing) = entries.get(alias) { + if existing != &path { + miette::bail!( + "conflicting module alias '{}' resolves to both '{}' and '{}'", + alias, + existing, + path + ); + } + return Ok(()); + } + entries.insert(alias.to_string(), path); + Ok(()) } fn portable_path(path: &Path) -> String { diff --git a/tools/sgpm/tests/integration.rs b/tools/sgpm/tests/integration.rs index 9dc23d72..461555e3 100644 --- a/tools/sgpm/tests/integration.rs +++ b/tools/sgpm/tests/integration.rs @@ -2123,6 +2123,122 @@ fn sgpm_check_exposes_dependency_library_module_map() { let _ = fs::remove_dir_all(dir); } +#[test] +fn sgpm_check_exposes_transitive_dependency_library_module_map() { + let dir = temp_dir("check_transitive_module_map"); + let transitive = dir.join("transitive"); + let direct = dir.join("direct"); + let app = dir.join("app"); + write_lib_pkg(&transitive, "transitive"); + write_lib_pkg(&direct, "direct"); + fs::write( + direct.join("src/lib.sg"), + "import transitive;\ndef imported_value() -> i64 { transitive::imported_value() }\n", + ) + .unwrap(); + fs::write( + direct.join("Sengoo.toml"), + "[package]\nname = 'direct'\nversion = '0.1.0'\nedition = '2026'\n\n[lib]\npath = 'src/lib.sg'\n\n[dependencies]\ntransitive = { path = '../transitive' }\n", + ) + .unwrap(); + write_pkg(&app, "app", &[("direct", "../direct")]); + + let record = dir.join("record.txt"); + let fake = fake_sgc(&dir); + let output = Command::new(sgpm()) + .args([ + "check", + "--manifest-path", + app.join("Sengoo.toml").to_str().unwrap(), + ]) + .current_dir(&dir) + .env("SGPM_SGC", fake) + .env("SGPM_RECORD", &record) + .output() + .expect("run sgpm check"); + + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let log = fs::read_to_string(record).unwrap().replace('\\', "/"); + let app_check = log + .lines() + .find(|line| line.contains("/app :: check")) + .expect("app check log entry"); + let direct_entry = direct + .join("src/lib.sg") + .to_string_lossy() + .replace('\\', "/"); + let transitive_entry = transitive + .join("src/lib.sg") + .to_string_lossy() + .replace('\\', "/"); + assert!( + app_check.contains(&format!("direct={direct_entry}")), + "root package should receive its direct dependency module:\n{app_check}" + ); + assert!( + app_check.contains(&format!("transitive={transitive_entry}")), + "root package should receive modules imported by dependency sources:\n{app_check}" + ); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn sgpm_check_rejects_conflicting_transitive_module_aliases() { + let dir = temp_dir("check_conflicting_transitive_aliases"); + let left_leaf = dir.join("left_leaf"); + let right_leaf = dir.join("right_leaf"); + let left = dir.join("left"); + let right = dir.join("right"); + let app = dir.join("app"); + write_lib_pkg(&left_leaf, "left_leaf"); + write_lib_pkg(&right_leaf, "right_leaf"); + write_lib_pkg(&left, "left"); + write_lib_pkg(&right, "right"); + fs::write( + left.join("Sengoo.toml"), + "[package]\nname = 'left'\nversion = '0.1.0'\nedition = '2026'\n\n[lib]\npath = 'src/lib.sg'\n\n[dependencies]\nshared = { package = 'left_leaf', path = '../left_leaf' }\n", + ) + .unwrap(); + fs::write( + right.join("Sengoo.toml"), + "[package]\nname = 'right'\nversion = '0.1.0'\nedition = '2026'\n\n[lib]\npath = 'src/lib.sg'\n\n[dependencies]\nshared = { package = 'right_leaf', path = '../right_leaf' }\n", + ) + .unwrap(); + write_pkg(&app, "app", &[("left", "../left"), ("right", "../right")]); + + let record = dir.join("record.txt"); + let fake = fake_sgc(&dir); + let output = Command::new(sgpm()) + .args([ + "check", + "--manifest-path", + app.join("Sengoo.toml").to_str().unwrap(), + ]) + .current_dir(&dir) + .env("SGPM_SGC", fake) + .env("SGPM_RECORD", &record) + .output() + .expect("run sgpm check"); + + assert!( + !output.status.success(), + "conflicting aliases must fail closed" + ); + let stderr = String::from_utf8_lossy(&output.stderr).replace('\\', "/"); + assert!( + stderr.contains("conflicting module alias 'shared'") + && stderr.contains("left_leaf/src/lib.sg") + && stderr.contains("right_leaf/src/lib.sg"), + "stderr should identify the alias and both conflicting sources:\n{stderr}" + ); + let _ = fs::remove_dir_all(dir); +} + #[test] fn sgpm_check_maps_dual_target_dependency_to_library_entry() { let dir = temp_dir("check_dual_target_module_map"); diff --git a/tools/sgpm/tests/realworld_e2e.rs b/tools/sgpm/tests/realworld_e2e.rs index a4fc4f2b..9a134be5 100644 --- a/tools/sgpm/tests/realworld_e2e.rs +++ b/tools/sgpm/tests/realworld_e2e.rs @@ -106,6 +106,109 @@ fn workflow_step_block<'a>(workflow: &'a str, step_name: &str) -> &'a str { &rest[..next] } +fn senline_protocol_canary_fragments() -> Vec { + let mut state = 0x243f_6a88_85a3_08d3_u64; + (0..256) + .flat_map(|index| { + let mut bytes = [0_u8; 16]; + for byte in &mut bytes { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + *byte = state as u8; + } + let hex = bytes + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + [format!("rejected_canary_{index:03}_{hex}"), hex] + }) + .collect() +} + +#[test] +fn senline_worker_publish_archive_contains_no_rejected_protocol_canaries() { + let dir = temp_dir("senline_worker_publish_archive"); + let package = dir.join("senline-domain-worker"); + copy_dir_filtered(&realworld_fixture("senline-domain-worker"), &package); + + let output = Command::new(sgpm()) + .args(["publish", "--dry-run", "--locked"]) + .current_dir(&package) + .output() + .expect("run locked Senline worker publish dry-run"); + assert!( + output.status.success(), + "Senline worker publish stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let archive = package.join("target/package/senline_domain_worker-0.1.0.tar.gz"); + assert!( + archive.is_file(), + "worker package archive must exist for leakage scanning: {}", + archive.display() + ); + assert!( + fs::metadata(&archive).unwrap().len() > 0, + "worker package archive must not be empty" + ); + + let archive_file = fs::File::open(&archive).expect("open Senline worker package archive"); + let mut archive_reader = Archive::new(GzDecoder::new(archive_file)); + let canaries = senline_protocol_canary_fragments(); + let mut entry_count = 0_usize; + let mut content_bytes = 0_usize; + for entry in archive_reader + .entries() + .expect("read worker package entries") + { + let mut entry = entry.expect("decode worker package entry"); + let path = entry + .path() + .expect("decode worker package entry path") + .to_string_lossy() + .replace('\\', "/"); + assert!( + !path.is_empty(), + "worker package entry path must not be empty" + ); + let mut contents = Vec::new(); + entry + .read_to_end(&mut contents) + .unwrap_or_else(|error| panic!("read worker package entry {path}: {error}")); + entry_count += 1; + content_bytes += contents.len(); + for canary in &canaries { + assert!( + !path + .as_bytes() + .windows(canary.len()) + .any(|window| window == canary.as_bytes()), + "rejected protocol canary leaked into package path {path}" + ); + assert!( + !contents + .windows(canary.len()) + .any(|window| window == canary.as_bytes()), + "rejected protocol canary leaked into package entry {path}" + ); + } + } + assert!( + entry_count > 0, + "worker package archive must contain entries" + ); + assert!( + content_bytes > 0, + "worker package archive entries must contain bytes" + ); + + let _ = fs::remove_dir_all(dir); +} + + #[test] fn realworld_locked_loop_uses_real_toolchain_binaries() { if !native_toolchain_available() { diff --git a/tools/sgpm/tests/toolchain_distribution.rs b/tools/sgpm/tests/toolchain_distribution.rs index b540803d..4e540178 100644 --- a/tools/sgpm/tests/toolchain_distribution.rs +++ b/tools/sgpm/tests/toolchain_distribution.rs @@ -1,3 +1,4 @@ +use serde_json::json; use serde_json::Value; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet}; @@ -553,3 +554,436 @@ fn frontend_baseline_points_at_an_exact_retained_ci_report() { "retained CI report should not contain bootstrap reconstruction notes" ); } + +// --- senline dogfood distribution tests --- + +fn temp_dir(tag: &str) -> PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock should be after epoch") + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "sengoo-distribution-{tag}-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&path).expect("create distribution test directory"); + path +} + +fn run_package_toolchain( + root: &Path, + output_dir: &Path, + cargo_target_dir: &Path, + environment: &[(&str, &str)], +) -> std::process::Output { + let mut command = Command::new(powershell()); + command + .args(["-NoLogo", "-NoProfile", "-NonInteractive", "-File"]) + .arg( + workspace_root() + .join("scripts") + .join("package-toolchain.ps1"), + ) + .arg("-NoBuild") + .arg("-RepoRoot") + .arg(root) + .arg("-OutputDir") + .arg(output_dir) + .arg("-CargoTargetDir") + .arg(cargo_target_dir); + for (name, value) in environment { + command.env(name, value); + } + command.output().expect("run package-toolchain.ps1") +} + +fn distribution_manifest() -> Value { + json!({ + "schema_version": 2, + "version": "0.1.0-repro-test", + "target": "x86_64-pc-windows-msvc", + "build_hash": "111111111111", + "source_revision": "1111111111111111111111111111111111111111", + "source_dirty": false, + "artifact_provenance": "built-by-package-toolchain", + "release_eligible": true, + "build_manifest_id": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "tools": ["sgc", "sgpm", "sgfmt", "sglsp"], + "tool_versions": { + "sgc": "sgc 0.1.0 (111111111111)", + "sgpm": "sgpm 0.1.0 (111111111111)", + "sgfmt": "sgfmt 0.1.0 (111111111111)", + "sglsp": "sglsp 0.1.0 (111111111111)" + }, + "stdlib_modules": ["io.sg", "json.sg"], + "runtime_sources": ["runtime.c", "runtime_json.c"], + "native_runtime": { + "abi_version": 1, + "target": "x86_64-pc-windows-msvc", + "library": "share/sengoo/runtime/x86_64-pc-windows-msvc/sengoo_runtime.lib", + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "link_args": ["kernel32.lib", "bcrypt.lib"], + "dynamic_dependencies": ["vcruntime140.dll", "ucrtbase.dll"] + }, + "payload_checksum_file": "payloads.sha256", + "payloads": [ + { + "path": "bin/sgc.exe", + "sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "size": 100 + }, + { + "path": "share/sengoo/runtime/x86_64-pc-windows-msvc/sengoo_runtime.lib", + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "size": 200 + } + ], + "archive_file": "sengoo-0.1.0-repro-test-x86_64-pc-windows-msvc.zip", + "checksum_file": "sengoo-0.1.0-repro-test-x86_64-pc-windows-msvc.zip.sha256", + "runner_os": "Windows", + "runner_image": "windows-2025", + "smoke_evidence": "build A", + "license_included": true, + "generated_at_utc": "2026-07-15T00:00:00Z" + }) +} + +fn powershell() -> &'static str { + if cfg!(windows) { + "powershell.exe" + } else { + "pwsh" + } +} + +fn run_manifest_comparator(left: &Path, right: &Path, output: &Path) -> std::process::Output { + Command::new(powershell()) + .args(["-NoLogo", "-NoProfile", "-NonInteractive", "-File"]) + .arg( + workspace_root() + .join("scripts") + .join("compare-distribution-manifests.ps1"), + ) + .arg("-LeftManifest") + .arg(left) + .arg("-RightManifest") + .arg(right) + .arg("-OutputDir") + .arg(output) + .output() + .expect("run distribution manifest comparator") +} + +fn write_json(path: &Path, value: &Value) { + fs::write( + path, + serde_json::to_vec_pretty(value).expect("serialize JSON"), + ) + .expect("write JSON fixture"); +} + +#[test] +fn distribution_manifest_comparator_allows_only_documented_provenance_differences() { + let root = temp_dir("comparator-allowed"); + let left_path = root.join("left.json"); + let right_path = root.join("right.json"); + let evidence_dir = root.join("evidence"); + let left = distribution_manifest(); + let mut right = left.clone(); + right["generated_at_utc"] = json!("2026-07-15T00:01:00Z"); + right["runner_os"] = json!("Windows-retry"); + right["runner_image"] = json!("windows-2025.1"); + right["smoke_evidence"] = json!("build B"); + right["tools"].as_array_mut().unwrap().reverse(); + right["stdlib_modules"].as_array_mut().unwrap().reverse(); + right["runtime_sources"].as_array_mut().unwrap().reverse(); + right["native_runtime"]["dynamic_dependencies"] + .as_array_mut() + .unwrap() + .reverse(); + right["payloads"].as_array_mut().unwrap().reverse(); + write_json(&left_path, &left); + write_json(&right_path, &right); + + let output = run_manifest_comparator(&left_path, &right_path, &evidence_dir); + assert!( + output.status.success(), + "comparator rejected documented differences\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let comparison: Value = serde_json::from_slice( + &fs::read(evidence_dir.join("comparison.json")).expect("comparison evidence"), + ) + .expect("parse comparison evidence"); + assert_eq!(comparison["status"], "reproducible"); + assert_eq!( + comparison["left"]["normalized_sha256"], + comparison["right"]["normalized_sha256"] + ); + assert_eq!( + comparison["excluded_fields"], + json!([ + "generated_at_utc", + "runner_os", + "runner_image", + "smoke_evidence" + ]) + ); + assert_eq!( + comparison["excluded_differences"] + .as_array() + .expect("excluded differences") + .len(), + 4 + ); + assert!(evidence_dir.join("normalized-a.json").is_file()); + assert!(evidence_dir.join("normalized-b.json").is_file()); + + let _ = fs::remove_dir_all(root); +} + +#[test] +fn distribution_manifest_comparator_rejects_identity_schema_and_path_drift() { + type Mutation = (&'static str, &'static str, Box); + + let mutations: Vec = vec![ + ( + "payload hash drift", + "payloads", + Box::new(|value| value["payloads"][0]["sha256"] = json!("c".repeat(64))), + ), + ( + "runtime ABI drift", + "native_runtime.abi_version", + Box::new(|value| value["native_runtime"]["abi_version"] = json!(2)), + ), + ( + "ordered link argument drift", + "native_runtime.link_args", + Box::new(|value| { + value["native_runtime"]["link_args"] + .as_array_mut() + .unwrap() + .reverse() + }), + ), + ( + "dynamic dependency drift", + "native_runtime.dynamic_dependencies", + Box::new(|value| { + value["native_runtime"]["dynamic_dependencies"][0] = json!("other.dll") + }), + ), + ( + "source revision drift", + "source_revision", + Box::new(|value| value["source_revision"] = json!("2".repeat(40))), + ), + ( + "tool version drift", + "tool_versions", + Box::new(|value| value["tool_versions"]["sgc"] = json!("sgc 0.1.1 (111111111111)")), + ), + ( + "unknown top-level field", + "unknown manifest field", + Box::new(|value| value["unexpected"] = json!(true)), + ), + ( + "missing required field", + "missing manifest field", + Box::new(|value| { + value.as_object_mut().unwrap().remove("license_included"); + }), + ), + ( + "absolute payload path", + "normalized relative path", + Box::new(|value| value["payloads"][0]["path"] = json!("C:/checkout/sgc.exe")), + ), + ( + "duplicate payload path", + "duplicate payload path", + Box::new(|value| { + value["payloads"][1]["path"] = value["payloads"][0]["path"].clone(); + }), + ), + ]; + + for (label, expected_error, mutate) in mutations { + let root = temp_dir(&label.replace(' ', "-")); + let left_path = root.join("left.json"); + let right_path = root.join("right.json"); + let evidence_dir = root.join("evidence"); + let left = distribution_manifest(); + let mut right = left.clone(); + mutate(&mut right); + write_json(&left_path, &left); + write_json(&right_path, &right); + + let output = run_manifest_comparator(&left_path, &right_path, &evidence_dir); + assert!( + !output.status.success(), + "{label} unexpectedly compared as reproducible" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains(expected_error), + "{label} did not report `{expected_error}`\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + stderr + ); + + let _ = fs::remove_dir_all(root); + } +} + +#[test] +fn distribution_packages_and_verifies_the_target_native_runtime_payload() { + let root = workspace_root(); + let package = fs::read_to_string(root.join("scripts/package-toolchain.ps1")) + .expect("read package-toolchain.ps1"); + assert!( + package.contains("sengoo_runtime.lib") && package.contains("libsengoo_runtime.a"), + "packaging must select the target-native runtime static library" + ); + assert!( + package.contains("schema_version = 2") + && package.contains("native_runtime") + && package.contains("abi_version = 1") + && package.contains("build_manifest_id"), + "manifest v2 must bind runtime ABI, target, payload, and build identity" + ); + assert!( + package.contains("payloads.sha256") && package.contains("Get-FileHash"), + "packaging must emit per-file SHA-256 evidence" + ); + assert!( + package.contains("source_revision = $sourceRevision") + && package.contains("source_dirty") + && package.contains("artifact_provenance") + && package.contains("release_eligible"), + "packaging must distinguish source identity from unverified prebuilt artifacts" + ); + + let powershell = + fs::read_to_string(root.join("scripts/install.ps1")).expect("read install.ps1"); + assert!( + powershell.contains("payloads.sha256") + && powershell.contains("Get-FileHash") + && powershell.contains("payload checksum mismatch"), + "PowerShell installation must verify every packaged payload before copying" + ); + + let shell = fs::read_to_string(root.join("scripts/install.sh")).expect("read install.sh"); + assert!( + shell.contains("payloads.sha256") + && shell.contains("sha256sum -c") + && shell.contains("shasum -a 256 -c"), + "POSIX installation must verify every packaged payload before copying" + ); + + let workflow = fs::read_to_string(root.join(".github/workflows/toolchain-distribution.yml")) + .expect("read toolchain-distribution.yml"); + assert!( + !workflow.contains("package-toolchain.ps1 -Version $version -NoBuild") + && workflow.contains("release_eligible"), + "release packaging must build its own artifacts and reject non-release provenance" + ); +} + +#[test] +fn distribution_workflow_compares_independent_windows_and_linux_builds() { + let root = workspace_root(); + let workflow = fs::read_to_string(root.join(".github/workflows/toolchain-distribution.yml")) + .expect("read toolchain-distribution.yml"); + let package_script = fs::read_to_string(root.join("scripts/package-toolchain.ps1")) + .expect("read package-toolchain.ps1"); + + assert!( + workflow.contains("reproducible: true") + && workflow.contains("sengoo-cargo-package-a-") + && workflow.contains("sengoo-cargo-package-b-"), + "Windows and Linux packaging must use independent Cargo target directories for A/B builds" + ); + // Path remapping lives in package-toolchain.ps1 (via CARGO_ENCODED_RUSTFLAGS) + // so workflow YAML no longer inlines --remap-path-prefix= flags. + assert!( + package_script.contains("--remap-path-prefix=") + && package_script.contains("CARGO_ENCODED_RUSTFLAGS"), + "package-toolchain must remap source and target paths for reproducible artifacts" + ); + assert!( + workflow.contains("compare-distribution-manifests.ps1") + && workflow.contains("target/repro-evidence/") + && workflow.contains("normalized-a.json") + && workflow.contains("comparison.json"), + "independent manifests must be normalized and compared with retained evidence" + ); + assert!( + workflow.contains("Install reproducibility build B (POSIX)") + && workflow.contains("Install reproducibility build B (Windows)") + && workflow.contains("target/install-smoke-repro-b"), + "both independently built archives must pass checksum-verifying installation" + ); + assert!( + workflow.contains("Upload reproducibility evidence") + && workflow.contains("sengoo-reproducibility-${{ matrix.artifact }}"), + "build B and the normalized comparison must be retained outside release publication inputs" + ); +} + +#[test] +fn package_toolchain_rejects_a_target_that_does_not_match_the_host() { + let temp = temp_dir("package-host-target"); + let wrong_target = if cfg!(windows) { + "x86_64-unknown-linux-gnu" + } else { + "x86_64-pc-windows-msvc" + }; + let output = run_package_toolchain( + &workspace_root(), + &temp.join("dist"), + &temp.join("cargo-target"), + &[("SENGOO_DIST_TARGET", wrong_target)], + ); + assert!( + !output.status.success(), + "cross-host target spoofing must fail" + ); + assert!( + String::from_utf8_lossy(&output.stderr).contains("does not match host target"), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(!temp.join("dist").exists()); + let _ = fs::remove_dir_all(temp); +} + +#[test] +fn package_toolchain_rejects_source_revision_override_that_is_not_head() { + let temp = temp_dir("package-source-override"); + let output = run_package_toolchain( + &workspace_root(), + &temp.join("dist"), + &temp.join("cargo-target"), + &[( + "SENGOO_SOURCE_REVISION", + "ffffffffffffffffffffffffffffffffffffffff", + )], + ); + assert!( + !output.status.success(), + "mismatched source revision must fail" + ); + assert!( + String::from_utf8_lossy(&output.stderr).contains("must equal repository HEAD"), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(!temp.join("dist").exists()); + let _ = fs::remove_dir_all(temp); +} diff --git a/tools/stdlib/README.md b/tools/stdlib/README.md index 5ba6d6b0..34e68d2a 100644 --- a/tools/stdlib/README.md +++ b/tools/stdlib/README.md @@ -220,9 +220,22 @@ as exact `i64` when representable. Builders create object, array, string, number, bool, and null values inside a document. `JsonDoc.serialize(buffer)` writes compact valid JSON, and -`JsonDoc.close()` releases runtime-owned handles. Parser diagnostics are -available through `json_last_error_code()`, `json_last_error_offset()`, and -`json_last_error_copy(buffer)`. The current runtime enforces conservative +`JsonDoc.close()` releases runtime-owned handles. `JsonDoc.new_string(&str)` +retains its C-string-compatible behavior; decoded or dynamic owned strings +that may contain `U+0000` use `JsonDoc.new_string_from_string(&String)`, which +copies the owned string's explicit byte length, validates its handle and UTF-8, +and accepts at most 1 MiB per string. Parser diagnostics are +available through `json_last_error_code()`, `json_last_error_kind()`, +`json_last_error_offset()`, and `json_last_error_copy(buffer)`. Stable error +kinds are `JSON_ERROR_KIND_NONE()` (`0`), +`JSON_ERROR_KIND_UNCLASSIFIED()` (`1`), +`JSON_ERROR_KIND_DUPLICATE_FIELD()` (`2`), +`JSON_ERROR_KIND_INVALID_UNICODE()` (`3`), and +`JSON_ERROR_KIND_TRAILING_BYTES()` (`4`). The code, kind, offset, and message +describe only the most recent JSON operation and must be copied before another +JSON call. Messages are human diagnostics and must not be parsed for program +control flow. Existing callers may continue to use the stable status code, +offset, and message APIs unchanged. The current runtime enforces conservative limits of 1 MiB input bytes, 64 nesting levels, and 4096 nodes; failed parses return no closeable partial document handle. diff --git a/tools/stdlib/ffi.sg b/tools/stdlib/ffi.sg index b5dd632c..b414527a 100644 --- a/tools/stdlib/ffi.sg +++ b/tools/stdlib/ffi.sg @@ -22,6 +22,10 @@ extern "C" { fn sengoo_ffi_buffer_len(buffer_handle: i64) -> i64; fn sengoo_ffi_buffer_capacity(buffer_handle: i64) -> i64; fn sengoo_ffi_buffer_used_len(buffer_handle: i64) -> i64; + fn sengoo_ffi_buffer_get_u8(buffer_handle: i64, index: i64) -> i64; + fn sengoo_ffi_buffer_set_u8(buffer_handle: i64, index: i64, value: i64) -> i64; + fn sengoo_ffi_buffer_read_u32_be(buffer_handle: i64, offset: i64) -> i64; + fn sengoo_ffi_buffer_write_u32_be(buffer_handle: i64, offset: i64, value: i64) -> i64; fn sengoo_ffi_buffer_ptr(buffer_handle: i64) -> i64; fn sengoo_ffi_buffer_copy_out(buffer_handle: i64, out_buffer: i64, out_capacity: i64) -> i64; fn sengoo_ffi_buffer_copy_in(buffer_handle: i64, src_ptr: i64, src_len: i64) -> i64; @@ -108,6 +112,8 @@ def ffi_i64_value_result(value: i64) -> Result { def ffi_status_from_raw(code: i64) -> i64 { if code == 0 { 0 + } else if code <= -1 && code >= -19 { + 0 - code } else if code == -2001 { 2 } else if code == -2002 { @@ -121,6 +127,14 @@ def ffi_status_from_raw(code: i64) -> i64 { } } +def ffi_bool_result(value: i64) -> Result { + if value >= 0 { + Result { is_ok: true, value: value != 0, error: 0 } + } else { + Result { is_ok: false, value: false, error: ffi_status_from_raw(value) } + } +} + def ffi_byte_count_result(value: i64) -> Result { if value >= 0 { Result { is_ok: true, value: value, error: 0 } @@ -311,6 +325,22 @@ impl Buffer { sengoo_ffi_buffer_used_len(self.handle) } + def get_u8(self, index: i64) -> Result { + ffi_byte_count_result(sengoo_ffi_buffer_get_u8(self.handle, index)) + } + + def set_u8(self, index: i64, value: i64) -> Result { + ffi_bool_result(sengoo_ffi_buffer_set_u8(self.handle, index, value)) + } + + def read_u32_be(self, offset: i64) -> Result { + ffi_byte_count_result(sengoo_ffi_buffer_read_u32_be(self.handle, offset)) + } + + def write_u32_be(self, offset: i64, value: i64) -> Result { + ffi_bool_result(sengoo_ffi_buffer_write_u32_be(self.handle, offset, value)) + } + def ptr(self) -> i64 { sengoo_ffi_buffer_ptr(self.handle) } @@ -380,12 +410,14 @@ impl Buffer { } def free(self) -> bool { - sengoo_ffi_buffer_free(self.handle) == 0 + if self.handle == 0 { true } else { sengoo_ffi_buffer_free(self.handle) == 0 } } } impl Drop for Buffer { def drop(&mut self) { - sengoo_ffi_buffer_free(self.handle); + if self.handle != 0 { + sengoo_ffi_buffer_free(self.handle); + } } } diff --git a/tools/stdlib/io.sg b/tools/stdlib/io.sg index 624c36e0..e0e40b64 100644 --- a/tools/stdlib/io.sg +++ b/tools/stdlib/io.sg @@ -2,6 +2,9 @@ extern "C" { fn sengoo_stdlib_str_ptr(value: &str) -> i64; fn sengoo_io_stdin_read(buffer: i64, capacity: i64) -> i64; fn sengoo_io_stdin_read_line(buffer: i64, capacity: i64) -> i64; + fn sengoo_io_protocol_binary_mode() -> i64; + fn sengoo_io_stdin_read_exact(buffer_handle: i64, offset: i64, len: i64) -> i64; + fn sengoo_io_stdout_write_all(buffer_handle: i64, offset: i64, len: i64) -> i64; fn sengoo_io_stdout_write(data: i64, len: i64) -> i64; fn sengoo_io_stderr_write(data: i64, len: i64) -> i64; fn sengoo_io_stdout_flush() -> i64; @@ -34,6 +37,18 @@ def io_stdin_read(buffer: Buffer) -> Result { io_stdin_read_raw(buffer.ptr(), buffer.len()) } +def io_protocol_binary_mode() -> Result { + io_bool_result(sengoo_io_protocol_binary_mode()) +} + +def io_stdin_read_exact(buffer: Buffer, offset: i64, len: i64) -> Result { + io_i64_result(sengoo_io_stdin_read_exact(buffer.handle, offset, len)) +} + +def io_stdout_write_all(buffer: Buffer, offset: i64, len: i64) -> Result { + io_i64_result(sengoo_io_stdout_write_all(buffer.handle, offset, len)) +} + def io_stdin_read_line_raw(buffer_ptr: i64, capacity: i64) -> Result { io_i64_result(sengoo_io_stdin_read_line(buffer_ptr, capacity)) } diff --git a/tools/stdlib/json.sg b/tools/stdlib/json.sg index 4667f468..3dde2d93 100644 --- a/tools/stdlib/json.sg +++ b/tools/stdlib/json.sg @@ -1,6 +1,8 @@ extern "C" { fn sengoo_json_parse_text(data: i64, len: i64) -> i64; fn sengoo_json_parse_buffer(buffer: i64, input_len: i64) -> i64; + fn sengoo_json_parse_text_strict(data: i64, len: i64) -> i64; + fn sengoo_json_parse_buffer_strict(buffer: i64, input_len: i64) -> i64; fn sengoo_json_doc_object_new() -> i64; fn sengoo_json_doc_close(handle: i64) -> i64; fn sengoo_json_doc_root(handle: i64) -> i64; @@ -9,12 +11,18 @@ extern "C" { fn sengoo_json_doc_new_object(handle: i64) -> i64; fn sengoo_json_doc_new_array(handle: i64) -> i64; fn sengoo_json_doc_new_string(handle: i64, value: i64) -> i64; + fn sengoo_json_doc_new_string_len(handle: i64, value: i64, value_len: i64) -> i64; + fn sengoo_json_doc_new_string_from_string(handle: i64, string_handle: i64) -> i64; fn sengoo_json_doc_new_bool(handle: i64, value: i64) -> i64; fn sengoo_json_doc_new_number(handle: i64, value: f64) -> i64; fn sengoo_json_doc_new_null(handle: i64) -> i64; fn sengoo_json_value_kind(doc_handle: i64, node_id: i64) -> i64; fn sengoo_json_object_has(doc_handle: i64, node_id: i64, key: i64) -> i64; + fn sengoo_json_object_has_len(doc_handle: i64, node_id: i64, key: i64, key_len: i64) -> i64; + fn sengoo_json_object_len(doc_handle: i64, node_id: i64) -> i64; + fn sengoo_json_object_key_copy(doc_handle: i64, node_id: i64, index: i64, buffer: i64) -> i64; fn sengoo_json_object_get(doc_handle: i64, node_id: i64, key: i64) -> i64; + fn sengoo_json_object_get_len(doc_handle: i64, node_id: i64, key: i64, key_len: i64) -> i64; fn sengoo_json_object_set(doc_handle: i64, node_id: i64, key: i64, value_doc_handle: i64, value_node_id: i64) -> i64; fn sengoo_json_array_len(doc_handle: i64, node_id: i64) -> i64; fn sengoo_json_array_get(doc_handle: i64, node_id: i64, index: i64) -> i64; @@ -25,6 +33,7 @@ extern "C" { fn sengoo_json_number_i64(doc_handle: i64, node_id: i64) -> i64; fn sengoo_json_number_f64(doc_handle: i64, node_id: i64) -> f64; fn sengoo_json_last_error_code() -> i64; + fn sengoo_json_last_error_kind() -> i64; fn sengoo_json_last_error_offset() -> i64; fn sengoo_json_last_error_copy(buffer: i64) -> i64; } @@ -50,6 +59,12 @@ def JSON_KIND_STRING() -> i64 { 3 } def JSON_KIND_ARRAY() -> i64 { 4 } def JSON_KIND_OBJECT() -> i64 { 5 } +def JSON_ERROR_KIND_NONE() -> i64 { 0 } +def JSON_ERROR_KIND_UNCLASSIFIED() -> i64 { 1 } +def JSON_ERROR_KIND_DUPLICATE_FIELD() -> i64 { 2 } +def JSON_ERROR_KIND_INVALID_UNICODE() -> i64 { 3 } +def JSON_ERROR_KIND_TRAILING_BYTES() -> i64 { 4 } + def json_error_or_unknown() -> i64 { if sengoo_json_last_error_code() == 0 { 1 @@ -128,6 +143,14 @@ def json_parse_buffer(buffer: Buffer, input_len: i64) -> Result { json_doc_result(sengoo_json_parse_buffer(buffer.handle, input_len)) } +def json_parse_strict(text: &str) -> Result { + json_doc_result(sengoo_json_parse_text_strict(sengoo_stdlib_str_ptr(text), text.len())) +} + +def json_parse_buffer_strict(buffer: Buffer, input_len: i64) -> Result { + json_doc_result(sengoo_json_parse_buffer_strict(buffer.handle, input_len)) +} + def json_doc_object() -> Result { json_doc_result(sengoo_json_doc_object_new()) } @@ -136,6 +159,10 @@ def json_last_error_code() -> i64 { sengoo_json_last_error_code() } +def json_last_error_kind() -> i64 { + sengoo_json_last_error_kind() +} + def json_last_error_offset() -> i64 { sengoo_json_last_error_offset() } @@ -173,6 +200,13 @@ impl JsonDoc { json_value_result(self.handle, sengoo_json_doc_new_string(self.handle, sengoo_stdlib_str_ptr(value))) } + def new_string_from_string(self, value: &String) -> Result { + json_value_result( + self.handle, + sengoo_json_doc_new_string_from_string(self.handle, value.handle) + ) + } + def new_bool(self, value: bool) -> Result { json_value_result(self.handle, sengoo_json_doc_new_bool(self.handle, if value { 1 } else { 0 })) } @@ -208,7 +242,15 @@ impl JsonValue { } def object_has(self, key: &str) -> bool { - self.object_has_raw(sengoo_stdlib_str_ptr(key)) + sengoo_json_object_has_len(self.doc_handle, self.node_id, sengoo_stdlib_str_ptr(key), key.len()) != 0 + } + + def object_len(self) -> Result { + json_count_result(sengoo_json_object_len(self.doc_handle, self.node_id)) + } + + def object_key_copy(self, index: i64, buffer: Buffer) -> Result { + json_count_result(sengoo_json_object_key_copy(self.doc_handle, self.node_id, index, buffer.handle)) } def object_get_raw(self, key_ptr: i64) -> Result { @@ -216,7 +258,10 @@ impl JsonValue { } def object_get(self, key: &str) -> Result { - self.object_get_raw(sengoo_stdlib_str_ptr(key)) + json_value_result( + self.doc_handle, + sengoo_json_object_get_len(self.doc_handle, self.node_id, sengoo_stdlib_str_ptr(key), key.len()) + ) } def object_set_raw(self, key_ptr: i64, value: JsonValue) -> Result { diff --git a/tools/stdlib/runtime.c b/tools/stdlib/runtime.c index 1f9a7343..9ab6073c 100644 --- a/tools/stdlib/runtime.c +++ b/tools/stdlib/runtime.c @@ -1,5 +1,17 @@ #define _CRT_SECURE_NO_WARNINGS +/* When compiling as ISO C11 (e.g. clang -std=c11), glibc hides POSIX APIs such + * as lstat unless a feature-test macro is set. Keep this before any system + * headers so native runtime probes and sgc-linked programs both see lstat. */ +#if !defined(_WIN32) +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE 1 +#endif +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 700 +#endif +#endif + #include #include #include @@ -24,6 +36,7 @@ extern long long sengoo_string_as_str_ptr(long long handle); #ifdef _WIN32 #include +#include #include #include #else @@ -942,7 +955,8 @@ static int sengoo_buffer_slot_ensure_capacity(size_t min_slots) { static long long sengoo_buffer_alloc_handle(SengooFfiBuffer* buffer) { size_t index = 0; for (; index < g_buffer_slot_count; ++index) { - if (!g_buffer_slots[index].alive) { + if (!g_buffer_slots[index].alive && + sengoo_runtime_next_handle_generation(g_buffer_slots[index].generation) != 0) { break; } } @@ -954,13 +968,15 @@ static long long sengoo_buffer_alloc_handle(SengooFfiBuffer* buffer) { } SengooBufferSlot* slot = &g_buffer_slots[index]; + uint32_t generation = sengoo_runtime_next_handle_generation(slot->generation); + long long handle = sengoo_runtime_encode_handle(generation, index); + if (handle == 0) { + return 0; + } slot->buffer = buffer; slot->alive = 1; - slot->generation += 1; - if (slot->generation == 0) { - slot->generation = 1; - } - return ((long long)slot->generation << 32) | (long long)(index + 1); + slot->generation = generation; + return handle; } static int sengoo_buffer_decode_handle(long long handle, size_t* out_index, uint32_t* out_generation) { @@ -1085,6 +1101,103 @@ long long sengoo_ffi_buffer_used_len(long long buffer_handle) { return (long long)buffer->used_len; } +long long sengoo_ffi_buffer_commit_used_len(long long buffer_handle, long long used_len) { + SengooFfiBuffer* buffer = sengoo_ffi_buffer_from_handle(buffer_handle); + if (!buffer) { + return -SENGOO_STATUS_INVALID_HANDLE; + } + if (used_len < 0 || (unsigned long long)used_len > (unsigned long long)buffer->capacity) { + return -SENGOO_STATUS_INVALID_ARGUMENT; + } + buffer->used_len = (size_t)used_len; + return 1; +} + +long long sengoo_ffi_buffer_get_u8(long long buffer_handle, long long index) { + SengooFfiBuffer* buffer = sengoo_ffi_buffer_from_handle(buffer_handle); + if (!buffer) { + return -SENGOO_STATUS_INVALID_HANDLE; + } + if (index < 0 || (unsigned long long)index >= (unsigned long long)buffer->used_len) { + return -SENGOO_STATUS_INVALID_ARGUMENT; + } + return (long long)buffer->bytes[(size_t)index]; +} + +long long sengoo_ffi_buffer_set_u8(long long buffer_handle, long long index, long long value) { + SengooFfiBuffer* buffer = sengoo_ffi_buffer_from_handle(buffer_handle); + if (!buffer) { + return -SENGOO_STATUS_INVALID_HANDLE; + } + if (index < 0 || value < 0 || value > 255 || + (unsigned long long)index >= (unsigned long long)buffer->capacity) { + return -SENGOO_STATUS_INVALID_ARGUMENT; + } + size_t byte_index = (size_t)index; + if (byte_index > buffer->used_len) { + memset(buffer->bytes + buffer->used_len, 0, byte_index - buffer->used_len); + } + buffer->bytes[byte_index] = (unsigned char)value; + if (buffer->used_len <= byte_index) { + buffer->used_len = byte_index + 1; + } + return 1; +} + +long long sengoo_ffi_buffer_read_u32_be(long long buffer_handle, long long offset) { + SengooFfiBuffer* buffer = sengoo_ffi_buffer_from_handle(buffer_handle); + if (!buffer) { + return -SENGOO_STATUS_INVALID_HANDLE; + } + if (offset < 0) { + return -SENGOO_STATUS_INVALID_ARGUMENT; + } + if (offset > LLONG_MAX - 4) { + return -SENGOO_STATUS_OVERFLOW; + } + if ((unsigned long long)offset > (unsigned long long)buffer->used_len || + buffer->used_len - (size_t)offset < 4) { + return -SENGOO_STATUS_INVALID_ARGUMENT; + } + const unsigned char* bytes = buffer->bytes + (size_t)offset; + uint32_t value = ((uint32_t)bytes[0] << 24) | + ((uint32_t)bytes[1] << 16) | + ((uint32_t)bytes[2] << 8) | + (uint32_t)bytes[3]; + return (long long)value; +} + +long long sengoo_ffi_buffer_write_u32_be(long long buffer_handle, long long offset, long long value) { + SengooFfiBuffer* buffer = sengoo_ffi_buffer_from_handle(buffer_handle); + if (!buffer) { + return -SENGOO_STATUS_INVALID_HANDLE; + } + if (offset < 0 || value < 0) { + return -SENGOO_STATUS_INVALID_ARGUMENT; + } + if (offset > LLONG_MAX - 4 || (unsigned long long)value > UINT32_MAX) { + return -SENGOO_STATUS_OVERFLOW; + } + if ((unsigned long long)offset > (unsigned long long)buffer->capacity || + buffer->capacity - (size_t)offset < 4) { + return -SENGOO_STATUS_INVALID_ARGUMENT; + } + if ((size_t)offset > buffer->used_len) { + memset(buffer->bytes + buffer->used_len, 0, (size_t)offset - buffer->used_len); + } + unsigned char* bytes = buffer->bytes + (size_t)offset; + uint32_t encoded = (uint32_t)value; + bytes[0] = (unsigned char)(encoded >> 24); + bytes[1] = (unsigned char)(encoded >> 16); + bytes[2] = (unsigned char)(encoded >> 8); + bytes[3] = (unsigned char)encoded; + size_t end = (size_t)offset + 4; + if (buffer->used_len < end) { + buffer->used_len = end; + } + return 1; +} + long long sengoo_ffi_buffer_ptr(long long buffer_handle) { sengoo_ffi_clear_error_state(); SengooFfiBuffer* buffer = sengoo_ffi_buffer_from_handle(buffer_handle); @@ -3230,6 +3343,198 @@ long long sengoo_io_stdin_read(long long out_buffer, long long out_capacity) { return (long long)read; } +long long sengoo_io_protocol_binary_mode(void) { +#ifdef _WIN32 + if (_setmode(_fileno(stdin), _O_BINARY) == -1 || + _setmode(_fileno(stdout), _O_BINARY) == -1) { + return -SENGOO_STATUS_IO; + } +#endif + return 0; +} + +static int sengoo_io_buffer_range( + SengooFfiBuffer* buffer, + long long offset, + long long len, + int require_initialized, + size_t* out_offset, + size_t* out_len) { + if (!buffer || offset < 0 || len < 0) { + return 0; + } + if ((unsigned long long)offset > (unsigned long long)buffer->capacity) { + return 0; + } + size_t start = (size_t)offset; + size_t count = (size_t)len; + size_t limit = require_initialized ? buffer->used_len : buffer->capacity; + if (start > limit || count > limit - start) { + return 0; + } + *out_offset = start; + *out_len = count; + return 1; +} + +long long sengoo_runtime_read_exact( + SengooRuntimeReadFn read_fn, + void* context, + unsigned char* destination, + size_t expected) { + if (!read_fn || (expected > 0 && !destination)) { + return -SENGOO_STATUS_INVALID_ARGUMENT; + } + if (expected == 0) { + return 0; + } + if (expected > (size_t)LLONG_MAX) { + return -SENGOO_STATUS_OVERFLOW; + } + + unsigned char* pending = (unsigned char*)malloc(expected); + if (!pending) { + return -SENGOO_STATUS_OUT_OF_MEMORY; + } + + size_t total = 0; + while (total < expected) { + size_t remaining = expected - total; + SengooRuntimeReadResult result = read_fn( + context, + pending + total, + remaining); + if (result.error || result.count > remaining) { + free(pending); + return -SENGOO_STATUS_IO; + } + if (result.count > 0) { + total += result.count; + if (total == expected) { + break; + } + if (result.eof) { + free(pending); + return -SENGOO_STATUS_IO; + } + continue; + } + if (result.eof) { + free(pending); + return total == 0 ? 0 : -SENGOO_STATUS_IO; + } + free(pending); + return -SENGOO_STATUS_IO; + } + + memcpy(destination, pending, expected); + free(pending); + return (long long)expected; +} + +static SengooRuntimeReadResult sengoo_io_stdin_read_some( + void* context, + unsigned char* destination, + size_t capacity) { + FILE* stream = (FILE*)context; + SengooRuntimeReadResult result; + result.count = fread(destination, 1, capacity, stream); + result.eof = feof(stream) ? 1 : 0; + result.error = ferror(stream) ? 1 : 0; + return result; +} + +long long sengoo_io_stdin_read_exact(long long buffer_handle, long long offset, long long len) { + SengooFfiBuffer* buffer = sengoo_ffi_buffer_from_handle(buffer_handle); + if (!buffer) { + return -SENGOO_STATUS_INVALID_HANDLE; + } + size_t start = 0; + size_t expected = 0; + if (!sengoo_io_buffer_range(buffer, offset, len, 0, &start, &expected)) { + return -SENGOO_STATUS_INVALID_ARGUMENT; + } + if (expected == 0) { + return 0; + } + long long status = sengoo_runtime_read_exact( + sengoo_io_stdin_read_some, + stdin, + buffer->bytes + start, + expected); + if (status <= 0) { + return status; + } + if (start > buffer->used_len) { + memset(buffer->bytes + buffer->used_len, 0, start - buffer->used_len); + } + if (buffer->used_len < start + expected) { + buffer->used_len = start + expected; + } + return (long long)expected; +} + +long long sengoo_runtime_write_all( + SengooRuntimeWriteFn write_fn, + void* context, + const unsigned char* source, + size_t expected) { + if (!write_fn || (expected > 0 && !source)) { + return -SENGOO_STATUS_INVALID_ARGUMENT; + } + if (expected == 0) { + return 0; + } + if (expected > (size_t)LLONG_MAX) { + return -SENGOO_STATUS_OVERFLOW; + } + + size_t total = 0; + while (total < expected) { + size_t remaining = expected - total; + SengooRuntimeWriteResult result = write_fn( + context, + source + total, + remaining); + if (result.error || result.count == 0 || result.count > remaining) { + return -SENGOO_STATUS_IO; + } + total += result.count; + } + return (long long)total; +} + +static SengooRuntimeWriteResult sengoo_io_stdout_write_some( + void* context, + const unsigned char* source, + size_t capacity) { + FILE* stream = (FILE*)context; + SengooRuntimeWriteResult result; + result.count = fwrite(source, 1, capacity, stream); + result.error = ferror(stream) ? 1 : 0; + return result; +} + +long long sengoo_io_stdout_write_all(long long buffer_handle, long long offset, long long len) { + SengooFfiBuffer* buffer = sengoo_ffi_buffer_from_handle(buffer_handle); + if (!buffer) { + return -SENGOO_STATUS_INVALID_HANDLE; + } + size_t start = 0; + size_t expected = 0; + if (!sengoo_io_buffer_range(buffer, offset, len, 1, &start, &expected)) { + return -SENGOO_STATUS_INVALID_ARGUMENT; + } + if (expected == 0) { + return 0; + } + return sengoo_runtime_write_all( + sengoo_io_stdout_write_some, + stdout, + buffer->bytes + start, + expected); +} + long long sengoo_io_stdin_read_line(long long out_buffer, long long out_capacity) { char* out = (char*)(intptr_t)out_buffer; if (out_capacity < 0 || (out_capacity > 0 && !out)) { @@ -4221,7 +4526,8 @@ long long sengoo_opaque_handle_new(void* ptr) { } size_t index = g_opaque_handle_slot_count; for (size_t i = 0; i < g_opaque_handle_slot_count; ++i) { - if (!g_opaque_handle_slots[i].alive) { + if (!g_opaque_handle_slots[i].alive && + sengoo_runtime_next_handle_generation(g_opaque_handle_slots[i].generation) != 0) { index = i; break; } @@ -4233,13 +4539,15 @@ long long sengoo_opaque_handle_new(void* ptr) { g_opaque_handle_slot_count += 1; } SengooOpaqueHandleSlot* slot = &g_opaque_handle_slots[index]; + uint32_t generation = sengoo_runtime_next_handle_generation(slot->generation); + long long handle = sengoo_runtime_encode_handle(generation, index); + if (handle == 0) { + return 0; + } slot->ptr = ptr; slot->alive = 1; - slot->generation += 1; - if (slot->generation == 0) { - slot->generation = 1; - } - return ((long long)slot->generation << 32) | (long long)(index + 1); + slot->generation = generation; + return handle; } void* sengoo_opaque_handle_get(long long handle) { diff --git a/tools/stdlib/runtime_process.c b/tools/stdlib/runtime_process.c index 6c430e3a..7986ff16 100644 --- a/tools/stdlib/runtime_process.c +++ b/tools/stdlib/runtime_process.c @@ -1159,7 +1159,8 @@ static int sengoo_process_handle_slot_ensure_capacity(size_t min_slots) { static long long sengoo_process_handle_alloc(SengooProcessHandleState* state) { size_t index = 0; for (; index < g_process_handle_slot_count; ++index) { - if (!g_process_handle_slots[index].alive) { + if (!g_process_handle_slots[index].alive && + sengoo_runtime_next_handle_generation(g_process_handle_slots[index].generation) != 0) { break; } } @@ -1170,13 +1171,15 @@ static long long sengoo_process_handle_alloc(SengooProcessHandleState* state) { g_process_handle_slot_count += 1; } SengooProcessHandleSlot* slot = &g_process_handle_slots[index]; + uint32_t generation = sengoo_runtime_next_handle_generation(slot->generation); + long long handle = sengoo_runtime_encode_handle(generation, index); + if (handle == 0) { + return -(long long)SENGOO_STATUS_OUT_OF_MEMORY; + } slot->state = state; slot->alive = 1; - slot->generation += 1; - if (slot->generation == 0) { - slot->generation = 1; - } - return ((long long)slot->generation << 32) | (long long)(index + 1); + slot->generation = generation; + return handle; } static SengooProcessHandleState* sengoo_process_handle_resolve(long long handle) { diff --git a/tools/stdlib/runtime_shared.h b/tools/stdlib/runtime_shared.h index bb10f28a..e359f1de 100644 --- a/tools/stdlib/runtime_shared.h +++ b/tools/stdlib/runtime_shared.h @@ -6,6 +6,20 @@ #define SENGOO_RUNTIME_ABI_VERSION 1 #define SENGOO_COLLECTIONS_ABI_VERSION 1 +#define SENGOO_RUNTIME_HANDLE_GENERATION_MAX UINT32_C(0x7fffffff) + +/* Keep generated handles positive and retire a slot before generation reuse. */ +static inline uint32_t sengoo_runtime_next_handle_generation(uint32_t current) { + return current < SENGOO_RUNTIME_HANDLE_GENERATION_MAX ? current + 1 : 0; +} + +static inline long long sengoo_runtime_encode_handle(uint32_t generation, size_t index) { + if (generation == 0 || generation > SENGOO_RUNTIME_HANDLE_GENERATION_MAX || index >= UINT32_MAX) { + return 0; + } + uint64_t encoded = ((uint64_t)generation << 32) | (uint64_t)(index + 1); + return (long long)encoded; +} enum { SENGOO_RUNTIME_MAX_BUFFER_BYTES = 64 * 1024 * 1024, @@ -38,12 +52,58 @@ enum { SENGOO_STATUS_INVALID_UTF8 = 20 }; +enum { + SENGOO_JSON_ERROR_KIND_NONE = 0, + SENGOO_JSON_ERROR_KIND_UNCLASSIFIED = 1, + SENGOO_JSON_ERROR_KIND_DUPLICATE_FIELD = 2, + SENGOO_JSON_ERROR_KIND_INVALID_UNICODE = 3, + SENGOO_JSON_ERROR_KIND_TRAILING_BYTES = 4 +}; + typedef struct { unsigned char* bytes; size_t capacity; size_t used_len; } SengooFfiBuffer; +/* Internal native-I/O seam. This is not part of the Sengoo stdlib surface. */ +typedef struct { + size_t count; + int eof; + int error; +} SengooRuntimeReadResult; + +typedef SengooRuntimeReadResult (*SengooRuntimeReadFn)( + void* context, + unsigned char* destination, + size_t capacity +); + +long long sengoo_runtime_read_exact( + SengooRuntimeReadFn read_fn, + void* context, + unsigned char* destination, + size_t expected +); + +typedef struct { + size_t count; + int error; +} SengooRuntimeWriteResult; + +typedef SengooRuntimeWriteResult (*SengooRuntimeWriteFn)( + void* context, + const unsigned char* source, + size_t capacity +); + +long long sengoo_runtime_write_all( + SengooRuntimeWriteFn write_fn, + void* context, + const unsigned char* source, + size_t expected +); + typedef void (*SengooMoveFn)(void* destination, void* source); typedef void (*SengooDropFn)(void* value); typedef int (*SengooCloneFn)(void* destination, const void* source); diff --git a/tools/stdlib/runtime_string.c b/tools/stdlib/runtime_string.c index 2410a50d..2106559e 100644 --- a/tools/stdlib/runtime_string.c +++ b/tools/stdlib/runtime_string.c @@ -70,7 +70,8 @@ static int sengoo_string_slot_ensure_capacity(size_t min_slots) { static long long sengoo_string_alloc_handle(SengooOwnedString* owned) { size_t index = 0; for (; index < g_string_slot_count; ++index) { - if (!g_string_slots[index].alive) { + if (!g_string_slots[index].alive && + sengoo_runtime_next_handle_generation(g_string_slots[index].generation) != 0) { break; } } @@ -82,13 +83,15 @@ static long long sengoo_string_alloc_handle(SengooOwnedString* owned) { } SengooStringSlot* slot = &g_string_slots[index]; + uint32_t generation = sengoo_runtime_next_handle_generation(slot->generation); + long long handle = sengoo_runtime_encode_handle(generation, index); + if (handle == 0) { + return -(long long)SENGOO_STATUS_OUT_OF_MEMORY; + } slot->owned = owned; slot->alive = 1; - slot->generation += 1; - if (slot->generation == 0) { - slot->generation = 1; - } - return ((long long)slot->generation << 32) | (long long)(index + 1); + slot->generation = generation; + return handle; } static int sengoo_string_decode_handle(long long handle, size_t* out_index, uint32_t* out_generation) {