diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index c3684056..9df66bd6 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -180,6 +180,7 @@ jobs: tests.test_adaptive_evidence_contracts \ tests.test_adaptive_environment_profile \ tests.test_adaptive_bundle_registry \ + tests.test_adaptive_algorithm_scout \ tests.test_adaptive_managed_output \ tests.test_adaptive_qualification \ tests.test_adaptive_recommendation \ @@ -249,6 +250,7 @@ jobs: tests.test_adaptive_vision_roadmap_generator \ tests.test_adaptive_environment_profile \ tests.test_adaptive_bundle_registry \ + tests.test_adaptive_algorithm_scout \ tests.test_adaptive_managed_output \ tests.test_adaptive_qualification \ tests.test_adaptive_recommendation \ @@ -332,6 +334,7 @@ jobs: tests/test_generated_cli_reference.py \ tests/test_ssot_capability_coverage.py \ tests/test_adaptive_vision_roadmap_generator.py \ + tests/test_adaptive_algorithm_scout.py \ tests/test_adaptive_evidence_activation.py \ tests/test_adaptive_qualification.py \ tests/test_adaptive_recommendation.py \ diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b33a0982..311fc7a8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,6 +43,7 @@ jobs: tests.test_adaptive_evidence_activation \ tests.test_adaptive_environment_profile \ tests.test_adaptive_bundle_registry \ + tests.test_adaptive_algorithm_scout \ tests.test_adaptive_managed_output \ tests.test_adaptive_recommendation \ tests.test_adaptive_processing \ @@ -69,6 +70,8 @@ jobs: required = { "yolozu/data/manifest/adaptive_vision_roadmap.json", "yolozu/data/schemas/adaptive_vision_roadmap.schema.json", + "yolozu/data/schemas/algorithm_scout_sources.schema.json", + "yolozu/data/schemas/algorithm_scout_report.schema.json", "yolozu/data/schemas/local_artifact_inventory.schema.json", "yolozu/data/schemas/qualification_report.schema.json", "yolozu/data/schemas/evidence_activation_record.schema.json", @@ -198,6 +201,7 @@ jobs: tests.test_adaptive_evidence_activation \ tests.test_adaptive_environment_profile \ tests.test_adaptive_bundle_registry \ + tests.test_adaptive_algorithm_scout \ tests.test_adaptive_managed_output \ tests.test_adaptive_recommendation \ tests.test_adaptive_processing \ @@ -222,6 +226,8 @@ jobs: required = { "yolozu/data/manifest/adaptive_vision_roadmap.json", "yolozu/data/schemas/adaptive_vision_roadmap.schema.json", + "yolozu/data/schemas/algorithm_scout_sources.schema.json", + "yolozu/data/schemas/algorithm_scout_report.schema.json", "yolozu/data/schemas/local_artifact_inventory.schema.json", "yolozu/data/schemas/qualification_report.schema.json", "yolozu/data/schemas/evidence_activation_record.schema.json", diff --git a/CHANGELOG.md b/CHANGELOG.md index a5e81f91..08d0f892 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add Experimental dry-run-by-default pinned local image processing through MCP. - Verify adaptive routing boundaries across source, candidate artifacts, and installed MCP calls. - Register the existing model zoo as non-promoted adaptive Candidate baselines. +- Add an Experimental bounded official-source algorithm scout and nonselectable candidate inbox. ### Fixed - Keep repository-wrapper prediction and TTA/TTT log paths anchored to the checkout when invoked from another working directory. diff --git a/README.md b/README.md index 2f916752..ddb5f684 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,12 @@ non-promoted Candidate baselines matching the existing model zoo: YOLOX-S, Detectron2 Faster R-CNN R50-FPN 1x, and MMDetection Faster R-CNN R50-FPN 1x. Their pinned weights are fetchable metadata, but their adaptive execution binding is explicitly unbound. Loading and validation do not import a model runtime. +Experimental `yolozu scout-algorithms` now validates one canonical official-source +allowlist and, only with `--collect`, writes a dated candidate inbox. The default +command is a network-free, write-free plan. Collected pages are untrusted metadata; +raw documents are not retained, and the inbox cannot be loaded as an +AlgorithmBundle registry or used as qualification, support, recommendation, +adoption, or promotion evidence. Explicit workspace catalogs remain operator-asserted and nonselectable. The Experimental `yolozu qualify-image-pipeline` command now implements pinned no-follow input and asset preflight, a frozen repeat/soak protocol, bounded child-process @@ -164,6 +170,8 @@ qualification evidence or proof of human adoption. The [baseline bundle registry report](reports/adaptive_baseline_bundle_registry_2026-08-26.md) records the current three-way state boundary. The earlier +[algorithm scout foundation report](reports/adaptive_algorithm_scout_foundation_2026-08-26.md) +records the monitored-source, retention, parser, and nonselection boundaries. The earlier [installed-artifact verification report](reports/adaptive_routing_installed_verification_2026-08-26.md) checks the same boundary from source, sdist, wheel, and installed MCP calls. Its positive selector and executor cases use internal fixtures; they do not qualify a diff --git a/Readme_jp.md b/Readme_jp.md index 7f662496..f75a8fb1 100644 --- a/Readme_jp.md +++ b/Readme_jp.md @@ -128,7 +128,12 @@ privacy-safe な live `environment_profile` を返します。probe failure は MMDetection Faster R-CNN R50-FPN 1xの3件を、未昇格のCandidate baselineとして 登録しています。固定済みweightは取得可能なmetadataですが、adaptive execution bindingは明示的にunboundです。model runtimeをimportせずに検証して読み込めます。明示したworkspace -catalogはoperator-assertedのままで、選択対象にはなりません。POSIX専用の +catalogはoperator-assertedのままで、選択対象にはなりません。 +Experimental `yolozu scout-algorithms` はcanonicalなofficial-source allowlistだけを +検証し、`--collect`を明示した場合だけ日付付きcandidate inboxを作ります。defaultは +network-freeかつwrite-freeのplanです。取得した内容はuntrusted metadataとして扱い、raw +documentは保持しません。このinboxをAlgorithmBundle registryとしてloadしたり、qualification、 +support、recommendation、adoption、promotionの証拠として使うことはできません。POSIX専用の Experimental `yolozu qualify-image-pipeline` commandは、pinned no-follow input/asset preflight、固定したrepeat/soak protocol、child processのbounded cancellation、 unactivatedな`qualification_report.json`のatomic publicationを実装しています。 @@ -162,6 +167,8 @@ qualification evidenceにもhuman adoptionの証明にもなりません。 [baseline bundle registry report](reports/adaptive_baseline_bundle_registry_2026-08-26.md) に現在の3段階の境界を記録しています。先行する +[algorithm scout foundation report](reports/adaptive_algorithm_scout_foundation_2026-08-26.md) +にはmonitored-source、retention、parser、nonselectionの境界を記録しています。先行する [installed-artifact verification report](reports/adaptive_routing_installed_verification_2026-08-26.md) では、source、sdist、wheel、installed MCP callで同じ境界を確認しています。positiveな selector/executor caseは内部fixtureによるもので、実bundleのqualificationやselectedな diff --git a/docs/README.md b/docs/README.md index 8a8d0ab5..0f392b8b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -56,6 +56,9 @@ The environment-qualified local image-processing program targets an Experimental unbound execution. The evidence stream and runner maps are empty, so the default response records `maturity_disallowed`, abstains, and cannot run a real adaptive model. +- Monitored-source inbox: [`algorithm_intake/README.md`](algorithm_intake/README.md). + `yolozu scout-algorithms` is network-free and write-free by default. Explicit + collection produces only a dated nonselectable candidate report. ## Primary Focus diff --git a/docs/adaptive_image_routing.md b/docs/adaptive_image_routing.md index b89fc9f6..5eec26f8 100644 --- a/docs/adaptive_image_routing.md +++ b/docs/adaptive_image_routing.md @@ -69,6 +69,16 @@ fails with an actionable error and does not create synthetic or no-op qualification evidence. It is not a selector, model adapter, or general image-processing capability. +The Experimental `yolozu scout-algorithms` surface is a separate candidate inbox. +It accepts only `docs/algorithm_intake/sources.json`. Without `--collect`, it +validates that file and prints a bounded JSON plan without network access or writes. +Collection uses credential-free HTTPS on port 443 for the exact scheme/host/path +allowlist, retains summaries and provenance rather than raw source documents, and +records unavailable fields as `unknown`. A failed or missed source remains explicit. +The report kind is not an AlgorithmBundle registry interface contract, and the +selector cannot load it. Discovery therefore cannot register, qualify, recommend, +execute, or promote a model. + `selected` means that one registered pipeline survived every hard filter, matched one active trusted qualification record for the exact measured configuration, and ranked first under the requested policy. `abstained` is a valid routing outcome when diff --git a/docs/algorithm_intake/README.md b/docs/algorithm_intake/README.md new file mode 100644 index 00000000..253ee7c8 --- /dev/null +++ b/docs/algorithm_intake/README.md @@ -0,0 +1,53 @@ +# Experimental algorithm scout + +`yolozu scout-algorithms` is a maintainer-only monitored-source inbox. It does +not search the whole web or establish which algorithm is latest or best. The +only accepted source file is `docs/algorithm_intake/sources.json`. + +The safe default validates the allowlist and prints a bounded JSON plan. It does +not open a network connection or create the output directory. + +```bash +yolozu scout-algorithms \ + --sources docs/algorithm_intake/sources.json \ + --output-dir reports/algorithm_scout \ + --collection-date 2026-08-26 \ + --trigger workflow_dispatch +``` + +`--collect` enables the only network and write path. Collection accepts exact +allowlisted HTTPS scheme/host/path records on port 443. It rejects credentials, +queries, fragments, IP literals, non-public DNS or peer addresses, and redirects +outside the same explicit allowlist. No caller headers, cookies, or tokens are +accepted. TLS 1.2 is the minimum protocol version; normal certificate-chain, +hostname, and SNI validation remain enabled. A source gets 30 seconds, the run +gets 12 minutes, and the surrounding +15-minute workflow must keep three minutes for report finalization and failure +handling. + +Fetched HTML, text, JSON, and PDF are untrusted input. The parser applies the +document, process, memory, time, and byte caps recorded in the Bead and report. +It never follows URLs found in fetched content, executes code, downloads weights, +installs packages, or copies raw pages into the repository. The dated managed +output retains bounded summaries, content provenance, explicit failed or missed +status, and `unknown` for unavailable metadata. A prior candidate absent from the +current response is retained only as `historical`, never relabeled as freshly +collected. + +Exit codes are: + +- `0`: a valid report with no enabled-source failure; +- `3`: a valid report with at least one failed or missed source; +- `2`: invalid or unsafe input rejected before collection; and +- `1`: bounded internal or finalization failure. + +Retention follows repository history policy for the dated metadata reports. Raw +source documents, headers, response bodies, local paths, credentials, and parser +temporary files are not retained. Repeated runs merge candidate history and +deduplicate by source URL plus version or revision. + +The output kind is `yolozu_algorithm_scout_report` with +`selectability=inbox_only`. It is deliberately different from the AlgorithmBundle +registry interface contract. A later reviewed screening and registry action is +required before any candidate can enter lifecycle, qualification, selection, or +execution paths. diff --git a/docs/algorithm_intake/sources.json b/docs/algorithm_intake/sources.json new file mode 100644 index 00000000..e9728cc9 --- /dev/null +++ b/docs/algorithm_intake/sources.json @@ -0,0 +1,126 @@ +{ + "kind": "yolozu_algorithm_scout_sources", + "schema_version": 1, + "sources": [ + { + "source_id": "detectron2-releases", + "enabled": true, + "location": { + "scheme": "https", + "host": "api.github.com", + "path": "/repos/facebookresearch/detectron2/releases" + }, + "redirect_allowlist": [], + "identity": { + "project": "facebookresearch/detectron2", + "model": "release inbox" + }, + "tasks": [ + "object_detection", + "instance_segmentation" + ], + "availability": { + "local": "unknown", + "hosted": "unknown" + }, + "license": { + "status": "unknown", + "expression": "unknown" + }, + "weights": { + "status": "unknown" + }, + "runtime_hints": [] + }, + { + "source_id": "mmdetection-releases", + "enabled": true, + "location": { + "scheme": "https", + "host": "api.github.com", + "path": "/repos/open-mmlab/mmdetection/releases" + }, + "redirect_allowlist": [], + "identity": { + "project": "open-mmlab/mmdetection", + "model": "release inbox" + }, + "tasks": [ + "object_detection", + "instance_segmentation" + ], + "availability": { + "local": "unknown", + "hosted": "unknown" + }, + "license": { + "status": "unknown", + "expression": "unknown" + }, + "weights": { + "status": "unknown" + }, + "runtime_hints": [] + }, + { + "source_id": "ultralytics-releases", + "enabled": true, + "location": { + "scheme": "https", + "host": "api.github.com", + "path": "/repos/ultralytics/ultralytics/releases" + }, + "redirect_allowlist": [], + "identity": { + "project": "ultralytics/ultralytics", + "model": "release inbox" + }, + "tasks": [ + "object_detection", + "instance_segmentation", + "pose_estimation" + ], + "availability": { + "local": "unknown", + "hosted": "unknown" + }, + "license": { + "status": "unknown", + "expression": "unknown" + }, + "weights": { + "status": "unknown" + }, + "runtime_hints": [] + }, + { + "source_id": "yolox-releases", + "enabled": true, + "location": { + "scheme": "https", + "host": "api.github.com", + "path": "/repos/Megvii-BaseDetection/YOLOX/releases" + }, + "redirect_allowlist": [], + "identity": { + "project": "Megvii-BaseDetection/YOLOX", + "model": "release inbox" + }, + "tasks": [ + "object_detection" + ], + "availability": { + "local": "unknown", + "hosted": "unknown" + }, + "license": { + "status": "unknown", + "expression": "unknown" + }, + "weights": { + "status": "unknown" + }, + "runtime_hints": [] + } + ] +} diff --git a/docs/generated/cli_reference.md b/docs/generated/cli_reference.md index c6b5a7e8..8acad998 100644 --- a/docs/generated/cli_reference.md +++ b/docs/generated/cli_reference.md @@ -6,10 +6,10 @@ Keep narrative docs short and link here for the full command surface. ## Top-level `yolozu --help` ```text -usage: yolozu [-h] [--version] {guide,doctor,dr,list,fetch,export,export-dataset,predict-images,eval-coco,calibrate,eval-long-tail,long-tail-recipe,benchmark,parity,predictions,validate,eval-instance-seg,onnxrt,resources,migrate,import,train,train-orchestrate,test,demo,qualify-image-pipeline,activate-qualification-evidence,registry,completion,comp} ... +usage: yolozu [-h] [--version] {guide,doctor,dr,list,fetch,export,export-dataset,predict-images,eval-coco,calibrate,eval-long-tail,long-tail-recipe,benchmark,parity,predictions,validate,eval-instance-seg,onnxrt,resources,migrate,import,train,train-orchestrate,test,demo,qualify-image-pipeline,activate-qualification-evidence,scout-algorithms,registry,completion,comp} ... positional arguments: - {guide,doctor,dr,list,fetch,export,export-dataset,predict-images,eval-coco,calibrate,eval-long-tail,long-tail-recipe,benchmark,parity,predictions,validate,eval-instance-seg,onnxrt,resources,migrate,import,train,train-orchestrate,test,demo,qualify-image-pipeline,activate-qualification-evidence,registry,completion,comp} + {guide,doctor,dr,list,fetch,export,export-dataset,predict-images,eval-coco,calibrate,eval-long-tail,long-tail-recipe,benchmark,parity,predictions,validate,eval-instance-seg,onnxrt,resources,migrate,import,train,train-orchestrate,test,demo,qualify-image-pipeline,activate-qualification-evidence,scout-algorithms,registry,completion,comp} guide Show beginner-friendly routes and copy-paste commands. doctor (dr) Check the environment. Use --explain for beginner-friendly next actions. list List registries and built-in catalogs. @@ -38,6 +38,7 @@ positional arguments: Measure one exact local image bundle (Experimental). activate-qualification-evidence Review one exact qualification report; dry-run unless --approve is set. + scout-algorithms Plan or collect a bounded monitored-source candidate inbox (Experimental). registry AI-first tool registry: list/show/validate/run tools from the canonical manifest. completion (comp) Print shell completion script (bash/zsh). @@ -284,6 +285,7 @@ Contact: develop@toppymicros.com | run_trt_pipeline | experimental | tools/run_trt_pipeline.py | Orchestrate the YOLO26 TensorRT pipeline (engine build → predictions export → parity → eval_suite → latency report). | | run_ttt_compare | research | tools/run_ttt_compare.py | Run a fail-closed seeded TTT comparison with full checkpoint preflight, real COCO evaluation, adaptation-cost counters, and a concise detector-response boilerplate for selected foreground class/box consistency. | | run_ttt_evidence_suite | research | tools/run_ttt_evidence_suite.py | Run and aggregate a fail-closed five-method clean/shifted TTT matrix for at least three seeds. | +| scout_algorithms | experimental | tools/scout_algorithms.py | Plan by default or collect a bounded read-only Experimental inbox from exact official HTTPS sources; discovered data cannot execute, download weights, mutate the bundle registry, qualify, or promote anything. | | smoke | stable | scripts/smoke.sh | Run one-command offline smoke flow (doctor -> validate dataset -> validate predictions -> eval-coco dry-run -> synthgen intake smoke) with optional deep walkthrough checks. | | smoke_synthgen | experimental | tools/smoke_synthgen.py | Run deterministic SynthGen intake smoke, or generate and qualify a fresh cross-repo handoff with strict QA and loader checks. | | summarize_bop19_pose_evidence | research | tools/summarize_bop19_pose_evidence.py | Combine official BOP19 VSD/MSSD/MSPD scores with matched rotation, translation, ADD, ADD-S, symmetry-aware pose success, and optional independent reproduction. | diff --git a/docs/generated/web_docs/commands.html b/docs/generated/web_docs/commands.html index e4bc38a9..f2d50e0f 100644 --- a/docs/generated/web_docs/commands.html +++ b/docs/generated/web_docs/commands.html @@ -41,7 +41,7 @@
Generated from tools/manifest.json

Command reference

-

135 manifest entries with declared maturity, inputs, +

136 manifest entries with declared maturity, inputs, examples, implementation paths, and documentation links. The manifest remains the source of truth.

@@ -1763,6 +1763,22 @@

Examples

Implementation · ttt_protocol.md · 15_ttt_tent_mim.tex · ttt_evidence_2026-07-27.md

+
+ scout_algorithms — Plan by default or collect a bounded read-only Experimental inbox from exact official HTTPS sources; discovered data cannot execute, download weights, mutate the bundle registry, qualify, or promote anything. +
+ experimental + tools/scout_algorithms.py +
+

Inputs

+ +

Examples

+ 
python3 tools/scout_algorithms.py --sources docs/algorithm_intake/sources.json --output-dir reports/algorithm_scout --collection-date 2026-08-26 --trigger workflow_dispatch
+python3 tools/scout_algorithms.py --sources docs/algorithm_intake/sources.json --output-dir reports/algorithm_scout --collection-date 2026-08-26 --trigger workflow_dispatch --collect
+python3 tools/scout_algorithms.py --help
+

Implementation · README.md · Readme_jp.md · README.md · adaptive_image_routing.md · README.md · algorithm_scout_report.schema.json · algorithm_scout_sources.schema.json · 04_cli_reference.tex · adaptive_algorithm_scout_foundation_2026-08-26.md

+
+
smoke — Run one-command offline smoke flow (doctor -> validate dataset -> validate predictions -> eval-coco dry-run -> synthgen intake smoke) with optional deep walkthrough checks. diff --git a/docs/generated/web_docs/provenance.json b/docs/generated/web_docs/provenance.json index 5989d0a4..e1bcc36f 100644 --- a/docs/generated/web_docs/provenance.json +++ b/docs/generated/web_docs/provenance.json @@ -3,8 +3,8 @@ "examples": 4, "failure_guides": 8, "glossary_terms": 10, - "schemas": 46, - "tools": 135 + "schemas": 48, + "tools": 136 }, "generated_files": [ "assets/docs.js", @@ -26,21 +26,22 @@ "schema_version": 1, "source_hashes": { "CONTRIBUTING.md": "d80fb82f1e97f616c5e10b15adaf189acf8ede8df631d01e77af5c6526b312f1", - "README.md": "40e999dc5777f6b52d74252a76a53e025bd6897c05788ab69d194fce32df3e96", + "README.md": "01fd4f2c5ed06a82a142991b026c215e1d5966345a6d5e6b3f56e7e0073a9216", "RELEASE.md": "b021e735a55712edc24a7f76e13c3c811a36f04e0bfd4dc7ceb0800e99e80e29", - "Readme_jp.md": "5438f6badc6a5a7fd2d2e03ceafc4c7909cea421c4590e674ac147015faae151", + "Readme_jp.md": "09a7d2abb98ca8d11c7617b736c3604a8b20a2c785f4246a4e3dd24d52e74d91", "Readme_zh.md": "c4630845f9915401468d79cc5a9b5f5c28600a6147e0a83d2351afcae162a5dd", "data/smoke/README.md": "8763fbc7545484a77b99a077cded552b03680109cec70fcd4c56bc2ed2cd4284", "data/smoke/predictions/predictions_dummy.json": "e028f2d9fb9f567527cd2b38f94ad7ed366665cec840af61bec62c736013e6fe", "data/smoke/synthgen_minishard/README.md": "af36faf0dae4d7e7950931dbd079090e844a871cd9ac9bfa2b8a2b2fa8ac3af0", "deploy/docker/README.md": "7a280c69fa7e5e54e035e94f155d0f46ff667740dc114192d9ca78c5a73677c4", "deploy/runpod/README.md": "de43d7c3c97d17c6bac13e43fd48726ed23a27892df740ff6b2d440f55e50e2c", - "docs/README.md": "88ff68c2e91ebae5389f337539c8bb90e625422508be0840da363927d6a16f49", + "docs/README.md": "b4075289310ca1f7a765b1bd3b668eb68596be289d0e2bf8093dd7633cac6ce8", "docs/adapter_contract.md": "24c180923cbba2f259c916c0b468559585f4ad6b667fa709ac5b42d632c825d8", "docs/adapter_strategy.md": "827913eb4930193db89bc7a57b6be2eeda99843e4d4f8ee857831909d8a5c83a", "docs/adapter_templates.md": "34c35ac8a115412886449937f9bd593223b67467428f19cdbb791facea3ca570", - "docs/adaptive_image_routing.md": "e3d2567c5bbe19ca5d1c1abba589157fde0528ab46a3eb6c8b739edecac85a8e", + "docs/adaptive_image_routing.md": "34ce4d0117ea81b7add85de068279e96ee97779cc01d9c134f0f798e2073a2b6", "docs/ai_first.md": "ce47dfe4c485f18e5484d720c412c0190f6cfdbe9fd26cf91794f37d3791a829", + "docs/algorithm_intake/README.md": "81a904672f04e35bc01aee4d40390c33b9e9731bfcfcd78ecf01f905c6a19b51", "docs/assets/instance_seg_coco_instances_demo.png": "c126d1fb8939c06778d071a2e9cc8abd12b07c17c304b7fd7f6327e67f385bf5", "docs/assets/ttt_method_results_source.json": "cfc5dbd684a6de9ed026617ca9fdfff1bc9f2d357916fb05daedc95f4ccef073", "docs/assets/ttt_method_results_summary.png": "750ec8a38ae871211638ab7161a8b561fa999981e55d5fd47abb5c13ac8c5553", @@ -110,12 +111,14 @@ "docs/runpod_gpu_validation_split.md": "4449e134d04ab7faa3dbf0d54f6729766aa75ec5962301c64ac0bd559bf2e26f", "docs/sar_design_spec.md": "a15382fc2cd6964b4021eb3641799ed83dc7ff591a09adff62a5280e2ffe2121", "docs/sar_evaluation.md": "8e8c340002c2fff76ca1487b9f4ed9a1bb72be2c0f95e2f510550db18d33724b", - "docs/schema_governance.md": "b9e9d29913e308851cbfa8fc48dbda7e024adf7cb422c746acaf5ce499ae8a77", + "docs/schema_governance.md": "571d807a8030068b6dbf302ab821c837ac1cbb6b36f777db409ef100d88cb51d", "docs/schemas/adaptive_vision_roadmap.schema.json": "dfa549c0955d45fb06306caf54eb4a4ada97593343f72cfa6cffd9c6103f716b", "docs/schemas/ai_generate_config.schema.json": "067bef1c1ee16894ccbf89b2c45b1e071fe5f5d80cfa7dcd631664dad21a1e27", "docs/schemas/ai_review_config.schema.json": "fa8ab6f989ab0e7e5207eae20d449bf588a1ef5a05cc170c0d28669566fad7f5", "docs/schemas/algorithm_bundle_registry.schema.json": "e5579bce048b66414326108bf4dbac7ee9573aef834cdb6f382d85eab45fd8be", "docs/schemas/algorithm_bundle_spec.schema.json": "cc5287060d9a83bc1c236defea96f5178654e936404a3fb888378b1e2e01bfbd", + "docs/schemas/algorithm_scout_report.schema.json": "c5035cb590f295b0c8b319862849b1f0b9a4640a8ed332f27ee9e6f481120661", + "docs/schemas/algorithm_scout_sources.schema.json": "69246963f0cb7cc0503e4bd624b4a48efea6bae4ce50f67f2298d1e55904807b", "docs/schemas/artifact_research_qualification.schema.json": "66b0ebcb1f36b32c90cd83ed1671eaf49dc44f1e61110eebf8d0e2f10885129a", "docs/schemas/bop19_tless_pose_qualification.schema.json": "d5606fbf95a4f58ccf1d447e9769f2245f27bfc85fb0de91c22d66116794ae18", "docs/schemas/bop_conversion_report.schema.json": "00e0136d07689429155556bd69fe7e15744331e52f4135e4a090e1b760dd37de", @@ -159,7 +162,7 @@ "docs/schemas/training_run_summary.schema.json": "dbd984c1cb32da9c1f42c2c8475a34a0751352db821d68bc7152d2efeac9be6f", "docs/score_calibration.md": "f2e3c4653c9d024f26443aa091d97cecbf7419c89c2633503ad15998d0868016", "docs/security_scorecard_governance.md": "02c955fe1e1706f8b6888fca3d925b810fed79c1a6d772d9873d7e7d66338e9e", - "docs/ssot_capability_coverage_audit.md": "57fb0158f7568a79969b74c30fb4259fb50b30a76042d7c9f78746a2976b9d3a", + "docs/ssot_capability_coverage_audit.md": "aac67ae87aaefc22eb49cef5342a4beadebe494b10d82e2ac66c192a200610ed", "docs/synthgen_contract.md": "f4afdf2243841c9d1acbf74ed360a6b6c630d6681749b138028e9be896b3a189", "docs/synthgen_intake.md": "2ec1a5d0e73bfa67dcf9b9d2bcbf732b59464acf6de1b01a814efa4af95ea042", "docs/synthgen_repo_integration.md": "d76af65c445a852bb6e1dabea3ec4fd5fbfadd2fcfba16057030c8e777281739", @@ -180,7 +183,7 @@ "docs/yolo_detr_support.md": "52159c7102c61a61108c6ef91e31cd4def3b8aff60175896387b1d7893d551cc", "examples/infer_rust/README.md": "f2c88091777499a3c23643a3e3b9ce45178b382fcd4300b49a19b155f3a1e744", "manual/chapters/02_installation.tex": "861d5aeaadcd20cc150dab598da92c0dadbec6485e0ab0837f84410139c5ab73", - "manual/chapters/04_cli_reference.tex": "d79fb36e6983b3ac05ff119ee9f68db44cd373207da2d360aa8a1e4e7c76de92", + "manual/chapters/04_cli_reference.tex": "398bd8522d8c526a2c5989c638688d1f714754c74a1f760e626dd3b62eb9bf49", "manual/chapters/05_workflows_eval_export.tex": "391acfc847f3c1606d38d019d1835f7db0e6ff3e9253e5159bacb6fbe0dd6d91", "manual/chapters/07_training_run_contract.tex": "7a681bd60fc67683e0590f7d28c5e2b8b6cd689e8a3266aa3531b37908f8932f", "manual/chapters/09_parity_bench_protocols.tex": "5cdd7965ddee2b9e8f308d2d63050c3ca01284c462bfb2bec2df1bc8b6652127", @@ -193,6 +196,7 @@ "manual/chapters/20_llm_mcp_integrations.tex": "d21cc39dcdc85fa996608cd97160732efce55122f04b341808508b5269e022bd", "recipes/synthgen/README.md": "d957e334a652ef46fc79a8a90c434d7240f4d316abc5a63f202af8204859ae9e", "release.sh": "20059faf4c482cc1350c8a6b4041e8c75a972fd1b2fad68cd2cace761a6b097c", + "reports/adaptive_algorithm_scout_foundation_2026-08-26.md": "c237849a2990c83b64390d7c005b9e084526b86bb13490f876ffa2afa98dac55", "reports/adaptive_vision_roadmap.md": "90624c439ad5c044c8445d176528a2a6fb988d8aae902e75bc08321ede472d32", "reports/artifact_research_evidence_2026-07-28.md": "225e40fb0fe78d516d845586220611a919530a1e337d8fc14c8ca82f65b7212b", "reports/bop19_tless_official_evidence_2026-07-30.md": "3fca9a8aec3075f0c7f9a41d5174c454529d351e949ed768dc1b907704aa4ada", @@ -221,7 +225,7 @@ "scripts/download_coco_instances_tiny.py": "722fadc65813317703dfd8ddfe39650a46c364942cd1325d9e740da688428006", "scripts/fresh_install_journey.sh": "fb155222d2f107159642836a4b5d0b8917c6836e97c87d5c89caca1652e7d2e8", "scripts/pre_pr_quality.sh": "4b96242d3015465758882235b29f39d072e8e8ba8009ea1d52764ef800e3ce74", - "scripts/pre_push.sh": "418cab9ecd2dd7846d49674532fed394f00419e8d6bb34e2107ce75c12648f8b", + "scripts/pre_push.sh": "23631b6d324ecb342363a4c5e4f3914d9e2aeaec459ca65def42fa7a6ba4d75f", "scripts/prepare_ttt_domain_shift_target.py": "d238643654edfd74d343b77eca7bf0ebbc8eb52d3cdfd20f329625ba779c08f6", "scripts/run_external_runtime_gpu_qualification.sh": "2744077ee3f012cef031eb96576311dc1d9f3835a109896c904adc79daaf682d", "scripts/smoke.sh": "64f7c9533a39ff35671093263c3f598af5b6fee45fc99bf720c6eadd9b06face", @@ -295,7 +299,7 @@ "tools/hpo_sweep.py": "969e2747753e7ffec3ee7c52692ed81a7d27f66378401ebbd8eb64840ba206e0", "tools/import_yolo_data_yaml.py": "a7e90969b9dd87fd603819ca8104de8d8b65e03eb5b53effdb50fb4da37b80f1", "tools/make_subset_dataset.py": "1d964b7f4e04944ad6b8191237861cadeee07c3a744c389e9d7e034cdff540a1", - "tools/manifest.json": "79265414cae6389954e230e661ea3e4cfd266917de051d70a51bff2976f0f992", + "tools/manifest.json": "57049058e22bd3fc22a5b028fe2a592bc5ff6d4a0aa02bfa961a281d912e2f07", "tools/measure_trt_latency.py": "7495d6e8f443b3ac03eb5e8bb4c411dd939f95d74bbe919f7a1ac6f7c06b97f0", "tools/normalize_predictions.py": "cae6eb84307d44defbddc66c5856e9c0f7981db065693a6fa396fcf75125df89", "tools/orchestrate_train.py": "5c9d7e0be705a6b0cbee0349872869140fad251bd8a4899a9c077ff30e386f1b", @@ -330,6 +334,7 @@ "tools/run_trt_pipeline.py": "c638900f0d95ff170fccdcfeddeb2145693e2d3e1c5bb8e3c7b7952584deea51", "tools/run_ttt_compare.py": "5c09476671987643698a885d461fc5b59b89ef8d83596883b35b2397d647c8c6", "tools/run_ttt_evidence_suite.py": "4fc1e7a5fa9aed5a6c2165b3937a3d421da6f7a4ac16a17db24ab0b041b95ea1", + "tools/scout_algorithms.py": "b645a36c35b1cebf892de337bdeac2b5fabc0171269e9b8f7b528498d190a6c2", "tools/smoke_synthgen.py": "7c8e47edb2b29311a6caf579cbcde40066371f219fa217ec6ba61f7e04e51af3", "tools/summarize_bop19_pose_evidence.py": "ea69ae60c19507ca720efba05c4bd9e72c038594310a6bb27c49edc14bb26d60", "tools/summarize_bop_tless_qualification.py": "6337c0d03edcf9283636a16169c93f89c7401714665f608e6eb482f9b73f8dde", diff --git a/docs/generated/web_docs/schemas.html b/docs/generated/web_docs/schemas.html index 0a4e4f04..51e1d8c1 100644 --- a/docs/generated/web_docs/schemas.html +++ b/docs/generated/web_docs/schemas.html @@ -41,7 +41,7 @@
Generated from docs/schemas/*.json

Schema browser

-

46 checked-in JSON Schemas covering predictions, +

48 checked-in JSON Schemas covering predictions, evaluation reports, training handoff, registry records, and Research artifacts. Open the source schema for complete field constraints.

@@ -122,6 +122,32 @@

YOLOZU AlgorithmBundleSpec v1

Open the complete JSON Schema

+
+

YOLOZU Experimental algorithm scout report

+

Checked-in YOLOZU JSON Schema.

+
+
Schema ID
https://www.toppymicros.com/yolozu/schemas/algorithm_scout_report.schema.json
+
Root properties
automation_caveat, candidates, collection_date, completed_at, kind, maturity, retention, schema_version, selectability, source_policy, sources, started_at, summary, timezone, trigger
+
Root required
automation_caveat, candidates, collection_date, completed_at, kind, maturity, retention, schema_version, selectability, source_policy, sources, started_at, summary, timezone, trigger
+
+

Open the complete JSON Schema

+
+ +
+

YOLOZU algorithm scout source allowlist

+

Checked-in YOLOZU JSON Schema.

+
+
Schema ID
https://www.toppymicros.com/yolozu/schemas/algorithm_scout_sources.schema.json
+
Root properties
kind, schema_version, sources
+
Root required
kind, schema_version, sources
+
+

Open the complete JSON Schema

+
+
diff --git a/docs/generated/web_docs/search-index.json b/docs/generated/web_docs/search-index.json index 08a5f8f0..7f2199ad 100644 --- a/docs/generated/web_docs/search-index.json +++ b/docs/generated/web_docs/search-index.json @@ -853,6 +853,13 @@ "summary": "Run and aggregate a fail-closed five-method clean/shifted TTT matrix for at least three seeds.", "title": "run_ttt_evidence_suite" }, + { + "href": "commands.html#tool-scout-algorithms", + "kind": "command", + "search_text": "scout_algorithms Plan by default or collect a bounded read-only Experimental inbox from exact official HTTPS sources; discovered data cannot execute, download weights, mutate the bundle registry, qualify, or promote anything. experimental tools/scout_algorithms.py --sources --output-dir --collection-date --trigger --workspace --collect", + "summary": "Plan by default or collect a bounded read-only Experimental inbox from exact official HTTPS sources; discovered data cannot execute, download weights, mutate the bundle registry, qualify, or promote anything.", + "title": "scout_algorithms" + }, { "href": "commands.html#tool-smoke", "kind": "command", @@ -1028,6 +1035,20 @@ "summary": "Immutable fetchable-model, optional bound-runner, artifact, vocabulary, and execution facts for Experimental adaptive routing.", "title": "YOLOZU AlgorithmBundleSpec v1" }, + { + "href": "schemas.html#schema-algorithm-scout-report-schema", + "kind": "schema", + "search_text": "YOLOZU Experimental algorithm scout report Checked-in YOLOZU JSON Schema. automation_caveat candidates collection_date completed_at kind maturity retention schema_version selectability source_policy sources started_at summary timezone trigger docs/schemas/algorithm_scout_report.schema.json", + "summary": "Checked-in YOLOZU JSON Schema.", + "title": "YOLOZU Experimental algorithm scout report" + }, + { + "href": "schemas.html#schema-algorithm-scout-sources-schema", + "kind": "schema", + "search_text": "YOLOZU algorithm scout source allowlist Checked-in YOLOZU JSON Schema. kind schema_version sources docs/schemas/algorithm_scout_sources.schema.json", + "summary": "Checked-in YOLOZU JSON Schema.", + "title": "YOLOZU algorithm scout source allowlist" + }, { "href": "schemas.html#schema-artifact-research-qualification-schema", "kind": "schema", diff --git a/docs/schema_governance.md b/docs/schema_governance.md index bedfba1e..4409acd6 100644 --- a/docs/schema_governance.md +++ b/docs/schema_governance.md @@ -22,6 +22,10 @@ byte-identical pairs. They do not have a third copy under `schemas/`: `yolozu/data/schemas/algorithm_bundle_spec.schema.json` - `docs/schemas/algorithm_bundle_registry.schema.json` and `yolozu/data/schemas/algorithm_bundle_registry.schema.json` +- `docs/schemas/algorithm_scout_sources.schema.json` and + `yolozu/data/schemas/algorithm_scout_sources.schema.json` +- `docs/schemas/algorithm_scout_report.schema.json` and + `yolozu/data/schemas/algorithm_scout_report.schema.json` - `docs/schemas/bundle_lifecycle_record.schema.json` and `yolozu/data/schemas/bundle_lifecycle_record.schema.json` - `docs/schemas/support_profile_spec.schema.json` and diff --git a/docs/schemas/algorithm_scout_report.schema.json b/docs/schemas/algorithm_scout_report.schema.json new file mode 100644 index 00000000..38704240 --- /dev/null +++ b/docs/schemas/algorithm_scout_report.schema.json @@ -0,0 +1,427 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://www.toppymicros.com/yolozu/schemas/algorithm_scout_report.schema.json", + "title": "YOLOZU Experimental algorithm scout report", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "schema_version", + "maturity", + "selectability", + "collection_date", + "trigger", + "started_at", + "completed_at", + "timezone", + "source_policy", + "retention", + "automation_caveat", + "sources", + "candidates", + "summary" + ], + "properties": { + "kind": { + "const": "yolozu_algorithm_scout_report" + }, + "schema_version": { + "const": 1 + }, + "maturity": { + "const": "experimental" + }, + "selectability": { + "const": "inbox_only" + }, + "collection_date": { + "$ref": "#/$defs/date" + }, + "trigger": { + "enum": [ + "schedule", + "workflow_dispatch" + ] + }, + "started_at": { + "$ref": "#/$defs/utc" + }, + "completed_at": { + "$ref": "#/$defs/utc" + }, + "timezone": { + "const": "UTC" + }, + "source_policy": { + "type": "object", + "additionalProperties": false, + "required": [ + "canonical_sources", + "scope", + "coverage", + "redirect_limit", + "network" + ], + "properties": { + "canonical_sources": { + "const": "docs/algorithm_intake/sources.json" + }, + "scope": { + "const": "explicit_official_allowlist_only" + }, + "coverage": { + "const": "monitored_sources_not_latest_world" + }, + "redirect_limit": { + "const": 3 + }, + "network": { + "const": "https_443_collect_only" + } + } + }, + "retention": { + "type": "object", + "additionalProperties": false, + "required": [ + "raw_documents", + "candidate_metadata", + "history_reports_scanned_max" + ], + "properties": { + "raw_documents": { + "const": "not_retained" + }, + "candidate_metadata": { + "const": "repository_policy" + }, + "history_reports_scanned_max": { + "type": "integer", + "minimum": 1, + "maximum": 104 + } + } + }, + "automation_caveat": { + "type": "string", + "minLength": 1, + "maxLength": 512 + }, + "sources": { + "type": "array", + "minItems": 1, + "maxItems": 128, + "items": { + "$ref": "#/$defs/source_result" + } + }, + "candidates": { + "type": "array", + "maxItems": 10000, + "items": { + "$ref": "#/$defs/candidate" + } + }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": [ + "enabled_sources", + "collected_sources", + "failed_or_missed_sources", + "candidate_count", + "current_candidate_count", + "historical_candidate_count", + "decoded_total_bytes", + "network_used", + "writes_performed" + ], + "properties": { + "enabled_sources": { + "$ref": "#/$defs/nonnegative" + }, + "collected_sources": { + "$ref": "#/$defs/nonnegative" + }, + "failed_or_missed_sources": { + "$ref": "#/$defs/nonnegative" + }, + "candidate_count": { + "$ref": "#/$defs/nonnegative" + }, + "current_candidate_count": { + "$ref": "#/$defs/nonnegative" + }, + "historical_candidate_count": { + "$ref": "#/$defs/nonnegative" + }, + "decoded_total_bytes": { + "type": "integer", + "minimum": 0, + "maximum": 536870912 + }, + "network_used": { + "const": true + }, + "writes_performed": { + "const": true + } + } + } + }, + "$defs": { + "date": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + }, + "utc": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + }, + "nonnegative": { + "type": "integer", + "minimum": 0 + }, + "status": { + "enum": [ + "available", + "unavailable", + "unknown" + ] + }, + "source_result": { + "type": "object", + "required": [ + "source_id", + "source_url", + "collected_at", + "timezone", + "collection_status" + ], + "properties": { + "source_id": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "source_url": { + "type": "string", + "pattern": "^https://[^/?#:]+/[^?#]*$", + "maxLength": 4096 + }, + "collected_at": { + "$ref": "#/$defs/utc" + }, + "timezone": { + "const": "UTC" + }, + "collection_status": { + "enum": [ + "collected", + "failed", + "missed" + ] + }, + "failure_code": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "candidate_count": { + "$ref": "#/$defs/nonnegative" + }, + "provenance": { + "type": "object" + } + }, + "additionalProperties": false + }, + "candidate": { + "type": "object", + "additionalProperties": false, + "required": [ + "candidate_key", + "source_id", + "source_url", + "identity", + "version", + "revision", + "release_date", + "tasks", + "availability", + "license", + "weights", + "runtime_hints", + "collection_status", + "collected_at", + "timezone", + "first_seen_date", + "last_seen_date", + "history" + ], + "properties": { + "candidate_key": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "source_id": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "source_url": { + "type": "string", + "pattern": "^https://[^/?#:]+/[^?#]*$", + "maxLength": 4096 + }, + "identity": { + "type": "object", + "additionalProperties": false, + "required": [ + "project", + "model" + ], + "properties": { + "project": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "model": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "revision": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "release_date": { + "type": "string", + "minLength": 1, + "maxLength": 10 + }, + "tasks": { + "type": "array", + "maxItems": 32, + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64 + } + }, + "availability": { + "type": "object", + "additionalProperties": false, + "required": [ + "local", + "hosted" + ], + "properties": { + "local": { + "$ref": "#/$defs/status" + }, + "hosted": { + "$ref": "#/$defs/status" + } + } + }, + "license": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "expression" + ], + "properties": { + "status": { + "enum": [ + "approved", + "rejected", + "review_required", + "unknown" + ] + }, + "expression": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } + }, + "weights": { + "type": "object", + "additionalProperties": false, + "required": [ + "status" + ], + "properties": { + "status": { + "$ref": "#/$defs/status" + } + } + }, + "runtime_hints": { + "type": "array", + "maxItems": 32, + "items": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + }, + "collection_status": { + "enum": [ + "collected", + "historical" + ] + }, + "collected_at": { + "$ref": "#/$defs/utc" + }, + "timezone": { + "const": "UTC" + }, + "first_seen_date": { + "$ref": "#/$defs/date" + }, + "last_seen_date": { + "$ref": "#/$defs/date" + }, + "history": { + "type": "array", + "maxItems": 104, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "collection_date", + "collected_at", + "content_sha256" + ], + "properties": { + "collection_date": { + "$ref": "#/$defs/date" + }, + "collected_at": { + "$ref": "#/$defs/utc" + }, + "content_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + } + } + } + } + } +} diff --git a/docs/schemas/algorithm_scout_sources.schema.json b/docs/schemas/algorithm_scout_sources.schema.json new file mode 100644 index 00000000..33ff07fd --- /dev/null +++ b/docs/schemas/algorithm_scout_sources.schema.json @@ -0,0 +1,185 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://www.toppymicros.com/yolozu/schemas/algorithm_scout_sources.schema.json", + "title": "YOLOZU algorithm scout source allowlist", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "schema_version", + "sources" + ], + "properties": { + "kind": { + "const": "yolozu_algorithm_scout_sources" + }, + "schema_version": { + "const": 1 + }, + "sources": { + "type": "array", + "minItems": 1, + "maxItems": 128, + "items": { + "$ref": "#/$defs/source" + } + } + }, + "$defs": { + "location": { + "type": "object", + "additionalProperties": false, + "required": [ + "scheme", + "host", + "path" + ], + "properties": { + "scheme": { + "const": "https" + }, + "host": { + "type": "string", + "pattern": "^[a-zA-Z0-9.-]{1,253}$" + }, + "path": { + "type": "string", + "pattern": "^/[^?#]{0,4095}$" + } + } + }, + "status": { + "enum": [ + "available", + "unavailable", + "unknown" + ] + }, + "source": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_id", + "enabled", + "location", + "redirect_allowlist", + "identity", + "tasks", + "availability", + "license", + "weights", + "runtime_hints" + ], + "properties": { + "source_id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]{0,63}$" + }, + "enabled": { + "type": "boolean" + }, + "location": { + "$ref": "#/$defs/location" + }, + "redirect_allowlist": { + "type": "array", + "maxItems": 3, + "items": { + "$ref": "#/$defs/location" + } + }, + "identity": { + "type": "object", + "additionalProperties": false, + "required": [ + "project", + "model" + ], + "properties": { + "project": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "model": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } + }, + "tasks": { + "type": "array", + "maxItems": 32, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64 + } + }, + "availability": { + "type": "object", + "additionalProperties": false, + "required": [ + "local", + "hosted" + ], + "properties": { + "local": { + "$ref": "#/$defs/status" + }, + "hosted": { + "$ref": "#/$defs/status" + } + } + }, + "license": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "expression" + ], + "properties": { + "status": { + "enum": [ + "approved", + "rejected", + "review_required", + "unknown" + ] + }, + "expression": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } + }, + "weights": { + "type": "object", + "additionalProperties": false, + "required": [ + "status" + ], + "properties": { + "status": { + "$ref": "#/$defs/status" + } + } + }, + "runtime_hints": { + "type": "array", + "maxItems": 32, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } + } + } + } +} diff --git a/docs/ssot_capability_coverage_audit.md b/docs/ssot_capability_coverage_audit.md index c5fe0daf..867e0a1e 100644 --- a/docs/ssot_capability_coverage_audit.md +++ b/docs/ssot_capability_coverage_audit.md @@ -19,6 +19,7 @@ Adaptive deterministic selector update: 2026-08-25 Adaptive pinned processing update: 2026-08-26 Adaptive installed-artifact verification update: 2026-08-26 Adaptive baseline bundle registration update: 2026-08-26 +Adaptive monitored-source scout update: 2026-08-26 The corresponding 30-run diagnostic artifacts and checkpoints are fixed in the [2026-07-27 prerelease](https://github.com/ToppyMicroServices/YOLOZU/releases/tag/ttt-evidence-2026-07-27) @@ -39,7 +40,7 @@ instead of promoting it by inference. | Capability | Maturity | Implementation | CLI | Manifest / packaged copy | Docs | Tests / evidence | Result / follow-up | |---|---|---|---|---|---|---|---| | Predictions validation/evaluation | Stable | `yolozu/api.py`, `yolozu/predictions/`, `yolozu/eval/` | `yolozu validate`, `eval-coco`, `eval-instance-seg`, `parity` | `validate_predictions`, `eval_coco`, `eval_instance_segmentation`, and `yolozu`; source and packaged manifests/schemas match | `python_api.md`, `predictions_schema.md`, `external_inference.md` | `tests/test_public_api.py`, `tests/test_eval_cli_guardrails.py`, `tests/test_predictions.py`, `data/smoke/predictions/predictions_dummy.json` | Aligned. `eval-coco` is strict by default, explicit repair records warnings, bounded subsets count/exclude known unselected predictions, and the typed in-process API ships `py.typed`. | -| Environment-qualified adaptive local vision | Experimental MCP recommendation and pinned processing; qualification, reviewed activation, and pure-selection foundations implemented | Strict typed validators, live environment profiling, trusted registry loading, pinned input/artifact readers, the frozen qualification protocol, child-process cancellation, bounded managed publication, reviewed append-only activation transitions, a file-free selector, read-only recommendation, and dry-run-by-default pinned processing exist. `process_images` requires a complete selected decision, repeats stable current-state checks, and restricts explicit work to registered code-owned network-free host or matching isolated routes. Three model-zoo entries are registered as non-promoted Candidate bundle metadata with unbound execution. No registered adaptive model runner, model adapter, real execution, or OS-isolation claim is made. | `yolozu doctor` returns `environment_profile`; qualification emits an unactivated report; activation requires review plus `--approve`; MCP `recommend_image_pipeline` defaults to abstention; MCP `process_images` defaults to no-write dry-run and publishes only the exact managed tree after explicit execution | Candidate-only registry/lifecycle SSOT, empty support/evidence streams, schemas, roadmap, synchronized manifests, and generated 27-tool MCP reference are packaged | `doctor_diagnostics.md`, `schema_governance.md`, `adaptive_image_routing.md`, `ai_first.md`, `roadmap.md`, `production_readiness.md`, `reports/adaptive_baseline_bundle_registry_2026-08-26.md`, `reports/adaptive_qualification_foundation_2026-08-25.md`, `reports/adaptive_routing_installed_verification_2026-08-26.md`, `reports/adaptive_vision_roadmap.md` | `tests/test_adaptive_routing_e2e.py` asserts exact public abstention/rejection and fixture separation; bundle-registry tests map all three records to the model zoo; candidate-artifact and installed-surface tests cover package inventory, MCP discovery, and calls outside the checkout; processing tests cover pinned lifecycle, bounded publication, failure cleanup, and isolation gates | Beads `YOLOZU-ll2.81` is the live planning source. Fixtures are interface tests, not performance evidence. The three records remain Candidate, the public evidence stream and runner maps remain empty, and default recommendation abstains with `maturity_disallowed`. No real bundle qualification, model support, execution, or human adoption is claimed. | +| Environment-qualified adaptive local vision | Experimental MCP recommendation and pinned processing; qualification, reviewed activation, pure-selection, and monitored-source foundations implemented | Strict typed validators, live environment profiling, trusted registry loading, pinned input/artifact readers, bounded qualification/publication, reviewed activation, pure selection, read-only recommendation, dry-run-by-default pinned processing, and a bounded official-source scout exist. Three model-zoo entries remain non-promoted Candidate metadata with unbound execution. Scout output is a separate inbox and cannot load as a bundle registry. No registered adaptive model runner, model adapter, or real execution claim is made. | `yolozu scout-algorithms` is network-free/write-free by default and collects only with `--collect`; qualification emits an unactivated report; activation requires review plus `--approve`; MCP recommendation abstains by default; MCP processing defaults to no-write dry-run | Candidate-only registry/lifecycle SSOT, empty support/evidence streams, scout source/report schemas, roadmap, synchronized manifests, and generated 27-tool MCP reference are packaged | `algorithm_intake/README.md`, `adaptive_image_routing.md`, `schema_governance.md`, `production_readiness.md`, `reports/adaptive_algorithm_scout_foundation_2026-08-26.md`, `reports/adaptive_baseline_bundle_registry_2026-08-26.md`, `reports/adaptive_routing_installed_verification_2026-08-26.md`, `reports/adaptive_vision_roadmap.md` | `tests/test_adaptive_algorithm_scout.py` covers offline network/parser/resource bounds, history integrity, exit codes, and nonselection; routing/installed tests retain the existing abstention and package boundaries | Beads `YOLOZU-ll2.81` is the live planning source. Scout and routing fixtures are interface tests, not performance evidence. The three records remain Candidate, public evidence and runner maps remain empty, and default recommendation abstains with `maturity_disallowed`. No real bundle qualification, support, execution, or human adoption is claimed. | | Dataset I/O and mask-only label derivation | Explicitly deferred as standalone capabilities | `yolozu/datasets/dataset.py`, `dataset_contract.py`, `dataset_validator.py`, `tools/make_subset_dataset.py`, `rtdetr_pose/rtdetr_pose/train_dataset.py` | `yolozu validate dataset`, `doctor train-dataset`, `export-dataset`, `import`, sidecar-safe subset helper | Covered by stable `yolozu` and dataset preparation entries; no separate mask-derivation command | `yolozu_spec.md`, `dataset_contract.md`, `dataset_processing_matrix.md`, `training_inference_export.md`, `reports/dataset_preflight_2026-07-27.md`, `reports/dataset_roundtrip_2026-07-27.md` | Dataset/doctor/export/subset tests; `data/smoke/`, `data/coco128/`, `data/conversion_tiny_coco/`, `data/real_multitask_fewshot/` | Empty splits fail closed; doctor and validator share strict checks; installed-wheel COCO round trips preserve counts/classes and bbox geometry within the recorded no-clipping tolerance. Subsets retain mask/depth/keypoint/object-pose sidecars and variable-resolution aux arrays collate after task-appropriate resizing. Real keypoint/semantic upstream qualification and fixture license review remain explicitly unqualified; standalone maturity remains deferred. | | RT-DETR pose reference trainer | Stable reference lane | `rtdetr_pose/rtdetr_pose/`, `rtdetr_pose/tools/train_minimal.py` | `yolozu train` | Stable `yolozu` entry; packaged copy matches | `training_backend_interface.md`, `training_capability_matrix.md`, `run_contract.md` | `tests/test_rtdetr_pose_adapter.py`, `rtdetr_pose/tests/test_train_minimal_integration.py` | Aligned. Runtime/GPU qualification remains environment-specific. | | Reference backbone and neck boundary | Stable within the reference trainer | `rtdetr_pose/rtdetr_pose/backbone_interface.py`, `models/backbones/` | `yolozu train --config ...` | Covered by stable `yolozu`; no independent public command | `yolozu_spec.md`, `training_backend_interface.md` | `tests/test_backbone_shapes.py`, `tests/test_rtdetr_backbone_neck_parity.py` | Aligned at the adapter boundary; no repository-wide model-family claim. | @@ -53,17 +54,17 @@ instead of promoting it by inference. | TTA and TTT | TTA is Experimental; TTT is Research | `yolozu/tta/`, `yolozu/response_selection.py`, `tools/export_predictions.py`, `tools/run_ttt_evidence_suite.py` | Opt-in `--tta` has default postprocess and `rtdetr_pose` model-branch modes; `--method detector_response` is the concise selected-foreground compare; opt-in `--ttt` updates parameters or explicitly abstains below the minimum selection count | `export_predictions` is stable at entrypoint level; optional TTA/TTT features retain narrower maturity | `tta_support_matrix.md`, `training_inference_export.md`, `ttt_protocol.md`, `research_lanes.md` | Independently reproduced zero-delta matrix plus a one-checkpoint 10-image detection-native diagnostic with non-zero clean/shifted deltas; abstention tests assert zero backward, optimizer steps, and parameter drift | Detection-native class/box consistency produced a bounded positive observation with no guard stops. Abstention is structurally verified, but neither efficacy nor an optimal threshold is established; maturity remains Research. | | Hessian refinement | Research | `yolozu/calibration/hessian_solver.py`, `tools/refine_predictions_hessian.py` | `refine_predictions_hessian`, `qualify_artifact_research` | Research manifest entries and schemas; packaged manifest matches | `hessian_solver.md`, `research_lanes.md`, `reports/artifact_research_evidence_2026-07-28.md` | `tests/test_hessian_solver.py`, `tests/test_refine_predictions_hessian_cli.py`, `tests/test_qualify_artifact_research_cli.py`; three deterministic COCO128 repetitions | Wrapped output now satisfies the predictions interface contract and reports measured latency/hashes. All 1,280 detections per repetition were `no_signal`, metrics were unchanged, and promotion remains `hold`. | | Searchable web onboarding | Stable generated documentation over per-capability maturity labels | `tools/generate_web_docs.py`, `docs/web_docs_content.json` | Self-contained strict CLI journey plus stable typed Python example | Stable `generate_web_docs` entry; source and packaged manifests match | `web_docs_plan.md`, generated `web_docs/start.html`, `python_api.md` | `tests/test_web_docs_generation.py`, `tests/test_web_docs_candidate_wheel.py`; adversarial path/URL/output checks plus candidate wheel outside checkout through the installed console script | Aligned. Sources are repository-confined, replacement requires owned provenance, all referenced SSOT files are hashed, and CI fails unless the canonical path completes real COCOeval. The dependency-free dry run remains an explicitly non-metric fallback outside that gate. | -| Installed CLI and mixed-lane entrypoints | Mixed; maturity is per entrypoint, with narrower sub-lane rules | `yolozu/cli.py`, `cli_entry.py`, `cli_commands.py` | 30 canonical commands/aliases in current top-level help | 135 entries: 60 stable, 54 experimental, 21 research; source and packaged copies match | `generated/cli_reference.md`, `tools_index.md`, `manifest_declarative_spec.md` | Per-entrypoint help/manifest audit and manual audit are required quality gates | Stable parent maturity is explicitly non-transitive; generated reference and manifest descriptions repeat that boundary. | +| Installed CLI and mixed-lane entrypoints | Mixed; maturity is per entrypoint, with narrower sub-lane rules | `yolozu/cli.py`, `cli_entry.py`, `cli_commands.py` | 31 canonical commands/aliases in current top-level help | 136 entries: 60 stable, 55 experimental, 21 research; source and packaged copies match | `generated/cli_reference.md`, `tools_index.md`, `manifest_declarative_spec.md` | Per-entrypoint help/manifest audit and manual audit are required quality gates | Stable parent maturity is explicitly non-transitive; generated reference and manifest descriptions repeat that boundary. | ## Confirmed checks - `tools/manifest.json` and `yolozu/data/manifest/tools_manifest.json` are byte-identical. -- Strict manifest validation passes for all 135 entries. +- Strict manifest validation passes for all 136 entries. - Per-entrypoint help audit scans the current declared Python tool set with zero execution errors and zero missing flags. -- Manual CLI drift audit passes for the current 28-command/alias top-level surface. +- Manual CLI drift audit passes for the current 31-command/alias top-level surface. - Public docs example audit passes 114 shell examples. - The generated benchmark support matrix is current for 7 formats, 7 tasks, and 49 rows. -- The generated web-docs bundle is current for 135 tools and 46 JSON Schemas. +- The generated web-docs bundle is current for 136 tools and 48 JSON Schemas. - Public PyPI `yolozu==4.5.1` completed the fresh-install stable lane in all 10 Linux/macOS jobs for Python 3.10 through 3.14 in [workflow run 29421807474](https://github.com/ToppyMicroServices/YOLOZU/actions/runs/29421807474). diff --git a/manual/chapters/04_cli_reference.tex b/manual/chapters/04_cli_reference.tex index 104a5beb..c62c6a58 100644 --- a/manual/chapters/04_cli_reference.tex +++ b/manual/chapters/04_cli_reference.tex @@ -60,6 +60,7 @@ \section{Common Commands} \cmd{yolozu benchmark} & Compare benchmark/parity lanes under pinned settings. & \cmd{yolozu benchmark --help} & benchmark report JSON & No \\ \cmd{yolozu qualify-image-pipeline} & Measure an exact registered Experimental image bundle. & \cmd{yolozu qualify-image-pipeline --help} & unactivated managed qualification report & No \\ \cmd{yolozu activate-qualification-evidence} & Review one exact qualification report; dry-run by default. & \cmd{yolozu activate-qualification-evidence --help} & gate report or approved append-only activation event & No \\ +\cmd{yolozu scout-algorithms} & Plan or collect an Experimental official-source candidate inbox. & \cmd{yolozu scout-algorithms --help} & no-write plan or dated nonselectable report & Only with \cmd{--collect} \\ \cmd{yolozu parity} & Compare two predictions artifacts for drift. & \cmd{yolozu parity --help} & parity JSON & No \\ \cmd{yolozu train} & Run config-driven training or external training wrappers. & \cmd{yolozu train --help} & run bundle under \path{runs/} & No \\ \cmd{yolozu train-orchestrate} & Plan or execute a small multi-backend training batch. & \cmd{yolozu train-orchestrate --help} & \path{training_orchestration_report.json} & No \\ diff --git a/reports/adaptive_algorithm_scout_foundation_2026-08-26.md b/reports/adaptive_algorithm_scout_foundation_2026-08-26.md new file mode 100644 index 00000000..617970bd --- /dev/null +++ b/reports/adaptive_algorithm_scout_foundation_2026-08-26.md @@ -0,0 +1,72 @@ +# Adaptive algorithm scout foundation + +Recorded: 2026-08-26 (Asia/Tokyo) + +Bead: `YOLOZU-ll2.81.3.1` + +## Result + +The repository now has an Experimental read-only algorithm scout. Its default +mode validates the canonical official-source allowlist and prints a JSON plan +without network access or writes. `--collect` is the only path that can fetch or +publish a dated managed report. + +The canonical allowlist monitors four public release endpoints: Detectron2, +MMDetection, Ultralytics, and YOLOX. The endpoints were read back through the +official GitHub API on 2026-08-26. This is a small monitored set, not evidence of +exhaustive coverage of current vision research. + +A live smoke collection through `SafeHttpsTransport` ran from +2026-08-25T17:45:10Z to 2026-08-25T17:45:12Z (2026-08-26 02:45 JST). All four +sources completed, producing 74 deduplicated release records from 595,016 decoded +bytes. The managed report and checksum were written only to a temporary workspace +and were not committed. This proves endpoint and transport interoperability at +that time; it does not qualify any release or make it selectable. + +## Trust and network boundary + +`SafeHttpsTransport` accepts structured scheme/host/path records only. It rejects +credentials, queries, fragments, IP literals, non-443 destinations, non-public +IPv4/IPv6 DNS results, changed peer addresses, and redirects not separately +allowlisted. The socket connects to a vetted resolved address while TLS chain, +hostname, and SNI verification continue to use the original host. TLS 1.2 is the +minimum protocol version. Callers cannot provide headers, cookies, or tokens. + +The implementation caps connect, read, per-source, and whole-collection time. It +also caps headers, transferred bytes, decoded bytes per source, and decoded bytes +for the run. Deadline exhaustion stops new fetches, records the remaining sources +as missed, finalizes a valid report, and returns exit 3. + +## Untrusted parser boundary + +Only HTML, UTF-8 text, JSON, and PDF are accepted. Archive/media magic, +content-type confusion, active or embedded HTML, DTD/entity declarations, and PDF +embedded files fail closed. HTML and JSON are bounded before an unbounded document +tree is built. PDF inspection runs in a fresh secret-free POSIX child process +group with network disabled, wall/CPU/address-space/PID/file/IPC limits, bounded +handoff, process-tree termination, reaping, and exact private temporary-directory +cleanup. + +The report stores source identity, version/revision or `unknown`, release date, +task scope, local/hosted availability, license status, weight status, runtime +hints, collection status/timezone, bounded parser facts, and content SHA-256. It +does not retain raw pages, release bodies, papers, archives, weights, or embedded +instructions. + +## Selection boundary + +The output kind is `yolozu_algorithm_scout_report` and has +`selectability=inbox_only`. It has no `bundles` field and is rejected by the +AlgorithmBundle registry validator. Discovery cannot register, qualify, +recommend, execute, or promote a candidate. Those state changes remain separate +reviewed Beads work. + +## Verification scope + +Offline fixtures cover plan/no-write behavior, CLI help and exit codes, URL and +address rejection, DNS/peer and redirect checks, header/body/decompression caps, +content confusion, HTML/JSON/PDF bounds, explicit failure and deadline reports, +history deduplication with prior-only candidates marked `historical`, output symlink rejection before network access, managed +checksums, and the nonselection boundary. These fixtures verify the interface and +failure policy. They are not algorithm, runtime, performance, quality, support, +or adoption evidence. diff --git a/scripts/pre_push.sh b/scripts/pre_push.sh index 45079a4c..b8a0dfee 100755 --- a/scripts/pre_push.sh +++ b/scripts/pre_push.sh @@ -84,6 +84,7 @@ else tests.test_adaptive_evidence_contracts \ tests.test_adaptive_environment_profile \ tests.test_adaptive_bundle_registry \ + tests.test_adaptive_algorithm_scout \ tests.test_adaptive_managed_output \ tests.test_adaptive_recommendation \ tests.test_adaptive_processing \ diff --git a/tests/test_adaptive_algorithm_scout.py b/tests/test_adaptive_algorithm_scout.py new file mode 100644 index 00000000..118df551 --- /dev/null +++ b/tests/test_adaptive_algorithm_scout.py @@ -0,0 +1,820 @@ +import gzip +import inspect +import json +import ssl +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from unittest import TestCase, main, mock + +from yolozu.adaptive.algorithm_scout import ( + AlgorithmScoutError, + DocumentParserLimits, + _apply_pdf_resource_limits, + _disable_child_process_creation, + _parse_document, + _parse_html, + _parse_pdf, + _validate_document_magic, + build_scout_plan, + collect_algorithm_candidates, +) +from yolozu.adaptive.bundles import validate_algorithm_bundle_registry +from yolozu.adaptive.safe_https import ( + FetchedDocument, + HttpsLocation, + SafeHttpsError, + SafeHttpsTransport, + TransportLimits, + _is_public_address, + _tls_dial, +) + + +REPO_ROOT = Path(__file__).resolve().parents[1] +SOURCES = "docs/algorithm_intake/sources.json" +PUBLIC_IP = "93.184.216.34" + + +def _sleep_pdf_worker(connection, body, temp_dir, limits) -> None: + del connection, body, temp_dir, limits + import os + + os.setsid() + time.sleep(5) + + +def _large_ipc_pdf_worker(connection, body, temp_dir, limits) -> None: + del body, temp_dir, limits + import os + + os.setsid() + connection.send_bytes(b"xx") + connection.close() + + +def _rss_pdf_worker(connection, body, temp_dir, limits) -> None: + del connection, body, temp_dir, limits + import os + + os.setsid() + allocation = bytearray(2 * 1024 * 1024) + allocation[0] = 1 + time.sleep(5) + + +def _cpu_pdf_worker(connection, body, temp_dir, limits) -> None: + del connection, body, temp_dir + import os + + os.setsid() + _apply_pdf_resource_limits(limits) + while True: + pass + + +def _pid_pdf_worker(connection, body, temp_dir, limits) -> None: + del body, temp_dir, limits + import os + + os.setsid() + _disable_child_process_creation() + try: + os.fork() + except PermissionError: + connection.send_bytes(b'{"code":"pdf_pid_limit","ok":false}') + connection.close() + + +def _temp_pdf_worker(connection, body, temp_dir, limits) -> None: + del body + import os + import resource + import signal + + os.setsid() + signal.signal(signal.SIGXFSZ, signal.SIG_IGN) + resource.setrlimit(resource.RLIMIT_FSIZE, (limits.pdf_temp_bytes, limits.pdf_temp_bytes)) + try: + with open(Path(temp_dir) / "overflow", "wb") as output: + output.write(b"xx") + output.flush() + except OSError: + connection.send_bytes(b'{"code":"pdf_temp_limit","ok":false}') + connection.close() + + +class _FakeSocket: + def __init__(self, response: bytes, *, peer: str = PUBLIC_IP) -> None: + self.response = bytearray(response) + self.peer = peer + self.sent = b"" + self.closed = False + self.timeouts: list[float] = [] + + def sendall(self, data: bytes) -> None: + self.sent += data + + def recv(self, size: int) -> bytes: + if not self.response: + return b"" + result = bytes(self.response[:size]) + del self.response[:size] + return result + + def settimeout(self, value: float) -> None: + self.timeouts.append(value) + + def getpeername(self) -> tuple[str, int]: + return self.peer, 443 + + def close(self) -> None: + self.closed = True + + +def _response( + body: bytes, + *, + status: int = 200, + content_type: str = "application/json", + extra: bytes = b"", +) -> bytes: + return ( + f"HTTP/1.1 {status} Test\r\n" + f"Content-Type: {content_type}\r\n" + f"Content-Length: {len(body)}\r\n" + ).encode("ascii") + extra + b"\r\n" + body + + +class TestSafeHttpsTransport(TestCase): + def _transport( + self, + responses: list[bytes], + *, + allowlist: tuple[HttpsLocation, ...] | None = None, + peer: str = PUBLIC_IP, + limits: TransportLimits | None = None, + ) -> tuple[SafeHttpsTransport, list[tuple[str, str]], list[_FakeSocket]]: + location = HttpsLocation(host="example.com", path="/release") + calls: list[tuple[str, str]] = [] + sockets: list[_FakeSocket] = [] + + def dialer(ip: str, host: str, connect: float, read: float) -> _FakeSocket: + self.assertGreater(connect, 0) + self.assertGreater(read, 0) + calls.append((ip, host)) + sock = _FakeSocket(responses[len(sockets)], peer=peer) + sockets.append(sock) + return sock + + return ( + SafeHttpsTransport( + allowlist=allowlist or (location,), + limits=limits, + resolver=lambda host: (PUBLIC_IP,), + dialer=dialer, + ), + calls, + sockets, + ) + + def test_fetch_connects_to_vetted_ip_with_original_tls_host(self) -> None: + location = HttpsLocation(host="example.com", path="/release") + transport, calls, sockets = self._transport([_response(b'{"ok":true}')]) + result = transport.fetch(location) + self.assertEqual(result.body, b'{"ok":true}') + self.assertEqual(calls, [(PUBLIC_IP, "example.com")]) + self.assertIn(b"Host: example.com\r\n", sockets[0].sent) + self.assertNotIn(b"Cookie:", sockets[0].sent) + self.assertNotIn(b"Authorization:", sockets[0].sent) + self.assertNotIn("headers", inspect.signature(transport.fetch).parameters) + + def test_system_dialer_requires_tls_1_2_or_newer(self) -> None: + raw = _FakeSocket(b"") + + class _Context: + minimum_version: ssl.TLSVersion | None = None + server_hostname: str | None = None + + def wrap_socket(self, sock: _FakeSocket, *, server_hostname: str) -> _FakeSocket: + self.server_hostname = server_hostname + return sock + + context = _Context() + with ( + mock.patch( + "yolozu.adaptive.safe_https.socket.create_connection", + return_value=raw, + ), + mock.patch( + "yolozu.adaptive.safe_https.ssl.create_default_context", + return_value=context, + ), + ): + wrapped = _tls_dial(PUBLIC_IP, "example.com", 5, 15) + self.assertIs(wrapped, raw) + self.assertEqual(context.minimum_version, ssl.TLSVersion.TLSv1_2) + self.assertEqual(context.server_hostname, "example.com") + self.assertEqual(raw.timeouts, [15]) + + def test_redirect_revalidates_exact_allowlist_and_detects_loop(self) -> None: + first = HttpsLocation(host="example.com", path="/release") + second = HttpsLocation(host="example.org", path="/final") + redirect = _response( + b"", + status=302, + content_type="text/plain", + extra=b"Location: https://example.org/final\r\n", + ) + transport, calls, _ = self._transport( + [redirect, _response(b"done", content_type="text/plain")], + allowlist=(first, second), + ) + transport._resolver = lambda host: (PUBLIC_IP,) + result = transport.fetch(first) + self.assertEqual(result.final_location, second) + self.assertEqual(result.redirect_count, 1) + self.assertEqual([host for _, host in calls], ["example.com", "example.org"]) + + loop_response = _response( + b"", + status=302, + content_type="text/plain", + extra=b"Location: https://example.com/release\r\n", + ) + looping, _, _ = self._transport([loop_response], allowlist=(first,)) + with self.assertRaisesRegex(SafeHttpsError, "redirect_loop"): + looping.fetch(first) + + def test_redirect_rejects_query_credentials_port_and_off_allowlist(self) -> None: + base = HttpsLocation(host="example.com", path="/release") + cases = ( + "https://example.com/final?token=secret", + "https://user:secret@example.com/final", + "https://example.com:444/final", + "https://127.0.0.1/final", + ) + for raw in cases: + with self.subTest(raw=raw): + with self.assertRaises(SafeHttpsError) as raised: + HttpsLocation.from_redirect(raw, base=base) + self.assertNotIn("secret", str(raised.exception)) + self.assertNotIn("token", str(raised.exception)) + off = _response( + b"", + status=302, + content_type="text/plain", + extra=b"Location: https://example.org/final\r\n", + ) + transport, _, _ = self._transport([off], allowlist=(base,)) + with self.assertRaisesRegex(SafeHttpsError, "redirect_not_allowlisted"): + transport.fetch(base) + with self.assertRaisesRegex(SafeHttpsError, "url_path_invalid"): + HttpsLocation.from_mapping( + {"scheme": "https", "host": "example.com", "path": "/release?token=secret"} + ) + + def test_all_non_public_address_classes_and_peer_change_fail(self) -> None: + denied = ( + "0.0.0.0", + "10.0.0.1", + "100.64.0.1", + "127.0.0.1", + "169.254.1.1", + "192.0.2.1", + "198.18.0.1", + "198.51.100.1", + "203.0.113.1", + "224.0.0.1", + "240.0.0.1", + "::", + "::1", + "fe80::1", + "fc00::1", + "ff00::1", + "2001:db8::1", + ) + for value in denied: + with self.subTest(value=value): + self.assertFalse(_is_public_address(value)) + location = HttpsLocation(host="example.com", path="/release") + transport = SafeHttpsTransport( + allowlist=(location,), resolver=lambda host: ("127.0.0.1",) + ) + with self.assertRaisesRegex(SafeHttpsError, "address_not_public"): + transport.fetch(location) + changed, _, _ = self._transport( + [_response(b"ok", content_type="text/plain")], peer="1.1.1.1" + ) + with self.assertRaisesRegex(SafeHttpsError, "peer_changed"): + changed.fetch(location) + + def test_header_transfer_and_decompression_one_over_limits(self) -> None: + location = HttpsLocation(host="example.com", path="/release") + limits = TransportLimits( + max_header_bytes=1024, + max_transferred_bytes=1024, + max_decoded_bytes=1024, + ) + huge_header = b"HTTP/1.1 200 OK\r\nX-Test: " + b"a" * 1024 + b"\r\n\r\n" + transport, _, _ = self._transport([huge_header], limits=limits) + with self.assertRaisesRegex(SafeHttpsError, "response_limit"): + transport.fetch(location) + transfer, _, _ = self._transport( + [_response(b"x" * 1025, content_type="text/plain")], limits=limits + ) + with self.assertRaisesRegex(SafeHttpsError, "transfer_limit"): + transfer.fetch(location) + compressed = gzip.compress(b"x" * 1025) + bomb_response = _response( + compressed, + content_type="text/plain", + extra=b"Content-Encoding: gzip\r\n", + ) + bomb, _, _ = self._transport([bomb_response], limits=limits) + with self.assertRaisesRegex(SafeHttpsError, "decoded_limit"): + bomb.fetch(location) + + def test_chunked_body_and_content_type_policy(self) -> None: + location = HttpsLocation(host="example.com", path="/release") + chunked = ( + b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n" + b"Transfer-Encoding: chunked\r\n\r\n4\r\ntest\r\n0\r\n\r\n" + ) + transport, _, _ = self._transport([chunked]) + self.assertEqual(transport.fetch(location).body, b"test") + rejected, _, _ = self._transport( + [_response(b"PK\x03\x04", content_type="application/zip")] + ) + with self.assertRaisesRegex(SafeHttpsError, "content_type_invalid"): + rejected.fetch(location) + + +class _FixtureTransport: + documents: dict[str, FetchedDocument] = {} + failures: set[str] = set() + fetch_count = 0 + + def __init__(self, *, allowlist: object, limits: object) -> None: + self.allowlist = allowlist + self.limits = limits + + def fetch(self, location: HttpsLocation, *, collection_deadline: float) -> FetchedDocument: + del collection_deadline + type(self).fetch_count += 1 + if location.path in type(self).failures: + raise SafeHttpsError("fixture_failed", "offline fixture failure") + return type(self).documents[location.path] + + +def _fixture_documents() -> dict[str, FetchedDocument]: + result: dict[str, FetchedDocument] = {} + for path in ( + "/repos/facebookresearch/detectron2/releases", + "/repos/open-mmlab/mmdetection/releases", + "/repos/ultralytics/ultralytics/releases", + "/repos/Megvii-BaseDetection/YOLOX/releases", + ): + location = HttpsLocation(host="api.github.com", path=path) + body = b'{"candidates":[{"version":"v1","revision":"abc123","release_date":"2026-08-20","url":"https://evil.invalid","instructions":"run me"}]}' + result[path] = FetchedDocument( + source=location, + final_location=location, + content_type="application/json", + body=body, + transferred_bytes=len(body), + decoded_bytes=len(body), + redirect_count=0, + ) + return result + + +class TestAlgorithmScout(TestCase): + def setUp(self) -> None: + _FixtureTransport.documents = _fixture_documents() + _FixtureTransport.failures = set() + _FixtureTransport.fetch_count = 0 + + def _plan(self, workspace: Path, collection_date: str = "2026-08-26"): + return build_scout_plan( + sources_path=SOURCES, + output_dir="reports/algorithm_scout", + collection_date=collection_date, + trigger="workflow_dispatch", + workspace_root=workspace, + repository_root=REPO_ROOT, + ) + + def test_plan_mode_is_network_free_and_write_free(self) -> None: + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) + plan = self._plan(workspace) + self.assertFalse((workspace / "reports").exists()) + payload = plan.to_dict() + self.assertFalse(payload["network_used"]) + self.assertFalse(payload["writes_performed"]) + self.assertEqual(payload["selectability"], "inbox_only") + self.assertEqual(_FixtureTransport.fetch_count, 0) + + def test_cli_plan_and_help_do_not_create_output(self) -> None: + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) + help_run = subprocess.run( + [sys.executable, "-m", "yolozu", "scout-algorithms", "--help"], + cwd=REPO_ROOT, + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(help_run.returncode, 0, help_run.stderr) + self.assertIn("--collect", help_run.stdout) + plan_run = subprocess.run( + [ + sys.executable, + "-m", + "yolozu", + "scout-algorithms", + "--sources", + SOURCES, + "--output-dir", + "reports/algorithm_scout", + "--collection-date", + "2026-08-26", + "--trigger", + "workflow_dispatch", + "--workspace", + str(workspace), + ], + cwd=REPO_ROOT, + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(plan_run.returncode, 0, plan_run.stderr) + self.assertFalse((workspace / "reports").exists()) + self.assertFalse(json.loads(plan_run.stdout)["network_used"]) + + def test_invalid_inputs_exit_two_without_leaking_token_value(self) -> None: + run = subprocess.run( + [ + sys.executable, + "-m", + "yolozu", + "scout-algorithms", + "--sources", + "https://example.invalid/releases?token=secret", + "--output-dir", + "../outside", + "--collection-date", + "2026-99-99", + "--trigger", + "workflow_dispatch", + ], + cwd=REPO_ROOT, + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(run.returncode, 2) + self.assertNotIn("secret", run.stderr) + self.assertNotIn("token", run.stderr) + + def test_collect_writes_one_dated_report_and_deduplicates_history(self) -> None: + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) + plan = self._plan(workspace) + first, code, path = collect_algorithm_candidates( + plan, + workspace_root=workspace, + transport_factory=_FixtureTransport, + now_utc=lambda: "2026-08-26T01:02:03Z", + monotonic=lambda: 0.0, + ) + self.assertEqual(code, 0) + self.assertTrue(path.is_file()) + self.assertEqual(len(first["candidates"]), 4) + self.assertNotIn("evil.invalid", path.read_text(encoding="utf-8")) + self.assertNotIn("run me", path.read_text(encoding="utf-8")) + detectron_path = "/repos/facebookresearch/detectron2/releases" + prior_document = _FixtureTransport.documents[detectron_path] + replacement = b'{"candidates":[{"version":"v2","release_date":"2026-08-26"}]}' + _FixtureTransport.documents[detectron_path] = FetchedDocument( + source=prior_document.source, + final_location=prior_document.final_location, + content_type="application/json", + body=replacement, + transferred_bytes=len(replacement), + decoded_bytes=len(replacement), + redirect_count=0, + ) + second, second_code, second_path = collect_algorithm_candidates( + plan, + workspace_root=workspace, + transport_factory=_FixtureTransport, + now_utc=lambda: "2026-08-26T02:03:04Z", + monotonic=lambda: 0.0, + ) + self.assertEqual(second_code, 0) + self.assertEqual(second_path, path) + self.assertEqual(len(second["candidates"]), 5) + self.assertEqual(second["summary"]["current_candidate_count"], 4) + self.assertEqual(second["summary"]["historical_candidate_count"], 1) + historical = [ + item for item in second["candidates"] if item["collection_status"] == "historical" + ] + self.assertEqual(historical[0]["version"], "v1") + self.assertTrue((path.parent / "checksums.json").is_file()) + + def test_failed_and_deadline_missed_sources_finalize_report_with_exit_three(self) -> None: + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) + plan = self._plan(workspace) + _FixtureTransport.failures = {"/repos/open-mmlab/mmdetection/releases"} + report, code, path = collect_algorithm_candidates( + plan, + workspace_root=workspace, + transport_factory=_FixtureTransport, + now_utc=lambda: "2026-08-26T01:02:03Z", + monotonic=lambda: 0.0, + ) + self.assertEqual(code, 3) + self.assertTrue(path.is_file()) + failed = [item for item in report["sources"] if item["collection_status"] == "failed"] + self.assertEqual(failed[0]["failure_code"], "fixture_failed") + + ticks = iter([0.0, 721.0, 721.0, 721.0, 721.0]) + deadline_report, deadline_code, _ = collect_algorithm_candidates( + self._plan(workspace, "2026-08-27"), + workspace_root=workspace, + transport_factory=_FixtureTransport, + now_utc=lambda: "2026-08-27T01:02:03Z", + monotonic=lambda: next(ticks), + ) + self.assertEqual(deadline_code, 3) + self.assertTrue(all(item["collection_status"] == "missed" for item in deadline_report["sources"])) + + def test_scout_report_is_not_an_algorithm_bundle_registry(self) -> None: + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) + report, _, _ = collect_algorithm_candidates( + self._plan(workspace), + workspace_root=workspace, + transport_factory=_FixtureTransport, + now_utc=lambda: "2026-08-26T01:02:03Z", + monotonic=lambda: 0.0, + ) + with self.assertRaises(ValueError): + validate_algorithm_bundle_registry(report) + self.assertEqual(report["selectability"], "inbox_only") + self.assertNotIn("bundles", report) + + def test_output_symlink_is_rejected_before_network(self) -> None: + with tempfile.TemporaryDirectory() as directory, tempfile.TemporaryDirectory() as external: + workspace = Path(directory) + (workspace / "reports").symlink_to(Path(external), target_is_directory=True) + with self.assertRaisesRegex(AlgorithmScoutError, "output_dir_invalid"): + collect_algorithm_candidates( + self._plan(workspace), + workspace_root=workspace, + transport_factory=_FixtureTransport, + now_utc=lambda: "2026-08-26T01:02:03Z", + monotonic=lambda: 0.0, + ) + self.assertEqual(_FixtureTransport.fetch_count, 0) + + def test_tampered_prior_report_is_rejected_before_network(self) -> None: + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) + plan = self._plan(workspace) + _, _, path = collect_algorithm_candidates( + plan, + workspace_root=workspace, + transport_factory=_FixtureTransport, + now_utc=lambda: "2026-08-26T01:02:03Z", + monotonic=lambda: 0.0, + ) + payload = json.loads(path.read_text(encoding="utf-8")) + payload["candidates"][0]["identity"]["model"] = "tampered" + path.write_text(json.dumps(payload), encoding="utf-8") + _FixtureTransport.fetch_count = 0 + with self.assertRaisesRegex(AlgorithmScoutError, "prior_report_invalid"): + collect_algorithm_candidates( + plan, + workspace_root=workspace, + transport_factory=_FixtureTransport, + now_utc=lambda: "2026-08-26T02:03:04Z", + monotonic=lambda: 0.0, + ) + self.assertEqual(_FixtureTransport.fetch_count, 0) + + +class TestBoundedScoutParsers(TestCase): + def _document(self, body: bytes, content_type: str) -> FetchedDocument: + location = HttpsLocation(host="example.com", path="/document") + return FetchedDocument( + source=location, + final_location=location, + content_type=content_type, + body=body, + transferred_bytes=len(body), + decoded_bytes=len(body), + redirect_count=0, + ) + + def test_html_each_limit_and_one_over_fails_incrementally(self) -> None: + exact = _parse_html( + b"x", + DocumentParserLimits( + html_nodes=2, + html_tokens=3, + html_depth=1, + html_retained_text_bytes=1, + ), + ) + self.assertEqual(exact["html_nodes"], 2) + self.assertEqual(exact["html_text_bytes"], 1) + cases = ( + (DocumentParserLimits(html_nodes=1), b"

", "html_node_limit"), + (DocumentParserLimits(html_tokens=2), b"x", "html_token_limit"), + (DocumentParserLimits(html_depth=1), b"

x

", "html_depth_limit"), + (DocumentParserLimits(html_retained_text_bytes=1), b"xx", "html_text_limit"), + ) + for limits, body, code in cases: + with self.subTest(code=code): + with self.assertRaisesRegex(AlgorithmScoutError, code): + _parse_html(body, limits) + with self.assertRaisesRegex(AlgorithmScoutError, "html_active_content"): + _parse_html(b"", DocumentParserLimits()) + with self.assertRaisesRegex(AlgorithmScoutError, "html_declaration_forbidden"): + _parse_html(b"", DocumentParserLimits()) + + def test_json_depth_and_node_one_over_use_bounded_parser(self) -> None: + location = HttpsLocation(host="example.com", path="/release") + source = build_scout_plan( + sources_path=SOURCES, + output_dir="reports/scout", + collection_date="2026-08-26", + trigger="schedule", + workspace_root=REPO_ROOT, + repository_root=REPO_ROOT, + ).enabled_sources[0] + deep = b"[" * 65 + b"0" + b"]" * 65 + document = FetchedDocument(location, location, "application/json", deep, len(deep), len(deep), 0) + with self.assertRaisesRegex(ValueError, "depth limit"): + _parse_document( + source, + document, + collected_at="2026-08-26T00:00:00Z", + collection_date="2026-08-26", + limits=DocumentParserLimits(), + ) + exact_depth = b"[" * 64 + b"0" + b"]" * 64 + exact_document = FetchedDocument( + location, + location, + "application/json", + exact_depth, + len(exact_depth), + len(exact_depth), + 0, + ) + _parse_document( + source, + exact_document, + collected_at="2026-08-26T00:00:00Z", + collection_date="2026-08-26", + limits=DocumentParserLimits(), + ) + exact_nodes = b"[" + b",".join([b"0"] * 100_000) + b"]" + exact_document = FetchedDocument( + location, + location, + "application/json", + exact_nodes, + len(exact_nodes), + len(exact_nodes), + 0, + ) + _parse_document( + source, + exact_document, + collected_at="2026-08-26T00:00:00Z", + collection_date="2026-08-26", + limits=DocumentParserLimits(), + ) + many = b"[" + b",".join([b"0"] * 100_001) + b"]" + document = FetchedDocument(location, location, "application/json", many, len(many), len(many), 0) + with self.assertRaisesRegex(ValueError, "node limit"): + _parse_document( + source, + document, + collected_at="2026-08-26T00:00:00Z", + collection_date="2026-08-26", + limits=DocumentParserLimits(), + ) + + def test_content_confusion_archives_media_and_binary_text_fail(self) -> None: + cases = ( + (b"PK\x03\x04data", "text/plain", "archive_forbidden"), + (b"\x89PNG\r\n\x1a\n", "text/plain", "media_forbidden"), + (b"not json", "application/json", "content_magic_mismatch"), + (b"not pdf", "application/pdf", "content_magic_mismatch"), + (b"\x00binary", "text/plain", "content_magic_mismatch"), + ) + for body, content_type, code in cases: + with self.subTest(code=code): + with self.assertRaisesRegex(AlgorithmScoutError, code): + _validate_document_magic(self._document(body, content_type)) + + def test_pdf_page_character_embedded_and_wall_limits_cleanup(self) -> None: + exact = _parse_pdf( + b"%PDF-1.4 /Type /Page (abc) Tj %%EOF", + DocumentParserLimits(pdf_pages=1, pdf_characters=3), + ) + self.assertEqual(exact["pdf_pages"], 1) + self.assertEqual(exact["pdf_extracted_characters"], 3) + with self.assertRaisesRegex(AlgorithmScoutError, "pdf_page_limit"): + _parse_pdf( + b"%PDF-1.4 /Type /Page /Type /Page %%EOF", + DocumentParserLimits(pdf_pages=1), + ) + with self.assertRaisesRegex(AlgorithmScoutError, "pdf_character_limit"): + _parse_pdf( + b"%PDF-1.4 (abcd) Tj %%EOF", + DocumentParserLimits(pdf_characters=3), + ) + with self.assertRaisesRegex(AlgorithmScoutError, "pdf_embedded_file"): + _parse_pdf(b"%PDF-1.4 /EmbeddedFiles %%EOF", DocumentParserLimits()) + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(AlgorithmScoutError, "pdf_wall_limit"): + _parse_pdf( + b"%PDF-1.4", + DocumentParserLimits(pdf_wall_seconds=1), + _worker=_sleep_pdf_worker, + _temp_parent=directory, + ) + self.assertEqual(list(Path(directory).iterdir()), []) + with self.assertRaisesRegex(AlgorithmScoutError, "pdf_ipc_limit"): + _parse_pdf( + b"%PDF-1.4", + DocumentParserLimits(pdf_ipc_bytes=1), + _worker=_large_ipc_pdf_worker, + _temp_parent=directory, + ) + self.assertEqual(list(Path(directory).iterdir()), []) + with self.assertRaisesRegex(AlgorithmScoutError, "pdf_rss_limit"): + _parse_pdf( + b"%PDF-1.4", + DocumentParserLimits(pdf_rss_bytes=1), + _worker=_rss_pdf_worker, + _temp_parent=directory, + ) + self.assertEqual(list(Path(directory).iterdir()), []) + with self.assertRaisesRegex(AlgorithmScoutError, "pdf_cpu_limit"): + _parse_pdf( + b"%PDF-1.4", + DocumentParserLimits(pdf_cpu_seconds=1), + _worker=_cpu_pdf_worker, + _temp_parent=directory, + ) + self.assertEqual(list(Path(directory).iterdir()), []) + with self.assertRaisesRegex(AlgorithmScoutError, "pdf_pid_limit"): + _parse_pdf( + b"%PDF-1.4", + DocumentParserLimits(), + _worker=_pid_pdf_worker, + _temp_parent=directory, + ) + self.assertEqual(list(Path(directory).iterdir()), []) + with self.assertRaisesRegex(AlgorithmScoutError, "pdf_temp_limit"): + _parse_pdf( + b"%PDF-1.4", + DocumentParserLimits(pdf_temp_bytes=1), + _worker=_temp_pdf_worker, + _temp_parent=directory, + ) + self.assertEqual(list(Path(directory).iterdir()), []) + + def test_pdf_resource_caps_cannot_be_raised_by_caller(self) -> None: + defaults = DocumentParserLimits() + for field in ( + "pdf_wall_seconds", + "pdf_cpu_seconds", + "pdf_rss_bytes", + "pdf_pids", + "pdf_temp_bytes", + "pdf_ipc_bytes", + ): + with self.subTest(field=field): + values = {field: getattr(defaults, field) + 1} + with self.assertRaises(ValueError): + DocumentParserLimits(**values) + with mock.patch("resource.setrlimit") as setter: + _apply_pdf_resource_limits(defaults) + self.assertEqual(setter.call_count, 4) + + +if __name__ == "__main__": + main() diff --git a/tests/test_candidate_artifact_ai_surface.py b/tests/test_candidate_artifact_ai_surface.py index b28e847b..08d80bbe 100644 --- a/tests/test_candidate_artifact_ai_surface.py +++ b/tests/test_candidate_artifact_ai_surface.py @@ -102,6 +102,8 @@ def test_git_archive_to_clean_venv_outside_checkout(self) -> None: for required in ( "yolozu/adaptive/isolation.py", "yolozu/adaptive/processing.py", + "yolozu/adaptive/algorithm_scout.py", + "yolozu/adaptive/safe_https.py", "yolozu/data/adaptive_routing/bundle_specs.json", "yolozu/data/adaptive_routing/bundle_lifecycle.jsonl", "yolozu/data/adaptive_routing/support_profiles.jsonl", @@ -110,6 +112,8 @@ def test_git_archive_to_clean_venv_outside_checkout(self) -> None: "yolozu/data/schemas/qualification_workload_profile.schema.json", "yolozu/data/schemas/environment_profile.schema.json", "yolozu/data/schemas/selection_decision.schema.json", + "yolozu/data/schemas/algorithm_scout_sources.schema.json", + "yolozu/data/schemas/algorithm_scout_report.schema.json", "yolozu/data/integrations/mcp_actions_tool_reference.json", ): self.assertIn(required, sdist_names) @@ -209,6 +213,8 @@ def test_git_archive_to_clean_venv_outside_checkout(self) -> None: "mcp_reference": data.joinpath("integrations").joinpath("mcp_actions_tool_reference.json").is_file(), "adaptive_processing": package.joinpath("adaptive").joinpath("processing.py").is_file(), "adaptive_isolation": package.joinpath("adaptive").joinpath("isolation.py").is_file(), + "adaptive_algorithm_scout": package.joinpath("adaptive").joinpath("algorithm_scout.py").is_file(), + "adaptive_safe_https": package.joinpath("adaptive").joinpath("safe_https.py").is_file(), "adaptive_registry": data.joinpath("adaptive_routing").joinpath("bundle_specs.json").is_file(), "adaptive_lifecycle": data.joinpath("adaptive_routing").joinpath("bundle_lifecycle.jsonl").is_file(), "adaptive_support": data.joinpath("adaptive_routing").joinpath("support_profiles.jsonl").is_file(), @@ -217,6 +223,8 @@ def test_git_archive_to_clean_venv_outside_checkout(self) -> None: "adaptive_workload_schema": data.joinpath("schemas").joinpath("qualification_workload_profile.schema.json").is_file(), "adaptive_environment_schema": data.joinpath("schemas").joinpath("environment_profile.schema.json").is_file(), "adaptive_selection_schema": data.joinpath("schemas").joinpath("selection_decision.schema.json").is_file(), + "adaptive_scout_sources_schema": data.joinpath("schemas").joinpath("algorithm_scout_sources.schema.json").is_file(), + "adaptive_scout_report_schema": data.joinpath("schemas").joinpath("algorithm_scout_report.schema.json").is_file(), "py_typed": package.joinpath("py.typed").is_file(), "numpy_available": importlib.util.find_spec("numpy") is not None, }, sort_keys=True)) @@ -259,6 +267,7 @@ def test_git_archive_to_clean_venv_outside_checkout(self) -> None: ) for key in ( "adaptive_environment_schema", + "adaptive_algorithm_scout", "adaptive_evidence", "adaptive_isolation", "adaptive_job_schema", @@ -266,6 +275,9 @@ def test_git_archive_to_clean_venv_outside_checkout(self) -> None: "adaptive_processing", "adaptive_registry", "adaptive_selection_schema", + "adaptive_scout_report_schema", + "adaptive_scout_sources_schema", + "adaptive_safe_https", "adaptive_support", "adaptive_workload_schema", "manifest", @@ -282,6 +294,17 @@ def test_git_archive_to_clean_venv_outside_checkout(self) -> None: yolozu_entry = venv_dir / executable_dir / ( "yolozu.exe" if os.name == "nt" else "yolozu" ) + scout_help = self._run( + [str(yolozu_entry), "scout-algorithms", "--help"], + cwd=consumer, + env=clean_env, + ) + self.assertEqual( + scout_help.returncode, + 0, + msg=f"installed scout help failed:\n{scout_help.stdout}\n{scout_help.stderr}", + ) + self.assertIn("--collect", scout_help.stdout) strict_validation = self._run( [ str(yolozu_entry), diff --git a/tests/test_schema_governance.py b/tests/test_schema_governance.py index b1483f92..9ddf41df 100644 --- a/tests/test_schema_governance.py +++ b/tests/test_schema_governance.py @@ -26,6 +26,8 @@ def test_adaptive_routing_schema_copies_and_browser_are_current(self): "environment_profile_json": "environment_profile.schema.json", "algorithm_bundle_spec_json": "algorithm_bundle_spec.schema.json", "algorithm_bundle_registry_json": "algorithm_bundle_registry.schema.json", + "algorithm_scout_report_json": "algorithm_scout_report.schema.json", + "algorithm_scout_sources_json": "algorithm_scout_sources.schema.json", "bundle_lifecycle_record_json": "bundle_lifecycle_record.schema.json", "support_profile_spec_json": "support_profile_spec.schema.json", "support_profile_record_json": "support_profile_record.schema.json", @@ -112,6 +114,8 @@ def test_adaptive_evidence_contracts_are_required_by_ci_and_release(self): self.assertIn(suite, build) self.assertIn(suite, pre_push) for resource in ( + "yolozu/data/schemas/algorithm_scout_sources.schema.json", + "yolozu/data/schemas/algorithm_scout_report.schema.json", "yolozu/data/schemas/local_artifact_inventory.schema.json", "yolozu/data/schemas/qualification_report.schema.json", "yolozu/data/schemas/evidence_activation_record.schema.json", diff --git a/tools/manifest.json b/tools/manifest.json index fe85a07d..1b8b7121 100644 --- a/tools/manifest.json +++ b/tools/manifest.json @@ -88,6 +88,15 @@ "schema": "docs/schemas/algorithm_bundle_spec.schema.json", "summary": "Immutable pipeline, artifact, runtime, vocabulary, and execution facts; presence is not qualification or availability evidence." }, + "algorithm_scout_report_json": { + "producer": "python3 tools/scout_algorithms.py --sources docs/algorithm_intake/sources.json --output-dir reports/algorithm_scout --collection-date --trigger --collect", + "schema": "docs/schemas/algorithm_scout_report.schema.json", + "summary": "Dated Experimental monitored-source inbox metadata; this interface contract is not loadable as a bundle registry and is not qualification, support, recommendation, adoption, or promotion evidence." + }, + "algorithm_scout_sources_json": { + "schema": "docs/schemas/algorithm_scout_sources.schema.json", + "summary": "Canonical credential-free HTTPS scheme/host/path allowlist for the bounded algorithm scout." + }, "artifact_research_qualification_json": { "producer": "python3 tools/qualify_artifact_research.py", "schema": "docs/schemas/artifact_research_qualification.schema.json", @@ -17305,6 +17314,127 @@ "reproducibility" ] }, + { + "contracts": { + "consumes": [ + "algorithm_scout_sources_json" + ], + "produces": [ + "algorithm_scout_report_json", + "managed_output_checksums_json" + ] + }, + "docs": [ + "README.md", + "Readme_jp.md", + "docs/README.md", + "docs/adaptive_image_routing.md", + "docs/algorithm_intake/README.md", + "docs/schemas/algorithm_scout_report.schema.json", + "docs/schemas/algorithm_scout_sources.schema.json", + "manual/chapters/04_cli_reference.tex", + "reports/adaptive_algorithm_scout_foundation_2026-08-26.md" + ], + "effects": { + "fixed_writes": [], + "writes": [ + { + "description": "With --collect only, atomically publish one dated bounded candidate inbox and checksums manifest inside the workspace.", + "flag": "--output-dir", + "kind": "dir", + "scope": "tree" + } + ] + }, + "entrypoint": "tools/scout_algorithms.py", + "examples": [ + { + "command": "python3 tools/scout_algorithms.py --sources docs/algorithm_intake/sources.json --output-dir reports/algorithm_scout --collection-date 2026-08-26 --trigger workflow_dispatch", + "description": "Validate the canonical allowlist and print a network-free, write-free JSON plan." + }, + { + "command": "python3 tools/scout_algorithms.py --sources docs/algorithm_intake/sources.json --output-dir reports/algorithm_scout --collection-date 2026-08-26 --trigger workflow_dispatch --collect", + "description": "Collect only bounded metadata from the explicit official-source allowlist and publish one dated inbox report." + }, + { + "command": "python3 tools/scout_algorithms.py --help", + "description": "Inspect the Experimental monitored-source interface contract." + } + ], + "id": "scout_algorithms", + "inputs": [ + { + "flag": "--sources", + "kind": "file", + "name": "sources", + "required": true + }, + { + "flag": "--output-dir", + "kind": "dir", + "name": "output_dir", + "required": true + }, + { + "flag": "--collection-date", + "kind": "string", + "name": "collection_date", + "required": true + }, + { + "flag": "--trigger", + "kind": "string", + "name": "trigger", + "required": true + }, + { + "default": ".", + "flag": "--workspace", + "kind": "dir", + "name": "workspace", + "required": false + }, + { + "flag": "--collect", + "kind": "string", + "name": "collect", + "required": false + } + ], + "maturity": "experimental", + "outputs": [ + { + "default": "//algorithm_scout_report.json", + "description": "Dated candidate inbox with explicit failed/unknown fields and no retained raw source document.", + "kind": "file", + "name": "algorithm_scout_report" + }, + { + "default": "//checksums.json", + "description": "Code-owned checksum manifest for the dated report.", + "kind": "file", + "name": "checksums" + } + ], + "platform": { + "cpu_ok": true, + "gpu_required": false, + "linux_ok": true, + "macos_ok": true + }, + "requires": { + "network": true + }, + "runner": "python3", + "summary": "Plan by default or collect a bounded read-only Experimental inbox from exact official HTTPS sources; discovered data cannot execute, download weights, mutate the bundle registry, qualify, or promote anything.", + "tags": [ + "adaptive-inference", + "algorithm-scout", + "candidate", + "experimental", + "provenance" + ] + }, { "contracts": { "consumes": [ diff --git a/tools/scout_algorithms.py b/tools/scout_algorithms.py new file mode 100644 index 00000000..d79f56a7 --- /dev/null +++ b/tools/scout_algorithms.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +"""Repository wrapper for the Experimental algorithm scout CLI.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from yolozu.cli_entry import main + + +if __name__ == "__main__": + raise SystemExit(main(["scout-algorithms", *sys.argv[1:]])) diff --git a/tools/yolozu.py b/tools/yolozu.py index 257b4e66..03010b97 100644 --- a/tools/yolozu.py +++ b/tools/yolozu.py @@ -53,6 +53,7 @@ "parity", "predictions", "qualify-image-pipeline", + "scout-algorithms", "resources", "test", "train", @@ -1415,6 +1416,10 @@ def _parse_args(argv: list[str]) -> argparse.Namespace: "qualify-image-pipeline", "Delegate to yolozu package CLI qualification command.", ), + ( + "scout-algorithms", + "Delegate to the Experimental monitored-source candidate inbox command.", + ), ("validate", "Delegate to yolozu package CLI validate command."), ("onnxrt", "Delegate to yolozu package CLI onnxrt command."), ("resources", "Delegate to yolozu package CLI resources command."), diff --git a/yolozu/adaptive/algorithm_scout.py b/yolozu/adaptive/algorithm_scout.py new file mode 100644 index 00000000..0798d4b5 --- /dev/null +++ b/yolozu/adaptive/algorithm_scout.py @@ -0,0 +1,1341 @@ +"""Read-only monitored-source inbox for Experimental algorithm intake.""" + +from __future__ import annotations + +import hashlib +import errno +import json +import multiprocessing +import os +import re +import resource +import shutil +import signal +import socket +import stat +import struct +import sys +import tempfile +import time +from dataclasses import dataclass +from datetime import date, datetime, timezone +from html.parser import HTMLParser +from pathlib import Path, PurePosixPath +from typing import Any, Callable, Iterable, Mapping + +from .canonical import canonical_json_v1 +from .control_records import load_bounded_json, load_bounded_json_bytes +from .managed_output import ( + ManagedOutputLimits, + ManagedOutputTransaction, + validate_managed_output_destination, +) +from .safe_https import ( + FetchedDocument, + HttpsLocation, + SafeHttpsError, + SafeHttpsTransport, + TransportLimits, +) + +__all__ = [ + "AlgorithmScoutError", + "AlgorithmScoutSource", + "DocumentParserLimits", + "ScoutPlan", + "build_scout_plan", + "collect_algorithm_candidates", + "load_algorithm_scout_sources", +] + + +CANONICAL_SOURCES_PATH = PurePosixPath("docs/algorithm_intake/sources.json") +REPORT_KIND = "yolozu_algorithm_scout_report" +REPORT_SCHEMA_VERSION = 1 +COLLECTION_TIMEOUT_SECONDS = 12 * 60 +WORKFLOW_RESERVE_SECONDS = 3 * 60 +TOTAL_DECODED_BYTES = 512 * 1024 * 1024 +MAX_SOURCES = 128 +MAX_PRIOR_REPORTS = 104 +MAX_CANDIDATES = 10_000 +MAX_HISTORY_ITEMS = 104 +REPORT_MAX_BYTES = 4 * 1024 * 1024 +_SOURCE_ID_RE = re.compile(r"\A[a-z0-9](?:[a-z0-9._-]{0,62}[a-z0-9])?\Z") +_VERSION_RE = re.compile(r"\A[A-Za-z0-9][A-Za-z0-9._+:/-]{0,127}\Z") +_DATE_RE = re.compile(r"\A[0-9]{4}-[0-9]{2}-[0-9]{2}\Z") +_UTC_RE = re.compile(r"\A[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z\Z") +_UNKNOWN = "unknown" + + +class AlgorithmScoutError(ValueError): + """One bounded, non-sensitive scout failure.""" + + def __init__(self, code: str, detail: str) -> None: + safe = detail.encode("utf-8", "replace")[:512].decode("utf-8", "ignore") + super().__init__(f"{code}: {safe}") + self.code = code + + +def _fail(code: str, detail: str) -> AlgorithmScoutError: + return AlgorithmScoutError(code, detail) + + +def _bounded_text(value: object, *, field: str, maximum: int, unknown: bool = False) -> str: + if value is None and unknown: + return _UNKNOWN + if not isinstance(value, str): + raise _fail("source_config_invalid", f"{field} must be a string") + if not value or len(value.encode("utf-8")) > maximum or any(ord(char) < 0x20 for char in value): + raise _fail("source_config_invalid", f"{field} is empty or exceeds its bound") + return value + + +def _bounded_string_list( + value: object, + *, + field: str, + maximum_items: int, + maximum_item_bytes: int, +) -> tuple[str, ...]: + if not isinstance(value, list) or len(value) > maximum_items: + raise _fail("source_config_invalid", f"{field} must be a bounded array") + normalized = tuple( + _bounded_text(item, field=field, maximum=maximum_item_bytes) for item in value + ) + if len(set(normalized)) != len(normalized): + raise _fail("source_config_invalid", f"{field} contains duplicates") + return normalized + + +def _validate_calendar_date(value: str) -> str: + if not isinstance(value, str) or _DATE_RE.fullmatch(value) is None: + raise _fail("collection_date_invalid", "collection date must be exact YYYY-MM-DD") + try: + parsed = date.fromisoformat(value) + except ValueError as exc: + raise _fail("collection_date_invalid", "collection date is not Gregorian") from exc + if parsed.isoformat() != value: + raise _fail("collection_date_invalid", "collection date spelling is not canonical") + return value + + +def _utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _validate_utc(value: str) -> str: + if _UTC_RE.fullmatch(value) is None: + raise _fail("timestamp_invalid", "timestamp must be exact RFC3339 UTC seconds") + try: + parsed = datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ") + except ValueError as exc: + raise _fail("timestamp_invalid", "timestamp is not Gregorian UTC") from exc + if parsed.strftime("%Y-%m-%dT%H:%M:%SZ") != value: + raise _fail("timestamp_invalid", "timestamp spelling is not canonical") + return value + + +def _parse_utc(value: str) -> datetime: + _validate_utc(value) + return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace( + tzinfo=timezone.utc + ) + + +@dataclass(frozen=True) +class AlgorithmScoutSource: + source_id: str + enabled: bool + location: HttpsLocation + redirects: tuple[HttpsLocation, ...] + project_identity: str + model_identity: str + tasks: tuple[str, ...] + local_availability: str + hosted_availability: str + license_status: str + license_expression: str + weight_status: str + runtime_hints: tuple[str, ...] + + @property + def allowlist(self) -> tuple[HttpsLocation, ...]: + return (self.location, *self.redirects) + + +@dataclass(frozen=True) +class ScoutPlan: + sources_path: str + output_dir: str + collection_date: str + trigger: str + enabled_sources: tuple[AlgorithmScoutSource, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "kind": "yolozu_algorithm_scout_plan", + "schema_version": 1, + "maturity": "experimental", + "mode": "plan", + "collection_date": self.collection_date, + "trigger": self.trigger, + "sources_path": self.sources_path, + "output_dir": self.output_dir, + "enabled_source_count": len(self.enabled_sources), + "sources": [ + { + "source_id": source.source_id, + "location": source.location.to_mapping(), + "redirect_allowlist": [item.to_mapping() for item in source.redirects], + } + for source in self.enabled_sources + ], + "network_used": False, + "writes_performed": False, + "selectability": "inbox_only", + "limits": { + "collection_timeout_seconds": COLLECTION_TIMEOUT_SECONDS, + "workflow_finalization_reserve_seconds": WORKFLOW_RESERVE_SECONDS, + "decoded_total_bytes": TOTAL_DECODED_BYTES, + }, + } + + +@dataclass(frozen=True) +class DocumentParserLimits: + html_nodes: int = 200_000 + html_tokens: int = 400_000 + html_depth: int = 128 + html_retained_text_bytes: int = 4 * 1024 * 1024 + pdf_pages: int = 64 + pdf_characters: int = 1_000_000 + pdf_wall_seconds: int = 10 + pdf_cpu_seconds: int = 8 + pdf_rss_bytes: int = 512 * 1024 * 1024 + pdf_pids: int = 4 + pdf_temp_bytes: int = 64 * 1024 * 1024 + pdf_ipc_bytes: int = 8 * 1024 * 1024 + + def __post_init__(self) -> None: + maxima = DocumentParserLimits.__dataclass_fields__ + defaults = { + name: field.default for name, field in maxima.items() + } + for name, maximum in defaults.items(): + value = getattr(self, name) + if isinstance(value, bool) or not isinstance(value, int) or value <= 0 or value > maximum: + raise ValueError(f"{name} must be a positive integer no larger than {maximum}") + + +def _source_from_record(value: object) -> AlgorithmScoutSource: + required = { + "source_id", + "enabled", + "location", + "redirect_allowlist", + "identity", + "tasks", + "availability", + "license", + "weights", + "runtime_hints", + } + if not isinstance(value, dict) or set(value) != required: + raise _fail("source_config_invalid", "source record fields are invalid") + source_id = _bounded_text(value["source_id"], field="source_id", maximum=64) + if _SOURCE_ID_RE.fullmatch(source_id) is None: + raise _fail("source_config_invalid", "source_id is not canonical") + if type(value["enabled"]) is not bool: + raise _fail("source_config_invalid", "enabled must be boolean") + location = HttpsLocation.from_mapping(value["location"]) + raw_redirects = value["redirect_allowlist"] + if not isinstance(raw_redirects, list) or len(raw_redirects) > 3: + raise _fail("source_config_invalid", "redirect_allowlist exceeds three locations") + redirects = tuple(HttpsLocation.from_mapping(item) for item in raw_redirects) + if len(set(redirects)) != len(redirects) or location in redirects: + raise _fail("source_config_invalid", "redirect_allowlist contains duplicates") + identity = value["identity"] + if not isinstance(identity, dict) or set(identity) != {"project", "model"}: + raise _fail("source_config_invalid", "identity fields are invalid") + availability = value["availability"] + if not isinstance(availability, dict) or set(availability) != {"local", "hosted"}: + raise _fail("source_config_invalid", "availability fields are invalid") + license_record = value["license"] + if not isinstance(license_record, dict) or set(license_record) != {"status", "expression"}: + raise _fail("source_config_invalid", "license fields are invalid") + weights = value["weights"] + if not isinstance(weights, dict) or set(weights) != {"status"}: + raise _fail("source_config_invalid", "weight fields are invalid") + status_values = {"available", "unavailable", "unknown"} + license_values = {"approved", "rejected", "review_required", "unknown"} + local = _bounded_text(availability["local"], field="availability.local", maximum=32) + hosted = _bounded_text(availability["hosted"], field="availability.hosted", maximum=32) + license_status = _bounded_text(license_record["status"], field="license.status", maximum=32) + weight_status = _bounded_text(weights["status"], field="weights.status", maximum=32) + if local not in status_values or hosted not in status_values or weight_status not in status_values: + raise _fail("source_config_invalid", "availability/weight status is invalid") + if license_status not in license_values: + raise _fail("source_config_invalid", "license status is invalid") + return AlgorithmScoutSource( + source_id=source_id, + enabled=bool(value["enabled"]), + location=location, + redirects=redirects, + project_identity=_bounded_text(identity["project"], field="identity.project", maximum=128), + model_identity=_bounded_text(identity["model"], field="identity.model", maximum=128), + tasks=_bounded_string_list(value["tasks"], field="tasks", maximum_items=32, maximum_item_bytes=64), + local_availability=local, + hosted_availability=hosted, + license_status=license_status, + license_expression=_bounded_text( + license_record["expression"], field="license.expression", maximum=128 + ), + weight_status=weight_status, + runtime_hints=_bounded_string_list( + value["runtime_hints"], + field="runtime_hints", + maximum_items=32, + maximum_item_bytes=128, + ), + ) + + +def load_algorithm_scout_sources( + sources_path: str | Path, + *, + repository_root: str | Path, +) -> tuple[AlgorithmScoutSource, ...]: + root = Path(repository_root).resolve(strict=True) + canonical = root.joinpath(*CANONICAL_SOURCES_PATH.parts) + supplied = Path(sources_path) + if supplied.is_absolute(): + candidate = supplied + else: + candidate = root / supplied + try: + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise _fail("sources_invalid", "canonical source file is unavailable") from exc + if resolved != canonical.resolve(strict=True): + raise _fail("sources_invalid", "only the canonical source file is accepted") + relative = canonical.relative_to(root) + current = root + for component in relative.parts: + current = current / component + if current.is_symlink(): + raise _fail("sources_invalid", "canonical source path cannot contain symlinks") + payload = load_bounded_json(canonical, label="algorithm scout sources") + if not isinstance(payload, dict) or set(payload) != {"kind", "schema_version", "sources"}: + raise _fail("source_config_invalid", "canonical source document fields are invalid") + if payload["kind"] != "yolozu_algorithm_scout_sources" or payload["schema_version"] != 1: + raise _fail("source_config_invalid", "canonical source document identity is invalid") + raw_sources = payload["sources"] + if not isinstance(raw_sources, list) or not 1 <= len(raw_sources) <= MAX_SOURCES: + raise _fail("source_config_invalid", "canonical source count is invalid") + sources = tuple(_source_from_record(item) for item in raw_sources) + if len({item.source_id for item in sources}) != len(sources): + raise _fail("source_config_invalid", "source IDs must be unique") + locations = [location for source in sources for location in source.allowlist] + if len(set(locations)) != len(locations): + raise _fail("source_config_invalid", "allowlisted locations must be globally unique") + if not any(source.enabled for source in sources): + raise _fail("source_config_invalid", "at least one source must be enabled") + return sources + + +def _relative_output_dir(value: str) -> str: + if not isinstance(value, str) or not value: + raise _fail("output_dir_invalid", "output directory is required") + path = PurePosixPath(value) + if ( + path.is_absolute() + or value.endswith("/") + or "\\" in value + or len(value.encode("utf-8")) > 4096 + or any(part in {"", ".", ".."} for part in path.parts) + or any(part.startswith(".") for part in path.parts) + ): + raise _fail("output_dir_invalid", "output directory must be a visible workspace-relative path") + return "/".join(path.parts) + + +def _workspace(value: str | Path) -> Path: + candidate = Path(value) + try: + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise _fail("workspace_invalid", "workspace is unavailable") from exc + if candidate.is_symlink() or not resolved.is_dir(): + raise _fail("workspace_invalid", "workspace must be a non-symlink directory") + return resolved + + +def build_scout_plan( + *, + sources_path: str | Path, + output_dir: str, + collection_date: str, + trigger: str, + workspace_root: str | Path = ".", + repository_root: str | Path | None = None, +) -> ScoutPlan: + _workspace(workspace_root) + repo = Path(repository_root).resolve(strict=True) if repository_root is not None else Path(__file__).resolve().parents[2] + sources = load_algorithm_scout_sources(sources_path, repository_root=repo) + if trigger not in {"schedule", "workflow_dispatch"}: + raise _fail("trigger_invalid", "trigger must be schedule or workflow_dispatch") + return ScoutPlan( + sources_path=CANONICAL_SOURCES_PATH.as_posix(), + output_dir=_relative_output_dir(output_dir), + collection_date=_validate_calendar_date(collection_date), + trigger=trigger, + enabled_sources=tuple(source for source in sources if source.enabled), + ) + + +class _BoundedHtmlParser(HTMLParser): + _VOID = frozenset({"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}) + + def __init__(self, limits: DocumentParserLimits) -> None: + super().__init__(convert_charrefs=True) + self.limits = limits + self.nodes = 0 + self.tokens = 0 + self.depth = 0 + self.max_depth_observed = 0 + self.text_bytes = 0 + self.in_title = False + self.title_parts: list[str] = [] + + def _token(self) -> None: + self.tokens += 1 + if self.tokens > self.limits.html_tokens: + raise _fail("html_token_limit", "HTML token count exceeds its cap") + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + del attrs + self._token() + self.nodes += 1 + if self.nodes > self.limits.html_nodes: + raise _fail("html_node_limit", "HTML node count exceeds its cap") + lowered = tag.lower() + if lowered in {"script", "object", "embed", "iframe"}: + raise _fail("html_active_content", "active or embedded HTML content is forbidden") + if lowered not in self._VOID: + self.depth += 1 + self.max_depth_observed = max(self.max_depth_observed, self.depth) + if self.depth > self.limits.html_depth: + raise _fail("html_depth_limit", "HTML nesting depth exceeds its cap") + if lowered == "title": + self.in_title = True + + def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + self.handle_starttag(tag, attrs) + if tag.lower() not in self._VOID: + self.depth = max(0, self.depth - 1) + + def handle_endtag(self, tag: str) -> None: + self._token() + if tag.lower() == "title": + self.in_title = False + if tag.lower() not in self._VOID: + self.depth = max(0, self.depth - 1) + + def handle_data(self, data: str) -> None: + self._token() + if data: + self.nodes += 1 + if self.nodes > self.limits.html_nodes: + raise _fail("html_node_limit", "HTML node count exceeds its cap") + self.text_bytes += len(data.encode("utf-8")) + if self.text_bytes > self.limits.html_retained_text_bytes: + raise _fail("html_text_limit", "HTML retained text exceeds its cap") + if self.in_title and sum(len(part.encode("utf-8")) for part in self.title_parts) < 512: + self.title_parts.append(data) + + def handle_decl(self, decl: str) -> None: + self._token() + lowered = decl.lower() + if ( + not lowered.startswith("doctype html") + or "[" in lowered + or " system " in lowered + or " public " in lowered + ): + raise _fail("html_declaration_forbidden", "HTML DTD/entity declarations are forbidden") + + def unknown_decl(self, data: str) -> None: + del data + raise _fail("html_declaration_forbidden", "HTML declarations are forbidden") + + def handle_comment(self, data: str) -> None: + del data + self._token() + + def summary(self) -> dict[str, Any]: + title = " ".join("".join(self.title_parts).split()) + return { + "document_title": title[:512] if title else _UNKNOWN, + "html_nodes": self.nodes, + "html_tokens": self.tokens, + "html_max_depth_observed": self.max_depth_observed, + "html_text_bytes": self.text_bytes, + } + + +def _parse_html(body: bytes, limits: DocumentParserLimits) -> dict[str, Any]: + try: + text = body.decode("utf-8") + except UnicodeDecodeError as exc: + raise _fail("document_utf8_invalid", "HTML must be UTF-8") from exc + lowered = text.lower() + if any(marker in lowered for marker in (" None: + def denied(*args: object, **kwargs: object) -> object: + del args, kwargs + raise PermissionError("network disabled in PDF parser") + + socket.socket = denied # type: ignore[assignment] + socket.create_connection = denied # type: ignore[assignment] + socket.getaddrinfo = denied # type: ignore[assignment] + + +def _disable_child_process_creation() -> None: + def denied(*args: object, **kwargs: object) -> object: + del args, kwargs + raise PermissionError("process creation disabled in PDF parser") + + for name in ( + "fork", + "forkpty", + "posix_spawn", + "posix_spawnp", + "system", + ): + if hasattr(os, name): + setattr(os, name, denied) + + +def _apply_pdf_resource_limits(limits: DocumentParserLimits) -> None: + for kind, soft, hard in ( + (resource.RLIMIT_CPU, limits.pdf_cpu_seconds, limits.pdf_cpu_seconds), + (resource.RLIMIT_AS, limits.pdf_rss_bytes, limits.pdf_rss_bytes), + (resource.RLIMIT_FSIZE, limits.pdf_temp_bytes, limits.pdf_temp_bytes), + (resource.RLIMIT_NPROC, limits.pdf_pids, limits.pdf_pids), + ): + try: + resource.setrlimit(kind, (soft, hard)) + except (OSError, ValueError): + # The parent still enforces wall/RSS and the worker never executes + # untrusted code or exposes a process-creation path. + continue + + +def _process_rss_bytes(pid: int) -> int | None: + if sys.platform.startswith("linux"): + try: + fields = Path(f"/proc/{pid}/statm").read_text(encoding="ascii").split() + return int(fields[1]) * int(os.sysconf("SC_PAGE_SIZE")) + except (OSError, ValueError, IndexError): + return None + if sys.platform == "darwin": + try: + import ctypes + + library = ctypes.CDLL("/usr/lib/libproc.dylib") + buffer = ctypes.create_string_buffer(256) + size = library.proc_pidinfo(pid, 4, 0, buffer, len(buffer)) + if size < 16: + return None + _virtual, resident = struct.unpack_from("QQ", buffer.raw) + return int(resident) + except (OSError, ValueError): + return None + return None + + +def _pdf_worker( + connection: Any, + body: bytes, + temp_dir: str, + limits: DocumentParserLimits, +) -> None: + try: + os.setsid() + os.environ.clear() + os.chdir(temp_dir) + null_input = os.open(os.devnull, os.O_RDONLY) + null_output = os.open(os.devnull, os.O_WRONLY) + try: + os.dup2(null_input, 0) + os.dup2(null_output, 1) + os.dup2(null_output, 2) + finally: + os.close(null_input) + os.close(null_output) + _disable_child_network() + _disable_child_process_creation() + _apply_pdf_resource_limits(limits) + if not body.startswith(b"%PDF-"): + raise _fail("pdf_magic_mismatch", "PDF magic does not match content type") + if b"%%EOF" not in body[-2048:]: + raise _fail("pdf_malformed", "PDF end marker is missing") + if b"/EmbeddedFile" in body or b"/EmbeddedFiles" in body: + raise _fail("pdf_embedded_file", "PDF embedded files are forbidden") + pages = len(re.findall(rb"/Type\s*/Page\b", body)) + if pages > limits.pdf_pages: + raise _fail("pdf_page_limit", "PDF page count exceeds its cap") + characters = 0 + for match in re.finditer(rb"\(([^()]{0,1048576})\)\s*T[Jj]", body): + characters += len(match.group(1).decode("latin-1")) + if characters > limits.pdf_characters: + raise _fail("pdf_character_limit", "PDF extracted character count exceeds its cap") + payload = { + "ok": True, + "summary": { + "pdf_pages": pages, + "pdf_extracted_characters": characters, + "embedded_files": False, + }, + } + except Exception as exc: + code = exc.code if isinstance(exc, AlgorithmScoutError) else "pdf_parser_failed" + payload = {"ok": False, "code": code} + try: + encoded = canonical_json_v1(payload) + if len(encoded) <= limits.pdf_ipc_bytes: + connection.send_bytes(encoded) + finally: + connection.close() + + +def _terminate_process_group(process: multiprocessing.Process) -> None: + if not process.is_alive(): + process.join(timeout=0.2) + return + try: + os.killpg(process.pid, signal.SIGTERM) + except (ProcessLookupError, PermissionError): + process.terminate() + process.join(timeout=1) + if process.is_alive(): + try: + os.killpg(process.pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + process.kill() + process.join(timeout=1) + + +def _parse_pdf( + body: bytes, + limits: DocumentParserLimits, + *, + _worker: Callable[..., None] = _pdf_worker, + _temp_parent: str | None = None, +) -> dict[str, Any]: + if os.name != "posix": + raise _fail("pdf_platform_unsupported", "PDF parser isolation requires POSIX") + temp_dir = tempfile.mkdtemp(prefix="yolozu-scout-pdf-", dir=_temp_parent) + context = multiprocessing.get_context("spawn") + parent, child = context.Pipe(duplex=False) + process = context.Process(target=_worker, args=(child, body, temp_dir, limits), daemon=False) + wait_started = time.monotonic() + process.start() + child.close() + try: + while not parent.poll(0.05): + rss = _process_rss_bytes(process.pid) + if rss is not None and rss > limits.pdf_rss_bytes: + _terminate_process_group(process) + raise _fail("pdf_rss_limit", "PDF parser exceeded its RSS cap") + if time.monotonic() - wait_started >= limits.pdf_wall_seconds: + _terminate_process_group(process) + raise _fail("pdf_wall_limit", "PDF parser exceeded its wall-clock cap") + try: + encoded = parent.recv_bytes(limits.pdf_ipc_bytes) + except OSError as exc: + raise _fail("pdf_ipc_limit", "PDF parser exceeded its IPC cap") from exc + except EOFError as exc: + process.join(timeout=0.2) + if process.exitcode in { + -getattr(signal, "SIGXCPU", signal.SIGKILL), + -signal.SIGKILL, + }: + raise _fail("pdf_cpu_limit", "PDF parser exceeded its CPU cap") from exc + raise _fail("pdf_parser_failed", "PDF parser ended without a bounded result") from exc + process.join(timeout=1) + if process.is_alive(): + _terminate_process_group(process) + raise _fail("pdf_reap_failed", "PDF parser did not exit after returning") + payload = load_bounded_json_bytes(encoded, label="PDF parser result") + if not isinstance(payload, dict) or payload.get("ok") is not True: + code = payload.get("code") if isinstance(payload, dict) else "pdf_parser_failed" + raise _fail(str(code), "PDF parser rejected the document") + summary = payload.get("summary") + if not isinstance(summary, dict): + raise _fail("pdf_parser_failed", "PDF parser result is invalid") + return summary + finally: + parent.close() + _terminate_process_group(process) + shutil.rmtree(temp_dir, ignore_errors=False) + + +def _validate_document_magic(document: FetchedDocument) -> None: + body = document.body + stripped = body.lstrip() + lowered = stripped[:64].lower() + if body.startswith((b"PK\x03\x04", b"\x1f\x8b", b"Rar!", b"7z\xbc\xaf\x27\x1c")): + raise _fail("archive_forbidden", "archive content is forbidden") + if body.startswith((b"\x89PNG\r\n\x1a\n", b"\xff\xd8\xff", b"GIF87a", b"GIF89a")): + raise _fail("media_forbidden", "media content is forbidden") + content_type = document.content_type + if content_type == "application/pdf": + if not body.startswith(b"%PDF-"): + raise _fail("content_magic_mismatch", "PDF magic does not match content type") + elif content_type == "application/json": + if not stripped.startswith((b"{", b"[")): + raise _fail("content_magic_mismatch", "JSON magic does not match content type") + elif content_type in {"text/html", "application/xhtml+xml"}: + if not lowered.startswith((b" str: + if value is None and unknown: + return _UNKNOWN + if not isinstance(value, str) or not value or len(value.encode("utf-8")) > maximum: + return _UNKNOWN if unknown else "" + if any(ord(character) < 0x20 for character in value): + return _UNKNOWN if unknown else "" + return value + + +def _candidate_from_source( + source: AlgorithmScoutSource, + *, + collected_at: str, + collection_date: str, + content_sha256: str, + untrusted: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + item = untrusted or {} + version = _candidate_text( + item.get("version", item.get("tag_name")), maximum=128 + ) + revision = _candidate_text(item.get("revision"), maximum=128) + if version != _UNKNOWN and _VERSION_RE.fullmatch(version) is None: + version = _UNKNOWN + if revision != _UNKNOWN and _VERSION_RE.fullmatch(revision) is None: + revision = _UNKNOWN + raw_release_date = item.get("release_date", item.get("published_at")) + if isinstance(raw_release_date, str) and len(raw_release_date) >= 10: + raw_release_date = raw_release_date[:10] + release_date = _candidate_text(raw_release_date, maximum=10) + if release_date != _UNKNOWN: + try: + release_date = _validate_calendar_date(release_date) + except AlgorithmScoutError: + release_date = _UNKNOWN + project = _candidate_text(item.get("project"), maximum=128) + model = _candidate_text(item.get("model"), maximum=128) + if project == _UNKNOWN: + project = source.project_identity + if model == _UNKNOWN: + model = source.model_identity + identity = version if version != _UNKNOWN else revision + if identity == _UNKNOWN: + identity = _UNKNOWN + key_record = { + "source_url": source.location.to_url(), + "version_or_revision": identity, + } + candidate_key = hashlib.sha256(canonical_json_v1(key_record)).hexdigest() + return { + "candidate_key": candidate_key, + "source_id": source.source_id, + "source_url": source.location.to_url(), + "identity": {"project": project, "model": model}, + "version": version, + "revision": revision, + "release_date": release_date, + "tasks": list(source.tasks), + "availability": { + "local": source.local_availability, + "hosted": source.hosted_availability, + }, + "license": { + "status": source.license_status, + "expression": source.license_expression, + }, + "weights": {"status": source.weight_status}, + "runtime_hints": list(source.runtime_hints), + "collection_status": "collected", + "collected_at": collected_at, + "timezone": "UTC", + "first_seen_date": collection_date, + "last_seen_date": collection_date, + "history": [ + { + "collection_date": collection_date, + "collected_at": collected_at, + "content_sha256": content_sha256, + } + ], + } + + +def _parse_document( + source: AlgorithmScoutSource, + document: FetchedDocument, + *, + collected_at: str, + collection_date: str, + limits: DocumentParserLimits, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + _validate_document_magic(document) + digest = hashlib.sha256(document.body).hexdigest() + if document.content_type in {"text/html", "application/xhtml+xml"}: + summary = _parse_html(document.body, limits) + candidates = [ + _candidate_from_source( + source, + collected_at=collected_at, + collection_date=collection_date, + content_sha256=digest, + ) + ] + elif document.content_type == "application/json": + payload = load_bounded_json_bytes(document.body, label="untrusted scout JSON") + raw_candidates: Iterable[Mapping[str, Any]] + if isinstance(payload, dict) and isinstance(payload.get("candidates"), list): + raw_candidates = [item for item in payload["candidates"] if isinstance(item, dict)][:MAX_CANDIDATES] + elif isinstance(payload, dict): + raw_candidates = [payload] + elif isinstance(payload, list): + raw_candidates = [item for item in payload if isinstance(item, dict)][:MAX_CANDIDATES] + else: + raw_candidates = [{}] + candidates = [ + _candidate_from_source( + source, + collected_at=collected_at, + collection_date=collection_date, + content_sha256=digest, + untrusted=item, + ) + for item in raw_candidates + ] + summary = {"json_nodes_bound": 100_000, "json_depth_bound": 64} + elif document.content_type == "application/pdf": + summary = _parse_pdf(document.body, limits) + candidates = [ + _candidate_from_source( + source, + collected_at=collected_at, + collection_date=collection_date, + content_sha256=digest, + ) + ] + else: + try: + document.body.decode("utf-8") + except UnicodeDecodeError as exc: + raise _fail("document_utf8_invalid", "plain text must be UTF-8") from exc + summary = {"text_bytes": len(document.body)} + candidates = [ + _candidate_from_source( + source, + collected_at=collected_at, + collection_date=collection_date, + content_sha256=digest, + ) + ] + return candidates, { + "content_sha256": digest, + "content_type": document.content_type, + "transferred_bytes": document.transferred_bytes, + "decoded_bytes": document.decoded_bytes, + "redirect_count": document.redirect_count, + "parser_summary": summary, + } + + +def _read_prior_candidates(workspace: Path, output_dir: str) -> list[dict[str, Any]]: + root = workspace.joinpath(*PurePosixPath(output_dir).parts) + if not root.exists(): + return [] + if root.is_symlink() or not root.is_dir(): + raise _fail("output_dir_invalid", "existing output directory is unsafe") + dated = sorted( + ( + item + for item in root.iterdir() + if item.is_dir() and not item.is_symlink() and _DATE_RE.fullmatch(item.name) + ), + key=lambda item: item.name, + )[-MAX_PRIOR_REPORTS:] + candidates: list[dict[str, Any]] = [] + for directory in dated: + try: + validate_managed_output_destination( + root=workspace, + destination=f"{output_dir}/{directory.name}", + limits=ManagedOutputLimits( + max_files=2, + max_file_bytes=REPORT_MAX_BYTES, + max_total_bytes=REPORT_MAX_BYTES + 4096, + ), + force=True, + ) + except ValueError as exc: + raise _fail( + "prior_report_invalid", + "prior scout managed output failed integrity validation", + ) from exc + report_path = directory / "algorithm_scout_report.json" + if report_path.is_symlink() or not report_path.is_file(): + continue + payload = load_bounded_json(report_path, label="prior algorithm scout report") + if ( + not isinstance(payload, dict) + or payload.get("kind") != REPORT_KIND + or payload.get("schema_version") != REPORT_SCHEMA_VERSION + or not isinstance(payload.get("candidates"), list) + ): + raise _fail("prior_report_invalid", "prior scout report is invalid") + for item in payload["candidates"]: + candidates.append(_validate_prior_candidate(item)) + if len(candidates) > MAX_CANDIDATES: + raise _fail("history_limit", "prior candidate history exceeds its cap") + return candidates + + +def _prior_text(value: object, *, field: str, maximum: int) -> str: + if ( + not isinstance(value, str) + or not value + or len(value.encode("utf-8")) > maximum + or any(ord(character) < 0x20 for character in value) + ): + raise _fail("prior_report_invalid", f"prior {field} is invalid") + return value + + +def _validate_prior_candidate(value: object) -> dict[str, Any]: + fields = { + "candidate_key", + "source_id", + "source_url", + "identity", + "version", + "revision", + "release_date", + "tasks", + "availability", + "license", + "weights", + "runtime_hints", + "collection_status", + "collected_at", + "timezone", + "first_seen_date", + "last_seen_date", + "history", + } + if not isinstance(value, dict) or set(value) != fields: + raise _fail("prior_report_invalid", "prior candidate fields are invalid") + candidate_key = _prior_text(value["candidate_key"], field="candidate key", maximum=64) + if len(candidate_key) != 64 or any(character not in "0123456789abcdef" for character in candidate_key): + raise _fail("prior_report_invalid", "prior candidate key is invalid") + source_id = _prior_text(value["source_id"], field="source ID", maximum=64) + if _SOURCE_ID_RE.fullmatch(source_id) is None: + raise _fail("prior_report_invalid", "prior source ID is invalid") + source_url = _prior_text(value["source_url"], field="source URL", maximum=4096) + try: + source_location = HttpsLocation.from_redirect( + source_url, + base=HttpsLocation(host="invalid.example", path="/"), + ) + except SafeHttpsError as exc: + raise _fail("prior_report_invalid", "prior source URL is invalid") from exc + if source_location.to_url() != source_url: + raise _fail("prior_report_invalid", "prior source URL is not canonical") + identity = value["identity"] + if not isinstance(identity, dict) or set(identity) != {"project", "model"}: + raise _fail("prior_report_invalid", "prior identity is invalid") + normalized_identity = { + "project": _prior_text(identity["project"], field="project identity", maximum=128), + "model": _prior_text(identity["model"], field="model identity", maximum=128), + } + version = _prior_text(value["version"], field="version", maximum=128) + revision = _prior_text(value["revision"], field="revision", maximum=128) + for label, item in (("version", version), ("revision", revision)): + if item != _UNKNOWN and _VERSION_RE.fullmatch(item) is None: + raise _fail("prior_report_invalid", f"prior {label} is invalid") + release_date = _prior_text(value["release_date"], field="release date", maximum=10) + if release_date != _UNKNOWN: + _validate_calendar_date(release_date) + try: + tasks = _bounded_string_list( + value["tasks"], field="prior tasks", maximum_items=32, maximum_item_bytes=64 + ) + runtime_hints = _bounded_string_list( + value["runtime_hints"], + field="prior runtime hints", + maximum_items=32, + maximum_item_bytes=128, + ) + except AlgorithmScoutError as exc: + raise _fail("prior_report_invalid", "prior candidate arrays are invalid") from exc + availability = value["availability"] + if not isinstance(availability, dict) or set(availability) != {"local", "hosted"}: + raise _fail("prior_report_invalid", "prior availability is invalid") + allowed_status = {"available", "unavailable", "unknown"} + normalized_availability = { + name: _prior_text(availability[name], field=f"availability {name}", maximum=32) + for name in ("local", "hosted") + } + if any(item not in allowed_status for item in normalized_availability.values()): + raise _fail("prior_report_invalid", "prior availability status is invalid") + license_record = value["license"] + if not isinstance(license_record, dict) or set(license_record) != {"status", "expression"}: + raise _fail("prior_report_invalid", "prior license is invalid") + normalized_license = { + "status": _prior_text(license_record["status"], field="license status", maximum=32), + "expression": _prior_text( + license_record["expression"], field="license expression", maximum=128 + ), + } + if normalized_license["status"] not in {"approved", "rejected", "review_required", "unknown"}: + raise _fail("prior_report_invalid", "prior license status is invalid") + weights = value["weights"] + if not isinstance(weights, dict) or set(weights) != {"status"}: + raise _fail("prior_report_invalid", "prior weight status is invalid") + weight_status = _prior_text(weights["status"], field="weight status", maximum=32) + if weight_status not in allowed_status: + raise _fail("prior_report_invalid", "prior weight status is invalid") + collection_status = _prior_text( + value["collection_status"], field="collection status", maximum=16 + ) + if collection_status not in {"collected", "historical"}: + raise _fail("prior_report_invalid", "prior collection status is invalid") + collected_at = _validate_utc( + _prior_text(value["collected_at"], field="collected timestamp", maximum=20) + ) + if value["timezone"] != "UTC": + raise _fail("prior_report_invalid", "prior timezone is invalid") + first_seen = _validate_calendar_date( + _prior_text(value["first_seen_date"], field="first seen date", maximum=10) + ) + last_seen = _validate_calendar_date( + _prior_text(value["last_seen_date"], field="last seen date", maximum=10) + ) + if date.fromisoformat(last_seen) < date.fromisoformat(first_seen): + raise _fail("prior_report_invalid", "prior candidate dates are reversed") + raw_history = value["history"] + if not isinstance(raw_history, list) or len(raw_history) > MAX_HISTORY_ITEMS: + raise _fail("prior_report_invalid", "prior candidate history is invalid") + history: list[dict[str, str]] = [] + for item in raw_history: + if not isinstance(item, dict) or set(item) != {"collection_date", "collected_at", "content_sha256"}: + raise _fail("prior_report_invalid", "prior history item is invalid") + digest = _prior_text(item["content_sha256"], field="content digest", maximum=64) + if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest): + raise _fail("prior_report_invalid", "prior content digest is invalid") + history.append( + { + "collection_date": _validate_calendar_date( + _prior_text(item["collection_date"], field="history date", maximum=10) + ), + "collected_at": _validate_utc( + _prior_text(item["collected_at"], field="history timestamp", maximum=20) + ), + "content_sha256": digest, + } + ) + version_or_revision = version if version != _UNKNOWN else revision + expected_key = hashlib.sha256( + canonical_json_v1( + {"source_url": source_url, "version_or_revision": version_or_revision} + ) + ).hexdigest() + if candidate_key != expected_key: + raise _fail("prior_report_invalid", "prior candidate key does not match identity") + return { + "candidate_key": candidate_key, + "source_id": source_id, + "source_url": source_url, + "identity": normalized_identity, + "version": version, + "revision": revision, + "release_date": release_date, + "tasks": list(tasks), + "availability": normalized_availability, + "license": normalized_license, + "weights": {"status": weight_status}, + "runtime_hints": list(runtime_hints), + "collection_status": collection_status, + "collected_at": collected_at, + "timezone": "UTC", + "first_seen_date": first_seen, + "last_seen_date": last_seen, + "history": history, + } + + +def _merge_candidates( + prior: Iterable[Mapping[str, Any]], + current: Iterable[Mapping[str, Any]], +) -> list[dict[str, Any]]: + prior_items = list(prior) + current_items = list(current) + current_keys = { + str(item.get("candidate_key")) + for item in current_items + if isinstance(item.get("candidate_key"), str) + } + merged: dict[str, dict[str, Any]] = {} + for raw in [*prior_items, *current_items]: + key = raw.get("candidate_key") + if not isinstance(key, str) or len(key) != 64: + continue + candidate = json.loads(json.dumps(raw, ensure_ascii=False)) + existing = merged.get(key) + if existing is None: + history = candidate.get("history") + candidate["history"] = history[-MAX_HISTORY_ITEMS:] if isinstance(history, list) else [] + merged[key] = candidate + continue + histories = [ + item + for item in [*(existing.get("history") or []), *(candidate.get("history") or [])] + if isinstance(item, dict) + ] + unique = { + ( + str(item.get("collection_date")), + str(item.get("collected_at")), + str(item.get("content_sha256")), + ): item + for item in histories + } + candidate["history"] = [ + unique[key] + for key in sorted(unique, key=lambda value: tuple(part.encode("utf-8") for part in value)) + ][-MAX_HISTORY_ITEMS:] + first_dates = [ + value + for value in (existing.get("first_seen_date"), candidate.get("first_seen_date")) + if isinstance(value, str) and _DATE_RE.fullmatch(value) + ] + last_dates = [ + value + for value in (existing.get("last_seen_date"), candidate.get("last_seen_date")) + if isinstance(value, str) and _DATE_RE.fullmatch(value) + ] + candidate["first_seen_date"] = min(first_dates) if first_dates else candidate.get("first_seen_date") + candidate["last_seen_date"] = max(last_dates) if last_dates else candidate.get("last_seen_date") + merged[key] = candidate + if len(merged) > MAX_CANDIDATES: + raise _fail("candidate_limit", "candidate inbox exceeds its cap") + for key, candidate in merged.items(): + candidate["collection_status"] = ( + "collected" if key in current_keys else "historical" + ) + return [merged[key] for key in sorted(merged, key=lambda item: item.encode("ascii"))] + + +def _ensure_directory_chain(workspace: Path, output_dir: str) -> None: + _open_output_chain(workspace, output_dir, create=True) + + +def _validate_existing_output_chain(workspace: Path, output_dir: str) -> None: + _open_output_chain(workspace, output_dir, create=False) + + +def _open_output_chain(workspace: Path, output_dir: str, *, create: bool) -> None: + nofollow = getattr(os, "O_NOFOLLOW", 0) + directory = getattr(os, "O_DIRECTORY", 0) + cloexec = getattr(os, "O_CLOEXEC", 0) + if os.name != "posix" or not nofollow or not directory: + raise _fail("platform_unsupported", "workspace output requires POSIX no-follow directories") + descriptor = os.open(workspace, os.O_RDONLY | directory | nofollow | cloexec) + try: + for component in PurePosixPath(output_dir).parts: + try: + child = os.open( + component, + os.O_RDONLY | directory | nofollow | cloexec, + dir_fd=descriptor, + ) + except FileNotFoundError: + if not create: + return + try: + os.mkdir(component, mode=0o700, dir_fd=descriptor) + except FileExistsError: + # A concurrent creator may win this race; the no-follow open below + # revalidates the component before it is used. + pass + child = os.open( + component, + os.O_RDONLY | directory | nofollow | cloexec, + dir_fd=descriptor, + ) + except OSError as exc: + if exc.errno in {errno.ELOOP, errno.ENOTDIR}: + raise _fail( + "output_dir_invalid", + "output path contains an unsafe component", + ) from exc + raise + info = os.fstat(child) + if not stat.S_ISDIR(info.st_mode): + os.close(child) + raise _fail("output_dir_invalid", "output path component is not a directory") + os.close(descriptor) + descriptor = child + finally: + os.close(descriptor) + + +def collect_algorithm_candidates( + plan: ScoutPlan, + *, + workspace_root: str | Path = ".", + transport_factory: Callable[..., SafeHttpsTransport] = SafeHttpsTransport, + parser_limits: DocumentParserLimits | None = None, + now_utc: Callable[[], str] = _utc_now, + monotonic: Callable[[], float] = time.monotonic, +) -> tuple[dict[str, Any], int, Path]: + workspace = _workspace(workspace_root) + limits = parser_limits or DocumentParserLimits() + started_at = _validate_utc(now_utc()) + deadline = monotonic() + COLLECTION_TIMEOUT_SECONDS + _validate_existing_output_chain(workspace, plan.output_dir) + prior = _read_prior_candidates(workspace, plan.output_dir) + locations = tuple( + location for source in plan.enabled_sources for location in source.allowlist + ) + transport = transport_factory(allowlist=locations, limits=TransportLimits()) + source_results: list[dict[str, Any]] = [] + current_candidates: list[dict[str, Any]] = [] + decoded_total = 0 + failures = 0 + for source in plan.enabled_sources: + collected_at = _validate_utc(now_utc()) + base = { + "source_id": source.source_id, + "source_url": source.location.to_url(), + "collected_at": collected_at, + "timezone": "UTC", + } + if monotonic() >= deadline: + failures += 1 + source_results.append({**base, "collection_status": "missed", "failure_code": "collection_deadline"}) + continue + try: + document = transport.fetch(source.location, collection_deadline=deadline) + decoded_total += document.decoded_bytes + if decoded_total > TOTAL_DECODED_BYTES: + raise _fail("decoded_total_limit", "collection decoded-byte total exceeds its cap") + candidates, provenance = _parse_document( + source, + document, + collected_at=collected_at, + collection_date=plan.collection_date, + limits=limits, + ) + current_candidates.extend(candidates) + if len(current_candidates) > MAX_CANDIDATES: + raise _fail("candidate_limit", "candidate inbox exceeds its cap") + source_results.append( + { + **base, + "collection_status": "collected", + "candidate_count": len(candidates), + "provenance": provenance, + } + ) + except (AlgorithmScoutError, SafeHttpsError, OSError, ValueError) as exc: + failures += 1 + code = getattr(exc, "code", "collection_failed") + source_results.append({**base, "collection_status": "failed", "failure_code": str(code)[:64]}) + candidates = _merge_candidates(prior, current_candidates) + completed_at = _validate_utc(now_utc()) + if _parse_utc(completed_at) < _parse_utc(started_at): + raise _fail("timestamp_reversed", "report completion precedes collection start") + report = { + "kind": REPORT_KIND, + "schema_version": REPORT_SCHEMA_VERSION, + "maturity": "experimental", + "selectability": "inbox_only", + "collection_date": plan.collection_date, + "trigger": plan.trigger, + "started_at": started_at, + "completed_at": completed_at, + "timezone": "UTC", + "source_policy": { + "canonical_sources": CANONICAL_SOURCES_PATH.as_posix(), + "scope": "explicit_official_allowlist_only", + "coverage": "monitored_sources_not_latest_world", + "redirect_limit": 3, + "network": "https_443_collect_only", + }, + "retention": { + "raw_documents": "not_retained", + "candidate_metadata": "repository_policy", + "history_reports_scanned_max": MAX_PRIOR_REPORTS, + }, + "automation_caveat": "Discovery metadata is an untrusted inbox signal, not qualification, support, recommendation, adoption, or promotion evidence.", + "sources": source_results, + "candidates": candidates, + "summary": { + "enabled_sources": len(plan.enabled_sources), + "collected_sources": sum(item["collection_status"] == "collected" for item in source_results), + "failed_or_missed_sources": failures, + "candidate_count": len(candidates), + "current_candidate_count": sum( + item["collection_status"] == "collected" for item in candidates + ), + "historical_candidate_count": sum( + item["collection_status"] == "historical" for item in candidates + ), + "decoded_total_bytes": decoded_total, + "network_used": True, + "writes_performed": True, + }, + } + encoded = canonical_json_v1(report) + if len(encoded) > REPORT_MAX_BYTES: + raise _fail("report_limit", "final report exceeds its byte cap") + _ensure_directory_chain(workspace, plan.output_dir) + destination = f"{plan.output_dir}/{plan.collection_date}" + with ManagedOutputTransaction( + root=workspace, + destination=destination, + declared_paths=("algorithm_scout_report.json",), + limits=ManagedOutputLimits( + max_files=2, + max_file_bytes=REPORT_MAX_BYTES, + max_total_bytes=REPORT_MAX_BYTES + 4096, + ), + force=True, + ) as transaction: + transaction.write_bytes("algorithm_scout_report.json", encoded) + transaction.commit() + return report, 3 if failures else 0, workspace / destination / "algorithm_scout_report.json" diff --git a/yolozu/adaptive/safe_https.py b/yolozu/adaptive/safe_https.py new file mode 100644 index 00000000..62f579a6 --- /dev/null +++ b/yolozu/adaptive/safe_https.py @@ -0,0 +1,554 @@ +"""Bounded HTTPS-only transport for code-owned adaptive intake services. + +The transport accepts structured, pre-allowlisted locations. It does not +accept caller headers, cookies, credentials, query strings, or fragments. +Fetched bytes remain an untrusted in-memory document for a caller-owned +bounded parser; this module never executes or persists them. +""" + +from __future__ import annotations + +import ipaddress +import socket +import ssl +import time +import zlib +from dataclasses import dataclass +from typing import Callable, Iterable, Mapping, Protocol +from urllib.parse import urljoin, urlsplit + +__all__ = [ + "FetchedDocument", + "HttpsLocation", + "SafeHttpsError", + "SafeHttpsTransport", + "TransportLimits", +] + + +_REDIRECT_STATUSES = {301, 302, 303, 307, 308} +_ALLOWED_CONTENT_TYPES = { + "application/json", + "application/pdf", + "application/xhtml+xml", + "text/html", + "text/plain", +} +_DENIED_NETWORKS = tuple( + ipaddress.ip_network(value) + for value in ( + "192.0.2.0/24", + "198.18.0.0/15", + "198.51.100.0/24", + "203.0.113.0/24", + "2001:db8::/32", + ) +) + + +class SafeHttpsError(ValueError): + """A bounded failure that never includes a rejected raw URL.""" + + def __init__(self, code: str, detail: str) -> None: + safe_detail = detail.encode("utf-8", "replace")[:512].decode( + "utf-8", "ignore" + ) + super().__init__(f"{code}: {safe_detail}") + self.code = code + + +def _fail(code: str, detail: str) -> SafeHttpsError: + return SafeHttpsError(code, detail) + + +@dataclass(frozen=True, order=True) +class HttpsLocation: + """One credential-free canonical HTTPS location.""" + + host: str + path: str + scheme: str = "https" + + def __post_init__(self) -> None: + if self.scheme != "https": + raise _fail("url_scheme_invalid", "only https is allowed") + host = self.host.rstrip(".").lower() + if not host or len(host.encode("ascii", "ignore")) != len(host): + raise _fail("url_host_invalid", "host must be bounded ASCII DNS text") + if len(host) > 253 or any( + not label + or len(label) > 63 + or label[0] == "-" + or label[-1] == "-" + or any(character not in "abcdefghijklmnopqrstuvwxyz0123456789-" for character in label) + for label in host.split(".") + ): + raise _fail("url_host_invalid", "host is not a canonical DNS name") + try: + ipaddress.ip_address(host.strip("[]")) + except ValueError: + pass + else: + raise _fail("url_ip_literal", "IP literals are forbidden") + if ( + not self.path.startswith("/") + or "\\" in self.path + or "?" in self.path + or "#" in self.path + or "\x00" in self.path + or not self.path.isascii() + or any(ord(character) <= 0x20 or ord(character) == 0x7F for character in self.path) + or len(self.path.encode("utf-8")) > 4096 + or any(segment in {".", ".."} for segment in self.path.split("/")) + ): + raise _fail("url_path_invalid", "path is not canonical and bounded") + for index, character in enumerate(self.path): + if character == "%" and ( + index + 2 >= len(self.path) + or any( + item not in "0123456789abcdefABCDEF" + for item in self.path[index + 1 : index + 3] + ) + ): + raise _fail("url_path_invalid", "path percent encoding is invalid") + object.__setattr__(self, "host", host) + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "HttpsLocation": + if not isinstance(value, Mapping) or set(value) != {"scheme", "host", "path"}: + raise _fail( + "url_record_invalid", + "location must contain only scheme, host, and path", + ) + if not all(isinstance(value[key], str) for key in value): + raise _fail("url_record_invalid", "location fields must be strings") + return cls( + scheme=str(value["scheme"]), + host=str(value["host"]), + path=str(value["path"]), + ) + + @classmethod + def from_redirect(cls, raw: str, *, base: "HttpsLocation") -> "HttpsLocation": + if not isinstance(raw, str) or len(raw.encode("utf-8", "replace")) > 4096: + raise _fail("redirect_invalid", "redirect location is invalid") + combined = urlsplit(urljoin(base.to_url(), raw)) + if ( + combined.scheme != "https" + or combined.username is not None + or combined.password is not None + or combined.query + or combined.fragment + ): + raise _fail( + "redirect_not_allowed", + "redirect must remain credential-free and query-free HTTPS", + ) + try: + port = combined.port + except ValueError as exc: + raise _fail("redirect_not_allowed", "redirect port is invalid") from exc + if port not in (None, 443): + raise _fail("redirect_not_allowed", "redirect port must be 443") + if combined.hostname is None: + raise _fail("redirect_not_allowed", "redirect host is missing") + return cls(host=combined.hostname, path=combined.path or "/") + + def to_url(self) -> str: + return f"https://{self.host}{self.path}" + + def to_mapping(self) -> dict[str, str]: + return {"scheme": "https", "host": self.host, "path": self.path} + + +@dataclass(frozen=True) +class TransportLimits: + connect_timeout_seconds: int = 5 + read_timeout_seconds: int = 15 + per_source_timeout_seconds: int = 30 + max_redirects: int = 3 + max_header_bytes: int = 64 * 1024 + max_transferred_bytes: int = 16 * 1024 * 1024 + max_decoded_bytes: int = 32 * 1024 * 1024 + + def __post_init__(self) -> None: + bounds = { + "connect_timeout_seconds": (self.connect_timeout_seconds, 1, 5), + "read_timeout_seconds": (self.read_timeout_seconds, 1, 15), + "per_source_timeout_seconds": (self.per_source_timeout_seconds, 1, 30), + "max_redirects": (self.max_redirects, 0, 3), + "max_header_bytes": (self.max_header_bytes, 1024, 64 * 1024), + "max_transferred_bytes": ( + self.max_transferred_bytes, + 1024, + 16 * 1024 * 1024, + ), + "max_decoded_bytes": ( + self.max_decoded_bytes, + 1024, + 32 * 1024 * 1024, + ), + } + for name, (value, minimum, maximum) in bounds.items(): + if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum: + raise ValueError(f"{name} must be in {minimum}..{maximum}") + + +@dataclass(frozen=True) +class FetchedDocument: + source: HttpsLocation + final_location: HttpsLocation + content_type: str + body: bytes + transferred_bytes: int + decoded_bytes: int + redirect_count: int + + +class _SocketLike(Protocol): + def sendall(self, data: bytes) -> None: + raise NotImplementedError + + def recv(self, size: int) -> bytes: + raise NotImplementedError + + def settimeout(self, value: float) -> None: + raise NotImplementedError + + def getpeername(self) -> tuple[object, ...]: + raise NotImplementedError + + def close(self) -> None: + raise NotImplementedError + + +Resolver = Callable[[str], tuple[str, ...]] +Dialer = Callable[[str, str, float, float], _SocketLike] + + +def _is_public_address(value: str) -> bool: + try: + address = ipaddress.ip_address(value) + except ValueError: + return False + if ( + not address.is_global + or address.is_loopback + or address.is_private + or address.is_link_local + or address.is_multicast + or address.is_reserved + or address.is_unspecified + ): + return False + return not any(address in network for network in _DENIED_NETWORKS) + + +def _system_resolver(host: str) -> tuple[str, ...]: + try: + records = socket.getaddrinfo(host, 443, type=socket.SOCK_STREAM) + except OSError as exc: + raise _fail("dns_failed", "allowlisted host resolution failed") from exc + addresses = tuple(sorted({str(record[4][0]) for record in records})) + if not addresses: + raise _fail("dns_failed", "allowlisted host resolved to no addresses") + return addresses + + +def _tls_dial(ip: str, host: str, connect_timeout: float, read_timeout: float) -> _SocketLike: + raw = socket.create_connection((ip, 443), timeout=connect_timeout) + try: + context = ssl.create_default_context() + context.minimum_version = ssl.TLSVersion.TLSv1_2 + wrapped = context.wrap_socket(raw, server_hostname=host) + wrapped.settimeout(read_timeout) + return wrapped + except Exception: + raw.close() + raise + + +class _Reader: + def __init__(self, connection: _SocketLike, initial: bytes, deadline: float, read_timeout: int) -> None: + self.connection = connection + self.buffer = bytearray(initial) + self.deadline = deadline + self.read_timeout = read_timeout + + def _recv(self) -> bytes: + remaining = self.deadline - time.monotonic() + if remaining <= 0: + raise _fail("source_timeout", "per-source collection deadline expired") + self.connection.settimeout(min(float(self.read_timeout), remaining)) + try: + return self.connection.recv(64 * 1024) + except (OSError, TimeoutError) as exc: + raise _fail("read_failed", "bounded HTTPS response read failed") from exc + + def until(self, delimiter: bytes, maximum: int) -> bytes: + while True: + index = self.buffer.find(delimiter) + if index >= 0: + end = index + len(delimiter) + if end > maximum: + raise _fail("response_limit", "response framing exceeds its byte cap") + result = bytes(self.buffer[:end]) + del self.buffer[:end] + return result + if len(self.buffer) >= maximum: + raise _fail("response_limit", "response framing exceeds its byte cap") + chunk = self._recv() + if not chunk: + raise _fail("response_invalid", "response ended before framing completed") + self.buffer.extend(chunk) + + def exact(self, count: int) -> bytes: + while len(self.buffer) < count: + chunk = self._recv() + if not chunk: + raise _fail("response_invalid", "response body ended early") + self.buffer.extend(chunk) + result = bytes(self.buffer[:count]) + del self.buffer[:count] + return result + + def to_eof(self, maximum: int) -> bytes: + result = bytearray(self.buffer) + self.buffer.clear() + if len(result) > maximum: + raise _fail("transfer_limit", "transferred body exceeds its byte cap") + while True: + chunk = self._recv() + if not chunk: + return bytes(result) + result.extend(chunk) + if len(result) > maximum: + raise _fail("transfer_limit", "transferred body exceeds its byte cap") + + +def _parse_headers(data: bytes) -> tuple[int, dict[str, list[str]]]: + try: + text = data.decode("iso-8859-1") + except UnicodeDecodeError as exc: + raise _fail("response_invalid", "response headers are invalid") from exc + lines = text[:-4].split("\r\n") + if not lines or any(line.startswith((" ", "\t")) for line in lines[1:]): + raise _fail("response_invalid", "response header folding is forbidden") + status_parts = lines[0].split(" ", 2) + if len(status_parts) < 2 or not status_parts[0].startswith("HTTP/1."): + raise _fail("response_invalid", "response status line is invalid") + try: + status = int(status_parts[1]) + except ValueError as exc: + raise _fail("response_invalid", "response status is invalid") from exc + headers: dict[str, list[str]] = {} + for line in lines[1:]: + if not line: + continue + if ":" not in line: + raise _fail("response_invalid", "response header is malformed") + name, value = line.split(":", 1) + lowered = name.strip().lower() + if not lowered or any(character not in "abcdefghijklmnopqrstuvwxyz0123456789-" for character in lowered): + raise _fail("response_invalid", "response header name is invalid") + headers.setdefault(lowered, []).append(value.strip()) + return status, headers + + +def _single_header(headers: Mapping[str, list[str]], name: str) -> str | None: + values = headers.get(name, []) + if len(values) > 1: + raise _fail("response_invalid", f"duplicate {name} header is forbidden") + return values[0] if values else None + + +def _read_body(reader: _Reader, headers: Mapping[str, list[str]], maximum: int) -> bytes: + transfer = _single_header(headers, "transfer-encoding") + length = _single_header(headers, "content-length") + if transfer is not None and length is not None: + raise _fail("response_invalid", "ambiguous response body framing") + if transfer is not None: + if transfer.lower() != "chunked": + raise _fail("response_invalid", "unsupported transfer encoding") + body = bytearray() + chunks = 0 + while True: + chunks += 1 + if chunks > 200_000: + raise _fail("response_limit", "chunk count exceeds its cap") + line = reader.until(b"\r\n", 1024) + token = line[:-2] + if not token or b";" in token or any(character not in b"0123456789abcdefABCDEF" for character in token): + raise _fail("response_invalid", "chunk framing is invalid") + size = int(token, 16) + if size == 0: + trailer = reader.until(b"\r\n", 1024) + if trailer != b"\r\n": + raise _fail("response_invalid", "response trailers are forbidden") + return bytes(body) + if len(body) + size > maximum: + raise _fail("transfer_limit", "transferred body exceeds its byte cap") + body.extend(reader.exact(size)) + if reader.exact(2) != b"\r\n": + raise _fail("response_invalid", "chunk terminator is invalid") + if length is not None: + if not length.isascii() or not length.isdigit(): + raise _fail("response_invalid", "content length is invalid") + count = int(length) + if count > maximum: + raise _fail("transfer_limit", "transferred body exceeds its byte cap") + return reader.exact(count) + return reader.to_eof(maximum) + + +def _decode_body(body: bytes, encoding: str | None, maximum: int) -> bytes: + if encoding is None or encoding.lower() == "identity": + if len(body) > maximum: + raise _fail("decoded_limit", "decoded body exceeds its byte cap") + return body + normalized = encoding.lower() + if normalized == "gzip": + decoder = zlib.decompressobj(16 + zlib.MAX_WBITS) + elif normalized == "deflate": + decoder = zlib.decompressobj() + else: + raise _fail("content_encoding_invalid", "unsupported content encoding") + try: + decoded = decoder.decompress(body, maximum + 1) + if len(decoded) > maximum or decoder.unconsumed_tail: + raise _fail("decoded_limit", "decoded body exceeds its byte cap") + decoded += decoder.flush(maximum + 1 - len(decoded)) + except zlib.error as exc: + raise _fail("content_encoding_invalid", "compressed response is invalid") from exc + if len(decoded) > maximum: + raise _fail("decoded_limit", "decoded body exceeds its byte cap") + if not decoder.eof or decoder.unused_data: + raise _fail("content_encoding_invalid", "compressed response has invalid trailing data") + return decoded + + +class SafeHttpsTransport: + """Fetch only vetted addresses for an exact structured allowlist.""" + + def __init__( + self, + *, + allowlist: Iterable[HttpsLocation], + limits: TransportLimits | None = None, + resolver: Resolver | None = None, + dialer: Dialer | None = None, + ) -> None: + allowed = frozenset(allowlist) + if not allowed: + raise ValueError("allowlist must not be empty") + self.allowlist = allowed + self.limits = limits or TransportLimits() + self._resolver = resolver or _system_resolver + self._dialer = dialer or _tls_dial + + def _resolve(self, host: str) -> tuple[str, ...]: + addresses = tuple(self._resolver(host)) + if not addresses or any(not _is_public_address(item) for item in addresses): + raise _fail("address_not_public", "host resolved to a forbidden address class") + return tuple(sorted(set(addresses))) + + def fetch( + self, + location: HttpsLocation, + *, + collection_deadline: float | None = None, + ) -> FetchedDocument: + if location not in self.allowlist: + raise _fail("url_not_allowlisted", "requested location is not allowlisted") + deadline = time.monotonic() + self.limits.per_source_timeout_seconds + if collection_deadline is not None: + deadline = min(deadline, collection_deadline) + source = location + current = location + seen: set[HttpsLocation] = set() + redirects = 0 + while True: + if current in seen: + raise _fail("redirect_loop", "redirect loop detected") + seen.add(current) + if time.monotonic() >= deadline: + raise _fail("source_timeout", "per-source collection deadline expired") + addresses = self._resolve(current.host) + connection: _SocketLike | None = None + last_error: Exception | None = None + for address in addresses: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + connection = self._dialer( + address, + current.host, + min(float(self.limits.connect_timeout_seconds), remaining), + min(float(self.limits.read_timeout_seconds), remaining), + ) + peer = str(connection.getpeername()[0]) + if peer not in addresses or not _is_public_address(peer): + connection.close() + connection = None + raise _fail("peer_changed", "TLS peer address was not the vetted DNS result") + break + except SafeHttpsError: + raise + except (OSError, ssl.SSLError, TimeoutError) as exc: + last_error = exc + if connection is not None: + connection.close() + connection = None + if connection is None: + raise _fail("connect_failed", "all vetted HTTPS addresses failed") from last_error + try: + request = ( + f"GET {current.path} HTTP/1.1\r\n" + f"Host: {current.host}\r\n" + "User-Agent: YOLOZU-Algorithm-Scout/1\r\n" + "Accept: application/json, application/pdf, text/html, text/plain\r\n" + "Accept-Encoding: gzip, deflate\r\n" + "Connection: close\r\n\r\n" + ).encode("ascii") + connection.sendall(request) + reader = _Reader(connection, b"", deadline, self.limits.read_timeout_seconds) + header_bytes = reader.until(b"\r\n\r\n", self.limits.max_header_bytes) + status, headers = _parse_headers(header_bytes) + if status in _REDIRECT_STATUSES: + raw_location = _single_header(headers, "location") + if raw_location is None: + raise _fail("redirect_invalid", "redirect location is missing") + redirects += 1 + if redirects > self.limits.max_redirects: + raise _fail("redirect_limit", "redirect count exceeds its cap") + target = HttpsLocation.from_redirect(raw_location, base=current) + if target not in self.allowlist: + raise _fail("redirect_not_allowlisted", "redirect target is not allowlisted") + current = target + continue + if status != 200: + raise _fail("http_status", "allowlisted source returned a non-success status") + raw_content_type = _single_header(headers, "content-type") + if raw_content_type is None: + raise _fail("content_type_invalid", "content type is missing") + content_type = raw_content_type.split(";", 1)[0].strip().lower() + if content_type not in _ALLOWED_CONTENT_TYPES: + raise _fail("content_type_invalid", "content type is not an allowed document type") + transferred = _read_body(reader, headers, self.limits.max_transferred_bytes) + decoded = _decode_body( + transferred, + _single_header(headers, "content-encoding"), + self.limits.max_decoded_bytes, + ) + return FetchedDocument( + source=source, + final_location=current, + content_type=content_type, + body=decoded, + transferred_bytes=len(transferred), + decoded_bytes=len(decoded), + redirect_count=redirects, + ) + finally: + connection.close() diff --git a/yolozu/cli_commands.py b/yolozu/cli_commands.py index ef9fde3c..c3481cf9 100644 --- a/yolozu/cli_commands.py +++ b/yolozu/cli_commands.py @@ -1187,6 +1187,56 @@ def _cmd_activate_qualification_evidence(args: argparse.Namespace) -> int: return 0 if outcome.status in {"dry_run_ready", "applied"} else 2 +def _cmd_scout_algorithms(args: argparse.Namespace) -> int: + from yolozu.adaptive.algorithm_scout import ( + AlgorithmScoutError, + build_scout_plan, + collect_algorithm_candidates, + ) + + try: + plan = build_scout_plan( + sources_path=str(args.sources), + output_dir=str(args.output_dir), + collection_date=str(args.collection_date), + trigger=str(args.trigger), + workspace_root=str(args.workspace), + ) + except (AlgorithmScoutError, OSError, ValueError) as exc: + code = getattr(exc, "code", "input_invalid") + print(json.dumps({"ok": False, "error": str(code), "exit_code": 2}), file=sys.stderr) + return 2 + if not bool(args.collect): + print(json.dumps(plan.to_dict(), ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + try: + _report, exit_code, report_path = collect_algorithm_candidates( + plan, + workspace_root=str(args.workspace), + ) + except AlgorithmScoutError as exc: + input_codes = { + "history_limit", + "output_dir_invalid", + "prior_report_invalid", + "workspace_invalid", + } + exit_code = 2 if exc.code in input_codes else 1 + print( + json.dumps({"ok": False, "error": exc.code, "exit_code": exit_code}), + file=sys.stderr, + ) + return exit_code + except Exception: + print( + json.dumps({"ok": False, "error": "bounded_internal_failure", "exit_code": 1}), + file=sys.stderr, + ) + return 1 + print(str(report_path)) + return int(exit_code) + + def _cmd_doctor_import(args: argparse.Namespace) -> int: import time diff --git a/yolozu/cli_entry.py b/yolozu/cli_entry.py index 29667d3f..2027a92e 100644 --- a/yolozu/cli_entry.py +++ b/yolozu/cli_entry.py @@ -29,6 +29,7 @@ _cmd_predictions, _cmd_qualify_image_pipeline, _cmd_activate_qualification_evidence, + _cmd_scout_algorithms, _cmd_validate, _cmd_eval_instance_seg, _cmd_onnxrt_export, @@ -1369,6 +1370,42 @@ def main(argv: list[str] | None = None) -> int: help="Apply the validated atomic append; omission is always dry-run.", ) + scout = sub.add_parser( + "scout-algorithms", + help="Plan or collect a bounded monitored-source candidate inbox (Experimental).", + ) + scout.add_argument( + "--sources", + required=True, + help="Canonical docs/algorithm_intake/sources.json; no other source file is accepted.", + ) + scout.add_argument( + "--output-dir", + required=True, + help="Explicit workspace-confined parent for dated candidate reports.", + ) + scout.add_argument( + "--collection-date", + required=True, + help="Workflow-supplied Gregorian date in exact YYYY-MM-DD form.", + ) + scout.add_argument( + "--trigger", + required=True, + choices=("schedule", "workflow_dispatch"), + help="Validated workflow trigger identity.", + ) + scout.add_argument( + "--workspace", + default=".", + help="Workspace boundary (default: current directory).", + ) + scout.add_argument( + "--collect", + action="store_true", + help="Enable the only network/write path; omission prints a no-write JSON plan.", + ) + reg = sub.add_parser("registry", help="AI-first tool registry: list/show/validate/run tools from the canonical manifest.") reg_sub = reg.add_subparsers(dest="registry_command", required=True) reg_validate = reg_sub.add_parser("validate", help="Validate the canonical tool manifest (repo checkout required).") @@ -1456,6 +1493,8 @@ def main(argv: list[str] | None = None) -> int: return _cmd_qualify_image_pipeline(args) if args.command == "activate-qualification-evidence": return _cmd_activate_qualification_evidence(args) + if args.command == "scout-algorithms": + return _cmd_scout_algorithms(args) if args.command == "list": if args.list_command == "models": return _cmd_list_models(args) diff --git a/yolozu/data/manifest/tools_manifest.json b/yolozu/data/manifest/tools_manifest.json index fe85a07d..1b8b7121 100644 --- a/yolozu/data/manifest/tools_manifest.json +++ b/yolozu/data/manifest/tools_manifest.json @@ -88,6 +88,15 @@ "schema": "docs/schemas/algorithm_bundle_spec.schema.json", "summary": "Immutable pipeline, artifact, runtime, vocabulary, and execution facts; presence is not qualification or availability evidence." }, + "algorithm_scout_report_json": { + "producer": "python3 tools/scout_algorithms.py --sources docs/algorithm_intake/sources.json --output-dir reports/algorithm_scout --collection-date --trigger --collect", + "schema": "docs/schemas/algorithm_scout_report.schema.json", + "summary": "Dated Experimental monitored-source inbox metadata; this interface contract is not loadable as a bundle registry and is not qualification, support, recommendation, adoption, or promotion evidence." + }, + "algorithm_scout_sources_json": { + "schema": "docs/schemas/algorithm_scout_sources.schema.json", + "summary": "Canonical credential-free HTTPS scheme/host/path allowlist for the bounded algorithm scout." + }, "artifact_research_qualification_json": { "producer": "python3 tools/qualify_artifact_research.py", "schema": "docs/schemas/artifact_research_qualification.schema.json", @@ -17305,6 +17314,127 @@ "reproducibility" ] }, + { + "contracts": { + "consumes": [ + "algorithm_scout_sources_json" + ], + "produces": [ + "algorithm_scout_report_json", + "managed_output_checksums_json" + ] + }, + "docs": [ + "README.md", + "Readme_jp.md", + "docs/README.md", + "docs/adaptive_image_routing.md", + "docs/algorithm_intake/README.md", + "docs/schemas/algorithm_scout_report.schema.json", + "docs/schemas/algorithm_scout_sources.schema.json", + "manual/chapters/04_cli_reference.tex", + "reports/adaptive_algorithm_scout_foundation_2026-08-26.md" + ], + "effects": { + "fixed_writes": [], + "writes": [ + { + "description": "With --collect only, atomically publish one dated bounded candidate inbox and checksums manifest inside the workspace.", + "flag": "--output-dir", + "kind": "dir", + "scope": "tree" + } + ] + }, + "entrypoint": "tools/scout_algorithms.py", + "examples": [ + { + "command": "python3 tools/scout_algorithms.py --sources docs/algorithm_intake/sources.json --output-dir reports/algorithm_scout --collection-date 2026-08-26 --trigger workflow_dispatch", + "description": "Validate the canonical allowlist and print a network-free, write-free JSON plan." + }, + { + "command": "python3 tools/scout_algorithms.py --sources docs/algorithm_intake/sources.json --output-dir reports/algorithm_scout --collection-date 2026-08-26 --trigger workflow_dispatch --collect", + "description": "Collect only bounded metadata from the explicit official-source allowlist and publish one dated inbox report." + }, + { + "command": "python3 tools/scout_algorithms.py --help", + "description": "Inspect the Experimental monitored-source interface contract." + } + ], + "id": "scout_algorithms", + "inputs": [ + { + "flag": "--sources", + "kind": "file", + "name": "sources", + "required": true + }, + { + "flag": "--output-dir", + "kind": "dir", + "name": "output_dir", + "required": true + }, + { + "flag": "--collection-date", + "kind": "string", + "name": "collection_date", + "required": true + }, + { + "flag": "--trigger", + "kind": "string", + "name": "trigger", + "required": true + }, + { + "default": ".", + "flag": "--workspace", + "kind": "dir", + "name": "workspace", + "required": false + }, + { + "flag": "--collect", + "kind": "string", + "name": "collect", + "required": false + } + ], + "maturity": "experimental", + "outputs": [ + { + "default": "//algorithm_scout_report.json", + "description": "Dated candidate inbox with explicit failed/unknown fields and no retained raw source document.", + "kind": "file", + "name": "algorithm_scout_report" + }, + { + "default": "//checksums.json", + "description": "Code-owned checksum manifest for the dated report.", + "kind": "file", + "name": "checksums" + } + ], + "platform": { + "cpu_ok": true, + "gpu_required": false, + "linux_ok": true, + "macos_ok": true + }, + "requires": { + "network": true + }, + "runner": "python3", + "summary": "Plan by default or collect a bounded read-only Experimental inbox from exact official HTTPS sources; discovered data cannot execute, download weights, mutate the bundle registry, qualify, or promote anything.", + "tags": [ + "adaptive-inference", + "algorithm-scout", + "candidate", + "experimental", + "provenance" + ] + }, { "contracts": { "consumes": [ diff --git a/yolozu/data/schemas/algorithm_scout_report.schema.json b/yolozu/data/schemas/algorithm_scout_report.schema.json new file mode 100644 index 00000000..38704240 --- /dev/null +++ b/yolozu/data/schemas/algorithm_scout_report.schema.json @@ -0,0 +1,427 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://www.toppymicros.com/yolozu/schemas/algorithm_scout_report.schema.json", + "title": "YOLOZU Experimental algorithm scout report", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "schema_version", + "maturity", + "selectability", + "collection_date", + "trigger", + "started_at", + "completed_at", + "timezone", + "source_policy", + "retention", + "automation_caveat", + "sources", + "candidates", + "summary" + ], + "properties": { + "kind": { + "const": "yolozu_algorithm_scout_report" + }, + "schema_version": { + "const": 1 + }, + "maturity": { + "const": "experimental" + }, + "selectability": { + "const": "inbox_only" + }, + "collection_date": { + "$ref": "#/$defs/date" + }, + "trigger": { + "enum": [ + "schedule", + "workflow_dispatch" + ] + }, + "started_at": { + "$ref": "#/$defs/utc" + }, + "completed_at": { + "$ref": "#/$defs/utc" + }, + "timezone": { + "const": "UTC" + }, + "source_policy": { + "type": "object", + "additionalProperties": false, + "required": [ + "canonical_sources", + "scope", + "coverage", + "redirect_limit", + "network" + ], + "properties": { + "canonical_sources": { + "const": "docs/algorithm_intake/sources.json" + }, + "scope": { + "const": "explicit_official_allowlist_only" + }, + "coverage": { + "const": "monitored_sources_not_latest_world" + }, + "redirect_limit": { + "const": 3 + }, + "network": { + "const": "https_443_collect_only" + } + } + }, + "retention": { + "type": "object", + "additionalProperties": false, + "required": [ + "raw_documents", + "candidate_metadata", + "history_reports_scanned_max" + ], + "properties": { + "raw_documents": { + "const": "not_retained" + }, + "candidate_metadata": { + "const": "repository_policy" + }, + "history_reports_scanned_max": { + "type": "integer", + "minimum": 1, + "maximum": 104 + } + } + }, + "automation_caveat": { + "type": "string", + "minLength": 1, + "maxLength": 512 + }, + "sources": { + "type": "array", + "minItems": 1, + "maxItems": 128, + "items": { + "$ref": "#/$defs/source_result" + } + }, + "candidates": { + "type": "array", + "maxItems": 10000, + "items": { + "$ref": "#/$defs/candidate" + } + }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": [ + "enabled_sources", + "collected_sources", + "failed_or_missed_sources", + "candidate_count", + "current_candidate_count", + "historical_candidate_count", + "decoded_total_bytes", + "network_used", + "writes_performed" + ], + "properties": { + "enabled_sources": { + "$ref": "#/$defs/nonnegative" + }, + "collected_sources": { + "$ref": "#/$defs/nonnegative" + }, + "failed_or_missed_sources": { + "$ref": "#/$defs/nonnegative" + }, + "candidate_count": { + "$ref": "#/$defs/nonnegative" + }, + "current_candidate_count": { + "$ref": "#/$defs/nonnegative" + }, + "historical_candidate_count": { + "$ref": "#/$defs/nonnegative" + }, + "decoded_total_bytes": { + "type": "integer", + "minimum": 0, + "maximum": 536870912 + }, + "network_used": { + "const": true + }, + "writes_performed": { + "const": true + } + } + } + }, + "$defs": { + "date": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + }, + "utc": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + }, + "nonnegative": { + "type": "integer", + "minimum": 0 + }, + "status": { + "enum": [ + "available", + "unavailable", + "unknown" + ] + }, + "source_result": { + "type": "object", + "required": [ + "source_id", + "source_url", + "collected_at", + "timezone", + "collection_status" + ], + "properties": { + "source_id": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "source_url": { + "type": "string", + "pattern": "^https://[^/?#:]+/[^?#]*$", + "maxLength": 4096 + }, + "collected_at": { + "$ref": "#/$defs/utc" + }, + "timezone": { + "const": "UTC" + }, + "collection_status": { + "enum": [ + "collected", + "failed", + "missed" + ] + }, + "failure_code": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "candidate_count": { + "$ref": "#/$defs/nonnegative" + }, + "provenance": { + "type": "object" + } + }, + "additionalProperties": false + }, + "candidate": { + "type": "object", + "additionalProperties": false, + "required": [ + "candidate_key", + "source_id", + "source_url", + "identity", + "version", + "revision", + "release_date", + "tasks", + "availability", + "license", + "weights", + "runtime_hints", + "collection_status", + "collected_at", + "timezone", + "first_seen_date", + "last_seen_date", + "history" + ], + "properties": { + "candidate_key": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "source_id": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "source_url": { + "type": "string", + "pattern": "^https://[^/?#:]+/[^?#]*$", + "maxLength": 4096 + }, + "identity": { + "type": "object", + "additionalProperties": false, + "required": [ + "project", + "model" + ], + "properties": { + "project": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "model": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "revision": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "release_date": { + "type": "string", + "minLength": 1, + "maxLength": 10 + }, + "tasks": { + "type": "array", + "maxItems": 32, + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64 + } + }, + "availability": { + "type": "object", + "additionalProperties": false, + "required": [ + "local", + "hosted" + ], + "properties": { + "local": { + "$ref": "#/$defs/status" + }, + "hosted": { + "$ref": "#/$defs/status" + } + } + }, + "license": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "expression" + ], + "properties": { + "status": { + "enum": [ + "approved", + "rejected", + "review_required", + "unknown" + ] + }, + "expression": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } + }, + "weights": { + "type": "object", + "additionalProperties": false, + "required": [ + "status" + ], + "properties": { + "status": { + "$ref": "#/$defs/status" + } + } + }, + "runtime_hints": { + "type": "array", + "maxItems": 32, + "items": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + }, + "collection_status": { + "enum": [ + "collected", + "historical" + ] + }, + "collected_at": { + "$ref": "#/$defs/utc" + }, + "timezone": { + "const": "UTC" + }, + "first_seen_date": { + "$ref": "#/$defs/date" + }, + "last_seen_date": { + "$ref": "#/$defs/date" + }, + "history": { + "type": "array", + "maxItems": 104, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "collection_date", + "collected_at", + "content_sha256" + ], + "properties": { + "collection_date": { + "$ref": "#/$defs/date" + }, + "collected_at": { + "$ref": "#/$defs/utc" + }, + "content_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + } + } + } + } + } +} diff --git a/yolozu/data/schemas/algorithm_scout_sources.schema.json b/yolozu/data/schemas/algorithm_scout_sources.schema.json new file mode 100644 index 00000000..33ff07fd --- /dev/null +++ b/yolozu/data/schemas/algorithm_scout_sources.schema.json @@ -0,0 +1,185 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://www.toppymicros.com/yolozu/schemas/algorithm_scout_sources.schema.json", + "title": "YOLOZU algorithm scout source allowlist", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "schema_version", + "sources" + ], + "properties": { + "kind": { + "const": "yolozu_algorithm_scout_sources" + }, + "schema_version": { + "const": 1 + }, + "sources": { + "type": "array", + "minItems": 1, + "maxItems": 128, + "items": { + "$ref": "#/$defs/source" + } + } + }, + "$defs": { + "location": { + "type": "object", + "additionalProperties": false, + "required": [ + "scheme", + "host", + "path" + ], + "properties": { + "scheme": { + "const": "https" + }, + "host": { + "type": "string", + "pattern": "^[a-zA-Z0-9.-]{1,253}$" + }, + "path": { + "type": "string", + "pattern": "^/[^?#]{0,4095}$" + } + } + }, + "status": { + "enum": [ + "available", + "unavailable", + "unknown" + ] + }, + "source": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_id", + "enabled", + "location", + "redirect_allowlist", + "identity", + "tasks", + "availability", + "license", + "weights", + "runtime_hints" + ], + "properties": { + "source_id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]{0,63}$" + }, + "enabled": { + "type": "boolean" + }, + "location": { + "$ref": "#/$defs/location" + }, + "redirect_allowlist": { + "type": "array", + "maxItems": 3, + "items": { + "$ref": "#/$defs/location" + } + }, + "identity": { + "type": "object", + "additionalProperties": false, + "required": [ + "project", + "model" + ], + "properties": { + "project": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "model": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } + }, + "tasks": { + "type": "array", + "maxItems": 32, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1, + "maxLength": 64 + } + }, + "availability": { + "type": "object", + "additionalProperties": false, + "required": [ + "local", + "hosted" + ], + "properties": { + "local": { + "$ref": "#/$defs/status" + }, + "hosted": { + "$ref": "#/$defs/status" + } + } + }, + "license": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "expression" + ], + "properties": { + "status": { + "enum": [ + "approved", + "rejected", + "review_required", + "unknown" + ] + }, + "expression": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } + }, + "weights": { + "type": "object", + "additionalProperties": false, + "required": [ + "status" + ], + "properties": { + "status": { + "$ref": "#/$defs/status" + } + } + }, + "runtime_hints": { + "type": "array", + "maxItems": 32, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } + } + } + } +}