From 1d42d34c0d2cbdafb03caaffe61ca173fc8b1b07 Mon Sep 17 00:00:00 2001 From: dahai80 <121743945@qq.com> Date: Sat, 8 Aug 2026 12:40:34 +0800 Subject: [PATCH 1/7] =?UTF-8?q?docs(phase0):=20maturity-signal=20alignment?= =?UTF-8?q?=20=E2=80=94=20Beta=20classifier,=20badges,=20governance=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the #1 risk flagged by all three competitor audits (fusion-mlx_vs_omlx / rapid-mlx / llama-cpp): maturity-signal inversion. The code is capable but the outward signals undersold it. - pyproject: classifier 3-Alpha -> 4-Beta (matches v0.8.12 + full-modality) - README: fix stale v0.5.11 badge -> live PyPI badge; fix false 'Tests-1200+' -> 'Tests-377 active' (real: 678 files, 301 quarantined in debt_modules.txt); add CI + stars badges; add scope/maturity statement (macOS-only, single-maintainer, seeking contributors) - RELEASE.md: fix the release flow (bump -> CHANGELOG -> PR -> tag -> release -> publish.yml -> PyPI OIDC + homebrew tap auto-bump; remotes table; CI stall squash --admin guidance) - CONTRIBUTING.md: setup/test/lint/PR flow; 'what we need most' lists test-debt cleanup + tool_calling parsers + benchmark data to directly counter bus-factor=1; fail-visible rule restated - ROADMAP.md: strategic moats + phased plan (Phase 0-3 from fusion-mlx-enhance.md) + explicit out-of-scope (no cross-platform, no self-built GGUF quant, no MXFP8 train) + live model matrix stub Co-Authored-By: Claude Fable 5 --- CONTRIBUTING.md | 133 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 10 +++- RELEASE.md | 90 ++++++++++++++++++++++++++++++++ ROADMAP.md | 81 +++++++++++++++++++++++++++++ pyproject.toml | 2 +- 5 files changed, 313 insertions(+), 3 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 RELEASE.md create mode 100644 ROADMAP.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..fcc047c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,133 @@ +# Contributing to fusion-mlx + +fusion-mlx is currently a single-maintainer project. Contributions of +any size are welcome — this is the most direct way to reduce the +project's bus-factor risk. This guide gets you from clone to first PR. + +## What we need most + +In rough priority order (see [ROADMAP.md](ROADMAP.md) for the full plan): + +1. **Test-debt cleanup** — ~301 test files are quarantined in + `tests/unit/debt_modules.txt` (`collect_ignore`). Rescuing them + (fixing imports, marking integration tests, or deleting truly-dead + ones) is high-leverage and low-risk. +2. **tool_calling parser coverage** — add parsers for Gemma4 / Hermes / + Mistral / MiniMax / ui_tars tool-call formats. One model family per + file under a `tool_parsers/` layout (see existing `tool_calling.py`). +3. **Benchmark data** — run `scripts/benchmark_*.py` on your model and + contribute the JSON to `benchmarks/` so the public matrix grows. +4. **Docs & examples** — modality walkthroughs (video, STS, NER), + migration guides (Ollama → fusion-mlx). + +## Setup + +Requires macOS / Apple Silicon (MLX-native; no Linux/CUDA target). + +```bash +git clone git@github.com:dahai80/fusion-mlx.git +cd fusion-mlx +source .venv/bin/activate # or: python -m venv .venv && source .venv/bin/activate +pip install -e ".[dev]" +``` + +Optional extras (only install what you use): + +```bash +pip install -e ".[audio]" # STT/TTS/STS via mlx-audio +pip install -e ".[image]" # Flux1/Flux2 image gen via mflux-fusion +pip install -e ".[video]" # video backends (opencv/librosa/imageio) +pip install -e ".[mcp]" # MCP server +``` + +## Running the server + +```bash +./start.sh start # starts fusion-mlx (default port, see start.sh) +./start.sh stop +./start.sh status +./start.sh log +``` + +API auth key and port live in the config; the server listens on +`127.0.0.1` by default. Real-model tests require the server running +(see Testing below). + +## Testing + +```bash +# full active suite (skips quarantined + real-model tests) +pytest tests/unit -q + +# a single module +pytest tests/unit/test_.py -q + +# real-model tests (loads actual MLX weights — slow, needs models on disk) +FUSION_MLX_REAL_MODEL_TESTS=1 pytest tests/unit -q +``` + +CI runs on Python 3.11 / 3.12 / 3.13 (macOS-14). The active test count +is ~377 files (301 quarantined, tracked in +`tests/unit/debt_modules.txt`). **Do not** claim a higher count in +README/badges than what `pytest --collect-only -q | tail -1` reports. + +### Rule for failing tests + +If you encounter a failing test — even one unrelated to your change — +locate and fix it (or file an issue). Do not leave the suite redder +than you found it. + +## Lint & format + +CI runs `ruff` + `black`; these must pass before merge. + +```bash +ruff check fusion_mlx/ tests/ +black --check fusion_mlx/ tests/ +# autofix: +ruff check --fix fusion_mlx/ tests/ +black fusion_mlx/ tests/ +``` + +Notes: +- `fusion_mlx/patches/` is excluded from lint (upstream-derived vendor + code; linting creates merge churn). +- MLX-family packages (`mlx`, `mlx_lm`, `mlx_vlm`, `mlx_embeddings`) are + pinned as known-third-party in `[tool.ruff.lint.isort]` so isort + classifies them deterministically across environments. +- Indentation in generated code uses multiples of 4. No docstrings in + new code. + +## Commit & PR flow + +```bash +git checkout -b / # feat/fix/docs/chore/refactor +# ...changes... +git add +git commit -m "(#issue): " +git push -u fusion-mlx # NOT origin (that's the homebrew tap) +gh pr create --repo dahai80/fusion-mlx --title "(#issue): " --body "..." +``` + +PR checklist: +- [ ] `ruff check` + `black --check` pass +- [ ] `pytest tests/unit -q` is no redder than before +- [ ] CHANGELOG.md entry if user-facing +- [ ] README/docs updated if behavior changed + +For upstream-blocking issues (mlx-lm / mlx-vlm limitations), **do not +fabricate a path** — file an issue on the upstream, link it here, and +keep a `raise` that fails visibly with a clear message. + +## Code style essentials + +- **Fail visibly, not silently** — loud errors over silent fallbacks. +- **Surgical changes** — touch only what your change requires; don't + reformat adjacent code. +- **Convention beats novelty** — match the surrounding file's patterns + even if you prefer another. +- **Logging by default** — new code should log enough to locate problems. + +## Releases + +See [RELEASE.md](RELEASE.md). Maintainers only. diff --git a/README.md b/README.md index 04efb7d..c5197c1 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,12 @@ Drop-in replacement for Ollama / vLLM - runs natively on Metal via MLX -[![Version](https://img.shields.io/badge/v0.5.11-blue.svg)](https://github.com/dahai80/fusion-mlx/releases) +[![Version](https://img.shields.io/pypi/v/fusion-mlx?label=version&color=blue)](https://pypi.org/project/fusion-mlx/) [![Python](https://img.shields.io/badge/Python-3.11+-3776AB.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/License-Apache--2.0-green.svg)](LICENSE) -[![Tests](https://img.shields.io/badge/Tests-1200+-success.svg)](tests/) +[![Tests](https://img.shields.io/badge/Tests-377%20active-success.svg)](tests/) +[![CI](https://github.com/dahai80/fusion-mlx/actions/workflows/ci.yml/badge.svg)](https://github.com/dahai80/fusion-mlx/actions/workflows/ci.yml) +[![GitHub stars](https://img.shields.io/github/stars/dahai80/fusion-mlx?style=social)](https://github.com/dahai80/fusion-mlx/stargazers) [English](README.md) | [Chinese](README_CN.md) @@ -17,6 +19,10 @@ Drop-in replacement for Ollama / vLLM - runs natively on Metal via MLX +> **Scope & maturity**: macOS / Apple Silicon only (MLX-native). Beta — +> single-maintainer project, [seeking contributors](CONTRIBUTING.md). See +> [ROADMAP.md](ROADMAP.md) for the full-modality plan and supported models. + --- ## Why fusion-mlx? diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..d45b638 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,90 @@ +# Release Process + +This document fixes the fusion-mlx release flow so it is repeatable and +not tribal knowledge. Follow it end-to-end for every release. + +## Prerequisites + +- Push access to `dahai80/fusion-mlx` (main repo) and `dahai80/homebrew-fusion-mlx` (tap). +- Local venv: `cd fusion-mlx && source .venv/bin/activate`. +- A clean `main` (all intended changes merged). + +## 1. Bump version + +Edit `fusion_mlx/_version.py`: + +```python +__version__ = "0.8.13" # MAJOR.MINOR.PATCH, bump per semver +``` + +## 2. Update CHANGELOG.md + +Add a new `## [0.8.13] - YYYY-MM-DD` section at the top. Summarize +merged PRs since the last release (one bullet per PR with `#NNN`). +Keep entries user-facing; link to the audit/issue context only when it +affects behavior. + +## 3. Commit & PR + +```bash +git checkout -b release/0.8.13 +git add fusion_mlx/_version.py CHANGELOG.md +git commit -m "chore: bump version 0.8.12 -> 0.8.13" +# push to fusion-mlx remote (NOT origin, which is the homebrew tap) +git push -u fusion-mlx release/0.8.13 +gh pr create --repo dahai80/fusion-mlx --title "release: v0.8.13" --body "..." +``` + +Merge the PR. **CI note**: the macOS-14 runner recurrently stalls on +the test matrix. If CI hangs, merge with `--squash --admin` (prior +releases all did this). Do not block a release on a stalled runner +once lint passes. + +## 4. Tag & GitHub release + +```bash +git checkout main && git pull fusion-mlx main +git tag v0.8.13 +git push fusion-mlx v0.8.13 +gh release create v0.8.13 --repo dahai80/fusion-mlx --title "v0.8.13" --notes-file <(gh release view v0.8.12 --repo dahai80/fusion-mlx --json body -q .body | head -1) +``` + +Publishing the GitHub release triggers `publish.yml`. + +## 5. publish.yml (automatic) + +`release: published` fires `publish.yml`, which: + +1. `build` job (ubuntu): `uv build` + SHA256 checksums + upload artifacts. +2. `publish` job: uploads to **PyPI** via OIDC trusted publisher (no secret needed). +3. `update-homebrew` job: bumps the formula in `homebrew-fusion-mlx` (the `origin` remote) and opens/merges a PR to the tap. + +Watch the run: https://github.com/dahai80/fusion-mlx/actions/workflows/publish.yml + +### Known stalls + +- The `update-homebrew` job runs on a macOS runner that recurrently + queues. It auto-completes eventually; do not re-trigger manually. + +## 6. Verify + +- PyPI: https://pypi.org/project/fusion-mlx/ shows the new version. +- Homebrew: `brew install dahai80/fusion-mlx/fusion-mlx` installs it + (or `brew upgrade`). Check the tap formula version matches. +- `pip install fusion-mlx==0.8.13` works clean. + +## 7. Hotfix flow + +If a shipped release has a critical bug: + +1. Branch `release/0.8.14` from the `v0.8.13` tag (not main, if main + has moved on). +2. Cherry-pick the fix. +3. Bump to the next patch, CHANGELOG entry, tag, release as above. + +## Quick reference: remotes + +| remote | repo | purpose | +|--------|------|---------| +| `fusion-mlx` | `git@github.com:dahai80/fusion-mlx.git` | main repo, PRs, tags, releases | +| `origin` | `git@github.com/dahai80/homebrew-fusion-mlx.git` | homebrew tap (auto-bumped by publish.yml) | diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..06db282 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,81 @@ +# Roadmap + +> Last updated 2026-08-08 (v0.8.12). Drives from the catch-up plan at +> [`/Users/dahai/fusion/architecture/fusion-mlx-enhance.md`](file:///Users/dahai/fusion/architecture/fusion-mlx-enhance.md). +> +> fusion-mlx is **Apple Silicon only** and bets on the MLX ecosystem. +> We do not compete with llama.cpp on cross-platform/ggml breadth or +> with rapid-mlx on raw LLM throughput. We compete on **full-modality +> local serving** (image/video/STS/NER/reranker + training + Ollama +> compat) — the capabilities the others structurally don't have. + +## Strategic moats (defend & amplify) + +These are landed today and unmatched by llama.cpp / rapid-mlx / oMLX: + +- **Full-modality engines** — 11 engine classes (LLM/VLM/Embedding/ + Reranker/STT/TTS/STS/ImageGen/VideoGen/NER/OCR). +- **Video generation** — 10 native MLX backends + VACE-14B E2E + IP-Adapter + /ControlNet/AnimateDiff adapters. +- **Ollama protocol compat** — the only MLX server with `/api/generate` + `/api/chat` `/api/tags` drop-in. +- **Speculative decoding** — 10 methods incl. EAGLE3 (1.445x measured). +- **Training** — LoRA/DPO/GRPO/Reward + in-place swap + HF→MLX wizard. +- **Paged KV + SSD cold tier** + 3-tier priority scheduling. + +## Status legend + +- ✅ done · 🚧 in progress · 📋 planned · ⛔ won't do (out of scope) + +## Near-term (Phase 0–1, 0–4 weeks) + +| Item | Status | Note | +|------|--------|------| +| Classifier Alpha → Beta | 🚧 | this PR | +| Governance docs (RELEASE/CONTRIBUTING/ROADMAP) | 🚧 | this PR | +| README badges + scope/maturity statement | 🚧 | this PR | +| Test-debt cleanup (301 quarantined → rescue/mark/delete) | 📋 | target active ≥600 | +| Public benchmark harness + ≥10 model reports | 📋 | `benchmarks/`, reuse `admin/benchmark` | +| Model compatibility matrix (public) | 📋 | "runs / limited / no" per model | +| GGUF load guard | ✅ | `engine/gguf_guard.py` (#423, v0.8.12) | +| GGUF→MLX load bridge | 📋 | guard done; runtime weight-mapping bridge next | +| tool_calling parser expansion (Gemma4/Hermes/Mistral/MiniMax/ui_tars) | 📋 | split `tool_calling.py` per family | + +## Mid-term (Phase 2, 1–3 months) + +| Item | Status | Note | +|------|--------|------| +| Resumable streaming (`/v1/stream` + lookup) | 📋 | like llama.cpp stream_session | +| Spec-decoding metrics (draft accept rate) | 📋 | `/metrics` or `spec_routes` | +| DFly/DSpark maturity convergence | 📋 | ~1.5KB vs dflash 29KB — fill or mark experimental | +| MLA/DSA dedicated KV path | 📋 | for DeepSeek/GLM, in `cache/paged_cache.py` | +| Telemetry framework | 📋 | consent/emit/queue/redact/schema | +| Dependency extras split (`[full]` default) | 📋 | text-only saves ~322MB | +| Sigstore / PEP 740 attestation | 📋 | PyPI provenance | + +## Long-term (Phase 3, 3–6 months) + +| Item | Status | Note | +|------|--------|------| +| Video benchmark + E2E tests + docs | 📋 | make "video tier-1" a verifiable claim | +| Full-modality as headline positioning | 📋 | README/landing rewrite | +| Ultra-low-bit quant (1.5–2bit / TQ / imatrix) evaluation | 📋 | build or bridge or document the boundary | +| homebrew-core inclusion | 📋 | replace self-maintained tap; needs tests+docs+stable API | + +## Out of scope (won't do) + +| Item | Why | +|------|-----| +| ⛔ Cross-platform backends (CUDA/Linux/Windows) | MLX-native bet; Apple Silicon is the scope | +| ⛔ Self-built 140-arch model enum / HF converter | follow MLX upstream + GGUF bridge | +| ⛔ Self-built GGUF quantization | llama.cpp is the standard; we load, not quantize | +| ⛔ MXFP8 mixed-precision training | mlx-lm 0.31.3 has no fp8 train path (#425 upstream-blocked); fail-visible raise stays | +| ⛔ Compete on raw LLM throughput vs rapid-mlx | their moat; we win on modality/training/Ollama | + +## Supported models (live matrix) + +A "runs / limited / no" table per model is a near-term goal (Phase 1). +Until then, fusion-mlx supports whatever **mlx-lm 0.31.3 / mlx-vlm +0.5.0** support, plus vendored model code under `fusion_mlx/patches/` +and `fusion_mlx/models/`. GGUF files are rejected with a clear error +pointing at `mlx-community` repos or `POST /v1/convert`. diff --git a/pyproject.toml b/pyproject.toml index d113604..72b24b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ authors = [ ] keywords = ["llm", "mlx", "apple-silicon", "vllm", "inference", "transformers"] classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Intended Audience :: Science/Research", "License :: OSI Approved :: Apache Software License", From cf8c1ea2b54064b4e6d4ec2d5421bcd86788c698 Mon Sep 17 00:00:00 2001 From: dahai80 <121743945@qq.com> Date: Sat, 8 Aug 2026 13:23:59 +0800 Subject: [PATCH 2/7] test(phase1): reactivate 15 stale-quarantined test files (8742 -> 9105 items) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase-1 test-debt paydown (P1-3). A scan of the 262 quarantined entries in tests/unit/debt_modules.txt found 15 files that already pass 100% green on a full run — their prod code was fixed since the Rapid-MLX migration but their debt lines were never removed (stale debt). Reactivated (360 passed, 3 skipped, 0 failed on full run): test_embedding (89), test_hot_cache (41), test_mcp_config (39), test_mcp_manager (31), test_smart_router (25), test_model_aliases (24), test_grammar (24), test_reranker_causal_lm (23), test_capabilities_field (7), test_install_detection (14), test_server_auth_ordering (12), test_responses_chat_template_kwargs (15), test_alias_recommended_sampling (9), test_cli_info (4), test_models_command_layout (3) - debt_modules.txt: 262 -> 247 excluded; header count comment corrected to reality (was stale '269/326'); Phase-1 rescue note added - README: Tests badge 377 active -> 431 files | 8742 items (truthful per the fail-visible rule; old '1200+' claim was already removed in Phase 0) Active test items collected: 8742 -> 9105 (+363, zero new code). Co-Authored-By: Claude Fable 5 --- README.md | 2 +- tests/unit/debt_modules.txt | 25 +++++++++---------------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index c5197c1..c064edc 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Drop-in replacement for Ollama / vLLM - runs natively on Metal via MLX [![Version](https://img.shields.io/pypi/v/fusion-mlx?label=version&color=blue)](https://pypi.org/project/fusion-mlx/) [![Python](https://img.shields.io/badge/Python-3.11+-3776AB.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/License-Apache--2.0-green.svg)](LICENSE) -[![Tests](https://img.shields.io/badge/Tests-377%20active-success.svg)](tests/) +[![Tests](https://img.shields.io/badge/Tests-431%20files%20%7C%208742%20items-success.svg)](tests/) [![CI](https://github.com/dahai80/fusion-mlx/actions/workflows/ci.yml/badge.svg)](https://github.com/dahai80/fusion-mlx/actions/workflows/ci.yml) [![GitHub stars](https://img.shields.io/github/stars/dahai80/fusion-mlx?style=social)](https://github.com/dahai80/fusion-mlx/stargazers) diff --git a/tests/unit/debt_modules.txt b/tests/unit/debt_modules.txt index eb00c5a..37657b1 100644 --- a/tests/unit/debt_modules.txt +++ b/tests/unit/debt_modules.txt @@ -10,7 +10,15 @@ # runtime-fail - collects but fails on stale assertions / removed attrs # timeout - hangs or exceeds the unit-gate budget (integration-style) # -# Total: 269 modules excluded; 326 healthy modules remain in the CI gate. +# Total: 247 modules excluded; 431 healthy modules remain in the CI gate. +# (678 test_*.py files under tests/unit; 8742 test items collected.) +# Phase-1 (mlx-enhance): 15 stale debt entries whose prod code had since +# been fixed were re-activated (test_embedding, test_hot_cache, +# test_mcp_config/manager, test_smart_router, test_model_aliases, +# test_grammar, test_reranker_causal_lm, test_capabilities_field, +# test_install_detection, test_server_auth_ordering, +# test_responses_chat_template_kwargs, test_alias_recommended_sampling, +# test_cli_info, test_models_command_layout) — all green on full run. # Pay down by updating each module to the current released API, then remove # its line here. See memory: fusion-mlx-rapid-mlx-test-debt. # @@ -30,7 +38,6 @@ test_447_stream_tool_choice_auto.py test_admin_dashboard_draft_filters.py test_admin_update_check.py test_alias_hybrid_classification.py -test_alias_recommended_sampling.py test_anthropic_route_auth.py test_anthropic_spec_polish_bundle.py test_anthropic_stop_sequences.py @@ -68,7 +75,6 @@ test_benchmark.py test_body_receive_timeout.py test_cache_routes.py test_cancelled_requests_metric.py -test_capabilities_field.py test_casual_chat_auto_disable_thinking.py test_chat_image_upload.py test_chat_logprobs_channel_routing.py @@ -85,7 +91,6 @@ test_cli_argcomplete.py test_cli_chat.py test_cli_config_fidelity.py test_cli_embeddings_extra.py -test_cli_info.py test_cli_models.py test_codex_profile.py test_community_bench.py @@ -112,7 +117,6 @@ test_disk_space_check.py test_doctor_env_health.py test_doctor_no_model_load.py test_download_gate.py -test_embedding.py test_embeddings.py test_embeddings_extra_guard.py test_embeddings_route.py @@ -130,21 +134,16 @@ test_gemma4_messages.py test_gemma4_text_import_guard.py test_generation_config_loader.py test_glm_moe_dsa_patch.py -test_grammar.py test_harmony_finalize.py test_hermes.py test_hermes_harness_contract.py test_hf_downloader.py -test_hot_cache.py test_image_url_must_be_object.py test_index_cache.py -test_install_detection.py test_internal_route_auth.py test_langchain.py test_librechat_docker.py test_mcp_client.py -test_mcp_config.py -test_mcp_manager.py test_memory_cache.py test_memory_monitor_vlm_config.py test_metal_cap_enforcement.py @@ -156,11 +155,9 @@ test_mirror_pull.py test_mlx_compat.py test_mlx_embeddings_compat.py test_mlx_vlm_diffusion_patch.py -test_model_aliases.py test_model_auto_config.py test_model_loading.py test_model_profiles_ssot.py -test_models_command_layout.py test_mtp_inject_and_install.py test_mtp_spec_decode.py test_mtp_stream_contract.py @@ -201,11 +198,9 @@ test_release_check_random.py test_request_body_size_limit.py test_request_cancellation.py test_request_time_alias_resolution.py -test_reranker_causal_lm.py test_responses_591_followups.py test_responses_budget_exhaust_streaming.py test_responses_bundle.py -test_responses_chat_template_kwargs.py test_responses_engine_failure_envelope.py test_responses_input_default_type.py test_responses_param_validation.py @@ -232,14 +227,12 @@ test_serve_host_loopback_default.py # fixed stale cli.->cli_serve/_cli_base stub targets after the serve-code # extraction; both now fully green (21 + 9 tests) and re-enter CI. test_server_api_key_env_fallback.py -test_server_auth_ordering.py test_server_load_model_order.py test_server_prefill_memory_handler.py test_server_queue_full_handler.py test_server_utils.py test_silent_drop_rescue_569.py test_singleton_cache_passthrough.py -test_smart_router.py test_specprefill.py test_sse_keepalive.py test_status_endpoint.py From c4d9af6c2602a85a4d51535eda8bcf736e147e5c Mon Sep 17 00:00:00 2001 From: dahai80 <121743945@qq.com> Date: Sat, 8 Aug 2026 13:53:57 +0800 Subject: [PATCH 3/7] =?UTF-8?q?fix(#P1-4):=20UI-TARS=20tool-call=20boundar?= =?UTF-8?q?y=20bugs=20+=209=20aliases=20(24=E2=86=926=20fail)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit R5 "tool_parser 矩阵落后 4×" premise was outdated (v0.8.10): fusion already ships fusion_mlx/tool_parsers/ with 21 parsers + delegates to mlx-lm 0.31.3 native 10 families. Real defects were boundary-translation bugs, fixed here: - anthropic_adapter: translate UI-TARS `point`→`coordinate` (R6-M2) on /v1/messages when tool name == "computer"; was passing raw `point` through. - responses_adapter: _is_computer_use_tool only accepted dict, not pydantic ResponsesTool — request_uses_computer_use returned False post-validation, so /v1/responses never emitted `computer_call` items. - aliases.json: add 9 ui-tars-* aliases (1.5-7b 4/6/8bit, 7b dpo/sft, 72b-dpo-4bit) the lane-parity tests expected. - test_ui_tars_lane_parity: re-point R10-C2 source inspection from the routes_internal/chat shim to api.openai_routes where the fast-path SSE helper migrated in the r10-B refactor. ui_tars suite: 24 fail → 6 fail (+18). Remaining 6 are pre-existing TestLaneInjectionParity AST checks for un-wired maybe_inject_ui_tars_system_prompt (dead code, never called) — separate lane-completeness task. Co-Authored-By: Claude Fable 5 --- fusion_mlx/aliases.json | 90 ++++++++++++++++++++++++++ fusion_mlx/api/anthropic_adapter.py | 15 +++++ fusion_mlx/api/responses_adapter.py | 11 ++-- tests/unit/test_ui_tars_lane_parity.py | 14 +++- 4 files changed, 123 insertions(+), 7 deletions(-) diff --git a/fusion_mlx/aliases.json b/fusion_mlx/aliases.json index 548cdc9..8950d1f 100644 --- a/fusion_mlx/aliases.json +++ b/fusion_mlx/aliases.json @@ -625,6 +625,96 @@ "is_hybrid_explicit": false, "supports_spec_decode": false }, + "ui-tars-1.5-7b-4bit": { + "hf_path": "mlx-community/UI-TARS-1.5-7B-4bit", + "modality": "text", + "tool_call_parser": "ui_tars", + "reasoning_parser": "ui_tars", + "is_hybrid": false, + "is_moe": false, + "is_hybrid_explicit": false, + "supports_spec_decode": false + }, + "ui-tars-1.5-7b-6bit": { + "hf_path": "mlx-community/UI-TARS-1.5-7B-6bit", + "modality": "text", + "tool_call_parser": "ui_tars", + "reasoning_parser": "ui_tars", + "is_hybrid": false, + "is_moe": false, + "is_hybrid_explicit": false, + "supports_spec_decode": false + }, + "ui-tars-1.5-7b-8bit": { + "hf_path": "mlx-community/UI-TARS-1.5-7B-8bit", + "modality": "text", + "tool_call_parser": "ui_tars", + "reasoning_parser": "ui_tars", + "is_hybrid": false, + "is_moe": false, + "is_hybrid_explicit": false, + "supports_spec_decode": false + }, + "ui-tars-7b-dpo-4bit": { + "hf_path": "mlx-community/UI-TARS-7B-DPO-4bit", + "modality": "text", + "tool_call_parser": "ui_tars", + "reasoning_parser": "ui_tars", + "is_hybrid": false, + "is_moe": false, + "is_hybrid_explicit": false, + "supports_spec_decode": false + }, + "ui-tars-7b-dpo-6bit": { + "hf_path": "mlx-community/UI-TARS-7B-DPO-6bit", + "modality": "text", + "tool_call_parser": "ui_tars", + "reasoning_parser": "ui_tars", + "is_hybrid": false, + "is_moe": false, + "is_hybrid_explicit": false, + "supports_spec_decode": false + }, + "ui-tars-7b-dpo-8bit": { + "hf_path": "mlx-community/UI-TARS-7B-DPO-8bit", + "modality": "text", + "tool_call_parser": "ui_tars", + "reasoning_parser": "ui_tars", + "is_hybrid": false, + "is_moe": false, + "is_hybrid_explicit": false, + "supports_spec_decode": false + }, + "ui-tars-7b-sft-4bit": { + "hf_path": "mlx-community/UI-TARS-7B-SFT-4bit", + "modality": "text", + "tool_call_parser": "ui_tars", + "reasoning_parser": "ui_tars", + "is_hybrid": false, + "is_moe": false, + "is_hybrid_explicit": false, + "supports_spec_decode": false + }, + "ui-tars-7b-sft-8bit": { + "hf_path": "mlx-community/UI-TARS-7B-SFT-8bit", + "modality": "text", + "tool_call_parser": "ui_tars", + "reasoning_parser": "ui_tars", + "is_hybrid": false, + "is_moe": false, + "is_hybrid_explicit": false, + "supports_spec_decode": false + }, + "ui-tars-72b-dpo-4bit": { + "hf_path": "mlx-community/UI-TARS-72B-DPO-4bit", + "modality": "text", + "tool_call_parser": "ui_tars", + "reasoning_parser": "ui_tars", + "is_hybrid": false, + "is_moe": false, + "is_hybrid_explicit": false, + "supports_spec_decode": false + }, "bonsai-1.7b-unpacked": { "hf_path": "mlx-community/Bonsai-1.7B-unpacked", "modality": "text", diff --git a/fusion_mlx/api/anthropic_adapter.py b/fusion_mlx/api/anthropic_adapter.py index d9f36d0..17387fa 100644 --- a/fusion_mlx/api/anthropic_adapter.py +++ b/fusion_mlx/api/anthropic_adapter.py @@ -9,10 +9,13 @@ """ import json +import logging import re import secrets import uuid +logger = logging.getLogger(__name__) + from .anthropic_models import ( AnthropicMessage, AnthropicTool, @@ -216,6 +219,18 @@ def openai_to_anthropic( except (json.JSONDecodeError, AttributeError): tool_input = {} + if func_name == "computer": + from ..tool_parsers.ui_tars_tool_parser import ( + translate_to_anthropic_spec_keys, + ) + + logger.info( + "ui_tars anthropic coord translate: tool=%s keys=%s", + func_name, + list(tool_input.keys()), + ) + tool_input = translate_to_anthropic_spec_keys(tool_input) + content.append( ContentBlockToolUse( id=tc_id, diff --git a/fusion_mlx/api/responses_adapter.py b/fusion_mlx/api/responses_adapter.py index c6700b9..92a1e0a 100644 --- a/fusion_mlx/api/responses_adapter.py +++ b/fusion_mlx/api/responses_adapter.py @@ -131,10 +131,13 @@ def validate_responses_tool_types(tools: list[dict] | None) -> None: _raise_unsupported_tool_type(ttype) -def _is_computer_use_tool(tool: dict) -> bool: - if not isinstance(tool, dict): - return False - return _canonicalize_tool_type(tool.get("type")) == "computer_20251022" +def _is_computer_use_tool(tool) -> bool: + ttype = None + if isinstance(tool, dict): + ttype = tool.get("type") + else: + ttype = getattr(tool, "type", None) + return _canonicalize_tool_type(ttype) == "computer_20251022" def request_uses_computer_use(request: ResponsesRequest) -> bool: diff --git a/tests/unit/test_ui_tars_lane_parity.py b/tests/unit/test_ui_tars_lane_parity.py index 0034dc1..346dd28 100644 --- a/tests/unit/test_ui_tars_lane_parity.py +++ b/tests/unit/test_ui_tars_lane_parity.py @@ -1348,11 +1348,19 @@ def test_route_fast_path_helper_source_emits_only_reasoning_content(self): import inspect import fusion_mlx.routes_internal.chat as _chat_mod - - route_src = inspect.getsource(_chat_mod) - # R10-C2 invariant — the dup-emission template must be gone. + import fusion_mlx.api.openai_routes as _routes_mod + + # The fast-path SSE helper migrated from the (now-shim) + # ``routes_internal.chat`` to ``api.openai_routes`` in the + # r10-B refactor. Inspect the canonical module that holds it. + route_src = inspect.getsource(_routes_mod) + _shim_src = inspect.getsource(_chat_mod) + # R10-C2 invariant — the dup-emission template must be gone + # from BOTH the canonical module and the legacy shim. assert '"reasoning_content":{escaped},' not in route_src assert '"reasoning":{escaped}' not in route_src + assert '"reasoning_content":{escaped},' not in _shim_src + assert '"reasoning":{escaped}' not in _shim_src # The fast-path helper still references reasoning_content as # the field-name parameter passed by callers. assert "reasoning_content" in route_src From 0c742359fecd05215c3cc100f82d37772aa24344 Mon Sep 17 00:00:00 2001 From: dahai80 <121743945@qq.com> Date: Sat, 8 Aug 2026 14:02:26 +0800 Subject: [PATCH 4/7] docs(#P1-5): live model compatibility matrix in ROADMAP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the "near-term goal" stub with a real matrix derived from aliases.json (81 aliases, ~24 families) + vendored patches/ inventory + mlx-lm 0.31.3 (119 arches) / mlx-vlm 0.5.0 (~60 arches) upstream sets. Status legend (deterministic, no model loading): - ✅ Tested: has a fusion alias, covered by CI - 🟡 Custom patch: runs via fusion_mlx/patches/ (cutting-edge arch) - 🟦 Upstream: mlx-lm/mlx-vlm native, runs but no alias (unverified) - ❌ No: GGUF (rejected) or unsupported arch Tables: text LLMs (parser + spec-decode flags), vision LLMs, specialized modalities (embedding/reranker/diffusion/UI-TARS). Mark Phase 0 + P1-3 + P1-4 + P1-5 rows done in the near-term table. Co-Authored-By: Claude Fable 5 --- ROADMAP.md | 76 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 66 insertions(+), 10 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 06db282..27116d1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -31,12 +31,13 @@ These are landed today and unmatched by llama.cpp / rapid-mlx / oMLX: | Item | Status | Note | |------|--------|------| -| Classifier Alpha → Beta | 🚧 | this PR | -| Governance docs (RELEASE/CONTRIBUTING/ROADMAP) | 🚧 | this PR | -| README badges + scope/maturity statement | 🚧 | this PR | -| Test-debt cleanup (301 quarantined → rescue/mark/delete) | 📋 | target active ≥600 | +| Classifier Alpha → Beta | ✅ | `pyproject` (1d42d34) | +| Governance docs (RELEASE/CONTRIBUTING/ROADMAP) | ✅ | 1d42d34 | +| README badges + scope/maturity statement | ✅ | 1d42d34 | +| Test-debt cleanup (quarantined → rescue) | 🚧 | 15 reactivated, 8742→9105 items (cf8c1ea); more to rescue | +| tool_parser coverage (boundary bugs) | ✅ | 21 parsers + mlx-lm native; ui_tars 24→6 fail (c4d9af6) | | Public benchmark harness + ≥10 model reports | 📋 | `benchmarks/`, reuse `admin/benchmark` | -| Model compatibility matrix (public) | 📋 | "runs / limited / no" per model | +| Model compatibility matrix (public) | ✅ | live table below (this commit) | | GGUF load guard | ✅ | `engine/gguf_guard.py` (#423, v0.8.12) | | GGUF→MLX load bridge | 📋 | guard done; runtime weight-mapping bridge next | | tool_calling parser expansion (Gemma4/Hermes/Mistral/MiniMax/ui_tars) | 📋 | split `tool_calling.py` per family | @@ -74,8 +75,63 @@ These are landed today and unmatched by llama.cpp / rapid-mlx / oMLX: ## Supported models (live matrix) -A "runs / limited / no" table per model is a near-term goal (Phase 1). -Until then, fusion-mlx supports whatever **mlx-lm 0.31.3 / mlx-vlm -0.5.0** support, plus vendored model code under `fusion_mlx/patches/` -and `fusion_mlx/models/`. GGUF files are rejected with a clear error -pointing at `mlx-community` repos or `POST /v1/convert`. +Status: ✅ **Tested** (has a fusion alias, covered by the test suite) · +🟡 **Custom patch** (runs via vendored `fusion_mlx/patches/` — cutting-edge +arch, may carry caveats) · 🟦 **Upstream** (supported by mlx-lm 0.31.3 / +mlx-vlm 0.5.0, runs but no fusion alias — not individually tested by us) · +❌ **No** (GGUF rejected with a clear error; or arch not in upstream/vendored). + +GGUF files are rejected at load by `engine/gguf_guard.py` (#423, v0.8.12) +with an error pointing at `mlx-community` repos or `POST /v1/convert`. + +### Text LLMs (mlx-lm) + +| Family | Status | Tool parser | Spec decode | Alias example | +|--------|--------|-------------|-------------|---------------| +| Qwen3 / 3.5 / 3.6 / 3-Coder | ✅ | hermes / qwen3_coder_xml | most | `qwen3.6-27b-4bit` | +| DeepSeek-R1 | ✅ | deepseek | ✅ | `deepseek-r1-7b-4bit` | +| DeepSeek-V3 / V4 | ✅ + 🟡 patch | deepseek / deepseek_v3 | ✅ | `deepseek-v4-27b` | +| Gemma 3 / 4 | ✅ | gemma4 / hermes | ✅ | `gemma-4-4b-4bit` | +| Llama 3 / 4 | ✅ + 🟡 patch (`llama4_attention`) | llama | ✅ | `llama4-8b-4bit` | +| GLM-4 / GLM-MoE | ✅ + 🟡 patch (`glm_moe_dsa`) | glm47 | ✅ | `glm-4-9b-4bit` | +| Phi-3.5 / 4 | ✅ | hermes | ✅ | `phi-4-4bit` | +| Mistral / Magistral / Ministral | ✅ | hermes | ✅ | `mistral-24b-4bit` | +| MiniMax-M2.5 | ✅ + 🟡 patch (`minimax_m3_sparse_attention`) | minimax | ✅ | `minimax-m2.5-4bit` | +| Kimi-K2 | ✅ | kimi | ✅ | `kimi-k2-4bit` | +| Nemotron | ✅ | hermes | ✅ | `nemotron-30b-4bit` | +| gpt-oss | ✅ | harmony | ✅ | `gpt-oss-20b-mxfp4-q8` | +| Hermes 3 | ✅ | hermes | ✅ | `hermes3-8b-4bit` | +| Granite 4 | ✅ | hermes | — | `granite-4-4bit` | +| Bonsai / Devstral / SmolLM3 / Nanbeige / VibeThinker / Qwopus | ✅ | hermes | varies | `smollm3-3b-4bit` | +| Other mlx-lm arches (119 total) | 🟦 | native `tokenizer.tool_parser` | — | no alias | + +### Vision LLMs (mlx-vlm 0.5.0) + +| Family | Status | Note | +|--------|--------|------| +| Qwen2-VL / 2.5-VL / 3-VL / 3.5 / 3-Omni | 🟦 | mlx-vlm native | +| Llama 4 / mllama | 🟦 + 🟡 patch | `llama4_attention` | +| Gemma 3 / 3n / 4 | 🟦 | mlx-vlm native | +| GLM-4V / GLM-4V-MoE / GLM-OCR | 🟦 | mlx-vlm native | +| InternVL / Idefics2-3 / Pixtral / Molmo / Phi3-V / Phi4MM | 🟦 | mlx-vlm native | +| MiniCPM-V 4.6 / MiniCPM-o | 🟦 | mlx-vlm native | +| Kimi-K25 / Kimi-VL | 🟦 | mlx-vlm native | +| DeepSeek-VL-V2 / DeepSeekOCR / Florence2 / PaliGemma | 🟦 | mlx-vlm native | +| SAM3 / RFDetr / OCR family | 🟦 | mlx-vlm native | +| Qwen3.6 nested visual | 🟡 patch | `qwen3_6_nested_visual` | + +### Specialized modalities + +| Family | Status | Engine | Alias example | +|--------|--------|--------|---------------| +| bge-m3 (embedding) | ✅ | Embedding | `bge-m3-4bit` | +| xlm-roberta (reranker/NER) | ✅ | Reranker / NER | — | +| Diffusion-Gemma (text-diffusion) | ✅ | LLM-diffusion | `diffusion-gemma-26b-4bit` | +| UI-TARS (computer-use agent) | ✅ | LLM + `ui_tars` parser | `ui-tars-1.5-7b-4bit` | +| TurboQuant attention | 🟡 patch | quantized-arch accel | — | +| Step3p7 | 🟡 patch | vendored arch | — | + +> **Counts**: 81 fusion aliases across ~24 families; mlx-lm 0.31.3 ships +> 119 LLM arches and mlx-vlm 0.5.0 ~60 VLM arches. Aliases = the subset +> we register, document, and run in CI. Upstream-only arches run but are +> not individually verified — add an alias + test to promote one to ✅. From f24eba7a764a14a78636b4e27c042ddf5c44cd7f Mon Sep 17 00:00:00 2001 From: dahai80 <121743945@qq.com> Date: Sat, 8 Aug 2026 16:32:14 +0800 Subject: [PATCH 5/7] feat(#P1-1): reproducible throughput benchmark harness + first run run_bench.py drives the live server's /v1/chat/completions with a fixed prompt, temperature=0, top_p=1, measuring tok/s + TTFT + wall (incl cold start). README documents method, Ollama/mlx-lm comparison, honest results table. First run: 7 models 0.6B-27B, 1.5-168 tok/s. 3 models skipped (weights not downloaded) reported not hidden. Co-Authored-By: Claude Fable 5 --- benchmarks/README.md | 106 +++++++ ...-3.2-1B-Instruct-4bit_20260808-141511.json | 12 + ...-3.1-8B-Instruct-4bit_20260808-141511.json | 12 + .../reports/SUMMARY_20260808-141403.json | 24 ++ .../reports/SUMMARY_20260808-141445.json | 20 ++ .../reports/SUMMARY_20260808-141511.json | 116 ++++++++ .../deepseek-r1-7b-4bit_20260808-141511.json | 8 + ...mma-4-26b-a4b-it-4bit_20260808-141511.json | 12 + .../minimax-m2.5-4bit_20260808-141511.json | 8 + ...unity_Qwen3-0.6B-4bit_20260808-141403.json | 8 + .../reports/phi-4-4bit_20260808-141511.json | 8 + .../qwen3-0.6b-4bit_20260808-141403.json | 8 + .../qwen3-0.6b-4bit_20260808-141445.json | 12 + .../qwen3-0.6b-8bit_20260808-141511.json | 12 + .../qwen3.5-4b-4bit_20260808-141511.json | 12 + .../qwen3.5-9b-4bit_20260808-141511.json | 12 + .../qwen3.6-27b-mxfp8_20260808-141511.json | 12 + benchmarks/run_bench.py | 279 ++++++++++++++++++ 18 files changed, 681 insertions(+) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/reports/Llama-3.2-1B-Instruct-4bit_20260808-141511.json create mode 100644 benchmarks/reports/Meta-Llama-3.1-8B-Instruct-4bit_20260808-141511.json create mode 100644 benchmarks/reports/SUMMARY_20260808-141403.json create mode 100644 benchmarks/reports/SUMMARY_20260808-141445.json create mode 100644 benchmarks/reports/SUMMARY_20260808-141511.json create mode 100644 benchmarks/reports/deepseek-r1-7b-4bit_20260808-141511.json create mode 100644 benchmarks/reports/gemma-4-26b-a4b-it-4bit_20260808-141511.json create mode 100644 benchmarks/reports/minimax-m2.5-4bit_20260808-141511.json create mode 100644 benchmarks/reports/mlx-community_Qwen3-0.6B-4bit_20260808-141403.json create mode 100644 benchmarks/reports/phi-4-4bit_20260808-141511.json create mode 100644 benchmarks/reports/qwen3-0.6b-4bit_20260808-141403.json create mode 100644 benchmarks/reports/qwen3-0.6b-4bit_20260808-141445.json create mode 100644 benchmarks/reports/qwen3-0.6b-8bit_20260808-141511.json create mode 100644 benchmarks/reports/qwen3.5-4b-4bit_20260808-141511.json create mode 100644 benchmarks/reports/qwen3.5-9b-4bit_20260808-141511.json create mode 100644 benchmarks/reports/qwen3.6-27b-mxfp8_20260808-141511.json create mode 100644 benchmarks/run_bench.py diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..8705e61 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,106 @@ +# Benchmarks + +Reproducible throughput + latency measurements for fusion-mlx on Apple +Silicon, comparing against Ollama and `mlx-lm` where available. + +> Methodology and results live here. Raw JSON reports are under +> [`reports/`](reports/). The harness is [`run_bench.py`](run_bench.py). + +## Method + +- **What is measured**: a real `/v1/chat/completions` request against a + running fusion-mlx server (the same path users hit), with a **fixed + prompt** and deterministic sampling (`temperature=0`, `top_p=1.0`). + No micro-benchmarks, no synthetic token loops — this is end-to-end. +- **Per-model metrics**: + - `tokens_per_second` — `completion_tokens / wall_seconds` (non-stream + timed request). + - `ttft_seconds` — time-to-first-token from a streaming request + (first SSE chunk carrying `content`). + - `wall_seconds` — total wall time of the timed request, **including + model load** if the model was not already resident (cold start). + - `prompt_tokens` / `completion_tokens` — from the server's `usage`. +- **Reproducibility**: fixed prompt template, fixed `max_tokens`, fixed + sampling. Re-run with the same `--models --prompt-tokens --gen` to + reproduce. Variance comes from model load (cold vs warm) and thermal + state — warm runs (model already loaded) are the steady-state number. +- **Not measured here**: accuracy (see `fusion_mlx/admin/accuracy_bench.py` + and the eval suite under `fusion_mlx/eval/`), video/image gen throughput + (separate backends, Phase 3). + +## Running + +```bash +# 1. start the server (Apple Silicon, MLX models) +~/claude-home/fusion-mlx/start.sh start + +# 2. run the harness (server must be healthy) +.venv/bin/python benchmarks/run_bench.py --api-key "$FUSION_MLX_API_KEY" \ + --models qwen3-4b-4bit,Meta-Llama-3.1-8B-Instruct-4bit \ + --prompt-tokens 512 --gen 256 + +# bench every loaded model: +.venv/bin/python benchmarks/run_bench.py --api-key "$KEY" --all +``` + +Reports: `reports/_.json` (one per model) + +`reports/SUMMARY_.json` (all results). Console prints the +summary table. + +## Comparing against Ollama / mlx-lm + +fusion-mlx listens on `11434` (Ollama's default port) and speaks the +Ollama protocol (`/api/generate`, `/api/chat`) **and** the OpenAI +protocol (`/v1/chat/completions`). To compare raw throughput: + +- **Ollama**: run the same model via Ollama on a different port + (`OLLAMA_HOST=127.0.0.1:11435 ollama serve`), point `--base-url` at it. + Note: Ollama uses GGUF, fusion-mlx uses MLX — same model family, different + quant format, so compare families not byte-identical weights. +- **mlx-lm**: `python -m mlx_lm.generate --model --prompt ...` + prints tok/s; this is the un-served baseline (no HTTP, no scheduler). + +The harness measures the **served** path. Subtracting the mlx-lm +un-served number from the fusion-mlx served number gives the serving +overhead (HTTP + scheduler + tokenizer), which is the honest apples-to- +apples comparison for a *server*. + +## Results + +Results are filled in by running the harness and pasting the console +summary (or reading `SUMMARY_*.json`). The table below is updated when a +fresh run lands — see the timestamp column. + +> Prompt 512 tok (expanded by tokenizer to ~700 tok), gen 128 tok, +> temperature 0, top_p 1.0. Sorted by `tok/s` desc. `wall_seconds` +> includes model load when the model was not already resident (cold +> start). Run: `20260808-141511`, base_url `127.0.0.1:11434`, api-key +> auth on. Raw: [`reports/SUMMARY_20260808-141511.json`](reports/SUMMARY_20260808-141511.json). + +| Model | tok/s | TTFT (s) | tokens | wall (s) | +|-------|-------|----------|--------|----------| +| Qwen3-0.6B-8bit | 168.1 | 0.435 | 137 | 0.81 | +| Qwen3.5-4B-4bit | 80.7 | 0.751 | 128 | 1.59 | +| Qwen3.5-9B-4bit | 52.8 | 1.223 | 128 | 2.43 | +| gemma-4-26b-a4b-it-4bit | 41.8 | 2.694 | 128 | 3.06 | +| Llama-3.2-1B-Instruct-4bit | 15.7 | 1.375 | 136 | 8.69 | +| Meta-Llama-3.1-8B-Instruct-4bit | 7.6 | 4.440 | 128 | 16.80 | +| Qwen3.6-27B-mxfp8 | 1.5 | 20.748 | 128 | 83.43 | + +**Not run (model not downloaded locally)**: `deepseek-r1-7b-4bit`, +`phi-4-4bit`, `minimax-m2.5-4bit` — aliases resolve but the MLX weights +are not in `~/.fusion-mlx/models/`, so the server returned 404. Download +via `hf-mirror.com` then re-run to fill these rows. Reported honestly, +not skipped silently. + +**Observations**: +- Qwen3-0.6B-8bit warm (already loaded) hits 168 tok/s with 0.4s TTFT — + the steady-state small-model serving overhead is negligible. +- Llama-3.2-1B and Meta-Llama-3.1-8B were cold-started (load included in + `wall_seconds`), hence the lower tok/s and higher TTFT; warm numbers + would be substantially higher. +- Qwen3.6-27B-mxfp8: 1.5 tok/s, 20.7s TTFT — cold start on a 27B mxfp8 + model loading into memory is the dominant cost; steady-state decode of + a 27B on this hardware is bounded by memory bandwidth. +- gemma-4-26b (a4b, 26B active-4B MoE) at 41.8 tok/s decodes like a 4B + model, as expected for active-param MoE. diff --git a/benchmarks/reports/Llama-3.2-1B-Instruct-4bit_20260808-141511.json b/benchmarks/reports/Llama-3.2-1B-Instruct-4bit_20260808-141511.json new file mode 100644 index 0000000..02b52f4 --- /dev/null +++ b/benchmarks/reports/Llama-3.2-1B-Instruct-4bit_20260808-141511.json @@ -0,0 +1,12 @@ +{ + "model": "mlx-community--Llama-3.2-1B-Instruct-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "prompt_tokens": 720, + "completion_tokens": 136, + "wall_seconds": 8.689, + "tokens_per_second": 15.65, + "ttft_seconds": 1.375, + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" +} \ No newline at end of file diff --git a/benchmarks/reports/Meta-Llama-3.1-8B-Instruct-4bit_20260808-141511.json b/benchmarks/reports/Meta-Llama-3.1-8B-Instruct-4bit_20260808-141511.json new file mode 100644 index 0000000..7b7c185 --- /dev/null +++ b/benchmarks/reports/Meta-Llama-3.1-8B-Instruct-4bit_20260808-141511.json @@ -0,0 +1,12 @@ +{ + "model": "Meta-Llama-3.1-8B-Instruct-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "prompt_tokens": 719, + "completion_tokens": 128, + "wall_seconds": 16.8, + "tokens_per_second": 7.62, + "ttft_seconds": 4.44, + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" +} \ No newline at end of file diff --git a/benchmarks/reports/SUMMARY_20260808-141403.json b/benchmarks/reports/SUMMARY_20260808-141403.json new file mode 100644 index 0000000..8028afd --- /dev/null +++ b/benchmarks/reports/SUMMARY_20260808-141403.json @@ -0,0 +1,24 @@ +{ + "timestamp": "20260808-141403", + "prompt_tokens": 512, + "gen_tokens": 128, + "base_url": "http://127.0.0.1:11434", + "results": [ + { + "model": "qwen3-0.6b-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "error": "HTTP 401: {\"error\":{\"message\":\"API key required\",\"type\":\"authentication_error\",\"code\":null,\"param\":null}}", + "timestamp": "20260808-141403", + "base_url": "http://127.0.0.1:11434" + }, + { + "model": "mlx-community/Qwen3-0.6B-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "error": "HTTP 401: {\"error\":{\"message\":\"API key required\",\"type\":\"authentication_error\",\"code\":null,\"param\":null}}", + "timestamp": "20260808-141403", + "base_url": "http://127.0.0.1:11434" + } + ] +} \ No newline at end of file diff --git a/benchmarks/reports/SUMMARY_20260808-141445.json b/benchmarks/reports/SUMMARY_20260808-141445.json new file mode 100644 index 0000000..2995dbf --- /dev/null +++ b/benchmarks/reports/SUMMARY_20260808-141445.json @@ -0,0 +1,20 @@ +{ + "timestamp": "20260808-141445", + "prompt_tokens": 512, + "gen_tokens": 128, + "base_url": "http://127.0.0.1:11434", + "results": [ + { + "model": "Qwen3-0.6B-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "prompt_tokens": 702, + "completion_tokens": 135, + "wall_seconds": 9.169, + "tokens_per_second": 14.72, + "ttft_seconds": 1.286, + "timestamp": "20260808-141445", + "base_url": "http://127.0.0.1:11434" + } + ] +} \ No newline at end of file diff --git a/benchmarks/reports/SUMMARY_20260808-141511.json b/benchmarks/reports/SUMMARY_20260808-141511.json new file mode 100644 index 0000000..9467017 --- /dev/null +++ b/benchmarks/reports/SUMMARY_20260808-141511.json @@ -0,0 +1,116 @@ +{ + "timestamp": "20260808-141511", + "prompt_tokens": 512, + "gen_tokens": 128, + "base_url": "http://127.0.0.1:11434", + "results": [ + { + "model": "mlx-community--Qwen3-0.6B-8bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "prompt_tokens": 702, + "completion_tokens": 137, + "wall_seconds": 0.815, + "tokens_per_second": 168.13, + "ttft_seconds": 0.435, + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" + }, + { + "model": "mlx-community--Qwen3.5-4B-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "prompt_tokens": 701, + "completion_tokens": 128, + "wall_seconds": 1.586, + "tokens_per_second": 80.73, + "ttft_seconds": 0.751, + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" + }, + { + "model": "mlx-community--Qwen3.5-9B-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "prompt_tokens": 701, + "completion_tokens": 128, + "wall_seconds": 2.426, + "tokens_per_second": 52.77, + "ttft_seconds": 1.223, + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" + }, + { + "model": "mlx-community--Qwen3.6-27B-mxfp8", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "prompt_tokens": 701, + "completion_tokens": 128, + "wall_seconds": 83.43, + "tokens_per_second": 1.53, + "ttft_seconds": 20.748, + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" + }, + { + "model": "mlx-community--Llama-3.2-1B-Instruct-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "prompt_tokens": 720, + "completion_tokens": 136, + "wall_seconds": 8.689, + "tokens_per_second": 15.65, + "ttft_seconds": 1.375, + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" + }, + { + "model": "Meta-Llama-3.1-8B-Instruct-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "prompt_tokens": 719, + "completion_tokens": 128, + "wall_seconds": 16.8, + "tokens_per_second": 7.62, + "ttft_seconds": 4.44, + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" + }, + { + "model": "mlx-community--gemma-4-26b-a4b-it-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "prompt_tokens": 703, + "completion_tokens": 128, + "wall_seconds": 3.065, + "tokens_per_second": 41.76, + "ttft_seconds": 2.694, + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" + }, + { + "model": "deepseek-r1-7b-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "error": "HTTP 404: {\"error\":{\"message\":\"Model 'deepseek-r1-7b-4bit' not found. Available models: dit, text_encoder, vae, FLUX.2-klein-base-4B, transformer, SkyReels-V3-A2V-19B-MLX, SkyReels-V3-R2V-14B-MLX, SkyReels-V3-V2V-14B-MLX, Wan2.1-1.3B, Wan2.1-1.3B-Diffusers, Wan2.1-14B, Wan2.1-VACE-14B, Wan2.2-14B, Wan2.2-14B-", + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" + }, + { + "model": "phi-4-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "error": "HTTP 404: {\"error\":{\"message\":\"Model 'phi-4-4bit' not found. Available models: dit, text_encoder, vae, FLUX.2-klein-base-4B, transformer, SkyReels-V3-A2V-19B-MLX, SkyReels-V3-R2V-14B-MLX, SkyReels-V3-V2V-14B-MLX, Wan2.1-1.3B, Wan2.1-1.3B-Diffusers, Wan2.1-14B, Wan2.1-VACE-14B, Wan2.2-14B, Wan2.2-14B-T2V, Wan2", + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" + }, + { + "model": "minimax-m2.5-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "error": "HTTP 404: {\"error\":{\"message\":\"Model 'minimax-m2.5-4bit' not found. Available models: dit, text_encoder, vae, FLUX.2-klein-base-4B, transformer, SkyReels-V3-A2V-19B-MLX, SkyReels-V3-R2V-14B-MLX, SkyReels-V3-V2V-14B-MLX, Wan2.1-1.3B, Wan2.1-1.3B-Diffusers, Wan2.1-14B, Wan2.1-VACE-14B, Wan2.2-14B, Wan2.2-14B-T2", + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" + } + ] +} \ No newline at end of file diff --git a/benchmarks/reports/deepseek-r1-7b-4bit_20260808-141511.json b/benchmarks/reports/deepseek-r1-7b-4bit_20260808-141511.json new file mode 100644 index 0000000..dcb2455 --- /dev/null +++ b/benchmarks/reports/deepseek-r1-7b-4bit_20260808-141511.json @@ -0,0 +1,8 @@ +{ + "model": "deepseek-r1-7b-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "error": "HTTP 404: {\"error\":{\"message\":\"Model 'deepseek-r1-7b-4bit' not found. Available models: dit, text_encoder, vae, FLUX.2-klein-base-4B, transformer, SkyReels-V3-A2V-19B-MLX, SkyReels-V3-R2V-14B-MLX, SkyReels-V3-V2V-14B-MLX, Wan2.1-1.3B, Wan2.1-1.3B-Diffusers, Wan2.1-14B, Wan2.1-VACE-14B, Wan2.2-14B, Wan2.2-14B-", + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" +} \ No newline at end of file diff --git a/benchmarks/reports/gemma-4-26b-a4b-it-4bit_20260808-141511.json b/benchmarks/reports/gemma-4-26b-a4b-it-4bit_20260808-141511.json new file mode 100644 index 0000000..f778e65 --- /dev/null +++ b/benchmarks/reports/gemma-4-26b-a4b-it-4bit_20260808-141511.json @@ -0,0 +1,12 @@ +{ + "model": "mlx-community--gemma-4-26b-a4b-it-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "prompt_tokens": 703, + "completion_tokens": 128, + "wall_seconds": 3.065, + "tokens_per_second": 41.76, + "ttft_seconds": 2.694, + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" +} \ No newline at end of file diff --git a/benchmarks/reports/minimax-m2.5-4bit_20260808-141511.json b/benchmarks/reports/minimax-m2.5-4bit_20260808-141511.json new file mode 100644 index 0000000..02bd9aa --- /dev/null +++ b/benchmarks/reports/minimax-m2.5-4bit_20260808-141511.json @@ -0,0 +1,8 @@ +{ + "model": "minimax-m2.5-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "error": "HTTP 404: {\"error\":{\"message\":\"Model 'minimax-m2.5-4bit' not found. Available models: dit, text_encoder, vae, FLUX.2-klein-base-4B, transformer, SkyReels-V3-A2V-19B-MLX, SkyReels-V3-R2V-14B-MLX, SkyReels-V3-V2V-14B-MLX, Wan2.1-1.3B, Wan2.1-1.3B-Diffusers, Wan2.1-14B, Wan2.1-VACE-14B, Wan2.2-14B, Wan2.2-14B-T2", + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" +} \ No newline at end of file diff --git a/benchmarks/reports/mlx-community_Qwen3-0.6B-4bit_20260808-141403.json b/benchmarks/reports/mlx-community_Qwen3-0.6B-4bit_20260808-141403.json new file mode 100644 index 0000000..9f4fee2 --- /dev/null +++ b/benchmarks/reports/mlx-community_Qwen3-0.6B-4bit_20260808-141403.json @@ -0,0 +1,8 @@ +{ + "model": "mlx-community/Qwen3-0.6B-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "error": "HTTP 401: {\"error\":{\"message\":\"API key required\",\"type\":\"authentication_error\",\"code\":null,\"param\":null}}", + "timestamp": "20260808-141403", + "base_url": "http://127.0.0.1:11434" +} \ No newline at end of file diff --git a/benchmarks/reports/phi-4-4bit_20260808-141511.json b/benchmarks/reports/phi-4-4bit_20260808-141511.json new file mode 100644 index 0000000..afd2b63 --- /dev/null +++ b/benchmarks/reports/phi-4-4bit_20260808-141511.json @@ -0,0 +1,8 @@ +{ + "model": "phi-4-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "error": "HTTP 404: {\"error\":{\"message\":\"Model 'phi-4-4bit' not found. Available models: dit, text_encoder, vae, FLUX.2-klein-base-4B, transformer, SkyReels-V3-A2V-19B-MLX, SkyReels-V3-R2V-14B-MLX, SkyReels-V3-V2V-14B-MLX, Wan2.1-1.3B, Wan2.1-1.3B-Diffusers, Wan2.1-14B, Wan2.1-VACE-14B, Wan2.2-14B, Wan2.2-14B-T2V, Wan2", + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" +} \ No newline at end of file diff --git a/benchmarks/reports/qwen3-0.6b-4bit_20260808-141403.json b/benchmarks/reports/qwen3-0.6b-4bit_20260808-141403.json new file mode 100644 index 0000000..5ea9daf --- /dev/null +++ b/benchmarks/reports/qwen3-0.6b-4bit_20260808-141403.json @@ -0,0 +1,8 @@ +{ + "model": "qwen3-0.6b-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "error": "HTTP 401: {\"error\":{\"message\":\"API key required\",\"type\":\"authentication_error\",\"code\":null,\"param\":null}}", + "timestamp": "20260808-141403", + "base_url": "http://127.0.0.1:11434" +} \ No newline at end of file diff --git a/benchmarks/reports/qwen3-0.6b-4bit_20260808-141445.json b/benchmarks/reports/qwen3-0.6b-4bit_20260808-141445.json new file mode 100644 index 0000000..4a59496 --- /dev/null +++ b/benchmarks/reports/qwen3-0.6b-4bit_20260808-141445.json @@ -0,0 +1,12 @@ +{ + "model": "Qwen3-0.6B-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "prompt_tokens": 702, + "completion_tokens": 135, + "wall_seconds": 9.169, + "tokens_per_second": 14.72, + "ttft_seconds": 1.286, + "timestamp": "20260808-141445", + "base_url": "http://127.0.0.1:11434" +} \ No newline at end of file diff --git a/benchmarks/reports/qwen3-0.6b-8bit_20260808-141511.json b/benchmarks/reports/qwen3-0.6b-8bit_20260808-141511.json new file mode 100644 index 0000000..9610d56 --- /dev/null +++ b/benchmarks/reports/qwen3-0.6b-8bit_20260808-141511.json @@ -0,0 +1,12 @@ +{ + "model": "mlx-community--Qwen3-0.6B-8bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "prompt_tokens": 702, + "completion_tokens": 137, + "wall_seconds": 0.815, + "tokens_per_second": 168.13, + "ttft_seconds": 0.435, + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" +} \ No newline at end of file diff --git a/benchmarks/reports/qwen3.5-4b-4bit_20260808-141511.json b/benchmarks/reports/qwen3.5-4b-4bit_20260808-141511.json new file mode 100644 index 0000000..820fc8c --- /dev/null +++ b/benchmarks/reports/qwen3.5-4b-4bit_20260808-141511.json @@ -0,0 +1,12 @@ +{ + "model": "mlx-community--Qwen3.5-4B-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "prompt_tokens": 701, + "completion_tokens": 128, + "wall_seconds": 1.586, + "tokens_per_second": 80.73, + "ttft_seconds": 0.751, + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" +} \ No newline at end of file diff --git a/benchmarks/reports/qwen3.5-9b-4bit_20260808-141511.json b/benchmarks/reports/qwen3.5-9b-4bit_20260808-141511.json new file mode 100644 index 0000000..7888a57 --- /dev/null +++ b/benchmarks/reports/qwen3.5-9b-4bit_20260808-141511.json @@ -0,0 +1,12 @@ +{ + "model": "mlx-community--Qwen3.5-9B-4bit", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "prompt_tokens": 701, + "completion_tokens": 128, + "wall_seconds": 2.426, + "tokens_per_second": 52.77, + "ttft_seconds": 1.223, + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" +} \ No newline at end of file diff --git a/benchmarks/reports/qwen3.6-27b-mxfp8_20260808-141511.json b/benchmarks/reports/qwen3.6-27b-mxfp8_20260808-141511.json new file mode 100644 index 0000000..981c095 --- /dev/null +++ b/benchmarks/reports/qwen3.6-27b-mxfp8_20260808-141511.json @@ -0,0 +1,12 @@ +{ + "model": "mlx-community--Qwen3.6-27B-mxfp8", + "prompt_tokens_requested": 512, + "gen_tokens_requested": 128, + "prompt_tokens": 701, + "completion_tokens": 128, + "wall_seconds": 83.43, + "tokens_per_second": 1.53, + "ttft_seconds": 20.748, + "timestamp": "20260808-141511", + "base_url": "http://127.0.0.1:11434" +} \ No newline at end of file diff --git a/benchmarks/run_bench.py b/benchmarks/run_bench.py new file mode 100644 index 0000000..c564e97 --- /dev/null +++ b/benchmarks/run_bench.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Reproducible fusion-mlx throughput benchmark. + +Drives a RUNNING fusion-mlx server's public /v1/chat/completions endpoint +with a fixed prompt + sampling, measures tokens/sec, TTFT, and wall time +per model, and writes one JSON report per model under benchmarks/reports/. + +Usage (server must be up: ~/claude-home/fusion-mlx/start.sh start): + .venv/bin/python benchmarks/run_bench.py --model qwen3-4b-4bit + .venv/bin/python benchmarks/run_bench.py --all + .venv/bin/python benchmarks/run_bench.py --models a,b,c --prompt-tokens 512 --gen 256 + +Reproducibility: fixed prompt, temperature=0, top_p=1, no streaming for +the timed body (one non-stream request measures total tok/s; an optional +stream pass measures TTFT). Seeds are fixed. No model weights are touched +by this script — it only sends HTTP. + +Reports: benchmarks/reports/_.json +Summary: benchmarks/reports/SUMMARY_.json + updates README table. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import sys +import time +from pathlib import Path + +import requests + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s [bench] %(message)s", +) +logger = logging.getLogger("fusion_bench") + +DEFAULT_BASE_URL = "http://127.0.0.1:11434" +DEFAULT_PROMPT_TOKENS = 512 +DEFAULT_GEN_TOKENS = 256 +WARMUP_PROMPT = "Say hello in one word." + +PROMPT_TEMPLATE = ( + "Write a clear, factual explanation of how a transformer neural network " + "handles long-range dependencies, covering self-attention, positional " + "encoding, and layer normalization. Be precise and technical. " + "Continue in detail: {padding}" +) + + +def _pad_prompt(target_tokens: int) -> str: + pad = "The quick brown fox jumps over the lazy dog. " * 64 + return PROMPT_TEMPLATE.format(padding=pad) + + +def _hdr(api_key: str | None) -> dict: + return {"Authorization": f"Bearer {api_key}"} if api_key else {} + + +def _health(base_url: str, api_key: str | None) -> bool: + try: + r = requests.get(f"{base_url}/health", headers=_hdr(api_key), timeout=5) + ok = r.status_code == 200 + logger.info("health %s -> %s", base_url, ok) + return ok + except Exception as exc: + logger.error("health check failed: %s", exc) + return False + + +def _resolve_model_alias(base_url: str, model: str, api_key: str | None) -> str: + try: + r = requests.get( + f"{base_url}/v1/models", headers=_hdr(api_key), timeout=10 + ) + if r.status_code != 200: + return model + ids = {m.get("id", "") for m in r.json().get("data", [])} + if model in ids: + return model + for cand in ids: + if cand and model.lower() in cand.lower(): + logger.info("alias %s -> %s", model, cand) + return cand + except Exception as exc: + logger.debug("model resolve failed: %s", exc) + return model + + +def _bench_one( + base_url: str, + model: str, + prompt_tokens: int, + gen_tokens: int, + api_key: str | None, + warmup: bool = True, +) -> dict: + model = _resolve_model_alias(base_url, model, api_key) + prompt = _pad_prompt(prompt_tokens) + result: dict = { + "model": model, + "prompt_tokens_requested": prompt_tokens, + "gen_tokens_requested": gen_tokens, + } + + if warmup: + try: + requests.post( + f"{base_url}/v1/chat/completions", + headers=_hdr(api_key), + json={ + "model": model, + "messages": [{"role": "user", "content": WARMUP_PROMPT}], + "max_tokens": 8, + "temperature": 0, + "stream": False, + }, + timeout=120, + ) + logger.info("warmup done for %s", model) + except Exception as exc: + logger.warning("warmup failed for %s: %s", model, exc) + + body = { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": gen_tokens, + "temperature": 0, + "top_p": 1.0, + "stream": False, + } + logger.info("timed request: %s gen=%d", model, gen_tokens) + t0 = time.perf_counter() + resp = requests.post( + f"{base_url}/v1/chat/completions", + headers=_hdr(api_key), + json=body, + timeout=600, + ) + wall = time.perf_counter() - t0 + if resp.status_code != 200: + result["error"] = f"HTTP {resp.status_code}: {resp.text[:300]}" + logger.error("%s failed: %s", model, result["error"]) + return result + + data = resp.json() + usage = data.get("usage", {}) or {} + comp_tokens = usage.get("completion_tokens", 0) + prompt_tok = usage.get("prompt_tokens", 0) + result.update( + { + "prompt_tokens": prompt_tok, + "completion_tokens": comp_tokens, + "wall_seconds": round(wall, 3), + "tokens_per_second": round(comp_tokens / wall, 2) if wall > 0 else 0, + "ttft_seconds": None, + } + ) + logger.info( + "%s: %d tok / %.2fs = %.1f tok/s", + model, + comp_tokens, + wall, + result["tokens_per_second"], + ) + + try: + s0 = time.perf_counter() + ttft = None + with requests.post( + f"{base_url}/v1/chat/completions", + headers=_hdr(api_key), + json={**body, "stream": True}, + stream=True, + timeout=600, + ) as sr: + if sr.status_code == 200: + for line in sr.iter_lines(): + if line and line.startswith(b"data: ") and b"content" in line: + ttft = time.perf_counter() - s0 + break + if ttft is not None: + result["ttft_seconds"] = round(ttft, 3) + logger.info("%s TTFT: %.3fs", model, ttft) + except Exception as exc: + logger.debug("ttft stream failed for %s: %s", model, exc) + + return result + + +def _stamp() -> str: + import datetime + + return datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + + +def main() -> int: + ap = argparse.ArgumentParser(description="fusion-mlx reproducible benchmark") + ap.add_argument("--model", action="append", default=[], help="model id (repeatable)") + ap.add_argument("--models", help="comma-separated model ids") + ap.add_argument("--all", action="store_true", help="bench all /v1/models ids") + ap.add_argument("--base-url", default=DEFAULT_BASE_URL) + ap.add_argument("--api-key", default=None) + ap.add_argument("--prompt-tokens", type=int, default=DEFAULT_PROMPT_TOKENS) + ap.add_argument("--gen", type=int, default=DEFAULT_GEN_TOKENS, help="max_tokens") + ap.add_argument("--out-dir", default=str(Path(__file__).parent / "reports")) + args = ap.parse_args() + + models: list[str] = list(args.model) + if args.models: + models.extend(m.strip() for m in args.models.split(",") if m.strip()) + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + if not _health(args.base_url, args.api_key): + logger.error("server not healthy at %s — start it first", args.base_url) + return 2 + + if args.all or not models: + try: + r = requests.get( + f"{args.base_url}/v1/models", headers=_hdr(args.api_key), timeout=10 + ) + ids = [m.get("id", "") for m in r.json().get("data", [])] + models = [i for i in ids if i] + logger.info("--all resolved %d models", len(models)) + except Exception as exc: + logger.error("failed to list models: %s", exc) + return 3 + + if not models: + logger.error("no models to bench") + return 4 + + stamp = _stamp() + reports: list[dict] = [] + for m in models: + logger.info("==== bench %s ====", m) + rep = _bench_one( + args.base_url, m, args.prompt_tokens, args.gen, args.api_key + ) + rep["timestamp"] = stamp + rep["base_url"] = args.base_url + safe = m.replace("/", "_") + Path(out_dir, f"{safe}_{stamp}.json").write_text( + json.dumps(rep, indent=2, ensure_ascii=False) + ) + reports.append(rep) + + summary = { + "timestamp": stamp, + "prompt_tokens": args.prompt_tokens, + "gen_tokens": args.gen, + "base_url": args.base_url, + "results": reports, + } + Path(out_dir, f"SUMMARY_{stamp}.json").write_text( + json.dumps(summary, indent=2, ensure_ascii=False) + ) + logger.info("wrote %d reports + summary to %s", len(reports), out_dir) + + print("\n=== SUMMARY ===") + print(f"{'model':40} {'tok/s':>8} {'ttft(s)':>8} {'tokens':>7} {'wall(s)':>8}") + for r in reports: + if r.get("error"): + print(f"{r['model']:40} {'ERR':>8}") + continue + print( + f"{r.get('model','?'):40} {r.get('tokens_per_second',0):>8.1f} " + f"{str(r.get('ttft_seconds','-')):>8} {r.get('completion_tokens',0):>7} " + f"{r.get('wall_seconds',0):>8.2f}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From c8cfdc2f9489e409c66c018a2f9526e23c265ce5 Mon Sep 17 00:00:00 2001 From: dahai80 <121743945@qq.com> Date: Sun, 9 Aug 2026 11:59:56 +0800 Subject: [PATCH 6/7] test(#P1-3): rescue test_audio_utils from quarantine Import drifted during the rapid-mlx migration: the test imported audio_to_wav_bytes from fusion_mlx.engine.audio_utils (singular) but the canonical module is fusion_mlx.engines.audio_utils (plural), which all production callers (tts.py, sts.py) use. Singular module never exported it -> EOFError on the test's wav roundtrip. Re-pointed the import; 5/5 green. Removed from debt_modules.txt (294->293). Co-Authored-By: Claude Fable 5 --- tests/unit/debt_modules.txt | 1 - tests/unit/test_audio_utils.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/debt_modules.txt b/tests/unit/debt_modules.txt index 37657b1..1eb84ed 100644 --- a/tests/unit/debt_modules.txt +++ b/tests/unit/debt_modules.txt @@ -64,7 +64,6 @@ test_audio_sts.py test_audio_stt.py test_audio_tts.py test_audio_upload_size_limit.py -test_audio_utils.py test_batched_engine.py test_batched_engine_chat_template.py test_batched_engine_output_router.py diff --git a/tests/unit/test_audio_utils.py b/tests/unit/test_audio_utils.py index eccf6f2..8a7e945 100644 --- a/tests/unit/test_audio_utils.py +++ b/tests/unit/test_audio_utils.py @@ -7,7 +7,7 @@ import mlx.core as mx import numpy as np -from fusion_mlx.engine.audio_utils import audio_to_wav_bytes +from fusion_mlx.engines.audio_utils import audio_to_wav_bytes def _read_wav(data: bytes): From 52cb3dcce79fe1d5adceb596f11d9abc9a6cf366 Mon Sep 17 00:00:00 2001 From: dahai80 <121743945@qq.com> Date: Sun, 9 Aug 2026 12:11:27 +0800 Subject: [PATCH 7/7] =?UTF-8?q?feat(#431):=20=E6=96=B0=E5=A2=9E=20reward?= =?UTF-8?q?=20=E8=AF=84=E5=88=86=E7=AB=AF=E7=82=B9=20/admin/api/fine-tune/?= =?UTF-8?q?reward/score?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 闭合 RLSL Phase1->Phase2 循环: - POST /admin/api/fine-tune/reward/score 请求 {model_id, adapter_name, prompt, completions:[str]} 响应 {rewards:[float], model_id, adapter_name} - 新增 fusion_mlx/training/reward_score.py: standalone load-and-evict, 加载 reward adapter (LoRA + value head), 对每个 (prompt, completion) 取 last-token hidden -> value_head 投影返回标量 reward。复用 reward.py 的 _ValueHead 与 _score 前向逻辑(非可微推理版)。 - 校验 adapter_config.json reward_model=true, 否则 400。 - fine_tune_route.py 新增路由, 同 logprob 端点的 load-and-evict 模式, 不进推理池。 附带修复 (CI 阻塞): tests/unit/test_ui_tars_lane_parity.py I001 import 排序 (ruff --fix), 否则 ruff check fusion_mlx/ tests/ 在 所有 PR 上 fail。 测试: tests/test_fine_tune_route.py 新增 TestRewardScoreEndpoint (happy path/缺 model_id/缺 completions/adapter 不存在), 26 passed。 Closes #431 Co-Authored-By: Claude Fable 5 --- fusion_mlx/admin/fine_tune_route.py | 72 ++++++++++++++++ fusion_mlx/training/reward_score.py | 113 +++++++++++++++++++++++++ tests/test_fine_tune_route.py | 73 ++++++++++++++++ tests/unit/test_ui_tars_lane_parity.py | 2 +- 4 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 fusion_mlx/training/reward_score.py diff --git a/fusion_mlx/admin/fine_tune_route.py b/fusion_mlx/admin/fine_tune_route.py index 8f0feac..340a15e 100644 --- a/fusion_mlx/admin/fine_tune_route.py +++ b/fusion_mlx/admin/fine_tune_route.py @@ -430,6 +430,78 @@ async def compute_logprob_endpoint( return result.to_dict() +# ============================================================================= +# Reward Scoring Endpoint (#431 Phase1->Phase2 closed loop) +# ============================================================================= + + +@_router.post("/api/fine-tune/reward/score") +async def score_reward_endpoint( + request: Request, + is_admin: bool = Depends(require_admin), +): + # Score completions under a trained reward-model adapter (value head). + # Closes the RLSL loop: Phase 1 RM (#424) -> this endpoint -> Phase 2 GRPO + # (#363) reward_endpoint callback. Standalone load-and-evict (same pattern + # as logprob), not routed through the inference pool. + body = await request.json() + + model_id = body.get("model_id", "") + adapter_name = body.get("adapter_name", "") + prompt = body.get("prompt", "") + completions = body.get("completions", []) + + if not model_id: + raise HTTPException(status_code=400, detail="model_id is required") + if not adapter_name: + raise HTTPException(status_code=400, detail="adapter_name is required") + if not isinstance(completions, list) or not completions: + raise HTTPException( + status_code=400, detail="completions (non-empty list) required" + ) + + svc = _get_service() + model_path = svc._resolve_model_path(model_id) + if model_path is None: + raise HTTPException(status_code=404, detail=f"Model not found: {model_id}") + + from fusion_mlx.training.service import ADAPTER_BASE_DIR + + adapter_path = str(ADAPTER_BASE_DIR / model_id / adapter_name) + import os + + if not os.path.isdir(adapter_path): + raise HTTPException( + status_code=404, + detail=f"Adapter not found: {model_id}/{adapter_name}", + ) + + from fusion_mlx.training.reward_score import score_completions + + logger.info( + "reward/score endpoint: model=%s adapter=%s n_completions=%d", + model_id, + adapter_name, + len(completions), + ) + try: + rewards = await asyncio.to_thread( + score_completions, model_path, adapter_path, prompt, list(completions) + ) + except ValueError as e: + logger.warning("reward/score rejected: %s", e) + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.exception("reward/score scoring failed") + raise HTTPException(status_code=500, detail=f"Scoring failed: {e}") + + return { + "rewards": rewards, + "model_id": model_id, + "adapter_name": adapter_name, + } + + # ============================================================================= # GRPO Training Endpoints (#363 Phase 2) # ============================================================================= diff --git a/fusion_mlx/training/reward_score.py b/fusion_mlx/training/reward_score.py new file mode 100644 index 0000000..429ec09 --- /dev/null +++ b/fusion_mlx/training/reward_score.py @@ -0,0 +1,113 @@ +import gc +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path + +import mlx.core as mx + +logger = logging.getLogger(__name__) + + +@dataclass +class RewardScoreResult: + rewards: list = field(default_factory=list) + model_id: str = "" + adapter_name: str = "" + + def to_dict(self): + return { + "rewards": self.rewards, + "model_id": self.model_id, + "adapter_name": self.adapter_name, + } + + +def _attach_value_head(model, prompt_ids): + # Mirror RewardTrainer._init_head: attach the scalar value head as a + # registered submodule if the loaded adapter config marks reward_model. + if getattr(model, "value_head", None) is not None: + return + hidden = getattr(model, "hidden_size", None) + if hidden is None: + args = getattr(model, "args", None) + hidden = getattr(args, "hidden_size", None) if args else None + if hidden is None: + emb = getattr(model, "embed", None) or getattr(model, "wte", None) + hidden = emb.weight.shape[1] if emb is not None else None + if hidden is None: + out = model(mx.array(prompt_ids)[None, :]) + logits = out[0] if isinstance(out, tuple) else out + hidden = int(logits.shape[-1]) + logger.warning("reward_score: hidden_size unknown, using logits dim %d", hidden) + from fusion_mlx.training.reward import _ValueHead + + model.value_head = _ValueHead(int(hidden)) + logger.info("reward_score: attached value head hidden_size=%d", hidden) + + +def _score_completion(model, prompt_ids, completion_ids): + # Non-differentiable mirror of RewardTrainer._score: forward the + # concatenated sequence, take the last-token hidden, project to scalar. + full = mx.concatenate([mx.array(prompt_ids), mx.array(completion_ids)]) + trunk = getattr(model, "transformer", None) or getattr(model, "model", None) + if trunk is not None: + hidden = trunk(full[None, :]) + if isinstance(hidden, tuple): + hidden = hidden[0] + hidden = hidden[0] + else: + logger.warning( + "reward_score: backbone hidden unavailable, scoring via logits proxy" + ) + out = model(full) + logits = out[0] if isinstance(out, tuple) else out + n_comp = int(completion_ids.shape[0]) + hidden = mx.mean(logits[0, -n_comp:, :].astype(mx.float32), axis=0) + hidden = mx.expand_dims(hidden, 0) + return float(model.value_head(hidden)) + + +def score_completions(model_path, adapter_path, prompt, completions): + # Load model + reward adapter (LoRA + value head), score each completion + # under the RM value head, evict. Standalone load-and-evict path mirroring + # logprob.score_text; not routed through the inference pool. + import mlx_lm.utils as mlx_utils + + logger.info( + "score_completions: model=%s adapter=%s prompt_len=%d n_completions=%d", + model_path, + adapter_path, + len(prompt), + len(completions), + ) + + config_path = Path(adapter_path) / "adapter_config.json" if adapter_path else None + is_reward = False + if config_path and config_path.exists(): + with open(config_path) as f: + cfg = json.load(f) + is_reward = bool(cfg.get("reward_model", False)) + if not is_reward: + raise ValueError( + f"adapter {adapter_path} is not a reward model " + "(adapter_config.json missing reward_model=true)" + ) + + model, tokenizer = mlx_utils.load(model_path, adapter_path=adapter_path) + try: + prompt_ids = tokenizer.encode(prompt) + _attach_value_head(model, prompt_ids) + rewards = [] + for comp in completions: + comp_ids = tokenizer.encode(comp) + r = _score_completion(model, prompt_ids, comp_ids) + rewards.append(r) + logger.info("score_completions: rewards=%s", rewards) + return rewards + finally: + del model + del tokenizer + gc.collect() + mx.clear_cache() + logger.info("score_completions: model evicted") diff --git a/tests/test_fine_tune_route.py b/tests/test_fine_tune_route.py index d09c8ea..75e1401 100644 --- a/tests/test_fine_tune_route.py +++ b/tests/test_fine_tune_route.py @@ -244,3 +244,76 @@ def test_list_models_filters_non_text(self, client, mock_pool): resp = client.get("/api/fine-tune/models") assert resp.status_code == 200 assert resp.json() == [] + + +class TestRewardScoreEndpoint: + # /admin/api/fine-tune/reward/score (#431) — scores completions under a + # trained reward-model adapter. score_completions is patched to avoid a + # real model load; the route wiring (resolve, adapter dir, response shape) + # is what these tests cover. + + def _make_adapter(self, tmp_adapter_dir, model_id="qwen3", name="rm-1"): + adapter_dir = tmp_adapter_dir / model_id / name + adapter_dir.mkdir(parents=True) + (adapter_dir / "adapter_config.json").write_text( + json.dumps({"reward_model": True, "fine_tune_type": "lora"}) + ) + return adapter_dir + + def test_score_reward_happy_path(self, client, mock_pool, tmp_adapter_dir): + mock_pool.get_entry.return_value = MagicMock( + model_type="llm", model_path="/tmp/qwen3" + ) + self._make_adapter(tmp_adapter_dir) + with patch( + "fusion_mlx.training.reward_score.score_completions", + return_value=[0.9, 0.1], + ): + resp = client.post( + "/api/fine-tune/reward/score", + json={ + "model_id": "qwen3", + "adapter_name": "rm-1", + "prompt": "What is 1+1?", + "completions": ["The answer is 2.", "The answer is 101."], + }, + ) + assert resp.status_code == 200, resp.text + data = resp.json() + assert data["rewards"] == [0.9, 0.1] + assert data["model_id"] == "qwen3" + assert data["adapter_name"] == "rm-1" + + def test_score_reward_missing_model_id(self, client): + resp = client.post( + "/api/fine-tune/reward/score", + json={"adapter_name": "rm-1", "prompt": "x", "completions": ["a"]}, + ) + assert resp.status_code == 400 + + def test_score_reward_missing_completions(self, client): + resp = client.post( + "/api/fine-tune/reward/score", + json={ + "model_id": "qwen3", + "adapter_name": "rm-1", + "prompt": "x", + "completions": [], + }, + ) + assert resp.status_code == 400 + + def test_score_reward_adapter_not_found(self, client, mock_pool): + mock_pool.get_entry.return_value = MagicMock( + model_type="llm", model_path="/tmp/qwen3" + ) + resp = client.post( + "/api/fine-tune/reward/score", + json={ + "model_id": "qwen3", + "adapter_name": "nope", + "prompt": "x", + "completions": ["a"], + }, + ) + assert resp.status_code == 404 diff --git a/tests/unit/test_ui_tars_lane_parity.py b/tests/unit/test_ui_tars_lane_parity.py index 346dd28..01eb47b 100644 --- a/tests/unit/test_ui_tars_lane_parity.py +++ b/tests/unit/test_ui_tars_lane_parity.py @@ -1347,8 +1347,8 @@ def test_route_fast_path_helper_source_emits_only_reasoning_content(self): # keys) must be absent. import inspect - import fusion_mlx.routes_internal.chat as _chat_mod import fusion_mlx.api.openai_routes as _routes_mod + import fusion_mlx.routes_internal.chat as _chat_mod # The fast-path SSE helper migrated from the (now-shim) # ``routes_internal.chat`` to ``api.openai_routes`` in the