Replace tokenizers-cpp with our own Rust tokenizer FFI and add zero-shot classification

.github/workflows/release.yml

@@ -7,14 +7,19 @@ on:
     # Tag == the ort-server version (no `v` prefix), matching the lemond backend
     # version pin, so the release download URL resolves. `v*` kept for back-compat.
     tags: ["[0-9]*", "v*"]
+  pull_request:
   workflow_dispatch:
     inputs:
       version:
         description: "Version string for artifact names (e.g. 0.1.0)"
         required: true
 
 permissions:
-  contents: write
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   build:
@@ -30,7 +35,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      # tokenizers-cpp builds a small Rust static lib via cargo.
+      # third_party/tok_ffi is a small Rust static lib, built via cargo.
       - uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: "1.88"  # pinned: the tokenizer stack is statically linked
@@ -51,20 +56,62 @@ jobs:
         shell: bash
         run: |
           V="${{ github.event.inputs.version }}"
+          if [ -z "$V" ] && [ "${{ github.event_name }}" = "pull_request" ]; then
+            V="0.0.0-pr${{ github.event.number }}"
+          fi
           [ -z "$V" ] && V="${GITHUB_REF_NAME#v}"
+          case "$V" in
+            */*) echo "FAIL: version '$V' contains a slash"; exit 1 ;;
+          esac
+          if [ "${{ github.ref_type }}" = "tag" ]; then
+            P=$(sed -n 's/^project(ort-server VERSION \([0-9.]*\).*/\1/p' CMakeLists.txt)
+            [ "$V" = "$P" ] || { echo "FAIL: tag '$V' != CMakeLists.txt VERSION '$P'"; exit 1; }
+          fi
           echo "version=$V" >> "$GITHUB_OUTPUT"
 
       # Use the runner's default CMake (Windows ships 4.x for VS 2026 support).
       # Pass CMAKE_POLICY_VERSION_MINIMUM via bash (PowerShell mangles "3.5" ->
-      # "3") so CMake 4.x accepts the old cmake_minimum_required in sentencepiece
-      # / protobuf / etc.
+      # "3") so CMake 4.x accepts any old cmake_minimum_required in a fetched
+      # dependency.
       - name: Configure
         shell: bash
         run: cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5
 
       - name: Build
         run: cmake --build build --config Release --parallel
 
+      - name: Test tok_ffi
+        shell: bash
+        run: cargo test --release --locked --manifest-path third_party/tok_ffi/Cargo.toml --target-dir build/tok_ffi
+
+      # Prebuilt binaries: `cargo install` would compile both tools from source
+      # on all four runners, which costs more than the build it guards.
+      - name: Install license tools
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-about,cargo-deny
+
+      # Fail the build on a license this project has not accepted, BEFORE the
+      # archive exists. A dependency bump that pulls in copyleft stops here.
+      - name: Check crate licenses
+        shell: bash
+        run: cargo deny --manifest-path third_party/tok_ffi/Cargo.toml check licenses
+
+      # 80 crates in Cargo.lock are reached through tok_ffi. MIT and Apache-2.0
+      # both require the notice to ship with a binary distribution, and that
+      # list changes on every bump, so generate it rather than maintain it.
+      - name: Generate crate license notices
+        shell: bash
+        run: |
+          cargo about generate \
+            --manifest-path third_party/tok_ffi/Cargo.toml \
+            third_party/tok_ffi/about.hbs > rust-crates-NOTICES.html
+          # An empty or near-empty file means the template or the target list
+          # stopped matching. That must fail here, not ship.
+          [ -s rust-crates-NOTICES.html ] || { echo "FAIL: notices file is empty"; exit 1; }
+          grep -q "license-text" rust-crates-NOTICES.html || {
+            echo "FAIL: notices file holds no license texts"; exit 1; }
+
       - name: Stage
         shell: bash
         run: |
@@ -87,6 +134,8 @@ jobs:
           # ORT texts come from the exact version-pinned upstream package.
           cp build/_deps/onnxruntime-src/LICENSE stage/licenses/onnxruntime-LICENSE
           cp build/_deps/onnxruntime-src/ThirdPartyNotices.txt stage/licenses/onnxruntime-ThirdPartyNotices.txt
+          # The generated Rust crate list (see "Generate crate license notices").
+          cp rust-crates-NOTICES.html stage/licenses/
 
       - name: Package
         shell: bash
@@ -115,7 +164,8 @@ jobs:
           fi
           [ -f smoke/LICENSE ] || { echo "FAIL: LICENSE missing from archive"; exit 1; }
           [ -f smoke/THIRD_PARTY_NOTICES.md ] || { echo "FAIL: THIRD_PARTY_NOTICES.md missing"; exit 1; }
-          for lf in onnxruntime-LICENSE onnxruntime-ThirdPartyNotices.txt $(ls third_party_licenses); do
+          for lf in onnxruntime-LICENSE onnxruntime-ThirdPartyNotices.txt \
+                    rust-crates-NOTICES.html $(ls third_party_licenses); do
             [ -f "smoke/licenses/$lf" ] || { echo "FAIL: licenses/$lf missing from archive"; exit 1; }
           done
           # Full contract suite: normalization, config.json inference, token
@@ -136,7 +186,9 @@ jobs:
           fi
           python3 test/check_deps.py "$BIN"
 
+      # A pull request build proves the code; it does not produce a download.
       - uses: actions/upload-artifact@v4
+        if: github.event_name != 'pull_request'
         with:
           name: ort-server-${{ matrix.platform }}
           path: ort-server-*.*
@@ -145,6 +197,9 @@ jobs:
     needs: build
     if: startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-latest
+    # The only job that publishes, and so the only one that needs write.
+    permissions:
+      contents: write
     steps:
       - uses: actions/download-artifact@v4
         with: { path: dist, merge-multiple: true }

.gitignore

@@ -5,3 +5,4 @@ dist/
 *.tar.gz
 build*/
 *.log
+third_party/tok_ffi/target/

CMakeLists.txt

@@ -1,5 +1,6 @@
 cmake_minimum_required(VERSION 3.24)
-project(ort-server CXX)
+# The release tag must equal VERSION (release.yml checks); /health reports it.
+project(ort-server VERSION 0.4.0 LANGUAGES CXX)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -53,38 +54,92 @@ FetchContent_Declare(onnxruntime
 FetchContent_MakeAvailable(onnxruntime)
 set(ORT_ROOT "${onnxruntime_SOURCE_DIR}")
 
-# --- tokenizers-cpp (runtime HF tokenizer from tokenizer.json) -------------
+# --- tokenizer FFI (runtime HF tokenizer from tokenizer.json) --------------
 # Loads the model's own tokenizer.json (the exact HuggingFace tokenizer), so any
 # HF text classifier runs from a STANDARD ONNX export + its tokenizer.json — no
-# per-model in-graph baking, guaranteed tokenization parity. Builds a small Rust
-# static lib via cargo (present on CI runners and any dev with rustup); this is a
-# build-time dependency only — users of the prebuilt binary never see Rust.
-if(WIN32)
-  set(TOKENIZERS_CPP_MSVC_RUNTIME_LIBRARY "MD" CACHE STRING "" FORCE)
+# per-model in-graph baking, guaranteed tokenization parity.
+#
+# third_party/tok_ffi is a ~100-line C ABI over the HuggingFace `tokenizers`
+# crate. It passes through the character offsets the zero-shot router needs to
+# tell which tokens belong to which label (docs/ZERO-SHOT-ROUTER.md). Building
+# it needs cargo; users of the prebuilt binary never see Rust.
+find_program(CARGO_EXECUTABLE cargo
+  HINTS "$ENV{USERPROFILE}/.cargo/bin" "$ENV{HOME}/.cargo/bin")
+if(NOT CARGO_EXECUTABLE)
+  message(FATAL_ERROR
+    "cargo not found. ort-server builds a small Rust static library from "
+    "third_party/tok_ffi. Install rustup (https://rustup.rs) and re-run; this "
+    "is a build-time requirement only.")
+endif()
+
+# Rust links the dynamic CRT (/MD) unless +crt-static is forced on, so pin the
+# C++ side to /MD too
+# ONNX Runtime as well.
+if(MSVC)
+  set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDLL")
 endif()
-FetchContent_Declare(tokenizers_cpp
-  GIT_REPOSITORY https://github.com/mlc-ai/tokenizers-cpp.git
-  GIT_TAG c586c52f93f7b060753bd2388eb96a105cb7374d)  # pinned main, 2026-07
-FetchContent_MakeAvailable(tokenizers_cpp)
+
+set(TOK_FFI_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/tok_ffi")
+# Build under the binary dir so a configure-and-build never writes to the source
+# tree (and so a fresh build dir gets a fresh crate build).
+set(TOK_FFI_TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}/tok_ffi")
+set(TOK_FFI_LIB
+  "${TOK_FFI_TARGET_DIR}/release/${CMAKE_STATIC_LIBRARY_PREFIX}tok_ffi${CMAKE_STATIC_LIBRARY_SUFFIX}")
+
+add_custom_command(
+  OUTPUT "${TOK_FFI_LIB}"
+  # --locked: build the versions Cargo.lock pins, and fail if the lock file is
+  # stale, rather than silently resolving something else. The release job
+  # already builds this way; so must a local build, or they are not the same
+  # binary.
+  COMMAND "${CARGO_EXECUTABLE}" build --release --locked
+          --manifest-path "${TOK_FFI_DIR}/Cargo.toml"
+          --target-dir "${TOK_FFI_TARGET_DIR}"
+  # Cargo.lock belongs here: `cargo update` rewrites it without touching
+  # Cargo.toml, and without this line the stale static library looks current.
+  DEPENDS "${TOK_FFI_DIR}/src/lib.rs"
+          "${TOK_FFI_DIR}/Cargo.toml"
+          "${TOK_FFI_DIR}/Cargo.lock"
+          "${TOK_FFI_DIR}/tok_ffi.h"
+  COMMENT "Building tokenizer FFI (Rust)"
+  VERBATIM)
+add_custom_target(tok_ffi_build DEPENDS "${TOK_FFI_LIB}")
 
 # --- Target -----------------------------------------------------------------
 add_executable(ort-server src/main.cpp)
+add_dependencies(ort-server tok_ffi_build)
 target_compile_definitions(ort-server PRIVATE
+  ORT_SERVER_VERSION="${PROJECT_VERSION}"
   ORT_SERVER_ONNXRUNTIME_VERSION="${ONNXRUNTIME_VERSION}")
 target_include_directories(ort-server PRIVATE
   "${ORT_ROOT}/include"
-  "${tokenizers_cpp_SOURCE_DIR}/include")
+  "${TOK_FFI_DIR}")
 target_link_libraries(ort-server PRIVATE
   httplib::httplib
   nlohmann_json::nlohmann_json
-  tokenizers_cpp)
+  "${TOK_FFI_LIB}")
 
 if(WIN32)
+  # The Rust standard library inside tok_ffi.lib needs these. Regenerate
+  # the list with:
+  #   cargo rustc --release --manifest-path third_party/tok_ffi/Cargo.toml -- \
+  #     --print native-static-libs
+  target_link_libraries(ort-server PRIVATE
+    kernel32 ntdll userenv ws2_32 dbghelp)
   target_link_libraries(ort-server PRIVATE "${ORT_ROOT}/lib/onnxruntime.lib")
   add_custom_command(TARGET ort-server POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E copy_if_different
       "${ORT_ROOT}/lib/onnxruntime.dll" "$<TARGET_FILE_DIR:ort-server>")
 else()
+  # Same story as the WIN32 branch: libtok_ffi.a carries the Rust standard
+  # library's code but not the platform libraries it calls. These resolve by
+  # accident on glibc >= 2.34, where libpthread and libdl folded into libc, and
+  # fail to link on older glibc (CentOS 7, Ubuntu 18.04) or on musl. Regenerate
+  # the list with:
+  #   cargo rustc --release --manifest-path third_party/tok_ffi/Cargo.toml -- \
+  #     --print native-static-libs
+  find_package(Threads REQUIRED)
+  target_link_libraries(ort-server PRIVATE Threads::Threads ${CMAKE_DL_LIBS} m)
   file(GLOB _ort_libs "${ORT_ROOT}/lib/libonnxruntime.*")
   target_link_libraries(ort-server PRIVATE ${_ort_libs})
 endif()

README.md

@@ -29,18 +29,22 @@ architectures with different segment/special-token conventions (XLNet, for
 instance, puts its classifier token last with a distinct segment id). The
 supported set is an explicit allowlist checked against `config.json`'s
 `model_type` at startup — an unsupported model is **refused**, not served with
-silently wrong scores.
+silently wrong scores. The allowlist is **per task**: zero-shot routing uses a
+different input convention entirely, so it has its own list and the two do not
+mix in either direction.
 
 - `model.onnx` is a plain export (`input_ids`/`attention_mask`[/`token_type_ids`] → logits) — the ordinary `optimum` export, no in-graph baking, no custom ops.
-- The server tokenizes at runtime by loading the model's `tokenizer.json` via [mlc-ai/tokenizers-cpp](https://github.com/mlc-ai/tokenizers-cpp), which is the same Rust tokenizer `transformers` uses.
+- The server tokenizes at runtime by loading the model's `tokenizer.json` through `third_party/tok_ffi`, a small C ABI over the [HuggingFace `tokenizers`](https://github.com/huggingface/tokenizers) crate — the same Rust tokenizer `transformers` uses.
 
-**What "parity" means here, precisely.** The tokenizer returns token ids only, so
-the attention mask, segment ids, padding handling and special-token filtering are
+**What "parity" means here, precisely.** The tokenizer's ids are exact, but the
+attention mask, segment ids, padding handling and special-token filtering are
 reconstructed by this server to match the HuggingFace pipeline. That reconstruction
 is verified against HF-computed reference scores on every release
 (`test/fixtures/*/golden.json`), but it is only *valid* for the single-sequence
 encoder families in the allowlist below — it is not a general guarantee for "any
-HuggingFace classifier".
+HuggingFace classifier". The tokenizer half of that claim has its own harness,
+`test/parity_tokenizer.py`, which compares ids **and character offsets** against
+HuggingFace case by case; run it whenever the tokenizer moves.
 - The output contract (task, labels, normalization, token budget) is inferred
   from the export's own `config.json` + `tokenizer_config.json`. `problem_type`
   is only a training-time hint, so manifest-less inference **assumes
@@ -72,12 +76,47 @@ error, and the model's output dimension must match `id2label` at inference time)
 }
 ```
 
+### Zero-shot routing
+
+`task: "zero-shot-classification"` takes its labels from **each request**
+instead of from a manifest, and scores them against the text on the fly:
+
+```
+POST /classify
+{"text": "Write a Python function that merges two sorted lists.",
+ "labels": ["code generation", "math reasoning", "creative writing"]}
+```
+
+Such a model reads the labels as part of its input text and needs two extra
+pooling inputs, built here from the tokenizer's character offsets. It has its
+own architecture allowlist, its own token budget, its own request limits and its
+own truncation rule, all described in
+**[docs/ZERO-SHOT-ROUTER.md](docs/ZERO-SHOT-ROUTER.md)**.
+
+A ready-to-serve model directory is
+[`lemonade-sdk/LFM2.5-Encoder-350M-Prompt-Router-ONNX`](https://huggingface.co/lemonade-sdk/LFM2.5-Encoder-350M-Prompt-Router-ONNX),
+converted from
+[`LiquidAI/LFM2.5-Encoder-350M-Prompt-Router`](https://huggingface.co/LiquidAI/LFM2.5-Encoder-350M-Prompt-Router)
+with `tools/zero_shot_router/export.py`.
+
 ## HTTP contract
 
 | Method | Path | Body | Response |
 |--------|------|------|----------|
-| GET | `/health` | — | `200` when the model is loaded and ready |
-| POST | `/classify` | `{"text": "...", "top_k": N?}` | `{"labels": {"<label>": <score in [0,1]>, ...}}` — `top_k` omitted or `0` returns all labels |
+| GET | `/health` | — | `200` when the model is loaded and ready: `{"status": "ok", "version": "<ort-server>", "onnxruntime": "<ORT>"}` |
+| POST | `/classify` | `{"text": "...", "top_k": N?, "labels": [...]?}` | `{"labels": {"<label>": <score in [0,1]>, ...}, "truncated": <bool>}` — `top_k` omitted or `0` returns all labels |
+
+`labels` is required for a `zero-shot-classification` model and rejected for any
+other; both mismatches are a `400` rather than a silently ignored field.
+
+`truncated` is on every response, for any task: `true` when the text did not fit
+the token window and was cut, `false` when it fitted. What the cut drops
+depends on the path: the encoder tasks keep the head and drop the end, while
+`zero-shot-classification` keeps the head and the tail and drops the middle.
+
+Servers before 0.4.0 cut without reporting it: they send no `truncated`, and
+their `/health` carries no `version`. A missing field therefore means "unknown",
+never "fitted".
 
 Future capabilities (same server, new endpoints): `POST /embed`, `POST /rerank`.
 
@@ -103,7 +142,7 @@ cmake --build build --config Release
 ./build/ort-server --model-path <model-dir> --port 8100
 ```
 
-CMake fetches ONNX Runtime, tokenizers-cpp, cpp-httplib, and nlohmann/json (see `CMakeLists.txt`). tokenizers-cpp builds a small Rust static lib, so **cargo/rustup must be on PATH** to build ort-server from source (build-time only — users of the prebuilt binary need nothing).
+CMake fetches ONNX Runtime, cpp-httplib, and nlohmann/json (see `CMakeLists.txt`) and builds `third_party/tok_ffi` in-tree. `tok_ffi` is a Rust static lib, so **cargo/rustup must be on PATH** to build ort-server from source (build-time only — users of the prebuilt binary need nothing).
 
 ## Releases
 

THIRD_PARTY_NOTICES.md

@@ -9,18 +9,21 @@ at build time).
 | Component | License | Text in archive | Distribution |
 |-----------|---------|-----------------|--------------|
 | [ONNX Runtime](https://github.com/microsoft/onnxruntime) © Microsoft Corporation | MIT | `licenses/onnxruntime-LICENSE`, `licenses/onnxruntime-ThirdPartyNotices.txt` | Shared library bundled in the archive |
-| [tokenizers-cpp](https://github.com/mlc-ai/tokenizers-cpp) © tokenizers-cpp contributors | Apache-2.0 | `licenses/tokenizers-cpp-LICENSE` | Statically linked |
-| [HuggingFace tokenizers](https://github.com/huggingface/tokenizers) © HuggingFace | Apache-2.0 | `licenses/huggingface-tokenizers-LICENSE` | Statically linked (via tokenizers-cpp) |
+| [HuggingFace tokenizers](https://github.com/huggingface/tokenizers) © HuggingFace | Apache-2.0 | `licenses/huggingface-tokenizers-LICENSE` | Statically linked (via `third_party/tok_ffi`) |
 | [rust-onig / onig_sys](https://github.com/rust-onig/rust-onig) © rust-onig contributors | MIT | `licenses/rust-onig-LICENSE` | Statically linked (via HF tokenizers) |
 | [Oniguruma](https://github.com/kkos/oniguruma) © K. Kosako | BSD-2-Clause | `licenses/oniguruma-LICENSE` | Statically linked (vendored by onig_sys) |
-| [SentencePiece](https://github.com/google/sentencepiece) © Google | Apache-2.0 | `licenses/sentencepiece-LICENSE` | Statically linked (via tokenizers-cpp) |
-| [Abseil](https://github.com/abseil/abseil-cpp) © Google | Apache-2.0 | `licenses/abseil-cpp-LICENSE` | Statically linked (vendored by sentencepiece) |
-| [protobuf-lite](https://github.com/protocolbuffers/protobuf) © Google | BSD-3-Clause | `licenses/protobuf-lite-LICENSE` | Statically linked (vendored by sentencepiece) |
-| [Darts-clone](https://github.com/s-yata/darts-clone) © Susumu Yata | BSD-2-Clause | `licenses/darts_clone-LICENSE` | Statically linked (vendored by sentencepiece) |
-| [esaxx](https://github.com/hillbig/esaxx) © Daisuke Okanohara | MIT | `licenses/esaxx-LICENSE` | Statically linked (vendored by sentencepiece) |
-| [msgpack-cxx](https://github.com/msgpack/msgpack-c) © msgpack contributors | BSL-1.0 | `licenses/msgpack-LICENSE` | Header-only, compiled in (via tokenizers-cpp) |
+| [esaxx](https://github.com/hillbig/esaxx) © Daisuke Okanohara | MIT | `licenses/esaxx-LICENSE` | Statically linked (vendored by the `esaxx-rs` crate) |
 | [cpp-httplib](https://github.com/yhirose/cpp-httplib) © Yuji Hirose | MIT | `licenses/cpp-httplib-LICENSE` | Header-only, compiled in |
 | [nlohmann/json](https://github.com/nlohmann/json) © Niels Lohmann | MIT | `licenses/nlohmann-json-LICENSE` | Header-only, compiled in |
+| Rust crates reached through `third_party/tok_ffi` (80 in `Cargo.lock`, including the tokenizer crates named above) | MIT / Apache-2.0 / Unicode-3.0 | `licenses/rust-crates-NOTICES.html` | Statically linked; **generated** per release from `Cargo.lock` |
+
+The last row is a count rather than a list because it is generated at release
+time from `Cargo.lock`: `licenses/rust-crates-NOTICES.html` in the archive is
+the authoritative list of those crates and carries their full license texts. The
+release job will not package an archive without it, and fails outright if a
+dependency arrives under a license this project has not accepted. The accepted
+set lives in `third_party/tok_ffi/about.toml` and `deny.toml`.
 
 Exact dependency versions are pinned in `CMakeLists.txt` (SHA256 / commit
-SHAs) and recorded in `VERSIONS.md`.
+SHAs) and recorded in `VERSIONS.md`; the Rust side is pinned in
+`third_party/tok_ffi/Cargo.lock` and built `--locked`.