diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..88c6f36 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,45 @@ +# Keep the build context tight — Docker uploads the whole working +# directory to the daemon otherwise, which is slow on big repos. + +# VCS + tooling +.git +.github +.gitignore +.lycheeignore + +# Local builds / artefacts +node_modules +data +target +rust/target +*.log +coverage +.nyc_output + +# Docs and large assets we don't need inside the image +docs +research +*.png +*.jpg +*.pdf +SEARCH_README.md + +# Non-runtime artefacts +api-patterns.json +limitations-found.json +*.html +*.css +*.jsx + +# Tests / scripts that don't ship in the container +js/tests +js/scripts +rust/src +rust/tests +rust/Cargo.* +rust/changelog.d +rust/README.md +rust/rust-toolchain.toml +rust/rustfmt.toml +examples +experiments diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml index 83d4009..2c57386 100644 --- a/.github/workflows/js.yml +++ b/.github/workflows/js.yml @@ -1,15 +1,16 @@ name: JS -# Single, unified pipeline that subsumes the previous test.yml, pages.yml, -# and links.yml workflows. Structured to fail fast: cheap checks -# (syntax-check, unit tests, link audit) gate the slower jobs (e2e against -# a locally-served build, Pages deploy, e2e against the deployed site). +# Single, unified JavaScript pipeline that subsumes the previous test.yml, +# pages.yml, links.yml, and release.yml responsibilities. Structured to fail +# fast: cheap checks (syntax-check, unit tests, link audit) gate the slower jobs +# (e2e against a locally-served build, Pages deploy, e2e against the deployed +# site, npm publish, Docker publish). # # Layout adopted from link-foundation/js-ai-driven-development-pipeline-template -# (release.yml), trimmed for this repo: -# * no npm publish path — this is a static site, not a library -# * no changeset PR / manual release jobs -# * no Docker publish job +# (release.yml), kept in this repository as js.yml per PR review: +# * npm publish is gated on package.json version bumps +# * Docker publish targets ghcr.io +# * changeset PR and manual release jobs are deferred # The template's `detect-changes` matrix is intentionally omitted: every job # below already either runs in <1 minute or is gated by a `needs:` chain, # so partial skipping wouldn't meaningfully improve wall time and would @@ -22,6 +23,11 @@ on: pull_request: types: [opened, synchronize, reopened] workflow_dispatch: + inputs: + force_publish: + description: 'Publish even if package.json version is unchanged' + type: boolean + default: false # Cancel in-flight runs for the same ref so the latest commit wins on # `main` and on PR force-pushes. Matches the template's policy. @@ -58,9 +64,11 @@ jobs: - uses: actions/setup-node@v6 with: node-version: '20.x' + - name: Install dependencies + run: npm ci - name: Run unit and integration suites - # `run-unit-tests.mjs` distinguishes gating (zero-dep node:test - # suites: routing.test.mjs, ipa.test.mjs) from informational + # `run-unit-tests.mjs` distinguishes gating node:test suites + # (including dependency-backed config/cache tests) from informational # (live-Wikidata cache/transformer scripts). Only gating failures # set a non-zero exit code. run: node js/scripts/run-unit-tests.mjs @@ -116,11 +124,7 @@ jobs: with: node-version: '20.x' - name: Install dependencies - # We don't have a deep dependency tree (only @playwright/test as a - # devDep), so a vanilla `npm install` is fast and produces a clean - # node_modules. `npm ci` would require a committed lockfile that - # matches package.json exactly — overkill for this repo. - run: npm install + run: npm ci - name: Install Playwright browsers run: npx playwright install --with-deps chromium - name: Run e2e suite (boots js/scripts/serve-static.mjs) @@ -191,7 +195,7 @@ jobs: with: node-version: '20.x' - name: Install dependencies - run: npm install + run: npm ci - name: Install Playwright browsers run: npx playwright install --with-deps chromium - name: Wait for new content to propagate @@ -208,3 +212,105 @@ jobs: name: playwright-report-deployed path: playwright-report/ retention-days: 7 + + # === npm + Docker release (main only) ===================================== + + detect-version-bump: + name: Detect package.json version bump + runs-on: ubuntu-latest + timeout-minutes: 5 + needs: [unit-tests, e2e-local, link-check] + if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + outputs: + should_publish: ${{ steps.diff.outputs.should_publish }} + version: ${{ steps.diff.outputs.version }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 2 + - id: diff + name: Compare versions + run: node js/scripts/check-package-version.mjs + env: + FORCE_PUBLISH: ${{ inputs.force_publish }} + + publish-npm: + name: npm publish + runs-on: ubuntu-latest + timeout-minutes: 15 + needs: [detect-version-bump, unit-tests] + if: needs.detect-version-bump.outputs.should_publish == 'true' + permissions: + contents: read + id-token: write + environment: + name: npm + url: https://www.npmjs.com/package/human-language + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: '20.x' + registry-url: 'https://registry.npmjs.org' + - name: Install dependencies + run: npm ci + - name: npm publish --provenance + run: npm publish --provenance --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + publish-docker: + name: Docker publish (ghcr.io) + runs-on: ubuntu-latest + timeout-minutes: 25 + needs: [detect-version-bump, unit-tests] + if: needs.detect-version-bump.outputs.should_publish == 'true' + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v6 + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/human-language + tags: | + type=raw,value=latest + type=raw,value=${{ needs.detect-version-bump.outputs.version }} + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + push: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + create-release: + name: Create GitHub release + runs-on: ubuntu-latest + timeout-minutes: 5 + needs: [detect-version-bump, publish-npm, publish-docker] + if: needs.detect-version-bump.outputs.should_publish == 'true' + permissions: + contents: write + steps: + - uses: actions/checkout@v6 + - name: Create release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ needs.detect-version-bump.outputs.version }} + run: | + gh release create "v${VERSION}" \ + --title "v${VERSION}" \ + --generate-notes \ + --latest diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000..8f1505d --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,118 @@ +name: Rust + +# Lint, test, and release pipeline for the Rust crate that lives under +# `rust/`. Mirrors the cadence of `js.yml`: PRs are gated on fmt+clippy+test; +# `main` pushes additionally publish a crates.io release when the version +# in `rust/Cargo.toml` advances. +# +# Adopted from rust-ai-driven-development-pipeline-template, trimmed to +# the surface this repo actually uses. + +on: + push: + branches: + - main + paths: + - 'rust/**' + - '.github/workflows/rust.yml' + pull_request: + types: [opened, synchronize, reopened] + paths: + - 'rust/**' + - '.github/workflows/rust.yml' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref == 'refs/heads/main' }} + +permissions: + contents: read + +defaults: + run: + working-directory: rust + +jobs: + fmt: + name: cargo fmt --check + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + - run: cargo fmt --all -- --check + + clippy: + name: cargo clippy + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: [fmt] + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + with: + workspaces: rust + - run: cargo clippy --all-targets -- -D warnings + + test: + name: cargo test + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + needs: [fmt] + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: rust + - run: cargo test --all-targets + + # === Publish to crates.io on version-advancing pushes to main ============= + + publish-check: + name: Detect Cargo.toml version bump + runs-on: ubuntu-latest + timeout-minutes: 5 + needs: [clippy, test] + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + outputs: + should_publish: ${{ steps.diff.outputs.should_publish }} + version: ${{ steps.diff.outputs.version }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 2 + - id: diff + name: Compare versions + working-directory: . + run: node rust/scripts/check-cargo-version.mjs + + publish: + name: cargo publish + runs-on: ubuntu-latest + timeout-minutes: 15 + needs: [publish-check] + if: needs.publish-check.outputs.should_publish == 'true' + environment: + name: crates-io + url: https://crates.io/crates/human-language + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: rust + - name: cargo publish --token $CARGO_REGISTRY_TOKEN + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + run: cargo publish --locked diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..282f019 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,51 @@ +# syntax=docker/dockerfile:1.7 +# +# Multi-stage build for the `human-language` microservice. +# +# Stage 1 ("deps") installs production npm dependencies into a fresh layer +# so the resulting image only contains what the runtime needs. +# +# Stage 2 ("runtime") starts from a minimal node base image, drops to a +# non-root user, and runs `human-language serve`. Configuration is read +# from environment variables (HUMAN_LANGUAGE_*) — see js/src/config.js. +# +# Build: +# docker build -t human-language:dev . +# Run: +# docker run --rm -p 8080:8080 human-language:dev + +ARG NODE_VERSION=22.11.0 + +FROM node:${NODE_VERSION}-alpine AS deps +WORKDIR /app +COPY package.json package-lock.json* ./ +# `npm ci --omit=dev` will fail if no lockfile exists, so prefer install. +RUN if [ -f package-lock.json ]; then npm ci --omit=dev; else npm install --omit=dev; fi + +FROM node:${NODE_VERSION}-alpine AS runtime +WORKDIR /app + +# Drop privileges. The `node:alpine` image already ships a `node` user. +USER node + +# Copy only the runtime surface: production deps, source, and metadata. +COPY --chown=node:node --from=deps /app/node_modules ./node_modules +COPY --chown=node:node package.json ./ +COPY --chown=node:node js ./js + +ENV NODE_ENV=production \ + HUMAN_LANGUAGE_HOST=0.0.0.0 \ + HUMAN_LANGUAGE_PORT=8080 \ + HUMAN_LANGUAGE_CACHE_TYPE=file \ + HUMAN_LANGUAGE_CACHE_DIR=/app/data/wikidata-cache + +EXPOSE 8080 + +# A small inline healthcheck so orchestrators can see the server is live +# without depending on `curl` being in the image. We use Node's built-in +# fetch so no extra binaries are needed. +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD node -e "fetch('http://127.0.0.1:'+(process.env.HUMAN_LANGUAGE_PORT||8080)+'/healthz').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" + +ENTRYPOINT ["node", "js/src/cli.js"] +CMD ["serve"] diff --git a/docs/case-studies/issue-37/README.md b/docs/case-studies/issue-37/README.md new file mode 100644 index 0000000..32cf7cd --- /dev/null +++ b/docs/case-studies/issue-37/README.md @@ -0,0 +1,102 @@ +# Case Study — Issue #37 + +> **Issue:** [link-assistant/human-language#37](https://github.com/link-assistant/human-language/issues/37) +> "Make it more universally accessible." +> **Pull request:** [#38](https://github.com/link-assistant/human-language/pull/38) +> **Branch:** `issue-37-0d7db2803f2c` + +## Summary + +Issue #37 continues the vision of #33 (unified SPA) by removing the +remaining "JavaScript-and-GitHub-Pages only" constraint. Today every +useful piece of logic in this repository is reachable only by opening +`app.html` in a browser. The issue asks us to: + +1. Expose the same core logic as a **library on npm** (for JavaScript + consumers) so other projects — most concretely + [`link-assistant/meta-expression`](https://github.com/link-assistant/meta-expression) + and [`link-assistant/calculator`](https://github.com/link-assistant/calculator) — + can reuse it without scraping the SPA. +2. Mirror the same core logic as a **crate on crates.io** so Rust + consumers (and WASM consumers) have parity. +3. Expose the same public API as an HTTP **microservice in Docker** so + any language can call it. +4. Adopt the configuration / encoding / data-storage primitives of the + `link-foundation` ecosystem: + [`lino-arguments`](https://github.com/link-foundation/lino-arguments), + [`lino-objects-codec`](https://github.com/link-foundation/lino-objects-codec), + [`links-notation`](https://github.com/link-foundation/links-notation), + [`link-cli`](https://github.com/link-foundation/link-cli) / + [`doublets-rs`](https://github.com/linksplatform/doublets-rs) / + [`doublets-web`](https://github.com/linksplatform/doublets-web). +5. Match the CI/CD blueprints of + [`js-ai-driven-development-pipeline-template`](https://github.com/link-foundation/js-ai-driven-development-pipeline-template) + and + [`rust-ai-driven-development-pipeline-template`](https://github.com/link-foundation/rust-ai-driven-development-pipeline-template). + +## Documents in this case study + +| File | Purpose | +| --- | --- | +| [`requirements.md`](./requirements.md) | Verbatim list of every requirement extracted from the issue body, with the status of each in PR #38. | +| [`external-research.md`](./external-research.md) | Notes on the eleven external repositories referenced from the issue body, plus other prior art for the same problem (universal accessibility via library + WASM + microservice). | +| [`known-components.md`](./known-components.md) | Short catalogue of libraries / data sources that we plan to depend on (or imitate) per requirement. | +| [`ci-template-comparison.md`](./ci-template-comparison.md) | File-by-file comparison of our `.github/workflows/` against `js-ai-driven-development-pipeline-template` and `rust-ai-driven-development-pipeline-template`; lists which jobs land in PR #38 and which are deferred. | +| [`solution-plans.md`](./solution-plans.md) | The chosen plan per requirement, plus the alternatives that were considered. | +| [`architecture.md`](./architecture.md) | Prose-and-ASCII diagram of the four surfaces (library, Rust crate, Docker microservice, existing SPA) and where they share code. | +| [`timeline.md`](./timeline.md) | Chronological reconstruction of the work that landed in PR #38. | + +## The fix in one sentence + +The core text-to-Q/P logic, the Wikidata client, the in-memory and +IndexedDB caches and the locale / IPA helpers were already pure +modules — PR #38 declares the package public, adds typed entry points +(`./` for the library, `./server` for the microservice, `./cli` for the +CLI), drops a `Dockerfile` that runs the same `./server` module, and +adds a Rust crate under `rust/` that mirrors the JS public API one +function at a time (the transformer port is the first; the rest follow +in subsequent PRs because each requires a Wikidata-shaped fixture). + +## What ships in this PR + +- **JS library** — `package.json` is no longer `private`, declares + `main` / `types` / `exports`, ships `bin` for the CLI, and is ready + to publish as `human-language` on npm (the actual publish step is + gated inside `.github/workflows/js.yml` — see + [`ci-template-comparison.md`](./ci-template-comparison.md)). +- **JS CLI** — `js/src/cli.js` exposes + `human-language transform "Albert Einstein was born in Ulm"` and + uses `lino-arguments` for CLI > env > defaults precedence. +- **JS microservice** — `js/src/server.js` exposes + `POST /transform`, `GET /entity/:id`, `GET /property/:id`, + `GET /healthz`. Same precedence rules as the CLI. +- **Dockerfile** — a small multi-stage image that runs + `node js/src/server.js` and exposes port 8080. +- **Rust crate** — `rust/Cargo.toml` declares `human-language` with + both `[lib]` and `[[bin]]`, depends on `lino-arguments` and + `lino-objects-codec`, and ports the tokenizer / n-gram generator / + property-indicator detector, routing helpers, locale helpers, and + LiNo formatting from `js/src/`. The Wikidata HTTP client is sketched + out behind a `reqwest` feature flag; the library currently exposes + deterministic operations only. +- **CI/CD** — `.github/workflows/js.yml` and + `.github/workflows/rust.yml` are adapted from the two templates. + JS release jobs live in `js.yml` per PR review. +- **Case study** — this folder. + +## What does not ship in this PR + +These are tracked as follow-up issues so the PR stays reviewable: + +- Actual npm OIDC trusted-publishing config (requires a repo-level + toggle outside Git). +- Actual crates.io token (requires a `CARGO_REGISTRY_TOKEN` secret). +- Docker Hub publishing (PR #38 publishes the Docker image to GHCR). +- Full Wikidata HTTP client in Rust (port deferred until the JS + fixture suite is exported). +- Inbound Links Notation parsing for HTTP requests. PR #38 serializes + transformer output and cache entries as LiNo, but does not yet accept + LiNo request bodies. + +See [`solution-plans.md`](./solution-plans.md) for the per-requirement +status. diff --git a/docs/case-studies/issue-37/architecture.md b/docs/case-studies/issue-37/architecture.md new file mode 100644 index 0000000..f030477 --- /dev/null +++ b/docs/case-studies/issue-37/architecture.md @@ -0,0 +1,135 @@ +# Architecture after issue #37 + +After PR #38, the project exposes **four surfaces** that all share the +same JavaScript modules under `js/src/` (plus, for Rust consumers, a +parallel implementation under `rust/`). + +``` +┌───────────────────────────────────────────────────────────────┐ +│ Surfaces │ +│ │ +│ ┌─────────────┐ ┌────────────┐ ┌─────────────┐ ┌────────────┐ │ +│ │ SPA │ │ npm │ │ Docker │ │ crates.io │ │ +│ │ (app.html) │ │ library │ │ microservice│ │ crate │ │ +│ │ │ │ (Node/Bun) │ │ (HTTP/JSON) │ │ (Rust) │ │ +│ └──────┬──────┘ └─────┬──────┘ └──────┬──────┘ └──────┬─────┘ │ +│ │ │ │ │ │ +└────────┼──────────────┼───────────────┼───────────────┼───────┘ + │ │ │ │ + ▼ ▼ ▼ ▼ +┌───────────────────────────────────────┐ ┌───────────────────────┐ +│ JS core modules │ │ Rust core │ +│ (js/src/) │ │ (rust/src/) │ +│ │ │ │ +│ ┌─────────────────────────────────┐ │ │ ┌─────────────────┐ │ +│ │ transformation/ │ │ │ │ tokenize.rs │ │ +│ │ text-to-qp-transformer.js │◀──┼─┤ │ tokenize, │ │ +│ │ lino-format.js │ │ │ │ generate_ngr │ │ +│ └─────────────────────────────────┘ │ │ │ is_property… │ │ +│ ┌─────────────────────────────────┐ │ │ └─────────────────┘ │ +│ │ wikidata-api[-browser].js │ │ │ ┌─────────────────┐ │ +│ │ unified-cache[-browser].js │ │ │ │ lino.rs │ │ +│ │ persistent-cache.js │ │ │ │ settings.rs │ │ +│ └─────────────────────────────────┘ │ │ │ │ │ +│ ┌─────────────────────────────────┐ │ │ └─────────────────┘ │ +│ │ app/routing.js │◀──┼─┤ ┌─────────────────┐ │ +│ │ app/ipa.js │ │ │ │ routing.rs │ │ +│ │ settings.js │ │ │ │ │ │ +│ └─────────────────────────────────┘ │ │ └─────────────────┘ │ +│ ┌─────────────────────────────────┐ │ │ ┌─────────────────┐ │ +│ │ index.js ◀── re-exports │ │ │ │ lib.rs │ │ +│ │ server.js (new) │ │ │ │ bin/cli.rs │ │ +│ │ cli.js (new) │ │ │ └─────────────────┘ │ +│ │ config.js (new) │ │ │ │ +│ └─────────────────────────────────┘ │ │ │ +└───────────────────────────────────────┘ └───────────────────────┘ + ▲ ▲ + │ depends on │ depends on + │ │ + ┌──────────┴────────────┐ ┌───────────┴─────────────┐ + │ npm deps │ │ crate deps │ + │ lino-objects-codec │ │ lino-arguments │ + │ lino-arguments │ │ lino-objects-codec │ + │ │ │ reqwest (feature) │ + └───────────────────────┘ │ serde / serde_json │ + │ │ + └─────────────────────────┘ +``` + +## Module ownership + +| Module path | Owner | Surface(s) that import it | +| --- | --- | --- | +| `js/src/index.js` | npm library | npm | +| `js/src/server.js` | server | Docker, npm `./server` import | +| `js/src/cli.js` | CLI | npm `bin`, Docker (entrypoint shim) | +| `js/src/config.js` | shared | server, CLI | +| `js/src/transformation/text-to-qp-transformer.js` | core | SPA, npm, server, CLI | +| `js/src/transformation/lino-format.js` | core | npm `transform({ format:'lino' })`, server `Accept: text/plain;codec=lino` | +| `js/src/wikidata-api-browser.js` | core | SPA, npm | +| `js/src/wikidata-api.js` | core | server, CLI | +| `js/src/unified-cache-browser.js` | core | SPA, npm | +| `js/src/unified-cache.js` | core | server, CLI | +| `js/src/persistent-cache.js` | core | server, CLI | +| `js/src/app/routing.js` | core | SPA, npm | +| `js/src/app/ipa.js` | core | SPA, npm | +| `js/src/settings.js` | core | SPA, npm | +| `js/src/app/shell.jsx` + `app/modes/*.jsx` | SPA-only | SPA (not re-exported) | +| `js/src/statements.jsx` | SPA-only | SPA | +| `js/src/loading.jsx` | SPA-only | SPA | +| `js/src/app/tests-panel.jsx` | SPA-only | SPA | +| `rust/src/tokenize.rs` | Rust core | crate `[lib]`, `[[bin]]` | +| `rust/src/routing.rs` | Rust core | crate `[lib]` | +| `rust/src/settings.rs` | Rust core | crate `[lib]` | +| `rust/src/lino.rs` | Rust core | crate `[lib]`, `[[bin]]` | +| `rust/src/lib.rs` | Rust core | crate `[lib]` | +| `rust/src/bin/cli.rs` | Rust CLI | crate `[[bin]]` | + +## Data flow (POST /transform under Docker) + +``` +HTTP POST { text, options } + │ + ▼ +js/src/server.js ─── parse JSON, validate + │ + ▼ +js/src/transformation/text-to-qp-transformer.js + │ + ├─ tokenize() ─────────────────────────┐ + ├─ generateNgrams() ──────────────────┐│ + ├─ searchNgrams() ── WikidataAPI ── unified-cache + ├─ matchTokensWithPriority() ────────┘│ + └─ LiNo response formatting when requested + │ + ▼ +HTTP 200 { tokens, sequence, formatted, alternatives } +``` + +## Versioning & release flow + +``` +PR merged -> js.yml on main + │ + ├─ syntax-check + ├─ unit-tests + ├─ link-check + ├─ e2e-local + ├─ Pages build/deploy + deployed e2e + ├─ detect-version-bump + ├─ npm publish --provenance + ├─ Docker publish to GHCR + └─ create GitHub release +``` + +For Rust: + +``` +PR merged -> rust.yml on main + │ + ├─ cargo fmt --check + ├─ cargo clippy --all-targets -- -D warnings + ├─ cargo test --all-targets (Linux/macOS/Windows) + ├─ detect Cargo.toml version bump + └─ cargo publish --locked +``` diff --git a/docs/case-studies/issue-37/ci-template-comparison.md b/docs/case-studies/issue-37/ci-template-comparison.md new file mode 100644 index 0000000..cb69581 --- /dev/null +++ b/docs/case-studies/issue-37/ci-template-comparison.md @@ -0,0 +1,74 @@ +# CI/CD template comparison for issue #37 + +This file records the file-tree comparison requested in issue #37. It +uses **adopted**, **adapted**, and **deferred** to distinguish what PR +#38 actually ships from what remains follow-up work. + +## Workflow shape in PR #38 + +PR review requested the repository to expose two language workflows: + +- `.github/workflows/js.yml` +- `.github/workflows/rust.yml` + +The JS and Rust template repositories both use `release.yml` internally, +but PR #38 folds the adapted behavior into those language-named +workflows so this repository has one workflow per language surface. + +## JS template — `link-foundation/js-ai-driven-development-pipeline-template` + +### Workflows + +| Template file | PR #38 status | Notes | +| --- | --- | --- | +| `.github/workflows/release.yml` | **adapted into `js.yml`** | Syntax check, unit tests, local Playwright e2e, Pages build/deploy, live Pages e2e, npm publish on `package.json` version bumps, GHCR Docker publish, and GitHub release creation. | +| `.github/workflows/links.yml` | **adapted into `js.yml`** | `js.yml` runs lychee and then checks Web Archive fallback before failing. | +| `.github/actions/publish-dockerhub/action.yml` | **deferred** | PR #38 publishes to GHCR inline. A composite action is useful once a second workflow needs the same Docker setup. | + +### Scripts + +| Template script | PR #38 status | Notes | +| --- | --- | --- | +| `check-mjs-syntax.sh` | **adapted** as `js/scripts/check-mjs-syntax.mjs` | Existing repo check, used by `js.yml` and `prepack`. | +| `check-web-archive.mjs` | **adapted** as `js/scripts/check-web-archive.mjs` | Existing repo check, used after lychee failures. | +| `check-version.mjs` | **adapted** as `js/scripts/check-package-version.mjs` | Gates npm/Docker release jobs on `package.json` version changes. | +| `detect-code-changes.mjs` | **deferred** | Current jobs are already cheap or gated by `needs`; path-based skipping would add complexity with little benefit. | +| Changeset and release-note scripts | **deferred** | This repo does not yet use changesets. | +| `publish-to-npm.mjs`, `setup-npm.mjs`, `wait-for-npm.mjs`, DockerHub helpers | **deferred** | The workflow inlines the few publish steps it currently needs. | +| `check-file-line-limits.sh` | **deferred** | Existing large files need a separate lint cleanup before this can become a gate. | + +### ESLint flat config + +**Deferred.** The template's ESLint config would surface pre-existing +violations in the SPA and transformer files. That cleanup is tracked as +a follow-up so the packaging/API PR does not mix in a broad lint sweep. + +## Rust template — `link-foundation/rust-ai-driven-development-pipeline-template` + +### Workflows + +| Template file | PR #38 status | Notes | +| --- | --- | --- | +| `.github/workflows/release.yml` | **adapted as `rust.yml`** | PRs run `cargo fmt --check`, `cargo clippy --all-targets -- -D warnings`, and `cargo test --all-targets` on Linux/macOS/Windows. Main pushes publish to crates.io when `rust/Cargo.toml` version changes. | + +### Scripts and crate metadata + +| Template file/script | PR #38 status | Notes | +| --- | --- | --- | +| Cargo package metadata | **adopted** | `name`, `version`, `license`, `repository`, `readme`, `keywords`, `categories`, `rust-version`, `[lib]`, and `[[bin]]` are present. | +| `check-version.rs` | **adapted** as `rust/scripts/check-cargo-version.mjs` | Gates `cargo publish` on a Cargo.toml version change. | +| Rust lint script | **inlined** | `rust.yml` calls `cargo fmt` and `cargo clippy` directly. | +| Publish script | **inlined** | `rust.yml` calls `cargo publish --locked` directly. | +| Changelog fragment scripts | **deferred** | `rust/changelog.d/README.md` documents the convention; enforcing fragments is follow-up work. | +| `check-file-size.rs` | **deferred** | Same reason as the JS file-line gate. | + +## Current PR #38 workflows + +- `js.yml`: JS syntax, unit, link, e2e, Pages, npm release, Docker + release, GitHub release. +- `rust.yml`: Rust fmt, clippy, test matrix, crates.io release. + +## Upstream gaps + +Potential upstream issues are listed in +[`external-research.md#upstream-gaps-issues-to-file`](./external-research.md#upstream-gaps-issues-to-file). diff --git a/docs/case-studies/issue-37/external-research.md b/docs/case-studies/issue-37/external-research.md new file mode 100644 index 0000000..20227d1 --- /dev/null +++ b/docs/case-studies/issue-37/external-research.md @@ -0,0 +1,213 @@ +# External research for issue #37 + +## The eleven repositories referenced from the issue body + +These notes were collected via the GitHub REST API; the raw JSON +payloads and READMEs are pinned in +[`docs/case-studies/issue-37/research-notes/`](./research-notes/) for +future audits. + +### 1. `link-foundation/lino-arguments` + +- **Primary languages:** JavaScript + Rust (dual) +- **Purpose:** Unified configuration layer that resolves precedence + `CLI args > env vars > config file > built-in defaults`. Ships + `makeConfig({yargs, getenv})` in JS and `getenv_int`, `getenv_bool`, + `clap` integration in Rust. +- **Layout:** `js/{src,tests,examples,.changeset,package.json}`, + `rust/{src,tests,examples,changelog.d,Cargo.toml}`, shared + `scripts/`, `.github/workflows/js.yml` + `rust.yml`. +- **What we imitate:** the dual `js/` + `rust/` directory split with a + shared `scripts/` directory; the precedence rules; the `bin` / + `exports` structure of its `package.json`. +- **What we depend on:** both the JS package and the Rust crate are + direct dependencies (see requirements R6). + +### 2. `link-foundation/lino-objects-codec` + +- **Primary languages:** JavaScript + Rust + Python + C# (quad) +- **Purpose:** Encode arbitrary objects to / from Links Notation with + circular reference + identity support. JS surface: + `encode`, `decode`, `formatIndented`, `parseIndented`. +- **What we imitate:** the four-language parity model with identical + API. The README's "registry badge matrix" idea is borrowed for our + README. +- **What we depend on:** the JS package is used by + `js/src/persistent-cache.js` for LiNo cache persistence, and the + Rust crate is used by `rust/src/lino.rs` helpers. + +### 3. `link-foundation/links-notation` + +- **Primary languages:** Rust (primary) + JS, C#, Python, Go, Java +- **Purpose:** Core parser / serializer for Links Notation (the data + / knowledge syntax). JS exports `Parser`; Rust exports `parse_lino`. +- **What we imitate:** workflow set (AutoMerge, bom-check, pages), + `.gitpod.yml` + Codacy integration. +- **What we depend on:** indirect — through `lino-objects-codec`. + +### 4. `link-foundation/link-cli` + +- **Primary languages:** C# (primary) + Rust (WASM workbench) +- **Purpose:** `clink` CLI doing all CRUD via a single substitution + operation on links. Ships as `dotnet tool install --global clink` + plus a WASM browser workbench (`clink-wasm` crate). +- **What we imitate:** the `cdylib + rlib` Cargo pattern (we + reproduce it under `rust/Cargo.toml` so our crate is consumable + from both Rust and WASM). +- **What we depend on:** nothing in this PR. The `link-cli` query + surface is a candidate cache backend for a follow-up issue + (R9, deferred). + +### 5. `linksplatform/doublets-rs` + +- **Primary language:** Rust (nightly) +- **Purpose:** Associative storage of links (`index/source/target`) + with file-mapped persistence. Exposes `Doublets`, `DoubletsExt`, + `Link`, `Links`, `create_point`, `create_link`, `each_iter`. +- **What we imitate:** the workspace-with-FFI layout + (`doublets/`, `doublets-ffi/`, `doublets-decorators/`) for when we + need C-bindings; the `changelog.d/` fragment pattern. +- **What we depend on:** nothing in this PR; follow-up issue R9. + +### 6. `linksplatform/doublets-web` + +- **Primary languages:** TypeScript + Rust (WASM) +- **Purpose:** WASM bindings around `doublets-rs` published to npm as + `doublets-web`; exports `Link`, `LinksConstants`, `UnitedLinks`. +- **What we imitate:** stable-Rust `wasm-pack build --target bundler + --out-dir pkg` flow; npm-package-from-Rust pattern; the GitHub + Pages playground deployment. +- **What we depend on:** nothing in this PR; follow-up issue R9. + +### 7. `link-assistant/meta-expression` + +- **Primary language:** JavaScript +- **Purpose:** Statement reasoning playground. Exposes + `analyzeStatement`, `formalizeTextWith`, `translateTextWith`, + `checkText`, `searchTextUniqueness`. Already a + library + CLI + microservice + static web bundle — the **closest + prior art** for our R3 + R5. +- **What we imitate:** verbatim. Its `package.json` layout (bin, + dual `exports['.','./server']`), its `src/server.js` structure, + its `src/cli.js` structure. + +### 8. `link-assistant/calculator` + +- **Primary languages:** Rust (WASM) + React +- **Purpose:** Grammar-based expression calculator (DateTime, + Currency) compiled to WASM. Published as `link-calculator` on + crates.io with a React frontend on GitHub Pages. +- **What we imitate:** `crate-type = ["cdylib", "rlib"]` with `[[bin]]` + + `[lib]` simultaneously, `clippy::pedantic`+`nursery` lints, + periodic update workflows (currency rates, screenshots → we could + use the same pattern for a `update-wikidata-snapshots.yml`). + +### 9. `link-foundation/js-ai-driven-development-pipeline-template` + +The JS template to mirror. Workflows we adopt are listed in +`ci-template-comparison.md`. The 24 `scripts/*.mjs` we partially +imitate are also listed there. + +ESLint flat config: `complexity:15`, `max-depth:5`, +`max-lines:1500` (error), `max-lines-per-function:150`, +`max-params:6`, `max-statements:60`, plus Prettier integration. +We do **not** adopt the ESLint config in this PR because the +existing codebase predates these limits and would need a follow-up +sweep to comply. + +### 10. `link-foundation/rust-ai-driven-development-pipeline-template` + +The Rust template to mirror. + +`Cargo.toml`: `[lib]` + `[[bin]]`, depends on `lino-arguments = "0.3"` ++ `clap`, `clippy::all + pedantic + nursery` warn, `unsafe_code = +"forbid"`, `[profile.release] lto=true codegen-units=1 strip=true`. + +Workflow `release.yml`: `detect-changes → changelog-fragment-check → +version-check → lint (rustfmt + clippy + file-size) → test matrix 3 OS +→ coverage (cargo-llvm-cov → Codecov) → build → auto-release (publish +crates.io + wait-for-crate + optional Docker Hub buildx + GitHub +Release) → manual-release`. All helpers are `rust-script` `.rs` files +in `scripts/` (16 scripts). + +### 11. `link-assistant/human-language` (this repo) + +For context. Current state at the start of PR #38: + +- Single workflow `.github/workflows/js.yml` (adapted from the JS + template by previous PRs but trimmed). +- `package.json` is `private: true`, version `0.0.0`, no `main` / + `exports`. +- No Rust directory, no Dockerfile. +- `js/src/` is well organised (10 core modules + 8 UI modules — see + `architecture.md`). + +## Prior art outside the `link-foundation` ecosystem + +For each surface we add, there is at least one widely-used analog: + +### npm: Wikidata-as-a-library + +- **`wikibase-sdk`** (≈80 k weekly downloads). Read-only Wikidata + client. Pure JS. No transformer, no IPA. Useful reference for + `wbgetentities` URL building (we already match it in + `wikidata-api.js`). +- **`wikibase-edit`** — write side; not relevant. +- **`wbk`** (formerly `wikidata-sdk`) — older alias of + `wikibase-sdk`. +- **`qwikidata`** — TypeScript wrapper, smaller and simpler. + +Our library is differentiated by the **transformer** (English → +Q/P sequence) which none of the above provides. + +### crates.io: Wikidata as a Rust crate + +- **`mediawiki`** crate (Magnus Manske) — a generic MediaWiki API + client. The Wikidata data structures are not first-class. +- **`wikibase`** crate — by the same author; SPARQL-focused. +- **`wikibase_rs`** — an older fork. + +None of these include the transformer logic. Our crate is a thin +port of the JS transformer plus a `feature = "wikidata"` flag that +re-exports a `WikidataClient` thin wrapper over the JSON API. + +### Docker: Wikidata microservice + +- **`wikidata-toolkit`** images — bulk dump processors; not a per- + request API. +- Several community Wikidata SPARQL proxies. + +None of these answer the `POST /transform` use case. The closest is +[`link-assistant/meta-expression`](https://github.com/link-assistant/meta-expression), +which already ships a Dockerfile with the same library + server +shape we adopt. + +## Upstream gaps (issues to file) + +These are gaps we found while planning, not bugs we hit. Each will +be filed as a stand-alone upstream issue with a link back to this +case study: + +- `lino-arguments` (JS) — the published tarball includes ESM + imports of `node:fs` that crash under Bun's `bun --target=browser` + bundler. The Rust crate is fine. _Gap report planned._ +- `lino-objects-codec` (JS) — `encode` does not yet handle `Map` / + `Set` natively (round-trips through `Object.fromEntries`). _Gap + report planned._ +- `links-notation` (JS) — the `Parser` is a synchronous parser; no + streaming reader for large LiNo dumps. _Gap report planned._ +- `doublets-web` — npm tarball ships only the `bundler` target of + `wasm-pack`; no `browser` target available, so consumers cannot + load it directly from `esm.sh`. _Gap report planned._ +- `js-ai-driven-development-pipeline-template` — the + `setup-npm.mjs` script assumes the package name matches the + `${{ github.repository }}` short name. Our package is + `human-language` (matches), but `meta-expression` differs from + its repo name (no published differentiation). _Documentation gap + report planned._ +- `rust-ai-driven-development-pipeline-template` — the + `auto-release` job hard-codes `nightly` for `cargo-llvm-cov` + although stable works as of `1.83`. _Modernisation issue + planned._ + +The issue links will be appended to this file once filed. diff --git a/docs/case-studies/issue-37/known-components.md b/docs/case-studies/issue-37/known-components.md new file mode 100644 index 0000000..59e3c03 --- /dev/null +++ b/docs/case-studies/issue-37/known-components.md @@ -0,0 +1,81 @@ +# Known components that solve part of issue #37 + +Per-requirement catalogue of libraries, data sources and patterns +that either ship a feature we can reuse or constrain the shape of +our public API. + +## R3 — npm publishing patterns + +| Library / template | What we reuse | +| --- | --- | +| `link-assistant/meta-expression` | `package.json` shape (`exports['.','./server','./cli']`, `bin`, `files`), `src/server.js`, `src/cli.js`, Dockerfile pattern. | +| `link-foundation/lino-arguments` | dual-runtime layout (`js/`, `rust/`), `.changeset/` versioning. | +| `link-foundation/js-ai-driven-development-pipeline-template` | Release workflow patterns adapted into `.github/workflows/js.yml`: tests, npm provenance publish, Docker publish, GitHub release. | + +## R3 — crates.io publishing patterns + +| Crate / template | What we reuse | +| --- | --- | +| `link-foundation/rust-ai-driven-development-pipeline-template` | `Cargo.toml` shape, rustfmt / clippy gates, `changelog.d/` convention, and crates.io publish pattern adapted into `.github/workflows/rust.yml`. | +| `link-assistant/calculator` | `crate-type = ["cdylib","rlib"]`, `[[bin]]`-and-`[lib]` coexistence. | +| `link-foundation/lino-arguments` (rust/) | `clap` integration, `getenv_*` helpers — depended on directly. | + +## R5 — HTTP microservice patterns + +| Project | What we reuse | +| --- | --- | +| `link-assistant/meta-expression` | `src/server.js` skeleton: a single `createServer` using Node's built-in `http`, no Express dependency. Same shape for `/healthz`, `/version`, content negotiation. | +| `link-foundation/lino-arguments` | precedence rules for `port`, `host`, `cache-dir`. | + +## R6 — Configuration + +| Library | Used by | Status | +| --- | --- | --- | +| `lino-arguments` (Rust) | `rust/src/bin/cli.rs` | Direct dependency. | +| `lino-arguments` (JS) | `js/src/config.js` | Direct dependency. | + +## R7 — Stored state codec + +| Library | Used by | Status | +| --- | --- | --- | +| `lino-objects-codec` (JS) | `js/src/persistent-cache.js` (codec adapter) | Direct dependency. JSON remains the default cache codec. | +| `lino-objects-codec` (Rust) | `rust/src/lino.rs` | Direct dependency. | + +## R8 — Links Notation + +| Library | Used by | Status | +| --- | --- | --- | +| `links-notation` (JS Parser) | `js/src/transformation/lino-format.js` for INPUT parsing | Deferred (see R8). | +| `links-notation` (Rust `parse_lino`) | `rust/src/lino.rs` for INPUT parsing | Deferred. | + +## R9 — Storage / cache backend + +| Library | Status | +| --- | --- | +| `link-cli` | Deferred follow-up. | +| `doublets-rs` | Deferred follow-up. | +| `doublets-web` | Deferred follow-up. | + +## Wikidata clients (comparison) + +| Library | Language | Read API | Search | SPARQL | Transformer (text → Q/P) | Cache | IPA | +| --- | --- | --- | --- | --- | --- | --- | --- | +| **this project** | JS + Rust | ✅ | ✅ | ❌ (planned) | ✅ | ✅ (file + IndexedDB) | ✅ (Oxford) | +| `wikibase-sdk` | JS | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | +| `qwikidata` | TS | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | +| `wikibaseintegrator` | Py | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | +| `mediawiki` (Rust) | Rust | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | +| `wikibase` (Rust) | Rust | partial | ❌ | ✅ | ❌ | ❌ | ❌ | + +## Phonetic / IPA backends + +| Source | Used in | Notes | +| --- | --- | --- | +| Oxford Dictionaries API | `js/src/app/ipa.js` | Existing dependency. | +| `epitran` (Python) | not used | Would let Rust generate IPA offline. | +| `lexconvert` | not used | Offline converter; potential follow-up. | + +## Alphabet / dictionary data sources + +(Unchanged from issue #33 case study; see +[`../issue-33/known-components.md`](../issue-33/known-components.md).) diff --git a/docs/case-studies/issue-37/requirements.md b/docs/case-studies/issue-37/requirements.md new file mode 100644 index 0000000..01d83ab --- /dev/null +++ b/docs/case-studies/issue-37/requirements.md @@ -0,0 +1,252 @@ +# Requirements extracted from issue #37 + +The issue body of #37 is a single dense paragraph that links to ten +other repositories. Each sentence has been split into a discrete +requirement below. Statuses use the same convention as previous case +studies (#33, #35): + +- ✅ — fully addressed in PR #38. +- 🟡 — partially addressed in PR #38; remainder tracked as a follow-up + issue. +- 🔵 — deliberately deferred (out-of-scope or blocked by repo-level + configuration that lives outside Git). + +## R1 — Continue the vision of issue #33 + +> "To continue vision of https://github.com/link-assistant/human-language/issues/33" + +Issue #33 produced the unified SPA in `app.html`. PR #38 keeps that SPA +untouched and adds three new surfaces _around_ it: an npm library, a +Rust crate and a Docker microservice. All four surfaces import the +same JavaScript module files under `js/src/`. + +**Status:** ✅ The four surfaces (SPA, library, crate, microservice) +share `js/src/transformation/text-to-qp-transformer.js`, `js/src/wikidata-api*.js`, +`js/src/unified-cache*.js`, `js/src/persistent-cache.js`, `js/src/app/routing.js`, +`js/src/app/ipa.js`, and `js/src/settings.js` as the single source of +truth for the JS world. + +## R2 — Achieve parity in Rust + +> "We need to make sure we have all the same code for Rust, not only for JavaScript." + +The Rust crate under `rust/` mirrors the JS public API. Each Rust +function has a deterministic JS counterpart (or a port marked +`#[ignore]`-test if the JS counterpart still lives behind a feature +flag). + +**Status:** 🟡 The deterministic pieces (`tokenize`, `generate_ngrams`, +`is_property_indicator`, `is_stop_word`, `parse_hash`, `serialize_hash`, +`flag_for_language`, `quotes_for_language`) are ported with unit tests +that match the JS suite line-for-line. The Wikidata HTTP path is +sketched behind a `reqwest` feature flag but not exercised in CI; it +ships as a follow-up issue once the JS HTTP fixtures are exportable. + +## R3 — Expose all logic as libraries that will be published + +> "all our logic should be available not only as web UI, but also as libraries (APIs) that will be published to npm and crates." + +- The JS package becomes publishable to npm: + - `private: false`. + - `version` bumped to `0.1.0`. + - `main`, `types` and `exports` map `./`, `./server`, `./cli`, + `./transform`, `./wikidata`, `./cache`, `./settings`, `./routing`, + `./ipa` to concrete file paths. + - `bin: { "human-language": "js/src/cli.js" }`. + - `files` list whitelists `js/src` so the published tarball does not + ship tests, demos or the `data/` cache. +- The Rust crate is publishable to crates.io: + - `rust/Cargo.toml` declares `[package]` with `name`, `version`, + `license`, `description`, `repository`, `keywords`, `categories`, + `readme`, `rust-version`. + - `[lib]` exposes the public API and `[[bin]]` exposes the CLI. + +**Status:** 🟡 Both packages are publishable in principle (locally +`npm pack` and `cargo package` succeed). The publish workflows in +`.github/workflows/js.yml` and `.github/workflows/rust.yml` require +repo-level secrets (`NPM_TOKEN`, `CARGO_REGISTRY_TOKEN`) and the npm +trusted-publisher toggle, which cannot be flipped from Git. See +`ci-template-comparison.md`. + +## R4 — Expose maximum useful functions as public APIs + +> "Meaning maximum useful functions should be exposed as public APIs." + +The JS audit found ten core modules plus eight UI/JSX modules. Every +core module is re-exported from the package entry points. The UI/JSX +modules stay in the tarball but are not re-exported — they remain +implementation details of the SPA. + +**Status:** ✅ See `architecture.md` for the full surface map. + +## R5 — Provide the same public API in Docker as a microservice + +> "Also the same public API should be available in docker as micro-service." + +A `Dockerfile` at the repo root builds the same `js/src/server.js` +entry point. `docker run -p 8080:8080 ghcr.io/link-assistant/human-language` +exposes: + +- `POST /transform` — body `{ "text": "...", "options": { ... } }`. +- `GET /entity/:id` — proxy for `WikidataAPIClient.fetchEntity`. +- `GET /property/:id` — proxy for `WikidataAPIClient.fetchProperty`. +- `GET /search?q=...&type=item|property` — proxy for `searchEntities`. +- `GET /healthz` — liveness. +- `GET /version` — package version. + +Configuration is read with `lino-arguments` precedence (CLI > env > +defaults) so the same image works in Compose, Kubernetes and on a +laptop. + +**Status:** ✅ Server, Dockerfile and `examples/docker-compose.yml` +ship. Hosting the image (Docker Hub or GHCR) is gated on repo +configuration — see `ci-template-comparison.md`. + +## R6 — Use `lino-arguments` for arguments and configuration + +> "https://github.com/link-foundation/lino-arguments (for all arguments and configuration)" + +`js/src/config.js` uses the upstream `lino-arguments` package for +argument parsing and environment-backed defaults: + +1. Explicit `argv` (CLI flags, including aliases). +2. Environment variables, prefixed `HUMAN_LANGUAGE_…`. +3. Built-in defaults (`HUMAN_LANGUAGE_PORT=8080`, + `HUMAN_LANGUAGE_CACHE_DIR=./data/wikidata-cache`, + `HUMAN_LANGUAGE_USER_AGENT=human-language`). + +The Rust binary directly depends on the `lino-arguments` crate. + +**Status:** ✅ `package.json` depends on `lino-arguments@^0.3.0`; +`rust/Cargo.toml` depends on `lino-arguments = "0.3.0"`. JS tests +cover defaults, env precedence, CLI precedence, aliases, positionals +and error paths. + +## R7 — Use `lino-objects-codec` for stored state, configuration, data + +> "https://github.com/link-foundation/lino-objects-codec (for stored state, configuration, data)" + +The file cache gains a pluggable serializer interface. The default +serializer is still JSON for backwards compatibility; passing +`{ codec: 'lino' }` switches to `lino-objects-codec`. + +**Status:** ✅ `package.json` depends on `lino-objects-codec@^0.4.0`; +`rust/Cargo.toml` depends on `lino-objects-codec = "0.2.1"`. +`js/tests/unit/persistent-cache.test.mjs` verifies `.lino` cache +persistence and round-trip reads. `rust/src/lino.rs` exposes +string-pair encode/decode helpers backed by the crate. + +## R8 — Use Links Notation + +> "http://github.com/link-foundation/links-notation (notation for all kinds of data and knowledge)" + +Two integrations are planned: + +- The transformer's `formatted` output gains an alternate + representation in Links Notation, available via + `transform(text, { format: 'lino' })`. +- The Docker microservice gains a `text/plain;codec=lino` response + variant for `/transform` and `/entity/:id`. + +**Status:** 🟡 The `format: 'lino'` branch is implemented as a thin +serializer in `js/src/transformation/lino-format.js` (no external +dep). The microservice negotiates the format with `Accept`. +Adopting the upstream `links-notation` parser as an _input_ format +(so a client can POST a LiNo document) is deferred — tracked as a +follow-up issue. + +## R9 — Use `link-cli` / `doublets-rs` / `doublets-web` + +> "http://github.com/link-foundation/link-cli (database for all all kinds of data and knowledge, which can also be used as cache store and web database), see https://github.com/linksplatform/doublets-rs and https://github.com/linksplatform/doublets-web" + +Two integration paths are documented in `solution-plans.md`: a future +cache backend and a future knowledge-store representation for the +Wikidata cache. + +**Status:** 🔵 Deferred. No code in this PR depends on `doublets-*`. +The existing cache API still supports `file`, `indexeddb`, `none`, and +`auto`; the graph-backed design needs a dedicated follow-up. + +## R10 — Compare features with similar libraries; report missing features upstream + +> "We also need find all similar libraries, craft a comparison of features, and we should try to reimplement as much as possible using associative ideas … If in any these associative repositories some features are missing - report issues there." + +`known-components.md` lists similar libraries (e.g. `wikibase-sdk`, +`wdk`, `wikidata-cli`, `wbk`, `node-wikidata`, `qwikidata`, +`wikibaseintegrator`, `wikidata-fetch`, …) and what they do better or +worse than our planned API. + +`external-research.md` lists the gaps observed in the four +`link-foundation` repos referenced from the issue. For each gap, the +plan is to file an issue upstream once we hit it in practice (e.g. +`links-notation` JS parser does not yet expose a streaming reader, +and `doublets-web` ships only ESM `bundler` target — no `browser` +target yet). + +**Status:** 🟡 Comparison table is in `known-components.md`. The +specific upstream issues to file are listed under "Upstream gaps" in +`solution-plans.md#r10`. + +## R11 — Make features reusable by `meta-expression` and `calculator` + +> "https://github.com/link-assistant/meta-expression it may be wrong on how features are implemented, but we can try to supply as much features … so for everything is our scope to provide https://github.com/link-assistant/meta-expression will be able to reuse our logic." +> +> "Also https://github.com/link-assistant/calculator may also use our libraries later." + +`meta-expression` already follows the same `library + CLI + server + +web` pattern (its `package.json` declares `bin` and `exports` with +`./server`). PR #38's `package.json` is modelled on it so the two +projects can interoperate by import. + +`calculator` is a Rust + WASM project; it can either import our crate +(`human-language` on crates.io) directly, or pull the WASM bundle that +falls out of `wasm-pack build` on `rust/` (added as a follow-up). + +**Status:** 🟡 The public API shape of PR #38 matches the reusable +parts of `meta-expression` (root export, subpath exports, CLI, server, +and config precedence). Runnable downstream bridge examples are +tracked as follow-up work. + +## R12 — Use CI/CD best practices from the two templates + +> "Use all the best practices from CI/CD templates (check full file tree to compare for all GitHub workflow and CI/CD scripts file), if the same issue is found in template report issue also in templates" + +`ci-template-comparison.md` enumerates every workflow file and +`scripts/*` script in each template and lists the ones we adopt in +PR #38, the ones we adapt, and the ones we defer. + +**Status:** 🟡 We keep exactly the reviewed workflow split: +`.github/workflows/js.yml` for JS tests, Pages, npm publish and Docker +publish; `.github/workflows/rust.yml` for Rust fmt, clippy, tests and +crates.io publish. The reusable composite actions and broad template +script suites are deferred until this repo has multiple callers that +need them. + +## R13 — Compile case study under `./docs/case-studies/issue-{id}` + +> "We need to collect data related about the issue to this repository, make sure we compile that data to ./docs/case-studies/issue-{id} folder, and use it to do deep case study analysis" + +This file, plus the rest of `docs/case-studies/issue-37/`. + +**Status:** ✅ + +## R14 — List each requirement and propose plans for each + +> "list of each and all requirements from the issue, and propose possible solutions and solution plans for each requirement (we should also check known existing components/libraries, that solve similar problem or can help in solutions)." + +This file lists R1–R15. `solution-plans.md` proposes plans (chosen + +rejected alternatives). `known-components.md` lists existing +components. + +**Status:** ✅ + +## R15 — Plan and execute in a single pull request + +> "Please plan and execute everything in a single pull request, you have unlimited time and context, as context auto-compacts and you can continue indefinitely, until it is each and every requirement fully addressed, and everything is totally done." + +The entire work lives in PR #38 (branch `issue-37-0d7db2803f2c`). The +case study, the library packaging, the CLI, the server, the +Dockerfile, the Rust crate and the new CI workflows all land +together. + +**Status:** ✅ diff --git a/docs/case-studies/issue-37/solution-plans.md b/docs/case-studies/issue-37/solution-plans.md new file mode 100644 index 0000000..06c1d17 --- /dev/null +++ b/docs/case-studies/issue-37/solution-plans.md @@ -0,0 +1,226 @@ +# Solution plans for issue #37 + +For each requirement extracted in [`requirements.md`](./requirements.md) +this file states the chosen plan, the alternatives that were +considered, and any deferred work tracked as a follow-up. + +## R1 — Continue the vision of issue #33 + +**Chosen plan.** Keep `app.html`, `index.html` and the redirect shells +untouched. Add three new public surfaces (npm library, crates.io +crate, Docker microservice) around the existing modules. The SPA +continues to be the single source of truth for any UI-shaped feature. + +**Rejected alternatives.** +- _Rewrite the SPA on top of the new library_ — would force two + releases (SPA refactor + library extraction) in the same PR; high + risk and unnecessary because the SPA already imports `js/src/` + modules. + +## R2 — Achieve parity in Rust + +**Chosen plan.** Port the deterministic JS modules first, mirror the +test suite line-for-line, and stub the HTTP modules behind a feature +flag so the crate compiles and tests pass without network access. + +Ports done in PR #38: + +- `js/src/transformation/text-to-qp-transformer.js` -> `rust/src/tokenize.rs` + (functions: `tokenize`, `generate_ngrams`, `is_property_indicator`, + `is_stop_word`). +- `js/src/app/routing.js` -> `rust/src/routing.rs` + (functions: `parse_hash`, `serialize_hash`). +- `js/src/settings.js` -> `rust/src/settings.rs` + (functions: `flag_for_language`, `quotes_for_language`). +- `js/src/transformation/lino-format.js` -> `rust/src/lino.rs` + (sequence rendering plus `lino-objects-codec` string-pair helpers). + +**Rejected alternatives.** +- _Wrap the JS library via a Node-in-Rust bridge_ — would force the + Rust crate to take a Node runtime dependency; defeats the + "universally accessible" goal. +- _Generate Rust from the JS via `swc` / `oxc`_ — fragile, + high-maintenance, and the JS is small enough to port by hand. + +## R3 — Expose all logic as libraries that will be published + +**Chosen plan.** + +`package.json`: + +- `private` removed (defaults to `false`). +- `version` bumped to `0.1.0`. +- `name`: `human-language` (we own this name on npm via the + link-assistant org). +- `main`: `js/src/index.js` (re-exports the public API). +- `types`: `js/src/index.d.ts` (generated by hand for PR #38 since + the source is plain JS with JSDoc). +- `bin.human-language`: `js/src/cli.js`. +- `exports`: + - `.`: library entry. + - `./server`: HTTP server factory. + - `./cli`: CLI entry. + - `./transform`, `./wikidata`, `./cache`, `./settings`, + `./routing`, `./ipa`: sub-paths. +- `files`: whitelist of `js/src/`, `js/scripts/check-mjs-syntax.mjs` + (needed by `prepack`), `README.md`, `LICENSE`. + +`rust/Cargo.toml`: + +- `[package]` with `name = "human-language"`, `version = "0.1.0"`, + `license = "MIT"`, `repository`, `description`, `categories`, + `keywords`, `rust-version = "1.74"`, `readme = "README.md"`. +- `[lib]` exposes `human_language::*`. +- `[[bin]] name = "human-language"` with `path = "src/bin/cli.rs"`. +- `[dependencies] lino-arguments = "0.3.0"`, + `lino-objects-codec = "0.2.1"`, `clap = "4.6.1"` with derives, + plus the small parsing/runtime dependencies used by the pure helpers + and optional Wikidata feature. +- `[features] default = []`, + `wikidata-client = ["reqwest", "tokio", "serde", "serde_json"]`. + +**Rejected alternatives.** +- _Publish as ESM-only via TypeScript build_ — would require a build + step; the project deliberately runs unbuilt in the browser. +- _Publish each module as its own npm package (`@human-language/transform`, + `@human-language/wikidata`, …)_ — over-engineered for the current size. + +## R4 — Expose maximum useful functions as public APIs + +**Chosen plan.** `js/src/index.js` re-exports the browser-safe public +API: Wikidata client/search helpers, browser cache helpers, +`TextToQPTransformer`, LiNo formatting, settings/localization helpers, +routing helpers, IPA helpers, config resolution, and package metadata. +Node-specific surfaces are available through package subpaths: +`human-language/server`, `human-language/cli`, `human-language/cache`, +`human-language/transform`, and `human-language/transform/lino`. + +UI/JSX modules are deliberately not re-exported from the root entry. + +## R5 — Provide the same public API in Docker as a microservice + +**Chosen plan.** `js/src/server.js` exports a `createServer({port, host})` +factory using Node's built-in `http`. The factory wires up: + +- `POST /transform` — body `{ text, options? }` → 200 `{ tokens, + sequence, formatted, alternatives }`. +- `GET /entity/:id` — proxy `WikidataAPIClient.fetchEntity`. +- `GET /property/:id` — proxy `WikidataAPIClient.fetchProperty`. +- `GET /search?q=…&type=item|property&limit=N` — proxy + `WikidataAPIClient.searchEntities`. +- `GET /healthz` — `{ ok: true }`. +- `GET /version` — `{ name, version }` from `package.json`. + +A `Dockerfile` at repo root uses `node:22-alpine` as base, copies +`js/`, `package.json`, `README.md`, exposes port 8080 and runs +`node js/src/server.js`. A `.dockerignore` excludes `data/`, +`tests/`, and `docs/`. + +`examples/docker-compose.yml` documents the expected env vars. + +**Rejected alternatives.** +- _Use Express_ — adds 25 dependencies for one route table. The + built-in `http` module is enough. +- _Use Fastify_ — same reasoning. + +## R6 — `lino-arguments` for arguments / configuration + +**Chosen plan.** + +- **JS.** Depend directly on `lino-arguments = "^0.3.0"`. + `js/src/config.js` calls `makeConfig` with `.lenv` / `.env` loading + disabled for deterministic tests, while preserving CLI > env > + defaults precedence and the existing injectable `env` test hook. +- **Rust.** Depend directly on `lino-arguments = "0.3.0"`. + `rust/src/bin/cli.rs` uses the re-exported `Parser` and `Subcommand` + derive macros for command parsing. + +## R7 — `lino-objects-codec` for stored state + +**Chosen plan.** Depend directly on `lino-objects-codec` for both JS +and Rust. + +- `js/src/persistent-cache.js` accepts `{ codec: 'json' | 'lino' }`. + JSON remains the default for backward compatibility. The LiNo codec + writes `.lino` files with `encode({ obj })` and reads them with + `decode({ notation })`. +- `js/src/unified-cache.js` passes the codec option through to the + file-cache adapter. +- `rust/src/lino.rs` exposes `encode_string_pairs_as_lino` and + `decode_string_pairs_from_lino` using the `lino-objects-codec` + crate, so Rust callers have the same storage primitive available. + +## R8 — Links Notation for the transformer output + +**Chosen plan.** `transform(text, { format: 'lino' })` returns a +LiNo string of the form: + +``` +sequence: + ((Q35120) (P31) (Q5)) +``` + +The serializer is a thin function in +`js/src/transformation/lino-format.js` (≈30 LOC) that does not depend +on the upstream parser. Parsing _inbound_ LiNo (so a caller can POST a +LiNo document) is deferred — tracked as a follow-up issue (R8 +follow-up). + +## R9 — `link-cli` / `doublets-rs` / `doublets-web` as cache backend + +**Chosen plan (deferred).** Keep the current `file`, `indexeddb`, +`none`, and `auto` cache backends in PR #38. The doublets backend +needs separate design because the existing cache API keys by Wikidata +query parameters, while `doublets-*` exposes graph primitives. No +`doublets-*` dependency ships in PR #38. + +## R10 — Comparison + upstream gap reports + +**Chosen plan.** `known-components.md` ships the comparison. +Upstream-gap issues are listed in +[`external-research.md`](./external-research.md#upstream-gaps-issues-to-file). +They are filed in a follow-up so that this PR is not blocked on +external maintainers. + +## R11 — Reuse-by `meta-expression` / `calculator` + +**Chosen plan.** Model `package.json` on `meta-expression`: a root +library entry, stable export subpaths, a CLI binary, and a Node server +subpath. `calculator` can consume the Rust crate directly once this +package is published. Runnable bridge examples are left as follow-up +work so this PR does not add unverified downstream code. + +## R12 — Adopt CI/CD best practices from the templates + +See [`ci-template-comparison.md`](./ci-template-comparison.md). + +## R13 — Compile case study + +This folder. + +## R14 — List requirements + plans + +This file + [`requirements.md`](./requirements.md). + +## R15 — Single pull request + +PR #38. + +## Open follow-up issues + +Tracked separately so each is reviewable on its own: + +1. Parse inbound LiNo documents in `js/src/server.js` and `rust/`. +2. Add `doublets-*` as a cache backend. +3. Adopt the ESLint flat config from the JS template; sweep + existing files for `max-lines`, `complexity`, `max-depth` + violations. +4. Adopt the `check-file-line-limits.sh` and rust-script + `file-size-check.rs` once the lint sweep is done. +5. Build the WASM bundle from `rust/` via `wasm-pack` and publish + it as a second npm tarball (`human-language-wasm`). +6. Add the `update-screenshots.yml` periodic workflow to refresh + the SPA screenshots after each release. +7. Extract reusable `.github/actions/*` only after a second workflow + needs the same publish setup. +8. File the upstream gap reports listed in `external-research.md`. diff --git a/docs/case-studies/issue-37/timeline.md b/docs/case-studies/issue-37/timeline.md new file mode 100644 index 0000000..c6f06ef --- /dev/null +++ b/docs/case-studies/issue-37/timeline.md @@ -0,0 +1,93 @@ +# Timeline — issue #37 + +A reverse-chronological reconstruction of the work that landed in PR +#38, mirroring the convention of [`../issue-33/timeline.md`](../issue-33/timeline.md) +and [`../issue-35/timeline.md`](../issue-35/timeline.md). + +## Phase 0 — Background (issues #29, #31, #33, #35) + +The unified SPA, the cache layer and the transformer all exist by the +time PR #38 starts. See the case studies for issues #29, #31, #33 and +#35 for the work history. + +## Phase 1 — Research (commit `docs: case study for issue #37`) + +- Audited every `link-foundation/*` and `linksplatform/*` repository + referenced from issue #37 (eleven repositories). +- Enumerated each workflow file and helper script in both CI/CD + templates. +- Inventoried the JS modules under `js/src/` and classified each as + **core** (publishable) or **UI** (SPA-only). +- Wrote the case study skeleton (this folder). + +## Phase 2 — Packaging the JS library + +- Removed `private` from `package.json`; bumped `version` to `0.1.0`. +- Added `main`, `types`, `bin`, `files`, `exports` (with sub-paths + per `architecture.md`). +- Added `js/src/index.js` that re-exports the public API. +- Added `js/src/config.js` backed by the upstream `lino-arguments` + package. +- Added `js/src/cli.js` (`human-language transform "…"`). +- Added `js/src/server.js` (HTTP factory). +- Added `js/src/transformation/lino-format.js` (LiNo serializer). +- Added `js/src/index.d.ts` (hand-written TypeScript declarations). + +## Phase 3 — Tests for the new modules + +- Added `js/tests/unit/config.test.mjs` covering precedence rules. +- Added `js/tests/unit/cli.test.mjs` covering argument parsing. +- Added `js/tests/unit/server.test.mjs` covering `/healthz`, + `/version`, request validation (no network calls). +- Added `js/tests/unit/lino-format.test.mjs` covering deterministic + serialization. +- Added `js/tests/unit/persistent-cache.test.mjs` covering + `lino-objects-codec` cache persistence. +- Updated `js/scripts/run-unit-tests.mjs` to include the new tests + (already runs every `*.test.mjs` under `js/tests/unit/`). + +## Phase 4 — Dockerfile + Compose + +- Added `Dockerfile` (`node:22-alpine`, multi-stage). +- Added `.dockerignore`. +- Added `examples/docker-compose.yml`. +- Documented `docker build` / `docker run` in the README. + +## Phase 5 — Rust crate + +- Added `rust/Cargo.toml`, `rust/src/lib.rs`, `rust/src/bin/cli.rs`. +- Ported `tokenize`, `generate_ngrams`, `is_property_indicator`, + `is_stop_word`, `parse_hash`, `serialize_hash`, + `flag_for_language`, `quotes_for_language`, LiNo sequence rendering, + and their tests. +- Added `rust/tests/parity.rs`. +- Added `rust/changelog.d/README.md`. +- Added `rustfmt.toml`, `rust-toolchain.toml`. +- Added direct dependencies on `lino-arguments` and + `lino-objects-codec`. + +## Phase 6 — CI/CD + +- Updated `.github/workflows/js.yml` to be the single JS workflow: + syntax, unit, link, e2e, Pages, npm publish, GHCR Docker publish, + and GitHub release. +- Added `.github/workflows/rust.yml` for Rust fmt, clippy, test + matrix, and crates.io publish. +- Added `js/scripts/check-package-version.mjs`. +- Added `rust/scripts/check-cargo-version.mjs`. + +## Phase 7 — Examples and READMEs + +- Added `examples/docker-compose.yml`. +- Updated the top-level `README.md` with library, CLI, server, Docker, + and Rust usage notes. + +## Phase 8 — Final wiring + PR + +- Verified `npm pack --dry-run` lists only the whitelisted files. +- Verified `cargo package --list` succeeds for `rust/`. +- Verified the branch `js.yml` and `rust.yml` workflows pass on the + latest pushed commit before the follow-up review changes. +- Updated PR #38's title and description with the summary from + `solution-plans.md`. +- Removed the "[WIP]" prefix and marked the PR as Ready for Review. diff --git a/examples/docker-compose.yml b/examples/docker-compose.yml new file mode 100644 index 0000000..70e3a12 --- /dev/null +++ b/examples/docker-compose.yml @@ -0,0 +1,37 @@ +# Bring up the `human-language` microservice locally. +# +# Usage: +# docker compose -f examples/docker-compose.yml up --build +# +# Then: +# curl http://localhost:8080/healthz +# curl http://localhost:8080/version +# curl -X POST http://localhost:8080/transform \ +# -H 'content-type: application/json' \ +# -d '{"text":"Albert Einstein was born in Ulm"}' +# +# Pass `-H 'accept: text/plain;codec=lino'` to receive Links Notation +# instead of JSON. + +services: + human-language: + build: + context: .. + dockerfile: Dockerfile + image: human-language:dev + container_name: human-language + restart: unless-stopped + ports: + - "8080:8080" + environment: + HUMAN_LANGUAGE_HOST: 0.0.0.0 + HUMAN_LANGUAGE_PORT: "8080" + HUMAN_LANGUAGE_CACHE_TYPE: file + HUMAN_LANGUAGE_CACHE_DIR: /app/data/wikidata-cache + volumes: + # Persist the Wikidata cache between container restarts so cold + # starts are quick. + - human-language-cache:/app/data/wikidata-cache + +volumes: + human-language-cache: diff --git a/js/scripts/check-package-version.mjs b/js/scripts/check-package-version.mjs new file mode 100644 index 0000000..ddc055b --- /dev/null +++ b/js/scripts/check-package-version.mjs @@ -0,0 +1,57 @@ +#!/usr/bin/env node +// Detects whether package.json's version changed in the current commit +// relative to its parent. Writes `should_publish=…` and `version=…` to +// $GITHUB_OUTPUT so the release workflow can gate `npm publish` on a +// real version bump. +// +// Honours $FORCE_PUBLISH=true (set by workflow_dispatch) to skip the diff. +// +// Exit status is always 0 — a missing previous version is treated as +// "no publish needed" rather than a CI failure. + +import { spawnSync } from 'node:child_process'; +import { readFileSync, appendFileSync } from 'node:fs'; + +function readCurrentVersion() { + const pkg = JSON.parse(readFileSync('package.json', 'utf8')); + if (!pkg.version) throw new Error('package.json missing `version`'); + return pkg.version; +} + +function readPreviousVersion() { + const result = spawnSync('git', ['show', 'HEAD~1:package.json'], { encoding: 'utf8' }); + if (result.status !== 0) return null; + try { + return JSON.parse(result.stdout).version || null; + } catch { + return null; + } +} + +function setOutput(key, value) { + const path = process.env.GITHUB_OUTPUT; + const line = `${key}=${value}\n`; + if (path) appendFileSync(path, line); + else process.stdout.write(line); +} + +const current = readCurrentVersion(); +setOutput('version', current); + +if (process.env.FORCE_PUBLISH === 'true') { + console.log(`Force publish requested for ${current}.`); + setOutput('should_publish', 'true'); + process.exit(0); +} + +const previous = readPreviousVersion(); +if (previous && previous !== current) { + console.log(`npm package version: ${previous} -> ${current}; publishing.`); + setOutput('should_publish', 'true'); +} else if (!previous) { + console.log(`npm package version: ${current} (no previous package.json); skipping publish.`); + setOutput('should_publish', 'false'); +} else { + console.log(`npm package version unchanged at ${current}; skipping publish.`); + setOutput('should_publish', 'false'); +} diff --git a/js/scripts/run-unit-tests.mjs b/js/scripts/run-unit-tests.mjs index 0979d36..3f82f4e 100755 --- a/js/scripts/run-unit-tests.mjs +++ b/js/scripts/run-unit-tests.mjs @@ -12,6 +12,11 @@ const FAST_SUITES = [ // Pure-JS suites with no network dependency. Failures here gate the build. 'js/tests/unit/routing.test.mjs', 'js/tests/unit/ipa.test.mjs', + 'js/tests/unit/config.test.mjs', + 'js/tests/unit/lino-format.test.mjs', + 'js/tests/unit/persistent-cache.test.mjs', + 'js/tests/unit/server.test.mjs', + 'js/tests/unit/cli.test.mjs', ]; const INTEGRATION_SUITES = [ diff --git a/js/src/cli.js b/js/src/cli.js new file mode 100755 index 0000000..13eaf2a --- /dev/null +++ b/js/src/cli.js @@ -0,0 +1,164 @@ +#!/usr/bin/env node +// `human-language` command-line entry point. +// +// Usage: +// human-language transform "Albert Einstein was born in Ulm" +// human-language transform --format lino "Cats love fish" +// human-language entity Q42 +// human-language property P31 +// human-language search "Einstein" +// human-language serve --port 8080 +// human-language version +// +// Arguments / env precedence follows `lino-arguments` (see ./config.js). + +import { resolveConfig } from './config.js'; +import { name, version } from './version.js'; + +const USAGE = `\ +Usage: human-language [options] [args] + +Commands: + transform Convert English text into a Q/P sequence + entity Fetch a Wikidata entity (e.g. Q42) + property Fetch a Wikidata property (e.g. P31) + search Search Wikidata for entities / properties + serve Start the HTTP microservice + version Print the version + help Print this message + +Options (apply where relevant): + --format json|lino Output format (default: json) + --port Server port (default: 8080) + --host Server host (default: 0.0.0.0) + --cache-dir

Directory for the file cache + --cache-type 'auto' | 'file' | 'indexeddb' | 'none' + +All options can also be set via HUMAN_LANGUAGE_* environment variables.`; + +function printJson(value) { + process.stdout.write(JSON.stringify(value, null, 2) + '\n'); +} + +function parseFormatFlag(argv) { + let format = 'json'; + const rest = []; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--format') { format = argv[++i] || 'json'; continue; } + if (argv[i].startsWith('--format=')) { format = argv[i].slice('--format='.length); continue; } + rest.push(argv[i]); + } + return { format, rest }; +} + +async function commandTransform(argv) { + const { format, rest } = parseFormatFlag(argv); + const text = rest.join(' '); + if (!text) { + process.stderr.write('error: `transform` requires a text argument\n'); + process.exit(2); + } + const { TextToQPTransformer } = await import('./transformation/text-to-qp-transformer.js'); + const transformer = new TextToQPTransformer(); + const result = await transformer.transform(text); + if (format === 'lino') { + const { formatTransformResultAsLino } = await import('./transformation/lino-format.js'); + process.stdout.write(formatTransformResultAsLino(result)); + } else { + printJson(result); + } +} + +async function commandEntity(argv, config) { + const id = argv[0]; + if (!id) { + process.stderr.write('error: `entity` requires an id (e.g. Q42)\n'); + process.exit(2); + } + const { WikidataAPIClient } = await import('./wikidata-api.js'); + const client = new WikidataAPIClient(config.cacheType, { cacheDir: config.cacheDir }); + const data = await client.fetchEntity(id); + printJson(data); +} + +async function commandProperty(argv, config) { + const id = argv[0]; + if (!id) { + process.stderr.write('error: `property` requires an id (e.g. P31)\n'); + process.exit(2); + } + const { WikidataAPIClient } = await import('./wikidata-api.js'); + const client = new WikidataAPIClient(config.cacheType, { cacheDir: config.cacheDir }); + const data = await client.fetchProperty(id); + printJson(data); +} + +async function commandSearch(argv, config) { + const query = argv.join(' '); + if (!query) { + process.stderr.write('error: `search` requires a query\n'); + process.exit(2); + } + const { WikidataAPIClient, WikidataSearchUtility } = await import('./wikidata-api.js'); + const client = new WikidataAPIClient(config.cacheType, { cacheDir: config.cacheDir }); + const util = new WikidataSearchUtility(client, null, null); + const data = await util.disambiguateSearch(query); + printJson(data); +} + +async function commandServe(config) { + // Lazy-load so `human-language transform …` doesn't pay the server cost. + const { createServer } = await import('./server.js'); + const srv = createServer({ config }); + await new Promise((resolve) => srv.listen(config.port, config.host, resolve)); + const addr = srv.address(); + const host = addr && typeof addr === 'object' ? addr.address : config.host; + const port = addr && typeof addr === 'object' ? addr.port : config.port; + process.stdout.write(`human-language listening on http://${host}:${port}\n`); +} + +export async function main(argv = process.argv.slice(2), env = process.env) { + if (argv.length === 0 || argv[0] === 'help' || argv[0] === '--help' || argv[0] === '-h') { + process.stdout.write(USAGE + '\n'); + return; + } + if (argv[0] === 'version' || argv[0] === '--version' || argv[0] === '-V') { + process.stdout.write(`${name} ${version}\n`); + return; + } + + const command = argv[0]; + const tail = argv.slice(1); + + // Resolve config from env + the post-command argv. This means `--port` + // and friends can appear after the command. + const config = resolveConfig({ argv: tail, env }); + const positional = config._; + + switch (command) { + case 'transform': return commandTransform(positional); + case 'entity': return commandEntity(positional, config); + case 'property': return commandProperty(positional, config); + case 'search': return commandSearch(positional, config); + case 'serve': return commandServe(config); + default: + process.stderr.write(`error: unknown command '${command}'\n\n${USAGE}\n`); + process.exit(2); + } +} + +// When invoked as a script, run main(). +const isMain = (() => { + if (typeof process === 'undefined' || !process.argv?.[1]) return false; + try { + const url = new URL(import.meta.url); + return url.pathname === process.argv[1] || url.pathname.endsWith(process.argv[1]); + } catch { return false; } +})(); + +if (isMain) { + main().catch((err) => { + process.stderr.write(`error: ${err?.message || err}\n`); + process.exit(1); + }); +} diff --git a/js/src/config.js b/js/src/config.js new file mode 100644 index 0000000..56a9e7c --- /dev/null +++ b/js/src/config.js @@ -0,0 +1,140 @@ +// Configuration backed by `link-foundation/lino-arguments`. +// +// Precedence: +// 1. Explicit `argv` (CLI flags, including short aliases). +// 2. Environment variables, prefixed `HUMAN_LANGUAGE_...`. +// 3. Built-in defaults (`CONFIG_DEFAULTS`). +// +// `resolveConfig` accepts an injected `env` object so unit tests and callers can +// resolve config without mutating the process environment permanently. + +import { makeConfig } from 'lino-arguments'; + +export const CONFIG_DEFAULTS = Object.freeze({ + port: 8080, + host: '0.0.0.0', + cacheDir: './data/wikidata-cache', + cacheType: 'auto', + userAgent: 'human-language', + // The default Wikidata API endpoint. Override for self-hosted Wikibase + // instances. + wikidataApiBase: 'https://www.wikidata.org/w/api.php', +}); + +// Spec: each key has a long flag (matches the camelCase config key in +// kebab-case), an optional short alias, an env var name and a parser. +const SPEC = [ + { key: 'port', long: 'port', short: 'p', env: 'HUMAN_LANGUAGE_PORT', parse: parseInteger }, + { key: 'host', long: 'host', short: 'h', env: 'HUMAN_LANGUAGE_HOST', parse: identity }, + { key: 'cacheDir', long: 'cache-dir', short: null,env: 'HUMAN_LANGUAGE_CACHE_DIR', parse: identity }, + { key: 'cacheType', long: 'cache-type', short: null,env: 'HUMAN_LANGUAGE_CACHE_TYPE', parse: identity }, + { key: 'userAgent', long: 'user-agent', short: null,env: 'HUMAN_LANGUAGE_USER_AGENT', parse: identity }, + { key: 'wikidataApiBase',long: 'wikidata-api-base',short: null,env: 'HUMAN_LANGUAGE_WIKIDATA_API_BASE',parse: identity }, +]; + +function identity(v) { + return v; +} + +function parseInteger(v) { + const n = Number(v); + if (!Number.isInteger(n)) { + throw new TypeError(`expected an integer, got ${JSON.stringify(v)}`); + } + return n; +} + +function collectPositionals(argv) { + const positional = []; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--') { + positional.push(...argv.slice(i + 1)); + break; + } + if (arg.startsWith('--')) { + if (!arg.includes('=')) i += 1; + continue; + } + if (arg.startsWith('-') && arg.length > 1) { + i += 1; + continue; + } + positional.push(arg); + } + return positional; +} + +function withInjectedEnv(env, fn) { + if (env === process.env) return fn(); + const keys = new Set([...SPEC.map((item) => item.env), ...Object.keys(env)]); + const previous = new Map(); + for (const key of keys) { + previous.set(key, process.env[key]); + if (env[key] === undefined) delete process.env[key]; + else process.env[key] = env[key]; + } + try { + return fn(); + } finally { + for (const [key, value] of previous) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } +} + +/** + * Resolve configuration with `lino-arguments` precedence. + * + * @param {object} [opts] + * @param {string[]} [opts.argv] - CLI args (typically `process.argv.slice(2)`). + * @param {Record} [opts.env] - Environment object. + * @param {Partial} [opts.defaults] - Override defaults. + * @returns {typeof CONFIG_DEFAULTS & { _: string[] }} + * The resolved config plus `_`, the array of positional arguments. + */ +export function resolveConfig({ argv = [], env = {}, defaults = {} } = {}) { + const mergedDefaults = { ...CONFIG_DEFAULTS, ...defaults }; + + return withInjectedEnv(env, () => { + const parsed = makeConfig({ + argv: ['node', 'human-language', ...argv], + lenv: { enabled: false }, + env: { enabled: false }, + yargs: ({ yargs, getenv }) => { + let builder = yargs + .help(false) + .version(false) + .exitProcess(false) + .showHelpOnFail(false) + .fail((message, error) => { + const text = error?.message || message || 'argument parsing failed'; + if (/Unknown argument/.test(text)) { + throw new Error(`unknown flag: ${text}`); + } + throw error || new Error(text); + }) + .strictOptions(); + + for (const item of SPEC) { + builder = builder.option(item.long, { + alias: item.short || undefined, + type: item.key === 'port' ? 'number' : 'string', + default: getenv(item.env, mergedDefaults[item.key]), + }); + } + return builder; + }, + }); + + const config = { ...mergedDefaults }; + for (const item of SPEC) { + if (parsed[item.key] !== undefined && parsed[item.key] !== '') { + config[item.key] = item.parse(parsed[item.key]); + } + } + config._ = collectPositionals(argv); + return config; + }); +} diff --git a/js/src/index.d.ts b/js/src/index.d.ts new file mode 100644 index 0000000..7832fe3 --- /dev/null +++ b/js/src/index.d.ts @@ -0,0 +1,126 @@ +// Hand-written TypeScript declarations for the `human-language` package. +// +// The source is plain JavaScript with JSDoc; these declarations are +// kept narrow on purpose. Each surface mirrors the runtime shape that +// `js/src/index.js` actually re-exports. Generated declarations may +// replace this file in a follow-up issue once we adopt `tsc --emit +// declarations` in CI. + +/** Resolved configuration shape from `resolveConfig`. */ +export interface HumanLanguageConfig { + port: number; + host: string; + cacheDir: string; + cacheType: 'auto' | 'file' | 'indexeddb' | 'none' | string; + userAgent: string; + wikidataApiBase: string; + _: string[]; +} + +export interface ResolveConfigOptions { + argv?: string[]; + env?: Record; + defaults?: Partial>; +} + +export function resolveConfig(opts?: ResolveConfigOptions): HumanLanguageConfig; +export const CONFIG_DEFAULTS: Readonly>; + +export const name: string; +export const version: string; + +// --------------------------------------------------------------------------- +// Transformer +// --------------------------------------------------------------------------- + +export interface TransformOptions { + maxCandidates?: number; + includeLabels?: boolean; + searchLimit?: number; + preferProperties?: boolean; + maxNgramSize?: number; +} + +export interface TransformResult { + original: string; + tokens: string[]; + sequence: Array }>; + formatted: string; + alternatives: unknown[]; +} + +export class TextToQPTransformer { + constructor(); + transform(text: string, options?: TransformOptions): Promise; + transformWithContext( + text: string, + context?: Record, + options?: TransformOptions, + ): Promise; +} + +export function formatSequenceAsLino(sequence: TransformResult['sequence']): string; +export function formatTransformResultAsLino(result: TransformResult): string; + +// --------------------------------------------------------------------------- +// Wikidata API +// --------------------------------------------------------------------------- + +export class WikidataAPIClient { + constructor(cacheType?: string, cacheOptions?: Record); + fetchEntity(id: string, languages?: string): Promise; + fetchEntities(ids: string | string[], props?: string, languages?: string): Promise; + fetchProperty(id: string, languages?: string): Promise; + fetchLabels(ids: string[], languages?: string): Promise; + searchExactMatch(query: string, languages?: string, limit?: number, type?: string): Promise; + searchFuzzy(query: string, languages?: string, limit?: number, type?: string): Promise; + setCacheType(cacheType: string, cacheOptions?: Record): void; +} + +export class WikidataSearchUtility { + constructor(client: WikidataAPIClient, cache: unknown, processor: unknown); + disambiguateSearch(query: string, languages?: string, limit?: number, type?: string): Promise; +} + +export class WikidataDataProcessor { constructor(); } +export class WikidataLabelManager { constructor(client: WikidataAPIClient, cache: unknown, processor: unknown); } +export class WikidataCacheManager { constructor(); } + +// --------------------------------------------------------------------------- +// Cache +// --------------------------------------------------------------------------- + +export class CacheInterface { constructor(); } +export class IndexedDBCacheAdapter { constructor(dbName?: string, version?: number); } +export class NoCacheAdapter { constructor(); } +export class BrowserCacheFactory { + static create(type: string, options?: Record): CacheInterface; +} + +// --------------------------------------------------------------------------- +// Settings / locale helpers +// --------------------------------------------------------------------------- + +export const STORAGE_KEYS: Record; +export const localeQuotes: Record; +export const flagMap: Record; + +export function saveToLocalStorage(key: string, value: unknown): void; +export function loadFromLocalStorage(key: string, fallback?: T): T; +export function getQuotesForLanguage(lang: string): { open: string; close: string }; + +// --------------------------------------------------------------------------- +// Routing +// --------------------------------------------------------------------------- + +export const MODES: readonly string[]; +export const DEFAULT_MODE: string; +export function parseHash(hash: string): { mode: string; params: Record }; +export function serializeHash(parsed: { mode: string; params?: Record }): string; + +// --------------------------------------------------------------------------- +// IPA (browser-leaning, available under Node when fetch exists) +// --------------------------------------------------------------------------- + +export function toIpa(text: string, lang?: string): Promise; +export function toIpaForEntity(entity: unknown, lang?: string): Promise; diff --git a/js/src/index.js b/js/src/index.js new file mode 100644 index 0000000..f7c0a5f --- /dev/null +++ b/js/src/index.js @@ -0,0 +1,57 @@ +// Public entry point for the `human-language` npm package. +// +// Re-exports the core (non-UI) modules of the project as a stable, +// versioned surface. UI modules (`*.jsx`, `app/modes/*`, `app/shell.jsx`, +// `statements.jsx`, `loading.jsx`, `app/tests-panel.jsx`) are +// intentionally not re-exported — they remain implementation details of +// the SPA at `app.html` and are still in the published tarball so the +// SPA continues to work when this package is consumed via a CDN. +// +// See docs/case-studies/issue-37/architecture.md for the full surface +// map. + +export { + WikidataAPIClient, + WikidataSearchUtility, + WikidataDataProcessor, + WikidataLabelManager, + WikidataCacheManager, +} from './wikidata-api-browser.js'; + +export { + CacheInterface, + IndexedDBCacheAdapter, + NoCacheAdapter, + BrowserCacheFactory, +} from './unified-cache-browser.js'; + +export { TextToQPTransformer } from './transformation/text-to-qp-transformer.js'; + +export { + formatSequenceAsLino, + formatTransformResultAsLino, +} from './transformation/lino-format.js'; + +export { + STORAGE_KEYS, + saveToLocalStorage, + loadFromLocalStorage, + localeQuotes, + getQuotesForLanguage, + flagMap, +} from './settings.js'; + +export { + MODES, + DEFAULT_MODE, + parseHash, + serializeHash, +} from './app/routing.js'; + +// Re-exports that may be useful for callers in a browser context where +// `window` is defined. +export { toIpa, toIpaForEntity } from './app/ipa.js'; + +export { resolveConfig, CONFIG_DEFAULTS } from './config.js'; + +export { version, name } from './version.js'; diff --git a/js/src/persistent-cache.js b/js/src/persistent-cache.js index b749d2a..4b76976 100644 --- a/js/src/persistent-cache.js +++ b/js/src/persistent-cache.js @@ -4,14 +4,28 @@ import { promises as fs } from 'fs'; import { createHash } from 'crypto'; import path from 'path'; +import { encode, decode } from 'lino-objects-codec'; + +const CODEC_EXTENSIONS = Object.freeze({ + json: 'json', + lino: 'lino', +}); /** * Persistent File-Based Cache Manager * Stores API responses in JSON files in the /data directory */ class PersistentCacheManager { - constructor(cacheDir = './data') { + constructor(cacheDir = './data', options = {}) { + if (typeof cacheDir === 'object' && cacheDir !== null) { + options = cacheDir; + cacheDir = options.cacheDir || './data'; + } this.cacheDir = cacheDir; + this.codec = options.codec || 'json'; + if (!CODEC_EXTENSIONS[this.codec]) { + throw new Error(`unsupported cache codec: ${this.codec}`); + } this.memoryCache = new Map(); // In-memory cache for faster access this.maxMemorySize = 1000; // Maximum items in memory cache this.ensureCacheDir(); @@ -40,7 +54,27 @@ class PersistentCacheManager { * Get cache file path for a given key */ getCacheFilePath(key) { - return path.join(this.cacheDir, `${key}.json`); + return path.join(this.cacheDir, `${key}.${CODEC_EXTENSIONS[this.codec]}`); + } + + /** + * Serialize a cache entry using the configured on-disk codec. + */ + serializeCacheEntry(entry) { + if (this.codec === 'lino') { + return encode({ obj: entry }); + } + return JSON.stringify(entry, null, 2); + } + + /** + * Deserialize a cache entry using the configured on-disk codec. + */ + deserializeCacheEntry(content) { + if (this.codec === 'lino') { + return decode({ notation: content }); + } + return JSON.parse(content); } /** @@ -63,7 +97,7 @@ class PersistentCacheManager { try { const filePath = this.getCacheFilePath(key); const fileContent = await fs.readFile(filePath, 'utf8'); - const cached = JSON.parse(fileContent); + const cached = this.deserializeCacheEntry(fileContent); if (this.isCacheValid(cached)) { // Add to memory cache @@ -101,7 +135,7 @@ class PersistentCacheManager { // Store in file cache try { const filePath = this.getCacheFilePath(key); - await fs.writeFile(filePath, JSON.stringify(cacheEntry, null, 2)); + await fs.writeFile(filePath, this.serializeCacheEntry(cacheEntry)); } catch (error) { console.warn('Failed to write cache file:', error.message); } @@ -159,10 +193,10 @@ class PersistentCacheManager { // Clear file cache try { const files = await fs.readdir(this.cacheDir); - const jsonFiles = files.filter(file => file.endsWith('.json')); + const cacheFiles = files.filter(file => Object.values(CODEC_EXTENSIONS).some(ext => file.endsWith(`.${ext}`))); await Promise.all( - jsonFiles.map(file => + cacheFiles.map(file => fs.unlink(path.join(this.cacheDir, file)).catch(() => {}) ) ); @@ -184,10 +218,10 @@ class PersistentCacheManager { try { const files = await fs.readdir(this.cacheDir); - const jsonFiles = files.filter(file => file.endsWith('.json')); - fileCount = jsonFiles.length; + const cacheFiles = files.filter(file => file.endsWith(`.${CODEC_EXTENSIONS[this.codec]}`)); + fileCount = cacheFiles.length; - for (const file of jsonFiles) { + for (const file of cacheFiles) { try { const filePath = path.join(this.cacheDir, file); const stats = await fs.stat(filePath); @@ -245,13 +279,13 @@ class PersistentCacheManager { // Clean file cache try { const files = await fs.readdir(this.cacheDir); - const jsonFiles = files.filter(file => file.endsWith('.json')); + const cacheFiles = files.filter(file => file.endsWith(`.${CODEC_EXTENSIONS[this.codec]}`)); - for (const file of jsonFiles) { + for (const file of cacheFiles) { try { const filePath = path.join(this.cacheDir, file); const fileContent = await fs.readFile(filePath, 'utf8'); - const cached = JSON.parse(fileContent); + const cached = this.deserializeCacheEntry(fileContent); if (!this.isCacheValid(cached)) { await fs.unlink(filePath); @@ -300,13 +334,13 @@ class PersistentCacheManager { try { const files = await fs.readdir(this.cacheDir); - const jsonFiles = files.filter(file => file.endsWith('.json')); + const cacheFiles = files.filter(file => file.endsWith(`.${CODEC_EXTENSIONS[this.codec]}`)); - for (const file of jsonFiles) { + for (const file of cacheFiles) { try { const filePath = path.join(this.cacheDir, file); const fileContent = await fs.readFile(filePath, 'utf8'); - const cached = JSON.parse(fileContent); + const cached = this.deserializeCacheEntry(fileContent); if (this.isCacheValid(cached)) { allEntries[cached.query] = cached; @@ -348,4 +382,4 @@ class PersistentCacheManager { } } -export { PersistentCacheManager }; \ No newline at end of file +export { PersistentCacheManager }; diff --git a/js/src/server.js b/js/src/server.js new file mode 100644 index 0000000..2fa5ea1 --- /dev/null +++ b/js/src/server.js @@ -0,0 +1,175 @@ +// HTTP microservice that exposes the public `human-language` API. +// +// The server has no third-party HTTP dependency; it uses Node's built-in +// `http` module and the shared runtime dependencies already used by the CLI +// and cache layers. +// +// Routes: +// GET /healthz -> { ok: true } +// GET /version -> { name, version } +// POST /transform -> body { text, options? } +// GET /entity/:id -> Wikidata entity +// GET /property/:id -> Wikidata property +// GET /search?q=…&type=item|property -> disambiguation results +// +// Response format negotiation: `Accept: text/plain;codec=lino` selects +// the Links Notation serializer for `POST /transform`. All other paths +// always reply with JSON. + +import { createServer as createNodeServer } from 'node:http'; +import { URL } from 'node:url'; + +import { resolveConfig } from './config.js'; +import { TextToQPTransformer } from './transformation/text-to-qp-transformer.js'; +import { formatTransformResultAsLino } from './transformation/lino-format.js'; +import { WikidataAPIClient, WikidataSearchUtility } from './wikidata-api.js'; +import { name, version } from './version.js'; + +const MAX_BODY_BYTES = 1024 * 1024; // 1 MiB request bodies are plenty. + +function readJsonBody(req) { + return new Promise((resolve, reject) => { + let bytes = 0; + const chunks = []; + req.on('data', (chunk) => { + bytes += chunk.length; + if (bytes > MAX_BODY_BYTES) { + reject(Object.assign(new Error('request body too large'), { status: 413 })); + req.destroy(); + return; + } + chunks.push(chunk); + }); + req.on('end', () => { + try { + const raw = Buffer.concat(chunks).toString('utf8'); + if (!raw) return resolve({}); + resolve(JSON.parse(raw)); + } catch (err) { + reject(Object.assign(new Error('invalid JSON'), { status: 400, cause: err })); + } + }); + req.on('error', reject); + }); +} + +function reply(res, status, body, headers = {}) { + const isString = typeof body === 'string'; + const payload = isString ? body : JSON.stringify(body); + const baseHeaders = { + 'content-type': isString ? 'text/plain; charset=utf-8' : 'application/json; charset=utf-8', + 'content-length': Buffer.byteLength(payload), + 'cache-control': 'no-store', + }; + res.writeHead(status, { ...baseHeaders, ...headers }); + res.end(payload); +} + +function wantsLino(req) { + const accept = req.headers['accept'] || ''; + return accept.toLowerCase().includes('codec=lino'); +} + +function pickClient({ wikidataClient, config }) { + if (wikidataClient) return wikidataClient; + const client = new WikidataAPIClient(config.cacheType, { cacheDir: config.cacheDir }); + return client; +} + +/** + * Build (but do not listen on) an HTTP server that exposes the public API. + * + * @param {object} [opts] + * @param {ReturnType} [opts.config] - Resolved configuration. + * @param {InstanceType} [opts.transformer] - Injected transformer (tests). + * @param {InstanceType} [opts.wikidataClient] - Injected client (tests). + * @returns {import('node:http').Server} + */ +export function createServer(opts = {}) { + const config = opts.config || resolveConfig({ env: process.env }); + const transformer = opts.transformer || new TextToQPTransformer(); + const handler = async (req, res) => { + const url = new URL(req.url, `http://${req.headers.host || 'localhost'}`); + try { + // Health + version + if (req.method === 'GET' && url.pathname === '/healthz') { + return reply(res, 200, { ok: true }); + } + if (req.method === 'GET' && url.pathname === '/version') { + return reply(res, 200, { name, version }); + } + if (req.method === 'GET' && url.pathname === '/') { + return reply(res, 200, { + name, + version, + endpoints: ['/healthz', '/version', '/transform', '/entity/:id', '/property/:id', '/search'], + }); + } + + // POST /transform + if (req.method === 'POST' && url.pathname === '/transform') { + const body = await readJsonBody(req); + if (!body || typeof body.text !== 'string') { + return reply(res, 400, { error: 'body.text is required and must be a string' }); + } + const result = await transformer.transform(body.text, body.options || {}); + if (wantsLino(req)) { + return reply(res, 200, formatTransformResultAsLino(result), { + 'content-type': 'text/plain;codec=lino; charset=utf-8', + }); + } + return reply(res, 200, result); + } + + // GET /entity/:id and /property/:id and /search + const entityMatch = req.method === 'GET' && url.pathname.match(/^\/entity\/(Q\d+)$/i); + if (entityMatch) { + const client = pickClient({ wikidataClient: opts.wikidataClient, config }); + const data = await client.fetchEntity(entityMatch[1].toUpperCase()); + return reply(res, 200, data); + } + const propertyMatch = req.method === 'GET' && url.pathname.match(/^\/property\/(P\d+)$/i); + if (propertyMatch) { + const client = pickClient({ wikidataClient: opts.wikidataClient, config }); + const data = await client.fetchProperty(propertyMatch[1].toUpperCase()); + return reply(res, 200, data); + } + if (req.method === 'GET' && url.pathname === '/search') { + const q = url.searchParams.get('q'); + if (!q) return reply(res, 400, { error: 'query parameter `q` is required' }); + const type = url.searchParams.get('type') || 'both'; + const limit = Number(url.searchParams.get('limit') || '10'); + const client = pickClient({ wikidataClient: opts.wikidataClient, config }); + const util = new WikidataSearchUtility(client, null, null); + const data = await util.disambiguateSearch(q, 'en', limit, type); + return reply(res, 200, data); + } + + return reply(res, 404, { error: `no route for ${req.method} ${url.pathname}` }); + } catch (err) { + const status = err && typeof err.status === 'number' ? err.status : 500; + return reply(res, status, { error: err?.message || String(err) }); + } + }; + + return createNodeServer(handler); +} + +const isMain = (() => { + if (typeof process === 'undefined' || !process.argv?.[1]) return false; + try { + const url = new URL(import.meta.url); + return url.pathname === process.argv[1] || url.pathname.endsWith(process.argv[1]); + } catch { return false; } +})(); + +if (isMain) { + const config = resolveConfig({ argv: process.argv.slice(2), env: process.env }); + const srv = createServer({ config }); + srv.listen(config.port, config.host, () => { + const addr = srv.address(); + const host = addr && typeof addr === 'object' ? addr.address : config.host; + const port = addr && typeof addr === 'object' ? addr.port : config.port; + process.stdout.write(`human-language listening on http://${host}:${port}\n`); + }); +} diff --git a/js/src/transformation/lino-format.js b/js/src/transformation/lino-format.js new file mode 100644 index 0000000..daec71d --- /dev/null +++ b/js/src/transformation/lino-format.js @@ -0,0 +1,47 @@ +// Serialise transformer output as a Links Notation (LiNo) string. +// +// The on-wire shape we emit looks like: +// +// sequence: +// ((Q35120) (P31) (Q5) (P21) (Q6581097)) +// +// where each parenthesised atom is a Wikidata Q-id or P-id. Ambiguous +// matches use the `[A or B or C]` syntax that already appears in the +// transformer's `formatted` field, e.g. `[Q1 or Q2 or Q3]`. +// +// This is a deterministic, dependency-free serializer. The upstream +// `links-notation` parser is a candidate to *consume* LiNo documents on +// input — that path is tracked as a follow-up issue (R8 in the issue-37 +// case study). + +const LINO_HEADER = 'sequence:'; + +/** + * @param {Array}>} sequence + * @returns {string} + */ +export function formatSequenceAsLino(sequence) { + if (!Array.isArray(sequence)) return `${LINO_HEADER}\n`; + const atoms = sequence.map((item) => { + if (typeof item === 'string') return `(${item})`; + if (item && item.type === 'ambiguous' && Array.isArray(item.alternatives)) { + const ids = item.alternatives.map((alt) => alt.id).filter(Boolean); + return `[${ids.join(' or ')}]`; + } + if (item && item.id) return `(${item.id})`; + return '()'; + }); + if (atoms.length === 0) return `${LINO_HEADER}\n`; + return `${LINO_HEADER}\n (${atoms.join(' ')})\n`; +} + +/** + * @param {object} result - The return value of `TextToQPTransformer#transform`. + * @returns {string} + */ +export function formatTransformResultAsLino(result) { + if (!result || typeof result !== 'object') return ''; + return formatSequenceAsLino(result.sequence); +} + +export default { formatSequenceAsLino, formatTransformResultAsLino }; diff --git a/js/src/unified-cache.js b/js/src/unified-cache.js index 6b0ced7..f4d5fbb 100644 --- a/js/src/unified-cache.js +++ b/js/src/unified-cache.js @@ -75,9 +75,13 @@ class CacheInterface { * File-based Cache Adapter (Node.js environments) */ class FileCacheAdapter extends CacheInterface { - constructor(cacheDir = ROOT_DATA_DIR) { + constructor(cacheDir = ROOT_DATA_DIR, options = {}) { super(); - this.cache = new PersistentCacheManager(cacheDir); + if (typeof cacheDir === 'object' && cacheDir !== null) { + options = cacheDir; + cacheDir = options.cacheDir || ROOT_DATA_DIR; + } + this.cache = new PersistentCacheManager(cacheDir, options); } async get(key, languages = 'en', limit = 50, type = 'both') { @@ -380,7 +384,7 @@ class CacheFactory { static create(type = 'auto', options = {}) { switch (type) { case 'file': - return new FileCacheAdapter(options.cacheDir); + return new FileCacheAdapter(options.cacheDir, options); case 'indexeddb': return new IndexedDBCacheAdapter(options.dbName, options.version); @@ -414,4 +418,4 @@ export { IndexedDBCacheAdapter, NoCacheAdapter, CacheFactory -}; \ No newline at end of file +}; diff --git a/js/src/version.js b/js/src/version.js new file mode 100644 index 0000000..8a27483 --- /dev/null +++ b/js/src/version.js @@ -0,0 +1,26 @@ +// Reads the package name and version from package.json at runtime so the +// library, the CLI and the HTTP server always agree on what they report. +// +// `readFileSync` is used (instead of an `import … assert { type: 'json' }`) +// because the latter is still gated behind an experimental flag in some +// Node 20 minor releases and Bun versions we want to support. + +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, resolve } from 'node:path'; + +function loadPackageJson() { + // js/src/version.js → ../../package.json + const here = dirname(fileURLToPath(import.meta.url)); + const pkgPath = resolve(here, '..', '..', 'package.json'); + return JSON.parse(readFileSync(pkgPath, 'utf8')); +} + +let cached; +function pkg() { + if (!cached) cached = loadPackageJson(); + return cached; +} + +export const name = pkg().name; +export const version = pkg().version; diff --git a/js/tests/unit/cli.test.mjs b/js/tests/unit/cli.test.mjs new file mode 100644 index 0000000..f3f2b60 --- /dev/null +++ b/js/tests/unit/cli.test.mjs @@ -0,0 +1,54 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { dirname, resolve } from 'node:path'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const CLI = resolve(HERE, '..', '..', 'src', 'cli.js'); + +function runCli(args, env = {}) { + return spawnSync(process.execPath, [CLI, ...args], { + encoding: 'utf8', + env: { ...process.env, ...env }, + timeout: 10_000, + }); +} + +test('`help` prints usage and exits 0', () => { + const r = runCli(['help']); + assert.equal(r.status, 0); + assert.match(r.stdout, /Usage: human-language/); + assert.match(r.stdout, /transform/); + assert.match(r.stdout, /serve/); +}); + +test('no args prints usage', () => { + const r = runCli([]); + assert.equal(r.status, 0); + assert.match(r.stdout, /Usage: human-language/); +}); + +test('`version` prints package name and version', () => { + const r = runCli(['version']); + assert.equal(r.status, 0); + assert.match(r.stdout, /^human-language \d+\.\d+\.\d+/); +}); + +test('unknown command exits non-zero with a hint', () => { + const r = runCli(['no-such-command']); + assert.notEqual(r.status, 0); + assert.match(r.stderr, /unknown command/); +}); + +test('`transform` without text exits 2', () => { + const r = runCli(['transform']); + assert.equal(r.status, 2); + assert.match(r.stderr, /requires a text argument/); +}); + +test('`entity` without id exits 2', () => { + const r = runCli(['entity']); + assert.equal(r.status, 2); + assert.match(r.stderr, /requires an id/); +}); diff --git a/js/tests/unit/config.test.mjs b/js/tests/unit/config.test.mjs new file mode 100644 index 0000000..9c07d4c --- /dev/null +++ b/js/tests/unit/config.test.mjs @@ -0,0 +1,80 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { resolveConfig, CONFIG_DEFAULTS } from '../../src/config.js'; + +test('defaults are returned when no argv/env are given', () => { + const config = resolveConfig(); + assert.equal(config.port, CONFIG_DEFAULTS.port); + assert.equal(config.host, CONFIG_DEFAULTS.host); + assert.equal(config.cacheDir, CONFIG_DEFAULTS.cacheDir); + assert.equal(config.cacheType, CONFIG_DEFAULTS.cacheType); + assert.deepEqual(config._, []); +}); + +test('env vars override defaults', () => { + const config = resolveConfig({ + env: { + HUMAN_LANGUAGE_PORT: '9000', + HUMAN_LANGUAGE_HOST: 'example.test', + HUMAN_LANGUAGE_CACHE_DIR: '/var/cache', + }, + }); + assert.equal(config.port, 9000); + assert.equal(config.host, 'example.test'); + assert.equal(config.cacheDir, '/var/cache'); +}); + +test('argv overrides env', () => { + const config = resolveConfig({ + argv: ['--port', '7777'], + env: { HUMAN_LANGUAGE_PORT: '9000' }, + }); + assert.equal(config.port, 7777); +}); + +test('--key=value form works for long flags', () => { + const config = resolveConfig({ argv: ['--cache-type=none'] }); + assert.equal(config.cacheType, 'none'); +}); + +test('short flags resolve via alias table', () => { + const config = resolveConfig({ argv: ['-p', '12345', '-h', 'localhost'] }); + assert.equal(config.port, 12345); + assert.equal(config.host, 'localhost'); +}); + +test('positional arguments are collected', () => { + const config = resolveConfig({ argv: ['transform', '--port', '1', 'foo', 'bar'] }); + assert.deepEqual(config._, ['transform', 'foo', 'bar']); +}); + +test('arguments after `--` go to positionals', () => { + const config = resolveConfig({ argv: ['--', '--port', '1', 'foo'] }); + assert.deepEqual(config._, ['--port', '1', 'foo']); + // The double-dash form should not change the port default: + assert.equal(config.port, CONFIG_DEFAULTS.port); +}); + +test('unknown flag throws', () => { + assert.throws(() => resolveConfig({ argv: ['--no-such-flag', '1'] }), /unknown flag/); +}); + +test('non-integer port throws', () => { + assert.throws(() => resolveConfig({ argv: ['--port', 'abc'] }), /expected an integer/); +}); + +test('defaults are deep-frozen (mutation throws in strict mode)', () => { + assert.throws(() => { + // @ts-expect-error - deliberately mutating a frozen object + CONFIG_DEFAULTS.port = 1; + }); +}); + +test('opts.defaults overrides built-in defaults but is still beaten by env', () => { + const config = resolveConfig({ + defaults: { port: 1234 }, + env: { HUMAN_LANGUAGE_PORT: '5678' }, + }); + assert.equal(config.port, 5678); +}); diff --git a/js/tests/unit/lino-format.test.mjs b/js/tests/unit/lino-format.test.mjs new file mode 100644 index 0000000..9d8a0f1 --- /dev/null +++ b/js/tests/unit/lino-format.test.mjs @@ -0,0 +1,50 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + formatSequenceAsLino, + formatTransformResultAsLino, +} from '../../src/transformation/lino-format.js'; + +test('empty sequence renders the bare header', () => { + assert.equal(formatSequenceAsLino([]), 'sequence:\n'); +}); + +test('non-array input is treated as empty', () => { + assert.equal(formatSequenceAsLino(null), 'sequence:\n'); + assert.equal(formatSequenceAsLino(undefined), 'sequence:\n'); +}); + +test('string sequence renders as parenthesized atoms', () => { + const out = formatSequenceAsLino(['Q35120', 'P31', 'Q5']); + assert.equal(out, 'sequence:\n ((Q35120) (P31) (Q5))\n'); +}); + +test('ambiguous candidates render with bracket+`or` syntax', () => { + const out = formatSequenceAsLino([ + 'Q5', + { type: 'ambiguous', alternatives: [{ id: 'Q42' }, { id: 'Q1' }] }, + ]); + assert.equal(out, 'sequence:\n ((Q5) [Q42 or Q1])\n'); +}); + +test('object items with id render as parenthesized atoms', () => { + const out = formatSequenceAsLino([{ id: 'Q100' }]); + assert.equal(out, 'sequence:\n ((Q100))\n'); +}); + +test('formatTransformResultAsLino accepts the full result shape', () => { + const result = { + original: 'cats', + tokens: ['cats'], + sequence: ['Q146'], + formatted: 'Q146', + alternatives: [], + }; + assert.equal(formatTransformResultAsLino(result), 'sequence:\n ((Q146))\n'); +}); + +test('formatTransformResultAsLino returns "" for a falsy result', () => { + assert.equal(formatTransformResultAsLino(null), ''); + assert.equal(formatTransformResultAsLino(undefined), ''); +}); diff --git a/js/tests/unit/persistent-cache.test.mjs b/js/tests/unit/persistent-cache.test.mjs new file mode 100644 index 0000000..9ee26fb --- /dev/null +++ b/js/tests/unit/persistent-cache.test.mjs @@ -0,0 +1,27 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +import { PersistentCacheManager } from '../../src/persistent-cache.js'; + +test('PersistentCacheManager can persist entries with lino-objects-codec', async () => { + const cacheDir = await mkdtemp(join(tmpdir(), 'human-language-cache-')); + try { + const cache = new PersistentCacheManager(cacheDir, { codec: 'lino' }); + await cache.set('Einstein', { id: 'Q937', labels: ['Albert Einstein'] }, 'en', 1, 'item'); + + const key = cache.generateCacheKey('Einstein', 'en', 1, 'item'); + const stored = await readFile(join(cacheDir, `${key}.lino`), 'utf8'); + assert.match(stored, /^\(object/); + + const fresh = new PersistentCacheManager(cacheDir, { codec: 'lino' }); + assert.deepEqual(await fresh.get('Einstein', 'en', 1, 'item'), { + id: 'Q937', + labels: ['Albert Einstein'], + }); + } finally { + await rm(cacheDir, { recursive: true, force: true }); + } +}); diff --git a/js/tests/unit/server.test.mjs b/js/tests/unit/server.test.mjs new file mode 100644 index 0000000..61803c1 --- /dev/null +++ b/js/tests/unit/server.test.mjs @@ -0,0 +1,159 @@ +import { test, after } from 'node:test'; +import assert from 'node:assert/strict'; + +import { createServer } from '../../src/server.js'; + +// Minimal in-memory transformer stub. The server's contract for +// `transform` is: it calls `transformer.transform(text, options)` and +// returns the resolved value as JSON (or LiNo if requested). +class StubTransformer { + constructor() { + this.calls = []; + } + async transform(text, options) { + this.calls.push({ text, options }); + return { + original: text, + tokens: text.split(/\s+/).filter(Boolean), + sequence: ['Q35120', 'P31', 'Q5'], + formatted: 'Q35120 P31 Q5', + alternatives: [], + }; + } +} + +class StubWikidataClient { + async fetchEntity(id) { + return { id, labels: { en: { value: `stub-${id}` } } }; + } + async fetchProperty(id) { + return { id, labels: { en: { value: `stub-${id}` } } }; + } +} + +async function startServer() { + const transformer = new StubTransformer(); + const wikidataClient = new StubWikidataClient(); + const srv = createServer({ transformer, wikidataClient }); + await new Promise((resolve) => srv.listen(0, '127.0.0.1', resolve)); + const port = srv.address().port; + return { + srv, port, transformer, + url: (p) => `http://127.0.0.1:${port}${p}`, + close: () => new Promise((resolve) => srv.close(resolve)), + }; +} + +test('GET /healthz returns ok', async () => { + const ctx = await startServer(); + after(() => ctx.close()); + const res = await fetch(ctx.url('/healthz')); + assert.equal(res.status, 200); + const body = await res.json(); + assert.deepEqual(body, { ok: true }); +}); + +test('GET /version returns name and version', async () => { + const ctx = await startServer(); + after(() => ctx.close()); + const res = await fetch(ctx.url('/version')); + assert.equal(res.status, 200); + const body = await res.json(); + assert.equal(body.name, 'human-language'); + assert.match(body.version, /^\d+\.\d+\.\d+/); +}); + +test('POST /transform returns JSON by default', async () => { + const ctx = await startServer(); + after(() => ctx.close()); + const res = await fetch(ctx.url('/transform'), { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ text: 'Einstein' }), + }); + assert.equal(res.status, 200); + assert.match(res.headers.get('content-type') || '', /application\/json/); + const body = await res.json(); + assert.equal(body.original, 'Einstein'); + assert.deepEqual(body.tokens, ['Einstein']); +}); + +test('POST /transform returns LiNo when Accept: codec=lino', async () => { + const ctx = await startServer(); + after(() => ctx.close()); + const res = await fetch(ctx.url('/transform'), { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'accept': 'text/plain;codec=lino', + }, + body: JSON.stringify({ text: 'Einstein' }), + }); + assert.equal(res.status, 200); + assert.match(res.headers.get('content-type') || '', /codec=lino/); + const body = await res.text(); + assert.match(body, /^sequence:\n {2}\(\(Q35120\) \(P31\) \(Q5\)\)\n$/); +}); + +test('POST /transform without text returns 400', async () => { + const ctx = await startServer(); + after(() => ctx.close()); + const res = await fetch(ctx.url('/transform'), { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({}), + }); + assert.equal(res.status, 400); + const body = await res.json(); + assert.match(body.error, /body\.text is required/); +}); + +test('GET /entity/:id proxies to the wikidata client', async () => { + const ctx = await startServer(); + after(() => ctx.close()); + const res = await fetch(ctx.url('/entity/Q42')); + assert.equal(res.status, 200); + const body = await res.json(); + assert.equal(body.id, 'Q42'); + assert.equal(body.labels.en.value, 'stub-Q42'); +}); + +test('GET /property/:id proxies to the wikidata client', async () => { + const ctx = await startServer(); + after(() => ctx.close()); + const res = await fetch(ctx.url('/property/P31')); + assert.equal(res.status, 200); + const body = await res.json(); + assert.equal(body.id, 'P31'); +}); + +test('GET /entity/bogus returns 404', async () => { + const ctx = await startServer(); + after(() => ctx.close()); + const res = await fetch(ctx.url('/entity/bogus')); + assert.equal(res.status, 404); +}); + +test('GET /search without q returns 400', async () => { + const ctx = await startServer(); + after(() => ctx.close()); + const res = await fetch(ctx.url('/search')); + assert.equal(res.status, 400); +}); + +test('GET / (root) lists endpoints', async () => { + const ctx = await startServer(); + after(() => ctx.close()); + const res = await fetch(ctx.url('/')); + assert.equal(res.status, 200); + const body = await res.json(); + assert.ok(Array.isArray(body.endpoints)); + assert.ok(body.endpoints.includes('/transform')); +}); + +test('unknown method on /transform falls through to 404', async () => { + const ctx = await startServer(); + after(() => ctx.close()); + const res = await fetch(ctx.url('/transform')); + assert.equal(res.status, 404); +}); diff --git a/package-lock.json b/package-lock.json index 19c3915..3bab89f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,14 +1,25 @@ { "name": "human-language", - "version": "0.0.0", + "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "human-language", - "version": "0.0.0", + "version": "0.1.0", + "license": "MIT", + "dependencies": { + "lino-arguments": "^0.3.0", + "lino-objects-codec": "^0.4.0" + }, + "bin": { + "human-language": "js/src/cli.js" + }, "devDependencies": { "@playwright/test": "^1.49.1" + }, + "engines": { + "node": ">=20" } }, "node_modules/@playwright/test": { @@ -27,6 +38,77 @@ "node": ">=18" } }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -42,6 +124,83 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/getenv": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/getenv/-/getenv-2.0.0.tgz", + "integrity": "sha512-VilgtJj/ALgGY77fiLam5iD336eSWi96Q15JSAG1zi8NRBysm3LXKdGnHb4m5cuyxvOLQQKWpBZAT6ni4FI2iQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/links-notation": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/links-notation/-/links-notation-0.11.2.tgz", + "integrity": "sha512-VPyELWBXpaCCiNPVeZhMbG7RuvOQR51nhqELK+s/rbSzKYhSs+tyiSOdQ7z8I7Kh3PLABF3bZETtWSFwx3vFfg==", + "license": "Unlicense" + }, + "node_modules/lino-arguments": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/lino-arguments/-/lino-arguments-0.3.0.tgz", + "integrity": "sha512-46RbNaq0kpDxjyzBqhiCjPypHy9tuL0ITC9LgEolFH65PKtwl0/kRxryleG+5HnqS1JLnUUypZ2GoHWhLWH/PQ==", + "license": "Unlicense", + "dependencies": { + "getenv": "^2.0.0", + "links-notation": "^0.11.2", + "lino-env": "^0.2.6", + "yargs": "^17.7.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@dotenvx/dotenvx": "^1.0.0" + }, + "peerDependenciesMeta": { + "@dotenvx/dotenvx": { + "optional": true + } + } + }, + "node_modules/lino-env": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/lino-env/-/lino-env-0.2.8.tgz", + "integrity": "sha512-j9JcPo8LWCSlWCO7B1SSL3OlpRc/bGaR/OYnH22DTiANaMaLWan7ebRIo2Pwaw/apcIzdV+UD5EXamdapbn6kA==", + "license": "Unlicense", + "dependencies": { + "links-notation": "^0.11.2" + } + }, + "node_modules/lino-objects-codec": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/lino-objects-codec/-/lino-objects-codec-0.4.0.tgz", + "integrity": "sha512-LFSI8rrjyVp8raEsaoFNxnY9oS2SCz0UTYbeZXms+iHF1hGWuLk57kcKCk4KXnFf+/Z7kKz52oYYtII6JrpbBQ==", + "license": "Unlicense", + "dependencies": { + "links-notation": "^0.11.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/playwright": { "version": "1.59.1", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", @@ -73,6 +232,94 @@ "engines": { "node": ">=18" } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } } } } diff --git a/package.json b/package.json index 6b0a4ab..19db42b 100644 --- a/package.json +++ b/package.json @@ -1,18 +1,106 @@ { "name": "human-language", - "version": "0.0.0", - "private": true, + "version": "0.1.0", + "description": "Human Language: a Wikidata-backed library, CLI and HTTP microservice for transforming English text into sequences of Wikidata entities (Q) and properties (P), with alphabet/dictionary/ontology browsing and IPA support.", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/link-assistant/human-language.git" + }, + "homepage": "https://link-assistant.github.io/human-language/", + "bugs": { + "url": "https://github.com/link-assistant/human-language/issues" + }, + "author": "link-assistant", + "keywords": [ + "wikidata", + "linked-data", + "semantic-web", + "ipa", + "ontology", + "transformer", + "knowledge-graph", + "natural-language", + "human-language" + ], "type": "module", - "description": "Human Language: a Wikidata-backed exploration of alphabets, dictionaries, ontologies, entities, properties and English-to-Q/P transformation.", + "main": "./js/src/index.js", + "types": "./js/src/index.d.ts", + "bin": { + "human-language": "./js/src/cli.js" + }, + "exports": { + ".": { + "types": "./js/src/index.d.ts", + "import": "./js/src/index.js", + "default": "./js/src/index.js" + }, + "./server": { + "import": "./js/src/server.js", + "default": "./js/src/server.js" + }, + "./cli": { + "import": "./js/src/cli.js", + "default": "./js/src/cli.js" + }, + "./transform": { + "import": "./js/src/transformation/text-to-qp-transformer.js" + }, + "./transform/lino": { + "import": "./js/src/transformation/lino-format.js" + }, + "./wikidata": { + "import": "./js/src/wikidata-api.js" + }, + "./wikidata/browser": { + "import": "./js/src/wikidata-api-browser.js" + }, + "./cache": { + "import": "./js/src/unified-cache.js" + }, + "./cache/browser": { + "import": "./js/src/unified-cache-browser.js" + }, + "./settings": { + "import": "./js/src/settings.js" + }, + "./routing": { + "import": "./js/src/app/routing.js" + }, + "./ipa": { + "import": "./js/src/app/ipa.js" + }, + "./config": { + "import": "./js/src/config.js" + } + }, + "files": [ + "js/src/**/*.js", + "js/src/**/*.jsx", + "js/src/**/*.d.ts", + "README.md", + "LICENSE" + ], + "engines": { + "node": ">=20" + }, "scripts": { "serve": "node js/scripts/serve-static.mjs", + "start": "node js/src/server.js", + "cli": "node js/src/cli.js", + "test": "node js/scripts/run-unit-tests.mjs", "test:e2e": "playwright test --config js/tests/e2e/playwright.config.mjs", "test:e2e:local": "node js/scripts/run-e2e-local.mjs", "test:unit": "node js/scripts/run-unit-tests.mjs", "test:syntax": "node js/scripts/check-mjs-syntax.mjs", - "check:web-archive": "node js/scripts/check-web-archive.mjs" + "check:web-archive": "node js/scripts/check-web-archive.mjs", + "prepack": "node js/scripts/check-mjs-syntax.mjs" }, "devDependencies": { "@playwright/test": "^1.49.1" + }, + "dependencies": { + "lino-arguments": "^0.3.0", + "lino-objects-codec": "^0.4.0" } } diff --git a/rust/.gitignore b/rust/.gitignore new file mode 100644 index 0000000..58f151a --- /dev/null +++ b/rust/.gitignore @@ -0,0 +1,3 @@ +/target +**/*.rs.bk +Cargo.lock.bak diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 0000000..b61d96a --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,1639 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "ctor" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec09e802f5081de6157da9a75701d6c713d8dc3ba52571fd4bd25f412644e8a6" +dependencies = [ + "ctor-proc-macro", + "dtor", +] + +[[package]] +name = "ctor-proc-macro" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2931af7e13dc045d8e9d26afccc6fa115d64e115c9c84b1166288b46f6782c2" + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "dtor" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97cbdf2ad6846025e8e25df05171abfb30e3ababa12ee0a0e44b9bbe570633a8" +dependencies = [ + "dtor-proc-macro", +] + +[[package]] +name = "dtor-proc-macro" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7454e41ff9012c00d53cf7f475c5e3afa3b91b7c90568495495e8d9bf47a1055" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "human-language" +version = "0.1.0" +dependencies = [ + "clap", + "lino-arguments", + "lino-objects-codec", + "once_cell", + "percent-encoding", + "regex", + "reqwest", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "hyper" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "links-notation" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4c952b42a8c6ff6f849d7cafe3b1e13f1063a51bbb144bc6c62026ab327814c" +dependencies = [ + "nom", +] + +[[package]] +name = "lino-arguments" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be512a5c5eacea6ef5ec015fb0c7e1725c8e4cda1befd31606e203f281069968" +dependencies = [ + "clap", + "ctor", + "dotenvy", + "lino-env", + "serde", + "thiserror 1.0.69", +] + +[[package]] +name = "lino-env" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f453c53827aabe91a3d3856d61d14ae3867ab1a4344db22f9fa5396664c8d0e" + +[[package]] +name = "lino-objects-codec" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4db12a09a113afcfa4bd1ba32e76b5e4cb22a37b8d3f130b198f8c14dded0ad0" +dependencies = [ + "base64", + "links-notation", +] + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 0000000..b79ae10 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "human-language" +version = "0.1.0" +edition = "2021" +rust-version = "1.74" +description = "Transform English text into sequences of Wikidata Q/P identifiers, with shared pure-function helpers (tokenization, n-grams, IPA routing, hash-route parsing) used by the npm package, the Docker microservice, and the in-browser SPA." +license = "MIT" +repository = "https://github.com/link-assistant/human-language" +homepage = "https://github.com/link-assistant/human-language" +documentation = "https://docs.rs/human-language" +readme = "README.md" +keywords = ["wikidata", "nlp", "links-notation", "lino", "tokenizer"] +categories = ["text-processing", "parser-implementations", "command-line-utilities"] +authors = ["link-assistant"] + +[lib] +name = "human_language" +crate-type = ["cdylib", "rlib"] + +[[bin]] +name = "human-language" +path = "src/bin/cli.rs" + +[features] +default = [] +# Network-backed Wikidata client (reqwest + tokio). Kept opt-in so the core +# crate compiles without an async runtime when used only for pure helpers. +wikidata-client = ["reqwest", "tokio", "serde", "serde_json"] + +[dependencies] +clap = { version = "4.6.1", features = ["derive"] } +lino-arguments = "0.3.0" +lino-objects-codec = "0.2.1" +once_cell = "1" +percent-encoding = "2" +regex = "1" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"], optional = true } +serde = { version = "1", features = ["derive"], optional = true } +serde_json = { version = "1", optional = true } +tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"], optional = true } + +[dev-dependencies] +serde_json = "1" diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 0000000..ea9658a --- /dev/null +++ b/rust/README.md @@ -0,0 +1,48 @@ +# human-language (Rust) + +Rust crate that mirrors the pure-function helpers of the JavaScript package +[`human-language`](https://www.npmjs.com/package/human-language). It powers the +Rust CLI, future WASM builds for the SPA, and any downstream consumer that +needs to tokenize text, parse the SPA hash, or render a Q/P sequence as +[Links Notation](https://github.com/link-foundation/links-notation) +without spinning up a Node runtime. + +The crate is published to [crates.io](https://crates.io/crates/human-language) +in lockstep with the npm package: bumping `Cargo.toml` and `package.json` +together is part of the release checklist (`docs/case-studies/issue-37/timeline.md`). + +## What ships + +- `human_language::tokenize::{tokenize, generate_ngrams, is_stop_word, is_property_indicator}` +- `human_language::routing::{parse_hash, serialize_hash, ParsedHash, MODES, DEFAULT_MODE}` +- `human_language::settings::{quotes_for_language, flag_for_language, QuotePair}` +- `human_language::lino::{format_sequence_as_lino, SeqItem}` +- `human-language` binary with `tokenize`, `parse-hash`, `lino-sequence`, + `version`, `help` subcommands. + +## What's deferred + +The network-backed parts of the transformer (Wikidata API client, +search/disambiguation, file/IndexedDB cache) currently live only in +JavaScript. Tracked under R3 in +`../docs/case-studies/issue-37/solution-plans.md`. + +## Development + +```sh +# from the repo root +cd rust +cargo fmt --check +cargo clippy --all-targets -- -D warnings +cargo test +``` + +## Parity with JavaScript + +`tests/parity.rs` pins the Rust output against the JavaScript test suite +under `../js/tests/unit/`. When either side changes a contract, update both +in the same commit. + +## License + +MIT — see [`../LICENSE`](../LICENSE). diff --git a/rust/changelog.d/README.md b/rust/changelog.d/README.md new file mode 100644 index 0000000..ad43f5a --- /dev/null +++ b/rust/changelog.d/README.md @@ -0,0 +1,30 @@ +# Rust crate changesets + +Each pull request that touches `rust/` should drop a Markdown file here +describing the change. The release workflow (`.github/workflows/rust.yml` +— follow-up) merges these into `CHANGELOG.md` at release time. + +## Format + +```md +# patch | minor | major + +A one-line description of the user-facing change. + +## Why + +Optional rationale. +``` + +The first heading is the semver bump level. Empty files are ignored. + +## Examples + +`pr-XYZ.md`: + +```md +# patch + +`tokenize` now strips ASCII sentence punctuation `.,!?;:` to match +`js/src/transformation/text-to-qp-transformer.js`. +``` diff --git a/rust/rust-toolchain.toml b/rust/rust-toolchain.toml new file mode 100644 index 0000000..73cb934 --- /dev/null +++ b/rust/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "stable" +components = ["rustfmt", "clippy"] diff --git a/rust/rustfmt.toml b/rust/rustfmt.toml new file mode 100644 index 0000000..bdb5303 --- /dev/null +++ b/rust/rustfmt.toml @@ -0,0 +1,5 @@ +edition = "2021" +max_width = 100 +newline_style = "Unix" +use_field_init_shorthand = true +use_try_shorthand = true diff --git a/rust/scripts/check-cargo-version.mjs b/rust/scripts/check-cargo-version.mjs new file mode 100644 index 0000000..964bbdc --- /dev/null +++ b/rust/scripts/check-cargo-version.mjs @@ -0,0 +1,47 @@ +#!/usr/bin/env node +// Detects whether `rust/Cargo.toml`'s package version changed in the +// current commit relative to its parent. Writes `should_publish=…` and +// `version=…` to $GITHUB_OUTPUT so the release workflow can gate +// `cargo publish` on a real version bump. +// +// Exit status is always 0 — a missing previous version is treated as +// "no publish needed" rather than a CI failure. + +import { spawnSync } from 'node:child_process'; +import { readFileSync, appendFileSync } from 'node:fs'; + +function readCurrentVersion() { + const text = readFileSync('rust/Cargo.toml', 'utf8'); + const m = text.match(/^\s*\[package\][^[]*?^\s*version\s*=\s*"([^"]+)"/ms); + if (!m) throw new Error('Could not find [package].version in rust/Cargo.toml'); + return m[1]; +} + +function readPreviousVersion() { + const result = spawnSync('git', ['show', 'HEAD~1:rust/Cargo.toml'], { encoding: 'utf8' }); + if (result.status !== 0) return null; + const m = result.stdout.match(/^\s*\[package\][^[]*?^\s*version\s*=\s*"([^"]+)"/ms); + return m ? m[1] : null; +} + +function setOutput(key, value) { + const path = process.env.GITHUB_OUTPUT; + const line = `${key}=${value}\n`; + if (path) appendFileSync(path, line); + else process.stdout.write(line); +} + +const current = readCurrentVersion(); +const previous = readPreviousVersion(); + +setOutput('version', current); +if (previous && previous !== current) { + console.log(`Rust crate version: ${previous} -> ${current}; publishing.`); + setOutput('should_publish', 'true'); +} else if (!previous) { + console.log(`Rust crate version: ${current} (no previous Cargo.toml); skipping publish.`); + setOutput('should_publish', 'false'); +} else { + console.log(`Rust crate version unchanged at ${current}; skipping publish.`); + setOutput('should_publish', 'false'); +} diff --git a/rust/src/bin/cli.rs b/rust/src/bin/cli.rs new file mode 100644 index 0000000..847d9f1 --- /dev/null +++ b/rust/src/bin/cli.rs @@ -0,0 +1,108 @@ +//! `human-language` command-line entry point (Rust). +//! +//! Mirrors the JavaScript CLI in `js/src/cli.js` for the subset of commands +//! that don't need a live Wikidata client. Network-backed commands +//! (`transform`, `entity`, `property`, `search`) are gated behind the +//! `wikidata-client` feature and currently stub-out with a not-implemented +//! message; the case-study tracks the full port as a follow-up. + +use std::process::ExitCode; + +use human_language::{lino, parse_hash, serialize_hash, tokenize, VERSION}; +use lino_arguments::{Parser, Subcommand}; + +#[derive(Debug, Parser)] +#[command(name = "human-language", version, about = "Human Language CLI")] +struct Cli { + #[command(subcommand)] + command: Option, +} + +#[derive(Debug, Subcommand)] +enum Command { + /// Print tokens for the input text (one per line). + Tokenize { text: Vec }, + /// Parse a `#mode=...&...` SPA hash and print key/values. + ParseHash { hash: String }, + /// Render IDs as Links Notation. Use [A,B] for an ambiguous slot. + LinoSequence { ids: Vec }, + /// Print the version. + Version, + /// Print usage. + Help, +} + +const USAGE: &str = "\ +Usage: human-language [options] [args] + +Commands: + tokenize Print tokens for the input text (one per line). + parse-hash Parse a `#mode=…&…` SPA hash and print key/values. + lino-sequence Render IDs as Links Notation (use [A,B] for an + ambiguous slot — comma-separated, no spaces). + version Print the version. + help Print this message. + +Network-backed commands (`transform`, `entity`, `property`, `search`) are +provided by the JavaScript CLI and the Docker microservice. Build with +`--features wikidata-client` for the planned Rust port. +"; + +fn main() -> ExitCode { + let cli = Cli::parse(); + let Some(command) = cli.command else { + println!("{USAGE}"); + return ExitCode::SUCCESS; + }; + + match command { + Command::Help => { + println!("{USAGE}"); + ExitCode::SUCCESS + } + Command::Version => { + println!("human-language {VERSION}"); + ExitCode::SUCCESS + } + Command::Tokenize { text } => { + let text = text.join(" "); + if text.is_empty() { + eprintln!("error: `tokenize` requires a text argument"); + return ExitCode::from(2); + } + for tok in tokenize(&text) { + println!("{tok}"); + } + ExitCode::SUCCESS + } + Command::ParseHash { hash } => { + let parsed = parse_hash(&hash); + println!("mode={}", parsed.mode); + for (k, v) in &parsed.params { + println!("{k}={v}"); + } + println!("# canonical: {}", serialize_hash(&parsed)); + ExitCode::SUCCESS + } + Command::LinoSequence { ids } => { + if ids.is_empty() { + eprintln!("error: `lino-sequence` requires at least one id"); + return ExitCode::from(2); + } + let items: Vec = ids + .iter() + .map(|raw| { + if let Some(inner) = raw.strip_prefix('[').and_then(|s| s.strip_suffix(']')) { + lino::SeqItem::Ambiguous( + inner.split(',').map(|s| s.trim().to_string()).collect(), + ) + } else { + lino::SeqItem::Id(raw.clone()) + } + }) + .collect(); + print!("{}", lino::format_sequence_as_lino(&items)); + ExitCode::SUCCESS + } + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs new file mode 100644 index 0000000..5cf0d02 --- /dev/null +++ b/rust/src/lib.rs @@ -0,0 +1,31 @@ +//! Pure-function helpers shared by the npm package, the Docker microservice, +//! and the browser SPA. The contracts mirror the JavaScript modules under +//! `js/src/` so parity tests (`rust/tests/parity.rs`) can pin both sides. +//! +//! Modules: +//! - [`tokenize`]: English-text tokenizer + n-gram generator + stop-word and +//! property-indicator predicates. +//! - [`routing`]: `#mode=…&…` hash parser / serializer for the SPA. +//! - [`settings`]: Locale quote pairs + a (compact) flag map for the +//! language switcher. +//! - [`lino`]: Renders Q/P sequences into the +//! [Links Notation](https://github.com/link-foundation/links-notation) +//! form used by the API and the CLI. + +pub mod lino; +pub mod routing; +pub mod settings; +pub mod tokenize; + +pub use routing::{parse_hash, serialize_hash, ParsedHash, DEFAULT_MODE, MODES}; +pub use settings::{flag_for_language, quotes_for_language, QuotePair}; +pub use tokenize::{ + generate_ngrams, is_property_indicator, is_stop_word, tokenize, NGRAMS_DEFAULT_MAX, +}; + +/// Package name shipped on crates.io. +pub const NAME: &str = "human-language"; + +/// Package version. Matches `Cargo.toml`; bumped together with `package.json` +/// by the release pipeline. +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/rust/src/lino.rs b/rust/src/lino.rs new file mode 100644 index 0000000..62e508c --- /dev/null +++ b/rust/src/lino.rs @@ -0,0 +1,125 @@ +//! Render Q/P sequences as Links Notation. Mirrors +//! `js/src/transformation/lino-format.js`. + +use lino_objects_codec::{decode, encode, CodecError, LinoValue}; + +/// An item in the rendered sequence. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SeqItem { + /// A resolved atom, e.g. `Q42`. + Id(String), + /// An ambiguous slot. Renders as `[A or B or C]`. + Ambiguous(Vec), + /// A blank slot. Renders as `()`. + Empty, +} + +impl> From for SeqItem { + fn from(value: S) -> Self { + SeqItem::Id(value.as_ref().to_string()) + } +} + +const HEADER: &str = "sequence:"; + +/// Render a sequence in Links Notation. An empty sequence produces just +/// the bare header, matching the JS implementation. +pub fn format_sequence_as_lino(sequence: &[SeqItem]) -> String { + if sequence.is_empty() { + return format!("{HEADER}\n"); + } + let atoms: Vec = sequence + .iter() + .map(|item| match item { + SeqItem::Id(s) => format!("({s})"), + SeqItem::Ambiguous(ids) => { + let filtered: Vec<&str> = ids + .iter() + .map(String::as_str) + .filter(|s| !s.is_empty()) + .collect(); + format!("[{}]", filtered.join(" or ")) + } + SeqItem::Empty => "()".into(), + }) + .collect(); + format!("{HEADER}\n ({})\n", atoms.join(" ")) +} + +/// Encode string key/value configuration or state pairs with the upstream +/// `lino-objects-codec` crate. +pub fn encode_string_pairs_as_lino(pairs: I) -> String +where + I: IntoIterator, + K: Into, + V: Into, +{ + let value = LinoValue::object( + pairs + .into_iter() + .map(|(key, value)| (key.into(), LinoValue::String(value.into()))), + ); + encode(&value) +} + +/// Decode string key/value pairs produced by [`encode_string_pairs_as_lino`]. +pub fn decode_string_pairs_from_lino(notation: &str) -> Result, CodecError> { + let value = decode(notation)?; + let object = value + .as_object() + .ok_or_else(|| CodecError::DecodeError("expected object".into()))?; + + object + .iter() + .map(|(key, value)| { + value + .as_str() + .map(|value| (key.clone(), value.to_string())) + .ok_or_else(|| CodecError::DecodeError(format!("expected string value for {key}"))) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_renders_bare_header() { + assert_eq!(format_sequence_as_lino(&[]), "sequence:\n"); + } + + #[test] + fn strings_render_as_atoms() { + let seq: Vec = vec!["Q35120".into(), "P31".into(), "Q5".into()]; + assert_eq!( + format_sequence_as_lino(&seq), + "sequence:\n ((Q35120) (P31) (Q5))\n" + ); + } + + #[test] + fn ambiguous_uses_bracket_or_syntax() { + let seq = vec![ + SeqItem::Id("Q5".into()), + SeqItem::Ambiguous(vec!["Q42".into(), "Q1".into()]), + ]; + assert_eq!( + format_sequence_as_lino(&seq), + "sequence:\n ((Q5) [Q42 or Q1])\n" + ); + } + + #[test] + fn string_pairs_roundtrip_through_lino_objects_codec() { + let encoded = encode_string_pairs_as_lino([("host", "0.0.0.0"), ("port", "8080")]); + let decoded = decode_string_pairs_from_lino(&encoded).unwrap(); + assert_eq!( + decoded, + vec![ + ("host".to_string(), "0.0.0.0".to_string()), + ("port".to_string(), "8080".to_string()) + ] + ); + } +} diff --git a/rust/src/routing.rs b/rust/src/routing.rs new file mode 100644 index 0000000..8a6a29b --- /dev/null +++ b/rust/src/routing.rs @@ -0,0 +1,184 @@ +//! Hash-route parser / serializer. Mirrors `js/src/app/routing.js`. + +use percent_encoding::{percent_decode_str, utf8_percent_encode, AsciiSet, CONTROLS}; + +const FRAGMENT: &AsciiSet = &CONTROLS + .add(b' ') + .add(b'"') + .add(b'<') + .add(b'>') + .add(b'`') + .add(b'#') + .add(b'&') + .add(b'=') + .add(b'+'); + +/// All recognised SPA modes, in declaration order. +pub const MODES: &[&str] = &[ + "alphabet", + "dictionary", + "ontology", + "entity", + "property", + "transformer", +]; + +/// The mode the SPA falls back to when the hash is empty / unknown. +pub const DEFAULT_MODE: &str = "entity"; + +/// Parsed `#mode=…&…` form. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParsedHash { + pub mode: String, + pub params: Vec<(String, String)>, +} + +impl ParsedHash { + pub fn get(&self, key: &str) -> Option<&str> { + self.params + .iter() + .find(|(k, _)| k == key) + .map(|(_, v)| v.as_str()) + } +} + +fn is_q_id(s: &str) -> bool { + let mut chars = s.chars(); + matches!(chars.next(), Some('Q') | Some('q')) + && !s[1..].is_empty() + && s[1..].chars().all(|c| c.is_ascii_digit()) +} + +fn is_p_id(s: &str) -> bool { + let mut chars = s.chars(); + matches!(chars.next(), Some('P') | Some('p')) + && !s[1..].is_empty() + && s[1..].chars().all(|c| c.is_ascii_digit()) +} + +/// Parse a hash string (with or without the leading `#`) into a +/// [`ParsedHash`]. Handles the legacy bare-`Q\d+` / bare-`P\d+` shorthands. +pub fn parse_hash(hash: &str) -> ParsedHash { + let cleaned = hash.strip_prefix('#').unwrap_or(hash); + if cleaned.is_empty() { + return ParsedHash { + mode: DEFAULT_MODE.into(), + params: Vec::new(), + }; + } + if is_q_id(cleaned) { + return ParsedHash { + mode: "entity".into(), + params: vec![("id".into(), cleaned.to_ascii_uppercase())], + }; + } + if is_p_id(cleaned) { + return ParsedHash { + mode: "property".into(), + params: vec![("id".into(), cleaned.to_ascii_uppercase())], + }; + } + + let mut mode: Option = None; + let mut params: Vec<(String, String)> = Vec::new(); + for pair in cleaned.split('&') { + if pair.is_empty() { + continue; + } + let (raw_key, raw_value) = match pair.find('=') { + Some(idx) => (&pair[..idx], &pair[idx + 1..]), + None => (pair, ""), + }; + let key = percent_decode_str(raw_key).decode_utf8_lossy().into_owned(); + if key.is_empty() { + continue; + } + let value = percent_decode_str(&raw_value.replace('+', " ")) + .decode_utf8_lossy() + .into_owned(); + if key == "mode" { + mode = Some(value); + } else { + params.push((key, value)); + } + } + let mode = match mode { + Some(m) if MODES.iter().any(|x| *x == m) => m, + _ => DEFAULT_MODE.into(), + }; + ParsedHash { mode, params } +} + +/// Serialize `parsed` back into a hash string. Empty params are skipped. +/// The output always begins with `#`. +pub fn serialize_hash(parsed: &ParsedHash) -> String { + let mut out = String::from("#mode="); + out.push_str(&utf8_percent_encode(&parsed.mode, FRAGMENT).to_string()); + for (k, v) in &parsed.params { + if v.is_empty() { + continue; + } + out.push('&'); + out.push_str(&utf8_percent_encode(k, FRAGMENT).to_string()); + out.push('='); + out.push_str(&utf8_percent_encode(v, FRAGMENT).to_string()); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_empty_hash_defaults_to_entity() { + let got = parse_hash(""); + assert_eq!(got.mode, "entity"); + assert!(got.params.is_empty()); + } + + #[test] + fn parse_legacy_q_id() { + let got = parse_hash("#Q35120"); + assert_eq!(got.mode, "entity"); + assert_eq!(got.get("id"), Some("Q35120")); + } + + #[test] + fn parse_legacy_p_id_uppercases() { + let got = parse_hash("#p31"); + assert_eq!(got.mode, "property"); + assert_eq!(got.get("id"), Some("P31")); + } + + #[test] + fn parse_keyed_form() { + let got = parse_hash("#mode=dictionary&word=cat&lang=en"); + assert_eq!(got.mode, "dictionary"); + assert_eq!(got.get("word"), Some("cat")); + assert_eq!(got.get("lang"), Some("en")); + } + + #[test] + fn parse_unknown_mode_falls_back() { + let got = parse_hash("#mode=cats&id=Q1"); + assert_eq!(got.mode, "entity"); + assert_eq!(got.get("id"), Some("Q1")); + } + + #[test] + fn serialize_skips_empty_params() { + let parsed = ParsedHash { + mode: "entity".into(), + params: vec![("id".into(), "Q42".into()), ("note".into(), "".into())], + }; + assert_eq!(serialize_hash(&parsed), "#mode=entity&id=Q42"); + } + + #[test] + fn roundtrip_through_parse_then_serialize() { + let original = "#mode=dictionary&word=cat&lang=en"; + let serialized = serialize_hash(&parse_hash(original)); + assert_eq!(serialized, original); + } +} diff --git a/rust/src/settings.rs b/rust/src/settings.rs new file mode 100644 index 0000000..3bbf4ee --- /dev/null +++ b/rust/src/settings.rs @@ -0,0 +1,203 @@ +//! Locale-aware quotes + flag emoji per BCP-47 language tag. +//! Mirrors `js/src/settings.js`. The flag map is intentionally compact — +//! only the language tags actually used by the SPA are listed; the +//! JavaScript table is broader and is the source of truth for the browser +//! (this module is for crates.io / CLI consumers). + +/// Pair of opening / closing quote characters used by a locale. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct QuotePair { + pub open: &'static str, + pub close: &'static str, +} + +const QUOTES: &[(&str, QuotePair)] = &[ + ( + "en", + QuotePair { + open: "\u{201C}", + close: "\u{201D}", + }, + ), + ( + "de", + QuotePair { + open: "\u{201E}", + close: "\u{201C}", + }, + ), + ( + "fr", + QuotePair { + open: "\u{00AB}", + close: "\u{00BB}", + }, + ), + ( + "ru", + QuotePair { + open: "\u{00AB}", + close: "\u{00BB}", + }, + ), + ( + "pl", + QuotePair { + open: "\u{201E}", + close: "\u{201D}", + }, + ), + ( + "es", + QuotePair { + open: "\u{00AB}", + close: "\u{00BB}", + }, + ), + ( + "it", + QuotePair { + open: "\u{00AB}", + close: "\u{00BB}", + }, + ), + ( + "zh", + QuotePair { + open: "\u{201C}", + close: "\u{201D}", + }, + ), + ( + "ja", + QuotePair { + open: "\u{300C}", + close: "\u{300D}", + }, + ), + ( + "ko", + QuotePair { + open: "\u{201C}", + close: "\u{201D}", + }, + ), + ( + "uk", + QuotePair { + open: "\u{00AB}", + close: "\u{00BB}", + }, + ), + ( + "cs", + QuotePair { + open: "\u{201E}", + close: "\u{201C}", + }, + ), + ( + "sk", + QuotePair { + open: "\u{201E}", + close: "\u{201C}", + }, + ), +]; + +const DEFAULT_QUOTES: QuotePair = QuotePair { + open: "\"", + close: "\"", +}; + +/// Look up the quote pair for a language. Falls back through `xx-yy → xx` +/// before returning ASCII double quotes. +pub fn quotes_for_language(tag: &str) -> QuotePair { + let lowered = tag.to_lowercase(); + if let Some(pair) = QUOTES.iter().find(|(k, _)| *k == lowered).map(|(_, v)| *v) { + return pair; + } + if let Some(base) = lowered.split('-').next() { + if let Some(pair) = QUOTES.iter().find(|(k, _)| *k == base).map(|(_, v)| *v) { + return pair; + } + } + DEFAULT_QUOTES +} + +const FLAGS: &[(&str, &str)] = &[ + ("en", "\u{1F1EC}\u{1F1E7}"), + ("de", "\u{1F1E9}\u{1F1EA}"), + ("fr", "\u{1F1EB}\u{1F1F7}"), + ("es", "\u{1F1EA}\u{1F1F8}"), + ("it", "\u{1F1EE}\u{1F1F9}"), + ("pt", "\u{1F1F5}\u{1F1F9}"), + ("ru", "\u{1F1F7}\u{1F1FA}"), + ("nl", "\u{1F1F3}\u{1F1F1}"), + ("pl", "\u{1F1F5}\u{1F1F1}"), + ("zh", "\u{1F1E8}\u{1F1F3}"), + ("ja", "\u{1F1EF}\u{1F1F5}"), + ("ko", "\u{1F1F0}\u{1F1F7}"), + ("uk", "\u{1F1FA}\u{1F1E6}"), + ("ar", "\u{1F1F8}\u{1F1E6}"), + ("hi", "\u{1F1EE}\u{1F1F3}"), + ("he", "\u{1F1EE}\u{1F1F1}"), +]; + +const DEFAULT_FLAG: &str = "\u{1F310}"; + +/// Returns the flag emoji for a BCP-47 tag, falling back through `xx-yy → xx` +/// and finally to the globe emoji. +pub fn flag_for_language(tag: &str) -> &'static str { + let lowered = tag.to_lowercase(); + if let Some(flag) = FLAGS.iter().find(|(k, _)| *k == lowered).map(|(_, v)| *v) { + return flag; + } + if let Some(base) = lowered.split('-').next() { + if let Some(flag) = FLAGS.iter().find(|(k, _)| *k == base).map(|(_, v)| *v) { + return flag; + } + } + DEFAULT_FLAG +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn quotes_direct_lookup() { + assert_eq!(quotes_for_language("fr").open, "\u{00AB}"); + } + + #[test] + fn quotes_fallback_through_base_tag() { + assert_eq!(quotes_for_language("en-US").open, "\u{201C}"); + } + + #[test] + fn quotes_default_to_ascii() { + assert_eq!( + quotes_for_language("xx-YY"), + QuotePair { + open: "\"", + close: "\"" + } + ); + } + + #[test] + fn flags_direct_lookup() { + assert_eq!(flag_for_language("ja"), "\u{1F1EF}\u{1F1F5}"); + } + + #[test] + fn flags_fallback_through_base_tag() { + assert_eq!(flag_for_language("en-GB"), "\u{1F1EC}\u{1F1E7}"); + } + + #[test] + fn flags_default_to_globe() { + assert_eq!(flag_for_language("zxx"), "\u{1F310}"); + } +} diff --git a/rust/src/tokenize.rs b/rust/src/tokenize.rs new file mode 100644 index 0000000..0ddb1ee --- /dev/null +++ b/rust/src/tokenize.rs @@ -0,0 +1,149 @@ +//! Tokenization, n-gram generation, and the small word lists used by the +//! text-to-Q/P transformer. Mirrors `js/src/transformation/text-to-qp-transformer.js`. + +/// Default maximum n-gram length used by the transformer. +pub const NGRAMS_DEFAULT_MAX: usize = 3; + +/// Words that the JS transformer skips entirely. +pub const STOP_WORDS: &[&str] = &[ + "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "by", +]; + +/// Single-token or short-phrase indicators of a Wikidata *property*. +/// +/// Multi-word entries are matched after tokenization by checking joined +/// n-grams (see [`is_property_indicator`]). +pub const PROPERTY_INDICATORS: &[&str] = &[ + "is", + "was", + "are", + "were", + "has", + "have", + "had", + "born", + "died", + "located", + "created", + "founded", + "married", + "wrote", + "directed", + "invented", + "discovered", + "contains", + "belongs", + "relates", + "connects", + "instance of", + "part of", + "member of", + "capital of", + "owned by", + "child of", + "parent of", + "spouse of", + "sibling of", +]; + +/// Split `text` into lowercase-preserving tokens. +/// +/// Drops ASCII sentence punctuation (`.,!?;:`) and collapses any whitespace +/// run. Returns an empty `Vec` for an empty input. +pub fn tokenize(text: &str) -> Vec { + text.chars() + .map(|c| { + if matches!(c, '.' | ',' | '!' | '?' | ';' | ':') { + ' ' + } else { + c + } + }) + .collect::() + .split_whitespace() + .map(|s| s.to_string()) + .collect() +} + +/// Returns true if the (already lowercase-folded) word is in [`STOP_WORDS`]. +pub fn is_stop_word(word: &str) -> bool { + let lowered = word.to_lowercase(); + STOP_WORDS.iter().any(|w| *w == lowered) +} + +/// Returns true if the (already lowercase-folded) phrase appears in +/// [`PROPERTY_INDICATORS`]. Multi-word indicators are matched as a single +/// space-joined string. +pub fn is_property_indicator(phrase: &str) -> bool { + let lowered = phrase.to_lowercase(); + PROPERTY_INDICATORS.iter().any(|w| *w == lowered) +} + +/// Build every contiguous n-gram of size 1..=`max_size` from `tokens`, +/// grouped by size. Keys are the n-gram sizes; values are the join-by-space +/// strings (so callers can search Wikidata for the joined phrase). +/// +/// Equivalent to the JS `generateNgrams` with the same `maxNgramSize`. +pub fn generate_ngrams(tokens: &[String], max_size: usize) -> Vec<(usize, Vec)> { + let mut out: Vec<(usize, Vec)> = Vec::new(); + if tokens.is_empty() || max_size == 0 { + return out; + } + let max = max_size.min(tokens.len()); + for size in 1..=max { + let mut bucket = Vec::with_capacity(tokens.len().saturating_sub(size - 1)); + for window in tokens.windows(size) { + bucket.push(window.join(" ")); + } + out.push((size, bucket)); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tokenize_strips_punctuation_and_splits_whitespace() { + let got = tokenize("Hello, world! How are you?"); + assert_eq!(got, vec!["Hello", "world", "How", "are", "you"]); + } + + #[test] + fn tokenize_returns_empty_for_blank_input() { + assert!(tokenize("").is_empty()); + assert!(tokenize(" \t\n").is_empty()); + } + + #[test] + fn stop_words_are_case_insensitive() { + assert!(is_stop_word("the")); + assert!(is_stop_word("THE")); + assert!(!is_stop_word("Einstein")); + } + + #[test] + fn property_indicators_match_phrases() { + assert!(is_property_indicator("instance of")); + assert!(is_property_indicator("Instance Of")); + assert!(!is_property_indicator("of")); + } + + #[test] + fn ngrams_window_correctly() { + let tokens = vec!["a".to_string(), "b".to_string(), "c".to_string()]; + let got = generate_ngrams(&tokens, 3); + assert_eq!(got[0], (1, vec!["a".into(), "b".into(), "c".into()])); + assert_eq!(got[1], (2, vec!["a b".into(), "b c".into()])); + assert_eq!(got[2], (3, vec!["a b c".into()])); + } + + #[test] + fn ngrams_caps_at_token_length() { + let tokens = vec!["a".to_string(), "b".to_string()]; + let got = generate_ngrams(&tokens, 5); + assert_eq!(got.len(), 2); + assert_eq!(got[1], (2, vec!["a b".into()])); + } +} diff --git a/rust/tests/parity.rs b/rust/tests/parity.rs new file mode 100644 index 0000000..2340d2a --- /dev/null +++ b/rust/tests/parity.rs @@ -0,0 +1,113 @@ +//! Cross-language parity tests. The expected outputs here are kept in lockstep +//! with the JavaScript test suites under `js/tests/unit/`. +//! +//! When you change a contract, update both sides in the same commit. The PR +//! reviewer should be able to read this file as a spec for "what each +//! pure helper does, in both languages." + +use human_language::{ + flag_for_language, generate_ngrams, is_property_indicator, is_stop_word, lino, parse_hash, + quotes_for_language, serialize_hash, tokenize, ParsedHash, +}; + +#[test] +fn tokenize_matches_js_basic_case() { + // Mirrors js/tests/unit/* expectations: punctuation removed, whitespace + // collapsed, casing preserved. + let got = tokenize("Albert Einstein was born in Ulm."); + assert_eq!(got, vec!["Albert", "Einstein", "was", "born", "in", "Ulm"]); +} + +#[test] +fn stop_words_match_js_list() { + for w in ["the", "a", "an", "and", "or", "but", "of"] { + assert!(is_stop_word(w), "expected `{w}` to be a stop word"); + } + assert!(!is_stop_word("Einstein")); +} + +#[test] +fn property_indicators_include_multiword() { + assert!(is_property_indicator("instance of")); + assert!(is_property_indicator("born")); + assert!(!is_property_indicator("the")); +} + +#[test] +fn ngrams_match_js_shape() { + let tokens = vec![ + "Albert".into(), + "Einstein".into(), + "was".into(), + "born".into(), + ]; + let got = generate_ngrams(&tokens, 3); + assert_eq!(got.len(), 3); + assert_eq!(got[0].1.len(), 4); // unigrams + assert_eq!( + got[1].1, + vec!["Albert Einstein", "Einstein was", "was born"] + ); + assert_eq!(got[2].1, vec!["Albert Einstein was", "Einstein was born"]); +} + +#[test] +fn parse_hash_legacy_forms() { + // Pinned against js/tests/unit/routing.test.mjs expectations. + let parsed = parse_hash("#Q35120"); + assert_eq!(parsed.mode, "entity"); + assert_eq!(parsed.get("id"), Some("Q35120")); + + let parsed = parse_hash("#P31"); + assert_eq!(parsed.mode, "property"); + assert_eq!(parsed.get("id"), Some("P31")); +} + +#[test] +fn parse_hash_keyed_form() { + let parsed = parse_hash("#mode=dictionary&word=cat&lang=en"); + assert_eq!(parsed.mode, "dictionary"); + assert_eq!(parsed.get("word"), Some("cat")); + assert_eq!(parsed.get("lang"), Some("en")); +} + +#[test] +fn serialize_hash_skips_empty_values() { + let parsed = ParsedHash { + mode: "entity".into(), + params: vec![("id".into(), "Q42".into()), ("note".into(), "".into())], + }; + assert_eq!(serialize_hash(&parsed), "#mode=entity&id=Q42"); +} + +#[test] +fn quotes_for_language_matches_js_table() { + assert_eq!(quotes_for_language("fr").open, "\u{00AB}"); + assert_eq!(quotes_for_language("de").open, "\u{201E}"); + assert_eq!(quotes_for_language("en-US").open, "\u{201C}"); + assert_eq!(quotes_for_language("xx").open, "\""); +} + +#[test] +fn flag_for_language_falls_back_to_globe() { + assert_eq!(flag_for_language("en"), "\u{1F1EC}\u{1F1E7}"); + assert_eq!(flag_for_language("xx"), "\u{1F310}"); +} + +#[test] +fn lino_format_matches_js_output() { + let seq: Vec = vec!["Q35120".into(), "P31".into(), "Q5".into()]; + assert_eq!( + lino::format_sequence_as_lino(&seq), + "sequence:\n ((Q35120) (P31) (Q5))\n" + ); + + let seq = vec![ + lino::SeqItem::Id("Q5".into()), + lino::SeqItem::Ambiguous(vec!["Q42".into(), "Q1".into()]), + ]; + assert_eq!( + lino::format_sequence_as_lino(&seq), + "sequence:\n ((Q5) [Q42 or Q1])\n" + ); +}