From e755d72ef16e276e54ce0b8ccca68818243b4418 Mon Sep 17 00:00:00 2001 From: Ne0x <24698438+ne0x86@users.noreply.github.com> Date: Tue, 27 Oct 2026 13:05:10 +0100 Subject: [PATCH 1/3] Suggest models in the model fields, give each tier its own default effort and say plainly what a project's options do - The Model field of every tier on the Accounts page, and the model fields of the form that adds an API key, point to a datalist, so the owner picks a name or types any other. Claude Code and Codex suggest the new "models" of data/tiers.json (the aliases haiku, sonnet and opus; the Codex models that its own catalog lists for choosing). An account on an API key suggests what models.dev lists for its provider, read by the web UI through the same fetch as its other lookups: at most once a day, waiting 5 seconds, asking for gzip with a User-Agent of its own (models.dev answers 403 without one), a failure kept 10 minutes, at most 1000 models for each provider, and only the models that call tools, have a price and are not deprecated. A local model server gets none. - The default efforts are medium for fast, high for balanced and xhigh for deep, for Claude Code and for Codex. README and DESIGN say why they differ and where the suggestions come from. - A project's language says what it is ("Language of its pull requests and issues", default "Same as the account", with a hint), as the account's own field does. The hints and the help of Tests too, Dependencies too, Merge after my approval and Watch after the merge say what each adds, and that the watch looks only at the checks of the merged commit and at the health address: logs, Loki and the error tracker feed the audits. - The two selects of a project's settings get a row each, so that a closed select does not cut its text short. Co-Authored-By: Claude Sonnet 5.5 --- CONTRIBUTING.md | 12 +- DESIGN.md | 31 +- README.md | 15 +- src/lastcall/catalog.py | 89 +++++ src/lastcall/data/tiers.json | 13 +- src/lastcall/engines/base.py | 1 + src/lastcall/engines/opencode.py | 12 +- src/lastcall/locales/en.json | 28 +- src/lastcall/locales/es.json | 26 +- src/lastcall/tiers.py | 16 +- src/lastcall/web/__init__.py | 93 ++++- src/lastcall/web/static/style.css | 1 + src/lastcall/web/templates/accounts.html | 16 +- src/lastcall/web/templates/projects.html | 3 +- tests/test_catalog.py | 206 +++++++++++ tests/test_docs.py | 64 ++++ tests/test_i18n.py | 48 +++ tests/test_keyed.py | 6 +- tests/test_recipes.py | 2 +- tests/test_tiers.py | 99 ++++++ tests/test_web.py | 430 ++++++++++++++++++++++- 21 files changed, 1151 insertions(+), 60 deletions(-) create mode 100644 src/lastcall/catalog.py create mode 100644 tests/test_catalog.py create mode 100644 tests/test_tiers.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 60b30e2..81a1ffc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,9 +56,10 @@ engine is one module and a few small changes: - the domains its CLI needs (`egress`); - where the CLI keeps its login (`home`), the variable that points it there (`home_env`), and its own login command. An engine that runs on API keys or on a local model server has no login: it implements `Keyed` - instead, its `providers` (each with where to create a key and where its models are listed) and `with_key`, - which returns the engine on an account's key. The key reaches the step only as a file in `/ctx` - (`Invocation.files`), never on the command line or in the environment; + instead, its `providers` (each with where to create a key, where its models are listed and, when models.dev + has them, its id there, so that the Accounts page suggests its models) and `with_key`, which returns the + engine on an account's key. The key reaches the step only as a file in `/ctx` (`Invocation.files`), never on + the command line or in the environment; - how a step becomes a command line (`invocation`), and how the output becomes events and a result (`parse` and `result`). 2. **If the CLI can read the account's quota without calling the model,** add `quota_invocation` and @@ -66,8 +67,9 @@ engine is one module and a few small changes: engine is `Keyed`: then lastcall's ledger reads what its steps cost, so `result` must report `cost_usd`. A window that limits only some models names the model in `Window.model`. 3. **Register it** in `src/lastcall/engines/__init__.py`. -4. **Pick its models** for each tier (fast, balanced, deep) in `src/lastcall/data/tiers.json`. An engine on API - keys may have none there: each of its accounts names its models. +4. **Pick its models** for each tier (fast, balanced, deep) in `src/lastcall/data/tiers.json`, and the names that + its CLI takes under `models`, which the Accounts page suggests. An engine on API keys may have none there: + each of its accounts names its models. 5. **Install its CLI**, at a pinned version, in `runner/Dockerfile`, and add its login to `runner/login.sh`, or, for an engine with no login, a branch that says so. A test checks that every engine has one there. 6. **Regenerate `egress/filter`.** The command is in `tests/test_egress.py`. diff --git a/DESIGN.md b/DESIGN.md index 2a2e7cd..ce360e6 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -272,12 +272,12 @@ accounts: - id: claude-max engine: claude-code login: cli # official login inside the account's credential volume - tiers: {fast: haiku/medium, balanced: sonnet/high, deep: opus/high} + tiers: {fast: haiku/medium, balanced: sonnet/high, deep: opus/xhigh} spend: {policy: before-reset, reserve_pct: 10} - id: chatgpt-plus engine: codex login: cli - tiers: {fast: gpt-6-luna/high, balanced: gpt-6.1-sol/high, deep: gpt-6.1-sol/high} + tiers: {fast: gpt-6-luna/medium, balanced: gpt-6.1-sol/high, deep: gpt-6.1-sol/xhigh} spend: {policy: before-reset, reserve_pct: 10} - id: openrouter # v0.5 engine: opencode @@ -301,6 +301,33 @@ goes back to the default. The data file also lists the effort levels that each e `--effort` (low to max), Codex's `model_reasoning_effort` (minimal to max) and OpenCode's `--variant`, whose levels depend on the model. More effort finds more, and spends more of the quota or the budget. +The default efforts differ by tier because the work does: `fast` is only asked for by a repository's own reviewers, +so it takes `medium`; `balanced` runs the reviewers, at `high`; and `deep` runs the judge, the implementers and the +improvement plans, where quality matters most and lastcall spends quota that would expire anyway, at `xhigh`. Both +engines' effort lists include `xhigh`. + +**Suggested models.** The model fields of that page suggest names with a ``: the owner picks one or types +any other, since the catalogs change every month. Where the names come from depends on the account: + +- An account on a CLI login gets the `models` of its engine in the data file, the names that its CLI takes: Claude + Code's aliases `haiku`, `sonnet` and `opus`, which mean the newest model of their family on the Anthropic API, + and the models that Codex's own catalog lists for choosing and has not replaced with a newer one + (`codex debug models --bundled`). `fable` is left out on purpose: `claude -p` can bill a Fable request to usage + credits without asking, and lastcall never spends paid usage. +- An account on an API key (OpenCode on OpenRouter, Anthropic or OpenAI) gets the models of its provider from + models.dev's public catalog, `https://models.dev/api.json`, where OpenCode reads its prices too. The id of the + provider there is `Provider.catalog`. The web UI asks through the same `fetch` as its other lookups, with a + `User-Agent` of its own (models.dev answers 403 to the default one) and gzip (the catalog weighs 6 MB, or 0.6 MB + packed), and waits 5 seconds at most. One request answers for every provider, and the answer stays in memory for + a day, since the Accounts page asks on every load. A failure is kept for 10 minutes: meanwhile the fields suggest + nothing and the page works as before. Of each provider it keeps at most 1000 models, the ones that a step can + run on: they call tools, have a price (the budget needs one) and are not deprecated, and their ids are ones that + an account may name. +- The form that adds an API key cannot follow the provider that it has chosen, since the page runs no script: its + model fields suggest the models of every provider together (OpenRouter writes the maker first, as in + `qwen/qwen3-coder`). +- A local model server (Ollama) gets no suggestions: the web UI cannot reach it. + **Several accounts on one project.** Different providers find different problems (section 16). A review step can therefore fan out across accounts, and one judge merges the results. diff --git a/README.md b/README.md index 09c84a3..a5c6095 100644 --- a/README.md +++ b/README.md @@ -113,9 +113,13 @@ spending limit at the provider too. **Models and effort.** Every step asks for a tier: the reviewers for *balanced*; the judge, the implementers and the improvement plans for *deep*; your repository's own reviewers for the one they name. Each engine has a default -model and effort per tier, such as opus at `high` effort for Claude Code's *deep*. On the Accounts page, under +model and effort per tier, such as opus at `xhigh` effort for Claude Code's *deep*. The efforts differ because the +work does: *deep* runs at `xhigh` because quality matters most there and the quota would expire anyway, *balanced* at +`high`, and *fast*, which only your repository's own reviewers ask for, at `medium`. On the Accounts page, under *Models and effort*, or with `lastcall account tier`, you can change both for one account, subscriptions included. -More effort finds more, and spends more of the quota. +The Model field suggests names and takes any other you type: the ones that the engine's CLI takes or, for an API key, +the models that [models.dev](https://models.dev) lists for its provider. More effort finds more, and spends more of +the quota. The rest of lastcall only knows the engine contract. Adding another tool is one module and a few small changes: [CONTRIBUTING.md](CONTRIBUTING.md#adding-an-engine) lists them. @@ -387,7 +391,9 @@ in lastcall only, never in `.lastcall.yml`, so that nobody can turn them on from later run, on the current code. - **Watch after the merge.** For a day after a merge, lastcall looks at the checks of the merged commit and, if you give one, at a health address every 10 minutes. When the checks fail where they passed before, or the address - fails 3 times in a row, it opens a pull request that reverts the change, pauses the project and tells you. + fails 3 times in a row, it opens a pull request that reverts the change, pauses the project and tells you. Nothing + else takes part in the watch: container logs, Loki and your error tracker feed the audits, which use them to find + the causes of errors. Both need the GitHub App to read checks and commit statuses, which Apps created by lastcall 0.7.2 or later have; the Projects page says what to do for an older App. @@ -457,7 +463,8 @@ With a local model on Ollama, it goes to no model provider. The names and versions of your dependencies go to [osv.dev](https://osv.dev) in every audit, and to [deps.dev](https://deps.dev) for upgrades. The agents' steps reach nothing beyond the allowlist in [egress/filter](egress/filter), and errors from production are masked before they reach a model. lastcall itself -sends no telemetry. +sends no telemetry. Opening the Accounts page makes it ask [models.dev](https://models.dev), about once a day, for +the names of the models that an API key's provider offers, in a request that carries nothing of yours. ### What if a file in my repository tricks the agent? diff --git a/src/lastcall/catalog.py b/src/lastcall/catalog.py new file mode 100644 index 0000000..ce0eb5a --- /dev/null +++ b/src/lastcall/catalog.py @@ -0,0 +1,89 @@ +"""models.dev's public catalog of models (https://models.dev/api.json), where OpenCode reads what each model costs. +The web UI reads from it which models an API key's provider offers, to suggest them in the model fields +(DESIGN.md, section 6). + +The catalog is one JSON object, `{provider id: {"models": {model id: {...}}}}`. Of each model lastcall looks at three +things: `tool_call`, since a step runs on tools; `cost`, since the monthly budget counts what each step costs, so a +model with no price cannot run; and `status`, since a deprecated model is not worth suggesting. It suggests only the +ids that an account's model field takes (`accounts.MODEL`). Anything else in the catalog is ignored, and an answer +that does not look like this counts as no models. + +The answer weighs 6 MB, or 0.6 MB gzipped, so the request asks for gzip. The reading is bounded: neither the answer +nor what it unpacks to may pass `MAX_BYTES`, and a provider keeps `MAX_MODELS` models at most. +""" + +from __future__ import annotations + +import json +import urllib.request +import zlib +from collections.abc import Iterable + +from lastcall import __version__ +from lastcall.accounts import MODEL +from lastcall.github import Fetch + +URL = "https://models.dev/api.json" +AGENT = f"lastcall/{__version__} (+https://github.com/ne0x86/lastcall)" # without a User-Agent, models.dev says 403 +MAX_BYTES = 32 * 1024 * 1024 # the most that the answer may weigh, packed or unpacked +MAX_MODELS = 1000 # the most that one provider keeps: OpenRouter lists the most, about 300 + + +def request(timeout: float) -> urllib.request.Request: + req = urllib.request.Request(URL) + req.add_header("User-Agent", AGENT) + req.add_header("Accept", "application/json") + req.add_header("Accept-Encoding", "gzip") + req.timeout = timeout # read by github.urlopen; a fake fetch ignores it + return req + + +def models(fetch: Fetch, providers: Iterable[str], timeout: float) -> dict[str, tuple[str, ...]]: + """The models that models.dev lists for each of `providers`, sorted, from one request to `fetch`: the ones that a + step can run on, `MAX_MODELS` at most. A provider that the catalog does not list gets none. Raises what `fetch` + raises, or ValueError, or zlib.error, when the answer cannot be read.""" + status, headers, raw = fetch(request(timeout)) + if status != 200: + raise ValueError(f"models.dev answered {status}") + if len(raw) > MAX_BYTES: + raise ValueError("the answer of models.dev is too big") + if any(name.lower() == "content-encoding" and value.strip().lower() == "gzip" for name, value in headers.items()): + raw = gunzip(raw) + catalog = json.loads(raw) + if not isinstance(catalog, dict): + raise ValueError("models.dev did not answer with its catalog") + return {name: suggestions(catalog.get(name)) for name in providers} + + +def gunzip(packed: bytes) -> bytes: + """What a gzipped answer unpacks to. A few KB can unpack to gigabytes, so more than `MAX_BYTES` is refused.""" + inflater = zlib.decompressobj(wbits=zlib.MAX_WBITS | 16) + unpacked = inflater.decompress(packed, MAX_BYTES + 1) + if len(unpacked) > MAX_BYTES: + raise ValueError("the answer of models.dev is too big") + return unpacked + + +def suggestions(provider: object) -> tuple[str, ...]: + """The ids of the models in a provider's entry of the catalog that a step can run on, in alphabetical order.""" + listed = provider.get("models") if isinstance(provider, dict) else None + if not isinstance(listed, dict): + return () + found = [name for name, model in listed.items() if isinstance(name, str) and usable(name, model)] + return tuple(sorted(found, key=lambda name: (name.casefold(), name))[:MAX_MODELS]) + + +def usable(name: str, model: object) -> bool: + """Whether a step can run on this model and an account may name it: it calls tools, it has a price (free is a + price) and it is not deprecated.""" + if not isinstance(model, dict) or not MODEL.fullmatch(name): + return False + return model.get("tool_call") is True and model.get("status") != "deprecated" and priced(model.get("cost")) + + +def priced(cost: object) -> bool: + """Whether a cost entry of the catalog has the price of what goes in and of what comes out.""" + if not isinstance(cost, dict): + return False + prices = [cost.get("input"), cost.get("output")] + return all(isinstance(price, int | float) and not isinstance(price, bool) for price in prices) diff --git a/src/lastcall/data/tiers.json b/src/lastcall/data/tiers.json index 97873e6..4ed4d6b 100644 --- a/src/lastcall/data/tiers.json +++ b/src/lastcall/data/tiers.json @@ -1,14 +1,19 @@ { - "_about": "Default model and effort for each tier, per engine (DESIGN.md, section 6), and the effort levels that each engine's CLI takes. Catalogs change every month: update this file, not the code. An account can override any tier's model and effort.", + "_about": "Default model and effort for each tier, per engine (DESIGN.md, section 6), the model names that each engine's CLI takes and the effort levels that it takes. Catalogs change every month: update this file, not the code. An account can override any tier's model and effort. The effort follows the work: fast is only asked for by a repository's own reviewers, balanced runs the reviewers and deep runs the judge, the implementers and the improvement plans, where quality matters most.", "claude-code": { "fast": ["haiku", "medium"], "balanced": ["sonnet", "high"], - "deep": ["opus", "high"] + "deep": ["opus", "xhigh"] }, "codex": { - "fast": ["gpt-6-luna", "high"], + "fast": ["gpt-6-luna", "medium"], "balanced": ["gpt-6.1-sol", "high"], - "deep": ["gpt-6.1-sol", "high"] + "deep": ["gpt-6.1-sol", "xhigh"] + }, + "models": { + "_about": "Names that each engine's CLI takes: the Accounts page suggests them for the model of a tier, and an owner can type any other. Claude Code: the aliases, which mean the newest model of their family on the Anthropic API. `fable` is left out on purpose, since `claude -p` can bill a Fable request to usage credits without asking. Codex: the models that its own catalog lists for choosing (`codex debug models --bundled`) and has not replaced with a newer one, in the order of its picker. An API key's models come from models.dev instead, and a local model server's are the owner's.", + "claude-code": ["haiku", "sonnet", "opus"], + "codex": ["gpt-6.1-sol", "gpt-6-astra", "gpt-6-sol", "gpt-6-luna"] }, "efforts": { "_about": "Lowest first. Claude Code's --effort; Codex's model_reasoning_effort; OpenCode's --variant, whose levels depend on the model.", diff --git a/src/lastcall/engines/base.py b/src/lastcall/engines/base.py index 1b0820a..87030cd 100644 --- a/src/lastcall/engines/base.py +++ b/src/lastcall/engines/base.py @@ -102,6 +102,7 @@ class Provider: title: str # as people know it: "OpenRouter" keys: str # where the owner creates an API key; "" for a local model server, which needs none models: str # where the owner finds the names of its models + catalog: str = "" # its id in models.dev, whose catalog lists its models; "" when there is none to ask @runtime_checkable diff --git a/src/lastcall/engines/opencode.py b/src/lastcall/engines/opencode.py index 037c283..386d946 100644 --- a/src/lastcall/engines/opencode.py +++ b/src/lastcall/engines/opencode.py @@ -37,19 +37,27 @@ KEY_FILE = f"{CONTEXT_DIR}/key" LOCAL = "ollama" # local models: an OpenAI-compatible server, by default the compose service `ollama` LOCAL_URL = "http://ollama:11434/v1" -PROVIDERS = ( - Provider("openrouter", "OpenRouter", "https://openrouter.ai/settings/keys", "https://openrouter.ai/models"), +PROVIDERS = ( # the models.dev id of each API is its name; a local model server has no catalog to ask + Provider( + "openrouter", + "OpenRouter", + "https://openrouter.ai/settings/keys", + "https://openrouter.ai/models", + "openrouter", + ), Provider( "anthropic", "Anthropic API", "https://platform.claude.com/settings/keys", "https://platform.claude.com/docs/en/models/overview", + "anthropic", ), Provider( "openai", "OpenAI API", "https://platform.openai.com/api-keys", "https://developers.openai.com/api/docs/models", + "openai", ), Provider(LOCAL, "Ollama", "", "https://ollama.com/library"), ) diff --git a/src/lastcall/locales/en.json b/src/lastcall/locales/en.json index 3ef18d0..47bce12 100644 --- a/src/lastcall/locales/en.json +++ b/src/lastcall/locales/en.json @@ -293,8 +293,9 @@ "web.project.weight.high": "High (×2)", "web.project.weight.top": "Highest (×3)", "web.project.weight.custom": "×{weight}", - "web.project.language": "Writes in", - "web.project.language.account": "The account's", + "web.project.language": "Language of its pull requests and issues", + "web.project.language.account": "Same as the account", + "web.project.language_hint": "“Same as the account” uses the language of whichever account works on it, set on the Accounts page.", "web.project.merge_on_approve": "Merge after my approval", "web.project.watch": "Watch after the merge", "web.project.tests": "Tests too", @@ -426,7 +427,7 @@ "web.accounts.id": "Name", "web.accounts.id_hint": "Lowercase letters, digits and dashes, such as claude-max.", "web.accounts.engine": "Subscription", - "web.accounts.language_label": "Writes in", + "web.accounts.language_label": "Language of its pull requests and issues", "web.accounts.reserve_label": "Reserve for you, % of every week", "web.accounts.enabled": "lastcall may spend it", "web.accounts.submit": "Add", @@ -459,6 +460,7 @@ "web.accounts.model_hint": "The model of every step, as the provider's list names it. lastcall counts what each step costs, so the model needs a price in that list.", "web.accounts.review_label": "Reviewers' model (optional)", "web.accounts.review_hint": "A cheaper model for the reviewers. Empty: the model above.", + "web.accounts.model_suggest": "The field suggests what models.dev lists for all the providers, and OpenRouter writes the maker first, as in qwen/qwen3-coder. Pick one from the provider you chose, or type any other.", "web.accounts.budget_label": "Monthly budget, USD", "web.accounts.budget_hint": "What its steps may cost each calendar month (UTC), whatever its policy. A step that is already running may go a little over it: keep the provider's own limit too.", "web.accounts.policy_label": "When it spends", @@ -617,6 +619,8 @@ "web.tiers.keyed_hint": "OpenCode passes the effort as the model's variant: choose one that your model offers, or keep Default.", "web.tiers.save": "Save models and effort", "web.tiers.need_models": "Give a model for each level: an account on an API key or a local model has no default.", + "web.tiers.suggest.cli": "The Model field suggests the names that {tool} takes. Pick one, or type any other.", + "web.tiers.suggest.catalog": "The Model field suggests what models.dev lists for {provider}: models that use tools and have a price. Pick one, or type any other.", "web.time.now": "just now", "web.time.ago": "{span} ago", @@ -681,7 +685,7 @@ "web.help.common.terminal.title": "Where to type it", "web.help.accounts.check.title": "Reading the quota", "web.help.settings.loki_sources.title": "Sources", - "web.help.projects.turns.title": "Tests and upgrades in turn", + "web.help.projects.turns.title": "Tests and dependencies in turn", "web.help.projects.tickets.title": "Labelled issues", "web.help.projects.score.title": "Points", "web.help.projects.size.title": "Tokens of code", @@ -717,12 +721,12 @@ "web.help.dashboard.status": "Status: stopped early means that before a step the account had reached your reserve or its monthly budget, had paid usage on, or had no quota reading, so the run ended there. Interrupted means lastcall itself stopped while the run was going, for example at a restart. Failed means an error stopped it.", "web.help.projects.active": "Active projects take part in bursts. Even then, a burst skips a project while it has 2 lastcall pull requests waiting for you, or when the same account audited it in the last 7 days, unless you ticked ideas or labelled issues for it. The run buttons work even when the project is inactive. Click the state at the top of a card to turn bursts on or off; the “Active” box in Settings does the same.", "web.help.projects.weight": "When an account has quota to spend, lastcall gives each active project points (never audited by that account, days and new commits since its last audit, pending findings, errors in production, what you asked for) and starts with the highest. Priority multiplies those points: High (×2) doubles them, Low (×0.5) halves them. It does not make a run longer.", - "web.help.projects.language": "The language of this project's pull requests, issues and reports. With “The account's”, lastcall writes in the language of whichever account does the run, set on the Accounts page. It does not change these pages or your notifications.", + "web.help.projects.language": "The language of this project's pull requests, issues and reports. With “Same as the account”, lastcall writes in the language of whichever account does the run, set on the Accounts page. It does not change these pages or your notifications.", "web.help.projects.max_prs": "The most pull requests one run opens on this project, whatever it does: an audit, tests, dependency upgrades, the ideas of a plan or labelled issues. What does not fit waits for a later run.", - "web.help.projects.merge_on_approve": "On GitHub, approving a pull request does not merge it. With this on, lastcall merges its own pull requests (never a revert) once someone who can write to the repository approves the latest commit, nobody with write access asks for changes, and no check fails or is still running. If one no longer merges cleanly, lastcall closes it and a later run makes the change again on the current code.", - "web.help.projects.watch": "For a day after one of lastcall's pull requests is merged, lastcall watches the checks of the merged commit and the health address below. If the checks fail where they passed before, or the address fails 3 times in a row, it opens a pull request that reverts the change, pauses the project (unticks “Active”) and tells you. It never merges the revert: you decide.", + "web.help.projects.merge_on_approve": "On GitHub, approving a pull request does not merge it. With this on, lastcall merges its own pull requests (never a revert) at its next look at GitHub, which is every 10 minutes: once someone who can write to the repository has approved the latest commit, nobody with write access asks for changes, and no check fails or is still running. If GitHub does not let it merge yet, for example because a rule of a protected branch is not met, lastcall waits. If one no longer merges cleanly, lastcall closes it and a later run makes the change again on the current code. Without this, you merge them yourself.", + "web.help.projects.watch": "For a day after one of lastcall's pull requests is merged, lastcall watches two things only: the checks of the merged commit and, if you give one, the health address below. If the checks fail where they passed before, or the address fails 3 times in a row, it opens a pull request that reverts the change, pauses the project (unticks “Active”) and tells you. It never merges the revert: you decide. Container logs, Loki and your error tracker take no part in the watch: they feed the audits, which use them to find the causes of errors.", "web.help.projects.health_url": "Used only while “Watch after the merge” is on and a merge is being watched. lastcall asks it every 10 minutes; any answer with a status from 200 to 399 within 10 seconds counts as working.", - "web.help.projects.turns": "Bursts on this project take turns: an audit, then tests, then dependency upgrades (each only if ticked), then an audit again. Tests too adds tests for behavior no test checks. Dependencies too upgrades what needs more than a new version number: known vulnerabilities, deprecated packages, major versions behind. Ideas you ticked in a plan and issues labelled lastcall always go first.", + "web.help.projects.turns": "The ideas you ticked in a plan and the issues labelled lastcall always go first. After them, a burst on this project audits it. Tick “Tests too”, “Dependencies too” or both, and the bursts take turns: an audit, then a run that writes tests, then one that upgrades dependencies (each only if ticked), then an audit again. A tests run opens pull requests that only add tests, for behavior no test checks. A dependencies run opens pull requests that upgrade what needs more than a new version number (a known vulnerability, a deprecated package, a major version behind, a runtime past its end of life) and fix the code that the new version breaks; plain minor and patch upgrades are left to bots such as Dependabot. The buttons “Write tests now” and “Upgrade dependencies now” run them at once, ticked or not.", "web.help.projects.signals": "Its short name in the tracker's URLs. lastcall reads this project's unresolved errors with a read-only token: a project with errors in the last day ranks higher, and its audits see the error's type, its message with personal data masked, and the lines of your own code. Request data, user data and local variables never reach a model.", "web.help.projects.account": "Any free account takes the account that is not running anything and has the most quota left before its reset. Choose an account to use that one.", "web.help.projects.account_all": "With the option for all free accounts, every free account reviews the project and the one with the most quota left judges the findings and writes the fixes. Only an audit can use them all: for Improve now, tests and upgrades it means any free account.", @@ -792,14 +796,14 @@ "web.project.legend.when": "When lastcall works on it", "web.project.legend.writes": "What it writes and may do", "web.project.active_hint": "Bursts may pick it. The run buttons work either way.", - "web.project.merge_hint": "lastcall merges its own pull requests once they are approved and their checks pass.", - "web.project.watch_hint": "For a day after a merge, if the checks or the health address break, lastcall proposes a revert and pauses the project.", + "web.project.merge_hint": "When you approve one of its pull requests on GitHub, lastcall merges it once its checks pass. Without this, you merge it yourself.", + "web.project.watch_hint": "For a day after a merge, lastcall watches only the merged commit's checks and the health address below, not your logs or error tracker. If either breaks, it proposes a revert and pauses the project.", "web.project.checks.lead": "“{merge}” and “{watch}” need the GitHub App to read checks and commit statuses, and this App cannot yet.", "web.project.checks.open": "Open the App's permissions on GitHub.", "web.project.checks.set": "Set “Checks” and “Commit statuses” to “Read-only” and save.", "web.project.checks.accept": "Accept the new permissions where GitHub asks for it, on the App's installation: Settings › Applications › Installed GitHub Apps › Configure, or the organization's settings.", - "web.project.tests_turn": "In turn with the audits, it adds tests for behavior that no test checks.", - "web.project.deps_turn": "In turn with the audits, it upgrades what needs more than a new version number.", + "web.project.tests_turn": "Some bursts write tests instead of auditing: pull requests that add tests for behavior no test checks.", + "web.project.deps_turn": "Some bursts upgrade dependencies instead of auditing: pull requests for what needs more than a new version number.", "web.project.run_hints": "What each button does", "web.settings.events_hint": "They apply once you add a destination above.", "web.nav.breadcrumb": "Breadcrumb", diff --git a/src/lastcall/locales/es.json b/src/lastcall/locales/es.json index 833f31c..6adb8bb 100644 --- a/src/lastcall/locales/es.json +++ b/src/lastcall/locales/es.json @@ -293,8 +293,9 @@ "web.project.weight.high": "Alta (×2)", "web.project.weight.top": "Máxima (×3)", "web.project.weight.custom": "×{weight}", - "web.project.language": "Escribe en", - "web.project.language.account": "El de la cuenta", + "web.project.language": "Idioma de sus pull requests e issues", + "web.project.language.account": "Igual que la cuenta", + "web.project.language_hint": "«Igual que la cuenta» usa el idioma de la cuenta que trabaje en él, el que se elige en Cuentas.", "web.project.merge_on_approve": "Fusionar tras mi aprobación", "web.project.watch": "Vigilar tras la fusión", "web.project.tests": "También tests", @@ -426,7 +427,7 @@ "web.accounts.id": "Nombre", "web.accounts.id_hint": "Minúsculas, dígitos y guiones, como claude-max.", "web.accounts.engine": "Suscripción", - "web.accounts.language_label": "Escribe en", + "web.accounts.language_label": "Idioma de sus pull requests e issues", "web.accounts.reserve_label": "Reserva para ti, % de cada semana", "web.accounts.enabled": "lastcall puede gastarla", "web.accounts.submit": "Añadir", @@ -459,6 +460,7 @@ "web.accounts.model_hint": "El modelo de todos los pasos, tal como lo nombra la lista del proveedor. lastcall cuenta lo que cuesta cada paso, así que el modelo necesita un precio en esa lista.", "web.accounts.review_label": "Modelo de los revisores (opcional)", "web.accounts.review_hint": "Un modelo más barato para los revisores. Vacío: el de arriba.", + "web.accounts.model_suggest": "El campo sugiere lo que models.dev lista para todos los proveedores, y OpenRouter escribe primero el fabricante, como en qwen/qwen3-coder. Elige uno del proveedor que seleccionaste, o escribe cualquier otro.", "web.accounts.budget_label": "Presupuesto mensual, USD", "web.accounts.budget_hint": "Lo que pueden costar sus pasos cada mes natural (UTC), sea cual sea su política. Un paso que ya está en marcha puede pasarse un poco: mantén también el límite del proveedor.", "web.accounts.policy_label": "Cuándo gasta", @@ -617,6 +619,8 @@ "web.tiers.keyed_hint": "OpenCode pasa el esfuerzo como variante del modelo: elige una que tu modelo ofrezca, o deja Por defecto.", "web.tiers.save": "Guardar modelos y esfuerzo", "web.tiers.need_models": "Da un modelo para cada nivel: una cuenta con API key o modelo local no tiene valor por defecto.", + "web.tiers.suggest.cli": "El campo Modelo sugiere los nombres que acepta {tool}. Elige uno o escribe cualquier otro.", + "web.tiers.suggest.catalog": "El campo Modelo sugiere lo que models.dev lista para {provider}: modelos que usan herramientas y tienen precio. Elige uno o escribe cualquier otro.", "web.time.now": "ahora mismo", "web.time.ago": "hace {span}", @@ -717,12 +721,12 @@ "web.help.dashboard.status": "Estado: parada antes de tiempo significa que, antes de un paso, la cuenta había llegado a tu reserva o a su presupuesto mensual, tenía activo el uso de pago o no tenía lectura de cuota, así que la ejecución terminó ahí. Interrumpida significa que lastcall se detuvo mientras la ejecución estaba en marcha, por ejemplo al reiniciarse. Fallida significa que un error la detuvo.", "web.help.projects.active": "Los proyectos activos entran en las ráfagas. Aun así, una ráfaga se salta un proyecto mientras tenga 2 pull requests de lastcall esperándote, o si la misma cuenta lo auditó en los últimos 7 días, salvo que hayas marcado ideas o etiquetado issues para él. Los botones para lanzar una ejecución funcionan aunque el proyecto esté inactivo. Haz clic en el estado, arriba en la tarjeta, para activar o desactivar las ráfagas; la casilla «Activo» de Ajustes hace lo mismo.", "web.help.projects.weight": "Cuando una cuenta tiene cuota que gastar, lastcall da puntos a cada proyecto activo (nunca auditado por esa cuenta, días y commits nuevos desde su última auditoría, hallazgos pendientes, errores en producción, lo que hayas pedido) y empieza por el que más tiene. La prioridad multiplica esos puntos: Alta los duplica y Baja los reduce a la mitad. No hace la ejecución más larga.", - "web.help.projects.language": "El idioma de los pull requests, issues e informes de este proyecto. Con «El de la cuenta», lastcall escribe en el idioma de la cuenta que haga la ejecución, el que se elige en Cuentas. No cambia estas páginas ni tus notificaciones.", + "web.help.projects.language": "El idioma de los pull requests, issues e informes de este proyecto. Con «Igual que la cuenta», lastcall escribe en el idioma de la cuenta que haga la ejecución, el que se elige en Cuentas. No cambia estas páginas ni tus notificaciones.", "web.help.projects.max_prs": "El máximo de pull requests que abre una ejecución en este proyecto, haga lo que haga: una auditoría, tests, actualizaciones de dependencias, las ideas de un plan o issues etiquetados. Lo que no cabe espera a una ejecución posterior.", - "web.help.projects.merge_on_approve": "En GitHub, aprobar un pull request no lo fusiona. Con esto activado, lastcall fusiona sus propios pull requests (nunca una reversión) cuando alguien con permiso de escritura aprueba el último commit, nadie con ese permiso pide cambios y ningún check falla ni sigue en curso. Si uno ya no se fusiona limpio, lastcall lo cierra y una ejecución posterior rehace el cambio sobre el código actual.", - "web.help.projects.watch": "Durante un día después de fusionar uno de los pull requests de lastcall, lastcall vigila los checks del commit fusionado y la dirección de salud de abajo. Si los checks fallan donde antes pasaban, o la dirección falla 3 veces seguidas, abre un pull request que deshace el cambio, pausa el proyecto (desmarca «Activo») y te avisa. Nunca fusiona esa reversión: decides tú.", + "web.help.projects.merge_on_approve": "En GitHub, aprobar un pull request no lo fusiona. Con esto activado, lastcall fusiona sus propios pull requests (nunca una reversión) en su siguiente consulta a GitHub, que es cada 10 minutos: cuando alguien con permiso de escritura en el repositorio ha aprobado el último commit, nadie con ese permiso pide cambios y ningún check falla ni sigue en curso. Si GitHub todavía no deja fusionarlo, por ejemplo porque no se cumple una regla de una rama protegida, lastcall espera. Si uno ya no se fusiona limpio, lastcall lo cierra y una ejecución posterior rehace el cambio sobre el código actual. Sin esto, los fusionas tú.", + "web.help.projects.watch": "Durante un día después de fusionar uno de los pull requests de lastcall, lastcall vigila solo dos cosas: los checks del commit fusionado y, si das una, la dirección de salud de abajo. Si los checks fallan donde antes pasaban, o la dirección falla 3 veces seguidas, abre un pull request que deshace el cambio, pausa el proyecto (desmarca «Activo») y te avisa. Nunca fusiona esa reversión: decides tú. Los logs de contenedores, Loki y tu tracker de errores no intervienen en la vigilancia: alimentan las auditorías, que los usan para encontrar las causas de los errores.", "web.help.projects.health_url": "Solo se usa con «Vigilar tras la fusión» activado y mientras se vigila una fusión. lastcall la consulta cada 10 minutos; cualquier respuesta con estado de 200 a 399 en menos de 10 segundos cuenta como que funciona.", - "web.help.projects.turns": "Las ráfagas en este proyecto se turnan: una auditoría, luego tests y luego actualizaciones de dependencias (cada una solo si está marcada), y otra vez una auditoría. «También tests» añade tests para el comportamiento que ningún test comprueba. «También dependencias» actualiza lo que necesita algo más que un número de versión nuevo: vulnerabilidades conocidas, paquetes obsoletos, versiones mayores atrasadas. Las ideas que marcaste en un plan y los issues con la etiqueta lastcall siempre van primero.", + "web.help.projects.turns": "Las ideas que marcaste en un plan y los issues con la etiqueta lastcall siempre van primero. Después, una ráfaga en este proyecto lo audita. Marca «También tests», «También dependencias» o las dos y las ráfagas se turnan: una auditoría, luego una ejecución que escribe tests, luego otra que actualiza dependencias (cada una solo si está marcada) y otra vez una auditoría. Una ejecución de tests abre pull requests que solo añaden tests, para el comportamiento que ningún test comprueba. Una ejecución de dependencias abre pull requests que actualizan lo que necesita algo más que un número de versión nuevo (una vulnerabilidad conocida, un paquete obsoleto, una versión mayor atrasada, un runtime que ya no tiene soporte) y arreglan el código que rompe la versión nueva; las actualizaciones menores y de parche sin más se dejan a bots como Dependabot. Los botones «Escribir tests ahora» y «Actualizar dependencias ahora» las lanzan al momento, estén marcadas o no.", "web.help.projects.signals": "Su nombre corto en las URLs de Sentry o GlitchTip. lastcall lee los errores sin resolver de este proyecto con un token de solo lectura: un proyecto con errores en el último día sube en la cola, y sus auditorías ven el tipo de error, el mensaje con los datos personales ocultos y las líneas de tu propio código. Los datos de la petición, del usuario y las variables locales nunca llegan a un modelo.", "web.help.projects.account": "«Cualquier cuenta libre» usa la cuenta que no está ejecutando nada y tiene más cuota sobrante antes de su reinicio. Elige una cuenta para usar esa.", "web.help.projects.account_all": "Con la opción de todas las cuentas libres, todas revisan el proyecto y la que tiene más cuota sobrante juzga los hallazgos y escribe los arreglos. Solo una auditoría puede usarlas todas: para Mejorar ahora, tests y dependencias equivale a cualquier cuenta libre.", @@ -792,14 +796,14 @@ "web.project.legend.when": "Cuándo trabaja lastcall en él", "web.project.legend.writes": "Qué escribe y qué puede hacer", "web.project.active_hint": "Las ráfagas pueden elegirlo. Los botones de ejecutar funcionan igual.", - "web.project.merge_hint": "lastcall fusiona sus propios pull requests en cuanto los aprueban y pasan sus checks.", - "web.project.watch_hint": "Durante un día tras una fusión, si fallan los checks o la dirección de salud, lastcall propone una reversión y pausa el proyecto.", + "web.project.merge_hint": "Cuando apruebas uno de sus pull requests en GitHub, lastcall lo fusiona en cuanto pasan sus checks. Sin esto, lo fusionas tú.", + "web.project.watch_hint": "Durante un día tras una fusión, lastcall vigila solo los checks del commit fusionado y la dirección de salud de abajo, no tus logs ni tu tracker de errores. Si alguno falla, propone una reversión y pausa el proyecto.", "web.project.checks.lead": "«{merge}» y «{watch}» necesitan que la GitHub App lea los checks y los estados de commit, y esta App todavía no puede.", "web.project.checks.open": "Abre los permisos de la App en GitHub.", "web.project.checks.set": "Pon «Checks» y «Commit statuses» en «Read-only» y guarda.", "web.project.checks.accept": "Acepta los nuevos permisos donde GitHub te lo pida, en la instalación de la App: Settings › Applications › Installed GitHub Apps › Configure, o en los ajustes de la organización.", - "web.project.tests_turn": "Por turnos con las auditorías, añade tests para el comportamiento que ningún test comprueba.", - "web.project.deps_turn": "Por turnos con las auditorías, actualiza lo que necesita algo más que un número de versión nuevo.", + "web.project.tests_turn": "Algunas ráfagas escriben tests en lugar de auditar: pull requests que añaden tests para el comportamiento que ningún test comprueba.", + "web.project.deps_turn": "Algunas ráfagas actualizan dependencias en lugar de auditar: pull requests para lo que necesita algo más que un número de versión nuevo.", "web.project.run_hints": "Qué hace cada botón", "web.settings.events_hint": "Se aplican cuando añadas un destino arriba.", "web.nav.breadcrumb": "Ruta de navegación", diff --git a/src/lastcall/tiers.py b/src/lastcall/tiers.py index 181e128..299e19d 100644 --- a/src/lastcall/tiers.py +++ b/src/lastcall/tiers.py @@ -8,19 +8,31 @@ Tier = Literal["fast", "balanced", "deep"] TIERS: tuple[Tier, ...] = ("fast", "balanced", "deep") +SECTIONS = ("_about", "models", "efforts") # the keys of tiers.json that are not an engine's tiers def defaults(engine: str) -> dict[str, tuple[str, str | None]]: """tier -> (model, effort), from lastcall/data/tiers.json.""" data = _data() - if engine not in data or engine == "efforts": + if engine not in data or engine in SECTIONS: raise ValueError(f"no default tiers for engine {engine!r}") return {tier: (str(model), str(effort) if effort else None) for tier, (model, effort) in data[engine].items()} +def models(engine: str) -> tuple[str, ...]: + """The model names that an engine's CLI takes, which the web UI suggests for a tier's model; none for an engine + with no list (an API key's models come from models.dev, `lastcall.catalog`).""" + return _listed("models", engine) + + def efforts(engine: str) -> tuple[str, ...]: """The effort levels that an engine's CLI takes, the lowest first; none for an engine that takes none.""" - return tuple(str(level) for level in _data()["efforts"].get(engine, ())) + return _listed("efforts", engine) + + +def _listed(section: str, engine: str) -> tuple[str, ...]: + names = _data()[section].get(engine, ()) if engine != "_about" else () + return tuple(str(name) for name in names) def _data() -> dict[str, Any]: diff --git a/src/lastcall/web/__init__.py b/src/lastcall/web/__init__.py index eeb34f5..d81acc6 100644 --- a/src/lastcall/web/__init__.py +++ b/src/lastcall/web/__init__.py @@ -32,7 +32,7 @@ from flask.typing import ResponseReturnValue from werkzeug.middleware.proxy_fix import ProxyFix -from lastcall import __version__, cron, destinations, engines, i18n, logs, loki, projects, signals +from lastcall import __version__, catalog, cron, destinations, engines, i18n, logs, loki, projects, signals from lastcall.accounts import ACCOUNT_ID, MODEL, Account, policies, tier_models from lastcall.engines.base import Keyed from lastcall.github import PERMISSIONS, App, Fetch, GitHub, GitHubError, lacking, permissions_of, urlopen @@ -45,6 +45,7 @@ from lastcall.store import AccountRow, Store from lastcall.supervisor import STEP from lastcall.tiers import TIERS, defaults, efforts +from lastcall.tiers import models as cli_models from lastcall.web import auth, dashboard, project_views COOKIE = "lastcall_session" @@ -66,6 +67,10 @@ RETRY_SECONDS = 60.0 # a gap (the owner is fixing it) or a miss is kept only a minute, so that the notice goes soon GRANT_TIMEOUT = 5 # seconds that the Projects page waits for GitHub's answer on one installation MAX_GRANTS = 100 +CATALOG_SECONDS = 86_400.0 # the models that models.dev lists for an API's provider, kept a day: they weigh 6 MB +CATALOG_RETRY_SECONDS = 600.0 # a miss is kept 10 minutes, so that a models.dev that is down costs one wait +CATALOG_TIMEOUT = 5 # seconds that the Accounts page waits for models.dev +KEYS_LIST = "models-keys" # the datalist of the form that adds an API key: the models of every provider MAX_ADDED = 100_000 # the most that a "N projects added" notice says: the number is in the address CREDENTIAL = re.compile(r"(?i)\b(bearer|basic)\s+[^\"']*|\b(authorization|header value)\W+.*") QUICKSTART = "https://github.com/ne0x86/lastcall#-quickstart" @@ -109,8 +114,9 @@ def create_app( """The web UI's application. `github` gives the GitHub client (by default, from the App in the secret store); `background` runs slow work, such as reading the installed repositories, outside the request. `read_quota` reads an account's quota in the sandbox; without it, the web UI cannot check a new account's login. - `exchange` turns GitHub's one-time code into the App's credentials; `fetch` asks Telegram for a bot's chats, and - GitHub whether an owner is a person or an organization, and what the App's installation was granted.""" + `exchange` turns GitHub's one-time code into the App's credentials; `fetch` asks Telegram for a bot's chats, + GitHub whether an owner is a person or an organization, and what the App's installation was granted, and + models.dev which models an API key's provider offers.""" app = Flask(__name__) app.config["MAX_CONTENT_LENGTH"] = 64 * 1024 # forms are small if trust_proxy: @@ -125,7 +131,8 @@ def create_app( app_states: dict[str, float] = {} # manifest forms shown, by state, with when they were shown owner_answers: dict[str, tuple[str, float]] = {} # what GitHub said an owner is ("user", "org", "missing"), when grant_answers: dict[str, tuple[str | None, float]] = {} # permissions_page of an owner's installation, and when - cache_lock = threading.Lock() # for app_states, owner_answers and grant_answers: waitress answers on 8 threads + catalog_answers: dict[str, tuple[tuple[str, ...], float]] = {} # a provider's models in models.dev, until when + cache_lock = threading.Lock() # for the four above: waitress answers on 8 threads quota_errors: dict[str, tuple[float, str]] = {} # the last failed quota reading of each account, and when def connect() -> GitHub | None: @@ -341,6 +348,32 @@ def permissions_link(repo: str) -> str: del grant_answers[next(iter(grant_answers))] return found or "" + def catalog_models(provider: str) -> tuple[str, ...]: + """The models that models.dev lists for a provider of API keys (its id there), from the last day when it was + asked: the Accounts page asks on every load, and the answer weighs 6 MB. One request answers for every + provider. A failure is kept 10 minutes, so that a models.dev that is down costs the page one wait in that + time, not one at every load; meanwhile it suggests no models. Bounded: one entry for each provider that an + engine offers, and `catalog.MAX_MODELS` models in each.""" + asked = catalog_providers() + if provider not in asked: + return () + now = clock() + with cache_lock: + known = catalog_answers.get(provider) + if known is not None and now < known[1]: + return known[0] + try: + found = catalog.models(fetch, asked, CATALOG_TIMEOUT) + until = now + CATALOG_SECONDS + except Exception as e: # an optional lookup never takes the page down + log(f"Web UI: models.dev gave no models: {brief(e)}") + found, until = {}, now + CATALOG_RETRY_SECONDS + with cache_lock: + for name in asked: + catalog_answers.pop(name, None) # asked again: it goes to the end, as the newest + catalog_answers[name] = (found.get(name, ()), until) + return found.get(provider, ()) + @app.post("/projects/add") @signed_in def add_project() -> ResponseReturnValue: @@ -951,13 +984,23 @@ def show_accounts( message = t(f"web.done.{done}") if done in known and not error else "" all_engines = [engines.get(name) for name in sorted(engines.BUILTIN)] sources = [(e, p) for e in all_engines if isinstance(e, Keyed) for p in e.providers] + lists: dict[str, tuple[str, ...]] = {} # the datalists of the page, by id: what its model fields suggest + cards = [account_card(row, now, working, lists) for row in store.accounts()] + keys = [(f"{e.name}/{p.name}", p) for e, p in sources if p.keys] + # Without a script, the form that adds an API key cannot follow the provider that it has chosen: its model + # fields suggest the models of every provider, and OpenRouter writes the maker first (qwen/qwen3-coder). + everyone = {name for _, p in keys for name in catalog_models(p.catalog)} + if everyone: + lists[KEYS_LIST] = tuple(sorted(everyone, key=lambda name: (name.casefold(), name))) return page( "accounts.html", status, - cards=[account_card(row, now, working) for row in store.accounts()], + cards=cards, held="down" if down else ("paused" if paused else ""), engines=[(e.name, f"{e.provider} ({e.title})") for e in all_engines if e.login], - keys=[(f"{e.name}/{p.name}", p) for e, p in sources if p.keys], + keys=keys, + datalists=lists, + key_list=KEYS_LIST if everyone else "", local=next(((f"{e.name}/{p.name}", p) for e, p in sources if not p.keys), None), writing=[(name, dashboard.language_name(g.lang, name).capitalize()) for name in project_views.LANGUAGES], known=project_views.LANGUAGES, @@ -993,7 +1036,32 @@ def quota_error(account_id: str) -> str: return "" return failed[1] - def account_card(row: AccountRow, now: float, working: Mapping[str, dict[str, Any]]) -> dict[str, Any]: + def suggestions(row: AccountRow, lists: dict[str, tuple[str, ...]]) -> dict[str, str]: + """The datalist that suggests models in the fields of an account's tiers, and the note that says where its + names come from: for an account on a CLI login, the names that the CLI takes, from lastcall's data file; for + an account on an API key, what models.dev lists for its provider. Empty when there is nothing to suggest: a + local model server (the web UI cannot reach it), a models.dev that did not answer, an engine that is gone.""" + if row.engine not in engines.BUILTIN: + return {} + engine = engines.get(row.engine) + if not row.keyed: + names, key, note = cli_models(row.engine), row.engine, t("web.tiers.suggest.cli", tool=engine.title) + elif isinstance(engine, Keyed): + source = next((p for p in engine.providers if p.name == row.provider), None) + if source is None: + return {} + names, key = catalog_models(source.catalog), f"{row.engine}-{source.name}" + note = t("web.tiers.suggest.catalog", provider=source.title) + else: + return {} + if not names: + return {} + lists[f"models-{key}"] = names + return {"list": f"models-{key}", "note": note} + + def account_card( + row: AccountRow, now: float, working: Mapping[str, dict[str, Any]], lists: dict[str, tuple[str, ...]] + ) -> dict[str, Any]: try: base = defaults(row.engine) except ValueError: # an engine with no defaults, such as opencode: the account names its models @@ -1008,6 +1076,7 @@ def account_card(row: AccountRow, now: float, working: Mapping[str, dict[str, An "policies": [], "needs_key": False, "levels": efforts(row.engine), + "suggest": suggestions(row, lists), "tiers": [ { "tier": tier, @@ -1236,6 +1305,16 @@ def not_found(_: Exception) -> ResponseReturnValue: return app +def catalog_providers() -> tuple[str, ...]: + """The ids in models.dev of the providers that the engines on API keys offer: what the web UI asks it about.""" + found: list[str] = [] + for name in sorted(engines.BUILTIN): + engine = engines.get(name) + if isinstance(engine, Keyed): + found += [p.catalog for p in engine.providers if p.catalog] + return tuple(dict.fromkeys(found)) + + def brief(error: BaseException) -> str: """An exception for the log: its type and a short line of its message. Never its repr, which can hold request data, and a header's value or an authorization is masked wherever it shows in the message.""" diff --git a/src/lastcall/web/static/style.css b/src/lastcall/web/static/style.css index c462e13..6fe4c17 100644 --- a/src/lastcall/web/static/style.css +++ b/src/lastcall/web/static/style.css @@ -1111,6 +1111,7 @@ fieldset.has-help > legend { float: left; margin: 0; } .project-settings fieldset { gap: var(--s3); padding: var(--s3); background: var(--surface); border-radius: var(--r-md); } .project-settings .check-row { min-height: 0; } .project-settings .check-row .hint { margin-top: 0; } +.project-settings .fields { grid-template-columns: minmax(0, 1fr); } /* a select a row: a closed select cuts its text short */ /* Add a project: the manual field, then suggestions as rows. */ .add-card .inline { margin-bottom: var(--s2); } diff --git a/src/lastcall/web/templates/accounts.html b/src/lastcall/web/templates/accounts.html index 7a2cacc..5e1cf8b 100644 --- a/src/lastcall/web/templates/accounts.html +++ b/src/lastcall/web/templates/accounts.html @@ -125,7 +125,7 @@

{{ t("web.nav.accounts") }}

- +
{% if loop.first %}{{ help(hb, "accounts-effort", t("web.tiers.effort"), ["accounts.effort"], true) }}{% endif %}
@@ -138,6 +138,9 @@

{{ t("web.nav.accounts") }}

{%- endfor %}

{{ t("web.tiers.keyed_hint") if row.keyed else t("web.tiers.default_hint") }}

+ {%- if c.suggest %} +

{{ c.suggest.note }}

+ {%- endif %}
@@ -228,12 +231,12 @@

{{ t("web.accounts.kind.key") }}

- -

{{ t("web.accounts.model_hint") }}

+ +

{{ t("web.accounts.model_hint") }}{% if key_list %} {{ t("web.accounts.model_suggest") }}{% endif %}

{{ help(hb, "accounts-review", t("web.accounts.review_label"), ["accounts.review_model"], true) }}
- +

{{ t("web.accounts.review_hint") }}

@@ -311,5 +314,10 @@

{{ t("web.accounts.kind.local") }}

{%- endif %}
+{#- The datalists that the model fields point to (`list`): one for each kind of account on the page, whatever the number of + its cards. A browser shows them as suggestions and lets the owner type any other name. -#} +{%- for list_id, names in datalists.items() %} +{% for name in names %}{% endfor %} +{%- endfor %} {{ bubbles(hb) }} {% endblock %} diff --git a/src/lastcall/web/templates/projects.html b/src/lastcall/web/templates/projects.html index b63480f..fc024a9 100644 --- a/src/lastcall/web/templates/projects.html +++ b/src/lastcall/web/templates/projects.html @@ -233,9 +233,10 @@

{{ t("web.project.remove_title", repo=p.repo) }}
{{ help(hb, "projects-language", t("web.project.language"), ["projects.language"], true) }}
- {%- for o in p.languages %}{% endfor %} +

{{ t("web.project.language_hint") }}

{{ help(hb, "projects-max_prs", t("web.project.max_prs"), ["projects.max_prs"], true) }}
diff --git a/tests/test_catalog.py b/tests/test_catalog.py new file mode 100644 index 0000000..78fac72 --- /dev/null +++ b/tests/test_catalog.py @@ -0,0 +1,206 @@ +"""models.dev's catalog, as the web UI reads it: which models a provider offers that a step can run on. The tests +never ask models.dev: `ModelsDev` is a `fetch` that answers from a small catalog of the shape that it has.""" + +import gzip +import json +import urllib.request +import zlib +from collections.abc import Mapping +from typing import Any + +import pytest + +from lastcall import __version__, catalog, engines +from lastcall.accounts import MODEL +from lastcall.engines.base import Keyed +from lastcall.github import Fetch, Response + + +def model(**fields: Any) -> dict[str, Any]: + """One model of a provider: it calls tools and has a price, until a field says otherwise.""" + return {"name": "A model", "tool_call": True, "cost": {"input": 1.5, "output": 7.5}, **fields} + + +CATALOG: dict[str, Any] = { + "openrouter": { + "id": "openrouter", + "name": "OpenRouter", + "models": { + "qwen/qwen3-coder": model(), + "Qwen/Qwen3-coder-flash": model(), # the order is alphabetical whatever the case + "qwen/qwen3-coder:free": model(cost={"input": 0, "output": 0}), # free is a price + "anthropic/claude-sonnet-5.5": model(), + "~anthropic/claude-sonnet-latest": model(), # an alias that an account cannot name: it starts with ~ + "legacy/old-model": model(status="deprecated"), + "beta/new-model": model(status="beta"), # not deprecated: it stays + "writer/no-tools": model(tool_call=False), + "writer/tools-unknown": {"cost": {"input": 1, "output": 2}}, + "writer/no-price": {"tool_call": True}, + "writer/half-price": model(cost={"input": 1}), + "writer/odd-price": model(cost={"input": "1", "output": True}), + }, + }, + "anthropic": {"id": "anthropic", "models": {"claude-sonnet-5-5": model(), "claude-opus-5-5": model()}}, + "openai": {"id": "openai", "models": {"gpt-6.1-sol": model(), "gpt-image-2": model(tool_call=False)}}, + "ollama-cloud": {"id": "ollama-cloud", "models": {"kimi-k3": model()}}, # not a provider that lastcall asks about +} +URL = "https://models.dev/api.json" +Answer = Mapping[str, Any] | Exception | tuple[int, bytes] + + +class ModelsDev: + """A `fetch` that answers what models.dev would: the catalog (gzipped, as it is when it is asked for gzip), or the + exception that a network that fails raises, or a status and a body. It records every request it gets.""" + + def __init__(self, answer: Answer = CATALOG, packed: bool = True) -> None: + self.answer, self.packed = answer, packed + self.asked: list[urllib.request.Request] = [] + + def __call__(self, req: urllib.request.Request) -> Response: + self.asked.append(req) + assert req.full_url == URL, req.full_url + if isinstance(self.answer, Exception): + raise self.answer + if isinstance(self.answer, tuple): + status, body = self.answer + return status, {}, body + body = json.dumps(self.answer).encode() + if self.packed: + return 200, {"Content-Type": "application/json", "Content-Encoding": "gzip"}, gzip.compress(body) + return 200, {"Content-Type": "application/json"}, body + + +def says(body: bytes, headers: Mapping[str, str] | None = None) -> Fetch: + """A `fetch` that answers 200 with this body and these headers, whatever it is asked.""" + + def fetch(req: urllib.request.Request) -> Response: + return 200, dict(headers or {}), body + + return fetch + + +def listed(provider: str) -> tuple[str, ...]: + return catalog.models(ModelsDev(), [provider], 5)[provider] + + +def test_it_keeps_the_models_that_a_step_can_run_on() -> None: + assert listed("openrouter") == ( + "anthropic/claude-sonnet-5.5", + "beta/new-model", + "qwen/qwen3-coder", + "Qwen/Qwen3-coder-flash", + "qwen/qwen3-coder:free", + ) + assert listed("anthropic") == ("claude-opus-5-5", "claude-sonnet-5-5") + assert listed("openai") == ("gpt-6.1-sol",) + + +def test_every_name_that_it_keeps_is_one_that_an_account_takes() -> None: + assert all(MODEL.fullmatch(name) for name in listed("openrouter")) + assert "~anthropic/claude-sonnet-latest" not in listed("openrouter") + + +def test_one_request_answers_for_every_provider_and_a_missing_provider_has_none() -> None: + fetch = ModelsDev() + found = catalog.models(fetch, ["openrouter", "anthropic", "openai", "nobody"], 5) + assert len(fetch.asked) == 1 + assert found["nobody"] == () and found["anthropic"] == ("claude-opus-5-5", "claude-sonnet-5-5") + assert "ollama-cloud" not in found # it keeps only the providers that it is asked about + + +def test_the_request_says_who_asks_wants_gzip_and_waits_5_seconds_and_carries_nothing_of_the_owner() -> None: + fetch = ModelsDev() + catalog.models(fetch, ["openai"], 5) + (req,) = fetch.asked + assert req.full_url == "https://models.dev/api.json" and req.get_method() == "GET" and req.data is None + assert req.get_header("User-agent") == f"lastcall/{__version__} (+https://github.com/ne0x86/lastcall)" + assert req.get_header("Accept-encoding") == "gzip" and getattr(req, "timeout", None) == 5 + assert not req.has_header("Authorization") and not req.has_header("Cookie") + + +@pytest.mark.parametrize("packed", [True, False]) +def test_a_gzipped_answer_and_a_plain_one_read_the_same(packed: bool) -> None: + found = catalog.models(ModelsDev(packed=packed), ["anthropic"], 5) + assert found == {"anthropic": ("claude-opus-5-5", "claude-sonnet-5-5")} + + +def test_the_header_that_says_gzip_is_read_whatever_its_case() -> None: + fetch = says(gzip.compress(json.dumps(CATALOG).encode()), {"content-encoding": " GZip "}) + assert catalog.models(fetch, ["openai"], 5) == {"openai": ("gpt-6.1-sol",)} + + +@pytest.mark.parametrize( + "answer", + [ + (404, b"{}"), + (403, gzip.compress(b"Forbidden")), # what models.dev says to a request with no User-Agent + (500, b""), + (200, b""), + (200, b"a captive portal"), + (200, b'["not", "the", "catalog"]'), + (200, b"null"), + OSError("no route to host"), + TimeoutError("timed out"), + ], +) +def test_an_answer_that_is_not_the_catalog_is_an_error(answer: Answer) -> None: + with pytest.raises((OSError, ValueError)): + catalog.models(ModelsDev(answer), ["openrouter"], 5) + + +def test_a_gzip_that_is_broken_is_an_error() -> None: + with pytest.raises(zlib.error): + catalog.models(says(b"this is not gzip", {"Content-Encoding": "gzip"}), ["openrouter"], 5) + + +@pytest.mark.parametrize( + "entry", + [None, [], "openrouter", 3, {}, {"models": None}, {"models": []}, {"models": "x"}, {"models": {"a/b": "text"}}], +) +def test_a_provider_that_does_not_look_like_one_has_no_models(entry: object) -> None: + assert catalog.models(ModelsDev({"p": entry}), ["p"], 5) == {"p": ()} + + +def test_what_else_the_catalog_says_about_a_model_does_not_matter() -> None: + odd = {"p": {"models": {"a/b": model(cost={"input": 1, "output": 2, "cache_read": None, "tiers": [{"x": 1}]})}}} + assert catalog.models(ModelsDev(odd), ["p"], 5) == {"p": ("a/b",)} + + +def test_a_provider_keeps_at_most_max_models_the_first_in_alphabetical_order(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(catalog, "MAX_MODELS", 3) + many = {"p": {"models": {f"vendor/m{n:02}": model() for n in range(20, 0, -1)}}} + assert catalog.models(ModelsDev(many), ["p"], 5) == {"p": ("vendor/m01", "vendor/m02", "vendor/m03")} + + +def test_an_answer_over_the_limit_is_refused_packed_or_not(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(catalog, "MAX_BYTES", 1000) + big = {"p": {"models": {f"vendor/model-{n}": model() for n in range(100)}}} + for packed in (True, False): + with pytest.raises(ValueError, match="too big"): + catalog.models(ModelsDev(big, packed=packed), ["p"], 5) + + +def test_a_few_kilobytes_that_unpack_to_megabytes_are_refused_while_they_unpack( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(catalog, "MAX_BYTES", 1024 * 1024) + bomb = gzip.compress(b" " * (20 * 1024 * 1024)) # 20 MB of spaces, 20 KB packed: under the limit as it comes + assert len(bomb) < 100 * 1024 + with pytest.raises(ValueError, match="too big"): + catalog.models(says(bomb, {"Content-Encoding": "gzip"}), ["openrouter"], 5) + + +def test_an_answer_of_exactly_the_limit_is_read(monkeypatch: pytest.MonkeyPatch) -> None: + body = json.dumps(CATALOG).encode() + monkeypatch.setattr(catalog, "MAX_BYTES", len(body)) # its size, plain or unpacked + assert catalog.models(says(body), ["openai"], 5) == {"openai": ("gpt-6.1-sol",)} + packed = says(gzip.compress(body), {"Content-Encoding": "gzip"}) + assert catalog.models(packed, ["openai"], 5) == {"openai": ("gpt-6.1-sol",)} + + +def test_each_api_provider_of_opencode_says_its_id_in_models_dev_and_a_local_server_has_none() -> None: + engine = engines.get("opencode") + assert isinstance(engine, Keyed) + found = {p.name: p.catalog for p in engine.providers} + assert found == {"openrouter": "openrouter", "anthropic": "anthropic", "openai": "openai", "ollama": ""} + assert all(p.keys for p in engine.providers if p.catalog) # an API takes a key; a local server has none diff --git a/tests/test_docs.py b/tests/test_docs.py index 4a59adf..e550d04 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -3,6 +3,7 @@ import json import re from pathlib import Path +from typing import Any ROOT = Path(__file__).resolve().parents[1] README = (ROOT / "README.md").read_text(encoding="utf-8") @@ -103,3 +104,66 @@ def test_the_documents_describe_the_state_chip_and_the_trash_button() -> None: assert '"/projects///active"' in source and '"/projects///remove"' in source cli = (ROOT / "src/lastcall/cli.py").read_text(encoding="utf-8") assert "lastcall project remove " in cli and "cli project remove " in README + + +def tiers_json() -> dict[str, Any]: + found: dict[str, Any] = json.loads((ROOT / "src/lastcall/data/tiers.json").read_text(encoding="utf-8")) + return found + + +def test_the_readme_example_of_a_default_is_the_one_in_data_tiers_json() -> None: + model, effort = tiers_json()["claude-code"]["deep"] + assert f"such as {model} at `{effort}` effort for Claude Code's *deep*" in " ".join(README.split()) + + +def test_the_documents_say_why_the_efforts_differ_and_name_the_ones_of_data_tiers_json() -> None: + data = tiers_json() + efforts = {tier: data["claude-code"][tier][1] for tier in ("fast", "balanced", "deep")} + assert efforts == {tier: data["codex"][tier][1] for tier in efforts} # one reason for both engines + assert len(set(efforts.values())) == 3 # they differ + readme = " ".join(README.split()) + assert "The efforts differ because the work does" in readme + assert f"*deep* runs at `{efforts['deep']}`" in readme and f"*balanced* at `{efforts['balanced']}`" in readme + assert f"at `{efforts['fast']}`." in readme + section = " ".join(DESIGN.split("### Tiers and models")[1].split("### Engines")[0].split()) + assert "The default efforts differ by tier because the work does" in section + assert f"so it takes `{efforts['fast']}`" in section and f"at `{efforts['balanced']}`" in section + assert f"at `{efforts['deep']}`" in section and "where quality matters most" in section + + +def test_the_documents_say_where_the_suggested_models_come_from() -> None: + import lastcall.web as site + from lastcall import catalog + + section = " ".join(DESIGN.split("### Tiers and models")[1].split("### Engines")[0].split()) + for words in ( + "**Suggested models.**", + "``", + "the `models` of its engine in the data file", + "https://models.dev/api.json", + "`Provider.catalog`", + "A local model server (Ollama) gets no suggestions", + "`fable` is left out on purpose", + ): + assert words in section, words + for name in tiers_json()["models"]["claude-code"]: + assert f"`{name}`" in section, name # the aliases that the data file lists + # the numbers that it gives are the code's + assert (site.CATALOG_TIMEOUT, site.CATALOG_SECONDS, site.CATALOG_RETRY_SECONDS) == (5, 86_400, 600) + assert "waits 5 seconds at most" in section and "stays in memory for a day" in section + assert "A failure is kept for 10 minutes" in section + assert catalog.MAX_MODELS == 1000 and "at most 1000 models" in section + assert "ask [models.dev](https://models.dev), about once a day, for the names of the models" in " ".join( + README.split() + ) + + +def test_the_documents_say_that_only_the_checks_and_the_health_address_take_part_in_the_watch() -> None: + after = " ".join(README.split("**After the merge.**")[1].split("\n## ")[0].split()) + assert ( + "Nothing else takes part in the watch: container logs, Loki and your error tracker feed the audits" in after + ) + deployment = " ".join(DESIGN.split("## 11. Delivery, approval and after the merge")[1].split("## 12.")[0].split()) + assert ( + "Errors in production feed the audits since v0.5, and container logs and their state since v0.6" in deployment + ) diff --git a/tests/test_i18n.py b/tests/test_i18n.py index 1064ea7..2f18c6f 100644 --- a/tests/test_i18n.py +++ b/tests/test_i18n.py @@ -95,3 +95,51 @@ def test_the_texts_about_the_checks_call_the_two_settings_as_the_projects_page_d permissions = text(lang, "web.help.github.permissions") # what the manifest asks for, and why for label in (merge, watch): assert label in lead and label in permissions, (lang, label) + + +def test_the_help_of_a_projects_language_quotes_the_option_where_it_really_is() -> None: + for lang in ("en", "es"): + assert text(lang, "web.project.language.account") in text(lang, "web.help.projects.language"), lang + + +def test_the_help_of_the_turns_calls_the_two_boxes_and_the_two_buttons_as_the_page_does() -> None: + for lang in ("en", "es"): + turns = text(lang, "web.help.projects.turns") + for key in ("web.project.tests", "web.project.deps", "web.project.tests_now", "web.project.deps_now"): + assert text(lang, key) in turns, (lang, key) + + +def test_the_help_of_the_watch_calls_the_active_box_as_the_page_does() -> None: + for lang in ("en", "es"): + assert text(lang, "web.project.active") in text(lang, "web.help.projects.watch"), lang + + +def test_the_hint_and_the_help_of_the_watch_both_say_that_logs_do_not_take_part() -> None: + said = { + "en": ("not your logs or error tracker", "take no part in the watch"), + "es": ("no tus logs ni tu tracker de errores", "no intervienen en la vigilancia"), + } + for lang, (hint, bubble) in said.items(): + assert hint in text(lang, "web.project.watch_hint"), lang + assert bubble in text(lang, "web.help.projects.watch"), lang + + +def test_the_help_about_effort_names_no_default_so_that_the_data_file_alone_decides_them() -> None: + for lang, catalog in MESSAGES.items(): + for key, template in catalog.items(): + if key.startswith(("web.tiers.", "web.help.accounts.")): + assert "xhigh" not in template, (lang, key) + + +def test_the_suggestions_say_where_they_come_from_in_both_languages() -> None: + for lang in ("en", "es"): + assert "models.dev" in text(lang, "web.tiers.suggest.catalog", provider="OpenRouter"), lang + assert "Codex" in text(lang, "web.tiers.suggest.cli", tool="Codex"), lang + assert "qwen/qwen3-coder" in text(lang, "web.accounts.model_suggest"), lang + + +def test_the_default_language_of_a_project_fits_a_closed_select_on_a_phone() -> None: + """A closed select cuts its text short: about 30 characters fit in a phone's, and the field is cut at 24 to leave + room for the narrower ones.""" + for lang in ("en", "es"): + assert len(text(lang, "web.project.language.account")) <= 24, lang diff --git a/tests/test_keyed.py b/tests/test_keyed.py index 783b661..2472381 100644 --- a/tests/test_keyed.py +++ b/tests/test_keyed.py @@ -65,7 +65,7 @@ def test_models_come_per_tier_and_keep_the_default_effort() -> None: assert tier_models("") == {} assert openrouter().tiers()["deep"] == ("qwen/qwen3-coder", None) claude = Account("claude", "claude-code", models={"deep": "sonnet"}) - assert claude.tiers()["deep"] == ("sonnet", "high") # the account's own model, the default effort + assert claude.tiers()["deep"] == ("sonnet", "xhigh") # the account's own model, the default effort assert claude.tiers()["fast"] == ("haiku", "medium") @@ -93,7 +93,7 @@ def test_an_account_chooses_the_model_and_effort_of_each_tier( row = store.account("max") assert row is not None and row.account().tiers() == claude.tiers() assert cli.main(["account", "tier", "max", "deep", "--effort", "-"]) == 0 # back to the default - assert "deep opus, effort high\n" in capsys.readouterr().out + assert "deep opus, effort xhigh\n" in capsys.readouterr().out assert cli.main(["account", "tier", "max", "balanced", "--model", "-"]) == 0 assert cli.main(["account", "tier", "max", "fast", "--effort", "low"]) == 0 assert "fast haiku, effort low (its own)" in capsys.readouterr().out @@ -287,7 +287,7 @@ def test_models_of_a_subscription(monkeypatch: pytest.MonkeyPatch, tmp_path: Pat == 0 ) row = Store(tmp_path / "state" / "lastcall.db").account("claude") - assert row is not None and row.account().tiers()["deep"] == ("sonnet", "high") + assert row is not None and row.account().tiers()["deep"] == ("sonnet", "xhigh") models: Mapping[str, str] = row.models assert set(models) == {"fast", "balanced", "deep"} diff --git a/tests/test_recipes.py b/tests/test_recipes.py index 39c8fa2..3429748 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -58,7 +58,7 @@ def test_schemas_are_strict(name: str) -> None: def test_default_tiers() -> None: - assert defaults("claude-code")["deep"] == ("opus", "high") + assert defaults("claude-code")["deep"] == ("opus", "xhigh") assert set(defaults("codex")) == {"fast", "balanced", "deep"} with pytest.raises(ValueError, match="no default tiers"): defaults("scripted") diff --git a/tests/test_tiers.py b/tests/test_tiers.py new file mode 100644 index 0000000..c54be30 --- /dev/null +++ b/tests/test_tiers.py @@ -0,0 +1,99 @@ +"""The data file of tiers: the default model and effort of each tier, the model names that each CLI takes, and the +effort levels that it takes.""" + +import json +from importlib import resources + +import pytest + +from lastcall.accounts import MODEL, Account +from lastcall.tiers import SECTIONS, TIERS, defaults, efforts, models + +ENGINES = ("claude-code", "codex") # the engines on a CLI login: the ones with default tiers + + +def test_the_default_effort_follows_the_work() -> None: + for engine in ENGINES: + assert {tier: effort for tier, (_, effort) in defaults(engine).items()} == { + "fast": "medium", # only a repository's own reviewers ask for it + "balanced": "high", # the reviewers + "deep": "xhigh", # the judge, the implementers and the improvement plans + } + + +def test_the_default_models_are_the_ones_of_each_engine() -> None: + assert {tier: model for tier, (model, _) in defaults("claude-code").items()} == { + "fast": "haiku", + "balanced": "sonnet", + "deep": "opus", + } + assert {tier: model for tier, (model, _) in defaults("codex").items()} == { + "fast": "gpt-6-luna", + "balanced": "gpt-6.1-sol", + "deep": "gpt-6.1-sol", + } + + +def test_every_default_effort_is_one_that_the_cli_takes() -> None: + for engine in ENGINES: + for tier, (_, effort) in defaults(engine).items(): + assert effort in efforts(engine), (engine, tier) + assert "xhigh" in efforts("claude-code") and "xhigh" in efforts("codex") + + +def test_the_efforts_of_an_engine_go_from_the_lowest_up() -> None: + for engine in ("claude-code", "codex", "opencode"): + levels = efforts(engine) + assert levels == tuple(sorted(levels, key=("minimal", "low", "medium", "high", "xhigh", "max").index)), engine + + +def test_the_cli_of_an_account_gets_the_default_effort_of_each_of_its_tiers() -> None: + assert Account("max", "claude-code").tiers() == { + "fast": ("haiku", "medium"), + "balanced": ("sonnet", "high"), + "deep": ("opus", "xhigh"), + } + assert Account("chatgpt", "codex").tiers() == { + "fast": ("gpt-6-luna", "medium"), + "balanced": ("gpt-6.1-sol", "high"), + "deep": ("gpt-6.1-sol", "xhigh"), + } + own = Account("chatgpt", "codex", models={"deep": "gpt-6-astra"}, efforts={"fast": "low"}).tiers() + assert own["deep"] == ("gpt-6-astra", "xhigh") and own["fast"] == ( + "gpt-6-luna", + "low", + ) # a model keeps its tier's + + +def test_models_lists_what_each_cli_takes() -> None: + assert models("claude-code") == ("haiku", "sonnet", "opus") + assert models("codex") == ("gpt-6.1-sol", "gpt-6-astra", "gpt-6-sol", "gpt-6-luna") + + +def test_every_default_model_is_one_that_it_suggests() -> None: + for engine in ENGINES: + for tier, (model, _) in defaults(engine).items(): + assert model in models(engine), (engine, tier) + + +def test_every_suggestion_is_a_name_that_an_account_takes_and_none_is_there_twice() -> None: + for engine in ENGINES: + assert all(MODEL.fullmatch(name) for name in models(engine)), engine + assert len(set(models(engine))) == len(models(engine)), engine + + +def test_an_engine_with_no_list_suggests_nothing() -> None: + # an API key's models come from models.dev, a local server's are the owner's, and a stand-in has none + for engine in ("opencode", "scripted", "nothing", "_about", "models", "efforts"): + assert models(engine) == (), engine + + +def test_the_sections_of_the_file_are_not_engines() -> None: + for section in SECTIONS: + with pytest.raises(ValueError, match="no default tiers"): + defaults(section) + data = json.loads(resources.files("lastcall.data").joinpath("tiers.json").read_text(encoding="utf-8")) + for engine in set(data) - set(SECTIONS): + assert set(data[engine]) == set(TIERS), engine # what is left is an engine, with its three tiers + assert set(data["models"]) - {"_about"} <= set(data) - set(SECTIONS) # a list only for an engine that has tiers + assert all(data[section]["_about"] for section in ("models", "efforts")) # each section says what it is diff --git a/tests/test_web.py b/tests/test_web.py index 88cde8c..2b1e25b 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -3,7 +3,7 @@ import json import re import threading -from collections.abc import Callable, Collection, Mapping +from collections.abc import Callable, Collection, Iterable, Mapping from dataclasses import replace from pathlib import Path from typing import Any, cast @@ -18,6 +18,7 @@ from werkzeug.test import TestResponse import lastcall.web as site +from lastcall import catalog from lastcall.accounts import Account from lastcall.github import App, GitHub, Response from lastcall.notify import Notifier @@ -40,6 +41,7 @@ hash_password, start_session, ) +from test_catalog import CATALOG, ModelsDev, model from test_loki import FakeLoki from test_notify import Outbox from test_projects import APP, FakeGitHub @@ -59,6 +61,12 @@ def __call__(self) -> float: return self.now +def offline(req: Request) -> Response: + """The `fetch` of a test that gives none: nothing reaches the network (models.dev is never asked from a test), and + a lookup that needs it fails as it does on a host with no way out.""" + raise OSError(f"offline: {req.full_url}") + + class Web: def __init__( self, tmp_path: Path, origins: tuple[str, ...] = (), github: FakeGitHub | None = None, **options: Any @@ -79,6 +87,7 @@ def __init__( github=(lambda: cast(GitHub, github)) if github else None, background=options.pop("background", lambda work: work()), notifier=Notifier(self.secrets, self.store, send=self.outbox, log=self.logs.append), + fetch=options.pop("fetch", offline), **options, ) self.client = self.app.test_client() @@ -743,7 +752,7 @@ def test_models_and_effort_from_the_web(web: Web) -> None: assert row is not None and row.account().tiers() == { "fast": ("haiku", "medium"), "balanced": ("claude-sonnet-5-5", "high"), - "deep": ("opus", "high"), + "deep": ("opus", "xhigh"), } for bad in ({"effort-deep": "ultra"}, {"model-deep": "opus; rm -rf /"}): assert web.client.post("/accounts/max/tiers", data={**form, **bad}, headers=SITE).status_code == 400 @@ -3067,3 +3076,420 @@ def test_without_popover_support_neither_the_question_nor_the_trash_button_shows assert ".trash, .bubble.confirm { display: none; }" in mine # never a Remove button one click away in a card # an author's display on a popover overrides the browser's, and would show the closed question assert not re.search(r"\.bubble\.confirm\s*\{[^}]*\bdisplay:", css.replace(mine, "")) + + +# the model fields suggest names: the CLI's own, or models.dev's for an API key (test_catalog's fakes never ask it) + + +def account_html(page: str, account: str) -> str: + return page.split(f'id="account-{account}"')[1].split("")[0] + + +def datalists(page: str) -> dict[str, list[str]]: + """The datalists of a page, each with the values of its options.""" + return { + found.group(1): [html.unescape(v) for v in re.findall(r'', found.group(2))] + for found in re.finditer(r'(.*?)', page, re.S) + } + + +def lists_of(section: str, fields: Iterable[str]) -> list[str | None]: + """The datalist that each of these inputs points to with `list`: None for an input with none.""" + found: list[str | None] = [] + for name in fields: + tag = re.search(rf']*name="{name}"[^>]*>', section) + assert tag, name + attribute = re.search(r' list="([^"]+)"', tag.group(0)) + found.append(attribute.group(1) if attribute else None) + return found + + +def models_web(tmp_path: Path, fetch: ModelsDev | None = None, accounts: bool = True) -> tuple[Web, ModelsDev]: + """A signed-in web UI whose `fetch` is models.dev's fake, with one account of each kind.""" + from lastcall.accounts import tier_models + + fetch = fetch or ModelsDev() + web = Web(tmp_path, fetch=fetch) + web.set_up() + if not accounts: + return web, fetch + web.store.add_account("max", "claude-code") + web.store.add_account("chatgpt", "codex") + for name, provider in (("router", "openrouter"), ("direct", "anthropic"), ("sky", "openai")): + web.store.add_account( + name, "opencode", provider=provider, models=tier_models("a-model"), policy="budget", budget_usd=20 + ) + web.store.add_account("local", "opencode", provider="ollama", models=tier_models("qwen3-coder"), policy="manual") + return web, fetch + + +# what test_catalog's catalog lists that a step can run on, in the order of the datalists +OPENROUTER = ["anthropic/claude-sonnet-5.5", "beta/new-model", "qwen/qwen3-coder"] +OPENROUTER += ["Qwen/Qwen3-coder-flash", "qwen/qwen3-coder:free"] +EVERY_PROVIDER = sorted( + [*OPENROUTER, "claude-opus-5-5", "claude-sonnet-5-5", "gpt-6.1-sol"], key=lambda n: (n.lower(), n) +) + + +def test_the_model_field_of_each_tier_suggests_the_models_of_its_own_engine(tmp_path: Path) -> None: + web, fetch = models_web(tmp_path) + page = web.client.get("/accounts").get_data(as_text=True) + found = datalists(page) + assert found["models-claude-code"] == ["haiku", "sonnet", "opus"] # lastcall's own list, from data/tiers.json + assert found["models-codex"] == ["gpt-6.1-sol", "gpt-6-astra", "gpt-6-sol", "gpt-6-luna"] + assert found["models-opencode-openrouter"] == OPENROUTER # what models.dev lists, as test_catalog filters it + assert found["models-opencode-anthropic"] == ["claude-opus-5-5", "claude-sonnet-5-5"] + assert found["models-opencode-openai"] == ["gpt-6.1-sol"] + assert found["models-keys"] == EVERY_PROVIDER + assert len(found) == 6 # none for the local model server: the web UI cannot reach it + fields = [f"model-{tier}" for tier in TIERS] + for account, wanted in ( + ("max", "models-claude-code"), + ("chatgpt", "models-codex"), + ("router", "models-opencode-openrouter"), + ("direct", "models-opencode-anthropic"), + ("sky", "models-opencode-openai"), + ): + assert lists_of(account_html(page, account), fields) == [wanted] * 3, account + assert lists_of(account_html(page, "local"), fields) == [None] * 3 + assert all(page.count(f'') == 1 for name in found) # once each, whatever points to it + assert len(fetch.asked) == 1 # one request, for every provider + + +def test_two_accounts_of_a_kind_share_the_one_datalist_of_their_provider(tmp_path: Path) -> None: + from lastcall.accounts import tier_models + + web, _ = models_web(tmp_path) + web.store.add_account( + "router-2", "opencode", provider="openrouter", models=tier_models("a-model"), policy="budget", budget_usd=5 + ) + web.store.add_account("max-2", "claude-code") + page = web.client.get("/accounts").get_data(as_text=True) + assert page.count('') == 1 + assert page.count('') == 1 + assert page.count('list="models-opencode-openrouter"') == 6 and page.count('list="models-claude-code"') == 6 + + +def test_the_fold_says_where_its_suggestions_come_from(tmp_path: Path) -> None: + web, _ = models_web(tmp_path) + page = web.client.get("/accounts").get_data(as_text=True) + said = { + "max": "The Model field suggests the names that Claude Code takes. Pick one, or type any other.", + "chatgpt": "The Model field suggests the names that Codex takes. Pick one, or type any other.", + "router": "The Model field suggests what models.dev lists for OpenRouter: models that use tools and have a " + "price. Pick one, or type any other.", + "direct": "The Model field suggests what models.dev lists for Anthropic API: models that use tools and " + "have a price. Pick one, or type any other.", + } + for account, text in said.items(): + assert f'

{text}

' in account_html(page, account), account + assert "The Model field suggests" not in account_html(page, "local") + spanish = web.client.get("/accounts", headers={"Accept-Language": "es"}).get_data(as_text=True) + assert ( + "El campo Modelo sugiere los nombres que acepta Claude Code. Elige uno o escribe cualquier otro." in spanish + ) + assert ( + "El campo Modelo sugiere lo que models.dev lista para OpenRouter: modelos que usan herramientas y tienen " + in (account_html(spanish, "router")) + ) + + +def test_the_form_that_adds_an_api_key_suggests_the_models_of_every_provider(tmp_path: Path) -> None: + web, _ = models_web(tmp_path, accounts=False) # the form is there with no card above it + english = web.client.get("/accounts").get_data(as_text=True) + spanish = web.client.get("/accounts", headers={"Accept-Language": "es"}).get_data(as_text=True) + assert datalists(english) == {"models-keys": EVERY_PROVIDER} + for page in (english, spanish): + form = page.split('
')[1].split('
')[1] + assert lists_of(local, ["model"]) == [None] # a local model server has no list to suggest from + hint = english.split('id="key-model-hint"')[1].split("

")[0] + assert ( + "The field suggests what models.dev lists for all the providers, and OpenRouter writes the maker first" + in hint + ) + assert "Pick one from the provider you chose, or type any other." in hint + assert "El campo sugiere lo que models.dev lista para todos los proveedores" in spanish + + +def test_without_models_dev_the_form_that_adds_a_key_is_as_it_was(web: Web) -> None: + web.set_up() # the web fixture's fetch is the offline one of the tests + page = web.client.get("/accounts").get_data(as_text=True) + assert "')[1].split('
None: + web, _ = models_web(tmp_path) + page = web.client.get("/accounts").get_data(as_text=True) + for account in ("max", "chatgpt"): # the same for both: medium for fast, high for balanced, xhigh for deep + found = re.findall(r'', account_html(page, account)) + assert found == ["medium", "high", "xhigh"], account + spanish = web.client.get("/accounts", headers={"Accept-Language": "es"}).get_data(as_text=True) + found = re.findall(r'', account_html(spanish, "chatgpt")) + assert found == ["medium", "high", "xhigh"] + + +def test_models_dev_is_asked_once_a_day_for_every_provider_with_a_wait_of_5_seconds(tmp_path: Path) -> None: + web, fetch = models_web(tmp_path) + for _ in range(3): + assert web.client.get("/accounts").status_code == 200 + assert len(fetch.asked) == 1 # three loads, one request + (req,) = fetch.asked + assert site.CATALOG_TIMEOUT == 5 and getattr(req, "timeout", None) == 5 + assert (req.get_header("User-agent") or "").startswith("lastcall/") and req.get_header( + "Accept-encoding" + ) == "gzip" + web.clock.now += DAY - 1 + web.client.get("/accounts") + assert len(fetch.asked) == 1 + web.clock.now += 2 + fetch.answer = {"openrouter": {"models": {"new/model": model()}}} # models.dev changed its mind + page = web.client.get("/accounts").get_data(as_text=True) + assert len(fetch.asked) == 2 + found = datalists(page) + assert found["models-opencode-openrouter"] == ["new/model"] and found["models-keys"] == ["new/model"] + assert "models-opencode-anthropic" not in found # it lists none now, and is not asked again before the day is out + web.clock.now += DAY - 1 + web.client.get("/accounts") + assert len(fetch.asked) == 2 + + +def test_only_the_accounts_page_asks_models_dev(tmp_path: Path) -> None: + web, fetch = models_web(tmp_path) + for path in ("/", "/projects", "/settings", "/github", "/login", "/healthz"): + web.client.get(path) + assert fetch.asked == [] + web.client.get("/accounts") + assert len(fetch.asked) == 1 + + +FAILURES: list[Exception | tuple[int, bytes]] = [ + OSError("no route to host"), + TimeoutError("timed out"), + RuntimeError("boom"), + (403, b"Forbidden"), + (500, b""), + (200, b"a captive portal"), + (200, b"[]"), + (200, b"null"), +] + + +@pytest.mark.parametrize("answer", FAILURES) +def test_when_models_dev_does_not_answer_the_fields_suggest_nothing_and_the_page_goes_on( + tmp_path: Path, answer: Exception | tuple[int, bytes] +) -> None: + web, _ = models_web(tmp_path, ModelsDev(answer)) + response = web.client.get("/accounts") + page = response.get_data(as_text=True) + assert response.status_code == 200 + assert sorted(datalists(page)) == ["models-claude-code", "models-codex"] # what lastcall knows itself stays + assert lists_of(account_html(page, "router"), ["model-fast", "model-deep"]) == [None, None] + form = page.split('
')[1].split('
None: + web, fetch = models_web(tmp_path, ModelsDev(OSError("no route to host"))) + for _ in range(3): + assert web.client.get("/accounts").status_code == 200 + assert len(fetch.asked) == 1 # three loads, one wait + web.clock.now += 599 + web.client.get("/accounts") + assert len(fetch.asked) == 1 + web.clock.now += 2 + fetch.answer = CATALOG + page = web.client.get("/accounts").get_data(as_text=True) + assert len(fetch.asked) == 2 and datalists(page)["models-opencode-openrouter"] == OPENROUTER + web.clock.now += 3600 # an answer is kept the whole day + web.client.get("/accounts") + assert len(fetch.asked) == 2 + + +def test_what_models_dev_says_is_bounded(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(catalog, "MAX_MODELS", 3) + many = { + name: {"models": {f"{name}/m{n:02}": model() for n in range(10, 0, -1)}} + for name in ("openrouter", "anthropic", "openai", "ollama-cloud", "mistral", "groq") + } + web, fetch = models_web(tmp_path, ModelsDev(many)) + found = datalists(web.client.get("/accounts").get_data(as_text=True)) + assert found["models-opencode-openrouter"] == ["openrouter/m01", "openrouter/m02", "openrouter/m03"] # 3 at most + assert len(found["models-keys"]) == 9 # of the three providers that an engine offers: not of the other three + assert not any("mistral" in name or "groq" in name for name in found["models-keys"]) + assert len(fetch.asked) == 1 + + +def test_a_catalog_over_the_limit_is_refused_and_the_page_goes_on( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(catalog, "MAX_BYTES", 500) + web, _ = models_web(tmp_path) + response = web.client.get("/accounts") + found = datalists(response.get_data(as_text=True)) + assert response.status_code == 200 and "models-opencode-openrouter" not in found + assert any("too big" in line for line in web.logs) + + +# the settings of a project say plainly what each option adds, and an account says what its language is for + +HINTS = { + "en": { + "tests": "Some bursts write tests instead of auditing: pull requests that add tests for behavior no test " + "checks.", + "deps": "Some bursts upgrade dependencies instead of auditing: pull requests for what needs more than a new " + "version number.", + "merge": "When you approve one of its pull requests on GitHub, lastcall merges it once its checks pass. " + "Without this, you merge it yourself.", + "watch": "For a day after a merge, lastcall watches only the merged commit's checks and the health address " + "below, not your logs or error tracker. If either breaks, it proposes a revert and pauses the project.", + }, + "es": { + "tests": "Algunas ráfagas escriben tests en lugar de auditar: pull requests que añaden tests para el " + "comportamiento que ningún test comprueba.", + "deps": "Algunas ráfagas actualizan dependencias en lugar de auditar: pull requests para lo que necesita " + "algo más que un número de versión nuevo.", + "merge": "Cuando apruebas uno de sus pull requests en GitHub, lastcall lo fusiona en cuanto pasan sus " + "checks. Sin esto, lo fusionas tú.", + "watch": "Durante un día tras una fusión, lastcall vigila solo los checks del commit fusionado y la " + "dirección de salud de abajo, no tus logs ni tu tracker de errores. Si alguno falla, propone una reversión " + "y pausa el proyecto.", + }, +} + + +def bubble(page: str, name: str) -> str: + """The text of a help bubble, as a reader sees it.""" + found = re.search(rf'
', page, re.S) + assert found, name + return html.unescape(re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", found.group(0)))).strip() + + +@pytest.mark.parametrize("lang", ["en", "es"]) +def test_each_project_option_says_plainly_what_it_adds(github_web: Web, lang: str) -> None: + web = github_web + web.store.add_project("acme/api") + page = web.client.get("/projects", headers={"Accept-Language": lang}).get_data(as_text=True) + found = re.findall( + r'

(.*?)

', card(page, "acme/api"), re.S + ) + assert {name: html.unescape(text) for name, text in found} == HINTS[lang] + + +@pytest.mark.parametrize( + ("lang", "label", "option", "hint"), + [ + ( + "en", + "Language of its pull requests and issues", + "Same as the account", + "“Same as the account” uses the language of whichever account works on it, set on the Accounts page.", + ), + ( + "es", + "Idioma de sus pull requests e issues", + "Igual que la cuenta", + "«Igual que la cuenta» usa el idioma de la cuenta que trabaje en él, el que se elige en Cuentas.", + ), + ], +) +def test_a_projects_language_says_what_it_is_and_what_its_default_means( + github_web: Web, lang: str, label: str, option: str, hint: str +) -> None: + web = github_web + web.store.add_project("acme/api") + page = web.client.get("/projects", headers={"Accept-Language": lang}).get_data(as_text=True) + mine = card(page, "acme/api") + assert f' + + {%- endif %} + + {%- endfor %} + + {%- else %} +

{{ t("web.logs.read_none") }}

+ {%- endif %} +
+ + {%- if v.suggested %} +
+
<{{ level }} class="part-title" id="{{ kind }}-suggested-title">{{ t("web.logs.suggested_title") }}{{ help(hb, "settings-logs_suggested", t("web.logs.suggested_title"), ["settings.logs_suggested"], true) }}
+
+ {{ hidden(extra) }} +
+ +
+
+
    + {%- for s in v.suggested %} + {%- set n = loop.index %} +
  • +
    + + +
    +

    {{ t("web.logs.for_project", project=s.suggested) }} + {%- if s.state %} {{ s.state }}{% endif %} + {%- for host in s.hosts %} {{ host }}{% endfor %}

    + {%- if kind == "loki" %}{{ clue(s, kind) }}{% endif %} + + +
    • + {%- endfor %} +
+
+
+ + +
+
+
+ {%- endif %} + + {%- if v.total %} +
+ {{ tn("web." ~ kind ~ ".others", v.total) }} +
+ {%- if v.filterable %} + + {%- endif %} + {%- if v.query %} +

{{ tn("web.logs.matching", v.total, shown=v.others|length, query=v.query) }} {{ t("web.suggest.show_all") }}

+ {%- endif %} + {%- if v.others %} +
+ {{ hidden(extra) }} +

{{ t("web.logs.others_hint") }}

{{ help(hb, "settings-logs_project", t("web.logs.project"), ["settings.logs_project"], true) }}
+
    + {%- for o in v.others %} + {%- set n = loop.index %} +
  • +
    + {{ identity(o) }} + {{ clue(o, kind) }} + {%- if o.gone %} +

    {{ t("web.logs.gone", project=o.gone) }}

    + {%- endif %} +
    + + +
    • + {%- endfor %} +
+ + {%- for repo in v.repos %}{% endfor %} + + +
+ {%- endif %} +
+
+ {%- endif %} +
+{%- endmacro %} diff --git a/src/lastcall/web/templates/projects.html b/src/lastcall/web/templates/projects.html index b63480f..4f79920 100644 --- a/src/lastcall/web/templates/projects.html +++ b/src/lastcall/web/templates/projects.html @@ -11,7 +11,7 @@

{{ t("web.projects.title") }}

{{ t("web.projects.intro") }}

- {%- if v.projects %}

{{ t("web.add.title") }}

{% endif %} + {%- if v.known %}

{{ t("web.add.title") }}

{% endif %}
{% if done %}

{{ done }}

{% endif %} @@ -73,10 +73,13 @@

{{ t("web.suggest.title") }}

+ {#- the search and the filter of the cards stay as they are -#} + {%- if v.search %}{% endif %} + {%- if v.show != "all" %}{% endif %} {%- endif %} {%- if v.query %} -

{{ tn("web.suggest.matching", v.total, shown=v.shown, query=v.query) }} {{ t("web.suggest.show_all") }}

+

{{ tn("web.suggest.matching", v.total, shown=v.shown, query=v.query) }} {{ t("web.suggest.show_all") }}

{%- endif %} {%- if v.shown %}
@@ -119,11 +122,39 @@

{{ t("web.suggest.title") }}

{%- endmacro %} -{% if not v.projects %} +{% if not v.known %}

{{ t("web.projects.empty") if v.github_ready else t("web.projects.none") }}

{{ add_section() }} {% else %}
+ {#- With many projects: a search by name and the filters, above the cards. Each link keeps the search and the filter of the + suggestions below, and all of them bring the page back here (#find). Their counts are those of the current search. -#} + {%- if v.can_search %} +
+ + +
+ + +
+ {%- if v.show != "all" %}{% endif %} + {%- if v.query %}{% endif %} + + + {%- if v.search or v.show != "all" %} +

{{ tn("web.projects.showing", v.known, shown=v.projects|length) }}

+ {%- endif %} +
+ {%- endif %} + {%- if not v.projects %} +

{{ t("web.projects.no_match") }} {{ t("web.projects.show_all") }}

+ {%- else %}
{%- for p in v.projects %} {%- set n = loop.index %} @@ -183,12 +214,22 @@

{{ t("web.project.remove_title", repo=p.repo) }}{{ p.busy }}

{%- elif v.accounts|length > 1 %} + {#- With many projects the group starts folded, like the settings below it, so that the cards are short: its title is + the summary that opens it, and its "?" moves inside, after a line that says what the group does. -#} + {%- if v.many %} +
+ {{ t("web.project.run_now") }} + {%- endif %}
-
+
+ {%- if v.many %} +

{{ t("web.project.run_summary") }}

{{ help(hb, "projects-run_now", t("web.project.run_now"), ["projects.run_now"], true) }}
+ {%- else %} {{ t("web.project.run_now") }}{{ help(hb, "projects-run_now", t("web.project.run_now"), ["projects.run_now"]) }} + {%- endif %}
-
{{ help(hb, "projects-account", t("web.project.account"), account_keys) }}
+
{{ help(hb, "projects-account", t("web.project.account"), account_keys, v.many) }}
@@ -200,12 +241,15 @@

{{ t("web.project.remove_title", repo=p.repo) }}{{ t("web.project.deps_now") }}

    -

  • {{ t("web.project.improve_hint") }}

    {{ help(hb, "projects-improve", t("web.project.improve"), ["projects.improve"]) }}
    • +

  • {{ t("web.project.improve_hint") }}

    {{ help(hb, "projects-improve", t("web.project.improve"), ["projects.improve"], v.many) }}

  • {{ t("web.project.tests_hint") }}

  • {{ t("web.project.deps_hint") }}

+ {%- if v.many %} +
+ {%- endif %} {%- else %}

{{ t("web.project.no_accounts") }}

{%- endif %} @@ -270,6 +314,7 @@

{{ t("web.project.remove_title", repo=p.repo) }} {%- endfor %}

+ {%- endif %}
{{ add_section() }} {% endif %} diff --git a/src/lastcall/web/templates/settings.html b/src/lastcall/web/templates/settings.html index 4243fef..4feb0de 100644 --- a/src/lastcall/web/templates/settings.html +++ b/src/lastcall/web/templates/settings.html @@ -2,6 +2,7 @@ {% block title %}{{ t("web.settings.title") }}{% endblock %} {% block main %} {%- from "_help.html" import help, bubbles with context %} +{%- from "_lists.html" import lists with context %} {%- set hb = namespace(items=[], seen=[]) %} {%- set event_help = ("decisions", "merged", "reverted", "paid", "quota", "burst") %}
@@ -317,36 +318,15 @@

{{ t("web.logs.title") }}

  • {{ t("web.logs.step1") }}
  • {{ t("web.logs.step2") }}
    • + {%- if logs_error %} + + {%- endif %} {%- if v.logs.error %} - {%- elif not v.logs.rows %} + {%- elif not v.logs.count %}

    {{ t("web.logs.none") }}

    {%- else %} -
    - -
      - {%- for c in v.logs.rows %} -
    • -

      {{ c.name }} {{ c.state }}

      -

      {{ c.image }}

      - {%- if c.label %} -

      {{ t("web.logs.from_label", project=c.label) }}

      - {%- else %} - - -
      {{ help(hb, "settings-logs_project", t("web.logs.project"), ["settings.logs_project"]) }}
      - - {%- if c.suggested %}

      {{ t("web.logs.suggested", project=c.suggested) }}

      {% endif %} - {%- if c.gone %}

      {{ t("web.logs.gone", project=c.gone) }}

      {% endif %} - {%- endif %} -
      • - {%- endfor %} -
    - -
    + {{ lists(hb, "logs", v.logs, url_for('logs_settings'), "h4", (), [("loki_q", v.loki.query)] if v.loki.query else []) }} {%- endif %}
    {{ t("web.logs.label_title") }} @@ -416,32 +396,11 @@

    {{ t("web.logs.title") }}

    - {%- if v.loki.label and not v.loki.rows %} + {%- if v.loki.label and not v.loki.count %}

    {{ t("web.loki.none") }}

    - {%- elif v.loki.rows %} -
    - - -

    {{ t("web.help.settings.loki_sources.title") }}

    {{ help(hb, "settings-loki_sources", t("web.help.settings.loki_sources.title"), ["settings.loki_sources"]) }}
    -
      - {%- for s in v.loki.rows %} -
    • -

      {{ s.name }}{% for host in s.hosts %} {{ host }}{% endfor %}

      - {%- if s.compose_project %}

      {{ t("web.loki.compose", project=s.compose_project) }}

      {% endif %} - - - - - {%- if s.suggested %}

      {{ t("web.logs.suggested", project=s.suggested) }}

      {% endif %} - {%- if s.gone %}

      {{ t("web.logs.gone", project=s.gone) }}

      {% endif %} -
      • - {%- endfor %} -
    - -
    + {%- elif v.loki.count %} +

    {{ t("web.help.settings.loki_sources.title") }}

    {{ help(hb, "settings-loki_sources", t("web.help.settings.loki_sources.title"), ["settings.loki_sources"]) }}
    + {{ lists(hb, "loki", v.loki, url_for('loki_sources'), "h5", [("label", v.loki.label)], [("logs_q", v.logs.query)] if v.logs.query else []) }} {%- endif %} {%- endif %}
    diff --git a/tests/test_lists.py b/tests/test_lists.py new file mode 100644 index 0000000..72fd3a5 --- /dev/null +++ b/tests/test_lists.py @@ -0,0 +1,769 @@ +"""Long lists in the web UI. Settings shows the containers of the host and the sources in Loki in three parts, so that +dozens of them stay short: what lastcall reads now, what it suggests, and the rest in a fold with a filter by name and +one field to give each a project. The Projects page gets a search by name, filters with their counts, and, with many +projects, a Run now that starts folded.""" + +import html +import re +from pathlib import Path +from typing import Any + +from lastcall import logs, loki +from lastcall.recipes.audit import AuditReport +from lastcall.recipes.improve import ImproveReport +from lastcall.web import project_views +from test_loki import FakeLoki +from test_projects import FakeGitHub +from test_web import NOW, OTHERS, SITE, Web, help_buttons_inside_a_paragraph, suggest_many + +ROOT = Path(__file__).resolve().parents[1] +PROJECTS = ("acme/shop", "acme/blog", "acme/api", "acme/Worker") + + +# the containers of a host + +NOT_MATCHED = "redis:7" + + +def box(name: str, image: str = NOT_MATCHED, compose: str = "", label: str = "") -> dict[str, str]: + return {"name": name, "image": image, "state": "running", "compose_project": compose, "label": label} + + +def host() -> list[dict[str, str]]: + """A host with dozens of containers: lastcall's own, one with a label, three that match a project, and 30 more.""" + return [ + box("lastcall-lastcall-1", "ghcr.io/ne0x86/lastcall-service:0.7.4", "lastcall"), + box("lastcall-supervisor-1", "ghcr.io/ne0x86/lastcall-supervisor:0.7.4", "lastcall"), + box("proxy-1", "tinyproxy", "lastcall"), # in lastcall's Compose project, whatever its name + box("blog", "nginx:1.27", label="ACME/blog"), + box("shop-web-1", "ghcr.io/acme/shop:1.2", "shop"), + box("shop-worker-1", "python:3.12", "shop"), + box("api-1", "ghcr.io/acme/api:3"), + *(box(f"cache-{n:02d}") for n in range(1, 31)), + ] + + +def settings_web(tmp_path: Path, listed: list[dict[str, str]] | None = None, **options: Any) -> Web: + """A signed-in web UI whose host lists `listed`, with the four projects.""" + found = host() if listed is None else listed + web = Web(tmp_path, containers=lambda: found, **options) + web.set_up() + for repo in PROJECTS: + web.store.add_project(repo) + return web + + +def section(page: str, name: str) -> str: + """The card of a section of the Settings page: "logs" or "loki".""" + return page.split(f'id="{name}">')[1].split("")[0] + + +def part(page: str, kind: str, which: str) -> str: + """One of the three parts of a list in the settings: "read", "suggested" or "others"; "" when it is not there.""" + body = section(page, kind).split("Or with a label in the compose file")[0] # what follows the lists is not theirs + marks = [ + ("read", f'id="{kind}-read-title"'), + ("suggested", f'id="{kind}-suggested-title"'), + ("others", '
    list[str]: + """The containers or sources that a part lists: those with a row or a tile of their own.""" + found = re.findall(r'([^<]+)|class="suggestion-name">([^<]+)', piece) + return [row or tile for row, tile in found] + + +def hidden(form: str) -> dict[str, str]: + """What a form sends without the owner touching it: its hidden fields.""" + found = re.findall(r'', form) + return {name: html.unescape(value) for name, value in found} + + +def row_of(form: dict[str, str], name: str) -> str: + """The number of the row that a form of the page gives to a container or a source.""" + return next(key for key, value in form.items() if value == name).removeprefix("name-") + + +def form_of(piece: str, css: str) -> str: + found = re.search(rf'(.*?)', piece, re.S) + assert found, css + return found.group(1) + + +def get(web: Web, path: str = "/settings", lang: str = "en") -> str: + return web.client.get(path, headers={"Accept-Language": lang}).get_data(as_text=True) + + +def post(web: Web, path: str, **fields: Any) -> Any: + return web.client.post(path, data={"csrf": web.csrf(), **fields}, headers=SITE) + + +def helps(page: str, topic: str) -> int: + """How many "?" buttons open the bubble of `topic` (its close button has a popovertarget too).""" + return len(re.findall(rf'popovertarget="help-projects-{topic}"(?! popovertargetaction)', page)) + + +def clean(page: str) -> None: + """What the content security policy would block, and a help button where it must not be.""" + assert " style=" not in page and "Read now

    " in read + assert 'acme/blog · its label decides' in read + assert 'acme/api · chosen here' in read + assert read.count('class="row-actions"') == 1 and 'aria-label="Stop reading cache-01"' in read + assert "Stop reading blog" not in read + # suggested: a checkbox with the container, and the project that fits + assert names(suggested) == ["shop-web-1", "shop-worker-1", "api-1"] + assert suggested.count("for acme/shop") == 2 and suggested.count("for acme/api") == 1 + assert re.findall(r'type="checkbox" name="tick" value="(\d)"', suggested) == ["1", "2", "3"] + assert ">Read the 3 suggested" in suggested and ">Read the ticked" in suggested + # the rest, in a fold that says how many: 29 caches (cache-01 is read), with the one that was said no to + assert "29 other containers" in others and '
    ' in others + assert names(others) == [f"cache-{n:02d}" for n in range(2, 31)] + assert others.count('list="logs-projects"') == 29 and 'name="logs_q"' in others # many: a filter by name + # one list of projects for the whole section, not one in each row + assert card.count("') == 1 + clean(page) + # nothing is read until the owner chooses it: a suggestion is no choice + listed = host() + assert logs.names_for("acme/shop", listed, logs.chosen(web.store)) == [] + assert logs.names_for("acme/blog", listed, logs.chosen(web.store)) == ["blog"] + assert logs.names_for("acme/api", listed, logs.chosen(web.store)) == ["cache-01"] + spanish = get(web, lang="es") + assert ">Se lee ahora" in spanish and ">Sugeridos" in spanish + assert "29 otros contenedores" in spanish and "· lo decide su etiqueta" in spanish + assert ">Leer los 3 sugeridos" in spanish and ">Dejar de leer" in spanish + + +def test_the_parts_that_have_nothing_to_show_are_short(tmp_path: Path) -> None: + web = settings_web(tmp_path, [box("cache-01"), box("cache-02")]) + page = get(web) + assert "Nothing is read yet." in part(page, "logs", "read") + assert part(page, "logs", "suggested") == "" and "2 other containers" in page + assert 'name="logs_q"' not in page # two: no filter to look for them + nothing = settings_web(tmp_path / "none", [box("lastcall-lastcall-1", compose="lastcall")]) + assert "No containers on this machine besides lastcall's own." in get(nothing) + assert 'class="parts"' not in get(nothing) + only = settings_web(tmp_path / "one", [box("shop-web-1", "ghcr.io/acme/shop")]) + assert ">Read the suggested one" in get(only) + assert '
    None: + web = settings_web(tmp_path) + form = hidden(form_of(part(get(web), "logs", "suggested"), "pick")) + assert (form["name-1"], form["project-1"], form["name-3"], form["project-3"]) == ( + "shop-web-1", + "acme/shop", + "api-1", + "acme/api", + ) + response = post(web, "/settings/logs", **{**form, "tick": ["1", "3"], "pick": "selected"}) + assert response.headers["Location"] == "/settings?done=logs#logs" + assert logs.chosen(web.store) == {"shop-web-1": "acme/shop", "api-1": "acme/api"} + page = get(web) + assert names(part(page, "logs", "read")) == ["blog", "shop-web-1", "api-1"] # read now, and not suggested again + assert names(part(page, "logs", "suggested")) == ["shop-worker-1"] + # "all" does not look at what was ticked + form = hidden(form_of(part(page, "logs", "suggested"), "pick")) + assert post(web, "/settings/logs", **{**form, "tick": ["9"], "pick": "suggested"}).status_code == 302 + assert logs.chosen(web.store) == {"shop-web-1": "acme/shop", "api-1": "acme/api", "shop-worker-1": "acme/shop"} + assert 'id="logs-suggested-title"' not in get(web) # nothing left to suggest: no such part + assert "Saved. lastcall reads the logs of the containers that have a project." in get(web, "/settings?done=logs") + + +def test_ticking_nothing_says_so_and_changes_nothing(tmp_path: Path) -> None: + web = settings_web(tmp_path) + form = hidden(form_of(part(get(web), "logs", "suggested"), "pick")) + for tick in ([], ["7"]): # none, or a row that the form does not have + response = post(web, "/settings/logs", **{**form, "tick": tick, "pick": "selected"}) + page = response.get_data(as_text=True) + assert response.status_code == 400 and "Tick at least one container." in section(page, "logs") + assert 'role="alert"' in section(page, "logs") and logs.chosen(web.store) == {} + spanish = web.client.post( + "/settings/logs", + data={"csrf": web.csrf(), **form, "pick": "selected"}, + headers={**SITE, "Accept-Language": "es"}, + ) + assert "Marca al menos un contenedor." in spanish.get_data(as_text=True) + + +def test_skipping_a_suggestion_or_stopping_a_container_says_no_so_that_it_is_not_suggested_again( + tmp_path: Path, +) -> None: + web = settings_web(tmp_path) + logs.choose(web.store, {"cache-01": "acme/api"}) + page = get(web) + form = hidden(form_of(part(page, "logs", "suggested"), "pick")) + assert post(web, "/settings/logs", **{**form, "tick": ["2"], "pick": "skip"}).status_code == 302 + assert logs.chosen(web.store) == {"cache-01": "acme/api", "shop-worker-1": ""} + page = get(web) + assert names(part(page, "logs", "suggested")) == ["shop-web-1", "api-1"] + assert "shop-worker-1" in names(part(page, "logs", "others")) and "shop-worker-1" not in part( + page, "logs", "read" + ) + # Stop reading is a small form of its own: it posts the container and nothing else but the button + stop = hidden(form_of(part(page, "logs", "read"), "row-actions")) + assert stop == {"csrf": stop["csrf"], "pick": "stop", "name-1": "cache-01"} + assert post(web, "/settings/logs", **{k: v for k, v in stop.items() if k != "csrf"}).status_code == 302 + assert logs.chosen(web.store) == {"cache-01": "", "shop-worker-1": ""} + page = get(web) + assert names(part(page, "logs", "read")) == ["blog"] and "cache-01" in names(part(page, "logs", "others")) + assert logs.names_for("acme/api", host(), logs.chosen(web.store)) == [] + + +def test_a_project_typed_for_a_container_is_checked_against_the_projects_whatever_the_case(tmp_path: Path) -> None: + web = settings_web(tmp_path) + form = hidden(form_of(part(get(web), "logs", "others"), "others-form")) + assert form["name-1"] == "cache-01" and form["name-30"] == "cache-30" + # the page offers the projects once, to type against; the field says which container it is for + assert '' in get(web) and 'aria-label="Project for cache-01"' in get(web) + typed = {"project-1": " ACME/blog ", "project-2": "acme/WORKER", "project-3": ""} + response = post(web, "/settings/logs", **{**form, **typed, "pick": "others"}) + assert response.headers["Location"] == "/settings?done=logs#logs" + assert logs.chosen(web.store) == { + "cache-01": "acme/blog", + "cache-02": "acme/Worker", + } # as the Projects page writes them + # one that is not a project: nothing is saved, the page says which, and keeps what was typed, with the fold open + before = logs.chosen(web.store) + wrong = post( + web, "/settings/logs", **{**form, "project-4": "acme/shp", "project-5": "acme/API", "pick": "others"} + ) + page = wrong.get_data(as_text=True) + assert wrong.status_code == 400 and logs.chosen(web.store) == before + assert "“acme/shp” is not one of your projects." in section(page, "logs") + fold = part(page, "logs", "others") + assert fold.startswith('
    ') and 'value="acme/shp"' in fold + assert 'value="acme/API"' in fold # what was typed in the others is back: the owner fixes one + clean(page) + spanish = web.client.post( + "/settings/logs", + data={"csrf": web.csrf(), **form, "project-4": "acme/shp", "pick": "others"}, + headers={**SITE, "Accept-Language": "es"}, + ) + assert "«acme/shp» no es uno de tus proyectos." in spanish.get_data(as_text=True) + # what the owner typed is the owner's own: escaped, and given back no longer than a repository's name + odd = post(web, "/settings/logs", **{**form, "project-6": "x", "project-7": "x" * 300, "pick": "others"}) + page = odd.get_data(as_text=True) + assert odd.status_code == 400 and "<b>x</b>" in page and "x" not in page + assert f'value="{"x" * 140}"' in page and "x" * 141 not in page + clean(page) + + +def test_the_other_containers_are_filtered_by_name_from_the_address(tmp_path: Path) -> None: + web = settings_web(tmp_path) + page = get(web, "/settings?logs_q=CACHE-1&loki_q=zz") # whatever the case + fold = part(page, "logs", "others") + assert fold.startswith('
    ') # a filter that is set: no match is hidden + assert names(fold) == [f"cache-{n}" for n in range(10, 20)] + assert "10 of 30 match “CACHE-1”." in fold and 'Show all' in fold + assert '