diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4cf2ac97..ee7c610e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -23,9 +23,18 @@ jobs: extra_nix_config: | experimental-features = nix-command flakes - - name: Run image, health, FTS, and persistence tests + - name: Run image, health, authentication, FTS, and persistence tests run: ./scripts/stack-test.sh + - name: Upload sanitized stack diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: stack-test-diagnostics + path: stack-test-artifacts/ + if-no-files-found: error + retention-days: 7 + - name: Publish images if: github.event_name == 'push' && github.ref == 'refs/heads/master' env: diff --git a/.github/workflows/starintel-schema-lock.yml b/.github/workflows/starintel-schema-lock.yml new file mode 100644 index 00000000..aa4d3dd4 --- /dev/null +++ b/.github/workflows/starintel-schema-lock.yml @@ -0,0 +1,29 @@ +name: Canonical StarIntel schema lock + +on: + pull_request: + push: + branches: + - master + - dev + - "agent/**" + paths: + - "schema/starintel-schema.lock.json" + - "scripts/check-starintel-schema-lock.py" + - ".github/workflows/starintel-schema-lock.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + research-node-schema: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: Verify canonical research-node schema + run: python scripts/check-starintel-schema-lock.py diff --git a/README.org b/README.org index 5a644add..c56cd5b6 100644 --- a/README.org +++ b/README.org @@ -1,119 +1,207 @@ -#+title: Readme +#+title: StarIntel Server +#+options: toc:2 -* Star Server -Starintel server is the new API server for interacting with the starintel system. +* StarIntel Server +StarIntel Server is the Common Lisp runtime for storing, routing, querying, and +recursively enriching StarIntel documents. +It combines: -- Features: - - rabbitmq - - actor model - - hackable +- CouchDB for durable documents, views, and full-text search. +- RabbitMQ topic exchanges for document and actor traffic. +- Sento actors for local concurrency, supervision boundaries, timers, and + message passing. +- A Ningle/Clack/Hunchentoot HTTP API. +- The =star-cl= document library and StarIntel specification adapters. +- Nix-built binaries, tests, and container images. +This repository is an experimental operator system, not a hardened public SaaS +service. -** *WARNINGS* -*** Project status -This repo is a mess! -This project is experimental, DO NOT EXPOSE TO THE WEB. +#+begin_quote +*Do not expose the HTTP API or RabbitMQ directly to the public Internet.* -*** Consumers and actors +The HTTP API currently has no authentication or authorization and sends +=Access-Control-Allow-Origin: *=. Put it behind an authenticated reverse proxy, +restrict network access, and treat the Lisp init file as trusted executable +code. +#+end_quote -In a future version the consumer that inserts will revert back to cl-gserver actor when I figure a good proto and learn more about sento. +** What the server does -Something along the lines of +A document normally moves through this pipeline: -#+begin_src lisp -(tell *couchdb* (:db "starintel" :id "0HJY....." :document (as-json (spec:create-user :dataset "github" :name "lost-rob0t")))) -#+end_src - -I will have to do more research to better utlize the sento system. +#+begin_example +HTTP/client/actor + | + v +documents.ingest. + | + v +CouchDB insert + _id/_rev enrichment + | + v +documents.new. + | + +--> local actor via TELL + +--> remote actor via RabbitMQ topic route + +--> derived documents and relations + | + +--> documents.ingest. (durable recursion) + `--> documents.new. (event-only fan-out) +#+end_example -** Documentation -*** Containers -See [[file:DOCKER.md][the Nix-built Compose stack guide]] for image builds, -secret setup, migration, backup, and upgrades. -*** HTTP -For http documentation: [[file:./docs/http-api-docs.org][Api Documentation]] -*** Setting up dev env -star-server uses nixpkgs to managment the dev shell, which is like a venv for this project. +Actors can emit more StarIntel documents, relations, targets, and actor events. +Those outputs can trigger more actors. This is *dataflow recursion*: the graph +expands through messages rather than recursive function calls. -it is not required +** Quick start: Nix-built Compose stack -First install the [[https://nixos.org/download/][Nix package manager]] and [[https://direnv.net/][Direnv]]. +Requirements: Nix with flakes, Docker Engine, Docker Compose v2, =curl=, =jq=, +and =openssl=. -Also Ensure you have [[https://www.quicklisp.org/beta/][Quicklisp]] installed also. +#+begin_src sh +cp .env.example .env +install -d -m 0700 secrets +openssl rand -base64 32 > secrets/couchdb_password +openssl rand -base64 48 > secrets/couchdb_secret +openssl rand -hex 24 | tr '[:lower:]' '[:upper:]' > secrets/erlang_cookie +openssl rand -base64 32 > secrets/rabbitmq_password +chmod 0600 secrets/* -#+Name: Setup dev env -#+begin_src sh :async :results output replace -git clone --recurse-submodules https://github.com/lost-rob0t/starintel-gserver.git star-server && cd star-server; -direnv allow . +nix run .#load-images +docker compose up --detach --wait +curl --fail http://127.0.0.1:5000/health #+end_src -Now nix should be pulling down everything and you will be placed inside a new shell with everything needed in $PATH. -*** Compiling From Source without nix +Default local endpoints: + +| Service | Address | +|---------------------+--------------------------| +| StarIntel HTTP API | http://127.0.0.1:5000 | +| CouchDB | http://127.0.0.1:5984 | +| RabbitMQ AMQP | 127.0.0.1:5672 | +| RabbitMQ management | http://127.0.0.1:15672 | + +See [[file:DOCKER.md][DOCKER.md]] for image builds, secrets, persistence, migration, backup, FTS, +and upgrades. + +** Submit a document + +The body must contain a =dtype=. The route chooses the RabbitMQ routing key but +does not currently inject or validate the body dtype. + +#+begin_src sh +curl --fail \ + --header 'Content-Type: application/json' \ + --request POST \ + --data '{ + "_id": "example-note", + "dataset": "demo", + "dtype": "note", + "sources": ["manual"], + "version": "0.8.0", + "dateAdded": 0, + "dateUpdated": 0, + "content": "first document" + }' \ + http://127.0.0.1:5000/new/document/note +#+end_src -Ensure you have [[https://www.quicklisp.org/beta/][Quicklisp]]. +The API acknowledges queue publication, not CouchDB persistence. Read the +document after the ingest consumer has processed it: -#+Name: Compile from source -#+begin_src shell :async :results output replace -git clone --recurse-submodules https://github.com/lost-rob0t/starintel-gserver.git star-server && cd star-server; -make build -make install +#+begin_src sh +curl --fail http://127.0.0.1:5000/document/example-note | jq #+end_src -*** Usage -#+Name: Usage -#+begin_src sh :async :results output replace -./star-server +** Build and test + +#+begin_src sh +nix build +nix run .#star-unit-tests #+end_src -#+RESULTS: Usage -#+begin_example -NAME: - star-server - Starintel unified API and document consuming service. +With CouchDB and RabbitMQ available: -USAGE: - star-server [global-options] [] [command-options] [arguments ...] +#+begin_src sh +nix run .#star-integration-tests +#+end_src -OPTIONS: - --help display usage information and exit - --version display version and exit +Full Nix image, health, FTS, restart, and persistence test: -COMMANDS: - start start the server +#+begin_src sh +./scripts/stack-test.sh +#+end_src -AUTHORS: - nsaspy +See [[file:docs/testing.md][docs/testing.md]]. -LICENSE: - GPL v3 +** Run from Common Lisp -#+end_example +The pinned Nix build is the supported reproducible path. For interactive +development: +#+begin_src sh +nix develop +sbcl --load run.lisp +#+end_src -#+Name: start -#+begin_src shell :async :results output replace -./star-server start --help +Build the executable: + +#+begin_src sh +nix build +./result/bin/star-server start --init ./example_configs/init.lisp #+end_src -#+RESULTS: start +The executable accepts: + #+begin_example -NAME: - star-server start - start the server +star-server start -i PATH +star-server start --init PATH +#+end_example -USAGE: - star-server [global-options] start [options] [arguments ...] +The same path can be supplied through =STAR_SERVER_INIT_FILE=. -OPTIONS: - --help display usage information and exit - --version display version and exit - -d, --debugger Enable Remote debugging - -i, --init Path to init file [default: ./init.lisp] [env: $STAR_SERVER_INIT_FILE] +** Documentation map -AUTHORS: - nsaspy +| Document | Contents | +|----------+----------| +| [[file:docs/index.org][docs/index.org]] | Documentation index and implementation status | +| [[file:docs/architecture.org][docs/architecture.org]] | Runtime structure, startup order, concurrency, and repository layout | +| [[file:docs/actors.org][docs/actors.org]] | Creating, registering, targeting, scheduling, and operating actors | +| [[file:docs/messaging.org][docs/messaging.org]] | RabbitMQ exchanges, queues, routing keys, recursion, delivery, and loop control | +| [[file:docs/document-spec.org][docs/document-spec.org]] | StarIntel 0.9 and legacy 0.8 documents, types, relations, IDs, metadata, and provenance | +| [[file:docs/configuration.org][docs/configuration.org]] | Environment, init files, secrets, advanced examples, and tuning | +| [[file:docs/http-api-docs.org][docs/http-api-docs.org]] | HTTP endpoint reference and examples | +| [[file:DOCKER.md][DOCKER.md]] | Nix-built container stack and operations | +| [[file:docs/testing.md][docs/testing.md]] | Unit, integration, and stack tests | -LICENSE: - GPL v3 +** Runtime status -#+end_example +The documentation distinguishes three states: + +- *Active*: loaded by =source/starintel-gserver.asd= and started by =star::main=. +- *Present but not active*: code exists in the repository but is not loaded by + the ASDF system or is not started by the current startup path. +- *Stub/experimental*: API or actor code exists but is incomplete. + +Important current limits: + +- HTTP ingestion does not enforce the strict StarIntel 0.9 schema. +- The server still uses legacy flat 0.8 constructors in parts of the actor code. +- =source/actor-systems/user-finder.lisp= and =user-hunt.lisp= are not loaded by + the ASDF system. +- The matcher actor framework is experimental; the URL extractor is loaded, but + a complete global pattern-dispatch loop is not wired. +- =/new/event/:id= is a stub. +- =*http-api-base-path*= and the HTTP certificate/key variables are not applied + by =start-http-api=. +- The second =/dataset-size= route definition replaces or shadows the first, + depending on Ningle route behavior. + +These are documented facts, not supported guarantees. + +** License + +See [[file:LICENSE][LICENSE]]. diff --git a/docker-compose.yml b/docker-compose.yml index 8b4b7958..389a6764 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,9 +99,16 @@ services: RABBITMQ_USER: ${RABBITMQ_USER:-starintel} RABBITMQ_PASSWORD_FILE: /run/secrets/rabbitmq_password HTTP_API_LISTEN_ADDRESS: 0.0.0.0 + STAR_AUTH_MODE: api-key + STAR_AUTH_DATABASE: ${STAR_AUTH_DATABASE:-starintel-gserver-auth} + STAR_AUTH_PEPPER_FILE: /run/secrets/auth_pepper + STAR_AUTH_BOOTSTRAP_SECRET_FILE: /run/secrets/auth_bootstrap_secret + STAR_AUTH_ALLOWED_ORIGINS: ${STAR_AUTH_ALLOWED_ORIGINS:-} secrets: - couchdb_password - rabbitmq_password + - auth_pepper + - auth_bootstrap_secret ports: - "${STAR_SERVER_BIND_ADDRESS:-127.0.0.1}:${STAR_SERVER_PORT:-5000}:5000" networks: @@ -127,6 +134,10 @@ secrets: file: ${CREDENTIALS_DIR:-./secrets}/erlang_cookie rabbitmq_password: file: ${CREDENTIALS_DIR:-./secrets}/rabbitmq_password + auth_pepper: + file: ${CREDENTIALS_DIR:-./secrets}/auth_pepper + auth_bootstrap_secret: + file: ${CREDENTIALS_DIR:-./secrets}/auth_bootstrap_secret volumes: clouseau_index: diff --git a/docker/star-server-entrypoint.sh b/docker/star-server-entrypoint.sh index ffdb6c68..0d613fa7 100755 --- a/docker/star-server-entrypoint.sh +++ b/docker/star-server-entrypoint.sh @@ -7,13 +7,23 @@ load_secret() { eval "file=\${$file_variable:-}" if [ -n "$file" ]; then + if [ ! -r "$file" ]; then + printf '%s\n' "Secret file for ${name} is not readable: ${file}" >&2 + exit 1 + fi value="$(cat "$file")" + if [ -z "$value" ]; then + printf '%s\n' "Secret file for ${name} is empty: ${file}" >&2 + exit 1 + fi export "$name=$value" fi } load_secret COUCHDB_PASSWORD load_secret RABBITMQ_PASSWORD +load_secret STAR_AUTH_PEPPER +load_secret STAR_AUTH_BOOTSTRAP_SECRET exec su-exec 65532:65532 \ /bin/star-server start -i /etc/starintel/init.lisp diff --git a/docker/star-server-init.lisp b/docker/star-server-init.lisp index 3c76fb63..f2de7285 100644 --- a/docker/star-server-init.lisp +++ b/docker/star-server-init.lisp @@ -1,14 +1,47 @@ (in-package :star) -(setf *couchdb-host* (or (uiop:getenv "COUCHDB_HOST") *couchdb-host*) +(setf *couchdb-host* + (or (uiop:getenv "COUCHDB_HOST") *couchdb-host*) *couchdb-default-database* (or (uiop:getenv "COUCHDB_DATABASE") *couchdb-default-database*) - *couchdb-user* (or (uiop:getenv "COUCHDB_USER") *couchdb-user*) + *couchdb-auth-database* + (or (uiop:getenv "STAR_AUTH_DATABASE") *couchdb-auth-database*) + *couchdb-user* + (or (uiop:getenv "COUCHDB_USER") *couchdb-user*) *couchdb-password* (or (uiop:getenv "COUCHDB_PASSWORD") *couchdb-password*) - *rabbit-address* (or (uiop:getenv "RABBITMQ_ADDRESS") *rabbit-address*) - *rabbit-user* (or (uiop:getenv "RABBITMQ_USER") *rabbit-user*) + *rabbit-address* + (or (uiop:getenv "RABBITMQ_ADDRESS") *rabbit-address*) + *rabbit-user* + (or (uiop:getenv "RABBITMQ_USER") *rabbit-user*) *rabbit-password* (or (uiop:getenv "RABBITMQ_PASSWORD") *rabbit-password*) *http-api-address* - (or (uiop:getenv "HTTP_API_LISTEN_ADDRESS") *http-api-address*)) + (or (uiop:getenv "HTTP_API_LISTEN_ADDRESS") *http-api-address*) + *http-cors-allowed-origins* + (or (split-comma-setting (uiop:getenv "STAR_AUTH_ALLOWED_ORIGINS")) + *http-cors-allowed-origins*) + *auth-mode* + (or (uiop:getenv "STAR_AUTH_MODE") *auth-mode*) + *auth-pepper* + (or (uiop:getenv "STAR_AUTH_PEPPER") *auth-pepper*) + *auth-bootstrap-secret* + (or (uiop:getenv "STAR_AUTH_BOOTSTRAP_SECRET") + *auth-bootstrap-secret*) + *auth-dev-bypass* + (not (null + (environment-boolean + "STAR_AUTH_DEV_BYPASS" + *auth-dev-bypass*))) + *auth-rotation-overlap-max-seconds* + (environment-integer + "STAR_AUTH_MAX_ROTATION_OVERLAP_SECONDS" + *auth-rotation-overlap-max-seconds*) + *auth-default-request-timeout-ms* + (environment-integer + "STAR_AUTH_DEFAULT_REQUEST_TIMEOUT_MS" + *auth-default-request-timeout-ms*) + *auth-max-request-timeout-ms* + (environment-integer + "STAR_AUTH_MAX_REQUEST_TIMEOUT_MS" + *auth-max-request-timeout-ms*)) diff --git a/docs/actors.org b/docs/actors.org new file mode 100644 index 00000000..f669ade4 --- /dev/null +++ b/docs/actors.org @@ -0,0 +1,307 @@ +#+title: Creating and operating actors +#+options: toc:3 + +* Actors + +StarIntel actors are message-driven transforms and operators. They consume +targets or documents and may emit documents, relations, updates, targets, or +actor events. + +** Actor kinds + +| Kind | Transport | Use | +|------+-----------+-----| +| Local Sento actor | In-process mailbox | Fast transforms, routing, timers, DB coordination | +| Rabbit actor service | RabbitMQ queue | Independent process/language, isolation, horizontal scale | +| Pattern subscriber | Local =tell= after a predicate match | Route selected documents to local actors | +| Recurring target actor | Timer plus local/remote delivery | Periodic collection or refresh | +| Actor-event receiver | Local or Rabbit event message | Append operational events to event DB | + +** Minimal local actor + +Create a file such as =actors/domain-enricher.lisp=: + +#+begin_src lisp +(in-package :star.actors) + +(define-actor (*domain-enricher* *sys*) + (lambda (message) + (let* ((source-id (jsown:val message "_id")) + (dataset (or (jsown:val-safe message "dataset") "default")) + (domain (jsown:val-safe message "target"))) + (when (and domain (not (string= domain ""))) + (let ((document + (jsown:new-js + ("_id" (format nil "domain:~a" (string-downcase domain))) + ("dataset" dataset) + ("dtype" "domain") + ("sources" (list "domain-enricher")) + ("version" "0.8.0") + ("dateAdded" (spec:unix-now)) + ("dateUpdated" (spec:unix-now)) + ("recordType" "A") + ("record" domain) + ("resolvedAddresses" (list))))) + (publish + *producer-agent* + :body (jsown:to-json document) + :routing-key "documents.ingest.domain" + :properties (list (cons :type "domain"))) + (log-actor-event + "domain-enricher" + :event-type "emitted" + :details (format nil "Emitted domain ~a" domain) + :source-id source-id)))))) + +(nhooks:add-hook + star:*actors-start-hook* + (lambda () + (register-actor "domain-enricher" *domain-enricher*))) +#+end_src + +The =define-actor= macro: + +- defines the actor variable; +- defines a =START-= function; +- creates the actor when the start hook runs; +- adds that start function to =*actors-start-hook*=. + +The macro accepts a =:register= keyword, but the current implementation does not +use it. Register the actor explicitly in a second hook as shown above. + +** Load a custom actor from the init file + +The init file runs before =start-actors=. It can load actor definitions whose +start functions will later run on =*actors-start-hook*=: + +#+begin_src lisp +(in-package :star) + +(load #P"/etc/starintel/actors/domain-enricher.lisp") +#+end_src + +The file is trusted executable code. Never load actor files from an +uncontrolled writable directory. + +For a built-in actor, add the file to the serial component order in +=source/starintel-gserver.asd= before files that call its exported functions. + +** Register and resolve actor names + +Registration maps an operator-facing string name to a Sento actor reference: + +#+begin_src lisp +(star.actors:register-actor "domain-enricher" + star.actors::*domain-enricher*) + +(star.actors:get-dest-actor "domain-enricher") +#+end_src + +Actor names in target documents must match the registered string exactly. + +** Target document + +Legacy target fields: + +| Field | Type | Meaning | +|-------+------+---------| +| =actor= | string | Registered local actor name or external actor name | +| =target= | string | Target value or source document ID | +| =delay= | integer | Recurrence interval in seconds | +| =recurring= | boolean | Schedule repeated delivery | +| =options= | array | Actor-defined options | +| common metadata | object fields | =_id=, dataset, dtype, sources, version, timestamps | + +Submit a target through HTTP: + +#+begin_src sh +curl --fail \ + --header 'Content-Type: application/json' \ + --request POST \ + --data '{ + "dataset": "demo", + "target": "example.org", + "delay": 3600, + "recurring": true, + "options": [{"record_types": ["A", "AAAA", "MX"]}] + }' \ + http://127.0.0.1:5000/new/target/domain-enricher +#+end_src + +The route sets =dtype= to =target= and =actor= from the URL. + +** Target routing + +When a persisted target arrives: + +1. the Rabbit target consumer tells the local target-router actor; +2. the router looks up =actor= in the actor index; +3. if found, it tells the local actor; +4. if missing, it publishes to + =actors..new.target= on the =documents= exchange; +5. if =recurring= is true on the first delivery, the router schedules a timer + and later repeats the local/remote route. + +This allows the same target format to address in-process and external actors. + +** Actor communication + +*** Local actor to local actor + +#+begin_src lisp +(tell destination message) +#+end_src + +Use this when both actors share the process. It avoids serialization and +RabbitMQ. + +*** Local actor to distributed pipeline + +#+begin_src lisp +(publish *producer-agent* + :body json + :routing-key "documents.ingest.user" + :properties (list (cons :type "user"))) +#+end_src + +Use the =documents.ingest.*= path for durable derived documents. + +*** Local actor event + +#+begin_src lisp +(log-actor-event + "domain-enricher" + :event-type "completed" + :details "Resolved target" + :source-id source-document-id) +#+end_src + +*** External actor + +Bind a queue to =actors..new.target=. Publish outputs back to the +=documents= exchange. See [[file:messaging.org][messaging.org]]. + +** Actor output contract + +An actor may return no documents. When it emits a document: + +- include a non-empty =dtype=; +- preserve the source dataset unless intentionally crossing datasets; +- generate a stable ID when the result is deterministic; +- append provenance rather than replacing it; +- publish to ingest for persistence; +- emit relations as first-class documents; +- make retries safe; +- do not ACK an external Rabbit delivery before required writes/publications + complete. + +** Creating relations + +The current =star-cl= constructor uses keyword arguments: + +#+begin_src lisp +(spec:new-relation + dataset + source-id + target-id + :predicate "derived-from" + :note "Created by domain-enricher") +#+end_src + +Do not use an old positional fourth predicate argument. Some experimental actor +files still contain calls written for an older signature. + +** Pattern subscribers + +A pattern contains: + +- a name; +- a predicate function; +- a list of actor references; +- a transient flag intended to describe generated messages. + +#+begin_src lisp +(add-pattern + (define-pattern + ("message-links" (list *domain-enricher*)) + (lambda (document) + (string= (jsown:val document "dtype") "message")))) +#+end_src + +When =notify-subs= is called and the predicate returns true, it tells each +subscriber the document. + +The matcher framework is currently experimental. The URL extractor starts from +a hook, but a complete actor that feeds every new document through +=*document-patterns*= is not evident in the active runtime. Do not assume that +adding a pattern alone creates a Rabbit subscription. + +** Recursive actor design + +A recursive actor should treat each invocation as one bounded graph expansion +step. + +Recommended message contract: + +#+begin_src json +{ + "extensions": { + "star_server": { + "trace_id": "01J...", + "root_id": "root-id", + "parent_id": "source-id", + "actor_path": ["seed", "domain-enricher"], + "depth": 1, + "hop": 2, + "max_depth": 8, + "max_hops": 32, + "input_hash": "sha256:...", + "output_hash": "sha256:..." + } + } +} +#+end_src + +Before emitting more work: + +1. verify =depth < max_depth= and =hop < max_hops=; +2. reject a non-repeatable actor already present in =actor_path=; +3. compute a deterministic output ID; +4. stop at a fixed point when output hash equals prior output hash; +5. add a provenance relation; +6. increment metadata; +7. emit a terminal actor event when stopping. + +The server deep-merges nested update objects, so this extension object can be +updated without deleting sibling metadata. + +** Error handling + +A local actor should catch errors it can classify, emit an actor event, and +either: + +- drop a permanently invalid message; +- emit an update marking the source failed; +- create a retry target with delay; +- let the actor runtime surface an unexpected failure. + +Do not create a zero-delay recursive retry loop. + +External actor services should use dead-letter exchanges or a retry queue with +backoff. The core server does not declare those queues for you. + +** Testing an actor + +Unit-test the receive function as a pure transform where possible. Integration +tests should verify: + +1. target publication; +2. actor delivery; +3. derived document publication; +4. CouchDB persistence; +5. relation/provenance creation; +6. duplicate delivery idempotency; +7. depth/hop termination; +8. recurring target behavior. + +Run the repository test commands from [[file:testing.md][testing.md]]. diff --git a/docs/architecture.org b/docs/architecture.org new file mode 100644 index 00000000..0e46de67 --- /dev/null +++ b/docs/architecture.org @@ -0,0 +1,204 @@ +#+title: StarIntel Server architecture +#+options: toc:3 + +* Architecture + +StarIntel Server is an event-driven Common Lisp service. CouchDB is the durable +corpus, RabbitMQ is the distributed message bus, and Sento actors are the local +message-passing runtime. + +** Component map + +#+begin_example + +----------------------+ +HTTP / CLI / services ->| HTTP API | + | Ningle + Clack | + +----------+-----------+ + | + | publish + v + +---------+----------+ + | RabbitMQ | + | topic: documents | + +----+----------+----+ + | | + ingest/update | actor-specific target + | | + v v + +---------+--+ +---+----------------+ + | Consumers | | External actors | + | lparallel | | Rabbit consumers | + +-----+------+ +--------------------+ + | + v + +-----+------+ + | CouchDB | + | documents | + | views/FTS | + +-----+------+ + | + | post-insert documents.new. + v + +-----+--------------------+ + | Sento actor system | + | local actors, timers, | + | index, DB agents, events | + +--------------------------+ +#+end_example + +** Startup order + +=star::main= performs the following operations in order: + +1. Resolve and load the init file. The init file is executable Common Lisp. +2. Create the global lparallel kernel with =*ingest-workers*= workers. +3. Initialize CouchDB: + - create the main database if missing; + - upsert design documents loaded from =source/views/*.json=; + - create the actor-event database if missing. +4. Start the Sento actor system. +5. Connect the pinned RabbitMQ producer agent. +6. Start internal CouchDB, actor-index, target, and timer actors. +7. Run =*actors-start-hook*= to start and register application actors. +8. Start the HTTP API and its RabbitMQ connection. +9. Start RabbitMQ ingest, update, and target consumers. +10. Start two actor-event RabbitMQ consumers. +11. Join all threads so the process remains alive. + +Startup is fail-fast: an uncaught initialization error terminates the process. + +** Concurrency model + +*** Sento actors + +Actors receive one message at a time through their mailbox. Actor references are +stored in an agent-backed name index. Local actor communication uses =tell= and +does not serialize through RabbitMQ. + +The actor system uses a pinned dispatcher sized from =*ingest-workers*=. The +producer and CouchDB agents are pinned so their mutable connection state is only +touched by one actor thread. + +*** lparallel consumers + +Each Rabbit consumer worker owns a RabbitMQ connection and channel. A worker: + +1. reads one delivery; +2. submits its handler to an lparallel channel; +3. waits for the handler result; +4. lets the handler ACK or NACK the delivery. + +Rabbit QoS prefetch is 200 per worker connection. + +*** HTTP threads + +Hunchentoot is started with: + +- maximum thread count: 50; +- maximum accept count: 100; +- request timeout: 300 seconds. + +CouchDB HTTP requests use a pool with 20 maximum open and 10 maximum idle +connections. + +** Durable data + +*** Main database + +=*couchdb-default-database*= defaults to =starintel=. It stores StarIntel +documents and CouchDB design documents. + +*** Actor-event database + +=*couchdb-event-log-database*= defaults to =starintel-event-source=. It stores +actor-event records separately from the main corpus. + +*** RabbitMQ state + +The Compose stack persists RabbitMQ state in =rabbitmq_data=. Document messages +are not a substitute for CouchDB backups. + +*** Full-text index + +Clouseau indexes are derived data. The Compose stack persists them in +=clouseau_index= for faster restart, but they can be rebuilt from CouchDB. + +** Repository layout + +| Path | Purpose | Runtime status | +|------+---------+----------------| +| =source/starintel-gserver.asd= | Loaded systems and component order | Active source of truth | +| =source/main.lisp= | CLI command, startup, REPL entry, debugger helper | Active | +| =source/gserver-settings.lisp= | Global defaults and environment reads | Active | +| =source/init-loader.lisp= | Init-file resolution, bootstrap, and load | Active | +| =source/actors.lisp= | Actor system, index, DB actors, target router, producer | Active | +| =source/rabbit.lisp= | Routing constants, ingest/update/target handlers | Active | +| =source/consumers/= | Generic and Rabbit stream consumers | Active | +| =source/producers/= | Persistent Rabbit producer abstraction | Active | +| =source/databases/couchdb.lisp= | DB initialization, JSON conversion, views | Active | +| =source/frontends/http-api.lisp= | HTTP routes and server startup | Active | +| =source/actor-systems/event-actor.lisp= | Actor-event storage and consumer | Active | +| =source/actor-systems/matcher-actor.lisp= | Pattern subscriptions and URL extractor | Experimental | +| =source/actor-systems/user-finder.lisp= | WhatsMyName actor | Not loaded by ASDF | +| =source/actor-systems/user-hunt.lisp= | Older WhatsMyName consumer | Not loaded by ASDF | +| =source/views/= | CouchDB map/reduce and FTS design documents | Installed at startup | +| =cli/= | API client and command-line client systems | Built separately | +| =ui/= | Separate Common Lisp UI system | Built separately | +| =source/migrations/= | Migration system | Built separately | +| =t/= | Unit and integration test code | Test-only | +| =nix/=, =flake.nix= | Reproducible packages, binaries, and OCI images | Build system | +| =docker/=, =docker-compose.yml= | Runtime images, entrypoints, and local stack | Deployment | +| =scripts/= | Stack acceptance test and utilities | Operations | +| =example_configs/= | Init-file examples | Operator input | +| =docs/= | Operator and developer documentation | Documentation | + +** Loaded actor systems + +The ASDF system loads only: + +- =event-actor.lisp= +- =matcher-actor.lisp= + +The user-finder and user-hunt files are not loaded. Adding a file to the +repository does not activate it. Add it to the ASDF component list or load it +from a trusted init file, then register its startup hook. + +** CouchDB views + +At load time, =gserver-settings.lisp= reads every JSON file under +=source/views/=. During startup, =init-db= creates or updates each design +document using its =_id= and current =_rev=. + +The Lisp view wrapper supports: + +- =limit=, =skip=, =descending=; +- =key=, =keys=, =start-key=, =end-key=; +- =include-docs=; +- =update=, including ="lazy"=; +- =reduce=, =group=, and =group-level=. + +Passing conflicting key modes raises an error. + +View wrappers cover messages, social posts, datasets, timelines, organizations, +people, relations, targets, users, events, hosts, emails, domains, networks, +URLs, and breaches. + +** Failure boundaries + +- Rabbit handler success must ACK the delivery. +- Unexpected ingest/update errors NACK and requeue. +- CouchDB insert conflict during initial ingest is NACKed without requeue. +- Update conflicts are refetched and retried up to two times before requeue. +- Rabbit publish is bounded by =*publish-timeout-seconds*=, default 5 seconds. +- Actor exceptions stay inside the Sento actor boundary according to Sento + behavior; actor-specific recovery policy is not centralized in this repo. +- HTTP publication success means the message reached the producer call. It does + not prove downstream persistence. + +** Security boundaries + +The current server has no built-in HTTP authentication, authorization, rate +limiting, tenant isolation, or request schema enforcement. The init file can run +arbitrary code. SLYNK provides remote Lisp evaluation. Use network isolation, +least-privilege Rabbit/Couch credentials, an authenticated reverse proxy, and +loopback-only debugger binding. diff --git a/docs/configuration.org b/docs/configuration.org new file mode 100644 index 00000000..d5daa8ac --- /dev/null +++ b/docs/configuration.org @@ -0,0 +1,271 @@ +#+title: StarIntel Server configuration +#+options: toc:3 + +* Configuration + +StarIntel Server has three effective configuration layers: + +1. compiled defaults and environment reads in =source/gserver-settings.lisp=; +2. an executable Common Lisp init file loaded before startup; +3. container environment and secret-file expansion in + =docker/star-server-entrypoint.sh=. + +The init file runs last and can override global variables. + +** Init file selection + +Resolution order: + +1. =--init PATH= or =-i PATH=; +2. =STAR_SERVER_INIT_FILE=; +3. =./init.lisp=. + +When the selected file does not exist, the loader copies +=example_configs/init.lisp= to that path, then loads it. + +The init file is arbitrary Common Lisp and has the same privileges as the +server process. + +** Environment variables read by the Lisp runtime + +| Variable | Default | Runtime variable | +|----------+---------+------------------| +| =COUCHDB_HOST= | =127.0.0.1= | =star:*couchdb-host*= | +| =COUCHDB_DATABASE= | =starintel= | =star:*couchdb-default-database*= | +| =COUCHDB_USER= | =admin= | =star:*couchdb-user*= | +| =COUCHDB_PASSWORD= | empty | =star:*couchdb-password*= | +| =HTTP_API_LISTEN_ADDRESS= | =localhost= | =star:*http-api-address*= | +| =RABBITMQ_ADDRESS= | =localhost= | =star:*rabbit-address*= | +| =RABBITMQ_USER= | =guest= | =star:*rabbit-user*= | +| =RABBITMQ_PASSWORD= | empty | =star:*rabbit-password*= | +| =STAR_SERVER_INIT_FILE= | =./init.lisp= | init path | + +Current fixed defaults unless overridden in Lisp: + +| Runtime variable | Default | +|------------------+---------| +| =*couchdb-port*= | 5984 | +| =*couchdb-scheme*= | ="http"= | +| =*http-api-port*= | 5000 | +| =*rabbit-port*= | 5672 | +| =*ingest-workers*= | 4 | +| =*bulk-max-documents*= | 500 | +| =*couchdb-event-log-database*= | ="starintel-event-source"= | +| =*slynk-port*= | 4009 | +| =*publish-timeout-seconds*= | 5 | + +The Compose variables =STAR_SERVER_PORT=, =COUCHDB_PORT=, and =RABBITMQ_PORT= +control host-side port mappings. They do not change the internal Lisp ports. + +** Secret files in containers + +The container entrypoint reads: + +- =COUCHDB_PASSWORD_FILE= and exports =COUCHDB_PASSWORD=; +- =RABBITMQ_PASSWORD_FILE= and exports =RABBITMQ_PASSWORD=. + +It then drops to UID/GID 65532 and starts: + +#+begin_example +/bin/star-server start -i /etc/starintel/init.lisp +#+end_example + +Outside that image, =*_FILE= variables are not read by the Lisp code. + +** Complete init-file example + +#+begin_src lisp +(in-package :star) + +;;; CouchDB +(setf *couchdb-host* "127.0.0.1" + *couchdb-port* 5984 + *couchdb-scheme* "http" + *couchdb-user* "admin" + *couchdb-password* (or (uiop:getenv "COUCHDB_PASSWORD") "") + *couchdb-default-database* "starintel" + *couchdb-event-log-database* "starintel-event-source") + +;;; RabbitMQ +(setf *rabbit-address* "127.0.0.1" + *rabbit-port* 5672 + *rabbit-user* "starintel" + *rabbit-password* (or (uiop:getenv "RABBITMQ_PASSWORD") "")) + +;;; HTTP +(setf *http-api-address* "127.0.0.1" + *http-api-port* 5000 + *bulk-max-documents* 500) + +;;; Concurrency +(setf *ingest-workers* 8 + star.actors:*publish-timeout-seconds* 5) + +;;; Logging +(ensure-directories-exist #P"logs/") +(log:config :daily "logs/star-server.log" :file2 :sane) + +;;; Custom actors are loaded now and started later by hooks. +(load #P"/absolute/path/to/actors/domain-enricher.lisp") +#+end_src + +** Local development + +#+begin_src lisp +(in-package :star) + +(setf *couchdb-host* "127.0.0.1" + *rabbit-address* "127.0.0.1" + *http-api-address* "127.0.0.1" + *ingest-workers* 4) + +(log:config :sane) +#+end_src + +Keep the API, Rabbit management UI, CouchDB, and SLYNK on loopback. + +** Docker internal networking + +The Compose image receives service DNS names from =docker-compose.yml=. The +mounted init file should not replace them with host-loopback values: + +#+begin_src lisp +(in-package :star) + +(setf *couchdb-host* (or (uiop:getenv "COUCHDB_HOST") "couchdb") + *rabbit-address* (or (uiop:getenv "RABBITMQ_ADDRESS") "rabbitmq") + *http-api-address* (or (uiop:getenv "HTTP_API_LISTEN_ADDRESS") "0.0.0.0")) +#+end_src + +** Remote CouchDB and RabbitMQ + +#+begin_src lisp +(in-package :star) + +(setf *couchdb-host* "couch.internal.example" + *couchdb-port* 6984 + *couchdb-scheme* "https" + *couchdb-user* "starintel-writer" + *couchdb-password* (uiop:getenv "COUCHDB_PASSWORD") + *rabbit-address* "rabbit.internal.example" + *rabbit-port* 5671 + *rabbit-user* "starintel" + *rabbit-password* (uiop:getenv "RABBITMQ_PASSWORD")) +#+end_src + +Caveat: this repository's Rabbit client setup does not visibly configure TLS +parameters. Changing the port to 5671 alone does not establish a verified TLS +connection. Terminate through a trusted private tunnel/proxy or add explicit +TLS support to the client before relying on it. + +CouchDB's scheme is configurable, but certificate verification behavior depends +on =cl-couch=/Dexador configuration. + +** High-throughput ingest + +#+begin_src lisp +(in-package :star) + +(setf *ingest-workers* 16 + *bulk-max-documents* 2000 + star.actors:*publish-timeout-seconds* 10) +#+end_src + +Tune carefully: + +- each Rabbit consumer worker opens its own connection/channel; +- each worker uses prefetch 200; +- the actor pinned dispatcher size follows =*ingest-workers*=; +- the HTTP CouchDB pool remains 20 open connections unless code is changed; +- larger bulk limits increase HTTP handler time and partial-failure volume; +- the producer remains serialized through one pinned producer agent. + +Scale external actors horizontally before turning one server into an +unbounded fan-out engine. + +** Low-memory deployment + +#+begin_src lisp +(in-package :star) + +(setf *ingest-workers* 2 + *bulk-max-documents* 100) +#+end_src + +Compose example: + +#+begin_example +CLOUSEAU_JAVA_OPTS=-Xms128m -Xmx512m +#+end_example + +The server schedules a full SBCL garbage collection once per hour. + +** Enable SLYNK deliberately + +#+begin_src lisp +(in-package :star) + +(setf *slynk-port* 4009) +(start-debugger) +#+end_src + +SLYNK is remote code execution. Bind it through SSH/Unix forwarding or a +loopback-only environment. The helper does not implement authentication. + +** Custom CouchDB view + +Add a JSON design document under =source/views/=. It is read when the Lisp +system loads and upserted at database initialization. + +Example: + +#+begin_src json +{ + "_id": "_design/custom", + "views": { + "by_actor": { + "map": "function(doc) { if (doc.extensions && doc.extensions.star_server) emit(doc.extensions.star_server.actor, null); }" + } + } +} +#+end_src + +Because view files are collected at Lisp load time, rebuild/reload the system +after adding a file. + +** Reverse proxy + +The server's built-in HTTP TLS variables and =*http-api-base-path*= are not +currently applied by =start-http-api=. Terminate TLS and authentication at a +reverse proxy. + +Minimum proxy controls: + +- authenticated access; +- request body limit; +- rate limiting; +- timeout shorter than or aligned with the 300-second backend timeout; +- explicit allowed origins instead of forwarding wildcard CORS; +- network denial for direct backend access; +- audit logs without sensitive document bodies. + +** Multi-dataset operation + +=dataset= is a document field, not a separate CouchDB database. The current +server stores all datasets in =*couchdb-default-database*= and exposes views that +filter/count by dataset. + +Use separate server/database deployments when you need hard isolation. Dataset +strings alone provide no authorization boundary. + +** Configuration traps + +- =*star-server-version*= currently says =0.0.1= while ASDF and container + packages use =0.1.0=. +- HTTP =POST /new/document/:dtype= does not set the body =dtype=. +- The init template can overwrite environment-derived values. +- =*http-api-cert*= and =*http-api-key*= are unused by the server startup. +- =*http-api-base-path*= is unused; routes are mounted at root. +- A missing init file is created automatically, which can hide a wrong path. +- =COUCHDB_PASSWORD_FILE= and =RABBITMQ_PASSWORD_FILE= work only through the + container entrypoint. diff --git a/docs/document-spec.org b/docs/document-spec.org new file mode 100644 index 00000000..f9bb078d --- /dev/null +++ b/docs/document-spec.org @@ -0,0 +1,390 @@ +#+title: StarIntel document specification in star-server +#+options: toc:3 + +* Document specification + +StarIntel Server currently sits between two document generations: + +1. *StarIntel 0.9.0*: the strict, schema-driven format implemented by the + validators in =lost-rob0t/star-cl= and =lost-rob0t/starintel-doc=. +2. *Legacy flat 0.8.0*: Common Lisp classes and constructors still used by parts + of this server and its actor examples. + +The server reports the =star-cl= document version from =GET /=, but the HTTP and +RabbitMQ ingest handlers do not currently call the strict 0.9 validator. A +=document= with a non-empty =dtype= can reach CouchDB even when it does not +conform to 0.9. + +** StarIntel 0.9 envelope + +The 0.9 adapters require: + +- a JSON object; +- =schema_version= exactly ="0.9.0"=; +- a canonical =dtype= from the schema; +- fields and data types defined by the schema; +- RFC 3339 date-time strings where the schema requires =date-time=; +- no undeclared fields in closed schema sections. + +The adapters preserve unknown extension data and missing optional fields when +the schema permits them. + +Conceptual envelope: + +#+begin_src json +{ + "schema_version": "0.9.0", + "dtype": "person", + "_id": "01J...", + "dataset": "investigation-a", + "sources": ["manual"], + "data": { + "fname": "Ada", + "lname": "Lovelace" + }, + "extensions": { + "star_server": { + "trace_id": "01J...", + "root_id": "01J...", + "parent_id": null, + "actor_path": [] + } + } +} +#+end_src + +Use the canonical schema distributed by the StarIntel conformance/spec +repository. The adapters locate it through: + +| Variable | Meaning | +|----------+---------| +| =STARINTEL_SCHEMA= | Explicit path to =starintel-doc-v0.9.0.schema.json= | +| =STARINTEL_CONFORMANCE_ROOT= | Root containing =schemas/starintel-doc-v0.9.0.schema.json= | + +The current server does not consume either variable directly; =star-cl='s +validator does. + +** Rejected 0.9 aliases + +The 0.9 adapters explicitly reject these dtype aliases rather than silently +normalizing them: + +#+begin_example +organization +organisation +investigation_target +social_media_post +email_message +financial_observation +research_pass +dataset_manifest +actor_manifest +legal_case +lobbying_filing +campaign_finance +#+end_example + +Use the canonical dtype from the schema inventory. + +** Legacy flat 0.8 common fields + +The legacy Common Lisp =document= class serializes Lisp slot names to camelCase, +while preserving leading underscore keys. + +| JSON field | Type | Meaning | +|------------+------+---------| +| =_id= | string | Document identity; ULID or deterministic hash | +| =_rev= | string, optional | CouchDB revision; emitted only when syntactically valid | +| =dataset= | string | Logical corpus/dataset | +| =dtype= | string | Object type | +| =sources= | array | Provenance labels/URIs/actor names | +| =version= | string | Legacy document version, normally ="0.8.0"= | +| =dateAdded= | integer | Unix timestamp | +| =dateUpdated= | integer | Unix timestamp | + +When encoding legacy classes, missing strings become =""=, integers become =0=, +lists become =[]=, booleans become =false=, and unknown values may become JSON +=null=. + +** Legacy object types + +These are the concrete classes currently exported by =star-cl=. They are not a +substitute for the canonical 0.9 schema. + +*** person + +| Field | Type | +|-------+------| +| =fname= | string | +| =mname= | string | +| =lname= | string | +| =bio= | string | +| =dob= | string | +| =region= | string | +| =misc= | array | +| =etype= | string, default =person= | +| =eid= | string | + +ID strategy: ULID. + +*** org + +| Field | Type | +|-------+------| +| =reg= | string | +| =name= | string | +| =bio= | string | +| =country= | string | +| =website= | string | +| =etype= | string, default =org= | +| =eid= | string | + +ID strategy: hash of name, registration, and country. + +*** relation + +| Field | Type | Meaning | +|-------+------+---------| +| =source= | string | Source document ID | +| =target= | string | Target document ID | +| =predicate= | string | Directed edge label | +| =note= | string | Human/actor note | + +ID strategy: ULID. Relations are directed. + +*** target + +| Field | Type | +|-------+------| +| =actor= | string | Local registered actor or external actor name | +| =target= | string | Target value/document ID | +| =delay= | integer | Recurrence interval in seconds | +| =recurring= | boolean | Re-deliver on timer | +| =options= | array | Actor-defined values | + +ID strategy: hash of dataset, target, and actor. + +*** domain + +| Field | Type | +|-------+------| +| =recordType= | string | +| =record= | string | +| =resolvedAddresses= | array | + +ID strategy: hash of record and record type. + +*** host + +| Field | Type | +|-------+------| +| =hostname= | string | +| =ip= | string | +| =os= | string | +| =ports= | array of service-like values | + +ID strategy: hash of IP. + +*** service + +Embedded object fields: =port= integer, =name= string, =ver= string. + +*** network + +Fields: =org= string, =subnet= string, =asn= integer. ID is derived from ASN and +organization. + +*** url + +Fields: =url= string, =path= string, =query= string, =content= string. ID is +derived from URL and content. + +*** email + +Fields: =user= string, =domain= string, =password= string. ID includes password +only when non-empty. + +#+begin_quote +Do not store credential material unless the collection and storage are lawful, +authorized, necessary, and access-controlled. The current server has no +field-level encryption or authorization. +#+end_quote + +*** emailmessage + +Fields: =body=, =subject=, =to=, =from=, =headers= strings; =cc= and =bcc= +arrays. + +*** user + +Fields: =url=, =name=, =platform=, =bio= strings and =misc= array. + +*** breach + +Fields: =total= integer, =description= string, =url= string. + +*** message + +Fields: + +- =content= string +- =platform= string +- =user= string +- =isReply= boolean +- =media= array +- =messageId= string +- =replyTo= string +- =group= string +- =channel= string +- =mentions= array + +*** socialmpost + +Fields: + +- =content=, =user=, =url=, =title=, =group=, =replyTo= strings +- =replies=, =media=, =links=, =tags= arrays +- =replyCount=, =repostCount= integers + +*** phone + +Fields: =number=, =carrier=, =status=, =phoneType= strings. + +*** geo and address + +=geo=: =lat=, =long=, =alt= numbers. + +=address= extends geo with =city=, =state=, =postal=, =country=, =street=, and +=street2=. + +** Relation predicates + +The current =star-cl= relation constructor validates predicates against an +allowlist. Groups include: + +- identity: =same-as=, =duplicate-of=, =aka=, =alias-of=, =username-of=, + =email-of=, =phone-of=, =account-of=; +- people/org: =member-of=, =employed-by=, =contractor-for=, =works-with=, + =manages=, =reports-to=; +- ownership/control: =owns=, =owned-by=, =controls=, =controlled-by=, + =operates=, =operated-by=, =administers=, =administered-by=; +- registration: =registered-to=, =registrant-of=, =whois-registrant-of=, + =whois-admin-of=, =whois-tech-of=; +- location: =located-at=, =geolocated-at=, =seen-at=; +- communication: =communicates-with=, =contacted=, =contacted-by=, =mentions=, + =replies-to=, =follows=; +- web: =links-to=, =redirects-to=, =canonical-url-of=, =hosts=, =hosted-by=, + =served-by=; +- DNS/network: =resolves-to=, =ptr-to=, =has-a=, =has-aaaa=, =has-cname=, + =has-ns=, =has-mx=, =has-txt=, =has-spf=, =has-dkim=, =has-dmarc=, + =has-soa=, =behind-cdn=, =belongs-to-asn=, =served-from=, + =shares-ip-with=, =shares-asn-with=; +- services: =hosts-service=, =listens-on=, =exposes-port=, =runs=, =runs-on=; +- credential/breach: =leaked-in=, =credential-for=, =compromised-by=; +- provenance/evidence: =observed-on=, =observed-by=, =collected-from=, + =extracted-from=, =derived-from=, =downloaded-from=, =uploaded-to=, + =created-by=, =modified-by=, =hashes-to=, =matches-hash=, =evidence-of=, + =indicates=; +- security/research: =attributed-to=, =uses=, =targets=, =exploits=, + =mitigates=, =c2-for=, =in-scope-of=, =out-of-scope-of=, + =discovered-by=, =scanned-by=, =has-finding=, =vulnerable-to=; +- generic: =related-to=. + +Constructor: + +#+begin_src lisp +(spec:new-relation + "dataset" + "source-id" + "target-id" + :predicate "derived-from" + :note "Generated by actor") +#+end_src + +** ID strategies + +The legacy constructors mix two identity models: + +- *ULID*: person, relation, default document when the server normalizes a + missing ID. +- *Deterministic hash*: org, domain, network, host, URL, email, user, message, + social post, phone, target, geo/address. + +Deterministic IDs support idempotent recursive actors, but the current default +hash algorithm in =star-cl= is MD5. Treat it as a deduplication key, not a +cryptographic integrity proof. Use SHA-256 in provenance extensions when +integrity matters. + +** CouchDB fields + +The ingest handler: + +1. generates a ULID when =_id= is missing or empty; +2. writes the JSON document; +3. replaces/sets =_id= from CouchDB's response; +4. adds =_rev=; +5. republishes the enriched document as =documents.new.=. + +The update handler requires =_id= and =dtype=. It fetches the current document, +deep-merges the patch, applies the latest =_rev=, retries conflicts, and emits +the new revision. + +** Actor event document + +Current actor-event fields: + +| Field | Type | +|-------+------| +| =_id= | ULID string | +| =timestamp= | Unix integer | +| =dtype= | intended =actorevent= | +| =actorName= | string | +| =eventType= | string | +| =details= | string | +| =sourceId= | string | + +Actor events are stored in the separate event database. + +** Recursive provenance convention + +No server component automatically adds recursive provenance. Use a stable +extension contract across actors: + +#+begin_src json +{ + "extensions": { + "star_server": { + "trace_id": "01J...", + "root_id": "root-document-id", + "parent_id": "source-document-id", + "actor": "dns-resolver", + "actor_path": ["seed", "domain-enricher", "dns-resolver"], + "depth": 2, + "hop": 3, + "max_depth": 8, + "max_hops": 32, + "derivation": "dns-resolution", + "input_hash": "sha256:...", + "output_hash": "sha256:...", + "emitted_at": "2026-07-26T20:00:00Z" + } + } +} +#+end_src + +Also emit a relation from source to derived output. Metadata alone does not +replace graph edges. + +** Migration guidance + +For new producers: + +1. emit strict 0.9 when the deployed server path validates it; +2. keep one canonical dtype and field spelling; +3. put actor/runtime-specific data under =extensions=; +4. do not silently convert aliases; +5. test round-trip preservation against every language adapter; +6. document any legacy 0.8 bridge at the boundary. + +For the current server, verify downstream consumers before switching an active +pipeline from flat 0.8 fields to nested 0.9 =data=. diff --git a/docs/http-api-docs.org b/docs/http-api-docs.org index ec9ae087..d9284172 100644 --- a/docs/http-api-docs.org +++ b/docs/http-api-docs.org @@ -1,246 +1,296 @@ -#+title: Http docs +#+title: StarIntel HTTP API +#+options: toc:3 -* Http Api Docs -** Search +* HTTP API -*** Search Documents +Default base URL: =http://127.0.0.1:5000= -Example usage +The API currently has no authentication or authorization and allows every CORS +origin. Do not expose it directly to an untrusted network. -#+begin_src http -GET http://127.0.0.1:5000/search?q=content:"hello"&limit=10&include_docs=true -Accept: application/json -#+end_src +All implemented routes are mounted at =/=; =*http-api-base-path*= is not used. -earches for documents matching the specified query. +** Response behavior -Parameters: -- q: (required) The search query string -- limit: (optional) Maximum number of results to return (default 25) -- bookmark: (optional) A bookmark from a previous search to start from for pagination -- sort: (optional) Field(s) to sort the results by -- include_docs: (optional) If "true", includes the full document content in the results - -Returns: -A JSON array of matching documents, with the following fields: -- id: The document ID -- order: An array with the sort value (if specified) and relevance score -- fields: An object with the stored field values (if include_docs is not "true") -- doc: The full document (if include_docs is "true") - -The response also includes: -- total_rows: The total number of matching documents -- bookmark: A bookmark that can be used for pagination of subsequent requests -** Targets -*** Get Targets for Actor - -Example usage - -#+begin_src http -GET http://127.0.0.1:5000/targets/01731aa61e40224a127259541c8d71da -Accept: application/json +Most successful document/query responses are raw JSON from JSOWN or CouchDB. +Structured status responses look like: + +#+begin_src json +{"msg":"OK","status":"info"} #+end_src -Retrieves the targets for the specified actor. +CouchDB wrapper errors commonly map to: -** Documents +| Status | Meaning | +|--------+---------| +| 400 | Bad request | +| 404 | Document/view result not found | +| 409 | CouchDB revision conflict | +| 500 | Unhandled server error | +| 504 | CouchDB/socket timeout | -*** Create Target +Rabbit publish failure from document/target routes maps to 502; publish timeout +maps to 504. -Example usage +** Core endpoints -#+begin_src http -POST http://127.0.0.1:5000/new/target/01731aa61e40224a127259541c8d71da -Content-Type: application/json +*** GET /health -{ - "_id": "example_target_id", - "dataset": "example_dataset", - "dtype": "target", - "sources": [], - "version": "0.7.2", - "date_updated": 1621234567, - "date_added": 1621234567, - "actor": "01731aa61e40224a127259541c8d71da", - "target": "example_target", - "delay": 0, - "recurring": false, - "options": [] -} +Returns process-level health: + +#+begin_src sh +curl --fail http://127.0.0.1:5000/health +#+end_src + +This does not deeply verify CouchDB, RabbitMQ, or Clouseau. + +*** GET / + +Returns server metadata: + +#+begin_src sh +curl --fail http://127.0.0.1:5000/ | jq +#+end_src + +Fields include document spec version, default dataset/database, actor-event +database, server name, and server version. + +*** POST /new/document/:dtype + +Publishes one JSON document to =documents.ingest.=. + +#+begin_src sh +curl --fail \ + --header 'Content-Type: application/json' \ + --request POST \ + --data '{ + "_id": "note-1", + "dataset": "demo", + "dtype": "note", + "sources": ["manual"], + "version": "0.8.0", + "dateAdded": 0, + "dateUpdated": 0, + "content": "hello" + }' \ + http://127.0.0.1:5000/new/document/note #+end_src -#+RESULTS: -: HTTP/1.1 200 OK -: Date: Mon, 20 May 2024 09:58:43 GMT -: Server: Hunchentoot 1.3.0 -: Transfer-Encoding: chunked -: Content-Type: text/html; charset=utf-8 -: -: { "_id": "example_target_id", "dataset": "example_dataset", "dtype": "target", "sources": [], "version": "0.7.2", "date_updated": 1621234567, "date_added": 1621234567, "actor": "01731aa61e40224a127259541c8d71da", "target": "example_target", "delay": 0, "recurring": false, "options": []} +Important: -Creates a new target document for the specified actor. +- the URL dtype selects the routing key; +- the handler does not inject the URL dtype into the body; +- the ingest consumer requires a non-empty body =dtype=; +- success means publication, not completed persistence; +- strict 0.9 validation is not called. -*** /new/document/ +*** POST /documents/bulk -Example usage +Publishes an array of documents. Each element must have =dtype=. -#+begin_src http -POST http://127.0.0.1:5000/new/document/person -Content-Type: application/json +#+begin_src sh +curl --fail \ + --header 'Content-Type: application/json' \ + --request POST \ + --data '[ + {"_id":"note-1","dataset":"demo","dtype":"note","content":"one"}, + {"_id":"note-2","dataset":"demo","dtype":"note","content":"two"} + ]' \ + http://127.0.0.1:5000/documents/bulk | jq +#+end_src +Response: +#+begin_src json { - "_id": "example_person_id", - "dataset": "example_dataset", - "dtype": "person", - "sources": ["manual"], - "version": "0.7.2", - "date_updated": 1621234567, - "date_added": 1621234567, - "fname": "John", - "mname": "Doe", - "lname": "Smith", - "bio": "Example bio", - "dob": "1990-01-01", - "race": "Example race", - "region": "Example region", - "misc": [""], - "etype": "person", - "eid": "example_eid" + "total": 2, + "succeeded": 2, + "failed": 0 } #+end_src -#+RESULTS: -: HTTP/1.1 200 OK -: Date: Mon, 20 May 2024 10:11:37 GMT -: Server: Hunchentoot 1.3.0 -: Transfer-Encoding: chunked -: Content-Type: text/html; charset=utf-8 -: -: { "_id": "example_person_id", "dataset": "example_dataset", "dtype": "person", "sources": ["manual"], "version": "0.7.2", "date_updated": 1621234567, "date_added": 1621234567, "fname": "John", "mname": "Doe", "lname": "Smith", "bio": "Example bio", "dob": "1990-01-01", "race": "Example race", "region": "Example region", "misc": [""], "etype": "person", "eid": "example_eid"} +Failures are per element and can produce partial acceptance. The default maximum +is 500 documents. -Creates a new document speficied by dtype +*** GET /document/:id -Note: it is emitted onto the message queue from processing +#+begin_src sh +curl --fail http://127.0.0.1:5000/document/note-1 | jq +#+end_src -*** /document -Example usage +Reads from the configured main CouchDB database. -#+begin_src http -GET http://127.0.0.1:5000/document/example_document_id -Accept: application/json -#+end_src +*** DELETE /document/:id -Retrieves the document with the specified ID. +The server fetches the current revision then deletes the document: -*** Messages -**** /documents/messages-by-user -Retrieve messages by user. -Example usage: -#+begin_src http -GET http://127.0.0.1:5000/documents/messages/by-user?user=john_doe&limit=10&descending=true +#+begin_src sh +curl --fail --request DELETE \ + http://127.0.0.1:5000/document/note-1 | jq #+end_src -#+RESULTS: -: HTTP/1.1 200 OK -: Date: Wed, 22 May 2024 00:31:04 GMT -: Server: Hunchentoot 1.3.0 -: Transfer-Encoding: chunked -: Content-Type: text/html; charset=utf-8 -: -: [] +Deletion does not emit a RabbitMQ deletion event. + +*** POST /new/target/:actor + +Sets =dtype= to =target=, sets =actor= from the URL, and publishes to target +ingest. + +#+begin_src sh +curl --fail \ + --header 'Content-Type: application/json' \ + --request POST \ + --data '{ + "dataset": "demo", + "target": "example.org", + "delay": 3600, + "recurring": true, + "options": [] + }' \ + http://127.0.0.1:5000/new/target/domain-enricher +#+end_src +*** GET /targets/:actor -Returns messages for the specified user, sorted by the dateAdded field in descending order. -Parameters: -+ user (required): The user to retrieve messages for. -+ limit (optional, default 50): The maximum number of messages to return. -+ start_key (optional): The starting key for the range of messages to return. -+ end_key (optional): The ending key for the range of messages to return. -+ descending (optional, default false): Whether to return messages in descending order. -+ skip (optional, default 0): The number of messages to skip. - -The start_key and end_key parameters should be valid JSON strings representing the key range. -The response is a JSON array of message documents. -Note: Refer to the Starintel specification for example message objects. -**** /documents/messages/by-platform -Retrieve messages by platform. -#+begin_src http -GET http://127.0.0.1:5000/documents/messages/by-platform?platform=discord&limit=10&descending=true -#+end_src -#+RESULTS: -: HTTP/1.1 200 OK -: Date: Wed, 22 May 2024 00:34:38 GMT -: Server: Hunchentoot 1.3.0 -: Transfer-Encoding: chunked -: Content-Type: text/html; charset=utf-8 -: -: [{"_id":"fce27626f2e2ed302b06fea3662e825f","_rev":"1-6992719df874825ab365e43d887391d4","dataset":"starintel","dtype":"message","sources":["discordwatch"],"version":"0.7.2","dateUpdated":1715906816,"dateAdded":1715906816180,"content":"they aren't configured to use the fonts","platform":"discord","user":"__gerg","isReply":[],"media":[],"messageId":"","replyTo":"","group":"Nix/NixOS (unofficial)","channel":"general","mentions":[]},{"_id":"fe7e1194359777b7e8b1dbd8b9477d04","_rev":"1-e82025832a08752c9523ffb6ffec0bf7","dataset":"starintel","dtype":"message","sources":["discordwatch"],"version":"0.7.2","dateUpdated":1715906770,"dateAdded":1715906770880,"content":"I'm aware","platform":"discord","user":"amyipdev","isReply":[],"media":[],"messageId":"","replyTo":"","group":"Nix/NixOS (unofficial)","channel":"Unable to get persistence to actually keep files, cannot use system as a result","mentions":[]},{"_id":"fe209ae93b7a332d98de3c348125b972","_rev":"1-8c017c1b61d9598996e5dd7d68076978","dataset":"starintel","dtype":"message","sources":["discordwatch"],"version":"0.7.2","dateUpdated":1715906654,"dateAdded":1715906654285,"content":"sleep is a scam","platform":"discord","user":"sudzsalmon","isReply":[],"media":[],"messageId":"","replyTo":"","group":"Daydream Society","channel":"general","mentions":[]},{"_id":"fc497f99907ac33142f62e2b77037f8f","_rev":"1-628005d952267d533920e0b8f7c2cddc","dataset":"starintel","dtype":"message","sources":["discordwatch"],"version":"0.7.2","dateUpdated":1715906622,"dateAdded":1715906622575,"content":"Where would she have been upset? A vbulletin forum?","platform":"discord","user":"evilmofo","isReply":[],"media":[],"messageId":"","replyTo":"","group":"DEFCON","channel":"linecon","mentions":[]},{"_id":"ff7c388068fa855cc9b6a60f1b86fde7","_rev":"1-6487f93c484d147ed9692b9ae89c40f8","dataset":"starintel","dtype":"message","sources":["discordwatch"],"version":"0.7.2","dateUpdated":1715906603,"dateAdded":1715906603427,"content":"sinope \u00BB one of the things I like about his channel is his earnest and respectful attitude towards the subject matter","platform":"discord","user":"/v/craft chat","isReply":[],"media":[],"messageId":"","replyTo":"","group":"s.s. /v/ minecraft","channel":"gaem-chat","mentions":[]},{"_id":"ff397ff6b42338692fba48edebc5bb82","_rev":"1-e09925814e5c5ddeb93b2c34e4ed525c","dataset":"starintel","dtype":"message","sources":["discordwatch"],"version":"0.7.2","dateUpdated":1715906398,"dateAdded":1715906398133,"content":"LAYS\n\nFentanyl flavored","platform":"discord","user":"canislycora","isReply":[],"media":[],"messageId":"","replyTo":"","group":"Bluelight.org","channel":"\uD83D\uDDEF\u2502the-lounge","mentions":[]},{"_id":"ff0e7a39d43a844d106bf88eb11722e4","_rev":"1-e9364ec4a3c6b0f73d60ce600b3446d8","dataset":"starintel","dtype":"message","sources":["discordwatch"],"version":"0.7.2","dateUpdated":1715906393,"dateAdded":1715906393579,"content":"my next question I guess is, is the extension list automatically updated or manually maintained?","platform":"discord","user":"ashtefere","isReply":[],"media":[],"messageId":"","replyTo":"","group":"Nix/NixOS (unofficial)","channel":"general","mentions":[]},{"_id":"fdea3948045b94c0ec4db2798d30e111","_rev":"1-706d7e642854330a4f8b3d7646d93182","dataset":"starintel","dtype":"message","sources":["discordwatch"],"version":"0.7.2","dateUpdated":1715906326,"dateAdded":1715906326159,"content":"**Designated Bottomfragger** (76561198137734744) > lightbringer was wild","platform":"discord","user":"Bob's All Gamemodes","isReply":[],"media":[],"messageId":"","replyTo":"","group":"Bob's BattleBit Community","channel":"all-gamemodes-chat","mentions":[]},{"_id":"fc5802e3366cd7b6a5e318cae204b545","_rev":"1-7202ff9567315331ee6b194e14868a58","dataset":"starintel","dtype":"message","sources":["discordwatch"],"version":"0.7.2","dateUpdated":1715906199,"dateAdded":1715906199246,"content":"i choose to believe the singer from Tool personally chewed me out through pirated Napster files","platform":"discord","user":"specksgalore","isReply":[],"media":[],"messageId":"","replyTo":"","group":"DEFCON","channel":"linecon","mentions":[]},{"_id":"fe0b44fcc54f1ae41ca7a0821aeee620","_rev":"1-ebb09652814a75830f290db94f555737","dataset":"starintel","dtype":"message","sources":["discordwatch"],"version":"0.7.2","dateUpdated":1715906148,"dateAdded":1715906148747,"content":"the only place that I know how to affect sound that you're playing with sunvox, is in the sunvox project.","platform":"discord","user":"polylokh_39446","isReply":[],"media":[],"messageId":"","replyTo":"","group":"Nim","channel":"gamedev","mentions":[]}] -Returns messages for the specified platform, sorted by the dateAdded field in descending order. -Parameters: +Returns persisted targets from the =targets/by_actor= CouchDB view: -+ platform (required): The platform to retrieve messages for. -+ limit (optional, default 50): The maximum number of messages to return. -+ start_key (optional): The starting key for the range of messages to return. -+ end_key (optional): The ending key for the range of messages to return. -+ descending (optional, default false): Whether to return messages in descending order. -+ skip (optional, default 0): The number of messages to skip. - -The start_key and end_key parameters should be valid JSON strings representing the key range. -The response is a JSON array of message documents. -Note: Refer to the Starintel specification for example message objects. -**** /documents/messages/by-group -Example usage - -#+begin_src http -GET http://127.0.0.1:5000/documents/messages/by-group?group=&limit=&start_key=&end_key=&descending=&skip= +#+begin_src sh +curl --fail \ + http://127.0.0.1:5000/targets/domain-enricher | jq #+end_src +** Full-text search + +*** GET /search Parameters: -- group (required): The group name to filter messages by. -- limit (optional, default: 50): The maximum number of messages to return. -- start_key (optional): The starting key for the range of messages to return. -- end_key (optional): The ending key for the range of messages to return. -- descending (optional, default: false): Whether to return messages in descending order. -- skip (optional, default: 0): The number of messages to skip from the beginning. -Response: -The response is a JSON array containing the matching message documents, sorted by the specified criteria. Each message document follows the starintel message spec format. Refer to the starintel-spec documentation for detailed information about the message document structure -*** SocialMPosts -**** /documents/socialmpost/by-user -Example usage +| Parameter | Required | Default | Meaning | +|-----------+----------+---------+---------| +| =q= | yes in practice | none | Clouseau/Lucene query string | +| =limit= | no | 25 | Maximum rows | +| =bookmark= | no | none | CouchDB FTS bookmark | +| =sort= | no | none | Sort expression accepted by CouchDB FTS | -#+begin_src http -GET http://127.0.0.1:5000/documents/socialmpost/by-user?user=&limit=&start_key=&end_key=&descending=&skip= +The handler always sets =include_docs=true=. + +#+begin_src sh +curl --fail --get \ + --data-urlencode 'q=content:"hello"' \ + --data-urlencode 'limit=10' \ + http://127.0.0.1:5000/search | jq #+end_src +Search requires the =_design/search= FTS design document and Clouseau. -Parameters: -- user (required): The username to filter social posts by. -- limit (optional, default: 50): The maximum number of social posts to return. -- start_key (optional): The starting key for the range of social posts to return. -- end_key (optional): The ending key for the range of social posts to return. -- descending (optional, default: false): Whether to return social posts in descending order. -- skip (optional, default: 0): The number of social posts to skip from the beginning. +** Common view query parameters -Response: -The response is a JSON array containing the matching socialmpost documents, sorted by the specified criteria. Each social post document follows the starintel social post spec format. Refer to the starintel-spec documentation for detailed information about the social post document structure. -*** Neighbors -**** Get Neighbors +Many view routes accept: -Example usage +| Parameter | Default | Meaning | +|-----------+---------+---------| +| =limit= | 50 | Row limit | +| =start_key= | none | JSON-encoded CouchDB start key | +| =end_key= | none | JSON-encoded CouchDB end key | +| =descending= | false | Reverse result order | +| =skip= | 0 | Skip rows | +| =reduce= | route-specific | Request reduce/group behavior | -#+begin_src http -POST http://127.0.0.1:5000/relations/neighbors -Content-Type: application/json +=start_key= and =end_key= are parsed as JSON. Quote string keys correctly: -{ - "docs": ["doc_id1", "doc_id2", "doc_id3"], - "n": 2 -} +#+begin_src sh +curl --get \ + --data-urlencode 'user=alice' \ + --data-urlencode 'start_key="alice"' \ + --data-urlencode 'limit=20' \ + http://127.0.0.1:5000/documents/messages/by-user #+end_src -#+RESULTS: +** Message and social-post views + +| Method | Route | Key parameters | +|--------+-------+----------------| +| GET | =/documents/messages/by-user= | =user= | +| GET | =/documents/messages/by-channel= | =group=, =channel=, optional =reduce= | +| GET | =/documents/messages/by-groups= | legacy/duplicate channel-style query | +| GET | =/documents/messages/by-platform= | =platform= | +| GET | =/documents/messages/groups= | pagination parameters | +| GET | =/documents/socialmpost/by-user= | =user= | + +The spelling =socialmpost= is the implemented route. + +** Dataset view + +*** GET /dataset-size + +The source declares this route twice. The later declaration exposes general +view pagination/reduce parameters and can shadow the earlier simple +=dataset== variant. + +Treat the current route as unstable until the duplicate definition is removed. + +** Host views + +| Route | Key | +|-------+-----| +| =/documents/hosts/by-ip= | =ip= | +| =/documents/hosts/by-port= | =port= integer | +| =/documents/hosts/by-service= | =service= | + +** Email views + +| Route | Key | +|-------+-----| +| =/documents/emails/by-email= | =email= | +| =/documents/emails/by-domain= | =domain= | +| =/documents/emails/with-password= | no key | + +** Domain views + +| Route | Key | +|-------+-----| +| =/documents/domains/by-record= | =record= | +| =/documents/domains/by-resolved-address= | =ip= | + +** User views + +| Route | Key | +|-------+-----| +| =/documents/users/by-name= | =name= | +| =/documents/users/by-platform= | =platform= | + +** Network views + +| Route | Key | +|-------+-----| +| =/documents/networks/by-asn= | =asn= integer | +| =/documents/networks/by-org= | =org= | + +** URL views + +| Route | Key | +|-------+-----| +| =/documents/urls/by-url= | =url= | +| =/documents/urls/by-domain= | =domain= | + +** Breach views + +| Route | Key | +|-------+-----| +| =/documents/breaches/by-size= | no explicit key; sort/paginate by size | + +** Stub endpoint + +=/new/event/:id= exists in the route source but has no implementation. Do not +use it. Send events through =log-actor-event= locally or the =events= exchange. + +** HTTP operational notes -Retrieves the neighbors of the specified documents up to the given level. +- CORS is wildcard. +- Request bodies are loaded in memory. +- Bulk requests are sequentially published inside one HTTP request. +- The producer is serialized through one pinned agent. +- Default publish timeout is five seconds. +- Hunchentoot request timeout is 300 seconds. +- No OpenAPI document is generated. +- No endpoint currently validates the strict StarIntel 0.9 schema. diff --git a/docs/http-auth-kv-lease-boundary.org b/docs/http-auth-kv-lease-boundary.org new file mode 100644 index 00000000..3c3f40e6 --- /dev/null +++ b/docs/http-auth-kv-lease-boundary.org @@ -0,0 +1,327 @@ +#+title: HTTP authentication threat model — KV lease boundary +#+options: toc:3 +#+status: DESIGN + +* Normative status + +This document is a normative part of the StarIntel Server HTTP authentication +threat model. It defines the trust boundary between authenticated HTTP callers, +actor components, target-lease operations, and the key-value lease store. + +It is design only. It does not claim that the current runtime implements these +controls. + +* Protected lease assets + +The lease subsystem protects: + +- target ownership and assignment state; +- lease identifiers; +- tenant, dataset, actor, and target bindings; +- lease holder principal and actor identity; +- expiration and renewal state; +- fencing tokens or monotonically increasing lease generations; +- compare-and-swap versions; +- retry, release, and revocation state; +- KV namespace layout and service credentials; +- lease audit records and correlation identifiers. + +Lease records MUST NOT contain API-key secrets, bearer tokens, credential +verifiers, salts, peppers, raw Authorization headers, or reusable CouchDB or +RabbitMQ credentials. + +* Actors and attacker profiles + +Relevant attacker profiles include: + +- an unauthenticated remote caller attempting to acquire or renew leases; +- an authenticated principal with =targets:read= but not =targets:lease=; +- an actor credential scoped to another actor, tenant, dataset, or target; +- a compromised actor replaying a stale lease or renewal request; +- two actor instances attempting to hold the same target concurrently; +- an internal-network attacker with direct KV connectivity; +- an imported library caller invoking lease functions without an HTTP security + context; +- an administrator or service with over-broad KV credentials; +- a denial-of-service attacker exhausting lease keys, renewal work, or storage. + +* Trust boundary and flow + +The required flow is: + +#+begin_example +HTTP request or trusted actor call + -> credential verification + -> principal resolution + -> targets:lease capability decision + -> tenant/dataset/actor/target scope checks + -> canonical lease request + -> KV lease adapter using service credentials + -> atomic acquire/renew/release operation + -> non-secret result and audit event +#+end_example + +The KV lease store is an infrastructure service boundary. It MUST NOT authenticate +StarIntel end users or actors directly from request-supplied fields. The +StarIntel authorization layer decides whether the operation is allowed before +calling the KV adapter. + +The KV adapter uses a dedicated service credential with access only to the lease +namespace required by this deployment. HTTP API keys, actor API keys, OIDC +tokens, and mTLS certificates MUST NOT be forwarded into KV values, keys, logs, +or connection metadata. + +Direct KV clients bypass the HTTP authorization boundary. Production network and +KV access controls MUST prevent unapproved direct clients. Approved internal +clients require their own service identity, namespace restriction, operation +restriction, and audit policy. + +* Principal and scope binding + +Lease acquisition, renewal, and release require =targets:lease=. + +The authorization decision MUST bind all applicable values before the KV call: + +- authenticated =principal_id=; +- non-secret =credential_id=; +- =tenant_id=; +- =dataset_id=; +- canonical =actor_name=; +- canonical =target_id=; +- requested operation: acquire, renew, or release; +- request =correlation_id=; +- authorization =decision_id=. + +The actor value in the route, request body, credential binding, target record, +and existing lease MUST agree after canonicalization. A mismatch denies. + +An actor-scoped credential cannot acquire, renew, inspect, or release another +actor's lease. Tenant and dataset boundaries are mandatory even when target IDs +are globally unique. + +Request-supplied identifiers do not create authority. The server resolves and +canonicalizes the target and actor before authorization and before generating a +KV key. + +* Lease record contract + +A conceptual lease record contains only non-secret state: + +#+begin_src json +{ + "lease_id": "lease_01...", + "tenant_id": "tenant-main", + "dataset_id": "dataset-a", + "actor_name": "usernamegen", + "target_id": "target_01...", + "holder_principal_id": "prn_actor_01...", + "holder_credential_id": "key_01...", + "generation": 42, + "acquired_at": "2026-07-30T20:00:00Z", + "expires_at": "2026-07-30T20:01:00Z", + "decision_id": "dec_01...", + "correlation_id": "corr_01..." +} +#+end_src + +The KV key MUST be generated from canonical, bounded identifiers through one +closed function. Clients MUST NOT provide arbitrary KV keys, prefixes, queries, +or namespaces. + +The lease value MUST be validated on every read and mutation. Unknown fields may +be preserved only under a versioned migration contract; malformed or +unsupported lease versions fail closed. + +* Atomicity and fencing + +Lease acquisition MUST be atomic. A read followed by an unconditional write is +not sufficient. + +The adapter MUST use the strongest primitive supported by the selected KV store, +such as compare-and-swap, create-if-absent with TTL, a transaction, or a lease +primitive. + +Every successful acquisition MUST produce a monotonically increasing generation +or fencing token. Downstream work that can mutate authoritative state SHOULD +carry and validate this token so an expired or partitioned former holder cannot +continue writing after another actor acquires the target. + +Renewal MUST succeed only when all of these match the current record: + +- lease identifier; +- holder principal; +- actor name; +- target identity; +- current generation or compare-and-swap version; +- unexpired lease state according to server-side policy. + +Release MUST use the same ownership checks. A caller cannot release a lease by +knowing only a target ID or lease ID. + +* Time and expiration + +The service controls lease duration. Clients may request a duration only within a +bounded server policy. + +Expiration calculations MUST use the KV service's atomic TTL semantics or a +server-controlled time source. Client clocks are not authoritative. + +Clock skew, delayed requests, retries, process suspension, and network partitions +MUST NOT allow two valid holders. When certainty is unavailable, acquisition or +renewal fails closed and the caller retries under bounded backoff. + +A renewal response received after local timeout MUST not be assumed successful. +The actor must read or reacquire using the lease contract before continuing +exclusive work. + +* Replay and stale-holder threats + +Threats: + +- replaying a previous acquire or renewal request; +- continuing work after lease expiration; +- renewing with a stale generation; +- reusing a lease identifier after release; +- delivering delayed RabbitMQ work to a former lease holder; +- restoring an old KV snapshot that resurrects stale leases. + +Controls: + +- unique opaque lease identifiers; +- generation or fencing tokens; +- bounded TTL; +- compare-and-swap renewal; +- explicit holder and actor binding; +- idempotency keys for retried API operations where required; +- downstream generation validation for authoritative writes; +- snapshot and recovery procedures that invalidate or safely re-establish leases; +- audit correlation across lease and RabbitMQ events. + +RabbitMQ delivery is not proof of lease ownership. Consumers performing leased +work MUST validate current lease state or a trustworthy fencing token before +committing authoritative results. + +* Direct KV access and service credentials + +The KV service account MUST have least-privilege access to the lease namespace. +It SHOULD NOT have access to unrelated application data. + +Service credentials MUST be stored through the deployment secret mechanism, not +inside StarIntel documents, lease records, source files, logs, or error bodies. + +KV credentials require independent creation, rotation, revocation, recovery, and +audit procedures. Rotating an HTTP or actor API key does not rotate the KV +service credential, and rotating the KV credential does not alter principal +grants. + +Network policy SHOULD restrict KV connectivity to approved StarIntel service +instances. Direct administrative access must be separately authenticated and +audited. + +* Imported library callers + +Imported Common Lisp callers do not gain lease authority because they execute in +the StarIntel process. + +Security-sensitive lease functions MUST accept an explicit request security +context or a separately defined trusted-internal context. A missing context, +=nil= context, caller package, stack location, or local process address MUST NOT +imply =targets:lease= or administrator authority. + +Trusted internal callers still provide canonical tenant, dataset, actor, and +target bindings and produce audit events. + +* Failure behavior + +| Condition | Result | +|-----------+--------| +| Missing or invalid credential | 401 authentication failure before KV access | +| Valid principal without =targets:lease= | 403 access denial before KV access | +| Tenant, dataset, actor, or target mismatch | 403 access denial | +| Existing unexpired lease held elsewhere | 409 lease conflict or stable equivalent | +| Stale generation or compare-and-swap failure | 409 lease conflict | +| Malformed lease request | 400 or 422 stable client error | +| KV unavailable or inconsistent | 503 and fail closed | +| Per-principal or global lease rate limit | 429 | +| Internal malformed lease record | fail closed, alert, and quarantine according to recovery policy | + +External errors MUST not reveal KV keys, namespace layout, service credentials, +raw backend errors, connection strings, stack traces, or another principal's +private metadata. + +* Abuse and availability controls + +The lease boundary MUST enforce: + +- bounded identifier and request sizes; +- bounded lease durations; +- bounded acquire and renewal rates per principal, credential, actor, target, and + source context; +- bounded concurrent KV operations; +- bounded retries with jitter; +- no unbounded lease queues; +- quotas on active leases per actor and principal; +- rejection before KV work when authentication or authorization fails; +- circuit breaking and explicit degraded health when KV is unavailable; +- audit aggregation that does not allow failure floods to exhaust storage. + +Rate limiting MUST not convert a denied caller into an allowed one, and lockout +policy MUST not permit permanent denial of service through spoofed failures. + +* Audit requirements + +Every acquire, renew, release, conflict, stale-generation rejection, scope denial, +backend failure, administrative override, and recovery action records: + +- timestamp; +- correlation and decision identifiers; +- principal class and identifier; +- non-secret credential identifier; +- tenant, dataset, actor, and target scope; +- operation; +- lease identifier where safe; +- generation or fencing token where safe; +- decision and result class; +- latency and backend health state. + +Audit records MUST NOT contain bearer material, verifier data, peppers, KV +service secrets, raw Authorization headers, or full arbitrary request bodies. + +Administrative force-release or recovery requires a separately authenticated +administrator credential, high-priority audit, and a reason or incident +identifier. + +* Residual risks + +Residual risks include: + +- a compromised authorized actor can abuse every lease in its granted scope; +- KV or host compromise can mutate lease state outside application policy; +- network partitions and delayed work require correct fencing enforcement in + downstream writers; +- a backend without strong atomic primitives may not be suitable for exclusive + leases; +- incorrectly canonicalized target or actor identifiers can split one logical + lease into multiple keys; +- restore procedures can resurrect stale state if generations are not handled; +- administrator abuse cannot be fully prevented, only constrained and audited; +- availability attacks can still deny lease progress within configured bounds. + +* Required verification + +Implementation is not complete until tests prove: + +- every acquire, renew, and release path requires =targets:lease=; +- tenant, dataset, actor, and target scope isolation; +- route/body/credential/target actor mismatch denial; +- atomic single-holder acquisition under concurrency; +- generation or fencing-token monotonicity; +- stale renewal and stale release denial; +- expired holder cannot commit authoritative work when fencing is enforced; +- duplicate and replayed requests are safe; +- KV failure and malformed records fail closed; +- no end-user credential reaches KV storage or logs; +- direct library callers without context deny; +- rate limits, quotas, retry bounds, and TTL bounds hold; +- administrative override is separately authenticated and audited; +- snapshot/restore behavior cannot silently resurrect valid stale leases. diff --git a/docs/http-auth-threat-model.org b/docs/http-auth-threat-model.org new file mode 100644 index 00000000..14189867 --- /dev/null +++ b/docs/http-auth-threat-model.org @@ -0,0 +1,596 @@ +#+title: HTTP authentication threat model +#+options: toc:3 +#+status: DESIGN + +* Status and scope + +This document defines the security model that MUST exist before StarIntel Server +HTTP authentication is considered implemented. It is a design contract, not a +claim about current runtime behavior. + +Current runtime status: + +- the HTTP API has no authentication or authorization enforcement; +- CORS permits every origin; +- callers are not represented as principals; +- RabbitMQ and CouchDB trust boundaries are not derived from HTTP identity; +- =*couchdb-auth-database*= is configuration residue, not an authentication + design. + +Until the implementation satisfies this document, deployments MUST place the +HTTP API behind a trusted private boundary and MUST NOT expose it as a public +multi-tenant service. + +* Security objectives + +The HTTP authentication system MUST: + +1. authenticate every protected request to one explicit principal; +2. authorize every operation by capability and scope using default deny; +3. prevent credentials, bearer material, hashes, and peppers from entering logs, + traces, metrics labels, error bodies, or StarIntel documents; +4. bind audit records to a stable principal, credential identifier, request + correlation identifier, decision, resource scope, and result; +5. support credential rotation and revocation without restarting the service; +6. keep HTTP identity from becoming implicit RabbitMQ or CouchDB authority; +7. reject ambiguous or partially configured security modes; +8. make local-development bypass explicit, narrow, observable, and impossible to + enable accidentally in non-development operation. + +Availability is a security objective. Authentication and authorization controls +MUST resist resource exhaustion and MUST fail closed when their backing state is +unavailable or inconsistent. + +* Protected assets + +** Credentials and cryptographic material + +- API key secret material; +- non-secret API key identifiers; +- password-derived or key-derived verifier records; +- server-side peppers and pepper version metadata; +- OIDC client secrets and issuer metadata when OIDC is added; +- mTLS private keys, trust anchors, and revocation state when mTLS is added; +- bootstrap credentials and recovery material; +- session or exchange tokens if introduced later. + +** Authorization state + +- principal records; +- principal status and tenant membership; +- capability grants and denials; +- dataset, actor, target, and tenant scopes; +- credential-to-principal bindings; +- rotation, revocation, and expiration state; +- development bypass configuration; +- administrator and break-glass assignments. + +** Operational data + +- StarIntel documents and relations; +- target definitions, leases, and dispatch state; +- search results and query history; +- actor manifests and actor control operations; +- dataset metadata and cross-dataset links; +- audit records and security-event records; +- RabbitMQ routing topology and message bodies; +- CouchDB documents, revisions, design documents, and credentials; +- configuration, environment variables, crash dumps, and diagnostic output. + +* Principal classes + +The authentication layer recognizes these principal classes. Their detailed +contract is defined in [[file:http-principal-capability-contract.org][the principal and capability contract]]. + +| Principal class | Purpose | Typical credential | +|-----------------+---------+--------------------| +| Human user | Interactive operator or analyst | API key in v0.1; OIDC later | +| API client | Script, CLI, integration, or application | API key | +| Service instance | Long-running external service | API key in v0.1; mTLS later | +| Actor component | Actor-specific service or runtime component | scoped API key in v0.1; mTLS later | +| Administrator | Security and deployment administration | separately issued administrator credential | + +Administrator is a security role with a dedicated credential and audit policy, +not merely a boolean field added to an ordinary credential. + +Anonymous is not a principal class for protected routes. Public routes, if any, +MUST be explicitly enumerated and MUST not inherit protected-route behavior. + +* Attacker profiles + +** Unauthenticated remote attacker + +Capabilities: + +- send arbitrary HTTP requests; +- vary headers, methods, encodings, bodies, origins, and request rates; +- replay observed requests; +- probe error behavior and timing; +- attempt credential guessing, identifier enumeration, and resource exhaustion. + +** Authenticated low-privilege principal + +Capabilities: + +- use a valid credential within a narrow grant; +- submit malicious documents and query parameters; +- attempt horizontal access across tenants, datasets, actors, or targets; +- attempt vertical privilege escalation; +- exploit differences between HTTP, RabbitMQ, CouchDB, and library callers. + +** Compromised API client or actor + +Capabilities: + +- use stolen credential material until expiration or revocation; +- generate high-rate validly authenticated requests; +- exploit broad scopes or reusable credentials; +- attempt to turn one actor credential into general document or administrative + access. + +** Malicious or compromised administrator + +Capabilities: + +- create, rotate, revoke, and broaden credentials; +- modify authorization state; +- access security telemetry and configuration; +- attempt to suppress or alter audit evidence. + +The design cannot fully prevent an authorized administrator from abusing granted +power. It MUST make such actions explicit, separately authenticated, narrowly +scoped where possible, and auditable. + +** Internal-network attacker + +Capabilities: + +- reach RabbitMQ, CouchDB, or the HTTP listener from a trusted network segment; +- exploit weak service credentials or permissive bind addresses; +- publish messages directly, bypassing HTTP authorization; +- read or mutate backing stores if infrastructure credentials are compromised. + +** Dependency or supply-chain attacker + +Capabilities: + +- influence Common Lisp dependencies, container images, Nix inputs, build + artifacts, or CI behavior; +- introduce logging or serialization behavior that exposes credentials; +- weaken cryptographic verification or authorization decisions. + +** Local host attacker + +Capabilities depend on host privileges and may include reading process +environments, memory, files, sockets, crash dumps, and container runtime state. +Root or equivalent host compromise is outside the boundary of guarantees, but +the service MUST minimize unnecessary secret exposure. + +* Trust boundaries and data flows + +** Boundary A: client to HTTP listener + +The client presents a credential over an authenticated transport. The HTTP +listener terminates transport security, parses the request, establishes a +correlation identifier, authenticates the credential, resolves the principal, +and performs authorization before executing protected behavior. + +Required properties: + +- production credentials MUST only be accepted over HTTPS or an equivalently + protected local transport; +- authentication occurs before request-specific data access or message publish; +- request body parsing limits apply before expensive processing; +- credential material is removed from diagnostic context immediately after + verification; +- all protected routes share one enforcement boundary. + +** Boundary B: HTTP authorization to application operation + +The authorization decision consumes: + +- principal identifier and class; +- credential identifier and status; +- requested capability; +- tenant, dataset, actor, target, and resource identifiers; +- route and HTTP method; +- relevant request context. + +The decision returns allow or deny plus a machine-readable reason. Missing, +ambiguous, unsupported, or stale authorization state MUST deny. + +** Boundary C: HTTP process to RabbitMQ + +RabbitMQ publication is an internal service action. A successful HTTP decision +MUST be recorded before publication. HTTP credentials MUST NOT be copied into +message bodies, routing keys, or RabbitMQ headers. + +Messages originating from HTTP SHOULD carry only non-secret provenance: + +- request correlation identifier; +- authenticated principal identifier; +- credential identifier; +- authorization decision identifier; +- originating tenant and dataset scope. + +RabbitMQ consumers MUST NOT treat these fields as self-authenticating. The broker +connection and publisher identity establish transport trust; provenance fields +support policy checks and audit correlation. + +Direct RabbitMQ publishers bypass the HTTP boundary. They require their own +service identity, broker authorization, message validation, and audit policy. + +** Boundary D: HTTP process to CouchDB + +The server uses a service credential to access CouchDB. End-user credentials MUST +NOT be forwarded to CouchDB. Application authorization MUST be completed before +CouchDB reads or writes. + +CouchDB documents containing principal, credential, or authorization state MUST +be isolated from ordinary StarIntel document APIs. General document read/search +capabilities MUST NOT imply access to authentication records. + +** Boundary E: imported library callers + +Code may call internal Common Lisp functions without HTTP. Such callers do not +inherit HTTP authorization automatically. Security-sensitive operations MUST +accept an explicit security context or use a separate trusted-internal API with +a documented caller boundary. + +No function may infer administrator authority from =nil= context, local process +location, package membership, or caller stack. + +** Boundary F: configuration and secret providers + +Environment variables, mounted secret files, process supervisors, container +secrets, and future external secret managers feed security configuration into +the process. + +Configuration parsing MUST reject: + +- unknown security modes; +- missing required secrets; +- empty peppers or bootstrap secrets; +- development bypass combined with a non-loopback listener or production mode; +- duplicate credential identifiers; +- unsupported hash or key versions. + +* Authentication threats and controls + +** Credential guessing and enumeration + +Threats: + +- brute-force secret guessing; +- discovery of valid key identifiers through timing or error differences; +- high-rate distributed attempts; +- account or principal enumeration. + +Controls: + +- opaque high-entropy secret material; +- constant-time verifier comparison after identifier lookup; +- uniform external failure response for unknown, revoked, expired, and invalid + credentials; +- per-source and per-key-identifier rate limits; +- progressive abuse throttling without permanent denial-of-service lockout; +- audit events with redacted identifiers and correlation IDs; +- no distinction between nonexistent principal and invalid credential in client + responses. + +** Credential replay + +API keys are bearer credentials and can be replayed while valid. Controls: + +- mandatory protected transport; +- narrow capabilities and scopes; +- short operational rotation intervals for automated clients; +- immediate revocation support; +- optional expiration; +- anomaly detection based on source, rate, and operation profile; +- future mTLS for service and actor identities where replay resistance is + required. + +** Credential disclosure + +Threat sources include logs, exception messages, tracing, shell history, +configuration dumps, process listings, crash dumps, browser storage, copied curl +commands, and audit records. + +Controls: + +- authorization headers and credential fields are always redacted; +- error envelopes never echo credentials or verifier data; +- structured logging uses allow-listed security fields; +- metrics labels never contain principal-provided secrets; +- API responses return secret material only once at credential creation; +- stored records contain only key identifier, verifier, salt, pepper version, + metadata, and authorization bindings; +- documentation and examples use unmistakably fake credentials. + +** Verifier compromise + +Stored API key secrets MUST NOT be reversible. Each credential record stores a +salted password-hash-style verifier using a memory-hard algorithm such as +Argon2id. A server-side pepper, versioned independently from records, is included +in verification. + +Controls: + +- unique salt per credential; +- parameters recorded with the verifier; +- pepper stored outside CouchDB; +- pepper version recorded, not pepper value; +- rehash on successful authentication when parameters or active pepper version + change; +- staged pepper rotation with bounded overlap; +- revocation of all affected credentials if verifier and pepper material are both + compromised. + +** Confused credential type + +An OIDC token, API key, bootstrap token, actor credential, or internal service +credential MUST not be accepted by the wrong verifier path. + +Credential parsing MUST use explicit prefixes and version fields. Unsupported +prefixes or versions deny without fallback. + +* Authorization threats and controls + +** Default-allow and missing-policy behavior + +Every protected route maps to an explicit capability and resource extractor. +Missing mapping, missing principal, missing scope, malformed scope, policy load +failure, or unknown capability MUST deny. + +There is no wildcard grant except the explicitly audited =admin= capability. +Even =admin= does not bypass authentication, audit, tenant invariants, or secret +redaction. + +** Horizontal privilege escalation + +Controls: + +- tenant is resolved from authenticated authorization state, not only request + input; +- requested dataset, actor, and target identifiers are checked against grants; +- object lookup and authorization use the same canonical identifiers; +- cross-dataset and cross-tenant relations require an explicit capability and + both endpoint scopes; +- bulk operations authorize every document or reject the entire batch according + to a documented atomicity rule. + +** Vertical privilege escalation + +Controls: + +- capability names are closed and versioned; +- credential-management operations require administrator or narrowly delegated + security capabilities; +- principals cannot edit their own grants through general document APIs; +- administrator credentials are separately issued and SHOULD not be used for + routine data access; +- capability changes emit immutable audit events. + +** Actor and target confusion + +Actor credentials MUST be scoped to named actor identities and permitted target +operations. A credential with =targets:lease= for one actor does not receive +=targets:dispatch=, general document write, or access to another actor. + +Target ownership, lease holder, and actor name MUST be canonicalized before the +authorization decision. + +** Search as a data-exfiltration path + +Search authorization is not satisfied solely by =search:read=. Results MUST be +filtered to datasets and tenants the principal may read. Search indexes MUST not +expose authentication databases, audit secrets, or deleted/revoked credential +material. + +Timing, counts, facets, bookmarks, and error differences can leak existence. +Where strict tenant isolation is required, queries MUST execute against a scope +that cannot include unauthorized records rather than filtering only after +retrieval. + +* Availability and abuse threats + +** Authentication endpoint exhaustion + +Controls: + +- bounded request body and header sizes; +- bounded credential identifier length; +- rate limits before expensive verifier computation; +- bounded concurrent expensive verifications; +- negative-result caching keyed by a non-secret digest where safe; +- circuit breaking for unavailable authorization storage; +- no unbounded queues. + +** Intentional lockout + +Permanent lockout based only on failed attempts enables denial of service. +Instead use temporary progressive throttling, source-aware controls, credential +revocation for confirmed compromise, and administrator-visible abuse events. + +** Bulk operation amplification + +Authentication does not replace existing bulk backpressure. The principal and +credential identifier MUST participate in per-principal quotas. Authentication +failures MUST not enqueue work. + +** Audit flooding + +Security-event generation MUST be bounded and aggregated where necessary, while +preserving enough detail to investigate distributed attacks. Audit storage +failure MUST be visible. For high-risk administrative writes, inability to write +the required audit event MUST fail the operation closed. + +* Failure behavior + +| Condition | External status | External code | Internal action | +|-----------+-----------------+---------------+-----------------| +| Missing credential | 401 | =authentication_required= | audit sampled failure | +| Invalid, unknown, revoked, or expired credential | 401 | =invalid_credential= | full security audit with safe identifiers | +| Valid credential, missing capability or scope | 403 | =access_denied= | authorization-denial audit | +| Authentication rate limit | 429 | =authentication_rate_limited= | abuse event and retry metadata | +| Authorization state unavailable | 503 | =authorization_unavailable= | fail closed and alert | +| Malformed authorization header | 400 or 401 by parser contract | stable non-secret code | no parser details returned | + +Responses MUST include a correlation identifier. They MUST NOT include principal +existence, credential status, verifier details, policy internals, stack traces, +or raw conditions. + +* Audit requirements + +Every successful protected request records: + +- event timestamp; +- request correlation identifier; +- decision identifier; +- principal identifier and class; +- non-secret credential identifier; +- capability requested; +- canonical tenant, dataset, actor, target, and resource scopes; +- route template and method; +- allow decision; +- result class and latency; +- source context according to privacy and retention policy. + +Every authentication failure records only safe evidence: + +- correlation identifier; +- presented key identifier when syntactically valid, otherwise a bounded digest; +- source context; +- failure class internal to security telemetry; +- throttling state. + +Credential create, rotate, revoke, expire, grant, deny, bootstrap, recovery, and +development-bypass events are mandatory and high-priority. + +Audit records MUST be append-oriented, access-controlled separately from general +documents, retained under an explicit policy, and protected against silent +modification or deletion. + +* Secret lifecycle + +** Bootstrap + +The first administrator credential MUST be created through an explicit bootstrap +operation requiring: + +- an empty security store; +- a one-time bootstrap secret supplied out of band; +- a loopback or explicitly trusted administrative channel; +- immediate invalidation of the bootstrap secret; +- mandatory audit output; +- refusal to bootstrap when any administrator already exists unless a documented + recovery procedure is invoked. + +The service MUST NOT ship a default administrator credential. + +** Creation + +API key creation generates secret material with a cryptographically secure random +source. The complete credential is returned exactly once. Subsequent reads return +only metadata and the non-secret key identifier. + +** Rotation + +Rotation creates a new credential record and supports a bounded overlap period. +The old and new credential have independent identifiers, verifiers, expiration, +and audit history. Rotation MUST NOT mutate secret material in place. + +** Revocation + +Revocation is immediate at the authorization store and invalidates relevant +caches. Revoked credentials remain represented as tombstoned metadata for audit +and identifier non-reuse. + +** Recovery + +Recovery uses a separately documented break-glass process. It MUST not weaken +normal authentication, reuse ordinary API keys, or silently recreate prior +secret material. + +* Local-development bypass + +A development bypass MAY exist only if all of these conditions are enforced: + +- security mode is explicitly =development=; +- an explicit bypass flag is set; +- listener address is loopback or a Unix-domain socket; +- no forwarded-client or proxy trust is enabled; +- environment is not marked production, staging, CI release, or container + publication mode; +- startup emits a high-severity warning and persistent health status; +- every bypassed request is assigned a fixed development principal and audited; +- bypass cannot grant administrator credential-management operations; +- configuration validation aborts startup if any condition is violated. + +A single environment variable MUST NOT be sufficient to enable bypass in a +network-accessible deployment. + +* CORS and browser boundary + +CORS is not authentication. Once authentication is implemented: + +- wildcard origins MUST not be used for credential-bearing browser requests; +- allowed origins MUST be explicit configuration; +- browser clients MUST not store long-lived API keys in insecure web storage; +- preflight responses MUST expose only required methods and headers; +- future cookie or browser-session authentication requires separate CSRF and + session threat modeling. + +Non-browser API clients are unaffected by CORS enforcement and still require +normal authentication. + +* Residual risks + +The v0.1 API-key design retains these risks: + +- bearer credentials can be replayed after theft; +- compromised authorized clients can perform all operations within their grants; +- host compromise can expose in-memory secrets and peppers; +- direct RabbitMQ or CouchDB compromise bypasses HTTP policy; +- administrator abuse cannot be fully prevented; +- traffic analysis can reveal request timing and size even with transport + encryption; +- authorization mistakes in route-to-capability mapping remain possible and + require exhaustive tests. + +Future mTLS can reduce service credential replay risk. Future OIDC can improve +human identity lifecycle. Neither replaces capability and scope authorization. + +* Verification requirements + +Implementation is not complete until tests demonstrate: + +- every protected route has an explicit capability mapping; +- unmapped routes deny; +- missing, invalid, revoked, expired, and wrong-type credentials deny uniformly; +- credentials and bearer material never appear in logs or errors; +- tenant, dataset, actor, and target scope isolation; +- administrator operations require separately authenticated authority; +- rate limits and queue bounds hold under concurrency; +- verifier and pepper rotation behavior; +- revocation invalidates caches immediately; +- authorization-store failure denies; +- development bypass startup constraints; +- direct internal callers cannot gain authority from missing context; +- audit events exist for security-sensitive allow and deny decisions. + +* Security review triggers + +This threat model MUST be reviewed when any of these change: + +- a new credential type is introduced; +- OIDC, mTLS, cookies, or sessions are added; +- a new principal class or capability is added; +- multi-tenancy or cross-tenant relations change; +- RabbitMQ or CouchDB trust boundaries change; +- the HTTP listener becomes publicly exposed by default; +- authorization state moves to another database or cache; +- browser credential handling is introduced; +- audit retention or storage changes; +- imported library callers gain security-sensitive operations. diff --git a/docs/http-authentication-runtime.org b/docs/http-authentication-runtime.org new file mode 100644 index 00000000..75a79cbd --- /dev/null +++ b/docs/http-authentication-runtime.org @@ -0,0 +1,362 @@ +#+title: HTTP API-key authentication runtime +#+options: toc:3 + +* Status + +StarIntel Server protects every HTTP route by default except: + +- =GET /health=; +- =GET /=; +- =POST /auth/bootstrap=. + +The runtime uses opaque API keys, an immutable request security context, a +separate CouchDB authentication database, exact-origin CORS configuration, and +uniform authentication failures. + +The broader security model is defined in: + +- [[file:http-auth-threat-model.org][HTTP authentication threat model]]; +- [[file:http-principal-capability-contract.org][principal and capability contract]]; +- [[file:http-auth-kv-lease-boundary.org][KV lease authentication boundary]]. + +This implementation establishes identity and administrator lifecycle controls. +Fine-grained route capability and resource-scope enforcement remains follow-on +work. Until that enforcement lands, any valid non-administrator key can reach +ordinary protected routes; lifecycle mutation routes require an administrator +principal or =admin= scope. + +* Credential format + +The v0.1 API key has this shape: + +#+begin_example +star_sk_v1__ +#+end_example + +- =credential-id= is a non-secret ULID used for record lookup; +- =secret= is 32 random bytes represented as 64 hexadecimal characters; +- the full key is bearer material and must be protected like a password; +- unsupported prefixes, versions, malformed fields, unknown identifiers, and + incorrect secrets all return the same external authentication failure. + +The server stores no recoverable API-key secret. A credential record contains: + +- credential identifier; +- owner and principal type; +- scope strings; +- status; +- random salt; +- salted and peppered verifier; +- creation and optional expiry timestamps; +- disable and revocation timestamps; +- rotation parent, successor, and overlap expiration metadata; +- CouchDB revision metadata. + +Credential records live in =STAR_AUTH_DATABASE=, not in the intelligence +document database. General document, search, export, and graph routes do not read +the authentication database. + +* Hashing and comparison + +The verifier is derived from: + +#+begin_example +SHA-256(server-pepper || credential-salt || random-secret) +#+end_example + +The implementation uses a constant-time comparison function at the verifier +boundary. The server pepper is loaded separately from CouchDB and is never stored +in a credential record. + +This SHA-256 construction is the implemented v0.1 verifier, not the final +memory-hard Argon2id design described by the threat model. Migrating verifier +algorithms requires a versioned credential-record format and rehash/rotation +path. + +* Request authentication + +Send the key as a bearer token: + +#+begin_example +Authorization: Bearer star_sk_v1__ +#+end_example + +A valid request creates one immutable security context containing: + +- principal identifier; +- principal type; +- non-secret credential identifier; +- scope strings; +- correlation identifier; +- request deadline; +- authentication timestamp. + +The context is propagated into asynchronous bulk jobs. RabbitMQ publication +receives only non-secret provenance fields: principal identifier and type, +credential identifier, scopes, correlation identifier, and deadline. Raw bearer +material, verifier data, salts, and the server pepper are never propagated. + +Clients may provide a bounded =X-Correlation-ID=. The server generates one when +it is missing or invalid. + +Clients may provide =X-Request-Timeout-Ms=. Values outside the configured bound +fall back to the server default rather than creating an unbounded deadline. + +* Uniform authentication failure + +Every missing, malformed, unknown, incorrect, expired, disabled, revoked, or +rotation-expired credential returns: + +#+begin_src json +{ + "status": "error", + "code": "invalid_credential", + "msg": "Authentication failed", + "correlation_id": "..." +} +#+end_src + +The status is =401=. The response includes =WWW-Authenticate= and +=Cache-Control: no-store=. It does not reveal whether the credential identifier +exists or which internal check failed. + +* Bootstrap + +Bootstrap creates the first administrator credential and is allowed only while +the authentication store is empty. + +Request: + +#+begin_example +POST /auth/bootstrap +Content-Type: application/json +X-Star-Bootstrap-Secret: + +{"owner":"initial-administrator"} +#+end_example + +The bootstrap secret is configured through =STAR_AUTH_BOOTSTRAP_SECRET= or +=STAR_AUTH_BOOTSTRAP_SECRET_FILE=. A constant-time digest comparison checks the +presented value. + +A successful response has status =201= and returns the API key exactly once: + +#+begin_src json +{ + "api_key": "star_sk_v1_...", + "credential": { + "credential_id": "...", + "owner": "initial-administrator", + "principal_type": "administrator", + "scopes": ["admin"], + "status": "active" + }, + "correlation_id": "..." +} +#+end_src + +The response is marked =Cache-Control: no-store= and =Pragma: no-cache=. + +A second bootstrap attempt returns =409 bootstrap_complete=. An invalid +bootstrap secret returns =403 bootstrap_denied=. Neither response returns secret +or verifier material. + +* Credential lifecycle routes + +All lifecycle routes below require an authenticated administrator principal or a +credential with =admin= scope. + +** Create + +#+begin_example +POST /auth/credentials +Authorization: Bearer +Content-Type: application/json + +{ + "owner": "quasar-production", + "principal_type": "api_client", + "scopes": ["documents:read", "documents:write", "search:read"], + "expires_in_seconds": 2592000 +} +#+end_example + +The successful =201= response returns the new API key exactly once and includes +redacted metadata. + +** List metadata + +#+begin_example +GET /auth/credentials +Authorization: Bearer +#+end_example + +The response contains metadata only. It never contains raw keys, verifier bytes, +salts, or the pepper. + +** Rotate + +#+begin_example +POST /auth/credentials//rotate +Authorization: Bearer +Content-Type: application/json + +{"overlap_seconds":300} +#+end_example + +Rotation creates a distinct credential and returns its secret exactly once. The +old credential remains valid until the overlap expiration, then fails with the +same uniform =401 invalid_credential= response. The maximum overlap is bounded by +=STAR_AUTH_MAX_ROTATION_OVERLAP_SECONDS=. + +An overlap of zero invalidates the old credential immediately after the rotation +record is committed. + +** Revoke + +#+begin_example +POST /auth/credentials//revoke +Authorization: Bearer +#+end_example + +Revocation is permanent for the credential identifier. + +** Disable + +#+begin_example +POST /auth/credentials//disable +Authorization: Bearer +#+end_example + +Disabled credentials cannot authenticate. Re-enable is intentionally not +implemented in this lifecycle surface. + +** Inspect current context + +#+begin_example +GET /auth/context +Authorization: Bearer +#+end_example + +The response returns the current non-secret principal, credential, scopes, +correlation identifier, and deadline. It is intended for client verification and +diagnostics. + +* Revocation cache bound + +The v0.1 implementation has no successful-credential or authorization cache. +Every authenticated request reads the current credential record from the +configured store. + +Therefore the documented cache bound is: + +#+begin_example +0 seconds +#+end_example + +Once a revocation or disable update commits, the next verifier lookup observes +it. An already-running request that authenticated before the update retains its +immutable request context for that request only. Concurrent tests verify that +requests released after the revocation commit are rejected. + +Adding a cache later is a security-contract change. It requires a bounded TTL, +immediate invalidation, concurrency tests, and an updated documented revocation +bound. + +* Bulk jobs and service calls + +Large bulk requests capture the authenticated service context when the job is +accepted. Worker execution uses that captured context rather than ambient +unauthenticated state. + +Bulk-job status is visible only to: + +- the principal that submitted the job; or +- an administrator. + +Other principals receive =404 bulk_job_not_found= so job existence is not +exposed. + +The repository does not yet contain a KV lease adapter. The same service-call +context is the required input projection for future target-lease operations; a +future adapter must not infer authority from process location or a missing +context. + +* CORS + +Wildcard CORS is removed. Configure exact browser origins with a comma-separated +list: + +#+begin_example +STAR_AUTH_ALLOWED_ORIGINS=https://quasar.example,https://admin.example +#+end_example + +For an allowed origin, the server returns that exact origin and =Vary: Origin=. +Credential-bearing wildcard origins are never emitted. + +A preflight from an unconfigured origin returns =403 cors_origin_denied=. Requests +without an =Origin= header, such as normal CLI or service clients, are unaffected +by browser CORS policy and still require authentication. + +* Configuration + +| Environment variable | Default | Purpose | +|----------------------+---------+---------| +| =STAR_AUTH_MODE= | =api-key= | Authentication mode. Supported: =api-key= and constrained development =disabled=. | +| =STAR_AUTH_DATABASE= | =starintel-gserver-auth= | Separate CouchDB credential database. | +| =STAR_AUTH_PEPPER= | none | Server-side verifier pepper. | +| =STAR_AUTH_PEPPER_FILE= | none | File containing the verifier pepper. | +| =STAR_AUTH_BOOTSTRAP_SECRET= | none | One-time bootstrap secret. | +| =STAR_AUTH_BOOTSTRAP_SECRET_FILE= | none | File containing the bootstrap secret. | +| =STAR_AUTH_ALLOWED_ORIGINS= | empty | Comma-separated exact browser origins. | +| =STAR_AUTH_DEV_BYPASS= | false | Explicit development bypass flag. | +| =STAR_AUTH_MAX_ROTATION_OVERLAP_SECONDS= | =86400= | Maximum old/new key overlap. | +| =STAR_AUTH_DEFAULT_REQUEST_TIMEOUT_MS= | =30000= | Default propagated request deadline. | +| =STAR_AUTH_MAX_REQUEST_TIMEOUT_MS= | =600000= | Maximum accepted client timeout. | +| =COUCHDB_PASSWORD_FILE= | none | Runtime-readable CouchDB password file. | +| =RABBITMQ_PASSWORD_FILE= | none | Runtime-readable RabbitMQ password file. | + +Production =api-key= mode refuses to initialize without a non-empty pepper. +Unknown modes refuse startup. + +Authentication may be disabled only when all implemented checks pass: + +- =STAR_AUTH_MODE=disabled=; +- =STAR_AUTH_DEV_BYPASS=true=; +- the HTTP listener is loopback (=localhost=, =127.0.0.1=, or =::1=). + +A disabled-auth configuration on =0.0.0.0= fails startup. + +* Container deployment + +The Compose stack mounts these additional secrets into the server: + +- =auth_pepper=; +- =auth_bootstrap_secret=. + +The stack test proves: + +- public health remains available; +- unauthenticated protected access returns =401=; +- bootstrap returns one administrator key; +- bootstrap cannot run twice; +- authenticated context resolves the expected administrator; +- authenticated full-text search works; +- the intelligence document persists across restart; +- the administrator credential persists in the separate auth database and still + authenticates after restart. + +* Operational rules + +- Never commit, log, paste into issue comments, or store complete API keys in + StarIntel documents. +- Capture the one-time key response into a secret manager immediately. +- Use distinct principals and credentials per service and environment. +- Prefer narrow scopes even though fine-grained route authorization is not yet + enforced; the stored grants are the migration basis for that enforcement. +- Rotate a suspected credential, use the shortest safe overlap, then revoke the + old identifier. +- Treat a lost key as unrecoverable; create or rotate instead of attempting to + read the secret from storage. +- Protect CouchDB and the auth database from direct untrusted network access. +- Protect the server pepper independently from CouchDB backups. diff --git a/docs/http-principal-capability-contract.org b/docs/http-principal-capability-contract.org new file mode 100644 index 00000000..ba7e0eeb --- /dev/null +++ b/docs/http-principal-capability-contract.org @@ -0,0 +1,795 @@ +#+title: HTTP principal and capability contract +#+options: toc:3 +#+status: DESIGN + +* Purpose + +This document defines the stable application contract for authenticated HTTP +principals, credential records, capabilities, scopes, authorization decisions, +and audit context. + +It does not implement authentication. Runtime code MUST conform to this contract +when authentication is added. Security assumptions and attacker analysis are in +[[file:http-auth-threat-model.org][the HTTP authentication threat model]]. + +* Normative rules + +The words MUST, MUST NOT, SHOULD, SHOULD NOT, and MAY are normative. + +The authorization model is: + +#+begin_example +request + -> credential parser + -> credential verifier + -> principal resolver + -> capability + scope authorization + -> application operation + -> audit result +#+end_example + +Every protected request MUST produce one immutable request security context. +Missing or invalid context MUST deny. Authorization MUST be default deny. + +* Identifiers + +All identifiers are opaque strings at the API boundary. Runtime code MUST NOT +derive authority from identifier shape, prefix, ordering, or embedded user input. + +| Field | Meaning | Secret | +|-------+---------+--------| +| =principal_id= | Stable identity record | No | +| =credential_id= | Stable identifier for one issued credential | No | +| =tenant_id= | Security and data isolation boundary | No | +| =decision_id= | Identifier for one authorization decision | No | +| =correlation_id= | Identifier for one request flow | No | +| API key secret | Bearer secret used for verification | Yes | +| verifier | Stored one-way credential verifier | Sensitive | +| pepper | Server-side verifier secret | Yes | + +Identifiers MUST have bounded length and a canonical representation. Identifier +comparison MUST use the canonical form defined by the implementing module. + +Deleted or revoked credential identifiers MUST NOT be reused. + +* Principal record + +A principal record represents an identity independent of any one credential. +One principal MAY own multiple credentials. Revoking one credential does not +necessarily disable the principal. + +Conceptual record: + +#+begin_src json +{ + "principal_id": "prn_01...", + "principal_type": "api_client", + "display_name": "quasar-ui-production", + "status": "active", + "tenant_ids": ["tenant-main"], + "attributes": { + "service": "quasar-ui" + }, + "created_at": "2026-07-30T20:00:00Z", + "updated_at": "2026-07-30T20:00:00Z", + "disabled_at": null +} +#+end_src + +Required fields: + +| Field | Type | Rule | +|-------+------+------| +| =principal_id= | string | Stable and unique | +| =principal_type= | enum | One recognized principal class | +| =display_name= | string | Human-readable, not authoritative | +| =status= | enum | =active=, =disabled=, or =deleted= | +| =tenant_ids= | array | Explicit tenant memberships | +| =created_at= | timestamp | Immutable | +| =updated_at= | timestamp | Changes with record mutation | + +Attributes are descriptive metadata. Attributes MUST NOT grant capabilities +unless a separately versioned policy explicitly maps them into grants. + +* Principal classes + +** Human user + +Value: =human_user= + +Represents an interactive operator, analyst, reviewer, or administrator. + +v0.1 credential: + +- API key, intended for CLI or trusted native use. + +Future credential: + +- OIDC identity mapped by issuer, subject, and tenant rules. + +A human user MUST NOT share a credential with another human or service. Human +administrator access uses a separately issued administrator credential. + +** API client + +Value: =api_client= + +Represents a script, CLI integration, web backend, or application instance that +calls HTTP endpoints. + +API clients SHOULD receive the minimum capability and scope needed for one +integration purpose. Different deployments or environments SHOULD use separate +principals and credentials. + +** Service instance + +Value: =service_instance= + +Represents a long-running external service, collector, scheduler, bridge, or +backend. + +A service principal SHOULD be unique per service and environment. Horizontal +replicas MAY share a credential initially, but unique workload credentials are +preferred when lifecycle tooling supports them. + +Future mTLS identity MUST map certificate identity to an existing service +principal rather than creating implicit authority from certificate fields. + +** Actor component + +Value: =actor_component= + +Represents an actor runtime component or external actor service. + +Actor principals MUST include explicit actor scope. An actor principal does not +receive general document, search, target dispatch, or administrative authority +unless those capabilities are separately granted. + +** Administrator + +Value: =administrator= + +Represents a principal authorized for security and deployment administration. + +Administrator principals: + +- MUST use separately issued credentials; +- MUST be audited at high priority; +- SHOULD use short-lived or frequently rotated credentials; +- MUST NOT be created through general document APIs; +- MUST NOT be inferred from operating-system username, source address, or local + process location; +- MUST still satisfy tenant and operation invariants defined by administrative + APIs. + +* Credential types + +** v0.1 API key + +Credential type: =api_key_v1= + +External representation: + +#+begin_example +star_sk_v1__ +#+end_example + +Properties: + +- =star_sk= identifies a StarIntel secret key; +- =v1= selects the parser and verifier contract; +- =credential-id= is non-secret and supports record lookup; +- =secret= is high-entropy random bearer material; +- separators and alphabets MUST be unambiguous; +- total length MUST be bounded; +- malformed values deny before verifier work; +- unsupported versions deny without fallback. + +The exact entropy and encoding parameters MUST be constants in the implementing +module and covered by tests. Secret generation MUST use a cryptographically +secure random source. + +The complete key is returned only once, at creation. It MUST NOT be retrievable +later. + +Authorization header: + +#+begin_example +Authorization: Bearer star_sk_v1__ +#+end_example + +Only one credential is accepted per request. Multiple Authorization headers, +multiple bearer values, or conflicting credential sources MUST reject. + +** Future OIDC adapter + +Credential type: =oidc_v1= + +OIDC is intended for human identity. The adapter MUST: + +- configure an explicit issuer allowlist; +- validate signature, issuer, audience, expiration, and required claims; +- map issuer and subject to one existing principal; +- reject unprovisioned identities unless an explicit provisioning policy exists; +- never map email address alone to administrator authority; +- keep token claims descriptive unless policy explicitly consumes them; +- produce the same principal and authorization context shape as API keys. + +** Future mTLS adapter + +Credential type: =mtls_v1= + +mTLS is intended for service and actor identity. The adapter MUST: + +- validate the chain against configured trust anchors; +- validate validity period and intended usage; +- apply revocation policy; +- map a stable certificate identity to an existing principal; +- reject certificates that match no explicit binding; +- not infer capabilities directly from arbitrary subject fields; +- produce the same principal and authorization context shape as API keys. + +* Credential record + +Conceptual stored record: + +#+begin_src json +{ + "credential_id": "key_01...", + "credential_type": "api_key_v1", + "principal_id": "prn_01...", + "status": "active", + "verifier": "encoded-memory-hard-verifier", + "salt": "encoded-random-salt", + "pepper_version": "pepper-2026-01", + "hash_parameters": { + "algorithm": "argon2id", + "memory": 0, + "iterations": 0, + "parallelism": 0 + }, + "not_before": null, + "expires_at": null, + "created_at": "2026-07-30T20:00:00Z", + "last_used_at": null, + "revoked_at": null, + "rotation_parent_id": null, + "description": "Quasar production write credential" +} +#+end_src + +The numeric hash parameters above are placeholders for implementation-selected +values. They MUST be centrally configured, benchmarked for the deployment class, +and stored with each verifier for migration. + +Credential status values: + +| Status | Authentication result | +|--------+-----------------------| +| =active= | Continue verification and policy checks | +| =disabled= | Deny | +| =revoked= | Deny permanently | +| =expired= | Deny | +| =deleted= | Deny; retain tombstone metadata | + +Credential records MUST be stored outside ordinary StarIntel datasets and MUST +not be reachable through document, search, export, graph, or relation APIs. + +* Verification and hashing + +API key verification MUST: + +1. parse the prefix, version, identifier, and secret with strict bounds; +2. look up the credential record by non-secret identifier; +3. perform a constant-time verifier comparison; +4. include the configured pepper version in verifier derivation; +5. check credential and principal status; +6. check =not_before= and =expires_at=; +7. resolve grants only after successful verification; +8. erase or release secret-bearing intermediate values as soon as practical; +9. update last-used metadata asynchronously and without changing the decision; +10. return a uniform external failure for unknown, invalid, expired, disabled, + and revoked credentials. + +Unknown credential identifiers SHOULD follow a bounded dummy-verifier path so +identifier existence is not exposed by large timing differences. + +Pepper values MUST be loaded from a secret provider outside the authorization +database. Credential records contain only a pepper version identifier. + +* Capabilities + +Capability names are closed, lowercase, colon-delimited strings. Unknown +capabilities deny. Grants are additive, but an explicit deny overrides an allow. + +** Core capabilities + +| Capability | Permitted operation | +|------------+---------------------| +| =documents:read= | Read individual documents within scope | +| =documents:write= | Create or update documents within scope | +| =documents:delete= | Delete documents within scope | +| =documents:bulk= | Submit bounded bulk document operations within scope | +| =search:read= | Execute search constrained to readable scope | +| =targets:read= | Read target definitions and status within scope | +| =targets:dispatch= | Submit or dispatch targets to permitted actors | +| =targets:lease= | Acquire, renew, or release actor target leases | +| =datasets:read= | Read dataset metadata | +| =datasets:manage= | Create or modify dataset metadata and policy within delegated scope | +| =actors:read= | Read actor metadata and status | +| =actors:invoke= | Invoke permitted actor operations | +| =audit:read= | Read approved audit records | +| =credentials:read= | Read non-secret credential metadata | +| =credentials:create= | Issue credentials within delegated principal/scope rules | +| =credentials:rotate= | Rotate credentials | +| =credentials:revoke= | Revoke credentials | +| =principals:manage= | Create, disable, and update principals and grants | +| =admin= | Administrative superset, subject to explicit route mapping and audit | + +A route MUST map to one primary capability. Complex operations MAY require +additional capabilities. Having =documents:write= does not imply +=documents:delete=, =documents:bulk=, =targets:dispatch=, or =datasets:manage=. + +=admin= is not an escape hatch for unmapped routes. An unmapped route denies even +for administrators. + +* Grants and explicit denies + +Conceptual grant record: + +#+begin_src json +{ + "grant_id": "grant_01...", + "principal_id": "prn_01...", + "effect": "allow", + "capability": "documents:write", + "scope": { + "tenant_ids": ["tenant-main"], + "dataset_ids": ["hunter-biden"], + "actor_names": [], + "target_ids": [] + }, + "conditions": { + "expires_at": null + }, + "created_at": "2026-07-30T20:00:00Z", + "created_by": "prn_admin_01..." +} +#+end_src + +Effects: + +- =allow= grants a capability within the specified scope; +- =deny= removes authority within the specified scope and overrides matching + allows. + +Grant evaluation MUST be deterministic and independently testable. The initial +implementation SHOULD avoid a general-purpose policy language. Closed capability +and scope records are easier to review and harder to misconfigure. + +* Scope model + +Scopes restrict where a capability applies. An empty list means no resources, +not all resources. Wildcards MUST be explicit values and SHOULD be reserved for +administrators or tightly controlled service principals. + +** Tenant scope + +Every protected resource belongs to one tenant security boundary, even if the +initial deployment has one tenant. + +A principal MUST have membership in the tenant and a matching capability scope. +Request-supplied tenant values do not create membership. + +** Dataset scope + +Dataset scope applies to document read, write, delete, bulk, search, export, +graph, and relation operations. + +A document write MUST validate the document dataset against the authorized +scope. The server MUST not authorize against a route dataset and then accept a +conflicting dataset in the body. + +** Actor scope + +Actor scope applies to actor metadata, invocation, target dispatch, and target +leases. Actor names MUST be canonicalized before policy evaluation. + +** Target scope + +Target scope can restrict individual target identifiers or target groups. Target +scope does not imply actor scope, and actor scope does not imply target scope. +Operations requiring both MUST satisfy both. + +** Resource scope + +Specific resources MAY be represented by canonical resource identifiers. The +contract MUST avoid accepting raw CouchDB query fragments or arbitrary route +strings as authorization scopes. + +* Authorization request + +The application boundary submits a normalized authorization request: + +#+begin_src json +{ + "principal_id": "prn_01...", + "principal_type": "api_client", + "credential_id": "key_01...", + "capability": "documents:write", + "resource": { + "tenant_id": "tenant-main", + "dataset_id": "hunter-biden", + "actor_name": null, + "target_id": null, + "resource_id": "doc_01..." + }, + "operation": { + "route": "/new/document/:dtype", + "method": "POST", + "dtype": "person" + }, + "correlation_id": "corr_01..." +} +#+end_src + +The route is descriptive audit context. Authorization MUST use the explicit +capability and canonical resource fields, not string matching against the raw +URL. + +* Authorization decision + +Conceptual decision: + +#+begin_src json +{ + "decision_id": "dec_01...", + "allowed": true, + "reason": "matching_allow_grant", + "principal_id": "prn_01...", + "credential_id": "key_01...", + "capability": "documents:write", + "matched_grant_ids": ["grant_01..."], + "evaluated_at": "2026-07-30T20:00:00Z" +} +#+end_src + +The external client receives only a stable error code on denial. Grant IDs, +policy internals, principal existence, and credential state remain internal. + +Decision reasons are a closed internal enum, including: + +- =matching_allow_grant=; +- =explicit_deny=; +- =missing_capability=; +- =scope_mismatch=; +- =principal_disabled=; +- =credential_invalid=; +- =credential_expired=; +- =credential_revoked=; +- =policy_unavailable=; +- =route_unmapped=. + +* Request security context + +After authentication, the HTTP layer creates one immutable context for the +request: + +#+begin_src lisp +(defstruct request-security-context + correlation-id + decision-id + principal-id + principal-type + credential-id + tenant-ids + capability + resource-scope + authenticated-at) +#+end_src + +The exact Lisp representation may differ, but semantics MUST match. + +Rules: + +- the context contains no credential secret, verifier, salt, or pepper; +- the context is established once and passed explicitly; +- application functions MUST not recover authority from global mutable state; +- asynchronous jobs copy only required non-secret fields; +- RabbitMQ provenance uses a safe projection of this context; +- missing context denies security-sensitive operations. + +* Route mapping + +Every protected route MUST register: + +1. route template; +2. HTTP method; +3. required capability; +4. canonical resource extractor; +5. whether body fields participate in scope; +6. whether the operation requires high-priority audit; +7. rate-limit class. + +Example conceptual mapping: + +| Route | Method | Capability | Scope source | +|-------+--------+------------+--------------| +| =/document/:id= | GET | =documents:read= | resolved document tenant/dataset | +| =/document/:id= | DELETE | =documents:delete= | resolved document tenant/dataset | +| =/new/document/:dtype= | POST | =documents:write= | body tenant/dataset and path dtype | +| =/documents/bulk= | POST | =documents:bulk= and =documents:write= | every document tenant/dataset | +| =/search= | GET | =search:read= and readable datasets | query scope | +| =/targets/:actor= | GET | =targets:read= | actor and tenant scope | +| =/new/target/:actor= | POST | =targets:dispatch= | actor, target, tenant, dataset | +| =/bulk/jobs/:id= | GET | =documents:bulk= | job owner principal and dataset scope | + +Health endpoints MAY be public only when they expose no sensitive configuration. +Root metadata and capability-discovery endpoints require a deliberate public or +protected classification. + +A startup validation pass MUST fail if a registered protected route has no +mapping. + +* Bulk authorization + +Bulk requests MUST be bounded before policy evaluation. + +Initial atomicity rule: + +- validate and authorize every document before enqueue or publish; +- if any document fails authentication-independent validation or authorization, + reject the entire request; +- do not partially enqueue unauthorized batches; +- asynchronous jobs retain the submitting principal identifier, credential + identifier, decision identifier, and authorized dataset set; +- job status is readable only by the submitting principal or a principal with an + explicit administrative/audit grant; +- per-principal quotas use authenticated principal and credential identity, not + only source address. + +A future partial-success mode requires a separate capability and response +contract. + +* Search authorization + +Search MUST execute within authorized tenant and dataset scope. + +The implementation MUST NOT: + +- search all records and merely redact unauthorized document bodies afterward; +- expose unauthorized counts, facets, bookmarks, or sort behavior; +- include the authentication database or audit store; +- accept arbitrary index names or design documents from the client. + +The search request scope is intersected with the principal's readable scope. +An empty intersection returns an access denial or an explicitly documented empty +result behavior that does not leak resource existence. + +* Target and actor authorization + +Target dispatch requires: + +- =targets:dispatch=; +- matching tenant and dataset scope; +- matching actor scope; +- matching target scope when target restrictions exist. + +Target leasing requires =targets:lease= and actor identity binding. The actor +claim in the request body, URL, credential metadata, and resolved target MUST +agree after canonicalization. + +Actor components SHOULD receive separate credentials for lease and document +write behavior when operationally feasible. + +* Administrative operations + +Administrative routes MUST require explicitly mapped capabilities. =admin= MAY +satisfy those mappings, but all of these still apply: + +- separate administrator credential; +- high-priority audit before and after mutation; +- no secret returned except once during credential creation; +- reason or ticket metadata SHOULD be required for destructive actions; +- self-grant and self-audit suppression are prohibited; +- last-administrator removal MUST be prevented outside documented recovery; +- bootstrap state cannot be re-enabled through an ordinary request. + +Credential metadata reads MUST redact verifier, salt, pepper version when not +operationally required, hash parameters if sensitive to policy, and all secret +material. + +* Rate-limit classes + +Rate limiting uses principal identity after successful authentication and source +context before authentication. + +| Class | Examples | Relative cost | +|-------+----------+---------------| +| =public-light= | health | low | +| =auth-verify= | failed credential checks | expensive | +| =read= | document reads | low to medium | +| =search= | full-text search | high | +| =write= | document and target writes | medium | +| =bulk= | bulk enqueue | high and quota-bound | +| =admin= | principal and credential mutation | low volume, high sensitivity | + +Rate-limit decisions MUST not grant authority and MUST not reveal credential +existence. Limits and queue capacities MUST be bounded configuration values. + +* Error contract + +Authentication errors: + +#+begin_src json +{ + "status": "error", + "code": "invalid_credential", + "msg": "Authentication failed", + "correlation_id": "corr_01..." +} +#+end_src + +Authorization errors: + +#+begin_src json +{ + "status": "error", + "code": "access_denied", + "msg": "Access denied", + "correlation_id": "corr_01..." +} +#+end_src + +Rules: + +- unknown, invalid, disabled, expired, and revoked credentials share the same + external authentication failure; +- responses never include raw authorization headers, key identifiers unless + intentionally non-sensitive, verifier state, grant IDs, or policy internals; +- status codes follow the threat model; +- correlation identifiers are always present; +- internal logs record the safe decision reason. + +* Audit contract + +The security context projects into an audit event: + +#+begin_src json +{ + "event_type": "authorization_decision", + "correlation_id": "corr_01...", + "decision_id": "dec_01...", + "principal_id": "prn_01...", + "principal_type": "api_client", + "credential_id": "key_01...", + "capability": "documents:write", + "tenant_id": "tenant-main", + "dataset_id": "hunter-biden", + "actor_name": null, + "target_id": null, + "route": "/new/document/:dtype", + "method": "POST", + "decision": "allow", + "result": "accepted", + "timestamp": "2026-07-30T20:00:00Z" +} +#+end_src + +Audit events MUST NOT contain credential secrets, authorization headers, +verifiers, salts, peppers, full request bodies, or raw exception conditions. + +* Caching contract + +Authentication and authorization caching MAY be added with these constraints: + +- cache keys contain non-secret identifiers and version numbers; +- secret material is never a cache key or value; +- cached grants have a bounded lifetime; +- principal disable, grant change, and credential revocation invalidate relevant + cache entries immediately; +- authorization-store failure does not convert a stale cache miss into allow; +- cache behavior is covered by concurrency and revocation tests. + +* Configuration contract + +Required security mode values: + +- =disabled=: only allowed for explicitly validated loopback development; +- =api-key=: v0.1 production authentication; +- future =oidc=, =mtls=, or composed modes only after implementation. + +Suggested configuration names: + +#+begin_example +STAR_AUTH_MODE=api-key +STAR_AUTH_STORE_DATABASE=starintel-auth +STAR_AUTH_PEPPER_ACTIVE_VERSION=pepper-2026-01 +STAR_AUTH_PEPPER_FILE=/run/secrets/starintel-auth-pepper +STAR_AUTH_ALLOWED_ORIGINS=https://quasar.example +STAR_AUTH_DEV_BYPASS=false +#+end_example + +Names are provisional until implementation, but validation semantics are not: + +- production mode with authentication disabled MUST abort startup; +- API-key mode without an active pepper MUST abort startup; +- development bypass on a non-loopback listener MUST abort startup; +- wildcard browser origin with credential-bearing browser mode MUST abort startup + or be rejected as invalid configuration; +- unknown configuration values MUST abort startup. + +* Migration and compatibility + +Authentication introduction is a breaking deployment change even if route paths +remain stable. + +Migration plan: + +1. ship threat model and contract; +2. implement credential store, verifier, and principal records; +3. implement route mapping and default-deny middleware; +4. add bootstrap tooling; +5. add audit storage and redaction tests; +6. add explicit development mode; +7. update clients to send credentials; +8. change deployment defaults to require authentication; +9. remove or tightly constrain wildcard CORS; +10. document credential rotation and recovery. + +There MUST be no silent fallback from invalid credentials to anonymous or +development authority. + +* Required test matrix + +** Credential parsing and verification + +- valid API key authenticates; +- missing credential denies; +- malformed prefix denies; +- unsupported version denies; +- unknown identifier denies; +- wrong secret denies; +- disabled, revoked, expired, and not-yet-valid credentials deny; +- principal disabled denies all credentials; +- timing behavior does not trivially enumerate identifiers; +- logs and errors contain no secret material; +- pepper rotation and verifier rehash work; +- revocation invalidates cache immediately. + +** Authorization + +- every protected route has a mapping; +- unmapped route denies for ordinary and administrator principals; +- each capability permits only its mapped operations; +- explicit deny overrides allow; +- tenant, dataset, actor, target, and resource scopes isolate correctly; +- body and route scope mismatches deny; +- search cannot reveal unauthorized counts or records; +- bulk rejects the entire batch when one item is unauthorized; +- bulk job status is owner-scoped; +- internal callers without context deny. + +** Operational controls + +- rate limits are bounded and principal-aware; +- authorization store outage fails closed; +- audit write failure blocks high-risk administrative mutation; +- development bypass cannot start on a non-loopback listener; +- production mode cannot start with auth disabled; +- wildcard CORS is rejected for credential-bearing browser configuration; +- bootstrap cannot run twice; +- last administrator cannot be removed normally. + +* Change control + +Adding or renaming a principal class, credential type, capability, scope field, +decision reason, or security mode is a security-contract change. Such changes +MUST update: + +- this document; +- the threat model; +- route mappings; +- client capability discovery where applicable; +- migration notes; +- hermetic and integration tests. diff --git a/docs/index.org b/docs/index.org new file mode 100644 index 00000000..4ad0305d --- /dev/null +++ b/docs/index.org @@ -0,0 +1,54 @@ +#+title: StarIntel Server documentation +#+options: toc:2 + +* Documentation index + +This directory documents the implemented StarIntel Server runtime. It is written +from the code in =source/starintel-gserver.asd=, not from an aspirational +architecture. + +Security design documents are explicitly marked =DESIGN= and do not claim that +the corresponding controls are implemented. + +** Start here + +1. [[file:../README.org][README]] — install, run, submit a document, and find the main entry points. +2. [[file:architecture.org][Architecture]] — understand startup, components, concurrency, and the repository. +3. [[file:document-spec.org][Document specification]] — understand the data that moves through the system. +4. [[file:actors.org][Actors]] — build local actors and connect external actor services. +5. [[file:messaging.org][Messaging]] — understand routing, delivery, recursion, and loop control. +6. [[file:configuration.org][Configuration]] — configure local, container, remote, and tuned deployments. +7. [[file:http-api-docs.org][HTTP API]] — call the service. +8. [[file:http-authentication-runtime.org][HTTP authentication runtime]] — bootstrap, API-key lifecycle, CORS, request context, revocation behavior, and deployment. +9. [[file:../DOCKER.md][Docker/Nix stack]] and [[file:testing.md][testing]] — operate and verify it. +10. [[file:http-auth-threat-model.org][HTTP authentication threat model]] — protected assets, attackers, boundaries, controls, failures, and residual risks. Design contract. +11. [[file:http-auth-kv-lease-boundary.org][KV lease authentication boundary]] — target-lease assets, atomic ownership, fencing, replay controls, and the KV trust boundary. Normative design contract. +12. [[file:http-principal-capability-contract.org][HTTP principal and capability contract]] — principal classes, credentials, capabilities, scopes, decisions, and route mapping. Design contract. + +** Implementation status + +| Area | Status | Notes | +|------+--------+-------| +| CouchDB database initialization | Active | Creates main, actor-event, and separate authentication databases; upserts required design documents | +| RabbitMQ document ingest | Active | =documents.ingest.#= | +| RabbitMQ update ingest | Active | =documents.updated.#=; partial deep merge and conflict retry | +| Target routing | Active | Local Sento actor or =actors..new.target= | +| Actor event storage | Active | Local receiver and separate =events= exchange consumer | +| HTTP API-key authentication | Active | Default-deny bearer authentication, immutable request context, lifecycle routes, exact-origin CORS, and separate credential storage | +| Fine-grained route authorization | Not implemented | Stored scopes exist, but ordinary protected routes do not yet enforce capability/resource policy beyond administrator lifecycle routes and bulk-job ownership | +| Authentication revocation cache | Active | No credential cache; committed revoke/disable is visible to the next verifier lookup | +| KV target leases | Design only | No KV lease adapter exists; authenticated service context is propagated for future integration | +| Clouseau full-text search | Active in Compose | Search endpoint uses CouchDB FTS design document and now requires authentication | +| URL extractor pattern | Experimental | Actor starts; complete global pattern dispatcher is not evident | +| User-finder/user-hunt actors | Present, inactive | Files are not in the ASDF component list | +| HTTP event endpoint | Stub | =/new/event/:id= has no implementation | +| Strict StarIntel 0.9 ingest validation | Partially wired | HTTP boundary validates required envelope fields and schema version; Rabbit ingest remains a separate boundary | + +** Source-of-truth rule + +When documentation and code disagree, the current code wins. Correct the docs in +the same pull request as behavior changes. + +The document model itself lives in [[https://github.com/lost-rob0t/star-cl][lost-rob0t/star-cl]]. The Python adapter lives +in [[https://github.com/lost-rob0t/starintel-doc][lost-rob0t/starintel-doc]]. This server transports and stores those documents, +but it currently also carries legacy flat 0.8 behavior. \ No newline at end of file diff --git a/docs/messaging.org b/docs/messaging.org new file mode 100644 index 00000000..c0dd63ee --- /dev/null +++ b/docs/messaging.org @@ -0,0 +1,258 @@ +#+title: StarIntel messaging and recursive dataflow +#+options: toc:3 + +* Messaging + +RabbitMQ carries distributed document traffic. Sento mailboxes carry local actor +traffic. They are separate layers and have different guarantees. + +** Documents exchange + +The main exchange is a durable RabbitMQ topic exchange named =documents=. + +| Flow | Queue | Binding / emitted key | Meaning | +|------+-------+-----------------------+---------| +| Initial ingest | =documents.ingest= | bind =documents.ingest.#= | Persist a new document | +| Initial ingest by type | — | emit =documents.ingest.= | Standard producer key | +| Post-insert event | actor-specific | emit =documents.new.= | Document now has CouchDB =_id= and =_rev= | +| Update ingest | =documents.updates.ingest= | bind =documents.updated.#= | Persist a partial/full update | +| Update event | same topic family | emit =documents.updated.= | Updated document with latest =_rev= | +| Target intake | =documents.targets= | bind =documents.new.target.#= | Route a persisted target | +| Remote actor target | actor-defined | emit =actors..new.target= | Deliver target to an external actor | + +The target wildcard currently equals =documents.new.target.#=. A normal target +is published to =documents.ingest.target=, persisted, republished as +=documents.new.target=, then consumed by the target router. + +** Events exchange + +Actor events can also enter through a separate exchange and queue: + +| Exchange | Queue | Binding | +|----------+-------+---------| +| =events= | =events= | =event.#= | + +The consumer converts the event JSON to an =actor-event= and tells the local +actor-event receiver, which writes it to the event database. + +Local actors normally call =log-actor-event= directly and skip RabbitMQ. + +** Local actor messages + +*** Tell + +Use =sento.actor:tell= for one-way local messages: + +#+begin_src lisp +(tell actor-ref message) +#+end_src + +Actor lookup by operator name: + +#+begin_src lisp +(let ((actor (star.actors:get-dest-actor "domain-enricher"))) + (when actor + (tell actor document))) +#+end_src + +*** Internal database actors + +The server has local actors for CouchDB insert and get operations. Their message +formats are implementation-specific lists/plists. They are useful inside the +server but are not a stable network protocol. + +*** Timers + +Recurring target work uses =timer-wheel=. The target's =delay= is the interval +in seconds. A recurring target is scheduled on first delivery and later +deliveries are sent directly without scheduling another timer. + +** Delivery and acknowledgement + +The system provides *at-least-once processing*, not exactly-once processing. + +- A successful handler ACKs. +- Most errors NACK and requeue. +- A process crash can occur after an external side effect but before ACK. +- Multiple consumers or retries can observe the same document. +- Initial insert conflicts are treated as a duplicate/conflict and are not + requeued. +- Update handlers refetch the current CouchDB revision and retry conflicts. + +Every actor must therefore be idempotent. + +Recommended rules: + +1. Use deterministic IDs for deterministic findings. +2. Treat CouchDB conflict as a duplicate unless the payload is materially + different. +3. Put external side effects behind a durable outbox or their own idempotency + key. +4. Never rely on message ordering across queues. +5. Do not use Rabbit delivery count as recursion depth. + +** Durable versus event-only actor output + +*** Durable output + +Publish derived documents to =documents.ingest.= when they must be +stored: + +#+begin_src lisp +(star.actors:publish + star.actors:*producer-agent* + :body (jsown:to-json derived) + :routing-key (format nil "documents.ingest.~a" + (jsown:val derived "dtype")) + :properties (list (cons :type (jsown:val derived "dtype")))) +#+end_src + +This path assigns =_id= when absent, writes CouchDB, attaches =_rev=, and emits +=documents.new.=. + +*** Event-only output + +Publish to =documents.new.= only when the document is already durable or +when an actor deliberately wants an event that bypasses the ingest consumer. + +Several experimental actors currently publish directly to =documents.new.*=. +That does *not* persist a new document by itself. + +** Recursive nature of actors + +StarIntel recursion is a graph/dataflow property: + +#+begin_example +person + -> username candidate actor + -> user documents + -> account verifier actor + -> verified user documents + -> relation documents + -> graph/query actors + -> more targets +#+end_example + +Each output can become another input. An actor may: + +- derive zero, one, or many documents; +- attach relations between source and derived documents; +- create a target for itself or another actor; +- update the source document; +- emit an actor event; +- schedule the target again. + +The recursion may be local, distributed, delayed, and branching. There is no +global recursion controller in the current server. + +** Loop prevention + +A recursive actor network can become an accidental message amplifier. Every +actor chain should enforce a termination policy. + +*** Deterministic identity + +Use a stable ID from the source identity, actor identity, and derivation type. +The legacy =star-cl= constructors already hash many object types. For custom +documents: + +#+begin_example +derived-id = hash(root-id, source-id, actor-name, operation, normalized-value) +#+end_example + +*** Depth and hop budget + +The server does not add a hop counter automatically. Actors should carry a +shared extension object: + +#+begin_src json +{ + "extensions": { + "star_server": { + "trace_id": "01J...", + "root_id": "root-document-id", + "parent_id": "source-document-id", + "actor_path": ["seed", "domain-enricher", "dns-resolver"], + "depth": 2, + "hop": 3, + "max_depth": 8, + "max_hops": 32 + } + } +} +#+end_src + +Reject, dead-letter, or emit a terminal event when a budget is exceeded. + +*** Actor-path cycle check + +Do not invoke an actor again when its name is already present in =actor_path= +unless the actor explicitly supports fixed-point iteration. + +*** Fixed-point actors + +An actor that refines a document repeatedly should stop when its normalized +output hash equals the previous output hash. Store the hash in actor metadata. + +*** Recurring targets + +Recurring targets are intentional cycles. Set a positive delay, use a stable +target ID, and make each run idempotent. A delay of zero on a recurring target +is an operational hazard. + +** Recursive metadata propagation + +The server itself currently: + +- preserves fields present in a document; +- deep-merges nested objects during updates; +- adds CouchDB =_id= and =_rev=; +- does not automatically append actor provenance, sources, depth, or trace data. + +Actor code owns recursive metadata propagation. + +A derived document should: + +1. preserve the original =dataset= unless crossing datasets intentionally; +2. carry forward =sources= and append the actor/tool source; +3. set =parent_id= to the input document; +4. preserve =root_id= and =trace_id=; +5. increment =depth= and =hop=; +6. append the actor name to =actor_path=; +7. create a =relation= using =derived-from=, =extracted-from=, + =discovered-by=, or another allowed predicate; +8. call =log-actor-event= for important lifecycle events. + +** Transient messages + +A JSON document with =transient: true= is filtered out by the standard ingest +and update consumers. It can still be routed to actor-specific queues. + +Use transient messages for high-volume intermediate work that should not enter +CouchDB. Do not use them for evidence that must be reproducible. + +** External actor contract + +An external actor service normally: + +1. declares a durable queue; +2. binds it to =documents= with one or more topic keys; +3. parses one StarIntel JSON document per message; +4. performs idempotent work; +5. publishes durable outputs to =documents.ingest.=; +6. publishes partial updates to =documents.updated.=; +7. publishes actor events to =events= / =event.= when needed; +8. ACKs only after required side effects complete. + +Suggested queue naming: + +#+begin_example +actors..targets +actors..documents. +#+end_example + +Suggested target binding: + +#+begin_example +actors..new.target +#+end_example diff --git a/docs/relation-deduplication.org b/docs/relation-deduplication.org new file mode 100644 index 00000000..7bcd1726 --- /dev/null +++ b/docs/relation-deduplication.org @@ -0,0 +1,74 @@ +#+title: Relation database deduplication + +* Scope + +This migration repairs CouchDB relation documents before any graph/UI changes. +It treats a relation as the directed identity: + +#+begin_example +(dataset, source, predicate, target) +#+end_example + +The canonical CouchDB document ID is =relation:= followed by the SHA-256 digest +of a length-prefixed encoding of those four fields. Notes and sources are +merged. Legacy relation documents are tombstoned only after the canonical +upserts succeed. + +Both legacy flat 0.8 relations and 0.9 relations with fields under =data= are +supported. + +* Dry run + +The script reads the normal CouchDB environment variables used by star-server: + +#+begin_src sh +python3 scripts/dedupe_relations.py +#+end_src + +This only prints the plan. It does not write anything. + +Useful explicit invocation: + +#+begin_src sh +python3 scripts/dedupe_relations.py \ + --url http://127.0.0.1:5984 \ + --database starintel \ + --user admin \ + --password-file /run/secrets/couchdb_password +#+end_src + +* Apply + +Stop relation-producing actors or pause the RabbitMQ ingest queue first. Then: + +#+begin_src sh +python3 scripts/dedupe_relations.py --apply +#+end_src + +An automatic JSONL backup of every original relation document is written before +mutation. Use =--backup PATH= to choose the path. Do not use =--no-backup= unless +there is already a verified CouchDB snapshot. + +The write is two-phase: + +1. create or update canonical relation documents; +2. tombstone the superseded legacy IDs. + +If canonical upserts fail, deletion does not begin. + +* Verification + +Run the dry-run again after applying: + +#+begin_src sh +python3 scripts/dedupe_relations.py +#+end_src + +Expected values: + +- =duplicate_documents=: 0 +- =canonical_upserts=: 0 +- =legacy_deletions=: 0 +- =invalid_relations=: 0, unless malformed relations already existed + +Malformed relations are reported and never deleted automatically. diff --git a/example_configs/init.lisp b/example_configs/init.lisp index 5769cb05..5f846787 100644 --- a/example_configs/init.lisp +++ b/example_configs/init.lisp @@ -1,14 +1,63 @@ -;; Example config +;;;; StarIntel Server init file +;;;; +;;;; This file is executable Common Lisp. Load it only from a trusted, +;;;; root/operator-controlled path. + (in-package :star) -(format t "Starting starintel....") -(setq *rabbit-address* "rabbitmq") -(setq *couchdb-host* "bots.star.intel") -(setq *couchdb-default-database* "starintel") -;; You can invoke sylnk like so -;; (start-debugger) -;; Set log config path to logs -(log:config :daily "logs/gserver.log" + +(format t "~&Starting StarIntel Server configuration...~%") + +;;; CouchDB ----------------------------------------------------------------- + +(setf *couchdb-host* (or (uiop:getenv "COUCHDB_HOST") "127.0.0.1") + *couchdb-port* 5984 + *couchdb-scheme* "http" + *couchdb-user* (or (uiop:getenv "COUCHDB_USER") "admin") + *couchdb-password* (or (uiop:getenv "COUCHDB_PASSWORD") "") + *couchdb-default-database* + (or (uiop:getenv "COUCHDB_DATABASE") "starintel") + *couchdb-event-log-database* "starintel-event-source") + +;;; RabbitMQ --------------------------------------------------------------- + +(setf *rabbit-address* (or (uiop:getenv "RABBITMQ_ADDRESS") "127.0.0.1") + *rabbit-port* 5672 + *rabbit-user* (or (uiop:getenv "RABBITMQ_USER") "guest") + *rabbit-password* (or (uiop:getenv "RABBITMQ_PASSWORD") "")) + +;;; HTTP ------------------------------------------------------------------- + +(setf *http-api-address* + (or (uiop:getenv "HTTP_API_LISTEN_ADDRESS") "127.0.0.1") + *http-api-port* 5000 + *bulk-max-documents* 500) + +;;; Concurrency ------------------------------------------------------------ + +(setf *ingest-workers* 4 + star.actors:*publish-timeout-seconds* 5) + +;;; Logging ---------------------------------------------------------------- + +(ensure-directories-exist #P"logs/") +(log:config :daily "logs/star-server.log" :file2 :sane) -;; +;;; Optional local actors -------------------------------------------------- +;;; +;;; Actor definition files loaded here can add startup/registration functions +;;; to STAR:*ACTORS-START-HOOK*. The hook runs after the Sento actor system, +;;; producer agent, database actors, target timer, and target router exist. +;;; +;;; Example: +;;; (load #P"/etc/starintel/actors/domain-enricher.lisp") + +;;; Optional SLYNK debugger ------------------------------------------------ +;;; +;;; SLYNK is remote code execution. Use loopback/SSH forwarding only. +;;; +;;; (setf *slynk-port* 4009) +;;; (start-debugger) + +(format t "~&StarIntel configuration loaded.~%") diff --git a/schema/starintel-schema.lock.json b/schema/starintel-schema.lock.json new file mode 100644 index 00000000..1f2a31c0 --- /dev/null +++ b/schema/starintel-schema.lock.json @@ -0,0 +1,16 @@ +{ + "schema_version": "0.9.0", + "canonical_repository": "lost-rob0t/starintel-gpt-auto-dig", + "canonical_commit": "ff814ff63868286d68e21502122832802cd5e361", + "canonical_pull_request": 46, + "schema_path": "schemas/starintel-doc-v0.9.0.schema.json", + "expansion_path": "schemas/starintel-doc-v0.9.0.expansion.json", + "manifest_path": "schemas/starintel-doc-v0.9.0.manifest.json", + "required_dtypes": [ + "research-node" + ], + "research_node_required_fields": [ + "objective", + "status" + ] +} diff --git a/scripts/check-starintel-schema-lock.py b/scripts/check-starintel-schema-lock.py new file mode 100644 index 00000000..1169c463 --- /dev/null +++ b/scripts/check-starintel-schema-lock.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import hashlib +import json +import sys +import urllib.request +from pathlib import Path +from typing import Any + + +def fail(message: str) -> None: + raise SystemExit(message) + + +def load_json(url: str) -> dict[str, Any]: + with urllib.request.urlopen(url, timeout=30) as response: + return json.load(response) + + +def canonical_hash(value: Any) -> str: + payload = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def main() -> int: + lock_path = Path(sys.argv[1] if len(sys.argv) > 1 else "schema/starintel-schema.lock.json") + lock = json.loads(lock_path.read_text(encoding="utf-8")) + repository = lock["canonical_repository"] + commit = lock["canonical_commit"] + base_url = f"https://raw.githubusercontent.com/{repository}/{commit}" + + schema = load_json(f"{base_url}/{lock['schema_path']}") + expansion = load_json(f"{base_url}/{lock['expansion_path']}") + manifest = load_json(f"{base_url}/{lock['manifest_path']}") + + if schema.get("$id") != "https://spec.starintel.actor/schema/starintel-doc-v0.9.0.json": + fail("unexpected canonical schema id") + if manifest.get("schema_version") != lock["schema_version"]: + fail("manifest schema version does not match lock") + if expansion.get("schema_version") != lock["schema_version"]: + fail("expansion schema version does not match lock") + + branches = schema.get("allOf", []) + research_branch = next( + ( + branch + for branch in branches + if branch.get("if", {}).get("properties", {}).get("dtype", {}).get("const") + == "research-node" + ), + None, + ) + if research_branch is None: + fail("canonical schema is missing dtype research-node") + + data_schema = research_branch.get("then", {}).get("properties", {}).get("data", {}) + if data_schema.get("additionalProperties") is not False: + fail("research-node data must reject undeclared fields") + + required = set(data_schema.get("required", [])) + expected_required = set(lock["research_node_required_fields"]) + missing_required = sorted(expected_required - required) + if missing_required: + fail(f"research-node is missing required fields: {missing_required}") + + expansion_fields = set(expansion.get("dtype_fields", {}).get("research-node", [])) + missing_expansion_fields = sorted(expected_required - expansion_fields) + if missing_expansion_fields: + fail(f"research-node expansion is missing fields: {missing_expansion_fields}") + + if manifest.get("dtype_count") != len(expansion.get("dtype_fields", {})): + fail("schema manifest dtype count does not match expansion") + if manifest.get("expansion_content_hash") != canonical_hash(expansion): + fail("schema manifest expansion hash does not match canonical expansion") + + print( + "verified StarIntel", + lock["schema_version"], + "research-node schema at", + commit, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/dedupe_relations.py b/scripts/dedupe_relations.py new file mode 100644 index 00000000..b03869b1 --- /dev/null +++ b/scripts/dedupe_relations.py @@ -0,0 +1,432 @@ +#!/usr/bin/env python3 +"""Deduplicate StarIntel relation documents in CouchDB. + +Dry-run is the default. Pass --apply to write canonical relation documents and +then tombstone legacy duplicates. The canonical identity is the directed tuple: +(dataset, source, predicate, target). +""" + +from __future__ import annotations + +import argparse +import base64 +import copy +import hashlib +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable, Iterator, Mapping, MutableMapping, Sequence + +DEFAULT_PREDICATE = "related-to" +DEFAULT_BATCH_SIZE = 200 + + +class MigrationError(RuntimeError): + pass + + +@dataclass(frozen=True, order=True) +class RelationIdentity: + dataset: str + source: str + predicate: str + target: str + + def encoded(self) -> str: + fields = (self.dataset, self.source, self.predicate, self.target) + return "|".join(f"{len(value.encode('utf-8'))}:{value}" for value in fields) + + def document_id(self) -> str: + digest = hashlib.sha256(self.encoded().encode("utf-8")).hexdigest() + return f"relation:{digest}" + + +@dataclass +class MigrationPlan: + relation_documents: int + invalid_documents: list[dict[str, Any]] + groups: dict[str, list[dict[str, Any]]] + upserts: list[dict[str, Any]] + deletions: list[dict[str, Any]] + duplicate_documents: int + rewritten_singletons: int + + +class CouchDB: + def __init__( + self, + base_url: str, + database: str, + user: str | None, + password: str | None, + timeout: float, + ) -> None: + self.base_url = base_url.rstrip("/") + self.database = database + self.timeout = timeout + self.headers = {"Accept": "application/json"} + if user is not None: + token = base64.b64encode(f"{user}:{password or ''}".encode()).decode() + self.headers["Authorization"] = f"Basic {token}" + + def _url(self, path: str, query: Mapping[str, str] | None = None) -> str: + database = urllib.parse.quote(self.database, safe="") + url = f"{self.base_url}/{database}/{path.lstrip('/')}" + if query: + url = f"{url}?{urllib.parse.urlencode(query)}" + return url + + def request( + self, + method: str, + path: str, + *, + query: Mapping[str, str] | None = None, + body: Any | None = None, + ) -> Any: + headers = dict(self.headers) + data = None + if body is not None: + data = json.dumps(body, separators=(",", ":")).encode("utf-8") + headers["Content-Type"] = "application/json" + request = urllib.request.Request( + self._url(path, query), data=data, headers=headers, method=method + ) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + payload = response.read() + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + raise MigrationError( + f"CouchDB {method} {request.full_url} failed: {exc.code} {detail}" + ) from exc + except urllib.error.URLError as exc: + raise MigrationError( + f"CouchDB {method} {request.full_url} failed: {exc.reason}" + ) from exc + return json.loads(payload) if payload else None + + def all_documents(self) -> list[dict[str, Any]]: + response = self.request( + "GET", + "_all_docs", + query={"include_docs": "true", "conflicts": "true"}, + ) + rows = response.get("rows", []) + return [row["doc"] for row in rows if isinstance(row.get("doc"), dict)] + + def bulk(self, documents: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + if not documents: + return [] + response = self.request("POST", "_bulk_docs", body={"docs": documents}) + if not isinstance(response, list): + raise MigrationError(f"Unexpected _bulk_docs response: {response!r}") + errors = [row for row in response if row.get("error")] + if errors: + raise MigrationError( + "CouchDB bulk operation failed: " + + json.dumps(errors, sort_keys=True) + ) + return response + + +def nested_data(document: Mapping[str, Any]) -> Mapping[str, Any]: + value = document.get("data") + return value if isinstance(value, Mapping) else {} + + +def relation_value(document: Mapping[str, Any], key: str) -> Any: + if key in document: + return document.get(key) + return nested_data(document).get(key) + + +def clean_string(value: Any) -> str: + return value.strip() if isinstance(value, str) else "" + + +def is_relation(document: Mapping[str, Any]) -> bool: + return clean_string(document.get("dtype")).lower() == "relation" + + +def relation_identity(document: Mapping[str, Any]) -> RelationIdentity | None: + if not is_relation(document): + return None + source = clean_string(relation_value(document, "source")) + target = clean_string(relation_value(document, "target")) + if not source or not target: + return None + predicate = clean_string(relation_value(document, "predicate")) or DEFAULT_PREDICATE + dataset = clean_string(document.get("dataset")) + return RelationIdentity(dataset, source, predicate, target) + + +def unique_strings(values: Iterable[Any]) -> list[str]: + seen: set[str] = set() + result: list[str] = [] + for value in values: + if not isinstance(value, str): + continue + clean = value.strip() + if clean and clean not in seen: + seen.add(clean) + result.append(clean) + return result + + +def relation_notes(document: Mapping[str, Any]) -> Iterator[str]: + note = relation_value(document, "note") + if isinstance(note, str): + yield note + notes = relation_value(document, "notes") + if isinstance(notes, list): + yield from (value for value in notes if isinstance(value, str)) + + +def document_sort_key(document: Mapping[str, Any]) -> tuple[str, str]: + date_added = document.get("dateAdded", document.get("date_added", "")) + return str(date_added), str(document.get("_id", "")) + + +def set_relation_value(document: MutableMapping[str, Any], key: str, value: Any) -> None: + data = document.get("data") + if isinstance(data, MutableMapping) and key not in document: + data[key] = value + else: + document[key] = value + + +def set_dedupe_metadata( + document: MutableMapping[str, Any], + identity: RelationIdentity, + old_ids: list[str], + evidence_count: int, +) -> None: + if isinstance(document.get("data"), Mapping): + extensions = document.setdefault("extensions", {}) + if not isinstance(extensions, MutableMapping): + extensions = {} + document["extensions"] = extensions + star_server = extensions.setdefault("star_server", {}) + if not isinstance(star_server, MutableMapping): + star_server = {} + extensions["star_server"] = star_server + star_server["relation_identity"] = identity.encoded() + star_server["deduplicated_from"] = old_ids + star_server["evidence_count"] = evidence_count + else: + document["relationIdentity"] = identity.encoded() + document["deduplicatedFrom"] = old_ids + document["evidenceCount"] = evidence_count + + +def merge_relation_group( + identity: RelationIdentity, documents: Sequence[dict[str, Any]] +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + canonical_id = identity.document_id() + canonical_existing = next( + (document for document in documents if document.get("_id") == canonical_id), + None, + ) + base = canonical_existing or min(documents, key=document_sort_key) + merged = copy.deepcopy(base) + merged["_id"] = canonical_id + if canonical_existing is None: + merged.pop("_rev", None) + else: + merged["_rev"] = canonical_existing["_rev"] + + set_relation_value(merged, "source", identity.source) + set_relation_value(merged, "target", identity.target) + set_relation_value(merged, "predicate", identity.predicate) + merged["dataset"] = identity.dataset + + sources = unique_strings( + source + for document in documents + for source in ( + document.get("sources", []) + if isinstance(document.get("sources"), list) + else [] + ) + ) + if sources: + merged["sources"] = sources + + notes = unique_strings( + note for document in documents for note in relation_notes(document) + ) + if notes: + set_relation_value(merged, "note", "\n\n".join(notes)) + + old_ids = sorted( + str(document["_id"]) + for document in documents + if document.get("_id") != canonical_id + ) + set_dedupe_metadata(merged, identity, old_ids, len(documents)) + + deletions = [ + {"_id": document["_id"], "_rev": document["_rev"], "_deleted": True} + for document in documents + if document.get("_id") != canonical_id + ] + return merged, deletions + + +def build_plan( + documents: Sequence[dict[str, Any]], *, rewrite_singletons: bool = True +) -> MigrationPlan: + groups: dict[str, list[dict[str, Any]]] = defaultdict(list) + invalid: list[dict[str, Any]] = [] + relation_documents = 0 + + for document in documents: + if not is_relation(document): + continue + relation_documents += 1 + identity = relation_identity(document) + if identity is None: + invalid.append(document) + continue + groups[identity.document_id()].append(document) + + upserts: list[dict[str, Any]] = [] + deletions: list[dict[str, Any]] = [] + duplicate_documents = 0 + rewritten_singletons = 0 + + for canonical_id in sorted(groups): + group = groups[canonical_id] + duplicate_documents += max(0, len(group) - 1) + is_noncanonical_singleton = ( + len(group) == 1 and group[0].get("_id") != canonical_id + ) + if len(group) == 1 and not (rewrite_singletons and is_noncanonical_singleton): + continue + identity = relation_identity(group[0]) + assert identity is not None + merged, group_deletions = merge_relation_group(identity, group) + upserts.append(merged) + deletions.extend(group_deletions) + if is_noncanonical_singleton: + rewritten_singletons += 1 + + return MigrationPlan( + relation_documents=relation_documents, + invalid_documents=invalid, + groups=dict(groups), + upserts=upserts, + deletions=deletions, + duplicate_documents=duplicate_documents, + rewritten_singletons=rewritten_singletons, + ) + + +def batches(values: Sequence[dict[str, Any]], size: int) -> Iterator[list[dict[str, Any]]]: + for start in range(0, len(values), size): + yield list(values[start : start + size]) + + +def write_backup(path: Path, documents: Sequence[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as stream: + for document in documents: + if is_relation(document): + stream.write(json.dumps(document, sort_keys=True)) + stream.write("\n") + + +def apply_plan(database: CouchDB, plan: MigrationPlan, batch_size: int) -> None: + for batch in batches(plan.upserts, batch_size): + database.bulk(batch) + for batch in batches(plan.deletions, batch_size): + database.bulk(batch) + + +def read_password(args: argparse.Namespace) -> str | None: + if args.password is not None: + return args.password + if args.password_file: + return Path(args.password_file).read_text(encoding="utf-8").strip() + return None + + +def parser() -> argparse.ArgumentParser: + default_url = ( + f"{os.getenv('COUCHDB_SCHEME', 'http')}://" + f"{os.getenv('COUCHDB_HOST', '127.0.0.1')}:" + f"{os.getenv('COUCHDB_PORT', '5984')}" + ) + result = argparse.ArgumentParser(description=__doc__) + result.add_argument("--url", default=default_url) + result.add_argument("--database", default=os.getenv("COUCHDB_DATABASE", "starintel")) + result.add_argument("--user", default=os.getenv("COUCHDB_USER", "admin")) + result.add_argument("--password", default=os.getenv("COUCHDB_PASSWORD")) + result.add_argument("--password-file", default=os.getenv("COUCHDB_PASSWORD_FILE")) + result.add_argument("--timeout", type=float, default=30.0) + result.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE) + result.add_argument("--duplicates-only", action="store_true") + result.add_argument("--apply", action="store_true") + result.add_argument("--no-backup", action="store_true") + result.add_argument("--backup") + return result + + +def main(argv: Sequence[str] | None = None) -> int: + args = parser().parse_args(argv) + if args.batch_size < 1: + raise MigrationError("--batch-size must be positive") + + database = CouchDB( + args.url, + args.database, + args.user, + read_password(args), + args.timeout, + ) + documents = database.all_documents() + plan = build_plan(documents, rewrite_singletons=not args.duplicates_only) + + summary = { + "database": args.database, + "relation_documents": plan.relation_documents, + "identity_groups": len(plan.groups), + "duplicate_documents": plan.duplicate_documents, + "canonical_upserts": len(plan.upserts), + "legacy_deletions": len(plan.deletions), + "rewritten_singletons": plan.rewritten_singletons, + "invalid_relations": len(plan.invalid_documents), + "mode": "apply" if args.apply else "dry-run", + } + print(json.dumps(summary, indent=2, sort_keys=True)) + + if not args.apply: + return 0 + + if not args.no_backup: + backup_path = Path( + args.backup + or f"relation-dedupe-backup-{time.strftime('%Y%m%d-%H%M%S')}.jsonl" + ) + write_backup(backup_path, documents) + print(f"backup={backup_path}") + + apply_plan(database, plan, args.batch_size) + print("relation deduplication applied") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except MigrationError as exc: + print(f"error: {exc}", file=sys.stderr) + raise SystemExit(1) from exc diff --git a/scripts/stack-test.sh b/scripts/stack-test.sh index 96607a85..fb185560 100755 --- a/scripts/stack-test.sh +++ b/scripts/stack-test.sh @@ -3,8 +3,12 @@ set -Eeuo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" credentials_dir="$(mktemp -d)" -project_name="starintel-issue55-$$" +artifact_dir="${repo_root}/stack-test-artifacts" +project_name="starintel-auth-stack-$$" port_base=$((20000 + $$ % 20000)) +stage="initialize" +failure_line="" +failure_status="0" export CREDENTIALS_DIR="$credentials_dir" export COMPOSE_PROJECT_NAME="$project_name" @@ -13,14 +17,50 @@ export COUCHDB_PORT="$((port_base + 1))" export RABBITMQ_PORT="$((port_base + 2))" export RABBITMQ_MANAGEMENT_PORT="$((port_base + 3))" -couchdb_password="issue55-couchdb-$$" -rabbitmq_password="issue55-rabbitmq-$$" +mkdir -p "$artifact_dir" +rm -f "$artifact_dir"/* + +set_stage() { + stage="$1" + printf 'stage=%s\nstatus=running\n' "$stage" > "$artifact_dir/status.txt" + printf '==> %s\n' "$stage" +} + +record_error() { + failure_status="$?" + failure_line="$1" + return "$failure_status" +} +trap 'record_error "$LINENO"' ERR + +capture_diagnostics() { + { + printf 'stage=%s\n' "$stage" + printf 'status=failed\n' + printf 'exit_status=%s\n' "$failure_status" + printf 'line=%s\n' "$failure_line" + } > "$artifact_dir/status.txt" + + docker compose --project-directory "$repo_root" ps --all \ + > "$artifact_dir/compose-ps.txt" 2>&1 || true + docker compose --project-directory "$repo_root" logs --no-color star-server \ + > "$artifact_dir/star-server.log" 2>&1 || true + docker compose --project-directory "$repo_root" logs --no-color couchdb \ + > "$artifact_dir/couchdb.log" 2>&1 || true +} cleanup() { status=$? if ((status != 0)); then - docker compose --project-directory "$repo_root" ps || true - docker compose --project-directory "$repo_root" logs --no-color || true + if ((failure_status == 0)); then + failure_status="$status" + fi + capture_diagnostics + cat "$artifact_dir/status.txt" >&2 || true + cat "$artifact_dir/compose-ps.txt" >&2 || true + tail -n 200 "$artifact_dir/star-server.log" >&2 || true + else + printf 'stage=complete\nstatus=passed\n' > "$artifact_dir/status.txt" fi docker compose --project-directory "$repo_root" down \ --volumes --remove-orphans >/dev/null 2>&1 || true @@ -29,23 +69,77 @@ cleanup() { } trap cleanup EXIT +couchdb_password="stack-couchdb-$$" +rabbitmq_password="stack-rabbitmq-$$" +auth_pepper="stack-auth-pepper-$$-$(date +%s%N)" +auth_bootstrap_secret="stack-bootstrap-$$-$(date +%s%N)" + +set_stage "write-secrets" printf '%s\n' "$couchdb_password" > "$credentials_dir/couchdb_password" -printf '%s\n' "issue55-couchdb-secret-$$" > "$credentials_dir/couchdb_secret" -printf '%s\n' "ISSUE55ERLANGCOOKIE$$" > "$credentials_dir/erlang_cookie" +printf '%s\n' "stack-couchdb-secret-$$" > "$credentials_dir/couchdb_secret" +printf '%s\n' "STACKERLANGCOOKIE$$" > "$credentials_dir/erlang_cookie" printf '%s\n' "$rabbitmq_password" > "$credentials_dir/rabbitmq_password" +printf '%s\n' "$auth_pepper" > "$credentials_dir/auth_pepper" +printf '%s\n' "$auth_bootstrap_secret" > "$credentials_dir/auth_bootstrap_secret" chmod 0600 "$credentials_dir"/* cd "$repo_root" +set_stage "load-images" nix run .#load-images + +set_stage "validate-compose" docker compose config --quiet + +set_stage "start-stack" docker compose up --detach --wait --wait-timeout 300 -fixture_id="issue-55-fixture" -fixture_term="issue55searchfixture" +fixture_id="auth-stack-fixture" +fixture_term="authstacksearchfixture" couchdb_url="http://127.0.0.1:${COUCHDB_PORT}" server_url="http://127.0.0.1:${STAR_SERVER_PORT}" +set_stage "bootstrap-administrator" +bootstrap_response="$( + curl --fail --silent --show-error \ + --request POST \ + --header "Content-Type: application/json" \ + --header "X-Star-Bootstrap-Secret: ${auth_bootstrap_secret}" \ + --data '{"owner":"stack-administrator"}' \ + "${server_url}/auth/bootstrap" +)" +api_key="$(jq --exit-status --raw-output '.api_key' <<<"$bootstrap_response")" +[[ "$api_key" == star_sk_v1_* ]] +auth_header="Authorization: Bearer ${api_key}" + +set_stage "verify-unauthenticated-denial" +unauthenticated_status="$( + curl --silent --output /dev/null --write-out '%{http_code}' \ + --get --data-urlencode "q=content:${fixture_term}" \ + "${server_url}/search" +)" +[[ "$unauthenticated_status" == "401" ]] + +set_stage "verify-one-time-bootstrap" +second_bootstrap_status="$( + curl --silent --output /dev/null --write-out '%{http_code}' \ + --request POST \ + --header "Content-Type: application/json" \ + --header "X-Star-Bootstrap-Secret: ${auth_bootstrap_secret}" \ + --data '{"owner":"second-administrator"}' \ + "${server_url}/auth/bootstrap" +)" +[[ "$second_bootstrap_status" == "409" ]] + +set_stage "verify-authenticated-context" +curl --fail --silent --show-error \ + --header "$auth_header" \ + "${server_url}/auth/context" | + jq --exit-status \ + '.principal_id == "stack-administrator" and .principal_type == "administrator"' \ + >/dev/null + +set_stage "insert-search-fixture" curl --fail --silent --show-error \ --user "${COUCHDB_USER:-admin}:${couchdb_password}" \ --request PUT \ @@ -58,6 +152,7 @@ wait_for_search() { for _ in $(seq 1 60); do response="$( curl --fail --silent --show-error \ + --header "$auth_header" \ --get --data-urlencode "q=content:${fixture_term}" \ "${server_url}/search" || true )" @@ -67,7 +162,7 @@ wait_for_search() { fi sleep 2 done - printf 'fixture did not appear in full-text search\n' >&2 + printf 'fixture did not appear in authenticated full-text search\n' >&2 return 1 } @@ -100,16 +195,28 @@ wait_for_healthy_stack() { return 1 } +set_stage "verify-authenticated-search" wait_for_search +set_stage "restart-stack" docker compose restart + +set_stage "wait-after-restart" wait_for_healthy_stack +set_stage "verify-document-persistence" curl --fail --silent --show-error \ --user "${COUCHDB_USER:-admin}:${couchdb_password}" \ "${couchdb_url}/${COUCHDB_DATABASE:-starintel}/${fixture_id}" | jq --exit-status --arg term "$fixture_term" '.content == $term' >/dev/null +set_stage "verify-credential-persistence" +curl --fail --silent --show-error \ + --header "$auth_header" \ + "${server_url}/auth/context" | + jq --exit-status '.principal_id == "stack-administrator"' >/dev/null + +set_stage "verify-search-after-restart" wait_for_search -printf 'Nix-built Compose stack passed health, FTS, and restart persistence checks.\n' +printf 'Authenticated Nix-built stack passed bootstrap, denial, FTS, and restart persistence checks.\n' diff --git a/scripts/test_dedupe_relations.py b/scripts/test_dedupe_relations.py new file mode 100644 index 00000000..cde35156 --- /dev/null +++ b/scripts/test_dedupe_relations.py @@ -0,0 +1,108 @@ +import importlib.util +import pathlib +import sys +import unittest + +SCRIPT = pathlib.Path(__file__).with_name("dedupe_relations.py") +SPEC = importlib.util.spec_from_file_location("dedupe_relations", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = MODULE +assert SPEC.loader is not None +SPEC.loader.exec_module(MODULE) + + +class RelationDeduplicationTests(unittest.TestCase): + def relation(self, document_id, *, sources=None, note=""): + return { + "_id": document_id, + "_rev": f"1-{document_id}", + "dataset": "hunter-biden", + "dtype": "relation", + "source": "person:robert-hunter-biden", + "target": "email:hunter@example.test", + "predicate": "related", + "sources": sources or [], + "note": note, + } + + def test_directed_identity_is_stable(self): + document = self.relation("legacy") + identity = MODULE.relation_identity(document) + self.assertEqual( + identity.document_id(), + MODULE.relation_identity(document).document_id(), + ) + reversed_document = dict( + document, + source=document["target"], + target=document["source"], + ) + self.assertNotEqual( + identity.document_id(), + MODULE.relation_identity(reversed_document).document_id(), + ) + + def test_duplicate_group_merges_evidence_and_deletes_legacy_ids(self): + first = self.relation("01A", sources=["actor-a"], note="first") + second = self.relation( + "01B", + sources=["actor-b", "actor-a"], + note="second", + ) + plan = MODULE.build_plan([first, second]) + self.assertEqual(plan.duplicate_documents, 1) + self.assertEqual(len(plan.upserts), 1) + self.assertEqual(len(plan.deletions), 2) + merged = plan.upserts[0] + self.assertEqual(merged["sources"], ["actor-a", "actor-b"]) + self.assertEqual(merged["note"], "first\n\nsecond") + self.assertEqual(merged["evidenceCount"], 2) + + def test_existing_canonical_document_is_updated_not_deleted(self): + legacy = self.relation("01A") + identity = MODULE.relation_identity(legacy) + canonical = self.relation( + identity.document_id(), + sources=["canonical"], + ) + plan = MODULE.build_plan([legacy, canonical]) + self.assertEqual(plan.upserts[0]["_id"], identity.document_id()) + self.assertEqual(plan.upserts[0]["_rev"], canonical["_rev"]) + self.assertEqual([row["_id"] for row in plan.deletions], ["01A"]) + + def test_nested_09_relation_is_supported(self): + document = { + "_id": "01A", + "_rev": "1-a", + "schema_version": "0.9.0", + "dataset": "d", + "dtype": "relation", + "sources": ["manual"], + "data": { + "source": "a", + "target": "b", + "predicate": "owns", + }, + } + plan = MODULE.build_plan([document]) + merged = plan.upserts[0] + self.assertEqual(merged["data"]["source"], "a") + self.assertIn( + "relation_identity", + merged["extensions"]["star_server"], + ) + + def test_invalid_relation_is_reported_not_deleted(self): + invalid = { + "_id": "bad", + "_rev": "1-bad", + "dtype": "relation", + } + plan = MODULE.build_plan([invalid]) + self.assertEqual(plan.invalid_documents, [invalid]) + self.assertEqual(plan.upserts, []) + self.assertEqual(plan.deletions, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/source/actor-systems/event-actor.lisp b/source/actor-systems/event-actor.lisp index 52f84a4d..608fcc12 100644 --- a/source/actor-systems/event-actor.lisp +++ b/source/actor-systems/event-actor.lisp @@ -1,71 +1,271 @@ -(in-package #:star.actors) - (in-package :star.actors) -(defclass actor-event () - ((_id :initarg :id :accessor event-id :initform (cms-ulid:ulid)) - (timestamp :initarg :timestamp :accessor event-timestamp :initform (spec:unix-now)) - (dtype :initarg :timestamp :accessor doc-type :initform "actorevent") - (actor-name :initarg :actor-name :accessor event-actor-name :initform "") - (event-type :initarg :event-type :accessor event-type :initform "") - (details :initarg :details :accessor event-details :initform "") - (source-id :initarg :source-id :accessor event-source-document :initform ""))) +(defparameter +event-exchange+ "events") +(defparameter +event-queue+ "events") +(defparameter +event-routing-key+ "event.#") +(defparameter +event-dead-letter-exchange+ "events.dead-letter") +(defparameter +event-dead-letter-queue+ "events.quarantine") +(defparameter +event-dead-letter-routing-key+ "events.invalid") +(defparameter +event-persistence-timeout-seconds+ 10) +(define-condition invalid-actor-event (error) + ((reason + :initarg :reason + :reader invalid-actor-event-reason) + (payload + :initarg :payload + :reader invalid-actor-event-payload)) + (:report + (lambda (condition stream) + (format stream + "Invalid actor event: ~a" + (invalid-actor-event-reason condition))))) +(defclass actor-event () + ((_id + :initarg :id + :accessor event-id + :initform (cms-ulid:ulid) + :type string) + (timestamp + :initarg :timestamp + :accessor event-timestamp + :initform (spec:unix-now) + :type integer) + (dtype + :initarg :dtype + :accessor doc-type + :initform "actorevent" + :type string) + (actor-name + :initarg :actor-name + :accessor event-actor-name + :initform "" + :type string) + (component + :initarg :component + :accessor event-component + :initform "" + :type string) + (event-type + :initarg :event-type + :accessor event-type + :initform "" + :type string) + (details + :initarg :details + :accessor event-details + :initform "" + :type string) + (source-id + :initarg :source-id + :accessor event-source-document + :initform "" + :type string) + (trace-id + :initarg :trace-id + :accessor event-trace-id + :initform "" + :type string) + (generation + :initarg :generation + :accessor event-generation + :initform 0 + :type integer))) -(defun make-actor-event (&key actor-name event-type details source-id) - (make-instance 'actor-event - :actor-name actor-name - :event-type event-type - :details details - :source-id source-id)) +(defun make-actor-event (&key actor-name component event-type details source-id + trace-id (generation 0) timestamp dtype id) + (make-instance + 'actor-event + :id (or id (cms-ulid:ulid)) + :timestamp (or timestamp (spec:unix-now)) + :dtype (or dtype "actorevent") + :actor-name (or actor-name component "") + :component (or component actor-name "") + :event-type (or event-type "") + :details (or details "") + :source-id (or source-id "") + :trace-id (or trace-id "") + :generation generation)) +(defun non-empty-string-p (value) + (and (stringp value) (plusp (length value)))) +(defun validate-actor-event (event &optional payload) + (flet ((invalid (reason) + (error 'invalid-actor-event + :reason reason + :payload payload))) + (unless (non-empty-string-p (event-id event)) + (invalid "_id must be a non-empty string")) + (unless (and (integerp (event-timestamp event)) + (plusp (event-timestamp event))) + (invalid "timestamp must be a positive integer")) + (unless (string= "actorevent" (doc-type event)) + (invalid "dtype must be actorevent")) + (unless (or (non-empty-string-p (event-actor-name event)) + (non-empty-string-p (event-component event))) + (invalid "actorName or component is required")) + (unless (non-empty-string-p (event-type event)) + (invalid "eventType is required")) + (unless (and (integerp (event-generation event)) + (not (minusp (event-generation event)))) + (invalid "generation must be a non-negative integer")) + event)) +(defun actor-event-json-object (payload) + (let ((json + (etypecase payload + (string (jsown:parse payload)) + (list payload)))) + ;; Migration defaults for pre-contract event documents. + (unless (jsown:val-safe json "dtype") + (setf (jsown:val json "dtype") "actorevent")) + (unless (jsown:val-safe json "generation") + (setf (jsown:val json "generation") 0)) + (let ((actor-name (jsown:val-safe json "actorName")) + (component (jsown:val-safe json "component"))) + (when (and actor-name (not component)) + (setf (jsown:val json "component") actor-name)) + (when (and component (not actor-name)) + (setf (jsown:val json "actorName") component))) + json)) +(defun decode-actor-event (payload) + "Decode and validate an event through the StarIntel object codec." + (handler-case + (let* ((json (actor-event-json-object payload)) + (event + (star.databases.couchdb:from-json json 'actor-event))) + (validate-actor-event event payload)) + (invalid-actor-event (condition) + (error condition)) + (error (condition) + (error 'invalid-actor-event + :reason (princ-to-string condition) + :payload payload)))) -(define-actor (*actor-event-receiver* *sys*) - (lambda (event) - (let ((event-json (jsown:to-json (as-json event)))) - (tell *couchdb-inserts* (list :id (event-id event) :database star:*couchdb-event-log-database* :document event-json))))) +(defun encode-actor-event (event) + (jsown:to-json + (star.databases.couchdb:as-json + (validate-actor-event event)))) +(defun actor-event-insert-request (event) + (make-couchdb-insert-request + :database star:*couchdb-event-log-database* + :document-id (event-id event) + :document (encode-actor-event event))) +(defun persist-actor-event (event &optional (insert-actor *couchdb-inserts*)) + (sento.actor:ask-s + insert-actor + (actor-event-insert-request event) + :time-out +event-persistence-timeout-seconds+)) +(defun actor-event-settlement (event persistence-result) + (cond + ((not (typep persistence-result 'couchdb-result)) + (star.consumers:rabbit-nack + :reason :persistence-protocol-error + :requeue t + :value event + :error persistence-result)) + ((member (couchdb-result-status persistence-result) + '(:success :exists :conflict)) + (star.consumers:rabbit-ack + :reason + (if (eq :success (couchdb-result-status persistence-result)) + :persisted + :duplicate) + :value event)) + (t + (star.consumers:rabbit-nack + :reason :persistence-failed + :requeue t + :value event + :error (couchdb-result-error-message persistence-result))))) +(defun process-event-delivery (payload &key (persist-fn #'persist-actor-event)) + "Decode, validate, persist idempotently, and return a settlement decision." + (handler-case + (let ((event (decode-actor-event payload))) + (actor-event-settlement event (funcall persist-fn event))) + (invalid-actor-event (condition) + (star.consumers:rabbit-nack + :reason :invalid-event + :requeue nil + :error (invalid-actor-event-reason condition))) + (error (condition) + (star.consumers:rabbit-nack + :reason :event-handler-error + :requeue t + :error condition)))) -(defun handle-event-message (self message) - "Handler function for processing event messages." - (let* ((jdoc (jsown:parse (car message)))) - (log:trace "Got Event: ~a" (spec:decode jdoc 'actor-event)) - (tell *actor-event-receiver* - (make-instance 'actor-event - :id (jsown:val jdoc "_id") - :timestamp (jsown:val jdoc "timestamp") - :actor-name (jsown:val jdoc "actorName") - :event-type (jsown:val jdoc "eventType") - :details (jsown:val jdoc "details") - :source-id (jsown:val jdoc "sourceId"))))) +(define-actor (*actor-event-receiver* *sys*) + (lambda (event) + (tell *couchdb-inserts* (actor-event-insert-request event)))) +(defun handle-event-message (consumer message) + (declare (ignore consumer)) + (process-event-delivery (car message))) (defun start-event-consumer (n) - "Initialize and set up the event consumer." - (let ((consumer (star.consumers:create-rabbit-consumer - :name "event-consumers" - :n n - :host star:*rabbit-address* - :port star:*rabbit-port* - :username star:*rabbit-user* - :password star:*rabbit-password* - :queue-name "events" - :exchange-name "events" - :routing-key "event.#" - :test-fn #'star.rabbit::insertp - :handler-fn #'handle-event-message))) + "Start the durable event consumer with retry and dead-letter policy. + +Valid and duplicate events are ACKed. Invalid events are NACKed without requeue +and routed to the quarantine queue. Transient persistence failures are NACKed +with requeue so RabbitMQ can retry them." + (let ((consumer + (star.consumers:create-rabbit-consumer + :name "event-consumers" + :n n + :host star:*rabbit-address* + :port star:*rabbit-port* + :username star:*rabbit-user* + :password star:*rabbit-password* + :queue-name +event-queue+ + :queue-durable t + :exchange-name +event-exchange+ + :exchange-type "topic" + :exchange-durable t + :routing-key +event-routing-key+ + :dead-letter-exchange +event-dead-letter-exchange+ + :dead-letter-routing-key +event-dead-letter-routing-key+ + :dead-letter-queue +event-dead-letter-queue+ + :test-fn #'identity + :handler-fn #'handle-event-message))) (star.consumers:start-consumer consumer))) -(defun log-actor-event (actor-name &key event-type details source-id) - (log:debug "Told *actor-event-reciver*") - (tell *actor-event-receiver* (make-actor-event :actor-name actor-name :event-type event-type :details details :source-id source-id))) +(defun log-actor-event (actor-name &key event-type details source-id trace-id + component (generation 0)) + (tell + *actor-event-receiver* + (make-actor-event + :actor-name actor-name + :component component + :event-type event-type + :details details + :source-id source-id + :trace-id trace-id + :generation generation))) + +(nhooks:add-hook + star:*actors-start-hook* + (lambda () + (star.actors:register-actor + "actor-event-receiver" + *actor-event-receiver*))) -(nhooks:add-hook star:*actors-start-hook* - (lambda () (star.actors:register-actor "actor-event-receiver" *actor-event-receiver*))) +(eval-when (:compile-toplevel :load-toplevel :execute) + (export '(actor-event + invalid-actor-event + invalid-actor-event-reason + make-actor-event + validate-actor-event + decode-actor-event + encode-actor-event + process-event-delivery + event-component + event-trace-id + event-generation) + :star.actors)) diff --git a/source/actors/couchdb-service.lisp b/source/actors/couchdb-service.lisp new file mode 100644 index 00000000..4ea0781f --- /dev/null +++ b/source/actors/couchdb-service.lisp @@ -0,0 +1,349 @@ +(in-package :star.actors) + +(defparameter +couchdb-storage-dispatcher+ :pinned + "Dispatcher used for blocking CouchDB actor work.") + +(defstruct (couchdb-get-request + (:constructor make-couchdb-get-request + (&key + (database star:*couchdb-default-database*) + document-id + revision))) + database + document-id + revision) + +(defstruct (couchdb-insert-request + (:constructor make-couchdb-insert-request + (&key + (database star:*couchdb-default-database*) + document-id + document))) + database + document-id + document) + +(defstruct (couchdb-delete-request + (:constructor make-couchdb-delete-request + (&key + (database star:*couchdb-default-database*) + document-id + revision))) + database + document-id + revision) + +(defstruct (couchdb-result + (:constructor make-couchdb-result + (&key status operation database document-id revision value + error-type error-message))) + status + operation + database + document-id + revision + value + error-type + error-message) + +(defun make-couchdb-agent (context pool + &key error-fun + (dispatcher-id +couchdb-storage-dispatcher+)) + "Wrap the injected CouchDB POOL in an agent on the storage dispatcher." + (declare (ignore error-fun)) + (unless pool + (error "MAKE-COUCHDB-AGENT requires an injected CouchDB pool.")) + (make-agent (lambda () pool) context dispatcher-id)) + +(defun start-couchdb-agent (system + &optional + (pool star.databases.couchdb:*couchdb-pool*)) + "Start the CouchDB pool agent without creating an unused standalone client." + (setf *couchdb-agent* (make-couchdb-agent system pool))) + +(defun couchdb-agent-get (agent database document-id &optional revision) + (anypool:with-connection (client (couchdb-agent-client agent)) + (if revision + (cl-couch:get-document client database document-id revision) + (cl-couch:get-document client database document-id)))) + +(defun parse-couchdb-document (document) + (etypecase document + (string (jsown:parse document)) + (list document))) + +(defun delete-couchdb-document (client database document-id + &optional revision + &key + (get-fn #'cl-couch:get-document) + (delete-fn #'cl-couch:delete-document)) + "Delete DOCUMENT-ID using REVISION, fetching the current revision when absent." + (let* ((resolved-revision + (or revision + (let* ((document (funcall get-fn client database document-id)) + (parsed (parse-couchdb-document document))) + (jsown:val-safe parsed "_rev"))))) + (unless resolved-revision + (error "CouchDB document ~a/~a has no revision." + database document-id)) + (values (funcall delete-fn + client database document-id resolved-revision) + resolved-revision))) + +(defun couchdb-agent-delete (agent database document-id &optional revision) + (anypool:with-connection (client (couchdb-agent-client agent)) + (delete-couchdb-document client database document-id revision))) + +(defun normalize-couchdb-get-request (message) + (typecase message + (couchdb-get-request message) + (string + (make-couchdb-get-request :document-id message)) + (list + (make-couchdb-get-request + :database (or (getf message :database) + star:*couchdb-default-database*) + :document-id (or (getf message :document-id) + (getf message :id)) + :revision (or (getf message :revision) + (getf message :rev)))) + (t + (error "Unsupported CouchDB GET request: ~s" message)))) + +(defun normalize-couchdb-insert-request (message) + (typecase message + (couchdb-insert-request message) + (list + (make-couchdb-insert-request + :database (or (getf message :database) + star:*couchdb-default-database*) + :document-id (or (getf message :document-id) + (getf message :id)) + :document (getf message :document))) + (t + (error "Unsupported CouchDB INSERT request: ~s" message)))) + +(defun normalize-couchdb-delete-request (message) + (typecase message + (couchdb-delete-request message) + (list + (make-couchdb-delete-request + :database (or (getf message :database) + star:*couchdb-default-database*) + :document-id (or (getf message :document-id) + (getf message :id)) + :revision (or (getf message :revision) + (getf message :rev)))) + (t + (error "Unsupported CouchDB DELETE request: ~s" message)))) + +(defun ensure-couchdb-request-id (operation document-id) + (unless (and (stringp document-id) (plusp (length document-id))) + (error "CouchDB ~a request requires a non-empty document id." + operation))) + +(defun couchdb-error-result (operation database document-id condition) + (make-couchdb-result + :status :error + :operation operation + :database database + :document-id document-id + :error-type (string-downcase (princ-to-string (type-of condition))) + :error-message (princ-to-string condition))) + +(defun complete-couchdb-request (result) + "Return RESULT for ASK-S and explicitly reply only to a real async sender." + (when *sender* + (reply result *sender*)) + result) + +(defun make-couchdb-get-handler (agent &key (get-fn #'couchdb-agent-get)) + (lambda (message) + (let ((request nil)) + (complete-couchdb-request + (handler-case + (progn + (setf request (normalize-couchdb-get-request message)) + (ensure-couchdb-request-id + :get + (couchdb-get-request-document-id request)) + (make-couchdb-result + :status :success + :operation :get + :database (couchdb-get-request-database request) + :document-id (couchdb-get-request-document-id request) + :revision (couchdb-get-request-revision request) + :value (funcall get-fn + agent + (couchdb-get-request-database request) + (couchdb-get-request-document-id request) + (couchdb-get-request-revision request)))) + (dexador:http-request-not-found () + (make-couchdb-result + :status :not-found + :operation :get + :database (and request (couchdb-get-request-database request)) + :document-id (and request + (couchdb-get-request-document-id request)))) + (error (condition) + (couchdb-error-result + :get + (and request (couchdb-get-request-database request)) + (and request (couchdb-get-request-document-id request)) + condition))))))) + +(defun make-couchdb-insert-handler + (agent + &key + (exists-fn #'couchdb-document-exists-p) + (insert-fn #'couchdb-agent-insert)) + (lambda (message) + (let ((request nil)) + (complete-couchdb-request + (handler-case + (progn + (setf request (normalize-couchdb-insert-request message)) + (ensure-couchdb-request-id + :insert + (couchdb-insert-request-document-id request)) + (unless (couchdb-insert-request-document request) + (error "CouchDB INSERT request requires a document.")) + (if (funcall exists-fn + agent + (couchdb-insert-request-database request) + (couchdb-insert-request-document-id request)) + (make-couchdb-result + :status :exists + :operation :insert + :database (couchdb-insert-request-database request) + :document-id (couchdb-insert-request-document-id request)) + (make-couchdb-result + :status :success + :operation :insert + :database (couchdb-insert-request-database request) + :document-id (couchdb-insert-request-document-id request) + :value (funcall insert-fn + agent + (couchdb-insert-request-database request) + (couchdb-insert-request-document request))))) + (dexador:http-request-conflict () + (make-couchdb-result + :status :conflict + :operation :insert + :database (and request (couchdb-insert-request-database request)) + :document-id (and request + (couchdb-insert-request-document-id request)))) + (error (condition) + (couchdb-error-result + :insert + (and request (couchdb-insert-request-database request)) + (and request (couchdb-insert-request-document-id request)) + condition))))))) + +(defun make-couchdb-delete-handler (agent &key (delete-fn #'couchdb-agent-delete)) + (lambda (message) + (let ((request nil)) + (complete-couchdb-request + (handler-case + (progn + (setf request (normalize-couchdb-delete-request message)) + (ensure-couchdb-request-id + :delete + (couchdb-delete-request-document-id request)) + (multiple-value-bind (value revision) + (funcall delete-fn + agent + (couchdb-delete-request-database request) + (couchdb-delete-request-document-id request) + (couchdb-delete-request-revision request)) + (make-couchdb-result + :status :success + :operation :delete + :database (couchdb-delete-request-database request) + :document-id (couchdb-delete-request-document-id request) + :revision revision + :value value))) + (dexador:http-request-not-found () + (make-couchdb-result + :status :not-found + :operation :delete + :database (and request (couchdb-delete-request-database request)) + :document-id (and request + (couchdb-delete-request-document-id request)))) + (dexador:http-request-conflict () + (make-couchdb-result + :status :conflict + :operation :delete + :database (and request (couchdb-delete-request-database request)) + :document-id (and request + (couchdb-delete-request-document-id request)) + :revision (and request + (couchdb-delete-request-revision request)))) + (error (condition) + (couchdb-error-result + :delete + (and request (couchdb-delete-request-database request)) + (and request (couchdb-delete-request-document-id request)) + condition))))))) + +(defun start-couchdb-gets (system) + (setf *couchdb-gets* + (actor-of system + :name "*couchdb-gets*" + :dispatcher +couchdb-storage-dispatcher+ + :receive (make-couchdb-get-handler *couchdb-agent*)))) + +(defun start-couchdb-inserts (system) + (setf *couchdb-inserts* + (actor-of system + :name "*couchdb-inserts*" + :dispatcher +couchdb-storage-dispatcher+ + :receive (make-couchdb-insert-handler *couchdb-agent*)))) + +(defvar *couchdb-deletes* nil + "Actor responsible for deterministic CouchDB delete requests.") + +(defun start-couchdb-deletes (system) + (setf *couchdb-deletes* + (actor-of system + :name "*couchdb-deletes*" + :dispatcher +couchdb-storage-dispatcher+ + :receive (make-couchdb-delete-handler *couchdb-agent*)))) + +(defun start-couchdb-delete-actor-hook () + (start-couchdb-deletes *sys*)) + +(nhooks:add-hook star:*actors-start-hook* #'start-couchdb-delete-actor-hook) + +(eval-when (:compile-toplevel :load-toplevel :execute) + (export '(couchdb-get-request + make-couchdb-get-request + couchdb-get-request-database + couchdb-get-request-document-id + couchdb-get-request-revision + couchdb-insert-request + make-couchdb-insert-request + couchdb-insert-request-database + couchdb-insert-request-document-id + couchdb-insert-request-document + couchdb-delete-request + make-couchdb-delete-request + couchdb-delete-request-database + couchdb-delete-request-document-id + couchdb-delete-request-revision + couchdb-result + couchdb-result-status + couchdb-result-operation + couchdb-result-database + couchdb-result-document-id + couchdb-result-revision + couchdb-result-value + couchdb-result-error-type + couchdb-result-error-message + make-couchdb-get-handler + make-couchdb-insert-handler + make-couchdb-delete-handler + delete-couchdb-document + *couchdb-deletes* + start-couchdb-deletes) + :star.actors)) diff --git a/source/auth/core.lisp b/source/auth/core.lisp new file mode 100644 index 00000000..de1aa837 --- /dev/null +++ b/source/auth/core.lisp @@ -0,0 +1,469 @@ +(in-package :star.auth) + +(defparameter +api-key-prefix+ "star_sk_v1_") +(defparameter +api-key-version+ "v1") +(defparameter +credential-kind+ "api-key") + +(define-condition authentication-error (error) + ((code + :initarg :code + :initform "invalid_credential" + :reader authentication-error-code) + (message + :initarg :message + :initform "Authentication failed" + :reader authentication-error-message)) + (:report + (lambda (condition stream) + (format stream "~a" (authentication-error-message condition))))) + +(define-condition credential-lifecycle-error (error) + ((code + :initarg :code + :reader credential-lifecycle-error-code) + (message + :initarg :message + :reader credential-lifecycle-error-message)) + (:report + (lambda (condition stream) + (format stream "~a" (credential-lifecycle-error-message condition))))) + +(defstruct (request-principal + (:constructor %make-request-principal) + (:copier nil)) + (id nil :read-only t) + (type nil :read-only t) + (scopes nil :read-only t) + (credential-id nil :read-only t)) + +(defstruct (request-security-context + (:constructor %make-request-security-context) + (:copier nil)) + (principal nil :read-only t) + (correlation-id nil :read-only t) + (deadline nil :read-only t) + (authenticated-at nil :read-only t)) + +(defstruct (service-call-context + (:constructor %make-service-call-context) + (:copier nil)) + (principal-id nil :read-only t) + (principal-type nil :read-only t) + (credential-id nil :read-only t) + (scopes nil :read-only t) + (correlation-id nil :read-only t) + (deadline nil :read-only t)) + +(defstruct api-key-record + id + owner + principal-type + scopes + status + salt + verifier + created-at + expires-at + disabled-at + revoked-at + rotation-parent-id + superseded-by + overlap-expires-at + revision) + +(defclass credential-store () ()) + +(defgeneric credential-store-get (store credential-id)) +(defgeneric credential-store-put (store record)) +(defgeneric credential-store-update (store record)) +(defgeneric credential-store-list (store)) +(defgeneric credential-store-count (store)) + +(defvar *credential-store* nil) +(defvar *request-security-context* nil) +(defvar *auth-clock* #'get-universal-time) + +(defun auth-now () + (funcall *auth-clock*)) + +(defun signal-authentication-failure () + (error 'authentication-error + :code "invalid_credential" + :message "Authentication failed")) + +(defun signal-lifecycle-error (code message) + (error 'credential-lifecycle-error + :code code + :message message)) + +(defun string-octets (value) + (babel:string-to-octets value :encoding :utf-8)) + +(defun concatenate-octet-vectors (&rest vectors) + (let* ((length (reduce #'+ vectors :key #'length :initial-value 0)) + (result (make-array length :element-type '(unsigned-byte 8))) + (offset 0)) + (dolist (vector vectors result) + (replace result vector :start1 offset) + (incf offset (length vector))))) + +(defun constant-time-octets= (left right) + "Compare octet vectors without data-dependent early return. +Verifier inputs are fixed-length SHA-256 values at the authentication boundary." + (let* ((left-length (length left)) + (right-length (length right)) + (maximum (max left-length right-length)) + (difference (logxor left-length right-length))) + (dotimes (index maximum (zerop difference)) + (setf difference + (logior difference + (logxor (if (< index left-length) + (aref left index) + 0) + (if (< index right-length) + (aref right index) + 0))))))) + +(defvar *verifier-compare-function* #'constant-time-octets=) + +(defun sha256 (&rest vectors) + (ironclad:digest-sequence + :sha256 + (apply #'concatenate-octet-vectors vectors))) + +(defun random-hex (octet-count) + (ironclad:byte-array-to-hex-string + (ironclad:random-data octet-count))) + +(defun decode-hex (value) + (ironclad:hex-string-to-byte-array value)) + +(defun derive-verifier (secret-octets salt-hex pepper) + (sha256 (string-octets pepper) + (decode-hex salt-hex) + secret-octets)) + +(defun verifier-hex (secret-octets salt-hex pepper) + (ironclad:byte-array-to-hex-string + (derive-verifier secret-octets salt-hex pepper))) + +(defun fixed-secret-digest (value) + (sha256 (string-octets (or value "")))) + +(defun constant-time-secret= (left right) + (funcall *verifier-compare-function* + (fixed-secret-digest left) + (fixed-secret-digest right))) + +(defun split-on-character (string character) + (loop with start = 0 + for position = (position character string :start start) + collect (subseq string start position) + while position + do (setf start (1+ position)))) + +(defun valid-hex-string-p (value expected-length) + (and (stringp value) + (= (length value) expected-length) + (every (lambda (character) + (not (null (digit-char-p character 16)))) + value))) + +(defun parse-api-key (api-key) + "Return credential id and decoded secret. Signal one uniform failure otherwise." + (handler-case + (let ((parts (and (stringp api-key) + (split-on-character api-key #\_)))) + (unless (and (= (length parts) 5) + (string= (first parts) "star") + (string= (second parts) "sk") + (string= (third parts) +api-key-version+) + (plusp (length (fourth parts))) + (valid-hex-string-p + (fifth parts) + (* 2 star:*auth-key-secret-bytes*))) + (signal-authentication-failure)) + (values (fourth parts) + (decode-hex (fifth parts)))) + (authentication-error (condition) + (error condition)) + (error () + (signal-authentication-failure)))) + +(defun bearer-token (authorization-header) + (unless (and (stringp authorization-header) + (> (length authorization-header) 7) + (string-equal "Bearer " authorization-header :end2 7)) + (signal-authentication-failure)) + (let ((token (subseq authorization-header 7))) + (when (or (zerop (length token)) + (find #\Space token) + (find #\Tab token)) + (signal-authentication-failure)) + token)) + +(defun normalize-principal-type (value) + (string-downcase + (etypecase value + (string value) + (symbol (symbol-name value))))) + +(defun normalize-scopes (scopes) + (unless (and (listp scopes) + (every (lambda (scope) + (and (stringp scope) + (plusp (length scope)))) + scopes)) + (signal-lifecycle-error + "invalid_scopes" + "Scopes must be a list of non-empty strings")) + (remove-duplicates (copy-list scopes) :test #'string=)) + +(defun active-record-p (record now) + (and record + (eq :active (api-key-record-status record)) + (or (null (api-key-record-expires-at record)) + (> (api-key-record-expires-at record) now)) + (or (null (api-key-record-superseded-by record)) + (and (api-key-record-overlap-expires-at record) + (> (api-key-record-overlap-expires-at record) now))))) + +(defun record-principal (record) + (%make-request-principal + :id (api-key-record-owner record) + :type (api-key-record-principal-type record) + :scopes (copy-list (api-key-record-scopes record)) + :credential-id (api-key-record-id record))) + +(defun authenticate-api-key (api-key correlation-id deadline + &key (store *credential-store*)) + (unless store + (signal-authentication-failure)) + (multiple-value-bind (credential-id secret-octets) + (parse-api-key api-key) + (let* ((record (credential-store-get store credential-id)) + (now (auth-now))) + (unless (active-record-p record now) + (signal-authentication-failure)) + (let ((expected (decode-hex (api-key-record-verifier record))) + (actual (derive-verifier + secret-octets + (api-key-record-salt record) + star:*auth-pepper*))) + (unless (funcall *verifier-compare-function* expected actual) + (signal-authentication-failure))) + (%make-request-security-context + :principal (record-principal record) + :correlation-id correlation-id + :deadline deadline + :authenticated-at now)))) + +(defun authenticate-authorization-header (authorization-header correlation-id deadline + &key (store *credential-store*)) + (authenticate-api-key + (bearer-token authorization-header) + correlation-id + deadline + :store store)) + +(defun current-request-principal () + (and *request-security-context* + (request-security-context-principal *request-security-context*))) + +(defun current-principal-id () + (let ((principal (current-request-principal))) + (and principal (request-principal-id principal)))) + +(defun current-service-call-context () + (let ((context *request-security-context*)) + (when context + (let ((principal (request-security-context-principal context))) + (%make-service-call-context + :principal-id (request-principal-id principal) + :principal-type (request-principal-type principal) + :credential-id (request-principal-credential-id principal) + :scopes (copy-list (request-principal-scopes principal)) + :correlation-id (request-security-context-correlation-id context) + :deadline (request-security-context-deadline context)))))) + +(defun scope-granted-p (scope &optional (principal (current-request-principal))) + (and principal + (or (member "admin" (request-principal-scopes principal) :test #'string=) + (member scope (request-principal-scopes principal) :test #'string=)))) + +(defun administrator-principal-p (&optional (principal (current-request-principal))) + (and principal + (or (string= "administrator" (request-principal-type principal)) + (scope-granted-p "admin" principal)))) + +(defun make-api-key-material (owner principal-type scopes + &key expires-at rotation-parent-id) + (let* ((credential-id (cms-ulid:ulid)) + (secret-hex (random-hex star:*auth-key-secret-bytes*)) + (secret-octets (decode-hex secret-hex)) + (salt-hex (random-hex star:*auth-salt-bytes*)) + (record + (make-api-key-record + :id credential-id + :owner owner + :principal-type (normalize-principal-type principal-type) + :scopes (normalize-scopes scopes) + :status :active + :salt salt-hex + :verifier (verifier-hex secret-octets salt-hex star:*auth-pepper*) + :created-at (auth-now) + :expires-at expires-at + :rotation-parent-id rotation-parent-id))) + (values record + (format nil "~a~a_~a" + +api-key-prefix+ + credential-id + secret-hex)))) + +(defun validate-expiry (expires-in-seconds) + (cond + ((null expires-in-seconds) nil) + ((and (integerp expires-in-seconds) + (plusp expires-in-seconds)) + (+ (auth-now) expires-in-seconds)) + (t + (signal-lifecycle-error + "invalid_expiry" + "Expiration must be a positive number of seconds")))) + +(defun create-api-key (owner principal-type scopes + &key expires-in-seconds rotation-parent-id + (store *credential-store*)) + (unless (and (stringp owner) (plusp (length owner))) + (signal-lifecycle-error + "invalid_owner" + "Credential owner must be a non-empty string")) + (unless store + (signal-lifecycle-error + "auth_store_unavailable" + "Credential store is unavailable")) + (multiple-value-bind (record raw-key) + (make-api-key-material + owner + principal-type + scopes + :expires-at (validate-expiry expires-in-seconds) + :rotation-parent-id rotation-parent-id) + (credential-store-put store record) + (values record raw-key))) + +(defun bootstrap-api-key (presented-secret owner + &key (store *credential-store*)) + (unless store + (signal-lifecycle-error + "auth_store_unavailable" + "Credential store is unavailable")) + (unless (and star:*auth-bootstrap-secret* + (constant-time-secret= + presented-secret + star:*auth-bootstrap-secret*)) + (signal-lifecycle-error + "bootstrap_denied" + "Bootstrap denied")) + (unless (zerop (credential-store-count store)) + (signal-lifecycle-error + "bootstrap_complete" + "Bootstrap has already been completed")) + (create-api-key owner + "administrator" + (list "admin") + :store store)) + +(defun validate-overlap-seconds (overlap-seconds) + (unless (and (integerp overlap-seconds) + (<= 0 overlap-seconds star:*auth-rotation-overlap-max-seconds*)) + (signal-lifecycle-error + "invalid_overlap" + "Rotation overlap is outside the configured bound")) + overlap-seconds) + +(defun rotate-api-key (credential-id overlap-seconds + &key (store *credential-store*)) + (let* ((overlap (validate-overlap-seconds overlap-seconds)) + (record (and store + (credential-store-get store credential-id)))) + (unless record + (signal-lifecycle-error + "credential_not_found" + "Credential was not found")) + (unless (eq :active (api-key-record-status record)) + (signal-lifecycle-error + "credential_not_active" + "Credential is not active")) + (multiple-value-bind (replacement raw-key) + (make-api-key-material + (api-key-record-owner record) + (api-key-record-principal-type record) + (api-key-record-scopes record) + :expires-at (api-key-record-expires-at record) + :rotation-parent-id credential-id) + (credential-store-put store replacement) + (setf (api-key-record-superseded-by record) + (api-key-record-id replacement) + (api-key-record-overlap-expires-at record) + (+ (auth-now) overlap)) + (handler-case + (credential-store-update store record) + (error (condition) + (setf (api-key-record-status replacement) :revoked + (api-key-record-revoked-at replacement) (auth-now)) + (ignore-errors + (credential-store-update store replacement)) + (error condition))) + (values replacement raw-key)))) + +(defun revoke-api-key (credential-id &key (store *credential-store*)) + (let ((record (and store + (credential-store-get store credential-id)))) + (unless record + (signal-lifecycle-error + "credential_not_found" + "Credential was not found")) + (setf (api-key-record-status record) :revoked + (api-key-record-revoked-at record) (auth-now)) + (credential-store-update store record))) + +(defun disable-api-key (credential-id &key (store *credential-store*)) + (let ((record (and store + (credential-store-get store credential-id)))) + (unless record + (signal-lifecycle-error + "credential_not_found" + "Credential was not found")) + (setf (api-key-record-status record) :disabled + (api-key-record-disabled-at record) (auth-now)) + (credential-store-update store record))) + +(defun nullable-json-value (value) + (or value :null)) + +(defun api-key-metadata-json (record) + (jsown:new-js + ("credential_id" (api-key-record-id record)) + ("owner" (api-key-record-owner record)) + ("principal_type" (api-key-record-principal-type record)) + ("scopes" (copy-list (api-key-record-scopes record))) + ("status" (string-downcase + (symbol-name (api-key-record-status record)))) + ("created_at" (api-key-record-created-at record)) + ("expires_at" (nullable-json-value + (api-key-record-expires-at record))) + ("disabled_at" (nullable-json-value + (api-key-record-disabled-at record))) + ("revoked_at" (nullable-json-value + (api-key-record-revoked-at record))) + ("rotation_parent_id" (nullable-json-value + (api-key-record-rotation-parent-id record))) + ("superseded_by" (nullable-json-value + (api-key-record-superseded-by record))) + ("overlap_expires_at" (nullable-json-value + (api-key-record-overlap-expires-at record))))) + +(defun list-api-key-metadata (&key (store *credential-store*)) + (mapcar #'api-key-metadata-json + (credential-store-list store))) diff --git a/source/auth/immutability.lisp b/source/auth/immutability.lisp new file mode 100644 index 00000000..f8fd8f9f --- /dev/null +++ b/source/auth/immutability.lisp @@ -0,0 +1,87 @@ +(in-package :star.auth) + +(defparameter *raw-request-principal-id-reader* + (symbol-function 'request-principal-id)) +(defparameter *raw-request-principal-type-reader* + (symbol-function 'request-principal-type)) +(defparameter *raw-request-principal-scopes-reader* + (symbol-function 'request-principal-scopes)) +(defparameter *raw-request-principal-credential-id-reader* + (symbol-function 'request-principal-credential-id)) +(defparameter *raw-request-security-context-correlation-id-reader* + (symbol-function 'request-security-context-correlation-id)) +(defparameter *raw-service-call-context-principal-id-reader* + (symbol-function 'service-call-context-principal-id)) +(defparameter *raw-service-call-context-principal-type-reader* + (symbol-function 'service-call-context-principal-type)) +(defparameter *raw-service-call-context-credential-id-reader* + (symbol-function 'service-call-context-credential-id)) +(defparameter *raw-service-call-context-scopes-reader* + (symbol-function 'service-call-context-scopes)) +(defparameter *raw-service-call-context-correlation-id-reader* + (symbol-function 'service-call-context-correlation-id)) + +(defun copy-string-or-nil (value) + (and value (copy-seq value))) + +(defun defensive-request-principal-id (principal) + (copy-string-or-nil + (funcall *raw-request-principal-id-reader* principal))) + +(defun defensive-request-principal-type (principal) + (copy-string-or-nil + (funcall *raw-request-principal-type-reader* principal))) + +(defun defensive-request-principal-scopes (principal) + (mapcar #'copy-string-or-nil + (funcall *raw-request-principal-scopes-reader* principal))) + +(defun defensive-request-principal-credential-id (principal) + (copy-string-or-nil + (funcall *raw-request-principal-credential-id-reader* principal))) + +(defun defensive-request-security-context-correlation-id (context) + (copy-string-or-nil + (funcall *raw-request-security-context-correlation-id-reader* context))) + +(defun defensive-service-call-context-principal-id (context) + (copy-string-or-nil + (funcall *raw-service-call-context-principal-id-reader* context))) + +(defun defensive-service-call-context-principal-type (context) + (copy-string-or-nil + (funcall *raw-service-call-context-principal-type-reader* context))) + +(defun defensive-service-call-context-credential-id (context) + (copy-string-or-nil + (funcall *raw-service-call-context-credential-id-reader* context))) + +(defun defensive-service-call-context-scopes (context) + (mapcar #'copy-string-or-nil + (funcall *raw-service-call-context-scopes-reader* context))) + +(defun defensive-service-call-context-correlation-id (context) + (copy-string-or-nil + (funcall *raw-service-call-context-correlation-id-reader* context))) + +(eval-when (:load-toplevel :execute) + (setf (symbol-function 'request-principal-id) + #'defensive-request-principal-id + (symbol-function 'request-principal-type) + #'defensive-request-principal-type + (symbol-function 'request-principal-scopes) + #'defensive-request-principal-scopes + (symbol-function 'request-principal-credential-id) + #'defensive-request-principal-credential-id + (symbol-function 'request-security-context-correlation-id) + #'defensive-request-security-context-correlation-id + (symbol-function 'service-call-context-principal-id) + #'defensive-service-call-context-principal-id + (symbol-function 'service-call-context-principal-type) + #'defensive-service-call-context-principal-type + (symbol-function 'service-call-context-credential-id) + #'defensive-service-call-context-credential-id + (symbol-function 'service-call-context-scopes) + #'defensive-service-call-context-scopes + (symbol-function 'service-call-context-correlation-id) + #'defensive-service-call-context-correlation-id)) diff --git a/source/auth/store.lisp b/source/auth/store.lisp new file mode 100644 index 00000000..fdaddef0 --- /dev/null +++ b/source/auth/store.lisp @@ -0,0 +1,293 @@ +(in-package :star.auth) + +(defclass memory-credential-store (credential-store) + ((records + :initform (make-hash-table :test #'equal) + :reader memory-store-records) + (lock + :initform (bt:make-lock "memory-credential-store") + :reader memory-store-lock))) + +(defun make-memory-credential-store () + (make-instance 'memory-credential-store)) + +(defun copy-record-or-nil (record) + (and record (copy-api-key-record record))) + +(defmethod credential-store-get ((store memory-credential-store) credential-id) + (bt:with-lock-held ((memory-store-lock store)) + (copy-record-or-nil + (gethash credential-id (memory-store-records store))))) + +(defmethod credential-store-put ((store memory-credential-store) record) + (bt:with-lock-held ((memory-store-lock store)) + (when (gethash (api-key-record-id record) + (memory-store-records store)) + (signal-lifecycle-error + "credential_conflict" + "Credential identifier already exists")) + (setf (gethash (api-key-record-id record) + (memory-store-records store)) + (copy-api-key-record record))) + (copy-api-key-record record)) + +(defmethod credential-store-update ((store memory-credential-store) record) + (bt:with-lock-held ((memory-store-lock store)) + (unless (gethash (api-key-record-id record) + (memory-store-records store)) + (signal-lifecycle-error + "credential_not_found" + "Credential was not found")) + (setf (gethash (api-key-record-id record) + (memory-store-records store)) + (copy-api-key-record record))) + (copy-api-key-record record)) + +(defmethod credential-store-list ((store memory-credential-store)) + (bt:with-lock-held ((memory-store-lock store)) + (sort + (loop for record being the hash-values of (memory-store-records store) + collect (copy-api-key-record record)) + #'< + :key #'api-key-record-created-at))) + +(defmethod credential-store-count ((store memory-credential-store)) + (bt:with-lock-held ((memory-store-lock store)) + (hash-table-count (memory-store-records store)))) + +(defclass couchdb-credential-store (credential-store) + ((pool + :initarg :pool + :reader couchdb-store-pool) + (database + :initarg :database + :reader couchdb-store-database))) + +(defun make-auth-couchdb-pool () + (anypool:make-pool + :name "starintel-auth-couchdb-connections" + :connector + (lambda () + (let ((client + (cl-couch:new-couchdb + star:*couchdb-host* + star:*couchdb-port* + :scheme star:*couchdb-scheme*))) + (cl-couch:password-auth + client + star:*couchdb-user* + star:*couchdb-password*) + client)) + :disconnector + (lambda (client) + (setf (cl-couch:couchdb-headers client) nil)) + :max-open-count 10 + :max-idle-count 5)) + +(defun make-couchdb-credential-store () + (make-instance + 'couchdb-credential-store + :pool (make-auth-couchdb-pool) + :database star:*couchdb-auth-database*)) + +(defun status-string (status) + (string-downcase (symbol-name status))) + +(defun parse-status (status) + (intern (string-upcase status) :keyword)) + +(defun api-key-record-to-json (record) + (let ((document + (jsown:new-js + ("_id" (api-key-record-id record)) + ("kind" +credential-kind+) + ("owner" (api-key-record-owner record)) + ("principal_type" (api-key-record-principal-type record)) + ("scopes" (copy-list (api-key-record-scopes record))) + ("status" (status-string (api-key-record-status record))) + ("salt" (api-key-record-salt record)) + ("verifier" (api-key-record-verifier record)) + ("created_at" (api-key-record-created-at record)) + ("expires_at" (nullable-json-value + (api-key-record-expires-at record))) + ("disabled_at" (nullable-json-value + (api-key-record-disabled-at record))) + ("revoked_at" (nullable-json-value + (api-key-record-revoked-at record))) + ("rotation_parent_id" (nullable-json-value + (api-key-record-rotation-parent-id record))) + ("superseded_by" (nullable-json-value + (api-key-record-superseded-by record))) + ("overlap_expires_at" (nullable-json-value + (api-key-record-overlap-expires-at record)))))) + (when (api-key-record-revision record) + (setf (jsown:val document "_rev") + (api-key-record-revision record))) + document)) + +(defun null-json-value-p (value) + (or (null value) (eq value :null))) + +(defun json-value-or-nil (document key) + (let ((value (jsown:val-safe document key))) + (unless (null-json-value-p value) + value))) + +(defun json-to-api-key-record (value) + (let ((document (if (stringp value) + (jsown:parse value) + value))) + (make-api-key-record + :id (jsown:val document "_id") + :owner (jsown:val document "owner") + :principal-type (jsown:val document "principal_type") + :scopes (copy-list (or (jsown:val-safe document "scopes") nil)) + :status (parse-status (jsown:val document "status")) + :salt (jsown:val document "salt") + :verifier (jsown:val document "verifier") + :created-at (jsown:val document "created_at") + :expires-at (json-value-or-nil document "expires_at") + :disabled-at (json-value-or-nil document "disabled_at") + :revoked-at (json-value-or-nil document "revoked_at") + :rotation-parent-id + (json-value-or-nil document "rotation_parent_id") + :superseded-by (json-value-or-nil document "superseded_by") + :overlap-expires-at + (json-value-or-nil document "overlap_expires_at") + :revision (jsown:val-safe document "_rev")))) + +(defun update-record-revision-from-response (record response) + (let ((parsed (ignore-errors (jsown:parse response)))) + (when parsed + (setf (api-key-record-revision record) + (jsown:val-safe parsed "rev")))) + record) + +(defmethod credential-store-get ((store couchdb-credential-store) credential-id) + (anypool:with-connection (client (couchdb-store-pool store)) + (handler-case + (json-to-api-key-record + (cl-couch:get-document + client + (couchdb-store-database store) + credential-id)) + (dex:http-request-not-found () nil)))) + +(defmethod credential-store-put ((store couchdb-credential-store) record) + (anypool:with-connection (client (couchdb-store-pool store)) + (handler-case + (update-record-revision-from-response + record + (cl-couch:create-document + client + (couchdb-store-database store) + (jsown:to-json (api-key-record-to-json record)))) + (dex:http-request-conflict () + (signal-lifecycle-error + "credential_conflict" + "Credential identifier already exists"))))) + +(defmethod credential-store-update ((store couchdb-credential-store) record) + (unless (api-key-record-revision record) + (let ((current + (credential-store-get store (api-key-record-id record)))) + (unless current + (signal-lifecycle-error + "credential_not_found" + "Credential was not found")) + (setf (api-key-record-revision record) + (api-key-record-revision current)))) + (anypool:with-connection (client (couchdb-store-pool store)) + (handler-case + (update-record-revision-from-response + record + (cl-couch:create-document + client + (couchdb-store-database store) + (jsown:to-json (api-key-record-to-json record)))) + (dex:http-request-conflict () + (signal-lifecycle-error + "credential_conflict" + "Credential update conflicted"))))) + +(defmethod credential-store-list ((store couchdb-credential-store)) + (anypool:with-connection (client (couchdb-store-pool store)) + (let* ((view + (star.databases.couchdb:query-view + client + (couchdb-store-database store) + "auth" + "credentials" + :include-docs t + :limit 10000 + :reduce nil)) + (rows (or (jsown:val-safe view "rows") nil))) + (loop for row in rows + for document = (jsown:val row "doc") + collect (json-to-api-key-record document))))) + +(defmethod credential-store-count ((store couchdb-credential-store)) + (length (credential-store-list store))) + +(defun auth-design-document () + (jsown:new-js + ("_id" "_design/auth") + ("views" + (jsown:new-js + ("credentials" + (jsown:new-js + ("map" + "function(doc){if(doc.kind==='api-key'){emit(doc.created_at,null);}}"))))))) + +(defun ensure-auth-database (store) + (anypool:with-connection (client (couchdb-store-pool store)) + (handler-case + (cl-couch:get-database client (couchdb-store-database store)) + (dex:http-request-not-found () + (cl-couch:create-database client (couchdb-store-database store)))))) + +(defun ensure-auth-design-document (store) + (anypool:with-connection (client (couchdb-store-pool store)) + (let* ((database (couchdb-store-database store)) + (document (auth-design-document))) + (handler-case + (let* ((existing + (jsown:parse + (cl-couch:get-document + client database "_design/auth"))) + (revision (jsown:val existing "_rev"))) + (setf (jsown:val document "_rev") revision) + (cl-couch:create-document + client database (jsown:to-json document))) + (dex:http-request-not-found () + (cl-couch:create-document + client database (jsown:to-json document))))))) + +(defun loopback-address-p (address) + (member (string-downcase address) + '("localhost" "127.0.0.1" "::1") + :test #'string=)) + +(defun validate-auth-configuration () + (let ((mode (string-downcase star:*auth-mode*))) + (cond + ((string= mode "api-key") + (unless (and (stringp star:*auth-pepper*) + (plusp (length star:*auth-pepper*))) + (error "STAR_AUTH_PEPPER or STAR_AUTH_PEPPER_FILE is required"))) + ((string= mode "disabled") + (unless (and star:*auth-dev-bypass* + (loopback-address-p star:*http-api-address*)) + (error "Disabled authentication requires explicit loopback development bypass"))) + (t + (error "Unsupported STAR_AUTH_MODE: ~a" star:*auth-mode*)))) + t) + +(defun initialize-auth-store (&key force) + (validate-auth-configuration) + (when (or force (null *credential-store*)) + (setf *credential-store* (make-couchdb-credential-store))) + (when (typep *credential-store* 'couchdb-credential-store) + (ensure-auth-database *credential-store*) + (ensure-auth-design-document *credential-store*)) + *credential-store*) diff --git a/source/auth/verification-hardening.lisp b/source/auth/verification-hardening.lisp new file mode 100644 index 00000000..cad6021e --- /dev/null +++ b/source/auth/verification-hardening.lisp @@ -0,0 +1,53 @@ +(in-package :star.auth) + +(defparameter +dummy-verifier-salt+ + "00000000000000000000000000000000") + +(defparameter +dummy-verifier+ + "0000000000000000000000000000000000000000000000000000000000000000") + +(defun credential-verifier-material (record) + "Return verifier bytes and salt without exposing record existence through work. +Malformed stored material fails closed through the same dummy verifier path." + (handler-case + (if record + (values (decode-hex (api-key-record-verifier record)) + (api-key-record-salt record)) + (values (decode-hex +dummy-verifier+) + +dummy-verifier-salt+)) + (error () + (values (decode-hex +dummy-verifier+) + +dummy-verifier-salt+)))) + +(defun hardened-authenticate-api-key (api-key correlation-id deadline + &key (store *credential-store*)) + "Authenticate through one verifier-comparison path for known and unknown ids." + (unless store + (signal-authentication-failure)) + (multiple-value-bind (credential-id secret-octets) + (parse-api-key api-key) + (let* ((record (credential-store-get store credential-id)) + (now (auth-now))) + (multiple-value-bind (expected salt) + (credential-verifier-material record) + (let* ((actual + (handler-case + (derive-verifier secret-octets salt star:*auth-pepper*) + (error () + (derive-verifier + secret-octets + +dummy-verifier-salt+ + (or star:*auth-pepper* ""))))) + (verified + (funcall *verifier-compare-function* expected actual))) + (unless (and verified (active-record-p record now)) + (signal-authentication-failure)))) + (%make-request-security-context + :principal (record-principal record) + :correlation-id correlation-id + :deadline deadline + :authenticated-at now)))) + +(eval-when (:load-toplevel :execute) + (setf (symbol-function 'authenticate-api-key) + #'hardened-authenticate-api-key)) diff --git a/source/consumers/rabbit-settlement.lisp b/source/consumers/rabbit-settlement.lisp new file mode 100644 index 00000000..342abcfe --- /dev/null +++ b/source/consumers/rabbit-settlement.lisp @@ -0,0 +1,288 @@ +(in-package :star.consumers) + +(defstruct (rabbit-settlement + (:constructor make-rabbit-settlement + (&key action reason (requeue nil) value error))) + action + reason + requeue + value + error) + +(defun rabbit-ack (&key reason value) + (make-rabbit-settlement + :action :ack + :reason reason + :value value)) + +(defun rabbit-nack (&key reason (requeue nil) value error) + (make-rabbit-settlement + :action :nack + :reason reason + :requeue requeue + :value value + :error error)) + +(defun normalize-rabbit-settlement (result) + "Translate a consumer result into an explicit Rabbit settlement decision." + (typecase result + (rabbit-settlement result) + (t (rabbit-ack :reason :handler-complete :value result)))) + +(defun settle-rabbit-delivery (consumer delivery settlement + &key + (ack-fn #'cl-rabbit:basic-ack) + (nack-fn #'cl-rabbit:basic-nack)) + "Settle DELIVERY on the same connection and channel that received it." + (let* ((stream (consumer-stream consumer)) + (connection (rabbit-stream-connection stream)) + (delivery-tag (cdr delivery)) + (decision (normalize-rabbit-settlement settlement))) + (ecase (rabbit-settlement-action decision) + (:ack + (funcall ack-fn connection 1 delivery-tag :multiple nil)) + (:nack + (funcall nack-fn + connection + 1 + delivery-tag + :multiple nil + :requeue (rabbit-settlement-requeue decision)))) + decision)) + +(defclass settled-rabbit-queue-stream (rabbit-queue-stream) + ((queue-arguments + :initarg :queue-arguments + :initform nil + :accessor rabbit-stream-queue-arguments) + (dead-letter-exchange + :initarg :dead-letter-exchange + :initform nil + :accessor rabbit-stream-dead-letter-exchange) + (dead-letter-routing-key + :initarg :dead-letter-routing-key + :initform nil + :accessor rabbit-stream-dead-letter-routing-key) + (dead-letter-queue + :initarg :dead-letter-queue + :initform nil + :accessor rabbit-stream-dead-letter-queue)) + (:documentation "Rabbit stream with explicit settlement and dead-letter policy.")) + +(defun rabbit-dead-letter-arguments (stream) + (let ((exchange (rabbit-stream-dead-letter-exchange stream)) + (routing-key (rabbit-stream-dead-letter-routing-key stream))) + (append + (rabbit-stream-queue-arguments stream) + (when exchange + (list (cons "x-dead-letter-exchange" exchange))) + (when routing-key + (list (cons "x-dead-letter-routing-key" routing-key)))))) + +(defmethod open-stream ((stream settled-rabbit-queue-stream)) + (let* ((connection (cl-rabbit:new-connection)) + (socket (cl-rabbit:tcp-socket-new connection)) + (username (rabbit-stream-user stream)) + (password (rabbit-stream-password stream)) + (dead-letter-exchange + (rabbit-stream-dead-letter-exchange stream)) + (dead-letter-queue + (rabbit-stream-dead-letter-queue stream)) + (dead-letter-routing-key + (or (rabbit-stream-dead-letter-routing-key stream) "#"))) + (setf (rabbit-stream-connection stream) connection) + (cl-rabbit:socket-open + socket + (rabbit-stream-host stream) + (rabbit-stream-port stream)) + (when (or username password) + (cl-rabbit:login-sasl-plain + connection + (rabbit-stream-vhost stream) + username + password)) + (cl-rabbit:channel-open connection 1) + (cl-rabbit:basic-qos connection 1 :prefetch-count 200) + (cl-rabbit:exchange-declare + connection + 1 + (rabbit-stream-exchange stream) + (rabbit-exchange-type stream) + :durable (rabbit-exchange-durable-p stream)) + (when dead-letter-exchange + (cl-rabbit:exchange-declare + connection + 1 + dead-letter-exchange + "topic" + :durable t)) + (cl-rabbit:queue-declare + connection + 1 + :queue (rabbit-stream-queue-name stream) + :durable (rabbit-stream-queue-durable-p stream) + :arguments (rabbit-dead-letter-arguments stream)) + (cl-rabbit:queue-bind + connection + 1 + :queue (rabbit-stream-queue-name stream) + :exchange (rabbit-stream-exchange stream) + :routing-key (rabbit-stream-routing-key stream)) + (when (and dead-letter-exchange dead-letter-queue) + (cl-rabbit:queue-declare + connection + 1 + :queue dead-letter-queue + :durable t) + (cl-rabbit:queue-bind + connection + 1 + :queue dead-letter-queue + :exchange dead-letter-exchange + :routing-key dead-letter-routing-key)) + (cl-rabbit:basic-consume + connection + 1 + (rabbit-stream-queue-name stream) + :no-ack nil) + (setf (rabbit-stream-open-p stream) t))) + +(defun copy-rabbit-stream-options (stream) + (list + :queue-name (rabbit-stream-queue-name stream) + :exchange-name (rabbit-stream-exchange stream) + :exchange-type (rabbit-exchange-type stream) + :exchange-durable (rabbit-exchange-durable-p stream) + :queue-durable (rabbit-stream-queue-durable-p stream) + :routing-key (rabbit-stream-routing-key stream) + :host (rabbit-stream-host stream) + :port (rabbit-stream-port stream) + :vhost (rabbit-stream-vhost stream) + :username (rabbit-stream-user stream) + :password (rabbit-stream-password stream) + :queue-arguments + (and (typep stream 'settled-rabbit-queue-stream) + (rabbit-stream-queue-arguments stream)) + :dead-letter-exchange + (and (typep stream 'settled-rabbit-queue-stream) + (rabbit-stream-dead-letter-exchange stream)) + :dead-letter-routing-key + (and (typep stream 'settled-rabbit-queue-stream) + (rabbit-stream-dead-letter-routing-key stream)) + :dead-letter-queue + (and (typep stream 'settled-rabbit-queue-stream) + (rabbit-stream-dead-letter-queue stream)))) + +(defmethod start-consumer ((consumer rabbit-consumer)) + "Run each Rabbit worker with its own connection and settle every delivery." + (let ((create-thread-consumer + (lambda (thread-number) + (let* ((thread-consumer + (apply #'create-rabbit-consumer + :name + (format nil "~A-~D" + (consumer-name consumer) + thread-number) + :n 1 + :test-fn (consumer-filter consumer) + :handler-fn (consumer-fn consumer) + (copy-rabbit-stream-options + (consumer-stream consumer))))) + (open-stream (consumer-stream thread-consumer)) + (assert + (rabbit-stream-open-p (consumer-stream thread-consumer)) + nil + "RabbitMQ stream was not opened.") + (lambda () + (loop + for delivery = (consumer-read thread-consumer) + for settlement = + (handler-case + (progn + (consume thread-consumer delivery) + (normalize-rabbit-settlement + (receive-result + (consumer-channel thread-consumer)))) + (error (condition) + (rabbit-nack + :reason :handler-error + :requeue t + :error condition))) + do (settle-rabbit-delivery + thread-consumer + delivery + settlement))))))) + (loop for thread-number from 1 to (consumer-worker-count consumer) + do (bt:make-thread + (funcall create-thread-consumer thread-number) + :name (format nil "~A-~D" + (consumer-name consumer) + thread-number))))) + +(defun create-rabbit-consumer (&key + (name (error "Consumer name is required")) + (n 1) + (queue-name + (error "Queue name is required")) + (exchange-name "documents") + (exchange-type "topic") + (exchange-durable t) + (queue-durable t) + (queue-arguments nil) + (dead-letter-exchange nil) + (dead-letter-routing-key nil) + (dead-letter-queue nil) + (routing-key + (error "Routing key is required")) + (host "localhost") + (port 5672) + (vhost "/") + (username "guest") + (password "guest") + (test-fn #'identity) + (handler-fn + (error "Handler function is required"))) + "Create a Rabbit consumer with explicit durability and settlement policy." + (make-instance + 'rabbit-consumer + :name (string-downcase (string name)) + :stream + (make-instance + 'settled-rabbit-queue-stream + :queue-name queue-name + :exchange-name exchange-name + :exchange-type exchange-type + :exchange-durable exchange-durable + :queue-durable queue-durable + :queue-arguments queue-arguments + :dead-letter-exchange dead-letter-exchange + :dead-letter-routing-key dead-letter-routing-key + :dead-letter-queue dead-letter-queue + :routing-key routing-key + :host host + :port port + :vhost vhost + :user username + :password password) + :workers n + :fn handler-fn + :test-fn test-fn)) + +(eval-when (:compile-toplevel :load-toplevel :execute) + (export '(rabbit-settlement + make-rabbit-settlement + rabbit-settlement-action + rabbit-settlement-reason + rabbit-settlement-requeue + rabbit-settlement-value + rabbit-settlement-error + rabbit-ack + rabbit-nack + normalize-rabbit-settlement + settle-rabbit-delivery + settled-rabbit-queue-stream + rabbit-stream-queue-arguments + rabbit-stream-dead-letter-exchange + rabbit-stream-dead-letter-routing-key + rabbit-stream-dead-letter-queue) + :star.consumers)) diff --git a/source/databases/export.lisp b/source/databases/export.lisp new file mode 100644 index 00000000..86e1a86e --- /dev/null +++ b/source/databases/export.lisp @@ -0,0 +1,106 @@ +(in-package :star.databases.couchdb) + +(defparameter +dataset-export-page-size+ 100) +(defparameter +dataset-export-consistency+ :monotonic-key-scan) + +(defun dataset-export-end-key (dataset) + (list dataset (jsown:empty-object))) + +(defun dataset-export-temp-path (path) + (pathname + (format nil "~a.~d.~d.tmp" + (namestring (pathname path)) + (get-universal-time) + (random most-positive-fixnum)))) + +(defun dataset-export-error-type (condition) + (string-downcase (princ-to-string (type-of condition)))) + +(defun export-by-dataset* (client database dataset path + &key + (page-size +dataset-export-page-size+) + (query-fn #'query-view)) + "Export DATASET as JSON Lines using bounded, key-based CouchDB pagination. + +The export is a monotonic key scan rather than a database snapshot. Documents +created after the current cursor may be included; documents deleted before they +are read are omitted. A document key is emitted at most once per export. + +The target file is replaced only after the complete export has been flushed. +The return value is a plist containing :OK, counts, path, consistency mode, and +structured error details when the export fails." + (unless (and (integerp page-size) (plusp page-size)) + (error "PAGE-SIZE must be a positive integer, got ~s" page-size)) + (let* ((target-path (pathname path)) + (temporary-path (dataset-export-temp-path target-path)) + (total-exported 0) + (pages-exported 0) + (last-key nil) + (completed nil)) + (ensure-directories-exist target-path) + (unwind-protect + (handler-case + (progn + (with-open-file (out temporary-path + :direction :output + :if-exists :error + :if-does-not-exist :create + :external-format :utf-8) + (loop + for result = (funcall query-fn + client + database + "data" + "documents_by_dataset" + :start-key (or last-key (list dataset)) + :end-key (dataset-export-end-key dataset) + :skip (if last-key 1 0) + :limit page-size + :include-docs t + :reduce nil + :update (zerop pages-exported)) + for rows = (or (jsown:val-safe result "rows") nil) + do (when (null rows) + (return)) + (incf pages-exported) + (loop for row in rows + for document = (jsown:val-safe row "doc") + for row-key = (jsown:val-safe row "key") + do (unless document + (error "Dataset export row is missing doc: ~s" row)) + (unless row-key + (error "Dataset export row is missing key: ~s" row)) + (unless (equal dataset + (jsown:val-safe document "dataset")) + (error "Dataset export row belongs to ~s, expected ~s" + (jsown:val-safe document "dataset") + dataset)) + (write-string (jsown:to-json document) out) + (terpri out) + (incf total-exported) + (setf last-key row-key)) + (when (< (length rows) page-size) + (return))) + (finish-output out)) + (uiop:rename-file-overwriting-target temporary-path target-path) + (setf completed t) + (list :ok t + :dataset dataset + :path (namestring target-path) + :exported total-exported + :pages pages-exported + :page-size page-size + :consistency +dataset-export-consistency+)) + (error (condition) + (list :ok nil + :dataset dataset + :path (namestring target-path) + :exported total-exported + :pages pages-exported + :page-size page-size + :consistency +dataset-export-consistency+ + :error-type (dataset-export-error-type condition) + :error (princ-to-string condition)))) + (unless completed + (when (probe-file temporary-path) + (ignore-errors (delete-file temporary-path))))))) diff --git a/source/frontends/http-auth-job-routes.lisp b/source/frontends/http-auth-job-routes.lisp new file mode 100644 index 00000000..19b48bde --- /dev/null +++ b/source/frontends/http-auth-job-routes.lisp @@ -0,0 +1,22 @@ +(in-package :star.frontends.http-api) + +(defun handle-authenticated-bulk-status-route (params) + (with-http-boundary () + (let* ((job-id (query-value params "job-id")) + (job + (and job-id + (bt:with-lock-held (*bulk-ingest-lock*) + (gethash job-id *bulk-ingest-jobs*)))) + (principal-id (star.auth:current-principal-id))) + (unless (and job + (or (star.auth:administrator-principal-p) + (string= principal-id + (bulk-ingest-job-principal job)))) + (signal-http-input-error + 404 + "bulk_job_not_found" + "Bulk ingest job was not found")) + (jsown:to-json (bulk-job-info-json job))))) + +(setf (ningle:route *app* "/documents/bulk/:job-id" :method :get) + #'handle-authenticated-bulk-status-route) diff --git a/source/frontends/http-auth-routes.lisp b/source/frontends/http-auth-routes.lisp new file mode 100644 index 00000000..9e695a8a --- /dev/null +++ b/source/frontends/http-auth-routes.lisp @@ -0,0 +1,217 @@ +(in-package :star.frontends.http-api) + +(defun lifecycle-error-status (code) + (cond + ((member code + '("invalid_owner" "invalid_scopes" "invalid_expiry" + "invalid_overlap") + :test #'string=) + 422) + ((string= code "credential_not_found") 404) + ((member code + '("credential_conflict" "credential_not_active" + "bootstrap_complete") + :test #'string=) + 409) + ((string= code "bootstrap_denied") 403) + ((string= code "auth_store_unavailable") 503) + (t 400))) + +(defmacro with-credential-lifecycle-errors (&body body) + `(handler-case + (progn ,@body) + (star.auth:credential-lifecycle-error (condition) + (signal-http-input-error + (lifecycle-error-status + (star.auth:credential-lifecycle-error-code condition)) + (star.auth:credential-lifecycle-error-code condition) + (star.auth:credential-lifecycle-error-message condition))))) + +(defun require-auth-string (document field) + (let ((value (jsown:val-safe document field))) + (unless (and (stringp value) (plusp (length value))) + (signal-http-input-error + 422 + "invalid_auth_request" + (format nil "Field ~a must be a non-empty string" field))) + value)) + +(defun optional-positive-integer (document field) + (let ((value (jsown:val-safe document field))) + (cond + ((or (null value) (eq value :null)) nil) + ((and (integerp value) (plusp value)) value) + (t + (signal-http-input-error + 422 + "invalid_auth_request" + (format nil "Field ~a must be a positive integer" field)))))) + +(defun require-scope-array (document) + (let ((scopes (jsown:val-safe document "scopes"))) + (unless (and (json-array-p scopes) + (every (lambda (scope) + (and (stringp scope) + (plusp (length scope)))) + scopes)) + (signal-http-input-error + 422 + "invalid_auth_request" + "Field scopes must be an array of non-empty strings")) + scopes)) + +(defun add-no-store-header () + (setf (lack.response:response-headers *response*) + (append (lack.response:response-headers *response*) + (list :cache-control "no-store" + :pragma "no-cache")))) + +(defun credential-secret-response (record raw-key) + (add-no-store-header) + (jsown:to-json + (jsown:new-js + ("api_key" raw-key) + ("credential" (star.auth:api-key-metadata-json record)) + ("correlation_id" (current-correlation-id))))) + +(defun handle-auth-bootstrap-route (params) + (declare (ignore params)) + (with-http-boundary () + (with-credential-lifecycle-errors + (let* ((request (ningle:context :request)) + (headers (lack.request:request-headers request)) + (presented-secret + (request-header-value headers "X-Star-Bootstrap-Secret")) + (body (require-json-object (parse-json-request))) + (owner (or (jsown:val-safe body "owner") + "bootstrap-administrator"))) + (unless (and (stringp owner) (plusp (length owner))) + (signal-http-input-error + 422 + "invalid_auth_request" + "Field owner must be a non-empty string")) + (multiple-value-bind (record raw-key) + (star.auth:bootstrap-api-key presented-secret owner) + (setf (lack.response:response-status *response*) 201) + (credential-secret-response record raw-key)))))) + +(defun handle-auth-create-route (params) + (declare (ignore params)) + (with-http-boundary () + (require-administrator-context) + (with-credential-lifecycle-errors + (let* ((body (require-json-object (parse-json-request))) + (owner (require-auth-string body "owner")) + (principal-type + (require-auth-string body "principal_type")) + (scopes (require-scope-array body)) + (expires-in-seconds + (optional-positive-integer body "expires_in_seconds"))) + (multiple-value-bind (record raw-key) + (star.auth:create-api-key + owner + principal-type + scopes + :expires-in-seconds expires-in-seconds) + (setf (lack.response:response-status *response*) 201) + (credential-secret-response record raw-key)))))) + +(defun handle-auth-list-route (params) + (declare (ignore params)) + (with-http-boundary () + (require-administrator-context) + (with-credential-lifecycle-errors + (jsown:to-json + (star.auth:list-api-key-metadata))))) + +(defun credential-id-param (params) + (let ((credential-id (query-value params "credential-id"))) + (unless (and (stringp credential-id) + (plusp (length credential-id))) + (signal-http-input-error + 400 + "missing_path_parameter" + "Credential identifier is required")) + credential-id)) + +(defun handle-auth-rotate-route (params) + (with-http-boundary () + (require-administrator-context) + (with-credential-lifecycle-errors + (let* ((credential-id (credential-id-param params)) + (body (require-json-object (parse-json-request))) + (overlap-seconds + (or (jsown:val-safe body "overlap_seconds") 0))) + (unless (and (integerp overlap-seconds) + (not (minusp overlap-seconds))) + (signal-http-input-error + 422 + "invalid_auth_request" + "Field overlap_seconds must be a non-negative integer")) + (multiple-value-bind (record raw-key) + (star.auth:rotate-api-key credential-id overlap-seconds) + (setf (lack.response:response-status *response*) 201) + (credential-secret-response record raw-key)))))) + +(defun lifecycle-status-response (record message) + (jsown:to-json + (jsown:new-js + ("status" "ok") + ("msg" message) + ("credential" (star.auth:api-key-metadata-json record)) + ("correlation_id" (current-correlation-id))))) + +(defun handle-auth-revoke-route (params) + (with-http-boundary () + (require-administrator-context) + (with-credential-lifecycle-errors + (lifecycle-status-response + (star.auth:revoke-api-key (credential-id-param params)) + "Credential revoked")))) + +(defun handle-auth-disable-route (params) + (with-http-boundary () + (require-administrator-context) + (with-credential-lifecycle-errors + (lifecycle-status-response + (star.auth:disable-api-key (credential-id-param params)) + "Credential disabled")))) + +(defun handle-auth-context-route (params) + (declare (ignore params)) + (with-http-boundary () + (let* ((context star.auth:*request-security-context*) + (principal + (star.auth:request-security-context-principal context))) + (jsown:to-json + (jsown:new-js + ("principal_id" (star.auth:request-principal-id principal)) + ("principal_type" (star.auth:request-principal-type principal)) + ("credential_id" + (star.auth:request-principal-credential-id principal)) + ("scopes" (star.auth:request-principal-scopes principal)) + ("correlation_id" + (star.auth:request-security-context-correlation-id context)) + ("deadline" + (star.auth:request-security-context-deadline context))))))) + +(setf (ningle:route *app* "/auth/bootstrap" :method :post) + #'handle-auth-bootstrap-route) + +(setf (ningle:route *app* "/auth/credentials" :method :post) + #'handle-auth-create-route) + +(setf (ningle:route *app* "/auth/credentials" :method :get) + #'handle-auth-list-route) + +(setf (ningle:route *app* "/auth/credentials/:credential-id/rotate" :method :post) + #'handle-auth-rotate-route) + +(setf (ningle:route *app* "/auth/credentials/:credential-id/revoke" :method :post) + #'handle-auth-revoke-route) + +(setf (ningle:route *app* "/auth/credentials/:credential-id/disable" :method :post) + #'handle-auth-disable-route) + +(setf (ningle:route *app* "/auth/context" :method :get) + #'handle-auth-context-route) diff --git a/source/frontends/http-auth.lisp b/source/frontends/http-auth.lisp new file mode 100644 index 00000000..f650308e --- /dev/null +++ b/source/frontends/http-auth.lisp @@ -0,0 +1,207 @@ +(in-package :star.frontends.http-api) + +(defun env-header-key (name) + (intern + (format nil "HTTP-~a" + (string-upcase name)) + :keyword)) + +(defun env-header-value (env name) + (or (getf env (env-header-key name)) + (let ((headers (getf env :headers))) + (cond + ((hash-table-p headers) + (or (gethash (string-downcase name) headers) + (gethash name headers))) + ((and (listp headers) + (consp (first headers))) + (cdr (assoc name headers :test #'string-equal))) + (t nil))))) + +(defun bounded-correlation-id-p (value) + (and (stringp value) + (<= 1 (length value) 128) + (every (lambda (character) + (or (alphanumericp character) + (find character "-_.:"))) + value))) + +(defun request-correlation-id-from-env (env) + (let ((provided (env-header-value env "X-Correlation-ID"))) + (if (bounded-correlation-id-p provided) + provided + (new-correlation-id)))) + +(defun parse-request-timeout-ms (env) + (let ((raw (env-header-value env "X-Request-Timeout-Ms"))) + (if raw + (handler-case + (let ((value (parse-integer raw :junk-allowed nil))) + (if (<= 1 value star:*auth-max-request-timeout-ms*) + value + star:*auth-default-request-timeout-ms*)) + (error () star:*auth-default-request-timeout-ms*)) + star:*auth-default-request-timeout-ms*))) + +(defun request-deadline-from-env (env) + (+ (get-universal-time) + (ceiling (parse-request-timeout-ms env) 1000))) + +(defun public-auth-path-p (path) + (member path star:*auth-public-paths* :test #'string=)) + +(defun development-security-context (correlation-id deadline) + (star.auth::%make-request-security-context + :principal + (star.auth::%make-request-principal + :id "development-bypass" + :type "administrator" + :scopes (list "admin") + :credential-id "development-bypass") + :correlation-id correlation-id + :deadline deadline + :authenticated-at (get-universal-time))) + +(defun authenticate-request-env (env correlation-id deadline) + (let ((mode (string-downcase star:*auth-mode*))) + (cond + ((string= mode "api-key") + (star.auth:authenticate-authorization-header + (env-header-value env "Authorization") + correlation-id + deadline)) + ((and (string= mode "disabled") + star:*auth-dev-bypass*) + (development-security-context correlation-id deadline)) + (t + (star.auth:signal-authentication-failure))))) + +(defun authentication-error-response (correlation-id) + (list + 401 + (list :content-type "application/json" + :cache-control "no-store" + :x-correlation-id correlation-id + :www-authenticate "Bearer realm=\"starintel\"") + (list + (jsown:to-json + (jsown:new-js + ("status" "error") + ("code" "invalid_credential") + ("msg" "Authentication failed") + ("correlation_id" correlation-id)))))) + +(defun append-response-headers (response headers) + (if (and response (listp response) (second response)) + (list (first response) + (append (second response) headers) + (third response)) + response)) + +(defun authentication-middleware (app) + (lambda (env) + (let* ((path (or (getf env :path-info) "/")) + (method (getf env :request-method)) + (correlation-id (request-correlation-id-from-env env)) + (deadline (request-deadline-from-env env))) + (handler-case + (let* ((context + (unless (or (eq method :options) + (public-auth-path-p path)) + (authenticate-request-env + env correlation-id deadline))) + (star.auth:*request-security-context* context) + (*http-correlation-id* correlation-id) + (response (lack.component:call app env))) + (append-response-headers + response + (list :x-correlation-id correlation-id))) + (star.auth:authentication-error () + (authentication-error-response correlation-id)))))) + +(defun configured-origin-allowed-p (origin) + (and (stringp origin) + (member origin + star:*http-cors-allowed-origins* + :test #'string=))) + +(defun cors-headers-for-origin (origin) + (when (configured-origin-allowed-p origin) + (list :access-control-allow-origin origin + :access-control-allow-methods star:*http-cors-allowed-methods* + :access-control-allow-headers star:*http-cors-allowed-headers* + :access-control-max-age "600" + :vary "Origin"))) + +(defun cors-middleware (app) + "Apply configured credential-safe CORS. Wildcard origins are never emitted." + (lambda (env) + (let* ((method (getf env :request-method)) + (origin (env-header-value env "Origin")) + (headers (cors-headers-for-origin origin))) + (if (eq method :options) + (if headers + (list 204 + (append (list :content-type "text/plain") headers) + (list "")) + (list 403 + (list :content-type "application/json") + (list + (jsown:to-json + (jsown:new-js + ("status" "error") + ("code" "cors_origin_denied") + ("msg" "Origin is not allowed")))))) + (append-response-headers + (lack.component:call app env) + headers))))) + +(defmacro with-http-boundary (() &body body) + `(let ((*http-correlation-id* + (or *http-correlation-id* (new-correlation-id)))) + (set-default-headers) + (set-correlation-id-header) + (handler-case + (progn ,@body) + (http-input-error (condition) + (log:warn "HTTP input rejected correlation=~a code=~a: ~a" + (current-correlation-id) + (http-input-error-code condition) + condition) + (respond-http-input-error condition)) + (bt:timeout (condition) + (log:error "HTTP operation timed out correlation=~a: ~a" + (current-correlation-id) + condition) + (setf (lack.response:response-status *response*) 504) + (status-msg "Request deadline exceeded" + 'error + :code "request_timeout")) + (error (condition) + (log:error "HTTP internal error correlation=~a: ~a" + (current-correlation-id) + condition) + (setf (lack.response:response-status *response*) 500) + (status-msg "Internal Server Error" + 'error + :code "internal_error"))))) + +(defun request-principal (&optional request) + (declare (ignore request)) + (or (star.auth:current-principal-id) + "anonymous")) + +(defun require-administrator-context () + (unless (star.auth:administrator-principal-p) + (signal-http-input-error + 403 + "access_denied" + "Access denied")) + star.auth:*request-security-context*) + +(setf *cors-headers* nil) +(setf *server* + (lack:builder + :accesslog + (cors-middleware + (authentication-middleware *app*)))) diff --git a/source/frontends/http-boundary-core.lisp b/source/frontends/http-boundary-core.lisp new file mode 100644 index 00000000..1ae3dd67 --- /dev/null +++ b/source/frontends/http-boundary-core.lisp @@ -0,0 +1,300 @@ +(in-package :star.frontends.http-api) + +(defparameter +http-max-body-bytes+ (* 1024 1024)) +(defparameter +http-max-query-limit+ 100) + +(defvar *http-correlation-id* nil) + +(define-condition http-input-error (error) + ((status + :initarg :status + :reader http-input-error-status) + (code + :initarg :code + :reader http-input-error-code) + (message + :initarg :message + :reader http-input-error-message) + (info + :initarg :info + :initform nil + :reader http-input-error-info)) + (:report + (lambda (condition stream) + (format stream "~a" (http-input-error-message condition))))) + +(defun new-correlation-id () + (cms-ulid:ulid)) + +(defun current-correlation-id () + (or *http-correlation-id* + (new-correlation-id))) + +(defun set-correlation-id-header () + (setf (lack.response:response-headers *response*) + (append (lack.response:response-headers *response*) + (list :x-correlation-id (current-correlation-id))))) + +(defun status-msg (msg status &key info traceback code) + "Return a client-safe status envelope. TRACEBACK is intentionally ignored." + (declare (ignore traceback)) + (let ((json (jsown:new-js + ("msg" msg) + ("status" (string-downcase (symbol-name status))) + ("correlation_id" (current-correlation-id))))) + (when code + (setf (jsown:val json "code") code)) + (when info + (setf (jsown:val json "info") info)) + (jsown:to-json json))) + +(defun signal-http-input-error (status code message &optional info) + (error 'http-input-error + :status status + :code code + :message message + :info info)) + +(defun respond-http-input-error (condition) + (setf (lack.response:response-status *response*) + (http-input-error-status condition)) + (status-msg (http-input-error-message condition) + 'error + :code (http-input-error-code condition) + :info (http-input-error-info condition))) + +(defmacro with-http-boundary (() &body body) + `(let ((*http-correlation-id* (new-correlation-id))) + (set-default-headers) + (set-correlation-id-header) + (handler-case + (progn ,@body) + (http-input-error (condition) + (log:warn "HTTP input rejected correlation=~a code=~a: ~a" + (current-correlation-id) + (http-input-error-code condition) + condition) + (respond-http-input-error condition)) + (bt:timeout (condition) + (log:error "HTTP operation timed out correlation=~a: ~a" + (current-correlation-id) + condition) + (setf (lack.response:response-status *response*) 504) + (status-msg "Request deadline exceeded" + 'error + :code "request_timeout")) + (error (condition) + (log:error "HTTP internal error correlation=~a: ~a" + (current-correlation-id) + condition) + (setf (lack.response:response-status *response*) 500) + (status-msg "Internal Server Error" + 'error + :code "internal_error"))))) + +(defun json-object-p (value) + (and (consp value) + (eq (car value) :obj))) + +(defun json-array-p (value) + (or (null value) + (and (listp value) + (not (json-object-p value))))) + +(defun request-content-type (&optional (request (ningle:context :request))) + (or (ignore-errors (lack.request:request-content-type request)) + (getf (lack.request:request-env request) :content-type))) + +(defun json-content-type-p (content-type) + (when (stringp content-type) + (let ((normalized (string-downcase content-type))) + (or (search "application/json" normalized) + (search "+json" normalized))))) + +(defun request-body-octets (&optional (request (ningle:context :request))) + (let ((content (lack.request:request-content request))) + (etypecase content + ((simple-array (unsigned-byte 8) (*)) content) + (string (babel:string-to-octets content :encoding :utf-8)) + (vector content) + (null #())))) + +(defun parse-json-octets (octets content-type + &key (max-bytes +http-max-body-bytes+)) + (unless (json-content-type-p content-type) + (signal-http-input-error + 415 + "unsupported_media_type" + "Content-Type must be application/json")) + (when (> (length octets) max-bytes) + (signal-http-input-error + 413 + "request_body_too_large" + "Request body exceeds the configured limit" + (jsown:new-js ("maximum_bytes" max-bytes)))) + (handler-case + (let ((text (babel:octets-to-string octets :encoding :utf-8))) + ;; Validate with the packaged standards-oriented parser, then retain + ;; JSOWN as the service's existing internal document representation. + (yason:parse text) + (jsown:parse text)) + (error () + (signal-http-input-error + 400 + "malformed_json" + "Request body contains malformed JSON")))) + +(defun parse-json-request (&key (max-bytes +http-max-body-bytes+)) + (let* ((request (ningle:context :request)) + (content-type (request-content-type request)) + (octets (request-body-octets request))) + (parse-json-octets octets content-type :max-bytes max-bytes))) + +(defun require-json-object (value) + (unless (json-object-p value) + (signal-http-input-error + 400 + "json_object_required" + "Request body must be a JSON object")) + value) + +(defun require-json-array (value) + (unless (json-array-p value) + (signal-http-input-error + 400 + "json_array_required" + "Request body must be a JSON array")) + value) + +(defun non-empty-string-p (value) + (and (stringp value) + (plusp (length value)))) + +(defun require-document-string (document field &key index) + (let ((value (jsown:val-safe document field))) + (unless (non-empty-string-p value) + (signal-http-input-error + 422 + "invalid_document" + (if index + (format nil "Document at index ~d requires a non-empty ~a field" + index field) + (format nil "Document requires a non-empty ~a field" field)) + (jsown:new-js ("field" field) + ("index" (or index :null))))) + value)) + +(defun validate-schema-version (document &key index) + (let ((version (jsown:val-safe document "version")) + (expected starintel:+starintel-doc-version+)) + (unless version + (signal-http-input-error + 422 + "schema_version_required" + (if index + (format nil "Document at index ~d requires a version field" index) + "Document requires a version field"))) + (unless (string= (princ-to-string version) + (princ-to-string expected)) + (signal-http-input-error + 422 + "unsupported_schema_version" + "Document schema version is not supported" + (jsown:new-js ("expected" (princ-to-string expected)) + ("received" (princ-to-string version)) + ("index" (or index :null))))))) + +(defun validate-document-input (document &key path-dtype index) + (unless (json-object-p document) + (signal-http-input-error + 422 + "invalid_document" + (if index + (format nil "Document at index ~d must be a JSON object" index) + "Document must be a JSON object"))) + (require-document-string document "_id" :index index) + (require-document-string document "dataset" :index index) + (let ((dtype (require-document-string document "dtype" :index index))) + (when (and path-dtype + (not (string-equal dtype path-dtype))) + (signal-http-input-error + 422 + "dtype_mismatch" + "Document dtype does not match the route dtype" + (jsown:new-js ("path_dtype" path-dtype) + ("document_dtype" dtype)))) + (validate-schema-version document :index index) + document)) + +(defun query-value (params name) + (or (cdr (assoc name params :test #'string=)) + (cdr (assoc (intern (string-upcase name) :keyword) + params + :test #'eq)))) + +(defun require-query-string (params name) + (let ((value (query-value params name))) + (unless (non-empty-string-p value) + (signal-http-input-error + 400 + "missing_query_parameter" + (format nil "Query parameter ~a is required" name))) + value)) + +(defun bounded-query-integer (params name &key default (minimum 0) + (maximum +http-max-query-limit+)) + (let ((raw (query-value params name))) + (when (and (null raw) default) + (return-from bounded-query-integer default)) + (unless raw + (signal-http-input-error + 400 + "missing_query_parameter" + (format nil "Query parameter ~a is required" name))) + (let ((value + (handler-case + (parse-integer raw :junk-allowed nil) + (error () + (signal-http-input-error + 400 + "invalid_query_parameter" + (format nil "Query parameter ~a must be an integer" name)))))) + (unless (<= minimum value maximum) + (signal-http-input-error + 400 + "query_parameter_out_of_range" + (format nil "Query parameter ~a must be between ~d and ~d" + name minimum maximum))) + value))) + +(defun request-header-value (headers name) + (cond + ((hash-table-p headers) + (or (gethash (string-downcase name) headers) + (gethash name headers))) + ((and (listp headers) (consp (car headers))) + (cdr (assoc name headers :test #'string-equal))) + (t nil))) + +(defun request-principal (&optional (request (ningle:context :request))) + (let* ((headers (ignore-errors (lack.request:request-headers request))) + (authorization (request-header-value headers "authorization")) + (remote-address + (getf (lack.request:request-env request) :remote-addr))) + (cond + (authorization + (format nil "auth-~x" (sxhash authorization))) + ((non-empty-string-p remote-address) + (format nil "remote-~a" remote-address)) + (t "anonymous")))) + +(eval-when (:compile-toplevel :load-toplevel :execute) + (export '(http-input-error + http-input-error-status + http-input-error-code + json-object-p + json-array-p + parse-json-octets + validate-document-input + bounded-query-integer) + :star.frontends.http-api)) diff --git a/source/frontends/http-boundary-routes.lisp b/source/frontends/http-boundary-routes.lisp new file mode 100644 index 00000000..8810dbd7 --- /dev/null +++ b/source/frontends/http-boundary-routes.lisp @@ -0,0 +1,111 @@ +(in-package :star.frontends.http-api) + +(defun handle-new-document-route (params) + (with-http-boundary () + (let* ((path-dtype (query-value params "dtype")) + (document (require-json-object (parse-json-request)))) + (unless (non-empty-string-p path-dtype) + (signal-http-input-error + 400 + "missing_path_parameter" + "Route dtype is required")) + (validate-document-input document :path-dtype path-dtype) + (publish-document document) + (jsown:to-json document)))) + +(defun handle-new-target-route (params) + (with-http-boundary () + (let* ((actor (query-value params "actor")) + (document (require-json-object (parse-json-request)))) + (unless (non-empty-string-p actor) + (signal-http-input-error + 400 + "missing_path_parameter" + "Route actor is required")) + (setf (jsown:val document "dtype") "target" + (jsown:val document "actor") actor) + (validate-document-input document :path-dtype "target") + (publish-document document) + (jsown:to-json document)))) + +(defun handle-bulk-route (params) + (declare (ignore params)) + (with-http-boundary () + (let* ((documents (require-json-array (parse-json-request))) + (document-count (length documents))) + (when (> document-count star:*bulk-max-documents*) + (signal-http-input-error + 413 + "bulk_document_limit_exceeded" + "Bulk request exceeds the configured document limit" + (jsown:new-js ("requested" document-count) + ("maximum" star:*bulk-max-documents*)))) + (loop for document in documents + for index from 0 + do (validate-document-input document :index index)) + (if (eq :inline (bulk-request-mode document-count)) + (process-inline-bulk documents) + (let ((job (submit-bulk-ingest-job + documents + (request-principal)))) + (setf (lack.response:response-status *response*) 202) + (jsown:to-json + (jsown:new-js + ("status" "accepted") + ("job_id" (bulk-ingest-job-id job)) + ("total" document-count) + ("status_url" + (format nil "/documents/bulk/~a" + (bulk-ingest-job-id job))) + ("correlation_id" (current-correlation-id))))))))) + +(defun handle-bulk-status-route (params) + (with-http-boundary () + (let* ((job-id (query-value params "job-id")) + (job (and job-id + (bt:with-lock-held (*bulk-ingest-lock*) + (gethash job-id *bulk-ingest-jobs*))))) + (unless job + (signal-http-input-error + 404 + "bulk_job_not_found" + "Bulk ingest job was not found")) + (jsown:to-json (bulk-job-info-json job))))) + +(defun handle-search-route (params) + (with-http-boundary () + (let ((q (require-query-string params "q")) + (limit (bounded-query-integer + params "limit" :default 25 :minimum 1))) + (couchdb-handler (client *couchdb-pool*) + (let* ((db star:*couchdb-default-database*) + (bookmark (query-value params "bookmark")) + (sort (query-value params "sort")) + (query (jsown:new-js + ("q" q) + ("limit" limit) + ("include_docs" t)))) + (when sort + (setf (jsown:val query "sort") sort)) + (when bookmark + (setf (jsown:val query "bookmark") bookmark)) + (cl-couch:fts-search client + (jsown:to-json query) + db + "search" + "fts")))))) + +(setf (ningle:route *app* "/new/document/:dtype" :method :post) + #'handle-new-document-route) + +(setf (ningle:route *app* "/new/target/:actor" :method :post) + #'handle-new-target-route) + +(setf (ningle:route *app* "/documents/bulk" :method :post) + #'handle-bulk-route) + +(setf (ningle:route *app* "/documents/bulk/:job-id" :method :get) + #'handle-bulk-status-route) + +(setf (ningle:route *app* "/search" :method :get) + #'handle-search-route) diff --git a/source/frontends/http-bulk-jobs.lisp b/source/frontends/http-bulk-jobs.lisp new file mode 100644 index 00000000..ba0e85a4 --- /dev/null +++ b/source/frontends/http-bulk-jobs.lisp @@ -0,0 +1,257 @@ +(in-package :star.frontends.http-api) + +(defparameter +bulk-inline-document-limit+ 10) +(defparameter +bulk-inline-deadline-seconds+ 2) +(defparameter +bulk-max-pending-jobs+ 32) +(defparameter +bulk-max-pending-per-principal+ 4) +(defparameter +bulk-worker-count+ 4) + +(defvar *bulk-ingest-workers* nil) +(defvar *bulk-ingest-worker-system* nil) +(defvar *bulk-ingest-worker-index* 0) +(defvar *bulk-ingest-jobs* (make-hash-table :test #'equal)) +(defvar *bulk-pending-jobs* 0) +(defvar *bulk-pending-by-principal* (make-hash-table :test #'equal)) +(defvar *bulk-ingest-lock* (bt:make-lock "bulk-ingest-state")) +(defvar *service-call-context* nil) + +(defstruct (bulk-ingest-job + (:constructor make-bulk-ingest-job + (&key id principal documents correlation-id service-context + submitted-at (status :queued) (succeeded 0) (failed 0) + error-code))) + id + principal + documents + correlation-id + service-context + submitted-at + status + succeeded + failed + error-code) + +(defun bulk-job-info-json (job) + (jsown:new-js + ("job_id" (bulk-ingest-job-id job)) + ("status" (string-downcase + (symbol-name (bulk-ingest-job-status job)))) + ("total" (length (bulk-ingest-job-documents job))) + ("succeeded" (bulk-ingest-job-succeeded job)) + ("failed" (bulk-ingest-job-failed job)) + ("correlation_id" (bulk-ingest-job-correlation-id job)))) + +(defun release-bulk-job-slot (job) + (bt:with-lock-held (*bulk-ingest-lock*) + (decf *bulk-pending-jobs*) + (let* ((principal (bulk-ingest-job-principal job)) + (count (gethash principal *bulk-pending-by-principal* 0))) + (if (> count 1) + (setf (gethash principal *bulk-pending-by-principal*) + (1- count)) + (remhash principal *bulk-pending-by-principal*))))) + +(defun comma-separated-scopes (scopes) + (format nil "~{~a~^,~}" scopes)) + +(defun service-context-properties (dtype context) + (let ((properties (list (cons :type dtype)))) + (when context + (push (cons :correlation-id + (star.auth:service-call-context-correlation-id context)) + properties) + (push + (cons :headers + (list + (cons "x-star-principal-id" + (star.auth:service-call-context-principal-id context)) + (cons "x-star-principal-type" + (star.auth:service-call-context-principal-type context)) + (cons "x-star-credential-id" + (star.auth:service-call-context-credential-id context)) + (cons "x-star-scopes" + (comma-separated-scopes + (star.auth:service-call-context-scopes context))) + (cons "x-star-deadline" + (princ-to-string + (star.auth:service-call-context-deadline context))))) + properties)) + (nreverse properties))) + +(defun current-publish-service-context () + (or *service-call-context* + (star.auth:current-service-call-context))) + +(defun publish-document (document) + (let* ((dtype (jsown:val document "dtype")) + (routing-key (format nil star.rabbit:+ingest-fmt-key+ dtype)) + (context (current-publish-service-context))) + (star.actors:publish + star.actors:*producer-agent* + :body (jsown:to-json document) + :routing-key routing-key + :properties (service-context-properties dtype context)))) + +(defun execute-bulk-job (job &key (publish-fn #'publish-document)) + (setf (bulk-ingest-job-status job) :running) + (let ((*service-call-context* (bulk-ingest-job-service-context job))) + (handler-case + (progn + (loop for document in (bulk-ingest-job-documents job) + do (handler-case + (progn + (funcall publish-fn document) + (incf (bulk-ingest-job-succeeded job))) + (error (condition) + (log:error + "Bulk publish failed job=~a correlation=~a: ~a" + (bulk-ingest-job-id job) + (bulk-ingest-job-correlation-id job) + condition) + (incf (bulk-ingest-job-failed job))))) + (setf (bulk-ingest-job-status job) + (if (zerop (bulk-ingest-job-failed job)) + :completed + :completed-with-errors))) + (error (condition) + (log:error "Bulk job failed job=~a correlation=~a: ~a" + (bulk-ingest-job-id job) + (bulk-ingest-job-correlation-id job) + condition) + (setf (bulk-ingest-job-status job) :failed + (bulk-ingest-job-error-code job) "bulk_job_failed")))) + job) + +(defun bulk-worker-handler (job) + (unwind-protect + (execute-bulk-job job) + (release-bulk-job-slot job))) + +(defun make-bulk-worker (system index dispatcher) + (sento.actor-context:actor-of + system + :name (format nil "bulk-ingest-worker-~d" index) + :dispatcher dispatcher + :receive #'bulk-worker-handler)) + +(defun start-bulk-ingest-workers (&optional (system star.actors:*sys*)) + (unless system + (return-from start-bulk-ingest-workers nil)) + (bt:with-lock-held (*bulk-ingest-lock*) + (unless (and *bulk-ingest-workers* + (eq *bulk-ingest-worker-system* system)) + (setf *bulk-ingest-workers* + (loop for index below +bulk-worker-count+ + collect + (handler-case + (make-bulk-worker system index :pinned) + (error () + (make-bulk-worker system index :shared)))) + *bulk-ingest-worker-system* system + *bulk-ingest-worker-index* 0))) + *bulk-ingest-workers*) + +(defun start-bulk-ingest-workers-hook () + (start-bulk-ingest-workers star.actors:*sys*)) + +(nhooks:add-hook star:*actors-start-hook* #'start-bulk-ingest-workers-hook) + +(defun submit-bulk-ingest-job (documents principal + &key + (tell-fn #'sento.actor:tell) + (ensure-workers-fn + #'start-bulk-ingest-workers)) + "Reserve bounded queue capacity and enqueue one authenticated job." + (unless (funcall ensure-workers-fn) + (signal-http-input-error + 503 + "bulk_service_unavailable" + "Bulk ingest service is not available")) + (let ((job + (make-bulk-ingest-job + :id (cms-ulid:ulid) + :principal principal + :documents documents + :correlation-id (current-correlation-id) + :service-context (star.auth:current-service-call-context) + :submitted-at (get-universal-time)))) + (bt:with-lock-held (*bulk-ingest-lock*) + (when (>= *bulk-pending-jobs* +bulk-max-pending-jobs+) + (signal-http-input-error + 429 + "bulk_queue_full" + "Bulk ingest queue is full")) + (let ((principal-pending + (gethash principal *bulk-pending-by-principal* 0))) + (when (>= principal-pending +bulk-max-pending-per-principal+) + (signal-http-input-error + 429 + "principal_bulk_quota_exceeded" + "Principal has too many pending bulk jobs")) + (incf *bulk-pending-jobs*) + (setf (gethash principal *bulk-pending-by-principal*) + (1+ principal-pending))) + (setf (gethash (bulk-ingest-job-id job) *bulk-ingest-jobs*) job) + (let ((worker + (nth (mod *bulk-ingest-worker-index* + (length *bulk-ingest-workers*)) + *bulk-ingest-workers*))) + (incf *bulk-ingest-worker-index*) + (handler-case + (funcall tell-fn worker job) + (error (condition) + (decf *bulk-pending-jobs*) + (let ((current + (gethash principal *bulk-pending-by-principal* 0))) + (if (> current 1) + (setf (gethash principal *bulk-pending-by-principal*) + (1- current)) + (remhash principal *bulk-pending-by-principal*))) + (remhash (bulk-ingest-job-id job) *bulk-ingest-jobs*) + (log:error "Failed to enqueue bulk job correlation=~a: ~a" + (current-correlation-id) + condition) + (signal-http-input-error + 503 + "bulk_enqueue_failed" + "Bulk ingest job could not be queued"))))) + job)) + +(defun bulk-request-mode (document-count) + (if (<= document-count +bulk-inline-document-limit+) + :inline + :async)) + +(defun process-inline-bulk (documents) + (let ((succeeded 0) + (failed 0)) + (bt:with-timeout (+bulk-inline-deadline-seconds+) + (loop for document in documents + do (handler-case + (progn + (publish-document document) + (incf succeeded)) + (error (condition) + (log:error "Inline bulk publish failed correlation=~a: ~a" + (current-correlation-id) + condition) + (incf failed))))) + (jsown:to-json + (jsown:new-js + ("total" (length documents)) + ("succeeded" succeeded) + ("failed" failed) + ("correlation_id" (current-correlation-id)))))) + +(eval-when (:compile-toplevel :load-toplevel :execute) + (export '(bulk-request-mode + execute-bulk-job + submit-bulk-ingest-job + bulk-ingest-job + bulk-ingest-job-principal + bulk-ingest-job-service-context + bulk-ingest-job-status + bulk-ingest-job-succeeded + bulk-ingest-job-failed + service-context-properties) + :star.frontends.http-api)) diff --git a/source/gserver-settings.lisp b/source/gserver-settings.lisp index 1d2f0e0b..ba35df57 100644 --- a/source/gserver-settings.lisp +++ b/source/gserver-settings.lisp @@ -1,49 +1,126 @@ (in-package :star) -;;; Version info + +(defun read-secret-file (path) + (when (and path (probe-file path)) + (string-trim '(#\Space #\Tab #\Newline #\Return) + (uiop:read-file-string path)))) + +(defun environment-secret (value-variable file-variable) + (or (uiop:getenv value-variable) + (read-secret-file (uiop:getenv file-variable)))) + +(defun environment-boolean (name &optional default) + (let ((value (uiop:getenv name))) + (if value + (member (string-downcase value) + '("1" "true" "yes" "on") + :test #'string=) + default))) + +(defun environment-integer (name default) + (let ((value (uiop:getenv name))) + (if value + (parse-integer value :junk-allowed nil) + default))) + +(defun split-comma-setting (value) + (when value + (loop with start = 0 + for position = (position #\, value :start start) + for item = (string-trim '(#\Space #\Tab) + (subseq value start position)) + when (plusp (length item)) + collect item + while position + do (setf start (1+ position))))) + +;;; Version info (defparameter *star-server-version* "0.0.1") -;;;; ** Gserver Settings -;;;; *** Couchdb -(defparameter *couchdb-host* (or (uiop:getenv "COUCHDB_HOST") "127.0.0.1") "The Couchdb host to use. -Defaults to using ENV var $COUCHDB_HOST if set, or localhost ") -(defparameter *couchdb-port* 5984 "The Couchdb port to use. Defaults to 5984") -(defparameter *couchdb-default-database* (or (uiop:getenv "COUCHDB_DATABASE") "starintel") "the default database name to use.") - -(defparameter *couchdb-auth-database* "starintel-gserver-auth") -(defparameter *couchdb-scheme* "http" "what http scheme to use. set to http or https") -(defparameter *couchdb-user* (or (uiop:getenv "COUCHDB_USER") "admin") "couchdb user") -(defparameter *couchdb-password* (uiop:getenv "COUCHDB_PASSWORD") "couchdb user password") -;;;; By Default the views in starintel-gserver/views will be installed, but you can append your own to this setting to have it created at startup. -(defparameter *couchdb-views* (let ((files (uiop:directory-files (uiop:merge-pathnames* "views/" (asdf:system-source-directory :starintel-gserver))))) - (loop for file in files - collect (with-open-file (str file) - (let ((content (make-string (file-length str)))) - (read-sequence content str) - content)))) - "List of views to install into couchdb.") - -;;;; *** HTTP API -(defparameter *http-api-address* (or (uiop:getenv "HTTP_API_LISTEN_ADDRESS") "localhost") "the listen address") -(defparameter *http-api-port* 5000 "the port the api server listen on") -(defparameter *http-api-base-path* "/api" "the base url to use for the api endpoint") -(defparameter *http-cert-file* nil "path to the http api cert providing https") -(defparameter *http-key-file* nil "path to the http cert providing https") -(defparameter *http-scheme* 'http "use https or not.") - -;;;; *** RabbitMQ -(defparameter *rabbit-address* (or (uiop:getenv "RABBITMQ_ADDRESS") "localhost") "The address rabbitmq is running on.") -(defparameter *rabbit-port* 5672 "The port that rabbitmq is listening on.") -(defparameter *rabbit-user* (or (uiop:getenv "RABBITMQ_USER") "guest") "the username for rabbitmq") -(defparameter *rabbit-password* (uiop:getenv "RABBITMQ_PASSWORD") "the password for the rabbitmq user.") -(defparameter *slynk-port* 4009 "Port to use for SLYNK remote debugging") - -;;;; *** Actors -;;;; Hooks are implemented Via nhooks you can read documentation here for how to add hooks. https://github.com/atlas-engineer/nhooks -(defparameter *actors-start-hook* (make-instance 'nhooks:hook-void) "Actor startup hook.") -;;;; *** Patterns -;;;; Patterns are -(defparameter *document-patterns* () "A List of document patterns created by defpattern") -(defparameter *ingest-workers* 4 "Number of workers for handling documents, set to 4 by default.") -;;;; *** actor event log -(defparameter *couchdb-event-log-database* "starintel-event-source" "The name of the database to be used for event logs.") -;;;; *** Bulk operations -(defparameter *bulk-max-documents* 500 "Maximum number of documents allowed in a single bulk operation.") + +;;;; CouchDB +(defparameter *couchdb-host* + (or (uiop:getenv "COUCHDB_HOST") "127.0.0.1")) +(defparameter *couchdb-port* + (environment-integer "COUCHDB_PORT" 5984)) +(defparameter *couchdb-default-database* + (or (uiop:getenv "COUCHDB_DATABASE") "starintel")) +(defparameter *couchdb-auth-database* + (or (uiop:getenv "STAR_AUTH_DATABASE") "starintel-gserver-auth")) +(defparameter *couchdb-scheme* + (or (uiop:getenv "COUCHDB_SCHEME") "http")) +(defparameter *couchdb-user* + (or (uiop:getenv "COUCHDB_USER") "admin")) +(defparameter *couchdb-password* + (environment-secret "COUCHDB_PASSWORD" "COUCHDB_PASSWORD_FILE")) + +(defparameter *couchdb-views* + (let ((files + (uiop:directory-files + (uiop:merge-pathnames* + "views/" + (asdf:system-source-directory :starintel-gserver))))) + (loop for file in files + collect + (with-open-file (stream file) + (let ((content (make-string (file-length stream)))) + (read-sequence content stream) + content)))) + "View documents installed into the intelligence database at startup.") + +;;;; HTTP API +(defparameter *http-api-address* + (or (uiop:getenv "HTTP_API_LISTEN_ADDRESS") "localhost")) +(defparameter *http-api-port* + (environment-integer "HTTP_API_PORT" 5000)) +(defparameter *http-api-base-path* "/api") +(defparameter *http-cert-file* nil) +(defparameter *http-key-file* nil) +(defparameter *http-scheme* 'http) +(defparameter *http-cors-allowed-origins* + (split-comma-setting (uiop:getenv "STAR_AUTH_ALLOWED_ORIGINS"))) +(defparameter *http-cors-allowed-methods* + "GET, POST, PUT, PATCH, DELETE, OPTIONS") +(defparameter *http-cors-allowed-headers* + "Content-Type, Authorization, X-Correlation-ID, X-Request-Timeout-Ms, X-Star-Bootstrap-Secret") + +;;;; HTTP authentication +(defparameter *auth-mode* + (or (uiop:getenv "STAR_AUTH_MODE") "api-key")) +(defparameter *auth-pepper* + (environment-secret "STAR_AUTH_PEPPER" "STAR_AUTH_PEPPER_FILE")) +(defparameter *auth-bootstrap-secret* + (environment-secret + "STAR_AUTH_BOOTSTRAP_SECRET" + "STAR_AUTH_BOOTSTRAP_SECRET_FILE")) +(defparameter *auth-dev-bypass* + (not (null (environment-boolean "STAR_AUTH_DEV_BYPASS" nil)))) +(defparameter *auth-key-secret-bytes* 32) +(defparameter *auth-salt-bytes* 16) +(defparameter *auth-rotation-overlap-max-seconds* + (environment-integer "STAR_AUTH_MAX_ROTATION_OVERLAP_SECONDS" 86400)) +(defparameter *auth-default-request-timeout-ms* + (environment-integer "STAR_AUTH_DEFAULT_REQUEST_TIMEOUT_MS" 30000)) +(defparameter *auth-max-request-timeout-ms* + (environment-integer "STAR_AUTH_MAX_REQUEST_TIMEOUT_MS" 600000)) +(defparameter *auth-public-paths* + '("/health" "/" "/auth/bootstrap")) + +;;;; RabbitMQ +(defparameter *rabbit-address* + (or (uiop:getenv "RABBITMQ_ADDRESS") "localhost")) +(defparameter *rabbit-port* + (environment-integer "RABBITMQ_PORT" 5672)) +(defparameter *rabbit-user* + (or (uiop:getenv "RABBITMQ_USER") "guest")) +(defparameter *rabbit-password* + (environment-secret "RABBITMQ_PASSWORD" "RABBITMQ_PASSWORD_FILE")) +(defparameter *slynk-port* 4009) + +;;;; Actors and patterns +(defparameter *actors-start-hook* (make-instance 'nhooks:hook-void)) +(defparameter *document-patterns* nil) +(defparameter *ingest-workers* 4) + +;;;; Event log and bulk operations +(defparameter *couchdb-event-log-database* "starintel-event-source") +(defparameter *bulk-max-documents* 500) diff --git a/source/main.lisp b/source/main.lisp index 8a004ca7..703447b3 100644 --- a/source/main.lisp +++ b/source/main.lisp @@ -1,6 +1,5 @@ (in-package :starintel-gserver) - (defun server/options () (list (clingon:make-option @@ -12,84 +11,61 @@ :env-vars '("STAR_SERVER_INIT_FILE") :key :init-value))) +(defun initialize-runtime (init-file) + (safe-load-init init-file) + (log:info "Creating ~a worker threads" star:*ingest-workers*) + (setf lparallel:*kernel* + (lparallel:make-kernel star:*ingest-workers*)) + (star.databases.couchdb:init-db) + (star.auth:initialize-auth-store) + (star.actors:start-actors + :rabbit-host *rabbit-address* + :rabbit-vhost "/" + :rabbit-port *rabbit-port* + :rabbit-user *rabbit-user* + :rabbit-password *rabbit-password*) + (star.frontends.http-api::start-http-api) + (star.rabbit:start-consumers) + (star.actors:start-event-consumer 2)) -(defun server/handler (cmd) - (let ((debugger (clingon:getopt cmd :debugger)) - (init-file (clingon:getopt cmd :init-value))) - - (safe-load-init init-file) - - (log:info (format nil "Creating ~a worker threads" star:*ingest-workers*)) - (setf lparallel:*kernel* (lparallel:make-kernel star:*ingest-workers*)) - (star.databases.couchdb:init-db) - (star.actors:start-actors :rabbit-host *rabbit-address* - :rabbit-vhost "/" - :rabbit-port *rabbit-port* - :rabbit-user *rabbit-user* - :rabbit-password *rabbit-password*) - (star.frontends.http-api::start-http-api) - (star.rabbit:start-consumers) - (star.actors:start-event-consumer 2)) - +(defun server/handler (command) + (initialize-runtime (clingon:getopt command :init-value)) (loop for thread in (bt:all-threads) - if (not (equal thread (bt:current-thread))) + unless (equal thread (bt:current-thread)) do (bt:join-thread thread))) - - (defun server/command () - "Start server" (clingon:make-command :name "start" - :description "start the server" + :description "Start the server" :authors '("nsaspy ") :license "GPL v3" :options (server/options) :handler #'server/handler)) - (defun main/commands () - (list - (server/command))) - -(defun main/handler (cmd) - "Print usage/exit" - (clingon:print-usage-and-exit cmd t)) + (list (server/command))) +(defun main/handler (command) + (clingon:print-usage-and-exit command t)) (defun main/command () - (clingon:make-command :name "star-server" - :version *star-server-version* - :description "Starintel unified API and document consuming service." - :authors '("nsaspy ") - :license "GPL v3" - :handler #'main/handler - :sub-commands (main/commands))) + (clingon:make-command + :name "star-server" + :version *star-server-version* + :description "StarIntel unified API and document-consuming service." + :authors '("nsaspy ") + :license "GPL v3" + :handler #'main/handler + :sub-commands (main/commands))) (defun start-debugger () (format t "Creating slynk server on port: ~a" star:*slynk-port*) (slynk:create-server :port star:*slynk-port*)) - - (defun main () - (let ((app (main/command))) - (clingon:run app))) - + (clingon:run (main/command))) (defun repl/main (init-file) - "Load the server from the repl" - - (safe-load-init init-file) - (log:info (format nil "Creating ~a worker threads" star:*ingest-workers*)) - (setf lparallel:*kernel* (lparallel:make-kernel star:*ingest-workers*)) - (star.databases.couchdb:init-db) - (star.actors:start-actors :rabbit-host *rabbit-address* - :rabbit-vhost "/" - :rabbit-port *rabbit-port* - :rabbit-user *rabbit-user* - :rabbit-password *rabbit-password*) - (star.frontends.http-api::start-http-api) - (star.rabbit:start-consumers) - (star.actors:start-event-consumer 2)) - + "Load and start the server from the REPL." + (initialize-runtime init-file)) diff --git a/source/package.lisp b/source/package.lisp index deada3cd..86bdad8b 100644 --- a/source/package.lisp +++ b/source/package.lisp @@ -1,7 +1,7 @@ ;; [[file:../source.org::*Namespace setup][Namespace setup:2]] -(uiop:define-package :starintel-gserver +(uiop:define-package :starintel-gserver (:nicknames :star) - (:use :cl) + (:use :cl) (:export #:*rabbit-password* #:*rabbit-user* @@ -13,12 +13,26 @@ #:*http-api-base-path* #:*http-api-port* #:*http-api-address* + #:*http-cors-allowed-origins* + #:*http-cors-allowed-methods* + #:*http-cors-allowed-headers* #:*couchdb-default-database* + #:*couchdb-auth-database* #:*couchdb-host* #:*couchdb-port* #:*couchdb-user* #:*couchdb-password* #:*couchdb-scheme* + #:*auth-mode* + #:*auth-pepper* + #:*auth-bootstrap-secret* + #:*auth-dev-bypass* + #:*auth-key-secret-bytes* + #:*auth-salt-bytes* + #:*auth-rotation-overlap-max-seconds* + #:*auth-default-request-timeout-ms* + #:*auth-max-request-timeout-ms* + #:*auth-public-paths* #:main #:reload #:start-debugger @@ -35,71 +49,151 @@ #:*bulk-max-documents* #:repl/main)) - -(uiop:define-package :star.databases.couchdb - (:use :cl-couch :cl :star #:lparallel) - (:export :init-db - :init-views - :init-event-db - :get-targets* - :get-view-docs - :query-view - :map-view-results - :get-neighbors - :search-fts - :sort-docs-by-date - :messages-by-user - :messages-by-platform - :messages-by-group - :social-posts-by-user - :social-posts-by-group - :by-channel - :export-by-dataset* - :count-by-dtype - :dataset-size - :total-documents-since - :orgs-by-country - :orgs-by-name - :persons-by-name - :persons-by-region - :relations-edges - :relations-incoming-count - :relations-outgoing-count - :targets-actor-counts - :targets-by-actor - :targets-target-count - :users-by-platform - :users-by-name - :as-json - :format-key - :from-json - :*couchdb-pool* - :groups - :lazy - :events-by-actor - :document-events - :target-events - :hosts-by-ip - :hosts-by-port - :hosts-by-service - :emails-by-email - :emails-by-domain - :emails-with-password - :domains-by-record - :domains-by-resolved-address - :networks-by-asn - :networks-by-org - :breaches-by-size - :urls-by-url - :urls-by-path - :urls-by-domain) - (:documentation "doc")) +(uiop:define-package :star.databases.couchdb + (:use :cl-couch :cl :star #:lparallel) + (:export + #:init-db + #:init-views + #:init-event-db + #:get-targets* + #:get-view-docs + #:query-view + #:map-view-results + #:get-neighbors + #:search-fts + #:sort-docs-by-date + #:messages-by-user + #:messages-by-platform + #:messages-by-group + #:social-posts-by-user + #:social-posts-by-group + #:by-channel + #:export-by-dataset* + #:count-by-dtype + #:dataset-size + #:total-documents-since + #:orgs-by-country + #:orgs-by-name + #:persons-by-name + #:persons-by-region + #:relations-edges + #:relations-incoming-count + #:relations-outgoing-count + #:targets-actor-counts + #:targets-by-actor + #:targets-target-count + #:users-by-platform + #:users-by-name + #:as-json + #:format-key + #:from-json + #:*couchdb-pool* + #:groups + #:lazy + #:events-by-actor + #:document-events + #:target-events + #:hosts-by-ip + #:hosts-by-port + #:hosts-by-service + #:emails-by-email + #:emails-by-domain + #:emails-with-password + #:domains-by-record + #:domains-by-resolved-address + #:networks-by-asn + #:networks-by-org + #:breaches-by-size + #:urls-by-url + #:urls-by-path + #:urls-by-domain) + (:documentation "CouchDB persistence and query helpers.")) ;; Namespace setup:2 ends here +(uiop:define-package :star.auth + (:use :cl) + (:export + #:+api-key-prefix+ + #:authentication-error + #:authentication-error-code + #:authentication-error-message + #:credential-lifecycle-error + #:credential-lifecycle-error-code + #:credential-lifecycle-error-message + #:request-principal + #:request-principal-id + #:request-principal-type + #:request-principal-scopes + #:request-principal-credential-id + #:request-security-context + #:request-security-context-principal + #:request-security-context-correlation-id + #:request-security-context-deadline + #:request-security-context-authenticated-at + #:service-call-context + #:service-call-context-principal-id + #:service-call-context-principal-type + #:service-call-context-credential-id + #:service-call-context-scopes + #:service-call-context-correlation-id + #:service-call-context-deadline + #:api-key-record + #:api-key-record-id + #:api-key-record-owner + #:api-key-record-principal-type + #:api-key-record-scopes + #:api-key-record-status + #:api-key-record-salt + #:api-key-record-verifier + #:api-key-record-created-at + #:api-key-record-expires-at + #:api-key-record-disabled-at + #:api-key-record-revoked-at + #:api-key-record-rotation-parent-id + #:api-key-record-superseded-by + #:api-key-record-overlap-expires-at + #:api-key-record-revision + #:credential-store + #:memory-credential-store + #:couchdb-credential-store + #:make-memory-credential-store + #:make-couchdb-credential-store + #:credential-store-get + #:credential-store-put + #:credential-store-update + #:credential-store-list + #:credential-store-count + #:*credential-store* + #:*request-security-context* + #:*auth-clock* + #:*verifier-compare-function* + #:auth-now + #:constant-time-octets= + #:constant-time-secret= + #:parse-api-key + #:bearer-token + #:signal-authentication-failure + #:authenticate-api-key + #:authenticate-authorization-header + #:current-request-principal + #:current-principal-id + #:current-service-call-context + #:scope-granted-p + #:administrator-principal-p + #:create-api-key + #:bootstrap-api-key + #:rotate-api-key + #:revoke-api-key + #:disable-api-key + #:api-key-metadata-json + #:list-api-key-metadata + #:validate-auth-configuration + #:initialize-auth-store)) + ;; [[file:../source.org::*Namespace setup][Namespace setup:3]] -(uiop:define-package :star.rabbit - (:use :cl :star.consumers :sento.actor) - (:documentation "Rabitmq namespace") +(uiop:define-package :star.rabbit + (:use :cl :star.consumers :sento.actor) + (:documentation "RabbitMQ namespace") (:export #:with-rabbit-send #:with-rabbit-recv @@ -127,9 +221,10 @@ #:+updates-topic-key+)) ;; Namespace setup:3 ends here -(uiop:define-package :star.actors - (:use :cl :star.databases.couchdb :sento.agent :sento.actor :sento.actor-system :sento.actor-context) - (:documentation "doc") +(uiop:define-package :star.actors + (:use :cl :star.databases.couchdb :sento.agent :sento.actor + :sento.actor-system :sento.actor-context) + (:documentation "Actor runtime namespace") (:export #:register-actor #:*targets* @@ -160,12 +255,12 @@ #:event-source-document #:event-id)) - ;; [[file:../source.org::*Namespace setup][Namespace setup:4]] -(uiop:define-package :starintel-gserver-http-api +(uiop:define-package :starintel-gserver-http-api (:nicknames :star.frontends.http-api) - (:use :cl :ningle :anypool :star.databases.couchdb :star) - (:documentation "simple http api.") + (:use :cl :ningle :anypool :star.databases.couchdb :star) + (:documentation "StarIntel HTTP API.") (:export + #:*app* #:*default-headers*)) ;; Namespace setup:4 ends here diff --git a/source/starintel-gserver.asd b/source/starintel-gserver.asd index fbe5ec47..b89e7fa9 100644 --- a/source/starintel-gserver.asd +++ b/source/starintel-gserver.asd @@ -1,82 +1,66 @@ (asdf:defsystem :starintel-gserver - :version "0.1.0" - :description "hackable/moddable starintel acess api." - :author "nsaspy@airmail.cc" - :license "GPL v3" + :version "0.1.0" + :description "Hackable StarIntel document and actor service." + :author "nsaspy@airmail.cc" + :license "GPL v3" :serial t :build-operation program-op - :build-pathname "star-server" ;; shell name - :entry-point "star::main" ;; thunk + :build-pathname "star-server" + :entry-point "star::main" :in-order-to ((test-op (test-op "starintel-gserver-tests"))) - :components ( - (:file "consumers/package") - (:file "consumers/consumers") - (:file "producers/package") - (:file "producers/producers") - (:file "package") - (:file "gserver-settings") - (:file "databases/couchdb") - (:file "init-loader") - (:file "actors") - (:file "actor-systems/event-actor") - (:file "actor-systems/matcher-actor") - (:file "rabbit") - (:file "frontends/http-api") - (:file "main")) + :components + ((:file "consumers/package") + (:file "consumers/consumers") + (:file "consumers/rabbit-settlement") + (:file "producers/package") + (:file "producers/producers") + (:file "package") + (:file "gserver-settings") + (:file "databases/couchdb") + (:file "databases/export") + (:file "auth/core") + (:file "auth/verification-hardening") + (:file "auth/immutability") + (:file "auth/store") + (:file "init-loader") + (:file "actors") + (:file "actors/couchdb-service") + (:file "actor-systems/event-actor") + (:file "actor-systems/matcher-actor") + (:file "rabbit") + (:file "frontends/http-api") + (:file "frontends/http-boundary-core") + (:file "frontends/http-auth") + (:file "frontends/http-bulk-jobs") + (:file "frontends/http-boundary-routes") + (:file "frontends/http-auth-routes") + (:file "frontends/http-auth-job-routes") + (:file "main")) + :depends-on + (#:starintel + #:cl-couch + #:serapeum + #:alexandria + #:cl-rabbit + #:sento + #:babel + #:yason + #:ironclad + #:dexador + #:uuid + #:anypool + #:clack + #:lack/middleware/accesslog + #:clack-handler-hunchentoot + #:ningle + #:clingon + #:slynk + #:nhooks + #:lparallel + #:cl-stream + #:cl-ppcre + #:cms-ulid + #:bordeaux-threads)) - :depends-on (#:starintel - #:cl-couch - #:serapeum - #:alexandria - #:cl-rabbit - #:sento - #:babel - #:uuid - #:anypool - #:clack - #:lack/middleware/accesslog - #:clack-handler-hunchentoot - #:ningle - #:clingon - #:slynk - #:nhooks - #:lparallel - #:cl-stream - #:cl-ppcre - #:cms-ulid - #:bordeaux-threads)) -;;;; * Starintel Gserver -;;;;@include "gserver-settings.lisp" -;;;; * Warning -;;;; Please do not use starintel-gserver in production! it is a PROTYPE, or really any star-* thing. they are subject to change as i correct or find better ideas. -;;;; Somethings are fairly fine but its not optmized, its a working notepad as of now. -;;;; * About StarIntel Gserver -;;;; Starintel-gserver is a processing framework for starintel documents. -;;;; it was created after a need that a mess of random scripts, bots and a database wasnt enough. -;;;; I needed something to allow multiple bots to communicate, store, query and handle new targets. -;;;; As of <2024-09-04 Wed> This is the second iteration of the starintel service. -;;;; This version improves on starRouter with actors instead of what i called actors, are really just consumers in -;;;; star-gserver. Gserver started in a org-mode notebook while i scketched things out. -;;;; -;;;; - - - - -;;;;** What in the box -;;;; Actors, provided by cl-gserver, not written by me. i named it gserver becuase this was the "gserver" version of starRouter, which no longer used ZMQ. -;;;; Recurring Targets (NO GUARENTEE!) -;;;; Database actor -;;;; Extensibility provided by simple init file and hooks. -;;;; simple http api -;;;; rabbitmq consumers (akin to starRouter actors) -;;;; producers -;;;; -;;;;** Todo -;;;; -;;;; + [ ] Durable target scheduling -;;;; -;;;; + [ ] ZMQ api for bulk use -;;;; -;;;; + [ ] A simpler to use "plugin" system -;;;; +;;;; StarIntel Gserver is a processing framework for StarIntel documents. +;;;; Runtime documentation lives in ../docs and must track behavior changes. diff --git a/source/views/data.json b/source/views/data.json index 13ced50b..c68170d4 100644 --- a/source/views/data.json +++ b/source/views/data.json @@ -1 +1 @@ -{"_id":"_design/data","language":"javascript","views":{"total":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;emit(null,1);}","reduce":"_sum"},"count_by_dtype":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype)emit(doc.dtype,1);}","reduce":"_sum"},"count_vertices_by_dtype":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype&&doc.dtype!=='relation')emit(doc.dtype,1);}","reduce":"_sum"},"count_by_dataset":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dataset)emit(doc.dataset,1);}","reduce":"_sum"},"count_by_dataset_and_dtype":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dataset&&doc.dtype)emit([doc.dataset,doc.dtype],1);}","reduce":"_sum"},"count_by_dtype_and_etype":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype&&doc.etype)emit([doc.dtype,doc.etype],1);}","reduce":"_sum"},"docs_added_by_day":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(!doc.dateAdded)return;var d=new Date(doc.dateAdded*1000);var day=d.toISOString().slice(0,10);emit(day,1);}","reduce":"_sum"},"docs_updated_by_day":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(!doc.dateUpdated)return;var d=new Date(doc.dateUpdated*1000);var day=d.toISOString().slice(0,10);emit(day,1);}","reduce":"_sum"},"date_added_stats_by_dataset":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dataset&&doc.dateAdded)emit(doc.dataset,doc.dateAdded);}","reduce":"_stats"},"sources_presence":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;var has=(doc.sources&&doc.sources.length>0)?'has_sources':'no_sources';emit(has,1);}","reduce":"_sum"},"sources_count_hist":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;var n=(doc.sources&&doc.sources.length)?doc.sources.length:0;emit(n,1);}","reduce":"_sum"},"relations_by_predicate":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype==='relation'){var p=doc.predicate||'predicate_missing';emit(p,1);}}","reduce":"_sum"},"relations_out_degree":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype==='relation'&&doc.source)emit(doc.source,1);}","reduce":"_sum"},"relations_in_degree":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype==='relation'&&doc.target)emit(doc.target,1);}","reduce":"_sum"},"relations_self_loops":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype==='relation'&&doc.source&&doc.target&&doc.source===doc.target)emit('self_loop',1);}","reduce":"_sum"},"targets_by_actor":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype==='target'){var a=doc.actor||'actor_missing';emit(a,1);}}","reduce":"_sum"},"docs_by_platform":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(!doc.platform)return;emit(doc.platform,1);}","reduce":"_sum"},"urls_by_scheme":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype!=='url')return;var u=doc.url||doc.uri;if(!u||typeof u!=='string')return;var idx=u.indexOf('://');var scheme=(idx>0)?u.slice(0,idx).toLowerCase():'no_scheme';emit(scheme,1);}","reduce":"_sum"},"host_ports_by_number":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype!=='host')return;if(!doc.ports||!doc.ports.length)return;for(var i=0;i0)emit(missing,doc._id);}"}}} +{"_id":"_design/data","language":"javascript","views":{"total":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;emit(null,1);}","reduce":"_sum"},"count_by_dtype":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype)emit(doc.dtype,1);}","reduce":"_sum"},"count_vertices_by_dtype":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype&&doc.dtype!=='relation')emit(doc.dtype,1);}","reduce":"_sum"},"count_by_dataset":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dataset)emit(doc.dataset,1);}","reduce":"_sum"},"documents_by_dataset":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dataset)emit([doc.dataset,doc._id],null);}"},"count_by_dataset_and_dtype":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dataset&&doc.dtype)emit([doc.dataset,doc.dtype],1);}","reduce":"_sum"},"count_by_dtype_and_etype":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype&&doc.etype)emit([doc.dtype,doc.etype],1);}","reduce":"_sum"},"docs_added_by_day":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(!doc.dateAdded)return;var d=new Date(doc.dateAdded*1000);var day=d.toISOString().slice(0,10);emit(day,1);}","reduce":"_sum"},"docs_updated_by_day":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(!doc.dateUpdated)return;var d=new Date(doc.dateUpdated*1000);var day=d.toISOString().slice(0,10);emit(day,1);}","reduce":"_sum"},"date_added_stats_by_dataset":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dataset&&doc.dateAdded)emit(doc.dataset,doc.dateAdded);}","reduce":"_stats"},"sources_presence":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;var has=(doc.sources&&doc.sources.length>0)?'has_sources':'no_sources';emit(has,1);}","reduce":"_sum"},"sources_count_hist":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;var n=(doc.sources&&doc.sources.length)?doc.sources.length:0;emit(n,1);}","reduce":"_sum"},"relations_by_predicate":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype==='relation'){var p=doc.predicate||'predicate_missing';emit(p,1);}}","reduce":"_sum"},"relations_out_degree":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype==='relation'&&doc.source)emit(doc.source,1);}","reduce":"_sum"},"relations_in_degree":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype==='relation'&&doc.target)emit(doc.target,1);}","reduce":"_sum"},"relations_self_loops":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype==='relation'&&doc.source&&doc.target&&doc.source===doc.target)emit('self_loop',1);}","reduce":"_sum"},"targets_by_actor":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype==='target'){var a=doc.actor||'actor_missing';emit(a,1);}}","reduce":"_sum"},"docs_by_platform":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(!doc.platform)return;emit(doc.platform,1);}","reduce":"_sum"},"urls_by_scheme":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype!=='url')return;var u=doc.url||doc.uri;if(!u||typeof u!=='string')return;var idx=u.indexOf('://');var scheme=(idx>0)?u.slice(0,idx).toLowerCase():'no_scheme';emit(scheme,1);}","reduce":"_sum"},"host_ports_by_number":{"map":"function(doc){if(!doc||!doc._id)return;if(doc._id.indexOf('_design/')===0)return;if(doc.dtype!=='host')return;if(!doc.ports||!doc.ports.length)return;for(var i=0;i0)emit(missing,doc._id);}"}}} diff --git a/starintel-gserver-tests.asd b/starintel-gserver-tests.asd index 43c32596..d3f88c9a 100644 --- a/starintel-gserver-tests.asd +++ b/starintel-gserver-tests.asd @@ -1,34 +1,46 @@ (asdf:defsystem :starintel-gserver-tests - :version "0.1.0" - :description "Hermetic unit test suite for starintel-gserver" - :author "nsaspy@airmail.cc" - :license "GPL v3" + :version "0.1.0" + :description "Hermetic unit test suite for starintel-gserver" + :author "nsaspy@airmail.cc" + :license "GPL v3" :serial t - :depends-on (#:starintel-gserver - #:starintel-gserver-client - #:star-cli - #:star-ui - #:star-migrations - #:fiveam - #:dexador - #:bordeaux-threads - #:jsown) - :components ((:module "t" - :serial t - :components ((:file "package") - (:file "test-runner") - (:file "test-runner-test") - (:file "consumers-test") - (:file "init-loader-test") - (:file "target-routing-test") - (:file "system-load-test") - (:file "run-tests")))) - :perform (test-op (o c) - (declare (ignore o c)) - (uiop:symbol-call :star-server-tests :run-all-gserver-tests))) + :depends-on + (#:starintel-gserver + #:starintel-gserver-client + #:star-cli + #:star-ui + #:star-migrations + #:fiveam + #:dexador + #:bordeaux-threads + #:jsown) + :components + ((:module "t" + :serial t + :components + ((:file "package") + (:file "test-runner") + (:file "test-runner-test") + (:file "consumers-test") + (:file "init-loader-test") + (:file "target-routing-test") + (:file "system-load-test") + (:file "couchdb-actor-test") + (:file "event-actor-test") + (:file "dataset-export-test") + (:file "http-boundary-test") + (:file "http-auth-test") + (:file "http-auth-oracle-test") + (:file "http-auth-immutability-test") + (:file "run-tests")))) + :perform + (test-op (operation component) + (declare (ignore operation component)) + (uiop:symbol-call + :star-server-tests + :run-all-gserver-tests))) ;;;; Canonical unit entry point: ;;;; (asdf:test-system :starintel-gserver-tests) -;;;; -;;;; Service-backed coverage is intentionally isolated in +;;;; Service-backed coverage is isolated in ;;;; :starintel-gserver-integration-tests. diff --git a/t/couchdb-actor-test.lisp b/t/couchdb-actor-test.lisp new file mode 100644 index 00000000..383e7414 --- /dev/null +++ b/t/couchdb-actor-test.lisp @@ -0,0 +1,236 @@ +(in-package :star-server-tests) + +(def-suite couchdb-actor-tests + :description "Hermetic CouchDB actor request, reply, and delete tests") + +(in-suite couchdb-actor-tests) + +(defun make-couchdb-test-system () + (make-actor-system '(:dispatchers + (:storage (:workers 2 :strategy :random)) + :timeout-timer + (:resolution 50 :max-size 100)))) + +(defun wait-for-future-result (future &key (attempts 200) (delay 0.01)) + (loop repeat attempts + when (sento.future:complete-p future) + return (sento.future:fresult future) + do (sleep delay) + finally (return :not-ready))) + +(test couchdb-get-ask-s-with-and-without-revision + (let ((system (make-couchdb-test-system)) + (calls nil)) + (unwind-protect + (let* ((handler + (star.actors:make-couchdb-get-handler + nil + :get-fn + (lambda (agent database document-id revision) + (declare (ignore agent)) + (push (list database document-id revision) calls) + (format nil "~a/~a@~a" + database document-id (or revision "current"))))) + (actor + (actor-of system + :name "couchdb-get-ask-s-test" + :dispatcher :storage + :receive handler)) + (current + (sento.actor:ask-s + actor + (star.actors:make-couchdb-get-request + :database "intel" + :document-id "doc-current") + :time-out 2)) + (historical + (sento.actor:ask-s + actor + (star.actors:make-couchdb-get-request + :database "intel" + :document-id "doc-historical" + :revision "3-deadbeef") + :time-out 2))) + (is (eq :success (star.actors:couchdb-result-status current))) + (is (string= "intel/doc-current@current" + (star.actors:couchdb-result-value current))) + (is (eq :success (star.actors:couchdb-result-status historical))) + (is (string= "intel/doc-historical@3-deadbeef" + (star.actors:couchdb-result-value historical))) + (is (member '("intel" "doc-current" nil) calls :test #'equal)) + (is (member '("intel" "doc-historical" "3-deadbeef") + calls + :test #'equal))) + (ac:shutdown system)))) + +(test couchdb-get-async-asks-return-to-correct-callers + (let ((system (make-couchdb-test-system))) + (unwind-protect + (let* ((handler + (star.actors:make-couchdb-get-handler + nil + :get-fn + (lambda (agent database document-id revision) + (declare (ignore agent database revision)) + document-id))) + (actor + (actor-of system + :name "couchdb-get-async-test" + :dispatcher :storage + :receive handler)) + (future-a + (sento.actor:ask + actor + (star.actors:make-couchdb-get-request + :document-id "caller-a") + :time-out 2)) + (future-b + (sento.actor:ask + actor + (star.actors:make-couchdb-get-request + :document-id "caller-b") + :time-out 2)) + (result-a (wait-for-future-result future-a)) + (result-b (wait-for-future-result future-b))) + (is (not (eq :not-ready result-a))) + (is (not (eq :not-ready result-b))) + (is (string= "caller-a" + (star.actors:couchdb-result-value result-a))) + (is (string= "caller-b" + (star.actors:couchdb-result-value result-b)))) + (ac:shutdown system)))) + +(test existing-document-insert-completes-deterministically + (let ((system (make-couchdb-test-system))) + (unwind-protect + (let* ((handler + (star.actors:make-couchdb-insert-handler + nil + :exists-fn + (lambda (agent database document-id) + (declare (ignore agent database document-id)) + t) + :insert-fn + (lambda (&rest arguments) + (declare (ignore arguments)) + (error "Insert must not run for an existing document.")))) + (actor + (actor-of system + :name "couchdb-existing-insert-test" + :dispatcher :storage + :receive handler)) + (result + (sento.actor:ask-s + actor + (star.actors:make-couchdb-insert-request + :database "intel" + :document-id "already-there" + :document "{\"_id\":\"already-there\"}") + :time-out 2))) + (is (eq :exists (star.actors:couchdb-result-status result))) + (is (string= "already-there" + (star.actors:couchdb-result-document-id result)))) + (ac:shutdown system)))) + +(test tell-based-insert-does-not-reply-without-sender + (let ((system (make-couchdb-test-system)) + (insert-count 0) + (lock (bt:make-lock))) + (unwind-protect + (let* ((handler + (star.actors:make-couchdb-insert-handler + nil + :exists-fn + (lambda (agent database document-id) + (declare (ignore agent database document-id)) + (bt:with-lock-held (lock) + (plusp insert-count))) + :insert-fn + (lambda (agent database document) + (declare (ignore agent database document)) + (bt:with-lock-held (lock) + (incf insert-count)) + :inserted))) + (actor + (actor-of system + :name "couchdb-tell-insert-test" + :dispatcher :storage + :receive handler)) + (request + (star.actors:make-couchdb-insert-request + :database "events" + :document-id "event-1" + :document "{\"_id\":\"event-1\"}"))) + (tell actor request) + (loop repeat 100 + until (bt:with-lock-held (lock) + (= 1 insert-count)) + do (sleep 0.01)) + (is (= 1 (bt:with-lock-held (lock) insert-count))) + (let ((result (sento.actor:ask-s actor request :time-out 2))) + (is (eq :exists (star.actors:couchdb-result-status result))) + (is (= 1 (bt:with-lock-held (lock) insert-count))))) + (ac:shutdown system)))) + +(test delete-fetches-current-revision + (let ((get-calls 0) + (delete-arguments nil)) + (multiple-value-bind (value revision) + (star.actors:delete-couchdb-document + :fake-client + "intel" + "doc-1" + nil + :get-fn + (lambda (client database document-id) + (declare (ignore client)) + (incf get-calls) + (is (string= "intel" database)) + (is (string= "doc-1" document-id)) + "{\"_id\":\"doc-1\",\"_rev\":\"7-current\"}") + :delete-fn + (lambda (client database document-id current-revision) + (declare (ignore client)) + (setf delete-arguments + (list database document-id current-revision)) + :deleted)) + (is (eq :deleted value)) + (is (string= "7-current" revision)) + (is (= 1 get-calls)) + (is (equal '("intel" "doc-1" "7-current") + delete-arguments))))) + +(test delete-uses-provided-revision-without-fetch + (let ((get-called-p nil) + (delete-revision nil)) + (multiple-value-bind (value revision) + (star.actors:delete-couchdb-document + :fake-client + "intel" + "doc-2" + "9-explicit" + :get-fn + (lambda (&rest arguments) + (declare (ignore arguments)) + (setf get-called-p t) + (error "Revision fetch must not run.")) + :delete-fn + (lambda (client database document-id current-revision) + (declare (ignore client database document-id)) + (setf delete-revision current-revision) + :deleted)) + (is (eq :deleted value)) + (is (string= "9-explicit" revision)) + (is-false get-called-p) + (is (string= "9-explicit" delete-revision))))) + +(test couchdb-agent-uses-injected-pool + (let ((system (make-couchdb-test-system)) + (sentinel-pool (list :injected-pool))) + (unwind-protect + (let ((agent (star.actors::make-couchdb-agent + system sentinel-pool + :dispatcher-id :storage))) + (is (eq sentinel-pool + (sento.agent:agent-get agent #'identity)))) + (ac:shutdown system)))) diff --git a/t/dataset-export-test.lisp b/t/dataset-export-test.lisp new file mode 100644 index 00000000..a9f7d6e9 --- /dev/null +++ b/t/dataset-export-test.lisp @@ -0,0 +1,149 @@ +(in-package :star-server-tests) + +(def-suite dataset-export-tests + :description "Hermetic dataset export pagination and atomic replacement tests") + +(in-suite dataset-export-tests) + +(defun dataset-export-test-path () + (pathname + (format nil "/tmp/starintel-export-~d-~d.jsonl" + (get-universal-time) + (random most-positive-fixnum)))) + +(defun delete-test-file (path) + (when (probe-file path) + (delete-file path))) + +(defun make-export-document (dataset index) + (jsown:new-js + ("_id" (format nil "doc-~6,'0d" index)) + ("dataset" dataset) + ("dtype" "person"))) + +(defun make-export-row (document) + (jsown:new-js + ("key" (list (jsown:val document "dataset") + (jsown:val document "_id"))) + ("doc" document))) + +(defun make-export-query (documents &key fail-on-call) + (let ((rows (mapcar #'make-export-row documents)) + (calls nil) + (call-count 0)) + (values + (lambda (client database design-document view-name + &key limit start-key end-key skip include-docs reduce update + &allow-other-keys) + (declare (ignore client database end-key update)) + (incf call-count) + (push (list :design-document design-document + :view-name view-name + :start-key start-key + :skip skip + :limit limit + :include-docs include-docs + :reduce reduce) + calls) + (when (and fail-on-call (= call-count fail-on-call)) + (error "Injected export failure on call ~d" call-count)) + (let* ((cursor-id (second start-key)) + (cursor-position + (and cursor-id + (position cursor-id + rows + :test #'string= + :key (lambda (row) + (second (jsown:val row "key")))))) + (start (+ (or cursor-position 0) skip)) + (end (min (length rows) (+ start limit)))) + (jsown:new-js + ("rows" (if (< start (length rows)) + (subseq rows start end) + nil))))) + (lambda () + (nreverse calls))))) + +(defun read-export-lines (path) + (with-open-file (stream path :direction :input :external-format :utf-8) + (loop for line = (read-line stream nil nil) + while line + collect (jsown:parse line)))) + +(test dataset-export-boundary-sizes + (dolist (size '(0 1 99 100 101 200 201)) + (let* ((dataset (format nil "dataset-~d" size)) + (documents (loop for index below size + collect (make-export-document dataset index))) + (path (dataset-export-test-path))) + (unwind-protect + (multiple-value-bind (query-fn calls-fn) + (make-export-query documents) + (let* ((result + (star.databases.couchdb:export-by-dataset* + nil nil dataset path + :page-size 100 + :query-fn query-fn)) + (exported (read-export-lines path)) + (calls (funcall calls-fn))) + (is-true (getf result :ok)) + (is (= size (getf result :exported))) + (is (= size (length exported))) + (is (= (ceiling size 100) (getf result :pages))) + (is (string= "documents_by_dataset" + (getf (first calls) :view-name))) + (is (zerop (getf (first calls) :skip))) + (is-true (getf (first calls) :include-docs)) + (is-false (getf (first calls) :reduce)) + (loop for document in exported + do (is (string= dataset + (jsown:val document "dataset")))))) + (delete-test-file path))))) + +(test dataset-export-uses-key-cursor-after-first-page + (let* ((dataset "cursor-test") + (documents (loop for index below 101 + collect (make-export-document dataset index))) + (path (dataset-export-test-path))) + (unwind-protect + (multiple-value-bind (query-fn calls-fn) + (make-export-query documents) + (let ((result + (star.databases.couchdb:export-by-dataset* + nil nil dataset path + :page-size 100 + :query-fn query-fn))) + (is-true (getf result :ok)) + (let ((calls (funcall calls-fn))) + (is (= 2 (length calls))) + (is (= 1 (getf (second calls) :skip))) + (is (equal (list dataset "doc-000099") + (getf (second calls) :start-key)))))) + (delete-test-file path)))) + +(test interrupted-export-preserves-valid-target + (let* ((dataset "interrupted-test") + (documents (loop for index below 150 + collect (make-export-document dataset index))) + (path (dataset-export-test-path)) + (original (format nil "previous-valid-export~%"))) + (unwind-protect + (progn + (with-open-file (stream path + :direction :output + :if-exists :supersede + :if-does-not-exist :create) + (write-string original stream)) + (multiple-value-bind (query-fn calls-fn) + (make-export-query documents :fail-on-call 2) + (declare (ignore calls-fn)) + (let ((result + (star.databases.couchdb:export-by-dataset* + nil nil dataset path + :page-size 100 + :query-fn query-fn))) + (is-false (getf result :ok)) + (is (= 100 (getf result :exported))) + (is (string= original + (alexandria:read-file-into-string path)))))) + (delete-test-file path)))) diff --git a/t/event-actor-test.lisp b/t/event-actor-test.lisp new file mode 100644 index 00000000..8056de40 --- /dev/null +++ b/t/event-actor-test.lisp @@ -0,0 +1,198 @@ +(in-package :star-server-tests) + +(def-suite event-actor-tests + :description "Actor-event codec, idempotency, and Rabbit settlement tests") + +(in-suite event-actor-tests) + +(defun valid-event-payload (&key (id "event-1") + (timestamp 1770000000) + (dtype "actorevent") + (actor-name "crawler") + (component "url-fetcher") + (event-type "document.fetched") + (details "ok") + (source-id "source-1") + (trace-id "trace-1") + (generation 2)) + (jsown:to-json + (jsown:new-js + ("_id" id) + ("timestamp" timestamp) + ("dtype" dtype) + ("actorName" actor-name) + ("component" component) + ("eventType" event-type) + ("details" details) + ("sourceId" source-id) + ("traceId" trace-id) + ("generation" generation)))) + +(test actor-event-timestamp-and-dtype-initialize-independently + (let ((event + (star.actors:make-actor-event + :id "event-independent" + :timestamp 123456789 + :dtype "actorevent" + :actor-name "scheduler" + :event-type "target.started"))) + (is (= 123456789 (star.actors::event-timestamp event))) + (is (string= "actorevent" (star.actors::doc-type event))) + (is (string= "scheduler" (star.actors:event-component event))))) + +(test legacy-event-fixture-migrates-through-one-codec + (let* ((payload + "{\"_id\":\"legacy-1\",\"timestamp\":1770000001,\"actorName\":\"legacy-actor\",\"eventType\":\"legacy.event\",\"details\":\"migrated\",\"sourceId\":\"source-old\"}") + (event (star.actors:decode-actor-event payload))) + (is (string= "legacy-1" (star.actors::event-id event))) + (is (string= "actorevent" (star.actors::doc-type event))) + (is (string= "legacy-actor" (star.actors:event-component event))) + (is (= 0 (star.actors:event-generation event))) + (is (string= "source-old" + (star.actors::event-source-document event))))) + +(test valid-event-is-persisted-once-and-acked + (let ((persist-count 0) + (persisted-id nil)) + (let ((settlement + (star.actors:process-event-delivery + (valid-event-payload) + :persist-fn + (lambda (event) + (incf persist-count) + (setf persisted-id (star.actors::event-id event)) + (star.actors::make-couchdb-result + :status :success + :operation :insert + :document-id persisted-id))))) + (is (= 1 persist-count)) + (is (string= "event-1" persisted-id)) + (is (eq :ack + (star.consumers:rabbit-settlement-action settlement))) + (is (eq :persisted + (star.consumers:rabbit-settlement-reason settlement)))))) + +(test duplicate-event-delivery-is-idempotently-acked + (let ((persist-count 0)) + (let ((settlement + (star.actors:process-event-delivery + (valid-event-payload :id "duplicate-1") + :persist-fn + (lambda (event) + (declare (ignore event)) + (incf persist-count) + (star.actors::make-couchdb-result + :status :exists + :operation :insert + :document-id "duplicate-1"))))) + (is (= 1 persist-count)) + (is (eq :ack + (star.consumers:rabbit-settlement-action settlement))) + (is (eq :duplicate + (star.consumers:rabbit-settlement-reason settlement)))))) + +(test invalid-event-is-quarantined-and-settled + (let ((persist-called-p nil)) + (let ((settlement + (star.actors:process-event-delivery + "{\"_id\":\"invalid-1\",\"timestamp\":1770000002,\"actorName\":\"crawler\"}" + :persist-fn + (lambda (event) + (declare (ignore event)) + (setf persist-called-p t) + (error "Invalid events must not persist."))))) + (is-false persist-called-p) + (is (eq :nack + (star.consumers:rabbit-settlement-action settlement))) + (is-false + (star.consumers:rabbit-settlement-requeue settlement)) + (is (eq :invalid-event + (star.consumers:rabbit-settlement-reason settlement)))))) + +(test persistence-failure-is-requeued + (let ((settlement + (star.actors:process-event-delivery + (valid-event-payload :id "retry-1") + :persist-fn + (lambda (event) + (declare (ignore event)) + (star.actors::make-couchdb-result + :status :error + :operation :insert + :document-id "retry-1" + :error-message "CouchDB unavailable"))))) + (is (eq :nack + (star.consumers:rabbit-settlement-action settlement))) + (is-true + (star.consumers:rabbit-settlement-requeue settlement)) + (is (eq :persistence-failed + (star.consumers:rabbit-settlement-reason settlement))))) + +(test rabbit-settlement-uses-owning-connection-and-delivery-tag + (let* ((stream + (make-instance + 'star.consumers:settled-rabbit-queue-stream + :queue-name "events" + :exchange-name "events" + :routing-key "event.#" + :rabbit-connection :owning-connection)) + (consumer + (make-instance + 'star.consumers:rabbit-consumer + :name "settlement-test" + :workers 1 + :stream stream)) + (ack-arguments nil) + (nack-arguments nil)) + (star.consumers:settle-rabbit-delivery + consumer + (cons "{}" 41) + (star.consumers:rabbit-ack :reason :persisted) + :ack-fn + (lambda (connection channel delivery-tag &key multiple) + (setf ack-arguments + (list connection channel delivery-tag multiple))) + :nack-fn + (lambda (&rest arguments) + (setf nack-arguments arguments))) + (is (equal '(:owning-connection 1 41 nil) ack-arguments)) + (is (null nack-arguments)) + (star.consumers:settle-rabbit-delivery + consumer + (cons "{}" 42) + (star.consumers:rabbit-nack + :reason :invalid-event + :requeue nil) + :ack-fn + (lambda (&rest arguments) + (setf ack-arguments arguments)) + :nack-fn + (lambda (connection channel delivery-tag &key multiple requeue) + (setf nack-arguments + (list connection channel delivery-tag multiple requeue)))) + (is (equal '(:owning-connection 1 42 nil nil) + nack-arguments)))) + +(test event-consumer-declares-durable-dead-letter-policy + (let* ((consumer + (star.consumers:create-rabbit-consumer + :name "events-test" + :queue-name "events" + :exchange-name "events" + :routing-key "event.#" + :queue-durable t + :exchange-durable t + :dead-letter-exchange "events.dead-letter" + :dead-letter-routing-key "events.invalid" + :dead-letter-queue "events.quarantine" + :handler-fn #'identity)) + (stream (star.consumers:consumer-stream consumer))) + (is (typep stream 'star.consumers:settled-rabbit-queue-stream)) + (is-true (star.consumers:rabbit-stream-queue-durable-p stream)) + (is-true (star.consumers:rabbit-exchange-durable-p stream)) + (is (string= "events.dead-letter" + (star.consumers:rabbit-stream-dead-letter-exchange stream))) + (is (string= "events.invalid" + (star.consumers:rabbit-stream-dead-letter-routing-key stream))) + (is (string= "events.quarantine" + (star.consumers:rabbit-stream-dead-letter-queue stream))))) diff --git a/t/http-auth-immutability-test.lisp b/t/http-auth-immutability-test.lisp new file mode 100644 index 00000000..749d9b94 --- /dev/null +++ b/t/http-auth-immutability-test.lisp @@ -0,0 +1,49 @@ +(in-package :star-server-tests) + +(in-suite http-auth-tests) + +(test request-principal-accessors-return-defensive-copies + (let* ((principal + (star.auth::%make-request-principal + :id "principal-original" + :type "api_client" + :scopes '("documents:read" "search:read") + :credential-id "credential-original")) + (first-id (star.auth:request-principal-id principal)) + (first-scopes (star.auth:request-principal-scopes principal)) + (first-credential-id + (star.auth:request-principal-credential-id principal))) + (setf (char first-id 0) #\X + (char (first first-scopes) 0) #\X + (char first-credential-id 0) #\X) + (setf (cdr first-scopes) nil) + (is (string= "principal-original" + (star.auth:request-principal-id principal))) + (is (equal '("documents:read" "search:read") + (star.auth:request-principal-scopes principal))) + (is (string= "credential-original" + (star.auth:request-principal-credential-id principal))))) + +(test service-call-context-accessors-return-defensive-copies + (let* ((context + (star.auth::%make-service-call-context + :principal-id "service-original" + :principal-type "actor_component" + :credential-id "credential-original" + :scopes '("targets:lease") + :correlation-id "correlation-original" + :deadline 1000)) + (principal-id + (star.auth:service-call-context-principal-id context)) + (scopes (star.auth:service-call-context-scopes context)) + (correlation-id + (star.auth:service-call-context-correlation-id context))) + (setf (char principal-id 0) #\X + (char (first scopes) 0) #\X + (char correlation-id 0) #\X) + (is (string= "service-original" + (star.auth:service-call-context-principal-id context))) + (is (equal '("targets:lease") + (star.auth:service-call-context-scopes context))) + (is (string= "correlation-original" + (star.auth:service-call-context-correlation-id context))))) diff --git a/t/http-auth-oracle-test.lisp b/t/http-auth-oracle-test.lisp new file mode 100644 index 00000000..11bd262f --- /dev/null +++ b/t/http-auth-oracle-test.lisp @@ -0,0 +1,41 @@ +(in-package :star-server-tests) + +(in-suite http-auth-tests) + +(test unknown-credential-id-uses-the-verifier-comparison-path + (let* ((star:*auth-pepper* "unit-test-pepper") + (store (star.auth:make-memory-credential-store)) + (calls 0) + (unknown-key + (format nil "star_sk_v1_unknown_~a" + (make-string 64 :initial-element #\a)))) + (let ((star.auth:*verifier-compare-function* + (lambda (left right) + (incf calls) + (star.auth:constant-time-octets= left right)))) + (is (string= "invalid_credential" + (captured-authentication-code + (lambda () + (authenticate-test-key unknown-key store))))) + (is (= 1 calls))))) + +(test inactive-credential-still-uses-the-verifier-comparison-path + (let* ((star:*auth-pepper* "unit-test-pepper") + (store (star.auth:make-memory-credential-store)) + (calls 0)) + (multiple-value-bind (record raw-key) + (star.auth:create-api-key + "inactive-client" "api_client" '("documents:read") + :store store) + (star.auth:revoke-api-key + (star.auth:api-key-record-id record) + :store store) + (let ((star.auth:*verifier-compare-function* + (lambda (left right) + (incf calls) + (star.auth:constant-time-octets= left right)))) + (is (string= "invalid_credential" + (captured-authentication-code + (lambda () + (authenticate-test-key raw-key store))))) + (is (= 1 calls)))))) diff --git a/t/http-auth-test.lisp b/t/http-auth-test.lisp new file mode 100644 index 00000000..780e2993 --- /dev/null +++ b/t/http-auth-test.lisp @@ -0,0 +1,281 @@ +(in-package :star-server-tests) + +(def-suite http-auth-tests + :description "API-key authentication, lifecycle, redaction, and concurrency") + +(in-suite http-auth-tests) + +(defun captured-authentication-code (thunk) + (handler-case + (progn + (funcall thunk) + nil) + (star.auth:authentication-error (condition) + (star.auth:authentication-error-code condition)))) + +(defun captured-lifecycle-code (thunk) + (handler-case + (progn + (funcall thunk) + nil) + (star.auth:credential-lifecycle-error (condition) + (star.auth:credential-lifecycle-error-code condition)))) + +(defun altered-api-key (api-key) + (let* ((last-index (1- (length api-key))) + (last-character (char api-key last-index)) + (replacement (if (char= last-character #\0) #\1 #\0))) + (concatenate 'string + (subseq api-key 0 last-index) + (string replacement)))) + +(defun authenticate-test-key (raw-key store &optional (correlation-id "corr-auth")) + (star.auth:authenticate-authorization-header + (format nil "Bearer ~a" raw-key) + correlation-id + (+ (star.auth:auth-now) 30) + :store store)) + +(test missing-malformed-expired-disabled-revoked-and-incorrect-credentials-are-rejected + (let* ((now 1000) + (star.auth:*auth-clock* (lambda () now)) + (star:*auth-pepper* "unit-test-pepper") + (store (star.auth:make-memory-credential-store))) + (multiple-value-bind (valid-record valid-key) + (star.auth:create-api-key + "valid-client" "api_client" '("documents:read") :store store) + (declare (ignore valid-record)) + (dolist (header + (list nil + "" + "Basic abc" + "Bearer broken" + "Bearer star_sk_v2_bad_bad" + (format nil "Bearer ~a" (altered-api-key valid-key)))) + (is (string= "invalid_credential" + (captured-authentication-code + (lambda () + (star.auth:authenticate-authorization-header + header "corr-reject" 1030 :store store))))))) + (multiple-value-bind (expired-record expired-key) + (star.auth:create-api-key + "expired-client" "api_client" '("documents:read") + :expires-in-seconds 1 + :store store) + (declare (ignore expired-record)) + (incf now 2) + (is (string= "invalid_credential" + (captured-authentication-code + (lambda () + (authenticate-test-key expired-key store)))))) + (multiple-value-bind (disabled-record disabled-key) + (star.auth:create-api-key + "disabled-client" "api_client" '("documents:read") :store store) + (star.auth:disable-api-key + (star.auth:api-key-record-id disabled-record) + :store store) + (is (string= "invalid_credential" + (captured-authentication-code + (lambda () + (authenticate-test-key disabled-key store)))))) + (multiple-value-bind (revoked-record revoked-key) + (star.auth:create-api-key + "revoked-client" "api_client" '("documents:read") :store store) + (star.auth:revoke-api-key + (star.auth:api-key-record-id revoked-record) + :store store) + (is (string= "invalid_credential" + (captured-authentication-code + (lambda () + (authenticate-test-key revoked-key store)))))))) + +(test valid-credential-creates-immutable-request-principal + (let* ((star:*auth-pepper* "unit-test-pepper") + (store (star.auth:make-memory-credential-store))) + (multiple-value-bind (record raw-key) + (star.auth:create-api-key + "quasar-client" + "api_client" + '("documents:read" "search:read") + :store store) + (let* ((context (authenticate-test-key raw-key store "corr-valid")) + (principal + (star.auth:request-security-context-principal context))) + (is (string= "quasar-client" + (star.auth:request-principal-id principal))) + (is (string= "api_client" + (star.auth:request-principal-type principal))) + (is (string= (star.auth:api-key-record-id record) + (star.auth:request-principal-credential-id principal))) + (is (equal '("documents:read" "search:read") + (star.auth:request-principal-scopes principal))) + (is (string= "corr-valid" + (star.auth:request-security-context-correlation-id + context))))))) + +(test verifier-boundary-uses-constant-time-comparison + (let* ((star:*auth-pepper* "unit-test-pepper") + (store (star.auth:make-memory-credential-store)) + (calls 0)) + (multiple-value-bind (record raw-key) + (star.auth:create-api-key + "constant-time-client" "api_client" '("documents:read") + :store store) + (declare (ignore record)) + (let ((star.auth:*verifier-compare-function* + (lambda (left right) + (incf calls) + (star.auth:constant-time-octets= left right)))) + (authenticate-test-key raw-key store) + (is (= 1 calls)))))) + +(test rotation-honors-overlap-and-then-invalidates-old-secret + (let* ((now 2000) + (star.auth:*auth-clock* (lambda () now)) + (star:*auth-pepper* "unit-test-pepper") + (store (star.auth:make-memory-credential-store))) + (multiple-value-bind (original original-key) + (star.auth:create-api-key + "rotation-client" "service_instance" '("documents:write") + :store store) + (multiple-value-bind (replacement replacement-key) + (star.auth:rotate-api-key + (star.auth:api-key-record-id original) + 10 + :store store) + (is-true (authenticate-test-key original-key store)) + (is-true (authenticate-test-key replacement-key store)) + (is (string= (star.auth:api-key-record-id original) + (star.auth:api-key-record-rotation-parent-id replacement))) + (setf now 2010) + (is (string= "invalid_credential" + (captured-authentication-code + (lambda () + (authenticate-test-key original-key store))))) + (is-true (authenticate-test-key replacement-key store)))))) + +(test bootstrap-is-one-time-and-does-not-store-raw-secret + (let* ((star:*auth-pepper* "unit-test-pepper") + (star:*auth-bootstrap-secret* "bootstrap-secret") + (store (star.auth:make-memory-credential-store))) + (is (string= "bootstrap_denied" + (captured-lifecycle-code + (lambda () + (star.auth:bootstrap-api-key + "wrong-secret" "admin" :store store))))) + (multiple-value-bind (record raw-key) + (star.auth:bootstrap-api-key + "bootstrap-secret" "admin" :store store) + (let* ((metadata + (jsown:to-json + (star.auth:api-key-metadata-json record))) + (stored + (star.auth:credential-store-get + store + (star.auth:api-key-record-id record)))) + (is (search "star_sk_v1_" raw-key)) + (is (null (search raw-key metadata :test #'char=))) + (is (null (search "verifier" metadata :test #'char-equal))) + (is (null (search "salt" metadata :test #'char-equal))) + (is (not (string= raw-key + (star.auth:api-key-record-verifier stored)))))) + (is (string= "bootstrap_complete" + (captured-lifecycle-code + (lambda () + (star.auth:bootstrap-api-key + "bootstrap-secret" "other-admin" :store store))))))) + +(test revocation-has-zero-cache-bound-under-concurrency + (let* ((star:*auth-pepper* "unit-test-pepper") + (store (star.auth:make-memory-credential-store)) + (gate-lock (bt:make-lock "auth-revocation-gate")) + (start nil) + (results (make-array 16 :initial-element nil))) + (multiple-value-bind (record raw-key) + (star.auth:create-api-key + "concurrent-client" "api_client" '("documents:read") + :store store) + (let ((threads + (loop for index below (length results) + collect + (let ((slot index)) + (bt:make-thread + (lambda () + (loop until + (bt:with-lock-held (gate-lock) start) + do (sleep 0.001)) + (setf (aref results slot) + (if (captured-authentication-code + (lambda () + (authenticate-test-key raw-key store))) + :rejected + :accepted)))))))) + (star.auth:revoke-api-key + (star.auth:api-key-record-id record) + :store store) + (bt:with-lock-held (gate-lock) + (setf start t)) + (dolist (thread threads) + (bt:join-thread thread)) + (is (every (lambda (result) (eq result :rejected)) results)))))) + +(test cors-is-allowlisted-and-never-wildcard + (let ((star:*http-cors-allowed-origins* + '("https://quasar.example"))) + (is-true + (star.frontends.http-api::configured-origin-allowed-p + "https://quasar.example")) + (is-false + (star.frontends.http-api::configured-origin-allowed-p + "https://attacker.example")) + (let ((headers + (star.frontends.http-api::cors-headers-for-origin + "https://quasar.example"))) + (is (string= "https://quasar.example" + (getf headers :access-control-allow-origin))) + (is (null (member "*" headers :test #'equal)))))) + +(test authenticated-service-context-propagates-without-secret + (let* ((principal + (star.auth::%make-request-principal + :id "actor-service" + :type "actor_component" + :scopes '("targets:lease") + :credential-id "key-public-id")) + (context + (star.auth::%make-request-security-context + :principal principal + :correlation-id "corr-service" + :deadline 9999 + :authenticated-at 9000)) + (star.auth:*request-security-context* context) + (service-context (star.auth:current-service-call-context)) + (properties + (star.frontends.http-api::service-context-properties + "target" service-context)) + (headers (cdr (assoc :headers properties)))) + (is (string= "corr-service" + (cdr (assoc :correlation-id properties)))) + (is (string= "actor-service" + (cdr (assoc "x-star-principal-id" headers + :test #'string=)))) + (is (string= "key-public-id" + (cdr (assoc "x-star-credential-id" headers + :test #'string=)))) + (is (string= "9999" + (cdr (assoc "x-star-deadline" headers + :test #'string=)))) + (is (null (search "star_sk_" (prin1-to-string properties)))))) + +(test authentication-configuration-fails-closed + (let ((star:*auth-mode* "api-key") + (star:*auth-pepper* nil)) + (signals error (star.auth:validate-auth-configuration))) + (let ((star:*auth-mode* "disabled") + (star:*auth-dev-bypass* t) + (star:*http-api-address* "0.0.0.0")) + (signals error (star.auth:validate-auth-configuration))) + (let ((star:*auth-mode* "disabled") + (star:*auth-dev-bypass* t) + (star:*http-api-address* "127.0.0.1")) + (is-true (star.auth:validate-auth-configuration)))) diff --git a/t/http-boundary-test.lisp b/t/http-boundary-test.lisp new file mode 100644 index 00000000..235ffb8f --- /dev/null +++ b/t/http-boundary-test.lisp @@ -0,0 +1,234 @@ +(in-package :star-server-tests) + +(def-suite http-boundary-tests + :description "HTTP boundary validation, safe errors, and bulk backpressure tests") + +(in-suite http-boundary-tests) + +(defun make-boundary-document (&key (dtype "host") + (version starintel:+starintel-doc-version+) + (id "boundary-doc-1")) + (jsown:new-js + ("_id" id) + ("dataset" "boundary-tests") + ("dtype" dtype) + ("version" version))) + +(defun capture-http-input-error (thunk) + (handler-case + (progn + (funcall thunk) + nil) + (star.frontends.http-api:http-input-error (condition) + condition))) + +(test jsown-object-is-not-a-bulk-array + (let ((object (jsown:new-js ("dtype" "host"))) + (array (list (jsown:new-js ("dtype" "host"))))) + (is-true (star.frontends.http-api:json-object-p object)) + (is-false (star.frontends.http-api:json-array-p object)) + (is-true (star.frontends.http-api:json-array-p array)))) + +(test malformed-json-is-a-400-client-error + (let ((condition + (capture-http-input-error + (lambda () + (star.frontends.http-api:parse-json-octets + (babel:string-to-octets "{" :encoding :utf-8) + "application/json"))))) + (is-true condition) + (is (= 400 + (star.frontends.http-api:http-input-error-status condition))) + (is (string= "malformed_json" + (star.frontends.http-api:http-input-error-code condition))))) + +(test non-json-content-type-is-rejected + (let ((condition + (capture-http-input-error + (lambda () + (star.frontends.http-api:parse-json-octets + (babel:string-to-octets "{}" :encoding :utf-8) + "text/plain"))))) + (is (= 415 + (star.frontends.http-api:http-input-error-status condition))) + (is (string= "unsupported_media_type" + (star.frontends.http-api:http-input-error-code condition))))) + +(test oversized-request-body-is-rejected-before-parsing + (let ((condition + (capture-http-input-error + (lambda () + (star.frontends.http-api:parse-json-octets + (make-array 17 + :element-type '(unsigned-byte 8) + :initial-element 32) + "application/json" + :max-bytes 16))))) + (is (= 413 + (star.frontends.http-api:http-input-error-status condition))) + (is (string= "request_body_too_large" + (star.frontends.http-api:http-input-error-code condition))))) + +(test missing-and-mismatched-dtype-return-422 + (let* ((missing (make-boundary-document)) + (mismatch (make-boundary-document :dtype "email"))) + (jsown:remkey missing "dtype") + (let ((missing-condition + (capture-http-input-error + (lambda () + (star.frontends.http-api:validate-document-input + missing + :path-dtype "host")))) + (mismatch-condition + (capture-http-input-error + (lambda () + (star.frontends.http-api:validate-document-input + mismatch + :path-dtype "host"))))) + (is (= 422 + (star.frontends.http-api:http-input-error-status + missing-condition))) + (is (= 422 + (star.frontends.http-api:http-input-error-status + mismatch-condition))) + (is (string= "dtype_mismatch" + (star.frontends.http-api:http-input-error-code + mismatch-condition)))))) + +(test unsupported-schema-version-is-rejected + (let ((condition + (capture-http-input-error + (lambda () + (star.frontends.http-api:validate-document-input + (make-boundary-document :version "999.0") + :path-dtype "host"))))) + (is (= 422 + (star.frontends.http-api:http-input-error-status condition))) + (is (string= "unsupported_schema_version" + (star.frontends.http-api:http-input-error-code condition))))) + +(test client-status-envelopes-never-expose-tracebacks + (let* ((star.frontends.http-api::*http-correlation-id* "corr-test") + (body + (star.frontends.http-api::status-msg + "Bad Request" + 'error + :code "invalid_request" + :traceback "password=super-secret internal stack")) + (parsed (jsown:parse body))) + (is-false (jsown:keyp parsed "trace")) + (is (null (search "super-secret" body :test #'char-equal))) + (is (string= "corr-test" (jsown:val parsed "correlation_id"))))) + +(test numeric-query-validation-is-bounded + (dolist (raw '("not-a-number" "0" "101")) + (let ((condition + (capture-http-input-error + (lambda () + (star.frontends.http-api:bounded-query-integer + (list (cons "limit" raw)) + "limit" + :minimum 1 + :maximum 100))))) + (is (= 400 + (star.frontends.http-api:http-input-error-status condition))))) + (is (= 25 + (star.frontends.http-api:bounded-query-integer + nil + "limit" + :default 25 + :minimum 1 + :maximum 100)))) + +(test oversized-bulk-selects-asynchronous-dispatch + (is (eq :inline + (star.frontends.http-api:bulk-request-mode 10))) + (is (eq :async + (star.frontends.http-api:bulk-request-mode 11)))) + +(test asynchronous-bulk-submission-does-not-wait-for-worker + (let* ((fake-system (list :test-system)) + (sent-worker nil) + (sent-job nil) + (star.actors:*sys* fake-system) + (star.frontends.http-api::*bulk-ingest-workers* + (list :fake-worker)) + (star.frontends.http-api::*bulk-ingest-worker-system* fake-system) + (star.frontends.http-api::*bulk-ingest-worker-index* 0) + (star.frontends.http-api::*bulk-ingest-jobs* + (make-hash-table :test #'equal)) + (star.frontends.http-api::*bulk-pending-jobs* 0) + (star.frontends.http-api::*bulk-pending-by-principal* + (make-hash-table :test #'equal)) + (star.frontends.http-api::*bulk-ingest-lock* + (bt:make-lock "http-boundary-test-lock")) + (star.frontends.http-api::*http-correlation-id* "corr-async") + (documents + (loop for index below 11 + collect + (make-boundary-document + :id (format nil "async-doc-~d" index)))) + (started (get-internal-real-time)) + (job + (star.frontends.http-api:submit-bulk-ingest-job + documents + "principal-test" + :ensure-workers-fn + (lambda () + star.frontends.http-api::*bulk-ingest-workers*) + :tell-fn + (lambda (worker queued-job) + (setf sent-worker worker + sent-job queued-job)))) + (elapsed + (/ (- (get-internal-real-time) started) + internal-time-units-per-second))) + (is-true job) + (is (< elapsed 0.5)) + (is (eq :fake-worker sent-worker)) + (is (eq job sent-job)) + (is (eq :queued + (star.frontends.http-api:bulk-ingest-job-status job))))) + +(test per-principal-bulk-quota-is-enforced + (let* ((fake-system (list :test-system)) + (star.actors:*sys* fake-system) + (star.frontends.http-api::*bulk-ingest-workers* + (list :fake-worker)) + (star.frontends.http-api::*bulk-ingest-worker-system* fake-system) + (star.frontends.http-api::*bulk-ingest-worker-index* 0) + (star.frontends.http-api::*bulk-ingest-jobs* + (make-hash-table :test #'equal)) + (star.frontends.http-api::*bulk-pending-jobs* 0) + (star.frontends.http-api::*bulk-pending-by-principal* + (make-hash-table :test #'equal)) + (star.frontends.http-api::*bulk-ingest-lock* + (bt:make-lock "http-boundary-quota-lock")) + (star.frontends.http-api::*http-correlation-id* "corr-quota") + (documents (list (make-boundary-document))) + (ensure-workers + (lambda () + star.frontends.http-api::*bulk-ingest-workers*)) + (tell-noop + (lambda (worker job) + (declare (ignore worker job))))) + (dotimes (index 4) + (declare (ignore index)) + (star.frontends.http-api:submit-bulk-ingest-job + documents + "quota-principal" + :ensure-workers-fn ensure-workers + :tell-fn tell-noop)) + (let ((condition + (capture-http-input-error + (lambda () + (star.frontends.http-api:submit-bulk-ingest-job + documents + "quota-principal" + :ensure-workers-fn ensure-workers + :tell-fn tell-noop))))) + (is (= 429 + (star.frontends.http-api:http-input-error-status condition))) + (is (string= "principal_bulk_quota_exceeded" + (star.frontends.http-api:http-input-error-code + condition)))))) diff --git a/t/run-tests.lisp b/t/run-tests.lisp index f57ab75f..efd3ab4e 100644 --- a/t/run-tests.lisp +++ b/t/run-tests.lisp @@ -5,7 +5,12 @@ consumer-tests init-loader-tests target-routing-tests - system-load-tests)) + system-load-tests + couchdb-actor-tests + event-actor-tests + dataset-export-tests + http-boundary-tests + http-auth-tests)) (defun run-all-gserver-tests () "Run every hermetic unit suite and fail on empty, skipped, or failed tests." diff --git a/t/test-runner.lisp b/t/test-runner.lisp index 35d440a4..7435f12e 100644 --- a/t/test-runner.lisp +++ b/t/test-runner.lisp @@ -95,18 +95,19 @@ (defun run-required-suite (suite-name &key setup teardown) (let ((discovered (suite-test-names suite-name)) - (summary nil)) + (summary nil) + (results nil)) (when (zerop (length discovered)) (error "Required suite ~a discovered zero tests." suite-name)) (unwind-protect (progn (when setup (funcall setup)) - (setf summary - (summarize-suite suite-name - discovered - (fiveam:run suite-name))) + (setf results (fiveam:run suite-name) + summary (summarize-suite suite-name discovered results)) (print-suite-summary summary) + (when (plusp (suite-summary-failed summary)) + (fiveam:explain! results)) (validate-required-suite summary)) (when teardown (funcall teardown)))