feat(pipeline): add Docker path for the netflow pipeline toolchain

.dockerignore

@@ -0,0 +1,9 @@
+.git
+.env
+.env.*
+data
+datasets.json
+node_modules
+**/node_modules
+target
+vendor/nfdump

Dockerfile

@@ -0,0 +1,74 @@
+# syntax=docker/dockerfile:1
+
+FROM ubuntu:24.04 AS build-base
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update \
+    && apt-get install --yes --no-install-recommends \
+        autoconf \
+        automake \
+        bison \
+        build-essential \
+        ca-certificates \
+        cmake \
+        curl \
+        flex \
+        git \
+        libbz2-dev \
+        libtool \
+        pkg-config \
+        python3 \
+        tar \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /build
+
+FROM build-base AS nfdump-builder
+
+ARG NFDUMP_COMMIT
+
+# Clone the exact superproject gitlink commit. A copied submodule has a .git
+# pointer into the host repository, so it cannot support compile-nfdump.sh.
+RUN test -n "$NFDUMP_COMMIT" \
+    || { echo "NFDUMP_COMMIT must name the vendor/nfdump gitlink commit" >&2; exit 1; }
+RUN git clone https://github.com/flamboh/nfdump.git vendor/nfdump \
+    && git -C vendor/nfdump checkout --detach "$NFDUMP_COMMIT" \
+    && test "$(git -C vendor/nfdump rev-parse HEAD)" = "$NFDUMP_COMMIT"
+
+COPY vendor/scripts/compile-nfdump.sh vendor/scripts/compile-nfdump.sh
+RUN ./vendor/scripts/compile-nfdump.sh
+
+FROM build-base AS rust-builder
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+COPY rust-toolchain.toml rust-toolchain.toml
+RUN curl --proto '=https' --tlsv1.2 --silent --show-error --fail \
+        https://sh.rustup.rs \
+        | sh -s -- --yes --profile minimal --default-toolchain none \
+    && cargo --version
+
+COPY Cargo.toml Cargo.lock ./
+COPY tools/netflow-db/Cargo.toml tools/netflow-db/Cargo.toml
+COPY tools/netflow-db/src tools/netflow-db/src
+RUN cargo build --locked --release --package atlantis-netflow-db --bin netflow-db
+
+FROM ubuntu:24.04 AS runtime
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update \
+    && apt-get install --yes --no-install-recommends ca-certificates libbz2-1.0 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=rust-builder /build/target/release/netflow-db /usr/local/bin/netflow-db
+COPY --from=nfdump-builder /build/target/nfdump/libexec/nfdump /usr/local/bin/nfdump
+
+RUN --mount=type=bind,from=nfdump-builder,source=/build/target/nfdump/build/smoke/dummy_flows.nf,target=/tmp/dummy_flows.nf \
+    netflow-db contract-version >/dev/null \
+    && nfdump -V >/dev/null 2>&1 \
+    && nfdump -G none -r /tmp/dummy_flows.nf -q -o atlantis 'host 203.0.113.255' >/tmp/atlantis.bin \
+    && test "$(wc -c </tmp/atlantis.bin)" -eq 16 \
+    && rm /tmp/atlantis.bin
+
+WORKDIR /workspace
+ENTRYPOINT ["/usr/local/bin/netflow-db"]

README.md

@@ -34,8 +34,8 @@ First, install the [required tools](docs/user/requirements.md) — both Bun and
    ```bash
    ./scripts/netflow-db.sh pipeline \
      --dataset example \
-     --start-date 2025-02-01 \
-     --end-date 2025-02-01 \
+     --start-date <YYYY-MM-DD> \
+     --end-date <YYYY-MM-DD> \
      --nfdump target/nfdump/libexec/nfdump
    ```
 

docs/code/pipeline-contract.md

@@ -90,9 +90,9 @@ The `extract-window` command creates bounded SQLite or Parquet analysis artifact
 ```bash
 ./scripts/netflow-db.sh extract-window \
   --source-db data/uoregon/netflow.sqlite \
-  --output-dir data/uoregon/extracts/2025-06 \
-  --start 2025-06-01 \
-  --end 2025-07-01 \
+  --output-dir data/uoregon/extracts/<YYYY-MM> \
+  --start <YYYY-MM-DD> \
+  --end <YYYY-MM-DD> \
   --output sqlite \
   --output parquet
 ```

docs/user/README.md

@@ -11,9 +11,11 @@ Have this available:
 - nfcapd capture files on disk, in the layout that [dataset configuration](datasets.md#input-directory-layout) shows.
 - Approximately 30 minutes. The one-time tool builds take most of this time.
 
+The pipeline can run in Docker instead of a host Rust and C toolchain. The dashboard always runs natively.
+
 ## First use
 
-1. Install the [required tools](requirements.md).
+1. Install the [required tools](requirements.md). Choose the Docker or native pipeline path.
 2. [Install the project dependencies](setup-web.md).
 3. [Define a dataset](datasets.md) that points at your captures.
 4. [Build and verify the database](setup-pipeline.md).

docs/user/operations.md

@@ -30,8 +30,8 @@ Compare a rebuilt candidate with a trusted historical database before you publis
 ./scripts/netflow-db.sh compare \
   data/candidate/netflow.sqlite \
   data/example/historical.sqlite \
-  --start 2025-11-01 \
-  --end 2025-12-01
+  --start <YYYY-MM-DD> \
+  --end <YYYY-MM-DD>
 ```
 
 `--start` and `--end` are half-open local date or time boundaries. The default timezone is `America/Los_Angeles`.

docs/user/querying.md

@@ -52,8 +52,8 @@ FROM traffic_stats
 WHERE granularity = '1d'
   AND src_visibility = 'all'
   AND dst_visibility = 'all'
-  AND bucket_start >= strftime('%s', '2025-01-01')
-  AND bucket_start < strftime('%s', '2025-01-08')
+  AND bucket_start >= strftime('%s', '<YYYY-MM-DD>')
+  AND bucket_start < strftime('%s', '<YYYY-MM-DD>')
 GROUP BY bucket_start, source_id
 ORDER BY bucket_start, source_id;
 ```

docs/user/requirements.md

@@ -2,7 +2,15 @@
 
 This document lists the required tools by concern. Install the tools for the parts that you use.
 
-The repository has a `shell.nix` file that supplies every tool below except Git. With Nix, run `nix-shell` and skip the manual installation.
+For native setup, the repository has a `shell.nix` file that supplies every development tool below.
+
+On NixOS, run `nix-shell` and skip the manual installation.
+
+## Docker pipeline
+
+Running the pipeline with Docker needs only Git and Docker on the host. The image build supplies the Rust toolchain, the nfdump build tools, and the pinned fork, and it does not need initialized Git submodules.
+
+Docker covers the pipeline only; the dashboard runs natively.
 
 ## Dashboard
 
@@ -16,7 +24,7 @@ The dashboard and all `bun run` commands need these tools:
 
 Node.js is necessary even though Bun installs the packages. The development server runs under Node.js, and `bun install` needs Node.js on `PATH` to download the prebuilt SQLite driver. Without it, the install prints a `better-sqlite3` warning and the dashboard cannot open a database (see [Troubleshooting](troubleshooting.md)).
 
-## Pipeline
+## Native pipeline
 
 Building a database with `scripts/netflow-db.sh` also needs the Rust toolchain:
 
@@ -28,7 +36,7 @@ Building a database with `scripts/netflow-db.sh` also needs the Rust toolchain:
 
 rustup reads `rust-toolchain.toml` and installs the pinned Rust version automatically on the first build.
 
-## nfdump fork
+## Native nfdump fork
 
 Processing nfcapd captures also needs the build tools for the pinned nfdump fork. CSV input does not.
 

docs/user/setup-pipeline.md

@@ -2,29 +2,48 @@
 
 The pipeline is the Rust `atlantis-netflow-db` crate. It converts nfcapd or CSV input into a compatible SQLite database.
 
-`scripts/netflow-db.sh` is the pipeline entry point. It runs the crate with `cargo run --locked --release`, which compiles it when necessary. The first run compiles the Rust dependencies and takes several minutes. Set `NETFLOW_DB_BIN` to run a prebuilt binary instead.
-
 Use a new output database when you change selection rules or result semantics. The pipeline rejects incompatible reuse.
 
-## Build the nfdump fork
+## Choose a path
+
+| Path   | Entry point                    | Host requirements                                                                                                  |
+| ------ | ------------------------------ | ------------------------------------------------------------------------------------------------------------------ |
+| Docker | `scripts/netflow-db-docker.sh` | [Git and Docker](requirements.md#docker-pipeline)                                                                  |
+| Native | `scripts/netflow-db.sh`        | [Rust toolchain](requirements.md#native-pipeline) and the [nfdump build tools](requirements.md#native-nfdump-fork) |
+
+Complete the one-time setup for your path, then follow the rest of this document. The examples below use `./scripts/netflow-db.sh`, and the native path adds `--nfdump` to every command that reads nfcapd input. To run an example with Docker, substitute `./scripts/netflow-db-docker.sh` and add `--capture-root <path>` for commands that read captures.
+
+### Docker setup
+
+Container options come before the pipeline command:
+
+```bash
+./scripts/netflow-db-docker.sh --capture-root /absolute/path/to/captures pipeline ...
+```
+
+The wrapper builds the image when it is missing; pass `--build` to rebuild it after a source update. Each `--capture-root` mounts read-only at the same absolute path inside the container, so `root_path` in `datasets.json` needs no change. Output stays under the repository's `data/` directory, owned by you.
+
+On macOS, bind mounts over large capture trees are slower than native filesystem access.
+
+### Native setup
 
-nfcapd input needs the pinned ATLANTIS nfdump fork. A system nfdump installation does not work: the pipeline uses an output mode that only the fork has. CSV input does not need nfdump.
+`scripts/netflow-db.sh` runs the crate with `cargo run --locked --release`, which compiles it when necessary. Set `NETFLOW_DB_BIN` to run a prebuilt binary instead.
 
-The build needs the [nfdump fork tools](requirements.md#nfdump-fork). The build script checks for them and names any tool that is missing.
+nfcapd input also needs the pinned ATLANTIS nfdump fork. A system nfdump installation does not work: the pipeline uses an output mode that only the fork has. CSV input does not need nfdump.
 
 1. Initialize the Git submodules.
 
    ```bash
    git submodule update --init --recursive
    ```
 
-2. Build the fork.
+2. Build the fork. The script checks for the [build tools](requirements.md#native-nfdump-fork) and names any tool that is missing.
 
    ```bash
    ./vendor/scripts/compile-nfdump.sh
    ```
 
-The build stages the executable at `target/nfdump/libexec/nfdump`. The `target` directory is disposable and git-ignored. Pass this path to each pipeline command with `--nfdump`; the pipeline does not find it automatically.
+The build stages the executable at `target/nfdump/libexec/nfdump`. The `target` directory is disposable and git-ignored. Pass this path with `--nfdump` to every command that reads nfcapd input; the pipeline does not find it automatically.
 
 ## Process a dataset
 
@@ -35,12 +54,11 @@ Run a bounded import while you test the configuration:
 ```bash
 ./scripts/netflow-db.sh pipeline \
   --dataset example \
-  --start-date 2025-02-01 \
-  --end-date 2025-02-01 \
-  --nfdump target/nfdump/libexec/nfdump
+  --start-date <YYYY-MM-DD> \
+  --end-date <YYYY-MM-DD>
 ```
 
-The start date and end date are inclusive, so this command processes one day. If you omit the end date, the pipeline processes each day through the latest available day.
+The start date and end date are inclusive; use the same date for both to process a single day. If you omit the end date, the pipeline processes each day through the latest available day.
 
 Dataset mode calculates MAAD statistics by default. MAAD statistics describe the multifractal structure of the observed IPv4 address sets, and they power the address-structure charts. Use `--no-maad` to skip them.
 
@@ -53,12 +71,11 @@ Selection conditions use AND logic. The IP prefix can match the source endpoint
 ```bash
 ./scripts/netflow-db.sh pipeline \
   --dataset example \
-  --start-date 2025-02-01 \
-  --end-date 2025-02-01 \
+  --start-date <YYYY-MM-DD> \
+  --end-date <YYYY-MM-DD> \
   --database-path data/example-public/netflow.sqlite \
   --ip-prefix 192.0.2.0/24 \
-  --src-visibility literal \
-  --nfdump target/nfdump/libexec/nfdump
+  --src-visibility literal
 ```
 
 A selected population is a different database product. Thus, selection options require an explicit `--database-path`.
@@ -92,7 +109,7 @@ Put flow selection in the top-level `selection` object:
 }
 ```
 
-For nfcapd input, set the top-level `"nfdump"` value to `"target/nfdump/libexec/nfdump"`. You can also pass the same path with `--nfdump` when the configuration does not set it.
+On the native path, nfcapd input needs the fork path: set the top-level `"nfdump"` value to `"target/nfdump/libexec/nfdump"`, or pass `--nfdump` when the configuration does not set it.
 
 ## Common options
 

scripts/netflow-db-docker.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+IMAGE="${NETFLOW_DB_DOCKER_IMAGE:-atlantis-netflow-db:local}"
+
+usage() {
+	cat >&2 <<'EOF'
+usage: scripts/netflow-db-docker.sh [--build] [--capture-root PATH]... [--] <netflow-db args...>
+
+  --build              Rebuild the local image before running the command.
+  --capture-root PATH  Mount an absolute capture root read-only at the same path.
+
+Container options must appear before the netflow-db command.
+EOF
+}
+
+force_build=0
+capture_roots=()
+while (($# > 0)); do
+	case "$1" in
+	--build)
+		force_build=1
+		shift
+		;;
+	--capture-root)
+		if (($# < 2)); then
+			echo "--capture-root requires a path" >&2
+			usage
+			exit 2
+		fi
+		capture_roots+=("$2")
+		shift 2
+		;;
+	--)
+		shift
+		break
+		;;
+	*)
+		break
+		;;
+	esac
+done
+
+if (($# == 0)); then
+	usage
+	exit 2
+fi
+
+if ! command -v docker >/dev/null 2>&1; then
+	echo "docker is required; install Docker Engine or Docker Desktop" >&2
+	exit 1
+fi
+
+for capture_root in "${capture_roots[@]}"; do
+	if [[ "$capture_root" != /* || "$capture_root" == "/" ]]; then
+		echo "capture root must be an absolute directory other than /: $capture_root" >&2
+		exit 2
+	fi
+	if [[ "$capture_root" == *,* ]]; then
+		echo "capture root cannot contain a comma: $capture_root" >&2
+		exit 2
+	fi
+	if [[ ! -d "$capture_root" ]]; then
+		echo "capture root is not a directory: $capture_root" >&2
+		exit 2
+	fi
+done
+
+tree_entry="$(git -C "$ROOT_DIR" ls-tree HEAD -- vendor/nfdump)"
+read -r submodule_mode submodule_type nfdump_commit submodule_path <<<"$tree_entry"
+if [[ "$submodule_mode" != "160000" || "$submodule_type" != "commit" || "$submodule_path" != "vendor/nfdump" || ! "$nfdump_commit" =~ ^[0-9a-f]{40}$ ]]; then
+	echo "unable to read the pinned vendor/nfdump commit from the current Git tree" >&2
+	exit 1
+fi
+
+if ((force_build)) || ! docker image inspect "$IMAGE" >/dev/null 2>&1; then
+	docker build \
+		--build-arg "NFDUMP_COMMIT=$nfdump_commit" \
+		--tag "$IMAGE" \
+		"$ROOT_DIR"
+fi
+
+mkdir -p "$ROOT_DIR/data"
+
+docker_args=(
+	run
+	--rm
+	--user "$(id -u):$(id -g)"
+	--env HOME=/tmp
+	--workdir /workspace
+	--mount "type=bind,source=$ROOT_DIR/data,target=/workspace/data"
+)
+
+if [[ -f "$ROOT_DIR/datasets.json" ]]; then
+	docker_args+=(
+		--mount "type=bind,source=$ROOT_DIR/datasets.json,target=/workspace/datasets.json,readonly"
+	)
+fi
+
+for capture_root in "${capture_roots[@]}"; do
+	docker_args+=(
+		--mount "type=bind,source=$capture_root,target=$capture_root,readonly"
+	)
+done
+
+exec docker "${docker_args[@]}" "$IMAGE" "$@"

shell.nix

@@ -11,6 +11,8 @@ mkShell {
     pkgs.bison
     pkgs.bun
     pkgs.flex
+    pkgs.gcc
+    pkgs.git
     pkgs.gnumake
     pkgs.gnutar
     pkgs.libtool