From ea590e9782ef0bb0e4b365c5d912c9df37840864 Mon Sep 17 00:00:00 2001 From: flamboh Date: Sat, 22 Aug 2026 03:59:23 -0700 Subject: [PATCH 1/3] feat(pipeline): add Docker path for the netflow pipeline toolchain Adds a multi-stage Dockerfile and scripts/netflow-db-docker.sh so new users can run the pipeline with only Git and Docker installed, instead of rustup, a C toolchain, and the autotools stack. The image clones the nfdump fork at the superproject-pinned commit, so host submodules stay uninitialized. Also adds gcc and git to shell.nix so the Nix shell fully covers the native build, and documents the Docker path in docs/user. --- .dockerignore | 9 +++ Dockerfile | 74 ++++++++++++++++++++++++ docs/user/README.md | 4 +- docs/user/requirements.md | 17 +++++- docs/user/setup-pipeline.md | 47 +++++++++++++-- scripts/netflow-db-docker.sh | 107 +++++++++++++++++++++++++++++++++++ shell.nix | 2 + 7 files changed, 250 insertions(+), 10 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100755 scripts/netflow-db-docker.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e08ed06 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.env +.env.* +data +datasets.json +node_modules +**/node_modules +target +vendor/nfdump diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3befbb8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,74 @@ +# syntax=docker/dockerfile:1 + +FROM ubuntu:24.04 AS build-base + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install --yes --no-install-recommends \ + autoconf \ + automake \ + bison \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + flex \ + git \ + libbz2-dev \ + libtool \ + pkg-config \ + python3 \ + tar \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /build + +FROM build-base AS nfdump-builder + +ARG NFDUMP_COMMIT + +# Clone the exact superproject gitlink commit. A copied submodule has a .git +# pointer into the host repository, so it cannot support compile-nfdump.sh. +RUN test -n "$NFDUMP_COMMIT" \ + || { echo "NFDUMP_COMMIT must name the vendor/nfdump gitlink commit" >&2; exit 1; } +RUN git clone https://github.com/flamboh/nfdump.git vendor/nfdump \ + && git -C vendor/nfdump checkout --detach "$NFDUMP_COMMIT" \ + && test "$(git -C vendor/nfdump rev-parse HEAD)" = "$NFDUMP_COMMIT" + +COPY vendor/scripts/compile-nfdump.sh vendor/scripts/compile-nfdump.sh +RUN ./vendor/scripts/compile-nfdump.sh + +FROM build-base AS rust-builder + +ENV PATH="/root/.cargo/bin:${PATH}" + +COPY rust-toolchain.toml rust-toolchain.toml +RUN curl --proto '=https' --tlsv1.2 --silent --show-error --fail \ + https://sh.rustup.rs \ + | sh -s -- --yes --profile minimal --default-toolchain none \ + && cargo --version + +COPY Cargo.toml Cargo.lock ./ +COPY tools/netflow-db/Cargo.toml tools/netflow-db/Cargo.toml +COPY tools/netflow-db/src tools/netflow-db/src +RUN cargo build --locked --release --package atlantis-netflow-db --bin netflow-db + +FROM ubuntu:24.04 AS runtime + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install --yes --no-install-recommends ca-certificates libbz2-1.0 \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=rust-builder /build/target/release/netflow-db /usr/local/bin/netflow-db +COPY --from=nfdump-builder /build/target/nfdump/libexec/nfdump /usr/local/bin/nfdump + +RUN --mount=type=bind,from=nfdump-builder,source=/build/target/nfdump/build/smoke/dummy_flows.nf,target=/tmp/dummy_flows.nf \ + netflow-db contract-version >/dev/null \ + && nfdump -V >/dev/null 2>&1 \ + && nfdump -G none -r /tmp/dummy_flows.nf -q -o atlantis 'host 203.0.113.255' >/tmp/atlantis.bin \ + && test "$(wc -c &2 <<'EOF' +usage: scripts/netflow-db-docker.sh [--build] [--capture-root PATH]... [--] + + --build Rebuild the local image before running the command. + --capture-root PATH Mount an absolute capture root read-only at the same path. + +Container options must appear before the netflow-db command. +EOF +} + +force_build=0 +capture_roots=() +while (($# > 0)); do + case "$1" in + --build) + force_build=1 + shift + ;; + --capture-root) + if (($# < 2)); then + echo "--capture-root requires a path" >&2 + usage + exit 2 + fi + capture_roots+=("$2") + shift 2 + ;; + --) + shift + break + ;; + *) + break + ;; + esac +done + +if (($# == 0)); then + usage + exit 2 +fi + +if ! command -v docker >/dev/null 2>&1; then + echo "docker is required; install Docker Engine or Docker Desktop" >&2 + exit 1 +fi + +for capture_root in "${capture_roots[@]}"; do + if [[ "$capture_root" != /* || "$capture_root" == "/" ]]; then + echo "capture root must be an absolute directory other than /: $capture_root" >&2 + exit 2 + fi + if [[ "$capture_root" == *,* ]]; then + echo "capture root cannot contain a comma: $capture_root" >&2 + exit 2 + fi + if [[ ! -d "$capture_root" ]]; then + echo "capture root is not a directory: $capture_root" >&2 + exit 2 + fi +done + +tree_entry="$(git -C "$ROOT_DIR" ls-tree HEAD -- vendor/nfdump)" +read -r submodule_mode submodule_type nfdump_commit submodule_path <<<"$tree_entry" +if [[ "$submodule_mode" != "160000" || "$submodule_type" != "commit" || "$submodule_path" != "vendor/nfdump" || ! "$nfdump_commit" =~ ^[0-9a-f]{40}$ ]]; then + echo "unable to read the pinned vendor/nfdump commit from the current Git tree" >&2 + exit 1 +fi + +if ((force_build)) || ! docker image inspect "$IMAGE" >/dev/null 2>&1; then + docker build \ + --build-arg "NFDUMP_COMMIT=$nfdump_commit" \ + --tag "$IMAGE" \ + "$ROOT_DIR" +fi + +mkdir -p "$ROOT_DIR/data" + +docker_args=( + run + --rm + --user "$(id -u):$(id -g)" + --env HOME=/tmp + --workdir /workspace + --mount "type=bind,source=$ROOT_DIR/data,target=/workspace/data" +) + +if [[ -f "$ROOT_DIR/datasets.json" ]]; then + docker_args+=( + --mount "type=bind,source=$ROOT_DIR/datasets.json,target=/workspace/datasets.json,readonly" + ) +fi + +for capture_root in "${capture_roots[@]}"; do + docker_args+=( + --mount "type=bind,source=$capture_root,target=$capture_root,readonly" + ) +done + +exec docker "${docker_args[@]}" "$IMAGE" "$@" diff --git a/shell.nix b/shell.nix index 02bf100..4e27246 100644 --- a/shell.nix +++ b/shell.nix @@ -11,6 +11,8 @@ mkShell { pkgs.bison pkgs.bun pkgs.flex + pkgs.gcc + pkgs.git pkgs.gnumake pkgs.gnutar pkgs.libtool From 5fe3bc7f72bbaec4be3a79238a3fce7b31fbb15b Mon Sep 17 00:00:00 2001 From: flamboh Date: Sun, 23 Aug 2026 17:43:27 -0700 Subject: [PATCH 2/3] docs: tighten the Docker pipeline section and use date placeholders Condenses the Docker prose to a command plus the two facts a user needs (capture-root mounting and output location). Replaces concrete example dates with placeholders across the docs so readers substitute a date their captures actually cover. --- README.md | 4 ++-- docs/code/pipeline-contract.md | 6 +++--- docs/user/README.md | 2 +- docs/user/operations.md | 4 ++-- docs/user/querying.md | 4 ++-- docs/user/requirements.md | 13 +++++------- docs/user/setup-pipeline.md | 38 ++++++++++++---------------------- 7 files changed, 28 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 96c042a..adbee0b 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,8 @@ First, install the [required tools](docs/user/requirements.md) — both Bun and ```bash ./scripts/netflow-db.sh pipeline \ --dataset example \ - --start-date 2025-02-01 \ - --end-date 2025-02-01 \ + --start-date \ + --end-date \ --nfdump target/nfdump/libexec/nfdump ``` diff --git a/docs/code/pipeline-contract.md b/docs/code/pipeline-contract.md index 32c735d..0ae77d2 100644 --- a/docs/code/pipeline-contract.md +++ b/docs/code/pipeline-contract.md @@ -90,9 +90,9 @@ The `extract-window` command creates bounded SQLite or Parquet analysis artifact ```bash ./scripts/netflow-db.sh extract-window \ --source-db data/uoregon/netflow.sqlite \ - --output-dir data/uoregon/extracts/2025-06 \ - --start 2025-06-01 \ - --end 2025-07-01 \ + --output-dir data/uoregon/extracts/ \ + --start \ + --end \ --output sqlite \ --output parquet ``` diff --git a/docs/user/README.md b/docs/user/README.md index 41b02ab..50687f3 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -11,7 +11,7 @@ Have this available: - nfcapd capture files on disk, in the layout that [dataset configuration](datasets.md#input-directory-layout) shows. - Approximately 30 minutes. The one-time tool builds take most of this time. -The pipeline can run in Docker, so it does not require a host Rust or C toolchain. The dashboard still runs natively. +The pipeline can run in Docker instead of a host Rust and C toolchain. The dashboard always runs natively. ## First use diff --git a/docs/user/operations.md b/docs/user/operations.md index 462d5d0..d3d6039 100644 --- a/docs/user/operations.md +++ b/docs/user/operations.md @@ -30,8 +30,8 @@ Compare a rebuilt candidate with a trusted historical database before you publis ./scripts/netflow-db.sh compare \ data/candidate/netflow.sqlite \ data/example/historical.sqlite \ - --start 2025-11-01 \ - --end 2025-12-01 + --start \ + --end ``` `--start` and `--end` are half-open local date or time boundaries. The default timezone is `America/Los_Angeles`. diff --git a/docs/user/querying.md b/docs/user/querying.md index 131acf6..c03647c 100644 --- a/docs/user/querying.md +++ b/docs/user/querying.md @@ -52,8 +52,8 @@ FROM traffic_stats WHERE granularity = '1d' AND src_visibility = 'all' AND dst_visibility = 'all' - AND bucket_start >= strftime('%s', '2025-01-01') - AND bucket_start < strftime('%s', '2025-01-08') + AND bucket_start >= strftime('%s', '') + AND bucket_start < strftime('%s', '') GROUP BY bucket_start, source_id ORDER BY bucket_start, source_id; ``` diff --git a/docs/user/requirements.md b/docs/user/requirements.md index f0dbf0b..6d9c0cd 100644 --- a/docs/user/requirements.md +++ b/docs/user/requirements.md @@ -2,18 +2,15 @@ This document lists the required tools by concern. Install the tools for the parts that you use. -For native setup, the repository has a `shell.nix` file that supplies every development tool below, including Git and a C compiler. With Nix, run `nix-shell` and skip the manual installation. You still need Git on the host to clone the repository before the shell is available. +For native setup, the repository has a `shell.nix` file that supplies every development tool below. -## Docker pipeline - -The Docker pipeline path needs only these host tools: +On NixOS, run `nix-shell` and skip the manual installation. -- Git -- Docker Engine or Docker Desktop +## Docker pipeline -The first run builds the image and downloads Ubuntu packages, the pinned Rust toolchain, Rust crates, and the pinned nfdump fork. You do not need to initialize Git submodules or install a compiler on the host. +Running the pipeline with Docker needs only Git and Docker on the host. The image build supplies the Rust toolchain, the nfdump build tools, and the pinned fork, and it does not need initialized Git submodules. -Docker covers the pipeline only. Run the dashboard natively with the tools in the next section. +Docker covers the pipeline only; the dashboard runs natively. ## Dashboard diff --git a/docs/user/setup-pipeline.md b/docs/user/setup-pipeline.md index 35b2062..16aa61c 100644 --- a/docs/user/setup-pipeline.md +++ b/docs/user/setup-pipeline.md @@ -8,32 +8,20 @@ Use a new output database when you change selection rules or result semantics. T ## Run the pipeline with Docker -Install the [Docker pipeline requirements](requirements.md#docker-pipeline), then configure `datasets.json` as described in [Dataset configuration](datasets.md). Keep `root_path` as the absolute host path to the capture tree. - -Pass that path to the wrapper before the pipeline arguments: +Install the [Docker pipeline requirements](requirements.md#docker-pipeline), then complete the [dataset configuration](datasets.md). Pass each capture root before the pipeline arguments: ```bash ./scripts/netflow-db-docker.sh \ - --capture-root /data/netflow/example \ + --capture-root /absolute/path/to/captures \ pipeline \ --dataset example \ - --start-date 2025-02-01 \ - --end-date 2025-02-01 + --start-date \ + --end-date ``` -The wrapper builds `atlantis-netflow-db:local` if it is missing. Pass `--build` after a source update to rebuild it. The image build clones the nfdump fork at the commit pinned by the repository, so host submodules may remain uninitialized. - -The wrapper mounts: - -- `data/` at `/workspace/data` with write access. The container uses your user and group IDs, so generated databases remain owned by you. -- `datasets.json` at `/workspace/datasets.json` with read-only access. -- Each `--capture-root` path at the identical absolute path inside the container, with read-only access. Pass the option more than once if a command reads more than one capture root. - -The identical capture mount means `root_path` in `datasets.json` works without changes. The image installs the fork as `/usr/local/bin/nfdump`, which is the default `nfdump` found on `PATH`. Docker commands do not need `--nfdump`. - -Keep Docker output paths under the repository's `data/` directory and write them as relative paths such as `data/example/netflow.sqlite`. The wrapper does not mount other host output directories. It also uses the repository's `datasets.json`; pass `--datasets /workspace/datasets.json` if a command needs the path explicitly. +The wrapper builds the image when it is missing; pass `--build` to rebuild it after a source update. Each `--capture-root` mounts read-only at the same absolute path inside the container, so `root_path` in `datasets.json` needs no change. Output stays under the repository's `data/` directory, owned by you. The image carries the nfdump fork on `PATH`, so Docker commands do not need `--nfdump`. -Docker Desktop bind mounts can be slower than native filesystem access on macOS, especially for large capture trees. +On macOS, bind mounts over large capture trees are slower than native filesystem access. ## Build the pipeline natively @@ -68,12 +56,12 @@ Run a bounded import while you test the configuration: ```bash ./scripts/netflow-db.sh pipeline \ --dataset example \ - --start-date 2025-02-01 \ - --end-date 2025-02-01 \ + --start-date \ + --end-date \ --nfdump target/nfdump/libexec/nfdump ``` -The start date and end date are inclusive, so this command processes one day. If you omit the end date, the pipeline processes each day through the latest available day. +The start date and end date are inclusive; use the same date for both to process a single day. If you omit the end date, the pipeline processes each day through the latest available day. Dataset mode calculates MAAD statistics by default. MAAD statistics describe the multifractal structure of the observed IPv4 address sets, and they power the address-structure charts. Use `--no-maad` to skip them. @@ -86,8 +74,8 @@ Selection conditions use AND logic. The IP prefix can match the source endpoint ```bash ./scripts/netflow-db.sh pipeline \ --dataset example \ - --start-date 2025-02-01 \ - --end-date 2025-02-01 \ + --start-date \ + --end-date \ --database-path data/example-public/netflow.sqlite \ --ip-prefix 192.0.2.0/24 \ --src-visibility literal \ @@ -125,7 +113,7 @@ Put flow selection in the top-level `selection` object: } ``` -For native nfcapd input, set the top-level `"nfdump"` value to `"target/nfdump/libexec/nfdump"`. You can also pass the same path with `--nfdump` when the configuration does not set it. For Docker, omit the setting or use `"/usr/local/bin/nfdump"`. +For native nfcapd input, set the top-level `"nfdump"` value to `"target/nfdump/libexec/nfdump"`. You can also pass the same path with `--nfdump` when the configuration does not set it. ## Common options @@ -157,6 +145,6 @@ Run the compatibility check after the pipeline finishes: The command prints an `OK` line when the database is compatible. A failed requirement returns a nonzero exit status. -Docker users can run the same check by replacing `./scripts/netflow-db.sh` with `./scripts/netflow-db-docker.sh`. Verification reads only `data/`, so it does not need `--capture-root`. +Docker users run the same check with `./scripts/netflow-db-docker.sh`; verification reads only `data/`. For pipeline identity and export rules, read the [pipeline contract](../code/pipeline-contract.md). From 831f1c7bb825197b3911917968a6870446c789b2 Mon Sep 17 00:00:00 2001 From: flamboh Date: Sun, 23 Aug 2026 18:00:44 -0700 Subject: [PATCH 3/3] docs: restructure pipeline setup by phase instead of by path Splits the document into one-time setup (Docker or native) and shared usage, so the dataset, selection, configuration, and verification sections apply to both paths instead of reading as native-only. States the Docker substitution once and drops --nfdump from the examples, which removes the duplicated pipeline command and the per-section Docker footnotes. --- docs/user/setup-pipeline.md | 52 ++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/docs/user/setup-pipeline.md b/docs/user/setup-pipeline.md index 16aa61c..f120b50 100644 --- a/docs/user/setup-pipeline.md +++ b/docs/user/setup-pipeline.md @@ -1,37 +1,35 @@ # Pipeline setup -The pipeline is the Rust `atlantis-netflow-db` crate. It converts nfcapd or CSV input into a compatible SQLite database. You can run it with Docker or build it natively. - -`scripts/netflow-db-docker.sh` runs the container path. `scripts/netflow-db.sh` runs the native path with `cargo run --locked --release`, which compiles it when necessary. Set `NETFLOW_DB_BIN` to make the native wrapper run a prebuilt binary instead. +The pipeline is the Rust `atlantis-netflow-db` crate. It converts nfcapd or CSV input into a compatible SQLite database. Use a new output database when you change selection rules or result semantics. The pipeline rejects incompatible reuse. -## Run the pipeline with Docker +## Choose a path + +| Path | Entry point | Host requirements | +| ------ | ------------------------------ | ------------------------------------------------------------------------------------------------------------------ | +| Docker | `scripts/netflow-db-docker.sh` | [Git and Docker](requirements.md#docker-pipeline) | +| Native | `scripts/netflow-db.sh` | [Rust toolchain](requirements.md#native-pipeline) and the [nfdump build tools](requirements.md#native-nfdump-fork) | + +Complete the one-time setup for your path, then follow the rest of this document. The examples below use `./scripts/netflow-db.sh`, and the native path adds `--nfdump` to every command that reads nfcapd input. To run an example with Docker, substitute `./scripts/netflow-db-docker.sh` and add `--capture-root ` for commands that read captures. -Install the [Docker pipeline requirements](requirements.md#docker-pipeline), then complete the [dataset configuration](datasets.md). Pass each capture root before the pipeline arguments: +### Docker setup + +Container options come before the pipeline command: ```bash -./scripts/netflow-db-docker.sh \ - --capture-root /absolute/path/to/captures \ - pipeline \ - --dataset example \ - --start-date \ - --end-date +./scripts/netflow-db-docker.sh --capture-root /absolute/path/to/captures pipeline ... ``` -The wrapper builds the image when it is missing; pass `--build` to rebuild it after a source update. Each `--capture-root` mounts read-only at the same absolute path inside the container, so `root_path` in `datasets.json` needs no change. Output stays under the repository's `data/` directory, owned by you. The image carries the nfdump fork on `PATH`, so Docker commands do not need `--nfdump`. +The wrapper builds the image when it is missing; pass `--build` to rebuild it after a source update. Each `--capture-root` mounts read-only at the same absolute path inside the container, so `root_path` in `datasets.json` needs no change. Output stays under the repository's `data/` directory, owned by you. On macOS, bind mounts over large capture trees are slower than native filesystem access. -## Build the pipeline natively - -The native path uses `scripts/netflow-db.sh` and needs the [native pipeline requirements](requirements.md#native-pipeline). - -### Build the nfdump fork +### Native setup -nfcapd input needs the pinned ATLANTIS nfdump fork. A system nfdump installation does not work: the pipeline uses an output mode that only the fork has. CSV input does not need nfdump. +`scripts/netflow-db.sh` runs the crate with `cargo run --locked --release`, which compiles it when necessary. Set `NETFLOW_DB_BIN` to run a prebuilt binary instead. -The build needs the [native nfdump fork tools](requirements.md#native-nfdump-fork). The build script checks for them and names any tool that is missing. +nfcapd input also needs the pinned ATLANTIS nfdump fork. A system nfdump installation does not work: the pipeline uses an output mode that only the fork has. CSV input does not need nfdump. 1. Initialize the Git submodules. @@ -39,15 +37,15 @@ The build needs the [native nfdump fork tools](requirements.md#native-nfdump-for git submodule update --init --recursive ``` -2. Build the fork. +2. Build the fork. The script checks for the [build tools](requirements.md#native-nfdump-fork) and names any tool that is missing. ```bash ./vendor/scripts/compile-nfdump.sh ``` -The build stages the executable at `target/nfdump/libexec/nfdump`. The `target` directory is disposable and git-ignored. Pass this path to each pipeline command with `--nfdump`; the pipeline does not find it automatically. +The build stages the executable at `target/nfdump/libexec/nfdump`. The `target` directory is disposable and git-ignored. Pass this path with `--nfdump` to every command that reads nfcapd input; the pipeline does not find it automatically. -## Process a dataset natively +## Process a dataset First, complete the [dataset configuration](datasets.md). @@ -57,8 +55,7 @@ Run a bounded import while you test the configuration: ./scripts/netflow-db.sh pipeline \ --dataset example \ --start-date \ - --end-date \ - --nfdump target/nfdump/libexec/nfdump + --end-date ``` The start date and end date are inclusive; use the same date for both to process a single day. If you omit the end date, the pipeline processes each day through the latest available day. @@ -78,8 +75,7 @@ Selection conditions use AND logic. The IP prefix can match the source endpoint --end-date \ --database-path data/example-public/netflow.sqlite \ --ip-prefix 192.0.2.0/24 \ - --src-visibility literal \ - --nfdump target/nfdump/libexec/nfdump + --src-visibility literal ``` A selected population is a different database product. Thus, selection options require an explicit `--database-path`. @@ -113,7 +109,7 @@ Put flow selection in the top-level `selection` object: } ``` -For native nfcapd input, set the top-level `"nfdump"` value to `"target/nfdump/libexec/nfdump"`. You can also pass the same path with `--nfdump` when the configuration does not set it. +On the native path, nfcapd input needs the fork path: set the top-level `"nfdump"` value to `"target/nfdump/libexec/nfdump"`, or pass `--nfdump` when the configuration does not set it. ## Common options @@ -145,6 +141,4 @@ Run the compatibility check after the pipeline finishes: The command prints an `OK` line when the database is compatible. A failed requirement returns a nonzero exit status. -Docker users run the same check with `./scripts/netflow-db-docker.sh`; verification reads only `data/`. - For pipeline identity and export rules, read the [pipeline contract](../code/pipeline-contract.md).