diff --git a/container/config/detector_rule_config.yaml b/container/config/detector_rule_config.yaml new file mode 100644 index 00000000..148b715f --- /dev/null +++ b/container/config/detector_rule_config.yaml @@ -0,0 +1,9 @@ +detectors: + RuleDetector: + method_type: rule_detector + auto_config: false + # Rules run independently against every log line (no training needed). + rules: + # - rule: "R001 - TemplateNotFound" # alerts if the parser couldn't match the line to any template in templates.txt + - rule: "R003 - CheckForExceptions" # alerts if the log text contains "exception", "fail", "error" or "raise" + - rule: "R004 - ErrorLevelFound" # alerts if a parsed "Level" field equals "error" (not produced by this Nginx setup) diff --git a/container/config/detector_rule_settings.yaml b/container/config/detector_rule_settings.yaml new file mode 100644 index 00000000..c149456b --- /dev/null +++ b/container/config/detector_rule_settings.yaml @@ -0,0 +1,17 @@ +component_name: RuleDetector01 +component_type: RuleDetector # this is the component class name, the service will automatically search for the matching component config schema +log_level: "DEBUG" +log_dir: "/logs" + +# Manager Interface (Command Channel) +http_host: 0.0.0.0 + +# Engine Interface (Data Channel) +engine_addr: "ipc:///run/detector-rule.ipc" +engine_autostart: true + +# Output Destinations (where processed data is sent) +out_addr: + - "ipc:///run/output-rule.ipc" + +out_dial_timeout: 1000 diff --git a/container/config/parser_settings.yaml b/container/config/parser_settings.yaml index fb6a52f7..3a0dd84f 100644 --- a/container/config/parser_settings.yaml +++ b/container/config/parser_settings.yaml @@ -13,5 +13,6 @@ engine_autostart: true # Output Destinations (where processed data is sent) out_addr: - "ipc:///run/detector.ipc" + - "ipc:///run/detector-rule.ipc" out_dial_timeout: 1000 diff --git a/container/fluentout/fluent.conf b/container/fluentout/fluent.conf index 81867edc..20c857bc 100644 --- a/container/fluentout/fluent.conf +++ b/container/fluentout/fluent.conf @@ -26,6 +26,32 @@ +# Second source: the rule-based detector (detector-rule) dials its own +# ipc socket, since each is a 1:1 pynng.Pair0 connection and can't share +# the socket above with the NewValueDetector. + + @type nng_in + @id input_nng_rule + + @type protobuf + class_file /fluentd/etc/schemas_pb.rb + class_name DetectorSchema + protobuf_version protobuf3 + + uri ipc:///run/output-rule.ipc + tag nngrule.* + + + + @type file + path /fluentd/log/output-rule.%Y%m%d + compress text + append true + + flush_mode immediate + + + # # @type stdout # diff --git a/container/prometheus.yml b/container/prometheus.yml index e70525b4..b6744af5 100644 --- a/container/prometheus.yml +++ b/container/prometheus.yml @@ -4,4 +4,4 @@ global: scrape_configs: - job_name: 'prometheus' static_configs: - - targets: ['parser:8000', 'detector:8000'] + - targets: ['parser:8000', 'detector:8000', 'detector-rule:8000'] diff --git a/docker-compose.yml b/docker-compose.yml index 9ac07a5e..d2ad97a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,111 +1,124 @@ # version: "3" services: - fluentin: - #image: dm-fluentd:latest - build: - context: . - dockerfile: container/Dockerfile_fluentd - volumes: - - '$PWD/container/fluentin:/fluentd/etc' - - '/var/log/nginx:/fluentd/log' - - '$PWD/container/run:/run' - depends_on: - - parser - parser: - # image: detectmate:dev-0.1.6 - build: - context: . - args: - LIBRARY_EXTRAS: ${LIBRARY_EXTRAS:-full} - volumes: - - '$PWD/container/config:/config' - - '$PWD/container/logs:/logs' - - '$PWD/container/run:/run' - command: uv run detectmate --settings /config/parser_settings.yaml --config /config/parser_config.yaml - ports: - - "8001:8000" - depends_on: - - detector + fluentin: + #image: dm-fluentd:latest + build: + context: . + dockerfile: container/Dockerfile_fluentd + volumes: + - '$PWD/container/fluentin:/fluentd/etc' + - '/var/log/nginx:/fluentd/log' + - '$PWD/container/run:/run' + depends_on: + - parser + parser: + build: + context: . + args: + LIBRARY_EXTRAS: ${LIBRARY_EXTRAS:-full} + volumes: + - '$PWD/container/config:/config' + - '$PWD/container/logs:/logs' + - '$PWD/container/run:/run' + command: uv run detectmate --settings /config/parser_settings.yaml --config /config/parser_config.yaml + ports: + - "8001:8000" + depends_on: + - detector + - detector-rule - detector: - # image: detectmate:dev-0.1.6 - build: - context: . - args: - LIBRARY_EXTRAS: ${LIBRARY_EXTRAS:-full} - volumes: - - '$PWD/container/config:/config' - - '$PWD/container/logs:/logs' - - '$PWD/container/run:/run' - - '$PWD/container/state:/state' - command: uv run detectmate --settings /config/detector_settings.yaml --config /config/detector_config.yaml - ports: - - "8002:8000" - depends_on: - - fluentout + detector: + build: + context: . + args: + LIBRARY_EXTRAS: ${LIBRARY_EXTRAS:-full} + volumes: + - '$PWD/container/config:/config' + - '$PWD/container/logs:/logs' + - '$PWD/container/run:/run' + - '$PWD/container/state:/state' + command: uv run detectmate --settings /config/detector_settings.yaml --config /config/detector_config.yaml + ports: + - "8002:8000" + depends_on: + - fluentout - fluentout: - # image: dm-fluentd:latest - build: - context: . - dockerfile: container/Dockerfile_fluentd - volumes: - - '$PWD/container/fluentout:/fluentd/etc' - - '$PWD/container/fluentlogs:/fluentd/log' - - '$PWD/container/run:/run' + detector-rule: + build: + context: . + args: + LIBRARY_EXTRAS: ${LIBRARY_EXTRAS:-full} + volumes: + - '$PWD/container/config:/config' + - '$PWD/container/logs:/logs' + - '$PWD/container/run:/run' + command: uv run detectmate --settings /config/detector_rule_settings.yaml --config /config/detector_rule_config.yaml + ports: + - "8003:8000" + depends_on: + - fluentout - prometheus: - image: prom/prometheus:latest - container_name: prometheus - restart: unless-stopped - volumes: - - ./container/prometheus.yml:/etc/prometheus/prometheus.yml - - prometheus_data:/prometheus - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.console.libraries=/etc/prometheus/console_libraries' - - '--web.console.templates=/etc/prometheus/consoles' - - '--web.enable-lifecycle' - expose: - - 9090 + fluentout: + # image: dm-fluentd:latest + build: + context: . + dockerfile: container/Dockerfile_fluentd + volumes: + - '$PWD/container/fluentout:/fluentd/etc' + - '$PWD/container/fluentlogs:/fluentd/log' + - '$PWD/container/run:/run' - grafana: - image: grafana/grafana:latest - container_name: grafana - ports: - - "3000:3000" - environment: - - GF_SECURITY_ADMIN_PASSWORD=admin - depends_on: - - prometheus - volumes: - - ./container/grafana/prometheus.yml:/etc/grafana/provisioning/datasources/prometheus.yml - - ./container/grafana/provisioning/dashboards/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml - - ./container/grafana/dashboards:/var/lib/grafana/dashboards - - grafana_data:/var/lib/grafana + prometheus: + image: prom/prometheus:latest + container_name: prometheus + restart: unless-stopped + volumes: + - ./container/prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--web.enable-lifecycle' + expose: + - 9090 - # kafka: - # image: apache/kafka-native - # ports: - # - "9092:9092" - # environment: - # KAFKA_LISTENERS: CONTROLLER://localhost:9091,HOST://0.0.0.0:9092,DOCKER://0.0.0.0:9093 - # KAFKA_ADVERTISED_LISTENERS: DOCKER://kafka:9093,HOST://kafka:9092 - # KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,DOCKER:PLAINTEXT,HOST:PLAINTEXT - # - # # Settings required for KRaft mode - # KAFKA_NODE_ID: 1 - # KAFKA_PROCESS_ROLES: broker,controller - # KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER - # KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9091 - # - # # Listener to use for broker-to-broker communication - # KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER - # - # # Required for a single node cluster - # KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + grafana: + image: grafana/grafana:latest + container_name: grafana + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + depends_on: + - prometheus + volumes: + - ./container/grafana/prometheus.yml:/etc/grafana/provisioning/datasources/prometheus.yml + - ./container/grafana/provisioning/dashboards/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml + - ./container/grafana/dashboards:/var/lib/grafana/dashboards + - grafana_data:/var/lib/grafana + # kafka: + # image: apache/kafka-native + # ports: + # - "9092:9092" + # environment: + # KAFKA_LISTENERS: CONTROLLER://localhost:9091,HOST://0.0.0.0:9092,DOCKER://0.0.0.0:9093 + # KAFKA_ADVERTISED_LISTENERS: DOCKER://kafka:9093,HOST://kafka:9092 + # KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,DOCKER:PLAINTEXT,HOST:PLAINTEXT + # + # # Settings required for KRaft mode + # KAFKA_NODE_ID: 1 + # KAFKA_PROCESS_ROLES: broker,controller + # KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + # KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9091 + # + # # Listener to use for broker-to-broker communication + # KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER + # + # # Required for a single node cluster + # KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 volumes: prometheus_data: diff --git a/docs/docker-compose.md b/docs/docker-compose.md index 17652534..57a1c827 100644 --- a/docs/docker-compose.md +++ b/docs/docker-compose.md @@ -4,7 +4,7 @@ DetectMate Service ships a comprehensive Docker Compose configuration that inclu ![Illustration of a logpipeline](images/Detectmate.drawio.png "logpipeline") -Log file ingestion is handled by Fluentd. It supports reading from various systems and can convert the data to a specific format before sending it to any other target. Using the default setup, fluentd reads lines from a file and sends them to the DetectMate parser. The parser processes and forwards this data to the detector. If the detector finds an anomaly, it will send it to another Fluentd process that can communicate with various targets, such as Elasticsearch, Kafka, or a log file. +Log file ingestion is handled by Fluentd. It supports reading from various systems and can convert the data to a specific format before sending it to any other target. Using the default setup, fluentd reads lines from a file and sends them to the DetectMate parser. The parser processes the data and forwards it to **both** detector services shipped by default, `detector` and `detector-rule`. If either detector finds an anomaly, it sends it to the `fluentout` process, which can communicate with various targets, such as Elasticsearch, Kafka, or a log file. The following sections will explain the [docker-compose.yml](https://github.com/ait-detectmate/DetectMateService/blob/main/docker-compose.yml) service by service. @@ -42,13 +42,13 @@ Docker compose builds the `detectmate` image using the [Dockerfile](https://gith ### Volumes -All DetectMate container(`parser` and `detector`) need the following volumes mounted: +All DetectMate containers (`parser`, `detector` and `detector-rule`) need the following volumes mounted: - `container/config:/config` provides the DetectMate configuration. Each DetectMate Service needs one settings-file for the general service configuration and one config file for the specific DetectMate component. For parser these files are `parser_settings.yaml` for service settings and `parser_config.yaml` that holds the component-specific parameters. - `container/logs:/logs` exposes all log files that are generated by DetectMate services. Mostly needed for debugging. -- `container/run:/run` is mounted to hold the socket files for inter process communication. `fluentin` and `parser` communicate via the file `ipc:///run/parser.engine.ipc`. `parser` and `detector` communicate via the file `ipc:///run/detector.ipc`. +- `container/run:/run` is mounted to hold the socket files for inter process communication. `fluentin` and `parser` communicate via the file `ipc:///run/parser.engine.ipc`. `parser` fans the parsed data out to both detectors: `parser` and `detector` communicate via the file `ipc:///run/detector.ipc`, and `parser` and `detector-rule` communicate via `ipc:///run/detector-rule.ipc` (see `out_addr` in `parser_settings.yaml`). ### Ports @@ -56,20 +56,20 @@ The parser service exposes port **8001** (internally the service runs on port 80 ### Depends -This services depends on the `detector` service, because `detector` creates the socket file to establish communication. +This service depends on the `detector` and `detector-rule` services, because they create the socket files to establish communication. ## Detector -The detector service listens for parsed logs received from the `parser` service on a [Nano Message socket](https://nng.nanomsg.org/). It expects the data in the [ParserSchema format used by DetectMate](https://ait-detectmate.github.io/DetectMateLibrary/latest/schemas/) and sends anomalies using the [DetectorSchema](https://ait-detectmate.github.io/DetectMateLibrary/latest/schemas/) to another Nano Message socket provided by the `fluentout` service. +The detector service listens for parsed logs received from the `parser` service on a [Nano Message socket](https://nng.nanomsg.org/). It expects the data in the [ParserSchema format used by DetectMate](https://ait-detectmate.github.io/DetectMateLibrary/latest/schemas/) and sends anomalies using the [DetectorSchema](https://ait-detectmate.github.io/DetectMateLibrary/latest/schemas/) to another Nano Message socket provided by the `fluentout` service. This is the `NewValueDetector`, which needs training data before it can flag anomalies (see the [Getting Started](getting_started.md) tutorial). ### Building the image -Docker compose builds the `detectmate` image using the [Dockerfile](https://github.com/ait-detectmate/DetectMateService/blob/main/Dockerfile) in the root of the DetectMate repository. This image is the same for `parser` and `detector`. +Docker compose builds the `detectmate` image using the [Dockerfile](https://github.com/ait-detectmate/DetectMateService/blob/main/Dockerfile) in the root of the DetectMate repository. This image is the same for `parser`, `detector` and `detector-rule`. ### Volumes -All DetectMate container(`parser` and `detector`) need the following volumes mounted: +All DetectMate containers (`parser`, `detector` and `detector-rule`) need the following volumes mounted: - `container/config:/config` provides the DetectMate configuration. Each DetectMate Service needs one **settings** file for the general service configuration and one **config** file for the DetectMate component-specific parameters. For `detector` these files are `detector_settings.yaml` and `detector_config.yaml` respectively. @@ -77,20 +77,50 @@ All DetectMate container(`parser` and `detector`) need the following volumes mou - `container/run:/run` is mounted to hold the socket files for inter process communication. `parser` and `detector` communicate via the file `ipc:///run/detector.ipc` and `detector` and `fluentout` use the socket `ipc:///run/output.ipc`. +- `container/state:/state` persists the values `NewValueDetector` has learned during training, so it survives container restarts. + ### Ports The detector service exposes port **8002** (internally the service runs on port 8000). The management API server runs on this port. This service exposes metrics for the `prometheus` service and allows to start, stop and reconfigure the service. ### Depends -This services depends on the `fluentout` service, because `fluentout` creates the socket file to establish communication. +This service depends on the `fluentout` service, because `fluentout` creates the socket file to establish communication. + +## Detector (Rule-Based) + +The `detector-rule` service is a second detector that listens for the same parsed logs as `detector`, on its own socket. It runs the `RuleDetector`, which evaluates a fixed list of simple rules (e.g. "the parser found no matching template", "the log text contains a keyword like 'error'") against every log line instead of learning from training data. See the [Getting Started](getting_started.md) tutorial for a walkthrough that triggers one of its rules. + +### Building the image + +Docker compose builds the `detectmate` image using the same [Dockerfile](https://github.com/ait-detectmate/DetectMateService/blob/main/Dockerfile) used for `parser` and `detector`. + +### Volumes + +- `container/config:/config` provides `detector_rule_settings.yaml` and `detector_rule_config.yaml`. + +- `container/logs:/logs` exposes all log files that are generated by DetectMate services. Mostly needed for debugging. + +- `container/run:/run` is mounted to hold the socket files for inter process communication. `parser` and `detector-rule` communicate via `ipc:///run/detector-rule.ipc`, and `detector-rule` and `fluentout` use `ipc:///run/output-rule.ipc`. + +Unlike `detector`, `detector-rule` does not mount `container/state`, since the rule-based detector doesn't persist any learned state. + +### Ports + +The detector-rule service exposes port **8003** (internally the service runs on port 8000). + +### Depends + +This service depends on the `fluentout` service, because `fluentout` creates the socket file to establish communication. ## Fluentout -The `fluentout`service receives data from the `detector`, parses it from the proper detectmate protobuf format, and then writes the alerts to the logfiles `output.%Y%m%d`. +The `fluentout`service receives data from both `detector` and `detector-rule`, parses it from the proper detectmate protobuf format, and then writes the alerts to the logfiles `output.%Y%m%d` and `output-rule.%Y%m%d` respectively. While writing to a log file is just an example in this setup, you can refer to the [Fluentd documentation](https://docs.fluentd.org/) for guidance on writing alerts to other targets. +Since each detector's output socket is a 1:1 [Nano Message](https://nng.nanomsg.org/) `Pair0` connection, `fluentout` needs one ``/`` pair per detector rather than a single shared one - but both live in the same Fluentd process and container, so no second `fluentout` container is needed. + ### Building the image Docker Compose builds the `fluentout` image using the configuration in the file [container/Dockerfile_fluentd](https://github.com/ait-detectmate/DetectMateService/blob/main/container/Dockerfile_fluentd). This Dockerfile uses the official Fluentd image for the base and adds the necessary Fluentd plugins. @@ -99,22 +129,21 @@ Docker Compose builds the `fluentout` image using the configuration in the file `fluentout` mounts the following volumes using bind-mount. -- `container/fluentin:/fluentd/etc` provides the fluentd configuration. The file [container/fluentin/fluent.conf](https://github.com/ait-detectmate/DetectMateService/blob/main/container/fluentout/fluent.conf) can be used to reconfigure fluentd to write to other targets. - -- `container/fluentlogs:/fluentd/log` exposes the directory containing written log files. By default, `fluentout` writes all alerts to files named `output.%Y%m%d`. +- `container/fluentin:/fluentd/etc` provides the fluentd configuration. The file [container/fluentin/fluent.conf](https://github.com/ait-detectmate/DetectMateService/blob/main/container/fluentout/fluent.conf) contains one ``/`` pair for `detector` (`ipc:///run/output.ipc` → `output.%Y%m%d`) and one for `detector-rule` (`ipc:///run/output-rule.ipc` → `output-rule.%Y%m%d`), and can be used to reconfigure fluentd to write to other targets. -- `container/run:/run` is mounted to hold the socket files for inter process communication. `fluentout` and `detector` are talking in this configuration using [Nano Message Next Generation](https://nng.nanomsg.org/) via unix domain sockets for maximum performance. +- `container/fluentlogs:/fluentd/log` exposes the directory containing written log files. By default, `fluentout` writes alerts from `detector` to `output.%Y%m%d` and alerts from `detector-rule` to `output-rule.%Y%m%d`. +- `container/run:/run` is mounted to hold the socket files for inter process communication. `fluentout` talks to `detector` via `ipc:///run/output.ipc` and to `detector-rule` via `ipc:///run/output-rule.ipc`, both using [Nano Message Next Generation](https://nng.nanomsg.org/) via unix domain sockets for maximum performance. ## Prometheus -[Prometheus](https://prometheus.io/) is the monitoring server that collects performance metrics from the management api that runs on the `parser` and `detector` on **port 8000**. +[Prometheus](https://prometheus.io/) is the monitoring server that collects performance metrics from the management api that runs on the `parser`, `detector` and `detector-rule` on **port 8000**. ### Volumes `prometheus` mounts the following volumes: -- `container/prometheus.yml:/etc/prometheus/prometheus.yml` is the single prometheus configuration file, that tells prometheus to collect the metrics data from `parser`and `detector` at **port 8000***. +- `container/prometheus.yml:/etc/prometheus/prometheus.yml` is the single prometheus configuration file, that tells prometheus to collect the metrics data from `parser`, `detector` and `detector-rule` at **port 8000***. - `prometheus_data:/prometheus` holds the prometheus database. diff --git a/docs/getting_started.md b/docs/getting_started.md index 9215edcd..88732a18 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -185,13 +185,14 @@ Let's start the default pipeline, just to test it: ``` alice@ubuntu2404:~/DetectMateService$ sudo docker compose up -d -[+] up 6/6 - ✔ Container detectmateservice-fluentout-1 Started - ✔ Container prometheus Started s - ✔ Container grafana Started s - ✔ Container detectmateservice-detector-1 Started s - ✔ Container detectmateservice-parser-1 Started - ✔ Container detectmateservice-fluentin-1 Started +[+] up 7/7 + ✔ Container detectmateservice-fluentout-1 Started + ✔ Container prometheus Started s + ✔ Container grafana Started s + ✔ Container detectmateservice-detector-1 Started s + ✔ Container detectmateservice-detector-rule-1 Started s + ✔ Container detectmateservice-parser-1 Started + ✔ Container detectmateservice-fluentin-1 Started alice@ubuntu2404:~/DetectMateService$ ``` @@ -199,29 +200,33 @@ To check the status of the containers, we can use `docker compose ps`: ``` alice@ubuntu2404:~/DetectMateService$ sudo docker compose ps -NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS -detectmateservice-detector-1 detectmateservice-detector "uv run detectmate -…" detector 4 minutes ago Up 3 minutes 0.0.0.0:8002->8000/tcp, [::]:8002->8000/tcp -detectmateservice-fluentin-1 detectmateservice-fluentin "tini -- /bin/entryp…" fluentin 4 minutes ago Up 3 minutes 5140/tcp, 24224/tcp -detectmateservice-fluentout-1 detectmateservice-fluentout "tini -- /bin/entryp…" fluentout 4 minutes ago Up 3 minutes 5140/tcp, 24224/tcp -detectmateservice-parser-1 detectmateservice-parser "uv run detectmate -…" parser 4 minutes ago Up 3 minutes 0.0.0.0:8001->8000/tcp, [::]:8001->8000/tcp -grafana grafana/grafana:latest "/run.sh" grafana 4 minutes ago Up 3 minutes 0.0.0.0:3000->3000/tcp, [::]:3000->3000/tcp -prometheus prom/prometheus:latest "/bin/prometheus --c…" prometheus 4 minutes ago Up 3 minutes 9090/tcp +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +detectmateservice-detector-1 detectmateservice-detector "uv run detectmate -…" detector 4 minutes ago Up 3 minutes 0.0.0.0:8002->8000/tcp, [::]:8002->8000/tcp +detectmateservice-detector-rule-1 detectmateservice-detector-rule "uv run detectmate -…" detector-rule 4 minutes ago Up 3 minutes 0.0.0.0:8003->8000/tcp, [::]:8003->8000/tcp +detectmateservice-fluentin-1 detectmateservice-fluentin "tini -- /bin/entryp…" fluentin 4 minutes ago Up 3 minutes 5140/tcp, 24224/tcp +detectmateservice-fluentout-1 detectmateservice-fluentout "tini -- /bin/entryp…" fluentout 4 minutes ago Up 3 minutes 5140/tcp, 24224/tcp +detectmateservice-parser-1 detectmateservice-parser "uv run detectmate -…" parser 4 minutes ago Up 3 minutes 0.0.0.0:8001->8000/tcp, [::]:8001->8000/tcp +grafana grafana/grafana:latest "/run.sh" grafana 4 minutes ago Up 3 minutes 0.0.0.0:3000->3000/tcp, [::]:3000->3000/tcp +prometheus prom/prometheus:latest "/bin/prometheus --c…" prometheus 4 minutes ago Up 3 minutes 9090/tcp ``` +Notice that the pipeline has **two** detector services running side by side: `detector` (running `NewValueDetector`) and `detector-rule` (running `RuleDetector`). The `parser` sends every parsed log line to both, and `fluentout` writes each detector's alerts to its own output file (`output.%Y%m%d` and `output-rule.%Y%m%d`). + For now, we will shutdown all containers: ``` alice@ubuntu2404:~/DetectMateService$ sudo docker compose down -v [+] down 9/9 - ✔ Container grafana Removed - ✔ Container detectmateservice-fluentin-1 Removed - ✔ Container prometheus Removed - ✔ Container detectmateservice-parser-1 Removed - ✔ Container detectmateservice-detector-1 Removed - ✔ Container detectmateservice-fluentout-1 Removed - ✔ Volume detectmateservice_grafana_data Removed - ✔ Network detectmateservice_default Removed + ✔ Container grafana Removed + ✔ Container detectmateservice-fluentin-1 Removed + ✔ Container prometheus Removed + ✔ Container detectmateservice-parser-1 Removed + ✔ Container detectmateservice-detector-1 Removed + ✔ Container detectmateservice-detector-rule-1 Removed + ✔ Container detectmateservice-fluentout-1 Removed + ✔ Volume detectmateservice_grafana_data Removed + ✔ Network detectmateservice_default Removed ✔ Volume detectmateservice_prometheus_data Removed ``` @@ -252,7 +257,6 @@ services: - parser parser: - # image: detectmate:dev-0.1.6 build: . volumes: - '$PWD/container/config:/config' @@ -265,7 +269,6 @@ services: - detector detector: - # image: detectmate:dev-0.1.6 build: . volumes: - '$PWD/container/config:/config' @@ -278,6 +281,18 @@ services: depends_on: - fluentout + detector-rule: + build: . + volumes: + - '$PWD/container/config:/config' + - '$PWD/container/logs:/logs' + - '$PWD/container/run:/run' + command: uv run detectmate --settings /config/detector_rule_settings.yaml --config /config/detector_rule_config.yaml + ports: + - "8003:8000" + depends_on: + - fluentout + fluentout: # image: dm-fluentd:latest build: @@ -382,7 +397,7 @@ generate anomalies. ## DetectMate Config -The log pipeline uses two DetectMate services, parser and detector. The parser splits the log line into meaningful tokens, which the detector then uses to identify anomalies. We need to configure the parser and detector. Since the detector needs to know which tokens it receives from the parser so it can look for anomalies, the two configurations are closely related. +The log pipeline uses three DetectMate services: `parser`, `detector` and `detector-rule`. The parser splits the log line into meaningful tokens, which the two detectors then use to identify anomalies, each with a different strategy. We need to configure the parser and both detectors. Since the detectors need to know which tokens they receive from the parser so they can look for anomalies, all three configurations are closely related. ### Parser @@ -412,10 +427,16 @@ parsers: remove_spaces: false remove_punctuation: false lowercase: false - path_templates: /config/templates.txt # empty file because there are no templates necessary for apache access logs + path_templates: /config/templates.txt +``` + +`container/config/templates.txt` contains a single wildcard template that matches any well-formed access-log line: + +``` +<*> - - [<*>] "<*> <*> <*>" <*> <*> "<*>" "<*>" ``` -We don't need to modify that configuration, since it is compatible with the nginx access.log format and we can now continue with the configuration of the detector. +This means every normal Nginx request parses successfully. We will rely on this later: a log line that does **not** match this template is exactly what the rule-based detector's `TemplateNotFound` rule reacts to. We don't need to modify the parser configuration otherwise, since it is already compatible with the nginx access.log format. We can now continue with the configuration of the detectors. ### Detector @@ -435,30 +456,48 @@ detectors: Here, the `URL` token from the parsed data is monitored (`- pos: URL`), and the first two log lines are used for training (`data_use_training: 2`). Any subsequent log lines will be evaluated for anomalies and compared against the values seen during training on the first two log lines. +### Rule-Based Detector + +Next to the `NewValueDetector`, the pipeline also runs a second detector, `detector-rule`, using the `rule_detector` method. Unlike `NewValueDetector`, it needs no training: it evaluates a fixed list of simple rules against every log line, such as "no template was found by the parser" or "the log text contains a keyword like 'error' or 'exception'". Its configuration lives in `container/config/detector_rule_config.yaml`: + +``` +detectors: + RuleDetector: + method_type: rule_detector + auto_config: false + rules: + - rule: "R003 - CheckForExceptions" + - rule: "R004 - ErrorLevelFound" +``` +By default (when nothing is specified in the rule: block), R001, R003 and R004 are enabled. +With the config above only `R003` and `R004` are enabled, so the rule-based detector stays quiet while we work through the rest of this tutorial (plain Nginx access logs don't contain exception/error keywords, nor a `Level` field). At the very end of this tutorial, we will enable the `R001 - TemplateNotFound` rule and deliberately send a log line that cannot be parsed, to see the rule-based detector raise its own alert. + Now let's start the pipeline using `sudo docker compose up -d` and send two valid log lines with two different status values: ``` alice@ubuntu2404:~/DetectMateService$ sudo docker compose up -d [+] up 7/7 - ✔ Network detectmateservice_default Created - ✔ Container prometheus Started - ✔ Container detectmateservice-fluentout-1 Started - ✔ Container detectmateservice-detector-1 Started - ✔ Container grafana Started - ✔ Container detectmateservice-parser-1 Started - ✔ Container detectmateservice-fluentin-1 Started + ✔ Network detectmateservice_default Created + ✔ Container prometheus Started + ✔ Container detectmateservice-fluentout-1 Started + ✔ Container detectmateservice-detector-1 Started + ✔ Container detectmateservice-detector-rule-1 Started + ✔ Container grafana Started + ✔ Container detectmateservice-parser-1 Started + ✔ Container detectmateservice-fluentin-1 Started alice@ubuntu2404:~/DetectMateService$ sudo docker compose ps -NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS -detectmateservice-detector-1 detectmateservice-detector "uv run detectmate -…" detector 7 seconds ago Up 5 seconds 0.0.0.0:8002->8000/tcp, [::]:8002->8000/tcp -detectmateservice-fluentin-1 detectmateservice-fluentin "tini -- /bin/entryp…" fluentin 7 seconds ago Up 4 seconds 5140/tcp, 24224/tcp -detectmateservice-fluentout-1 detectmateservice-fluentout "tini -- /bin/entryp…" fluentout 8 seconds ago Up 6 seconds 5140/tcp, 24224/tcp -detectmateservice-parser-1 detectmateservice-parser "uv run detectmate -…" parser 7 seconds ago Up 5 seconds 0.0.0.0:8001->8000/tcp, [::]:8001->8000/tcp -grafana grafana/grafana:latest "/run.sh" grafana 7 seconds ago Up 5 seconds 0.0.0.0:3000->3000/tcp, [::]:3000->3000/tcp -prometheus prom/prometheus:latest "/bin/prometheus --c…" prometheus 8 seconds ago Up 6 seconds 9090/tcp +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +detectmateservice-detector-1 detectmateservice-detector "uv run detectmate -…" detector 7 seconds ago Up 5 seconds 0.0.0.0:8002->8000/tcp, [::]:8002->8000/tcp +detectmateservice-detector-rule-1 detectmateservice-detector-rule "uv run detectmate -…" detector-rule 7 seconds ago Up 5 seconds 0.0.0.0:8003->8000/tcp, [::]:8003->8000/tcp +detectmateservice-fluentin-1 detectmateservice-fluentin "tini -- /bin/entryp…" fluentin 7 seconds ago Up 4 seconds 5140/tcp, 24224/tcp +detectmateservice-fluentout-1 detectmateservice-fluentout "tini -- /bin/entryp…" fluentout 8 seconds ago Up 6 seconds 5140/tcp, 24224/tcp +detectmateservice-parser-1 detectmateservice-parser "uv run detectmate -…" parser 7 seconds ago Up 5 seconds 0.0.0.0:8001->8000/tcp, [::]:8001->8000/tcp +grafana grafana/grafana:latest "/run.sh" grafana 7 seconds ago Up 5 seconds 0.0.0.0:3000->3000/tcp, [::]:3000->3000/tcp +prometheus prom/prometheus:latest "/bin/prometheus --c…" prometheus 8 seconds ago Up 6 seconds 9090/tcp alice@ubuntu2404:~/DetectMateService$ ``` -**Wait a couple of minutes until parser and detector containers are up and running.** You can check by executing `sudo docker compose logs parser` or `sudo docker compose logs detector`. +**Wait a couple of minutes until parser and detector containers are up and running.** You can check by executing `sudo docker compose logs parser`, `sudo docker compose logs detector` or `sudo docker compose logs detector-rule`. When the containers are ready, the output of the component will show `Uvicorn running on` or any HTTP-requests for the `/metrics` endpoint: ``` @@ -514,7 +553,53 @@ alice@ubuntu2404:~/DetectMateService$ Great! We detected our first anomaly. +If you check `container/fluentlogs/output-rule.%Y%m%d` at this point, you'll find it empty (or missing entirely). The rule-based detector has stayed quiet the whole time, since none of the requests we sent contain an exception/error keyword. We'll change that in the next section. + + + +## Triggering the Rule-Based Detector + +So far, `detector-rule` has been running quietly next to `detector`, since none of our requests matched any of its enabled rules. To see it raise an alert, we'll enable the `R001 - TemplateNotFound` rule and then generate a log line that the parser cannot match against the template in `container/config/templates.txt`. + +Edit `container/config/detector_rule_config.yaml` and add the rule "R001 - TemplateNotFound": + +``` +detectors: + RuleDetector: + method_type: rule_detector + auto_config: false + rules: + - rule: "R001 - TemplateNotFound" + - rule: "R003 - CheckForExceptions" + - rule: "R004 - ErrorLevelFound" +``` + +Restart just the `detector-rule` service to pick up the change: + +``` +alice@ubuntu2404:~/DetectMateService$ sudo docker compose restart detector-rule +[+] Restarting 1/1 + ✔ Container detectmateservice-detector-rule-1 Started +``` + +Now append a contrived log line to `/var/log/nginx/access.log` that doesn't match the Nginx access log template at all, for example a line without the expected quotes and brackets: + +``` +alice@ubuntu2404:~/DetectMateService$ echo 'this line does not match the configured nginx log format at all' | sudo tee -a /var/log/nginx/access.log +``` + +Since this line can't be matched against `<*> - - [<*>] "<*> <*> <*>" <*> <*> "<*>" "<*>"`, the parser assigns it `EventID: -1`, which is exactly what the `TemplateNotFound` rule checks for. Have a look at `container/fluentlogs/output-rule.%Y%m%d`: + +``` +alice@ubuntu2404:~/DetectMateService$ sudo cat container/fluentlogs/output-rule.20260713.log +2026-07-13T12:03:55+00:00 nng.* {"__version__":"1.0.0","detectorID":"RuleDetector","detectorType":"rule_detector","alertID":"10","detectionTimestamp":1783944235,"logIDs":["5269c4a0-304c-431d-809b-ae34a15aa68b"],"score":1.0,"extractedTimestamps":[0],"description":"","receivedTimestamp":1783944235,"alertsObtain":{"R001 - TemplateNotFound":"No template found by parser"}} +``` + +The rule-based detector caught it: `"R001 - TemplateNotFound":"No template found by parser"`. Unlike `NewValueDetector`, it needed no training data at all — it was ready to alert from the very first log line. + +## Grafana UI + The Grafana UI is accessible at http://localhost:3000 (default login credentials in this demo are admin/admin) and the raw Prometheus metrics can be explored under "Drilldown" → "Metrics". Under "Dashboards" is a basic Dashboard with graphs for Throughput, Latency, Processing rate and Engine state. -This was a very basic example, but it shows how to easily deploy a full log data anomaly pipeline, including two DetectMate services, using a parser for the Nginx access log format, and how this is then used to flag an anomaly. +This was a very basic example, but it shows how to easily deploy a full log data anomaly pipeline, including two different DetectMate detectors, using a parser for the Nginx access log format, and how this is then used to flag anomalies. \ No newline at end of file diff --git a/docs/images/Detectmate.drawio.png b/docs/images/Detectmate.drawio.png index 25180aaf..021e5247 100644 Binary files a/docs/images/Detectmate.drawio.png and b/docs/images/Detectmate.drawio.png differ