Feature/add rule based detector to getting started

container/config/detector_rule_config.yaml

@@ -0,0 +1,9 @@
+detectors:
+  RuleDetector:
+   method_type: rule_detector
+   auto_config: false
+   # Rules run independently against every log line (no training needed).
+   rules:
+     # - rule: "R001 - TemplateNotFound"       # alerts if the parser couldn't match the line to any template in templates.txt
+     - rule: "R003 - CheckForExceptions"     # alerts if the log text contains "exception", "fail", "error" or "raise"
+     - rule: "R004 - ErrorLevelFound"        # alerts if a parsed "Level" field equals "error" (not produced by this Nginx setup)

container/config/detector_rule_settings.yaml

@@ -0,0 +1,17 @@
+component_name: RuleDetector01
+component_type: RuleDetector  # this is the component class name, the service will automatically search for the matching component config schema
+log_level: "DEBUG"
+log_dir: "/logs"
+
+# Manager Interface (Command Channel)
+http_host: 0.0.0.0
+
+# Engine Interface (Data Channel)
+engine_addr: "ipc:///run/detector-rule.ipc"
+engine_autostart: true
+
+# Output Destinations (where processed data is sent)
+out_addr:
+  - "ipc:///run/output-rule.ipc"
+
+out_dial_timeout: 1000

container/config/parser_settings.yaml

@@ -13,5 +13,6 @@ engine_autostart: true
 # Output Destinations (where processed data is sent)
 out_addr:
   - "ipc:///run/detector.ipc"
+  - "ipc:///run/detector-rule.ipc"
 
 out_dial_timeout: 1000

container/fluentout/fluent.conf

@@ -26,6 +26,32 @@
   </buffer>
 </match>
 
+# Second source: the rule-based detector (detector-rule) dials its own
+# ipc socket, since each is a 1:1 pynng.Pair0 connection and can't share
+# the socket above with the NewValueDetector.
+<source>
+  @type nng_in
+  @id input_nng_rule
+  <parse>
+    @type protobuf
+    class_file /fluentd/etc/schemas_pb.rb
+    class_name DetectorSchema
+    protobuf_version protobuf3
+  </parse>
+  uri ipc:///run/output-rule.ipc
+  tag nngrule.*
+</source>
+
+<match nngrule.**>
+  @type file
+  path /fluentd/log/output-rule.%Y%m%d
+  compress text
+  append true
+  <buffer>
+    flush_mode immediate
+  </buffer>
+</match>
+
 # <match nng.**>
 #   @type stdout
 # </match>

container/prometheus.yml

@@ -4,4 +4,4 @@ global:
 scrape_configs:
   - job_name: 'prometheus'
     static_configs:
-      - targets: ['parser:8000', 'detector:8000']
+      - targets: ['parser:8000', 'detector:8000', 'detector-rule:8000']

docker-compose.yml

@@ -1,111 +1,124 @@
 # version: "3"
 
 services:
-    fluentin:
-      #image: dm-fluentd:latest
-      build:
-        context: .
-        dockerfile: container/Dockerfile_fluentd
-      volumes:
-        - '$PWD/container/fluentin:/fluentd/etc'
-        - '/var/log/nginx:/fluentd/log'
-        - '$PWD/container/run:/run'
-      depends_on:
-        - parser
-    parser:
-      # image: detectmate:dev-0.1.6
-      build:
-        context: .
-        args:
-          LIBRARY_EXTRAS: ${LIBRARY_EXTRAS:-full}
-      volumes:
-        - '$PWD/container/config:/config'
-        - '$PWD/container/logs:/logs'
-        - '$PWD/container/run:/run'
-      command: uv run detectmate --settings /config/parser_settings.yaml --config /config/parser_config.yaml
-      ports:
-        - "8001:8000"
-      depends_on:
-        - detector
+  fluentin:
+    #image: dm-fluentd:latest
+    build:
+      context: .
+      dockerfile: container/Dockerfile_fluentd
+    volumes:
+      - '$PWD/container/fluentin:/fluentd/etc'
+      - '/var/log/nginx:/fluentd/log'
+      - '$PWD/container/run:/run'
+    depends_on:
+      - parser
+  parser:
+    build:
+      context: .
+      args:
+        LIBRARY_EXTRAS: ${LIBRARY_EXTRAS:-full}
+    volumes:
+      - '$PWD/container/config:/config'
+      - '$PWD/container/logs:/logs'
+      - '$PWD/container/run:/run'
+    command: uv run detectmate --settings /config/parser_settings.yaml --config /config/parser_config.yaml
+    ports:
+      - "8001:8000"
+    depends_on:
+      - detector
+      - detector-rule
 
-    detector:
-      # image: detectmate:dev-0.1.6
-      build:
-        context: .
-        args:
-          LIBRARY_EXTRAS: ${LIBRARY_EXTRAS:-full}
-      volumes:
-        - '$PWD/container/config:/config'
-        - '$PWD/container/logs:/logs'
-        - '$PWD/container/run:/run'
-        - '$PWD/container/state:/state'
-      command: uv run detectmate --settings /config/detector_settings.yaml --config /config/detector_config.yaml
-      ports:
-        - "8002:8000"
-      depends_on:
-        - fluentout
+  detector:
+    build:
+      context: .
+      args:
+        LIBRARY_EXTRAS: ${LIBRARY_EXTRAS:-full}
+    volumes:
+      - '$PWD/container/config:/config'
+      - '$PWD/container/logs:/logs'
+      - '$PWD/container/run:/run'
+      - '$PWD/container/state:/state'
+    command: uv run detectmate --settings /config/detector_settings.yaml --config /config/detector_config.yaml
+    ports:
+      - "8002:8000"
+    depends_on:
+      - fluentout
 
-    fluentout:
-      # image: dm-fluentd:latest
-      build:
-        context: .
-        dockerfile: container/Dockerfile_fluentd
-      volumes:
-        - '$PWD/container/fluentout:/fluentd/etc'
-        - '$PWD/container/fluentlogs:/fluentd/log'
-        - '$PWD/container/run:/run'
+  detector-rule:
+    build:
+      context: .
+      args:
+        LIBRARY_EXTRAS: ${LIBRARY_EXTRAS:-full}
+    volumes:
+      - '$PWD/container/config:/config'
+      - '$PWD/container/logs:/logs'
+      - '$PWD/container/run:/run'
+    command: uv run detectmate --settings /config/detector_rule_settings.yaml --config /config/detector_rule_config.yaml
+    ports:
+      - "8003:8000"
+    depends_on:
+      - fluentout
 
-    prometheus:
-      image: prom/prometheus:latest
-      container_name: prometheus
-      restart: unless-stopped
-      volumes:
-        - ./container/prometheus.yml:/etc/prometheus/prometheus.yml
-        - prometheus_data:/prometheus
-      command:
-        - '--config.file=/etc/prometheus/prometheus.yml'
-        - '--storage.tsdb.path=/prometheus'
-        - '--web.console.libraries=/etc/prometheus/console_libraries'
-        - '--web.console.templates=/etc/prometheus/consoles'
-        - '--web.enable-lifecycle'
-      expose:
-        - 9090
+  fluentout:
+    # image: dm-fluentd:latest
+    build:
+      context: .
+      dockerfile: container/Dockerfile_fluentd
+    volumes:
+      - '$PWD/container/fluentout:/fluentd/etc'
+      - '$PWD/container/fluentlogs:/fluentd/log'
+      - '$PWD/container/run:/run'
 
-    grafana:
-      image: grafana/grafana:latest
-      container_name: grafana
-      ports:
-        - "3000:3000"
-      environment:
-        - GF_SECURITY_ADMIN_PASSWORD=admin
-      depends_on:
-        - prometheus
-      volumes:
-        - ./container/grafana/prometheus.yml:/etc/grafana/provisioning/datasources/prometheus.yml
-        - ./container/grafana/provisioning/dashboards/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml
-        - ./container/grafana/dashboards:/var/lib/grafana/dashboards
-        - grafana_data:/var/lib/grafana
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    restart: unless-stopped
+    volumes:
+      - ./container/prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--web.enable-lifecycle'
+    expose:
+      - 9090
 
-          #    kafka:
-          #      image: apache/kafka-native
-          #      ports:
-          #        - "9092:9092"
-          #      environment:
-          #        KAFKA_LISTENERS: CONTROLLER://localhost:9091,HOST://0.0.0.0:9092,DOCKER://0.0.0.0:9093
-          #        KAFKA_ADVERTISED_LISTENERS: DOCKER://kafka:9093,HOST://kafka:9092
-          #        KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,DOCKER:PLAINTEXT,HOST:PLAINTEXT
-          #
-          #        # Settings required for KRaft mode
-          #        KAFKA_NODE_ID: 1
-          #        KAFKA_PROCESS_ROLES: broker,controller
-          #        KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
-          #        KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9091
-          #
-          #        # Listener to use for broker-to-broker communication
-          #        KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER
-          #
-          #        # Required for a single node cluster
-          #        KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    depends_on:
+      - prometheus
+    volumes:
+      - ./container/grafana/prometheus.yml:/etc/grafana/provisioning/datasources/prometheus.yml
+      - ./container/grafana/provisioning/dashboards/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml
+      - ./container/grafana/dashboards:/var/lib/grafana/dashboards
+      - grafana_data:/var/lib/grafana
+        #    kafka:
+        #      image: apache/kafka-native
+        #      ports:
+        #        - "9092:9092"
+        #      environment:
+        #        KAFKA_LISTENERS: CONTROLLER://localhost:9091,HOST://0.0.0.0:9092,DOCKER://0.0.0.0:9093
+        #        KAFKA_ADVERTISED_LISTENERS: DOCKER://kafka:9093,HOST://kafka:9092
+        #        KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,DOCKER:PLAINTEXT,HOST:PLAINTEXT
+        #
+        #        # Settings required for KRaft mode
+        #        KAFKA_NODE_ID: 1
+        #        KAFKA_PROCESS_ROLES: broker,controller
+        #        KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+        #        KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9091
+        #
+        #        # Listener to use for broker-to-broker communication
+        #        KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER
+        #
+        #        # Required for a single node cluster
+        #        KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
 
 volumes:
   prometheus_data: