Lesson 30 submission: EFK logging with regexp and grok parsing

lesson30/docker-compose.yml

@@ -0,0 +1,99 @@
+services:
+  ui:
+    image: ghcr.io/codeby-devops/ui:logging
+    container_name: ui
+    ports:
+      - "9292:9292/tcp"
+    environment:
+      - POST_SERVICE_HOST=post
+      - POST_SERVICE_PORT=5000
+      - COMMENT_SERVICE_HOST=comment
+      - COMMENT_SERVICE_PORT=9292
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: "127.0.0.1:24224"
+        tag: "service.ui"
+    depends_on:
+      - comment
+      - post
+      - fluentd
+    networks:
+      - logging
+
+  post:
+    image: ghcr.io/codeby-devops/post:logging
+    container_name: post
+    environment:
+      - POST_DATABASE_HOST=post_db
+      - POST_DATABASE=posts
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: "127.0.0.1:24224"
+        tag: "service.post"
+    depends_on:
+      - post_db
+      - fluentd
+    ports:
+      - 5000:5000
+    networks:
+      - logging
+
+  comment:
+    image: ghcr.io/codeby-devops/comment:logging
+    container_name: comment
+    environment:
+      - COMMENT_DATABASE_HOST=post_db
+      - COMMENT_DATABASE=comments
+    depends_on:
+      - post_db
+    networks:
+      - logging
+
+  post_db:
+    image: mongo:4.4
+    container_name: post_db
+    volumes:
+      - ./data/post_db:/data/db
+    networks:
+      - logging
+
+  elasticsearch:
+    container_name: elasticsearch
+    image: elasticsearch:6.8.20
+    environment:
+      - discovery.type=single-node
+      - ES_JAVA_OPTS=-Xms512m -Xmx512m
+    ports:
+      - "9200:9200"
+    networks:
+      - logging
+
+  kibana:
+    container_name: kibana
+    image: kibana:6.8.20
+    ports:
+      - "5601:5601"
+    depends_on:
+      - elasticsearch
+    networks:
+      - logging
+
+  fluentd:
+    container_name: fluentd
+    build:
+      context: ./fluentd-image
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"
+    volumes:
+      - ./fluentd/fluent.conf:/fluentd/etc/fluent.conf
+    depends_on:
+      - elasticsearch
+    networks:
+      - logging
+
+networks:
+  logging:
+    driver: bridge

lesson30/fluentd-image/Dockerfile

@@ -0,0 +1,9 @@
+FROM fluent/fluentd:v1.19-debian-1
+
+USER root
+RUN fluent-gem install fluent-plugin-elasticsearch -v 6.0.0 \
+    && gem uninstall elasticsearch -aIx || true \
+    && gem install elasticsearch -v 6.8.0 \
+    && fluent-gem install fluent-plugin-grok-parser
+
+USER fluent

lesson30/fluentd/fluent.conf

@@ -0,0 +1,53 @@
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+
+<filter service.post>
+  @type parser
+  format json
+  key_name log
+</filter>
+
+<filter service.ui>
+  @type parser
+  key_name log
+  <parse>
+    @type regexp
+    expression /^I,\s+\[(?<time>[^\]]+)\]\s+(?<level>\S+)\s+--\s+:\s+service=(?<service>\S+)\s+\|\s+event=(?<event>\S+)(?:\s+\|\s+path=(?<path>[^|]+))?\s+\|\s+request_id=(?<request_id>[^ ]+)(?:\s+\|\s+remote_addr=(?<remote_addr>[^ ]+))?(?:\s+\|\s+method=\s*(?<method>\S+))?(?:\s+\|\s+response_status=(?<response_status>\d+))?\s+\|\s+message='(?<message>[^']+)'\s+\|\s+params:\s*(?<params>.*)$/
+  </parse>
+</filter>
+
+<filter service.ui>
+  @type parser
+  key_name log
+  reserve_data true
+  emit_invalid_record_to_error false
+  <parse>
+    @type grok
+    grok_name_key parsed_by
+    grok_failure_key grok_failure
+    grok_pattern I,\s+\[%{TIMESTAMP_ISO8601:time}\s+#%{INT:pid}\]\s+%{LOGLEVEL:level}\s+--\s+:\s+service=%{WORD:service}\s+\|\s+event=%{WORD:event}\s+\|\s+request_id=%{UUID:request_id}\s+\|\s+message='%{DATA:message}'\s+\|\s+params:\s+%{GREEDYDATA:params}
+  </parse>
+</filter>
+
+<match *.**>
+  @type copy
+  <store>
+    @type elasticsearch
+    host elasticsearch
+    port 9200
+    logstash_format true
+    suppress_type_name true
+    logstash_prefix fluentd
+    logstash_dateformat %Y%m%d
+    include_tag_key true
+    type_name access_log
+    tag_key @log_name
+    flush_interval 1s
+  </store>
+  <store>
+    @type stdout
+  </store>
+</match>