From 7795e467ea49ebd5982a6ee3ded83d552dafe132 Mon Sep 17 00:00:00 2001 From: filip Date: Fri, 4 Jul 2025 15:33:32 +0200 Subject: [PATCH 1/6] adding fluent-bit role --- roles/debian/fluent-bit/README.md | 66 ++++++++++++++++++ roles/debian/fluent-bit/defaults/main.yml | 8 +++ roles/debian/fluent-bit/tasks/main.yml | 68 +++++++++++++++++++ .../templates/fluent-bit.config.yml.j2 | 3 + .../templates/fluent-bit.service.j2 | 4 ++ 5 files changed, 149 insertions(+) create mode 100644 roles/debian/fluent-bit/README.md create mode 100644 roles/debian/fluent-bit/defaults/main.yml create mode 100644 roles/debian/fluent-bit/tasks/main.yml create mode 100644 roles/debian/fluent-bit/templates/fluent-bit.config.yml.j2 create mode 100644 roles/debian/fluent-bit/templates/fluent-bit.service.j2 diff --git a/roles/debian/fluent-bit/README.md b/roles/debian/fluent-bit/README.md new file mode 100644 index 000000000..d32496c5f --- /dev/null +++ b/roles/debian/fluent-bit/README.md @@ -0,0 +1,66 @@ +# Fluent-bit + +## Description + +Deploy [Fluent-bit](https://github.com/fluent/fluent-bit) using ansible. + +### Requirements + +Role expects to be provided with the following information: +* `fluentbit_main_config` - the main Fluent-bit configuration + +### Example +Minimum Fluent-bit config that will ingest journal and some file logs, then ship them to AWS CloudWatch. + +```yaml +fluentbit_main_config: + service: + flush: 5 + log_level: info + +parsers: + - name: json + format: json + time_key: time + time_format: '%d/%b/%Y:%H:%M:%S %z' + + pipeline: + inputs: + - name: dummy + dummy: '{"endpoint":"localhost", "value":"something"}' + tag: dummy + filters: + - name: grep + match: '*' + logical_op: or + regex: + - value something + - value error + outputs: + - name: stdout + +``` + +For more details on setting up the Fluent-bit config, refer to official documentation: +https://docs.fluentbit.io/manual/installation/getting-started-with-fluent-bitexporter + + + + + +## Default variables +```yaml +--- +# Default variables for Fluent-bit role +# Construct the download URL using the version variable. +fluent_bit_repo_key_url: https://packages.fluentbit.io/fluentbit.key +fluent_bit_key_location: /usr/share/keyrings/fluentbit-keyring.asc +fluent_bit_apt_source: "deb [signed-by=/usr/share/keyrings/fluentbit-keyring.asc] https://packages.fluentbit.io/debian/{{ ansible_distribution_release }} {{ ansible_distribution_release }} main" +fluentbit_startup_command: /opt/fluent-bit/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.yml + +# fluent-bit configuration +fluentbit_main_config: {} +# Example config +``` + + diff --git a/roles/debian/fluent-bit/defaults/main.yml b/roles/debian/fluent-bit/defaults/main.yml new file mode 100644 index 000000000..3d161cb79 --- /dev/null +++ b/roles/debian/fluent-bit/defaults/main.yml @@ -0,0 +1,8 @@ +--- +# Default variables for Fluent-bit role +fluent_bit_repo_key_url: https://packages.fluentbit.io/fluentbit.key +fluent_bit_key_location: /usr/share/keyrings/fluentbit-keyring.asc +fluent_bit_apt_source: "deb [signed-by=/usr/share/keyrings/fluentbit-keyring.asc] https://packages.fluentbit.io/debian/{{ ansible_distribution_release }} {{ ansible_distribution_release }} main" +fluentbit_startup_command: /opt/fluent-bit/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.yml + +fluent-bit_configuration: {} diff --git a/roles/debian/fluent-bit/tasks/main.yml b/roles/debian/fluent-bit/tasks/main.yml new file mode 100644 index 000000000..cf9e91717 --- /dev/null +++ b/roles/debian/fluent-bit/tasks/main.yml @@ -0,0 +1,68 @@ +--- +- name: Add the Fluent Bit server GPG key to system keyring and source repository to apt sources. + block: + - name: Get the repository key. + ansible.builtin.get_url: + url: "{{ fluent_bit_repo_key_url }}" + dest: "{{ fluent_bit_key_location }}" + + - name: Set apt source. + ansible.builtin.apt_repository: + repo: "{{ fluent_bit_apt_source }}" + state: present + +- name: Install Fluent-bit. + ansible.builtin.apt: + name: fluent-bit + state: present + +- name: Get list of configuration files that come with the pacakge installation. + ansible.builtin.find: + path: /etc/fluent-bit + patterns: .*conf + register: conf + +- name: Remove configuration files that come with the pacakge installation. + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + with_items: "{{ conf.files }}" + loop_control: + label: "{{ item.path }}" + +- name: Ensure that systemd override directory exists. + ansible.builtin.file: + path: "/etc/systemd/system/fluent-bit.service.d/" + state: directory + mode: '0755' + +- name: Ensure systemd service file override is in place. + ansible.builtin.template: + src: fluent-bit.service.j2 + dest: "/etc/systemd/system/fluent-bit.service.d/override.conf" + mode: '0644' + register: config_service + +- name: Create or update main configuration file. + ansible.builtin.template: + src: fluent-bit.config.yml.j2 + dest: "/etc/fluent-bit/fluent-bit.yml" + mode: '0644' + register: config + +- name: Reload systemd daemon. + ansible.builtin.systemd_service: + daemon_reload: true + when: config_service.changed + +- name: Restart Fluent-bit to apply config updates. + ansible.builtin.service: + name: "fluent-bit.service" + state: restarted + when: config.changed + +- name: Check that service is enabled and started. + ansible.builtin.systemd_service: + name: fluent-bit.service + enabled: true + state: started diff --git a/roles/debian/fluent-bit/templates/fluent-bit.config.yml.j2 b/roles/debian/fluent-bit/templates/fluent-bit.config.yml.j2 new file mode 100644 index 000000000..2476a888b --- /dev/null +++ b/roles/debian/fluent-bit/templates/fluent-bit.config.yml.j2 @@ -0,0 +1,3 @@ +{{ ansible_managed | comment }} + +{{ fluentbit_main_config | to_nice_yaml(indent=2, sort_keys=false) }} diff --git a/roles/debian/fluent-bit/templates/fluent-bit.service.j2 b/roles/debian/fluent-bit/templates/fluent-bit.service.j2 new file mode 100644 index 000000000..bc0103255 --- /dev/null +++ b/roles/debian/fluent-bit/templates/fluent-bit.service.j2 @@ -0,0 +1,4 @@ +{# ExecStart is used twice so that the variable gets cleared first before applying our variable #} +[Service] +ExecStart= +ExecStart={{ fluentbit_startup_command }} From 8b0dbf060a31fed46f46f311230d3f9ed5a375d2 Mon Sep 17 00:00:00 2001 From: filip Date: Mon, 7 Jul 2025 08:52:04 +0200 Subject: [PATCH 2/6] changing main config var from curly to brackets --- roles/debian/fluent-bit/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/debian/fluent-bit/defaults/main.yml b/roles/debian/fluent-bit/defaults/main.yml index 3d161cb79..d9f84c217 100644 --- a/roles/debian/fluent-bit/defaults/main.yml +++ b/roles/debian/fluent-bit/defaults/main.yml @@ -5,4 +5,4 @@ fluent_bit_key_location: /usr/share/keyrings/fluentbit-keyring.asc fluent_bit_apt_source: "deb [signed-by=/usr/share/keyrings/fluentbit-keyring.asc] https://packages.fluentbit.io/debian/{{ ansible_distribution_release }} {{ ansible_distribution_release }} main" fluentbit_startup_command: /opt/fluent-bit/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.yml -fluent-bit_configuration: {} +fluent-bit_configuration: [] From b8e51e84cfaa4c33f565508b5b56e767fa59d904 Mon Sep 17 00:00:00 2001 From: filip Date: Mon, 7 Jul 2025 08:54:21 +0200 Subject: [PATCH 3/6] updating readme --- roles/debian/fluent-bit/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/debian/fluent-bit/README.md b/roles/debian/fluent-bit/README.md index d32496c5f..8a0d6076e 100644 --- a/roles/debian/fluent-bit/README.md +++ b/roles/debian/fluent-bit/README.md @@ -10,7 +10,7 @@ Role expects to be provided with the following information: * `fluentbit_main_config` - the main Fluent-bit configuration ### Example -Minimum Fluent-bit config that will ingest journal and some file logs, then ship them to AWS CloudWatch. +Minimum Fluent-bit config that will send a test log, filter it, and output to stdout. ```yaml fluentbit_main_config: From fd1798bbe4187e5b687e43329a200ebd71b21c2f Mon Sep 17 00:00:00 2001 From: filip Date: Mon, 7 Jul 2025 08:55:51 +0200 Subject: [PATCH 4/6] setting main config var to empty --- roles/debian/fluent-bit/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/debian/fluent-bit/defaults/main.yml b/roles/debian/fluent-bit/defaults/main.yml index d9f84c217..61b60c095 100644 --- a/roles/debian/fluent-bit/defaults/main.yml +++ b/roles/debian/fluent-bit/defaults/main.yml @@ -5,4 +5,4 @@ fluent_bit_key_location: /usr/share/keyrings/fluentbit-keyring.asc fluent_bit_apt_source: "deb [signed-by=/usr/share/keyrings/fluentbit-keyring.asc] https://packages.fluentbit.io/debian/{{ ansible_distribution_release }} {{ ansible_distribution_release }} main" fluentbit_startup_command: /opt/fluent-bit/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.yml -fluent-bit_configuration: [] +fluent-bit_configuration: "" From 9c9258d0cace89f2eea67537ed823ba914c930f9 Mon Sep 17 00:00:00 2001 From: filip Date: Mon, 7 Jul 2025 08:59:15 +0200 Subject: [PATCH 5/6] moving access error nginx logs into nginx dir --- roles/debian/nginx/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/debian/nginx/defaults/main.yml b/roles/debian/nginx/defaults/main.yml index d74c13d56..0d5568b55 100644 --- a/roles/debian/nginx/defaults/main.yml +++ b/roles/debian/nginx/defaults/main.yml @@ -14,8 +14,8 @@ nginx: worker_connections: 768 http: server_names_hash_bucket_size: 256 - access_log: /var/log/nginx-access.log - error_log: /var/log/nginx-error.log + access_log: /var/log/nginx/nginx-access.log + error_log: /var/log/nginx/nginx-error.log ssl_protocols: "TLSv1.2 TLSv1.3" sendfile: "on" keepalive_timeout: 65 From ebd575b647e0ae1ce2c56d2653cecfe362980876 Mon Sep 17 00:00:00 2001 From: filip Date: Mon, 7 Jul 2025 09:04:37 +0200 Subject: [PATCH 6/6] updating variable names to be more cleaner --- roles/debian/fluent-bit/README.md | 2 +- roles/debian/fluent-bit/defaults/main.yml | 4 ++-- roles/debian/fluent-bit/templates/fluent-bit.config.yml.j2 | 2 +- roles/debian/fluent-bit/templates/fluent-bit.service.j2 | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/debian/fluent-bit/README.md b/roles/debian/fluent-bit/README.md index 8a0d6076e..92e1f1846 100644 --- a/roles/debian/fluent-bit/README.md +++ b/roles/debian/fluent-bit/README.md @@ -56,7 +56,7 @@ https://docs.fluentbit.io/manual/installation/getting-started-with-fluent-bitexp fluent_bit_repo_key_url: https://packages.fluentbit.io/fluentbit.key fluent_bit_key_location: /usr/share/keyrings/fluentbit-keyring.asc fluent_bit_apt_source: "deb [signed-by=/usr/share/keyrings/fluentbit-keyring.asc] https://packages.fluentbit.io/debian/{{ ansible_distribution_release }} {{ ansible_distribution_release }} main" -fluentbit_startup_command: /opt/fluent-bit/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.yml +fluent_bit_startup_command: /opt/fluent-bit/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.yml # fluent-bit configuration fluentbit_main_config: {} diff --git a/roles/debian/fluent-bit/defaults/main.yml b/roles/debian/fluent-bit/defaults/main.yml index 61b60c095..c1b188412 100644 --- a/roles/debian/fluent-bit/defaults/main.yml +++ b/roles/debian/fluent-bit/defaults/main.yml @@ -3,6 +3,6 @@ fluent_bit_repo_key_url: https://packages.fluentbit.io/fluentbit.key fluent_bit_key_location: /usr/share/keyrings/fluentbit-keyring.asc fluent_bit_apt_source: "deb [signed-by=/usr/share/keyrings/fluentbit-keyring.asc] https://packages.fluentbit.io/debian/{{ ansible_distribution_release }} {{ ansible_distribution_release }} main" -fluentbit_startup_command: /opt/fluent-bit/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.yml +fluent_bit_startup_command: /opt/fluent-bit/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.yml -fluent-bit_configuration: "" +fluent_bit_configuration: "" diff --git a/roles/debian/fluent-bit/templates/fluent-bit.config.yml.j2 b/roles/debian/fluent-bit/templates/fluent-bit.config.yml.j2 index 2476a888b..fb477fcbb 100644 --- a/roles/debian/fluent-bit/templates/fluent-bit.config.yml.j2 +++ b/roles/debian/fluent-bit/templates/fluent-bit.config.yml.j2 @@ -1,3 +1,3 @@ {{ ansible_managed | comment }} -{{ fluentbit_main_config | to_nice_yaml(indent=2, sort_keys=false) }} +{{ fluent_bit_configuration | to_nice_yaml(indent=2, sort_keys=false) }} diff --git a/roles/debian/fluent-bit/templates/fluent-bit.service.j2 b/roles/debian/fluent-bit/templates/fluent-bit.service.j2 index bc0103255..fc078f89f 100644 --- a/roles/debian/fluent-bit/templates/fluent-bit.service.j2 +++ b/roles/debian/fluent-bit/templates/fluent-bit.service.j2 @@ -1,4 +1,4 @@ {# ExecStart is used twice so that the variable gets cleared first before applying our variable #} [Service] ExecStart= -ExecStart={{ fluentbit_startup_command }} +ExecStart={{ fluent_bit_startup_command }}