feat: add custom Fluentd filters and flexible volume support to lm-logs chart and mandatory clustername

charts/lm-logs/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 description: A Helm chart for sending k8s logs to Logic Monitor
 name: lm-logs
-version: 1.0.1
+version: 1.1.0
 maintainers:
   - email: dev@logicmonitor.com
     name: LogicMonitor

charts/lm-logs/README.md

@@ -13,6 +13,7 @@ helm install -n <namespace> \
 --set lm_company_name="<lm_company_name>" \
 --set lm_access_id="<lm_access_id>" \
 --set lm_access_key="<lm_access_key>" \
+--set kubernetes.cluster_name="<cluster_name>" \
 lm-logs logicmonitor/lm-logs
 ```
 
@@ -22,11 +23,13 @@ Install the lm-logs chart using a user defined secret for LM credentials.
 # For LMv1
 helm install lm-logs logicmonitor/lm-logs -n <namespace> \
   --set global.userDefinedSecret=lm-logs-credentials \
+  --set kubernetes.cluster_name="<cluster_name>" \
   --set authMode=lmv1
 
 # For Bearer
 helm install lm-logs logicmonitor/lm-logs -n <namespace> \
   --set global.userDefinedSecret=lm-logs-credentials \
+  --set kubernetes.cluster_name="<cluster_name>" \
   --set authMode=bearer
 ```
 
@@ -69,7 +72,7 @@ The following tables lists the configurable parameters of the lm-logs chart and
 | `affinity`                  | Affinity for pod assignment		                | `{}`  (evaluated as a template)                         |
 | `env`                       | Map to add extra environment variables	        | `{}`                                                    |
 | `kubernetes.multiline_start_regexp` | Regexp to match beginning of multiline	| `/^\[(\d{4}-)?\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}.*\]/` |
-| `kubernetes.cluster_name`       | ClusterName given while adding k8s cluster  	| `""`                                                |
+| `kubernetes.cluster_name`       | (Required) Cluster name given while adding k8s cluster. Must be set for both standalone lm-logs and lm-container with lm-logs.  	| `""`                                                |
 | `kubernetes.multiline_concat_key`       | Key to look for fluentD to concatenate multiline logs   	| `"log"`                     |
 
 
@@ -166,6 +169,7 @@ helm upgrade --install -n <namespace> \
 --set lm_company_name="<company>" \
 --set lm_access_id="<access_id>" \
 --set lm_access_key="<access_key"> \
+--set kubernetes.cluster_name="<cluster_name>" \
 --set env.FLUENT_CONTAINER_TAIL_PARSER_TYPE="cri" \
 --set kubernetes.multiline_concat_key="message" \
 lm-logs logicmonitor/lm-logs

charts/lm-logs/ci/base-sanity-values.yaml

@@ -2,6 +2,8 @@ lm_company_name: "dummy_company"
 lm_access_id: "dummy_id"
 lm_access_key: "dummy_key"
 authMode: "lmv1"
+kubernetes:
+  cluster_name: "dummy-cluster"
 global:
   account: "dummyaccount"
   accessID: "dummyid"

charts/lm-logs/templates/_helpers.tpl

@@ -134,18 +134,18 @@ optional: true
 {{- else -}}
   {{- fail "authMode must be 'lmv1' or 'bearer'." -}}
 {{- end -}}
+
+{{- /* 3) Enforce kubernetes.cluster_name presence (required for resource mapping). No global fallback. */ -}}
+{{- if not (and .Values.kubernetes .Values.kubernetes.cluster_name) -}}
+  {{- fail "kubernetes.cluster_name is required. Set lm-logs.kubernetes.cluster_name (or kubernetes.cluster_name when installing lm-logs standalone)." -}}
+{{- end -}}
 {{- end -}}
 
 {{/*
 Adding validations for clustername for lm-logs to contain only lower alphanumeric or '-' and start and end with an alphanumeric character
 */}}
 {{- define "kubernetes.cluster_name" -}}
-{{- $cluster := "" -}}
-{{- if .Values.kubernetes.cluster_name -}}
-{{- $cluster = .Values.kubernetes.cluster_name -}}
-{{- else if .Values.global.clusterName -}}
-{{- $cluster = .Values.global.clusterName -}}
-{{- end -}}
+{{- $cluster := .Values.kubernetes.cluster_name -}}
 {{- if ne $cluster "" -}}
 {{- if regexMatch "^[a-z0-9][a-z0-9-]*[a-z0-9]$" $cluster }}
 kubernetes.cluster_name {{ $cluster }}
@@ -159,12 +159,7 @@ kubernetes.cluster_name {{ $cluster }}
 User-agent for log-ingest requests
 */}}
 {{- define "logsource.userAgent" -}}
-{{- $cluster := "" -}}
-{{- if .Values.kubernetes.cluster_name -}}
-{{- $cluster = .Values.kubernetes.cluster_name -}}
-{{- else if .Values.global.clusterName -}}
-{{- $cluster = .Values.global.clusterName -}}
-{{- end -}}
+{{- $cluster := .Values.kubernetes.cluster_name -}}
 log_source lm-logs-fluentd (K8S; {{ $cluster }})
 {{- end -}}
 
@@ -367,3 +362,20 @@ systemd.conf: ""
 {{- end }}
 {{- end }}
 
+
+{{/*
+Validate fluent.extraFilters: only <filter> blocks allowed, tags must be balanced.
+*/}}
+{{- define "lm-logs.validateExtraFilters" -}}
+{{- $f := .Values.fluent.extraFilters -}}
+{{- if $f -}}
+  {{- if or (contains "<match" $f) (contains "<source" $f) (contains "<label" $f) -}}
+    {{- fail "fluent.extraFilters must only contain <filter> blocks. Found disallowed directive (<match>, <source>, or <label>). Use only <filter> blocks for custom filtering." -}}
+  {{- end -}}
+  {{- $open := regexFindAll "<filter" $f -1 -}}
+  {{- $close := regexFindAll "</filter>" $f -1 -}}
+  {{- if ne (len $open) (len $close) -}}
+    {{- fail "fluent.extraFilters has mismatched <filter>/</filter> tags. Ensure every <filter> has a closing </filter>." -}}
+  {{- end -}}
+{{- end -}}
+{{- end -}}

charts/lm-logs/templates/configmap.yaml

@@ -34,7 +34,7 @@ data:
           {{- if .Values.fluent.device_less_logs }}
           resource.service.name ${ENV['K8S_NODE_NAME']}
           {{- end }}
-          {{- if or .Values.kubernetes.cluster_name .Values.global.clusterName }}
+          {{- if .Values.kubernetes.cluster_name }}
           {{ include "kubernetes.cluster_name" . | nindent 10 }}
           {{- end }}
         </record>
@@ -54,14 +54,19 @@ data:
         skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
       </filter>
 
+      {{- if .Values.fluent.extraFilters }}
+      {{- include "lm-logs.validateExtraFilters" . }}
+{{ .Values.fluent.extraFilters | nindent 6 }}
+      {{- end }}
+
       <filter kubernetes.**>
         @type record_transformer
         enable_ruby
         <record>
           message ${record["log"]} ${record["message"]}
           timestamp ${record["time"]}
-          {{- if or .Values.kubernetes.cluster_name .Values.global.clusterName }}
-          {{- $cluster := .Values.kubernetes.cluster_name | default .Values.global.clusterName }}
+          {{- if .Values.kubernetes.cluster_name }}
+          {{- $cluster := .Values.kubernetes.cluster_name }}
           kubernetes.cluster_name {{ $cluster }}
           kubernetes ${record["kubernetes"] ? record["kubernetes"].merge({"cluster_name" => "{{ $cluster }}"}) : {"cluster_name" => "{{ $cluster }}"}}
           {{- end }}

charts/lm-logs/templates/daemonset.yaml

@@ -59,6 +59,9 @@ spec:
             - name: fluentbuffer
               mountPath: {{ .Values.fluent.buffer.file.mountPath | default "/fluentd/buffer" }}
             {{- end }}
+            {{- with .Values.extraVolumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
       volumes:
         - name: fluentconf
           configMap:
@@ -71,6 +74,9 @@ spec:
             sizeLimit: {{ .Values.fluent.buffer.file.sizeLimit }}
             {{- end }}
         {{- end }}
+        {{- with .Values.extraVolumes }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

charts/lm-logs/values.schema.json

@@ -305,6 +305,11 @@
         "resource_type": {
           "type" : "string"
         },
+        "extraFilters": {
+          "type": "string",
+          "default": "",
+          "description": "Raw Fluentd filter config injected after kubernetes_metadata enrichment inside <label @PROCESS_AFTER_CONCAT>. Use for grep include/exclude filters."
+        },
         "buffer": {
           "type": "object",
           "additionalProperties": false,
@@ -441,6 +446,22 @@
       },
       "uniqueItems": true
     },
+    "extraVolumes": {
+      "type": "array",
+      "default": [],
+      "description": "Additional volumes to add to the Fluentd DaemonSet pod.",
+      "items": {
+        "$ref": "#/definitions/io.k8s.api.core.v1.Volume"
+      }
+    },
+    "extraVolumeMounts": {
+      "type": "array",
+      "default": [],
+      "description": "Additional volume mounts to add to the Fluentd container.",
+      "items": {
+        "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount"
+      }
+    },
     "systemd": {
       "type": ["object", "null"],
       "default": null,

charts/lm-logs/values.yaml

@@ -43,6 +43,31 @@ fluent:
   device_less_logs: false
   include_metadata: true
   resource_type: ""
+
+  # Extra Fluentd filters injected after kubernetes_metadata enrichment,
+  # inside <label @PROCESS_AFTER_CONCAT>. Use for include/exclude filtering.
+  #
+  # Example – drop logs from kube-system namespace:
+  #   extraFilters: |
+  #     <filter kubernetes.**>
+  #       @type grep
+  #       <exclude>
+  #         key $.kubernetes.namespace_name
+  #         pattern /^kube-system$/
+  #       </exclude>
+  #     </filter>
+  #
+  # Example – keep only logs matching a pattern:
+  #   extraFilters: |
+  #     <filter kubernetes.**>
+  #       @type grep
+  #       <regexp>
+  #         key message
+  #         pattern /ERROR|WARN/
+  #       </regexp>
+  #     </filter>
+  extraFilters: ""
+
   buffer:
     type: memory
     memory:
@@ -107,3 +132,6 @@ volumeMounts:
   - name: journal
     mountPath: /var/log/journal
     readOnly: true
+
+extraVolumes: []
+extraVolumeMounts: []