diff --git a/examples/k8s_perf_test/run-test.sh b/examples/k8s_perf_test/run-test.sh new file mode 100755 index 00000000000..e63d7c70300 --- /dev/null +++ b/examples/k8s_perf_test/run-test.sh @@ -0,0 +1,23 @@ +#/bin/bash +export TEST_NAMESPACE="${TEST_NAMESPACE:-default}" +export FLUENTBIT_IMAGE_REPOSITORY=${FLUENTBIT_IMAGE_REPOSITORY:-ghcr.io/fluent/fluent-bit} +export FLUENTBIT_IMAGE_TAG=${FLUENTBIT_IMAGE_TAG:-latest} + +# update helm +helm repo add fluent https://fluent.github.io/helm-charts/ +helm repo update --fail-on-repo-update-fail + +echo "Installing fluent-bit via helm in namespace $TEST_NAMESPACE" +helm upgrade --install --debug --create-namespace --namespace "$TEST_NAMESPACE" fluent-bit fluent/fluent-bit \ + --values values.yaml \ + --set image.repository=${FLUENTBIT_IMAGE_REPOSITORY},image.tag=${FLUENTBIT_IMAGE_TAG} \ + --timeout "${HELM_FB_TIMEOUT:-5m0s}" \ + --wait + +export POD_NAME=$(kubectl get pods --namespace $TEST_NAMESPACE -l "app.kubernetes.io/name=fluent-bit,app.kubernetes.io/instance=fluent-bit" --field-selector status.phase=Running -o jsonpath="{.items[-1].metadata.name}") +echo "$POD_NAME" deployed, tailing logs +kubectl logs -n $TEST_NAMESPACE $POD_NAME -c logwriter -f + +#NOTE You can also follow -c fluent-bit for the fluent-bit logs if you'd like + +# To rest the test, just `helm uninstall fluent-bit` in your $TEST_NAMESPACE and re-run ./run-test.sh \ No newline at end of file diff --git a/examples/k8s_perf_test/values.yaml b/examples/k8s_perf_test/values.yaml new file mode 100644 index 00000000000..125ca62798f --- /dev/null +++ b/examples/k8s_perf_test/values.yaml @@ -0,0 +1,288 @@ +kind: Deployment +image: + pullPolicy: Always +replicaCount: 1 +rbac: + create: true +extraVolumeMounts: +- mountPath: /app/perftest/containers + name: perftest-volume +extraVolumes: +- name: perftest-volume + emptyDir: {} + +env: + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP +extraContainers: + - name: logwriter + image: python:3.12.6-bookworm + command: ["/bin/bash", "/fluent-bit/etc/conf/run-log-writer-test.sh"] + volumeMounts: + - name: config + mountPath: /fluent-bit/etc/conf + - mountPath: /app/perftest/containers + name: perftest-volume + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: NAMESPACE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + +podLabels: + label1: "test1" + label2: "test2" + label3: "test3" + label4: "test4" + +config: + service: | + [SERVICE] + Flush 0.25 + Grace 2 + Daemon Off + Log_Level info + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + Parsers_File /fluent-bit/etc/conf/custom_parsers.conf + + inputs: | + [INPUT] + name tail + read_from_head false + skip_long_lines on + path /app/perftest/containers/*.log + multiline.parser cri + Tag kube.* + #buffer_chunk_size 5M + #buffer_max_size 5M + Rotate_Wait 60 + Refresh_Interval 1 + mem_buf_limit 50MB + threaded on + #Inotify_Watcher false + + # If you want to bypass filters, you must pass an empty string, otherwise helm + # will give you some defaults + filters: "" + + filters-simple: | + [FILTER] + Name modify + Match kube* + Copy level severity + Remove _p + + [FILTER] + Name kubernetes + Alias get_k8s_metadata + Kube_Tag_Prefix kube.app.perftest.containers + Match kube* + K8S-Logging.Parser Off + K8S-Logging.Exclude Off + #Use_Kubelet On + #Kubelet_Host ${NODE_IP} + #Kubelet_Port 10250 + Buffer_Size 2MB + Labels On + Annotations Off + #Merge_Log On moves 'log' field to message + Merge_Log On + Merge_Log_Trim On + kube_meta_cache_ttl 15m + kube_meta_namespace_cache_ttl 15m + Namespace_labels On + Namespace_annotations Off + namespace_metadata_only Off + + filters-extended: | + [FILTER] + Name parser + Match kube* + Key_Name log + Reserve_Data True + Parser glog + Parser json + + [FILTER] + Name modify + Match kube* + Copy level severity + Remove _p + + [FILTER] + Name kubernetes + Alias get_k8s_metadata + Kube_Tag_Prefix kube.app.perftest.containers + Match kube* + K8S-Logging.Parser Off + K8S-Logging.Exclude Off + #Use_Kubelet On + #Kubelet_Host ${NODE_IP} + #Kubelet_Port 10250 + Buffer_Size 2MB + Labels On + Annotations Off + #Merge_Log On moves 'log' field to message + Merge_Log On + Merge_Log_Trim On + kube_meta_cache_ttl 15m + kube_meta_namespace_cache_ttl 15m + Namespace_labels On + Namespace_annotations On + namespace_metadata_only Off + + # We only want the metadata labels & host (compute resource), not all that other + # metadata, so we do this messy filtering below with the nest lift to move up + # meta info into kuberenetesmeta_*, then the lua script to set the compute resource tag, + # then we modify (delete) all 'kubernetesmeta' fields excluding kubernetesmeta_labels + [FILTER] + Name nest + Match kube* + Operation lift + Nested_under kubernetes + Add_prefix kubernetesmeta_ + + #[FILTER] + # lua not included, uses kubernetes_namespace data from record + # Name lua + # Alias fix_k8s_labels + # Match kube* + # Call fix_k8s_labels + # Script functions.lua + + [FILTER] + Name modify + Alias remove_unused_k8s_meta + Match kube* + Remove_regex kubernetesmeta_(?!labels) + Remove_regex kubernetes_namespace + + outputs: | + [OUTPUT] + Name null + Match * + + customParsers: | + [PARSER] + Name json + Format json + + [PARSER] + Name glog + Format regex + Regex ^(?\w)(?