From 82ba124a402a8fdd87abcf5e1cbb79ee5c2a0de4 Mon Sep 17 00:00:00 2001 From: Nikita Kumble Date: Fri, 19 Jun 2026 15:49:22 +0530 Subject: [PATCH 1/3] clusterlogforwarder code done --- .../templates/05-ClusterLogForwarder.yaml | 91 ++++++++++++++++++- .../054-cluster-logging-operator/values.yaml | 11 ++- .../054-cluster-logging-operator-app.yaml | 4 + 3 files changed, 103 insertions(+), 3 deletions(-) diff --git a/cluster-applications/054-cluster-logging-operator/templates/05-ClusterLogForwarder.yaml b/cluster-applications/054-cluster-logging-operator/templates/05-ClusterLogForwarder.yaml index 36c30a264..5072347c8 100644 --- a/cluster-applications/054-cluster-logging-operator/templates/05-ClusterLogForwarder.yaml +++ b/cluster-applications/054-cluster-logging-operator/templates/05-ClusterLogForwarder.yaml @@ -23,15 +23,48 @@ spec: namespace: mas-*-iot - container: rest-v2-sync-scheduler namespace: mas-*-monitor + - container: instana-agent + namespace: instana-agent +{{- if ne .Values.environment_type "prod" }} + - container: cron + namespace: mas-*-manage +{{- end }} +{{- if .Values.additional_exclude_containers }} +{{- range .Values.additional_exclude_containers }} + - container: {{ .container }} + namespace: {{ .namespace }} +{{- end }} +{{- end }} name: application-logs type: application filters: +{{- if ne .Values.environment_type "prod" }} + - drop: + - test: + - field: .level + matches: info|Info|INFO|DEBUG|Debug|debug + - test: + - field: .message + matches: '\[CID\-CRON\-[0-9]+\]' + - test: + - field: .message + matches: 'BMX[A-Z0-9]+I' + - test: + - field: .message + matches: 'BMXAA8229W' + name: noise + type: drop +{{- else }} - drop: - test: - field: .level matches: info|Info|INFO|DEBUG|Debug|debug + - test: + - field: .message + matches: 'BMX[A-Z0-9]+I' name: noise type: drop +{{- end }} - kubeAPIAudit: omitStages: [] rules: @@ -114,23 +147,71 @@ spec: namespace: mas-*-iot - container: rest-v2-sync-scheduler namespace: mas-*-monitor + - container: instana-agent + namespace: instana-agent +{{- if ne .Values.environment_type "prod" }} + - container: cron + namespace: mas-*-manage +{{- end }} +{{- if .Values.additional_exclude_containers }} +{{- range .Values.additional_exclude_containers }} + - container: {{ .container }} + namespace: {{ .namespace }} +{{- end }} +{{- end }} name: application-logs type: application serviceAccount: name: collector filters: + - name: detect-multiline-errors + type: detectMultilineException +{{- if ne .Values.environment_type "prod" }} - drop: - test: - field: .level matches: info|Info|INFO|DEBUG|Debug|debug + - test: + - field: .message + matches: '\[CID\-CRON\-[0-9]+\]' + - test: + - field: .message + matches: 'BMX[A-Z0-9]+I' + - test: + - field: .message + matches: 'BMXAA8229W' name: noise type: drop +{{- else }} + - drop: + - test: + - field: .level + matches: info|Info|INFO|DEBUG|Debug|debug + - test: + - field: .message + matches: 'BMX[A-Z0-9]+I' + name: noise + type: drop +{{- end }} - kubeAPIAudit: omitStages: [] rules: - level: Metadata name: sos-policy type: kubeAPIAudit + - prune: + notIn: + - .message + - .log_type + - .log_source + - .kubernetes.namespace_name + - .kubernetes.pod_name + - .kubernetes.container_name + - .kubernetes.labels.app + - .kubernetes.labels."mas.ibm.com/instanceId" + - .kubernetes.container_image + type: prune + name: prune-fields outputs: - name: cloudwatch-1 type: cloudwatch @@ -163,19 +244,25 @@ spec: pipelines: - name: app-logs filterRefs: + - detect-multiline-errors - noise + - prune-fields inputRefs: - application-logs outputRefs: - cloudwatch-1 - name: infra-logs + filterRefs: + - detect-multiline-errors + - prune-fields inputRefs: - infrastructure outputRefs: - cloudwatch-1 - name: audit-logs filterRefs: - - sos-policy + - sos-policy + - prune-fields inputRefs: - audit outputRefs: @@ -184,4 +271,4 @@ spec: - rsyslog-local {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/cluster-applications/054-cluster-logging-operator/values.yaml b/cluster-applications/054-cluster-logging-operator/values.yaml index 7226b4c6b..b6d37591a 100644 --- a/cluster-applications/054-cluster-logging-operator/values.yaml +++ b/cluster-applications/054-cluster-logging-operator/values.yaml @@ -3,5 +3,14 @@ cluster_id: xxx region_id: xxx aws_access_key_id: xxx aws_secret_access_key: xxx -cluster_logging_operator_channel: "stable-5.9" +# cluster_logging_operator_channel: "stable-5.9" +cluster_logging_operator_channel: "stable-6" use_syslog_forwarder: "false" +# Environment type: "prod" for production, "non-prod" for non-production environments +# This can be used to apply different configurations based on environment +# Must be explicitly set to either "prod" or "non-prod" +environment_type: "" +# Additional containers to exclude from log collection (optional) +# Provide via cluster-params to filter out noisy container logs when needed +additional_exclude_containers: [] + diff --git a/root-applications/ibm-mas-cluster-root/templates/054-cluster-logging-operator-app.yaml b/root-applications/ibm-mas-cluster-root/templates/054-cluster-logging-operator-app.yaml index 016591f0a..1674a2605 100644 --- a/root-applications/ibm-mas-cluster-root/templates/054-cluster-logging-operator-app.yaml +++ b/root-applications/ibm-mas-cluster-root/templates/054-cluster-logging-operator-app.yaml @@ -47,6 +47,10 @@ spec: log_forwarder_client_url: "{{ .Values.cluster_logging_operator.log_forwarder_client_url }}" log_forwarder_dlc_cert: "{{ .Values.cluster_logging_operator.log_forwarder_dlc_cert }}" syslog_forwarder_version: "{{ .Values.cluster_logging_operator.syslog_forwarder_version }}" + {{- if .Values.cluster_logging_operator.additional_exclude_containers }} + additional_exclude_containers: {{ .Values.cluster_logging_operator.additional_exclude_containers | toYaml | nindent 14 }} + {{- end }} + environment_type: "{{ .Values.cluster_logging_operator.environment_type }}" junitreporter: reporter_name: "cluster-logging-operator" cluster_id: "{{ .Values.cluster.id }}" From acc32f70f91f413b53ec65696d7ac5828bf5702a Mon Sep 17 00:00:00 2001 From: Nikita Kumble Date: Mon, 3 Aug 2026 13:22:46 +0530 Subject: [PATCH 2/3] minor update --- .../054-cluster-logging-operator/README.md | 9 +++++++++ .../templates/05-ClusterLogForwarder.yaml | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/cluster-applications/054-cluster-logging-operator/README.md b/cluster-applications/054-cluster-logging-operator/README.md index a4280fdb2..c4364978f 100644 --- a/cluster-applications/054-cluster-logging-operator/README.md +++ b/cluster-applications/054-cluster-logging-operator/README.md @@ -37,6 +37,15 @@ cluster_logging_operator: # Default: false use_syslog_forwarder: false + # Additional containers to exclude from log collection (optional) + # List of container/namespace pairs to filter out noisy logs + # Instana-agent logs are excluded by default + # Example: + # additional_exclude_containers: + # - container: example-container + # namespace: example-namespace + additional_exclude_containers: [] + # Setup log forwarding configuration (optional) # When true, configures additional log forwarding resources # Default: false diff --git a/cluster-applications/054-cluster-logging-operator/templates/05-ClusterLogForwarder.yaml b/cluster-applications/054-cluster-logging-operator/templates/05-ClusterLogForwarder.yaml index 5072347c8..ebb1ff3cd 100644 --- a/cluster-applications/054-cluster-logging-operator/templates/05-ClusterLogForwarder.yaml +++ b/cluster-applications/054-cluster-logging-operator/templates/05-ClusterLogForwarder.yaml @@ -23,6 +23,8 @@ spec: namespace: mas-*-iot - container: rest-v2-sync-scheduler namespace: mas-*-monitor + - container: run + namespace: mas-*-core - container: instana-agent namespace: instana-agent {{- if ne .Values.environment_type "prod" }} @@ -147,6 +149,8 @@ spec: namespace: mas-*-iot - container: rest-v2-sync-scheduler namespace: mas-*-monitor + - container: run + namespace: mas-*-core - container: instana-agent namespace: instana-agent {{- if ne .Values.environment_type "prod" }} From dd0dfb99e4200057a33feb3a93c0b63907d054b7 Mon Sep 17 00:00:00 2001 From: Nikita Kumble Date: Fri, 7 Aug 2026 15:23:48 +0530 Subject: [PATCH 3/3] operator channel set to 5.9 --- cluster-applications/054-cluster-logging-operator/values.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cluster-applications/054-cluster-logging-operator/values.yaml b/cluster-applications/054-cluster-logging-operator/values.yaml index b6d37591a..b8b75ea90 100644 --- a/cluster-applications/054-cluster-logging-operator/values.yaml +++ b/cluster-applications/054-cluster-logging-operator/values.yaml @@ -3,8 +3,7 @@ cluster_id: xxx region_id: xxx aws_access_key_id: xxx aws_secret_access_key: xxx -# cluster_logging_operator_channel: "stable-5.9" -cluster_logging_operator_channel: "stable-6" +cluster_logging_operator_channel: "stable-5.9" use_syslog_forwarder: "false" # Environment type: "prod" for production, "non-prod" for non-production environments # This can be used to apply different configurations based on environment