[minor] ClusterLogForwarder: noise filters, multiline detection, prune-fields, and dynamic container exclusions

cluster-applications/054-cluster-logging-operator/README.md

@@ -37,6 +37,15 @@ cluster_logging_operator:
   # Default: false
   use_syslog_forwarder: false
 
+  # Additional containers to exclude from log collection (optional)
+  # List of container/namespace pairs to filter out noisy logs
+  # Instana-agent logs are excluded by default
+  # Example:
+  # additional_exclude_containers:
+  #   - container: example-container
+  #     namespace: example-namespace
+  additional_exclude_containers: []
+
   # Setup log forwarding configuration (optional)
   # When true, configures additional log forwarding resources
   # Default: false

cluster-applications/054-cluster-logging-operator/templates/05-ClusterLogForwarder.yaml

@@ -23,15 +23,50 @@ spec:
             namespace: mas-*-iot
           - container: rest-v2-sync-scheduler
             namespace: mas-*-monitor
+          - container: run
+            namespace: mas-*-core
+          - container: instana-agent
+            namespace: instana-agent
+{{- if ne .Values.environment_type "prod" }}
+          - container: cron
+            namespace: mas-*-manage
+{{- end }}
+{{- if .Values.additional_exclude_containers }}
+{{- range .Values.additional_exclude_containers }}
+          - container: {{ .container }}
+            namespace: {{ .namespace }}
+{{- end }}
+{{- end }}
       name: application-logs
       type: application
   filters:
+{{- if ne .Values.environment_type "prod" }}
+    - drop:
+        - test:
+            - field: .level
+              matches: info|Info|INFO|DEBUG|Debug|debug
+        - test:
+            - field: .message
+              matches: '\[CID\-CRON\-[0-9]+\]'
+        - test:
+            - field: .message
+              matches: 'BMX[A-Z0-9]+I'
+        - test:
+            - field: .message
+              matches: 'BMXAA8229W'
+      name: noise
+      type: drop
+{{- else }}
     - drop:
         - test:
             - field: .level
               matches: info|Info|INFO|DEBUG|Debug|debug
+        - test:
+            - field: .message
+              matches: 'BMX[A-Z0-9]+I'
       name: noise
       type: drop
+{{- end }}
     - kubeAPIAudit:
         omitStages: []
         rules:
@@ -114,23 +149,73 @@ spec:
             namespace: mas-*-iot
           - container: rest-v2-sync-scheduler
             namespace: mas-*-monitor
+          - container: run
+            namespace: mas-*-core
+          - container: instana-agent
+            namespace: instana-agent
+{{- if ne .Values.environment_type "prod" }}
+          - container: cron
+            namespace: mas-*-manage
+{{- end }}
+{{- if .Values.additional_exclude_containers }}
+{{- range .Values.additional_exclude_containers }}
+          - container: {{ .container }}
+            namespace: {{ .namespace }}
+{{- end }}
+{{- end }}
       name: application-logs
       type: application
   serviceAccount:
     name: collector
   filters:
+    - name: detect-multiline-errors
+      type: detectMultilineException
+{{- if ne .Values.environment_type "prod" }}
     - drop:
         - test:
             - field: .level
               matches: info|Info|INFO|DEBUG|Debug|debug
+        - test:
+            - field: .message
+              matches: '\[CID\-CRON\-[0-9]+\]'
+        - test:
+            - field: .message
+              matches: 'BMX[A-Z0-9]+I'
+        - test:
+            - field: .message
+              matches: 'BMXAA8229W'
       name: noise
       type: drop
+{{- else }}
+    - drop:
+        - test:
+            - field: .level
+              matches: info|Info|INFO|DEBUG|Debug|debug
+        - test:
+            - field: .message
+              matches: 'BMX[A-Z0-9]+I'
+      name: noise
+      type: drop
+{{- end }}
     - kubeAPIAudit:
         omitStages: []
         rules:
           - level: Metadata
       name: sos-policy
       type: kubeAPIAudit
+    - prune:
+        notIn:
+          - .message
+          - .log_type
+          - .log_source
+          - .kubernetes.namespace_name
+          - .kubernetes.pod_name
+          - .kubernetes.container_name
+          - .kubernetes.labels.app
+          - .kubernetes.labels."mas.ibm.com/instanceId"
+          - .kubernetes.container_image
+      type: prune
+      name: prune-fields
   outputs:
     - name: cloudwatch-1
       type: cloudwatch
@@ -163,19 +248,25 @@ spec:
   pipelines:
     - name: app-logs
       filterRefs:
+        - detect-multiline-errors
         - noise
+        - prune-fields
       inputRefs:
         - application-logs
       outputRefs:
         - cloudwatch-1
     - name: infra-logs
+      filterRefs:
+        - detect-multiline-errors
+        - prune-fields
       inputRefs:
         - infrastructure
       outputRefs:
         - cloudwatch-1
     - name: audit-logs
       filterRefs:
-         - sos-policy
+        - sos-policy
+        - prune-fields
       inputRefs:
         - audit
       outputRefs:
@@ -184,4 +275,4 @@ spec:
         - rsyslog-local
 {{- end }}
 
-{{- end }}
+{{- end }}

cluster-applications/054-cluster-logging-operator/values.yaml

@@ -5,3 +5,11 @@ aws_access_key_id: xxx
 aws_secret_access_key: xxx
 cluster_logging_operator_channel: "stable-5.9"
 use_syslog_forwarder: "false"
+# Environment type: "prod" for production, "non-prod" for non-production environments
+# This can be used to apply different configurations based on environment
+# Must be explicitly set to either "prod" or "non-prod"
+environment_type: ""
+# Additional containers to exclude from log collection (optional)
+# Provide via cluster-params to filter out noisy container logs when needed
+additional_exclude_containers: []
+

root-applications/ibm-mas-cluster-root/templates/054-cluster-logging-operator-app.yaml

@@ -47,6 +47,10 @@ spec:
             log_forwarder_client_url: "{{ .Values.cluster_logging_operator.log_forwarder_client_url }}"
             log_forwarder_dlc_cert: "{{ .Values.cluster_logging_operator.log_forwarder_dlc_cert }}"
             syslog_forwarder_version: "{{ .Values.cluster_logging_operator.syslog_forwarder_version }}"
+            {{- if .Values.cluster_logging_operator.additional_exclude_containers }}
+            additional_exclude_containers: {{ .Values.cluster_logging_operator.additional_exclude_containers | toYaml | nindent 14 }}
+            {{- end }}
+            environment_type: "{{ .Values.cluster_logging_operator.environment_type }}"
             junitreporter:
               reporter_name: "cluster-logging-operator"
               cluster_id: "{{ .Values.cluster.id }}"