From d8c56d53e80abcb5cbd49afc4622eaa302450bb5 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Fri, 14 Aug 2026 11:04:35 +0200 Subject: [PATCH 01/19] add check_multi --- docs/checks/commands/check_multi.md | 74 +++++++ packaging/snclient.ini | 8 + pkg/snclient/check_multi.go | 301 ++++++++++++++++++++++++++++ pkg/snclient/check_multi_test.go | 220 ++++++++++++++++++++ pkg/snclient/checkdata.go | 78 +++++-- pkg/snclient/config.go | 36 +++- 6 files changed, 693 insertions(+), 24 deletions(-) create mode 100644 docs/checks/commands/check_multi.md create mode 100644 pkg/snclient/check_multi.go create mode 100644 pkg/snclient/check_multi_test.go diff --git a/docs/checks/commands/check_multi.md b/docs/checks/commands/check_multi.md new file mode 100644 index 00000000..68dda21b --- /dev/null +++ b/docs/checks/commands/check_multi.md @@ -0,0 +1,74 @@ +--- +title: multi +--- + +## check_multi + +Runs multiple checks and aggregates their status, output and performance data. + +In order to use this plugin, you need to enable 'CheckMulti' in the '[/modules]' section of the snclient.ini. + +- [Examples](#examples) +- [Argument Defaults](#argument-defaults) +- [Attributes](#attributes) + +## Implementation + +| Windows | Linux | FreeBSD | MacOSX | +|:------------------:|:------------------:|:------------------:|:------------------:| +| :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | + +## Examples + +### Inline Checks + + check_multi "check=check_process process=123" "check=check_process process=345" "warn=none" "crit=ok_count ne 2" + OK - 2 plugins checked, 2 ok + +### Config Section Checks + +Define checks in `snclient.ini` under `[/settings/check/multi/]`: + + [/settings/check/multi/mycheck] + check_process process=123 + check_process process=345 + +Run the configured multi check: + + check_multi "config=mycheck" "warn=none" "crit=ok_count ne 2" + OK - 2 plugins checked, 2 ok + +### External Script Config + + [/settings/check/multi/custom] + /opt/script/test.sh -H 123 + /opt/script/test2.sh -W 123 + +Run the configured multi check: + + check_multi "config=custom" "warn=problem_count gt 0" + +## Check Specific Arguments + +| Argument | Default | Description | +| --- | --- | --- | +| check | | Inline check command to execute (can be specified multiple times) | +| config | | Config section name under `/settings/check/multi/` to execute | + +## Attributes + +| Filter / Threshold | Default | Description | +| --- | --- | --- | +| warn | `warning_count > 0` | Warning threshold | +| crit | `critical_count > 0 \|\| unknown_count > 0` | Critical threshold | +| count | | Total number of checks executed | +| ok_count | | Number of checks in OK state | +| warning_count | | Number of checks in WARNING state | +| critical_count | | Number of checks in CRITICAL state | +| unknown_count | | Number of checks in UNKNOWN state | +| problem_count | | Number of checks in non-OK state | +| name | | Name/tag of the check | +| command | | Command executed | +| state | | Exit code of the check (0=OK, 1=WARNING, 2=CRITICAL, 3=UNKNOWN) | +| status | | Status text of the check (OK, WARNING, CRITICAL, UNKNOWN) | +| output | | Output of the check | diff --git a/packaging/snclient.ini b/packaging/snclient.ini index 51432c9b..3d8e20da 100644 --- a/packaging/snclient.ini +++ b/packaging/snclient.ini @@ -70,6 +70,9 @@ CheckWMI = disabled ; CheckLogFile - Controls whether check_logfile is allowed or not. CheckLogFile = disabled +; CheckMulti - Controls whether check_multi is allowed or not. +CheckMulti = enabled + [/settings/default] ; allowed hosts - Comma separated list of ips/networks/hostname allowed to connect. @@ -357,6 +360,11 @@ allowed pattern += /var/log/snclient/snclient.log max lines per file limit = 1000000 +[/settings/check/multi] +; max checks - Maximum number of checks check_multi can execute. +max checks = 20 + + ; External script settings - General settings for the external scripts module (CheckExternalScripts). [/settings/external scripts] diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go new file mode 100644 index 00000000..cee23603 --- /dev/null +++ b/pkg/snclient/check_multi.go @@ -0,0 +1,301 @@ +package snclient + +import ( + "context" + "fmt" + "strings" + + "github.com/consol-monitoring/snclient/pkg/utils" +) + +func init() { + AvailableChecks["check_multi"] = CheckEntry{"check_multi", NewCheckMulti} +} + +type CheckMulti struct { + checks []string + config string +} + +func NewCheckMulti() CheckHandler { + return &CheckMulti{ + checks: make([]string, 0), + } +} + +func (l *CheckMulti) Build() *CheckData { + return &CheckData{ + name: "check_multi", + description: "Runs multiple checks and aggregates their status, output and performance data.", + implemented: ALL, + disableFilter: true, + result: &CheckResult{ + State: CheckExitOK, + }, + args: map[string]CheckArgument{ + "check": {value: &l.checks, description: "Inline check command to execute (can be specified multiple times)"}, + "config": {value: &l.config, description: "Config section name under /settings/check/multi/ to execute"}, + }, + conditionAlias: map[string]map[string]string{ + "warning_count": {"warn_count": "warning_count"}, + "critical_count": {"crit_count": "critical_count"}, + }, + attributes: []CheckAttribute{ + {name: "count", description: "Total number of checks executed", unit: UNone}, + {name: "ok_count", description: "Number of checks in OK state", unit: UNone}, + {name: "warning_count", description: "Number of checks in WARNING state", unit: UNone}, + {name: "critical_count", description: "Number of checks in CRITICAL state", unit: UNone}, + {name: "unknown_count", description: "Number of checks in UNKNOWN state", unit: UNone}, + {name: "problem_count", description: "Number of checks in non-OK state", unit: UNone}, + {name: "name", description: "Name/tag of the check", unit: UNone}, + {name: "command", description: "Command executed", unit: UNone}, + {name: "state", description: "Exit code of the check (0=OK, 1=WARNING, 2=CRITICAL, 3=UNKNOWN)", unit: UNone}, + {name: "status", description: "Status text of the check (OK, WARNING, CRITICAL, UNKNOWN)", unit: UNone}, + {name: "output", description: "Output of the check", unit: UNone}, + }, + defaultWarning: "warning_count > 0", + defaultCritical: "critical_count > 0 || unknown_count > 0", + okSyntax: "%(status) - %(count) plugins checked, %(ok_count) ok", + topSyntax: "%(status) - %(count) plugins checked: %(ok_count) ok, %(warning_count) warning, %(critical_count) critical, %(unknown_count) unknown%(problem_list)", + detailSyntax: "[%(status)] %(name): %(output)", + emptySyntax: "%(status) - no checks executed", + emptyState: CheckExitUnknown, + exampleDefault: ` + check_multi "check=check_process process=123" "check=check_process process=345" "warn=none" "crit=ok_count ne 2" + OK - 2 plugins checked, 2 ok + `, + } +} + +type multiChildCheck struct { + tag string + cmdStr string + isInline bool +} + +func (l *CheckMulti) Check(ctx context.Context, snc *Agent, check *CheckData, _ []Argument) (*CheckResult, error) { + enabled, _, _ := snc.config.Section("/modules").GetBool("CheckMulti") + if !enabled { + return &CheckResult{ + State: CheckExitUnknown, + Output: "module CheckMulti is not enabled in /modules section", + }, nil + } + + maxChecks, ok, err := snc.config.Section("/settings/check/multi").GetInt("max checks") + if err != nil || !ok || maxChecks <= 0 { + maxChecks = 20 + } + + childChecks, res := l.buildChildChecks(snc) + if res != nil { + return res, nil + } + + if len(childChecks) == 0 { + return &CheckResult{ + State: CheckExitUnknown, + Output: "no checks or config specified", + }, nil + } + + if int64(len(childChecks)) > maxChecks { + return &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("number of checks (%d) exceeds max checks limit (%d)", len(childChecks), maxChecks), + }, nil + } + + return l.executeChildChecks(ctx, snc, check, childChecks) +} + +// buildChildChecks assembles the list of child checks from config section and inline args. +func (l *CheckMulti) buildChildChecks(snc *Agent) ([]multiChildCheck, *CheckResult) { + childChecks := []multiChildCheck{} + + if l.config != "" { + configChecks, res := l.buildConfigChecks(snc) + if res != nil { + return nil, res + } + childChecks = append(childChecks, configChecks...) + } + + for _, inlineCmd := range l.checks { + inlineCmd = strings.TrimSpace(inlineCmd) + if inlineCmd == "" { + continue + } + childChecks = append(childChecks, multiChildCheck{ + tag: "", + cmdStr: inlineCmd, + isInline: true, + }) + } + + return childChecks, nil +} + +// buildConfigChecks loads checks from the named config section. +func (l *CheckMulti) buildConfigChecks(snc *Agent) ([]multiChildCheck, *CheckResult) { + secName := "/settings/check/multi/" + l.config + sec := snc.config.Section(secName) + + if len(sec.keys) == 0 { + return nil, &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("no checks defined in config section %s", secName), + } + } + + childChecks := make([]multiChildCheck, 0, len(sec.keys)) + + for _, key := range sec.keys { + rawCmd, tag := l.resolveConfigEntry(snc, key, sec.data[key]) + childChecks = append(childChecks, multiChildCheck{ + tag: tag, + cmdStr: rawCmd, + isInline: false, + }) + } + + return childChecks, nil +} + +// resolveConfigEntry determines the raw command and tag for a single config section entry. +func (l *CheckMulti) resolveConfigEntry(snc *Agent, key, val string) (rawCmd, tag string) { + rawCmd = key + tag = "" + + if val == "" { + return rawCmd, tag + } + + if _, isKnown := snc.getCheck(key, false); isKnown { + return key + " " + val, key + } + + return val, key +} + +// executeChildChecks runs all child checks and aggregates results. +func (l *CheckMulti) executeChildChecks(ctx context.Context, snc *Agent, check *CheckData, childChecks []multiChildCheck) (*CheckResult, error) { + var count, okCount, warnCount, critCount, unknownCount int64 + + detailsList := make([]string, 0, len(childChecks)) + allMetrics := make([]*CheckMetric, 0) + + for idx, chk := range childChecks { + res, fatal := l.runChildCheck(ctx, snc, check, chk) + if fatal { + return res, nil + } + + count++ + switch res.State { + case CheckExitOK: + okCount++ + case CheckExitWarning: + warnCount++ + case CheckExitCritical: + critCount++ + default: + unknownCount++ + } + + tokens := utils.Tokenize(chk.cmdStr) + cmdName := chk.cmdStr + if len(tokens) > 0 { + cmdName = tokens[0] + } + + tag := chk.tag + if tag == "" { + tag = cmdName + } + + firstLine := strings.TrimSpace(strings.Split(res.Output, "\n")[0]) + detailsList = append(detailsList, fmt.Sprintf("[% 2d] %s %s", idx+1, tag, res.Output)) + + entry := map[string]string{ + "idx": fmt.Sprintf("%d", idx+1), + "name": tag, + "command": chk.cmdStr, + "state": fmt.Sprintf("%d", res.State), + "status": res.StateString(), + "output": firstLine, + "_state": fmt.Sprintf("%d", res.State), + "_count": "1", + } + check.listData = append(check.listData, entry) + + for _, m := range res.Metrics { + metricCopy := *m + metricCopy.Name = fmt.Sprintf("%s::%s", tag, m.Name) + allMetrics = append(allMetrics, &metricCopy) + } + } + + problemCount := warnCount + critCount + unknownCount + check.details = map[string]string{ + "count": fmt.Sprintf("%d", count), + "ok_count": fmt.Sprintf("%d", okCount), + "warning_count": fmt.Sprintf("%d", warnCount), + "warn_count": fmt.Sprintf("%d", warnCount), + "critical_count": fmt.Sprintf("%d", critCount), + "crit_count": fmt.Sprintf("%d", critCount), + "unknown_count": fmt.Sprintf("%d", unknownCount), + "problem_count": fmt.Sprintf("%d", problemCount), + } + + check.result.Metrics = allMetrics + check.result.Details = strings.Join(detailsList, "\n") + + return check.Finalize() +} + +// runChildCheck executes a single child check and returns its result. +// The second return value is true when the error is fatal and the caller should stop processing. +func (l *CheckMulti) runChildCheck(ctx context.Context, snc *Agent, check *CheckData, chk multiChildCheck) (*CheckResult, bool) { + tokens := utils.Tokenize(chk.cmdStr) + tokens, err := utils.TrimQuotesList(tokens) + + if err != nil || len(tokens) == 0 { + return &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("failed to parse check command: %s", chk.cmdStr), + }, true + } + + cmdName := tokens[0] + cmdArgs := tokens[1:] + + _, isKnown := snc.getCheck(cmdName, false) + + if chk.isInline && !isKnown { + return &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("unknown check command: %s (inline checks only support existing check commands)", cmdName), + }, true + } + + if isKnown { + return snc.RunCheckWithContext(ctx, cmdName, cmdArgs, 0, nil, false), false + } + + stdout, stderr, exitCode, _ := snc.runExternalCheckString(ctx, chk.cmdStr, int64(check.timeout)) + out := stdout + if stderr != "" { + if out != "" { + out += "\n" + } + out += "[" + stderr + "]" + } + res := &CheckResult{ + State: exitCode, + Output: out, + } + res.ParsePerformanceDataFromOutput() + + return res, false +} diff --git a/pkg/snclient/check_multi_test.go b/pkg/snclient/check_multi_test.go new file mode 100644 index 00000000..885bfb81 --- /dev/null +++ b/pkg/snclient/check_multi_test.go @@ -0,0 +1,220 @@ +package snclient + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCheckMultiInline(t *testing.T) { + config := ` +[/modules] +CheckMulti = enabled +` + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + // 1. Basic inline checks - all OK + res := snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'dummy ok 1'", + "check=check_dummy 0 'dummy ok 2'", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK") + assert.Contains(t, res.Output, "2 plugins checked, 2 ok") + assert.Contains(t, res.Details, "dummy ok 1") + assert.Contains(t, res.Details, "dummy ok 2") + + // 2. Inline checks with warning and critical (default thresholds) + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'dummy ok'", + "check=check_dummy 1 'dummy warn'", + }) + assert.Equalf(t, CheckExitWarning, res.State, "state WARNING") + assert.Contains(t, res.Output, "2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown") + + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'dummy ok'", + "check=check_dummy 2 'dummy crit'", + }) + assert.Equalf(t, CheckExitCritical, res.State, "state CRITICAL") + assert.Contains(t, res.Output, "2 plugins checked: 1 ok, 0 warning, 1 critical, 0 unknown") + + // 3. Custom conditions: warn=none crit=ok_count ne 2 + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'dummy 1'", + "check=check_dummy 0 'dummy 2'", + "warn=none", + "crit=ok_count ne 2", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK when ok_count == 2") + + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'dummy 1'", + "check=check_dummy 1 'dummy 2'", + "warn=none", + "crit=ok_count ne 2", + }) + assert.Equalf(t, CheckExitCritical, res.State, "state CRITICAL when ok_count != 2") + + // 4. Custom conditions: warn=problem_count gt 0 + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'dummy 1'", + "check=check_dummy 1 'dummy 2'", + "warn=problem_count gt 0", + "crit=none", + }) + assert.Equalf(t, CheckExitWarning, res.State, "state WARNING when problem_count > 0") + + // 5. Unknown/inline checks restriction (cannot run arbitrary external commands inline) + res = snc.RunCheck("check_multi", []string{ + "check=/bin/nonexistent_or_external_script -H 123", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN for unregistered inline command") + assert.Contains(t, res.Output, "unknown check command") + + // 6. Inline check with check_process or check_cpu + res = snc.RunCheck("check_multi", []string{ + "check=check_cpu warn=load=101 crit=load=102", + "warn=none", + "crit=ok_count ne 1", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK for check_cpu inline") + assert.Contains(t, res.Details, "check_cpu") + + // 7. Filter argument is disabled/rejected + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'ok'", + "filter=state=1", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when filter argument is used") + assert.Contains(t, res.Output, "filter is disabled for this check") +} + +func TestCheckMultiLimits(t *testing.T) { + config := ` +[/modules] +CheckMulti = enabled + +[/settings/check/multi] +max checks = 2 +` + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + // Under limit: 2 checks + res := snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'ok 1'", + "check=check_dummy 0 'ok 2'", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK for 2 checks") + + // Exceeds limit: 3 checks + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'ok 1'", + "check=check_dummy 0 'ok 2'", + "check=check_dummy 0 'ok 3'", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when exceeding max checks") + assert.Contains(t, res.Output, "exceeds max checks limit") +} + +func TestCheckMultiDisabled(t *testing.T) { + config := ` +[/modules] +CheckMulti = disabled +` + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + res := snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'ok 1'", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when module is disabled") + assert.Contains(t, res.Output, "module CheckMulti is not enabled") +} + +func TestCheckMultiConfigSection(t *testing.T) { + // Create a temporary shell script to test external scripts in config + tmpDir := t.TempDir() + script1 := filepath.Join(tmpDir, "test1.sh") + script2 := filepath.Join(tmpDir, "test2.sh") + + err := os.WriteFile(script1, []byte("#!/bin/sh\necho \"SCRIPT 1 OK | perf1=10;20;30\"\nexit 0\n"), 0o600) + require.NoError(t, err) + require.NoError(t, os.Chmod(script1, 0o700)) + err = os.WriteFile(script2, []byte("#!/bin/sh\necho \"SCRIPT 2 WARNING | perf2=50;40;60\"\nexit 1\n"), 0o600) + require.NoError(t, err) + require.NoError(t, os.Chmod(script2, 0o700)) + + config := fmt.Sprintf(` +[/modules] +CheckMulti = enabled + +[/settings/check/multi/mycheck] +check_dummy 0 ok1 +check_dummy 0 ok2 + +[/settings/check/multi/custom] +%s -H 123 +%s -W 123 + +[/settings/check/multi/named] +first = check_dummy 0 ok_first +second = %s -H 456 +`, script1, script2, script1) + + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + // Test config=mycheck (builtin checks in config) + res := snc.RunCheck("check_multi", []string{ + "config=mycheck", + "warn=none", + "crit=ok_count ne 2", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK for mycheck config") + assert.Contains(t, res.Output, "2 plugins checked, 2 ok") + + // Test config=custom (external scripts in config) + res = snc.RunCheck("check_multi", []string{ + "config=custom", + "warn=problem_count gt 0", + "crit=none", + }) + assert.Equalf(t, CheckExitWarning, res.State, "state WARNING for custom config") + assert.Contains(t, res.Output, "2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown") + assert.Contains(t, res.Details, "SCRIPT 1 OK") + assert.Contains(t, res.Details, "SCRIPT 2 WARNING") + + // Test config=named (named check tags in config) + res = snc.RunCheck("check_multi", []string{ + "config=named", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK for named config") + assert.Contains(t, res.Details, "first") + assert.Contains(t, res.Details, "second") + + // Test non-existing config + res = snc.RunCheck("check_multi", []string{ + "config=doesnotexist", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN for missing config section") + assert.Contains(t, res.Output, "no checks defined in config section") +} + +func TestCheckMultiIndex(t *testing.T) { + config := ` +[/modules] +CheckMulti = enabled +` + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + res := snc.RunCheck("check_index", []string{"filter=name = 'check_multi'"}) + assert.Equalf(t, CheckExitOK, res.State, "state OK for check_index") + assert.Contains(t, res.Output, "check_multi") +} diff --git a/pkg/snclient/checkdata.go b/pkg/snclient/checkdata.go index 9729acba..aff2ed01 100644 --- a/pkg/snclient/checkdata.go +++ b/pkg/snclient/checkdata.go @@ -102,6 +102,7 @@ type CheckData struct { defaultFilter string conditionAlias map[string]map[string]string // replacement map of equivalent condition values conditionColAlias map[string][]string // if there are filter for given column, apply to alias columns too + disableFilter bool // disable filter argument for checks where filtering listData makes no sense args map[string]CheckArgument extraArgs map[string]CheckArgument // internal, map of expanded args argsPassthrough bool // allow arbitrary arguments without complaining about unknown argument @@ -280,10 +281,12 @@ func (cd *CheckData) buildListMacros() map[string]string { okList := make([]string, 0) warnList := make([]string, 0) critList := make([]string, 0) + unknownList := make([]string, 0) count := int64(0) okCount := int64(0) warnCount := int64(0) critCount := int64(0) + unknownCount := int64(0) for _, entry := range cd.listData { weight := int64(1) if w, ok := entry["_count"]; ok { @@ -308,6 +311,9 @@ func (cd *CheckData) buildListMacros() map[string]string { case "2": critList = append(critList, expanded) critCount += weight + case "3": + unknownList = append(unknownList, expanded) + unknownCount += weight } } @@ -315,17 +321,21 @@ func (cd *CheckData) buildListMacros() map[string]string { cd.listCombine = ", " } result := map[string]string{ - "count": fmt.Sprintf("%d", count), - "list": strings.Join(list, cd.listCombine), - "ok_count": fmt.Sprintf("%d", okCount), - "ok_list": "", - "warn_count": fmt.Sprintf("%d", warnCount), - "warn_list": "", - "crit_count": fmt.Sprintf("%d", critCount), - "crit_list": "", - "problem_count": fmt.Sprintf("%d", warnCount+critCount), - "problem_list": "", - "detail_list": "", + "count": fmt.Sprintf("%d", count), + "list": strings.Join(list, cd.listCombine), + "ok_count": fmt.Sprintf("%d", okCount), + "ok_list": "", + "warn_count": fmt.Sprintf("%d", warnCount), + "warning_count": fmt.Sprintf("%d", warnCount), + "warn_list": "", + "crit_count": fmt.Sprintf("%d", critCount), + "critical_count": fmt.Sprintf("%d", critCount), + "crit_list": "", + "unknown_count": fmt.Sprintf("%d", unknownCount), + "unknown_list": "", + "problem_count": fmt.Sprintf("%d", warnCount+critCount+unknownCount), + "problem_list": "", + "detail_list": "", } problemList := []string{} @@ -342,6 +352,11 @@ func (cd *CheckData) buildListMacros() map[string]string { problemList = append(problemList, result["warn_list"]) detailList = append(detailList, result["warn_list"]) } + if len(unknownList) > 0 { + result["unknown_list"] = "unknown(" + strings.Join(unknownList, cd.listCombine) + ")" + problemList = append(problemList, result["unknown_list"]) + detailList = append(detailList, result["unknown_list"]) + } if len(okList) > 0 { result["ok_list"] = strings.Join(okList, cd.listCombine) detailList = append(detailList, result["ok_list"]) @@ -363,17 +378,21 @@ func (cd *CheckData) buildListMacrosFromSingleEntry() map[string]string { } result := map[string]string{ - "count": "1", - "list": expanded, - "ok_count": "0", - "ok_list": "", - "warn_count": "0", - "warn_list": "", - "crit_count": "0", - "crit_list": "", - "problem_count": "0", - "problem_list": "", - "detail_list": expanded, + "count": "1", + "list": expanded, + "ok_count": "0", + "ok_list": "", + "warn_count": "0", + "warning_count": "0", + "warn_list": "", + "crit_count": "0", + "critical_count": "0", + "crit_list": "", + "unknown_count": "0", + "unknown_list": "", + "problem_count": "0", + "problem_list": "", + "detail_list": expanded, } numWarn := 0 @@ -386,12 +405,21 @@ func (cd *CheckData) buildListMacrosFromSingleEntry() map[string]string { result["problem_list"] = expanded result["warn_list"] = expanded result["warn_count"] = "1" + result["warning_count"] = "1" + result["problem_count"] = "1" numWarn = 1 case "2": result["problem_list"] = expanded result["crit_list"] = expanded result["crit_count"] = "1" + result["critical_count"] = "1" + result["problem_count"] = "1" numCrit = 1 + case "3": + result["problem_list"] = expanded + result["unknown_list"] = expanded + result["unknown_count"] = "1" + result["problem_count"] = "1" } cd.buildCountMetrics(1, numCrit, numWarn) @@ -731,6 +759,9 @@ func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCr } cd.critThreshold = append(cd.critThreshold, cond) case "filter+": + if cd.disableFilter { + return nil, false, fmt.Errorf("%s is disabled for this check", keyword) + } applyDefaultFilter = false filter, err2 := cd.appendDefaultThreshold(keyword, argValue, cd.defaultFilter, cd.filter) if err2 != nil { @@ -738,6 +769,9 @@ func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCr } cd.filter = filter case "filter": + if cd.disableFilter { + return nil, false, fmt.Errorf("%s is disabled for this check", keyword) + } applyDefaultFilter = false cond, err2 := NewCondition(argValue, &cd.attributes) if err2 != nil { diff --git a/pkg/snclient/config.go b/pkg/snclient/config.go index ea0975e7..b2afb2a7 100644 --- a/pkg/snclient/config.go +++ b/pkg/snclient/config.go @@ -323,16 +323,44 @@ func (config *Config) ParseINI(configData, iniPath string, snc *Agent) error { continue } + isMultiSection := strings.HasPrefix(currentSection.name, "/settings/check/multi/") + // parse key and value val := strings.SplitN(line, "=", 2) + + // bare line (no '='): only allowed in check/multi sections, treated as raw command if len(val) < 2 { - parseErrors = append(parseErrors, fmt.Errorf("parse error in %s:%d: found key without '='", iniPath, lineNr)) + if !isMultiSection { + parseErrors = append(parseErrors, fmt.Errorf("parse error in %s:%d: found key without '='", iniPath, lineNr)) + + continue + } + if err := currentSection.SetRaw(line, ""); err != nil { + parseErrors = append(parseErrors, fmt.Errorf("config error in %s:%d: %s", iniPath, lineNr, err.Error())) + } + if len(currentComments) > 0 { + currentSection.comments[line] = currentComments + currentComments = make([]string, 0) + } continue } val[0] = strings.TrimSpace(val[0]) val[1] = strings.TrimSpace(val[1]) + // key contains space (e.g. 'check_process process=123'): also a raw command line in check/multi sections + if isMultiSection && strings.Contains(val[0], " ") { + if err := currentSection.SetRaw(line, ""); err != nil { + parseErrors = append(parseErrors, fmt.Errorf("config error in %s:%d: %s", iniPath, lineNr, err.Error())) + } + if len(currentComments) > 0 { + currentSection.comments[line] = currentComments + currentComments = make([]string, 0) + } + + continue + } + // silently skip UNKNOWN values which were placeholder in nsclient if val[1] == "UNKNOWN" { continue @@ -794,7 +822,11 @@ func (cs *ConfigSection) String() string { // none-multiline entries case 0, 1: if val == "" { - data = append(data, fmt.Sprintf("%s =", key)) + if strings.HasPrefix(cs.name, "/settings/check/multi/") { + data = append(data, key) + } else { + data = append(data, fmt.Sprintf("%s =", key)) + } } else { data = append(data, fmt.Sprintf("%s = %s", key, strings.Join(raw, ""))) } From 15e36f1591f559d8bc77f8bae510c918544409a6 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Fri, 14 Aug 2026 12:02:49 +0200 Subject: [PATCH 02/19] fix test, problem_count should be 1 here --- pkg/snclient/check_files_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/snclient/check_files_test.go b/pkg/snclient/check_files_test.go index fa9ffe7e..243e76dd 100644 --- a/pkg/snclient/check_files_test.go +++ b/pkg/snclient/check_files_test.go @@ -76,7 +76,7 @@ func TestCheckFiles(t *testing.T) { res = snc.RunCheck("check_files", []string{"path=./t/checksum.txt", "crit=md5_checksum != 3687C5D7106484CD61CDE867A2A999FA"}) assert.Equalf(t, CheckExitCritical, res.State, "CRITICAL") - assert.Contains(t, string(res.BuildPluginOutput()), "0/1 files") + assert.Contains(t, string(res.BuildPluginOutput()), "1/1 files") res = snc.RunCheck("check_files", []string{"path=./t/checksum.txt", "crit=sha1_checksum == 4EE4BFE9AA51E56A7BD5CCF4785C35A27EE022F8"}) assert.Equalf(t, CheckExitOK, res.State, "state OK") From c8b007d6f4191da7c9e4f540448c82622a740833 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Fri, 14 Aug 2026 13:25:04 +0200 Subject: [PATCH 03/19] fix windows tests --- README.md | 1 + pkg/snclient/check_multi_test.go | 42 ++++++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9b7586dc..f1df62ec 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ Further details are covered in the [documentation](https://omd.consol.de/docs/sn | **check_mailq** | | X | X | X | | **check_memory** | X | X | X | X | | **check_mount** | X | X | X | X | +| **check_multi** | X | X | X | X | | **check_network** | X | X | X | X | | **check_nsc_web** | X | X | X | X | | **check_ntp_offset** | X | X | X | X | diff --git a/pkg/snclient/check_multi_test.go b/pkg/snclient/check_multi_test.go index 885bfb81..ca7537b0 100644 --- a/pkg/snclient/check_multi_test.go +++ b/pkg/snclient/check_multi_test.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -138,17 +139,44 @@ CheckMulti = disabled } func TestCheckMultiConfigSection(t *testing.T) { - // Create a temporary shell script to test external scripts in config + // Create temporary scripts to test external scripts in config tmpDir := t.TempDir() - script1 := filepath.Join(tmpDir, "test1.sh") - script2 := filepath.Join(tmpDir, "test2.sh") + var scriptExt string + var script1Content, script2Content string - err := os.WriteFile(script1, []byte("#!/bin/sh\necho \"SCRIPT 1 OK | perf1=10;20;30\"\nexit 0\n"), 0o600) + if runtime.GOOS == "windows" { + scriptExt = ".ps1" + script1Content = `Write-Output "SCRIPT 1 OK | perf1=10;20;30" +exit 0 +` + script2Content = `Write-Output "SCRIPT 2 WARNING | perf2=50;40;60" +exit 1 +` + } else { + scriptExt = ".sh" + script1Content = `#!/bin/sh +echo "SCRIPT 1 OK | perf1=10;20;30" +exit 0 +` + script2Content = `#!/bin/sh +echo "SCRIPT 2 WARNING | perf2=50;40;60" +exit 1 +` + } + + script1 := filepath.Join(tmpDir, "test1"+scriptExt) + script2 := filepath.Join(tmpDir, "test2"+scriptExt) + + err := os.WriteFile(script1, []byte(script1Content), 0o600) require.NoError(t, err) - require.NoError(t, os.Chmod(script1, 0o700)) - err = os.WriteFile(script2, []byte("#!/bin/sh\necho \"SCRIPT 2 WARNING | perf2=50;40;60\"\nexit 1\n"), 0o600) + + err = os.WriteFile(script2, []byte(script2Content), 0o600) require.NoError(t, err) - require.NoError(t, os.Chmod(script2, 0o700)) + + if runtime.GOOS != "windows" { + require.NoError(t, os.Chmod(script1, 0o700)) + require.NoError(t, os.Chmod(script2, 0o700)) + } config := fmt.Sprintf(` [/modules] From 4fc5bc2a8819b04134f029ec68e6f81f4c749d05 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Fri, 14 Aug 2026 13:44:53 +0200 Subject: [PATCH 04/19] fix DOC_COMMANDS --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index db681a21..d616c7fe 100644 --- a/Makefile +++ b/Makefile @@ -737,6 +737,7 @@ DOC_COMMANDS=\ check_mailq \ check_memory \ check_mount \ + check_multi \ check_network \ check_ntp_offset \ check_omd \ From 461bc15628a28360a3f0321c1ec5244af5ceb613 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Fri, 14 Aug 2026 14:55:30 +0200 Subject: [PATCH 05/19] update docs --- docs/checks/commands/check_multi.md | 112 ++++++++++++++++++---------- pkg/snclient/check_multi.go | 43 +++++++++-- 2 files changed, 111 insertions(+), 44 deletions(-) diff --git a/docs/checks/commands/check_multi.md b/docs/checks/commands/check_multi.md index 68dda21b..53984adb 100644 --- a/docs/checks/commands/check_multi.md +++ b/docs/checks/commands/check_multi.md @@ -6,7 +6,27 @@ title: multi Runs multiple checks and aggregates their status, output and performance data. -In order to use this plugin, you need to enable 'CheckMulti' in the '[/modules]' section of the snclient.ini. + By default 'CheckMulti' is enabled, but you can disable it in the '[/modules]' section of the snclient_local.ini. + You can also set a limit for the number of checks that can be executed in the '[/settings/check/multi]' section + of the snclient_local.ini. + + When using the inline mode, you can only use available commands (run 'check_index' to get a full list). + + You can also define custom check sections in the config file, for example: + [/settings/check/multi/mycheck] + check_process process=123 + check_process process=345 + + This can be executed with 'check_multi "config=mycheck"'. + + It's also possible to use custom scripts in the config section, for example: + [/settings/check/multi/myscript] + /path/to/plugin1 + /path/to/plugin2 + /path/to/plugin3 + + This can be executed with 'check_multi "config=myscript"'. + - [Examples](#examples) - [Argument Defaults](#argument-defaults) @@ -20,55 +40,71 @@ In order to use this plugin, you need to enable 'CheckMulti' in the '[/modules]' ## Examples -### Inline Checks - - check_multi "check=check_process process=123" "check=check_process process=345" "warn=none" "crit=ok_count ne 2" - OK - 2 plugins checked, 2 ok +### Default Check -### Config Section Checks + check_multi "check=check_process 'process=firefox'" "check=check_memory 'crit=used_pct gt 80%'" + OK - 2 plugins checked, 2 ok | 'check_process::count'=1;;;0 'check_process::rss'=258686976B;;;0 ... + [ 1] check_process OK - all 1 processes are ok. + [ 2] check_memory OK - physical = 12.22 GiB/16.00 GiB (76.4%), swap = 1.95 GiB/3.00 GiB (65.0%) -Define checks in `snclient.ini` under `[/settings/check/multi/]`: + You can define warning/critical conditions based on the number of checks in a certain state (see attributes below): - [/settings/check/multi/mycheck] - check_process process=123 - check_process process=345 + check_multi "check=check_dummy 0 'OK'" "check=check_dummy 1 'WARNING'" "critical=problem_count gt 0" + CRITICAL - 2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown + [ 1] check_dummy OK + [ 2] check_dummy WARNING -Run the configured multi check: +### Example using NRPE and Naemon - check_multi "config=mycheck" "warn=none" "crit=ok_count ne 2" - OK - 2 plugins checked, 2 ok +Naemon Config -### External Script Config + define command{ + command_name check_nrpe + command_line $USER1$/check_nrpe -H $HOSTADDRESS$ -n -c $ARG1$ -a $ARG2$ + } - [/settings/check/multi/custom] - /opt/script/test.sh -H 123 - /opt/script/test2.sh -W 123 + define service { + host_name testhost + service_description check_multi + use generic-service + check_command check_nrpe!check_multi! + } -Run the configured multi check: +## Argument Defaults - check_multi "config=custom" "warn=problem_count gt 0" +| Argument | Default Value | +| ------------- | ----------------------------------------------------------------------------------------------------- | +| warning | warning_count > 0 | +| critical | critical_count > 0 \|\| unknown_count > 0 | +| empty-state | 3 (UNKNOWN) | +| empty-syntax | %(status) - no checks executed | +| top-syntax | %(status) - %(count) plugins checked: %(ok_count) ok, %(warning_count) warning, %(critical_count) critical, %(unknown_count) unknown%(problem_list) | +| ok-syntax | %(status) - %(count) plugins checked, %(ok_count) ok | +| detail-syntax | [%(status)] %(name): %(output) | ## Check Specific Arguments -| Argument | Default | Description | -| --- | --- | --- | -| check | | Inline check command to execute (can be specified multiple times) | -| config | | Config section name under `/settings/check/multi/` to execute | +| Argument | Description | +| -------- | ------------------------------------------------------------------------ | +| check | Check command to execute (can be specified multiple times) | +| config | Config section name under [/settings/check/multi/< section >] to execute | ## Attributes -| Filter / Threshold | Default | Description | -| --- | --- | --- | -| warn | `warning_count > 0` | Warning threshold | -| crit | `critical_count > 0 \|\| unknown_count > 0` | Critical threshold | -| count | | Total number of checks executed | -| ok_count | | Number of checks in OK state | -| warning_count | | Number of checks in WARNING state | -| critical_count | | Number of checks in CRITICAL state | -| unknown_count | | Number of checks in UNKNOWN state | -| problem_count | | Number of checks in non-OK state | -| name | | Name/tag of the check | -| command | | Command executed | -| state | | Exit code of the check (0=OK, 1=WARNING, 2=CRITICAL, 3=UNKNOWN) | -| status | | Status text of the check (OK, WARNING, CRITICAL, UNKNOWN) | -| output | | Output of the check | +### Filter Keywords + +these can be used in filters and thresholds (along with the default attributes): + +| Attribute | Description | +| -------------- | --------------------------------------------------------------- | +| count | Total number of checks executed | +| ok_count | Number of checks in OK state | +| warning_count | Number of checks in WARNING state | +| critical_count | Number of checks in CRITICAL state | +| unknown_count | Number of checks in UNKNOWN state | +| problem_count | Number of checks in non-OK state | +| name | Name/tag of the check | +| command | Command executed | +| state | Exit code of the check (0=OK, 1=WARNING, 2=CRITICAL, 3=UNKNOWN) | +| status | Status text of the check (OK, WARNING, CRITICAL, UNKNOWN) | +| output | Output of the check | diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go index cee23603..b75c276f 100644 --- a/pkg/snclient/check_multi.go +++ b/pkg/snclient/check_multi.go @@ -25,16 +25,38 @@ func NewCheckMulti() CheckHandler { func (l *CheckMulti) Build() *CheckData { return &CheckData{ - name: "check_multi", - description: "Runs multiple checks and aggregates their status, output and performance data.", + name: "check_multi", + description: `Runs multiple checks and aggregates their status, output and performance data. + + By default 'CheckMulti' is enabled, but you can disable it in the '[/modules]' section of the snclient_local.ini. + You can also set a limit for the number of checks that can be executed in the '[/settings/check/multi]' section + of the snclient_local.ini. + + When using the inline mode, you can only use available commands (run 'check_index' to get a full list). + + You can also define custom check sections in the config file, for example: + [/settings/check/multi/mycheck] + check_process process=123 + check_process process=345 + + This can be executed with 'check_multi "config=mycheck"'. + + It's also possible to use custom scripts in the config section, for example: + [/settings/check/multi/myscript] + /path/to/plugin1 + /path/to/plugin2 + /path/to/plugin3 + + This can be executed with 'check_multi "config=myscript"'. +`, implemented: ALL, disableFilter: true, result: &CheckResult{ State: CheckExitOK, }, args: map[string]CheckArgument{ - "check": {value: &l.checks, description: "Inline check command to execute (can be specified multiple times)"}, - "config": {value: &l.config, description: "Config section name under /settings/check/multi/ to execute"}, + "check": {value: &l.checks, description: "Check command to execute (can be specified multiple times)"}, + "config": {value: &l.config, description: "Config section name under [/settings/check/multi/< section >] to execute"}, }, conditionAlias: map[string]map[string]string{ "warning_count": {"warn_count": "warning_count"}, @@ -61,8 +83,17 @@ func (l *CheckMulti) Build() *CheckData { emptySyntax: "%(status) - no checks executed", emptyState: CheckExitUnknown, exampleDefault: ` - check_multi "check=check_process process=123" "check=check_process process=345" "warn=none" "crit=ok_count ne 2" - OK - 2 plugins checked, 2 ok + check_multi "check=check_process 'process=firefox'" "check=check_memory 'crit=used_pct gt 80%'" + OK - 2 plugins checked, 2 ok | 'check_process::count'=1;;;0 'check_process::rss'=258686976B;;;0 ... + [ 1] check_process OK - all 1 processes are ok. + [ 2] check_memory OK - physical = 12.22 GiB/16.00 GiB (76.4%), swap = 1.95 GiB/3.00 GiB (65.0%) + + You can define warning/critical conditions based on the number of checks in a certain state (see attributes below): + + check_multi "check=check_dummy 0 'OK'" "check=check_dummy 1 'WARNING'" "critical=problem_count gt 0" + CRITICAL - 2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown + [ 1] check_dummy OK + [ 2] check_dummy WARNING `, } } From 6542be3e8703e8d28115aa38d1a06e95528527a5 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Fri, 14 Aug 2026 15:41:09 +0200 Subject: [PATCH 06/19] improve docs --- docs/checks/commands/check_multi.md | 4 ++-- pkg/snclient/check_multi.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/checks/commands/check_multi.md b/docs/checks/commands/check_multi.md index 53984adb..9e987874 100644 --- a/docs/checks/commands/check_multi.md +++ b/docs/checks/commands/check_multi.md @@ -7,8 +7,8 @@ title: multi Runs multiple checks and aggregates their status, output and performance data. By default 'CheckMulti' is enabled, but you can disable it in the '[/modules]' section of the snclient_local.ini. - You can also set a limit for the number of checks that can be executed in the '[/settings/check/multi]' section - of the snclient_local.ini. + You can also set 'max checks' in the '[/settings/check/multi]' section of the snclient_local.ini, which limits + the number of checks that can be configured. When using the inline mode, you can only use available commands (run 'check_index' to get a full list). diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go index b75c276f..2da7ec4e 100644 --- a/pkg/snclient/check_multi.go +++ b/pkg/snclient/check_multi.go @@ -29,8 +29,8 @@ func (l *CheckMulti) Build() *CheckData { description: `Runs multiple checks and aggregates their status, output and performance data. By default 'CheckMulti' is enabled, but you can disable it in the '[/modules]' section of the snclient_local.ini. - You can also set a limit for the number of checks that can be executed in the '[/settings/check/multi]' section - of the snclient_local.ini. + You can also set 'max checks' in the '[/settings/check/multi]' section of the snclient_local.ini, which limits + the number of checks that can be configured. When using the inline mode, you can only use available commands (run 'check_index' to get a full list). From f8e8192862e2adc069453194f2cd141148fec868 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Mon, 17 Aug 2026 10:06:34 +0200 Subject: [PATCH 07/19] max checks documentation improvement --- packaging/snclient.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/snclient.ini b/packaging/snclient.ini index 3d8e20da..d4e82ce3 100644 --- a/packaging/snclient.ini +++ b/packaging/snclient.ini @@ -361,7 +361,7 @@ max lines per file limit = 1000000 [/settings/check/multi] -; max checks - Maximum number of checks check_multi can execute. +; max checks - Maximum number of checks that can be configured within check_multi (applies to config and inline). max checks = 20 From c99671c241b2cb9e0fb181c6e623ab33fde20372 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Tue, 18 Aug 2026 14:42:51 +0200 Subject: [PATCH 08/19] add _skip, fix potential duplicated output, add example --- docs/checks/commands/check_multi.md | 10 +++++++++- pkg/snclient/check_multi.go | 15 ++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/checks/commands/check_multi.md b/docs/checks/commands/check_multi.md index 9e987874..30de54de 100644 --- a/docs/checks/commands/check_multi.md +++ b/docs/checks/commands/check_multi.md @@ -47,13 +47,21 @@ Runs multiple checks and aggregates their status, output and performance data. [ 1] check_process OK - all 1 processes are ok. [ 2] check_memory OK - physical = 12.22 GiB/16.00 GiB (76.4%), swap = 1.95 GiB/3.00 GiB (65.0%) - You can define warning/critical conditions based on the number of checks in a certain state (see attributes below): + You can define 'warning' and 'critical' conditions based on the number of checks in a certain state (see attributes below): check_multi "check=check_dummy 0 'OK'" "check=check_dummy 1 'WARNING'" "critical=problem_count gt 0" CRITICAL - 2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown [ 1] check_dummy OK [ 2] check_dummy WARNING + You can also override the 'top-syntax' and use IF ELSE statements to get a certain output based on the results: + + check_multi "check=check_dummy 0 'OK'" "check=check_dummy 2 'CRITICAL'" \ + "top-syntax={{ if ok_count gt 0 }}OK - %(ok_count)/%(count) checks are OK {{ ELSE }}CRITICAL - all checks failed{{ END }}" + OK - 1/2 checks are OK + [ 1] check_dummy OK + [ 2] check_dummy CRITICAL + ### Example using NRPE and Naemon Naemon Config diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go index 2da7ec4e..0a503a91 100644 --- a/pkg/snclient/check_multi.go +++ b/pkg/snclient/check_multi.go @@ -88,12 +88,20 @@ func (l *CheckMulti) Build() *CheckData { [ 1] check_process OK - all 1 processes are ok. [ 2] check_memory OK - physical = 12.22 GiB/16.00 GiB (76.4%), swap = 1.95 GiB/3.00 GiB (65.0%) - You can define warning/critical conditions based on the number of checks in a certain state (see attributes below): + You can define 'warning' and 'critical' conditions based on the number of checks in a certain state (see attributes below): check_multi "check=check_dummy 0 'OK'" "check=check_dummy 1 'WARNING'" "critical=problem_count gt 0" CRITICAL - 2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown [ 1] check_dummy OK [ 2] check_dummy WARNING + + You can also override the 'top-syntax' and use IF ELSE statements to get a certain output based on the results: + + check_multi "check=check_dummy 0 'OK'" "check=check_dummy 2 'CRITICAL'" \ + "top-syntax={{ if ok_count gt 0 }}OK - %(ok_count)/%(count) checks are OK {{ ELSE }}CRITICAL - all checks failed{{ END }}" + OK - 1/2 checks are OK + [ 1] check_dummy OK + [ 2] check_dummy CRITICAL `, } } @@ -256,6 +264,7 @@ func (l *CheckMulti) executeChildChecks(ctx context.Context, snc *Agent, check * "status": res.StateString(), "output": firstLine, "_state": fmt.Sprintf("%d", res.State), + "_skip": "1", "_count": "1", } check.listData = append(check.listData, entry) @@ -314,9 +323,9 @@ func (l *CheckMulti) runChildCheck(ctx context.Context, snc *Agent, check *Check return snc.RunCheckWithContext(ctx, cmdName, cmdArgs, 0, nil, false), false } - stdout, stderr, exitCode, _ := snc.runExternalCheckString(ctx, chk.cmdStr, int64(check.timeout)) + stdout, stderr, exitCode, err := snc.runExternalCheckString(ctx, chk.cmdStr, int64(check.timeout)) out := stdout - if stderr != "" { + if stderr != "" && !strings.Contains(out, stderr) { if out != "" { out += "\n" } From c4fdee4452543bf992d8da0595a280d2560916f0 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Tue, 18 Aug 2026 14:47:40 +0200 Subject: [PATCH 09/19] fix citest --- pkg/snclient/check_multi.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go index 0a503a91..cfba891c 100644 --- a/pkg/snclient/check_multi.go +++ b/pkg/snclient/check_multi.go @@ -323,7 +323,7 @@ func (l *CheckMulti) runChildCheck(ctx context.Context, snc *Agent, check *Check return snc.RunCheckWithContext(ctx, cmdName, cmdArgs, 0, nil, false), false } - stdout, stderr, exitCode, err := snc.runExternalCheckString(ctx, chk.cmdStr, int64(check.timeout)) + stdout, stderr, exitCode, _ := snc.runExternalCheckString(ctx, chk.cmdStr, int64(check.timeout)) out := stdout if stderr != "" && !strings.Contains(out, stderr) { if out != "" { From f46c13a234a236b7be8df1af3277a1221be8ba35 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Tue, 18 Aug 2026 16:36:28 +0200 Subject: [PATCH 10/19] add unknown condition --- docs/checks/commands/check_multi.md | 15 ++--- pkg/snclient/check_multi.go | 15 ++--- pkg/snclient/check_multi_test.go | 34 +++++++++++ pkg/snclient/checkdata.go | 90 ++++++++++++++++++++++------- 4 files changed, 120 insertions(+), 34 deletions(-) diff --git a/docs/checks/commands/check_multi.md b/docs/checks/commands/check_multi.md index 30de54de..fa9b373f 100644 --- a/docs/checks/commands/check_multi.md +++ b/docs/checks/commands/check_multi.md @@ -49,10 +49,10 @@ Runs multiple checks and aggregates their status, output and performance data. You can define 'warning' and 'critical' conditions based on the number of checks in a certain state (see attributes below): - check_multi "check=check_dummy 0 'OK'" "check=check_dummy 1 'WARNING'" "critical=problem_count gt 0" - CRITICAL - 2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown - [ 1] check_dummy OK - [ 2] check_dummy WARNING + check_multi "check=check_dummy 0 'OK - check works'" "check=check_dummy 1 'WARNING - problem found'" "critical=problem_count gt 0" + CRITICAL - 2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown - warning(check_dummy: WARNING - problem found) + [ 1] check_dummy OK - check works + [ 2] check_dummy WARNING - problem found You can also override the 'top-syntax' and use IF ELSE statements to get a certain output based on the results: @@ -83,12 +83,13 @@ Naemon Config | Argument | Default Value | | ------------- | ----------------------------------------------------------------------------------------------------- | | warning | warning_count > 0 | -| critical | critical_count > 0 \|\| unknown_count > 0 | +| critical | critical_count > 0 | +| unknown | unknown_count > 0 | | empty-state | 3 (UNKNOWN) | | empty-syntax | %(status) - no checks executed | -| top-syntax | %(status) - %(count) plugins checked: %(ok_count) ok, %(warning_count) warning, %(critical_count) critical, %(unknown_count) unknown%(problem_list) | +| top-syntax | %(status) - %(count) plugins checked: %(ok_count) ok, %(warning_count) warning, %(critical_count) critical, %(unknown_count) unknown - %(problem_list) | | ok-syntax | %(status) - %(count) plugins checked, %(ok_count) ok | -| detail-syntax | [%(status)] %(name): %(output) | +| detail-syntax | %(name): %(output) | ## Check Specific Arguments diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go index cfba891c..3c9e6d44 100644 --- a/pkg/snclient/check_multi.go +++ b/pkg/snclient/check_multi.go @@ -76,10 +76,11 @@ func (l *CheckMulti) Build() *CheckData { {name: "output", description: "Output of the check", unit: UNone}, }, defaultWarning: "warning_count > 0", - defaultCritical: "critical_count > 0 || unknown_count > 0", + defaultCritical: "critical_count > 0", + defaultUnknown: "unknown_count > 0", okSyntax: "%(status) - %(count) plugins checked, %(ok_count) ok", - topSyntax: "%(status) - %(count) plugins checked: %(ok_count) ok, %(warning_count) warning, %(critical_count) critical, %(unknown_count) unknown%(problem_list)", - detailSyntax: "[%(status)] %(name): %(output)", + topSyntax: "%(status) - %(count) plugins checked: %(ok_count) ok, %(warning_count) warning, %(critical_count) critical, %(unknown_count) unknown - %(problem_list)", + detailSyntax: "%(name): %(output)", emptySyntax: "%(status) - no checks executed", emptyState: CheckExitUnknown, exampleDefault: ` @@ -90,10 +91,10 @@ func (l *CheckMulti) Build() *CheckData { You can define 'warning' and 'critical' conditions based on the number of checks in a certain state (see attributes below): - check_multi "check=check_dummy 0 'OK'" "check=check_dummy 1 'WARNING'" "critical=problem_count gt 0" - CRITICAL - 2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown - [ 1] check_dummy OK - [ 2] check_dummy WARNING + check_multi "check=check_dummy 0 'OK - check works'" "check=check_dummy 1 'WARNING - problem found'" "critical=problem_count gt 0" + CRITICAL - 2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown - warning(check_dummy: WARNING - problem found) + [ 1] check_dummy OK - check works + [ 2] check_dummy WARNING - problem found You can also override the 'top-syntax' and use IF ELSE statements to get a certain output based on the results: diff --git a/pkg/snclient/check_multi_test.go b/pkg/snclient/check_multi_test.go index ca7537b0..1c5d3d95 100644 --- a/pkg/snclient/check_multi_test.go +++ b/pkg/snclient/check_multi_test.go @@ -93,6 +93,40 @@ CheckMulti = enabled }) assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when filter argument is used") assert.Contains(t, res.Output, "filter is disabled for this check") + + // 8. Unknown threshold (default and custom) + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'ok'", + "check=check_dummy 3 'unknown check'", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when child check is unknown by default") + + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 0 'ok'", + "check=check_dummy 3 'unknown check'", + "unknown=unknown_count gt 0", + "warning=warning_count gt 0", + "critical=critical_count gt 0", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when custom unknown condition matches") + + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 1 'warn check'", + "check=check_dummy 3 'unknown check'", + "unknown=unknown_count gt 0", + "warning=warning_count gt 0", + "critical=critical_count gt 0", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN takes precedence over WARNING") + + res = snc.RunCheck("check_multi", []string{ + "check=check_dummy 2 'crit check'", + "check=check_dummy 3 'unknown check'", + "unknown=unknown_count gt 0", + "warning=warning_count gt 0", + "critical=critical_count gt 0", + }) + assert.Equalf(t, CheckExitCritical, res.State, "state CRITICAL takes precedence over UNKNOWN") } func TestCheckMultiLimits(t *testing.T) { diff --git a/pkg/snclient/checkdata.go b/pkg/snclient/checkdata.go index d4498eed..39d6e730 100644 --- a/pkg/snclient/checkdata.go +++ b/pkg/snclient/checkdata.go @@ -47,7 +47,8 @@ type CheckArgument struct { description string // used in help isFilter bool // if true, default filter is not used when this argument is set defaultCritical string // overrides default filter if argument is used - defaultWarning string // same for critical condition + defaultWarning string // same for warning condition + defaultUnknown string // same for unknown condition } // Implemented defines the available supported operating systems @@ -114,6 +115,8 @@ type CheckData struct { defaultWarning string critThreshold ConditionList defaultCritical string + unknownThreshold ConditionList + defaultUnknown string okThreshold ConditionList detailSyntax string topSyntax string @@ -160,13 +163,14 @@ func (cd *CheckData) Finalize() (*CheckResult, error) { log.Debugf("filter: %s", cd.filter.String()) log.Debugf("condition warning: %s", cd.warnThreshold.String()) log.Debugf("condition critical: %s", cd.critThreshold.String()) + log.Debugf("condition unknown: %s", cd.unknownThreshold.String()) log.Debugf("condition ok: %s", cd.okThreshold.String()) // Run thresholds once on cd.details. This is done separately than metrics or entries // cd.details are of type map[string]string, // same as elements of the slice cd.listData, but there is only one per check // This can possibly set a value to cd.details[_state] , influencing check state - log.Tracef("checking warning, critical, and ok thresholds on check details") - cd.Check(cd.details, cd.warnThreshold, cd.critThreshold, cd.okThreshold) + log.Tracef("checking warning, critical, unknown, and ok thresholds on check details") + cd.Check(cd.details, cd.warnThreshold, cd.critThreshold, cd.unknownThreshold, cd.okThreshold) log.Tracef("details:") logTraceASCIIMap(cd.details) @@ -217,7 +221,7 @@ func (cd *CheckData) finalizeOutput() (*CheckResult, error) { // each entry in the list data is individually checked // This can possibly set "_state" of each entry, influencing the final state - cd.Check(entry, cd.warnThreshold, cd.critThreshold, cd.okThreshold) + cd.Check(entry, cd.warnThreshold, cd.critThreshold, cd.unknownThreshold, cd.okThreshold) } } @@ -243,8 +247,8 @@ func (cd *CheckData) finalizeOutput() (*CheckResult, error) { cd.result.ApplyPerfSyntax(cd.perfSyntax, cd.timezone) // Run a separate check on the macros - log.Tracef("checking warning, critical, and ok thresholds on check macros") - cd.Check(finalMacros, cd.warnThreshold, cd.critThreshold, cd.okThreshold) + log.Tracef("checking warning, critical, unknown, and ok thresholds on check macros") + cd.Check(finalMacros, cd.warnThreshold, cd.critThreshold, cd.unknownThreshold, cd.okThreshold) log.Tracef("checking warning, critical, and ok thresholds on check metrics") cd.setStateFromMaps(finalMacros) @@ -489,9 +493,9 @@ func (cd *CheckData) setStateFromMaps(macros map[string]string) { cd.details["_state"] = fmt.Sprintf("%d", cd.result.State) } -// Check tries warn/crit/ok conditions against given data and sets result state. +// Check tries warn/crit/unknown/ok conditions against given data and sets result state. // The data argument can be anything that has the correct keys that conditions use -func (cd *CheckData) Check(data map[string]string, warnCond, critCond, okCond ConditionList) { +func (cd *CheckData) Check(data map[string]string, warnCond, critCond, unknownCond, okCond ConditionList) { data["_state"] = fmt.Sprintf("%d", CheckExitOK) for i := range warnCond { @@ -501,6 +505,13 @@ func (cd *CheckData) Check(data map[string]string, warnCond, critCond, okCond Co } } + for i := range unknownCond { + if res, ok := unknownCond[i].Match(data); res && ok { + log.Debugf("This given data matched the UNKNOWN condition: '%s' ", unknownCond[i].DetailedString()) + data["_state"] = fmt.Sprintf("%d", CheckExitUnknown) + } + } + for i := range critCond { if res, ok := critCond[i].Match(data); res && ok { log.Debugf("This given data matched the CRITICAL condition: '%s' ", critCond[i].DetailedString()) @@ -676,7 +687,7 @@ func (cd *CheckData) parseArgs(args []string) (argList []Argument, err error) { argList = make([]Argument, 0, len(args)) cd.expandArgDefinitions() - sanitized, defaultWarning, defaultCritical, applyDefaultFilter, err := cd.preParseArgs(args) + sanitized, defaultWarning, defaultCritical, defaultUnknown, applyDefaultFilter, err := cd.preParseArgs(args) if err != nil { return nil, err } @@ -687,7 +698,7 @@ func (cd *CheckData) parseArgs(args []string) (argList []Argument, err error) { argList = append(argList, Argument{key: arg.key, value: arg.value}) } } else { - argList, applyDefaultFilter, err = cd.processArgs(sanitized, defaultWarning, defaultCritical, applyDefaultFilter) + argList, applyDefaultFilter, err = cd.processArgs(sanitized, defaultWarning, defaultCritical, defaultUnknown, applyDefaultFilter) if err != nil { return nil, err } @@ -698,7 +709,7 @@ func (cd *CheckData) parseArgs(args []string) (argList []Argument, err error) { cd.timezone = timeZone } - err = cd.setFallbacks(applyDefaultFilter, defaultWarning, defaultCritical) + err = cd.setFallbacks(applyDefaultFilter, defaultWarning, defaultCritical, defaultUnknown) if err != nil { return nil, err } @@ -710,7 +721,7 @@ func (cd *CheckData) parseArgs(args []string) (argList []Argument, err error) { } //nolint:funlen,gocyclo // it is not complex, it is just a long list of options -func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCritical string, initialApplyDefaultFilter bool) (argList []Argument, applyDefaultFilter bool, err error) { +func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCritical, defaultUnknown string, initialApplyDefaultFilter bool) (argList []Argument, applyDefaultFilter bool, err error) { topSupplied := false okSupplied := false applyDefaultFilter = initialApplyDefaultFilter @@ -759,6 +770,18 @@ func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCr return nil, false, err2 } cd.critThreshold = append(cd.critThreshold, cond) + case "unknown+": + unknown, err2 := cd.appendDefaultThreshold(keyword, argValue, defaultUnknown, cd.unknownThreshold) + if err2 != nil { + return nil, false, err2 + } + cd.unknownThreshold = unknown + case "unknown": + cond, err2 := NewCondition(argValue, &cd.attributes) + if err2 != nil { + return nil, false, err2 + } + cd.unknownThreshold = append(cd.unknownThreshold, cond) case "filter+": if cd.disableFilter { return nil, false, fmt.Errorf("%s is disabled for this check", keyword) @@ -860,12 +883,13 @@ func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCr return argList, applyDefaultFilter, nil } -func (cd *CheckData) preParseArgs(args []string) (sanitized []Argument, defaultWarning, defaultCritical string, hasArgsFilter bool, err error) { +func (cd *CheckData) preParseArgs(args []string) (sanitized []Argument, defaultWarning, defaultCritical, defaultUnknown string, hasArgsFilter bool, err error) { sanitized = make([]Argument, 0) numArgs := len(args) applyDefaultFilter := true defaultWarning = cd.defaultWarning defaultCritical = cd.defaultCritical + defaultUnknown = cd.defaultUnknown for idx := 0; idx < numArgs; idx++ { argExpr := cd.removeQuotes(args[idx]) @@ -873,7 +897,7 @@ func (cd *CheckData) preParseArgs(args []string) (sanitized []Argument, defaultW keyword := cd.removeQuotes(split[0]) argValue, newIdx, err2 := cd.fetchNextArg(args, split, keyword, idx, numArgs) if err2 != nil { - return nil, "", "", false, err2 + return nil, "", "", "", false, err2 } idx = newIdx argValue = cd.removeQuotes(argValue) @@ -893,11 +917,14 @@ func (cd *CheckData) preParseArgs(args []string) (sanitized []Argument, defaultW if chkArg.defaultCritical != "" { defaultCritical = chkArg.defaultCritical } + if chkArg.defaultUnknown != "" { + defaultUnknown = chkArg.defaultUnknown + } } sanitized = append(sanitized, Argument{key: keyword, value: argValue, raw: argExpr}) } - return sanitized, defaultWarning, defaultCritical, applyDefaultFilter, nil + return sanitized, defaultWarning, defaultCritical, defaultUnknown, applyDefaultFilter, nil } // Threshold keywords do not necessarily have to match an attribute name. @@ -923,6 +950,14 @@ func (cd *CheckData) checkThresholdKeywordsAgainstAttributeNames() { } } + unknownKeywords, err := cd.unknownThreshold.GetListOfKeywords() + if err == nil && len(unknownKeywords) > 0 { + unknownKeywordsExtra := utils.SubtractSlice(unknownKeywords, attributeNames) + if len(unknownKeywordsExtra) > 0 { + log.Tracef("Unknown condition uses keyword(s) not present in the attributes, run with --help to get a list of attributes, extra keywords: %v", unknownKeywordsExtra) + } + } + okKeywords, err := cd.okThreshold.GetListOfKeywords() if err == nil && len(okKeywords) > 0 { okKeywordsExtra := utils.SubtractSlice(okKeywords, attributeNames) @@ -1087,8 +1122,8 @@ func (cd *CheckData) removeQuotes(str string) string { return str } -// setFallbacks sets default filter/warn/crit thresholds unless already set. -func (cd *CheckData) setFallbacks(applyDefaultFilter bool, defaultWarning, defaultCritical string) error { +// setFallbacks sets default filter/warn/crit/unknown thresholds unless already set. +func (cd *CheckData) setFallbacks(applyDefaultFilter bool, defaultWarning, defaultCritical, defaultUnknown string) error { if applyDefaultFilter && cd.defaultFilter != "" { cond, err := NewCondition(cd.defaultFilter, &cd.attributes) if err != nil { @@ -1097,16 +1132,20 @@ func (cd *CheckData) setFallbacks(applyDefaultFilter bool, defaultWarning, defau cd.filter = append(cd.filter, cond) } - // default warning/critical overridden from check arguments, ex. check_service + // default warning/critical/unknown overridden from check arguments, ex. check_service if defaultWarning != "" { cd.defaultWarning = defaultWarning } if defaultCritical != "" { cd.defaultCritical = defaultCritical } + if defaultUnknown != "" { + cd.defaultUnknown = defaultUnknown + } cd.warnThreshold = cd.applyDefaultThreshold(cd.defaultWarning, cd.warnThreshold) cd.critThreshold = cd.applyDefaultThreshold(cd.defaultCritical, cd.critThreshold) + cd.unknownThreshold = cd.applyDefaultThreshold(cd.defaultUnknown, cd.unknownThreshold) if cd.timeout == 0 { cd.timeout = DefaultCheckTimeout.Seconds() @@ -1156,6 +1195,7 @@ func (cd *CheckData) applyConditionColAlias() { cd.applyConditionColAliasList(cd.filter) cd.applyConditionColAliasList(cd.warnThreshold) cd.applyConditionColAliasList(cd.critThreshold) + cd.applyConditionColAliasList(cd.unknownThreshold) cd.applyConditionColAliasList(cd.okThreshold) } @@ -1201,6 +1241,7 @@ func (cd *CheckData) applyConditionAlias() { cd.applyConditionAliasList(cd.filter) cd.applyConditionAliasList(cd.warnThreshold) cd.applyConditionAliasList(cd.critThreshold) + cd.applyConditionAliasList(cd.unknownThreshold) cd.applyConditionAliasList(cd.okThreshold) } @@ -1237,6 +1278,9 @@ func (cd *CheckData) HasThreshold(name string) bool { if cd.hasThresholdCond(cd.critThreshold, name) { return true } + if cd.hasThresholdCond(cd.unknownThreshold, name) { + return true + } if cd.hasThresholdCond(cd.okThreshold, name) { return true } @@ -1244,16 +1288,18 @@ func (cd *CheckData) HasThreshold(name string) bool { return false } -// GetAllThresholdKeywords returns a list of all keywords used in warn/crit/ok thresholds. +// GetAllThresholdKeywords returns a list of all keywords used in warn/crit/unknown/ok thresholds. func (cd *CheckData) GetAllThresholdKeywords() []string { - keywords := make([]string, 0, len(cd.warnThreshold)+len(cd.critThreshold)+len(cd.okThreshold)) + keywords := make([]string, 0, len(cd.warnThreshold)+len(cd.critThreshold)+len(cd.unknownThreshold)+len(cd.okThreshold)) warnThresholdKeywords, _ := cd.warnThreshold.GetListOfKeywords() critThresholdKeywords, _ := cd.critThreshold.GetListOfKeywords() + unknownThresholdKeywords, _ := cd.unknownThreshold.GetListOfKeywords() okThresholdKeywords, _ := cd.okThreshold.GetListOfKeywords() keywords = append(keywords, warnThresholdKeywords...) keywords = append(keywords, critThresholdKeywords...) + keywords = append(keywords, unknownThresholdKeywords...) keywords = append(keywords, okThresholdKeywords...) utils.Deduplicate(keywords) @@ -1293,6 +1339,7 @@ func (cd *CheckData) SetDefaultThresholdUnit(defaultUnit string, names []string) } cd.VisitAll(cd.warnThreshold, setDefault) cd.VisitAll(cd.critThreshold, setDefault) + cd.VisitAll(cd.unknownThreshold, setDefault) cd.VisitAll(cd.okThreshold, setDefault) cd.VisitAll(cd.filter, setDefault) } @@ -1694,6 +1741,9 @@ func (cd *CheckData) helpDefaultArguments(format ShowHelp) string { if cd.defaultCritical != "" { defaultArgs = append(defaultArgs, defaultArg{name: "critical", defaults: cd.defaultCritical}) } + if cd.defaultUnknown != "" { + defaultArgs = append(defaultArgs, defaultArg{name: "unknown", defaults: cd.defaultUnknown}) + } defaultArgs = append( defaultArgs, defaultArg{name: "empty-state", defaults: fmt.Sprintf("%d (%s)", cd.emptyState, convert.StateString(cd.emptyState))}, From 48c30dfae81437e057271b6fe022db406ac3f007 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Tue, 18 Aug 2026 16:53:59 +0200 Subject: [PATCH 11/19] fix golangci lint --- pkg/snclient/check_multi.go | 28 +++++++++-------- pkg/snclient/checkdata.go | 60 ++++++++++++++++++++++--------------- 2 files changed, 51 insertions(+), 37 deletions(-) diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go index 3c9e6d44..52baf7d7 100644 --- a/pkg/snclient/check_multi.go +++ b/pkg/snclient/check_multi.go @@ -17,6 +17,20 @@ type CheckMulti struct { config string } +var checkMultiAttributes = []CheckAttribute{ + {name: "count", description: "Total number of checks executed", unit: UNone}, + {name: "ok_count", description: "Number of checks in OK state", unit: UNone}, + {name: "warning_count", description: "Number of checks in WARNING state", unit: UNone}, + {name: "critical_count", description: "Number of checks in CRITICAL state", unit: UNone}, + {name: "unknown_count", description: "Number of checks in UNKNOWN state", unit: UNone}, + {name: "problem_count", description: "Number of checks in non-OK state", unit: UNone}, + {name: "name", description: "Name/tag of the check", unit: UNone}, + {name: "command", description: "Command executed", unit: UNone}, + {name: "state", description: "Exit code of the check (0=OK, 1=WARNING, 2=CRITICAL, 3=UNKNOWN)", unit: UNone}, + {name: "status", description: "Status text of the check (OK, WARNING, CRITICAL, UNKNOWN)", unit: UNone}, + {name: "output", description: "Output of the check", unit: UNone}, +} + func NewCheckMulti() CheckHandler { return &CheckMulti{ checks: make([]string, 0), @@ -62,19 +76,7 @@ func (l *CheckMulti) Build() *CheckData { "warning_count": {"warn_count": "warning_count"}, "critical_count": {"crit_count": "critical_count"}, }, - attributes: []CheckAttribute{ - {name: "count", description: "Total number of checks executed", unit: UNone}, - {name: "ok_count", description: "Number of checks in OK state", unit: UNone}, - {name: "warning_count", description: "Number of checks in WARNING state", unit: UNone}, - {name: "critical_count", description: "Number of checks in CRITICAL state", unit: UNone}, - {name: "unknown_count", description: "Number of checks in UNKNOWN state", unit: UNone}, - {name: "problem_count", description: "Number of checks in non-OK state", unit: UNone}, - {name: "name", description: "Name/tag of the check", unit: UNone}, - {name: "command", description: "Command executed", unit: UNone}, - {name: "state", description: "Exit code of the check (0=OK, 1=WARNING, 2=CRITICAL, 3=UNKNOWN)", unit: UNone}, - {name: "status", description: "Status text of the check (OK, WARNING, CRITICAL, UNKNOWN)", unit: UNone}, - {name: "output", description: "Output of the check", unit: UNone}, - }, + attributes: checkMultiAttributes, defaultWarning: "warning_count > 0", defaultCritical: "critical_count > 0", defaultUnknown: "unknown_count > 0", diff --git a/pkg/snclient/checkdata.go b/pkg/snclient/checkdata.go index 39d6e730..9e08eea3 100644 --- a/pkg/snclient/checkdata.go +++ b/pkg/snclient/checkdata.go @@ -687,18 +687,20 @@ func (cd *CheckData) parseArgs(args []string) (argList []Argument, err error) { argList = make([]Argument, 0, len(args)) cd.expandArgDefinitions() - sanitized, defaultWarning, defaultCritical, defaultUnknown, applyDefaultFilter, err := cd.preParseArgs(args) + pre, err := cd.preParseArgs(args) if err != nil { return nil, err } + applyDefaultFilter := pre.applyDefaultFilter + // skip argument parsing for external scripts if _, ok := AvailableChecks[cd.name]; !ok && cd.argsPassthrough { - for _, arg := range sanitized { + for _, arg := range pre.sanitized { argList = append(argList, Argument{key: arg.key, value: arg.value}) } } else { - argList, applyDefaultFilter, err = cd.processArgs(sanitized, defaultWarning, defaultCritical, defaultUnknown, applyDefaultFilter) + argList, applyDefaultFilter, err = cd.processArgs(pre) if err != nil { return nil, err } @@ -709,7 +711,7 @@ func (cd *CheckData) parseArgs(args []string) (argList []Argument, err error) { cd.timezone = timeZone } - err = cd.setFallbacks(applyDefaultFilter, defaultWarning, defaultCritical, defaultUnknown) + err = cd.setFallbacks(applyDefaultFilter, pre.defaultWarning, pre.defaultCritical, pre.defaultUnknown) if err != nil { return nil, err } @@ -720,13 +722,13 @@ func (cd *CheckData) parseArgs(args []string) (argList []Argument, err error) { return argList, nil } -//nolint:funlen,gocyclo // it is not complex, it is just a long list of options -func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCritical, defaultUnknown string, initialApplyDefaultFilter bool) (argList []Argument, applyDefaultFilter bool, err error) { +//nolint:funlen,gocyclo,maintidx // it is not complex, it is just a long list of options +func (cd *CheckData) processArgs(pre *preParsedArgs) (argList []Argument, applyDefaultFilter bool, err error) { topSupplied := false okSupplied := false - applyDefaultFilter = initialApplyDefaultFilter + applyDefaultFilter = pre.applyDefaultFilter - for _, arg := range sanitized { + for _, arg := range pre.sanitized { keyword := arg.key argValue := arg.value argExpr := arg.raw @@ -747,7 +749,7 @@ func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCr } cd.okThreshold = append(cd.okThreshold, cond) case "warn+", "warning+": - warn, err2 := cd.appendDefaultThreshold(keyword, argValue, defaultWarning, cd.warnThreshold) + warn, err2 := cd.appendDefaultThreshold(keyword, argValue, pre.defaultWarning, cd.warnThreshold) if err2 != nil { return nil, false, err2 } @@ -759,7 +761,7 @@ func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCr } cd.warnThreshold = append(cd.warnThreshold, cond) case "crit+", "critical+": - crit, err2 := cd.appendDefaultThreshold(keyword, argValue, defaultCritical, cd.critThreshold) + crit, err2 := cd.appendDefaultThreshold(keyword, argValue, pre.defaultCritical, cd.critThreshold) if err2 != nil { return nil, false, err2 } @@ -771,7 +773,7 @@ func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCr } cd.critThreshold = append(cd.critThreshold, cond) case "unknown+": - unknown, err2 := cd.appendDefaultThreshold(keyword, argValue, defaultUnknown, cd.unknownThreshold) + unknown, err2 := cd.appendDefaultThreshold(keyword, argValue, pre.defaultUnknown, cd.unknownThreshold) if err2 != nil { return nil, false, err2 } @@ -883,13 +885,23 @@ func (cd *CheckData) processArgs(sanitized []Argument, defaultWarning, defaultCr return argList, applyDefaultFilter, nil } -func (cd *CheckData) preParseArgs(args []string) (sanitized []Argument, defaultWarning, defaultCritical, defaultUnknown string, hasArgsFilter bool, err error) { - sanitized = make([]Argument, 0) +type preParsedArgs struct { + sanitized []Argument + defaultWarning string + defaultCritical string + defaultUnknown string + applyDefaultFilter bool +} + +func (cd *CheckData) preParseArgs(args []string) (pre *preParsedArgs, err error) { + pre = &preParsedArgs{ + sanitized: make([]Argument, 0), + applyDefaultFilter: true, + defaultWarning: cd.defaultWarning, + defaultCritical: cd.defaultCritical, + defaultUnknown: cd.defaultUnknown, + } numArgs := len(args) - applyDefaultFilter := true - defaultWarning = cd.defaultWarning - defaultCritical = cd.defaultCritical - defaultUnknown = cd.defaultUnknown for idx := 0; idx < numArgs; idx++ { argExpr := cd.removeQuotes(args[idx]) @@ -897,7 +909,7 @@ func (cd *CheckData) preParseArgs(args []string) (sanitized []Argument, defaultW keyword := cd.removeQuotes(split[0]) argValue, newIdx, err2 := cd.fetchNextArg(args, split, keyword, idx, numArgs) if err2 != nil { - return nil, "", "", "", false, err2 + return pre, err2 } idx = newIdx argValue = cd.removeQuotes(argValue) @@ -909,22 +921,22 @@ func (cd *CheckData) preParseArgs(args []string) (sanitized []Argument, defaultW chkArg = &a } if chkArg != nil { - applyDefaultFilter = false + pre.applyDefaultFilter = false cd.hasArgsFilter = true if chkArg.defaultWarning != "" { - defaultWarning = chkArg.defaultWarning + pre.defaultWarning = chkArg.defaultWarning } if chkArg.defaultCritical != "" { - defaultCritical = chkArg.defaultCritical + pre.defaultCritical = chkArg.defaultCritical } if chkArg.defaultUnknown != "" { - defaultUnknown = chkArg.defaultUnknown + pre.defaultUnknown = chkArg.defaultUnknown } } - sanitized = append(sanitized, Argument{key: keyword, value: argValue, raw: argExpr}) + pre.sanitized = append(pre.sanitized, Argument{key: keyword, value: argValue, raw: argExpr}) } - return sanitized, defaultWarning, defaultCritical, defaultUnknown, applyDefaultFilter, nil + return pre, nil } // Threshold keywords do not necessarily have to match an attribute name. From c1c6944b64dae3acbdfbb889702b25d5f221a5ed Mon Sep 17 00:00:00 2001 From: lorenzg Date: Wed, 19 Aug 2026 16:53:07 +0200 Subject: [PATCH 12/19] adjust syntax, add new attributes, changed default output --- docs/checks/commands/check_multi.md | 46 +++--- pkg/snclient/check_multi.go | 240 ++++++++++++++++++---------- pkg/snclient/check_multi_test.go | 153 +++++++++--------- pkg/snclient/checkdata.go | 106 +++++++++--- pkg/snclient/config.go | 29 +--- 5 files changed, 339 insertions(+), 235 deletions(-) diff --git a/docs/checks/commands/check_multi.md b/docs/checks/commands/check_multi.md index fa9b373f..ce3213d0 100644 --- a/docs/checks/commands/check_multi.md +++ b/docs/checks/commands/check_multi.md @@ -14,16 +14,16 @@ Runs multiple checks and aggregates their status, output and performance data. You can also define custom check sections in the config file, for example: [/settings/check/multi/mycheck] - check_process process=123 - check_process process=345 + command[alias1] = check_process process=123 + command[alias2] = check_process process=345 This can be executed with 'check_multi "config=mycheck"'. It's also possible to use custom scripts in the config section, for example: [/settings/check/multi/myscript] - /path/to/plugin1 - /path/to/plugin2 - /path/to/plugin3 + command[alias1] = /path/to/plugin1 + command[alias2] = /path/to/plugin2 + command[alias3] = /path/to/plugin3 This can be executed with 'check_multi "config=myscript"'. @@ -42,25 +42,25 @@ Runs multiple checks and aggregates their status, output and performance data. ### Default Check - check_multi "check=check_process 'process=firefox'" "check=check_memory 'crit=used_pct gt 80%'" - OK - 2 plugins checked, 2 ok | 'check_process::count'=1;;;0 'check_process::rss'=258686976B;;;0 ... - [ 1] check_process OK - all 1 processes are ok. - [ 2] check_memory OK - physical = 12.22 GiB/16.00 GiB (76.4%), swap = 1.95 GiB/3.00 GiB (65.0%) + check_multi "command[check_process]=check_process 'process=firefox'" "command[check_memory]=check_memory 'type=physical' 'crit=used_pct gt 80%'" + OK - 2 plugins checked, 2 ok |'check_process::count'=1;;;0 ... 'check_memory::physical %'=78.7%;;;0;100 + [check_process] OK - all 1 processes are ok. + [check_memory] OK - physical = 12.59 GiB/16.00 GiB (78.7%) You can define 'warning' and 'critical' conditions based on the number of checks in a certain state (see attributes below): - check_multi "check=check_dummy 0 'OK - check works'" "check=check_dummy 1 'WARNING - problem found'" "critical=problem_count gt 0" - CRITICAL - 2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown - warning(check_dummy: WARNING - problem found) - [ 1] check_dummy OK - check works - [ 2] check_dummy WARNING - problem found + check_multi "command[check_dummy1]=check_dummy 0 'OK - check works'" "command[check_dummy2]=check_dummy 1 'WARNING - problem found'" "critical=problem_count gt 0" + CRITICAL - 2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown - warning(check_dummy2: WARNING - problem found) + [check_dummy1] OK - check works + [check_dummy2] WARNING - problem found You can also override the 'top-syntax' and use IF ELSE statements to get a certain output based on the results: - check_multi "check=check_dummy 0 'OK'" "check=check_dummy 2 'CRITICAL'" \ + check_multi "command[check_dummy1]=check_dummy 0 'OK'" "command[check_dummy2]=check_dummy 2 'CRITICAL'" \ "top-syntax={{ if ok_count gt 0 }}OK - %(ok_count)/%(count) checks are OK {{ ELSE }}CRITICAL - all checks failed{{ END }}" OK - 1/2 checks are OK - [ 1] check_dummy OK - [ 2] check_dummy CRITICAL + [check_dummy1] OK + [check_dummy2] CRITICAL ### Example using NRPE and Naemon @@ -93,10 +93,10 @@ Naemon Config ## Check Specific Arguments -| Argument | Description | -| -------- | ------------------------------------------------------------------------ | -| check | Check command to execute (can be specified multiple times) | -| config | Config section name under [/settings/check/multi/< section >] to execute | +| Argument | Description | +| -------- | ------------------------------------------------------------------------- | +| command | Check command to execute with mandatory unique tag, e.g. command[tag]=... | +| config | Config section name under [/settings/check/multi/< section >] to execute | ## Attributes @@ -112,8 +112,10 @@ these can be used in filters and thresholds (along with the default attributes): | critical_count | Number of checks in CRITICAL state | | unknown_count | Number of checks in UNKNOWN state | | problem_count | Number of checks in non-OK state | -| name | Name/tag of the check | +| name | Name/Tag of the check | +| tag | Alias for name | | command | Command executed | | state | Exit code of the check (0=OK, 1=WARNING, 2=CRITICAL, 3=UNKNOWN) | | status | Status text of the check (OK, WARNING, CRITICAL, UNKNOWN) | -| output | Output of the check | +| output | Check output | +| shortoutput | First line of the check output | diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go index 52baf7d7..c321b3fb 100644 --- a/pkg/snclient/check_multi.go +++ b/pkg/snclient/check_multi.go @@ -3,6 +3,7 @@ package snclient import ( "context" "fmt" + "maps" "strings" "github.com/consol-monitoring/snclient/pkg/utils" @@ -12,28 +13,35 @@ func init() { AvailableChecks["check_multi"] = CheckEntry{"check_multi", NewCheckMulti} } +type ( + checkMultiConfigKey struct{} + checkMultiDepthKey struct{} +) + type CheckMulti struct { - checks []string - config string + commands TaggedCommandList + config string } var checkMultiAttributes = []CheckAttribute{ - {name: "count", description: "Total number of checks executed", unit: UNone}, - {name: "ok_count", description: "Number of checks in OK state", unit: UNone}, - {name: "warning_count", description: "Number of checks in WARNING state", unit: UNone}, - {name: "critical_count", description: "Number of checks in CRITICAL state", unit: UNone}, - {name: "unknown_count", description: "Number of checks in UNKNOWN state", unit: UNone}, - {name: "problem_count", description: "Number of checks in non-OK state", unit: UNone}, - {name: "name", description: "Name/tag of the check", unit: UNone}, - {name: "command", description: "Command executed", unit: UNone}, - {name: "state", description: "Exit code of the check (0=OK, 1=WARNING, 2=CRITICAL, 3=UNKNOWN)", unit: UNone}, - {name: "status", description: "Status text of the check (OK, WARNING, CRITICAL, UNKNOWN)", unit: UNone}, - {name: "output", description: "Output of the check", unit: UNone}, + {name: "count", description: "Total number of checks executed"}, + {name: "ok_count", description: "Number of checks in OK state"}, + {name: "warning_count", description: "Number of checks in WARNING state"}, + {name: "critical_count", description: "Number of checks in CRITICAL state"}, + {name: "unknown_count", description: "Number of checks in UNKNOWN state"}, + {name: "problem_count", description: "Number of checks in non-OK state"}, + {name: "name", description: "Name/Tag of the check"}, + {name: "tag", description: "Alias for name"}, + {name: "command", description: "Command executed"}, + {name: "state", description: "Exit code of the check (0=OK, 1=WARNING, 2=CRITICAL, 3=UNKNOWN)"}, + {name: "status", description: "Status text of the check (OK, WARNING, CRITICAL, UNKNOWN)"}, + {name: "output", description: "Check output"}, + {name: "shortoutput", description: "First line of the check output"}, } func NewCheckMulti() CheckHandler { return &CheckMulti{ - checks: make([]string, 0), + commands: make(TaggedCommandList, 0), } } @@ -50,16 +58,16 @@ func (l *CheckMulti) Build() *CheckData { You can also define custom check sections in the config file, for example: [/settings/check/multi/mycheck] - check_process process=123 - check_process process=345 + command[alias1] = check_process process=123 + command[alias2] = check_process process=345 This can be executed with 'check_multi "config=mycheck"'. It's also possible to use custom scripts in the config section, for example: [/settings/check/multi/myscript] - /path/to/plugin1 - /path/to/plugin2 - /path/to/plugin3 + command[alias1] = /path/to/plugin1 + command[alias2] = /path/to/plugin2 + command[alias3] = /path/to/plugin3 This can be executed with 'check_multi "config=myscript"'. `, @@ -69,8 +77,8 @@ func (l *CheckMulti) Build() *CheckData { State: CheckExitOK, }, args: map[string]CheckArgument{ - "check": {value: &l.checks, description: "Check command to execute (can be specified multiple times)"}, - "config": {value: &l.config, description: "Config section name under [/settings/check/multi/< section >] to execute"}, + "command": {value: &l.commands, description: "Check command to execute with mandatory unique tag, e.g. command[tag]=..."}, + "config": {value: &l.config, description: "Config section name under [/settings/check/multi/< section >] to execute"}, }, conditionAlias: map[string]map[string]string{ "warning_count": {"warn_count": "warning_count"}, @@ -86,25 +94,25 @@ func (l *CheckMulti) Build() *CheckData { emptySyntax: "%(status) - no checks executed", emptyState: CheckExitUnknown, exampleDefault: ` - check_multi "check=check_process 'process=firefox'" "check=check_memory 'crit=used_pct gt 80%'" - OK - 2 plugins checked, 2 ok | 'check_process::count'=1;;;0 'check_process::rss'=258686976B;;;0 ... - [ 1] check_process OK - all 1 processes are ok. - [ 2] check_memory OK - physical = 12.22 GiB/16.00 GiB (76.4%), swap = 1.95 GiB/3.00 GiB (65.0%) + check_multi "command[check_process]=check_process 'process=firefox'" "command[check_memory]=check_memory 'type=physical' 'crit=used_pct gt 80%'" + OK - 2 plugins checked, 2 ok |'check_process::count'=1;;;0 ... 'check_memory::physical %'=78.7%;;;0;100 + [check_process] OK - all 1 processes are ok. + [check_memory] OK - physical = 12.59 GiB/16.00 GiB (78.7%) You can define 'warning' and 'critical' conditions based on the number of checks in a certain state (see attributes below): - check_multi "check=check_dummy 0 'OK - check works'" "check=check_dummy 1 'WARNING - problem found'" "critical=problem_count gt 0" - CRITICAL - 2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown - warning(check_dummy: WARNING - problem found) - [ 1] check_dummy OK - check works - [ 2] check_dummy WARNING - problem found + check_multi "command[check_dummy1]=check_dummy 0 'OK - check works'" "command[check_dummy2]=check_dummy 1 'WARNING - problem found'" "critical=problem_count gt 0" + CRITICAL - 2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown - warning(check_dummy2: WARNING - problem found) + [check_dummy1] OK - check works + [check_dummy2] WARNING - problem found You can also override the 'top-syntax' and use IF ELSE statements to get a certain output based on the results: - check_multi "check=check_dummy 0 'OK'" "check=check_dummy 2 'CRITICAL'" \ + check_multi "command[check_dummy1]=check_dummy 0 'OK'" "command[check_dummy2]=check_dummy 2 'CRITICAL'" \ "top-syntax={{ if ok_count gt 0 }}OK - %(ok_count)/%(count) checks are OK {{ ELSE }}CRITICAL - all checks failed{{ END }}" OK - 1/2 checks are OK - [ 1] check_dummy OK - [ 2] check_dummy CRITICAL + [check_dummy1] OK + [check_dummy2] CRITICAL `, } } @@ -124,6 +132,33 @@ func (l *CheckMulti) Check(ctx context.Context, snc *Agent, check *CheckData, _ }, nil } + depth, _ := ctx.Value(checkMultiDepthKey{}).(int) + if depth > 5 { + return &CheckResult{ + State: CheckExitUnknown, + Output: "recursion limit exceeded for check_multi", + }, nil + } + ctx = context.WithValue(ctx, checkMultiDepthKey{}, depth+1) + + activeConfigs, _ := ctx.Value(checkMultiConfigKey{}).(map[string]bool) + if activeConfigs == nil { + activeConfigs = make(map[string]bool) + } + + if l.config != "" { + if activeConfigs[l.config] { + return &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("loop detected: check_multi config %s is already running in the call chain", l.config), + }, nil + } + newActive := make(map[string]bool, len(activeConfigs)+1) + maps.Copy(newActive, activeConfigs) + newActive[l.config] = true + ctx = context.WithValue(ctx, checkMultiConfigKey{}, newActive) + } + maxChecks, ok, err := snc.config.Section("/settings/check/multi").GetInt("max checks") if err != nil || !ok || maxChecks <= 0 { maxChecks = 20 @@ -154,23 +189,36 @@ func (l *CheckMulti) Check(ctx context.Context, snc *Agent, check *CheckData, _ // buildChildChecks assembles the list of child checks from config section and inline args. func (l *CheckMulti) buildChildChecks(snc *Agent) ([]multiChildCheck, *CheckResult) { childChecks := []multiChildCheck{} + seenTags := make(map[string]bool) if l.config != "" { configChecks, res := l.buildConfigChecks(snc) if res != nil { return nil, res } - childChecks = append(childChecks, configChecks...) + for _, chk := range configChecks { + if seenTags[chk.tag] { + return nil, &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("duplicate command tag: %s", chk.tag), + } + } + seenTags[chk.tag] = true + childChecks = append(childChecks, chk) + } } - for _, inlineCmd := range l.checks { - inlineCmd = strings.TrimSpace(inlineCmd) - if inlineCmd == "" { - continue + for _, cmd := range l.commands { + if seenTags[cmd.Tag] { + return nil, &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("duplicate command tag: %s", cmd.Tag), + } } + seenTags[cmd.Tag] = true childChecks = append(childChecks, multiChildCheck{ - tag: "", - cmdStr: inlineCmd, + tag: cmd.Tag, + cmdStr: cmd.Command, isInline: true, }) } @@ -191,12 +239,45 @@ func (l *CheckMulti) buildConfigChecks(snc *Agent) ([]multiChildCheck, *CheckRes } childChecks := make([]multiChildCheck, 0, len(sec.keys)) + seenTags := make(map[string]bool) for _, key := range sec.keys { - rawCmd, tag := l.resolveConfigEntry(snc, key, sec.data[key]) + rawVal := sec.data[key] + if !strings.HasPrefix(key, "command[") || !strings.HasSuffix(key, "]") { + return nil, &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("invalid check_multi config entry: %s (must be in format command[tag]=)", key), + } + } + tag := strings.TrimSuffix(strings.TrimPrefix(key, "command["), "]") + if strings.ContainsAny(tag, DefaultNastyCharacters+"=") { + return nil, &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("command tag contains invalid characters: %s", tag), + } + } + if strings.TrimSpace(tag) == "" { + return nil, &CheckResult{ + State: CheckExitUnknown, + Output: "empty command tag in config section", + } + } + if strings.TrimSpace(rawVal) == "" { + return nil, &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("empty command for tag %s in config section", tag), + } + } + if seenTags[tag] { + return nil, &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("duplicate command tag: %s", tag), + } + } + seenTags[tag] = true childChecks = append(childChecks, multiChildCheck{ tag: tag, - cmdStr: rawCmd, + cmdStr: rawVal, isInline: false, }) } @@ -204,22 +285,6 @@ func (l *CheckMulti) buildConfigChecks(snc *Agent) ([]multiChildCheck, *CheckRes return childChecks, nil } -// resolveConfigEntry determines the raw command and tag for a single config section entry. -func (l *CheckMulti) resolveConfigEntry(snc *Agent, key, val string) (rawCmd, tag string) { - rawCmd = key - tag = "" - - if val == "" { - return rawCmd, tag - } - - if _, isKnown := snc.getCheck(key, false); isKnown { - return key + " " + val, key - } - - return val, key -} - // executeChildChecks runs all child checks and aggregates results. func (l *CheckMulti) executeChildChecks(ctx context.Context, snc *Agent, check *CheckData, childChecks []multiChildCheck) (*CheckResult, error) { var count, okCount, warnCount, critCount, unknownCount int64 @@ -227,49 +292,50 @@ func (l *CheckMulti) executeChildChecks(ctx context.Context, snc *Agent, check * detailsList := make([]string, 0, len(childChecks)) allMetrics := make([]*CheckMetric, 0) - for idx, chk := range childChecks { + hasEntryThresholds := check.HasThreshold("name") || check.HasThreshold("tag") || check.HasThreshold("command") || + check.HasThreshold("output") || check.HasThreshold("shortoutput") || check.HasThreshold("status") || check.HasThreshold("state") + + for _, chk := range childChecks { res, fatal := l.runChildCheck(ctx, snc, check, chk) if fatal { return res, nil } + tag := chk.tag + + firstLine := strings.TrimSpace(strings.Split(res.Output, "\n")[0]) + detailsList = append(detailsList, fmt.Sprintf("[%s] %s", tag, res.Output)) + + entryState := fmt.Sprintf("%d", res.State) + entry := map[string]string{ + "name": tag, + "tag": tag, + "command": chk.cmdStr, + "state": entryState, + "status": res.StateString(), + "shortoutput": firstLine, + "output": res.Output, + "_state": entryState, + "_skip": "1", + "_count": "1", + } + + if hasEntryThresholds { + check.Check(entry, check.warnThreshold, check.critThreshold, check.unknownThreshold, check.okThreshold) + } + count++ - switch res.State { - case CheckExitOK: + switch entry["_state"] { + case "0": okCount++ - case CheckExitWarning: + case "1": warnCount++ - case CheckExitCritical: + case "2": critCount++ default: unknownCount++ } - tokens := utils.Tokenize(chk.cmdStr) - cmdName := chk.cmdStr - if len(tokens) > 0 { - cmdName = tokens[0] - } - - tag := chk.tag - if tag == "" { - tag = cmdName - } - - firstLine := strings.TrimSpace(strings.Split(res.Output, "\n")[0]) - detailsList = append(detailsList, fmt.Sprintf("[% 2d] %s %s", idx+1, tag, res.Output)) - - entry := map[string]string{ - "idx": fmt.Sprintf("%d", idx+1), - "name": tag, - "command": chk.cmdStr, - "state": fmt.Sprintf("%d", res.State), - "status": res.StateString(), - "output": firstLine, - "_state": fmt.Sprintf("%d", res.State), - "_skip": "1", - "_count": "1", - } check.listData = append(check.listData, entry) for _, m := range res.Metrics { diff --git a/pkg/snclient/check_multi_test.go b/pkg/snclient/check_multi_test.go index 1c5d3d95..45c8d84e 100644 --- a/pkg/snclient/check_multi_test.go +++ b/pkg/snclient/check_multi_test.go @@ -19,114 +19,112 @@ CheckMulti = enabled snc := StartTestAgent(t, config) defer StopTestAgent(t, snc) - // 1. Basic inline checks - all OK + // 1. Basic inline checks with mandatory tags - all OK res := snc.RunCheck("check_multi", []string{ - "check=check_dummy 0 'dummy ok 1'", - "check=check_dummy 0 'dummy ok 2'", + "command[d1]=check_dummy 0 'dummy ok 1'", + "command[d2]=check_dummy 0 'dummy ok 2'", }) assert.Equalf(t, CheckExitOK, res.State, "state OK") assert.Contains(t, res.Output, "2 plugins checked, 2 ok") - assert.Contains(t, res.Details, "dummy ok 1") - assert.Contains(t, res.Details, "dummy ok 2") + assert.Contains(t, res.Details, "[d1] dummy ok 1") + assert.Contains(t, res.Details, "[d2] dummy ok 2") // 2. Inline checks with warning and critical (default thresholds) res = snc.RunCheck("check_multi", []string{ - "check=check_dummy 0 'dummy ok'", - "check=check_dummy 1 'dummy warn'", + "command[d1]=check_dummy 0 'dummy ok'", + "command[d2]=check_dummy 1 'dummy warn'", }) assert.Equalf(t, CheckExitWarning, res.State, "state WARNING") assert.Contains(t, res.Output, "2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown") res = snc.RunCheck("check_multi", []string{ - "check=check_dummy 0 'dummy ok'", - "check=check_dummy 2 'dummy crit'", + "command[d1]=check_dummy 0 'dummy ok'", + "command[d2]=check_dummy 2 'dummy crit'", }) assert.Equalf(t, CheckExitCritical, res.State, "state CRITICAL") assert.Contains(t, res.Output, "2 plugins checked: 1 ok, 0 warning, 1 critical, 0 unknown") // 3. Custom conditions: warn=none crit=ok_count ne 2 res = snc.RunCheck("check_multi", []string{ - "check=check_dummy 0 'dummy 1'", - "check=check_dummy 0 'dummy 2'", + "command[d1]=check_dummy 0 'dummy 1'", + "command[d2]=check_dummy 0 'dummy 2'", "warn=none", "crit=ok_count ne 2", }) assert.Equalf(t, CheckExitOK, res.State, "state OK when ok_count == 2") res = snc.RunCheck("check_multi", []string{ - "check=check_dummy 0 'dummy 1'", - "check=check_dummy 1 'dummy 2'", + "command[d1]=check_dummy 0 'dummy 1'", + "command[d2]=check_dummy 1 'dummy 2'", "warn=none", "crit=ok_count ne 2", }) assert.Equalf(t, CheckExitCritical, res.State, "state CRITICAL when ok_count != 2") - // 4. Custom conditions: warn=problem_count gt 0 + // 4. Custom condition on entry attribute: critical=name eq 'alias2' and state=2 res = snc.RunCheck("check_multi", []string{ - "check=check_dummy 0 'dummy 1'", - "check=check_dummy 1 'dummy 2'", - "warn=problem_count gt 0", - "crit=none", + "command[alias1]=check_dummy 2 'crit 1'", + "command[alias2]=check_dummy 0 'ok 2'", + "warn=none", + "crit=name eq 'alias2' and state=2", }) - assert.Equalf(t, CheckExitWarning, res.State, "state WARNING when problem_count > 0") + assert.Equalf(t, CheckExitOK, res.State, "state OK when alias2 is not in state 2") - // 5. Unknown/inline checks restriction (cannot run arbitrary external commands inline) res = snc.RunCheck("check_multi", []string{ - "check=/bin/nonexistent_or_external_script -H 123", + "command[alias1]=check_dummy 0 'ok 1'", + "command[alias2]=check_dummy 2 'crit 2'", + "warn=none", + "crit=name eq 'alias2' and state=2", }) - assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN for unregistered inline command") - assert.Contains(t, res.Output, "unknown check command") + assert.Equalf(t, CheckExitCritical, res.State, "state CRITICAL when alias2 is in state 2") - // 6. Inline check with check_process or check_cpu + // 5. Mandatory tag validation: missing tag & duplicate tag res = snc.RunCheck("check_multi", []string{ - "check=check_cpu warn=load=101 crit=load=102", - "warn=none", - "crit=ok_count ne 1", + "command=check_dummy 0 'ok'", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when command has no tag") + assert.Contains(t, res.Output, "command argument requires a unique tag") + + res = snc.RunCheck("check_multi", []string{ + "command[dup]=check_dummy 0 'ok 1'", + "command[dup]=check_dummy 0 'ok 2'", }) - assert.Equalf(t, CheckExitOK, res.State, "state OK for check_cpu inline") - assert.Contains(t, res.Details, "check_cpu") + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when command tag is duplicated") + assert.Contains(t, res.Output, "duplicate command tag: dup") + + // 6. Unknown/inline checks restriction (cannot run arbitrary external commands inline) + res = snc.RunCheck("check_multi", []string{ + "command[ext]=/bin/nonexistent_or_external_script -H 123", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN for unregistered inline command") + assert.Contains(t, res.Output, "unknown check command") // 7. Filter argument is disabled/rejected res = snc.RunCheck("check_multi", []string{ - "check=check_dummy 0 'ok'", + "command[d1]=check_dummy 0 'ok'", "filter=state=1", }) assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when filter argument is used") assert.Contains(t, res.Output, "filter is disabled for this check") - // 8. Unknown threshold (default and custom) + // 8. Severity hierarchy: UNKNOWN > CRITICAL > WARNING > OK res = snc.RunCheck("check_multi", []string{ - "check=check_dummy 0 'ok'", - "check=check_dummy 3 'unknown check'", + "command[d1]=check_dummy 0 'ok'", + "command[d2]=check_dummy 3 'unknown check'", }) assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when child check is unknown by default") res = snc.RunCheck("check_multi", []string{ - "check=check_dummy 0 'ok'", - "check=check_dummy 3 'unknown check'", - "unknown=unknown_count gt 0", - "warning=warning_count gt 0", - "critical=critical_count gt 0", - }) - assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when custom unknown condition matches") - - res = snc.RunCheck("check_multi", []string{ - "check=check_dummy 1 'warn check'", - "check=check_dummy 3 'unknown check'", - "unknown=unknown_count gt 0", - "warning=warning_count gt 0", - "critical=critical_count gt 0", + "command[d1]=check_dummy 1 'warn check'", + "command[d2]=check_dummy 3 'unknown check'", }) assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN takes precedence over WARNING") res = snc.RunCheck("check_multi", []string{ - "check=check_dummy 2 'crit check'", - "check=check_dummy 3 'unknown check'", - "unknown=unknown_count gt 0", - "warning=warning_count gt 0", - "critical=critical_count gt 0", + "command[d1]=check_dummy 2 'crit check'", + "command[d2]=check_dummy 3 'unknown check'", }) - assert.Equalf(t, CheckExitCritical, res.State, "state CRITICAL takes precedence over UNKNOWN") + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN takes precedence over CRITICAL") } func TestCheckMultiLimits(t *testing.T) { @@ -142,16 +140,16 @@ max checks = 2 // Under limit: 2 checks res := snc.RunCheck("check_multi", []string{ - "check=check_dummy 0 'ok 1'", - "check=check_dummy 0 'ok 2'", + "command[d1]=check_dummy 0 'ok 1'", + "command[d2]=check_dummy 0 'ok 2'", }) assert.Equalf(t, CheckExitOK, res.State, "state OK for 2 checks") // Exceeds limit: 3 checks res = snc.RunCheck("check_multi", []string{ - "check=check_dummy 0 'ok 1'", - "check=check_dummy 0 'ok 2'", - "check=check_dummy 0 'ok 3'", + "command[d1]=check_dummy 0 'ok 1'", + "command[d2]=check_dummy 0 'ok 2'", + "command[d3]=check_dummy 0 'ok 3'", }) assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when exceeding max checks") assert.Contains(t, res.Output, "exceeds max checks limit") @@ -166,7 +164,7 @@ CheckMulti = disabled defer StopTestAgent(t, snc) res := snc.RunCheck("check_multi", []string{ - "check=check_dummy 0 'ok 1'", + "command[d1]=check_dummy 0 'ok 1'", }) assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when module is disabled") assert.Contains(t, res.Output, "module CheckMulti is not enabled") @@ -217,17 +215,22 @@ exit 1 CheckMulti = enabled [/settings/check/multi/mycheck] -check_dummy 0 ok1 -check_dummy 0 ok2 +command[c1] = check_dummy 0 ok1 +command[c2] = check_dummy 0 ok2 [/settings/check/multi/custom] -%s -H 123 -%s -W 123 +command[s1] = %s -H 123 +command[s2] = %s -W 123 -[/settings/check/multi/named] -first = check_dummy 0 ok_first -second = %s -H 456 -`, script1, script2, script1) +[/settings/check/multi/loop] +command[sub] = check_multi config=loop + +[/settings/check/multi/loopA] +command[b] = check_multi config=loopB + +[/settings/check/multi/loopB] +command[a] = check_multi config=loopA +`, script1, script2) snc := StartTestAgent(t, config) defer StopTestAgent(t, snc) @@ -252,13 +255,19 @@ second = %s -H 456 assert.Contains(t, res.Details, "SCRIPT 1 OK") assert.Contains(t, res.Details, "SCRIPT 2 WARNING") - // Test config=named (named check tags in config) + // Test direct loop detection: check_multi config=loop + res = snc.RunCheck("check_multi", []string{ + "config=loop", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN for loop config") + assert.Contains(t, res.Output, "loop detected") + + // Test indirect loop detection: loopA -> loopB -> loopA res = snc.RunCheck("check_multi", []string{ - "config=named", + "config=loopA", }) - assert.Equalf(t, CheckExitOK, res.State, "state OK for named config") - assert.Contains(t, res.Details, "first") - assert.Contains(t, res.Details, "second") + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN for indirect loop") + assert.Contains(t, res.Output, "loop detected") // Test non-existing config res = snc.RunCheck("check_multi", []string{ diff --git a/pkg/snclient/checkdata.go b/pkg/snclient/checkdata.go index 9e08eea3..5d384362 100644 --- a/pkg/snclient/checkdata.go +++ b/pkg/snclient/checkdata.go @@ -42,6 +42,13 @@ const ( type CommaStringList []string +type TaggedCommand struct { + Tag string + Command string +} + +type TaggedCommandList []TaggedCommand + type CheckArgument struct { value any // reference to storage pointer description string // used in help @@ -478,10 +485,13 @@ func (cd *CheckData) setStateFromMaps(macros map[string]string) { } switch { - case macros["crit_count"] != "0": + case macros["unknown_count"] != "0" && macros["unknown_count"] != "": + cd.result.EscalateStatus(3) + macros["_state"] = "3" + case macros["crit_count"] != "0" && macros["crit_count"] != "": cd.result.EscalateStatus(2) macros["_state"] = "2" - case macros["warn_count"] != "0": + case macros["warn_count"] != "0" && macros["warn_count"] != "": cd.result.EscalateStatus(1) macros["_state"] = "1" } @@ -505,13 +515,6 @@ func (cd *CheckData) Check(data map[string]string, warnCond, critCond, unknownCo } } - for i := range unknownCond { - if res, ok := unknownCond[i].Match(data); res && ok { - log.Debugf("This given data matched the UNKNOWN condition: '%s' ", unknownCond[i].DetailedString()) - data["_state"] = fmt.Sprintf("%d", CheckExitUnknown) - } - } - for i := range critCond { if res, ok := critCond[i].Match(data); res && ok { log.Debugf("This given data matched the CRITICAL condition: '%s' ", critCond[i].DetailedString()) @@ -519,6 +522,13 @@ func (cd *CheckData) Check(data map[string]string, warnCond, critCond, unknownCo } } + for i := range unknownCond { + if res, ok := unknownCond[i].Match(data); res && ok { + log.Debugf("This given data matched the UNKNOWN condition: '%s' ", unknownCond[i].DetailedString()) + data["_state"] = fmt.Sprintf("%d", CheckExitUnknown) + } + } + for i := range okCond { if res, ok := okCond[i].Match(data); res && ok { log.Debugf("This given data matched the OK condition: '%s' ", okCond[i].DetailedString()) @@ -1028,9 +1038,13 @@ func (cd *CheckData) fetchNextArg(args, split []string, keyword string, idx, num if len(split) == 2 { return split[1], idx, nil } - arg, ok := cd.args[keyword] + lookupKey := keyword + if before, _, found := strings.Cut(keyword, "["); found && strings.HasSuffix(keyword, "]") { + lookupKey = before + } + arg, ok := cd.args[lookupKey] if !ok { - arg, ok = cd.extraArgs[keyword] + arg, ok = cd.extraArgs[lookupKey] if !ok { return "", idx, nil } @@ -1050,29 +1064,72 @@ func (cd *CheckData) fetchNextArg(args, split []string, keyword string, idx, num return args[idx], idx, nil } +// parseTaggedCommand handles parsing a TaggedCommandList argument (command[tag]=...). +func (cd *CheckData) parseTaggedCommand(argRef *TaggedCommandList, tag, argValue string) error { + tag = strings.TrimSpace(tag) + if tag == "" { + return fmt.Errorf("command argument requires a unique tag, e.g. command[tag]=") + } + + if strings.ContainsAny(tag, DefaultNastyCharacters+"=") { + return fmt.Errorf("command tag contains invalid characters: %s", tag) + } + + for _, existing := range *argRef { + if existing.Tag == tag { + return fmt.Errorf("duplicate command tag: %s", tag) + } + } + + *argRef = append(*argRef, TaggedCommand{ + Tag: tag, + Command: strings.TrimSpace(argValue), + }) + + return nil +} + // parseAnyArg parses args into the args map with custom arguments func (cd *CheckData) parseAnyArg(argExpr, keyword, argValue string) (bool, error) { - arg, ok := cd.args[keyword] + lookupKey := keyword + tag := "" + if before, rest, found := strings.Cut(keyword, "["); found && strings.HasSuffix(keyword, "]") { + lookupKey = before + tag = rest[:len(rest)-1] + } + + arg, ok := cd.args[lookupKey] if !ok { - arg, ok = cd.extraArgs[keyword] + arg, ok = cd.extraArgs[lookupKey] if !ok { return false, nil } } + if err := cd.parseArgValue(argExpr, keyword, argValue, tag, &arg); err != nil { + return true, err + } + + cd.hasArgsSupplied[keyword] = true + + return true, nil +} + +// parseArgValue dispatches an argument value into the correct typed storage reference. +func (cd *CheckData) parseArgValue(argExpr, keyword, argValue, tag string, arg *CheckArgument) error { //nolint:cyclop // many type cases are required here switch argRef := arg.value.(type) { + case *TaggedCommandList: + return cd.parseTaggedCommand(argRef, tag, argValue) case *[]string: if _, ok := cd.hasArgsSupplied[keyword]; !ok { // first time this arg occurs, empty default lists - empty := make([]string, 0) - *argRef = empty + *argRef = make([]string, 0) } *argRef = append(*argRef, argValue) case *CommaStringList: if _, ok := cd.hasArgsSupplied[keyword]; !ok { // first time this arg occurs, empty default lists - empty := make([]string, 0) - *argRef = empty + *argRef = make([]string, 0) } *argRef = append(*argRef, strings.Split(argValue, ",")...) case *string: @@ -1080,29 +1137,28 @@ func (cd *CheckData) parseAnyArg(argExpr, keyword, argValue string) (bool, error case *float64: f, err := strconv.ParseFloat(argValue, 64) if err != nil { - return true, fmt.Errorf("parseFloat %s: %s", argExpr, err.Error()) + return fmt.Errorf("parseFloat %s: %s", argExpr, err.Error()) } *argRef = f case *int64: i, err := strconv.ParseInt(argValue, 10, 64) if err != nil { - return true, fmt.Errorf("parseInt %s: %s", argExpr, err.Error()) + return fmt.Errorf("parseInt %s: %s", argExpr, err.Error()) } *argRef = i case *int: i, err := strconv.ParseInt(argValue, 10, 32) if err != nil { - return true, fmt.Errorf("parseInt %s: %s", argExpr, err.Error()) + return fmt.Errorf("parseInt %s: %s", argExpr, err.Error()) } *argRef = int(i) case *bool: if argValue == "" { - b := true - *argRef = b + *argRef = true } else { b, err := convert.BoolE(argValue) if err != nil { - return true, fmt.Errorf("parseBool %s: %s", argValue, err.Error()) + return fmt.Errorf("parseBool %s: %s", argValue, err.Error()) } *argRef = b } @@ -1110,9 +1166,7 @@ func (cd *CheckData) parseAnyArg(argExpr, keyword, argValue string) (bool, error log.Errorf("unsupported args type: %T in %s", argRef, argExpr) } - cd.hasArgsSupplied[keyword] = true - - return true, nil + return nil } // removeQuotes remove single/double quotes around string diff --git a/pkg/snclient/config.go b/pkg/snclient/config.go index 577bdf61..a14ff1f0 100644 --- a/pkg/snclient/config.go +++ b/pkg/snclient/config.go @@ -323,44 +323,17 @@ func (config *Config) ParseINI(configData, iniPath string, snc *Agent) error { continue } - isMultiSection := strings.HasPrefix(currentSection.name, "/settings/check/multi/") - // parse key and value val := strings.SplitN(line, "=", 2) - // bare line (no '='): only allowed in check/multi sections, treated as raw command if len(val) < 2 { - if !isMultiSection { - parseErrors = append(parseErrors, fmt.Errorf("parse error in %s:%d: found key without '='", iniPath, lineNr)) - - continue - } - if err := currentSection.SetRaw(line, ""); err != nil { - parseErrors = append(parseErrors, fmt.Errorf("config error in %s:%d: %s", iniPath, lineNr, err.Error())) - } - if len(currentComments) > 0 { - currentSection.comments[line] = currentComments - currentComments = make([]string, 0) - } + parseErrors = append(parseErrors, fmt.Errorf("parse error in %s:%d: found key without '='", iniPath, lineNr)) continue } val[0] = strings.TrimSpace(val[0]) val[1] = strings.TrimSpace(val[1]) - // key contains space (e.g. 'check_process process=123'): also a raw command line in check/multi sections - if isMultiSection && strings.Contains(val[0], " ") { - if err := currentSection.SetRaw(line, ""); err != nil { - parseErrors = append(parseErrors, fmt.Errorf("config error in %s:%d: %s", iniPath, lineNr, err.Error())) - } - if len(currentComments) > 0 { - currentSection.comments[line] = currentComments - currentComments = make([]string, 0) - } - - continue - } - // silently skip UNKNOWN values which were placeholder in nsclient if val[1] == "UNKNOWN" { continue From e13e652993f1bd6b8182f67af66ef1b71a035d8f Mon Sep 17 00:00:00 2001 From: lorenzg Date: Thu, 20 Aug 2026 10:20:16 +0200 Subject: [PATCH 13/19] remove duplication check here because config parser already deduplicates --- pkg/snclient/check_multi.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go index c321b3fb..d104d2b4 100644 --- a/pkg/snclient/check_multi.go +++ b/pkg/snclient/check_multi.go @@ -239,7 +239,6 @@ func (l *CheckMulti) buildConfigChecks(snc *Agent) ([]multiChildCheck, *CheckRes } childChecks := make([]multiChildCheck, 0, len(sec.keys)) - seenTags := make(map[string]bool) for _, key := range sec.keys { rawVal := sec.data[key] @@ -268,13 +267,6 @@ func (l *CheckMulti) buildConfigChecks(snc *Agent) ([]multiChildCheck, *CheckRes Output: fmt.Sprintf("empty command for tag %s in config section", tag), } } - if seenTags[tag] { - return nil, &CheckResult{ - State: CheckExitUnknown, - Output: fmt.Sprintf("duplicate command tag: %s", tag), - } - } - seenTags[tag] = true childChecks = append(childChecks, multiChildCheck{ tag: tag, cmdStr: rawVal, From 4bba3bee5347cb0fba7ba8ce8c66a98b6c0bb9e7 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Thu, 20 Aug 2026 13:59:25 +0200 Subject: [PATCH 14/19] improve max checks handling --- pkg/snclient/check_multi.go | 42 +++++++++++++++++++++----------- pkg/snclient/check_multi_test.go | 19 +++++++++++++-- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go index d104d2b4..1a2b7384 100644 --- a/pkg/snclient/check_multi.go +++ b/pkg/snclient/check_multi.go @@ -14,10 +14,16 @@ func init() { } type ( - checkMultiConfigKey struct{} - checkMultiDepthKey struct{} + checkMultiConfigKey struct{} + checkMultiDepthKey struct{} + checkMultiCounterKey struct{} ) +type checkMultiCounter struct { + count int64 + maxChecks int64 +} + type CheckMulti struct { commands TaggedCommandList config string @@ -141,6 +147,17 @@ func (l *CheckMulti) Check(ctx context.Context, snc *Agent, check *CheckData, _ } ctx = context.WithValue(ctx, checkMultiDepthKey{}, depth+1) + maxChecks, ok, err := snc.config.Section("/settings/check/multi").GetInt("max checks") + if err != nil || !ok || maxChecks <= 0 { + maxChecks = 20 + } + + counter, ok := ctx.Value(checkMultiCounterKey{}).(*checkMultiCounter) + if !ok { + counter = &checkMultiCounter{maxChecks: maxChecks} + ctx = context.WithValue(ctx, checkMultiCounterKey{}, counter) + } + activeConfigs, _ := ctx.Value(checkMultiConfigKey{}).(map[string]bool) if activeConfigs == nil { activeConfigs = make(map[string]bool) @@ -159,11 +176,6 @@ func (l *CheckMulti) Check(ctx context.Context, snc *Agent, check *CheckData, _ ctx = context.WithValue(ctx, checkMultiConfigKey{}, newActive) } - maxChecks, ok, err := snc.config.Section("/settings/check/multi").GetInt("max checks") - if err != nil || !ok || maxChecks <= 0 { - maxChecks = 20 - } - childChecks, res := l.buildChildChecks(snc) if res != nil { return res, nil @@ -176,13 +188,6 @@ func (l *CheckMulti) Check(ctx context.Context, snc *Agent, check *CheckData, _ }, nil } - if int64(len(childChecks)) > maxChecks { - return &CheckResult{ - State: CheckExitUnknown, - Output: fmt.Sprintf("number of checks (%d) exceeds max checks limit (%d)", len(childChecks), maxChecks), - }, nil - } - return l.executeChildChecks(ctx, snc, check, childChecks) } @@ -280,6 +285,7 @@ func (l *CheckMulti) buildConfigChecks(snc *Agent) ([]multiChildCheck, *CheckRes // executeChildChecks runs all child checks and aggregates results. func (l *CheckMulti) executeChildChecks(ctx context.Context, snc *Agent, check *CheckData, childChecks []multiChildCheck) (*CheckResult, error) { var count, okCount, warnCount, critCount, unknownCount int64 + counter, _ := ctx.Value(checkMultiCounterKey{}).(*checkMultiCounter) detailsList := make([]string, 0, len(childChecks)) allMetrics := make([]*CheckMetric, 0) @@ -288,6 +294,14 @@ func (l *CheckMulti) executeChildChecks(ctx context.Context, snc *Agent, check * check.HasThreshold("output") || check.HasThreshold("shortoutput") || check.HasThreshold("status") || check.HasThreshold("state") for _, chk := range childChecks { + counter.count++ + if counter.count > counter.maxChecks { + return &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("number of checks (%d) exceeds max checks limit (%d)", counter.count, counter.maxChecks), + }, nil + } + res, fatal := l.runChildCheck(ctx, snc, check, chk) if fatal { return res, nil diff --git a/pkg/snclient/check_multi_test.go b/pkg/snclient/check_multi_test.go index 45c8d84e..3756dd6c 100644 --- a/pkg/snclient/check_multi_test.go +++ b/pkg/snclient/check_multi_test.go @@ -133,7 +133,11 @@ func TestCheckMultiLimits(t *testing.T) { CheckMulti = enabled [/settings/check/multi] -max checks = 2 +max checks = 4 + +[/settings/check/multi/nested] +command[d1] = check_dummy 0 'nested 1' +command[d2] = check_dummy 0 'nested 2' ` snc := StartTestAgent(t, config) defer StopTestAgent(t, snc) @@ -145,14 +149,25 @@ max checks = 2 }) assert.Equalf(t, CheckExitOK, res.State, "state OK for 2 checks") - // Exceeds limit: 3 checks + // Exceeds limit: 5 checks res = snc.RunCheck("check_multi", []string{ "command[d1]=check_dummy 0 'ok 1'", "command[d2]=check_dummy 0 'ok 2'", "command[d3]=check_dummy 0 'ok 3'", + "command[d4]=check_dummy 0 'ok 4'", + "command[d5]=check_dummy 0 'ok 5'", }) assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when exceeding max checks") assert.Contains(t, res.Output, "exceeds max checks limit") + + // Nested checks share the same cumulative execution count. + res = snc.RunCheck("check_multi", []string{ + "command[d1]=check_dummy 0 'outer 1'", + "command[d2]=check_dummy 0 'outer 2'", + "command[nested]=check_multi config=nested", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN when nested checks exceed max checks") + assert.Contains(t, res.Details, "number of checks (5) exceeds max checks limit (4)") } func TestCheckMultiDisabled(t *testing.T) { From 72a5664126829cacc1e284636c49ddc3bee2ba43 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Thu, 20 Aug 2026 14:26:10 +0200 Subject: [PATCH 15/19] fix tests --- docs/checks/commands/check_multi.md | 8 +++---- pkg/snclient/check_multi.go | 35 +++++++++++++++++------------ 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/docs/checks/commands/check_multi.md b/docs/checks/commands/check_multi.md index ce3213d0..06678780 100644 --- a/docs/checks/commands/check_multi.md +++ b/docs/checks/commands/check_multi.md @@ -13,11 +13,11 @@ Runs multiple checks and aggregates their status, output and performance data. When using the inline mode, you can only use available commands (run 'check_index' to get a full list). You can also define custom check sections in the config file, for example: - [/settings/check/multi/mycheck] - command[alias1] = check_process process=123 - command[alias2] = check_process process=345 + [/settings/check/multi/mycheck] + command[alias1] = check_process process=123 + command[alias2] = check_process process=345 - This can be executed with 'check_multi "config=mycheck"'. + This can be executed with 'check_multi "config=mycheck"'. It's also possible to use custom scripts in the config section, for example: [/settings/check/multi/myscript] diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go index 1a2b7384..7e55d8f3 100644 --- a/pkg/snclient/check_multi.go +++ b/pkg/snclient/check_multi.go @@ -63,11 +63,11 @@ func (l *CheckMulti) Build() *CheckData { When using the inline mode, you can only use available commands (run 'check_index' to get a full list). You can also define custom check sections in the config file, for example: - [/settings/check/multi/mycheck] - command[alias1] = check_process process=123 - command[alias2] = check_process process=345 + [/settings/check/multi/mycheck] + command[alias1] = check_process process=123 + command[alias2] = check_process process=345 - This can be executed with 'check_multi "config=mycheck"'. + This can be executed with 'check_multi "config=mycheck"'. It's also possible to use custom scripts in the config section, for example: [/settings/check/multi/myscript] @@ -152,9 +152,8 @@ func (l *CheckMulti) Check(ctx context.Context, snc *Agent, check *CheckData, _ maxChecks = 20 } - counter, ok := ctx.Value(checkMultiCounterKey{}).(*checkMultiCounter) - if !ok { - counter = &checkMultiCounter{maxChecks: maxChecks} + if _, ok := ctx.Value(checkMultiCounterKey{}).(*checkMultiCounter); !ok { + counter := &checkMultiCounter{maxChecks: maxChecks} ctx = context.WithValue(ctx, checkMultiCounterKey{}, counter) } @@ -285,7 +284,6 @@ func (l *CheckMulti) buildConfigChecks(snc *Agent) ([]multiChildCheck, *CheckRes // executeChildChecks runs all child checks and aggregates results. func (l *CheckMulti) executeChildChecks(ctx context.Context, snc *Agent, check *CheckData, childChecks []multiChildCheck) (*CheckResult, error) { var count, okCount, warnCount, critCount, unknownCount int64 - counter, _ := ctx.Value(checkMultiCounterKey{}).(*checkMultiCounter) detailsList := make([]string, 0, len(childChecks)) allMetrics := make([]*CheckMetric, 0) @@ -294,12 +292,8 @@ func (l *CheckMulti) executeChildChecks(ctx context.Context, snc *Agent, check * check.HasThreshold("output") || check.HasThreshold("shortoutput") || check.HasThreshold("status") || check.HasThreshold("state") for _, chk := range childChecks { - counter.count++ - if counter.count > counter.maxChecks { - return &CheckResult{ - State: CheckExitUnknown, - Output: fmt.Sprintf("number of checks (%d) exceeds max checks limit (%d)", counter.count, counter.maxChecks), - }, nil + if res := l.incrementCheckMultiCounter(ctx); res != nil { + return res, nil } res, fatal := l.runChildCheck(ctx, snc, check, chk) @@ -369,6 +363,19 @@ func (l *CheckMulti) executeChildChecks(ctx context.Context, snc *Agent, check * return check.Finalize() } +func (l *CheckMulti) incrementCheckMultiCounter(ctx context.Context) *CheckResult { + counter, _ := ctx.Value(checkMultiCounterKey{}).(*checkMultiCounter) + counter.count++ + if counter.count > counter.maxChecks { + return &CheckResult{ + State: CheckExitUnknown, + Output: fmt.Sprintf("number of checks (%d) exceeds max checks limit (%d)", counter.count, counter.maxChecks), + } + } + + return nil +} + // runChildCheck executes a single child check and returns its result. // The second return value is true when the error is fatal and the caller should stop processing. func (l *CheckMulti) runChildCheck(ctx context.Context, snc *Agent, check *CheckData, chk multiChildCheck) (*CheckResult, bool) { From 83ca431d5f17e9f6e301f33253bb096ce3528dda Mon Sep 17 00:00:00 2001 From: lorenzg Date: Thu, 20 Aug 2026 16:11:02 +0200 Subject: [PATCH 16/19] fix conditions --- docs/checks/commands/check_multi.md | 2 +- pkg/snclient/check_multi.go | 42 +++++++++++++------- pkg/snclient/check_multi_test.go | 61 +++++++++++++++++++++++++++++ pkg/snclient/checkdata.go | 49 +++++++++++++++++------ pkg/snclient/checkmetric.go | 25 ++++++------ pkg/snclient/checkmetric_test.go | 19 +++++++++ pkg/snclient/config.go | 1 + 7 files changed, 161 insertions(+), 38 deletions(-) diff --git a/docs/checks/commands/check_multi.md b/docs/checks/commands/check_multi.md index 06678780..aed5a914 100644 --- a/docs/checks/commands/check_multi.md +++ b/docs/checks/commands/check_multi.md @@ -88,7 +88,7 @@ Naemon Config | empty-state | 3 (UNKNOWN) | | empty-syntax | %(status) - no checks executed | | top-syntax | %(status) - %(count) plugins checked: %(ok_count) ok, %(warning_count) warning, %(critical_count) critical, %(unknown_count) unknown - %(problem_list) | -| ok-syntax | %(status) - %(count) plugins checked, %(ok_count) ok | +| ok-syntax | {{ if problem_count gt 0 }}%(status) - %(count) plugins checked: %(ok_count) ok, %(warning_count) warning, %(critical_count) critical, %(unknown_count) unknown - %(problem_list){{ ELSE }}%(status) - %(count) plugins checked, %(ok_count) ok{{ END }} | | detail-syntax | %(name): %(output) | ## Check Specific Arguments diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go index 7e55d8f3..a2882cc4 100644 --- a/pkg/snclient/check_multi.go +++ b/pkg/snclient/check_multi.go @@ -6,6 +6,7 @@ import ( "maps" "strings" + "github.com/consol-monitoring/snclient/pkg/convert" "github.com/consol-monitoring/snclient/pkg/utils" ) @@ -94,11 +95,14 @@ func (l *CheckMulti) Build() *CheckData { defaultWarning: "warning_count > 0", defaultCritical: "critical_count > 0", defaultUnknown: "unknown_count > 0", - okSyntax: "%(status) - %(count) plugins checked, %(ok_count) ok", - topSyntax: "%(status) - %(count) plugins checked: %(ok_count) ok, %(warning_count) warning, %(critical_count) critical, %(unknown_count) unknown - %(problem_list)", - detailSyntax: "%(name): %(output)", - emptySyntax: "%(status) - no checks executed", - emptyState: CheckExitUnknown, + okSyntax: "{{ if problem_count gt 0 }}%(status) - %(count) plugins checked: " + + "%(ok_count) ok, %(warning_count) warning, %(critical_count) critical, " + + "%(unknown_count) unknown - %(problem_list){{ ELSE }}%(status) - " + + "%(count) plugins checked, %(ok_count) ok{{ END }}", + topSyntax: "%(status) - %(count) plugins checked: %(ok_count) ok, %(warning_count) warning, %(critical_count) critical, %(unknown_count) unknown - %(problem_list)", + detailSyntax: "%(name): %(output)", + emptySyntax: "%(status) - no checks executed", + emptyState: CheckExitUnknown, exampleDefault: ` check_multi "command[check_process]=check_process 'process=firefox'" "command[check_memory]=check_memory 'type=physical' 'crit=used_pct gt 80%'" OK - 2 plugins checked, 2 ok |'check_process::count'=1;;;0 ... 'check_memory::physical %'=78.7%;;;0;100 @@ -302,9 +306,10 @@ func (l *CheckMulti) executeChildChecks(ctx context.Context, snc *Agent, check * } tag := chk.tag + childOutput := res.BuildOutputString() - firstLine := strings.TrimSpace(strings.Split(res.Output, "\n")[0]) - detailsList = append(detailsList, fmt.Sprintf("[%s] %s", tag, res.Output)) + firstLine := strings.TrimSpace(strings.Split(childOutput, "\n")[0]) + detailsList = append(detailsList, fmt.Sprintf("[%s] %s", tag, childOutput)) entryState := fmt.Sprintf("%d", res.State) entry := map[string]string{ @@ -314,14 +319,16 @@ func (l *CheckMulti) executeChildChecks(ctx context.Context, snc *Agent, check * "state": entryState, "status": res.StateString(), "shortoutput": firstLine, - "output": res.Output, + "output": childOutput, "_state": entryState, "_skip": "1", "_count": "1", } if hasEntryThresholds { - check.Check(entry, check.warnThreshold, check.critThreshold, check.unknownThreshold, check.okThreshold) + thresholdEntry := maps.Clone(entry) + check.Check(thresholdEntry, check.warnThreshold, check.critThreshold, check.unknownThreshold, check.okThreshold) + check.result.EscalateStatus(convert.Int64(thresholdEntry["_state"])) } count++ @@ -338,11 +345,7 @@ func (l *CheckMulti) executeChildChecks(ctx context.Context, snc *Agent, check * check.listData = append(check.listData, entry) - for _, m := range res.Metrics { - metricCopy := *m - metricCopy.Name = fmt.Sprintf("%s::%s", tag, m.Name) - allMetrics = append(allMetrics, &metricCopy) - } + allMetrics = appendChildMetrics(allMetrics, res, tag) } problemCount := warnCount + critCount + unknownCount @@ -363,6 +366,17 @@ func (l *CheckMulti) executeChildChecks(ctx context.Context, snc *Agent, check * return check.Finalize() } +func appendChildMetrics(allMetrics []*CheckMetric, res *CheckResult, tag string) []*CheckMetric { + for _, m := range res.Metrics { + metricCopy := *m + metricCopy.Name = fmt.Sprintf("%s::%s", tag, m.Name) + metricCopy.SkipStateCheck = true + allMetrics = append(allMetrics, &metricCopy) + } + + return allMetrics +} + func (l *CheckMulti) incrementCheckMultiCounter(ctx context.Context) *CheckResult { counter, _ := ctx.Value(checkMultiCounterKey{}).(*checkMultiCounter) counter.count++ diff --git a/pkg/snclient/check_multi_test.go b/pkg/snclient/check_multi_test.go index 3756dd6c..6f48a4c3 100644 --- a/pkg/snclient/check_multi_test.go +++ b/pkg/snclient/check_multi_test.go @@ -37,6 +37,13 @@ CheckMulti = enabled assert.Equalf(t, CheckExitWarning, res.State, "state WARNING") assert.Contains(t, res.Output, "2 plugins checked: 1 ok, 1 warning, 0 critical, 0 unknown") + res = snc.RunCheck("check_multi", []string{ + "command[test1]=check_dummy 1 WARN", + "warn=none", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK when warning threshold is none") + assert.Contains(t, res.Output, "OK - 1 plugins checked: 0 ok, 1 warning, 0 critical, 0 unknown - test1: WARN") + res = snc.RunCheck("check_multi", []string{ "command[d1]=check_dummy 0 'dummy ok'", "command[d2]=check_dummy 2 'dummy crit'", @@ -127,6 +134,49 @@ CheckMulti = enabled assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN takes precedence over CRITICAL") } +func TestCheckMultiDefaultEnabled(t *testing.T) { + snc := StartTestAgent(t, "") + defer StopTestAgent(t, snc) + + res := snc.RunCheck("check_multi", []string{ + "command[d1]=check_dummy 0 'default enabled'", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK when CheckMulti is enabled by default") +} + +func TestCheckMultiPriorityThreshold(t *testing.T) { + config := ` +[/modules] +CheckMulti = enabled +` + snc := StartTestAgent(t, config) + defer StopTestAgent(t, snc) + + args := []string{ + "command[prio]=check_dummy 2 'priority critical'", + "command[dummy2]=check_dummy 0 'dummy 2'", + "command[dummy3]=check_dummy 0 'dummy 3'", + "warn=none", + "unknown=none", + "crit=name eq 'prio' and state ne '0'", + } + res := snc.RunCheck("check_multi", args) + assert.Equalf(t, CheckExitCritical, res.State, "state CRITICAL when prio is not OK") + assert.Contains(t, res.Details, "[prio] priority critical") + + res = snc.RunCheck("check_multi", []string{ + "command[prio]=check_dummy 0 'priority ok'", + "command[dummy2]=check_dummy 3 'dummy 2 unknown'", + "command[dummy3]=check_dummy 2 'dummy 3 critical'", + "warn=none", + "unknown=none", + "crit=name eq 'prio' and state ne '0'", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK when only non-prio checks are problems") + assert.Contains(t, res.Details, "[dummy2] dummy 2 unknown") + assert.Contains(t, res.Details, "[dummy3] dummy 3 critical") +} + func TestCheckMultiLimits(t *testing.T) { config := ` [/modules] @@ -245,6 +295,9 @@ command[b] = check_multi config=loopB [/settings/check/multi/loopB] command[a] = check_multi config=loopA + +[/settings/check/multi/inner] +command[leaf] = check_dummy 0 'nested detail' `, script1, script2) snc := StartTestAgent(t, config) @@ -270,6 +323,14 @@ command[a] = check_multi config=loopA assert.Contains(t, res.Details, "SCRIPT 1 OK") assert.Contains(t, res.Details, "SCRIPT 2 WARNING") + // Nested detail output is included in the parent output attribute. + res = snc.RunCheck("check_multi", []string{ + "command[nested]=check_multi config=inner", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK for nested detail output") + assert.Contains(t, res.BuildOutputString(), "nested detail") + assert.Contains(t, res.Details, "nested detail") + // Test direct loop detection: check_multi config=loop res = snc.RunCheck("check_multi", []string{ "config=loop", diff --git a/pkg/snclient/checkdata.go b/pkg/snclient/checkdata.go index 5d384362..67fbf88c 100644 --- a/pkg/snclient/checkdata.go +++ b/pkg/snclient/checkdata.go @@ -134,7 +134,7 @@ type CheckData struct { emptyStateSet bool details map[string]string listData []map[string]string - listCombine string // join string for detail list + listCombine string // join string for detail output listCombineSet bool // has the listCombine been set by user showAll bool // flag if check called with show-all addCountMetrics bool @@ -484,16 +484,27 @@ func (cd *CheckData) setStateFromMaps(macros map[string]string) { cd.result.EscalateStatus(3) } - switch { - case macros["unknown_count"] != "0" && macros["unknown_count"] != "": - cd.result.EscalateStatus(3) - macros["_state"] = "3" - case macros["crit_count"] != "0" && macros["crit_count"] != "": - cd.result.EscalateStatus(2) - macros["_state"] = "2" - case macros["warn_count"] != "0" && macros["warn_count"] != "": - cd.result.EscalateStatus(1) - macros["_state"] = "1" + // Only escalate based on counts if the user hasn't explicitly set the threshold. + // This respects explicit thresholds like "crit=none" which disable escalation. + if !cd.hasArgsSupplied["unknown"] && !cd.hasArgsSupplied["unknown+"] { + if macros["unknown_count"] != "0" && macros["unknown_count"] != "" { + cd.result.EscalateStatus(3) + macros["_state"] = "3" + } + } + + if !cd.hasArgsSupplied["crit"] && !cd.hasArgsSupplied["critical"] && !cd.hasArgsSupplied["crit+"] && !cd.hasArgsSupplied["critical+"] { + if macros["crit_count"] != "0" && macros["crit_count"] != "" { + cd.result.EscalateStatus(2) + macros["_state"] = "2" + } + } + + if !cd.hasArgsSupplied["warn"] && !cd.hasArgsSupplied["warning"] && !cd.hasArgsSupplied["warn+"] && !cd.hasArgsSupplied["warning+"] { + if macros["warn_count"] != "0" && macros["warn_count"] != "" { + cd.result.EscalateStatus(1) + macros["_state"] = "1" + } } if state, ok := cd.details["_state"]; ok { @@ -503,6 +514,12 @@ func (cd *CheckData) setStateFromMaps(macros map[string]string) { cd.details["_state"] = fmt.Sprintf("%d", cd.result.State) } +func (cd *CheckData) markCheckMultiThresholdSupplied(keyword string) { + if cd.name == "check_multi" { + cd.hasArgsSupplied[keyword] = true + } +} + // Check tries warn/crit/unknown/ok conditions against given data and sets result state. // The data argument can be anything that has the correct keys that conditions use func (cd *CheckData) Check(data map[string]string, warnCond, critCond, unknownCond, okCond ConditionList) { @@ -541,6 +558,10 @@ func (cd *CheckData) Check(data map[string]string, warnCond, critCond, unknownCo func (cd *CheckData) CheckMetrics(okCond ConditionList) { // each metric is ran through conditions individually for _, metric := range cd.result.Metrics { + if metric.SkipStateCheck { + continue + } + state := CheckExitOK if metric.CheckForThresholds(&metric.Warning) { @@ -764,36 +785,42 @@ func (cd *CheckData) processArgs(pre *preParsedArgs) (argList []Argument, applyD return nil, false, err2 } cd.warnThreshold = warn + cd.markCheckMultiThresholdSupplied(keyword) case "warn", "warning": cond, err2 := NewCondition(argValue, &cd.attributes) if err2 != nil { return nil, false, err2 } cd.warnThreshold = append(cd.warnThreshold, cond) + cd.markCheckMultiThresholdSupplied(keyword) case "crit+", "critical+": crit, err2 := cd.appendDefaultThreshold(keyword, argValue, pre.defaultCritical, cd.critThreshold) if err2 != nil { return nil, false, err2 } cd.critThreshold = crit + cd.markCheckMultiThresholdSupplied(keyword) case "crit", "critical": cond, err2 := NewCondition(argValue, &cd.attributes) if err2 != nil { return nil, false, err2 } cd.critThreshold = append(cd.critThreshold, cond) + cd.markCheckMultiThresholdSupplied(keyword) case "unknown+": unknown, err2 := cd.appendDefaultThreshold(keyword, argValue, pre.defaultUnknown, cd.unknownThreshold) if err2 != nil { return nil, false, err2 } cd.unknownThreshold = unknown + cd.markCheckMultiThresholdSupplied(keyword) case "unknown": cond, err2 := NewCondition(argValue, &cd.attributes) if err2 != nil { return nil, false, err2 } cd.unknownThreshold = append(cd.unknownThreshold, cond) + cd.markCheckMultiThresholdSupplied(keyword) case "filter+": if cd.disableFilter { return nil, false, fmt.Errorf("%s is disabled for this check", keyword) diff --git a/pkg/snclient/checkmetric.go b/pkg/snclient/checkmetric.go index c67f4f20..6398d177 100644 --- a/pkg/snclient/checkmetric.go +++ b/pkg/snclient/checkmetric.go @@ -14,18 +14,19 @@ import ( // CheckMetric contains a single performance value. type CheckMetric struct { - Name string // Name as used in the perf data string - Unit string // Unit of the value - Value any // Current value - ThresholdName string // if set, this will be added to the data before checking a conditions - Warning ConditionList // threshold used for warnings - WarningStr *string // set warnings from string - Critical ConditionList // threshold used for critical - CriticalStr *string // set critical from string - Min *float64 - Max *float64 - PerfConfig *PerfConfig // apply perf tweaks - Entry map[string]string // entry that this metric is generated from + Name string // Name as used in the perf data string + Unit string // Unit of the value + Value any // Current value + ThresholdName string // if set, this will be added to the data before checking a conditions + Warning ConditionList // threshold used for warnings + WarningStr *string // set warnings from string + Critical ConditionList // threshold used for critical + CriticalStr *string // set critical from string + SkipStateCheck bool // do not use warning or critical conditions for state + Min *float64 + Max *float64 + PerfConfig *PerfConfig // apply perf tweaks + Entry map[string]string // entry that this metric is generated from } // generates a naemon like string, including the perfdata diff --git a/pkg/snclient/checkmetric_test.go b/pkg/snclient/checkmetric_test.go index 8f17c69c..a1823b9f 100644 --- a/pkg/snclient/checkmetric_test.go +++ b/pkg/snclient/checkmetric_test.go @@ -19,3 +19,22 @@ func TestCheckMetricsString(t *testing.T) { assert.Equalf(t, check.expect, res, "CheckMetric.String() ->> %s", res) } } + +func TestCheckMetricsSkipStateCheck(t *testing.T) { + metric := &CheckMetric{ + Name: "value", + Value: 1, + Warning: ConditionList{{ + keyword: "value", + operator: Greater, + value: float64(0), + }}, + SkipStateCheck: true, + } + check := &CheckData{result: &CheckResult{Metrics: []*CheckMetric{metric}}} + + check.CheckMetrics(nil) + + assert.Equal(t, CheckExitOK, check.result.State) + assert.Equal(t, "'value'=1;0", metric.String()) +} diff --git a/pkg/snclient/config.go b/pkg/snclient/config.go index a14ff1f0..0cc47baa 100644 --- a/pkg/snclient/config.go +++ b/pkg/snclient/config.go @@ -48,6 +48,7 @@ var DefaultConfig = map[string]ConfigData{ "CheckSystem": "enabled", "CheckSystemUnix": "enabled", "CheckAlias": "enabled", + "CheckMulti": "enabled", "CheckExternalScripts": "enabled", "CheckDisk": "enabled", "CheckDriveIO": "enabled", From 41c17b185b072b2d002f1d03b85693ff7b68e463 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Thu, 20 Aug 2026 16:12:01 +0200 Subject: [PATCH 17/19] fix typo in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d616c7fe..fa60b351 100644 --- a/Makefile +++ b/Makefile @@ -278,7 +278,7 @@ citest: tools vendor # $(MAKE) benchmark # - # Race rondition tests + # Race condition tests # $(MAKE) racetest # From 492b904cf95a652dce66f00fa1a0f9eb01530fe1 Mon Sep 17 00:00:00 2001 From: lorenzg Date: Fri, 21 Aug 2026 08:44:56 +0200 Subject: [PATCH 18/19] remove leftover --- pkg/snclient/config.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/snclient/config.go b/pkg/snclient/config.go index 0cc47baa..4d577d55 100644 --- a/pkg/snclient/config.go +++ b/pkg/snclient/config.go @@ -796,11 +796,7 @@ func (cs *ConfigSection) String() string { // none-multiline entries case 0, 1: if val == "" { - if strings.HasPrefix(cs.name, "/settings/check/multi/") { - data = append(data, key) - } else { - data = append(data, fmt.Sprintf("%s =", key)) - } + data = append(data, fmt.Sprintf("%s =", key)) } else { data = append(data, fmt.Sprintf("%s = %s", key, strings.Join(raw, ""))) } From 141e60454ed44ba1d59355562960c080df6bdefb Mon Sep 17 00:00:00 2001 From: lorenzg Date: Fri, 21 Aug 2026 10:19:28 +0200 Subject: [PATCH 19/19] fix cfg duplicate, tags, literal output --- pkg/snclient/check_multi.go | 9 +++++---- pkg/snclient/check_multi_test.go | 30 ++++++++++++++++++++++++++++++ pkg/snclient/checkdata.go | 7 +++++++ pkg/snclient/checkresult.go | 26 +++++++++++++++++++++----- 4 files changed, 63 insertions(+), 9 deletions(-) diff --git a/pkg/snclient/check_multi.go b/pkg/snclient/check_multi.go index a2882cc4..ca0b5128 100644 --- a/pkg/snclient/check_multi.go +++ b/pkg/snclient/check_multi.go @@ -237,9 +237,9 @@ func (l *CheckMulti) buildChildChecks(snc *Agent) ([]multiChildCheck, *CheckResu // buildConfigChecks loads checks from the named config section. func (l *CheckMulti) buildConfigChecks(snc *Agent) ([]multiChildCheck, *CheckResult) { secName := "/settings/check/multi/" + l.config - sec := snc.config.Section(secName) + sec, ok := snc.config.sections[secName] - if len(sec.keys) == 0 { + if !ok || len(sec.keys) == 0 { return nil, &CheckResult{ State: CheckExitUnknown, Output: fmt.Sprintf("no checks defined in config section %s", secName), @@ -256,7 +256,7 @@ func (l *CheckMulti) buildConfigChecks(snc *Agent) ([]multiChildCheck, *CheckRes Output: fmt.Sprintf("invalid check_multi config entry: %s (must be in format command[tag]=)", key), } } - tag := strings.TrimSuffix(strings.TrimPrefix(key, "command["), "]") + tag := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(key, "command["), "]")) if strings.ContainsAny(tag, DefaultNastyCharacters+"=") { return nil, &CheckResult{ State: CheckExitUnknown, @@ -309,7 +309,8 @@ func (l *CheckMulti) executeChildChecks(ctx context.Context, snc *Agent, check * childOutput := res.BuildOutputString() firstLine := strings.TrimSpace(strings.Split(childOutput, "\n")[0]) - detailsList = append(detailsList, fmt.Sprintf("[%s] %s", tag, childOutput)) + literalOutput := check.result.LiteralizeDetails(fmt.Sprintf("[%s] %s", tag, childOutput)) + detailsList = append(detailsList, literalOutput) entryState := fmt.Sprintf("%d", res.State) entry := map[string]string{ diff --git a/pkg/snclient/check_multi_test.go b/pkg/snclient/check_multi_test.go index 6f48a4c3..4f0e733b 100644 --- a/pkg/snclient/check_multi_test.go +++ b/pkg/snclient/check_multi_test.go @@ -144,6 +144,26 @@ func TestCheckMultiDefaultEnabled(t *testing.T) { assert.Equalf(t, CheckExitOK, res.State, "state OK when CheckMulti is enabled by default") } +func TestCheckMultiPreservesLiteralChildOutput(t *testing.T) { + snc := StartTestAgent(t, "") + defer StopTestAgent(t, snc) + + res := snc.RunCheck("check_multi", []string{ + "command[child]=check_dummy 0 '%(count) {{ IF condition }}literal{{ END }}'", + }) + assert.Equalf(t, CheckExitOK, res.State, "state OK for literal child output") + assert.Contains(t, res.Details, "%(count) {{ IF condition }}literal{{ END }}") +} + +func TestTaggedNonCommandArgumentRejected(t *testing.T) { + snc := StartTestAgent(t, "") + defer StopTestAgent(t, snc) + + res := snc.RunCheck("check_files", []string{"path[x]=/tmp"}) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN for tagged non-command argument") + assert.Contains(t, res.Output, "does not support tags") +} + func TestCheckMultiPriorityThreshold(t *testing.T) { config := ` [/modules] @@ -298,6 +318,10 @@ command[a] = check_multi config=loopA [/settings/check/multi/inner] command[leaf] = check_dummy 0 'nested detail' + +[/settings/check/multi/duplicate] +command[foo] = check_dummy 0 first +command[ foo ] = check_dummy 0 second `, script1, script2) snc := StartTestAgent(t, config) @@ -331,6 +355,12 @@ command[leaf] = check_dummy 0 'nested detail' assert.Contains(t, res.BuildOutputString(), "nested detail") assert.Contains(t, res.Details, "nested detail") + res = snc.RunCheck("check_multi", []string{ + "config=duplicate", + }) + assert.Equalf(t, CheckExitUnknown, res.State, "state UNKNOWN for whitespace-duplicate config tags") + assert.Contains(t, res.Output, "duplicate command tag: foo") + // Test direct loop detection: check_multi config=loop res = snc.RunCheck("check_multi", []string{ "config=loop", diff --git a/pkg/snclient/checkdata.go b/pkg/snclient/checkdata.go index 67fbf88c..e996a979 100644 --- a/pkg/snclient/checkdata.go +++ b/pkg/snclient/checkdata.go @@ -1120,9 +1120,11 @@ func (cd *CheckData) parseTaggedCommand(argRef *TaggedCommandList, tag, argValue func (cd *CheckData) parseAnyArg(argExpr, keyword, argValue string) (bool, error) { lookupKey := keyword tag := "" + hasTag := false if before, rest, found := strings.Cut(keyword, "["); found && strings.HasSuffix(keyword, "]") { lookupKey = before tag = rest[:len(rest)-1] + hasTag = true } arg, ok := cd.args[lookupKey] @@ -1132,6 +1134,11 @@ func (cd *CheckData) parseAnyArg(argExpr, keyword, argValue string) (bool, error return false, nil } } + if hasTag { + if _, tagged := arg.value.(*TaggedCommandList); !tagged { + return false, fmt.Errorf("argument %s does not support tags", lookupKey) + } + } if err := cd.parseArgValue(argExpr, keyword, argValue, tag, &arg); err != nil { return true, err diff --git a/pkg/snclient/checkresult.go b/pkg/snclient/checkresult.go index bd5cba5b..f926c75d 100644 --- a/pkg/snclient/checkresult.go +++ b/pkg/snclient/checkresult.go @@ -2,6 +2,7 @@ package snclient import ( "bytes" + "fmt" "regexp" "strconv" "strings" @@ -29,11 +30,12 @@ var reValuesUnit = regexp.MustCompile(`^([0-9.]+)(.*?)$`) // CheckResult is the result of a single check run. type CheckResult struct { - State int64 // naemon exit code: OK=0, Warning=1, Critical=2, Unknown=3 - Output string // plugin output, should be human readable - Metrics []*CheckMetric // performance data metrics - Raw *CheckData // reference to the original check data, for use in inventory and other checks - Details string // additional details that should be printed on a new line after the main output, e.g. for showing top consuming processes + State int64 // naemon exit code: OK=0, Warning=1, Critical=2, Unknown=3 + Output string // plugin output, should be human readable + Metrics []*CheckMetric // performance data metrics + Raw *CheckData // reference to the original check data, for use in inventory and other checks + Details string // additional details that should be printed on a new line after the main output, e.g. for showing top consuming processes + literalDetails map[string]string } func (cr *CheckResult) Finalize(timezone *time.Location, macros ...map[string]string) { @@ -59,6 +61,20 @@ func (cr *CheckResult) Finalize(timezone *time.Location, macros ...map[string]st cr.Output = ReplaceMacros(cr.Output, timezone, macroSet...) } cr.Details = ReplaceMacros(cr.Details, timezone, macroSet...) + for placeholder, literal := range cr.literalDetails { + cr.Details = strings.ReplaceAll(cr.Details, placeholder, literal) + } + cr.literalDetails = nil +} + +func (cr *CheckResult) LiteralizeDetails(value string) string { + if cr.literalDetails == nil { + cr.literalDetails = make(map[string]string) + } + placeholder := fmt.Sprintf("\x00snclient-literal-%d\x00", len(cr.literalDetails)) + cr.literalDetails[placeholder] = value + + return placeholder } func (cr *CheckResult) ApplyPerfConfig(perfCfg []PerfConfig) error {