diff --git a/core/cmd/shell_local_test.go b/core/cmd/shell_local_test.go index f4ccd61e8b9..ea03760a47b 100644 --- a/core/cmd/shell_local_test.go +++ b/core/cmd/shell_local_test.go @@ -74,7 +74,7 @@ func genTestEVMRelayers(t *testing.T, cfg chainlink.GeneralConfig, ds sqlutil.Da f := chainlink.RelayerFactory{ Logger: lggr, LoopRegistry: plugins.NewLoopRegistry(lggr, cfg.AppID().String(), cfg.Feature().LogPoller(), cfg.Database(), - cfg.Mercury(), cfg.Pyroscope(), cfg.AutoPprof(), cfg.Tracing(), cfg.Telemetry(), nil, "", cfg.LOOPP()), + cfg.Mercury(), cfg.Pyroscope(), cfg.AutoPprof(), cfg.Tracing(), cfg.Telemetry(), cfg.Metering(), nil, "", cfg.LOOPP()), CapabilitiesRegistry: capabilities.NewRegistry(lggr), } diff --git a/core/config/app_config.go b/core/config/app_config.go index d8cb8df626b..3632308cd72 100644 --- a/core/config/app_config.go +++ b/core/config/app_config.go @@ -61,6 +61,7 @@ type AppConfig interface { WebServer() WebServer Tracing() Tracing Telemetry() Telemetry + Metering() Metering CRE() CRE CCV() CCV Billing() Billing diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index 3bd697e934f..817bc64a208 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -960,6 +960,28 @@ Enabled = false # Default # By default, we only forward the go runtime metrics. Empty means forward everything. Prefixes = ["go_"] # Default +# Metering configures durable resource metering emission and the coarse +# deployment/node identity dimensions stamped on emitted MeterRecords and +# MeterSnapshots. +[Metering] +# MeterRecordsEnabled enables durable MeterRecord emission for LOOP plugins. +MeterRecordsEnabled = false # Default +# MeterSnapshotsEnabled enables durable MeterSnapshot emission for LOOP plugins. +# Requires MeterRecordsEnabled = true. +MeterSnapshotsEnabled = false # Default +# Product is the deployment product identity dimension, e.g. 'cre'. +Product = 'cre' # Default +# Tenant is the human-readable tenant name, e.g. 'mainline'. +Tenant = '' # Default +# NumericTenantID is the numbered tenant identifier represented as a string. +NumericTenantID = '' # Default +# Environment is the deployment environment identity dimension, e.g. 'production'. +Environment = '' # Default +# Zone is the deployment zone identity dimension, e.g. 'wf-zone-a'. +Zone = '' # Default +# NodeID is the node's logical name, e.g. 'clp-cre-wf-zone-a-1' (not the CSA public key). +NodeID = '' # Default + [CRE.Streams] # WsURL is the websockets url for the streams sdk config WsURL = "streams.url" # Example diff --git a/core/config/metering_config.go b/core/config/metering_config.go new file mode 100644 index 00000000000..9f11e1c3419 --- /dev/null +++ b/core/config/metering_config.go @@ -0,0 +1,16 @@ +package config + +// Metering exposes durable resource-metering configuration: the emission +// toggles and the coarse deployment/node identity dimensions stamped on emitted +// MeterRecords and MeterSnapshots. These are passed via loop.EnvConfig to every LOOP +// plugin. +type Metering interface { + MeterRecordsEnabled() bool + MeterSnapshotsEnabled() bool + Product() string + Tenant() string + NumericTenantID() string + Environment() string + Zone() string + NodeID() string +} diff --git a/core/config/toml/types.go b/core/config/toml/types.go index 2fe42cda761..b405134b02b 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -64,6 +64,7 @@ type Core struct { Mercury Mercury `toml:",omitempty"` Capabilities Capabilities `toml:",omitempty"` Telemetry Telemetry `toml:",omitempty"` + Metering Metering `toml:",omitempty"` Workflows Workflows `toml:",omitempty"` CRE CreConfig `toml:",omitempty"` Billing Billing `toml:",omitempty"` @@ -112,6 +113,7 @@ func (c *Core) SetFrom(f *Core) { c.JobDistributor.setFrom(&f.JobDistributor) c.Tracing.setFrom(&f.Tracing) c.Telemetry.setFrom(&f.Telemetry) + c.Metering.setFrom(&f.Metering) c.CRE.setFrom(&f.CRE) c.Billing.setFrom(&f.Billing) c.BridgeStatusReporter.setFrom(&f.BridgeStatusReporter) @@ -3163,6 +3165,75 @@ func (b *Telemetry) ValidateConfig() (err error) { return err } +// Metering configures durable resource metering emission and the coarse +// deployment/node identity dimensions stamped on emitted MeterRecords and +// MeterSnapshots. These are passed via loop.EnvConfig to every LOOP plugin. +type Metering struct { + // MeterRecordsEnabled enables durable MeterRecord emission for LOOP plugins. + MeterRecordsEnabled *bool + // MeterSnapshotsEnabled enables durable MeterSnapshot emission. Requires + // MeterRecordsEnabled to be true. + MeterSnapshotsEnabled *bool + // Product is the deployment product identity dimension, e.g. "cre". + Product *string + // Tenant is the human-readable tenant name, e.g. "mainline". + Tenant *string + // NumericTenantID is the numbered tenant identifier as a string. + NumericTenantID *string + // Environment is the deployment environment dimension, e.g. "production". + Environment *string + // Zone is the deployment zone dimension, e.g. "wf-zone-a". + Zone *string + // NodeID is the node's logical name, e.g. "clp-cre-wf-zone-a-1" (NOT the CSA + // public key) + NodeID *string +} + +func (b *Metering) setFrom(f *Metering) { + if v := f.MeterRecordsEnabled; v != nil { + b.MeterRecordsEnabled = v + } + if v := f.MeterSnapshotsEnabled; v != nil { + b.MeterSnapshotsEnabled = v + } + if v := f.Product; v != nil { + b.Product = v + } + if v := f.Tenant; v != nil { + b.Tenant = v + } + if v := f.NumericTenantID; v != nil { + b.NumericTenantID = v + } + if v := f.Environment; v != nil { + b.Environment = v + } + if v := f.Zone; v != nil { + b.Zone = v + } + if v := f.NodeID; v != nil { + b.NodeID = v + } +} + +func (b *Metering) ValidateConfig() (err error) { + if b.MeterSnapshotsEnabled != nil && *b.MeterSnapshotsEnabled && (b.MeterRecordsEnabled == nil || !*b.MeterRecordsEnabled) { + err = errors.Join(err, configutils.ErrInvalid{ + Name: "MeterSnapshotsEnabled", + Value: true, + Msg: "requires MeterRecordsEnabled to be true", + }) + } + if b.MeterRecordsEnabled != nil && *b.MeterRecordsEnabled && (b.NodeID == nil || *b.NodeID == "") { + err = errors.Join(err, configutils.ErrInvalid{ + Name: "NodeID", + Value: "", + Msg: "must be non-empty when MeterRecordsEnabled is true (an empty NodeID collapses per-node snapshot dedup scope DON-wide)", + }) + } + return err +} + type PrometheusBridge struct { Enabled *bool Prefixes []string diff --git a/core/config/toml/types_test.go b/core/config/toml/types_test.go index bae3d4b1be6..89c19b3f1d4 100644 --- a/core/config/toml/types_test.go +++ b/core/config/toml/types_test.go @@ -860,3 +860,67 @@ func durationPtr(d time.Duration) *commonconfig.Duration { cd := *commonconfig.MustNewDuration(d) return &cd } + +func TestMetering_ValidateConfig(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + config *Metering + expectError bool + errorMsg string + }{ + { + name: "disabled with all nil fields", + config: &Metering{}, + expectError: false, + }, + { + name: "records enabled with non-empty NodeID", + config: &Metering{ + MeterRecordsEnabled: new(true), + NodeID: new("clp-cre-wf-zone-a-1"), + }, + expectError: false, + }, + { + name: "records enabled with nil NodeID", + config: &Metering{ + MeterRecordsEnabled: new(true), + NodeID: nil, + }, + expectError: true, + errorMsg: "NodeID", + }, + { + name: "records enabled with empty NodeID", + config: &Metering{ + MeterRecordsEnabled: new(true), + NodeID: new(""), + }, + expectError: true, + errorMsg: "NodeID", + }, + { + name: "snapshots enabled without records enabled", + config: &Metering{ + MeterSnapshotsEnabled: new(true), + NodeID: new("clp-cre-wf-zone-a-1"), + }, + expectError: true, + errorMsg: "requires MeterRecordsEnabled to be true", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + err := tc.config.ValidateConfig() + if tc.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errorMsg) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/core/scripts/cre/environment/configs/examples/workflow-don-overrides.toml b/core/scripts/cre/environment/configs/examples/workflow-don-overrides.toml index 24884aff475..35c56bd55cc 100644 --- a/core/scripts/cre/environment/configs/examples/workflow-don-overrides.toml +++ b/core/scripts/cre/environment/configs/examples/workflow-don-overrides.toml @@ -64,28 +64,16 @@ #docker_file = "core/chainlink.Dockerfile" image = "chainlink-tmp:latest" user_config_overrides = """ + # [Telemetry], [Billing], and [Metering] are framework-managed and are + # REJECTED here (see validateUserConfigOverrides in system-tests/lib): + # the framework generates them itself, pointing telemetry at the + # chip-router so downstream subscribers (e.g. the CI test sink) see all + # events. To enable metering, set enable_metering = true on the nodeset. [Log] Level = 'debug' JSONConsole = true - [Telemetry] - Enabled = true - # assumes that CTF's observability stack is running on the local host - # 4317 is OTEL ingress port of the stack (LGTM) - # command: ctf obs up - Endpoint = 'host.docker.internal:4317' - # default service-name and port used by locally running Chip Ingress - # command: go run . env chip-ingress-stack start - ChipIngressEndpoint = 'chip-ingress:50051' - InsecureConnection = true - TraceSampleRatio = 1 - HeartbeatInterval = '30s' - # Remove this bit to allow downloading workflows from remote sources [CRE.WorkflowFetcher] URL = "file:///home/chainlink/workflows" - - [Billing] - URL = "host.docker.internal:2223" - TLSEnabled = false """ diff --git a/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml b/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml index 0401c754ad2..921a3e00cbf 100644 --- a/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml +++ b/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml @@ -40,6 +40,7 @@ name = "workflow" don_family = "test-don-family" don_types = ["workflow"] + enable_metering = true override_mode = "all" http_port_range_start = 10100 @@ -71,6 +72,7 @@ name = "capabilities" don_family = "test-don-family" don_types = ["capabilities"] + enable_metering = true exposes_remote_capabilities = true override_mode = "all" http_port_range_start = 10200 diff --git a/core/scripts/cre/environment/environment/chip_ingress_stack.go b/core/scripts/cre/environment/environment/chip_ingress_stack.go index eaac04360ef..f3d3bfca0fa 100644 --- a/core/scripts/cre/environment/environment/chip_ingress_stack.go +++ b/core/scripts/cre/environment/environment/chip_ingress_stack.go @@ -77,10 +77,14 @@ func schemaCommitRefFromGoMod(ctx context.Context, repoRoot, targetModule string // getSchemaSetFromGoMod resolves SchemaSets from chainlink-protos commits pinned in go.mod: // - workflows (chip-cre.json) for CRE/workflow telemetry // - node-platform (chip-schemas.json) for PluginRelayerConfigEmitter / common.v1.ChainPluginConfig +// - metering (chip-cll.meter.json) for durable resource metering (MeterRecord/MeterSnapshot on +// the cll.meter domain); without this, ChIP Ingress rejects those events at pre-publish encode +// time with "Subject 'cll-meter-metering.v1.MeterRecord' not found" and drops them silently. func getSchemaSetFromGoMod(ctx context.Context) ([]chipingressset.SchemaSet, error) { const ( workflowsModule = "github.com/smartcontractkit/chainlink-protos/workflows/go" nodePlatformModule = "github.com/smartcontractkit/chainlink-protos/node-platform" + meteringModule = "github.com/smartcontractkit/chainlink-protos/metering/go" ) repoRoot, err := filepath.Abs(relativePathToRepoRoot) @@ -100,6 +104,12 @@ func getSchemaSetFromGoMod(ctx context.Context) ([]chipingressset.SchemaSet, err } framework.L.Info().Msgf("Extracted commit ref for %s: %s (from version: %s)", nodePlatformModule, npRef, npVer) + meteringRef, meteringVer, err := schemaCommitRefFromGoMod(ctx, repoRoot, meteringModule) + if err != nil { + return nil, err + } + framework.L.Info().Msgf("Extracted commit ref for %s: %s (from version: %s)", meteringModule, meteringRef, meteringVer) + return []chipingressset.SchemaSet{ { URI: chainlinkProtosGitURI, @@ -113,6 +123,12 @@ func getSchemaSetFromGoMod(ctx context.Context) ([]chipingressset.SchemaSet, err SchemaDir: "node-platform", ConfigFile: "chip-schemas.json", }, + { + URI: chainlinkProtosGitURI, + Ref: meteringRef, + SchemaDir: "metering", + ConfigFile: "chip-cll.meter.json", + }, }, nil } @@ -1045,6 +1061,6 @@ func fetchAndRegisterProtosCmd() *cobra.Command { }) }, } - cmd.Flags().StringVarP(&chipIngressGRPCURL, "chip-ingress-grpc-url", "h", "localhost:"+chipingressset.DEFAULT_CHIP_INGRESS_GRPC_PORT, "Chip Ingress GRPC URL") + cmd.Flags().StringVarP(&chipIngressGRPCURL, "chip-ingress-grpc-url", "u", "localhost:"+chipingressset.DEFAULT_CHIP_INGRESS_GRPC_PORT, "Chip Ingress GRPC URL") return cmd } diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 88b3913ecb0..0eeedefeded 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -511,7 +511,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 4c8e6980e15..55db1483222 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1626,8 +1626,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index 645f864479f..23bc1ed0001 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -286,7 +286,7 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err } loopRegistry := plugins.NewLoopRegistry(globalLogger, cfg.AppID().String(), cfg.Feature().LogPoller(), cfg.Database(), cfg.Mercury(), cfg.Pyroscope(), cfg.AutoPprof(), cfg.Tracing(), cfg.Telemetry(), - beholderAuthHeaders, csaPubKeyHex, cfg.LOOPP()) + cfg.Metering(), beholderAuthHeaders, csaPubKeyHex, cfg.LOOPP()) relayerFactory := RelayerFactory{ Logger: opts.Logger, diff --git a/core/services/chainlink/config_general.go b/core/services/chainlink/config_general.go index 71fa265fc05..3b6c86522a9 100644 --- a/core/services/chainlink/config_general.go +++ b/core/services/chainlink/config_general.go @@ -587,6 +587,10 @@ func (g *generalConfig) Telemetry() coreconfig.Telemetry { return &telemetryConfig{s: g.c.Telemetry} } +func (g *generalConfig) Metering() coreconfig.Metering { + return &meteringConfig{s: g.c.Metering} +} + func (g *generalConfig) CRE() coreconfig.CRE { return &creConfig{s: g.secrets.CRE, c: g.c.CRE} } diff --git a/core/services/chainlink/config_metering.go b/core/services/chainlink/config_metering.go new file mode 100644 index 00000000000..c63a6b3e022 --- /dev/null +++ b/core/services/chainlink/config_metering.go @@ -0,0 +1,70 @@ +package chainlink + +import ( + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" + "github.com/smartcontractkit/chainlink/v2/core/config/toml" +) + +type meteringConfig struct { + s toml.Metering +} + +func (b *meteringConfig) MeterRecordsEnabled() bool { + if b.s.MeterRecordsEnabled == nil { + return false + } + return *b.s.MeterRecordsEnabled +} + +func (b *meteringConfig) MeterSnapshotsEnabled() bool { + if b.s.MeterSnapshotsEnabled == nil { + return false + } + return *b.s.MeterSnapshotsEnabled +} + +// Product returns the deployment product identity dimension. The parsed config +// defaults it to "cre" via docs.CoreDefaults so metering is never enabled with +// an empty product dimension; a zero-value toml.Metering that has not been run +// through setDefaults returns UnsetProduct (the nil-pointer fallback below). +func (b *meteringConfig) Product() string { + if b.s.Product == nil { + return resourcemanager.UnsetProduct + } + return *b.s.Product +} + +func (b *meteringConfig) Tenant() string { + if b.s.Tenant == nil { + return "" + } + return *b.s.Tenant +} + +func (b *meteringConfig) NumericTenantID() string { + if b.s.NumericTenantID == nil { + return "" + } + return *b.s.NumericTenantID +} + +func (b *meteringConfig) Environment() string { + if b.s.Environment == nil { + return "" + } + return *b.s.Environment +} + +func (b *meteringConfig) Zone() string { + if b.s.Zone == nil { + return "" + } + return *b.s.Zone +} + +func (b *meteringConfig) NodeID() string { + if b.s.NodeID == nil { + return "" + } + return *b.s.NodeID +} diff --git a/core/services/chainlink/config_metering_test.go b/core/services/chainlink/config_metering_test.go new file mode 100644 index 00000000000..71eb6eb0da7 --- /dev/null +++ b/core/services/chainlink/config_metering_test.go @@ -0,0 +1,52 @@ +package chainlink + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/smartcontractkit/chainlink/v2/core/config/toml" +) + +func TestMeteringConfig(t *testing.T) { + t.Parallel() + t.Run("defaults", func(t *testing.T) { + t.Parallel() + mc := meteringConfig{s: toml.Metering{}} + assert.False(t, mc.MeterRecordsEnabled()) + assert.False(t, mc.MeterSnapshotsEnabled()) + // A zero-value toml.Metering (not run through setDefaults) has a nil + // Product pointer, so Product() returns "unset". The parsed config + // applies a "cre" default via docs.CoreDefaults (covered by the + // LogConfiguration effective-TOML test), so metering is never enabled + // with an empty product dimension. + assert.Equal(t, "unset", mc.Product()) + assert.Empty(t, mc.Tenant()) + assert.Empty(t, mc.NumericTenantID()) + assert.Empty(t, mc.Environment()) + assert.Empty(t, mc.Zone()) + assert.Empty(t, mc.NodeID()) + }) + + t.Run("explicit values", func(t *testing.T) { + t.Parallel() + mc := meteringConfig{s: toml.Metering{ + MeterRecordsEnabled: new(true), + MeterSnapshotsEnabled: new(true), + Product: new("cre"), + Tenant: new("mainline"), + NumericTenantID: new("42"), + Environment: new("production"), + Zone: new("wf-zone-a"), + NodeID: new("clp-cre-wf-zone-a-1"), + }} + assert.True(t, mc.MeterRecordsEnabled()) + assert.True(t, mc.MeterSnapshotsEnabled()) + assert.Equal(t, "cre", mc.Product()) + assert.Equal(t, "mainline", mc.Tenant()) + assert.Equal(t, "42", mc.NumericTenantID()) + assert.Equal(t, "production", mc.Environment()) + assert.Equal(t, "wf-zone-a", mc.Zone()) + assert.Equal(t, "clp-cre-wf-zone-a-1", mc.NodeID()) + }) +} diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 9ae33c0e05a..88d6f39dcd9 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -589,6 +589,16 @@ func TestConfig_Marshal(t *testing.T) { Prefixes: []string{"ocr_"}, }, } + full.Metering = toml.Metering{ + MeterRecordsEnabled: new(true), + MeterSnapshotsEnabled: new(true), + Product: new("cre"), + Tenant: new("mainline"), + NumericTenantID: new("42"), + Environment: new("production"), + Zone: new("wf-zone-a"), + NodeID: new("clp-cre-wf-zone-a-1"), + } full.CRE = toml.CreConfig{ UseLocalTimeProvider: new(true), EnableDKGRecipient: new(false), diff --git a/core/services/chainlink/mocks/general_config.go b/core/services/chainlink/mocks/general_config.go index a1a857ced3a..8f2b34a2ccf 100644 --- a/core/services/chainlink/mocks/general_config.go +++ b/core/services/chainlink/mocks/general_config.go @@ -1523,6 +1523,53 @@ func (_c *GeneralConfig_Mercury_Call) RunAndReturn(run func() dataengine.Mercury return _c } +// Metering provides a mock function with no fields +func (_m *GeneralConfig) Metering() config.Metering { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Metering") + } + + var r0 config.Metering + if rf, ok := ret.Get(0).(func() config.Metering); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(config.Metering) + } + } + + return r0 +} + +// GeneralConfig_Metering_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Metering' +type GeneralConfig_Metering_Call struct { + *mock.Call +} + +// Metering is a helper method to define mock.On call +func (_e *GeneralConfig_Expecter) Metering() *GeneralConfig_Metering_Call { + return &GeneralConfig_Metering_Call{Call: _e.mock.On("Metering")} +} + +func (_c *GeneralConfig_Metering_Call) Run(run func()) *GeneralConfig_Metering_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *GeneralConfig_Metering_Call) Return(_a0 config.Metering) *GeneralConfig_Metering_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *GeneralConfig_Metering_Call) RunAndReturn(run func() config.Metering) *GeneralConfig_Metering_Call { + _c.Call.Return(run) + return _c +} + // OCR provides a mock function with no fields func (_m *GeneralConfig) OCR() config.OCR { ret := _m.Called() diff --git a/core/services/chainlink/testdata/config-empty-effective.toml b/core/services/chainlink/testdata/config-empty-effective.toml index f0e139720dd..cc77a2a4f6d 100644 --- a/core/services/chainlink/testdata/config-empty-effective.toml +++ b/core/services/chainlink/testdata/config-empty-effective.toml @@ -377,6 +377,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index e72250d2270..f12e60a6c0b 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -416,6 +416,16 @@ Foo = 'bar' Enabled = true Prefixes = ['ocr_'] +[Metering] +MeterRecordsEnabled = true +MeterSnapshotsEnabled = true +Product = 'cre' +Tenant = 'mainline' +NumericTenantID = '42' +Environment = 'production' +Zone = 'wf-zone-a' +NodeID = 'clp-cre-wf-zone-a-1' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index 049e0576280..6b233347d23 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -377,6 +377,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/core/services/cre/confidential_relay_peerid_test.go b/core/services/cre/confidential_relay_peerid_test.go index 4c64ccb6943..31e9df40d4c 100644 --- a/core/services/cre/confidential_relay_peerid_test.go +++ b/core/services/cre/confidential_relay_peerid_test.go @@ -50,6 +50,7 @@ func (s stubConfig) Workflows() config.Workflows { return nil } func (s stubConfig) CRE() config.CRE { return nil } func (s stubConfig) P2P() config.P2P { return s.p2p } func (s stubConfig) Sharding() config.Sharding { return nil } +func (s stubConfig) Metering() config.Metering { return nil } func peerIDFromByte(b byte) p2pkey.PeerID { var id p2pkey.PeerID diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index e33b2e89f52..1d1442f5075 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -18,11 +18,13 @@ import ( "github.com/smartcontractkit/chainlink-common/keystore/corekeys/p2pkey" "github.com/smartcontractkit/chainlink-common/keystore/corekeys/workflowkey" + "github.com/smartcontractkit/chainlink-common/pkg/beholder" "github.com/smartcontractkit/chainlink-common/pkg/billing" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/loop" nodeauthjwt "github.com/smartcontractkit/chainlink-common/pkg/nodeauth/jwt" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" commonsrv "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" @@ -292,6 +294,10 @@ func (s *Services) newSubservices( return srvs, nil } + // Build the syncer's base metering identity once from node metering config; + // the handler resolves the workflow DON id later from the don notifier. + meterIdentity := newSyncerMeterIdentity(cfg) + wfSyncer, billingClient, wfSyncerSrvcs, err := newWorkflowRegistrySyncer( cfg, relayerChainInterops, @@ -305,6 +311,7 @@ func (s *Services) newSubservices( opts.LimitsFactory, s.OrgResolver, s.GatewayConnectorWrapper, + meterIdentity, ) if err != nil { return nil, err @@ -333,6 +340,7 @@ type Config interface { CRE() config.CRE P2P() config.P2P Sharding() config.Sharding + Metering() config.Metering } // RelayerChainInterops is the minimal interface needed for relayer chain interops @@ -688,6 +696,30 @@ func newBillingClient(lggr logger.Logger, cfg Config, opts Opts) (metering.Billi return billing.NewWorkflowClient(lggr, cfg.Billing().URL(), workflowOpts...) } +// newSyncerMeterIdentity builds the syncer's base metering identity from node +// metering config. Product/tenant/environment/zone and logical node_id are read +// from [Metering]. The workflow DON id (don_id) is resolved later by the +// registry from the don notifier (the engine runs on the workflow DON), so it +// is intentionally left empty here. Service/resource_pool are stamped by +// syncer.NewSpecMeter. +func newSyncerMeterIdentity(cfg Config) resourcemanager.ResourceIdentity { + m := cfg.Metering() + if m == nil { + return resourcemanager.ResourceIdentity{} + } + id := resourcemanager.ResourceIdentity{ + Product: m.Product(), + Tenant: m.Tenant(), + NumericTenantID: m.NumericTenantID(), + Environment: m.Environment(), + Zone: m.Zone(), + } + if nodeID := m.NodeID(); nodeID != "" { + id.Don = &resourcemanager.DonIdentity{NodeID: nodeID} + } + return id +} + func newShardOrchestratorClient(cfg Config, lggr logger.Logger) (*shardorchestrator.Client, error) { shardID := cfg.Sharding().ShardIndex() if shardID == 0 { @@ -903,6 +935,7 @@ func newWorkflowRegistrySyncerV2( lf limits.Factory, orgResolver orgresolver.OrgResolver, gatewayConnectorWrapper *gatewayconnector.ServiceWrapper, + meterIdentity resourcemanager.ResourceIdentity, ) (syncerV2.WorkflowRegistrySyncer, []commonsrv.Service, error) { capCfg := cfg.Capabilities() wfReg := capCfg.WorkflowRegistry() @@ -979,6 +1012,10 @@ func newWorkflowRegistrySyncerV2( shardRoutingSteady = shardownership.NewSteadySignal(shardownership.WithSteadySignalMetrics(steadyMetrics)) } + meteringCfg := cfg.Metering() + meterRecordsEnabled := meteringCfg != nil && meteringCfg.MeterRecordsEnabled() + meterSnapshotsEnabled := meteringCfg != nil && meteringCfg.MeterSnapshotsEnabled() + handlerOpts := []syncerV2.EventHandlerOption{ syncerV2.WithBillingClient(billingClient), syncerV2.WithWorkflowRegistry(capCfg.WorkflowRegistry().Address(), selector), @@ -989,6 +1026,23 @@ func newWorkflowRegistrySyncerV2( syncerV2.WithShardRoutingSteady(shardRoutingSteady), } + // The spec meter (and its ResourceManager) exists only when metering is + // enabled; a handler without one emits nothing. The meter owns the RM + // lifecycle and snapshot registration as a sub-service of the handler. + if meterRecordsEnabled || meterSnapshotsEnabled { + rm := resourcemanager.NewResourceManager(lggr, resourcemanager.ResourceManagerConfig{ + MeterRecordsEnabled: meterRecordsEnabled, + MeterSnapshotsEnabled: meterSnapshotsEnabled, + Emitter: beholder.GetEmitter(), + SnapshotInterval: resourcemanager.DefaultSnapshotInterval, + }) + specMeter, smErr := syncerV2.NewSpecMeter(lggr, rm, meterIdentity, artifactsStore, orgResolver) + if smErr != nil { + return nil, nil, fmt.Errorf("unable to create workflow spec meter: %w", smErr) + } + handlerOpts = append(handlerOpts, syncerV2.WithSpecMeter(specMeter)) + } + mc := capCfg.WorkflowRegistry().ModuleCache() cacheEnabled := mc.Enabled() diskMonitorEnabled := mc.DiskMonitorEnabled() || cacheEnabled @@ -1138,6 +1192,7 @@ func newWorkflowRegistrySyncer( lf limits.Factory, orgResolver orgresolver.OrgResolver, gatewayConnectorWrapper *gatewayconnector.ServiceWrapper, + meterIdentity resourcemanager.ResourceIdentity, ) (syncerV2.WorkflowRegistrySyncer, metering.BillingClient, []commonsrv.Service, error) { capCfg := cfg.Capabilities() @@ -1186,6 +1241,7 @@ func newWorkflowRegistrySyncer( lf, orgResolver, gatewayConnectorWrapper, + meterIdentity, ) return syncer, billingClient, srvcs, err default: diff --git a/core/services/job/models.go b/core/services/job/models.go index 81115a9d256..ba9cc7836db 100644 --- a/core/services/job/models.go +++ b/core/services/job/models.go @@ -842,9 +842,13 @@ type WorkflowSpec struct { UpdatedAt time.Time `toml:"-" db:"updated_at"` SpecType WorkflowSpecType `toml:"spec_type" db:"spec_type"` Attributes []byte `db:"attributes"` - sdkWorkflow *sdk.WorkflowSpec - rawSpec []byte - config []byte + RegisteredAt int64 `toml:"-" db:"registered_at"` + // Source records which workflow metadata source produced this spec (e.g. + // "ContractWorkflowSource"). + Source string `toml:"-" db:"source"` + sdkWorkflow *sdk.WorkflowSpec + rawSpec []byte + config []byte } var ( diff --git a/core/services/standardcapabilities/standard_capabilities.go b/core/services/standardcapabilities/standard_capabilities.go index f5b373aa4ed..85cb44d829a 100644 --- a/core/services/standardcapabilities/standard_capabilities.go +++ b/core/services/standardcapabilities/standard_capabilities.go @@ -101,6 +101,24 @@ func NewStandardCapabilities( } } +// initialiseDependencies builds the StandardCapabilitiesDependencies delivered to +// the capability LOOP via Initialise. +func (s *StandardCapabilities) initialiseDependencies() core.StandardCapabilitiesDependencies { + return core.StandardCapabilitiesDependencies{ + Config: s.config, + Store: s.store, + CapabilityRegistry: s.CapabilitiesRegistry, + RelayerSet: s.relayerSet, + OracleFactory: s.oracleFactory, + GatewayConnector: s.gatewayConnector, + P2PKeystore: s.keystore, + OrgResolver: s.orgResolver, + CRESettings: s.creSettings, + TriggerEventStore: s.triggerEventStore, + CapabilityDonID: s.capabilityDonID, + } +} + func (s *StandardCapabilities) Start(ctx context.Context) error { return s.StartOnce("StandardCapabilities", func() error { envVars, err := plugins.ParseEnvFile(env.CapabilitiesPlugin.Env.Get()) @@ -137,19 +155,7 @@ func (s *StandardCapabilities) Start(ctx context.Context) error { return } - dependencies := core.StandardCapabilitiesDependencies{ - Config: s.config, - Store: s.store, - CapabilityRegistry: s.CapabilitiesRegistry, - RelayerSet: s.relayerSet, - OracleFactory: s.oracleFactory, - GatewayConnector: s.gatewayConnector, - P2PKeystore: s.keystore, - OrgResolver: s.orgResolver, - CRESettings: s.creSettings, - TriggerEventStore: s.triggerEventStore, - CapabilityDonID: s.capabilityDonID, - } + dependencies := s.initialiseDependencies() if err = s.capabilitiesLoop.Service.Initialise(cctx, dependencies); err != nil { s.log.Errorf("error initialising standard capabilities service: %v", err) s.setReadyErr(fmt.Errorf("initialising standard capabilities service: %w", err)) diff --git a/core/services/standardcapabilities/standard_capabilities_test.go b/core/services/standardcapabilities/standard_capabilities_test.go index 8662cc5bbe5..b0198649632 100644 --- a/core/services/standardcapabilities/standard_capabilities_test.go +++ b/core/services/standardcapabilities/standard_capabilities_test.go @@ -104,6 +104,66 @@ func TestStandardCapabilities_ForwardsPluginEnvFile(t *testing.T) { }) } +func TestStandardCapabilities_InitialiseDependenciesRoundTrip(t *testing.T) { + t.Parallel() + want := core.StandardCapabilitiesDependencies{ + Config: "test-config", + } + + std := NewStandardCapabilities( + logger.TestLogger(t), + "not/found/path/to/binary", + want.Config, + &capturingRegistrar{}, + want, + ) + + got := std.initialiseDependencies() + + require.Equal(t, want.Config, got.Config, "config should be re-delivered to LOOP Initialise dependencies") +} + +// TestStandardCapabilities_CapabilityDonIDDeliveredToLOOP asserts that the +// host-resolved capability DON ID is carried on the dependencies delivered to +// the capability LOOP at Initialise. Without this, a trigger producer silently +// falls back to the consumer workflow's DON for metering identity and event +// labels even when the node knows which DON it is serving. +func TestStandardCapabilities_CapabilityDonIDDeliveredToLOOP(t *testing.T) { + t.Parallel() + t.Run("nonzero DON ID round-trips when the DON is known", func(t *testing.T) { + t.Parallel() + const knownDonID = uint32(42) + std := NewStandardCapabilities( + logger.TestLogger(t), + "not/found/path/to/binary", + "{}", + &capturingRegistrar{}, + core.StandardCapabilitiesDependencies{CapabilityDonID: knownDonID}, + ) + + got := std.initialiseDependencies() + + require.Equal(t, knownDonID, got.CapabilityDonID, + "the host-resolved capability DON ID must reach the LOOP at Initialise") + }) + + t.Run("zero DON ID is preserved when the host could not resolve one", func(t *testing.T) { + t.Parallel() + std := NewStandardCapabilities( + logger.TestLogger(t), + "not/found/path/to/binary", + "{}", + &capturingRegistrar{}, + core.StandardCapabilitiesDependencies{}, + ) + + got := std.initialiseDependencies() + + require.Zero(t, got.CapabilityDonID, + "an unresolved DON ID stays zero so the LOOP can fall back to the workflow DON") + }) +} + const capturingRegistrarErr = "capturingRegistrar: stop after capture" // capturingRegistrar records the CmdConfig handed to RegisterLOOP and then diff --git a/core/services/workflows/artifacts/v2/orm.go b/core/services/workflows/artifacts/v2/orm.go index 31395bce8f3..27a28b18bef 100644 --- a/core/services/workflows/artifacts/v2/orm.go +++ b/core/services/workflows/artifacts/v2/orm.go @@ -3,6 +3,7 @@ package v2 import ( "context" "database/sql" + "errors" "time" "github.com/jmoiron/sqlx" @@ -20,8 +21,24 @@ type WorkflowSpecsDS interface { // GetWorkflowSpecByID returns the workflow spec for the given workflowID. GetWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSpec, error) - // DeleteWorkflowSpec deletes the workflow spec for the given workflow ID. - DeleteWorkflowSpec(ctx context.Context, id string) error + // ListWorkflowSpecs returns the persisted workflow specs. It projects only + // identity columns (workflow_id, workflow_owner, registered_at); + // other fields are left zero to keep returned batch size small + ListWorkflowSpecs(ctx context.Context) ([]*job.WorkflowSpec, error) + + // DeleteWorkflowSpec deletes the spec row for id. Idempotent: a missing row + // returns (nil, nil), not an error. The deleted row is returned so the + // caller can derive the generation-scoped metering event_id from the row's + // own persisted registered_at (every node emits the identical id regardless + // of local staleness). + DeleteWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSpec, error) + + // PauseWorkflowSpec tombstones the spec row: status becomes paused and the + // heavy artifact payload (hex binary + config) is cleared in the same + // statement, freeing storage while the row itself remains the durable record + // that this registration generation is still held (level-neutral for + // metering). Idempotent; pausing an absent or already-paused row is a no-op. + PauseWorkflowSpec(ctx context.Context, id string) error // DeleteWorkflowSpecs deletes workflow specs for the given workflow IDs in a single query. DeleteWorkflowSpecs(ctx context.Context, ids []string) error @@ -65,7 +82,9 @@ func (orm *orm) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) created_at, updated_at, spec_type, - attributes + attributes, + registered_at, + source ) VALUES ( :workflow, :config, @@ -79,7 +98,9 @@ func (orm *orm) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) :created_at, :updated_at, :spec_type, - :attributes + :attributes, + :registered_at, + :source ) ON CONFLICT (workflow_id) DO UPDATE SET workflow = EXCLUDED.workflow, @@ -93,7 +114,9 @@ func (orm *orm) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) created_at = EXCLUDED.created_at, updated_at = EXCLUDED.updated_at, spec_type = EXCLUDED.spec_type, - attributes = EXCLUDED.attributes + attributes = EXCLUDED.attributes, + registered_at = EXCLUDED.registered_at, + source = EXCLUDED.source RETURNING id ` @@ -121,6 +144,7 @@ func (orm *orm) GetWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSp ` var spec job.WorkflowSpec + // Note: "Get will return sql.ErrNoRows like row.Scan would" - sqlx@v1.4.0 err := orm.ds.GetContext(ctx, &spec, query, id) if err != nil { return nil, err @@ -129,27 +153,44 @@ func (orm *orm) GetWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSp return &spec, nil } -func (orm *orm) DeleteWorkflowSpec(ctx context.Context, id string) error { +// ListWorkflowSpecs returns all persisted workflow specs, projecting the +// identity columns needed by the orphan sweep and the metering snapshot path. +func (orm *orm) ListWorkflowSpecs(ctx context.Context) ([]*job.WorkflowSpec, error) { query := ` - DELETE FROM workflow_specs_v2 - WHERE workflow_id = $1 + SELECT workflow_id, workflow_owner, registered_at, source + FROM workflow_specs_v2 ` - result, err := orm.ds.ExecContext(ctx, query, id) - if err != nil { - return err + var specs []*job.WorkflowSpec + if err := orm.ds.SelectContext(ctx, &specs, query); err != nil { + return nil, err } - rowsAffected, err := result.RowsAffected() - if err != nil { - return err - } + return specs, nil +} - if rowsAffected == 0 { - return sql.ErrNoRows // No spec deleted +func (orm *orm) DeleteWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSpec, error) { + query := `DELETE FROM workflow_specs_v2 WHERE workflow_id = $1 + RETURNING id, workflow, config, workflow_id, workflow_owner, workflow_name, + workflow_tag, status, binary_url, config_url, created_at, updated_at, + spec_type, attributes, registered_at, source` + var spec job.WorkflowSpec + err := orm.ds.GetContext(ctx, &spec, query, id) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err } + return &spec, nil +} - return nil +func (orm *orm) PauseWorkflowSpec(ctx context.Context, id string) error { + query := `UPDATE workflow_specs_v2 + SET status = $2, workflow = '', config = '', updated_at = now() + WHERE workflow_id = $1` + _, err := orm.ds.ExecContext(ctx, query, id, job.WorkflowSpecStatusPaused) + return err } func (orm *orm) DeleteWorkflowSpecs(ctx context.Context, ids []string) error { diff --git a/core/services/workflows/artifacts/v2/orm_test.go b/core/services/workflows/artifacts/v2/orm_test.go index e83c7a48389..bd5a654910f 100644 --- a/core/services/workflows/artifacts/v2/orm_test.go +++ b/core/services/workflows/artifacts/v2/orm_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" @@ -188,14 +189,32 @@ func Test_DeleteWorkflowSpec(t *testing.T) { ConfigURL: "http://example.com/config", CreatedAt: time.Now(), SpecType: job.WASMFile, + RegisteredAt: 1717171717, + Source: "ContractWorkflowSource", } id, err := orm.UpsertWorkflowSpec(ctx, spec) require.NoError(t, err) require.NotZero(t, id) - err = orm.DeleteWorkflowSpec(ctx, spec.WorkflowID) + deletedSpec, err := orm.DeleteWorkflowSpec(ctx, spec.WorkflowID) require.NoError(t, err) + require.NotNil(t, deletedSpec) + + // The RETURNING clause must scan the full row back: the caller derives + // the metering delete event_id from the row's persisted registered_at. + assert.Equal(t, spec.WorkflowID, deletedSpec.WorkflowID) + assert.Equal(t, spec.WorkflowOwner, deletedSpec.WorkflowOwner) + assert.Equal(t, spec.WorkflowName, deletedSpec.WorkflowName) + assert.Equal(t, spec.WorkflowTag, deletedSpec.WorkflowTag) + assert.Equal(t, spec.Workflow, deletedSpec.Workflow) + assert.Equal(t, spec.Config, deletedSpec.Config) + assert.Equal(t, spec.Status, deletedSpec.Status) + assert.Equal(t, spec.BinaryURL, deletedSpec.BinaryURL) + assert.Equal(t, spec.ConfigURL, deletedSpec.ConfigURL) + assert.Equal(t, spec.SpecType, deletedSpec.SpecType) + assert.Equal(t, int64(1717171717), deletedSpec.RegisteredAt) + assert.Equal(t, "ContractWorkflowSource", deletedSpec.Source) // Verify the record is deleted from the database var dbSpec job.WorkflowSpec @@ -204,16 +223,16 @@ func Test_DeleteWorkflowSpec(t *testing.T) { require.Equal(t, sql.ErrNoRows, err) }) - t.Run("fails if no workflow spec exists", func(t *testing.T) { + t.Run("returns false for non-existent workflow spec", func(t *testing.T) { t.Parallel() db := pgtest.NewSqlxDB(t) ctx := t.Context() lggr := logger.TestLogger(t) orm := &orm{ds: db, lggr: lggr} - err := orm.DeleteWorkflowSpec(ctx, "non-existent-workflow-id") - require.Error(t, err) - require.Equal(t, sql.ErrNoRows, err) + deletedSpec, err := orm.DeleteWorkflowSpec(ctx, "non-existent-workflow-id") + require.NoError(t, err) + assert.Nil(t, deletedSpec) }) } @@ -262,6 +281,99 @@ func Test_DeleteWorkflowSpecs(t *testing.T) { require.NoError(t, orm.DeleteWorkflowSpecs(ctx, []string{})) } +func Test_PauseWorkflowSpec(t *testing.T) { + t.Parallel() + t.Run("tombstones the row: status paused, artifacts cleared, registered_at preserved", func(t *testing.T) { + t.Parallel() + db := pgtest.NewSqlxDB(t) + ctx := t.Context() + lggr := logger.TestLogger(t) + orm := &orm{ds: db, lggr: lggr} + + _, err := orm.UpsertWorkflowSpec(ctx, &job.WorkflowSpec{ + Workflow: "test_workflow", + Config: "test_config", + WorkflowID: "cid-123", + WorkflowOwner: "owner-123", + WorkflowName: "Test Workflow", + Status: job.WorkflowSpecStatusActive, + CreatedAt: time.Now(), + SpecType: job.WASMFile, + RegisteredAt: 1717171717, + }) + require.NoError(t, err) + + require.NoError(t, orm.PauseWorkflowSpec(ctx, "cid-123")) + + paused, err := orm.GetWorkflowSpec(ctx, "cid-123") + require.NoError(t, err) + assert.Equal(t, job.WorkflowSpecStatusPaused, paused.Status) + assert.Empty(t, paused.Workflow, "artifact payload must be cleared") + assert.Empty(t, paused.Config, "config payload must be cleared") + assert.Equal(t, int64(1717171717), paused.RegisteredAt, "registration generation must survive the tombstone") + assert.Equal(t, "owner-123", paused.WorkflowOwner) + + // Idempotent: pausing again is a no-op. + require.NoError(t, orm.PauseWorkflowSpec(ctx, "cid-123")) + }) + + t.Run("pausing an absent row is a no-op", func(t *testing.T) { + t.Parallel() + db := pgtest.NewSqlxDB(t) + lggr := logger.TestLogger(t) + orm := &orm{ds: db, lggr: lggr} + + require.NoError(t, orm.PauseWorkflowSpec(t.Context(), "non-existent-workflow-id")) + }) +} + +func Test_ListWorkflowSpecs(t *testing.T) { + t.Parallel() + db := pgtest.NewSqlxDB(t) + ctx := t.Context() + lggr := logger.TestLogger(t) + orm := &orm{ds: db, lggr: lggr} + + want := map[string]struct { + owner string + registeredAt int64 + source string + }{ + "wf-1": {"owner-1", 100, "ContractWorkflowSource"}, + "wf-2": {"owner-2", 200, "GRPCWorkflowSource"}, + "wf-3": {"owner-3", 0, ""}, // pre-migration row: registered_at and source defaulted + } + for id, w := range want { + _, err := orm.UpsertWorkflowSpec(ctx, &job.WorkflowSpec{ + Workflow: "binary", + Config: "config", + WorkflowID: id, + WorkflowOwner: w.owner, + WorkflowName: "wf", + Status: job.WorkflowSpecStatusActive, + CreatedAt: time.Now(), + SpecType: job.WASMFile, + RegisteredAt: w.registeredAt, + Source: w.source, + }) + require.NoError(t, err) + } + + specs, err := orm.ListWorkflowSpecs(ctx) + require.NoError(t, err) + require.Len(t, specs, len(want)) + for _, spec := range specs { + w, ok := want[spec.WorkflowID] + require.True(t, ok, "unexpected workflow_id %q", spec.WorkflowID) + assert.Equal(t, w.owner, spec.WorkflowOwner) + assert.Equal(t, w.registeredAt, spec.RegisteredAt) + assert.Equal(t, w.source, spec.Source) + // The projection deliberately omits the heavy artifact columns. + assert.Empty(t, spec.Workflow) + assert.Empty(t, spec.Config) + } +} + func Test_GetWorkflowSpec(t *testing.T) { t.Parallel() t.Run("gets a workflow spec by ID", func(t *testing.T) { @@ -293,7 +405,7 @@ func Test_GetWorkflowSpec(t *testing.T) { require.NoError(t, err) require.Equal(t, spec.Workflow, dbSpec.Workflow) - err = orm.DeleteWorkflowSpec(ctx, spec.WorkflowID) + _, err = orm.DeleteWorkflowSpec(ctx, spec.WorkflowID) require.NoError(t, err) }) diff --git a/core/services/workflows/artifacts/v2/store.go b/core/services/workflows/artifacts/v2/store.go index 0eeb694cc6a..deec21692ca 100644 --- a/core/services/workflows/artifacts/v2/store.go +++ b/core/services/workflows/artifacts/v2/store.go @@ -3,7 +3,6 @@ package v2 import ( "context" "crypto/sha256" - "database/sql" "encoding/base64" "encoding/hex" "errors" @@ -234,22 +233,24 @@ func (h *Store) GetWorkflowSpec(ctx context.Context, workflowID string) (*job.Wo return spec, err } +// ListWorkflowSpecs returns the persisted workflow specs (identity columns +// only). It backs the orphan sweep and the metering snapshot path. +func (h *Store) ListWorkflowSpecs(ctx context.Context) ([]*job.WorkflowSpec, error) { + return h.orm.ListWorkflowSpecs(ctx) +} + func (h *Store) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) (int64, error) { return h.orm.UpsertWorkflowSpec(ctx, spec) } -// DeleteWorkflowArtifacts removes the workflow spec from the database. If not found, returns nil. -func (h *Store) DeleteWorkflowArtifacts(ctx context.Context, workflowID string) error { - err := h.orm.DeleteWorkflowSpec(ctx, workflowID) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - h.lggr.Warnw("failed to delete workflow spec: not found", "workflowID", workflowID) - return nil - } - return fmt.Errorf("failed to delete workflow spec: %w", err) - } +// DeleteWorkflowArtifacts removes the workflow spec from the database. If not +// found, returns (nil, nil). +func (h *Store) DeleteWorkflowArtifacts(ctx context.Context, workflowID string) (*job.WorkflowSpec, error) { + return h.orm.DeleteWorkflowSpec(ctx, workflowID) +} - return nil +func (h *Store) PauseWorkflowArtifacts(ctx context.Context, workflowID string) error { + return h.orm.PauseWorkflowSpec(ctx, workflowID) } func (h *Store) DeleteWorkflowArtifactsBatch(ctx context.Context, workflowIDs []string) error { diff --git a/core/services/workflows/artifacts/v2/store_test.go b/core/services/workflows/artifacts/v2/store_test.go index 34fa6aabacd..c6eb7c0989f 100644 --- a/core/services/workflows/artifacts/v2/store_test.go +++ b/core/services/workflows/artifacts/v2/store_test.go @@ -85,7 +85,7 @@ func Test_Store_DeleteWorkflowArtifacts(t *testing.T) { require.NoError(t, err) // Delete the workflow artifacts by ID - err = h.DeleteWorkflowArtifacts(t.Context(), workflowID) + _, err = h.DeleteWorkflowArtifacts(t.Context(), workflowID) require.NoError(t, err) // Check that the workflow no longer exists diff --git a/core/services/workflows/syncer/v2/engine_registry.go b/core/services/workflows/syncer/v2/engine_registry.go index d1b3c90b938..82eba6d951d 100644 --- a/core/services/workflows/syncer/v2/engine_registry.go +++ b/core/services/workflows/syncer/v2/engine_registry.go @@ -23,7 +23,7 @@ type ServiceWithMetadata struct { services.Service } -// engineEntry holds the engine and its associated source for internal storage +// engineEntry holds the engine and its associated source for internal storage. type engineEntry struct { engine services.Service source string diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 36a4a6748cd..756ec5d5b8f 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -2,11 +2,13 @@ package v2 import ( "context" + "database/sql" "encoding/hex" "errors" "fmt" "io" "maps" + "strconv" "sync" "time" @@ -16,9 +18,11 @@ import ( "go.opentelemetry.io/otel/trace/noop" "github.com/smartcontractkit/chainlink-common/keystore/corekeys/workflowkey" + commoncap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-common/pkg/contexts" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" @@ -87,8 +91,14 @@ type eventHandler struct { workflowEncryptionKey workflowkey.Key workflowDonSubscriber capabilities.DonSubscriber billingClient metering.BillingClient - orgResolver orgresolver.OrgResolver - secretsFetcher v2.SecretsFetcher + + // specMeter owns all metering for durable workflow-spec storage (the + // ResourceManager lifecycle, identity, snapshots). Nil when metering is + // disabled: its handler-facing methods are nil-receiver safe no-ops. + specMeter *SpecMeter + + orgResolver orgresolver.OrgResolver + secretsFetcher v2.SecretsFetcher // localSecretOverrides is keyed by owner address; values are secret id -> secret value localSecretOverrides map[string]map[string]string @@ -152,6 +162,17 @@ func WithBillingClient(client metering.BillingClient) func(*eventHandler) { } } +// WithSpecMeter supplies the SpecMeter that emits metering.v1.MeterRecord +// events for the workflow_specs_v2 storage lifecycle. The handler runs it as a +// sub-service and reports storage transitions through EmitSpecDelta; all other +// metering concerns (ResourceManager lifecycle, identity, snapshots) live on +// the meter. A nil meter (metering disabled) is a valid no-op. +func WithSpecMeter(sm *SpecMeter) func(*eventHandler) { + return func(e *eventHandler) { + e.specMeter = sm + } +} + func WithShardExecutionGuard(client shardorchestrator.ClientInterface, shardingEnabled bool, shardID uint32) func(*eventHandler) { return func(e *eventHandler) { e.shardOrchestratorClient = client @@ -252,8 +273,10 @@ func WithModuleEngineVersion(v string) func(*eventHandler) { type WorkflowArtifactsStore interface { FetchWorkflowArtifacts(ctx context.Context, workflowID, binaryIdentifier, configIdentifier string) ([]byte, []byte, error) GetWorkflowSpec(ctx context.Context, workflowID string) (*job.WorkflowSpec, error) + ListWorkflowSpecs(ctx context.Context) ([]*job.WorkflowSpec, error) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) (int64, error) - DeleteWorkflowArtifacts(ctx context.Context, workflowID string) error + DeleteWorkflowArtifacts(ctx context.Context, workflowID string) (*job.WorkflowSpec, error) + PauseWorkflowArtifacts(ctx context.Context, workflowID string) error DeleteWorkflowArtifactsBatch(ctx context.Context, workflowIDs []string) error } @@ -305,7 +328,8 @@ func NewEventHandler( workflowArtifactsStore: workflowArtifacts, workflowEncryptionKey: workflowEncryptionKey, workflowDonSubscriber: workflowDonSubscriber, - tracer: noop.NewTracerProvider().Tracer(""), // default to noop, enable via WithDebugMode + // default, enable via WithDebugMode + tracer: noop.NewTracerProvider().Tracer(""), } metricsInst, metricsErr := newMetrics() if metricsErr != nil { @@ -319,12 +343,17 @@ func NewEventHandler( eh.Service, eh.eng = services.Config{ Name: "EventHandler", - // The workflow store is started and stopped alongside the handler. + // The workflow store and the spec meter are started and stopped + // alongside the handler. NewSubServices: func(logger.Logger) []services.Service { - if eh.workflowStore == nil { - return nil + var subs []services.Service + if eh.workflowStore != nil { + subs = append(subs, eh.workflowStore) + } + if eh.specMeter != nil { + subs = append(subs, eh.specMeter) } - return []services.Service{eh.workflowStore} + return subs }, Start: eh.start, Close: eh.close, @@ -333,18 +362,30 @@ func NewEventHandler( return eh, nil } -func (h *eventHandler) start(_ context.Context) error { +func (h *eventHandler) start(context.Context) error { if h.moduleLRU != nil { h.moduleLRU.Start() } return nil } +// SetWorkflowDon supplies the launcher-resolved workflow DON identity for +// metering. Called by the registry after WaitForDon, before any event is +// dispatched; the value is static for the life of the node. +func (h *eventHandler) SetWorkflowDon(don commoncap.DON) { + h.specMeter.SetWorkflowDon(don) +} + func (h *eventHandler) close() error { if h.moduleLRU != nil { h.moduleLRU.Close() } es := h.engineRegistry.PopAll() + // No metering is emitted on close: meter records anchor on workflow-spec + // storage transitions, not engine lifecycle, so stopping an engine at + // shutdown leaves the persisted spec (and therefore its metered level) + // untouched. A spec that is genuinely released stops being snapshotted + // (the spec meter sub-service unregisters itself after this hook runs). cs := make([]io.Closer, 0, len(es)+1) cs = append(cs, h.engineLimiters) for _, e := range es { @@ -443,7 +484,7 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { var err error defer func() { - if err2 := events.EmitWorkflowStatusChangedEventV2(ctx, cma.Labels(), toCommonHead(event.Head), string(event.Name), payload.BinaryURL, payload.ConfigURL, err); err2 != nil { + if err2 := events.EmitWorkflowStatusChangedEventV2(ctx, cma.Labels(), toCommonHead(event.Head), string(event.Name), payload.BinaryURL, payload.ConfigURL, customerFacingError(err)); err2 != nil { h.lggr.Errorf("failed to emit status changed event: %+v", err2) } }() @@ -468,21 +509,20 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { wfID := payload.WorkflowID.Hex() - // Get workflow spec from database to get owner and name info for organization lookup - // Alternative: wire through workflowOwner into the Event, but that requires a lot more surgery - spec, err := h.workflowArtifactsStore.GetWorkflowSpec(ctx, wfID) + // Get workflow spec from database to get owner and name info for organization lookup, + // and to check if the spec exists to determine if a MeterRecord is warranted. var wfOwner, wfName, orgID string - if err != nil { - // Workflow spec not found, proceed with deletion but without event metadata - h.lggr.Warnw("Workflow spec not found during deletion, proceeding without org info", "workflowID", wfID, "error", err) - } else { + if spec, gerr := h.workflowArtifactsStore.GetWorkflowSpec(ctx, wfID); gerr == nil && spec != nil { wfOwner = spec.WorkflowOwner wfName = spec.WorkflowName - if wfOwner != "" { - orgID, err = h.fetchOrganizationID(ctx, wfOwner) - if err != nil { - h.lggr.Warnw("Failed to get organization from linking service", "workflowOwner", wfOwner, "error", err) - } + } else if gerr != nil && !errors.Is(gerr, sql.ErrNoRows) { + h.lggr.Errorw("failed to read workflow spec during deletion, proceeding without metadata", "workflowID", wfID, "error", gerr) + } + if wfOwner != "" { + if resolvedOrgID, orgErr := h.fetchOrganizationID(ctx, wfOwner); orgErr != nil { + h.lggr.Warnw("Failed to get organization from linking service", "workflowOwner", wfOwner, "error", orgErr) + } else { + orgID = resolvedOrgID } } ctx = contexts.WithCRE(ctx, contexts.CRE{Org: orgID, Owner: wfOwner, Workflow: wfID}) @@ -506,7 +546,7 @@ func (h *eventHandler) Handle(ctx context.Context, event Event) error { } }() - if herr = h.workflowDeletedEvent(ctx, payload); herr != nil { + if herr = h.workflowDeletedEvent(ctx, payload, wfOwner); herr != nil { if errors.Is(herr, ErrDrainInProgress) { logCustMsg(ctx, cma, fmt.Sprintf("workflow deletion deferred: %v", herr), h.lggr) } else { @@ -558,13 +598,18 @@ func (h *eventHandler) workflowRegisteredEvent( // - existing registration that has been updated with a new status spec, err := h.workflowArtifactsStore.GetWorkflowSpec(ctx, payload.WorkflowID.Hex()) switch { - case err != nil: + case errors.Is(err, sql.ErrNoRows): newSpec, innerErr := h.createWorkflowSpec(ctx, payload) if innerErr != nil { return innerErr } + h.specMeter.EmitSpecDelta(ctx, 1, payload.WorkflowID.Hex(), hex.EncodeToString(payload.WorkflowOwner), + resourcemanager.EventID("workflow-spec-register", payload.WorkflowID.Hex(), strconv.FormatUint(payload.CreatedAt, 10))) + spec = newSpec + case err != nil: + return fmt.Errorf("failed to get workflow spec: %w", err) case spec.WorkflowID != payload.WorkflowID.Hex() || spec.WorkflowOwner != hex.EncodeToString(payload.WorkflowOwner) || spec.WorkflowName != payload.WorkflowName: @@ -572,13 +617,43 @@ func (h *eventHandler) workflowRegisteredEvent( if innerErr != nil { return innerErr } + // A different spec's artifacts were persisted under this key: the newly + // stored spec is a fresh durable resource, so emit a +1 delta. + h.specMeter.EmitSpecDelta(ctx, 1, payload.WorkflowID.Hex(), hex.EncodeToString(payload.WorkflowOwner), + resourcemanager.EventID("workflow-spec-register", payload.WorkflowID.Hex(), strconv.FormatUint(payload.CreatedAt, 10))) + spec = newSpec + case status == job.WorkflowSpecStatusActive && spec.Workflow == "": + // Activating a paused tombstone: the artifact payload was cleared at + // pause time, so refetch and re-persist it. Level-neutral for metering + // (the registration generation was never released), so no delta. + newSpec, innerErr := h.createWorkflowSpec(ctx, payload) + if innerErr != nil { + return innerErr + } spec = newSpec case spec.Status != status: spec.Status = status if _, innerErr := h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, spec); innerErr != nil { return fmt.Errorf("failed to update workflow spec: %w", innerErr) } + // Status-only flip: no artifact-persistence transition, no delta. + } + + // backfill registered_at, source when necessary + backfill := false + if spec.RegisteredAt == 0 && payload.CreatedAt > 0 { + spec.RegisteredAt = int64(payload.CreatedAt) //nolint:gosec // G115: CreatedAt is a timestamp that cannot overflow int64 + backfill = true + } + if spec.Source == "" && payload.Source != "" { + spec.Source = payload.Source + backfill = true + } + if backfill { + if _, err := h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, spec); err != nil { + h.lggr.Warnw("failed to backfill registered_at/source", "workflowID", spec.WorkflowID, "err", err) + } } // Next, let's synchronize the engine. @@ -669,6 +744,8 @@ func (h *eventHandler) createWorkflowSpec(ctx context.Context, payload WorkflowR BinaryURL: payload.BinaryURL, ConfigURL: payload.ConfigURL, Attributes: payload.Attributes, + RegisteredAt: int64(payload.CreatedAt), //nolint:gosec // G115: CreatedAt is a timestamp that cannot overflow int64 + Source: payload.Source, } if _, err = h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, entry); err != nil { @@ -793,32 +870,17 @@ func (h *eventHandler) createEngineModule( return engineModule } -// workflowPausedEvent handles the WorkflowPausedEvent event type. This method must remain idempotent. -func (h *eventHandler) workflowPausedEvent( - ctx context.Context, - payload WorkflowPausedEvent, -) error { - return h.workflowDeletedEvent(ctx, WorkflowDeletedEvent{WorkflowID: payload.WorkflowID}) -} - -// workflowDeletedEvent handles the WorkflowDeletedEvent event type. This method must remain idempotent. -func (h *eventHandler) workflowDeletedEvent( - ctx context.Context, - payload WorkflowDeletedEvent, -) error { - // The order in the handler is slightly different to the order in `tryEngineCleanup`. - // This is because the engine requires its corresponding DB record to be present to be successfully - // closed. - // At the same time, popping the engine should occur last to allow deletes to be retried if any of the - // prior steps fail. - workflowID := payload.WorkflowID.Hex() - e, ok := h.engineRegistry.Get(payload.WorkflowID) +// stopEngine drains (returning ErrDrainInProgress while executions remain) and +// closes the engine for workflowID if one is registered. +// Returns the drainable handle (nil-able) for drain-completed metrics. +func (h *eventHandler) stopEngine(ctx context.Context, workflowID types.WorkflowID) (DrainableService, error) { + e, ok := h.engineRegistry.Get(workflowID) var drainable DrainableService - var isDrainable bool if ok { + var isDrainable bool if drainable, isDrainable = e.Service.(DrainableService); isDrainable { if started := drainable.Drain(); started { - h.lggr.Infow("initiated drain for workflow engine", "workflowID", workflowID) + h.lggr.Infow("initiated drain for workflow in workflow engine", "workflowID", workflowID.String()) if h.metrics != nil { h.metrics.incrementDrainStarted(ctx) } @@ -829,32 +891,92 @@ func (h *eventHandler) workflowDeletedEvent( h.metrics.incrementDeleteDeferred(ctx, "drain_in_progress") } h.lggr.Infow("workflow deletion deferred: active executions still running", - "workflowID", workflowID, + "workflowID", workflowID.String(), "activeExecutions", active) - return fmt.Errorf("%w: %d active executions still running", ErrDrainInProgress, active) + return nil, fmt.Errorf("%w: %d active executions still running", ErrDrainInProgress, active) } } if innerErr := e.Close(); innerErr != nil && !errors.Is(innerErr, services.ErrAlreadyStopped) { - return fmt.Errorf("failed to close workflow engine: %w", innerErr) + return nil, fmt.Errorf("failed to close workflow engine: %w", innerErr) } } + return drainable, nil +} - if err := h.workflowArtifactsStore.DeleteWorkflowArtifacts(ctx, payload.WorkflowID.Hex()); err != nil { +// releaseSpecStorage deletes the persisted spec row and, iff a row was +// actually removed, emits the -1 generation delta. RowsAffected is the +// exactly-once gate: redeliveries observe no row and emit nothing; a transient +// DELETE error returns before emission so the retry emits when the delete +// lands. The event_id is derived AFTER the delete from the row's persisted +// registered_at, so all nodes emit the identical id. Module-cache cleanup +// always runs. owner may be empty; org resolution is fail-open. +func (h *eventHandler) releaseSpecStorage(ctx context.Context, workflowID, owner string) error { + deletedSpec, err := h.workflowArtifactsStore.DeleteWorkflowArtifacts(ctx, workflowID) + if err != nil { return fmt.Errorf("failed to delete workflow artifacts: %w", err) } + if deletedSpec != nil { + parts := []string{workflowID} + if deletedSpec.RegisteredAt > 0 { + parts = append(parts, strconv.FormatInt(deletedSpec.RegisteredAt, 10)) + } + h.specMeter.EmitSpecDelta(ctx, -1, workflowID, owner, resourcemanager.EventID("workflow-spec-delete", parts...)) + } + h.cleanupModuleCache(workflowID) + return nil +} - h.cleanupModuleCache(payload.WorkflowID.Hex()) +// workflowPausedEvent handles WorkflowPaused. Idempotent. +// +// Pause is level-neutral for metering: the spec row survives as a tombstone +// (status=paused, artifact payload cleared to free storage) so the +// registration generation stays metered until deletion — mirroring the +// on-chain registry, which retains the paused registration. Emitting per-cycle +// pause/activate deltas is impossible without drift: no DON-consistent +// per-occurrence discriminator exists, so a -1 here would force the +// re-activation +1 to reuse the register event_id and be dropped by consumer +// dedup. Deltas therefore fire only at generation boundaries (register/delete). +func (h *eventHandler) workflowPausedEvent( + ctx context.Context, + payload WorkflowPausedEvent, +) error { + workflowID := payload.WorkflowID.Hex() + if _, err := h.stopEngine(ctx, payload.WorkflowID); err != nil { + return err + } + if err := h.workflowArtifactsStore.PauseWorkflowArtifacts(ctx, workflowID); err != nil { + return fmt.Errorf("failed to pause workflow artifacts: %w", err) + } + h.cleanupModuleCache(workflowID) + if _, err := h.engineRegistry.Pop(payload.WorkflowID); err != nil && !errors.Is(err, ErrNotFound) { + return err + } + return nil +} - _, err := h.engineRegistry.Pop(payload.WorkflowID) +// workflowDeletedEvent handles the WorkflowDeletedEvent event type. This method must remain idempotent. +func (h *eventHandler) workflowDeletedEvent( + ctx context.Context, + payload WorkflowDeletedEvent, + owner string, +) error { + workflowID := payload.WorkflowID.Hex() + drainable, err := h.stopEngine(ctx, payload.WorkflowID) + if err != nil { + return err + } + if err = h.releaseSpecStorage(ctx, workflowID, owner); err != nil { + return err + } + _, err = h.engineRegistry.Pop(payload.WorkflowID) if errors.Is(err, ErrNotFound) { return nil } if err != nil { return err } - - if isDrainable { + if drainable != nil { startedAt, exists := drainable.DrainStartedAt() if exists && h.metrics != nil { h.metrics.recordDrainCompleted(ctx, time.Since(startedAt)) @@ -863,6 +985,11 @@ func (h *eventHandler) workflowDeletedEvent( return nil } +// ListWorkflowSpecs backs the orphan sweep. +func (h *eventHandler) ListWorkflowSpecs(ctx context.Context) ([]*job.WorkflowSpec, error) { + return h.workflowArtifactsStore.ListWorkflowSpecs(ctx) +} + // tryEngineCleanup attempts to stop the workflow engine for the given workflow ID. Does nothing if the // workflow engine is not running. func (h *eventHandler) tryEngineCleanup(workflowID types.WorkflowID) error { diff --git a/core/services/workflows/syncer/v2/handler_metering_test.go b/core/services/workflows/syncer/v2/handler_metering_test.go new file mode 100644 index 00000000000..9b24f0a0d82 --- /dev/null +++ b/core/services/workflows/syncer/v2/handler_metering_test.go @@ -0,0 +1,815 @@ +package v2 + +import ( + "context" + "errors" + "math/big" + "strconv" + "sync" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/smartcontractkit/chainlink-common/keystore/corekeys/workflowkey" + "github.com/smartcontractkit/chainlink-common/pkg/custmsg" + commonlogger "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" + "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" + "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + pkgworkflows "github.com/smartcontractkit/chainlink-common/pkg/workflows" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" + "github.com/smartcontractkit/chainlink/v2/core/capabilities" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/confidentialrelay" + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/job" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/ratelimiter" + workflowstore "github.com/smartcontractkit/chainlink/v2/core/services/workflows/store" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncerlimiter" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" + v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" +) + +// meteringwfID is the workflow ID that GenerateWorkflowID produces from the +// stub store's FetchWorkflowArtifacts return values (binary="binary", +// config="config") with owner 0xaabbccdd and name "wf-name". Tests that use +// persistUpserts and register as Active must use this ID so tryEngineCreate's +// validation passes. +var meteringwfID = func() types.WorkflowID { + owner := []byte{0xaa, 0xbb, 0xcc, 0xdd} + wfIDBytes, _ := pkgworkflows.GenerateWorkflowID(owner, "wf-name", []byte("binary"), []byte("config"), "") + return types.WorkflowID(wfIDBytes) +}() + +// recordingEmitter is a fake resourcemanager.Emitter that decodes and stores +// every emitted MeterRecord. If err is set, Emit fails instead. +type recordingEmitter struct { + mu sync.Mutex + err error + records []*meteringpb.MeterRecord +} + +func (r *recordingEmitter) Emit(_ context.Context, body []byte, _ ...any) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.err != nil { + return r.err + } + var record meteringpb.MeterRecord + if err := proto.Unmarshal(body, &record); err != nil { + return err + } + r.records = append(r.records, &record) + return nil +} + +func (r *recordingEmitter) Records() []*meteringpb.MeterRecord { + r.mu.Lock() + defer r.mu.Unlock() + records := make([]*meteringpb.MeterRecord, len(r.records)) + copy(records, r.records) + return records +} + +func newMeteringResourceManager(t *testing.T, enabled bool, emitter resourcemanager.Emitter) *resourcemanager.ResourceManager { + t.Helper() + return resourcemanager.NewResourceManager(commonlogger.Test(t), resourcemanager.ResourceManagerConfig{ + MeterRecordsEnabled: enabled, + Emitter: emitter, + }) +} + +// recordingSnapshotEmitter is a fake resourcemanager.Emitter that decodes and +// stores every emitted MeterSnapshot (one per active resource). Used to drive +// the Meterable snapshot path. +type recordingSnapshotEmitter struct { + mu sync.Mutex + snapshots []*meteringpb.MeterSnapshot +} + +func (r *recordingSnapshotEmitter) Emit(_ context.Context, body []byte, _ ...any) error { + r.mu.Lock() + defer r.mu.Unlock() + var snapshot meteringpb.MeterSnapshot + if err := proto.Unmarshal(body, &snapshot); err != nil { + return err + } + r.snapshots = append(r.snapshots, &snapshot) + return nil +} + +func (r *recordingSnapshotEmitter) Snapshots() []*meteringpb.MeterSnapshot { + r.mu.Lock() + defer r.mu.Unlock() + snapshots := make([]*meteringpb.MeterSnapshot, len(r.snapshots)) + copy(snapshots, r.snapshots) + return snapshots +} + +// newTestSpecMeter builds a SpecMeter over rm and artifactsStore, or returns +// nil (the no-op meter) when rm is nil. +func newTestSpecMeter(t *testing.T, rm *resourcemanager.ResourceManager, artifactsStore WorkflowArtifactsStore, org orgresolver.OrgResolver) *SpecMeter { + t.Helper() + if rm == nil { + return nil + } + sm, err := NewSpecMeter(logger.TestLogger(t), rm, resourcemanager.ResourceIdentity{}, artifactsStore, org) + require.NoError(t, err) + return sm +} + +func newMeteringTestHandler(t *testing.T, artifactsStore WorkflowArtifactsStore, rm *resourcemanager.ResourceManager) *eventHandler { + t.Helper() + lggr := logger.TestLogger(t) + lf := limits.Factory{Logger: lggr} + registry := capabilities.NewRegistry(lggr) + registry.SetLocalRegistry(&capabilities.TestMetadataRegistry{}) + limiters, err := v2.NewLimiters(lf, nil) + require.NoError(t, err) + rl, err := ratelimiter.NewRateLimiter(rlConfig) + require.NoError(t, err) + workflowLimits, err := syncerlimiter.NewWorkflowLimits(lggr, wlConfig, lf) + require.NoError(t, err) + + h, err := NewEventHandler( + lggr, + workflowstore.NewInMemoryStore(lggr, clockwork.NewFakeClock()), + nil, + true, + registry, + &confidentialrelay.ExecutionHandlers{}, + NewEngineRegistry(), + custmsg.NewLabeler(), + limiters, + nil, + rl, + workflowLimits, + artifactsStore, + workflowkey.MustNewXXXTestingOnly(big.NewInt(1)), + &testDonNotifier{}, + WithSpecMeter(newTestSpecMeter(t, rm, artifactsStore, nil)), + WithEngineFactoryFn(mockEngineFactory), + ) + require.NoError(t, err) + return h +} + +// requireSpecDelta asserts that record is a workflow-syncer-v2 spec delta: a +// METER_ACTION_UPDATE carrying a single utilization with the given signed value +// and resource_id (= workflow_id). event_id must be the deterministic, +// cross-node-identical id wantEventID (an opaque string that is never +// format-validated). +func requireSpecDelta(t *testing.T, record *meteringpb.MeterRecord, value, workflowID, wantEventID string) { + t.Helper() + require.NotNil(t, record.Identity) + assert.Equal(t, "workflow-syncer-v2", record.Identity.Service) + assert.Equal(t, "workflow_specs_v2", record.Identity.ResourcePool) + // Every syncer meter record is a signed level delta (UPDATE), never a + // RESERVE/RELEASE lifecycle edge. + assert.Equal(t, meteringpb.MeterAction_METER_ACTION_UPDATE, record.Action) + assert.NotNil(t, record.Timestamp) + require.Len(t, record.Utilizations, 1) + util := record.Utilizations[0] + assert.Equal(t, value, util.Value) + assert.Equal(t, "operations", util.ResourceType) + // resource_id = workflow_id for the syncer (no shared physical resource). + assert.Equal(t, workflowID, util.ResourceId) + // event_id is the deterministic reconciliation-derived id, identical on every + // workflow-DON node; it is an opaque string with no validated format. + assert.Equal(t, wantEventID, util.EventId) +} + +func Test_meterRecords(t *testing.T) { + t.Parallel() + + wfOwner := []byte{0xaa, 0xbb, 0xcc, 0xdd} + + t.Run("registered event persisting a new spec emits +1", func(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, true, emitter)) + + wfID := types.WorkflowID{1} + err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + }) + require.NoError(t, err) + + records := emitter.Records() + require.Len(t, records, 1) + requireSpecDelta(t, records[0], "1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-register", wfID.Hex(), "0")) + }) + + t.Run("reprocessed registered event emits the IDENTICAL event_id (cross-node dedup)", func(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + // The stub never returns a stored spec, so each call replays the + // new-spec path exactly as a reprocessed event (or a second node + // reconciling the same on-chain event) would. + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, true, emitter)) + + event := WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: types.WorkflowID{2}, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + CreatedAt: 123, + } + require.NoError(t, h.workflowRegisteredEvent(t.Context(), event)) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), event)) + + records := emitter.Records() + require.Len(t, records, 2) + require.Len(t, records[0].Utilizations, 1) + require.Len(t, records[1].Utilizations, 1) + // The same reconciliation event MUST yield the identical event_id, so the + // billing consumer dedups reprocessing and cross-node duplicates. It is + // derived deterministically from the on-chain workflowID + CreatedAt. + want := resourcemanager.EventID("workflow-spec-register", types.WorkflowID{2}.Hex(), "123") + assert.Equal(t, want, records[0].Utilizations[0].GetEventId()) + assert.Equal(t, records[0].Utilizations[0].GetEventId(), records[1].Utilizations[0].GetEventId()) + }) + + t.Run("activating an already-stored spec emits nothing (status-only update)", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{3} + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + WorkflowName: "wf-name", + }, + }, newMeteringResourceManager(t, true, emitter)) + + // Same workflow ID already stored, only the status changes: the spec + // stays stored, so no artifact-persistence transition and no delta. + err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + }) + require.NoError(t, err) + + assert.Empty(t, emitter.Records()) + }) + + t.Run("paused event emits nothing", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{4} + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + }, + }, newMeteringResourceManager(t, true, emitter)) + + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + + // Pause routes through the delete path, but the spec's release is + // realized by its absence from subsequent snapshots, not a delta. + assert.Empty(t, emitter.Records()) + }) + + t.Run("deleted event emits -1 after artifacts are deleted", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{5} + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + }, + }, newMeteringResourceManager(t, true, emitter)) + + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) + + records := emitter.Records() + require.Len(t, records, 1) + requireSpecDelta(t, records[0], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex())) + }) + + t.Run("no record when persisting a new spec fails", func(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{upsertErr: assert.AnError}, newMeteringResourceManager(t, true, emitter)) + + err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: types.WorkflowID{6}, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + }) + require.ErrorIs(t, err, assert.AnError) + assert.Empty(t, emitter.Records()) + }) + + t.Run("no record when deleting artifacts fails", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{7} + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + }, + deleteErr: assert.AnError, + }, newMeteringResourceManager(t, true, emitter)) + + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd") + require.ErrorIs(t, err, assert.AnError) + assert.Empty(t, emitter.Records()) + }) + + t.Run("no record while a delete is deferred by drain; exactly one on the successful retry", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{8} + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + }, + }, newMeteringResourceManager(t, true, emitter)) + + drainable := &mockDrainableEngine{} + drainable.activeExecutions.Store(1) + require.NoError(t, h.engineRegistry.Add(wfID, "test-source", drainable)) + + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd") + require.ErrorIs(t, err, ErrDrainInProgress) + assert.Empty(t, emitter.Records()) + + drainable.activeExecutions.Store(0) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) + + records := emitter.Records() + require.Len(t, records, 1) + requireSpecDelta(t, records[0], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex())) + }) + + t.Run("emit failure never fails event handling", func(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{9} + emitter := &recordingEmitter{err: errors.New("beholder unavailable")} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + }, + }, newMeteringResourceManager(t, true, emitter)) + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + })) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) + assert.Empty(t, emitter.Records()) + }) + + t.Run("disabled resource manager emits nothing", func(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, false, emitter)) + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: types.WorkflowID{10}, + WorkflowOwner: wfOwner, + WorkflowName: "wf-name", + })) + assert.Empty(t, emitter.Records()) + }) + + t.Run("snapshot emits one MeterSnapshot per persisted spec", func(t *testing.T) { + t.Parallel() + emitter := &recordingSnapshotEmitter{} + clock := clockwork.NewFakeClockAt(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + rm := resourcemanager.NewResourceManager(commonlogger.Test(t), resourcemanager.ResourceManagerConfig{ + MeterRecordsEnabled: true, + MeterSnapshotsEnabled: true, + Emitter: emitter, + SnapshotInterval: time.Minute, + Clock: clock, + }) + + // The snapshot enumerates persisted specs from the store, not running + // engines, so paused-but-stored specs are still accounted for. + wfID1 := types.WorkflowID{20} + wfID2 := types.WorkflowID{21} + store := &stubWorkflowArtifactsStore{ + specs: []*job.WorkflowSpec{ + {WorkflowID: wfID1.Hex(), WorkflowOwner: "aabbccdd"}, + {WorkflowID: wfID2.Hex(), WorkflowOwner: "aabbccdd"}, + }, + } + h := newMeteringTestHandler(t, store, rm) + unregister := rm.Register(h.specMeter) + t.Cleanup(unregister) + + servicetest.Run(t, rm) + require.NoError(t, clock.BlockUntilContext(t.Context(), 1)) + clock.Advance(time.Minute) + + require.Eventually(t, func() bool { + return len(emitter.Snapshots()) == 2 + }, time.Second, time.Millisecond) + snapshots := emitter.Snapshots() + require.Len(t, snapshots, 2) + + byWorkflowID := map[string]*meteringpb.MeterSnapshot{} + for _, snap := range snapshots { + require.NotNil(t, snap.Identity) + assert.Equal(t, "workflow-syncer-v2", snap.Identity.Service) + assert.Equal(t, "workflow_specs_v2", snap.Identity.ResourcePool) + require.Len(t, snap.Utilization, 1) + assert.Equal(t, "1", snap.Utilization[0].Value) + // resource_id = workflow_id fully identifies the resource; no labels. + byWorkflowID[snap.Utilization[0].ResourceId] = snap + } + require.NotNil(t, byWorkflowID[wfID1.Hex()], "snapshot must contain an entry for the first persisted spec") + require.NotNil(t, byWorkflowID[wfID2.Hex()], "snapshot must contain an entry for the second persisted spec") + }) +} + +// fakeOrgResolver is a no-network OrgResolver that maps workflow owners (hex) +// to organization IDs. It asserts that metering org resolution is exercised +// (previously org was entirely untested). +type fakeOrgResolver struct { + orgs map[string]string +} + +func (f *fakeOrgResolver) Get(_ context.Context, owner string) (string, error) { + if org, ok := f.orgs[owner]; ok { + return org, nil + } + return "", errors.New("owner not found") +} + +func (f *fakeOrgResolver) Start(context.Context) error { return nil } +func (f *fakeOrgResolver) Close() error { return nil } +func (f *fakeOrgResolver) HealthReport() map[string]error { return nil } +func (f *fakeOrgResolver) Ready() error { return nil } +func (f *fakeOrgResolver) Name() string { return "fake-org-resolver" } + +// newMeteringTestHandlerWithOrg is newMeteringTestHandler plus a wired +// OrgResolver, so tests can assert OrgId on emitted records. +func newMeteringTestHandlerWithOrg(t *testing.T, artifactsStore WorkflowArtifactsStore, rm *resourcemanager.ResourceManager, org orgresolver.OrgResolver) *eventHandler { + t.Helper() + lggr := logger.TestLogger(t) + lf := limits.Factory{Logger: lggr} + registry := capabilities.NewRegistry(lggr) + registry.SetLocalRegistry(&capabilities.TestMetadataRegistry{}) + limiters, err := v2.NewLimiters(lf, nil) + require.NoError(t, err) + rl, err := ratelimiter.NewRateLimiter(rlConfig) + require.NoError(t, err) + workflowLimits, err := syncerlimiter.NewWorkflowLimits(lggr, wlConfig, lf) + require.NoError(t, err) + + h, err := NewEventHandler( + lggr, + workflowstore.NewInMemoryStore(lggr, clockwork.NewFakeClock()), + nil, + true, + registry, + &confidentialrelay.ExecutionHandlers{}, + NewEngineRegistry(), + custmsg.NewLabeler(), + limiters, + nil, + rl, + workflowLimits, + artifactsStore, + workflowkey.MustNewXXXTestingOnly(big.NewInt(1)), + &testDonNotifier{}, + WithSpecMeter(newTestSpecMeter(t, rm, artifactsStore, org)), + WithEngineFactoryFn(mockEngineFactory), + WithOrgResolver(org), + ) + require.NoError(t, err) + return h +} + +// redeliveredDeleteSpecs returns a fresh stub seeded with a stored spec and the +// owner the metering tests use, so a delete + redelivered-delete pair can be +// driven through Handle (which pre-reads the spec itself). +func redeliveredDeleteStore() *stubWorkflowArtifactsStore { + return &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: types.WorkflowID{5}.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + }, + } +} + +// a redelivered delete (after the spec was already removed) must NOT emit a +// second -1. DeleteWorkflowArtifacts maps ErrNoRows→nil, so without gating on +// the pre-read the redelivery would double-emit. +func Test_meterRecords_RedeliveredDeleteEmitsExactlyOneDelta(t *testing.T) { + t.Parallel() + wfID := types.WorkflowID{5} + emitter := &recordingEmitter{} + store := redeliveredDeleteStore() + h := newMeteringTestHandler(t, store, newMeteringResourceManager(t, true, emitter)) + + deleteEvent := Event{Name: WorkflowDeleted, Data: WorkflowDeletedEvent{WorkflowID: wfID}} + require.NoError(t, h.Handle(t.Context(), deleteEvent)) + require.NoError(t, h.Handle(t.Context(), deleteEvent)) // redelivery + + records := emitter.Records() + require.Len(t, records, 1, "redelivered delete must not emit a second -1") + requireSpecDelta(t, records[0], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex())) +} + +// Pause and activate are level-neutral: neither emits a metering delta. +// The +1 from register stays as the sole record through a pause→activate cycle. +func Test_meterRecords_PauseActivateCycleIsLevelNeutral(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{persistUpserts: true}, newMeteringResourceManager(t, true, emitter)) + + wfID := meteringwfID + createdAt := uint64(123) + wantEventID := resourcemanager.EventID("workflow-spec-register", wfID.Hex(), strconv.FormatUint(createdAt, 10)) + payload := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: createdAt, + } + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), payload)) + require.Len(t, emitter.Records(), 1) + requireSpecDelta(t, emitter.Records()[0], "1", wfID.Hex(), wantEventID) + + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + require.Len(t, emitter.Records(), 1, "pause must not emit a delta") + + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(payload))) + require.Len(t, emitter.Records(), 1, "activate-after-pause must not emit a delta") +} + +// OrgId is resolved from the workflow owner and stamped on emitted records. +func Test_meterRecords_OrgIdResolvedOnRecords(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + orgR := &fakeOrgResolver{orgs: map[string]string{"aabbccdd": "org-42"}} + h := newMeteringTestHandlerWithOrg(t, &stubWorkflowArtifactsStore{}, newMeteringResourceManager(t, true, emitter), orgR) + + wfID := types.WorkflowID{50} + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: 1, + })) + + records := emitter.Records() + require.Len(t, records, 1) + require.Len(t, records[0].Utilizations, 1) + assert.Equal(t, "org-42", records[0].Utilizations[0].OrgId, "OrgId must be resolved and stamped on the record") +} + +// don_id is folded into the metering identity once resolved and stamped on both +// emitted records and snapshot entries. +func Test_meterRecords_DonIDOnRecordAndSnapshot(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + wfID := types.WorkflowID{60} + store := &stubWorkflowArtifactsStore{ + specs: []*job.WorkflowSpec{{WorkflowID: wfID.Hex(), WorkflowOwner: "aabbccdd"}}, + } + h := newMeteringTestHandler(t, store, newMeteringResourceManager(t, true, emitter)) + + // Simulate a resolved workflow DON id (as SetWorkflowDon would store). + resolvedDon := "7" + h.specMeter.resolvedDonID.Store(&resolvedDon) + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: 1, + })) + records := emitter.Records() + require.Len(t, records, 1) + require.NotNil(t, records[0].Identity.GetDon()) + assert.Equal(t, "7", records[0].Identity.GetDon().GetDonId(), "record identity must carry resolved don_id") + + entries := h.specMeter.GetUtilization(t.Context()) + require.Len(t, entries, 1) + require.NotNil(t, entries[0].Identity.Don) + assert.Equal(t, "7", entries[0].Identity.Don.DonID, "snapshot identity must carry resolved don_id") +} + +// a transient DB error on GetWorkflowSpec (not a missing row) must surface +// an error and emit no +1, so the event is retried instead of creating a +// spurious spec + delta. +func Test_meterRecords_TransientDBErrorOnGetSpecEmitsNoDelta(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{getSpecErr: errors.New("db connection lost")}, + newMeteringResourceManager(t, true, emitter)) + + err := h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: types.WorkflowID{70}, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: 1, + }) + require.Error(t, err) + assert.Empty(t, emitter.Records(), "transient DB error must not take the new-spec path or emit a +1") +} + +// A full register→pause→activate→delete cycle emits exactly two records: +// one +1 at register and one -1 at delete. Pause and activate emit nothing. +// The delete event_id carries the registered_at that flowed insert→RETURNING, +// proving the generation-scoped id is DON-consistent. +func Test_meterRecords_FullLifecycleEmitsExactlyTwoRecords(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{persistUpserts: true}, newMeteringResourceManager(t, true, emitter)) + + wfID := meteringwfID + createdAt := uint64(123) + payload := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: createdAt, + } + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), payload)) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(payload))) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) + + records := emitter.Records() + require.Len(t, records, 2, "exactly two records: +1 at register, -1 at delete") + requireSpecDelta(t, records[0], "1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-register", wfID.Hex(), strconv.FormatUint(createdAt, 10))) + requireSpecDelta(t, records[1], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex(), strconv.FormatUint(createdAt, 10))) +} + +// A transient delete error returns before emission (no -1); a successful retry +// emits exactly one -1. This fixes the lost-−1 hazard from the old pre-read gate. +func Test_meterRecords_TransientDeleteErrorThenRetryEmitsOnce(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + store := &stubWorkflowArtifactsStore{ + persistUpserts: true, + deleteErr: assert.AnError, + } + h := newMeteringTestHandler(t, store, newMeteringResourceManager(t, true, emitter)) + + wfID := types.WorkflowID{77} + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusPaused, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: 1, + })) + require.Len(t, emitter.Records(), 1, "register emits +1") + + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd") + require.ErrorIs(t, err, assert.AnError) + assert.Empty(t, emitter.Records()[1:], "failed delete must not emit a -1") + + store.deleteErr = nil + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) + + records := emitter.Records() + require.Len(t, records, 2, "exactly one -1 after successful retry") + requireSpecDelta(t, records[1], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex(), "1")) +} + +// A legacy row with RegisteredAt == 0 produces a delete event_id with no +// timestamp part (the fallback for pre-migration rows). +func Test_meterRecords_LegacyRowDeleteUsesFallbackID(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{ + spec: &job.WorkflowSpec{ + WorkflowID: types.WorkflowID{88}.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + RegisteredAt: 0, + }, + }, newMeteringResourceManager(t, true, emitter)) + + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: types.WorkflowID{88}}, "aabbccdd")) + + records := emitter.Records() + require.Len(t, records, 1) + requireSpecDelta(t, records[0], "-1", types.WorkflowID{88}.Hex(), + resourcemanager.EventID("workflow-spec-delete", types.WorkflowID{88}.Hex())) +} + +// Identical event sequences with RM nil / disabled / erroring emitter produce +// identical row+engine outcomes and zero handler errors (fail-open equivalence). +func Test_meterRecords_FailOpenEquivalence(t *testing.T) { + t.Parallel() + wfID := meteringwfID + owner := []byte{0xaa, 0xbb, 0xcc, 0xdd} + payload := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: owner, + WorkflowName: "wf-name", + CreatedAt: 1, + } + delEvt := WorkflowDeletedEvent{WorkflowID: wfID} + + runCycle := func(t *testing.T, rm *resourcemanager.ResourceManager, emitter resourcemanager.Emitter) (*stubWorkflowArtifactsStore, *eventHandler) { + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := newMeteringTestHandler(t, store, rm) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), payload)) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(payload))) + require.NoError(t, h.workflowDeletedEvent(t.Context(), delEvt, "aabbccdd")) + return store, h + } + + // No spec meter (metering disabled at construction) + nilStore, nilH := runCycle(t, nil, nil) + assert.Nil(t, nilH.specMeter) + assert.Nil(t, nilStore.spec, "spec should be deleted by the cycle") + + // RM disabled + disabledEmitter := &recordingEmitter{} + disabledRM := newMeteringResourceManager(t, false, disabledEmitter) + disabledStore, disabledH := runCycle(t, disabledRM, disabledEmitter) + assert.NotNil(t, disabledH.specMeter) + assert.Empty(t, disabledEmitter.Records(), "disabled RM emits nothing") + assert.Nil(t, disabledStore.spec, "spec should be deleted by the cycle") + + // RM enabled but erroring emitter + errEmitter := &recordingEmitter{err: errors.New("beholder unavailable")} + errRM := newMeteringResourceManager(t, true, errEmitter) + errStore, errH := runCycle(t, errRM, errEmitter) + assert.NotNil(t, errH.specMeter) + assert.Empty(t, errEmitter.Records(), "erroring emitter stores nothing") + assert.Nil(t, errStore.spec, "spec should be deleted by the cycle") +} + +// The orphan sweep releases a paused tombstone (no engine) with exactly one -1 +// carrying the generation delete-id. This is the sweep's new load-bearing case: +// workflows deleted on-chain while paused have a tombstone but no engine. The +// sweep dispatches an ordinary WorkflowDeleted event through Handle — the same +// path as reconciliation-generated deletes. +func Test_meterRecords_SweepReleasesPausedTombstone(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + h := newMeteringTestHandler(t, &stubWorkflowArtifactsStore{persistUpserts: true}, newMeteringResourceManager(t, true, emitter)) + + wfID := meteringwfID + createdAt := uint64(456) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + WorkflowName: "wf-name", + CreatedAt: createdAt, + })) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + // After pause the engine is popped; the tombstone has no engine → sweep path. + require.NoError(t, h.Handle(t.Context(), Event{Name: WorkflowDeleted, Data: WorkflowDeletedEvent{WorkflowID: wfID}})) + + records := emitter.Records() + require.Len(t, records, 2, "register +1 and sweep -1") + requireSpecDelta(t, records[0], "1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-register", wfID.Hex(), strconv.FormatUint(createdAt, 10))) + requireSpecDelta(t, records[1], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex(), strconv.FormatUint(createdAt, 10))) +} diff --git a/core/services/workflows/syncer/v2/handler_test.go b/core/services/workflows/syncer/v2/handler_test.go index c600e4e6a94..01f205b677a 100644 --- a/core/services/workflows/syncer/v2/handler_test.go +++ b/core/services/workflows/syncer/v2/handler_test.go @@ -2,12 +2,14 @@ package v2 import ( "context" + "database/sql" "encoding/base64" "encoding/hex" "errors" "fmt" "math/big" "net" + "strconv" "sync/atomic" "testing" "time" @@ -34,6 +36,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/actions/confidentialworkflow/server" "github.com/smartcontractkit/chainlink-common/pkg/contexts" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" @@ -1014,13 +1017,21 @@ func (m *mockArtifactStore) UpsertWorkflowSpec(ctx context.Context, spec *job.Wo return m.artifactStore.UpsertWorkflowSpec(ctx, spec) } -func (m *mockArtifactStore) DeleteWorkflowArtifacts(ctx context.Context, workflowID string) error { +func (m *mockArtifactStore) DeleteWorkflowArtifacts(ctx context.Context, workflowID string) (*job.WorkflowSpec, error) { if m.deleteWorkflowArtifactsErr != nil { - return m.deleteWorkflowArtifactsErr + return nil, m.deleteWorkflowArtifactsErr } return m.artifactStore.DeleteWorkflowArtifacts(ctx, workflowID) } +func (m *mockArtifactStore) PauseWorkflowArtifacts(ctx context.Context, workflowID string) error { + return m.artifactStore.PauseWorkflowArtifacts(ctx, workflowID) +} + +func (m *mockArtifactStore) ListWorkflowSpecs(ctx context.Context) ([]*job.WorkflowSpec, error) { + return m.artifactStore.ListWorkflowSpecs(ctx) +} + func (m *mockArtifactStore) DeleteWorkflowArtifactsBatch(ctx context.Context, workflowIDs []string) error { return m.artifactStore.DeleteWorkflowArtifactsBatch(ctx, workflowIDs) } @@ -1125,7 +1136,7 @@ func Test_workflowDeletedHandler(t *testing.T) { deleteEvent := WorkflowDeletedEvent{ WorkflowID: giveWFID, } - err = h.workflowDeletedEvent(ctx, deleteEvent) + err = h.workflowDeletedEvent(ctx, deleteEvent, "") require.NoError(t, err) // Verify the record is deleted in the database @@ -1180,7 +1191,7 @@ func Test_workflowDeletedHandler(t *testing.T) { deleteEvent := WorkflowDeletedEvent{ WorkflowID: giveWFID, } - err = h.workflowDeletedEvent(ctx, deleteEvent) + err = h.workflowDeletedEvent(ctx, deleteEvent, "") require.NoError(t, err) // Verify the record is deleted in the database @@ -1281,7 +1292,7 @@ func Test_workflowDeletedHandler(t *testing.T) { deleteEvent := WorkflowDeletedEvent{ WorkflowID: giveWFID, } - err = h.workflowDeletedEvent(ctx, deleteEvent) + err = h.workflowDeletedEvent(ctx, deleteEvent, "") require.Error(t, err, failWith) // Verify the record is still in the DB @@ -1295,29 +1306,72 @@ func Test_workflowDeletedHandler(t *testing.T) { } type stubWorkflowArtifactsStore struct { - spec *job.WorkflowSpec - deleteErr error - deleteCalls atomic.Int32 + spec *job.WorkflowSpec + specs []*job.WorkflowSpec + persistUpserts bool + upsertErr error + deleteErr error + listErr error + getSpecErr error + deleteCalls atomic.Int32 + pauseCalls atomic.Int32 + fetchCalls atomic.Int32 } func (s *stubWorkflowArtifactsStore) FetchWorkflowArtifacts(context.Context, string, string, string) ([]byte, []byte, error) { - return nil, nil, nil + s.fetchCalls.Add(1) + return []byte("binary"), []byte("config"), nil } func (s *stubWorkflowArtifactsStore) GetWorkflowSpec(context.Context, string) (*job.WorkflowSpec, error) { + if s.getSpecErr != nil { + return nil, s.getSpecErr + } if s.spec == nil { - return nil, errors.New("not found") + return nil, sql.ErrNoRows } return s.spec, nil } -func (s *stubWorkflowArtifactsStore) UpsertWorkflowSpec(context.Context, *job.WorkflowSpec) (int64, error) { +func (s *stubWorkflowArtifactsStore) UpsertWorkflowSpec(_ context.Context, spec *job.WorkflowSpec) (int64, error) { + if s.upsertErr != nil { + return 0, s.upsertErr + } + if s.persistUpserts { + cp := *spec + s.spec = &cp + } return 1, nil } -func (s *stubWorkflowArtifactsStore) DeleteWorkflowArtifacts(context.Context, string) error { +func (s *stubWorkflowArtifactsStore) ListWorkflowSpecs(context.Context) ([]*job.WorkflowSpec, error) { + if s.listErr != nil { + return nil, s.listErr + } + return s.specs, nil +} + +func (s *stubWorkflowArtifactsStore) DeleteWorkflowArtifacts(context.Context, string) (*job.WorkflowSpec, error) { s.deleteCalls.Add(1) - return s.deleteErr + if s.deleteErr != nil { + return nil, s.deleteErr + } + if s.spec == nil { + return nil, nil + } + deleted := *s.spec + s.spec = nil + return &deleted, nil +} + +func (s *stubWorkflowArtifactsStore) PauseWorkflowArtifacts(context.Context, string) error { + s.pauseCalls.Add(1) + if s.spec != nil { + s.spec.Status = job.WorkflowSpecStatusPaused + s.spec.Workflow = "" + s.spec.Config = "" + } + return nil } func (s *stubWorkflowArtifactsStore) DeleteWorkflowArtifactsBatch(context.Context, []string) error { @@ -1340,7 +1394,7 @@ func Test_workflowDeletedEvent_DrainInProgress(t *testing.T) { workflowArtifactsStore: artifactStore, } - err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}) + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}, "") require.Error(t, err) require.ErrorIs(t, err, ErrDrainInProgress) assert.Equal(t, int32(1), drainable.drainCalls.Load()) @@ -1366,7 +1420,7 @@ func Test_workflowDeletedEvent_IgnoresErrAlreadyStopped(t *testing.T) { workflowArtifactsStore: artifactStore, } - err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}) + err := h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: workflowID}, "") require.NoError(t, err) assert.Equal(t, int32(1), drainable.closeCalls.Load()) assert.Equal(t, int32(1), artifactStore.deleteCalls.Load()) @@ -1610,7 +1664,7 @@ func Test_Handler_OrganizationID(t *testing.T) { //nolint:paralleltest // behold }) // Mock ORM responses - mockORM.EXPECT().GetWorkflowSpec(mock.Anything, types.WorkflowID(giveWFID).Hex()).Return(nil, errors.New("not found")) + mockORM.EXPECT().GetWorkflowSpec(mock.Anything, types.WorkflowID(giveWFID).Hex()).Return(nil, sql.ErrNoRows) mockORM.EXPECT().UpsertWorkflowSpec(mock.Anything, mock.AnythingOfType("*job.WorkflowSpec")).Return(int64(1), nil) // Set up handler @@ -1706,7 +1760,7 @@ func Test_Handler_OrganizationID(t *testing.T) { //nolint:paralleltest // behold } mockDeleteORM.EXPECT().GetWorkflowSpec(mock.Anything, types.WorkflowID(giveWFID).Hex()).Return(spec, nil) - mockDeleteORM.EXPECT().DeleteWorkflowSpec(mock.Anything, types.WorkflowID(giveWFID).Hex()).Return(nil) + mockDeleteORM.EXPECT().DeleteWorkflowSpec(mock.Anything, types.WorkflowID(giveWFID).Hex()).Return(&job.WorkflowSpec{}, nil) deleteArtifactStore, err := artifacts.NewStore(lggr, mockDeleteORM, fetcher.FetcherFunc(), fetcher.RetrieverFunc(), clockwork.NewFakeClock(), workflowkey.Key{}, custmsg.NewLabeler(), lf, artifacts.WithConfig(artifacts.StoreConfig{ ArtifactStorageHost: "example.com", @@ -1874,3 +1928,216 @@ func (c *confidentialCap) Initialise(_ context.Context, _ core.StandardCapabilit } var _ server.ClientCapability = &confidentialCap{} + +// assertStubState checks the in-memory stub's spec state. +func assertStubState(t *testing.T, s *stubWorkflowArtifactsStore, exists bool, status job.WorkflowSpecStatus, artifactsEmpty bool, registeredAt int64) { + t.Helper() + if !exists { + assert.Nil(t, s.spec, "spec should not exist") + return + } + require.NotNil(t, s.spec, "spec should exist") + assert.Equal(t, status, s.spec.Status) + if artifactsEmpty { + assert.Empty(t, s.spec.Workflow, "workflow artifacts should be empty") + assert.Empty(t, s.spec.Config, "config artifacts should be empty") + } else { + assert.NotEmpty(t, s.spec.Workflow, "workflow artifacts should be present") + } + assert.Equal(t, registeredAt, s.spec.RegisteredAt) +} + +// Test_specStorage_StateMachine exercises the storage state machine across +// (row state × event) transitions, verifying row existence, Status, artifact +// presence, RegisteredAt, engine-registry contents, and fetch call counts. +func Test_specStorage_StateMachine(t *testing.T) { + t.Parallel() + + owner := []byte{0xaa, 0xbb, 0xcc, 0xdd} + binaryData := []byte("binary") + configData := []byte("config") + wfIDBytes, err := pkgworkflows.GenerateWorkflowID(owner, "wf-name", binaryData, configData, "") + require.NoError(t, err) + wfID := types.WorkflowID(wfIDBytes) + hexWorkflow := hex.EncodeToString(binaryData) + createdAt := uint64(999) + createdAtI64 := int64(createdAt) + + makeHandler := func(store *stubWorkflowArtifactsStore) *eventHandler { + return newMeteringTestHandler(t, store, nil) + } + activePayload := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: owner, + WorkflowName: "wf-name", + CreatedAt: createdAt, + } + + t.Run("absent + Activated → fetch, insert, engine started", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := makeHandler(store) + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(activePayload))) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, createdAtI64) + _, ok := h.engineRegistry.Get(wfID) + assert.True(t, ok, "engine should be started") + assert.Equal(t, int32(1), store.fetchCalls.Load(), "exactly one fetch") + }) + + t.Run("active + Paused → tombstone, engine popped; redelivery no-op", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := makeHandler(store) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), activePayload)) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + assertStubState(t, store, true, job.WorkflowSpecStatusPaused, true, createdAtI64) + _, ok := h.engineRegistry.Get(wfID) + assert.False(t, ok, "engine should be popped") + // Redelivery + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + assertStubState(t, store, true, job.WorkflowSpecStatusPaused, true, createdAtI64) + }) + + t.Run("tombstone + Activated → fetch, full restore, engine started", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := makeHandler(store) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), activePayload)) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + fetchBefore := store.fetchCalls.Load() + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(activePayload))) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, createdAtI64) + _, ok := h.engineRegistry.Get(wfID) + assert.True(t, ok, "engine should be started") + assert.Equal(t, fetchBefore+1, store.fetchCalls.Load(), "exactly one new fetch for restore") + }) + + t.Run("tombstone + Deleted → row removed, engine steps no-op", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := makeHandler(store) + require.NoError(t, h.workflowRegisteredEvent(t.Context(), activePayload)) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "aabbccdd")) + assertStubState(t, store, false, "", true, 0) + _, ok := h.engineRegistry.Get(wfID) + assert.False(t, ok, "engine should be absent") + }) + + t.Run("paused+artifacts + Activated → status flip, no fetch", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{ + persistUpserts: true, + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusPaused, + WorkflowOwner: "aabbccdd", + Workflow: hexWorkflow, + Config: string(configData), + RegisteredAt: createdAtI64, + }, + } + h := makeHandler(store) + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(activePayload))) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, createdAtI64) + }) + + t.Run("absent + Paused → no-op, no error", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{} + h := makeHandler(store) + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID})) + assertStubState(t, store, false, "", true, 0) + }) + + t.Run("absent + Deleted → no-op, no error", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{} + h := makeHandler(store) + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID}, "")) + assertStubState(t, store, false, "", true, 0) + }) + + t.Run("active RegisteredAt==0 + Activated (no status change) → backfill", func(t *testing.T) { + t.Parallel() + store := &stubWorkflowArtifactsStore{ + persistUpserts: true, + spec: &job.WorkflowSpec{ + WorkflowID: wfID.Hex(), + Status: job.WorkflowSpecStatusActive, + WorkflowOwner: "aabbccdd", + Workflow: hexWorkflow, + Config: string(configData), + RegisteredAt: 0, + }, + } + h := makeHandler(store) + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(activePayload))) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, createdAtI64) + }) +} + +// Test_handler_SourceParity_PauseActivateCycle runs a full register→pause→ +// activate→delete cycle twice with contract-style and gRPC-style source +// identifiers, asserting identical storage outcomes, engine-registry source +// scoping, and identical metering records/ids. +func Test_handler_SourceParity_PauseActivateCycle(t *testing.T) { + t.Parallel() + owner := []byte{0xaa, 0xbb, 0xcc, 0xdd} + binaryData := []byte("binary") + configData := []byte("config") + wfIDBytes, err := pkgworkflows.GenerateWorkflowID(owner, "wf-name", binaryData, configData, "") + require.NoError(t, err) + wfID := types.WorkflowID(wfIDBytes) + createdAt := uint64(777) + createdAtI64 := int64(createdAt) + sources := []string{ + "contract:42:0xabcdef1234567890", + "grpc:centralized-registry:v1", + } + + for _, source := range sources { + t.Run(source, func(t *testing.T) { + t.Parallel() + emitter := &recordingEmitter{} + store := &stubWorkflowArtifactsStore{persistUpserts: true} + h := newMeteringTestHandler(t, store, newMeteringResourceManager(t, true, emitter)) + + payload := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: wfID, + WorkflowOwner: owner, + WorkflowName: "wf-name", + CreatedAt: createdAt, + Source: source, + } + + require.NoError(t, h.workflowRegisteredEvent(t.Context(), payload)) + // Verify engine is registered with the correct source + entry, ok := h.engineRegistry.Get(wfID) + require.True(t, ok) + assert.Equal(t, source, entry.Source) + + require.NoError(t, h.workflowPausedEvent(t.Context(), WorkflowPausedEvent{WorkflowID: wfID, Source: source})) + assertStubState(t, store, true, job.WorkflowSpecStatusPaused, true, createdAtI64) + + require.NoError(t, h.workflowActivatedEvent(t.Context(), WorkflowActivatedEvent(payload))) + assertStubState(t, store, true, job.WorkflowSpecStatusActive, false, createdAtI64) + entry, ok = h.engineRegistry.Get(wfID) + require.True(t, ok) + assert.Equal(t, source, entry.Source) + + require.NoError(t, h.workflowDeletedEvent(t.Context(), WorkflowDeletedEvent{WorkflowID: wfID, Source: source}, "aabbccdd")) + assertStubState(t, store, false, "", true, 0) + + // Both sources produce identical metering: one +1, one -1, same ids + records := emitter.Records() + require.Len(t, records, 2) + requireSpecDelta(t, records[0], "1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-register", wfID.Hex(), strconv.FormatUint(createdAt, 10))) + requireSpecDelta(t, records[1], "-1", wfID.Hex(), + resourcemanager.EventID("workflow-spec-delete", wfID.Hex(), strconv.FormatUint(createdAt, 10))) + }) + } +} diff --git a/core/services/workflows/syncer/v2/helpers.go b/core/services/workflows/syncer/v2/helpers.go index e8be21d6d7f..60e33a9d07f 100644 --- a/core/services/workflows/syncer/v2/helpers.go +++ b/core/services/workflows/syncer/v2/helpers.go @@ -11,6 +11,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" eventsv2 "github.com/smartcontractkit/chainlink-protos/workflows/go/v2" + "github.com/smartcontractkit/chainlink/v2/core/services/job" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/ratelimiter" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncerlimiter" ) @@ -78,6 +79,12 @@ func (m *testEvtHandler) EmitActivationAbandoned(context.Context, Event, eventsv return nil } +func (m *testEvtHandler) ListWorkflowSpecs(context.Context) ([]*job.WorkflowSpec, error) { + return nil, nil +} + +func (m *testEvtHandler) SetWorkflowDon(capabilities.DON) {} + func (m *testEvtHandler) ClearEvents() { m.mux.Lock() defer m.mux.Unlock() diff --git a/core/services/workflows/syncer/v2/mocks/orm.go b/core/services/workflows/syncer/v2/mocks/orm.go index 4ea1067822c..6232e8d4a49 100644 --- a/core/services/workflows/syncer/v2/mocks/orm.go +++ b/core/services/workflows/syncer/v2/mocks/orm.go @@ -23,21 +23,33 @@ func (_m *ORM) EXPECT() *ORM_Expecter { } // DeleteWorkflowSpec provides a mock function with given fields: ctx, id -func (_m *ORM) DeleteWorkflowSpec(ctx context.Context, id string) error { +func (_m *ORM) DeleteWorkflowSpec(ctx context.Context, id string) (*job.WorkflowSpec, error) { ret := _m.Called(ctx, id) if len(ret) == 0 { panic("no return value specified for DeleteWorkflowSpec") } - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { + var r0 *job.WorkflowSpec + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*job.WorkflowSpec, error)); ok { + return rf(ctx, id) + } + if rf, ok := ret.Get(0).(func(context.Context, string) *job.WorkflowSpec); ok { r0 = rf(ctx, id) } else { - r0 = ret.Error(0) + if ret.Get(0) != nil { + r0 = ret.Get(0).(*job.WorkflowSpec) + } } - return r0 + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } // ORM_DeleteWorkflowSpec_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteWorkflowSpec' @@ -59,12 +71,12 @@ func (_c *ORM_DeleteWorkflowSpec_Call) Run(run func(ctx context.Context, id stri return _c } -func (_c *ORM_DeleteWorkflowSpec_Call) Return(_a0 error) *ORM_DeleteWorkflowSpec_Call { - _c.Call.Return(_a0) +func (_c *ORM_DeleteWorkflowSpec_Call) Return(_a0 *job.WorkflowSpec, _a1 error) *ORM_DeleteWorkflowSpec_Call { + _c.Call.Return(_a0, _a1) return _c } -func (_c *ORM_DeleteWorkflowSpec_Call) RunAndReturn(run func(context.Context, string) error) *ORM_DeleteWorkflowSpec_Call { +func (_c *ORM_DeleteWorkflowSpec_Call) RunAndReturn(run func(context.Context, string) (*job.WorkflowSpec, error)) *ORM_DeleteWorkflowSpec_Call { _c.Call.Return(run) return _c } @@ -175,6 +187,111 @@ func (_c *ORM_GetWorkflowSpec_Call) RunAndReturn(run func(context.Context, strin return _c } +// ListWorkflowSpecs provides a mock function with given fields: ctx +func (_m *ORM) ListWorkflowSpecs(ctx context.Context) ([]*job.WorkflowSpec, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for ListWorkflowSpecs") + } + + var r0 []*job.WorkflowSpec + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]*job.WorkflowSpec, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) []*job.WorkflowSpec); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*job.WorkflowSpec) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ORM_ListWorkflowSpecs_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListWorkflowSpecs' +type ORM_ListWorkflowSpecs_Call struct { + *mock.Call +} + +// ListWorkflowSpecs is a helper method to define mock.On call +// - ctx context.Context +func (_e *ORM_Expecter) ListWorkflowSpecs(ctx interface{}) *ORM_ListWorkflowSpecs_Call { + return &ORM_ListWorkflowSpecs_Call{Call: _e.mock.On("ListWorkflowSpecs", ctx)} +} + +func (_c *ORM_ListWorkflowSpecs_Call) Run(run func(ctx context.Context)) *ORM_ListWorkflowSpecs_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *ORM_ListWorkflowSpecs_Call) Return(_a0 []*job.WorkflowSpec, _a1 error) *ORM_ListWorkflowSpecs_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *ORM_ListWorkflowSpecs_Call) RunAndReturn(run func(context.Context) ([]*job.WorkflowSpec, error)) *ORM_ListWorkflowSpecs_Call { + _c.Call.Return(run) + return _c +} + +// PauseWorkflowSpec provides a mock function with given fields: ctx, id +func (_m *ORM) PauseWorkflowSpec(ctx context.Context, id string) error { + ret := _m.Called(ctx, id) + + if len(ret) == 0 { + panic("no return value specified for PauseWorkflowSpec") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ORM_PauseWorkflowSpec_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PauseWorkflowSpec' +type ORM_PauseWorkflowSpec_Call struct { + *mock.Call +} + +// PauseWorkflowSpec is a helper method to define mock.On call +// - ctx context.Context +// - id string +func (_e *ORM_Expecter) PauseWorkflowSpec(ctx interface{}, id interface{}) *ORM_PauseWorkflowSpec_Call { + return &ORM_PauseWorkflowSpec_Call{Call: _e.mock.On("PauseWorkflowSpec", ctx, id)} +} + +func (_c *ORM_PauseWorkflowSpec_Call) Run(run func(ctx context.Context, id string)) *ORM_PauseWorkflowSpec_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *ORM_PauseWorkflowSpec_Call) Return(_a0 error) *ORM_PauseWorkflowSpec_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ORM_PauseWorkflowSpec_Call) RunAndReturn(run func(context.Context, string) error) *ORM_PauseWorkflowSpec_Call { + _c.Call.Return(run) + return _c +} + // UpsertWorkflowSpec provides a mock function with given fields: ctx, spec func (_m *ORM) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) (int64, error) { ret := _m.Called(ctx, spec) diff --git a/core/services/workflows/syncer/v2/spec_meter.go b/core/services/workflows/syncer/v2/spec_meter.go new file mode 100644 index 00000000000..2eb269ce59b --- /dev/null +++ b/core/services/workflows/syncer/v2/spec_meter.go @@ -0,0 +1,199 @@ +package v2 + +import ( + "context" + "errors" + "strconv" + "sync/atomic" + + commoncap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + "github.com/smartcontractkit/chainlink-common/pkg/contexts" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/resourcemanager" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" + meteringpb "github.com/smartcontractkit/chainlink-protos/metering/go" + "github.com/smartcontractkit/chainlink/v2/core/services/job" +) + +// Service-level metering identity constants for the workflow syncer. Service is +// the stable service constant; ResourcePool identifies the workflow_specs_v2 +// pool. The billing unit (ResourceType) and per-resource id (ResourceID) are +// carried on each Utilization. The coarse deployment/node/DON dimensions +// (product, tenant, environment, zone, don_id, node_id) are supplied at +// construction via NewSpecMeter. +const ( + meterService = "workflow-syncer-v2" + meterResourcePool = "workflow_specs_v2" + meterResourceType = "operations" +) + +// SpecLister is the read-only view of durable workflow-spec storage the +// metering snapshot path needs: the identity columns of every persisted spec. +type SpecLister interface { + ListWorkflowSpecs(ctx context.Context) ([]*job.WorkflowSpec, error) +} + +// specMeter wraps ResourceManager lifecycle and metering identity concerns +// to stop leaking complexity into the event handler +type SpecMeter struct { + services.Service + eng *services.Engine + + rm *resourcemanager.ResourceManager + identity resourcemanager.ResourceIdentity + + // resolvedDonID holds the workflow DON id once set via SetWorkflowDon. + resolvedDonID atomic.Pointer[string] + + // unregister removes this meter from the ResourceManager's snapshot + // registry; set in start, called in close. Nil until started. + unregister func() + + orgResolver orgresolver.OrgResolver + specs SpecLister +} + +// NewSpecMeter returns a SpecMeter emitting through rm with the given coarse +// identity (product/tenant/environment/zone/node_id from node metering +// config). The syncer Service/ResourcePool constants are stamped here and +// always overwrite whatever the caller passes; the workflow DON id is supplied +// later via SetWorkflowDon. specs backs the snapshot enumeration. orgResolver +// may be nil (org_id is then always empty). +func NewSpecMeter( + lggr logger.Logger, + rm *resourcemanager.ResourceManager, + identity resourcemanager.ResourceIdentity, + specs SpecLister, + orgResolver orgresolver.OrgResolver, +) (*SpecMeter, error) { + if rm == nil { + return nil, errors.New("resource manager must be provided") + } + if specs == nil { + return nil, errors.New("spec lister must be provided") + } + identity.Service = meterService + identity.ResourcePool = meterResourcePool + + sm := &SpecMeter{ + rm: rm, + identity: identity, + orgResolver: orgResolver, + specs: specs, + } + sm.Service, sm.eng = services.Config{ + Name: "SpecMeter", + Start: sm.start, + Close: sm.close, + }.NewServiceEngine(lggr) + return sm, nil +} + +func (sm *SpecMeter) start(ctx context.Context) error { + if err := sm.rm.Start(ctx); err != nil { + return err + } + sm.unregister = sm.rm.Register(sm) + return nil +} + +func (sm *SpecMeter) close() error { + // Stop snapshotting this meter, then close the ResourceManager service. + if sm.unregister != nil { + sm.unregister() + sm.unregister = nil + } + return sm.rm.Close() +} + +// SetWorkflowDon supplies the launcher-resolved workflow DON identity. Called +// by the registry after WaitForDon, before any event is dispatched; the value +// is static for the life of the node. +func (sm *SpecMeter) SetWorkflowDon(don commoncap.DON) { + if sm == nil { + return + } + donID := strconv.FormatUint(uint64(don.ID), 10) + sm.resolvedDonID.Store(&donID) +} + +// EmitSpecDelta emits one metering.v1.MeterRecord (METER_ACTION_UPDATE) +// capturing a signed ±delta to the durable workflow_specs_v2 level for +// workflowID +func (sm *SpecMeter) EmitSpecDelta(ctx context.Context, delta int64, workflowID, owner, eventID string) { + if sm == nil { + return + } + orgID := contexts.CREValue(ctx).Org + if orgID == "" && sm.orgResolver != nil && owner != "" { + if resolved, err := sm.orgResolver.Get(ctx, owner); err != nil { + sm.eng.Warnw("failed to resolve org ID for metering", "owner", owner, "err", err) + } else { + orgID = resolved + } + } + // resource_id = workflow_id (the syncer meters one durable spec per workflow). + sm.rm.EmitDelta(ctx, sm.baseIdentity(), eventID, delta, resourcemanager.UtilizationFields{ + ResourceType: meterResourceType, + ResourceID: workflowID, + OrgID: orgID, + }) +} + +// baseIdentity returns the meter's identity with the workflow DON id folded in +// once it has been set via SetWorkflowDon. +func (sm *SpecMeter) baseIdentity() resourcemanager.ResourceIdentity { + id := sm.identity + if donID := sm.resolvedDonID.Load(); donID != nil { + nodeID := "" + if id.Don != nil { + nodeID = id.Don.NodeID + } + id.Don = &resourcemanager.DonIdentity{ + DonID: *donID, + NodeID: nodeID, + } + } + return id +} + +// GetUtilization implements resourcemanager.Meterable: it returns one +// SnapshotEntry per persisted workflow_specs_v2 spec. The durable resource is +// the stored spec, NOT the running engine. +// +// resource_id is the workflow_id; the organization is resolved fail-open from +// the spec's stored owner through the org resolver. Must fail open. +func (sm *SpecMeter) GetUtilization(ctx context.Context) []resourcemanager.SnapshotEntry { + specs, err := sm.specs.ListWorkflowSpecs(ctx) + if err != nil { + sm.eng.Warnw("failed to list persisted workflow specs for metering snapshot; skipping tick", "err", err) + return nil + } + base := sm.baseIdentity() + entries := make([]resourcemanager.SnapshotEntry, 0, len(specs)) + for _, spec := range specs { + if spec == nil { + continue + } + var orgID string + if sm.orgResolver != nil && spec.WorkflowOwner != "" { + if resolved, err := sm.orgResolver.Get(ctx, spec.WorkflowOwner); err != nil { + sm.eng.Warnw("failed to resolve org ID for metering snapshot", "owner", spec.WorkflowOwner, "err", err) + } else { + orgID = resolved + } + } + entries = append(entries, resourcemanager.SnapshotEntry{ + Identity: base, + Utilizations: []*meteringpb.Utilization{ + resourcemanager.NewUtilizationInt(1, resourcemanager.UtilizationFields{ + ResourceType: meterResourceType, + ResourceID: spec.WorkflowID, + OrgID: orgID, + }), + }, + }) + } + return entries +} diff --git a/core/services/workflows/syncer/v2/workflow_registry.go b/core/services/workflows/syncer/v2/workflow_registry.go index 63a92bec820..0ae28fdc30f 100644 --- a/core/services/workflows/syncer/v2/workflow_registry.go +++ b/core/services/workflows/syncer/v2/workflow_registry.go @@ -28,8 +28,10 @@ import ( "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" "github.com/smartcontractkit/chainlink-evm/pkg/config" eventsv2 "github.com/smartcontractkit/chainlink-protos/workflows/go/v2" + "github.com/smartcontractkit/chainlink/v2/core/services/job" "github.com/smartcontractkit/chainlink/v2/core/services/shardorchestrator" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncer/versioning" + wftypes "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" ) const name = "WorkflowRegistrySyncer" @@ -142,6 +144,8 @@ type evtHandler interface { Handle(ctx context.Context, event Event) error EmitActivationAbandoned(ctx context.Context, event Event, reason eventsv2.ActivationAbandonReason, activationErr error, retryCount int32) error + ListWorkflowSpecs(ctx context.Context) ([]*job.WorkflowSpec, error) + SetWorkflowDon(don capabilities.DON) } type donNotifier interface { @@ -418,11 +422,12 @@ func (w *workflowRegistry) Start(_ context.Context) error { return } w.lggr.Debugw("read from don received channel while waiting to start reconciliation sync") - _, err := w.workflowDonNotifier.WaitForDon(ctx) + don, err := w.workflowDonNotifier.WaitForDon(ctx) if err != nil { w.hooks.OnStartFailure(fmt.Errorf("failed to start workflow sync strategy: %w", err)) return } + w.handler.SetWorkflowDon(don) w.syncUsingReconciliationStrategy(ctx) }) @@ -818,7 +823,7 @@ func (w *workflowRegistry) filterWorkflowsByShard(ctx context.Context, workflows } // syncUsingReconciliationStrategy syncs workflow registry contract state by polling the workflow metadata state and comparing to local state. -// NOTE: In this mode paused states will be treated as a deleted workflow. Workflows will not be registered as paused. +// NOTE: Paused workflows are retained as tombstones (status=paused, artifact payload cleared). // This function processes each source independently to ensure that failure in one source doesn't affect workflows from other sources. func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) { ticker := w.getTicker(w.syncTickInterval) @@ -841,6 +846,23 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) totalWorkflowsFetched := 0 reconcileReport := newReconcileReport() + // Persisted specs are listed once per tick and reconciled against + // the union of every source's metadata after the loop (see + // reconcileOrphanedSpecs); a listing failure skips only orphan + // reconciliation this tick — engines still reconcile. + persistedSpecs, specsErr := w.handler.ListWorkflowSpecs(ctx) + if specsErr != nil { + persistedSpecs = nil + w.lggr.Warnw("failed to list persisted workflow specs; skipping orphaned-spec reconciliation this tick", "err", specsErr) + } + + // metadataUnion collects every source's (post-shard) workflow IDs; + // allSourcesHealthy stays true only when every source fetched and + // filtered successfully, so orphan reconciliation never judges + // rows against incomplete metadata. + metadataUnion := make(map[string]struct{}) + allSourcesHealthy := true + for _, source := range w.workflowSources { sourceName := source.Name() sourceIdentifier := source.SourceIdentifier() @@ -860,6 +882,7 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) w.metrics.recordSourceFetch(ctx, sourceName, len(workflows), duration, fetchErr) if fetchErr != nil { + allSourcesHealthy = false w.lggr.Errorw("Failed to fetch from source, skipping reconciliation for this source", "source", sourceName, "error", fetchErr, "durationMs", duration.Milliseconds()) // KEY: Skip this source entirely - no events generated, no deletions @@ -876,7 +899,8 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) if w.shardingEnabled { filteredWorkflowsMetadata, err = w.filterWorkflowsByShard(ctx, workflows) if err != nil { - w.lggr.Errorw("failed to filter workflows by shard", + allSourcesHealthy = false + w.lggr.Errorw("failed to filter workflows by shard, skipping reconciliation for this source", "err", err, "source", sourceName) continue @@ -889,6 +913,10 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) ) } + for _, wfMeta := range filteredWorkflowsMetadata { + metadataUnion[wfMeta.WorkflowID.Hex()] = struct{}{} + } + // Generate events only for this source's engines (using sourceIdentifier for engine registry lookups) events, genErr := w.generateReconciliationEvents(ctx, pendingEvents, filteredWorkflowsMetadata, head, sourceIdentifier) if genErr != nil { @@ -999,6 +1027,10 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) ) } + if allSourcesHealthy && len(w.workflowSources) > 0 { + w.reconcileOrphanedSpecs(ctx, persistedSpecs, metadataUnion) + } + w.metrics.recordFetchedWorkflows(ctx, totalWorkflowsFetched) w.lggr.Debugw("reconciled events", "report", reconcileReport) @@ -1020,6 +1052,47 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) } } +// reconcileOrphanedSpecs releases persisted specs whose workflow ID is absent +// from all sources' metadata, catching workflows deleted while the node was +// down — no engine exists at the next tick, so engine reconciliation can't +// generate the delete event. Liveness is checked against the union of all +// sources (not per-source) because workflow IDs are content-addressed and a row +// is live if any source lists it. The caller guarantees all sources succeeded; +// if any source errored, the sweep is skipped entirely. +func (w *workflowRegistry) reconcileOrphanedSpecs(ctx context.Context, specs []*job.WorkflowSpec, metadataUnion map[string]struct{}) { + if len(specs) == 0 { + return + } + for _, spec := range specs { + if spec == nil || spec.WorkflowID == "" { + continue + } + if _, live := metadataUnion[spec.WorkflowID]; live { + continue + } + wfIDBytes, derr := hex.DecodeString(spec.WorkflowID) + if derr != nil || len(wfIDBytes) != len(wftypes.WorkflowID{}) { + w.lggr.Warnw("orphaned-spec reconciliation: skipping unparseable persisted workflow_id", "workflowID", spec.WorkflowID, "err", derr) + continue + } + var wfID wftypes.WorkflowID + copy(wfID[:], wfIDBytes) + if _, engineFound := w.engineRegistry.Get(wfID); engineFound { + // Engine-owned: per-source reconciliation deletes it with drain + // machinery; this path only handles engine-less leftovers. + continue + } + w.lggr.Debugw("orphaned workflow spec absent from all sources' metadata; releasing", + "workflowID", spec.WorkflowID, "owner", spec.WorkflowOwner) + if herr := w.handleWithMetrics(ctx, Event{ + Name: WorkflowDeleted, + Data: WorkflowDeletedEvent{WorkflowID: wfID}, + }); herr != nil { + w.lggr.Warnw("failed to release orphaned workflow spec", "workflowID", spec.WorkflowID, "err", herr) + } + } +} + // getTicker returns the ticker that the workflowRegistry will use to poll for events. If the ticker // is nil, then a default ticker is returned. func (w *workflowRegistry) getTicker(d time.Duration) <-chan time.Time { diff --git a/core/services/workflows/syncer/v2/workflow_registry_test.go b/core/services/workflows/syncer/v2/workflow_registry_test.go index cf467132db1..2494791f232 100644 --- a/core/services/workflows/syncer/v2/workflow_registry_test.go +++ b/core/services/workflows/syncer/v2/workflow_registry_test.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" commonCap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" @@ -20,8 +21,10 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/types/query/primitives" "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" ringpb "github.com/smartcontractkit/chainlink-protos/ring/go" + eventsv2 "github.com/smartcontractkit/chainlink-protos/workflows/go/v2" "github.com/smartcontractkit/chainlink/v2/core/capabilities" "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/job" "github.com/smartcontractkit/chainlink/v2/core/services/shardorchestrator" wfTypes "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" @@ -1997,3 +2000,309 @@ func TestWorkflowRegistry_getTicker_WithTickerOverride(t *testing.T) { tickerCh := wr.getTicker(10 * time.Second) require.Equal(t, (<-chan time.Time)(customCh), tickerCh) } + +// orphanSweepFakeHandler is a minimal evtHandler that returns a fixed spec list +// and records every Handle call, so the reconciliation loop's orphaned-spec +// behavior can be tested end to end. +type orphanSweepFakeHandler struct { + mu sync.Mutex + specs []*job.WorkflowSpec + handled []Event +} + +func (h *orphanSweepFakeHandler) Close() error { return nil } +func (h *orphanSweepFakeHandler) Start(context.Context) error { return nil } +func (h *orphanSweepFakeHandler) Handle(_ context.Context, event Event) error { + h.mu.Lock() + defer h.mu.Unlock() + h.handled = append(h.handled, event) + return nil +} +func (h *orphanSweepFakeHandler) EmitActivationAbandoned(context.Context, Event, eventsv2.ActivationAbandonReason, error, int32) error { + return nil +} +func (h *orphanSweepFakeHandler) ListWorkflowSpecs(context.Context) ([]*job.WorkflowSpec, error) { + return h.specs, nil +} + +func (h *orphanSweepFakeHandler) SetWorkflowDon(commonCap.DON) {} +func (h *orphanSweepFakeHandler) Handled() []Event { + h.mu.Lock() + defer h.mu.Unlock() + cp := make([]Event, len(h.handled)) + copy(cp, h.handled) + return cp +} + +// OrphanDeletes returns the workflow IDs of every WorkflowDeleted event +// dispatched to the handler. In these tests the engine registry is empty, so +// every delete is sweep-generated. +func (h *orphanSweepFakeHandler) OrphanDeletes() []string { + h.mu.Lock() + defer h.mu.Unlock() + var ids []string + for _, evt := range h.handled { + if evt.Name != WorkflowDeleted { + continue + } + if payload, ok := evt.Data.(WorkflowDeletedEvent); ok { + ids = append(ids, payload.WorkflowID.Hex()) + } + } + return ids +} + +func newTestRegistryMetrics(t *testing.T) *metrics { + t.Helper() + m, err := newMetrics() + require.NoError(t, err) + return m +} + +// Orphaned specs — rows whose workflow ID is absent from the union of every +// source's metadata with no running engine — are released through the normal +// WorkflowDeleted event path. Liveness is judged against the union regardless +// of the row's Source attribution: IDs are content-addressed and may recur +// across sources, and pre-attribution rows carry no source at all. +func Test_reconcileOrphanedSpecs(t *testing.T) { + t.Parallel() + + liveID := wfTypes.WorkflowID{1} + orphanID := wfTypes.WorkflowID{2} + + t.Run("engine-less orphans are released regardless of attribution", func(t *testing.T) { + t.Parallel() + attributedOrphan := wfTypes.WorkflowID{3} + h := &orphanSweepFakeHandler{} + w := &workflowRegistry{lggr: logger.TestLogger(t), handler: h, engineRegistry: NewEngineRegistry(), metrics: newTestRegistryMetrics(t)} + + specs := []*job.WorkflowSpec{ + {WorkflowID: liveID.Hex(), WorkflowOwner: "aabbccdd", Source: "source-a"}, + {WorkflowID: orphanID.Hex(), WorkflowOwner: "aabbccdd"}, // pre-attribution row + {WorkflowID: attributedOrphan.Hex(), WorkflowOwner: "aabbccdd", Source: "source-a"}, + } + w.reconcileOrphanedSpecs(t.Context(), specs, map[string]struct{}{liveID.Hex(): {}}) + + assert.ElementsMatch(t, []string{orphanID.Hex(), attributedOrphan.Hex()}, h.OrphanDeletes(), + "both orphans are released; the live spec is retained") + }) + + t.Run("row listed by any source is retained even when its attribution differs", func(t *testing.T) { + t.Parallel() + // Workflow IDs are content-addressed and may recur across sources: a + // row written by source-a whose ID is currently listed by source-b is + // live and must not be released. + h := &orphanSweepFakeHandler{} + w := &workflowRegistry{lggr: logger.TestLogger(t), handler: h, engineRegistry: NewEngineRegistry(), metrics: newTestRegistryMetrics(t)} + + specs := []*job.WorkflowSpec{ + {WorkflowID: orphanID.Hex(), WorkflowOwner: "aabbccdd", Source: "source-a"}, + } + w.reconcileOrphanedSpecs(t.Context(), specs, map[string]struct{}{orphanID.Hex(): {}}) + + assert.Empty(t, h.Handled(), "a row present in the union must never be released") + }) + + t.Run("engine-owned orphan is left to engine reconciliation", func(t *testing.T) { + t.Parallel() + er := NewEngineRegistry() + require.NoError(t, er.Add(orphanID, "test-source", &mockService{})) + h := &orphanSweepFakeHandler{} + w := &workflowRegistry{lggr: logger.TestLogger(t), handler: h, engineRegistry: er, metrics: newTestRegistryMetrics(t)} + + specs := []*job.WorkflowSpec{ + {WorkflowID: orphanID.Hex(), WorkflowOwner: "aabbccdd"}, + } + w.reconcileOrphanedSpecs(t.Context(), specs, map[string]struct{}{liveID.Hex(): {}}) + + assert.Empty(t, h.Handled(), "engine-owned orphan must not be released here") + }) +} + +// fakeMetadataSource is a canned WorkflowMetadataSource for driving the +// reconciliation loop: it returns fixed metadata or a fixed error. +type fakeMetadataSource struct { + name string + metadata []WorkflowMetadataView + err error +} + +func (s *fakeMetadataSource) ListWorkflowMetadata(context.Context, commonCap.DON) ([]WorkflowMetadataView, *types.Head, error) { + if s.err != nil { + return nil, nil, s.err + } + return s.metadata, &types.Head{Height: "1"}, nil +} +func (s *fakeMetadataSource) Name() string { return s.name } +func (s *fakeMetadataSource) SourceIdentifier() string { return s.name } +func (s *fakeMetadataSource) Ready() error { return nil } + +// failingShardMappingClient always fails shard-mapping lookups, driving the +// shard-filter error path. +type failingShardMappingClient struct{} + +func (f *failingShardMappingClient) GetWorkflowShardMapping(context.Context, []string) (*ringpb.GetWorkflowShardMappingResponse, error) { + return nil, assert.AnError +} + +func (f *failingShardMappingClient) ReportWorkflowTriggerRegistration(context.Context, *ringpb.ReportWorkflowTriggerRegistrationRequest) (*ringpb.ReportWorkflowTriggerRegistrationResponse, error) { + return &ringpb.ReportWorkflowTriggerRegistrationResponse{Success: true}, nil +} + +func (f *failingShardMappingClient) Close() error { return nil } + +var _ shardorchestrator.ClientInterface = (*failingShardMappingClient)(nil) + +// newSweepLoopRegistry builds a workflowRegistry wired for driving +// syncUsingReconciliationStrategy directly: no contract source, injected fake +// sources, and an unbuffered ticker channel the test controls. +func newSweepLoopRegistry(t *testing.T, h evtHandler, er *EngineRegistry, sources []WorkflowMetadataSource, opts ...Option) (*workflowRegistry, chan time.Time) { + t.Helper() + tick := make(chan time.Time) + wr, err := NewWorkflowRegistry( + logger.TestLogger(t), + func(ctx context.Context, bytes []byte) (types.ContractReader, error) { return nil, nil }, + "", // no contract source + "test-chain-selector", + Config{QueryCount: 20, SyncStrategy: SyncStrategyReconciliation}, + h, + &testDonNotifier{don: commonCap.DON{ID: 1}}, + er, + append([]Option{WithTicker(tick)}, opts...)..., + ) + require.NoError(t, err) + wr.workflowSources = sources + return wr, tick +} + +// runOneSweepTick drives the reconciliation loop through at least one complete +// tick. The ticker channel is unbuffered, so the second send only succeeds +// once the loop has fully processed the first tick — including the orphan +// sweep decision — and returned to its select. +func runOneSweepTick(t *testing.T, wr *workflowRegistry, tick chan time.Time) { + t.Helper() + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan struct{}) + go func() { + defer close(done) + wr.syncUsingReconciliationStrategy(ctx) + }() + tick <- time.Now() + tick <- time.Now() // barrier: the first tick has been fully processed + cancel() + <-done +} + +// Test_syncLoop_OrphanedSpecs exercises orphaned-spec reconciliation end to +// end through syncUsingReconciliationStrategy: orphans are released per +// source, and a source failure only defers its own specs. +func Test_syncLoop_OrphanedSpecs(t *testing.T) { + t.Parallel() + + liveID := wfTypes.WorkflowID{1} + orphanID := wfTypes.WorkflowID{2} + liveMeta := WorkflowMetadataView{ + WorkflowID: liveID, + Owner: []byte{0xaa, 0xbb, 0xcc, 0xdd}, + Status: WorkflowStatusActive, + WorkflowName: "live-wf", + BinaryURL: "b1", + ConfigURL: "c1", + Source: "source-a", + } + + t.Run("engine-less orphan is released through the event path", func(t *testing.T) { + t.Parallel() + h := &orphanSweepFakeHandler{specs: []*job.WorkflowSpec{ + {WorkflowID: liveID.Hex(), WorkflowOwner: "aabbccdd", Source: "source-a"}, + {WorkflowID: orphanID.Hex(), WorkflowOwner: "aabbccdd", Source: "source-a"}, + }} + wr, tick := newSweepLoopRegistry(t, h, NewEngineRegistry(), []WorkflowMetadataSource{ + &fakeMetadataSource{name: "source-a", metadata: []WorkflowMetadataView{liveMeta}}, + }) + + runOneSweepTick(t, wr, tick) + + deletes := h.OrphanDeletes() + require.NotEmpty(t, deletes, "the orphan must be released") + for _, id := range deletes { + assert.Equal(t, orphanID.Hex(), id, "only the orphan is released") + } + }) + + t.Run("any failing source defers orphan cleanup", func(t *testing.T) { + t.Parallel() + // The union of source metadata is incomplete when a source fails: an + // absent row may belong to the failed source, so no row can be safely + // judged orphaned. Healthy sources still reconcile their engines. + okMeta := liveMeta + okMeta.Source = "source-ok" + h := &orphanSweepFakeHandler{specs: []*job.WorkflowSpec{ + {WorkflowID: liveID.Hex(), WorkflowOwner: "aabbccdd", Source: "source-ok"}, + {WorkflowID: orphanID.Hex(), WorkflowOwner: "aabbccdd", Source: "source-bad"}, + }} + wr, tick := newSweepLoopRegistry(t, h, NewEngineRegistry(), []WorkflowMetadataSource{ + &fakeMetadataSource{name: "source-ok", metadata: []WorkflowMetadataView{okMeta}}, + &fakeMetadataSource{name: "source-bad", err: assert.AnError}, + }) + + runOneSweepTick(t, wr, tick) + + assert.NotEmpty(t, h.Handled(), "healthy source still reconciles") + assert.Empty(t, h.OrphanDeletes(), "no spec is released while any source is failing") + }) + + t.Run("shard filter error skips the source entirely", func(t *testing.T) { + t.Parallel() + h := &orphanSweepFakeHandler{specs: []*job.WorkflowSpec{ + {WorkflowID: liveID.Hex(), WorkflowOwner: "aabbccdd", Source: "source-a"}, + {WorkflowID: orphanID.Hex(), WorkflowOwner: "aabbccdd", Source: "source-a"}, + }} + wr, tick := newSweepLoopRegistry(t, h, NewEngineRegistry(), []WorkflowMetadataSource{ + &fakeMetadataSource{name: "source-a", metadata: []WorkflowMetadataView{liveMeta}}, + }, + WithShardEnabled(true), + WithShardID(0), + WithShardOrchestratorClient(&failingShardMappingClient{}), + ) + + runOneSweepTick(t, wr, tick) + + assert.Empty(t, h.Handled(), "no events dispatched and no spec released when the shard filter fails") + }) + + t.Run("workflow filtered to another shard is swept", func(t *testing.T) { + t.Parallel() + myID := wfTypes.WorkflowID{3} + movedID := wfTypes.WorkflowID{4} + myMeta := liveMeta + myMeta.WorkflowID = myID + movedMeta := liveMeta + movedMeta.WorkflowID = movedID + h := &orphanSweepFakeHandler{specs: []*job.WorkflowSpec{ + {WorkflowID: myID.Hex(), WorkflowOwner: "aabbccdd", Source: "source-a"}, + {WorkflowID: movedID.Hex(), WorkflowOwner: "aabbccdd", Source: "source-a"}, + }} + wr, tick := newSweepLoopRegistry(t, h, NewEngineRegistry(), []WorkflowMetadataSource{ + &fakeMetadataSource{name: "source-a", metadata: []WorkflowMetadataView{myMeta, movedMeta}}, + }, + WithShardEnabled(true), + WithShardID(0), + WithShardOrchestratorClient(&mockShardMappingClient{mappings: map[string]uint32{ + myID.Hex(): 0, // stays on this shard + movedID.Hex(): 1, // moved to another shard + }}), + ) + + runOneSweepTick(t, wr, tick) + + // The moved workflow is absent from this shard's post-filter metadata + // and has no engine, so its spec is released here (-1); the receiving + // shard registers it (+1) under a distinct event_id — net level 0. + deletes := h.OrphanDeletes() + require.NotEmpty(t, deletes, "the shard-moved workflow must be swept") + for _, id := range deletes { + assert.Equal(t, movedID.Hex(), id, "only the moved workflow is released") + } + }) +} diff --git a/core/store/migrate/migrations/0302_workflow_specs_v2_registered_at.sql b/core/store/migrate/migrations/0302_workflow_specs_v2_registered_at.sql new file mode 100644 index 00000000000..9758ac3d9e6 --- /dev/null +++ b/core/store/migrate/migrations/0302_workflow_specs_v2_registered_at.sql @@ -0,0 +1,5 @@ +-- +goose Up +ALTER TABLE workflow_specs_v2 ADD COLUMN registered_at bigint NOT NULL DEFAULT 0; + +-- +goose Down +ALTER TABLE workflow_specs_v2 DROP COLUMN registered_at; diff --git a/core/store/migrate/migrations/0303_workflow_specs_v2_source.sql b/core/store/migrate/migrations/0303_workflow_specs_v2_source.sql new file mode 100644 index 00000000000..8e4020af371 --- /dev/null +++ b/core/store/migrate/migrations/0303_workflow_specs_v2_source.sql @@ -0,0 +1,9 @@ +-- +goose Up +-- Records which metadata source (e.g. ContractWorkflowSource) produced each +-- persisted workflow spec. +-- Pre-migration rows default to '' and are filled opportunistically on +-- the next event that touches them. +ALTER TABLE workflow_specs_v2 ADD COLUMN source text NOT NULL DEFAULT ''; + +-- +goose Down +ALTER TABLE workflow_specs_v2 DROP COLUMN source; diff --git a/core/web/resolver/testdata/config-empty-effective.toml b/core/web/resolver/testdata/config-empty-effective.toml index f0e139720dd..cc77a2a4f6d 100644 --- a/core/web/resolver/testdata/config-empty-effective.toml +++ b/core/web/resolver/testdata/config-empty-effective.toml @@ -377,6 +377,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index 6bd020acb91..3f09ea51413 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -395,6 +395,16 @@ Foo = 'bar' Enabled = true Prefixes = ['ocr_'] +[Metering] +MeterRecordsEnabled = true +MeterSnapshotsEnabled = true +Product = 'cre' +Tenant = 'mainline' +NumericTenantID = '42' +Environment = 'production' +Zone = 'wf-zone-a' +NodeID = 'clp-cre-wf-zone-a-1' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index c5ab5449c14..b7eb3ab1365 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -377,6 +377,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/deployment/go.mod b/deployment/go.mod index 65331723e55..484c64a75fc 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -451,7 +451,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/deployment/go.sum b/deployment/go.sum index 19a2092efc1..9141eade609 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1460,8 +1460,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 4f7a7d8487d..311d04b7510 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -2669,6 +2669,71 @@ Prefixes = ["go_"] # Default Prefixes is a set of filters to restrict which prometheus metrics are forwarded based on prefix matching. By default, we only forward the go runtime metrics. Empty means forward everything. +## Metering +```toml +[Metering] +MeterRecordsEnabled = false # Default +MeterSnapshotsEnabled = false # Default +Product = 'cre' # Default +Tenant = '' # Default +NumericTenantID = '' # Default +Environment = '' # Default +Zone = '' # Default +NodeID = '' # Default +``` +Metering configures durable resource metering emission and the coarse +deployment/node identity dimensions stamped on emitted MeterRecords and +MeterSnapshots. + +### MeterRecordsEnabled +```toml +MeterRecordsEnabled = false # Default +``` +MeterRecordsEnabled enables durable MeterRecord emission for LOOP plugins. + +### MeterSnapshotsEnabled +```toml +MeterSnapshotsEnabled = false # Default +``` +MeterSnapshotsEnabled enables durable MeterSnapshot emission for LOOP plugins. +Requires MeterRecordsEnabled = true. + +### Product +```toml +Product = 'cre' # Default +``` +Product is the deployment product identity dimension, e.g. 'cre'. + +### Tenant +```toml +Tenant = '' # Default +``` +Tenant is the human-readable tenant name, e.g. 'mainline'. + +### NumericTenantID +```toml +NumericTenantID = '' # Default +``` +NumericTenantID is the numbered tenant identifier represented as a string. + +### Environment +```toml +Environment = '' # Default +``` +Environment is the deployment environment identity dimension, e.g. 'production'. + +### Zone +```toml +Zone = '' # Default +``` +Zone is the deployment zone identity dimension, e.g. 'wf-zone-a'. + +### NodeID +```toml +NodeID = '' # Default +``` +NodeID is the node's logical name, e.g. 'clp-cre-wf-zone-a-1' (not the CSA public key). + ## CRE.Streams ```toml [CRE.Streams] diff --git a/go.mod b/go.mod index 832bd228d9c..04ffc6da9fd 100644 --- a/go.mod +++ b/go.mod @@ -99,6 +99,7 @@ require ( github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260804191526-b7a850ae7648 github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd @@ -355,7 +356,6 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-discovery v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-rules v0.0.0-20260505131349-78e491b80735 // indirect github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect github.com/smartcontractkit/chainlink-protos/svr v1.3.0 // indirect github.com/smartcontractkit/chainlink-tron/relayer v0.0.11-0.20260408092456-3c6369888d4a // indirect github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 // indirect diff --git a/go.sum b/go.sum index e8f911b149a..2f0d8cfd83a 100644 --- a/go.sum +++ b/go.sum @@ -1201,8 +1201,8 @@ github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546- github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36/go.mod h1:vL1bDgPSJjV0EqHYs4dDlR+EEE0cJchgvGLYXhwIjXY= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 h1:NXKTdIESAiCkVnPS6dyZP+NXVek3GzXa6P4uFAs0o8Y= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 64214917fe7..fbe4249cef3 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -62,7 +62,7 @@ require ( require ( github.com/bytecodealliance/wasmtime-go/v47 v47.0.0 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect ) require ( diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 0595119f03e..35845e9cfc7 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1447,8 +1447,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 203efd25685..11196f8766b 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -503,7 +503,7 @@ require ( github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 6a95a3adb7e..5a426246fc9 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1683,8 +1683,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/plugins/loop_registry.go b/plugins/loop_registry.go index ffcbfe1145f..76523a7894e 100644 --- a/plugins/loop_registry.go +++ b/plugins/loop_registry.go @@ -39,6 +39,7 @@ type LoopRegistry struct { autoPPROF config.AutoPprof cfgTracing config.Tracing cfgTelemetry config.Telemetry + cfgMetering config.Metering telemetryAuthHeaders map[string]string telemetryAuthPubKeyHex string cfgLOOPP config.LOOPP @@ -46,7 +47,7 @@ type LoopRegistry struct { func NewLoopRegistry(lggr logger.Logger, appID string, featureLogPoller bool, dbConfig config.Database, mercury dataengine.Mercury, pyroscope config.Pyroscope, autoPPROF config.AutoPprof, tracing config.Tracing, telemetry config.Telemetry, - telemetryAuthHeaders map[string]string, telemetryAuthPubKeyHex string, looppCfg config.LOOPP) *LoopRegistry { + metering config.Metering, telemetryAuthHeaders map[string]string, telemetryAuthPubKeyHex string, looppCfg config.LOOPP) *LoopRegistry { return &LoopRegistry{ registry: map[string]*RegisteredLoop{}, lggr: logger.Named(lggr, "LoopRegistry"), @@ -58,6 +59,7 @@ func NewLoopRegistry(lggr logger.Logger, appID string, featureLogPoller bool, db autoPPROF: autoPPROF, cfgTracing: tracing, cfgTelemetry: telemetry, + cfgMetering: metering, telemetryAuthHeaders: telemetryAuthHeaders, telemetryAuthPubKeyHex: telemetryAuthPubKeyHex, cfgLOOPP: looppCfg, @@ -175,6 +177,20 @@ func (m *LoopRegistry) Register(id string) (*RegisteredLoop, error) { envCfg.TelemetryPrometheusBridgeEnabled = m.cfgTelemetry.PrometheusBridge().Enabled() envCfg.TelemetryPrometheusBridgePrefixes = m.cfgTelemetry.PrometheusBridge().Prefixes() } + + // Metering config (emission toggles + deployment/node identity dimensions) + // is passed over the env channel to every LOOP plugin, rather than the + // standard-capabilities boundary. + if m.cfgMetering != nil { + envCfg.MeterRecordsEnabled = m.cfgMetering.MeterRecordsEnabled() + envCfg.MeterSnapshotsEnabled = m.cfgMetering.MeterSnapshotsEnabled() + envCfg.MeterProduct = m.cfgMetering.Product() + envCfg.MeterTenant = m.cfgMetering.Tenant() + envCfg.MeterNumericTenantID = m.cfgMetering.NumericTenantID() + envCfg.MeterEnvironment = m.cfgMetering.Environment() + envCfg.MeterZone = m.cfgMetering.Zone() + envCfg.MeterNodeID = m.cfgMetering.NodeID() + } m.lggr.Debugf("Registered loopp %q with port %d", id, envCfg.PrometheusPort) // Add auth header after logging config diff --git a/plugins/loop_registry_test.go b/plugins/loop_registry_test.go index 847d0a8a20b..132c4b8c2e6 100644 --- a/plugins/loop_registry_test.go +++ b/plugins/loop_registry_test.go @@ -122,6 +122,17 @@ func (m mockCfgTelemetry) PrometheusBridge() config.PrometheusBridge { return mockPrometheusBridge{} } +type mockCfgMetering struct{} + +func (m mockCfgMetering) MeterRecordsEnabled() bool { return true } +func (m mockCfgMetering) MeterSnapshotsEnabled() bool { return true } +func (m mockCfgMetering) Product() string { return "cre" } +func (m mockCfgMetering) Tenant() string { return "mainline" } +func (m mockCfgMetering) NumericTenantID() string { return "42" } +func (m mockCfgMetering) Environment() string { return "production" } +func (m mockCfgMetering) Zone() string { return "wf-zone-a" } +func (m mockCfgMetering) NodeID() string { return "clp-cre-wf-zone-a-1" } + type mockPrometheusBridge struct{} func (m mockPrometheusBridge) Enabled() bool { return true } @@ -215,6 +226,7 @@ func TestLoopRegistry_Register(t *testing.T) { mockCfgMercury := &mockCfgMercury{} mockCfgTracing := &mockCfgTracing{} mockCfgTelemetry := &mockCfgTelemetry{} + mockCfgMetering := &mockCfgMetering{} registry := make(map[string]*RegisteredLoop) // Create a LoopRegistry instance with mockCfgTracing @@ -227,6 +239,7 @@ func TestLoopRegistry_Register(t *testing.T) { cfgMercury: mockCfgMercury, cfgTracing: mockCfgTracing, cfgTelemetry: mockCfgTelemetry, + cfgMetering: mockCfgMetering, } // Test case 1: Register new loop @@ -280,6 +293,14 @@ func TestLoopRegistry_Register(t *testing.T) { require.Equal(t, 512, envCfg.TelemetryLogExportMaxBatchSize) require.Equal(t, 5*time.Second, envCfg.TelemetryLogExportInterval) require.Equal(t, 2048, envCfg.TelemetryLogMaxQueueSize) + require.True(t, envCfg.MeterRecordsEnabled) + require.True(t, envCfg.MeterSnapshotsEnabled) + require.Equal(t, "cre", envCfg.MeterProduct) + require.Equal(t, "mainline", envCfg.MeterTenant) + require.Equal(t, "42", envCfg.MeterNumericTenantID) + require.Equal(t, "production", envCfg.MeterEnvironment) + require.Equal(t, "wf-zone-a", envCfg.MeterZone) + require.Equal(t, "clp-cre-wf-zone-a-1", envCfg.MeterNodeID) require.Equal(t, []string{"event_id"}, envCfg.TelemetryMetricViewsDenyAttributes) require.NotNil(t, envCfg.TelemetryMetricCardinalityLimit) require.Equal(t, 100000, *envCfg.TelemetryMetricCardinalityLimit) diff --git a/system-tests/lib/cre/don/config/config.go b/system-tests/lib/cre/don/config/config.go index 811d0c1b240..eb1f324db4e 100644 --- a/system-tests/lib/cre/don/config/config.go +++ b/system-tests/lib/cre/don/config/config.go @@ -115,6 +115,7 @@ func PrepareNodeTOMLs( Topology: topology, Provider: creEnv.Provider, ChipRouterInternalGRPCURL: chipRouterInternalGRPCURL, + EnableMetering: localNodeSets[i].EnableMetering, }, configFactoryFunctions, ) @@ -186,6 +187,9 @@ func PrepareNodeTOMLs( for i := range localNodeSets { for j := range localNodeSets[i].NodeSpecs { if localNodeSets[i].NodeSpecs[j].Node.UserConfigOverrides != "" { + if err := validateUserConfigOverrides(localNodeSets[i].NodeSpecs[j].Node.UserConfigOverrides); err != nil { + return nil, errors.Wrapf(err, "invalid user_config_overrides for nodeset %q node %d", localNodeSets[i].Name, j) + } localNodeSets[i].NodeSpecs[j].Node.UserConfigOverrides = transformUserConfigOverrides( localNodeSets[i].NodeSpecs[j].Node.UserConfigOverrides, ) @@ -450,6 +454,10 @@ func addWorkerNodeConfig( URL: new("billing-platform-service:2223"), TLSEnabled: new(false), } + + if commonInputs.enableMetering { + existingConfig.Metering = meteringNodeConfig(donMetadata, m.Index) + } } // Preserve existing WorkflowRegistry config (e.g., AdditionalSourcesConfig from user_config_overrides) @@ -570,6 +578,23 @@ func addWorkerNodeConfig( return existingConfig, nil } +// meteringNodeConfig returns the [Metering] section for a worker node with +// local-cre deployment dimensions. NodeID is derived from the DON name and +// node index to be unique per node within the DON (required by +// Metering.ValidateConfig and per-node snapshot dedup). +func meteringNodeConfig(donMetadata *cre.DonMetadata, nodeIndex int) coretoml.Metering { + return coretoml.Metering{ + MeterRecordsEnabled: new(true), + MeterSnapshotsEnabled: new(true), + Product: new("cre"), + Tenant: new("local-cre"), + NumericTenantID: new("1"), + Environment: new("local"), + Zone: new(donMetadata.Name), + NodeID: new(donMetadata.Name + "-node-" + strconv.Itoa(nodeIndex)), + } +} + func addGatewayNodeConfig( existingConfig corechainlink.Config, ocrPeeringData cre.OCRPeeringData, @@ -672,6 +697,7 @@ type commonInputs struct { provider infra.Provider chipRouterInternalGRPCURL string + enableMetering bool } func gatherCommonInputs(input cre.GenerateConfigsInput) (*commonInputs, error) { @@ -716,6 +742,7 @@ func gatherCommonInputs(input cre.GenerateConfigsInput) (*commonInputs, error) { }, provider: input.Provider, chipRouterInternalGRPCURL: input.ChipRouterInternalGRPCURL, + enableMetering: input.EnableMetering, }, nil } @@ -971,6 +998,37 @@ func transformAdditionalSourceURLs(sources []coretoml.AdditionalWorkflowSource) return transformed } +// frameworkManagedSections are top-level node-config tables the framework +// generates itself (see baseNodeConfig/addWorkerNodeConfig): Telemetry points +// nodes at the chip-router (the test sink and other consumers subscribe to the +// router, so overriding ChipIngressEndpoint silently bypasses them), Billing +// at the local billing service, and Metering is gated by the nodeset-level +// enable_metering flag. Overriding any of them in user_config_overrides wins +// the last-one-wins config merge on the node and desynchronizes the +// environment from what the framework wired up. +var frameworkManagedSections = []string{"Telemetry", "Billing", "Metering"} + +// validateUserConfigOverrides rejects user_config_overrides that set +// framework-managed config tables. It parses the TOML rather than string +// matching so comments mentioning these sections stay legal. +func validateUserConfigOverrides(userConfig string) error { + var parsed map[string]any + if err := toml.Unmarshal([]byte(userConfig), &parsed); err != nil { + return errors.Wrap(err, "user_config_overrides is not valid TOML") + } + for _, section := range frameworkManagedSections { + if _, found := parsed[section]; found { + return errors.Errorf( + "[%s] is framework-managed and cannot be set via user_config_overrides; "+ + "the framework generates it (pointing telemetry at the chip-router). "+ + "For metering, set enable_metering = true on the nodeset instead", + section, + ) + } + } + return nil +} + // transformUserConfigOverrides transforms URLs in a user config overrides string to use // platform-specific Docker host addresses. This handles differences between macOS // (host.docker.internal) and Linux (172.17.0.1 or similar) Docker host resolution. diff --git a/system-tests/lib/cre/don/config/config_test.go b/system-tests/lib/cre/don/config/config_test.go new file mode 100644 index 00000000000..a1a06e21b9b --- /dev/null +++ b/system-tests/lib/cre/don/config/config_test.go @@ -0,0 +1,199 @@ +package config + +import ( + "strconv" + "testing" + + "github.com/Masterminds/semver/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink/system-tests/lib/cre" + "github.com/smartcontractkit/chainlink/system-tests/lib/infra" + corechainlink "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" +) + +func derefBool(p *bool) bool { + if p == nil { + return false + } + return *p +} + +func derefStr(p *string) string { + if p == nil { + return "" + } + return *p +} + +func TestMeteringNodeConfig(t *testing.T) { + t.Parallel() + + dm := &cre.DonMetadata{Name: "workflow"} + got := meteringNodeConfig(dm, 0) + + assert.True(t, derefBool(got.MeterRecordsEnabled)) + assert.True(t, derefBool(got.MeterSnapshotsEnabled)) + assert.Equal(t, "cre", derefStr(got.Product)) + assert.Equal(t, "local-cre", derefStr(got.Tenant)) + assert.Equal(t, "1", derefStr(got.NumericTenantID)) + assert.Equal(t, "local", derefStr(got.Environment)) + assert.Equal(t, "workflow", derefStr(got.Zone)) + assert.Equal(t, "workflow-node-0", derefStr(got.NodeID)) +} + +func TestMeteringNodeConfig_NodeIDUniqueness(t *testing.T) { + t.Parallel() + + dm := &cre.DonMetadata{Name: "capabilities"} + seen := make(map[string]struct{}) + for i := range 4 { + got := meteringNodeConfig(dm, i) + nodeID := derefStr(got.NodeID) + expected := "capabilities-node-" + strconv.Itoa(i) + assert.Equal(t, expected, nodeID) + _, dup := seen[nodeID] + assert.False(t, dup, "NodeID %q duplicated", nodeID) + seen[nodeID] = struct{}{} + } +} + +func TestValidateUserConfigOverrides(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + overrides string + wantErr string + }{ + { + name: "allows non-managed sections", + overrides: "[Log]\nLevel = 'debug'\n\n[CRE.WorkflowFetcher]\nURL = 'file:///home/chainlink/workflows'\n", + }, + { + name: "allows comments mentioning managed sections", + overrides: "# do not set [Telemetry] or [Metering] here\n[Log]\nLevel = 'debug'\n", + }, + { + name: "rejects Telemetry", + overrides: "[Telemetry]\nChipIngressEndpoint = 'chip-ingress:50051'\n", + wantErr: "[Telemetry] is framework-managed", + }, + { + name: "rejects Metering", + overrides: "[Metering]\nMeterRecordsEnabled = true\n", + wantErr: "[Metering] is framework-managed", + }, + { + name: "rejects Billing", + overrides: "[Billing]\nURL = 'host.docker.internal:2223'\n", + wantErr: "[Billing] is framework-managed", + }, + { + name: "rejects invalid TOML", + overrides: "[Log\nLevel = 'debug'\n", + wantErr: "not valid TOML", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + err := validateUserConfigOverrides(tc.overrides) + if tc.wantErr == "" { + assert.NoError(t, err) + } else { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + } + }) + } +} + +// TestAddWorkerNodeConfig_MeteringAndChipIngress is a thin integration test +// through addWorkerNodeConfig on a capabilities DON (not WorkflowDON/ShardDON) +// to verify: +// 1. [Metering] is populated when enableMetering is true +// 2. Telemetry.ChipIngressEndpoint equals the chip-router URL passed in +// +// topology is nil because the capabilities path never accesses it: the +// WorkflowDON and ShardDON branches are both skipped when Flags does not +// contain "workflow" or "shard". +func TestAddWorkerNodeConfig_MeteringAndChipIngress(t *testing.T) { + t.Parallel() + + const routerURL = "chip-router:50051" + + inputs := capabilitiesCommonInputs(routerURL) + inputs.enableMetering = true + + config, err := addWorkerNodeConfig(corechainlink.Config{}, nil, testOCRPeeringData(), inputs, capabilitiesDonMetadata(), &cre.NodeMetadata{ + Index: 1, + Roles: []string{cre.WorkerNode}, + }) + require.NoError(t, err) + + // [Metering] is present with the expected NodeID + require.NotNil(t, config.Metering.MeterRecordsEnabled) + assert.True(t, *config.Metering.MeterRecordsEnabled) + assert.Equal(t, "capabilities-node-1", *config.Metering.NodeID) + + // Telemetry.ChipIngressEndpoint equals the router URL — regression test for + // the original bug where user_config_overrides hardcoded chip-ingress:50051 + require.NotNil(t, config.Telemetry.ChipIngressEndpoint) + assert.Equal(t, routerURL, *config.Telemetry.ChipIngressEndpoint) +} + +// TestAddWorkerNodeConfig_MeteringDisabled verifies no [Metering] is injected +// when enableMetering is false, while ChipIngressEndpoint still points at the +// router. +func TestAddWorkerNodeConfig_MeteringDisabled(t *testing.T) { + t.Parallel() + + const routerURL = "chip-router:50051" + + config, err := addWorkerNodeConfig(corechainlink.Config{}, nil, testOCRPeeringData(), capabilitiesCommonInputs(routerURL), capabilitiesDonMetadata(), &cre.NodeMetadata{ + Index: 0, + Roles: []string{cre.WorkerNode}, + }) + require.NoError(t, err) + + assert.Nil(t, config.Metering.MeterRecordsEnabled, "Metering should not be set when disabled") + require.NotNil(t, config.Telemetry.ChipIngressEndpoint) + assert.Equal(t, routerURL, *config.Telemetry.ChipIngressEndpoint) +} + +func capabilitiesDonMetadata() *cre.DonMetadata { + return &cre.DonMetadata{ + Name: "capabilities", + Flags: []string{"capabilities"}, + } +} + +func capabilitiesCommonInputs(routerURL string) *commonInputs { + return &commonInputs{ + provider: infra.Provider{Type: "docker"}, + chipRouterInternalGRPCURL: routerURL, + registryChainID: 1337, + registryChainSelector: 1, + capabilityRegistry: versionedAddress{address: "0xdead", version: mustSemVer("1.0.0")}, + workflowRegistry: versionedAddress{address: "0xbeef", version: mustSemVer("1.0.0")}, + } +} + +func testOCRPeeringData() cre.OCRPeeringData { + return cre.OCRPeeringData{ + OCRBootstraperPeerID: "12D3KooWPjceQrSwdWXPyLLeABRXmuqt69Rg3sBYbU1Nft9HyQ6X", + OCRBootstraperHost: "bootstrap", + Port: 4222, + } +} + +func mustSemVer(v string) *semver.Version { + ver, err := semver.NewVersion(v) + if err != nil { + panic(err) + } + return ver +} diff --git a/system-tests/lib/cre/types.go b/system-tests/lib/cre/types.go index 13568bb97ad..2ce0687d527 100644 --- a/system-tests/lib/cre/types.go +++ b/system-tests/lib/cre/types.go @@ -450,6 +450,7 @@ type GenerateConfigsInput struct { Topology *Topology Provider infra.Provider ChipRouterInternalGRPCURL string + EnableMetering bool } func (g *GenerateConfigsInput) Validate() error { @@ -1284,6 +1285,10 @@ type NodeSet struct { // GatewayDonID is the gateway DON used for multi-gateway HTTP action routing on gateway nodesets. GatewayDonID string `toml:"gateway_don_id"` + // EnableMetering turns on framework-generated [Metering] node config for this + // nodeset's worker nodes (durable MeterRecord/MeterSnapshot emission). + EnableMetering bool `toml:"enable_metering"` + chainCapabilityIndex map[CapabilityFlag][]uint64 chainCapabilityIndexBuilt bool } diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index 5042b8a4d96..3a51b8f79ba 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -474,7 +474,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-rules v0.0.0-20260505131349-78e491b80735 // indirect github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-protos/orchestrator v0.11.0 // indirect github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd // indirect diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index 3e3b2880693..8a61aaae388 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -1597,8 +1597,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index ea4ff37ae0e..4dbd8df2ef5 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -249,7 +249,7 @@ require ( github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d // indirect github.com/smartcontractkit/chainlink-protos/data-feeds v0.1.1-0.20260501174546-2e8846986b36 // indirect github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea // indirect - github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/chainlink-solana/contracts v0.0.0-20260513123719-d347eaf314e1 // indirect github.com/smartcontractkit/chainlink-sui/codec v0.0.0-20260720132736-e99278bfdc96 // indirect diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index 3211c1ffea4..3d7b95b45c6 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -1802,8 +1802,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.202607011 github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea/go.mod h1:2ahgl5bI9+fMCD+dBC785Lak38Tb7ApdTe5I8a09Qp0= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305 h1:NJdGFhzT6zMaTod4QkBqVD2sg0I25iw1boOYtTpEwRo= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20260512230622-65f10f4cd305/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= -github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536 h1:ecQYtdRA+NQLXf0aKYUMfcn1TRhcQ4RZCZzzadFsbSs= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260729184203-90b4cdd48536/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/op-catalog v0.1.0 h1:hGEJFD2X3oNIPXQbtIPxCJyg5CcKglRCYBmESS+gmeQ= diff --git a/testdata/scripts/config/merge_raw_configs.txtar b/testdata/scripts/config/merge_raw_configs.txtar index 821f2f5a76d..0b79050ab19 100644 --- a/testdata/scripts/config/merge_raw_configs.txtar +++ b/testdata/scripts/config/merge_raw_configs.txtar @@ -524,6 +524,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/default.txtar b/testdata/scripts/node/validate/default.txtar index 534841af1bc..21ba967ff09 100644 --- a/testdata/scripts/node/validate/default.txtar +++ b/testdata/scripts/node/validate/default.txtar @@ -389,6 +389,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index ce093f64219..0ddff376935 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -450,6 +450,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index 1296b4e7d72..1011964d249 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -433,6 +433,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index 74b11162020..8ad9fe716ed 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -433,6 +433,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index 059d6ab65bd..1231e06b1a6 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -433,6 +433,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/fallback-override.txtar b/testdata/scripts/node/validate/fallback-override.txtar index 865b4aa0f56..51da7740280 100644 --- a/testdata/scripts/node/validate/fallback-override.txtar +++ b/testdata/scripts/node/validate/fallback-override.txtar @@ -535,6 +535,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar index b09445dc279..7739dc51fd6 100644 --- a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar +++ b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar @@ -418,6 +418,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index 1f9c08f31f0..be924e1de39 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -429,6 +429,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index e884c6f7efe..8ef8d7be402 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -430,6 +430,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200 diff --git a/testdata/scripts/node/validate/warnings.txtar b/testdata/scripts/node/validate/warnings.txtar index 07c178ad53b..09e11128a1c 100644 --- a/testdata/scripts/node/validate/warnings.txtar +++ b/testdata/scripts/node/validate/warnings.txtar @@ -412,6 +412,16 @@ MetricCardinalityLimit = 100000 Enabled = false Prefixes = ['go_'] +[Metering] +MeterRecordsEnabled = false +MeterSnapshotsEnabled = false +Product = 'cre' +Tenant = '' +NumericTenantID = '' +Environment = '' +Zone = '' +NodeID = '' + [Workflows] [Workflows.Limits] Global = 200