-
Notifications
You must be signed in to change notification settings - Fork 2k
adding resource manager for workflows #22845
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
7bafc6b
276ddee
937a244
8fd5251
d288672
1898360
58607d9
c662958
72b90d5
a500659
cc1ab09
1b333be
3d48b1c
af84741
9ca0e7f
54689c5
d17cd8b
bfb6a4b
a8829ad
27f3ad9
e6d049d
dcc0e66
4a4eb83
712d4fd
a3a78af
9192de6
2bb12c8
cc8ab0f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| package config | ||
|
|
||
| // Metering exposes durable resource-metering configuration: the emission | ||
| // toggles and the coarse deployment/node identity dimensions stamped on emitted | ||
| // MeterRecords and MeterSnapshots. These are passed via loop.EnvConfig to every LOOP | ||
| // plugin. | ||
| type Metering interface { | ||
| MeterRecordsEnabled() bool | ||
| MeterSnapshotsEnabled() bool | ||
| Product() string | ||
| Tenant() string | ||
| NumericTenantID() string | ||
| Environment() string | ||
| Zone() string | ||
| NodeID() string | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -64,6 +64,7 @@ type Core struct { | |
| Mercury Mercury `toml:",omitempty"` | ||
| Capabilities Capabilities `toml:",omitempty"` | ||
| Telemetry Telemetry `toml:",omitempty"` | ||
| Metering Metering `toml:",omitempty"` | ||
|
Comment on lines
66
to
+67
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I wonder if just
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. are
|
||
| Workflows Workflows `toml:",omitempty"` | ||
| CRE CreConfig `toml:",omitempty"` | ||
| Billing Billing `toml:",omitempty"` | ||
|
|
@@ -112,6 +113,7 @@ func (c *Core) SetFrom(f *Core) { | |
| c.JobDistributor.setFrom(&f.JobDistributor) | ||
| c.Tracing.setFrom(&f.Tracing) | ||
| c.Telemetry.setFrom(&f.Telemetry) | ||
| c.Metering.setFrom(&f.Metering) | ||
| c.CRE.setFrom(&f.CRE) | ||
| c.Billing.setFrom(&f.Billing) | ||
| c.BridgeStatusReporter.setFrom(&f.BridgeStatusReporter) | ||
|
|
@@ -3163,6 +3165,75 @@ func (b *Telemetry) ValidateConfig() (err error) { | |
| return err | ||
| } | ||
|
|
||
| // Metering configures durable resource metering emission and the coarse | ||
| // deployment/node identity dimensions stamped on emitted MeterRecords and | ||
| // MeterSnapshots. These are passed via loop.EnvConfig to every LOOP plugin. | ||
| type Metering struct { | ||
| // MeterRecordsEnabled enables durable MeterRecord emission for LOOP plugins. | ||
| MeterRecordsEnabled *bool | ||
| // MeterSnapshotsEnabled enables durable MeterSnapshot emission. Requires | ||
| // MeterRecordsEnabled to be true. | ||
| MeterSnapshotsEnabled *bool | ||
| // Product is the deployment product identity dimension, e.g. "cre". | ||
| Product *string | ||
| // Tenant is the human-readable tenant name, e.g. "mainline". | ||
| Tenant *string | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is this different from the cresettings
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. TenantID is also included, the human readable name is helpful for o11y and debugging |
||
| // NumericTenantID is the numbered tenant identifier as a string. | ||
| NumericTenantID *string | ||
| // Environment is the deployment environment dimension, e.g. "production". | ||
| Environment *string | ||
| // Zone is the deployment zone dimension, e.g. "wf-zone-a". | ||
| Zone *string | ||
| // NodeID is the node's logical name, e.g. "clp-cre-wf-zone-a-1" (NOT the CSA | ||
| // public key) | ||
| NodeID *string | ||
| } | ||
|
|
||
| func (b *Metering) setFrom(f *Metering) { | ||
| if v := f.MeterRecordsEnabled; v != nil { | ||
| b.MeterRecordsEnabled = v | ||
| } | ||
| if v := f.MeterSnapshotsEnabled; v != nil { | ||
| b.MeterSnapshotsEnabled = v | ||
| } | ||
| if v := f.Product; v != nil { | ||
| b.Product = v | ||
| } | ||
| if v := f.Tenant; v != nil { | ||
| b.Tenant = v | ||
| } | ||
| if v := f.NumericTenantID; v != nil { | ||
| b.NumericTenantID = v | ||
| } | ||
| if v := f.Environment; v != nil { | ||
| b.Environment = v | ||
| } | ||
| if v := f.Zone; v != nil { | ||
| b.Zone = v | ||
| } | ||
| if v := f.NodeID; v != nil { | ||
| b.NodeID = v | ||
| } | ||
| } | ||
|
|
||
| func (b *Metering) ValidateConfig() (err error) { | ||
| if b.MeterSnapshotsEnabled != nil && *b.MeterSnapshotsEnabled && (b.MeterRecordsEnabled == nil || !*b.MeterRecordsEnabled) { | ||
| err = errors.Join(err, configutils.ErrInvalid{ | ||
| Name: "MeterSnapshotsEnabled", | ||
| Value: true, | ||
| Msg: "requires MeterRecordsEnabled to be true", | ||
| }) | ||
| } | ||
| if b.MeterRecordsEnabled != nil && *b.MeterRecordsEnabled && (b.NodeID == nil || *b.NodeID == "") { | ||
| err = errors.Join(err, configutils.ErrInvalid{ | ||
| Name: "NodeID", | ||
| Value: "", | ||
| Msg: "must be non-empty when MeterRecordsEnabled is true (an empty NodeID collapses per-node snapshot dedup scope DON-wide)", | ||
| }) | ||
| } | ||
| return err | ||
| } | ||
|
|
||
| type PrometheusBridge struct { | ||
| Enabled *bool | ||
| Prefixes []string | ||
|
|
||
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You can get the numeric tenant ID from a cresetting that @prashantkumar1982 created just fyi
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is this the right use of it? cresettings are sourced from job specs, no? I'm wary of adding dependencies into multiple sources of config. The cap registry and TOML is already two places to manage and keep in-sync.