From 64e8914970ee575533223a0a7b596edcbe148b48 Mon Sep 17 00:00:00 2001 From: aminsammara Date: Mon, 27 Jul 2026 18:52:27 +0000 Subject: [PATCH 1/4] chore: add new client --- spartan/terraform/deploy-ethereum/variables.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/spartan/terraform/deploy-ethereum/variables.tf b/spartan/terraform/deploy-ethereum/variables.tf index b0eb38a82556..f0cd402ac1f1 100644 --- a/spartan/terraform/deploy-ethereum/variables.tf +++ b/spartan/terraform/deploy-ethereum/variables.tf @@ -6,6 +6,7 @@ variable "API_KEY_SECRET_NAMES" { "eth-sepolia-rpc-consumer-client2", "eth-sepolia-rpc-consumer-client3", "eth-sepolia-rpc-consumer-client4", + "eth-sepolia-rpc-consumer-client5", ] } From cb9009713e0e784293296b0f124df7ddad7a9c88 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Tue, 28 Jul 2026 09:20:01 +0000 Subject: [PATCH 2/4] feat: map pino log level to AWS severity --- yarn-project/foundation/src/config/env_var.ts | 1 + .../foundation/src/log/aws-logger-config.ts | 32 +++++++++++++++++++ .../foundation/src/log/pino-logger.ts | 18 ++++++++--- 3 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 yarn-project/foundation/src/log/aws-logger-config.ts diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index fa3c07ced9ef..09c3eacc19cc 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -100,6 +100,7 @@ export type EnvVar = | 'NETWORK' | 'NETWORK_CONFIG_LOCATION' | 'ALLOW_OVERRIDING_NETWORK_CONFIG' + | 'USE_AWS_LOGGING' | 'USE_GCLOUD_LOGGING' | 'OTEL_EXPORTER_OTLP_METRICS_ENDPOINT' | 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT' diff --git a/yarn-project/foundation/src/log/aws-logger-config.ts b/yarn-project/foundation/src/log/aws-logger-config.ts new file mode 100644 index 000000000000..a9c23e32c71c --- /dev/null +++ b/yarn-project/foundation/src/log/aws-logger-config.ts @@ -0,0 +1,32 @@ +import type { pino } from 'pino'; + +function getSeverity(label: string): { severityText: string; severityNumber: number } { + switch (label) { + case 'trace': + return { severityText: 'TRACE', severityNumber: 1 }; + case 'debug': + return { severityText: 'DEBUG', severityNumber: 5 }; + case 'verbose': + return { severityText: 'VERBOSE', severityNumber: 7 }; + case 'info': + return { severityText: 'INFO', severityNumber: 9 }; + case 'warn': + return { severityText: 'WARN', severityNumber: 13 }; + case 'error': + return { severityText: 'ERROR', severityNumber: 17 }; + case 'fatal': + return { severityText: 'FATAL', severityNumber: 21 }; + default: + return { severityText: 'UNSPECIFIED', severityNumber: 0 }; + } +} + +/** Pino configuration that adds OpenTelemetry severity fields understood by CloudWatch Logs. */ +export const AWSCloudLoggerConfig = { + messageKey: 'msg', + formatters: { + level(label: string, level: number): object { + return { level, ...getSeverity(label) }; + }, + }, +} satisfies pino.LoggerOptions; diff --git a/yarn-project/foundation/src/log/pino-logger.ts b/yarn-project/foundation/src/log/pino-logger.ts index ce264b40da56..b4a6a12b117e 100644 --- a/yarn-project/foundation/src/log/pino-logger.ts +++ b/yarn-project/foundation/src/log/pino-logger.ts @@ -7,6 +7,7 @@ import { inspect } from 'util'; import { compactArray } from '../collection/array.js'; import type { EnvVar } from '../config/index.js'; import { parseBooleanEnv } from '../config/parse-env.js'; +import { AWSCloudLoggerConfig } from './aws-logger-config.js'; import { convertBigintsToStrings } from './bigint-utils.js'; import { GoogleCloudLoggerConfig } from './gcloud-logger-config.js'; import { getLogLevelFromFilters, parseLogLevelEnvVar } from './log-filters.js'; @@ -134,8 +135,17 @@ export const [logLevel, logFilters] = parseLogLevelEnvVar(process.env.LOG_LEVEL, // Define custom logging levels for pino. const customLevels = { verbose: 25 }; -// Global pino options, tweaked for google cloud if running there. +// Global pino options, tweaked for the active cloud logging backend. const useGcloudLogging = parseBooleanEnv(process.env['USE_GCLOUD_LOGGING' satisfies EnvVar]); +const useAwsLogging = parseBooleanEnv(process.env['USE_AWS_LOGGING' satisfies EnvVar]); +const loggingConfig = useGcloudLogging + ? GoogleCloudLoggerConfig + : useAwsLogging + ? AWSCloudLoggerConfig + : { + formatters: {}, + messageKey: 'msg', + }; const redactedPaths = [ 'validatorPrivateKeys', @@ -156,7 +166,7 @@ const redactedPaths = [ const pinoOpts: pino.LoggerOptions = { customLevels, - messageKey: 'msg', + messageKey: loggingConfig.messageKey, useOnlyCustomLevels: false, level: logLevel, redact: { @@ -170,8 +180,8 @@ const pinoOpts: pino.LoggerOptions = { }, formatters: { log: obj => convertBigintsToStrings(obj) as Record, + ...loggingConfig.formatters, }, - ...(useGcloudLogging ? GoogleCloudLoggerConfig : {}), }; export const levels = { @@ -241,7 +251,7 @@ const pinoPrettyBaseOpts = { destination: 2, sync: true, colorize: useColor, - ignore: 'module,actor,instanceId,pid,hostname,trace_id,span_id,trace_flags,severity', + ignore: 'module,actor,instanceId,pid,hostname,trace_id,span_id,trace_flags,severity,severityText,severityNumber', customLevels: 'fatal:60,error:50,warn:40,info:30,verbose:25,debug:20,trace:10', customColors: 'fatal:bgRed,error:red,warn:yellow,info:green,verbose:magenta,debug:blue,trace:gray', minimumLevel: 'trace' as const, From 57f5f93a1ba6fac2d39a75c9c66b3ea8bfd6d96f Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Tue, 28 Jul 2026 11:36:58 +0000 Subject: [PATCH 3/4] chore(bench): scale inclusion-sweep prover agents to 100 The bench-inclusion-sweep environment capped prover-agent autoscaling at 10 replicas, which throttled proving throughput during high-TPS sweep runs. Raise PROVER_AGENT_KEDA_MAX_REPLICAS and the scaling band to 100 so the agent pool can keep pace with the offered load. --- spartan/environments/bench-inclusion-sweep.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spartan/environments/bench-inclusion-sweep.env b/spartan/environments/bench-inclusion-sweep.env index 393dadcbf755..d11fa127d34c 100644 --- a/spartan/environments/bench-inclusion-sweep.env +++ b/spartan/environments/bench-inclusion-sweep.env @@ -57,11 +57,11 @@ PROVER_TEST_VERIFICATION_DELAY_MS=60 PROVER_AGENT_KEDA_ENABLED=true PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS=REPLACE_WITH_GCP_SECRET PROVER_AGENT_KEDA_MIN_REPLICAS=0 -PROVER_AGENT_KEDA_MAX_REPLICAS=10 +PROVER_AGENT_KEDA_MAX_REPLICAS=100 PROVER_AGENT_KEDA_SCALING_BANDS='[ { queueSize = 0 - replicas = 10 + replicas = 100 } ]' From c4ec3ed1c63720634aa138b62a69438f6ffb2d40 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Tue, 28 Jul 2026 12:18:06 +0000 Subject: [PATCH 4/4] chore: delete unused envs --- spartan/environments/mainnet-v4.env | 81 --------------------- spartan/environments/testnet-v4.env | 105 ---------------------------- 2 files changed, 186 deletions(-) delete mode 100644 spartan/environments/mainnet-v4.env delete mode 100644 spartan/environments/testnet-v4.env diff --git a/spartan/environments/mainnet-v4.env b/spartan/environments/mainnet-v4.env deleted file mode 100644 index ad6bc007c6e8..000000000000 --- a/spartan/environments/mainnet-v4.env +++ /dev/null @@ -1,81 +0,0 @@ -NETWORK=${NETWORK:-mainnet} -L1_NETWORK=${L1_NETWORK:-mainnet} -ETHEREUM_CHAIN_ID=${ETHEREUM_CHAIN_ID:-1} - -GCP_REGION=us-west1-a -CLUSTER=aztec-gke-public -RESOURCE_PROFILE=prod -NAMESPACE=${NAMESPACE:-mainnet-v4} - -RPC_GATEWAY_ENABLED=false - -CREATE_ROLLUP_CONTRACTS=false -VERIFY_CONTRACTS=false -DEPLOY_INTERNAL_BOOTNODE=false -VALIDATOR_REPLICAS=0 -VALIDATOR_HA_REPLICAS=0 -RPC_REPLICAS=0 -FULL_NODE_REPLICAS=0 -FISHERMAN_REPLICAS=0 -BOT_TRANSFERS_REPLICAS=0 -BOT_SWAPS_REPLICAS=0 -BOT_CROSS_CHAIN_REPLICAS=0 - -PROVER_ENABLED=true -PROVER_REPLICAS=4 -PROVER_AGENTS_PER_PROVER=1 -PUBLISHERS_PER_PROVER=2 -PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000 - -# !!! -PROVER_NODE_DISABLE_PROOF_PUBLISH=false - -PROVER_AGENT_KEDA_ENABLED=true -PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS=REPLACE_WITH_GCP_SECRET -PROVER_AGENT_KEDA_MIN_REPLICAS=32 -PROVER_AGENT_KEDA_MAX_REPLICAS=100 -PROVER_AGENT_KEDA_SCALING_BANDS='[ - { - queueSize = 0 - replicas = 32 - }, - { - queueSize = 200 - replicas = 80 - } -]' - -RPC_RESOURCE_PROFILE=prod -PROVER_RESOURCE_PROFILE=prod-fat-agents - -LOG_LEVEL=info -USE_NETWORK_CONFIG=true - -PROVER_FAILED_PROOF_STORE=gs://aztec-develop/mainnet-v4/failed-proofs -L1_TX_FAILED_STORE=gs://aztec-develop/mainnet-v4/failed-l1-txs - -ETHEREUM_RPC_URLS=REPLACE_WITH_GCP_SECRET -ETHEREUM_CONSENSUS_HOST_URLS=REPLACE_WITH_GCP_SECRET -ETHEREUM_CONSENSUS_HOST_API_KEYS=REPLACE_WITH_GCP_SECRET -ETHEREUM_CONSENSUS_HOST_API_KEY_HEADERS=REPLACE_WITH_GCP_SECRET -LABS_INFRA_MNEMONIC_SECRET_NAME=mainnet-labs-mainnet-v4-mnemonic -LABS_INFRA_MNEMONIC=REPLACE_WITH_GCP_SECRET -OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET -# intentionally left blank -FUNDING_PRIVATE_KEY="" -ROLLUP_DEPLOYMENT_PRIVATE_KEY="" - -# Snapshots intentionally disabled: this deployment runs no RPC/full node -# (RPC_REPLICAS=0, FULL_NODE_REPLICAS=0), so there is nothing to snapshot. -# Leaving SNAPSHOT_BUCKET_DIRECTORY unset stops setup_gcp_secrets.sh from -# constructing STORE_SNAPSHOT_URL, which is what gates the aztec-snapshots cronjob. - -BLOB_BUCKET_DIRECTORY=${BLOB_BUCKET_DIRECTORY:-mainnet/blobs} -BLOB_FILE_STORE_URLS="," - -TX_FILE_STORE_ENABLED=true -TX_FILE_STORE_BUCKET_DIRECTORY=${TX_FILE_STORE_BUCKET_DIRECTORY:-mainnet/txs} -TX_COLLECTION_FILE_STORE_URLS="https://aztec-labs-snapshots.com/${TX_FILE_STORE_BUCKET_DIRECTORY}" - -R2_ACCESS_KEY_ID=REPLACE_WITH_GCP_SECRET -R2_SECRET_ACCESS_KEY=REPLACE_WITH_GCP_SECRET diff --git a/spartan/environments/testnet-v4.env b/spartan/environments/testnet-v4.env deleted file mode 100644 index 38b2418afd71..000000000000 --- a/spartan/environments/testnet-v4.env +++ /dev/null @@ -1,105 +0,0 @@ -CREATE_ETH_DEVNET=false -GCP_REGION=us-west1-a -CLUSTER=aztec-gke-public -RESOURCE_PROFILE=prod -NAMESPACE=${NAMESPACE:-testnet-v4} -NETWORK=testnet - -RPC_GATEWAY_ENABLED=false -PROVER_NODE_RPC_GATEWAY_ENABLED=true -RPC_GATEWAY_FRONTEND_STATIC_IP_NAME=testnet-v4-infra-rpc-frontend -RPC_GATEWAY_KONG_HELM_RELEASE_NAME=testnet-v4-infra-rpc-kong -RPC_GATEWAY_KONG_INGRESS_CLASS=testnet-v4-infra-rpc-kong -PROVER_NODE_RPC_GATEWAY_HOSTS='["prover-v4.testnet.rpc.aztec-labs.com"]' -PROVER_NODE_RPC_GATEWAY_API_KEY_SECRET_NAMES='["testnet-prover-rpc-consumer-client1"]' - -REAL_VERIFIER=true - -LABS_INFRA_MNEMONIC_SECRET_NAME="sepolia-labs-testnet-v4-mnemonic" - -# testnet intentionally diverges from the generated testnet network defaults (slashing round size 2 epochs vs -# 4), so the consensus-config enforcement must permit these overrides instead of failing at startup. -ALLOW_OVERRIDING_NETWORK_CONFIG=true -AZTEC_ENTRY_QUEUE_BOOTSTRAP_VALIDATOR_SET_SIZE=48 -AZTEC_ENTRY_QUEUE_BOOTSTRAP_FLUSH_SIZE=48 -AZTEC_ENTRY_QUEUE_FLUSH_SIZE_MIN=10 -AZTEC_ENTRY_QUEUE_FLUSH_SIZE_QUOTIENT=400 -AZTEC_ENTRY_QUEUE_MAX_FLUSH_SIZE=10 -AZTEC_SLOT_DURATION=72 -AZTEC_EPOCH_DURATION=32 -AZTEC_TARGET_COMMITTEE_SIZE=48 -AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=2 -AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=2 -AZTEC_PROOF_SUBMISSION_EPOCHS=1 -AZTEC_LOCAL_EJECTION_THRESHOLD=199000e18 -AZTEC_SLASHING_ROUND_SIZE_IN_EPOCHS=2 -AZTEC_SLASHING_QUORUM=33 -AZTEC_SLASHING_OFFSET_IN_ROUNDS=2 -AZTEC_SLASHING_LIFETIME_IN_ROUNDS=5 -AZTEC_SLASHING_EXECUTION_DELAY_IN_ROUNDS=2 -AZTEC_SLASHING_VETOER=\"0xdfe19Da6a717b7088621d8bBB66be59F2d78e924\" -AZTEC_MANA_TARGET=75000000 -AZTEC_PROVING_COST_PER_MANA=12500000 -AZTEC_SLASH_AMOUNT_SMALL=100000e18 -AZTEC_SLASH_AMOUNT_MEDIUM=250000e18 -AZTEC_SLASH_AMOUNT_LARGE=250000e18 -AZTEC_ACTIVATION_THRESHOLD=200000e18 -AZTEC_EJECTION_THRESHOLD=100000e18 -AZTEC_GOVERNANCE_PROPOSER_ROUND_SIZE=100 -AZTEC_GOVERNANCE_PROPOSER_QUORUM=60 - - -ETHEREUM_CHAIN_ID=11155111 -ETHEREUM_RPC_URLS=REPLACE_WITH_GCP_SECRET -ETHEREUM_CONSENSUS_HOST_URLS=REPLACE_WITH_GCP_SECRET -ETHEREUM_CONSENSUS_HOST_API_KEYS=REPLACE_WITH_GCP_SECRET -ETHEREUM_CONSENSUS_HOST_API_KEY_HEADERS=REPLACE_WITH_GCP_SECRET -FUNDING_PRIVATE_KEY=REPLACE_WITH_GCP_SECRET -LABS_INFRA_MNEMONIC=REPLACE_WITH_GCP_SECRET -ROLLUP_DEPLOYMENT_PRIVATE_KEY=REPLACE_WITH_GCP_SECRET -OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET -VERIFY_CONTRACTS=true -ETHERSCAN_API_KEY=REPLACE_WITH_GCP_SECRET -CREATE_ROLLUP_CONTRACTS=${CREATE_ROLLUP_CONTRACTS:-false} -USE_NETWORK_CONFIG=${USE_NETWORK_CONFIG:-true} -ROLLUP_VERSION="4127419662" - -TX_FILE_STORE_ENABLED=true -TX_FILE_STORE_BUCKET_DIRECTORY=${TX_FILE_STORE_BUCKET_DIRECTORY:-testnet/txs} -TX_COLLECTION_FILE_STORE_URLS="https://aztec-labs-snapshots.com/${TX_FILE_STORE_BUCKET_DIRECTORY}" -DEPLOY_INTERNAL_BOOTNODE=false - -BOT_TRANSFERS_REPLICAS=0 -BOT_SWAPS_REPLICAS=0 -BOT_CROSS_CHAIN_REPLICAS=0 -RPC_REPLICAS=0 -FULL_NODE_REPLICAS=0 - -P2P_TX_POOL_DELETE_TXS_AFTER_REORG=true -SEQ_MAX_TX_PER_CHECKPOINT=72 - - -VALIDATOR_REPLICAS=1 -VALIDATORS_PER_NODE=500 -VALIDATOR_PUBLISHERS_PER_REPLICA=4 -VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 -VALIDATOR_RESOURCE_PROFILE="prod" - -PUBLISHERS_PER_PROVER=2 -PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000 -PROVER_REPLICAS=4 -PROVER_RESOURCE_PROFILE="prod" -PROVER_AGENT_KEDA_ENABLED=true -PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS=REPLACE_WITH_GCP_SECRET -PROVER_AGENT_KEDA_MIN_REPLICAS=0 -PROVER_AGENT_KEDA_MAX_REPLICAS=8 -PROVER_AGENT_KEDA_SCALING_BANDS='[ - { - queueSize = 0 - replicas = 4 - }, - { - queueSize = 100 - replicas = 8 - } -]'