From 08d066532fc852876ccf644c0a02166568ecddf4 Mon Sep 17 00:00:00 2001 From: Brian Tyler Date: Fri, 3 Jul 2026 11:48:00 +0100 Subject: [PATCH 1/6] Implement job retention and cancellation features - Added JobRetentionOptions to configure cleanup of terminal jobs. - Introduced IJobRetentionStore interface for managing job retention. - Implemented ShedduellerJobRetentionService and DashboardJobRetentionService for background cleanup tasks. - Created CancelQueuedJobsRequest and related operations for canceling queued jobs in the store. - Developed PostgresJobRetentionOperation and CancelQueuedJobsOperation for PostgreSQL backend. - Enhanced tests for job retention and cancellation, including contract tests for job retention store. - Updated RegistrationTests to ensure proper registration of retention services. --- README.md | 2 + .../Components/Pages/Jobs.razor | 165 +++++++++++++++++ .../Internal/DashboardJobRetentionService.cs | 91 ++++++++++ .../ShedduellerDashboardLoggerMessages.cs | 23 +++ ...lerDashboardServiceCollectionExtensions.cs | 1 + .../Internal/Operations/CancelJobOperation.cs | 22 ++- .../Operations/CancelQueuedJobsOperation.cs | 156 ++++++++++++++++ .../Operations/MarkJobCompletedOperation.cs | 3 +- .../Operations/MarkJobFailedOperation.cs | 9 +- .../Operations/PostgresClaimedJobs.cs | 10 +- .../PostgresJobRetentionOperation.cs | 98 +++++++++++ .../Operations/PostgresWorkerOperations.cs | 5 +- .../RecoverExpiredLeasesOperation.cs | 3 + .../Operations/ReleaseJobOperation.cs | 1 + .../Internal/PostgresJobStore.cs | 19 ++ .../Internal/PostgresMigrator.cs | 10 +- .../Internal/PostgresNames.cs | 2 +- .../ShedduellerPostgresBuilderExtensions.cs | 1 + .../ShedduellerJobRetentionService.cs | 91 ++++++++++ .../ShedduellerWorkerLoggerMessages.cs | 23 +++ ...uellerWorkerServiceCollectionExtensions.cs | 10 ++ src/Sheddueller/IJobManager.cs | 6 + src/Sheddueller/JobRetentionOptions.cs | 37 ++++ .../Logging/ShedduellerLoggerMessages.cs | 8 + src/Sheddueller/Runtime/JobManager.cs | 11 ++ .../ShedduellerCommonStartupValidator.cs | 30 ++++ src/Sheddueller/ShedduellerOptions.cs | 5 + .../Storage/CancelQueuedJobsRequest.cs | 7 + src/Sheddueller/Storage/IJobRetentionStore.cs | 14 ++ src/Sheddueller/Storage/IJobStore.cs | 7 + .../Storage/JobRetentionCleanupRequest.cs | 47 +++++ .../Storage/JobRetentionCleanupResult.cs | 25 +++ .../DashboardEndpointTests.cs | 17 ++ .../Operations/CancelJobOperationTests.cs | 66 +++++++ .../Operations/JobRetentionOperationTests.cs | 43 +++++ .../PostgresMigrationTests.cs | 42 +++++ .../PostgresRegistrationTests.cs | 1 + .../PostgresJobRetentionStoreContractTests.cs | 18 ++ .../JobRetentionStoreContractContext.cs | 25 +++ .../JobRetentionStoreContractTests.cs | 166 ++++++++++++++++++ .../JobStoreContractTests.cs | 50 ++++++ test/Sheddueller.Tests/JobManagerTests.cs | 27 +++ test/Sheddueller.Tests/RecordingJobStore.cs | 15 ++ .../RegistrationTests.cs | 48 +++++ .../WorkerJobLoggerTests.cs | 5 + .../WorkerLoggingTests.cs | 5 + .../WorkerProgressTests.cs | 5 + 47 files changed, 1455 insertions(+), 20 deletions(-) create mode 100644 src/Sheddueller.Dashboard/Internal/DashboardJobRetentionService.cs create mode 100644 src/Sheddueller.Postgres/Internal/Operations/CancelQueuedJobsOperation.cs create mode 100644 src/Sheddueller.Postgres/Internal/Operations/PostgresJobRetentionOperation.cs create mode 100644 src/Sheddueller.Worker/Internal/ShedduellerJobRetentionService.cs create mode 100644 src/Sheddueller/JobRetentionOptions.cs create mode 100644 src/Sheddueller/Storage/CancelQueuedJobsRequest.cs create mode 100644 src/Sheddueller/Storage/IJobRetentionStore.cs create mode 100644 src/Sheddueller/Storage/JobRetentionCleanupRequest.cs create mode 100644 src/Sheddueller/Storage/JobRetentionCleanupResult.cs create mode 100644 test/Sheddueller.Postgres.Tests/Operations/JobRetentionOperationTests.cs create mode 100644 test/Sheddueller.Postgres.Tests/ProviderContracts/PostgresJobRetentionStoreContractTests.cs create mode 100644 test/Sheddueller.ProviderContracts/JobRetentionStoreContractContext.cs create mode 100644 test/Sheddueller.ProviderContracts/JobRetentionStoreContractTests.cs create mode 100644 test/Sheddueller.Tests/JobManagerTests.cs diff --git a/README.md b/README.md index cd5765f..84af0e0 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,8 @@ Run migrations during deployment or before starting workers against a new schema Use `UsePostgres(postgres => postgres.DataSource = dataSource)` when an application needs to share or own a prebuilt `NpgsqlDataSource`; in that mode, the application also owns disposal. +The operational store keeps active jobs plus a bounded searchable terminal window. By default, background retention cleanup keeps completed jobs for 24 hours and failed or canceled jobs for 7 days, then deletes those terminal job rows and their tags, concurrency groups, and events. Configure `ShedduellerOptions.JobRetention` to change the windows, set a state retention to `null` to keep that state indefinitely, or set `Enabled = false` to disable cleanup. + ## Enqueue Jobs Job methods return `Task` or `ValueTask` and receive the scheduler-owned `CancellationToken`. Use constructor-injected `ILogger` for durable job logs, `Job.Context` when a handler needs the job id or attempt number, and scheduler-supplied `IProgress` for durable progress updates. diff --git a/src/Sheddueller.Dashboard/Components/Pages/Jobs.razor b/src/Sheddueller.Dashboard/Components/Pages/Jobs.razor index 85e4af1..c70a94b 100644 --- a/src/Sheddueller.Dashboard/Components/Pages/Jobs.razor +++ b/src/Sheddueller.Dashboard/Components/Pages/Jobs.razor @@ -1,8 +1,10 @@ @page "/jobs" @inherits DashboardPageComponent @inject IJobInspectionReader Reader +@inject IJobManager JobManager @inject DashboardLiveUpdateStream LiveUpdates @inject NavigationManager Navigation +@inject IJSRuntime JsRuntime
@@ -10,6 +12,12 @@

Jobs

@HeaderText

+ +
@@ -42,6 +50,13 @@
+ @if (_actionMessage is not null) + { + + @_actionMessage + + } + @if (_error is not null) {
@@ -292,6 +307,7 @@ } button.jobs-clear-button, + button.jobs-clear-queued-button, button.jobs-load-more-button { display: inline-flex; align-items: center; @@ -310,15 +326,31 @@ text-transform: uppercase; } + button.jobs-clear-queued-button { + border-color: var(--sd-error); + background: var(--sd-error-container); + color: var(--sd-on-error-container); + } + + button.jobs-clear-queued-button .material-symbols-outlined { + font-size: 17px; + } + button.jobs-load-more-button { min-width: 160px; } button.jobs-clear-button:hover, + button.jobs-clear-queued-button:hover, button.jobs-load-more-button:hover { color: var(--sd-primary); } + button.jobs-clear-queued-button:hover:not(:disabled) { + border-color: var(--sd-error); + color: var(--sd-error); + } + .jobs-page button:disabled, .jobs-page input:disabled { cursor: progress; @@ -353,6 +385,26 @@ color: var(--sd-error); } + .jobs-inline-alert--success { + border-color: var(--sd-success); + background: var(--sd-success-container); + color: var(--sd-success); + } + + .jobs-inline-alert--success .material-symbols-outlined { + color: var(--sd-success); + } + + .jobs-inline-alert--warning { + border-color: var(--sd-warning); + background: var(--sd-warning-container); + color: var(--sd-warning); + } + + .jobs-inline-alert--warning .material-symbols-outlined { + color: var(--sd-warning); + } + .jobs-message p, .jobs-inline-alert { color: var(--sd-on-surface-variant); @@ -362,6 +414,14 @@ color: var(--sd-on-error-container); } + .jobs-inline-alert--success { + color: var(--sd-success); + } + + .jobs-inline-alert--warning { + color: var(--sd-warning); + } + .jobs-table-shell { display: flex; flex: 1 1 auto; @@ -730,6 +790,7 @@ .jobs-filter-bar .sd-table-search, .jobs-filter-bar .sd-table-select, button.jobs-clear-button, + button.jobs-clear-queued-button, .jobs-pagination { width: 100%; } @@ -751,6 +812,8 @@ @code { private static readonly TimeSpan FilterDebounce = TimeSpan.FromMilliseconds(300); private const int DefaultPageSize = 25; + private const string ClearQueuedConfirmationWord = "delete"; + private const string ClearQueuedPrompt = "This will cancel all queued, delayed, and retry-waiting jobs while preserving historical records.\n\nType delete to continue."; private readonly DashboardJobFilters _filters = new(); private readonly List _jobs = []; @@ -758,10 +821,13 @@ private JobInspectionQuery _lastQuery = new(PageSize: DefaultPageSize); private CancellationTokenSource? _filterDebounceCts; private string? _error; + private string? _actionMessage; private string? _pendingFilterNavigationUri; private int _queryVersion; + private JobsActionSeverity _actionSeverity = JobsActionSeverity.Success; private bool _isLoading; private bool _isLoadingMore; + private bool _isClearQueuedRunning; private string HeaderText => this._page is null ? "Reading scheduler job records." : this.ResultRangeText; @@ -771,6 +837,34 @@ ? "No jobs matched the current filters." : "No jobs are available."; + private string ActionAlertClass + => this._actionSeverity switch + { + JobsActionSeverity.Success => "jobs-inline-alert jobs-inline-alert--success", + JobsActionSeverity.Warning => "jobs-inline-alert jobs-inline-alert--warning", + _ => "jobs-inline-alert jobs-inline-alert--error", + }; + + private string ActionAlertIcon + => this._actionSeverity switch + { + JobsActionSeverity.Success => "check_circle", + JobsActionSeverity.Warning => "info", + _ => "warning", + }; + + private bool IsClearQueuedDisabled + => this._isClearQueuedRunning || this._isLoading || this._isLoadingMore; + + private string ClearQueuedButtonIcon + => this._isClearQueuedRunning ? "progress_activity" : "cancel"; + + private string ClearQueuedButtonText + => this._isClearQueuedRunning ? "Clearing Queued" : "Clear Queued"; + + private string ClearQueuedButtonTitle + => "Cancel all queued, delayed, and retry-waiting jobs. Type delete to confirm."; + private string ResultRangeText { get @@ -869,6 +963,46 @@ await this.LoadAsync(); } + private async Task ClearQueuedJobsAsync() + { + if (this._isClearQueuedRunning) + { + return; + } + + var confirmation = await JsRuntime.InvokeAsync("prompt", ClearQueuedPrompt); + if (!string.Equals(confirmation?.Trim(), ClearQueuedConfirmationWord, StringComparison.Ordinal)) + { + return; + } + + this._isClearQueuedRunning = true; + this.ClearActionAlert(); + + try + { + var canceledCount = await JobManager.CancelQueuedJobsAsync(); + if (canceledCount == 0) + { + this.SetActionWarning("No queued jobs were available to cancel."); + } + else + { + this.SetActionSuccess(CreateClearQueuedSuccessMessage(canceledCount)); + } + + await this.LiveRefresh.RefreshNowAsync(); + } + catch (Exception exception) + { + this.SetActionError(string.Create(CultureInfo.InvariantCulture, $"Clear queued jobs failed: {exception.Message}")); + } + finally + { + this._isClearQueuedRunning = false; + } + } + private async Task LoadAsync() { var query = this.CreateQuery(DefaultPageSize, continuationToken: null); @@ -1094,6 +1228,30 @@ this._page = page; } + private void SetActionSuccess(string message) + { + this._actionSeverity = JobsActionSeverity.Success; + this._actionMessage = message; + } + + private void SetActionWarning(string message) + { + this._actionSeverity = JobsActionSeverity.Warning; + this._actionMessage = message; + } + + private void SetActionError(string message) + { + this._actionSeverity = JobsActionSeverity.Error; + this._actionMessage = message; + } + + private void ClearActionAlert() + => this._actionMessage = null; + + internal static string CreateClearQueuedSuccessMessage(int canceledCount) + => string.Create(CultureInfo.InvariantCulture, $"Canceled {DashboardFormat.Count(canceledCount)} queued job(s)."); + private static string StatusFilterClass(JobState state) => string.Concat("jobs-status-option jobs-status-option--", DashboardFormat.StateCssModifier(state)); @@ -1143,4 +1301,11 @@ private string GroupFilterHref(string group) => DashboardJobFilterQuery.WithGroupHref(this._filters, group); + + private enum JobsActionSeverity + { + Success, + Warning, + Error, + } } diff --git a/src/Sheddueller.Dashboard/Internal/DashboardJobRetentionService.cs b/src/Sheddueller.Dashboard/Internal/DashboardJobRetentionService.cs new file mode 100644 index 0000000..56a0310 --- /dev/null +++ b/src/Sheddueller.Dashboard/Internal/DashboardJobRetentionService.cs @@ -0,0 +1,91 @@ +namespace Sheddueller.Dashboard.Internal; + +using System.Diagnostics.CodeAnalysis; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using Sheddueller.Storage; + +internal sealed class DashboardJobRetentionService( + IServiceProvider serviceProvider, + TimeProvider timeProvider, + IOptions options, + ILogger logger) : BackgroundService +{ + [SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Retention cleanup failures are diagnostic and should not stop the dashboard host.")] + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + try + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + await this.CleanupOnceAsync(stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + logger.DashboardJobRetentionCleanupFailed(exception); + } + + await Task.Delay(options.Value.JobRetention.CleanupInterval, stoppingToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + } + } + + private async ValueTask CleanupOnceAsync(CancellationToken cancellationToken) + { + var retention = options.Value.JobRetention; + if (!retention.Enabled) + { + return; + } + + var store = serviceProvider.GetService(); + if (store is null) + { + logger.DashboardJobRetentionStoreMissing(); + return; + } + + if (retention.CompletedRetention is null + && retention.FailedRetention is null + && retention.CanceledRetention is null) + { + return; + } + + var now = timeProvider.GetUtcNow(); + var request = new JobRetentionCleanupRequest( + retention.CompletedRetention is { } completedRetention ? now.Subtract(completedRetention) : null, + retention.FailedRetention is { } failedRetention ? now.Subtract(failedRetention) : null, + retention.CanceledRetention is { } canceledRetention ? now.Subtract(canceledRetention) : null, + retention.BatchSize); + + var totalDeleted = 0; + while (!cancellationToken.IsCancellationRequested) + { + var result = await store.CleanupTerminalJobsAsync(request, cancellationToken).ConfigureAwait(false); + totalDeleted += result.DeletedCount; + if (result.DeletedCount < retention.BatchSize) + { + break; + } + } + + if (totalDeleted > 0) + { + logger.DashboardJobRetentionCleaned(totalDeleted); + } + } +} diff --git a/src/Sheddueller.Dashboard/Internal/ShedduellerDashboardLoggerMessages.cs b/src/Sheddueller.Dashboard/Internal/ShedduellerDashboardLoggerMessages.cs index 4b2718b..13dcb20 100644 --- a/src/Sheddueller.Dashboard/Internal/ShedduellerDashboardLoggerMessages.cs +++ b/src/Sheddueller.Dashboard/Internal/ShedduellerDashboardLoggerMessages.cs @@ -44,4 +44,27 @@ public static partial void DashboardEventRetentionCleaned( public static partial void DashboardEventRetentionCleanupFailed( this ILogger logger, Exception exception); + + [LoggerMessage( + EventIdStart + 30, + LogLevel.Debug, + "Dashboard job retention cleanup skipped because no retention store is registered.")] + public static partial void DashboardJobRetentionStoreMissing( + this ILogger logger); + + [LoggerMessage( + EventIdStart + 31, + LogLevel.Information, + "Dashboard job retention cleanup deleted {DeletedCount} terminal jobs.")] + public static partial void DashboardJobRetentionCleaned( + this ILogger logger, + int deletedCount); + + [LoggerMessage( + EventIdStart + 32, + LogLevel.Warning, + "Dashboard job retention cleanup failed.")] + public static partial void DashboardJobRetentionCleanupFailed( + this ILogger logger, + Exception exception); } diff --git a/src/Sheddueller.Dashboard/ShedduellerDashboardServiceCollectionExtensions.cs b/src/Sheddueller.Dashboard/ShedduellerDashboardServiceCollectionExtensions.cs index 416d550..30155e1 100644 --- a/src/Sheddueller.Dashboard/ShedduellerDashboardServiceCollectionExtensions.cs +++ b/src/Sheddueller.Dashboard/ShedduellerDashboardServiceCollectionExtensions.cs @@ -45,6 +45,7 @@ public static IServiceCollection AddShedduellerDashboard( services.TryAddEnumerable(ServiceDescriptor.Singleton()); services.TryAddEnumerable(ServiceDescriptor.Singleton()); services.TryAddEnumerable(ServiceDescriptor.Singleton()); + services.TryAddEnumerable(ServiceDescriptor.Singleton()); return services; } diff --git a/src/Sheddueller.Postgres/Internal/Operations/CancelJobOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/CancelJobOperation.cs index 677077e..795d3a6 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/CancelJobOperation.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/CancelJobOperation.cs @@ -23,8 +23,8 @@ public static async ValueTask ExecuteAsync( var result = row.State switch { - JobState.Queued => await CancelQueuedJobAsync(context, connection, transaction, row, cancellationToken).ConfigureAwait(false), - JobState.Claimed => await RequestClaimedJobCancellationAsync(context, connection, transaction, row, cancellationToken).ConfigureAwait(false), + JobState.Queued => await CancelQueuedJobAsync(context, connection, transaction, row, request, cancellationToken).ConfigureAwait(false), + JobState.Claimed => await RequestClaimedJobCancellationAsync(context, connection, transaction, row, request, cancellationToken).ConfigureAwait(false), JobState.Completed or JobState.Failed or JobState.Canceled => JobCancellationResult.AlreadyFinished, _ => throw new InvalidOperationException($"Unsupported job state '{row.State}'."), }; @@ -38,6 +38,7 @@ private static async ValueTask CancelQueuedJobAsync( NpgsqlConnection connection, NpgsqlTransaction transaction, CancelJobRow row, + CancelJobRequest request, CancellationToken cancellationToken) { await PostgresOperationContext.ExecuteCountAsync( @@ -46,10 +47,14 @@ await PostgresOperationContext.ExecuteCountAsync( $""" update {context.Names.Jobs} set state = 'Canceled', - canceled_at_utc = transaction_timestamp() + canceled_at_utc = @canceled_at_utc where job_id = @job_id; """, - command => command.Parameters.AddWithValue("job_id", row.JobId), + command => + { + command.Parameters.AddWithValue("job_id", row.JobId); + command.Parameters.AddWithValue("canceled_at_utc", request.CanceledAtUtc); + }, cancellationToken) .ConfigureAwait(false); @@ -69,6 +74,7 @@ private static async ValueTask RequestClaimedJobCancellat NpgsqlConnection connection, NpgsqlTransaction transaction, CancelJobRow row, + CancelJobRequest request, CancellationToken cancellationToken) { if (row.CancellationRequestedAtUtc is not null) @@ -81,10 +87,14 @@ await PostgresOperationContext.ExecuteCountAsync( transaction, $""" update {context.Names.Jobs} - set cancellation_requested_at_utc = transaction_timestamp() + set cancellation_requested_at_utc = @cancellation_requested_at_utc where job_id = @job_id; """, - command => command.Parameters.AddWithValue("job_id", row.JobId), + command => + { + command.Parameters.AddWithValue("job_id", row.JobId); + command.Parameters.AddWithValue("cancellation_requested_at_utc", request.CanceledAtUtc); + }, cancellationToken) .ConfigureAwait(false); diff --git a/src/Sheddueller.Postgres/Internal/Operations/CancelQueuedJobsOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/CancelQueuedJobsOperation.cs new file mode 100644 index 0000000..a63b976 --- /dev/null +++ b/src/Sheddueller.Postgres/Internal/Operations/CancelQueuedJobsOperation.cs @@ -0,0 +1,156 @@ +namespace Sheddueller.Postgres.Internal.Operations; + +using Npgsql; + +using NpgsqlTypes; + +using Sheddueller.Storage; + +internal static class CancelQueuedJobsOperation +{ + public static async ValueTask ExecuteAsync( + PostgresOperationContext context, + CancelQueuedJobsRequest request, + CancellationToken cancellationToken) + { + await using var connection = await context.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + var canceledJobs = await CancelQueuedJobsAsync(context, connection, transaction, request, cancellationToken).ConfigureAwait(false); + + if (canceledJobs.Count > 0) + { + await InsertLifecycleEventsAsync(context, connection, transaction, canceledJobs, cancellationToken).ConfigureAwait(false); + await NotifyLifecycleEventsAsync(context, connection, transaction, canceledJobs, cancellationToken).ConfigureAwait(false); + } + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return canceledJobs.Count; + } + + private static async ValueTask> CancelQueuedJobsAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + CancelQueuedJobsRequest request, + CancellationToken cancellationToken) + { + await using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = + $""" + update {context.Names.Jobs} + set state = 'Canceled', + canceled_at_utc = @canceled_at_utc, + job_event_sequence = job_event_sequence + 1 + where state = 'Queued' + returning job_id, job_event_sequence, attempt_count; + """; + command.Parameters.AddWithValue("canceled_at_utc", request.CanceledAtUtc); + + var canceledJobs = new List(); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + canceledJobs.Add(new CanceledQueuedJob( + reader.GetGuid(0), + reader.GetInt64(1), + Guid.NewGuid(), + reader.GetInt32(2))); + } + + return canceledJobs; + } + + private static async ValueTask InsertLifecycleEventsAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + IReadOnlyList canceledJobs, + CancellationToken cancellationToken) + => await PostgresOperationContext.ExecuteCountAsync( + connection, + transaction, + $""" + insert into {context.Names.JobEvents} ( + job_id, + event_sequence, + event_id, + kind, + occurred_at_utc, + attempt_number, + log_level, + message, + progress_percent, + fields) + select + staged.job_id, + staged.event_sequence, + staged.event_id, + 'Lifecycle', + transaction_timestamp(), + staged.attempt_number, + null, + 'Canceled', + null, + null + from unnest( + @job_ids::uuid[], + @event_sequences::bigint[], + @event_ids::uuid[], + @attempt_numbers::integer[]) + as staged(job_id, event_sequence, event_id, attempt_number); + """, + command => + { + AddCanceledJobIdentityParameters(command, canceledJobs); + command.Parameters.Add("event_ids", NpgsqlDbType.Array | NpgsqlDbType.Uuid).Value = + canceledJobs.Select(static job => job.EventId).ToArray(); + command.Parameters.Add("attempt_numbers", NpgsqlDbType.Array | NpgsqlDbType.Integer).Value = + canceledJobs.Select(static job => job.AttemptNumber).ToArray(); + }, + cancellationToken) + .ConfigureAwait(false); + + private static async ValueTask NotifyLifecycleEventsAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + IReadOnlyList canceledJobs, + CancellationToken cancellationToken) + => await PostgresOperationContext.ExecuteCountAsync( + connection, + transaction, + """ + select pg_notify( + @job_event_channel, + @schema_name || '|' || replace(staged.job_id::text, '-', '') || '|' || staged.event_sequence::text) + from unnest( + @job_ids::uuid[], + @event_sequences::bigint[]) + as staged(job_id, event_sequence); + """, + command => + { + command.Parameters.AddWithValue("job_event_channel", PostgresNames.JobEventChannel); + command.Parameters.AddWithValue("schema_name", context.Options.SchemaName); + AddCanceledJobIdentityParameters(command, canceledJobs); + }, + cancellationToken) + .ConfigureAwait(false); + + private static void AddCanceledJobIdentityParameters( + NpgsqlCommand command, + IReadOnlyList canceledJobs) + { + command.Parameters.Add("job_ids", NpgsqlDbType.Array | NpgsqlDbType.Uuid).Value = + canceledJobs.Select(static job => job.JobId).ToArray(); + command.Parameters.Add("event_sequences", NpgsqlDbType.Array | NpgsqlDbType.Bigint).Value = + canceledJobs.Select(static job => job.EventSequence).ToArray(); + } + + private sealed record CanceledQueuedJob( + Guid JobId, + long EventSequence, + Guid EventId, + int AttemptNumber); +} diff --git a/src/Sheddueller.Postgres/Internal/Operations/MarkJobCompletedOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/MarkJobCompletedOperation.cs index 23de656..aa4225c 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/MarkJobCompletedOperation.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/MarkJobCompletedOperation.cs @@ -34,7 +34,7 @@ public static async ValueTask ExecuteAsync( $""" update {context.Names.Jobs} set state = 'Completed', - completed_at_utc = transaction_timestamp() + completed_at_utc = @completed_at_utc where job_id = @job_id and state = 'Claimed' and claimed_by_node_id = @node_id @@ -46,6 +46,7 @@ public static async ValueTask ExecuteAsync( command.Parameters.AddWithValue("job_id", request.JobId); command.Parameters.AddWithValue("node_id", request.NodeId); command.Parameters.AddWithValue("lease_token", request.LeaseToken); + command.Parameters.AddWithValue("completed_at_utc", request.CompletedAtUtc); }, cancellationToken) .ConfigureAwait(false); diff --git a/src/Sheddueller.Postgres/Internal/Operations/MarkJobFailedOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/MarkJobFailedOperation.cs index e541233..849c3d1 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/MarkJobFailedOperation.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/MarkJobFailedOperation.cs @@ -28,7 +28,14 @@ public static async ValueTask ExecuteAsync( } await PostgresJobGroups.DecrementGroupsAsync(context, connection, transaction, job.GroupKeys, cancellationToken).ConfigureAwait(false); - var lifecycleMessage = await PostgresClaimedJobs.ApplyFailedAttemptAsync(context, connection, transaction, job, request.Failure, cancellationToken) + var lifecycleMessage = await PostgresClaimedJobs.ApplyFailedAttemptAsync( + context, + connection, + transaction, + job, + request.FailedAtUtc, + request.Failure, + cancellationToken) .ConfigureAwait(false); await PostgresJobEvents.AppendAndNotifyInTransactionAsync( context, diff --git a/src/Sheddueller.Postgres/Internal/Operations/PostgresClaimedJobs.cs b/src/Sheddueller.Postgres/Internal/Operations/PostgresClaimedJobs.cs index bf1d278..dac5e39 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/PostgresClaimedJobs.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/PostgresClaimedJobs.cs @@ -123,12 +123,13 @@ public static async ValueTask ApplyFailedAttemptAsync( NpgsqlConnection connection, NpgsqlTransaction transaction, PostgresClaimedJob job, + DateTimeOffset failedAtUtc, JobFailureInfo failure, CancellationToken cancellationToken) { var retriesRemain = job.AttemptCount < job.MaxAttempts; if (retriesRemain - && await TryMarkSupersededByQueuedDuplicateAsync(context, connection, transaction, job, failure, cancellationToken) + && await TryMarkSupersededByQueuedDuplicateAsync(context, connection, transaction, job, failedAtUtc, failure, cancellationToken) .ConfigureAwait(false) is { } supersededMessage) { return supersededMessage; @@ -141,7 +142,7 @@ await PostgresOperationContext.ExecuteCountAsync( $""" update {context.Names.Jobs} set state = @state, - failed_at_utc = transaction_timestamp(), + failed_at_utc = @failed_at_utc, failure_type_name = @failure_type_name, failure_message = @failure_message, failure_stack_trace = @failure_stack_trace, @@ -157,6 +158,7 @@ await PostgresOperationContext.ExecuteCountAsync( { command.Parameters.AddWithValue("job_id", job.JobId); command.Parameters.AddWithValue("state", retriesRemain ? "Queued" : "Failed"); + command.Parameters.AddWithValue("failed_at_utc", failedAtUtc); command.Parameters.AddWithValue("failure_type_name", failure.ExceptionType); command.Parameters.AddWithValue("failure_message", failure.Message); command.Parameters.AddWithValue("failure_stack_trace", PostgresOperationContext.ToDbValue(failure.StackTrace)); @@ -177,6 +179,7 @@ await PostgresOperationContext.ExecuteCountAsync( NpgsqlConnection connection, NpgsqlTransaction transaction, PostgresClaimedJob job, + DateTimeOffset failedAtUtc, JobFailureInfo failure, CancellationToken cancellationToken) { @@ -198,7 +201,7 @@ await PostgresOperationContext.ExecuteCountAsync( $""" update {context.Names.Jobs} set state = 'Failed', - failed_at_utc = transaction_timestamp(), + failed_at_utc = @failed_at_utc, failure_type_name = @failure_type_name, failure_message = @failure_message, failure_stack_trace = @failure_stack_trace, @@ -213,6 +216,7 @@ await PostgresOperationContext.ExecuteCountAsync( command => { command.Parameters.AddWithValue("job_id", job.JobId); + command.Parameters.AddWithValue("failed_at_utc", failedAtUtc); command.Parameters.AddWithValue("failure_type_name", failure.ExceptionType); command.Parameters.AddWithValue("failure_message", failure.Message); command.Parameters.AddWithValue("failure_stack_trace", PostgresOperationContext.ToDbValue(failure.StackTrace)); diff --git a/src/Sheddueller.Postgres/Internal/Operations/PostgresJobRetentionOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/PostgresJobRetentionOperation.cs new file mode 100644 index 0000000..edd1ec6 --- /dev/null +++ b/src/Sheddueller.Postgres/Internal/Operations/PostgresJobRetentionOperation.cs @@ -0,0 +1,98 @@ +namespace Sheddueller.Postgres.Internal.Operations; + +using System.Globalization; + +using Npgsql; + +using NpgsqlTypes; + +using Sheddueller.Storage; + +internal static class PostgresJobRetentionOperation +{ + private const int RetentionAdvisoryLockKey = 7870834; + + public static async ValueTask ExecuteAsync( + PostgresOperationContext context, + JobRetentionCleanupRequest request, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + await using var connection = await context.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + var lockAcquired = await TryAcquireCleanupLockAsync(context, connection, transaction, cancellationToken).ConfigureAwait(false); + if (!lockAcquired) + { + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); + return new JobRetentionCleanupResult(0); + } + + var deletedCount = await DeleteTerminalJobsAsync(context, connection, transaction, request, cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + + return new JobRetentionCleanupResult(deletedCount); + } + + private static async ValueTask TryAcquireCleanupLockAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + CancellationToken cancellationToken) + { + await using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = "select pg_try_advisory_xact_lock(@lock_key, hashtext(@schema_name));"; + command.Parameters.AddWithValue("lock_key", RetentionAdvisoryLockKey); + command.Parameters.AddWithValue("schema_name", context.Options.SchemaName); + + return (bool)(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException("PostgreSQL did not return an advisory lock result.")); + } + + private static async ValueTask DeleteTerminalJobsAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + JobRetentionCleanupRequest request, + CancellationToken cancellationToken) + { + await using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = + $""" + with candidates as ( + select job_id + from {context.Names.Jobs} + where + (state = 'Completed' and @completed_before_utc is not null and completed_at_utc < @completed_before_utc) + or (state = 'Failed' and @failed_before_utc is not null and failed_at_utc < @failed_before_utc) + or (state = 'Canceled' and @canceled_before_utc is not null and canceled_at_utc < @canceled_before_utc) + order by coalesce(completed_at_utc, failed_at_utc, canceled_at_utc) asc, enqueue_sequence asc + limit @batch_size + for update skip locked + ), + deleted_jobs as ( + delete from {context.Names.Jobs} job + using candidates + where job.job_id = candidates.job_id + returning 1 + ) + select count(*) from deleted_jobs; + """; + AddNullableTimestampParameter(command, "completed_before_utc", request.CompletedBeforeUtc); + AddNullableTimestampParameter(command, "failed_before_utc", request.FailedBeforeUtc); + AddNullableTimestampParameter(command, "canceled_before_utc", request.CanceledBeforeUtc); + command.Parameters.Add("batch_size", NpgsqlDbType.Integer).Value = request.BatchSize; + + var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException("PostgreSQL did not return a terminal job cleanup count."); + return Convert.ToInt32(result, CultureInfo.InvariantCulture); + } + + private static void AddNullableTimestampParameter( + NpgsqlCommand command, + string name, + DateTimeOffset? value) + => command.Parameters.Add(name, NpgsqlDbType.TimestampTz).Value = value is { } timestamp ? timestamp : DBNull.Value; +} diff --git a/src/Sheddueller.Postgres/Internal/Operations/PostgresWorkerOperations.cs b/src/Sheddueller.Postgres/Internal/Operations/PostgresWorkerOperations.cs index 4c68d23..c7e0408 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/PostgresWorkerOperations.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/PostgresWorkerOperations.cs @@ -60,8 +60,8 @@ public static async ValueTask MarkCancellationObservedAsync( $""" update {context.Names.Jobs} set state = 'Canceled', - canceled_at_utc = transaction_timestamp(), - cancellation_observed_at_utc = transaction_timestamp(), + canceled_at_utc = @observed_at_utc, + cancellation_observed_at_utc = @observed_at_utc, claimed_by_node_id = null, claimed_at_utc = null, lease_token = null, @@ -78,6 +78,7 @@ public static async ValueTask MarkCancellationObservedAsync( command.Parameters.AddWithValue("job_id", request.JobId); command.Parameters.AddWithValue("node_id", request.NodeId); command.Parameters.AddWithValue("lease_token", request.LeaseToken); + command.Parameters.AddWithValue("observed_at_utc", request.ObservedAtUtc); }, cancellationToken) .ConfigureAwait(false); diff --git a/src/Sheddueller.Postgres/Internal/Operations/RecoverExpiredLeasesOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/RecoverExpiredLeasesOperation.cs index dc60c8a..3eed464 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/RecoverExpiredLeasesOperation.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/RecoverExpiredLeasesOperation.cs @@ -12,6 +12,8 @@ public static async ValueTask ExecuteAsync( await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); var expiredJobs = await PostgresClaimedJobs.ReadExpiredClaimsAsync(context, connection, transaction, cancellationToken) .ConfigureAwait(false); + var recoveredAtUtc = await PostgresOperationContext.ReadTransactionTimestampAsync(connection, transaction, cancellationToken) + .ConfigureAwait(false); foreach (var job in expiredJobs) { await PostgresJobGroups.DecrementGroupsAsync(context, connection, transaction, job.GroupKeys, cancellationToken).ConfigureAwait(false); @@ -20,6 +22,7 @@ public static async ValueTask ExecuteAsync( connection, transaction, job, + recoveredAtUtc, new JobFailureInfo("Sheddueller.LeaseExpired", "The job lease expired before the owning node renewed it.", null), cancellationToken) .ConfigureAwait(false); diff --git a/src/Sheddueller.Postgres/Internal/Operations/ReleaseJobOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/ReleaseJobOperation.cs index c8184e1..3ae78e8 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/ReleaseJobOperation.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/ReleaseJobOperation.cs @@ -37,6 +37,7 @@ public static async ValueTask ExecuteAsync( connection, transaction, job, + request.ReleasedAtUtc, supersededFailure, cancellationToken) .ConfigureAwait(false); diff --git a/src/Sheddueller.Postgres/Internal/PostgresJobStore.cs b/src/Sheddueller.Postgres/Internal/PostgresJobStore.cs index eefa7fc..a686cec 100644 --- a/src/Sheddueller.Postgres/Internal/PostgresJobStore.cs +++ b/src/Sheddueller.Postgres/Internal/PostgresJobStore.cs @@ -18,6 +18,7 @@ internal sealed class PostgresJobStore( IJobInspectionReader, IJobEventSink, IJobEventRetentionStore, + IJobRetentionStore, IScheduleInspectionReader, IConcurrencyGroupInspectionReader, INodeInspectionReader, @@ -107,6 +108,15 @@ public ValueTask CancelAsync( return CancelJobOperation.ExecuteAsync(this._context, request, cancellationToken); } + public ValueTask CancelQueuedJobsAsync( + CancelQueuedJobsRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + return CancelQueuedJobsOperation.ExecuteAsync(this._context, request, cancellationToken); + } + public ValueTask GetCancellationRequestedAtAsync( JobCancellationStatusRequest request, CancellationToken cancellationToken = default) @@ -240,6 +250,15 @@ public ValueTask CleanupAsync( CancellationToken cancellationToken = default) => PostgresJobInspectionOperation.CleanupAsync(this._context, retention, cancellationToken); + public ValueTask CleanupTerminalJobsAsync( + JobRetentionCleanupRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + return PostgresJobRetentionOperation.ExecuteAsync(this._context, request, cancellationToken); + } + public ValueTask SearchSchedulesAsync( ScheduleInspectionQuery query, CancellationToken cancellationToken = default) diff --git a/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs b/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs index d973c39..259f70f 100644 --- a/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs +++ b/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs @@ -308,9 +308,6 @@ constraint worker_nodes_current_execution_count_check check (current_execution_c on {this._names.Jobs} (idempotency_key) where state = 'Queued' and idempotency_key is not null; - create index if not exists idx_jobs_inspection_newest - on {this._names.Jobs} (enqueue_sequence desc); - create index if not exists idx_jobs_inspection_state_newest on {this._names.Jobs} (state, enqueue_sequence desc); @@ -351,13 +348,14 @@ constraint worker_nodes_current_execution_count_check check (current_execution_c create unique index if not exists idx_schedule_tags_schedule_key_ordinal on {this._names.ScheduleTags} (schedule_key, ordinal); - create index if not exists idx_job_events_job_sequence - on {this._names.JobEvents} (job_id, event_sequence); - create index if not exists idx_job_events_progress on {this._names.JobEvents} (job_id, event_sequence desc) where kind = 'Progress'; + drop index if exists {this._names.Schema}.idx_jobs_inspection_newest; + + drop index if exists {this._names.Schema}.idx_job_events_job_sequence; + create index if not exists idx_recurring_schedules_due on {this._names.RecurringSchedules} (next_fire_at_utc, schedule_key) where is_paused = false; diff --git a/src/Sheddueller.Postgres/Internal/PostgresNames.cs b/src/Sheddueller.Postgres/Internal/PostgresNames.cs index 619ed63..f383fa3 100644 --- a/src/Sheddueller.Postgres/Internal/PostgresNames.cs +++ b/src/Sheddueller.Postgres/Internal/PostgresNames.cs @@ -4,7 +4,7 @@ namespace Sheddueller.Postgres.Internal; internal sealed class PostgresNames { - public const int ExpectedSchemaVersion = 8; + public const int ExpectedSchemaVersion = 9; public const string WakeupChannel = "sheddueller_wakeup"; public const string JobEventChannel = "sheddueller_job_event"; diff --git a/src/Sheddueller.Postgres/ShedduellerPostgresBuilderExtensions.cs b/src/Sheddueller.Postgres/ShedduellerPostgresBuilderExtensions.cs index 9c9e1db..a7ab200 100644 --- a/src/Sheddueller.Postgres/ShedduellerPostgresBuilderExtensions.cs +++ b/src/Sheddueller.Postgres/ShedduellerPostgresBuilderExtensions.cs @@ -143,6 +143,7 @@ private static void RegisterProviderServices(IServiceCollection services) services.Replace(ServiceDescriptor.Singleton(serviceProvider => serviceProvider.GetRequiredService())); services.Replace(ServiceDescriptor.Singleton(serviceProvider => serviceProvider.GetRequiredService())); services.Replace(ServiceDescriptor.Singleton(serviceProvider => serviceProvider.GetRequiredService())); + services.Replace(ServiceDescriptor.Singleton(serviceProvider => serviceProvider.GetRequiredService())); services.Replace(ServiceDescriptor.Singleton(serviceProvider => serviceProvider.GetRequiredService())); services.Replace(ServiceDescriptor.Singleton(serviceProvider => serviceProvider.GetRequiredService())); services.Replace(ServiceDescriptor.Singleton(serviceProvider => serviceProvider.GetRequiredService())); diff --git a/src/Sheddueller.Worker/Internal/ShedduellerJobRetentionService.cs b/src/Sheddueller.Worker/Internal/ShedduellerJobRetentionService.cs new file mode 100644 index 0000000..c7bf2cf --- /dev/null +++ b/src/Sheddueller.Worker/Internal/ShedduellerJobRetentionService.cs @@ -0,0 +1,91 @@ +namespace Sheddueller.Worker.Internal; + +using System.Diagnostics.CodeAnalysis; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using Sheddueller.Storage; + +internal sealed class ShedduellerJobRetentionService( + IServiceProvider serviceProvider, + TimeProvider timeProvider, + IOptions options, + ILogger logger) : BackgroundService +{ + [SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Retention cleanup failures are diagnostic and should not stop the worker host.")] + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + try + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + await this.CleanupOnceAsync(stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + logger.WorkerJobRetentionCleanupFailed(exception); + } + + await Task.Delay(options.Value.JobRetention.CleanupInterval, stoppingToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + } + } + + private async ValueTask CleanupOnceAsync(CancellationToken cancellationToken) + { + var retention = options.Value.JobRetention; + if (!retention.Enabled) + { + return; + } + + var store = serviceProvider.GetService(); + if (store is null) + { + logger.WorkerJobRetentionStoreMissing(); + return; + } + + if (retention.CompletedRetention is null + && retention.FailedRetention is null + && retention.CanceledRetention is null) + { + return; + } + + var now = timeProvider.GetUtcNow(); + var request = new JobRetentionCleanupRequest( + retention.CompletedRetention is { } completedRetention ? now.Subtract(completedRetention) : null, + retention.FailedRetention is { } failedRetention ? now.Subtract(failedRetention) : null, + retention.CanceledRetention is { } canceledRetention ? now.Subtract(canceledRetention) : null, + retention.BatchSize); + + var totalDeleted = 0; + while (!cancellationToken.IsCancellationRequested) + { + var result = await store.CleanupTerminalJobsAsync(request, cancellationToken).ConfigureAwait(false); + totalDeleted += result.DeletedCount; + if (result.DeletedCount < retention.BatchSize) + { + break; + } + } + + if (totalDeleted > 0) + { + logger.WorkerJobRetentionCleaned(totalDeleted); + } + } +} diff --git a/src/Sheddueller.Worker/Internal/ShedduellerWorkerLoggerMessages.cs b/src/Sheddueller.Worker/Internal/ShedduellerWorkerLoggerMessages.cs index fdb43c1..36efc03 100644 --- a/src/Sheddueller.Worker/Internal/ShedduellerWorkerLoggerMessages.cs +++ b/src/Sheddueller.Worker/Internal/ShedduellerWorkerLoggerMessages.cs @@ -120,4 +120,27 @@ public static partial void WorkerPeriodicStoreWorkCompleted( string nodeId, int recoveredCount, int materializedCount); + + [LoggerMessage( + EventIdStart + 50, + LogLevel.Debug, + "Worker job retention cleanup skipped because no retention store is registered.")] + public static partial void WorkerJobRetentionStoreMissing( + this ILogger logger); + + [LoggerMessage( + EventIdStart + 51, + LogLevel.Information, + "Worker job retention cleanup deleted {DeletedCount} terminal jobs.")] + public static partial void WorkerJobRetentionCleaned( + this ILogger logger, + int deletedCount); + + [LoggerMessage( + EventIdStart + 52, + LogLevel.Warning, + "Worker job retention cleanup failed.")] + public static partial void WorkerJobRetentionCleanupFailed( + this ILogger logger, + Exception exception); } diff --git a/src/Sheddueller.Worker/ShedduellerWorkerServiceCollectionExtensions.cs b/src/Sheddueller.Worker/ShedduellerWorkerServiceCollectionExtensions.cs index 01e922c..d4468ca 100644 --- a/src/Sheddueller.Worker/ShedduellerWorkerServiceCollectionExtensions.cs +++ b/src/Sheddueller.Worker/ShedduellerWorkerServiceCollectionExtensions.cs @@ -8,6 +8,7 @@ namespace Microsoft.Extensions.DependencyInjection; using Sheddueller; using Sheddueller.Runtime; +using Sheddueller.Storage; using Sheddueller.Worker.Internal; /// @@ -34,11 +35,20 @@ public static IServiceCollection AddShedduellerWorker( services.TryAddEnumerable(ServiceDescriptor.Singleton()); services.TryAddEnumerable(ServiceDescriptor.Singleton()); TryAddStartupValidationHostedService(services); + TryAddJobRetentionHostedService(services); services.TryAddEnumerable(ServiceDescriptor.Singleton()); return services; } + private static void TryAddJobRetentionHostedService(IServiceCollection services) + { + if (services.Any(descriptor => descriptor.ServiceType == typeof(IJobRetentionStore))) + { + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + } + } + private static void TryAddStartupValidationHostedService(IServiceCollection services) { if (services.Any(descriptor => descriptor.ServiceType == typeof(ShedduellerHostedStartupValidationMarker))) diff --git a/src/Sheddueller/IJobManager.cs b/src/Sheddueller/IJobManager.cs index 7091617..d05b7bf 100644 --- a/src/Sheddueller/IJobManager.cs +++ b/src/Sheddueller/IJobManager.cs @@ -11,4 +11,10 @@ public interface IJobManager ValueTask CancelAsync( Guid jobId, CancellationToken cancellationToken = default); + + /// + /// Cancels all queued jobs. Running jobs are left untouched. + /// + ValueTask CancelQueuedJobsAsync( + CancellationToken cancellationToken = default); } diff --git a/src/Sheddueller/JobRetentionOptions.cs b/src/Sheddueller/JobRetentionOptions.cs new file mode 100644 index 0000000..f8b5c1a --- /dev/null +++ b/src/Sheddueller/JobRetentionOptions.cs @@ -0,0 +1,37 @@ +namespace Sheddueller; + +/// +/// Configures cleanup of terminal jobs from the operational store. +/// +public sealed class JobRetentionOptions +{ + /// + /// Gets or sets whether terminal job retention cleanup is enabled. + /// + public bool Enabled { get; set; } = true; + + /// + /// Gets or sets how long completed jobs remain in the operational store. Null retains them forever. + /// + public TimeSpan? CompletedRetention { get; set; } = TimeSpan.FromDays(1); + + /// + /// Gets or sets how long failed jobs remain in the operational store. Null retains them forever. + /// + public TimeSpan? FailedRetention { get; set; } = TimeSpan.FromDays(7); + + /// + /// Gets or sets how long canceled jobs remain in the operational store. Null retains them forever. + /// + public TimeSpan? CanceledRetention { get; set; } = TimeSpan.FromDays(7); + + /// + /// Gets or sets how often background retention cleanup runs. + /// + public TimeSpan CleanupInterval { get; set; } = TimeSpan.FromHours(1); + + /// + /// Gets or sets the maximum number of jobs deleted in one cleanup transaction. + /// + public int BatchSize { get; set; } = 1000; +} diff --git a/src/Sheddueller/Logging/ShedduellerLoggerMessages.cs b/src/Sheddueller/Logging/ShedduellerLoggerMessages.cs index b38f86b..7d8dfe7 100644 --- a/src/Sheddueller/Logging/ShedduellerLoggerMessages.cs +++ b/src/Sheddueller/Logging/ShedduellerLoggerMessages.cs @@ -40,6 +40,14 @@ public static partial void JobCancellationRequested( Guid jobId, string result); + [LoggerMessage( + EventIdStart + 11, + LogLevel.Debug, + "Canceled {CanceledCount} queued jobs.")] + public static partial void QueuedJobsCanceled( + this ILogger logger, + int canceledCount); + [LoggerMessage( EventIdStart + 20, LogLevel.Debug, diff --git a/src/Sheddueller/Runtime/JobManager.cs b/src/Sheddueller/Runtime/JobManager.cs index f318000..06d5625 100644 --- a/src/Sheddueller/Runtime/JobManager.cs +++ b/src/Sheddueller/Runtime/JobManager.cs @@ -17,4 +17,15 @@ public async ValueTask CancelAsync(Guid jobId, Cancellati return result; } + + public async ValueTask CancelQueuedJobsAsync(CancellationToken cancellationToken = default) + { + var canceledCount = await store.CancelQueuedJobsAsync( + new CancelQueuedJobsRequest(timeProvider.GetUtcNow()), + cancellationToken) + .ConfigureAwait(false); + logger.QueuedJobsCanceled(canceledCount); + + return canceledCount; + } } diff --git a/src/Sheddueller/Runtime/ShedduellerCommonStartupValidator.cs b/src/Sheddueller/Runtime/ShedduellerCommonStartupValidator.cs index 8b54744..25dfb56 100644 --- a/src/Sheddueller/Runtime/ShedduellerCommonStartupValidator.cs +++ b/src/Sheddueller/Runtime/ShedduellerCommonStartupValidator.cs @@ -31,6 +31,36 @@ public ValueTask ValidateAsync(CancellationToken cancellationToken) throw new InvalidOperationException("No Sheddueller job store provider has been registered."); } + ValidateJobRetentionOptions(value.JobRetention); + return ValueTask.CompletedTask; } + + private static void ValidateJobRetentionOptions(JobRetentionOptions options) + { + if (options.CompletedRetention is { } completedRetention && completedRetention <= TimeSpan.Zero) + { + throw new InvalidOperationException("ShedduellerOptions.JobRetention.CompletedRetention must be positive or null."); + } + + if (options.FailedRetention is { } failedRetention && failedRetention <= TimeSpan.Zero) + { + throw new InvalidOperationException("ShedduellerOptions.JobRetention.FailedRetention must be positive or null."); + } + + if (options.CanceledRetention is { } canceledRetention && canceledRetention <= TimeSpan.Zero) + { + throw new InvalidOperationException("ShedduellerOptions.JobRetention.CanceledRetention must be positive or null."); + } + + if (options.CleanupInterval <= TimeSpan.Zero) + { + throw new InvalidOperationException("ShedduellerOptions.JobRetention.CleanupInterval must be positive."); + } + + if (options.BatchSize <= 0) + { + throw new InvalidOperationException("ShedduellerOptions.JobRetention.BatchSize must be positive."); + } + } } diff --git a/src/Sheddueller/ShedduellerOptions.cs b/src/Sheddueller/ShedduellerOptions.cs index b46e9df..dbdcd93 100644 --- a/src/Sheddueller/ShedduellerOptions.cs +++ b/src/Sheddueller/ShedduellerOptions.cs @@ -50,6 +50,11 @@ public sealed class ShedduellerOptions /// public bool EnableJobLogCapture { get; set; } + /// + /// Gets the retention policy for terminal jobs in the operational store. + /// + public JobRetentionOptions JobRetention { get; } = new(); + /// /// Gets the effective stale worker node threshold. /// diff --git a/src/Sheddueller/Storage/CancelQueuedJobsRequest.cs b/src/Sheddueller/Storage/CancelQueuedJobsRequest.cs new file mode 100644 index 0000000..2126862 --- /dev/null +++ b/src/Sheddueller/Storage/CancelQueuedJobsRequest.cs @@ -0,0 +1,7 @@ +namespace Sheddueller.Storage; + +/// +/// Store request for canceling all queued jobs. +/// +public sealed record CancelQueuedJobsRequest( + DateTimeOffset CanceledAtUtc); diff --git a/src/Sheddueller/Storage/IJobRetentionStore.cs b/src/Sheddueller/Storage/IJobRetentionStore.cs new file mode 100644 index 0000000..eaf1209 --- /dev/null +++ b/src/Sheddueller/Storage/IJobRetentionStore.cs @@ -0,0 +1,14 @@ +namespace Sheddueller.Storage; + +/// +/// Cleans up terminal jobs from the operational store. +/// +public interface IJobRetentionStore +{ + /// + /// Deletes terminal jobs older than their configured cutoff timestamps. + /// + ValueTask CleanupTerminalJobsAsync( + JobRetentionCleanupRequest request, + CancellationToken cancellationToken = default); +} diff --git a/src/Sheddueller/Storage/IJobStore.cs b/src/Sheddueller/Storage/IJobStore.cs index 3dd38ce..7fa932d 100644 --- a/src/Sheddueller/Storage/IJobStore.cs +++ b/src/Sheddueller/Storage/IJobStore.cs @@ -68,6 +68,13 @@ ValueTask CancelAsync( CancelJobRequest request, CancellationToken cancellationToken = default); + /// + /// Cancels all queued jobs. + /// + ValueTask CancelQueuedJobsAsync( + CancelQueuedJobsRequest request, + CancellationToken cancellationToken = default); + /// /// Reads the cooperative cancellation request timestamp for a currently claimed job. /// diff --git a/src/Sheddueller/Storage/JobRetentionCleanupRequest.cs b/src/Sheddueller/Storage/JobRetentionCleanupRequest.cs new file mode 100644 index 0000000..3f21b48 --- /dev/null +++ b/src/Sheddueller/Storage/JobRetentionCleanupRequest.cs @@ -0,0 +1,47 @@ +namespace Sheddueller.Storage; + +/// +/// Store request for deleting terminal jobs older than configured cutoffs. +/// +public sealed record JobRetentionCleanupRequest +{ + /// + /// Initializes a new instance of the class. + /// + public JobRetentionCleanupRequest( + DateTimeOffset? completedBeforeUtc, + DateTimeOffset? failedBeforeUtc, + DateTimeOffset? canceledBeforeUtc, + int batchSize) + { + if (batchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(batchSize), batchSize, "Retention cleanup batch size must be positive."); + } + + this.CompletedBeforeUtc = completedBeforeUtc; + this.FailedBeforeUtc = failedBeforeUtc; + this.CanceledBeforeUtc = canceledBeforeUtc; + this.BatchSize = batchSize; + } + + /// + /// Gets the exclusive cutoff for completed jobs. Null keeps completed jobs. + /// + public DateTimeOffset? CompletedBeforeUtc { get; } + + /// + /// Gets the exclusive cutoff for failed jobs. Null keeps failed jobs. + /// + public DateTimeOffset? FailedBeforeUtc { get; } + + /// + /// Gets the exclusive cutoff for canceled jobs. Null keeps canceled jobs. + /// + public DateTimeOffset? CanceledBeforeUtc { get; } + + /// + /// Gets the maximum number of jobs to delete. + /// + public int BatchSize { get; } +} diff --git a/src/Sheddueller/Storage/JobRetentionCleanupResult.cs b/src/Sheddueller/Storage/JobRetentionCleanupResult.cs new file mode 100644 index 0000000..ec5a77e --- /dev/null +++ b/src/Sheddueller/Storage/JobRetentionCleanupResult.cs @@ -0,0 +1,25 @@ +namespace Sheddueller.Storage; + +/// +/// Result of a terminal job retention cleanup batch. +/// +public sealed record JobRetentionCleanupResult +{ + /// + /// Initializes a new instance of the class. + /// + public JobRetentionCleanupResult(int deletedCount) + { + if (deletedCount < 0) + { + throw new ArgumentOutOfRangeException(nameof(deletedCount), deletedCount, "Deleted count cannot be negative."); + } + + this.DeletedCount = deletedCount; + } + + /// + /// Gets the number of terminal jobs deleted. + /// + public int DeletedCount { get; } +} diff --git a/test/Sheddueller.Dashboard.Tests/DashboardEndpointTests.cs b/test/Sheddueller.Dashboard.Tests/DashboardEndpointTests.cs index 6c81c56..01d318a 100644 --- a/test/Sheddueller.Dashboard.Tests/DashboardEndpointTests.cs +++ b/test/Sheddueller.Dashboard.Tests/DashboardEndpointTests.cs @@ -22,6 +22,7 @@ namespace Sheddueller.Dashboard.Tests; using Shouldly; +using JobsPage = Sheddueller.Dashboard.Components.Pages.Jobs; using SchedulesPage = Sheddueller.Dashboard.Components.Pages.Schedules; public sealed class DashboardEndpointTests @@ -112,6 +113,8 @@ public async Task Jobs_KnownData_RendersSearchResults() html.ShouldContain("Operational Order"); html.ShouldContain("Newest First"); html.ShouldContain("Clear Filters"); + html.ShouldContain("Clear Queued"); + html.ShouldContain("Cancel all queued, delayed, and retry-waiting jobs. Type delete to confirm."); html.ShouldNotContain("Expand query filters"); html.ShouldNotContain("Execute Query"); html.ShouldNotContain("Query Parameters"); @@ -244,6 +247,13 @@ public void Schedules_TriggerActionMessages_FormatSuccessSkippedAndMissingCases( .ShouldBe("Schedule action failed: Schedule was not found."); } + [Fact] + public void Jobs_ClearQueuedActionMessage_FormatsCanceledCount() + { + JobsPage.CreateClearQueuedSuccessMessage(1234) + .ShouldBe("Canceled 1,234 queued job(s)."); + } + [Fact] public async Task ConcurrencyGroups_KnownData_RendersRegistry() { @@ -908,6 +918,13 @@ public ValueTask CancelAsync( return ValueTask.FromResult(JobCancellationResult.NotFound); } + + public ValueTask CancelQueuedJobsAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + return ValueTask.FromResult(2); + } } private sealed class StubScheduleInspectionReader : IScheduleInspectionReader, IRecurringScheduleManager diff --git a/test/Sheddueller.Postgres.Tests/Operations/CancelJobOperationTests.cs b/test/Sheddueller.Postgres.Tests/Operations/CancelJobOperationTests.cs index 659b64f..8d371b7 100644 --- a/test/Sheddueller.Postgres.Tests/Operations/CancelJobOperationTests.cs +++ b/test/Sheddueller.Postgres.Tests/Operations/CancelJobOperationTests.cs @@ -1,5 +1,7 @@ namespace Sheddueller.Postgres.Tests.Operations; +using System.Globalization; + using Sheddueller.Inspection.Jobs; using Sheddueller.Storage; @@ -63,6 +65,66 @@ public async Task Cancel_TerminalOrMissingJob_ReturnsExpectedResult() (await context.Store.CancelAsync(new CancelJobRequest(Guid.NewGuid(), DateTimeOffset.UtcNow))).ShouldBe(JobCancellationResult.NotFound); } + [Fact] + public async Task CancelQueuedJobs_QueuedState_MarksCanceledAndAppendsLifecycleEvents() + { + await using var context = await PostgresTestContext.CreateMigratedAsync(fixture); + var claimed = Guid.NewGuid(); + var retryWaiting = Guid.NewGuid(); + var completed = Guid.NewGuid(); + var failed = Guid.NewGuid(); + var alreadyCanceled = Guid.NewGuid(); + var claimable = Guid.NewGuid(); + var delayed = Guid.NewGuid(); + var canceledAtUtc = DateTimeOffset.Parse("2026-04-20T12:30:00Z", CultureInfo.InvariantCulture); + + await context.Store.EnqueueAsync(PostgresTestData.CreateRequest(claimed)); + await PostgresTestData.ClaimAsync(context.Store); + + await context.Store.EnqueueAsync(PostgresTestData.CreateRequest( + retryWaiting, + maxAttempts: 2, + retryBackoffKind: RetryBackoffKind.Fixed, + retryBaseDelay: TimeSpan.FromHours(1))); + var retryClaim = await PostgresTestData.ClaimAsync(context.Store); + (await context.Store.MarkFailedAsync(new FailJobRequest(retryWaiting, "node-1", retryClaim.LeaseToken, DateTimeOffset.UtcNow, PostgresTestData.CreateFailure()))).ShouldBeTrue(); + + await context.Store.EnqueueAsync(PostgresTestData.CreateRequest(completed)); + var completedClaim = await PostgresTestData.ClaimAsync(context.Store); + (await context.Store.MarkCompletedAsync(new CompleteJobRequest(completed, "node-1", completedClaim.LeaseToken, DateTimeOffset.UtcNow))).ShouldBeTrue(); + + await context.Store.EnqueueAsync(PostgresTestData.CreateRequest(failed)); + var failedClaim = await PostgresTestData.ClaimAsync(context.Store); + (await context.Store.MarkFailedAsync(new FailJobRequest(failed, "node-1", failedClaim.LeaseToken, DateTimeOffset.UtcNow, PostgresTestData.CreateFailure()))).ShouldBeTrue(); + + await context.Store.EnqueueAsync(PostgresTestData.CreateRequest(alreadyCanceled)); + (await context.Store.CancelAsync(new CancelJobRequest(alreadyCanceled, DateTimeOffset.UtcNow))).ShouldBe(JobCancellationResult.Canceled); + + await context.Store.EnqueueAsync(PostgresTestData.CreateRequest(claimable)); + await context.Store.EnqueueAsync(PostgresTestData.CreateRequest(delayed, notBeforeUtc: DateTimeOffset.UtcNow.AddHours(1))); + + (await context.Store.CancelQueuedJobsAsync(new CancelQueuedJobsRequest(canceledAtUtc))).ShouldBe(3); + + var claimableJob = await context.ReadJobAsync(claimable); + claimableJob.State.ShouldBe("Canceled"); + claimableJob.CanceledAtUtc.ShouldBe(canceledAtUtc); + var delayedJob = await context.ReadJobAsync(delayed); + delayedJob.State.ShouldBe("Canceled"); + delayedJob.CanceledAtUtc.ShouldBe(canceledAtUtc); + var retryJob = await context.ReadJobAsync(retryWaiting); + retryJob.State.ShouldBe("Canceled"); + retryJob.CanceledAtUtc.ShouldBe(canceledAtUtc); + (await context.ReadJobAsync(claimed)).State.ShouldBe("Claimed"); + (await context.ReadJobAsync(completed)).State.ShouldBe("Completed"); + (await context.ReadJobAsync(failed)).State.ShouldBe("Failed"); + (await context.ReadJobAsync(alreadyCanceled)).State.ShouldBe("Canceled"); + + (await ReadEventsAsync(context, claimable)).Count(IsCanceledLifecycleEvent).ShouldBe(1); + (await ReadEventsAsync(context, delayed)).Count(IsCanceledLifecycleEvent).ShouldBe(1); + (await ReadEventsAsync(context, retryWaiting)).Count(IsCanceledLifecycleEvent).ShouldBe(1); + (await ReadEventsAsync(context, alreadyCanceled)).Count(IsCanceledLifecycleEvent).ShouldBe(1); + } + private static async ValueTask> ReadEventsAsync( PostgresTestContext context, Guid jobId) @@ -77,4 +139,8 @@ private static async ValueTask> ReadEventsAsync( return events; } + + private static bool IsCanceledLifecycleEvent(JobEvent jobEvent) + => jobEvent.Kind == JobEventKind.Lifecycle + && string.Equals(jobEvent.Message, "Canceled", StringComparison.Ordinal); } diff --git a/test/Sheddueller.Postgres.Tests/Operations/JobRetentionOperationTests.cs b/test/Sheddueller.Postgres.Tests/Operations/JobRetentionOperationTests.cs new file mode 100644 index 0000000..c5482a2 --- /dev/null +++ b/test/Sheddueller.Postgres.Tests/Operations/JobRetentionOperationTests.cs @@ -0,0 +1,43 @@ +namespace Sheddueller.Postgres.Tests.Operations; + +using Sheddueller.Inspection.Jobs; +using Sheddueller.Storage; + +using Shouldly; + +public sealed class JobRetentionOperationTests(PostgresFixture fixture) : IClassFixture +{ + [Fact] + public async Task CleanupTerminalJobs_DeletedJob_RemovesCascadingRows() + { + await using var context = await PostgresTestContext.CreateMigratedAsync(fixture); + var jobId = Guid.NewGuid(); + await context.Store.EnqueueAsync(PostgresTestData.CreateRequest( + jobId, + groupKeys: ["group-a"], + tags: [new JobTag("tenant", "acme")])); + var claimed = await PostgresTestData.ClaimAsync(context.Store); + await context.Store.MarkCompletedAsync(new CompleteJobRequest( + jobId, + "node-1", + claimed.LeaseToken, + DateTimeOffset.UtcNow.AddDays(-2))); + + (await context.ReadJobTagsAsync(jobId)).ShouldNotBeEmpty(); + (await context.ReadJobGroupKeysAsync(jobId)).ShouldNotBeEmpty(); + (await context.CountJobEventsAsync(jobId)).ShouldBeGreaterThan(0); + + var result = await ((IJobRetentionStore)context.Store).CleanupTerminalJobsAsync( + new JobRetentionCleanupRequest( + DateTimeOffset.UtcNow.AddDays(-1), + failedBeforeUtc: null, + canceledBeforeUtc: null, + batchSize: 10)); + + result.DeletedCount.ShouldBe(1); + (await ((IJobInspectionReader)context.Store).GetJobAsync(jobId)).ShouldBeNull(); + (await context.ReadJobTagsAsync(jobId)).ShouldBeEmpty(); + (await context.ReadJobGroupKeysAsync(jobId)).ShouldBeEmpty(); + (await context.CountJobEventsAsync(jobId)).ShouldBe(0); + } +} diff --git a/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs b/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs index 12e0c9d..ef197ef 100644 --- a/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs +++ b/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs @@ -28,6 +28,25 @@ public async Task Migration_Reapplied_IsIdempotent() (await context.ReadSchemaVersionAsync()).ShouldBe(PostgresNames.ExpectedSchemaVersion); } + [Fact] + public async Task Migration_Reapplied_DropsRedundantIndexes() + { + await using var context = await PostgresTestContext.CreateMigratedAsync(fixture); + await ExecuteAsync( + context, + $""" + create index idx_jobs_inspection_newest on {context.Table("jobs")} (enqueue_sequence desc); + create index idx_job_events_job_sequence on {context.Table("job_events")} (job_id, event_sequence); + """); + (await IndexExistsAsync(context, "idx_jobs_inspection_newest")).ShouldBeTrue(); + (await IndexExistsAsync(context, "idx_job_events_job_sequence")).ShouldBeTrue(); + + await context.Provider.GetRequiredService().ApplyAsync(); + + (await IndexExistsAsync(context, "idx_jobs_inspection_newest")).ShouldBeFalse(); + (await IndexExistsAsync(context, "idx_job_events_job_sequence")).ShouldBeFalse(); + } + [Fact] public async Task Migration_FreshSchema_CreatesIndexedHandlerSearchColumn() { @@ -175,4 +194,27 @@ private static async ValueTask ScalarAsync( result.ShouldNotBeNull(); return result.ShouldBeOfType(); } + + private static async ValueTask ExecuteAsync( + PostgresTestContext context, + string commandText) + { + await using var command = context.DataSource.CreateCommand(commandText); + await command.ExecuteNonQueryAsync(); + } + + private static async ValueTask IndexExistsAsync( + PostgresTestContext context, + string indexName) + => await ScalarAsync( + context, + """ + select exists ( + select 1 + from pg_indexes + where schemaname = @schema_name + and indexname = @index_name + ); + """, + command => command.Parameters.AddWithValue("index_name", indexName)); } diff --git a/test/Sheddueller.Postgres.Tests/PostgresRegistrationTests.cs b/test/Sheddueller.Postgres.Tests/PostgresRegistrationTests.cs index bf027c0..309e3ef 100644 --- a/test/Sheddueller.Postgres.Tests/PostgresRegistrationTests.cs +++ b/test/Sheddueller.Postgres.Tests/PostgresRegistrationTests.cs @@ -27,6 +27,7 @@ public async Task UsePostgres_ConnectionString_RegistersProviderServices() provider.GetRequiredService().SchemaName.ShouldBe("sheddueller"); provider.GetRequiredService().ShouldBeSameAs(provider.GetRequiredService()); provider.GetRequiredService().ShouldBeSameAs(provider.GetRequiredService()); + provider.GetRequiredService().ShouldBeSameAs(provider.GetRequiredService()); provider.GetRequiredService().ShouldBeOfType(); provider.GetRequiredService().ShouldBeOfType(); } diff --git a/test/Sheddueller.Postgres.Tests/ProviderContracts/PostgresJobRetentionStoreContractTests.cs b/test/Sheddueller.Postgres.Tests/ProviderContracts/PostgresJobRetentionStoreContractTests.cs new file mode 100644 index 0000000..ecfe48e --- /dev/null +++ b/test/Sheddueller.Postgres.Tests/ProviderContracts/PostgresJobRetentionStoreContractTests.cs @@ -0,0 +1,18 @@ +namespace Sheddueller.Postgres.Tests.ProviderContracts; + +using Sheddueller.Inspection.Jobs; +using Sheddueller.ProviderContracts; +using Sheddueller.Storage; + +public sealed class PostgresJobRetentionStoreContractTests(PostgresFixture fixture) : JobRetentionStoreContractTests, IClassFixture +{ + protected override async ValueTask CreateRetentionContextAsync() + { + var context = await PostgresTestContext.CreateMigratedAsync(fixture); + return new JobRetentionStoreContractContext( + context.Store, + (IJobRetentionStore)context.Store, + (IJobInspectionReader)context.Store, + context); + } +} diff --git a/test/Sheddueller.ProviderContracts/JobRetentionStoreContractContext.cs b/test/Sheddueller.ProviderContracts/JobRetentionStoreContractContext.cs new file mode 100644 index 0000000..0cc74a5 --- /dev/null +++ b/test/Sheddueller.ProviderContracts/JobRetentionStoreContractContext.cs @@ -0,0 +1,25 @@ +namespace Sheddueller.ProviderContracts; + +using Sheddueller.Inspection.Jobs; +using Sheddueller.Storage; + +public sealed class JobRetentionStoreContractContext( + IJobStore store, + IJobRetentionStore retentionStore, + IJobInspectionReader reader, + IAsyncDisposable? asyncDisposable = null) : IAsyncDisposable +{ + public IJobStore Store { get; } = store; + + public IJobRetentionStore RetentionStore { get; } = retentionStore; + + public IJobInspectionReader Reader { get; } = reader; + + public async ValueTask DisposeAsync() + { + if (asyncDisposable is not null) + { + await asyncDisposable.DisposeAsync().ConfigureAwait(false); + } + } +} diff --git a/test/Sheddueller.ProviderContracts/JobRetentionStoreContractTests.cs b/test/Sheddueller.ProviderContracts/JobRetentionStoreContractTests.cs new file mode 100644 index 0000000..856411f --- /dev/null +++ b/test/Sheddueller.ProviderContracts/JobRetentionStoreContractTests.cs @@ -0,0 +1,166 @@ +namespace Sheddueller.ProviderContracts; + +using Sheddueller.Inspection.Jobs; +using Sheddueller.Serialization; +using Sheddueller.Storage; + +using Shouldly; + +public abstract class JobRetentionStoreContractTests +{ + protected abstract ValueTask CreateRetentionContextAsync(); + + [Fact] + public async Task CleanupTerminalJobs_TerminalCutoffs_DeletesOnlyEligibleTerminalJobs() + { + await using var context = await this.CreateRetentionContextAsync(); + var now = DateTimeOffset.UtcNow; + var old = now.AddDays(-10); + var recent = now.AddHours(-1); + + var oldCompleted = await CompleteJobAsync(context.Store, old); + var recentCompleted = await CompleteJobAsync(context.Store, recent); + var oldFailed = await FailJobAsync(context.Store, old); + var recentFailed = await FailJobAsync(context.Store, recent); + var oldCanceled = await CancelJobAsync(context.Store, old); + var recentCanceled = await CancelJobAsync(context.Store, recent); + var oldQueued = await EnqueueJobAsync(context.Store, now.AddDays(-30), priority: -100); + var oldClaimed = await EnqueueJobAsync(context.Store, now.AddDays(-30), priority: 100); + + (await ClaimAsync(context.Store)).JobId.ShouldBe(oldClaimed); + + var result = await context.RetentionStore.CleanupTerminalJobsAsync( + new JobRetentionCleanupRequest( + now.AddDays(-1), + now.AddDays(-1), + now.AddDays(-1), + 20)); + + result.DeletedCount.ShouldBe(3); + await AssertDeletedAsync(context.Reader, oldCompleted); + await AssertDeletedAsync(context.Reader, oldFailed); + await AssertDeletedAsync(context.Reader, oldCanceled); + await AssertRetainedAsync(context.Reader, recentCompleted); + await AssertRetainedAsync(context.Reader, recentFailed); + await AssertRetainedAsync(context.Reader, recentCanceled); + await AssertRetainedAsync(context.Reader, oldQueued); + await AssertRetainedAsync(context.Reader, oldClaimed); + } + + [Fact] + public async Task CleanupTerminalJobs_BatchSize_DeletesOneBatchAtATime() + { + await using var context = await this.CreateRetentionContextAsync(); + var cutoff = DateTimeOffset.UtcNow.AddDays(-1); + var old = cutoff.AddDays(-1); + + var first = await CompleteJobAsync(context.Store, old); + var second = await CompleteJobAsync(context.Store, old.AddMinutes(1)); + var third = await CompleteJobAsync(context.Store, old.AddMinutes(2)); + var request = new JobRetentionCleanupRequest(cutoff, failedBeforeUtc: null, canceledBeforeUtc: null, batchSize: 2); + + (await context.RetentionStore.CleanupTerminalJobsAsync(request)).DeletedCount.ShouldBe(2); + (await context.RetentionStore.CleanupTerminalJobsAsync(request)).DeletedCount.ShouldBe(1); + (await context.RetentionStore.CleanupTerminalJobsAsync(request)).DeletedCount.ShouldBe(0); + + await AssertDeletedAsync(context.Reader, first); + await AssertDeletedAsync(context.Reader, second); + await AssertDeletedAsync(context.Reader, third); + } + + [Fact] + public async Task CleanupTerminalJobs_NullCutoff_RetainsThatTerminalState() + { + await using var context = await this.CreateRetentionContextAsync(); + var oldCompleted = await CompleteJobAsync(context.Store, DateTimeOffset.UtcNow.AddDays(-10)); + + var result = await context.RetentionStore.CleanupTerminalJobsAsync( + new JobRetentionCleanupRequest( + completedBeforeUtc: null, + failedBeforeUtc: DateTimeOffset.UtcNow, + canceledBeforeUtc: DateTimeOffset.UtcNow, + batchSize: 20)); + + result.DeletedCount.ShouldBe(0); + await AssertRetainedAsync(context.Reader, oldCompleted); + } + + private static async ValueTask CompleteJobAsync( + IJobStore store, + DateTimeOffset completedAtUtc) + { + var jobId = await EnqueueJobAsync(store, completedAtUtc.AddMinutes(-1)); + var claimed = await ClaimAsync(store, "complete-node"); + claimed.JobId.ShouldBe(jobId); + (await store.MarkCompletedAsync(new CompleteJobRequest(jobId, "complete-node", claimed.LeaseToken, completedAtUtc))) + .ShouldBeTrue(); + return jobId; + } + + private static async ValueTask FailJobAsync( + IJobStore store, + DateTimeOffset failedAtUtc) + { + var jobId = await EnqueueJobAsync(store, failedAtUtc.AddMinutes(-1)); + var claimed = await ClaimAsync(store, "fail-node"); + claimed.JobId.ShouldBe(jobId); + (await store.MarkFailedAsync(new FailJobRequest(jobId, "fail-node", claimed.LeaseToken, failedAtUtc, new JobFailureInfo("TestException", "failed", null)))) + .ShouldBeTrue(); + return jobId; + } + + private static async ValueTask CancelJobAsync( + IJobStore store, + DateTimeOffset canceledAtUtc) + { + var jobId = await EnqueueJobAsync(store, canceledAtUtc.AddMinutes(-1)); + (await store.CancelAsync(new CancelJobRequest(jobId, canceledAtUtc))).ShouldBe(JobCancellationResult.Canceled); + return jobId; + } + + private static async ValueTask EnqueueJobAsync( + IJobStore store, + DateTimeOffset enqueuedAtUtc, + int priority = 0) + { + var jobId = Guid.NewGuid(); + await store.EnqueueAsync(new EnqueueJobRequest( + jobId, + priority, + typeof(JobRetentionContractService).AssemblyQualifiedName!, + nameof(JobRetentionContractService.RunAsync), + [typeof(CancellationToken).AssemblyQualifiedName!], + new SerializedJobPayload(SystemTextJsonJobPayloadSerializer.JsonContentType, "[]"u8.ToArray()), + ConcurrencyGroupKeys: [], + enqueuedAtUtc, + NotBeforeUtc: null, + MaxAttempts: 1)); + return jobId; + } + + private static async ValueTask ClaimAsync( + IJobStore store, + string nodeId = "node-1") + { + var claimedAt = DateTimeOffset.UtcNow; + return (await store.TryClaimNextAsync(new ClaimJobRequest(nodeId, claimedAt, claimedAt.AddMinutes(5)))) + .ShouldBeOfType() + .Job; + } + + private static async ValueTask AssertDeletedAsync( + IJobInspectionReader reader, + Guid jobId) + => (await reader.GetJobAsync(jobId)).ShouldBeNull(); + + private static async ValueTask AssertRetainedAsync( + IJobInspectionReader reader, + Guid jobId) + => (await reader.GetJobAsync(jobId)).ShouldNotBeNull(); + + private sealed class JobRetentionContractService + { + public Task RunAsync(CancellationToken cancellationToken) + => Task.CompletedTask; + } +} diff --git a/test/Sheddueller.ProviderContracts/JobStoreContractTests.cs b/test/Sheddueller.ProviderContracts/JobStoreContractTests.cs index a876c13..1f30405 100644 --- a/test/Sheddueller.ProviderContracts/JobStoreContractTests.cs +++ b/test/Sheddueller.ProviderContracts/JobStoreContractTests.cs @@ -512,6 +512,56 @@ public async Task Cancel_TerminalOrMissingJob_ReturnsNonMutatingResult() (await context.Store.CancelAsync(new CancelJobRequest(Guid.NewGuid(), DateTimeOffset.UtcNow))).ShouldBe(JobCancellationResult.NotFound); } + [Fact] + public async Task CancelQueuedJobs_QueuedState_CancelsAllQueuedAndLeavesOthersUntouched() + { + await using var context = await this.CreateContextAsync(); + var claimed = Guid.NewGuid(); + var retryWaiting = Guid.NewGuid(); + var completed = Guid.NewGuid(); + var failed = Guid.NewGuid(); + var alreadyCanceled = Guid.NewGuid(); + var claimable = Guid.NewGuid(); + var delayed = Guid.NewGuid(); + + await context.Store.EnqueueAsync(CreateRequest(claimed)); + await ClaimAsync(context.Store); + + await context.Store.EnqueueAsync(CreateRequest( + retryWaiting, + maxAttempts: 2, + retryBackoffKind: RetryBackoffKind.Fixed, + retryBaseDelay: TimeSpan.FromHours(1))); + var retryClaim = await ClaimAsync(context.Store); + (await context.Store.MarkFailedAsync(new FailJobRequest(retryWaiting, "node-1", retryClaim.LeaseToken, DateTimeOffset.UtcNow, CreateFailure()))).ShouldBeTrue(); + + await context.Store.EnqueueAsync(CreateRequest(completed)); + var completedClaim = await ClaimAsync(context.Store); + (await context.Store.MarkCompletedAsync(new CompleteJobRequest(completed, "node-1", completedClaim.LeaseToken, DateTimeOffset.UtcNow))).ShouldBeTrue(); + + await context.Store.EnqueueAsync(CreateRequest(failed)); + var failedClaim = await ClaimAsync(context.Store); + (await context.Store.MarkFailedAsync(new FailJobRequest(failed, "node-1", failedClaim.LeaseToken, DateTimeOffset.UtcNow, CreateFailure()))).ShouldBeTrue(); + + await context.Store.EnqueueAsync(CreateRequest(alreadyCanceled)); + (await context.Store.CancelAsync(new CancelJobRequest(alreadyCanceled, DateTimeOffset.UtcNow))).ShouldBe(JobCancellationResult.Canceled); + + await context.Store.EnqueueAsync(CreateRequest(claimable)); + await context.Store.EnqueueAsync(CreateRequest(delayed, notBeforeUtc: DateTimeOffset.UtcNow.AddHours(1))); + + (await context.Store.CancelQueuedJobsAsync(new CancelQueuedJobsRequest(DateTimeOffset.UtcNow))).ShouldBe(3); + + var reader = GetInspectionReader(context); + (await reader.GetJobAsync(claimable)).ShouldNotBeNull().Summary.State.ShouldBe(JobState.Canceled); + (await reader.GetJobAsync(delayed)).ShouldNotBeNull().Summary.State.ShouldBe(JobState.Canceled); + (await reader.GetJobAsync(retryWaiting)).ShouldNotBeNull().Summary.State.ShouldBe(JobState.Canceled); + (await reader.GetJobAsync(claimed)).ShouldNotBeNull().Summary.State.ShouldBe(JobState.Claimed); + (await reader.GetJobAsync(completed)).ShouldNotBeNull().Summary.State.ShouldBe(JobState.Completed); + (await reader.GetJobAsync(failed)).ShouldNotBeNull().Summary.State.ShouldBe(JobState.Failed); + (await reader.GetJobAsync(alreadyCanceled)).ShouldNotBeNull().Summary.State.ShouldBe(JobState.Canceled); + (await context.Store.TryClaimNextAsync(CreateClaimRequest("node-1"))).ShouldBeOfType(); + } + [Fact] public async Task ConcurrencyLimit_SetAndGet_RoundTripsConfiguredLimit() { diff --git a/test/Sheddueller.Tests/JobManagerTests.cs b/test/Sheddueller.Tests/JobManagerTests.cs new file mode 100644 index 0000000..da7c138 --- /dev/null +++ b/test/Sheddueller.Tests/JobManagerTests.cs @@ -0,0 +1,27 @@ +namespace Sheddueller.Tests; + +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; + +using Sheddueller.Runtime; + +using Shouldly; + +public sealed class JobManagerTests +{ + [Fact] + public async Task CancelQueuedJobs_CurrentTime_PassesRequestToStoreAndReturnsCount() + { + var now = new DateTimeOffset(2026, 4, 20, 12, 30, 0, TimeSpan.Zero); + var store = new RecordingJobStore + { + CancelQueuedJobsResult = 12, + }; + var manager = new JobManager(store, new FakeTimeProvider(now), NullLogger.Instance); + + var canceledCount = await manager.CancelQueuedJobsAsync(); + + canceledCount.ShouldBe(12); + store.CancelQueuedJobsRequests.ShouldHaveSingleItem().CanceledAtUtc.ShouldBe(now); + } +} diff --git a/test/Sheddueller.Tests/RecordingJobStore.cs b/test/Sheddueller.Tests/RecordingJobStore.cs index 7b6feb5..b9b86f6 100644 --- a/test/Sheddueller.Tests/RecordingJobStore.cs +++ b/test/Sheddueller.Tests/RecordingJobStore.cs @@ -7,6 +7,7 @@ internal sealed class RecordingJobStore : IJobStore private readonly List enqueuedRequests = []; private readonly List recurringScheduleRequests = []; private readonly List triggerRequests = []; + private readonly List cancelQueuedJobsRequests = []; private long nextSequence; public IReadOnlyList EnqueuedRequests => this.enqueuedRequests; @@ -15,10 +16,14 @@ internal sealed class RecordingJobStore : IJobStore public IReadOnlyList TriggerRequests => this.triggerRequests; + public IReadOnlyList CancelQueuedJobsRequests => this.cancelQueuedJobsRequests; + public RecurringScheduleUpsertResult CreateOrUpdateRecurringScheduleResult { get; set; } = RecurringScheduleUpsertResult.Created; public RecurringScheduleTriggerResult TriggerResult { get; set; } = new(RecurringScheduleTriggerStatus.NotFound); + public int CancelQueuedJobsResult { get; set; } + public EnqueueJobRequest GetRequest(Guid jobId) => this.enqueuedRequests.Single(request => request.JobId == jobId); @@ -85,6 +90,16 @@ public ValueTask CancelAsync( CancellationToken cancellationToken = default) => throw CreateUnsupportedException(); + public ValueTask CancelQueuedJobsAsync( + CancelQueuedJobsRequest request, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + this.cancelQueuedJobsRequests.Add(request); + + return ValueTask.FromResult(this.CancelQueuedJobsResult); + } + public ValueTask GetCancellationRequestedAtAsync( JobCancellationStatusRequest request, CancellationToken cancellationToken = default) diff --git a/test/Sheddueller.Worker.Tests/RegistrationTests.cs b/test/Sheddueller.Worker.Tests/RegistrationTests.cs index 7155f34..0572e53 100644 --- a/test/Sheddueller.Worker.Tests/RegistrationTests.cs +++ b/test/Sheddueller.Worker.Tests/RegistrationTests.cs @@ -30,6 +30,21 @@ public void AddShedduellerWorker_RegistersClientAndWorkerServices() provider.GetServices().ShouldContain(service => service.GetType() == typeof(ShedduellerWorker)); } + [Fact] + public void AddShedduellerWorker_RetentionStore_RegistersRetentionHostedService() + { + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddSingleton(serviceProvider => serviceProvider.GetRequiredService()); + services.AddSingleton(); + + services.AddShedduellerWorker(); + + using var provider = services.BuildServiceProvider(); + + provider.GetServices().ShouldContain(service => service.GetType() == typeof(ShedduellerJobRetentionService)); + } + [Fact] public async Task StartupValidation_InvalidWorkerOption_FailsStart() { @@ -50,6 +65,34 @@ public async Task StartupValidation_InvalidWorkerOption_FailsStart() exception.Message.ShouldContain("ShedduellerOptions.MaxConcurrentExecutionsPerNode must be positive."); } + [Fact] + public async Task StartupValidation_InvalidJobRetentionOption_FailsStart() + { + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddSingleton(serviceProvider => serviceProvider.GetRequiredService()); + services.AddShedduellerWorker(builder => builder.ConfigureOptions(options => options.JobRetention.BatchSize = 0)); + using var provider = services.BuildServiceProvider(); + + var exception = await Should.ThrowAsync(async () => + { + foreach (var hostedService in provider.GetServices()) + { + await hostedService.StartAsync(CancellationToken.None); + } + }); + + exception.Message.ShouldContain("ShedduellerOptions.JobRetention.BatchSize must be positive."); + } + + private sealed class RecordingRetentionStore : IJobRetentionStore + { + public ValueTask CleanupTerminalJobsAsync( + JobRetentionCleanupRequest request, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(new JobRetentionCleanupResult(0)); + } + private sealed class RecordingJobStore : IJobStore { public ValueTask EnqueueAsync( @@ -98,6 +141,11 @@ public ValueTask CancelAsync( CancellationToken cancellationToken = default) => ValueTask.FromResult(JobCancellationResult.NotFound); + public ValueTask CancelQueuedJobsAsync( + CancelQueuedJobsRequest request, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(0); + public ValueTask GetCancellationRequestedAtAsync( JobCancellationStatusRequest request, CancellationToken cancellationToken = default) diff --git a/test/Sheddueller.Worker.Tests/WorkerJobLoggerTests.cs b/test/Sheddueller.Worker.Tests/WorkerJobLoggerTests.cs index d3954a3..b37cc32 100644 --- a/test/Sheddueller.Worker.Tests/WorkerJobLoggerTests.cs +++ b/test/Sheddueller.Worker.Tests/WorkerJobLoggerTests.cs @@ -537,6 +537,11 @@ public ValueTask CancelAsync( CancellationToken cancellationToken = default) => ValueTask.FromResult(JobCancellationResult.NotFound); + public ValueTask CancelQueuedJobsAsync( + CancelQueuedJobsRequest request, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(0); + public ValueTask GetCancellationRequestedAtAsync( JobCancellationStatusRequest request, CancellationToken cancellationToken = default) diff --git a/test/Sheddueller.Worker.Tests/WorkerLoggingTests.cs b/test/Sheddueller.Worker.Tests/WorkerLoggingTests.cs index 464bceb..5e5773d 100644 --- a/test/Sheddueller.Worker.Tests/WorkerLoggingTests.cs +++ b/test/Sheddueller.Worker.Tests/WorkerLoggingTests.cs @@ -137,6 +137,11 @@ public ValueTask CancelAsync( CancellationToken cancellationToken = default) => ValueTask.FromResult(JobCancellationResult.NotFound); + public ValueTask CancelQueuedJobsAsync( + CancelQueuedJobsRequest request, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(0); + public ValueTask GetCancellationRequestedAtAsync( JobCancellationStatusRequest request, CancellationToken cancellationToken = default) diff --git a/test/Sheddueller.Worker.Tests/WorkerProgressTests.cs b/test/Sheddueller.Worker.Tests/WorkerProgressTests.cs index 105c531..1dc890c 100644 --- a/test/Sheddueller.Worker.Tests/WorkerProgressTests.cs +++ b/test/Sheddueller.Worker.Tests/WorkerProgressTests.cs @@ -217,6 +217,11 @@ public ValueTask CancelAsync( CancellationToken cancellationToken = default) => ValueTask.FromResult(JobCancellationResult.NotFound); + public ValueTask CancelQueuedJobsAsync( + CancelQueuedJobsRequest request, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(0); + public ValueTask GetCancellationRequestedAtAsync( JobCancellationStatusRequest request, CancellationToken cancellationToken = default) From 7323e60caa3dcd00321350fe2cd8e7af72beb3cc Mon Sep 17 00:00:00 2001 From: Brian Tyler Date: Fri, 3 Jul 2026 12:46:12 +0100 Subject: [PATCH 2/6] feat: enhance job inspection overview with detailed state summaries and pagination support --- .../PostgresJobInspectionOperation.cs | 429 +++++++++++++++++- .../InspectionContractTests.cs | 188 ++++++++ 2 files changed, 597 insertions(+), 20 deletions(-) diff --git a/src/Sheddueller.Postgres/Internal/Operations/PostgresJobInspectionOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/PostgresJobInspectionOperation.cs index 75dc554..0a7b2eb 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/PostgresJobInspectionOperation.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/PostgresJobInspectionOperation.cs @@ -26,19 +26,82 @@ public static async ValueTask GetOverviewAsync( { await using var connection = await context.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); var counts = await ReadStateCountsAsync(context, connection, cancellationToken).ConfigureAwait(false); - var running = await ReadSummaryPageAsync(context, connection, "where state = 'Claimed' order by claimed_at_utc desc nulls last, enqueue_sequence desc limit 10", static _ => { }, cancellationToken) + var nowUtc = await ReadCurrentTimestampAsync(connection, cancellationToken).ConfigureAwait(false); + var runningRows = await ReadSummaryRowsAsync( + context, + connection, + "where state = 'Claimed' order by claimed_at_utc desc nulls last, enqueue_sequence desc limit 10", + static _ => { }, + cancellationToken) + .ConfigureAwait(false); + var recentlyFailedRows = await ReadSummaryRowsAsync( + context, + connection, + "where state = 'Failed' order by failed_at_utc desc nulls last, enqueue_sequence desc limit 10", + static _ => { }, + cancellationToken) + .ConfigureAwait(false); + var queuedRows = await ReadSummaryRowsAsync( + context, + connection, + $""" + where state = 'Queued' + and (not_before_utc is null or not_before_utc <= @now_utc) + and not exists ( + select 1 + from {context.Names.JobConcurrencyGroups} job_group + left join {context.Names.ConcurrencyGroups} concurrency_group on concurrency_group.group_key = job_group.group_key + where job_group.job_id = job.job_id + and coalesce(concurrency_group.in_use_count, 0) >= coalesce(concurrency_group.configured_limit, 1) + ) + order by priority desc, enqueue_sequence asc + limit 10 + """, + command => command.Parameters.AddWithValue("now_utc", nowUtc), + cancellationToken) + .ConfigureAwait(false); + var delayedRows = await ReadSummaryRowsAsync( + context, + connection, + """ + where state = 'Queued' + and not_before_utc > @now_utc + and failed_at_utc is null + order by not_before_utc asc, enqueue_sequence asc + limit 10 + """, + command => command.Parameters.AddWithValue("now_utc", nowUtc), + cancellationToken) .ConfigureAwait(false); - var recentlyFailed = await ReadSummaryPageAsync(context, connection, "where state = 'Failed' order by failed_at_utc desc nulls last, enqueue_sequence desc limit 10", static _ => { }, cancellationToken) + var retryWaitingRows = await ReadSummaryRowsAsync( + context, + connection, + """ + where state = 'Queued' + and not_before_utc > @now_utc + and failed_at_utc is not null + order by not_before_utc asc, enqueue_sequence asc + limit 10 + """, + command => command.Parameters.AddWithValue("now_utc", nowUtc), + cancellationToken) .ConfigureAwait(false); - var queuedPage = await SearchJobsAsync(context, new JobInspectionQuery(States: [JobState.Queued], PageSize: 100), cancellationToken).ConfigureAwait(false); + + var allRows = runningRows + .Concat(recentlyFailedRows) + .Concat(queuedRows) + .Concat(delayedRows) + .Concat(retryWaitingRows) + .ToArray(); + var summaries = await CreateSummaryMapAsync(context, connection, allRows, nowUtc, cancellationToken).ConfigureAwait(false); return new JobInspectionOverview( counts, - running, - recentlyFailed, - [.. queuedPage.Jobs.Where(job => job.QueuePosition?.Kind == JobQueuePositionKind.Claimable).Take(10)], - [.. queuedPage.Jobs.Where(job => job.QueuePosition?.Kind == JobQueuePositionKind.Delayed).Take(10)], - [.. queuedPage.Jobs.Where(job => job.QueuePosition?.Kind == JobQueuePositionKind.RetryWaiting).Take(10)]); + SelectSummaries(runningRows, summaries), + SelectSummaries(recentlyFailedRows, summaries), + SelectSummaries(queuedRows, summaries), + SelectSummaries(delayedRows, summaries), + SelectSummaries(retryWaitingRows, summaries)); } public static async ValueTask SearchJobsAsync( @@ -163,14 +226,62 @@ private static async ValueTask> CreateSummar NpgsqlConnection connection, IReadOnlyList rows, CancellationToken cancellationToken) + => await CreateSummariesAsync(context, connection, rows, nowUtc: null, cancellationToken).ConfigureAwait(false); + + private static async ValueTask> CreateSummariesAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + IReadOnlyList rows, + DateTimeOffset? nowUtc, + CancellationToken cancellationToken) { - var jobs = new List(rows.Count); + if (rows.Count == 0) + { + return []; + } + + var jobIds = rows.Select(static row => row.JobId).Distinct().ToArray(); + var tagsByJobId = await ReadTagsByJobIdAsync(context, connection, jobIds, cancellationToken).ConfigureAwait(false); + var groupsByJobId = await ReadGroupsByJobIdAsync(context, connection, jobIds, cancellationToken).ConfigureAwait(false); + var latestProgressByJobId = await ReadLatestProgressByJobIdAsync(context, connection, jobIds, cancellationToken).ConfigureAwait(false); + var queuePositionsByJobId = await ReadQueuePositionsAsync(context, connection, rows, nowUtc, cancellationToken).ConfigureAwait(false); + + return CreateSummaries(rows, tagsByJobId, groupsByJobId, latestProgressByJobId, queuePositionsByJobId); + } + + private static async ValueTask> CreateSummaryMapAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + IReadOnlyList rows, + DateTimeOffset nowUtc, + CancellationToken cancellationToken) + { + var summaries = await CreateSummariesAsync(context, connection, rows, nowUtc, cancellationToken).ConfigureAwait(false); + var byJobId = new Dictionary(summaries.Count); + foreach (var summary in summaries) + { + byJobId[summary.JobId] = summary; + } + + return byJobId; + } + + private static List SelectSummaries( + IReadOnlyList rows, + IReadOnlyDictionary summaries) + { + if (rows.Count == 0) + { + return []; + } + + var selected = new List(rows.Count); foreach (var row in rows) { - jobs.Add(await CreateSummaryAsync(context, connection, row, cancellationToken).ConfigureAwait(false)); + selected.Add(summaries[row.JobId]); } - return jobs; + return selected; } public static async ValueTask GetJobAsync( @@ -309,7 +420,7 @@ private static async ValueTask> ReadStateCoun private static string CreateWhereClause(List conditions) => conditions.Count == 0 ? string.Empty : $"where {string.Join(" and ", conditions)}"; - private static async ValueTask> ReadSummaryPageAsync( + private static async ValueTask> ReadSummaryRowsAsync( PostgresOperationContext context, NpgsqlConnection connection, string clause, @@ -324,14 +435,7 @@ private static async ValueTask> ReadSummaryP """; configure(command); - var rows = await ReadRowsAsync(command, cancellationToken).ConfigureAwait(false); - var jobs = new List(rows.Count); - foreach (var row in rows) - { - jobs.Add(await CreateSummaryAsync(context, connection, row, cancellationToken).ConfigureAwait(false)); - } - - return jobs; + return await ReadRowsAsync(command, cancellationToken).ConfigureAwait(false); } private static async ValueTask CreateSummaryAsync( @@ -366,6 +470,291 @@ await GetQueuePositionAsync(context, row.JobId, cancellationToken).ConfigureAwai ScheduleOccurrenceKind = row.ScheduleOccurrenceKind, }; + private static List CreateSummaries( + IReadOnlyList rows, + IReadOnlyDictionary> tagsByJobId, + IReadOnlyDictionary> groupsByJobId, + IReadOnlyDictionary latestProgressByJobId, + IReadOnlyDictionary queuePositionsByJobId) + { + var jobs = new List(rows.Count); + foreach (var row in rows) + { + jobs.Add(new JobInspectionSummary( + row.JobId, + row.State, + row.ServiceType, + row.MethodName, + row.Priority, + row.EnqueueSequence, + row.EnqueuedAtUtc, + row.NotBeforeUtc, + row.AttemptCount, + row.MaxAttempts, + tagsByJobId.TryGetValue(row.JobId, out var tags) ? tags : [], + groupsByJobId.TryGetValue(row.JobId, out var groups) ? groups : [], + row.SourceScheduleKey, + latestProgressByJobId.GetValueOrDefault(row.JobId), + queuePositionsByJobId[row.JobId], + row.ClaimedAtUtc, + row.CompletedAtUtc, + row.FailedAtUtc, + row.CanceledAtUtc) + { + RetryCloneSourceJobId = row.RetryCloneSourceJobId, + CancellationRequestedAtUtc = row.CancellationRequestedAtUtc, + CancellationObservedAtUtc = row.CancellationObservedAtUtc, + ScheduleOccurrenceKind = row.ScheduleOccurrenceKind, + }); + } + + return jobs; + } + + private static async ValueTask>> ReadTagsByJobIdAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + Guid[] jobIds, + CancellationToken cancellationToken) + { + if (jobIds.Length == 0) + { + return new Dictionary>(); + } + + await using var command = connection.CreateCommand(); + command.CommandText = + $""" + select job_id, name, value + from {context.Names.JobTags} + where job_id = any(@job_ids) + order by job_id asc, ordinal asc, name asc, value asc; + """; + command.Parameters.AddWithValue("job_ids", jobIds); + + var tags = new Dictionary>(); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + var jobId = reader.GetGuid(0); + if (!tags.TryGetValue(jobId, out var jobTags)) + { + jobTags = []; + tags.Add(jobId, jobTags); + } + + jobTags.Add(new JobTag(reader.GetString(1), reader.GetString(2))); + } + + var result = new Dictionary>(tags.Count); + foreach (var pair in tags) + { + result.Add(pair.Key, pair.Value); + } + + return result; + } + + private static async ValueTask>> ReadGroupsByJobIdAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + Guid[] jobIds, + CancellationToken cancellationToken) + { + if (jobIds.Length == 0) + { + return new Dictionary>(); + } + + await using var command = connection.CreateCommand(); + command.CommandText = + $""" + select job_id, group_key + from {context.Names.JobConcurrencyGroups} + where job_id = any(@job_ids) + order by job_id asc, group_key asc; + """; + command.Parameters.AddWithValue("job_ids", jobIds); + + var groups = new Dictionary>(); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + var jobId = reader.GetGuid(0); + if (!groups.TryGetValue(jobId, out var jobGroups)) + { + jobGroups = []; + groups.Add(jobId, jobGroups); + } + + jobGroups.Add(reader.GetString(1)); + } + + var result = new Dictionary>(groups.Count); + foreach (var pair in groups) + { + result.Add(pair.Key, pair.Value); + } + + return result; + } + + private static async ValueTask> ReadLatestProgressByJobIdAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + Guid[] jobIds, + CancellationToken cancellationToken) + { + if (jobIds.Length == 0) + { + return new Dictionary(); + } + + await using var command = connection.CreateCommand(); + command.CommandText = + $""" + select distinct on (job_id) + job_id, + progress_percent, + message, + occurred_at_utc + from {context.Names.JobEvents} + where job_id = any(@job_ids) + and kind = 'Progress' + order by job_id asc, event_sequence desc; + """; + command.Parameters.AddWithValue("job_ids", jobIds); + + var progress = new Dictionary(); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + progress.Add( + reader.GetGuid(0), + new JobProgressSnapshot( + reader.IsDBNull(1) ? null : reader.GetDouble(1), + reader.IsDBNull(2) ? null : reader.GetString(2), + PostgresConversion.ToDateTimeOffset(reader.GetValue(3)))); + } + + return progress; + } + + private static async ValueTask> ReadQueuePositionsAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + IReadOnlyList rows, + DateTimeOffset? nowUtc, + CancellationToken cancellationToken) + { + var positions = new Dictionary(rows.Count); + var readyQueuedRows = new List(); + DateTimeOffset? resolvedNowUtc = null; + + foreach (var row in rows) + { + switch (row.State) + { + case JobState.Canceled: + positions[row.JobId] = new JobQueuePosition(row.JobId, JobQueuePositionKind.Canceled, Position: null, "Job was canceled."); + break; + + case JobState.Completed: + case JobState.Failed: + positions[row.JobId] = new JobQueuePosition(row.JobId, JobQueuePositionKind.Terminal, Position: null, "Job is terminal."); + break; + + case JobState.Claimed: + positions[row.JobId] = new JobQueuePosition(row.JobId, JobQueuePositionKind.Claimed, Position: null, "Job is currently claimed."); + break; + + case JobState.Queued: + resolvedNowUtc ??= nowUtc ?? await ReadCurrentTimestampAsync(connection, cancellationToken).ConfigureAwait(false); + if (row.NotBeforeUtc is { } notBeforeUtc && notBeforeUtc > resolvedNowUtc.Value) + { + positions[row.JobId] = row.FailedAtUtc is null + ? new JobQueuePosition(row.JobId, JobQueuePositionKind.Delayed, Position: null, $"Job is delayed until {notBeforeUtc:O}.") + : new JobQueuePosition(row.JobId, JobQueuePositionKind.RetryWaiting, Position: null, $"Job is waiting to retry until {notBeforeUtc:O}."); + } + else + { + readyQueuedRows.Add(row); + } + + break; + + default: + throw new ArgumentOutOfRangeException(nameof(rows), row.State, "Job state is not supported."); + } + } + + if (readyQueuedRows.Count == 0) + { + return positions; + } + + resolvedNowUtc ??= nowUtc ?? await ReadCurrentTimestampAsync(connection, cancellationToken).ConfigureAwait(false); + var readyJobIds = readyQueuedRows.Select(static row => row.JobId).Distinct().ToArray(); + var claimablePositions = await ReadClaimablePositionsAsync(context, connection, readyJobIds, resolvedNowUtc.Value, cancellationToken) + .ConfigureAwait(false); + + foreach (var row in readyQueuedRows) + { + positions[row.JobId] = claimablePositions.TryGetValue(row.JobId, out var position) + ? new JobQueuePosition(row.JobId, JobQueuePositionKind.Claimable, position, "Job is currently claimable.") + : new JobQueuePosition(row.JobId, JobQueuePositionKind.BlockedByConcurrency, Position: null, "Job is blocked by concurrency group limits."); + } + + return positions; + } + + private static async ValueTask> ReadClaimablePositionsAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + Guid[] jobIds, + DateTimeOffset nowUtc, + CancellationToken cancellationToken) + { + if (jobIds.Length == 0) + { + return new Dictionary(); + } + + await using var command = connection.CreateCommand(); + command.CommandText = + $""" + with claimable as ( + select + job.job_id, + row_number() over (order by job.priority desc, job.enqueue_sequence asc) as position + from {context.Names.Jobs} job + where job.state = 'Queued' + and (job.not_before_utc is null or job.not_before_utc <= @now_utc) + and not exists ( + select 1 + from {context.Names.JobConcurrencyGroups} job_group + left join {context.Names.ConcurrencyGroups} concurrency_group on concurrency_group.group_key = job_group.group_key + where job_group.job_id = job.job_id + and coalesce(concurrency_group.in_use_count, 0) >= coalesce(concurrency_group.configured_limit, 1) + ) + ) + select job_id, position + from claimable + where job_id = any(@job_ids); + """; + command.Parameters.AddWithValue("now_utc", nowUtc); + command.Parameters.AddWithValue("job_ids", jobIds); + + var positions = new Dictionary(); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + positions.Add(reader.GetGuid(0), reader.GetInt64(1)); + } + + return positions; + } + private static async ValueTask ReadLatestProgressAsync( PostgresOperationContext context, NpgsqlConnection connection, diff --git a/test/Sheddueller.ProviderContracts/InspectionContractTests.cs b/test/Sheddueller.ProviderContracts/InspectionContractTests.cs index 59b8483..de71d34 100644 --- a/test/Sheddueller.ProviderContracts/InspectionContractTests.cs +++ b/test/Sheddueller.ProviderContracts/InspectionContractTests.cs @@ -250,6 +250,191 @@ public async Task SearchJobs_DefaultSort_OrdersClaimedThenQueuedByClaimOrder() page.Jobs.Select(job => job.QueuePosition?.Position).ShouldBe([null, 1L, 2L, 3L]); } + [Fact] + public async Task SearchJobs_PagedQueuedResults_KeepGlobalQueuePositions() + { + await using var context = await this.CreateContextAsync(); + var jobIds = Enumerable.Range(0, 5).Select(_ => Guid.NewGuid()).ToArray(); + + foreach (var jobId in jobIds) + { + await context.Store.EnqueueAsync(CreateRequest(jobId)); + } + + var firstPage = await context.Reader.SearchJobsAsync(new JobInspectionQuery( + States: [JobState.Queued], + PageSize: 2)); + var secondPage = await context.Reader.SearchJobsAsync(new JobInspectionQuery( + States: [JobState.Queued], + PageSize: 2, + ContinuationToken: firstPage.ContinuationToken)); + + firstPage.Jobs.Select(job => job.JobId).ShouldBe([jobIds[0], jobIds[1]]); + firstPage.Jobs.Select(job => job.QueuePosition?.Position).ShouldBe([1L, 2L]); + secondPage.Jobs.Select(job => job.JobId).ShouldBe([jobIds[2], jobIds[3]]); + secondPage.Jobs.Select(job => job.QueuePosition?.Position).ShouldBe([3L, 4L]); + } + + [Fact] + public async Task SearchJobs_MixedStates_ReportsEquivalentQueuePositionKinds() + { + await using var context = await this.CreateContextAsync(); + var completed = Guid.NewGuid(); + var failed = Guid.NewGuid(); + var retryWaiting = Guid.NewGuid(); + var running = Guid.NewGuid(); + var blocked = Guid.NewGuid(); + var claimable = Guid.NewGuid(); + var delayed = Guid.NewGuid(); + var canceled = Guid.NewGuid(); + + await context.Store.EnqueueAsync(CreateRequest(completed)); + var completedClaim = await ClaimAsync(context.Store); + completedClaim.JobId.ShouldBe(completed); + (await context.Store.MarkCompletedAsync(new CompleteJobRequest(completed, "node-1", completedClaim.LeaseToken, DateTimeOffset.UtcNow))).ShouldBeTrue(); + + await context.Store.EnqueueAsync(CreateRequest(failed)); + var failedClaim = await ClaimAsync(context.Store); + failedClaim.JobId.ShouldBe(failed); + (await context.Store.MarkFailedAsync(new FailJobRequest(failed, "node-1", failedClaim.LeaseToken, DateTimeOffset.UtcNow, CreateFailure()))).ShouldBeTrue(); + + await context.Store.EnqueueAsync(CreateRequest( + retryWaiting, + maxAttempts: 2, + retryBackoffKind: RetryBackoffKind.Fixed, + retryBaseDelay: TimeSpan.FromHours(1))); + var retryClaim = await ClaimAsync(context.Store); + retryClaim.JobId.ShouldBe(retryWaiting); + (await context.Store.MarkFailedAsync(new FailJobRequest(retryWaiting, "node-1", retryClaim.LeaseToken, DateTimeOffset.UtcNow, CreateFailure()))).ShouldBeTrue(); + + await context.Store.EnqueueAsync(CreateRequest(running, priority: 100, groupKeys: ["shared"])); + (await ClaimAsync(context.Store)).JobId.ShouldBe(running); + await context.Store.EnqueueAsync(CreateRequest(blocked, priority: 100, groupKeys: ["shared"])); + await context.Store.EnqueueAsync(CreateRequest(claimable, priority: 50)); + await context.Store.EnqueueAsync(CreateRequest(delayed, notBeforeUtc: DateTimeOffset.UtcNow.AddHours(1))); + await context.Store.EnqueueAsync(CreateRequest(canceled)); + (await context.Store.CancelAsync(new CancelJobRequest(canceled, DateTimeOffset.UtcNow))).ShouldBe(JobCancellationResult.Canceled); + + var page = await context.Reader.SearchJobsAsync(new JobInspectionQuery(PageSize: 20)); + var jobsById = page.Jobs.ToDictionary(static job => job.JobId); + + jobsById[running].QueuePosition?.Kind.ShouldBe(JobQueuePositionKind.Claimed); + jobsById[claimable].QueuePosition?.Kind.ShouldBe(JobQueuePositionKind.Claimable); + jobsById[claimable].QueuePosition?.Position.ShouldBe(1); + jobsById[blocked].QueuePosition?.Kind.ShouldBe(JobQueuePositionKind.BlockedByConcurrency); + jobsById[retryWaiting].QueuePosition?.Kind.ShouldBe(JobQueuePositionKind.RetryWaiting); + jobsById[delayed].QueuePosition?.Kind.ShouldBe(JobQueuePositionKind.Delayed); + jobsById[completed].QueuePosition?.Kind.ShouldBe(JobQueuePositionKind.Terminal); + jobsById[failed].QueuePosition?.Kind.ShouldBe(JobQueuePositionKind.Terminal); + jobsById[canceled].QueuePosition?.Kind.ShouldBe(JobQueuePositionKind.Canceled); + } + + [Fact] + public async Task Overview_MixedStates_ReturnsIndependentSectionsWithHydratedSummaries() + { + await using var context = await this.CreateContextAsync(); + var failed = Guid.NewGuid(); + var retryWaiting = Guid.NewGuid(); + var running = Guid.NewGuid(); + var claimable = Guid.NewGuid(); + var delayed = Guid.NewGuid(); + + await context.Store.EnqueueAsync(CreateRequest( + failed, + groupKeys: ["overview-group"], + tags: [new JobTag("tenant", "acme")])); + await context.EventSink.AppendAsync(new AppendJobEventRequest( + failed, + JobEventKind.Progress, + AttemptNumber: 0, + Message: "halfway", + ProgressPercent: 50)); + var failedClaim = await ClaimAsync(context.Store); + failedClaim.JobId.ShouldBe(failed); + (await context.Store.MarkFailedAsync(new FailJobRequest(failed, "node-1", failedClaim.LeaseToken, DateTimeOffset.UtcNow, CreateFailure()))).ShouldBeTrue(); + + await context.Store.EnqueueAsync(CreateRequest( + retryWaiting, + maxAttempts: 2, + retryBackoffKind: RetryBackoffKind.Fixed, + retryBaseDelay: TimeSpan.FromHours(1))); + var retryClaim = await ClaimAsync(context.Store); + retryClaim.JobId.ShouldBe(retryWaiting); + (await context.Store.MarkFailedAsync(new FailJobRequest(retryWaiting, "node-1", retryClaim.LeaseToken, DateTimeOffset.UtcNow, CreateFailure()))).ShouldBeTrue(); + + await context.Store.EnqueueAsync(CreateRequest(running)); + (await ClaimAsync(context.Store)).JobId.ShouldBe(running); + await context.Store.EnqueueAsync(CreateRequest(claimable, priority: 10)); + await context.Store.EnqueueAsync(CreateRequest(delayed, notBeforeUtc: DateTimeOffset.UtcNow.AddHours(1))); + + var overview = await context.Reader.GetOverviewAsync(); + + var runningSummary = overview.RunningJobs.Single(job => job.JobId == running); + runningSummary.QueuePosition?.Kind.ShouldBe(JobQueuePositionKind.Claimed); + + var failedSummary = overview.RecentlyFailedJobs.Single(job => job.JobId == failed); + failedSummary.Tags.ShouldBe([new JobTag("tenant", "acme")]); + failedSummary.ConcurrencyGroupKeys.ShouldBe(["overview-group"]); + failedSummary.LatestProgress.ShouldNotBeNull().Message.ShouldBe("halfway"); + failedSummary.QueuePosition?.Kind.ShouldBe(JobQueuePositionKind.Terminal); + + var claimableSummary = overview.QueuedJobs.Single(job => job.JobId == claimable); + claimableSummary.QueuePosition?.Kind.ShouldBe(JobQueuePositionKind.Claimable); + claimableSummary.QueuePosition?.Position.ShouldBe(1); + overview.DelayedJobs.Single(job => job.JobId == delayed).QueuePosition?.Kind.ShouldBe(JobQueuePositionKind.Delayed); + overview.RetryWaitingJobs.Single(job => job.JobId == retryWaiting).QueuePosition?.Kind.ShouldBe(JobQueuePositionKind.RetryWaiting); + } + + [Fact] + public async Task Overview_DelayedBacklog_DoesNotHideClaimableJobs() + { + await using var context = await this.CreateContextAsync(); + var claimable = Guid.NewGuid(); + var delayedRequests = Enumerable + .Range(0, 105) + .Select(index => CreateRequest(Guid.NewGuid(), notBeforeUtc: DateTimeOffset.UtcNow.AddHours(1).AddMinutes(index))) + .ToArray(); + + await context.Store.EnqueueManyAsync(delayedRequests); + await context.Store.EnqueueAsync(CreateRequest(claimable, priority: 100)); + + var overview = await context.Reader.GetOverviewAsync(); + + overview.QueuedJobs.Select(job => job.JobId).ShouldContain(claimable); + overview.QueuedJobs.Single(job => job.JobId == claimable).QueuePosition?.Kind.ShouldBe(JobQueuePositionKind.Claimable); + overview.DelayedJobs.Count.ShouldBe(10); + } + + [Fact] + public async Task Overview_ClaimableBacklog_DoesNotHideWaitingSections() + { + await using var context = await this.CreateContextAsync(); + var retryWaiting = Guid.NewGuid(); + var delayed = Guid.NewGuid(); + var claimableRequests = Enumerable + .Range(0, 105) + .Select(index => CreateRequest(Guid.NewGuid(), priority: 100 - index)) + .ToArray(); + + await context.Store.EnqueueAsync(CreateRequest( + retryWaiting, + maxAttempts: 2, + retryBackoffKind: RetryBackoffKind.Fixed, + retryBaseDelay: TimeSpan.FromHours(1))); + var retryClaim = await ClaimAsync(context.Store); + retryClaim.JobId.ShouldBe(retryWaiting); + (await context.Store.MarkFailedAsync(new FailJobRequest(retryWaiting, "node-1", retryClaim.LeaseToken, DateTimeOffset.UtcNow, CreateFailure()))).ShouldBeTrue(); + await context.Store.EnqueueAsync(CreateRequest(delayed, notBeforeUtc: DateTimeOffset.UtcNow.AddHours(1))); + await context.Store.EnqueueManyAsync(claimableRequests); + + var overview = await context.Reader.GetOverviewAsync(); + + overview.QueuedJobs.Count.ShouldBe(10); + overview.QueuedJobs.All(job => job.QueuePosition?.Kind == JobQueuePositionKind.Claimable).ShouldBeTrue(); + overview.DelayedJobs.Select(job => job.JobId).ShouldContain(delayed); + overview.RetryWaitingJobs.Select(job => job.JobId).ShouldContain(retryWaiting); + } + [Fact] public async Task SearchJobs_NewestFirstSort_OrdersByNewestEnqueueSequence() { @@ -597,6 +782,9 @@ protected static UpsertRecurringScheduleRequest CreateSchedule( DateTimeOffset.UtcNow, tags); + private static JobFailureInfo CreateFailure() + => new("TestException", "failed", "stack"); + private static async ValueTask> ReadAllAsync(IJobInspectionReader reader, Guid jobId) { var events = new List(); From 30fb039e868a3dc002f51a8dfeee74456cdac2f5 Mon Sep 17 00:00:00 2001 From: Brian Tyler Date: Fri, 3 Jul 2026 13:03:06 +0100 Subject: [PATCH 3/6] feat: update node health metrics handling and improve database queries for claimed jobs --- .../Components/Pages/Nodes.razor | 50 +++++-- .../PostgresNodeInspectionOperation.cs | 139 +++++++++++------- .../Internal/PostgresMigrator.cs | 4 + .../Internal/PostgresNames.cs | 2 +- .../DashboardEndpointTests.cs | 22 ++- .../PostgresMigrationTests.cs | 21 +++ .../InspectionContractTests.cs | 85 ++++++++++- 7 files changed, 250 insertions(+), 73 deletions(-) diff --git a/src/Sheddueller.Dashboard/Components/Pages/Nodes.razor b/src/Sheddueller.Dashboard/Components/Pages/Nodes.razor index 250244f..82d8e9e 100644 --- a/src/Sheddueller.Dashboard/Components/Pages/Nodes.razor +++ b/src/Sheddueller.Dashboard/Components/Pages/Nodes.razor @@ -1,7 +1,6 @@ @page "/nodes" @inherits DashboardPageComponent @inject INodeInspectionReader NodeReader -@inject IMetricsInspectionReader MetricsReader
@@ -528,7 +527,7 @@ private readonly List _nodes = []; private readonly DashboardNodeFilters _filters = new(); private NodeInspectionPage? _nodePage; - private MetricsInspectionWindow? _metricsWindow; + private NodeHealthCounts? _healthCounts; private string? _loadError; private bool _isLoading; private bool _isLoadingMore; @@ -537,16 +536,16 @@ => this._filters.ApplyClientFilter(this._nodes); private int ActiveNodeCount - => this._metricsWindow?.ActiveNodeCount ?? this._nodes.Count(node => node.State == NodeHealthState.Active); + => this._healthCounts?.ActiveNodeCount ?? this._nodes.Count(node => node.State == NodeHealthState.Active); private int StaleNodeCount - => this._metricsWindow?.StaleNodeCount ?? this._nodes.Count(node => node.State == NodeHealthState.Stale); + => this._healthCounts?.StaleNodeCount ?? this._nodes.Count(node => node.State == NodeHealthState.Stale); private int DeadNodeCount - => this._metricsWindow?.DeadNodeCount ?? this._nodes.Count(node => node.State == NodeHealthState.Dead); + => this._healthCounts?.DeadNodeCount ?? this._nodes.Count(node => node.State == NodeHealthState.Dead); private int TotalNodeCount - => this.ActiveNodeCount + this.StaleNodeCount + this.DeadNodeCount; + => this._healthCounts?.TotalNodeCount ?? this.ActiveNodeCount + this.StaleNodeCount + this.DeadNodeCount; private string EmptyText => this._nodes.Count == 0 @@ -671,16 +670,30 @@ this._filters.ToQuery(pageSize, continuationToken), cancellationToken) .AsTask(); - var metricsTask = MetricsReader.GetMetricsAsync( - new MetricsInspectionQuery([TimeSpan.FromMinutes(5)]), - cancellationToken) - .AsTask(); + var activeCountTask = this.ReadNodeHealthCountAsync(NodeHealthState.Active, cancellationToken); + var staleCountTask = this.ReadNodeHealthCountAsync(NodeHealthState.Stale, cancellationToken); + var deadCountTask = this.ReadNodeHealthCountAsync(NodeHealthState.Dead, cancellationToken); - await Task.WhenAll(nodePageTask, metricsTask); + await Task.WhenAll(nodePageTask, activeCountTask, staleCountTask, deadCountTask); return new NodesSnapshot( nodePageTask.Result, - metricsTask.Result.Windows.FirstOrDefault()); + new NodeHealthCounts( + activeCountTask.Result, + staleCountTask.Result, + deadCountTask.Result)); + } + + private async Task ReadNodeHealthCountAsync( + NodeHealthState state, + CancellationToken cancellationToken) + { + var page = await NodeReader.SearchNodesAsync( + new NodeInspectionQuery(State: state, PageSize: 1), + cancellationToken) + .ConfigureAwait(false); + + return Convert.ToInt32(page.TotalCount, CultureInfo.InvariantCulture); } private void ApplySnapshot( @@ -694,7 +707,7 @@ this._nodes.AddRange(snapshot.Page.Nodes); this._nodePage = snapshot.Page; - this._metricsWindow = snapshot.MetricsWindow; + this._healthCounts = snapshot.HealthCounts; this._loadError = null; } @@ -742,5 +755,14 @@ private sealed record NodesSnapshot( NodeInspectionPage Page, - MetricsInspectionWindow? MetricsWindow); + NodeHealthCounts HealthCounts); + + private sealed record NodeHealthCounts( + int ActiveNodeCount, + int StaleNodeCount, + int DeadNodeCount) + { + public int TotalNodeCount + => this.ActiveNodeCount + this.StaleNodeCount + this.DeadNodeCount; + } } diff --git a/src/Sheddueller.Postgres/Internal/Operations/PostgresNodeInspectionOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/PostgresNodeInspectionOperation.cs index 08379d2..11ede33 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/PostgresNodeInspectionOperation.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/PostgresNodeInspectionOperation.cs @@ -56,16 +56,18 @@ private static async ValueTask ReadTotalCountAsync( { await using var command = connection.CreateCommand(); var conditions = new List(); - ConfigureFilters(command, conditions, query); + ConfigureFilters(conditions, query); command.CommandText = $""" - {NodeSummaryCteSql(context)} select count(*) - from summary + from {context.Names.WorkerNodes} node {CreateWhereClause(conditions)}; """; - command.Parameters.AddWithValue("stale_threshold", staleThreshold); - command.Parameters.AddWithValue("dead_threshold", deadThreshold); + if (query.State is not null) + { + command.Parameters.AddWithValue("stale_threshold", staleThreshold); + command.Parameters.AddWithValue("dead_threshold", deadThreshold); + } return Convert.ToInt64(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false), CultureInfo.InvariantCulture); } @@ -80,28 +82,49 @@ private static async ValueTask> ReadNodeSum { await using var command = connection.CreateCommand(); var conditions = new List(); - ConfigureFilters(command, conditions, query); + ConfigureFilters(conditions, query); if (query.ContinuationToken is not null) { - conditions.Add("summary.node_id > @after_node_id"); + conditions.Add("node.node_id > @after_node_id"); command.Parameters.AddWithValue("after_node_id", query.ContinuationToken); } command.CommandText = $""" - {NodeSummaryCteSql(context)} + with page_nodes as ( + select + node.node_id, + {CreateNodeStateSql("node")} as state, + node.first_seen_at_utc, + node.last_heartbeat_at_utc, + node.max_concurrent_executions_per_node, + node.current_execution_count + from {context.Names.WorkerNodes} node + {CreateWhereClause(conditions)} + order by node.node_id asc + limit @limit + ), + claimed_counts as ( + select + job.claimed_by_node_id as node_id, + count(*) as claimed_count + from {context.Names.Jobs} job + join page_nodes node on node.node_id = job.claimed_by_node_id + where job.state = 'Claimed' + and job.claimed_by_node_id is not null + group by job.claimed_by_node_id + ) select - summary.node_id, - summary.state, - summary.first_seen_at_utc, - summary.last_heartbeat_at_utc, - summary.claimed_count, - summary.max_concurrent_executions_per_node, - summary.current_execution_count - from summary - {CreateWhereClause(conditions)} - order by summary.node_id asc - limit @limit; + page_nodes.node_id, + page_nodes.state, + page_nodes.first_seen_at_utc, + page_nodes.last_heartbeat_at_utc, + coalesce(claimed_counts.claimed_count, 0), + page_nodes.max_concurrent_executions_per_node, + page_nodes.current_execution_count + from page_nodes + left join claimed_counts on claimed_counts.node_id = page_nodes.node_id + order by page_nodes.node_id asc; """; command.Parameters.AddWithValue("stale_threshold", staleThreshold); command.Parameters.AddWithValue("dead_threshold", deadThreshold); @@ -121,17 +144,34 @@ order by summary.node_id asc await using var command = connection.CreateCommand(); command.CommandText = $""" - {NodeSummaryCteSql(context)} + with selected_node as ( + select + node.node_id, + {CreateNodeStateSql("node")} as state, + node.first_seen_at_utc, + node.last_heartbeat_at_utc, + node.max_concurrent_executions_per_node, + node.current_execution_count + from {context.Names.WorkerNodes} node + where node.node_id = @node_id + ), + claimed_counts as ( + select count(*) as claimed_count + from {context.Names.Jobs} job + join selected_node node on node.node_id = job.claimed_by_node_id + where job.state = 'Claimed' + and job.claimed_by_node_id is not null + ) select - summary.node_id, - summary.state, - summary.first_seen_at_utc, - summary.last_heartbeat_at_utc, - summary.claimed_count, - summary.max_concurrent_executions_per_node, - summary.current_execution_count - from summary - where summary.node_id = @node_id; + selected_node.node_id, + selected_node.state, + selected_node.first_seen_at_utc, + selected_node.last_heartbeat_at_utc, + claimed_counts.claimed_count, + selected_node.max_concurrent_executions_per_node, + selected_node.current_execution_count + from selected_node + cross join claimed_counts; """; command.Parameters.AddWithValue("node_id", nodeId); command.Parameters.AddWithValue("stale_threshold", staleThreshold); @@ -190,42 +230,33 @@ select job_id } private static void ConfigureFilters( - NpgsqlCommand command, List conditions, NodeInspectionQuery query) { - if (query.State is { } state) + if (query.State is not { } state) { - conditions.Add("summary.state = @state"); - command.Parameters.AddWithValue("state", state.ToString()); + return; } + + conditions.Add(state switch + { + NodeHealthState.Active => "transaction_timestamp() - node.last_heartbeat_at_utc < @stale_threshold", + NodeHealthState.Stale => "transaction_timestamp() - node.last_heartbeat_at_utc >= @stale_threshold and transaction_timestamp() - node.last_heartbeat_at_utc < @dead_threshold", + NodeHealthState.Dead => "transaction_timestamp() - node.last_heartbeat_at_utc >= @dead_threshold", + _ => throw new ArgumentOutOfRangeException(nameof(query), query.State, "Node health state is invalid."), + }); } private static string CreateWhereClause(List conditions) => conditions.Count == 0 ? string.Empty : $"where {string.Join(" and ", conditions)}"; - private static string NodeSummaryCteSql(PostgresOperationContext context) + private static string CreateNodeStateSql(string nodeAlias) => $""" - with summary as ( - select - node.node_id, - case - when transaction_timestamp() - node.last_heartbeat_at_utc >= @dead_threshold then 'Dead' - when transaction_timestamp() - node.last_heartbeat_at_utc >= @stale_threshold then 'Stale' - else 'Active' - end as state, - node.first_seen_at_utc, - node.last_heartbeat_at_utc, - ( - select count(*) - from {context.Names.Jobs} job - where job.state = 'Claimed' - and job.claimed_by_node_id = node.node_id - ) as claimed_count, - node.max_concurrent_executions_per_node, - node.current_execution_count - from {context.Names.WorkerNodes} node - ) + case + when transaction_timestamp() - {nodeAlias}.last_heartbeat_at_utc >= @dead_threshold then 'Dead' + when transaction_timestamp() - {nodeAlias}.last_heartbeat_at_utc >= @stale_threshold then 'Stale' + else 'Active' + end """; private static void ValidateQuery(NodeInspectionQuery query) diff --git a/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs b/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs index 259f70f..8337efd 100644 --- a/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs +++ b/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs @@ -300,6 +300,10 @@ constraint worker_nodes_current_execution_count_check check (current_execution_c on {this._names.Jobs} (lease_expires_at_utc) where state = 'Claimed'; + create index if not exists idx_jobs_claimed_by_node + on {this._names.Jobs} (claimed_by_node_id, enqueue_sequence) + where state = 'Claimed' and claimed_by_node_id is not null; + create index if not exists idx_jobs_source_schedule_nonterminal on {this._names.Jobs} (source_schedule_key) where state in ('Queued', 'Claimed'); diff --git a/src/Sheddueller.Postgres/Internal/PostgresNames.cs b/src/Sheddueller.Postgres/Internal/PostgresNames.cs index f383fa3..0eefe6d 100644 --- a/src/Sheddueller.Postgres/Internal/PostgresNames.cs +++ b/src/Sheddueller.Postgres/Internal/PostgresNames.cs @@ -4,7 +4,7 @@ namespace Sheddueller.Postgres.Internal; internal sealed class PostgresNames { - public const int ExpectedSchemaVersion = 9; + public const int ExpectedSchemaVersion = 10; public const string WakeupChannel = "sheddueller_wakeup"; public const string JobEventChannel = "sheddueller_job_event"; diff --git a/test/Sheddueller.Dashboard.Tests/DashboardEndpointTests.cs b/test/Sheddueller.Dashboard.Tests/DashboardEndpointTests.cs index 01d318a..b908e68 100644 --- a/test/Sheddueller.Dashboard.Tests/DashboardEndpointTests.cs +++ b/test/Sheddueller.Dashboard.Tests/DashboardEndpointTests.cs @@ -323,6 +323,19 @@ public async Task Nodes_KnownData_RendersRegistry() AssertShellRefresh(html); } + [Fact] + public async Task Nodes_HealthSummary_RendersFromNodeCountsWithoutMetricsData() + { + await using var app = await CreateStartedDashboardAsync(registerMetricsReader: false); + var html = await GetOkHtmlAsync(app, "/sheddueller/nodes"); + + html.ShouldContain("Total Nodes"); + html.ShouldContain("3"); + html.ShouldContain("cluster-wide"); + html.ShouldContain("33.3%"); + html.ShouldContain("Showing 1-3 of 3 nodes"); + } + [Fact] public async Task Metrics_KnownData_RendersRollingHealth() { @@ -479,7 +492,8 @@ public async Task JobDetail_MissingJob_RendersNotFoundWithDisabledCancelAction() private static async Task CreateStartedDashboardAsync( bool prerender = true, bool mapWithWebApplication = true, - Action? configureDashboard = null) + Action? configureDashboard = null, + bool registerMetricsReader = true) { var builder = WebApplication.CreateBuilder(); builder.WebHost.UseTestServer(); @@ -490,7 +504,11 @@ private static async Task CreateStartedDashboardAsync( builder.Services.AddSingleton(serviceProvider => serviceProvider.GetRequiredService()); builder.Services.AddSingleton(); builder.Services.AddSingleton(); - builder.Services.AddSingleton(); + if (registerMetricsReader) + { + builder.Services.AddSingleton(); + } + builder.Services.AddSingleton(); builder.Services.AddShedduellerDashboard(options => { diff --git a/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs b/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs index ef197ef..556c5e9 100644 --- a/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs +++ b/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs @@ -117,6 +117,27 @@ from pg_indexes indexDefinition.ShouldContain("state = 'Queued'"); } + [Fact] + public async Task Migration_FreshSchema_CreatesClaimedJobsByNodeIndex() + { + await using var context = await PostgresTestContext.CreateMigratedAsync(fixture); + + var indexDefinition = await ScalarAsync( + context, + """ + select indexdef + from pg_indexes + where schemaname = @schema_name + and indexname = 'idx_jobs_claimed_by_node'; + """); + + var normalized = indexDefinition.ToLowerInvariant(); + normalized.ShouldContain("claimed_by_node_id"); + normalized.ShouldContain("enqueue_sequence"); + normalized.ShouldContain("state = 'claimed'"); + normalized.ShouldContain("claimed_by_node_id is not null"); + } + [Fact] public async Task Migration_FreshSchema_CreatesTagOrdinalColumnsAndIndexes() { diff --git a/test/Sheddueller.ProviderContracts/InspectionContractTests.cs b/test/Sheddueller.ProviderContracts/InspectionContractTests.cs index de71d34..dd87220 100644 --- a/test/Sheddueller.ProviderContracts/InspectionContractTests.cs +++ b/test/Sheddueller.ProviderContracts/InspectionContractTests.cs @@ -668,6 +668,45 @@ await context.Store.RecordWorkerNodeHeartbeatAsync(new WorkerNodeHeartbeatReques metrics.Windows[0].ActiveNodeCount.ShouldBeGreaterThanOrEqualTo(1); } + [Fact] + public async Task NodeSearch_MultipleClaimedJobsAcrossNodes_ReturnsPerNodeClaimedCounts() + { + await using var context = await this.CreateContextAsync(); + var nodeAFirst = Guid.NewGuid(); + var nodeASecond = Guid.NewGuid(); + var nodeBJob = Guid.NewGuid(); + + await context.Store.RecordWorkerNodeHeartbeatAsync(new WorkerNodeHeartbeatRequest( + "node-a", + DateTimeOffset.UtcNow, + MaxConcurrentExecutionsPerNode: 4, + CurrentExecutionCount: 2)); + await context.Store.RecordWorkerNodeHeartbeatAsync(new WorkerNodeHeartbeatRequest( + "node-b", + DateTimeOffset.UtcNow, + MaxConcurrentExecutionsPerNode: 4, + CurrentExecutionCount: 1)); + await context.Store.RecordWorkerNodeHeartbeatAsync(new WorkerNodeHeartbeatRequest( + "node-c", + DateTimeOffset.UtcNow, + MaxConcurrentExecutionsPerNode: 4, + CurrentExecutionCount: 0)); + await context.Store.EnqueueAsync(CreateRequest(nodeAFirst)); + await context.Store.EnqueueAsync(CreateRequest(nodeASecond)); + await context.Store.EnqueueAsync(CreateRequest(nodeBJob)); + + (await ClaimAsync(context.Store, "node-a")).JobId.ShouldBe(nodeAFirst); + (await ClaimAsync(context.Store, "node-a")).JobId.ShouldBe(nodeASecond); + (await ClaimAsync(context.Store, "node-b")).JobId.ShouldBe(nodeBJob); + + var page = await context.NodeReader.SearchNodesAsync(new NodeInspectionQuery(PageSize: 10)); + var nodesById = page.Nodes.ToDictionary(node => node.NodeId); + + nodesById["node-a"].ClaimedJobCount.ShouldBe(2); + nodesById["node-b"].ClaimedJobCount.ShouldBe(1); + nodesById["node-c"].ClaimedJobCount.ShouldBe(0); + } + [Fact] public async Task NodeSearch_StateFilter_ReturnsTotalMatchingCount() { @@ -688,9 +727,49 @@ await context.Store.RecordWorkerNodeHeartbeatAsync(new WorkerNodeHeartbeatReques new NodeInspectionQuery(State: NodeHealthState.Active, PageSize: 1)); page.Nodes.Count.ShouldBe(1); + page.Nodes.Select(node => node.NodeId).ShouldBe(["node-a"]); page.Nodes[0].State.ShouldBe(NodeHealthState.Active); page.TotalCount.ShouldBe(2L); page.ContinuationToken.ShouldNotBeNull(); + + var secondPage = await context.NodeReader.SearchNodesAsync( + new NodeInspectionQuery(State: NodeHealthState.Active, PageSize: 1, ContinuationToken: page.ContinuationToken)); + + secondPage.Nodes.Select(node => node.NodeId).ShouldBe(["node-b"]); + secondPage.TotalCount.ShouldBe(2L); + secondPage.ContinuationToken.ShouldBeNull(); + } + + [Fact] + public async Task NodeDetail_ClaimedJobs_ReturnsClaimedJobIdsForNode() + { + await using var context = await this.CreateContextAsync(); + var nodeAFirst = Guid.NewGuid(); + var nodeBJob = Guid.NewGuid(); + var nodeASecond = Guid.NewGuid(); + + await context.Store.RecordWorkerNodeHeartbeatAsync(new WorkerNodeHeartbeatRequest( + "node-a", + DateTimeOffset.UtcNow, + MaxConcurrentExecutionsPerNode: 4, + CurrentExecutionCount: 2)); + await context.Store.RecordWorkerNodeHeartbeatAsync(new WorkerNodeHeartbeatRequest( + "node-b", + DateTimeOffset.UtcNow, + MaxConcurrentExecutionsPerNode: 4, + CurrentExecutionCount: 1)); + await context.Store.EnqueueAsync(CreateRequest(nodeAFirst)); + await context.Store.EnqueueAsync(CreateRequest(nodeBJob)); + await context.Store.EnqueueAsync(CreateRequest(nodeASecond)); + + (await ClaimAsync(context.Store, "node-a")).JobId.ShouldBe(nodeAFirst); + (await ClaimAsync(context.Store, "node-b")).JobId.ShouldBe(nodeBJob); + (await ClaimAsync(context.Store, "node-a")).JobId.ShouldBe(nodeASecond); + + var detail = await context.NodeReader.GetNodeAsync("node-a"); + + detail.ShouldNotBeNull().Summary.ClaimedJobCount.ShouldBe(2); + detail.ClaimedJobIds.ShouldBe([nodeAFirst, nodeASecond]); } [Fact] @@ -728,8 +807,10 @@ await context.Store.RecordWorkerNodeHeartbeatAsync(new WorkerNodeHeartbeatReques secondPage.ContinuationToken.ShouldNotBeNull(); } - protected static async ValueTask ClaimAsync(IJobStore store) - => (await store.TryClaimNextAsync(new ClaimJobRequest("node-1", DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddSeconds(30)))) + protected static async ValueTask ClaimAsync( + IJobStore store, + string nodeId = "node-1") + => (await store.TryClaimNextAsync(new ClaimJobRequest(nodeId, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddSeconds(30)))) .ShouldBeOfType() .Job; From 3c271fb0ce0da250d3a5539e082b5ef4dcb0ef7a Mon Sep 17 00:00:00 2001 From: Brian Tyler Date: Fri, 3 Jul 2026 13:29:20 +0100 Subject: [PATCH 4/6] feat(metrics): add metrics tracking tables and rollup functionality - Introduced new tables: `metrics_buckets`, `metrics_histogram_bins`, and `metrics_rollup_state` to track job metrics. - Implemented logic for recording job events and updating metrics in the database. - Added cleanup functionality to remove old metrics data based on retention policy. - Created a caching mechanism for dashboard metrics to improve performance. - Added tests to ensure metrics rollup and caching work as expected. --- .../Components/Pages/Metrics.razor | 4 +- .../Internal/DashboardMetricsSnapshotCache.cs | 94 ++++ .../Internal/IDashboardMetricsReader.cs | 10 + ...lerDashboardServiceCollectionExtensions.cs | 2 + .../Operations/CancelQueuedJobsOperation.cs | 7 + .../Operations/EnqueueJobOperation.cs | 2 + .../Internal/Operations/PostgresJobEvents.cs | 6 +- .../PostgresMetricsInspectionOperation.cs | 278 ++++++----- .../Operations/PostgresMetricsRollups.cs | 445 ++++++++++++++++++ .../Internal/PostgresMigrator.cs | 40 ++ .../Internal/PostgresNames.cs | 11 +- .../DashboardMetricsSnapshotCacheTests.cs | 122 +++++ .../PostgresMetricsRollupTests.cs | 58 +++ .../PostgresMigrationTests.cs | 52 ++ .../InspectionContractTests.cs | 73 ++- 15 files changed, 1081 insertions(+), 123 deletions(-) create mode 100644 src/Sheddueller.Dashboard/Internal/DashboardMetricsSnapshotCache.cs create mode 100644 src/Sheddueller.Dashboard/Internal/IDashboardMetricsReader.cs create mode 100644 src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsRollups.cs create mode 100644 test/Sheddueller.Dashboard.Tests/DashboardMetricsSnapshotCacheTests.cs create mode 100644 test/Sheddueller.Postgres.Tests/PostgresMetricsRollupTests.cs diff --git a/src/Sheddueller.Dashboard/Components/Pages/Metrics.razor b/src/Sheddueller.Dashboard/Components/Pages/Metrics.razor index 8783e1f..86f3e10 100644 --- a/src/Sheddueller.Dashboard/Components/Pages/Metrics.razor +++ b/src/Sheddueller.Dashboard/Components/Pages/Metrics.razor @@ -1,6 +1,6 @@ @page "/metrics" @inherits DashboardPageComponent -@inject IMetricsInspectionReader Reader +@inject IDashboardMetricsReader Reader
@@ -975,7 +975,7 @@ @code { - private static readonly TimeSpan PageRefreshInterval = TimeSpan.FromSeconds(5); + private static readonly TimeSpan PageRefreshInterval = TimeSpan.FromSeconds(30); private static readonly TimeSpan[] DefaultMetricWindows = [TimeSpan.FromMinutes(5), TimeSpan.FromHours(1), TimeSpan.FromHours(24)]; diff --git a/src/Sheddueller.Dashboard/Internal/DashboardMetricsSnapshotCache.cs b/src/Sheddueller.Dashboard/Internal/DashboardMetricsSnapshotCache.cs new file mode 100644 index 0000000..c21a744 --- /dev/null +++ b/src/Sheddueller.Dashboard/Internal/DashboardMetricsSnapshotCache.cs @@ -0,0 +1,94 @@ +namespace Sheddueller.Dashboard.Internal; + +using System.Globalization; + +using Sheddueller.Inspection.Metrics; + +internal sealed class DashboardMetricsSnapshotCache( + IMetricsInspectionReader reader, + TimeProvider timeProvider) : IDashboardMetricsReader +{ + internal static readonly TimeSpan TimeToLive = TimeSpan.FromSeconds(30); + + private readonly Lock _gate = new(); + private readonly Dictionary _cache = []; + private readonly Dictionary> _inflight = []; + + public async ValueTask GetMetricsAsync( + MetricsInspectionQuery query, + CancellationToken cancellationToken = default) + { + var stableQuery = CreateStableQuery(query); + var key = MetricsCacheKey.From(stableQuery); + var nowUtc = timeProvider.GetUtcNow(); + Task readTask; + var ownsRead = false; + + lock (this._gate) + { + if (this._cache.TryGetValue(key, out var entry) + && nowUtc - entry.CachedAtUtc < TimeToLive) + { + return entry.Snapshot; + } + + if (!this._inflight.TryGetValue(key, out readTask!)) + { + readTask = this.ReadAndCacheAsync(key, stableQuery, cancellationToken); + this._inflight[key] = readTask; + ownsRead = true; + } + } + + try + { + return (await readTask.WaitAsync(cancellationToken).ConfigureAwait(false)).Snapshot; + } + finally + { + if (ownsRead) + { + lock (this._gate) + { + if (this._inflight.TryGetValue(key, out var current) && ReferenceEquals(current, readTask)) + { + this._inflight.Remove(key); + } + } + } + } + } + + private async Task ReadAndCacheAsync( + MetricsCacheKey key, + MetricsInspectionQuery query, + CancellationToken cancellationToken) + { + var snapshot = await reader.GetMetricsAsync(query, cancellationToken).ConfigureAwait(false); + var entry = new MetricsCacheEntry(snapshot, timeProvider.GetUtcNow()); + + lock (this._gate) + { + this._cache[key] = entry; + } + + return entry; + } + + private static MetricsInspectionQuery CreateStableQuery(MetricsInspectionQuery query) + => query.Windows is { Count: > 0 } windows + ? new MetricsInspectionQuery([.. windows]) + : new MetricsInspectionQuery(); + + private readonly record struct MetricsCacheKey(string Value) + { + public static MetricsCacheKey From(MetricsInspectionQuery query) + => query.Windows is { Count: > 0 } windows + ? new MetricsCacheKey(string.Join("|", windows.Select(static window => window.Ticks.ToString(CultureInfo.InvariantCulture)))) + : new MetricsCacheKey(""); + } + + private sealed record MetricsCacheEntry( + MetricsInspectionSnapshot Snapshot, + DateTimeOffset CachedAtUtc); +} diff --git a/src/Sheddueller.Dashboard/Internal/IDashboardMetricsReader.cs b/src/Sheddueller.Dashboard/Internal/IDashboardMetricsReader.cs new file mode 100644 index 0000000..d186e12 --- /dev/null +++ b/src/Sheddueller.Dashboard/Internal/IDashboardMetricsReader.cs @@ -0,0 +1,10 @@ +namespace Sheddueller.Dashboard.Internal; + +using Sheddueller.Inspection.Metrics; + +internal interface IDashboardMetricsReader +{ + ValueTask GetMetricsAsync( + MetricsInspectionQuery query, + CancellationToken cancellationToken = default); +} diff --git a/src/Sheddueller.Dashboard/ShedduellerDashboardServiceCollectionExtensions.cs b/src/Sheddueller.Dashboard/ShedduellerDashboardServiceCollectionExtensions.cs index 30155e1..9288827 100644 --- a/src/Sheddueller.Dashboard/ShedduellerDashboardServiceCollectionExtensions.cs +++ b/src/Sheddueller.Dashboard/ShedduellerDashboardServiceCollectionExtensions.cs @@ -40,6 +40,8 @@ public static IServiceCollection AddShedduellerDashboard( services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(serviceProvider => serviceProvider.GetRequiredService()); + services.TryAddSingleton(); + services.TryAddSingleton(serviceProvider => serviceProvider.GetRequiredService()); services.Replace(ServiceDescriptor.Singleton()); TryAddStartupValidationHostedService(services); services.TryAddEnumerable(ServiceDescriptor.Singleton()); diff --git a/src/Sheddueller.Postgres/Internal/Operations/CancelQueuedJobsOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/CancelQueuedJobsOperation.cs index a63b976..5167a42 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/CancelQueuedJobsOperation.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/CancelQueuedJobsOperation.cs @@ -20,6 +20,13 @@ public static async ValueTask ExecuteAsync( if (canceledJobs.Count > 0) { await InsertLifecycleEventsAsync(context, connection, transaction, canceledJobs, cancellationToken).ConfigureAwait(false); + await PostgresMetricsRollups.RecordCanceledJobsAsync( + context, + connection, + transaction, + [.. canceledJobs.Select(static job => job.JobId)], + cancellationToken) + .ConfigureAwait(false); await NotifyLifecycleEventsAsync(context, connection, transaction, canceledJobs, cancellationToken).ConfigureAwait(false); } diff --git a/src/Sheddueller.Postgres/Internal/Operations/EnqueueJobOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/EnqueueJobOperation.cs index 69f4ad1..e5a5f75 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/EnqueueJobOperation.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/EnqueueJobOperation.cs @@ -59,6 +59,8 @@ public static async ValueTask> ExecuteManyAsync( await InsertStagedGroupsAsync(context, connection, transaction, cancellationToken).ConfigureAwait(false); await InsertStagedTagsAsync(context, connection, transaction, cancellationToken).ConfigureAwait(false); await InsertStagedEventsAsync(context, connection, transaction, cancellationToken).ConfigureAwait(false); + await PostgresMetricsRollups.RecordStagedQueuedJobsAsync(context, connection, transaction, cancellationToken) + .ConfigureAwait(false); await NotifyStagedEventsAsync(context, connection, transaction, cancellationToken).ConfigureAwait(false); await context.NotifyAsync(connection, transaction, cancellationToken).ConfigureAwait(false); diff --git a/src/Sheddueller.Postgres/Internal/Operations/PostgresJobEvents.cs b/src/Sheddueller.Postgres/Internal/Operations/PostgresJobEvents.cs index 4506327..856ee4d 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/PostgresJobEvents.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/PostgresJobEvents.cs @@ -143,7 +143,7 @@ private static async ValueTask AppendInTransactionAsync( var occurredAtUtc = PostgresConversion.ToDateTimeOffset(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false) ?? throw new InvalidOperationException("PostgreSQL did not return a job event timestamp.")); - return new JobEvent( + var jobEvent = new JobEvent( eventId, request.JobId, eventSequence, @@ -154,6 +154,10 @@ private static async ValueTask AppendInTransactionAsync( request.Message, request.ProgressPercent, request.Fields); + await PostgresMetricsRollups.RecordJobEventAsync(context, connection, transaction, jobEvent, cancellationToken) + .ConfigureAwait(false); + + return jobEvent; } private static async ValueTask IncrementEventSequenceAsync( diff --git a/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsInspectionOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsInspectionOperation.cs index 7eebbe4..b4b4cd9 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsInspectionOperation.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsInspectionOperation.cs @@ -8,6 +8,10 @@ namespace Sheddueller.Postgres.Internal.Operations; internal static class PostgresMetricsInspectionOperation { + private const string QueueLatencyMetric = "queue_latency"; + private const string ExecutionDurationMetric = "execution_duration"; + private const string ScheduleFireLagMetric = "schedule_fire_lag"; + private static readonly TimeSpan[] DefaultMetricWindows = [TimeSpan.FromMinutes(5), TimeSpan.FromHours(1), TimeSpan.FromHours(24)]; public static async ValueTask GetAsync( @@ -24,10 +28,13 @@ public static async ValueTask GetAsync( } await using var connection = await context.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + await PostgresMetricsRollups.CleanupAsync(context, connection, cancellationToken).ConfigureAwait(false); + var current = await ReadCurrentCountsAsync(context, connection, staleThreshold, deadThreshold, cancellationToken).ConfigureAwait(false); + var metrics = new List(windows.Count); foreach (var window in windows) { - metrics.Add(await ReadWindowAsync(context, connection, window, staleThreshold, deadThreshold, cancellationToken).ConfigureAwait(false)); + metrics.Add(await ReadWindowAsync(context, connection, window, current, cancellationToken).ConfigureAwait(false)); } return new MetricsInspectionSnapshot(metrics); @@ -37,79 +44,40 @@ private static async ValueTask ReadWindowAsync( PostgresOperationContext context, NpgsqlConnection connection, TimeSpan window, - TimeSpan staleThreshold, - TimeSpan deadThreshold, + PostgresCurrentMetricsCounts current, CancellationToken cancellationToken) { - var counts = await ReadCountsAsync(context, connection, window, staleThreshold, deadThreshold, cancellationToken).ConfigureAwait(false); - var (queueLatencyP50, queueLatencyP95) = await ReadPercentilesAsync( - connection, - $""" - select extract(epoch from (claimed_at_utc - enqueued_at_utc)) * 1000 as value_ms - from {context.Names.Jobs} - where claimed_at_utc >= transaction_timestamp() - @window - and claimed_at_utc is not null - and claimed_at_utc >= enqueued_at_utc - """, - window, - cancellationToken) - .ConfigureAwait(false); - var (executionDurationP50, executionDurationP95) = await ReadPercentilesAsync( - connection, - $""" - select extract(epoch from (coalesce(completed_at_utc, failed_at_utc, canceled_at_utc) - claimed_at_utc)) * 1000 as value_ms - from {context.Names.Jobs} - where state in ('Completed', 'Failed', 'Canceled') - and claimed_at_utc is not null - and coalesce(completed_at_utc, failed_at_utc, canceled_at_utc) >= transaction_timestamp() - @window - and coalesce(completed_at_utc, failed_at_utc, canceled_at_utc) >= claimed_at_utc - """, - window, - cancellationToken) - .ConfigureAwait(false); - var (_, scheduleFireLagP95) = await ReadPercentilesAsync( - connection, - $""" - select extract(epoch from (enqueued_at_utc - scheduled_fire_at_utc)) * 1000 as value_ms - from {context.Names.Jobs} - where schedule_occurrence_kind = 'Automatic' - and scheduled_fire_at_utc is not null - and enqueued_at_utc >= transaction_timestamp() - @window - and enqueued_at_utc >= scheduled_fire_at_utc - """, - window, - cancellationToken) - .ConfigureAwait(false); - + var counts = await ReadWindowCountsAsync(context, connection, window, cancellationToken).ConfigureAwait(false); + var percentiles = await ReadWindowPercentilesAsync(context, connection, window, cancellationToken).ConfigureAwait(false); var minutes = Math.Max(window.TotalMinutes, double.Epsilon); + return new MetricsInspectionWindow( window, - counts.QueuedCount, - counts.ClaimedCount, - counts.FailedCount, - counts.CanceledCount, - counts.OldestQueuedAge, + current.QueuedCount, + current.ClaimedCount, + Convert.ToInt32(counts.FailedCount, CultureInfo.InvariantCulture), + Convert.ToInt32(counts.CanceledCount, CultureInfo.InvariantCulture), + current.OldestQueuedAge, counts.EnqueuedCount / minutes, counts.ClaimedStartedCount / minutes, counts.SucceededCount / minutes, counts.FailedCount / minutes, counts.CanceledCount / minutes, counts.RetryEventCount / minutes, - queueLatencyP50, - queueLatencyP95, - executionDurationP50, - executionDurationP95, - scheduleFireLagP95, - counts.SaturatedGroupCount, - counts.ActiveNodeCount, - counts.StaleNodeCount, - counts.DeadNodeCount); + percentiles.QueueLatencyP50, + percentiles.QueueLatencyP95, + percentiles.ExecutionDurationP50, + percentiles.ExecutionDurationP95, + percentiles.ScheduleFireLagP95, + current.SaturatedGroupCount, + current.ActiveNodeCount, + current.StaleNodeCount, + current.DeadNodeCount); } - private static async ValueTask ReadCountsAsync( + private static async ValueTask ReadCurrentCountsAsync( PostgresOperationContext context, NpgsqlConnection connection, - TimeSpan window, TimeSpan staleThreshold, TimeSpan deadThreshold, CancellationToken cancellationToken) @@ -117,30 +85,17 @@ private static async ValueTask ReadCountsAsync( await using var command = connection.CreateCommand(); command.CommandText = $""" - with current_counts as ( - select - count(*) filter (where state = 'Queued') as queued_count, - count(*) filter (where state = 'Claimed') as claimed_count, - max(transaction_timestamp() - enqueued_at_utc) filter (where state = 'Queued') as oldest_queued_age - from {context.Names.Jobs} - ), - terminal_counts as ( + with queued_counts as ( select - count(*) filter (where state = 'Completed' and completed_at_utc >= transaction_timestamp() - @window) as succeeded_count, - count(*) filter (where state = 'Failed' and failed_at_utc >= transaction_timestamp() - @window) as failed_count, - count(*) filter (where state = 'Canceled' and canceled_at_utc >= transaction_timestamp() - @window) as canceled_count + count(*) as queued_count, + max(transaction_timestamp() - enqueued_at_utc) as oldest_queued_age from {context.Names.Jobs} + where state = 'Queued' ), - event_counts as ( - select - count(*) filter (where kind = 'AttemptStarted' and occurred_at_utc >= transaction_timestamp() - @window) as claimed_started_count, - count(*) filter (where kind = 'AttemptFailed' and occurred_at_utc >= transaction_timestamp() - @window) as retry_event_count - from {context.Names.JobEvents} - ), - enqueue_counts as ( - select count(*) as enqueued_count + claimed_counts as ( + select count(*) as claimed_count from {context.Names.Jobs} - where enqueued_at_utc >= transaction_timestamp() - @window + where state = 'Claimed' ), saturated_groups as ( select count(*) as saturated_group_count @@ -155,89 +110,176 @@ node_counts as ( from {context.Names.WorkerNodes} ) select - current_counts.queued_count, - current_counts.claimed_count, - current_counts.oldest_queued_age, - terminal_counts.succeeded_count, - terminal_counts.failed_count, - terminal_counts.canceled_count, - event_counts.claimed_started_count, - event_counts.retry_event_count, - enqueue_counts.enqueued_count, + queued_counts.queued_count, + claimed_counts.claimed_count, + queued_counts.oldest_queued_age, saturated_groups.saturated_group_count, node_counts.active_node_count, node_counts.stale_node_count, node_counts.dead_node_count - from current_counts, terminal_counts, event_counts, enqueue_counts, saturated_groups, node_counts; + from queued_counts, claimed_counts, saturated_groups, node_counts; """; - command.Parameters.AddWithValue("window", window); command.Parameters.AddWithValue("stale_threshold", staleThreshold); command.Parameters.AddWithValue("dead_threshold", deadThreshold); await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) { - throw new InvalidOperationException("PostgreSQL did not return inspection metrics."); + throw new InvalidOperationException("PostgreSQL did not return current inspection metrics."); } - return new PostgresMetricsCounts( + return new PostgresCurrentMetricsCounts( Convert.ToInt32(reader.GetInt64(0), CultureInfo.InvariantCulture), Convert.ToInt32(reader.GetInt64(1), CultureInfo.InvariantCulture), reader.IsDBNull(2) ? null : reader.GetTimeSpan(2), Convert.ToInt32(reader.GetInt64(3), CultureInfo.InvariantCulture), Convert.ToInt32(reader.GetInt64(4), CultureInfo.InvariantCulture), Convert.ToInt32(reader.GetInt64(5), CultureInfo.InvariantCulture), - Convert.ToInt32(reader.GetInt64(6), CultureInfo.InvariantCulture), - Convert.ToInt32(reader.GetInt64(7), CultureInfo.InvariantCulture), - Convert.ToInt32(reader.GetInt64(8), CultureInfo.InvariantCulture), - Convert.ToInt32(reader.GetInt64(9), CultureInfo.InvariantCulture), - Convert.ToInt32(reader.GetInt64(10), CultureInfo.InvariantCulture), - Convert.ToInt32(reader.GetInt64(11), CultureInfo.InvariantCulture), - Convert.ToInt32(reader.GetInt64(12), CultureInfo.InvariantCulture)); + Convert.ToInt32(reader.GetInt64(6), CultureInfo.InvariantCulture)); } - private static async ValueTask<(TimeSpan? P50, TimeSpan? P95)> ReadPercentilesAsync( + private static async ValueTask ReadWindowCountsAsync( + PostgresOperationContext context, NpgsqlConnection connection, - string sourceSql, TimeSpan window, CancellationToken cancellationToken) { await using var command = connection.CreateCommand(); command.CommandText = $""" - with values_ms as ( - {sourceSql} - ) select - percentile_cont(0.5) within group (order by value_ms) as p50, - percentile_cont(0.95) within group (order by value_ms) as p95 - from values_ms; + coalesce(sum(enqueued_count), 0), + coalesce(sum(claimed_started_count), 0), + coalesce(sum(succeeded_count), 0), + coalesce(sum(failed_count), 0), + coalesce(sum(canceled_count), 0), + coalesce(sum(retry_event_count), 0) + from {context.Names.MetricsBuckets} + where bucket_started_at_utc >= {PostgresMetricsRollups.BucketStartedAtSql("transaction_timestamp() - @window")}; """; command.Parameters.AddWithValue("window", window); await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) { - return (null, null); + throw new InvalidOperationException("PostgreSQL did not return rolled-up inspection metrics."); } - return ( - reader.IsDBNull(0) ? null : TimeSpan.FromMilliseconds(reader.GetDouble(0)), - reader.IsDBNull(1) ? null : TimeSpan.FromMilliseconds(reader.GetDouble(1))); + return new PostgresWindowRollupCounts( + Convert.ToInt64(reader.GetValue(0), CultureInfo.InvariantCulture), + Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture), + Convert.ToInt64(reader.GetValue(2), CultureInfo.InvariantCulture), + Convert.ToInt64(reader.GetValue(3), CultureInfo.InvariantCulture), + Convert.ToInt64(reader.GetValue(4), CultureInfo.InvariantCulture), + Convert.ToInt64(reader.GetValue(5), CultureInfo.InvariantCulture)); } - private sealed record PostgresMetricsCounts( + private static async ValueTask ReadWindowPercentilesAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + TimeSpan window, + CancellationToken cancellationToken) + { + await using var command = connection.CreateCommand(); + command.CommandText = + $""" + with histogram as ( + select + metric, + bin_index, + sum(sample_count) as sample_count + from {context.Names.MetricsHistogramBins} + where bucket_started_at_utc >= {PostgresMetricsRollups.BucketStartedAtSql("transaction_timestamp() - @window")} + group by metric, bin_index + ), + ordered as ( + select + metric, + bin_index, + sum(sample_count) over (partition by metric order by bin_index asc) as cumulative_count, + sum(sample_count) over (partition by metric) as total_count + from histogram + ) + select + metric, + min(bin_index) filter (where cumulative_count >= ceiling(total_count * 0.50)) as p50_bin, + min(bin_index) filter (where cumulative_count >= ceiling(total_count * 0.95)) as p95_bin + from ordered + group by metric; + """; + command.Parameters.AddWithValue("window", window); + + TimeSpan? queueLatencyP50 = null; + TimeSpan? queueLatencyP95 = null; + TimeSpan? executionDurationP50 = null; + TimeSpan? executionDurationP95 = null; + TimeSpan? scheduleFireLagP95 = null; + + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + var metric = reader.GetString(0); + var p50 = ReadDurationBin(reader, 1); + var p95 = ReadDurationBin(reader, 2); + + switch (metric) + { + case QueueLatencyMetric: + queueLatencyP50 = p50; + queueLatencyP95 = p95; + break; + case ExecutionDurationMetric: + executionDurationP50 = p50; + executionDurationP95 = p95; + break; + case ScheduleFireLagMetric: + scheduleFireLagP95 = p95; + break; + } + } + + return new PostgresWindowPercentiles( + queueLatencyP50, + queueLatencyP95, + executionDurationP50, + executionDurationP95, + scheduleFireLagP95); + } + + private static TimeSpan? ReadDurationBin( + NpgsqlDataReader reader, + int ordinal) + { + if (reader.IsDBNull(ordinal)) + { + return null; + } + + var binIndex = Math.Clamp(reader.GetInt32(ordinal), 0, PostgresMetricsRollups.DurationHistogramThresholdsMs.Length - 1); + return TimeSpan.FromMilliseconds(PostgresMetricsRollups.DurationHistogramThresholdsMs[binIndex]); + } + + private sealed record PostgresCurrentMetricsCounts( int QueuedCount, int ClaimedCount, TimeSpan? OldestQueuedAge, - int SucceededCount, - int FailedCount, - int CanceledCount, - int ClaimedStartedCount, - int RetryEventCount, - int EnqueuedCount, int SaturatedGroupCount, int ActiveNodeCount, int StaleNodeCount, int DeadNodeCount); + + private sealed record PostgresWindowRollupCounts( + long EnqueuedCount, + long ClaimedStartedCount, + long SucceededCount, + long FailedCount, + long CanceledCount, + long RetryEventCount); + + private sealed record PostgresWindowPercentiles( + TimeSpan? QueueLatencyP50, + TimeSpan? QueueLatencyP95, + TimeSpan? ExecutionDurationP50, + TimeSpan? ExecutionDurationP95, + TimeSpan? ScheduleFireLagP95); } diff --git a/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsRollups.cs b/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsRollups.cs new file mode 100644 index 0000000..b072452 --- /dev/null +++ b/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsRollups.cs @@ -0,0 +1,445 @@ +namespace Sheddueller.Postgres.Internal.Operations; + +using Npgsql; + +using NpgsqlTypes; + +using Sheddueller.Storage; + +internal static class PostgresMetricsRollups +{ + public const int BucketSizeSeconds = 5; + public static readonly TimeSpan Retention = TimeSpan.FromDays(7); + + private const int CleanupAdvisoryLockKey = 7870835; + private static readonly TimeSpan CleanupInterval = TimeSpan.FromHours(1); + + internal static readonly long[] DurationHistogramThresholdsMs = + [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1_000, + 2_000, + 5_000, + 10_000, + 30_000, + 60_000, + 120_000, + 300_000, + 600_000, + 1_800_000, + 3_600_000, + 7_200_000, + 21_600_000, + 86_400_000, + ]; + + public static async ValueTask RecordJobEventAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + JobEvent jobEvent, + CancellationToken cancellationToken) + { + await using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = + $""" + with job as ( + select + job_id, + enqueued_at_utc, + claimed_at_utc, + completed_at_utc, + failed_at_utc, + canceled_at_utc, + scheduled_fire_at_utc, + schedule_occurrence_kind + from {context.Names.Jobs} + where job_id = @job_id + ), + counter_samples as ( + select {BucketStartedAtSql("job.enqueued_at_utc")} as bucket_started_at_utc, 1::bigint as enqueued_count, 0::bigint as claimed_started_count, 0::bigint as succeeded_count, 0::bigint as failed_count, 0::bigint as canceled_count, 0::bigint as retry_event_count + from job + where @event_kind = 'Lifecycle' + and @event_message = 'Queued' + union all + select {BucketStartedAtSql("coalesce(job.claimed_at_utc, @event_occurred_at_utc)")}, 0::bigint, 1::bigint, 0::bigint, 0::bigint, 0::bigint, 0::bigint + from job + where @event_kind = 'AttemptStarted' + union all + select {BucketStartedAtSql("@event_occurred_at_utc")}, 0::bigint, 0::bigint, 0::bigint, 0::bigint, 0::bigint, 1::bigint + from job + where @event_kind = 'AttemptFailed' + union all + select {BucketStartedAtSql("job.completed_at_utc")}, 0::bigint, 0::bigint, 1::bigint, 0::bigint, 0::bigint, 0::bigint + from job + where @event_kind = 'Lifecycle' + and @event_message = 'Completed' + and job.completed_at_utc is not null + union all + select {BucketStartedAtSql("job.failed_at_utc")}, 0::bigint, 0::bigint, 0::bigint, 1::bigint, 0::bigint, 0::bigint + from job + where @event_kind = 'Lifecycle' + and (@event_message = 'Failed' or @event_message like 'Failed;%') + and job.failed_at_utc is not null + union all + select {BucketStartedAtSql("job.canceled_at_utc")}, 0::bigint, 0::bigint, 0::bigint, 0::bigint, 1::bigint, 0::bigint + from job + where @event_kind = 'Lifecycle' + and @event_message = 'Canceled' + and job.canceled_at_utc is not null + ), + bucket_counts as ( + select + bucket_started_at_utc, + sum(enqueued_count) as enqueued_count, + sum(claimed_started_count) as claimed_started_count, + sum(succeeded_count) as succeeded_count, + sum(failed_count) as failed_count, + sum(canceled_count) as canceled_count, + sum(retry_event_count) as retry_event_count + from counter_samples + group by bucket_started_at_utc + ) + insert into {context.Names.MetricsBuckets} as bucket ( + bucket_started_at_utc, + enqueued_count, + claimed_started_count, + succeeded_count, + failed_count, + canceled_count, + retry_event_count) + select + bucket_started_at_utc, + enqueued_count, + claimed_started_count, + succeeded_count, + failed_count, + canceled_count, + retry_event_count + from bucket_counts + on conflict (bucket_started_at_utc) do update + set enqueued_count = bucket.enqueued_count + excluded.enqueued_count, + claimed_started_count = bucket.claimed_started_count + excluded.claimed_started_count, + succeeded_count = bucket.succeeded_count + excluded.succeeded_count, + failed_count = bucket.failed_count + excluded.failed_count, + canceled_count = bucket.canceled_count + excluded.canceled_count, + retry_event_count = bucket.retry_event_count + excluded.retry_event_count; + + with job as ( + select + job_id, + enqueued_at_utc, + claimed_at_utc, + completed_at_utc, + failed_at_utc, + canceled_at_utc, + scheduled_fire_at_utc, + schedule_occurrence_kind + from {context.Names.Jobs} + where job_id = @job_id + ), + histogram_samples as ( + select + 'queue_latency'::text as metric, + {BucketStartedAtSql("coalesce(job.claimed_at_utc, @event_occurred_at_utc)")} as bucket_started_at_utc, + extract(epoch from (coalesce(job.claimed_at_utc, @event_occurred_at_utc) - job.enqueued_at_utc)) * 1000 as value_ms + from job + where @event_kind = 'AttemptStarted' + and coalesce(job.claimed_at_utc, @event_occurred_at_utc) >= job.enqueued_at_utc + union all + select + 'schedule_fire_lag'::text, + {BucketStartedAtSql("job.enqueued_at_utc")}, + extract(epoch from (job.enqueued_at_utc - job.scheduled_fire_at_utc)) * 1000 + from job + where @event_kind = 'Lifecycle' + and @event_message = 'Queued' + and job.schedule_occurrence_kind = 'Automatic' + and job.scheduled_fire_at_utc is not null + and job.enqueued_at_utc >= job.scheduled_fire_at_utc + union all + select + 'execution_duration'::text, + {BucketStartedAtSql("job.completed_at_utc")}, + extract(epoch from (job.completed_at_utc - job.claimed_at_utc)) * 1000 + from job + where @event_kind = 'Lifecycle' + and @event_message = 'Completed' + and job.claimed_at_utc is not null + and job.completed_at_utc is not null + and job.completed_at_utc >= job.claimed_at_utc + union all + select + 'execution_duration'::text, + {BucketStartedAtSql("job.failed_at_utc")}, + extract(epoch from (job.failed_at_utc - job.claimed_at_utc)) * 1000 + from job + where @event_kind = 'Lifecycle' + and (@event_message = 'Failed' or @event_message like 'Failed;%') + and job.claimed_at_utc is not null + and job.failed_at_utc is not null + and job.failed_at_utc >= job.claimed_at_utc + union all + select + 'execution_duration'::text, + {BucketStartedAtSql("job.canceled_at_utc")}, + extract(epoch from (job.canceled_at_utc - job.claimed_at_utc)) * 1000 + from job + where @event_kind = 'Lifecycle' + and @event_message = 'Canceled' + and job.claimed_at_utc is not null + and job.canceled_at_utc is not null + and job.canceled_at_utc >= job.claimed_at_utc + ), + binned_samples as ( + select + sample.metric, + sample.bucket_started_at_utc, + coalesce( + ( + select threshold.ordinality::integer - 1 + from unnest(@duration_thresholds_ms::bigint[]) with ordinality as threshold(threshold_ms, ordinality) + where sample.value_ms <= threshold.threshold_ms + order by threshold.ordinality asc + limit 1 + ), + @last_bin_index) as bin_index, + count(*) as sample_count + from histogram_samples sample + where sample.value_ms >= 0 + group by sample.metric, sample.bucket_started_at_utc, bin_index + ) + insert into {context.Names.MetricsHistogramBins} as histogram ( + bucket_started_at_utc, + metric, + bin_index, + sample_count) + select + bucket_started_at_utc, + metric, + bin_index, + sample_count + from binned_samples + on conflict (bucket_started_at_utc, metric, bin_index) do update + set sample_count = histogram.sample_count + excluded.sample_count; + """; + command.Parameters.AddWithValue("job_id", jobEvent.JobId); + command.Parameters.Add("event_kind", NpgsqlDbType.Text).Value = PostgresConversion.ToText(jobEvent.Kind); + command.Parameters.Add("event_message", NpgsqlDbType.Text).Value = + PostgresOperationContext.ToDbValue(jobEvent.Message); + command.Parameters.Add("event_occurred_at_utc", NpgsqlDbType.TimestampTz).Value = jobEvent.OccurredAtUtc; + AddHistogramParameters(command); + + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + public static async ValueTask RecordStagedQueuedJobsAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + CancellationToken cancellationToken) + { + await using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = + $""" + with counter_samples as ( + select + {BucketStartedAtSql("job.enqueued_at_utc")} as bucket_started_at_utc, + count(*)::bigint as enqueued_count + from {context.Names.Jobs} job + join sheddueller_enqueue_results result on result.job_id = job.job_id + where result.was_enqueued = true + group by bucket_started_at_utc + ) + insert into {context.Names.MetricsBuckets} as bucket ( + bucket_started_at_utc, + enqueued_count) + select + bucket_started_at_utc, + enqueued_count + from counter_samples + on conflict (bucket_started_at_utc) do update + set enqueued_count = bucket.enqueued_count + excluded.enqueued_count; + + with histogram_samples as ( + select + {BucketStartedAtSql("job.enqueued_at_utc")} as bucket_started_at_utc, + extract(epoch from (job.enqueued_at_utc - job.scheduled_fire_at_utc)) * 1000 as value_ms + from {context.Names.Jobs} job + join sheddueller_enqueue_results result on result.job_id = job.job_id + where result.was_enqueued = true + and job.schedule_occurrence_kind = 'Automatic' + and job.scheduled_fire_at_utc is not null + and job.enqueued_at_utc >= job.scheduled_fire_at_utc + ), + binned_samples as ( + select + bucket_started_at_utc, + coalesce( + ( + select threshold.ordinality::integer - 1 + from unnest(@duration_thresholds_ms::bigint[]) with ordinality as threshold(threshold_ms, ordinality) + where sample.value_ms <= threshold.threshold_ms + order by threshold.ordinality asc + limit 1 + ), + @last_bin_index) as bin_index, + count(*) as sample_count + from histogram_samples sample + where sample.value_ms >= 0 + group by bucket_started_at_utc, bin_index + ) + insert into {context.Names.MetricsHistogramBins} as histogram ( + bucket_started_at_utc, + metric, + bin_index, + sample_count) + select + bucket_started_at_utc, + 'schedule_fire_lag', + bin_index, + sample_count + from binned_samples + on conflict (bucket_started_at_utc, metric, bin_index) do update + set sample_count = histogram.sample_count + excluded.sample_count; + """; + AddHistogramParameters(command); + + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + public static async ValueTask RecordCanceledJobsAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + IReadOnlyList jobIds, + CancellationToken cancellationToken) + { + if (jobIds.Count == 0) + { + return; + } + + await using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = + $""" + with canceled_jobs as ( + select + {BucketStartedAtSql("job.canceled_at_utc")} as bucket_started_at_utc, + count(*)::bigint as canceled_count + from {context.Names.Jobs} job + join unnest(@job_ids::uuid[]) canceled_job(job_id) on canceled_job.job_id = job.job_id + where job.canceled_at_utc is not null + group by bucket_started_at_utc + ) + insert into {context.Names.MetricsBuckets} as bucket ( + bucket_started_at_utc, + canceled_count) + select + bucket_started_at_utc, + canceled_count + from canceled_jobs + on conflict (bucket_started_at_utc) do update + set canceled_count = bucket.canceled_count + excluded.canceled_count; + """; + command.Parameters.Add("job_ids", NpgsqlDbType.Array | NpgsqlDbType.Uuid).Value = jobIds.ToArray(); + + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + public static async ValueTask CleanupAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + CancellationToken cancellationToken) + { + await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + var acquired = await TryAcquireCleanupLockAsync(context, connection, transaction, cancellationToken).ConfigureAwait(false); + if (!acquired) + { + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); + return; + } + + if (!await ShouldCleanupAsync(context, connection, transaction, cancellationToken).ConfigureAwait(false)) + { + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return; + } + + await PostgresOperationContext.ExecuteCountAsync( + connection, + transaction, + $""" + delete from {context.Names.MetricsBuckets} + where bucket_started_at_utc < transaction_timestamp() - @retention; + + update {context.Names.MetricsRollupState} + set last_cleanup_at_utc = transaction_timestamp() + where singleton_id = 1; + """, + command => command.Parameters.AddWithValue("retention", Retention), + cancellationToken) + .ConfigureAwait(false); + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } + + private static async ValueTask TryAcquireCleanupLockAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + CancellationToken cancellationToken) + { + await using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = "select pg_try_advisory_xact_lock(@lock_key, hashtext(@schema_name));"; + command.Parameters.AddWithValue("lock_key", CleanupAdvisoryLockKey); + command.Parameters.AddWithValue("schema_name", context.Options.SchemaName); + + return (bool)(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException("PostgreSQL did not return a metrics cleanup lock result.")); + } + + private static async ValueTask ShouldCleanupAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + CancellationToken cancellationToken) + { + await using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = + $""" + select last_cleanup_at_utc is null + or last_cleanup_at_utc <= transaction_timestamp() - @cleanup_interval + from {context.Names.MetricsRollupState} + where singleton_id = 1 + for update; + """; + command.Parameters.AddWithValue("cleanup_interval", CleanupInterval); + + return (bool)(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException("PostgreSQL did not return metrics cleanup state.")); + } + + private static void AddHistogramParameters(NpgsqlCommand command) + { + command.Parameters.Add("duration_thresholds_ms", NpgsqlDbType.Array | NpgsqlDbType.Bigint).Value = + DurationHistogramThresholdsMs; + command.Parameters.AddWithValue("last_bin_index", DurationHistogramThresholdsMs.Length - 1); + } + + internal static string BucketStartedAtSql(string timestampSql) + => $"to_timestamp(floor(extract(epoch from {timestampSql}) / {BucketSizeSeconds}) * {BucketSizeSeconds})"; +} diff --git a/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs b/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs index 8337efd..d0480b5 100644 --- a/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs +++ b/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs @@ -288,6 +288,43 @@ constraint worker_nodes_max_concurrency_check check (max_concurrent_executions_p constraint worker_nodes_current_execution_count_check check (current_execution_count >= 0) ); + create table if not exists {this._names.MetricsBuckets} ( + bucket_started_at_utc timestamptz primary key, + enqueued_count bigint not null default 0, + claimed_started_count bigint not null default 0, + succeeded_count bigint not null default 0, + failed_count bigint not null default 0, + canceled_count bigint not null default 0, + retry_event_count bigint not null default 0, + constraint metrics_buckets_enqueued_count_check check (enqueued_count >= 0), + constraint metrics_buckets_claimed_started_count_check check (claimed_started_count >= 0), + constraint metrics_buckets_succeeded_count_check check (succeeded_count >= 0), + constraint metrics_buckets_failed_count_check check (failed_count >= 0), + constraint metrics_buckets_canceled_count_check check (canceled_count >= 0), + constraint metrics_buckets_retry_event_count_check check (retry_event_count >= 0) + ); + + create table if not exists {this._names.MetricsHistogramBins} ( + bucket_started_at_utc timestamptz not null references {this._names.MetricsBuckets}(bucket_started_at_utc) on delete cascade, + metric text not null, + bin_index integer not null, + sample_count bigint not null, + primary key (bucket_started_at_utc, metric, bin_index), + constraint metrics_histogram_bins_metric_check check (metric in ('queue_latency', 'execution_duration', 'schedule_fire_lag')), + constraint metrics_histogram_bins_bin_index_check check (bin_index >= 0), + constraint metrics_histogram_bins_sample_count_check check (sample_count > 0) + ); + + create table if not exists {this._names.MetricsRollupState} ( + singleton_id smallint primary key, + last_cleanup_at_utc timestamptz null, + constraint metrics_rollup_state_singleton_id_check check (singleton_id = 1) + ); + + insert into {this._names.MetricsRollupState} (singleton_id, last_cleanup_at_utc) + values (1, null) + on conflict (singleton_id) do nothing; + create index if not exists idx_jobs_claim_scan on {this._names.Jobs} (priority desc, enqueue_sequence asc) where state = 'Queued'; @@ -369,6 +406,9 @@ constraint worker_nodes_current_execution_count_check check (current_execution_c create index if not exists idx_worker_nodes_last_heartbeat on {this._names.WorkerNodes} (last_heartbeat_at_utc); + + create index if not exists idx_metrics_histogram_bins_metric_bucket + on {this._names.MetricsHistogramBins} (metric, bucket_started_at_utc, bin_index); """, cancellationToken) .ConfigureAwait(false); diff --git a/src/Sheddueller.Postgres/Internal/PostgresNames.cs b/src/Sheddueller.Postgres/Internal/PostgresNames.cs index 0eefe6d..eb235fd 100644 --- a/src/Sheddueller.Postgres/Internal/PostgresNames.cs +++ b/src/Sheddueller.Postgres/Internal/PostgresNames.cs @@ -4,7 +4,7 @@ namespace Sheddueller.Postgres.Internal; internal sealed class PostgresNames { - public const int ExpectedSchemaVersion = 10; + public const int ExpectedSchemaVersion = 11; public const string WakeupChannel = "sheddueller_wakeup"; public const string JobEventChannel = "sheddueller_job_event"; @@ -22,6 +22,9 @@ public PostgresNames(string schemaName) this.ScheduleTags = this.Table("schedule_tags"); this.JobEvents = this.Table("job_events"); this.WorkerNodes = this.Table("worker_nodes"); + this.MetricsBuckets = this.Table("metrics_buckets"); + this.MetricsHistogramBins = this.Table("metrics_histogram_bins"); + this.MetricsRollupState = this.Table("metrics_rollup_state"); } public string SchemaName { get; } @@ -48,6 +51,12 @@ public PostgresNames(string schemaName) public string WorkerNodes { get; } + public string MetricsBuckets { get; } + + public string MetricsHistogramBins { get; } + + public string MetricsRollupState { get; } + public static string QuoteIdentifier(string identifier) => $"\"{identifier.Replace("\"", "\"\"", StringComparison.Ordinal)}\""; diff --git a/test/Sheddueller.Dashboard.Tests/DashboardMetricsSnapshotCacheTests.cs b/test/Sheddueller.Dashboard.Tests/DashboardMetricsSnapshotCacheTests.cs new file mode 100644 index 0000000..1e5d572 --- /dev/null +++ b/test/Sheddueller.Dashboard.Tests/DashboardMetricsSnapshotCacheTests.cs @@ -0,0 +1,122 @@ +namespace Sheddueller.Dashboard.Tests; + +using Microsoft.Extensions.Time.Testing; + +using Sheddueller.Dashboard.Internal; +using Sheddueller.Inspection.Metrics; + +using Shouldly; + +public sealed class DashboardMetricsSnapshotCacheTests +{ + [Fact] + public async Task GetMetricsAsync_RepeatedQueryWithinTtl_UsesCachedSnapshot() + { + var reader = new CountingMetricsReader(); + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 4, 26, 12, 0, 0, TimeSpan.Zero)); + var cache = new DashboardMetricsSnapshotCache(reader, timeProvider); + var query = new MetricsInspectionQuery([TimeSpan.FromMinutes(5)]); + + var first = await cache.GetMetricsAsync(query); + var second = await cache.GetMetricsAsync(query); + + reader.CallCount.ShouldBe(1); + first.Windows[0].QueuedCount.ShouldBe(1); + second.Windows[0].QueuedCount.ShouldBe(1); + } + + [Fact] + public async Task GetMetricsAsync_ExpiredEntry_ReadsAgain() + { + var reader = new CountingMetricsReader(); + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 4, 26, 12, 0, 0, TimeSpan.Zero)); + var cache = new DashboardMetricsSnapshotCache(reader, timeProvider); + var query = new MetricsInspectionQuery([TimeSpan.FromMinutes(5)]); + + await cache.GetMetricsAsync(query); + timeProvider.SetUtcNow(timeProvider.GetUtcNow().Add(DashboardMetricsSnapshotCache.TimeToLive)); + var second = await cache.GetMetricsAsync(query); + + reader.CallCount.ShouldBe(2); + second.Windows[0].QueuedCount.ShouldBe(2); + } + + [Fact] + public async Task GetMetricsAsync_ConcurrentSameQuery_CoalescesUnderlyingRead() + { + var reader = new BlockingMetricsReader(); + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 4, 26, 12, 0, 0, TimeSpan.Zero)); + var cache = new DashboardMetricsSnapshotCache(reader, timeProvider); + var query = new MetricsInspectionQuery([TimeSpan.FromMinutes(5)]); + + var first = cache.GetMetricsAsync(query).AsTask(); + var second = cache.GetMetricsAsync(query).AsTask(); + reader.CallCount.ShouldBe(1); + + reader.Complete(); + var snapshots = await Task.WhenAll(first, second); + + reader.CallCount.ShouldBe(1); + snapshots[0].Windows[0].QueuedCount.ShouldBe(42); + snapshots[1].Windows[0].QueuedCount.ShouldBe(42); + } + + private static MetricsInspectionSnapshot CreateSnapshot(int queuedCount) + => new( + [ + new( + TimeSpan.FromMinutes(5), + queuedCount, + ClaimedCount: 0, + FailedCount: 0, + CanceledCount: 0, + OldestQueuedAge: null, + EnqueueRatePerMinute: 0, + ClaimRatePerMinute: 0, + SuccessRatePerMinute: 0, + FailureRatePerMinute: 0, + CancellationRatePerMinute: 0, + RetryRatePerMinute: 0, + P50QueueLatency: null, + P95QueueLatency: null, + P50ExecutionDuration: null, + P95ExecutionDuration: null, + P95ScheduleFireLag: null, + SaturatedConcurrencyGroupCount: 0, + ActiveNodeCount: 0, + StaleNodeCount: 0, + DeadNodeCount: 0), + ]); + + private sealed class CountingMetricsReader : IMetricsInspectionReader + { + public int CallCount { get; private set; } + + public ValueTask GetMetricsAsync( + MetricsInspectionQuery query, + CancellationToken cancellationToken = default) + { + this.CallCount++; + return ValueTask.FromResult(CreateSnapshot(this.CallCount)); + } + } + + private sealed class BlockingMetricsReader : IMetricsInspectionReader + { + private readonly TaskCompletionSource _completion = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int CallCount { get; private set; } + + public ValueTask GetMetricsAsync( + MetricsInspectionQuery query, + CancellationToken cancellationToken = default) + { + this.CallCount++; + return new ValueTask(this._completion.Task); + } + + public void Complete() + => this._completion.SetResult(CreateSnapshot(42)); + } +} diff --git a/test/Sheddueller.Postgres.Tests/PostgresMetricsRollupTests.cs b/test/Sheddueller.Postgres.Tests/PostgresMetricsRollupTests.cs new file mode 100644 index 0000000..2e3d4df --- /dev/null +++ b/test/Sheddueller.Postgres.Tests/PostgresMetricsRollupTests.cs @@ -0,0 +1,58 @@ +namespace Sheddueller.Postgres.Tests; + +using Microsoft.Extensions.DependencyInjection; + +using Sheddueller.Inspection.Metrics; + +using Shouldly; + +public sealed class PostgresMetricsRollupTests(PostgresFixture fixture) : IClassFixture +{ + [Fact] + public async Task MetricsRead_OldRollupBuckets_RemovesExpiredBucketsAndHistogramBins() + { + await using var context = await PostgresTestContext.CreateMigratedAsync(fixture); + await InsertOldRollupAsync(context); + + await context.Provider.GetRequiredService() + .GetMetricsAsync(new MetricsInspectionQuery([TimeSpan.FromMinutes(5)])); + + (await CountAsync(context, "metrics_buckets")).ShouldBe(0L); + (await CountAsync(context, "metrics_histogram_bins")).ShouldBe(0L); + } + + private static async ValueTask InsertOldRollupAsync(PostgresTestContext context) + { + await using var command = context.DataSource.CreateCommand( + $""" + insert into {context.Table("metrics_buckets")} ( + bucket_started_at_utc, + enqueued_count) + values ( + transaction_timestamp() - interval '8 days', + 1); + + insert into {context.Table("metrics_histogram_bins")} ( + bucket_started_at_utc, + metric, + bin_index, + sample_count) + values ( + (select bucket_started_at_utc from {context.Table("metrics_buckets")} limit 1), + 'queue_latency', + 0, + 1); + """); + await command.ExecuteNonQueryAsync(); + } + + private static async ValueTask CountAsync( + PostgresTestContext context, + string tableName) + { + await using var command = context.DataSource.CreateCommand($"select count(*) from {context.Table(tableName)};"); + var result = await command.ExecuteScalarAsync(); + result.ShouldNotBeNull(); + return result.ShouldBeOfType(); + } +} diff --git a/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs b/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs index 556c5e9..3de348d 100644 --- a/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs +++ b/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs @@ -138,6 +138,42 @@ from pg_indexes normalized.ShouldContain("claimed_by_node_id is not null"); } + [Fact] + public async Task Migration_FreshSchema_CreatesMetricsRollupTables() + { + await using var context = await PostgresTestContext.CreateMigratedAsync(fixture); + + await AssertTableExistsAsync(context, "metrics_buckets"); + await AssertTableExistsAsync(context, "metrics_histogram_bins"); + await AssertTableExistsAsync(context, "metrics_rollup_state"); + + (await ScalarAsync( + context, + $"select count(*) from {context.Table("metrics_buckets")};")) + .ShouldBe(0L); + (await ScalarAsync( + context, + $"select count(*) from {context.Table("metrics_histogram_bins")};")) + .ShouldBe(0L); + (await ScalarAsync( + context, + $"select count(*) from {context.Table("metrics_rollup_state")};")) + .ShouldBe(1L); + + var indexDefinition = await ScalarAsync( + context, + """ + select indexdef + from pg_indexes + where schemaname = @schema_name + and indexname = 'idx_metrics_histogram_bins_metric_bucket'; + """); + + indexDefinition.ShouldContain("metric"); + indexDefinition.ShouldContain("bucket_started_at_utc"); + indexDefinition.ShouldContain("bin_index"); + } + [Fact] public async Task Migration_FreshSchema_CreatesTagOrdinalColumnsAndIndexes() { @@ -203,6 +239,22 @@ from information_schema.columns command => command.Parameters.AddWithValue("table_name", tableName))) .ShouldBeTrue(); + private static async Task AssertTableExistsAsync( + PostgresTestContext context, + string tableName) + => (await ScalarAsync( + context, + """ + select exists ( + select 1 + from information_schema.tables + where table_schema = @schema_name + and table_name = @table_name + ); + """, + command => command.Parameters.AddWithValue("table_name", tableName))) + .ShouldBeTrue(); + private static async ValueTask ScalarAsync( PostgresTestContext context, string commandText, diff --git a/test/Sheddueller.ProviderContracts/InspectionContractTests.cs b/test/Sheddueller.ProviderContracts/InspectionContractTests.cs index dd87220..c2908b7 100644 --- a/test/Sheddueller.ProviderContracts/InspectionContractTests.cs +++ b/test/Sheddueller.ProviderContracts/InspectionContractTests.cs @@ -668,6 +668,71 @@ await context.Store.RecordWorkerNodeHeartbeatAsync(new WorkerNodeHeartbeatReques metrics.Windows[0].ActiveNodeCount.ShouldBeGreaterThanOrEqualTo(1); } + [Fact] + public async Task Metrics_RollingActivity_ReturnsRolledUpRatesAndPercentiles() + { + await using var context = await this.CreateContextAsync(); + var completed = Guid.NewGuid(); + var failed = Guid.NewGuid(); + var canceled = Guid.NewGuid(); + var scheduled = Guid.NewGuid(); + + await context.Store.EnqueueAsync(CreateRequest(completed)); + var completedClaim = await ClaimAsync(context.Store); + await context.Store.MarkCompletedAsync(new CompleteJobRequest( + completed, + "node-1", + completedClaim.LeaseToken, + DateTimeOffset.UtcNow)); + + await context.Store.EnqueueAsync(CreateRequest(failed)); + var failedClaim = await ClaimAsync(context.Store); + await context.Store.MarkFailedAsync(new FailJobRequest( + failed, + "node-1", + failedClaim.LeaseToken, + DateTimeOffset.UtcNow, + CreateFailure())); + + await context.Store.EnqueueAsync(CreateRequest(canceled)); + (await context.Store.CancelAsync(new CancelJobRequest(canceled, DateTimeOffset.UtcNow))) + .ShouldBe(JobCancellationResult.Canceled); + + await context.Store.EnqueueAsync(CreateRequest( + scheduled, + sourceScheduleKey: "nightly", + scheduledFireAtUtc: DateTimeOffset.UtcNow.AddSeconds(-2), + scheduleOccurrenceKind: ScheduleOccurrenceKind.Automatic)); + + var metrics = await context.MetricsReader.GetMetricsAsync(new MetricsInspectionQuery([TimeSpan.FromMinutes(5)])); + var window = metrics.Windows.ShouldHaveSingleItem(); + + window.EnqueueRatePerMinute.ShouldBeGreaterThan(0); + window.ClaimRatePerMinute.ShouldBeGreaterThan(0); + window.SuccessRatePerMinute.ShouldBeGreaterThan(0); + window.FailureRatePerMinute.ShouldBeGreaterThan(0); + window.CancellationRatePerMinute.ShouldBeGreaterThan(0); + window.RetryRatePerMinute.ShouldBeGreaterThan(0); + window.P50QueueLatency.ShouldNotBeNull(); + window.P95ExecutionDuration.ShouldNotBeNull(); + window.P95ScheduleFireLag.ShouldNotBeNull(); + } + + [Fact] + public async Task Metrics_BulkQueuedCancellation_ReturnsRolledUpCancellationRate() + { + await using var context = await this.CreateContextAsync(); + + await context.Store.EnqueueAsync(CreateRequest(Guid.NewGuid())); + await context.Store.EnqueueAsync(CreateRequest(Guid.NewGuid())); + + (await context.Store.CancelQueuedJobsAsync(new CancelQueuedJobsRequest(DateTimeOffset.UtcNow))).ShouldBe(2); + + var metrics = await context.MetricsReader.GetMetricsAsync(new MetricsInspectionQuery([TimeSpan.FromMinutes(5)])); + + metrics.Windows.ShouldHaveSingleItem().CancellationRatePerMinute.ShouldBeGreaterThan(0); + } + [Fact] public async Task NodeSearch_MultipleClaimedJobsAcrossNodes_ReturnsPerNodeClaimedCounts() { @@ -828,7 +893,10 @@ protected static EnqueueJobRequest CreateRequest( IReadOnlyList? methodParameterTypes = null, SerializedJobPayload? serializedArguments = null, JobInvocationTargetKind invocationTargetKind = JobInvocationTargetKind.Instance, - IReadOnlyList? methodParameterBindings = null) + IReadOnlyList? methodParameterBindings = null, + string? sourceScheduleKey = null, + DateTimeOffset? scheduledFireAtUtc = null, + ScheduleOccurrenceKind? scheduleOccurrenceKind = null) => new( jobId, priority, @@ -842,7 +910,10 @@ protected static EnqueueJobRequest CreateRequest( maxAttempts, retryBackoffKind, retryBaseDelay, + SourceScheduleKey: sourceScheduleKey, + ScheduledFireAtUtc: scheduledFireAtUtc, Tags: tags, + ScheduleOccurrenceKind: scheduleOccurrenceKind, InvocationTargetKind: invocationTargetKind, MethodParameterBindings: methodParameterBindings); From 4fd37745356703ed1902c3f77808ed9920c6dd4f Mon Sep 17 00:00:00 2001 From: Brian Tyler Date: Fri, 3 Jul 2026 13:52:56 +0100 Subject: [PATCH 5/6] feat: Add settings management for Sheddueller cleanup configurations - Implemented a new Settings.razor component for managing cleanup settings in the Sheddueller dashboard. - Introduced ShedduellerCleanupConfiguration, JobRetentionCleanupConfiguration, JobEventCleanupConfiguration, and MetricsCleanupConfiguration records to encapsulate cleanup settings. - Created IShedduellerCleanupConfigurationStore interface for storing and retrieving cleanup configurations. - Developed PostgresCleanupConfigurationOperation for handling cleanup configuration operations in PostgreSQL. - Added unit tests for cleanup configuration operations and job retention service to ensure correct functionality. - Updated PostgresMigrationTests to verify the creation of the settings table in the database. - Enhanced PostgresRegistrationTests to include checks for the new cleanup configuration store. --- .../Components/DashboardLayout.razor | 4 + .../Components/Pages/Settings.razor | 587 ++++++++++++++++++ .../Components/_Imports.razor | 1 + .../Internal/DashboardJobRetentionService.cs | 45 +- .../Internal/JobEventRetentionService.cs | 44 +- .../PostgresCleanupConfigurationOperation.cs | 342 ++++++++++ .../PostgresMetricsInspectionOperation.cs | 4 +- .../Operations/PostgresMetricsRollups.cs | 13 +- .../Internal/PostgresJobStore.cs | 71 ++- .../Internal/PostgresMigrator.cs | 7 + .../Internal/PostgresNames.cs | 5 +- .../ShedduellerPostgresBuilderExtensions.cs | 1 + .../ShedduellerJobRetentionService.cs | 45 +- .../JobEventCleanupConfiguration.cs | 14 + .../JobRetentionCleanupConfiguration.cs | 29 + .../MetricsCleanupConfiguration.cs | 16 + .../ShedduellerCleanupConfiguration.cs | 9 + .../IShedduellerCleanupConfigurationStore.cs | 42 ++ .../DashboardEndpointTests.cs | 104 +++- .../JobEventRetentionServiceLoggingTests.cs | 57 ++ .../CleanupConfigurationOperationTests.cs | 131 ++++ .../PostgresMetricsRollupTests.cs | 20 + .../PostgresMigrationTests.cs | 22 + .../PostgresRegistrationTests.cs | 1 + .../JobRetentionServiceTests.cs | 102 +++ 25 files changed, 1681 insertions(+), 35 deletions(-) create mode 100644 src/Sheddueller.Dashboard/Components/Pages/Settings.razor create mode 100644 src/Sheddueller.Postgres/Internal/Operations/PostgresCleanupConfigurationOperation.cs create mode 100644 src/Sheddueller/JobEventCleanupConfiguration.cs create mode 100644 src/Sheddueller/JobRetentionCleanupConfiguration.cs create mode 100644 src/Sheddueller/MetricsCleanupConfiguration.cs create mode 100644 src/Sheddueller/ShedduellerCleanupConfiguration.cs create mode 100644 src/Sheddueller/Storage/IShedduellerCleanupConfigurationStore.cs create mode 100644 test/Sheddueller.Postgres.Tests/Operations/CleanupConfigurationOperationTests.cs create mode 100644 test/Sheddueller.Worker.Tests/JobRetentionServiceTests.cs diff --git a/src/Sheddueller.Dashboard/Components/DashboardLayout.razor b/src/Sheddueller.Dashboard/Components/DashboardLayout.razor index 02085f4..cd184a3 100644 --- a/src/Sheddueller.Dashboard/Components/DashboardLayout.razor +++ b/src/Sheddueller.Dashboard/Components/DashboardLayout.razor @@ -34,6 +34,10 @@ Metrics + + + Settings + @if (CurrentRefresh is { } refresh) diff --git a/src/Sheddueller.Dashboard/Components/Pages/Settings.razor b/src/Sheddueller.Dashboard/Components/Pages/Settings.razor new file mode 100644 index 0000000..f5677ff --- /dev/null +++ b/src/Sheddueller.Dashboard/Components/Pages/Settings.razor @@ -0,0 +1,587 @@ +@page "/settings" +@inherits DashboardPageComponent +@inject IServiceProvider Services +@inject Microsoft.Extensions.Options.IOptions ShedduellerOptions +@inject Microsoft.Extensions.Options.IOptions DashboardOptions + +
+
+
+

Settings

+

Persisted cluster cleanup controls.

+
+
+ + @if (_store is null) + { +
+ +
+

Persisted settings unavailable

+

The registered Sheddueller provider does not expose dashboard-editable cleanup settings.

+
+
+ } + else if (_loadError is not null) + { +
+ +
+

Settings failed to load

+

@_loadError

+
+
+ } + else if (_isLoading) + { +
+ +
+

Loading settings

+

Reading persisted cleanup settings.

+
+
+ } + else + { + @if (_actionMessage is not null) + { + + @_actionMessage + + } + +
+
+

Terminal Jobs

+ +
+ +
+ + + + + +
+
+ +
+
+

Job Events

+
+ +
+ + +
+
+ +
+
+

Metrics

+
+ +
+ + +
+
+ +
+ + +
+ } +
+ + + +@code { + private IShedduellerCleanupConfigurationStore? _store; + private SettingsFormModel _form = new(); + private string? _loadError; + private string? _actionMessage; + private bool _isLoading; + private bool _isSaving; + private bool _isActionError; + + private string SaveButtonText + => this._isSaving ? "Saving..." : "Save Settings"; + + private string ActionAlertClass + => this._isActionError + ? "settings-inline-alert settings-inline-alert--error" + : "settings-inline-alert settings-inline-alert--success"; + + private string ActionAlertIcon + => this._isActionError ? "warning" : "check_circle"; + + protected override async Task OnInitializedAsync() + { + this._store = Services.GetService(); + this._form = SettingsFormModel.FromConfiguration(this.CreateDefaultConfiguration()); + + if (this._store is not null) + { + await this.LoadAsync(); + } + } + + private async Task LoadAsync() + { + if (this._store is null) + { + return; + } + + this._isLoading = true; + this._loadError = null; + this._actionMessage = null; + + try + { + var configuration = await this._store.GetCleanupConfigurationAsync(this.CreateDefaultConfiguration()); + this._form = SettingsFormModel.FromConfiguration(configuration); + } + catch (Exception exception) + { + this._loadError = exception.Message; + } + finally + { + this._isLoading = false; + } + } + + private async Task SaveAsync() + { + if (this._store is null || this._isSaving) + { + return; + } + + this._isSaving = true; + this._actionMessage = null; + this._isActionError = false; + + try + { + var configuration = this._form.ToConfiguration(); + await this._store.SetCleanupConfigurationAsync(configuration); + this._form = SettingsFormModel.FromConfiguration(configuration); + this._actionMessage = "Cleanup settings saved."; + } + catch (Exception exception) + { + this._isActionError = true; + this._actionMessage = string.Concat("Save failed: ", exception.Message); + } + finally + { + this._isSaving = false; + } + } + + private ShedduellerCleanupConfiguration CreateDefaultConfiguration() + => new( + JobRetentionCleanupConfiguration.FromOptions(ShedduellerOptions.Value.JobRetention), + new JobEventCleanupConfiguration(DashboardOptions.Value.EventRetention, JobEventCleanupConfiguration.DefaultCleanupInterval), + MetricsCleanupConfiguration.Default); + + private sealed class SettingsFormModel + { + public bool JobRetentionEnabled { get; set; } + + public int CompletedRetentionHours { get; set; } + + public bool CompletedRetainForever { get; set; } + + public int FailedRetentionHours { get; set; } + + public bool FailedRetainForever { get; set; } + + public int CanceledRetentionHours { get; set; } + + public bool CanceledRetainForever { get; set; } + + public int JobRetentionCleanupIntervalMinutes { get; set; } + + public int JobRetentionBatchSize { get; set; } + + public int JobEventRetentionHours { get; set; } + + public int JobEventCleanupIntervalMinutes { get; set; } + + public int MetricsRetentionHours { get; set; } + + public int MetricsCleanupIntervalMinutes { get; set; } + + public static SettingsFormModel FromConfiguration(ShedduellerCleanupConfiguration configuration) + => new() + { + JobRetentionEnabled = configuration.JobRetention.Enabled, + CompletedRetentionHours = ToPositiveHours(configuration.JobRetention.CompletedRetention ?? TimeSpan.FromDays(1)), + CompletedRetainForever = configuration.JobRetention.CompletedRetention is null, + FailedRetentionHours = ToPositiveHours(configuration.JobRetention.FailedRetention ?? TimeSpan.FromDays(7)), + FailedRetainForever = configuration.JobRetention.FailedRetention is null, + CanceledRetentionHours = ToPositiveHours(configuration.JobRetention.CanceledRetention ?? TimeSpan.FromDays(7)), + CanceledRetainForever = configuration.JobRetention.CanceledRetention is null, + JobRetentionCleanupIntervalMinutes = ToPositiveMinutes(configuration.JobRetention.CleanupInterval), + JobRetentionBatchSize = configuration.JobRetention.BatchSize, + JobEventRetentionHours = ToPositiveHours(configuration.JobEvents.Retention), + JobEventCleanupIntervalMinutes = ToPositiveMinutes(configuration.JobEvents.CleanupInterval), + MetricsRetentionHours = ToPositiveHours(configuration.Metrics.Retention), + MetricsCleanupIntervalMinutes = ToPositiveMinutes(configuration.Metrics.CleanupInterval), + }; + + public ShedduellerCleanupConfiguration ToConfiguration() + => new( + new JobRetentionCleanupConfiguration( + this.JobRetentionEnabled, + this.CompletedRetainForever ? null : ToPositiveTimeSpanFromHours(this.CompletedRetentionHours, "Completed retention"), + this.FailedRetainForever ? null : ToPositiveTimeSpanFromHours(this.FailedRetentionHours, "Failed retention"), + this.CanceledRetainForever ? null : ToPositiveTimeSpanFromHours(this.CanceledRetentionHours, "Canceled retention"), + ToPositiveTimeSpanFromMinutes(this.JobRetentionCleanupIntervalMinutes, "Terminal job cleanup interval"), + RequirePositive(this.JobRetentionBatchSize, "Batch size")), + new JobEventCleanupConfiguration( + ToPositiveTimeSpanFromHours(this.JobEventRetentionHours, "Job event retention"), + ToPositiveTimeSpanFromMinutes(this.JobEventCleanupIntervalMinutes, "Job event cleanup interval")), + new MetricsCleanupConfiguration( + ToPositiveTimeSpanFromHours(this.MetricsRetentionHours, "Metrics retention"), + ToPositiveTimeSpanFromMinutes(this.MetricsCleanupIntervalMinutes, "Metrics cleanup interval"))); + + private static int ToPositiveHours(TimeSpan value) + => Math.Max(1, Convert.ToInt32(Math.Ceiling(value.TotalHours))); + + private static int ToPositiveMinutes(TimeSpan value) + => Math.Max(1, Convert.ToInt32(Math.Ceiling(value.TotalMinutes))); + + private static TimeSpan ToPositiveTimeSpanFromHours( + int value, + string label) + => TimeSpan.FromHours(RequirePositive(value, label)); + + private static TimeSpan ToPositiveTimeSpanFromMinutes( + int value, + string label) + => TimeSpan.FromMinutes(RequirePositive(value, label)); + + private static int RequirePositive( + int value, + string label) + { + if (value <= 0) + { + throw new InvalidOperationException(string.Concat(label, " must be positive.")); + } + + return value; + } + } +} diff --git a/src/Sheddueller.Dashboard/Components/_Imports.razor b/src/Sheddueller.Dashboard/Components/_Imports.razor index 7edf627..ab5fe9e 100644 --- a/src/Sheddueller.Dashboard/Components/_Imports.razor +++ b/src/Sheddueller.Dashboard/Components/_Imports.razor @@ -2,6 +2,7 @@ @using Microsoft.AspNetCore.Components @using Microsoft.AspNetCore.Components.Routing @using Microsoft.AspNetCore.Components.Web +@using Microsoft.Extensions.DependencyInjection @using Microsoft.JSInterop @using Sheddueller @using Sheddueller.Dashboard.Components diff --git a/src/Sheddueller.Dashboard/Internal/DashboardJobRetentionService.cs b/src/Sheddueller.Dashboard/Internal/DashboardJobRetentionService.cs index 56a0310..6a1cd99 100644 --- a/src/Sheddueller.Dashboard/Internal/DashboardJobRetentionService.cs +++ b/src/Sheddueller.Dashboard/Internal/DashboardJobRetentionService.cs @@ -15,6 +15,9 @@ internal sealed class DashboardJobRetentionService( IOptions options, ILogger logger) : BackgroundService { + private static readonly TimeSpan PersistedSettingsPollingInterval = TimeSpan.FromMinutes(1); + private DateTimeOffset? _lastCleanupAtUtc; + [SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Retention cleanup failures are diagnostic and should not stop the dashboard host.")] protected override async Task ExecuteAsync(CancellationToken stoppingToken) { @@ -22,9 +25,10 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { while (!stoppingToken.IsCancellationRequested) { + var delay = PersistedSettingsPollingInterval; try { - await this.CleanupOnceAsync(stoppingToken).ConfigureAwait(false); + delay = await this.CleanupIfDueAsync(stoppingToken).ConfigureAwait(false); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { @@ -35,7 +39,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) logger.DashboardJobRetentionCleanupFailed(exception); } - await Task.Delay(options.Value.JobRetention.CleanupInterval, stoppingToken).ConfigureAwait(false); + await Task.Delay(delay, stoppingToken).ConfigureAwait(false); } } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) @@ -43,29 +47,43 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } - private async ValueTask CleanupOnceAsync(CancellationToken cancellationToken) + private async ValueTask CleanupIfDueAsync(CancellationToken cancellationToken) { - var retention = options.Value.JobRetention; + var settingsStore = serviceProvider.GetService(); + var retention = settingsStore is null + ? JobRetentionCleanupConfiguration.FromOptions(options.Value.JobRetention) + : await settingsStore + .GetJobRetentionCleanupConfigurationAsync(JobRetentionCleanupConfiguration.FromOptions(options.Value.JobRetention), cancellationToken) + .ConfigureAwait(false); + var usesPersistedSettings = settingsStore is not null; if (!retention.Enabled) { - return; + return GetDelay(retention.CleanupInterval, usesPersistedSettings); + } + + var now = timeProvider.GetUtcNow(); + if (usesPersistedSettings + && this._lastCleanupAtUtc is { } lastCleanupAtUtc + && now - lastCleanupAtUtc < retention.CleanupInterval) + { + return GetDelay(retention.CleanupInterval - (now - lastCleanupAtUtc), usesPersistedSettings); } var store = serviceProvider.GetService(); if (store is null) { logger.DashboardJobRetentionStoreMissing(); - return; + return GetDelay(retention.CleanupInterval, usesPersistedSettings); } if (retention.CompletedRetention is null && retention.FailedRetention is null && retention.CanceledRetention is null) { - return; + this._lastCleanupAtUtc = now; + return GetDelay(retention.CleanupInterval, usesPersistedSettings); } - var now = timeProvider.GetUtcNow(); var request = new JobRetentionCleanupRequest( retention.CompletedRetention is { } completedRetention ? now.Subtract(completedRetention) : null, retention.FailedRetention is { } failedRetention ? now.Subtract(failedRetention) : null, @@ -87,5 +105,16 @@ private async ValueTask CleanupOnceAsync(CancellationToken cancellationToken) { logger.DashboardJobRetentionCleaned(totalDeleted); } + + this._lastCleanupAtUtc = now; + + return GetDelay(retention.CleanupInterval, usesPersistedSettings); } + + private static TimeSpan GetDelay( + TimeSpan requestedDelay, + bool usesPersistedSettings) + => usesPersistedSettings && requestedDelay > PersistedSettingsPollingInterval + ? PersistedSettingsPollingInterval + : requestedDelay; } diff --git a/src/Sheddueller.Dashboard/Internal/JobEventRetentionService.cs b/src/Sheddueller.Dashboard/Internal/JobEventRetentionService.cs index 811528a..9986432 100644 --- a/src/Sheddueller.Dashboard/Internal/JobEventRetentionService.cs +++ b/src/Sheddueller.Dashboard/Internal/JobEventRetentionService.cs @@ -12,9 +12,13 @@ namespace Sheddueller.Dashboard.Internal; internal sealed class JobEventRetentionService( IServiceProvider serviceProvider, + TimeProvider timeProvider, IOptions options, ILogger logger) : BackgroundService { + private static readonly TimeSpan PersistedSettingsPollingInterval = TimeSpan.FromMinutes(1); + private DateTimeOffset? _lastCleanupAtUtc; + [SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Retention cleanup failures are diagnostic and should not stop the dashboard host.")] protected override async Task ExecuteAsync(CancellationToken stoppingToken) { @@ -22,9 +26,10 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { while (!stoppingToken.IsCancellationRequested) { + var delay = PersistedSettingsPollingInterval; try { - await this.CleanupOnceAsync(stoppingToken).ConfigureAwait(false); + delay = await this.CleanupIfDueAsync(stoppingToken).ConfigureAwait(false); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { @@ -35,7 +40,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) logger.DashboardEventRetentionCleanupFailed(exception); } - await Task.Delay(TimeSpan.FromHours(1), stoppingToken).ConfigureAwait(false); + await Task.Delay(delay, stoppingToken).ConfigureAwait(false); } } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) @@ -43,19 +48,48 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } - private async ValueTask CleanupOnceAsync(CancellationToken cancellationToken) + private async ValueTask CleanupIfDueAsync(CancellationToken cancellationToken) { + var settingsStore = serviceProvider.GetService(); + var configuration = settingsStore is null + ? this.CreateDefaultConfiguration() + : await settingsStore + .GetJobEventCleanupConfigurationAsync(this.CreateDefaultConfiguration(), cancellationToken) + .ConfigureAwait(false); + var usesPersistedSettings = settingsStore is not null; + var now = timeProvider.GetUtcNow(); + if (usesPersistedSettings + && this._lastCleanupAtUtc is { } lastCleanupAtUtc + && now - lastCleanupAtUtc < configuration.CleanupInterval) + { + return GetDelay(configuration.CleanupInterval - (now - lastCleanupAtUtc), usesPersistedSettings); + } + var store = serviceProvider.GetService(); if (store is null) { logger.DashboardEventRetentionStoreMissing(); - return; + return GetDelay(configuration.CleanupInterval, usesPersistedSettings); } - var deleted = await store.CleanupAsync(options.Value.EventRetention, cancellationToken).ConfigureAwait(false); + var deleted = await store.CleanupAsync(configuration.Retention, cancellationToken).ConfigureAwait(false); if (deleted > 0) { logger.DashboardEventRetentionCleaned(deleted); } + + this._lastCleanupAtUtc = now; + + return GetDelay(configuration.CleanupInterval, usesPersistedSettings); } + + private JobEventCleanupConfiguration CreateDefaultConfiguration() + => new(options.Value.EventRetention, JobEventCleanupConfiguration.DefaultCleanupInterval); + + private static TimeSpan GetDelay( + TimeSpan requestedDelay, + bool usesPersistedSettings) + => usesPersistedSettings && requestedDelay > PersistedSettingsPollingInterval + ? PersistedSettingsPollingInterval + : requestedDelay; } diff --git a/src/Sheddueller.Postgres/Internal/Operations/PostgresCleanupConfigurationOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/PostgresCleanupConfigurationOperation.cs new file mode 100644 index 0000000..2a7ea2d --- /dev/null +++ b/src/Sheddueller.Postgres/Internal/Operations/PostgresCleanupConfigurationOperation.cs @@ -0,0 +1,342 @@ +namespace Sheddueller.Postgres.Internal.Operations; + +using System.Text.Json; + +using Npgsql; + +using NpgsqlTypes; + +using Sheddueller; + +internal static class PostgresCleanupConfigurationOperation +{ + private const string JobRetentionKey = "cleanup.job_retention"; + private const string JobEventsKey = "cleanup.job_events"; + private const string MetricsKey = "cleanup.metrics"; + + public static async ValueTask GetAsync( + PostgresOperationContext context, + ShedduellerCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken) + { + Validate(defaultConfiguration); + + await using var connection = await context.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + + var configuration = new ShedduellerCleanupConfiguration( + await ReadJobRetentionAsync(context, connection, transaction, defaultConfiguration.JobRetention, cancellationToken).ConfigureAwait(false), + await ReadJobEventsAsync(context, connection, transaction, defaultConfiguration.JobEvents, cancellationToken).ConfigureAwait(false), + await ReadMetricsAsync(context, connection, transaction, defaultConfiguration.Metrics, cancellationToken).ConfigureAwait(false)); + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + + return configuration; + } + + public static async ValueTask GetJobRetentionAsync( + PostgresOperationContext context, + JobRetentionCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken) + { + Validate(defaultConfiguration); + + await using var connection = await context.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + var configuration = await ReadJobRetentionAsync(context, connection, transaction, defaultConfiguration, cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + + return configuration; + } + + public static async ValueTask GetJobEventsAsync( + PostgresOperationContext context, + JobEventCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken) + { + Validate(defaultConfiguration); + + await using var connection = await context.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + var configuration = await ReadJobEventsAsync(context, connection, transaction, defaultConfiguration, cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + + return configuration; + } + + public static async ValueTask GetMetricsAsync( + PostgresOperationContext context, + MetricsCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken) + { + Validate(defaultConfiguration); + + await using var connection = await context.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + var configuration = await ReadMetricsAsync(context, connection, transaction, defaultConfiguration, cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + + return configuration; + } + + public static async ValueTask SetAsync( + PostgresOperationContext context, + ShedduellerCleanupConfiguration configuration, + CancellationToken cancellationToken) + { + Validate(configuration); + + await using var connection = await context.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + + await UpsertAsync(context, connection, transaction, JobRetentionKey, ToDocument(configuration.JobRetention), cancellationToken).ConfigureAwait(false); + await UpsertAsync(context, connection, transaction, JobEventsKey, ToDocument(configuration.JobEvents), cancellationToken).ConfigureAwait(false); + await UpsertAsync(context, connection, transaction, MetricsKey, ToDocument(configuration.Metrics), cancellationToken).ConfigureAwait(false); + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } + + private static async ValueTask ReadJobRetentionAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + JobRetentionCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken) + { + await SeedAsync(context, connection, transaction, JobRetentionKey, ToDocument(defaultConfiguration), cancellationToken).ConfigureAwait(false); + var document = await ReadAsync(context, connection, transaction, JobRetentionKey, cancellationToken).ConfigureAwait(false); + var configuration = document.ToConfiguration(); + Validate(configuration); + + return configuration; + } + + private static async ValueTask ReadJobEventsAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + JobEventCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken) + { + await SeedAsync(context, connection, transaction, JobEventsKey, ToDocument(defaultConfiguration), cancellationToken).ConfigureAwait(false); + var document = await ReadAsync(context, connection, transaction, JobEventsKey, cancellationToken).ConfigureAwait(false); + var configuration = document.ToConfiguration(); + Validate(configuration); + + return configuration; + } + + private static async ValueTask ReadMetricsAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + MetricsCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken) + { + await SeedAsync(context, connection, transaction, MetricsKey, ToDocument(defaultConfiguration), cancellationToken).ConfigureAwait(false); + var document = await ReadAsync(context, connection, transaction, MetricsKey, cancellationToken).ConfigureAwait(false); + var configuration = document.ToConfiguration(); + Validate(configuration); + + return configuration; + } + + private static async ValueTask SeedAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + string key, + TDocument document, + CancellationToken cancellationToken) + => await PostgresOperationContext.ExecuteCountAsync( + connection, + transaction, + $""" + insert into {context.Names.Settings} (setting_key, value, updated_at_utc) + values (@setting_key, @value, transaction_timestamp()) + on conflict (setting_key) do nothing; + """, + command => AddSettingParameters(command, key, document), + cancellationToken) + .ConfigureAwait(false); + + private static async ValueTask UpsertAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + string key, + TDocument document, + CancellationToken cancellationToken) + => await PostgresOperationContext.ExecuteCountAsync( + connection, + transaction, + $""" + insert into {context.Names.Settings} (setting_key, value, updated_at_utc) + values (@setting_key, @value, transaction_timestamp()) + on conflict (setting_key) do update + set value = excluded.value, + updated_at_utc = excluded.updated_at_utc; + """, + command => AddSettingParameters(command, key, document), + cancellationToken) + .ConfigureAwait(false); + + private static async ValueTask ReadAsync( + PostgresOperationContext context, + NpgsqlConnection connection, + NpgsqlTransaction transaction, + string key, + CancellationToken cancellationToken) + { + await using var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = + $""" + select value::text + from {context.Names.Settings} + where setting_key = @setting_key; + """; + command.Parameters.AddWithValue("setting_key", key); + + var json = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false) as string + ?? throw new InvalidOperationException($"PostgreSQL cleanup setting '{key}' was not found."); + try + { + return JsonSerializer.Deserialize(json) + ?? throw new InvalidOperationException($"PostgreSQL cleanup setting '{key}' was empty."); + } + catch (JsonException exception) + { + throw new InvalidOperationException($"PostgreSQL cleanup setting '{key}' is not valid JSON.", exception); + } + } + + private static void AddSettingParameters( + NpgsqlCommand command, + string key, + TDocument document) + { + command.Parameters.AddWithValue("setting_key", key); + command.Parameters.Add("value", NpgsqlDbType.Jsonb).Value = JsonSerializer.Serialize(document); + } + + private static JobRetentionSettingsDocument ToDocument(JobRetentionCleanupConfiguration configuration) + => new( + configuration.Enabled, + configuration.CompletedRetention?.Ticks, + configuration.FailedRetention?.Ticks, + configuration.CanceledRetention?.Ticks, + configuration.CleanupInterval.Ticks, + configuration.BatchSize); + + private static JobEventSettingsDocument ToDocument(JobEventCleanupConfiguration configuration) + => new(configuration.Retention.Ticks, configuration.CleanupInterval.Ticks); + + private static MetricsSettingsDocument ToDocument(MetricsCleanupConfiguration configuration) + => new(configuration.Retention.Ticks, configuration.CleanupInterval.Ticks); + + private static void Validate(ShedduellerCleanupConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + Validate(configuration.JobRetention); + Validate(configuration.JobEvents); + Validate(configuration.Metrics); + } + + private static void Validate(JobRetentionCleanupConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + + if (configuration.CompletedRetention is { } completedRetention && completedRetention <= TimeSpan.Zero) + { + throw new InvalidOperationException("Job retention completed retention must be positive or null."); + } + + if (configuration.FailedRetention is { } failedRetention && failedRetention <= TimeSpan.Zero) + { + throw new InvalidOperationException("Job retention failed retention must be positive or null."); + } + + if (configuration.CanceledRetention is { } canceledRetention && canceledRetention <= TimeSpan.Zero) + { + throw new InvalidOperationException("Job retention canceled retention must be positive or null."); + } + + if (configuration.CleanupInterval <= TimeSpan.Zero) + { + throw new InvalidOperationException("Job retention cleanup interval must be positive."); + } + + if (configuration.BatchSize <= 0) + { + throw new InvalidOperationException("Job retention batch size must be positive."); + } + } + + private static void Validate(JobEventCleanupConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + + if (configuration.Retention <= TimeSpan.Zero) + { + throw new InvalidOperationException("Job event retention must be positive."); + } + + if (configuration.CleanupInterval <= TimeSpan.Zero) + { + throw new InvalidOperationException("Job event cleanup interval must be positive."); + } + } + + private static void Validate(MetricsCleanupConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + + if (configuration.Retention <= TimeSpan.Zero) + { + throw new InvalidOperationException("Metrics retention must be positive."); + } + + if (configuration.CleanupInterval <= TimeSpan.Zero) + { + throw new InvalidOperationException("Metrics cleanup interval must be positive."); + } + } + + private sealed record JobRetentionSettingsDocument( + bool Enabled, + long? CompletedRetentionTicks, + long? FailedRetentionTicks, + long? CanceledRetentionTicks, + long CleanupIntervalTicks, + int BatchSize) + { + public JobRetentionCleanupConfiguration ToConfiguration() + => new( + this.Enabled, + ToNullableTimeSpan(this.CompletedRetentionTicks), + ToNullableTimeSpan(this.FailedRetentionTicks), + ToNullableTimeSpan(this.CanceledRetentionTicks), + TimeSpan.FromTicks(this.CleanupIntervalTicks), + this.BatchSize); + } + + private sealed record JobEventSettingsDocument( + long RetentionTicks, + long CleanupIntervalTicks) + { + public JobEventCleanupConfiguration ToConfiguration() + => new(TimeSpan.FromTicks(this.RetentionTicks), TimeSpan.FromTicks(this.CleanupIntervalTicks)); + } + + private sealed record MetricsSettingsDocument( + long RetentionTicks, + long CleanupIntervalTicks) + { + public MetricsCleanupConfiguration ToConfiguration() + => new(TimeSpan.FromTicks(this.RetentionTicks), TimeSpan.FromTicks(this.CleanupIntervalTicks)); + } + + private static TimeSpan? ToNullableTimeSpan(long? ticks) + => ticks is { } value ? TimeSpan.FromTicks(value) : null; +} diff --git a/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsInspectionOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsInspectionOperation.cs index b4b4cd9..c6240bc 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsInspectionOperation.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsInspectionOperation.cs @@ -4,6 +4,7 @@ namespace Sheddueller.Postgres.Internal.Operations; using Npgsql; +using Sheddueller; using Sheddueller.Inspection.Metrics; internal static class PostgresMetricsInspectionOperation @@ -17,6 +18,7 @@ internal static class PostgresMetricsInspectionOperation public static async ValueTask GetAsync( PostgresOperationContext context, MetricsInspectionQuery query, + MetricsCleanupConfiguration cleanupConfiguration, TimeSpan staleThreshold, TimeSpan deadThreshold, CancellationToken cancellationToken) @@ -28,7 +30,7 @@ public static async ValueTask GetAsync( } await using var connection = await context.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); - await PostgresMetricsRollups.CleanupAsync(context, connection, cancellationToken).ConfigureAwait(false); + await PostgresMetricsRollups.CleanupAsync(context, connection, cleanupConfiguration, cancellationToken).ConfigureAwait(false); var current = await ReadCurrentCountsAsync(context, connection, staleThreshold, deadThreshold, cancellationToken).ConfigureAwait(false); var metrics = new List(windows.Count); diff --git a/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsRollups.cs b/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsRollups.cs index b072452..f5c5494 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsRollups.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsRollups.cs @@ -4,15 +4,14 @@ namespace Sheddueller.Postgres.Internal.Operations; using NpgsqlTypes; +using Sheddueller; using Sheddueller.Storage; internal static class PostgresMetricsRollups { public const int BucketSizeSeconds = 5; - public static readonly TimeSpan Retention = TimeSpan.FromDays(7); private const int CleanupAdvisoryLockKey = 7870835; - private static readonly TimeSpan CleanupInterval = TimeSpan.FromHours(1); internal static readonly long[] DurationHistogramThresholdsMs = [ @@ -361,8 +360,11 @@ from canceled_jobs public static async ValueTask CleanupAsync( PostgresOperationContext context, NpgsqlConnection connection, + MetricsCleanupConfiguration configuration, CancellationToken cancellationToken) { + ArgumentNullException.ThrowIfNull(configuration); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); var acquired = await TryAcquireCleanupLockAsync(context, connection, transaction, cancellationToken).ConfigureAwait(false); if (!acquired) @@ -371,7 +373,7 @@ public static async ValueTask CleanupAsync( return; } - if (!await ShouldCleanupAsync(context, connection, transaction, cancellationToken).ConfigureAwait(false)) + if (!await ShouldCleanupAsync(context, connection, transaction, configuration.CleanupInterval, cancellationToken).ConfigureAwait(false)) { await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); return; @@ -388,7 +390,7 @@ delete from {context.Names.MetricsBuckets} set last_cleanup_at_utc = transaction_timestamp() where singleton_id = 1; """, - command => command.Parameters.AddWithValue("retention", Retention), + command => command.Parameters.AddWithValue("retention", configuration.Retention), cancellationToken) .ConfigureAwait(false); @@ -415,6 +417,7 @@ private static async ValueTask ShouldCleanupAsync( PostgresOperationContext context, NpgsqlConnection connection, NpgsqlTransaction transaction, + TimeSpan cleanupInterval, CancellationToken cancellationToken) { await using var command = connection.CreateCommand(); @@ -427,7 +430,7 @@ select last_cleanup_at_utc is null where singleton_id = 1 for update; """; - command.Parameters.AddWithValue("cleanup_interval", CleanupInterval); + command.Parameters.AddWithValue("cleanup_interval", cleanupInterval); return (bool)(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false) ?? throw new InvalidOperationException("PostgreSQL did not return metrics cleanup state.")); diff --git a/src/Sheddueller.Postgres/Internal/PostgresJobStore.cs b/src/Sheddueller.Postgres/Internal/PostgresJobStore.cs index a686cec..9658a12 100644 --- a/src/Sheddueller.Postgres/Internal/PostgresJobStore.cs +++ b/src/Sheddueller.Postgres/Internal/PostgresJobStore.cs @@ -19,6 +19,7 @@ internal sealed class PostgresJobStore( IJobEventSink, IJobEventRetentionStore, IJobRetentionStore, + IShedduellerCleanupConfigurationStore, IScheduleInspectionReader, IConcurrencyGroupInspectionReader, INodeInspectionReader, @@ -259,6 +260,51 @@ public ValueTask CleanupTerminalJobsAsync( return PostgresJobRetentionOperation.ExecuteAsync(this._context, request, cancellationToken); } + public ValueTask GetCleanupConfigurationAsync( + ShedduellerCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(defaultConfiguration); + + return PostgresCleanupConfigurationOperation.GetAsync(this._context, defaultConfiguration, cancellationToken); + } + + public ValueTask GetJobRetentionCleanupConfigurationAsync( + JobRetentionCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(defaultConfiguration); + + return PostgresCleanupConfigurationOperation.GetJobRetentionAsync(this._context, defaultConfiguration, cancellationToken); + } + + public ValueTask GetJobEventCleanupConfigurationAsync( + JobEventCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(defaultConfiguration); + + return PostgresCleanupConfigurationOperation.GetJobEventsAsync(this._context, defaultConfiguration, cancellationToken); + } + + public ValueTask GetMetricsCleanupConfigurationAsync( + MetricsCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(defaultConfiguration); + + return PostgresCleanupConfigurationOperation.GetMetricsAsync(this._context, defaultConfiguration, cancellationToken); + } + + public ValueTask SetCleanupConfigurationAsync( + ShedduellerCleanupConfiguration configuration, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(configuration); + + return PostgresCleanupConfigurationOperation.SetAsync(this._context, configuration, cancellationToken); + } + public ValueTask SearchSchedulesAsync( ScheduleInspectionQuery query, CancellationToken cancellationToken = default) @@ -302,11 +348,24 @@ public ValueTask SearchNodesAsync( public ValueTask GetMetricsAsync( MetricsInspectionQuery query, CancellationToken cancellationToken = default) - => PostgresMetricsInspectionOperation.GetAsync( - this._context, - query, - this._shedduellerOptions.Value.EffectiveStaleNodeThreshold, - this._shedduellerOptions.Value.EffectiveDeadNodeThreshold, - cancellationToken); + => this.GetMetricsCoreAsync(query, cancellationToken); + + private async ValueTask GetMetricsCoreAsync( + MetricsInspectionQuery query, + CancellationToken cancellationToken) + { + var cleanupConfiguration = await PostgresCleanupConfigurationOperation + .GetMetricsAsync(this._context, MetricsCleanupConfiguration.Default, cancellationToken) + .ConfigureAwait(false); + + return await PostgresMetricsInspectionOperation.GetAsync( + this._context, + query, + cleanupConfiguration, + this._shedduellerOptions.Value.EffectiveStaleNodeThreshold, + this._shedduellerOptions.Value.EffectiveDeadNodeThreshold, + cancellationToken) + .ConfigureAwait(false); + } } diff --git a/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs b/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs index d0480b5..f231c6d 100644 --- a/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs +++ b/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs @@ -325,6 +325,13 @@ constraint metrics_rollup_state_singleton_id_check check (singleton_id = 1) values (1, null) on conflict (singleton_id) do nothing; + create table if not exists {this._names.Settings} ( + setting_key text primary key, + value jsonb not null, + updated_at_utc timestamptz not null, + constraint settings_setting_key_check check (length(setting_key) > 0) + ); + create index if not exists idx_jobs_claim_scan on {this._names.Jobs} (priority desc, enqueue_sequence asc) where state = 'Queued'; diff --git a/src/Sheddueller.Postgres/Internal/PostgresNames.cs b/src/Sheddueller.Postgres/Internal/PostgresNames.cs index eb235fd..3e0f5ce 100644 --- a/src/Sheddueller.Postgres/Internal/PostgresNames.cs +++ b/src/Sheddueller.Postgres/Internal/PostgresNames.cs @@ -4,7 +4,7 @@ namespace Sheddueller.Postgres.Internal; internal sealed class PostgresNames { - public const int ExpectedSchemaVersion = 11; + public const int ExpectedSchemaVersion = 12; public const string WakeupChannel = "sheddueller_wakeup"; public const string JobEventChannel = "sheddueller_job_event"; @@ -25,6 +25,7 @@ public PostgresNames(string schemaName) this.MetricsBuckets = this.Table("metrics_buckets"); this.MetricsHistogramBins = this.Table("metrics_histogram_bins"); this.MetricsRollupState = this.Table("metrics_rollup_state"); + this.Settings = this.Table("settings"); } public string SchemaName { get; } @@ -57,6 +58,8 @@ public PostgresNames(string schemaName) public string MetricsRollupState { get; } + public string Settings { get; } + public static string QuoteIdentifier(string identifier) => $"\"{identifier.Replace("\"", "\"\"", StringComparison.Ordinal)}\""; diff --git a/src/Sheddueller.Postgres/ShedduellerPostgresBuilderExtensions.cs b/src/Sheddueller.Postgres/ShedduellerPostgresBuilderExtensions.cs index a7ab200..98e1dba 100644 --- a/src/Sheddueller.Postgres/ShedduellerPostgresBuilderExtensions.cs +++ b/src/Sheddueller.Postgres/ShedduellerPostgresBuilderExtensions.cs @@ -144,6 +144,7 @@ private static void RegisterProviderServices(IServiceCollection services) services.Replace(ServiceDescriptor.Singleton(serviceProvider => serviceProvider.GetRequiredService())); services.Replace(ServiceDescriptor.Singleton(serviceProvider => serviceProvider.GetRequiredService())); services.Replace(ServiceDescriptor.Singleton(serviceProvider => serviceProvider.GetRequiredService())); + services.Replace(ServiceDescriptor.Singleton(serviceProvider => serviceProvider.GetRequiredService())); services.Replace(ServiceDescriptor.Singleton(serviceProvider => serviceProvider.GetRequiredService())); services.Replace(ServiceDescriptor.Singleton(serviceProvider => serviceProvider.GetRequiredService())); services.Replace(ServiceDescriptor.Singleton(serviceProvider => serviceProvider.GetRequiredService())); diff --git a/src/Sheddueller.Worker/Internal/ShedduellerJobRetentionService.cs b/src/Sheddueller.Worker/Internal/ShedduellerJobRetentionService.cs index c7bf2cf..6865891 100644 --- a/src/Sheddueller.Worker/Internal/ShedduellerJobRetentionService.cs +++ b/src/Sheddueller.Worker/Internal/ShedduellerJobRetentionService.cs @@ -15,6 +15,9 @@ internal sealed class ShedduellerJobRetentionService( IOptions options, ILogger logger) : BackgroundService { + private static readonly TimeSpan PersistedSettingsPollingInterval = TimeSpan.FromMinutes(1); + private DateTimeOffset? _lastCleanupAtUtc; + [SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Retention cleanup failures are diagnostic and should not stop the worker host.")] protected override async Task ExecuteAsync(CancellationToken stoppingToken) { @@ -22,9 +25,10 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { while (!stoppingToken.IsCancellationRequested) { + var delay = PersistedSettingsPollingInterval; try { - await this.CleanupOnceAsync(stoppingToken).ConfigureAwait(false); + delay = await this.CleanupIfDueAsync(stoppingToken).ConfigureAwait(false); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { @@ -35,7 +39,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) logger.WorkerJobRetentionCleanupFailed(exception); } - await Task.Delay(options.Value.JobRetention.CleanupInterval, stoppingToken).ConfigureAwait(false); + await Task.Delay(delay, stoppingToken).ConfigureAwait(false); } } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) @@ -43,29 +47,43 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } - private async ValueTask CleanupOnceAsync(CancellationToken cancellationToken) + private async ValueTask CleanupIfDueAsync(CancellationToken cancellationToken) { - var retention = options.Value.JobRetention; + var settingsStore = serviceProvider.GetService(); + var retention = settingsStore is null + ? JobRetentionCleanupConfiguration.FromOptions(options.Value.JobRetention) + : await settingsStore + .GetJobRetentionCleanupConfigurationAsync(JobRetentionCleanupConfiguration.FromOptions(options.Value.JobRetention), cancellationToken) + .ConfigureAwait(false); + var usesPersistedSettings = settingsStore is not null; if (!retention.Enabled) { - return; + return GetDelay(retention.CleanupInterval, usesPersistedSettings); + } + + var now = timeProvider.GetUtcNow(); + if (usesPersistedSettings + && this._lastCleanupAtUtc is { } lastCleanupAtUtc + && now - lastCleanupAtUtc < retention.CleanupInterval) + { + return GetDelay(retention.CleanupInterval - (now - lastCleanupAtUtc), usesPersistedSettings); } var store = serviceProvider.GetService(); if (store is null) { logger.WorkerJobRetentionStoreMissing(); - return; + return GetDelay(retention.CleanupInterval, usesPersistedSettings); } if (retention.CompletedRetention is null && retention.FailedRetention is null && retention.CanceledRetention is null) { - return; + this._lastCleanupAtUtc = now; + return GetDelay(retention.CleanupInterval, usesPersistedSettings); } - var now = timeProvider.GetUtcNow(); var request = new JobRetentionCleanupRequest( retention.CompletedRetention is { } completedRetention ? now.Subtract(completedRetention) : null, retention.FailedRetention is { } failedRetention ? now.Subtract(failedRetention) : null, @@ -87,5 +105,16 @@ private async ValueTask CleanupOnceAsync(CancellationToken cancellationToken) { logger.WorkerJobRetentionCleaned(totalDeleted); } + + this._lastCleanupAtUtc = now; + + return GetDelay(retention.CleanupInterval, usesPersistedSettings); } + + private static TimeSpan GetDelay( + TimeSpan requestedDelay, + bool usesPersistedSettings) + => usesPersistedSettings && requestedDelay > PersistedSettingsPollingInterval + ? PersistedSettingsPollingInterval + : requestedDelay; } diff --git a/src/Sheddueller/JobEventCleanupConfiguration.cs b/src/Sheddueller/JobEventCleanupConfiguration.cs new file mode 100644 index 0000000..aab416f --- /dev/null +++ b/src/Sheddueller/JobEventCleanupConfiguration.cs @@ -0,0 +1,14 @@ +namespace Sheddueller; + +/// +/// Cluster-wide cleanup configuration for durable job events. +/// +public sealed record JobEventCleanupConfiguration( + TimeSpan Retention, + TimeSpan CleanupInterval) +{ + /// + /// Gets the default job-event cleanup interval. + /// + public static TimeSpan DefaultCleanupInterval { get; } = TimeSpan.FromHours(1); +} diff --git a/src/Sheddueller/JobRetentionCleanupConfiguration.cs b/src/Sheddueller/JobRetentionCleanupConfiguration.cs new file mode 100644 index 0000000..412583a --- /dev/null +++ b/src/Sheddueller/JobRetentionCleanupConfiguration.cs @@ -0,0 +1,29 @@ +namespace Sheddueller; + +/// +/// Cluster-wide cleanup configuration for terminal jobs. +/// +public sealed record JobRetentionCleanupConfiguration( + bool Enabled, + TimeSpan? CompletedRetention, + TimeSpan? FailedRetention, + TimeSpan? CanceledRetention, + TimeSpan CleanupInterval, + int BatchSize) +{ + /// + /// Creates cleanup configuration from process-level job retention options. + /// + public static JobRetentionCleanupConfiguration FromOptions(JobRetentionOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + return new JobRetentionCleanupConfiguration( + options.Enabled, + options.CompletedRetention, + options.FailedRetention, + options.CanceledRetention, + options.CleanupInterval, + options.BatchSize); + } +} diff --git a/src/Sheddueller/MetricsCleanupConfiguration.cs b/src/Sheddueller/MetricsCleanupConfiguration.cs new file mode 100644 index 0000000..05b2671 --- /dev/null +++ b/src/Sheddueller/MetricsCleanupConfiguration.cs @@ -0,0 +1,16 @@ +namespace Sheddueller; + +/// +/// Cluster-wide cleanup configuration for persisted metrics rollups. +/// +public sealed record MetricsCleanupConfiguration( + TimeSpan Retention, + TimeSpan CleanupInterval) +{ + /// + /// Gets the default metrics cleanup configuration. + /// + public static MetricsCleanupConfiguration Default { get; } = new( + TimeSpan.FromDays(7), + TimeSpan.FromHours(1)); +} diff --git a/src/Sheddueller/ShedduellerCleanupConfiguration.cs b/src/Sheddueller/ShedduellerCleanupConfiguration.cs new file mode 100644 index 0000000..581502c --- /dev/null +++ b/src/Sheddueller/ShedduellerCleanupConfiguration.cs @@ -0,0 +1,9 @@ +namespace Sheddueller; + +/// +/// Cluster-wide cleanup configuration for Sheddueller stores. +/// +public sealed record ShedduellerCleanupConfiguration( + JobRetentionCleanupConfiguration JobRetention, + JobEventCleanupConfiguration JobEvents, + MetricsCleanupConfiguration Metrics); diff --git a/src/Sheddueller/Storage/IShedduellerCleanupConfigurationStore.cs b/src/Sheddueller/Storage/IShedduellerCleanupConfigurationStore.cs new file mode 100644 index 0000000..8bbc8c8 --- /dev/null +++ b/src/Sheddueller/Storage/IShedduellerCleanupConfigurationStore.cs @@ -0,0 +1,42 @@ +namespace Sheddueller.Storage; + +/// +/// Stores cluster-wide cleanup configuration. +/// +public interface IShedduellerCleanupConfigurationStore +{ + /// + /// Gets the full cleanup configuration, seeding any missing settings from the supplied defaults. + /// + ValueTask GetCleanupConfigurationAsync( + ShedduellerCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default); + + /// + /// Gets terminal job cleanup configuration, seeding it from the supplied default when missing. + /// + ValueTask GetJobRetentionCleanupConfigurationAsync( + JobRetentionCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default); + + /// + /// Gets durable job-event cleanup configuration, seeding it from the supplied default when missing. + /// + ValueTask GetJobEventCleanupConfigurationAsync( + JobEventCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default); + + /// + /// Gets metrics cleanup configuration, seeding it from the supplied default when missing. + /// + ValueTask GetMetricsCleanupConfigurationAsync( + MetricsCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default); + + /// + /// Persists the full cleanup configuration. + /// + ValueTask SetCleanupConfigurationAsync( + ShedduellerCleanupConfiguration configuration, + CancellationToken cancellationToken = default); +} diff --git a/test/Sheddueller.Dashboard.Tests/DashboardEndpointTests.cs b/test/Sheddueller.Dashboard.Tests/DashboardEndpointTests.cs index b908e68..f7326af 100644 --- a/test/Sheddueller.Dashboard.Tests/DashboardEndpointTests.cs +++ b/test/Sheddueller.Dashboard.Tests/DashboardEndpointTests.cs @@ -369,6 +369,42 @@ public async Task Metrics_KnownData_RendersRollingHealth() AssertShellRefresh(html); } + [Fact] + public async Task Settings_KnownData_RendersCleanupControls() + { + await using var app = await CreateStartedDashboardAsync(registerCleanupSettingsStore: true); + var html = await GetOkHtmlAsync(app, "/sheddueller/settings"); + + html.ShouldContain("base href=\"http://localhost/sheddueller/\""); + html.ShouldContain("Settings"); + html.ShouldContain("Persisted cluster cleanup controls."); + html.ShouldContain("Terminal Jobs"); + html.ShouldContain("Cleanup enabled"); + html.ShouldContain("Completed retention"); + html.ShouldContain("Failed retention"); + html.ShouldContain("Canceled retention"); + html.ShouldContain("Job Events"); + html.ShouldContain("Event retention"); + html.ShouldContain("Metrics"); + html.ShouldContain("Metrics retention"); + html.ShouldContain("Save Settings"); + html.ShouldContain("Reload"); + html.ShouldContain("value=\"48\""); + html.ShouldContain("value=\"72\""); + html.ShouldContain("value=\"250\""); + } + + [Fact] + public async Task Settings_UnsupportedProvider_RendersUnavailableState() + { + await using var app = await CreateStartedDashboardAsync(); + var html = await GetOkHtmlAsync(app, "/sheddueller/settings"); + + html.ShouldContain("Settings"); + html.ShouldContain("Persisted settings unavailable"); + html.ShouldContain("does not expose dashboard-editable cleanup settings"); + } + [Fact] public async Task JobDetail_KnownJob_RendersDetailAndDefaultLogFilter() { @@ -493,7 +529,8 @@ private static async Task CreateStartedDashboardAsync( bool prerender = true, bool mapWithWebApplication = true, Action? configureDashboard = null, - bool registerMetricsReader = true) + bool registerMetricsReader = true, + bool registerCleanupSettingsStore = false) { var builder = WebApplication.CreateBuilder(); builder.WebHost.UseTestServer(); @@ -509,6 +546,11 @@ private static async Task CreateStartedDashboardAsync( builder.Services.AddSingleton(); } + if (registerCleanupSettingsStore) + { + builder.Services.AddSingleton(); + } + builder.Services.AddSingleton(); builder.Services.AddShedduellerDashboard(options => { @@ -651,6 +693,66 @@ private static string GetCancelButtonHtml(string html) return html[startIndex..(endIndex + "".Length)]; } + private sealed class StubCleanupConfigurationStore : IShedduellerCleanupConfigurationStore + { + private ShedduellerCleanupConfiguration _configuration = new( + new JobRetentionCleanupConfiguration( + Enabled: true, + CompletedRetention: null, + FailedRetention: TimeSpan.FromHours(48), + CanceledRetention: TimeSpan.FromHours(72), + CleanupInterval: TimeSpan.FromMinutes(15), + BatchSize: 250), + new JobEventCleanupConfiguration(TimeSpan.FromHours(12), TimeSpan.FromMinutes(20)), + new MetricsCleanupConfiguration(TimeSpan.FromHours(168), TimeSpan.FromMinutes(30))); + + public ValueTask GetCleanupConfigurationAsync( + ShedduellerCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + return ValueTask.FromResult(this._configuration); + } + + public ValueTask GetJobRetentionCleanupConfigurationAsync( + JobRetentionCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + return ValueTask.FromResult(this._configuration.JobRetention); + } + + public ValueTask GetJobEventCleanupConfigurationAsync( + JobEventCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + return ValueTask.FromResult(this._configuration.JobEvents); + } + + public ValueTask GetMetricsCleanupConfigurationAsync( + MetricsCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + return ValueTask.FromResult(this._configuration.Metrics); + } + + public ValueTask SetCleanupConfigurationAsync( + ShedduellerCleanupConfiguration configuration, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + this._configuration = configuration; + + return ValueTask.CompletedTask; + } + } + private sealed class StubJobInspectionReader : IJobInspectionReader { public static readonly Guid JobId = Guid.Parse("8c32d457-9e7a-42bb-8947-0c8fa54743be"); diff --git a/test/Sheddueller.Dashboard.Tests/JobEventRetentionServiceLoggingTests.cs b/test/Sheddueller.Dashboard.Tests/JobEventRetentionServiceLoggingTests.cs index 22198b3..f404136 100644 --- a/test/Sheddueller.Dashboard.Tests/JobEventRetentionServiceLoggingTests.cs +++ b/test/Sheddueller.Dashboard.Tests/JobEventRetentionServiceLoggingTests.cs @@ -27,6 +27,7 @@ public async Task Cleanup_NonZeroDeletedCount_LogsCleanupCount() .AddProvider(logs)); using var service = new JobEventRetentionService( serviceProvider, + TimeProvider.System, Options.Create(new ShedduellerDashboardOptions { EventRetention = TimeSpan.FromDays(1) }), loggerFactory.CreateLogger()); @@ -40,16 +41,72 @@ public async Task Cleanup_NonZeroDeletedCount_LogsCleanupCount() entry.MessageTemplate.ShouldBe("Dashboard job-event retention cleanup deleted {DeletedCount} events."); } + [Fact] + public async Task Cleanup_PersistedSettingsStore_UsesPersistedRetention() + { + using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var store = new RecordingRetentionStore(0); + var settingsStore = new RecordingCleanupConfigurationStore(new JobEventCleanupConfiguration( + TimeSpan.FromDays(3), + TimeSpan.FromHours(1))); + var services = new ServiceCollection(); + services.AddSingleton(store); + services.AddSingleton(settingsStore); + using var serviceProvider = services.BuildServiceProvider(); + using var service = new JobEventRetentionService( + serviceProvider, + TimeProvider.System, + Options.Create(new ShedduellerDashboardOptions { EventRetention = TimeSpan.FromDays(1) }), + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + + await service.StartAsync(cancellationTokenSource.Token); + await store.CleanupCalled.Task.WaitAsync(cancellationTokenSource.Token); + await service.StopAsync(cancellationTokenSource.Token); + + store.Retention.ShouldBe(TimeSpan.FromDays(3)); + } + private sealed class RecordingRetentionStore(int deletedCount) : IJobEventRetentionStore { public TaskCompletionSource CleanupCalled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TimeSpan? Retention { get; private set; } + public ValueTask CleanupAsync( TimeSpan retention, CancellationToken cancellationToken = default) { + this.Retention = retention; this.CleanupCalled.TrySetResult(); return ValueTask.FromResult(deletedCount); } } + + private sealed class RecordingCleanupConfigurationStore(JobEventCleanupConfiguration configuration) : IShedduellerCleanupConfigurationStore + { + public ValueTask GetCleanupConfigurationAsync( + ShedduellerCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(defaultConfiguration with { JobEvents = configuration }); + + public ValueTask GetJobRetentionCleanupConfigurationAsync( + JobRetentionCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(defaultConfiguration); + + public ValueTask GetJobEventCleanupConfigurationAsync( + JobEventCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(configuration); + + public ValueTask GetMetricsCleanupConfigurationAsync( + MetricsCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(defaultConfiguration); + + public ValueTask SetCleanupConfigurationAsync( + ShedduellerCleanupConfiguration configuration, + CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + } } diff --git a/test/Sheddueller.Postgres.Tests/Operations/CleanupConfigurationOperationTests.cs b/test/Sheddueller.Postgres.Tests/Operations/CleanupConfigurationOperationTests.cs new file mode 100644 index 0000000..f0242fa --- /dev/null +++ b/test/Sheddueller.Postgres.Tests/Operations/CleanupConfigurationOperationTests.cs @@ -0,0 +1,131 @@ +namespace Sheddueller.Postgres.Tests.Operations; + +using System.Globalization; + +using Microsoft.Extensions.DependencyInjection; + +using Sheddueller.Storage; + +using Shouldly; + +public sealed class CleanupConfigurationOperationTests(PostgresFixture fixture) : IClassFixture +{ + [Fact] + public async Task GetCleanupConfiguration_MissingSettings_SeedsDefaults() + { + await using var context = await PostgresTestContext.CreateMigratedAsync(fixture); + var store = context.Provider.GetRequiredService(); + var defaults = CreateConfiguration( + jobRetentionEnabled: true, + completedRetention: TimeSpan.FromDays(2), + failedRetention: TimeSpan.FromDays(3), + canceledRetention: TimeSpan.FromDays(4), + jobRetentionCleanupInterval: TimeSpan.FromMinutes(15), + batchSize: 500, + jobEventRetention: TimeSpan.FromHours(12), + jobEventCleanupInterval: TimeSpan.FromMinutes(20), + metricsRetention: TimeSpan.FromDays(10), + metricsCleanupInterval: TimeSpan.FromMinutes(30)); + + var configuration = await store.GetCleanupConfigurationAsync(defaults); + + configuration.ShouldBe(defaults); + (await CountSettingsAsync(context)).ShouldBe(3); + } + + [Fact] + public async Task SetCleanupConfiguration_ExistingSettings_ReturnsPersistedValues() + { + await using var context = await PostgresTestContext.CreateMigratedAsync(fixture); + var store = context.Provider.GetRequiredService(); + await store.GetCleanupConfigurationAsync(CreateConfiguration()); + var updated = new ShedduellerCleanupConfiguration( + new JobRetentionCleanupConfiguration( + Enabled: false, + CompletedRetention: null, + FailedRetention: TimeSpan.FromHours(48), + CanceledRetention: null, + CleanupInterval: TimeSpan.FromMinutes(5), + BatchSize: 50), + new JobEventCleanupConfiguration(TimeSpan.FromHours(6), TimeSpan.FromMinutes(7)), + new MetricsCleanupConfiguration(TimeSpan.FromHours(8), TimeSpan.FromMinutes(9))); + + await store.SetCleanupConfigurationAsync(updated); + var configuration = await store.GetCleanupConfigurationAsync(CreateConfiguration( + jobRetentionEnabled: true, + completedRetention: TimeSpan.FromDays(30), + failedRetention: TimeSpan.FromDays(30), + canceledRetention: TimeSpan.FromDays(30), + jobRetentionCleanupInterval: TimeSpan.FromHours(1), + batchSize: 1000, + jobEventRetention: TimeSpan.FromDays(30), + jobEventCleanupInterval: TimeSpan.FromHours(1), + metricsRetention: TimeSpan.FromDays(30), + metricsCleanupInterval: TimeSpan.FromHours(1))); + + configuration.ShouldBe(updated); + (await CountSettingsAsync(context)).ShouldBe(3); + } + + [Fact] + public async Task GetJobRetentionCleanupConfiguration_MissingSetting_SeedsOnlyJobRetention() + { + await using var context = await PostgresTestContext.CreateMigratedAsync(fixture); + var store = context.Provider.GetRequiredService(); + var defaults = CreateConfiguration().JobRetention; + + var configuration = await store.GetJobRetentionCleanupConfigurationAsync(defaults); + + configuration.ShouldBe(defaults); + (await CountSettingsAsync(context)).ShouldBe(1); + } + + [Fact] + public async Task SetCleanupConfiguration_InvalidValue_Throws() + { + await using var context = await PostgresTestContext.CreateMigratedAsync(fixture); + var store = context.Provider.GetRequiredService(); + var configuration = CreateConfiguration(batchSize: 0); + + var exception = await Should.ThrowAsync(() => + store.SetCleanupConfigurationAsync(configuration).AsTask()); + + exception.Message.ShouldContain("batch size must be positive"); + } + + private static ShedduellerCleanupConfiguration CreateConfiguration( + bool jobRetentionEnabled = true, + TimeSpan? completedRetention = null, + TimeSpan? failedRetention = null, + TimeSpan? canceledRetention = null, + TimeSpan? jobRetentionCleanupInterval = null, + int batchSize = 100, + TimeSpan? jobEventRetention = null, + TimeSpan? jobEventCleanupInterval = null, + TimeSpan? metricsRetention = null, + TimeSpan? metricsCleanupInterval = null) + => new( + new JobRetentionCleanupConfiguration( + jobRetentionEnabled, + completedRetention ?? TimeSpan.FromDays(1), + failedRetention ?? TimeSpan.FromDays(7), + canceledRetention ?? TimeSpan.FromDays(7), + jobRetentionCleanupInterval ?? TimeSpan.FromHours(1), + batchSize), + new JobEventCleanupConfiguration( + jobEventRetention ?? TimeSpan.FromDays(7), + jobEventCleanupInterval ?? TimeSpan.FromHours(1)), + new MetricsCleanupConfiguration( + metricsRetention ?? TimeSpan.FromDays(7), + metricsCleanupInterval ?? TimeSpan.FromHours(1))); + + private static async ValueTask CountSettingsAsync(PostgresTestContext context) + { + await using var command = context.DataSource.CreateCommand( + $"select count(*) from {context.Table("settings")};"); + var count = await command.ExecuteScalarAsync(); + count.ShouldNotBeNull(); + + return Convert.ToInt32(count, CultureInfo.InvariantCulture); + } +} diff --git a/test/Sheddueller.Postgres.Tests/PostgresMetricsRollupTests.cs b/test/Sheddueller.Postgres.Tests/PostgresMetricsRollupTests.cs index 2e3d4df..2c44796 100644 --- a/test/Sheddueller.Postgres.Tests/PostgresMetricsRollupTests.cs +++ b/test/Sheddueller.Postgres.Tests/PostgresMetricsRollupTests.cs @@ -2,7 +2,9 @@ namespace Sheddueller.Postgres.Tests; using Microsoft.Extensions.DependencyInjection; +using Sheddueller; using Sheddueller.Inspection.Metrics; +using Sheddueller.Storage; using Shouldly; @@ -21,6 +23,24 @@ await context.Provider.GetRequiredService() (await CountAsync(context, "metrics_histogram_bins")).ShouldBe(0L); } + [Fact] + public async Task MetricsRead_PersistedMetricsRetention_RetainsBucketsInsideConfiguredWindow() + { + await using var context = await PostgresTestContext.CreateMigratedAsync(fixture); + await context.Provider.GetRequiredService() + .SetCleanupConfigurationAsync(new ShedduellerCleanupConfiguration( + JobRetentionCleanupConfiguration.FromOptions(new JobRetentionOptions()), + new JobEventCleanupConfiguration(TimeSpan.FromDays(7), TimeSpan.FromHours(1)), + new MetricsCleanupConfiguration(TimeSpan.FromDays(10), TimeSpan.FromHours(1)))); + await InsertOldRollupAsync(context); + + await context.Provider.GetRequiredService() + .GetMetricsAsync(new MetricsInspectionQuery([TimeSpan.FromMinutes(5)])); + + (await CountAsync(context, "metrics_buckets")).ShouldBe(1L); + (await CountAsync(context, "metrics_histogram_bins")).ShouldBe(1L); + } + private static async ValueTask InsertOldRollupAsync(PostgresTestContext context) { await using var command = context.DataSource.CreateCommand( diff --git a/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs b/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs index 3de348d..2159a62 100644 --- a/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs +++ b/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs @@ -174,6 +174,28 @@ from pg_indexes indexDefinition.ShouldContain("bin_index"); } + [Fact] + public async Task Migration_FreshSchema_CreatesSettingsTable() + { + await using var context = await PostgresTestContext.CreateMigratedAsync(fixture); + + await AssertTableExistsAsync(context, "settings"); + + (await ScalarAsync( + context, + """ + select exists ( + select 1 + from information_schema.columns + where table_schema = @schema_name + and table_name = 'settings' + and column_name = 'value' + and data_type = 'jsonb' + ); + """)) + .ShouldBeTrue(); + } + [Fact] public async Task Migration_FreshSchema_CreatesTagOrdinalColumnsAndIndexes() { diff --git a/test/Sheddueller.Postgres.Tests/PostgresRegistrationTests.cs b/test/Sheddueller.Postgres.Tests/PostgresRegistrationTests.cs index 309e3ef..9c48077 100644 --- a/test/Sheddueller.Postgres.Tests/PostgresRegistrationTests.cs +++ b/test/Sheddueller.Postgres.Tests/PostgresRegistrationTests.cs @@ -28,6 +28,7 @@ public async Task UsePostgres_ConnectionString_RegistersProviderServices() provider.GetRequiredService().ShouldBeSameAs(provider.GetRequiredService()); provider.GetRequiredService().ShouldBeSameAs(provider.GetRequiredService()); provider.GetRequiredService().ShouldBeSameAs(provider.GetRequiredService()); + provider.GetRequiredService().ShouldBeSameAs(provider.GetRequiredService()); provider.GetRequiredService().ShouldBeOfType(); provider.GetRequiredService().ShouldBeOfType(); } diff --git a/test/Sheddueller.Worker.Tests/JobRetentionServiceTests.cs b/test/Sheddueller.Worker.Tests/JobRetentionServiceTests.cs new file mode 100644 index 0000000..fbbf142 --- /dev/null +++ b/test/Sheddueller.Worker.Tests/JobRetentionServiceTests.cs @@ -0,0 +1,102 @@ +namespace Sheddueller.Worker.Tests; + +using System.Globalization; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +using Sheddueller.Storage; +using Sheddueller.Worker.Internal; + +using Shouldly; + +public sealed class JobRetentionServiceTests +{ + [Fact] + public async Task Cleanup_PersistedSettingsStore_UsesPersistedJobRetention() + { + using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var now = DateTimeOffset.Parse("2026-04-20T12:00:00Z", CultureInfo.InvariantCulture); + var retentionStore = new RecordingRetentionStore(); + var settingsStore = new RecordingCleanupConfigurationStore(new JobRetentionCleanupConfiguration( + Enabled: true, + CompletedRetention: TimeSpan.FromDays(3), + FailedRetention: null, + CanceledRetention: null, + CleanupInterval: TimeSpan.FromHours(1), + BatchSize: 12)); + var services = new ServiceCollection(); + services.AddSingleton(retentionStore); + services.AddSingleton(settingsStore); + await using var serviceProvider = services.BuildServiceProvider(); + var options = new ShedduellerOptions(); + options.JobRetention.CompletedRetention = TimeSpan.FromDays(1); + using var service = new ShedduellerJobRetentionService( + serviceProvider, + new FixedTimeProvider(now), + Options.Create(options), + NullLogger.Instance); + + await service.StartAsync(cancellationTokenSource.Token); + await retentionStore.CleanupCalled.Task.WaitAsync(cancellationTokenSource.Token); + await service.StopAsync(cancellationTokenSource.Token); + + var request = retentionStore.Request.ShouldNotBeNull(); + request.CompletedBeforeUtc.ShouldBe(now.AddDays(-3)); + request.FailedBeforeUtc.ShouldBeNull(); + request.CanceledBeforeUtc.ShouldBeNull(); + request.BatchSize.ShouldBe(12); + } + + private sealed class RecordingRetentionStore : IJobRetentionStore + { + public TaskCompletionSource CleanupCalled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public JobRetentionCleanupRequest? Request { get; private set; } + + public ValueTask CleanupTerminalJobsAsync( + JobRetentionCleanupRequest request, + CancellationToken cancellationToken = default) + { + this.Request = request; + this.CleanupCalled.TrySetResult(); + + return ValueTask.FromResult(new JobRetentionCleanupResult(0)); + } + } + + private sealed class RecordingCleanupConfigurationStore(JobRetentionCleanupConfiguration configuration) : IShedduellerCleanupConfigurationStore + { + public ValueTask GetCleanupConfigurationAsync( + ShedduellerCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(defaultConfiguration with { JobRetention = configuration }); + + public ValueTask GetJobRetentionCleanupConfigurationAsync( + JobRetentionCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(configuration); + + public ValueTask GetJobEventCleanupConfigurationAsync( + JobEventCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(defaultConfiguration); + + public ValueTask GetMetricsCleanupConfigurationAsync( + MetricsCleanupConfiguration defaultConfiguration, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(defaultConfiguration); + + public ValueTask SetCleanupConfigurationAsync( + ShedduellerCleanupConfiguration configuration, + CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + } + + private sealed class FixedTimeProvider(DateTimeOffset now) : TimeProvider + { + public override DateTimeOffset GetUtcNow() + => now; + } +} From 06d8ae04402cf72cf004151228bc69319760ca0f Mon Sep 17 00:00:00 2001 From: Brian Tyler Date: Fri, 3 Jul 2026 14:25:29 +0100 Subject: [PATCH 6/6] feat: Introduce concurrency group default limit functionality - Added support for setting and clearing concurrency group default limits. - Introduced new operations: SetConcurrencyDefaultLimitOperation and ClearConcurrencyLimitOverrideOperation. - Updated the database schema to include effective_limit and default_limit columns in the concurrency_groups table. - Modified existing operations and queries to utilize effective_limit instead of configured_limit. - Enhanced the IConcurrencyGroupManager interface with methods for managing default limits. - Updated related tests to cover new functionality and ensure correct behavior of concurrency limits. --- README.md | 2 + .../Components/Pages/ConcurrencyGroups.razor | 337 +++++++++++++++++- .../ClearConcurrencyLimitOverrideOperation.cs | 29 ++ ...gresConcurrencyGroupInspectionOperation.cs | 30 +- .../Internal/Operations/PostgresJobGroups.cs | 6 +- .../PostgresJobInspectionOperation.cs | 8 +- .../PostgresMetricsInspectionOperation.cs | 2 +- .../SetConcurrencyDefaultLimitOperation.cs | 34 ++ .../Operations/TryClaimNextJobOperation.cs | 2 +- .../Internal/PostgresJobStore.cs | 18 + .../Internal/PostgresMigrator.cs | 17 + .../Internal/PostgresNames.cs | 2 +- src/Sheddueller/IConcurrencyGroupManager.cs | 19 +- .../ConcurrencyGroupInspectionSummary.cs | 13 +- .../Logging/ShedduellerLoggerMessages.cs | 17 + .../Runtime/ConcurrencyGroupManager.cs | 37 +- .../ClearConcurrencyLimitOverrideRequest.cs | 8 + src/Sheddueller/Storage/IJobStore.cs | 18 +- .../SetConcurrencyDefaultLimitRequest.cs | 9 + .../DashboardEndpointTests.cs | 67 +++- ...onfiguredConcurrencyLimitOperationTests.cs | 10 + .../SetConcurrencyLimitOperationTests.cs | 32 ++ .../PostgresMigrationTests.cs | 42 +++ .../PostgresTestContext.cs | 8 +- .../InspectionContractTests.cs | 24 ++ .../JobStoreContractTests.cs | 37 ++ .../ConcurrencyGroupManagerTests.cs | 73 ++++ test/Sheddueller.Tests/RecordingJobStore.cs | 36 +- .../RegistrationTests.cs | 10 + .../WorkerJobLoggerTests.cs | 10 + .../WorkerLoggingTests.cs | 10 + .../WorkerProgressTests.cs | 10 + 32 files changed, 936 insertions(+), 41 deletions(-) create mode 100644 src/Sheddueller.Postgres/Internal/Operations/ClearConcurrencyLimitOverrideOperation.cs create mode 100644 src/Sheddueller.Postgres/Internal/Operations/SetConcurrencyDefaultLimitOperation.cs create mode 100644 src/Sheddueller/Storage/ClearConcurrencyLimitOverrideRequest.cs create mode 100644 src/Sheddueller/Storage/SetConcurrencyDefaultLimitRequest.cs create mode 100644 test/Sheddueller.Tests/ConcurrencyGroupManagerTests.cs diff --git a/README.md b/README.md index 84af0e0..9ee23f3 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,8 @@ Use `UsePostgres(postgres => postgres.DataSource = dataSource)` when an applicat The operational store keeps active jobs plus a bounded searchable terminal window. By default, background retention cleanup keeps completed jobs for 24 hours and failed or canceled jobs for 7 days, then deletes those terminal job rows and their tags, concurrency groups, and events. Configure `ShedduellerOptions.JobRetention` to change the windows, set a state retention to `null` to keep that state indefinitely, or set `Enabled = false` to disable cleanup. +Concurrency group limits use a persisted override over a code-defined default over the built-in default of `1`. Use `IConcurrencyGroupManager.SetDefaultLimitAsync(...)` from startup or deployment seeding code so dashboard edits survive restarts. Use `SetLimitAsync(...)` for an explicit live override and `ClearLimitOverrideAsync(...)` to fall back to the code default. + ## Enqueue Jobs Job methods return `Task` or `ValueTask` and receive the scheduler-owned `CancellationToken`. Use constructor-injected `ILogger` for durable job logs, `Job.Context` when a handler needs the job id or attempt number, and scheduler-supplied `IProgress` for durable progress updates. diff --git a/src/Sheddueller.Dashboard/Components/Pages/ConcurrencyGroups.razor b/src/Sheddueller.Dashboard/Components/Pages/ConcurrencyGroups.razor index dc4b45a..c2c7964 100644 --- a/src/Sheddueller.Dashboard/Components/Pages/ConcurrencyGroups.razor +++ b/src/Sheddueller.Dashboard/Components/Pages/ConcurrencyGroups.razor @@ -1,6 +1,7 @@ @page "/concurrency-groups" @inherits DashboardPageComponent @inject IConcurrencyGroupInspectionReader Reader +@inject IServiceProvider Services
@@ -41,6 +42,13 @@ } + @if (_actionMessage is not null) + { + + @_actionMessage + + } + @@ -52,26 +60,36 @@ + + @if (CanEditLimits) + { + + } Group Key Effective Limit + Limit Source Current Occupancy Blocked Jobs Saturation State Last Updated + @if (CanEditLimits) + { + Actions + } @if (visibleGroups.Count == 0) { - @EmptyText + @EmptyText } else @@ -82,7 +100,20 @@ @group.GroupKey - @DashboardFormat.Count(group.EffectiveLimit) + + @if (IsEditing(group)) + { + + } + else + { + @DashboardFormat.Count(group.EffectiveLimit) + } + + + @LimitSourceText(group) + @DashboardFormat.Count(group.CurrentOccupancy) @DashboardFormat.Count(group.BlockedJobCount) @@ -95,6 +126,36 @@ + @if (CanEditLimits) + { + +
+ @if (IsEditing(group)) + { + + + } + else + { + + + } +
+ + } } } @@ -131,6 +192,7 @@ .groups-message, .groups-inline-alert { + display: flex; align-items: flex-start; gap: 12px; border: 1px solid var(--sd-outline-variant); @@ -165,8 +227,18 @@ color: var(--sd-on-error-container); } + .groups-inline-alert--success { + border-color: var(--sd-success); + background: var(--sd-success-container); + color: var(--sd-success); + } + + .groups-inline-alert--success .material-symbols-outlined { + color: var(--sd-success); + } + .groups-table { - min-width: 1120px; + min-width: 1320px; table-layout: fixed; white-space: nowrap; } @@ -179,6 +251,10 @@ width: 136px; } + .groups-table__col--source { + width: 148px; + } + .groups-table__col--state { width: 176px; } @@ -187,6 +263,10 @@ width: 204px; } + .groups-table__col--actions { + width: 116px; + } + .groups-table thead { position: sticky; top: 0; @@ -304,6 +384,86 @@ color: var(--sd-on-surface-variant); } + .groups-source { + display: inline-flex; + align-items: center; + border: 1px solid var(--sd-outline-variant); + border-radius: 2px; + padding: 2px 8px; + color: var(--sd-on-surface-variant); + font-size: 10px; + font-weight: 700; + letter-spacing: 0.05em; + line-height: 12px; + text-transform: uppercase; + } + + .groups-source--override { + border-color: var(--sd-primary); + background: color-mix(in srgb, var(--sd-primary) 12%, var(--sd-surface-lowest)); + color: var(--sd-primary); + } + + .groups-source--default { + border-color: var(--sd-outline); + background: var(--sd-surface-variant); + color: var(--sd-on-surface); + } + + .groups-actions { + display: flex; + align-items: center; + justify-content: flex-end; + gap: 4px; + } + + .groups-icon-button { + display: inline-flex; + width: 30px; + height: 30px; + align-items: center; + justify-content: center; + border: 1px solid var(--sd-outline-variant); + border-radius: 2px; + background: var(--sd-surface-lowest); + color: var(--sd-on-surface-variant); + cursor: pointer; + } + + .groups-icon-button--primary { + border-color: var(--sd-primary); + background: var(--sd-primary); + color: var(--sd-on-primary); + } + + .groups-icon-button .material-symbols-outlined { + font-size: 18px; + line-height: 18px; + } + + .groups-icon-button:disabled { + cursor: not-allowed; + opacity: 0.5; + } + + .groups-limit-input { + width: 92px; + min-height: 30px; + border: 1px solid var(--sd-outline-variant); + border-radius: 2px; + background: var(--sd-surface-lowest); + color: var(--sd-on-surface); + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; + padding: 4px 6px; + text-align: right; + } + + .groups-limit-input:focus-visible, + .groups-icon-button:focus-visible { + outline: 2px solid var(--sd-primary); + outline-offset: 1px; + } + .groups-timestamp { display: flex; flex-direction: column; @@ -340,14 +500,37 @@ private readonly List _groups = []; private readonly DashboardConcurrencyGroupFilters _filters = new(); + private IConcurrencyGroupManager? _manager; private ConcurrencyGroupInspectionPage? _page; private string? _loadError; + private string? _actionMessage; + private string? _editingGroupKey; + private int _limitInput = 1; private bool _isLoading; private bool _isLoadingMore; + private bool _isActionRunning; + private bool _isActionError; private IReadOnlyList FilteredGroups => this._filters.ApplyClientFilter(this._groups); + private bool CanEditLimits + => this._manager is not null; + + private int GroupTableColumnCount + => this.CanEditLimits ? 8 : 7; + + private string ActionAlertClass + => this._isActionError + ? "groups-inline-alert groups-inline-alert--error" + : "groups-inline-alert groups-inline-alert--success"; + + private string ActionAlertIcon + => this._isActionError ? "warning" : "check_circle"; + + private bool IsActionDisabled + => this._isActionRunning || this._isLoading || this._isLoadingMore; + private string EmptyText => this._groups.Count == 0 ? "No concurrency groups matched the current query." @@ -373,6 +556,7 @@ protected override async Task OnInitializedAsync() { + this._manager = Services.GetService(); this.InitializeLiveRefresh( this.RefreshCurrentQueryAsync, showRefreshing: false, @@ -385,6 +569,8 @@ private async Task SetSaturatedOnlyAsync(bool value) { this._filters.SaturatedOnly = value; + this.ClearActionAlert(); + this.CancelEdit(); await this.LoadAsync(); } @@ -392,12 +578,17 @@ private async Task SetHasBlockedJobsOnlyAsync(bool value) { this._filters.HasBlockedJobsOnly = value; + this.ClearActionAlert(); + this.CancelEdit(); await this.LoadAsync(); } private void SetGroupKeyFilter(string value) - => this._filters.GroupKey = value; + { + this._filters.GroupKey = value; + this.ClearActionAlert(); + } private async Task LoadAsync() { @@ -467,6 +658,110 @@ this._groups.AddRange(page.Groups); this._page = page; this._loadError = null; + if (this._editingGroupKey is not null && page.Groups.All(group => !string.Equals(group.GroupKey, this._editingGroupKey, StringComparison.Ordinal))) + { + this.CancelEdit(); + } + } + + private void StartEdit(ConcurrencyGroupInspectionSummary group) + { + if (this.IsActionDisabled) + { + return; + } + + this.ClearActionAlert(); + this._editingGroupKey = group.GroupKey; + this._limitInput = group.OverrideLimit ?? group.EffectiveLimit; + } + + private void CancelEdit() + => this._editingGroupKey = null; + + private async Task SaveLimitAsync(ConcurrencyGroupInspectionSummary group) + { + if (this._manager is null || this.IsActionDisabled || !this.IsEditing(group)) + { + return; + } + + if (this._limitInput <= 0) + { + this.SetActionFailure("Limit save failed: Limit must be positive."); + return; + } + + this._isActionRunning = true; + this.ClearActionAlert(); + + try + { + await this._manager.SetLimitAsync(group.GroupKey, this._limitInput); + this.CancelEdit(); + this.SetActionSuccess(string.Create(CultureInfo.InvariantCulture, $"Concurrency group {group.GroupKey} override set to {DashboardFormat.Count(this._limitInput)}.")); + await this.LiveRefresh.RefreshNowAsync(); + } + catch (Exception exception) + { + this.SetActionFailure(string.Create(CultureInfo.InvariantCulture, $"Limit save failed: {exception.Message}")); + } + finally + { + this._isActionRunning = false; + } + } + + private async Task ResetLimitAsync(ConcurrencyGroupInspectionSummary group) + { + if (this._manager is null || this.IsActionDisabled) + { + return; + } + + this._isActionRunning = true; + this.ClearActionAlert(); + + try + { + await this._manager.ClearLimitOverrideAsync(group.GroupKey); + if (this.IsEditing(group)) + { + this.CancelEdit(); + } + + this.SetActionSuccess(string.Create(CultureInfo.InvariantCulture, $"Concurrency group {group.GroupKey} override reset.")); + await this.LiveRefresh.RefreshNowAsync(); + } + catch (Exception exception) + { + this.SetActionFailure(string.Create(CultureInfo.InvariantCulture, $"Limit reset failed: {exception.Message}")); + } + finally + { + this._isActionRunning = false; + } + } + + private bool IsEditing(ConcurrencyGroupInspectionSummary group) + => string.Equals(this._editingGroupKey, group.GroupKey, StringComparison.Ordinal); + + private void ClearActionAlert() + { + this._actionMessage = null; + this._isActionError = false; + } + + private void SetActionSuccess(string message) + { + this._isActionError = false; + this._actionMessage = message; + } + + private void SetActionFailure(string message) + { + this._isActionError = true; + this._actionMessage = message; } private static string GroupRowClass(ConcurrencyGroupInspectionSummary group) @@ -505,4 +800,38 @@ private static double GetOccupancyRatio(ConcurrencyGroupInspectionSummary group) => group.EffectiveLimit <= 0 ? 0 : (double)group.CurrentOccupancy / group.EffectiveLimit; + + private static string LimitSourceText(ConcurrencyGroupInspectionSummary group) + => group.OverrideLimit is not null + ? "Override" + : group.DefaultLimit is not null ? "Code default" : "Built-in default"; + + private static string LimitSourceTitle(ConcurrencyGroupInspectionSummary group) + => group.OverrideLimit is not null + ? string.Create(CultureInfo.InvariantCulture, $"Override limit {DashboardFormat.Count(group.OverrideLimit.Value)}.") + : group.DefaultLimit is not null + ? string.Create(CultureInfo.InvariantCulture, $"Code default limit {DashboardFormat.Count(group.DefaultLimit.Value)}.") + : "Built-in default limit 1."; + + private static string LimitSourceClass(ConcurrencyGroupInspectionSummary group) + => group.OverrideLimit is not null + ? "groups-source groups-source--override" + : group.DefaultLimit is not null + ? "groups-source groups-source--default" + : "groups-source"; + + private static string LimitInputLabel(ConcurrencyGroupInspectionSummary group) + => string.Create(CultureInfo.InvariantCulture, $"Limit for concurrency group {group.GroupKey}"); + + private static string EditLimitLabel(ConcurrencyGroupInspectionSummary group) + => string.Create(CultureInfo.InvariantCulture, $"Edit limit for concurrency group {group.GroupKey}"); + + private static string SaveLimitLabel(ConcurrencyGroupInspectionSummary group) + => string.Create(CultureInfo.InvariantCulture, $"Save limit for concurrency group {group.GroupKey}"); + + private static string CancelEditLabel(ConcurrencyGroupInspectionSummary group) + => string.Create(CultureInfo.InvariantCulture, $"Cancel limit edit for concurrency group {group.GroupKey}"); + + private static string ResetLimitLabel(ConcurrencyGroupInspectionSummary group) + => string.Create(CultureInfo.InvariantCulture, $"Reset limit override for concurrency group {group.GroupKey}"); } diff --git a/src/Sheddueller.Postgres/Internal/Operations/ClearConcurrencyLimitOverrideOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/ClearConcurrencyLimitOverrideOperation.cs new file mode 100644 index 0000000..e9eb5e4 --- /dev/null +++ b/src/Sheddueller.Postgres/Internal/Operations/ClearConcurrencyLimitOverrideOperation.cs @@ -0,0 +1,29 @@ +namespace Sheddueller.Postgres.Internal.Operations; + +using Sheddueller.Storage; + +internal static class ClearConcurrencyLimitOverrideOperation +{ + public static async ValueTask ExecuteAsync( + PostgresOperationContext context, + ClearConcurrencyLimitOverrideRequest request, + CancellationToken cancellationToken) + { + await using var connection = await context.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + await PostgresOperationContext.ExecuteCountAsync( + connection, + transaction, + $""" + update {context.Names.ConcurrencyGroups} + set configured_limit = null, + updated_at_utc = transaction_timestamp() + where group_key = @group_key; + """, + command => command.Parameters.AddWithValue("group_key", request.GroupKey), + cancellationToken) + .ConfigureAwait(false); + await context.NotifyAsync(connection, transaction, cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Sheddueller.Postgres/Internal/Operations/PostgresConcurrencyGroupInspectionOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/PostgresConcurrencyGroupInspectionOperation.cs index f856bd6..9caf42c 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/PostgresConcurrencyGroupInspectionOperation.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/PostgresConcurrencyGroupInspectionOperation.cs @@ -84,6 +84,8 @@ private static async ValueTask> {GroupSummaryCteSql(context)} select summary.group_key, + summary.default_limit, + summary.override_limit, summary.effective_limit, summary.current_occupancy, summary.blocked_count, @@ -110,6 +112,8 @@ order by summary.group_key asc {GroupSummaryCteSql(context)} select summary.group_key, + summary.default_limit, + summary.override_limit, summary.effective_limit, summary.current_occupancy, summary.blocked_count, @@ -132,15 +136,21 @@ private static async ValueTask> await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) { - var limit = reader.GetInt32(1); - var occupancy = reader.GetInt32(2); + var defaultLimit = reader.IsDBNull(1) ? null : (int?)reader.GetInt32(1); + var overrideLimit = reader.IsDBNull(2) ? null : (int?)reader.GetInt32(2); + var limit = reader.GetInt32(3); + var occupancy = reader.GetInt32(4); groups.Add(new ConcurrencyGroupInspectionSummary( reader.GetString(0), limit, occupancy, - Convert.ToInt32(reader.GetInt64(3), CultureInfo.InvariantCulture), - reader.GetBoolean(4), - reader.IsDBNull(5) ? null : PostgresConversion.ToDateTimeOffset(reader.GetValue(5)))); + Convert.ToInt32(reader.GetInt64(5), CultureInfo.InvariantCulture), + reader.GetBoolean(6), + reader.IsDBNull(7) ? null : PostgresConversion.ToDateTimeOffset(reader.GetValue(7))) + { + DefaultLimit = defaultLimit, + OverrideLimit = overrideLimit, + }); } return groups; @@ -187,16 +197,18 @@ blocked as ( left join {context.Names.ConcurrencyGroups} concurrency_group on concurrency_group.group_key = job_group.group_key where job.state = 'Queued' and (job.not_before_utc is null or job.not_before_utc <= transaction_timestamp()) - and coalesce(concurrency_group.in_use_count, 0) >= coalesce(concurrency_group.configured_limit, 1) + and coalesce(concurrency_group.in_use_count, 0) >= coalesce(concurrency_group.effective_limit, 1) group by job_group.group_key ), summary as ( select group_keys.group_key, - coalesce(concurrency_group.configured_limit, 1) as effective_limit, + concurrency_group.default_limit, + concurrency_group.configured_limit as override_limit, + coalesce(concurrency_group.effective_limit, 1) as effective_limit, coalesce(concurrency_group.in_use_count, 0) as current_occupancy, coalesce(blocked.blocked_count, 0) as blocked_count, - coalesce(concurrency_group.in_use_count, 0) >= coalesce(concurrency_group.configured_limit, 1) as is_saturated, + coalesce(concurrency_group.in_use_count, 0) >= coalesce(concurrency_group.effective_limit, 1) as is_saturated, concurrency_group.updated_at_utc from group_keys left join {context.Names.ConcurrencyGroups} concurrency_group on concurrency_group.group_key = group_keys.group_key @@ -238,7 +250,7 @@ select job.job_id where job_group.group_key = @group_key and job.state = 'Queued' and (job.not_before_utc is null or job.not_before_utc <= transaction_timestamp()) - and coalesce(concurrency_group.in_use_count, 0) >= coalesce(concurrency_group.configured_limit, 1) + and coalesce(concurrency_group.in_use_count, 0) >= coalesce(concurrency_group.effective_limit, 1) order by job.priority desc, job.enqueue_sequence asc; """, groupKey, diff --git a/src/Sheddueller.Postgres/Internal/Operations/PostgresJobGroups.cs b/src/Sheddueller.Postgres/Internal/Operations/PostgresJobGroups.cs index 3cc0cf5..3460c06 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/PostgresJobGroups.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/PostgresJobGroups.cs @@ -71,7 +71,7 @@ public static async ValueTask TryReserveGroupsAsync( command.Transaction = transaction; command.CommandText = $""" - select group_key, configured_limit, in_use_count + select group_key, effective_limit, in_use_count from {context.Names.ConcurrencyGroups} where group_key = any(@group_keys) order by group_key asc @@ -85,9 +85,9 @@ order by group_key asc while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) { lockCount++; - var configuredLimit = reader.IsDBNull(1) ? 1 : reader.GetInt32(1); + var effectiveLimit = reader.GetInt32(1); var inUseCount = reader.GetInt32(2); - if (inUseCount >= configuredLimit) + if (inUseCount >= effectiveLimit) { return false; } diff --git a/src/Sheddueller.Postgres/Internal/Operations/PostgresJobInspectionOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/PostgresJobInspectionOperation.cs index 0a7b2eb..a9f1599 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/PostgresJobInspectionOperation.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/PostgresJobInspectionOperation.cs @@ -52,7 +52,7 @@ select 1 from {context.Names.JobConcurrencyGroups} job_group left join {context.Names.ConcurrencyGroups} concurrency_group on concurrency_group.group_key = job_group.group_key where job_group.job_id = job.job_id - and coalesce(concurrency_group.in_use_count, 0) >= coalesce(concurrency_group.configured_limit, 1) + and coalesce(concurrency_group.in_use_count, 0) >= coalesce(concurrency_group.effective_limit, 1) ) order by priority desc, enqueue_sequence asc limit 10 @@ -735,7 +735,7 @@ select 1 from {context.Names.JobConcurrencyGroups} job_group left join {context.Names.ConcurrencyGroups} concurrency_group on concurrency_group.group_key = job_group.group_key where job_group.job_id = job.job_id - and coalesce(concurrency_group.in_use_count, 0) >= coalesce(concurrency_group.configured_limit, 1) + and coalesce(concurrency_group.in_use_count, 0) >= coalesce(concurrency_group.effective_limit, 1) ) ) select job_id, position @@ -806,7 +806,7 @@ select 1 from {context.Names.JobConcurrencyGroups} job_group left join {context.Names.ConcurrencyGroups} concurrency_group on concurrency_group.group_key = job_group.group_key where job_group.job_id = job.job_id - and coalesce(concurrency_group.in_use_count, 0) >= coalesce(concurrency_group.configured_limit, 1) + and coalesce(concurrency_group.in_use_count, 0) >= coalesce(concurrency_group.effective_limit, 1) ) ) select position @@ -1138,7 +1138,7 @@ select 1 from {context.Names.JobConcurrencyGroups} job_group left join {context.Names.ConcurrencyGroups} concurrency_group on concurrency_group.group_key = job_group.group_key where job_group.job_id = job.job_id - and coalesce(concurrency_group.in_use_count, 0) >= coalesce(concurrency_group.configured_limit, 1) + and coalesce(concurrency_group.in_use_count, 0) >= coalesce(concurrency_group.effective_limit, 1) ) then 1 else 0 end asc, diff --git a/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsInspectionOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsInspectionOperation.cs index c6240bc..52997a4 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsInspectionOperation.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/PostgresMetricsInspectionOperation.cs @@ -102,7 +102,7 @@ select count(*) as claimed_count saturated_groups as ( select count(*) as saturated_group_count from {context.Names.ConcurrencyGroups} - where in_use_count >= coalesce(configured_limit, 1) + where in_use_count >= effective_limit ), node_counts as ( select diff --git a/src/Sheddueller.Postgres/Internal/Operations/SetConcurrencyDefaultLimitOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/SetConcurrencyDefaultLimitOperation.cs new file mode 100644 index 0000000..ea3f661 --- /dev/null +++ b/src/Sheddueller.Postgres/Internal/Operations/SetConcurrencyDefaultLimitOperation.cs @@ -0,0 +1,34 @@ +namespace Sheddueller.Postgres.Internal.Operations; + +using Sheddueller.Storage; + +internal static class SetConcurrencyDefaultLimitOperation +{ + public static async ValueTask ExecuteAsync( + PostgresOperationContext context, + SetConcurrencyDefaultLimitRequest request, + CancellationToken cancellationToken) + { + await using var connection = await context.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + await PostgresOperationContext.ExecuteCountAsync( + connection, + transaction, + $""" + insert into {context.Names.ConcurrencyGroups} (group_key, configured_limit, default_limit, in_use_count, updated_at_utc) + values (@group_key, null, @default_limit, 0, transaction_timestamp()) + on conflict (group_key) do update + set default_limit = excluded.default_limit, + updated_at_utc = excluded.updated_at_utc; + """, + command => + { + command.Parameters.AddWithValue("group_key", request.GroupKey); + command.Parameters.AddWithValue("default_limit", request.Limit); + }, + cancellationToken) + .ConfigureAwait(false); + await context.NotifyAsync(connection, transaction, cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Sheddueller.Postgres/Internal/Operations/TryClaimNextJobOperation.cs b/src/Sheddueller.Postgres/Internal/Operations/TryClaimNextJobOperation.cs index 71c11f7..8a9ef2d 100644 --- a/src/Sheddueller.Postgres/Internal/Operations/TryClaimNextJobOperation.cs +++ b/src/Sheddueller.Postgres/Internal/Operations/TryClaimNextJobOperation.cs @@ -120,7 +120,7 @@ select 1 from {context.Names.JobConcurrencyGroups} job_group join {context.Names.ConcurrencyGroups} concurrency_group on concurrency_group.group_key = job_group.group_key where job_group.job_id = job.job_id - and concurrency_group.in_use_count >= coalesce(concurrency_group.configured_limit, 1) + and concurrency_group.in_use_count >= concurrency_group.effective_limit ) order by job.priority desc, job.enqueue_sequence asc for update of job skip locked diff --git a/src/Sheddueller.Postgres/Internal/PostgresJobStore.cs b/src/Sheddueller.Postgres/Internal/PostgresJobStore.cs index 9658a12..bf26f30 100644 --- a/src/Sheddueller.Postgres/Internal/PostgresJobStore.cs +++ b/src/Sheddueller.Postgres/Internal/PostgresJobStore.cs @@ -154,6 +154,24 @@ public ValueTask SetConcurrencyLimitAsync( return SetConcurrencyLimitOperation.ExecuteAsync(this._context, request, cancellationToken); } + public ValueTask SetConcurrencyDefaultLimitAsync( + SetConcurrencyDefaultLimitRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + return SetConcurrencyDefaultLimitOperation.ExecuteAsync(this._context, request, cancellationToken); + } + + public ValueTask ClearConcurrencyLimitOverrideAsync( + ClearConcurrencyLimitOverrideRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + return ClearConcurrencyLimitOverrideOperation.ExecuteAsync(this._context, request, cancellationToken); + } + public ValueTask GetConfiguredConcurrencyLimitAsync( string groupKey, CancellationToken cancellationToken = default) diff --git a/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs b/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs index f231c6d..1979dc8 100644 --- a/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs +++ b/src/Sheddueller.Postgres/Internal/PostgresMigrator.cs @@ -174,12 +174,29 @@ alter table {this._names.JobTags} create table if not exists {this._names.ConcurrencyGroups} ( group_key text primary key, configured_limit integer null, + default_limit integer null, + effective_limit integer generated always as (coalesce(configured_limit, default_limit, 1)) stored, in_use_count integer not null, updated_at_utc timestamptz not null, constraint concurrency_groups_configured_limit_check check (configured_limit is null or configured_limit > 0), + constraint concurrency_groups_default_limit_check check (default_limit is null or default_limit > 0), constraint concurrency_groups_in_use_count_check check (in_use_count >= 0) ); + alter table {this._names.ConcurrencyGroups} + add column if not exists default_limit integer null; + + alter table {this._names.ConcurrencyGroups} + add column if not exists effective_limit integer generated always as (coalesce(configured_limit, default_limit, 1)) stored; + + alter table {this._names.ConcurrencyGroups} + drop constraint if exists concurrency_groups_configured_limit_check, + add constraint concurrency_groups_configured_limit_check check (configured_limit is null or configured_limit > 0), + drop constraint if exists concurrency_groups_default_limit_check, + add constraint concurrency_groups_default_limit_check check (default_limit is null or default_limit > 0), + drop constraint if exists concurrency_groups_in_use_count_check, + add constraint concurrency_groups_in_use_count_check check (in_use_count >= 0); + create table if not exists {this._names.RecurringSchedules} ( schedule_key text primary key, cron_expression text not null, diff --git a/src/Sheddueller.Postgres/Internal/PostgresNames.cs b/src/Sheddueller.Postgres/Internal/PostgresNames.cs index 3e0f5ce..00089d8 100644 --- a/src/Sheddueller.Postgres/Internal/PostgresNames.cs +++ b/src/Sheddueller.Postgres/Internal/PostgresNames.cs @@ -4,7 +4,7 @@ namespace Sheddueller.Postgres.Internal; internal sealed class PostgresNames { - public const int ExpectedSchemaVersion = 12; + public const int ExpectedSchemaVersion = 13; public const string WakeupChannel = "sheddueller_wakeup"; public const string JobEventChannel = "sheddueller_job_event"; diff --git a/src/Sheddueller/IConcurrencyGroupManager.cs b/src/Sheddueller/IConcurrencyGroupManager.cs index 9b3513d..aab183a 100644 --- a/src/Sheddueller/IConcurrencyGroupManager.cs +++ b/src/Sheddueller/IConcurrencyGroupManager.cs @@ -6,7 +6,7 @@ namespace Sheddueller; public interface IConcurrencyGroupManager { /// - /// Sets the configured limit for a concurrency group. + /// Sets the live override limit for a concurrency group. /// ValueTask SetLimitAsync( string groupKey, @@ -14,7 +14,22 @@ ValueTask SetLimitAsync( CancellationToken cancellationToken = default); /// - /// Gets the configured limit for a concurrency group, if one exists. + /// Sets the code-defined default limit for a concurrency group without clearing a live override. + /// + ValueTask SetDefaultLimitAsync( + string groupKey, + int limit, + CancellationToken cancellationToken = default); + + /// + /// Clears the live override limit for a concurrency group, falling back to the code-defined or built-in default. + /// + ValueTask ClearLimitOverrideAsync( + string groupKey, + CancellationToken cancellationToken = default); + + /// + /// Gets the live override limit for a concurrency group, if one exists. /// ValueTask GetConfiguredLimitAsync( string groupKey, diff --git a/src/Sheddueller/Inspection/ConcurrencyGroups/ConcurrencyGroupInspectionSummary.cs b/src/Sheddueller/Inspection/ConcurrencyGroups/ConcurrencyGroupInspectionSummary.cs index f3ae978..71fe24e 100644 --- a/src/Sheddueller/Inspection/ConcurrencyGroups/ConcurrencyGroupInspectionSummary.cs +++ b/src/Sheddueller/Inspection/ConcurrencyGroups/ConcurrencyGroupInspectionSummary.cs @@ -9,4 +9,15 @@ public sealed record ConcurrencyGroupInspectionSummary( int CurrentOccupancy, int BlockedJobCount, bool IsSaturated, - DateTimeOffset? UpdatedAtUtc); + DateTimeOffset? UpdatedAtUtc) +{ + /// + /// Gets the code-defined default limit, if one exists. + /// + public int? DefaultLimit { get; init; } + + /// + /// Gets the live override limit, if one exists. + /// + public int? OverrideLimit { get; init; } +} diff --git a/src/Sheddueller/Logging/ShedduellerLoggerMessages.cs b/src/Sheddueller/Logging/ShedduellerLoggerMessages.cs index 7d8dfe7..a873267 100644 --- a/src/Sheddueller/Logging/ShedduellerLoggerMessages.cs +++ b/src/Sheddueller/Logging/ShedduellerLoggerMessages.cs @@ -102,6 +102,23 @@ public static partial void ConcurrencyGroupLimitSet( string groupKey, int limit); + [LoggerMessage( + EventIdStart + 31, + LogLevel.Debug, + "Set concurrency group {GroupKey} default limit to {Limit}.")] + public static partial void ConcurrencyGroupDefaultLimitSet( + this ILogger logger, + string groupKey, + int limit); + + [LoggerMessage( + EventIdStart + 32, + LogLevel.Debug, + "Cleared concurrency group {GroupKey} limit override.")] + public static partial void ConcurrencyGroupLimitOverrideCleared( + this ILogger logger, + string groupKey); + [LoggerMessage( EventIdStart + 40, LogLevel.Warning, diff --git a/src/Sheddueller/Runtime/ConcurrencyGroupManager.cs b/src/Sheddueller/Runtime/ConcurrencyGroupManager.cs index b8f46cd..a8da76e 100644 --- a/src/Sheddueller/Runtime/ConcurrencyGroupManager.cs +++ b/src/Sheddueller/Runtime/ConcurrencyGroupManager.cs @@ -14,11 +14,7 @@ internal sealed class ConcurrencyGroupManager( public async ValueTask SetLimitAsync(string groupKey, int limit, CancellationToken cancellationToken = default) { SubmissionValidator.ValidateConcurrencyGroupKey(groupKey); - - if (limit <= 0) - { - throw new ArgumentOutOfRangeException(nameof(limit), limit, "Concurrency group limits must be positive."); - } + ValidateLimit(limit); await store .SetConcurrencyLimitAsync(new SetConcurrencyLimitRequest(groupKey, limit, timeProvider.GetUtcNow()), cancellationToken) @@ -27,10 +23,41 @@ await store logger.ConcurrencyGroupLimitSet(groupKey, limit); } + public async ValueTask SetDefaultLimitAsync(string groupKey, int limit, CancellationToken cancellationToken = default) + { + SubmissionValidator.ValidateConcurrencyGroupKey(groupKey); + ValidateLimit(limit); + + await store + .SetConcurrencyDefaultLimitAsync(new SetConcurrencyDefaultLimitRequest(groupKey, limit, timeProvider.GetUtcNow()), cancellationToken) + .ConfigureAwait(false); + wakeSignal.Notify(); + logger.ConcurrencyGroupDefaultLimitSet(groupKey, limit); + } + + public async ValueTask ClearLimitOverrideAsync(string groupKey, CancellationToken cancellationToken = default) + { + SubmissionValidator.ValidateConcurrencyGroupKey(groupKey); + + await store + .ClearConcurrencyLimitOverrideAsync(new ClearConcurrencyLimitOverrideRequest(groupKey, timeProvider.GetUtcNow()), cancellationToken) + .ConfigureAwait(false); + wakeSignal.Notify(); + logger.ConcurrencyGroupLimitOverrideCleared(groupKey); + } + public ValueTask GetConfiguredLimitAsync(string groupKey, CancellationToken cancellationToken = default) { SubmissionValidator.ValidateConcurrencyGroupKey(groupKey); return store.GetConfiguredConcurrencyLimitAsync(groupKey, cancellationToken); } + + private static void ValidateLimit(int limit) + { + if (limit <= 0) + { + throw new ArgumentOutOfRangeException(nameof(limit), limit, "Concurrency group limits must be positive."); + } + } } diff --git a/src/Sheddueller/Storage/ClearConcurrencyLimitOverrideRequest.cs b/src/Sheddueller/Storage/ClearConcurrencyLimitOverrideRequest.cs new file mode 100644 index 0000000..99c3940 --- /dev/null +++ b/src/Sheddueller/Storage/ClearConcurrencyLimitOverrideRequest.cs @@ -0,0 +1,8 @@ +namespace Sheddueller.Storage; + +/// +/// Store request for clearing a concurrency-group live override limit. +/// +public sealed record ClearConcurrencyLimitOverrideRequest( + string GroupKey, + DateTimeOffset UpdatedAtUtc); diff --git a/src/Sheddueller/Storage/IJobStore.cs b/src/Sheddueller/Storage/IJobStore.cs index 7fa932d..fb3a1f4 100644 --- a/src/Sheddueller/Storage/IJobStore.cs +++ b/src/Sheddueller/Storage/IJobStore.cs @@ -97,14 +97,28 @@ ValueTask RecordWorkerNodeHeartbeatAsync( CancellationToken cancellationToken = default); /// - /// Persists a configured concurrency-group limit. + /// Persists a live override concurrency-group limit. /// ValueTask SetConcurrencyLimitAsync( SetConcurrencyLimitRequest request, CancellationToken cancellationToken = default); /// - /// Gets the configured concurrency-group limit, if one exists. + /// Persists a code-defined concurrency-group default limit without clearing a live override. + /// + ValueTask SetConcurrencyDefaultLimitAsync( + SetConcurrencyDefaultLimitRequest request, + CancellationToken cancellationToken = default); + + /// + /// Clears a live override concurrency-group limit. + /// + ValueTask ClearConcurrencyLimitOverrideAsync( + ClearConcurrencyLimitOverrideRequest request, + CancellationToken cancellationToken = default); + + /// + /// Gets the live override concurrency-group limit, if one exists. /// ValueTask GetConfiguredConcurrencyLimitAsync( string groupKey, diff --git a/src/Sheddueller/Storage/SetConcurrencyDefaultLimitRequest.cs b/src/Sheddueller/Storage/SetConcurrencyDefaultLimitRequest.cs new file mode 100644 index 0000000..61ae304 --- /dev/null +++ b/src/Sheddueller/Storage/SetConcurrencyDefaultLimitRequest.cs @@ -0,0 +1,9 @@ +namespace Sheddueller.Storage; + +/// +/// Store request for setting a code-defined concurrency-group default limit. +/// +public sealed record SetConcurrencyDefaultLimitRequest( + string GroupKey, + int Limit, + DateTimeOffset UpdatedAtUtc); diff --git a/test/Sheddueller.Dashboard.Tests/DashboardEndpointTests.cs b/test/Sheddueller.Dashboard.Tests/DashboardEndpointTests.cs index f7326af..0af2340 100644 --- a/test/Sheddueller.Dashboard.Tests/DashboardEndpointTests.cs +++ b/test/Sheddueller.Dashboard.Tests/DashboardEndpointTests.cs @@ -278,6 +278,12 @@ public async Task ConcurrencyGroups_KnownData_RendersRegistry() html.ShouldContain("api_sync_workers"); html.ShouldContain("bg_maintenance"); html.ShouldContain("db_vacuum_ops"); + html.ShouldContain("Limit Source"); + html.ShouldContain("Override"); + html.ShouldContain("Code default"); + html.ShouldContain("Built-in default"); + html.ShouldNotContain("Edit Limit"); + html.ShouldNotContain("Reset Override"); html.ShouldContain("Saturated"); html.ShouldContain("High Load"); html.ShouldContain("Nominal"); @@ -288,6 +294,19 @@ public async Task ConcurrencyGroups_KnownData_RendersRegistry() AssertShellRefresh(html); } + [Fact] + public async Task ConcurrencyGroups_WithManager_RendersLimitEditActions() + { + await using var app = await CreateStartedDashboardAsync(registerConcurrencyGroupManager: true); + var html = await GetOkHtmlAsync(app, "/sheddueller/concurrency-groups"); + + html.ShouldContain("Actions"); + html.ShouldContain("Edit Limit"); + html.ShouldContain("Reset Override"); + html.ShouldContain("aria-label=\"Edit limit for concurrency group pool_etl_heavy\""); + html.ShouldContain("aria-label=\"Reset limit override for concurrency group pool_etl_heavy\""); + } + [Fact] public async Task Nodes_KnownData_RendersRegistry() { @@ -530,7 +549,8 @@ private static async Task CreateStartedDashboardAsync( bool mapWithWebApplication = true, Action? configureDashboard = null, bool registerMetricsReader = true, - bool registerCleanupSettingsStore = false) + bool registerCleanupSettingsStore = false, + bool registerConcurrencyGroupManager = false) { var builder = WebApplication.CreateBuilder(); builder.WebHost.UseTestServer(); @@ -540,6 +560,11 @@ private static async Task CreateStartedDashboardAsync( builder.Services.AddSingleton(serviceProvider => serviceProvider.GetRequiredService()); builder.Services.AddSingleton(serviceProvider => serviceProvider.GetRequiredService()); builder.Services.AddSingleton(); + if (registerConcurrencyGroupManager) + { + builder.Services.AddSingleton(); + } + builder.Services.AddSingleton(); if (registerMetricsReader) { @@ -1262,18 +1287,25 @@ private sealed class StubConcurrencyGroupInspectionReader : IConcurrencyGroupIns CurrentOccupancy: 50, BlockedJobCount: 12, IsSaturated: true, - UpdatedAtUtc), + UpdatedAtUtc) + { + DefaultLimit = 25, + OverrideLimit = 50, + }, new( "api_sync_workers", EffectiveLimit: 100, CurrentOccupancy: 85, BlockedJobCount: 0, IsSaturated: false, - UpdatedAtUtc.AddMinutes(-1)), + UpdatedAtUtc.AddMinutes(-1)) + { + DefaultLimit = 100, + }, new( "bg_maintenance", - EffectiveLimit: 10, - CurrentOccupancy: 2, + EffectiveLimit: 1, + CurrentOccupancy: 0, BlockedJobCount: 0, IsSaturated: false, UpdatedAtUtc.AddMinutes(-5)), @@ -1312,6 +1344,31 @@ public ValueTask SearchConcurrencyGroupsAsync( => ValueTask.FromResult(null); } + private sealed class StubConcurrencyGroupManager : IConcurrencyGroupManager + { + public ValueTask SetLimitAsync( + string groupKey, + int limit, + CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + + public ValueTask SetDefaultLimitAsync( + string groupKey, + int limit, + CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + + public ValueTask ClearLimitOverrideAsync( + string groupKey, + CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + + public ValueTask GetConfiguredLimitAsync( + string groupKey, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(null); + } + private sealed class StubNodeInspectionReader : INodeInspectionReader { private static readonly DateTimeOffset FirstSeenAtUtc = DateTimeOffset.Parse("2026-04-20T12:00:00Z", CultureInfo.InvariantCulture); diff --git a/test/Sheddueller.Postgres.Tests/Operations/GetConfiguredConcurrencyLimitOperationTests.cs b/test/Sheddueller.Postgres.Tests/Operations/GetConfiguredConcurrencyLimitOperationTests.cs index de58727..506b40d 100644 --- a/test/Sheddueller.Postgres.Tests/Operations/GetConfiguredConcurrencyLimitOperationTests.cs +++ b/test/Sheddueller.Postgres.Tests/Operations/GetConfiguredConcurrencyLimitOperationTests.cs @@ -23,4 +23,14 @@ public async Task GetConfiguredConcurrencyLimit_ConfiguredGroup_ReturnsLimit() (await context.Store.GetConfiguredConcurrencyLimitAsync("shared")).ShouldBe(4); } + + [Fact] + public async Task GetConfiguredConcurrencyLimit_DefaultOnly_ReturnsNull() + { + await using var context = await PostgresTestContext.CreateMigratedAsync(fixture); + + await context.Store.SetConcurrencyDefaultLimitAsync(new SetConcurrencyDefaultLimitRequest("shared", 4, DateTimeOffset.UtcNow)); + + (await context.Store.GetConfiguredConcurrencyLimitAsync("shared")).ShouldBeNull(); + } } diff --git a/test/Sheddueller.Postgres.Tests/Operations/SetConcurrencyLimitOperationTests.cs b/test/Sheddueller.Postgres.Tests/Operations/SetConcurrencyLimitOperationTests.cs index b2fbaac..158450b 100644 --- a/test/Sheddueller.Postgres.Tests/Operations/SetConcurrencyLimitOperationTests.cs +++ b/test/Sheddueller.Postgres.Tests/Operations/SetConcurrencyLimitOperationTests.cs @@ -31,4 +31,36 @@ public async Task SetConcurrencyLimit_ExistingOccupiedGroup_PreservesInUseCount( row.ConfiguredLimit.ShouldBe(2); row.InUseCount.ShouldBe(1); } + + [Fact] + public async Task SetConcurrencyDefaultLimit_ExistingOverride_PreservesOverrideAndSetsEffectiveOverride() + { + await using var context = await PostgresTestContext.CreateMigratedAsync(fixture); + + await context.Store.SetConcurrencyLimitAsync(new SetConcurrencyLimitRequest("shared", 5, DateTimeOffset.UtcNow)); + await context.Store.SetConcurrencyDefaultLimitAsync(new SetConcurrencyDefaultLimitRequest("shared", 2, DateTimeOffset.UtcNow)); + + var row = (await context.ReadConcurrencyGroupAsync("shared")).ShouldNotBeNull(); + row.ConfiguredLimit.ShouldBe(5); + row.DefaultLimit.ShouldBe(2); + row.EffectiveLimit.ShouldBe(5); + } + + [Fact] + public async Task ClearConcurrencyLimitOverride_DefaultLimit_FallsBackToDefaultAndPreservesInUseCount() + { + await using var context = await PostgresTestContext.CreateMigratedAsync(fixture); + await context.Store.SetConcurrencyDefaultLimitAsync(new SetConcurrencyDefaultLimitRequest("shared", 2, DateTimeOffset.UtcNow)); + await context.Store.SetConcurrencyLimitAsync(new SetConcurrencyLimitRequest("shared", 5, DateTimeOffset.UtcNow)); + await context.Store.EnqueueAsync(PostgresTestData.CreateRequest(Guid.NewGuid(), groupKeys: ["shared"])); + await PostgresTestData.ClaimAsync(context.Store); + + await context.Store.ClearConcurrencyLimitOverrideAsync(new ClearConcurrencyLimitOverrideRequest("shared", DateTimeOffset.UtcNow)); + + var row = (await context.ReadConcurrencyGroupAsync("shared")).ShouldNotBeNull(); + row.ConfiguredLimit.ShouldBeNull(); + row.DefaultLimit.ShouldBe(2); + row.EffectiveLimit.ShouldBe(2); + row.InUseCount.ShouldBe(1); + } } diff --git a/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs b/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs index 2159a62..5ffa1eb 100644 --- a/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs +++ b/test/Sheddueller.Postgres.Tests/PostgresMigrationTests.cs @@ -196,6 +196,48 @@ from information_schema.columns .ShouldBeTrue(); } + [Fact] + public async Task Migration_FreshSchema_CreatesConcurrencyEffectiveLimitColumn() + { + await using var context = await PostgresTestContext.CreateMigratedAsync(fixture); + + (await ScalarAsync( + context, + """ + select exists ( + select 1 + from information_schema.columns + where table_schema = @schema_name + and table_name = 'concurrency_groups' + and column_name = 'effective_limit' + and is_generated = 'ALWAYS' + ); + """)) + .ShouldBeTrue(); + + await ExecuteAsync( + context, + $""" + insert into {context.Table("concurrency_groups")} (group_key, configured_limit, default_limit, in_use_count, updated_at_utc) + values ('override', 5, 2, 0, transaction_timestamp()), + ('default', null, 3, 0, transaction_timestamp()), + ('built-in', null, null, 0, transaction_timestamp()); + """); + + (await ScalarAsync( + context, + $"select effective_limit from {context.Table("concurrency_groups")} where group_key = 'override';")) + .ShouldBe(5); + (await ScalarAsync( + context, + $"select effective_limit from {context.Table("concurrency_groups")} where group_key = 'default';")) + .ShouldBe(3); + (await ScalarAsync( + context, + $"select effective_limit from {context.Table("concurrency_groups")} where group_key = 'built-in';")) + .ShouldBe(1); + } + [Fact] public async Task Migration_FreshSchema_CreatesTagOrdinalColumnsAndIndexes() { diff --git a/test/Sheddueller.Postgres.Tests/PostgresTestContext.cs b/test/Sheddueller.Postgres.Tests/PostgresTestContext.cs index a180ce2..82934e3 100644 --- a/test/Sheddueller.Postgres.Tests/PostgresTestContext.cs +++ b/test/Sheddueller.Postgres.Tests/PostgresTestContext.cs @@ -313,7 +313,7 @@ public async ValueTask> ReadScheduleGroupKeysAsync(string { await using var command = this.DataSource.CreateCommand( $""" - select group_key, configured_limit, in_use_count + select group_key, configured_limit, default_limit, effective_limit, in_use_count from {this.Table("concurrency_groups")} where group_key = @group_key; """); @@ -328,7 +328,9 @@ public async ValueTask> ReadScheduleGroupKeysAsync(string return new PostgresConcurrencyGroupRow( reader.GetString(0), reader.IsDBNull(1) ? null : reader.GetInt32(1), - reader.GetInt32(2)); + reader.IsDBNull(2) ? null : reader.GetInt32(2), + reader.GetInt32(3), + reader.GetInt32(4)); } public async ValueTask CountJobsForScheduleAsync(string scheduleKey) @@ -460,4 +462,6 @@ internal sealed record PostgresScheduleRow( internal sealed record PostgresConcurrencyGroupRow( string GroupKey, int? ConfiguredLimit, + int? DefaultLimit, + int EffectiveLimit, int InUseCount); diff --git a/test/Sheddueller.ProviderContracts/InspectionContractTests.cs b/test/Sheddueller.ProviderContracts/InspectionContractTests.cs index c2908b7..c0b5dc7 100644 --- a/test/Sheddueller.ProviderContracts/InspectionContractTests.cs +++ b/test/Sheddueller.ProviderContracts/InspectionContractTests.cs @@ -595,12 +595,36 @@ public async Task ConcurrencyGroupView_SaturatedGroup_ShowsClaimedAndBlockedJobs saturatedPage.TotalCount.ShouldBe(1L); detail.ShouldNotBeNull(); detail.Summary.EffectiveLimit.ShouldBe(1); + detail.Summary.DefaultLimit.ShouldBeNull(); + detail.Summary.OverrideLimit.ShouldBeNull(); detail.Summary.CurrentOccupancy.ShouldBe(1); detail.Summary.IsSaturated.ShouldBeTrue(); detail.ClaimedJobIds.ShouldBe([running]); detail.BlockedJobIds.ShouldBe([blocked]); } + [Fact] + public async Task ConcurrencyGroupView_DefaultAndOverrideLimits_AreVisible() + { + await using var context = await this.CreateContextAsync(); + + await context.Store.SetConcurrencyDefaultLimitAsync(new SetConcurrencyDefaultLimitRequest("api", 2, DateTimeOffset.UtcNow)); + await context.Store.SetConcurrencyDefaultLimitAsync(new SetConcurrencyDefaultLimitRequest("etl", 3, DateTimeOffset.UtcNow)); + await context.Store.SetConcurrencyLimitAsync(new SetConcurrencyLimitRequest("etl", 5, DateTimeOffset.UtcNow)); + + var api = await context.ConcurrencyGroupReader.GetConcurrencyGroupAsync("api"); + var etl = await context.ConcurrencyGroupReader.GetConcurrencyGroupAsync("etl"); + + api.ShouldNotBeNull().Summary.ShouldSatisfyAllConditions( + summary => summary.DefaultLimit.ShouldBe(2), + summary => summary.OverrideLimit.ShouldBeNull(), + summary => summary.EffectiveLimit.ShouldBe(2)); + etl.ShouldNotBeNull().Summary.ShouldSatisfyAllConditions( + summary => summary.DefaultLimit.ShouldBe(3), + summary => summary.OverrideLimit.ShouldBe(5), + summary => summary.EffectiveLimit.ShouldBe(5)); + } + [Fact] public async Task ConcurrencyGroupSearch_GroupKeyFilter_IsExactAndReturnsTotalCount() { diff --git a/test/Sheddueller.ProviderContracts/JobStoreContractTests.cs b/test/Sheddueller.ProviderContracts/JobStoreContractTests.cs index 1f30405..5f7038a 100644 --- a/test/Sheddueller.ProviderContracts/JobStoreContractTests.cs +++ b/test/Sheddueller.ProviderContracts/JobStoreContractTests.cs @@ -576,6 +576,43 @@ public async Task ConcurrencyLimit_SetAndGet_RoundTripsConfiguredLimit() (await context.Store.GetConfiguredConcurrencyLimitAsync("shared")).ShouldBe(5); } + [Fact] + public async Task ConcurrencyLimit_DefaultOverrideAndClear_UsesEffectivePrecedence() + { + await using var context = await this.CreateContextAsync(); + var first = Guid.NewGuid(); + var second = Guid.NewGuid(); + var third = Guid.NewGuid(); + var fourth = Guid.NewGuid(); + + await context.Store.SetConcurrencyDefaultLimitAsync(new SetConcurrencyDefaultLimitRequest("shared", 2, ContractClock)); + (await context.Store.GetConfiguredConcurrencyLimitAsync("shared")).ShouldBeNull(); + + await context.Store.EnqueueAsync(CreateRequest(first, groupKeys: ["shared"])); + await context.Store.EnqueueAsync(CreateRequest(second, groupKeys: ["shared"])); + await context.Store.EnqueueAsync(CreateRequest(third, groupKeys: ["shared"])); + + var firstClaim = await ClaimAsync(context.Store); + firstClaim.JobId.ShouldBe(first); + var secondClaim = await ClaimAsync(context.Store); + secondClaim.JobId.ShouldBe(second); + (await context.Store.TryClaimNextAsync(CreateClaimRequest("node-1"))).ShouldBeOfType(); + + await context.Store.SetConcurrencyLimitAsync(new SetConcurrencyLimitRequest("shared", 3, ContractClock)); + (await ClaimAsync(context.Store)).JobId.ShouldBe(third); + + await context.Store.EnqueueAsync(CreateRequest(fourth, groupKeys: ["shared"])); + await context.Store.ClearConcurrencyLimitOverrideAsync(new ClearConcurrencyLimitOverrideRequest("shared", ContractClock)); + (await context.Store.GetConfiguredConcurrencyLimitAsync("shared")).ShouldBeNull(); + (await context.Store.TryClaimNextAsync(CreateClaimRequest("node-1"))).ShouldBeOfType(); + + (await context.Store.MarkCompletedAsync(new CompleteJobRequest(first, "node-1", firstClaim.LeaseToken, ContractClock))).ShouldBeTrue(); + (await context.Store.TryClaimNextAsync(CreateClaimRequest("node-1"))).ShouldBeOfType(); + + (await context.Store.MarkCompletedAsync(new CompleteJobRequest(second, "node-1", secondClaim.LeaseToken, ContractClock))).ShouldBeTrue(); + (await ClaimAsync(context.Store)).JobId.ShouldBe(fourth); + } + [Fact] public async Task RecurringSchedule_Lifecycle_CreateUpdatePauseResumeListDelete() { diff --git a/test/Sheddueller.Tests/ConcurrencyGroupManagerTests.cs b/test/Sheddueller.Tests/ConcurrencyGroupManagerTests.cs new file mode 100644 index 0000000..18f4367 --- /dev/null +++ b/test/Sheddueller.Tests/ConcurrencyGroupManagerTests.cs @@ -0,0 +1,73 @@ +namespace Sheddueller.Tests; + +using Microsoft.Extensions.DependencyInjection; + +using Sheddueller.Storage; + +using Shouldly; + +public sealed class ConcurrencyGroupManagerTests +{ + [Fact] + public async Task SetLimit_ValidGroup_PersistsOverrideLimit() + { + using var provider = CreateProvider(); + var store = provider.GetRequiredService(); + var manager = provider.GetRequiredService(); + + await manager.SetLimitAsync("api", 4); + + store.ConcurrencyLimitRequests.ShouldHaveSingleItem().ShouldSatisfyAllConditions( + request => request.GroupKey.ShouldBe("api"), + request => request.Limit.ShouldBe(4)); + } + + [Fact] + public async Task SetDefaultLimit_ValidGroup_PersistsDefaultLimit() + { + using var provider = CreateProvider(); + var store = provider.GetRequiredService(); + var manager = provider.GetRequiredService(); + + await manager.SetDefaultLimitAsync("api", 3); + + store.ConcurrencyDefaultLimitRequests.ShouldHaveSingleItem().ShouldSatisfyAllConditions( + request => request.GroupKey.ShouldBe("api"), + request => request.Limit.ShouldBe(3)); + store.ConcurrencyLimitRequests.ShouldBeEmpty(); + } + + [Fact] + public async Task ClearLimitOverride_ValidGroup_PersistsClearRequest() + { + using var provider = CreateProvider(); + var store = provider.GetRequiredService(); + var manager = provider.GetRequiredService(); + + await manager.ClearLimitOverrideAsync("api"); + + store.ClearConcurrencyLimitOverrideRequests.ShouldHaveSingleItem().GroupKey.ShouldBe("api"); + } + + [Fact] + public async Task SetLimit_NonPositiveLimit_DoesNotPersist() + { + using var provider = CreateProvider(); + var store = provider.GetRequiredService(); + var manager = provider.GetRequiredService(); + + await Should.ThrowAsync(async () => await manager.SetLimitAsync("api", 0)); + + store.ConcurrencyLimitRequests.ShouldBeEmpty(); + } + + private static ServiceProvider CreateProvider() + { + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddSingleton(serviceProvider => serviceProvider.GetRequiredService()); + services.AddSheddueller(); + + return services.BuildServiceProvider(); + } +} diff --git a/test/Sheddueller.Tests/RecordingJobStore.cs b/test/Sheddueller.Tests/RecordingJobStore.cs index b9b86f6..b385ac5 100644 --- a/test/Sheddueller.Tests/RecordingJobStore.cs +++ b/test/Sheddueller.Tests/RecordingJobStore.cs @@ -8,6 +8,9 @@ internal sealed class RecordingJobStore : IJobStore private readonly List recurringScheduleRequests = []; private readonly List triggerRequests = []; private readonly List cancelQueuedJobsRequests = []; + private readonly List concurrencyLimitRequests = []; + private readonly List concurrencyDefaultLimitRequests = []; + private readonly List clearConcurrencyLimitOverrideRequests = []; private long nextSequence; public IReadOnlyList EnqueuedRequests => this.enqueuedRequests; @@ -18,6 +21,12 @@ internal sealed class RecordingJobStore : IJobStore public IReadOnlyList CancelQueuedJobsRequests => this.cancelQueuedJobsRequests; + public IReadOnlyList ConcurrencyLimitRequests => this.concurrencyLimitRequests; + + public IReadOnlyList ConcurrencyDefaultLimitRequests => this.concurrencyDefaultLimitRequests; + + public IReadOnlyList ClearConcurrencyLimitOverrideRequests => this.clearConcurrencyLimitOverrideRequests; + public RecurringScheduleUpsertResult CreateOrUpdateRecurringScheduleResult { get; set; } = RecurringScheduleUpsertResult.Created; public RecurringScheduleTriggerResult TriggerResult { get; set; } = new(RecurringScheduleTriggerStatus.NotFound); @@ -118,7 +127,32 @@ public ValueTask RecordWorkerNodeHeartbeatAsync( public ValueTask SetConcurrencyLimitAsync( SetConcurrencyLimitRequest request, CancellationToken cancellationToken = default) - => throw CreateUnsupportedException(); + { + cancellationToken.ThrowIfCancellationRequested(); + this.concurrencyLimitRequests.Add(request); + + return ValueTask.CompletedTask; + } + + public ValueTask SetConcurrencyDefaultLimitAsync( + SetConcurrencyDefaultLimitRequest request, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + this.concurrencyDefaultLimitRequests.Add(request); + + return ValueTask.CompletedTask; + } + + public ValueTask ClearConcurrencyLimitOverrideAsync( + ClearConcurrencyLimitOverrideRequest request, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + this.clearConcurrencyLimitOverrideRequests.Add(request); + + return ValueTask.CompletedTask; + } public ValueTask GetConfiguredConcurrencyLimitAsync( string groupKey, diff --git a/test/Sheddueller.Worker.Tests/RegistrationTests.cs b/test/Sheddueller.Worker.Tests/RegistrationTests.cs index 0572e53..9db14ff 100644 --- a/test/Sheddueller.Worker.Tests/RegistrationTests.cs +++ b/test/Sheddueller.Worker.Tests/RegistrationTests.cs @@ -166,6 +166,16 @@ public ValueTask SetConcurrencyLimitAsync( CancellationToken cancellationToken = default) => ValueTask.CompletedTask; + public ValueTask SetConcurrencyDefaultLimitAsync( + SetConcurrencyDefaultLimitRequest request, + CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + + public ValueTask ClearConcurrencyLimitOverrideAsync( + ClearConcurrencyLimitOverrideRequest request, + CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + public ValueTask GetConfiguredConcurrencyLimitAsync( string groupKey, CancellationToken cancellationToken = default) diff --git a/test/Sheddueller.Worker.Tests/WorkerJobLoggerTests.cs b/test/Sheddueller.Worker.Tests/WorkerJobLoggerTests.cs index b37cc32..0ad9e9c 100644 --- a/test/Sheddueller.Worker.Tests/WorkerJobLoggerTests.cs +++ b/test/Sheddueller.Worker.Tests/WorkerJobLoggerTests.cs @@ -562,6 +562,16 @@ public ValueTask SetConcurrencyLimitAsync( CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public ValueTask SetConcurrencyDefaultLimitAsync( + SetConcurrencyDefaultLimitRequest request, + CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public ValueTask ClearConcurrencyLimitOverrideAsync( + ClearConcurrencyLimitOverrideRequest request, + CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + public ValueTask GetConfiguredConcurrencyLimitAsync( string groupKey, CancellationToken cancellationToken = default) diff --git a/test/Sheddueller.Worker.Tests/WorkerLoggingTests.cs b/test/Sheddueller.Worker.Tests/WorkerLoggingTests.cs index 5e5773d..8bc686a 100644 --- a/test/Sheddueller.Worker.Tests/WorkerLoggingTests.cs +++ b/test/Sheddueller.Worker.Tests/WorkerLoggingTests.cs @@ -162,6 +162,16 @@ public ValueTask SetConcurrencyLimitAsync( CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public ValueTask SetConcurrencyDefaultLimitAsync( + SetConcurrencyDefaultLimitRequest request, + CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public ValueTask ClearConcurrencyLimitOverrideAsync( + ClearConcurrencyLimitOverrideRequest request, + CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + public ValueTask GetConfiguredConcurrencyLimitAsync( string groupKey, CancellationToken cancellationToken = default) diff --git a/test/Sheddueller.Worker.Tests/WorkerProgressTests.cs b/test/Sheddueller.Worker.Tests/WorkerProgressTests.cs index 1dc890c..2774ebc 100644 --- a/test/Sheddueller.Worker.Tests/WorkerProgressTests.cs +++ b/test/Sheddueller.Worker.Tests/WorkerProgressTests.cs @@ -242,6 +242,16 @@ public ValueTask SetConcurrencyLimitAsync( CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public ValueTask SetConcurrencyDefaultLimitAsync( + SetConcurrencyDefaultLimitRequest request, + CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public ValueTask ClearConcurrencyLimitOverrideAsync( + ClearConcurrencyLimitOverrideRequest request, + CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + public ValueTask GetConfiguredConcurrencyLimitAsync( string groupKey, CancellationToken cancellationToken = default)