From db7739746bfccf2d5742f2de7bb51110fd950f94 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Wed, 29 Jul 2026 22:27:04 +0200 Subject: [PATCH 1/4] Keep the fwdata cache alive during long syncs The LcmCache's 30-minute sliding expiration only resets when a new FwDataMiniLcmApi first touches the memory cache, so a sync that runs longer than that can have the cache disposed out from under it. FwDataFactory.PreventEviction returns a disposable timer that touches the cache entry every 5 minutes; the sync service holds one per sync. Co-Authored-By: Claude Fable 5 --- backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs | 11 +++++++++++ .../FwLiteProjectSync/CrdtFwdataProjectSyncService.cs | 7 ++++++- .../FwHeadless/Services/SyncWorkerTestHarness.cs | 1 + 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs b/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs index 3691bede09..315ea49d2a 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs @@ -131,6 +131,17 @@ public IAsyncDisposable DeferCloseAsync(FwDataProject project) return Defer.Async(() => CloseProjectAsync(project)); } + /// + /// Keeps the project's LcmCache from being evicted until disposed, by periodically resetting its sliding expiration. + /// Use around long-running work that holds one api instance past the expiration window (e.g. a sync paused in a debugger). + /// + public IDisposable PreventEviction(FwDataProject project) + { + var key = CacheKey(project); + var period = TimeSpan.FromMinutes(5); + return new Timer(_ => cache.TryGetValue(key, out _), null, period, period); + } + public Task StartAsync(CancellationToken cancellationToken) { return Task.CompletedTask; diff --git a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs index 588803caba..dbbd64def7 100644 --- a/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs +++ b/backend/FwLite/FwLiteProjectSync/CrdtFwdataProjectSyncService.cs @@ -1,4 +1,5 @@ using System.Diagnostics; +using FwDataMiniLcmBridge; using FwDataMiniLcmBridge.Api; using LcmCrdt; using LexCore.Sync; @@ -11,7 +12,8 @@ namespace FwLiteProjectSync; public class CrdtFwdataProjectSyncService(MiniLcmImport miniLcmImport, ILogger logger, - MiniLcmApiValidationWrapperFactory validationWrapperFactory) + MiniLcmApiValidationWrapperFactory validationWrapperFactory, + FwDataFactory fwDataFactory) { public record DryRunSyncResult( int CrdtChanges, @@ -52,6 +54,9 @@ private async Task SyncOrImportInternal(IMiniLcmApi crdtApi, IMiniLc throw new InvalidOperationException($"Project id mismatch, CRDT Id: {crdt.ProjectData.FwProjectId}, FWData Id: {fwdata.ProjectId}"); } + // A sync that outlives the LcmCache's sliding expiration (e.g. paused in a debugger) would otherwise have it disposed mid-sync. + using var keepFwdataAlive = fwDataFactory.PreventEviction(fwdata.Project); + // Project snapshot logic/handling is done outside of this class so that Sync vs Import is explicit. // We still choose to explicitly verify a consistent state to avoid accidental misuse. var hasSyncedSuccessfully = ProjectSnapshotService.HasSyncedSuccessfully(fwdata.Project); diff --git a/backend/Testing/FwHeadless/Services/SyncWorkerTestHarness.cs b/backend/Testing/FwHeadless/Services/SyncWorkerTestHarness.cs index 2737455254..0f4567fdfc 100644 --- a/backend/Testing/FwHeadless/Services/SyncWorkerTestHarness.cs +++ b/backend/Testing/FwHeadless/Services/SyncWorkerTestHarness.cs @@ -252,6 +252,7 @@ private ServiceProvider BuildServiceProvider( MockBehavior.Strict, null!, NullLogger.Instance, + null!, null!); syncService From d4624badd93d97ba18ff575be07306c3b6f754da Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Thu, 30 Jul 2026 11:01:01 +0200 Subject: [PATCH 2/4] Add fwDataFactory ctor arg to sync service mock CrdtFwdataProjectSyncService gained a FwDataFactory constructor parameter; the SyncWorker test harness mock was still passing four args, so Moq could not find a matching constructor and every SyncWorkerTests case failed. Co-Authored-By: Claude Fable 5 --- backend/Testing/FwHeadless/Services/SyncWorkerTestHarness.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/Testing/FwHeadless/Services/SyncWorkerTestHarness.cs b/backend/Testing/FwHeadless/Services/SyncWorkerTestHarness.cs index 0f4567fdfc..2f6205b511 100644 --- a/backend/Testing/FwHeadless/Services/SyncWorkerTestHarness.cs +++ b/backend/Testing/FwHeadless/Services/SyncWorkerTestHarness.cs @@ -253,6 +253,7 @@ private ServiceProvider BuildServiceProvider( null!, NullLogger.Instance, null!, + null!, null!); syncService From 557b03e1672bfac5f2ccaea14ce34476ddcb3b56 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Thu, 30 Jul 2026 12:45:58 +0200 Subject: [PATCH 3/4] Make the cache keepalive refresh immediately and stay coupled to the expiration PreventEviction waited a full period before its first touch, so an entry with little of its 30-minute window left could be evicted before the first refresh. The refresh period was also an independent magic number that could silently exceed the sliding expiration if either changed. Fire the timer immediately (TimeSpan.Zero due-time), and derive the period from a shared CacheSlidingExpiration constant (half the window) so a refresh is guaranteed to land inside every window by construction. Co-Authored-By: Claude Fable 5 --- backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs b/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs index 315ea49d2a..58bfc4cd0f 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs @@ -20,6 +20,10 @@ public class FwDataFactory( IOptions config) : IDisposable, IHostedService { private bool _shuttingDown = false; + + // Sliding window before an idle LcmCache is evicted and disposed. PreventEviction refreshes inside this window. + private static readonly TimeSpan CacheSlidingExpiration = TimeSpan.FromMinutes(30); + public FwDataFactory(ILogger fwdataLogger, IMemoryCache cache, ILogger logger, @@ -51,7 +55,7 @@ private LcmCache GetProjectServiceCached(FwDataProject project) var projectService = cache.GetOrCreate(key, entry => { - entry.SlidingExpiration = TimeSpan.FromMinutes(30); + entry.SlidingExpiration = CacheSlidingExpiration; entry.RegisterPostEvictionCallback(OnLcmProjectCacheEviction, (logger, _projectCacheKeys)); logger.LogInformation("Loading project {ProjectFileName}", project.FileName); var projectService = projectLoader.LoadCache(project); @@ -138,8 +142,9 @@ public IAsyncDisposable DeferCloseAsync(FwDataProject project) public IDisposable PreventEviction(FwDataProject project) { var key = CacheKey(project); - var period = TimeSpan.FromMinutes(5); - return new Timer(_ => cache.TryGetValue(key, out _), null, period, period); + // Refresh now in case little of the window remains, then every half-window so a tick can't be missed. + var period = CacheSlidingExpiration / 2; + return new Timer(_ => cache.TryGetValue(key, out _), null, TimeSpan.Zero, period); } public Task StartAsync(CancellationToken cancellationToken) From 850aab12723b2e9e6ae73216343fc81ebe20cf75 Mon Sep 17 00:00:00 2001 From: Tim Haasdyk Date: Thu, 30 Jul 2026 12:51:21 +0200 Subject: [PATCH 4/4] Guard the keepalive tick against a disposed cache on shutdown The timer's cache lookup runs on a ThreadPool thread. If the shared MemoryCache is disposed during host shutdown while a sync is still finishing, TryGetValue throws ObjectDisposedException, and an unhandled throw there crashes the process. Swallow it so the keepalive stays best-effort, matching how a missing entry is already tolerated. Co-Authored-By: Claude Fable 5 --- backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs b/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs index 58bfc4cd0f..8dff6914bc 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/FwDataFactory.cs @@ -144,7 +144,13 @@ public IDisposable PreventEviction(FwDataProject project) var key = CacheKey(project); // Refresh now in case little of the window remains, then every half-window so a tick can't be missed. var period = CacheSlidingExpiration / 2; - return new Timer(_ => cache.TryGetValue(key, out _), null, TimeSpan.Zero, period); + return new Timer(_ => + { + // Best-effort: the shared cache can be disposed during shutdown while a sync is still finishing, + // and an unhandled throw on this timer thread would take the process down. + try { cache.TryGetValue(key, out _); } + catch (ObjectDisposedException) { } + }, null, TimeSpan.Zero, period); } public Task StartAsync(CancellationToken cancellationToken)