From e17d967b3fa8f1457a912d1602138ffdf3ecd6c4 Mon Sep 17 00:00:00 2001 From: Sluijsens Date: Mon, 9 Nov 2026 19:50:36 +0100 Subject: [PATCH] Master-poll: fail-open telt vanaf het laatste contact van de master (poll, push, registratie), niet alleen vanaf de laatste geslaagde poll Een push 'niet beschikbaar' werd bij een al langer falende poll binnen 30 s door de volgende mislukte poll teruggezet. Co-Authored-By: Claude Sonnet 5.5 --- .../MasterAvailabilityServiceTests.cs | 51 +++++++++++++++++-- .../Services/MasterAvailabilityService.cs | 7 ++- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/src/SlpModularCms.Modules.Availability.Tests/Services/MasterAvailabilityServiceTests.cs b/src/SlpModularCms.Modules.Availability.Tests/Services/MasterAvailabilityServiceTests.cs index 1e04e84..4220106 100644 --- a/src/SlpModularCms.Modules.Availability.Tests/Services/MasterAvailabilityServiceTests.cs +++ b/src/SlpModularCms.Modules.Availability.Tests/Services/MasterAvailabilityServiceTests.cs @@ -341,6 +341,7 @@ public class MasterAvailabilityServiceTests : IDisposable _repo.GetAsync().Returns(existing); _protector.Unprotect("enc:key").Returns("key"); await _svc.PushStatusAsync("key", false, "Down"); + existing.LastContactedAt = DateTimeOffset.UtcNow.AddMinutes(-10); // the push was a while ago: no contact since await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5)); @@ -349,6 +350,46 @@ public class MasterAvailabilityServiceTests : IDisposable status.DisableMessage.Should().BeNull(); } + [Fact] + public async Task APushJustReceived_IsNotUndoneByTheNextFailedPoll_EvenWhenPollingHasBeenFailingForHours() + { + // The ASF case: polling never succeeds (LastPolledAt old), the master pushes "not available" ... + var existing = RegisteredLongAgo(); + existing.LastPolledAt = null; + await _svc.PushStatusAsync("key", false, "Onderhoud"); + + // ... and the very next poll (seconds later) fails. + await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5)); + + var status = _svc.GetMasterStatus(); + status.IsAvailable.Should().BeFalse(); + status.DisableMessage.Should().Be("Onderhoud"); + } + + [Fact] + public async Task WithoutAnyContactForTheGracePeriod_ThePushedStatusStillFailsOpen() + { + var existing = RegisteredLongAgo(); + existing.LastPolledAt = null; + await _svc.PushStatusAsync("key", false, "Onderhoud"); + + existing.LastContactedAt = DateTimeOffset.UtcNow.AddMinutes(-5).AddSeconds(-1); // 5 minutes of silence + await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5)); + _svc.GetMasterStatus().IsAvailable.Should().BeTrue(); + } + + [Fact] + public async Task JustBeforeTheGracePeriodEnds_TheStatusStaysUnavailable() + { + var existing = RegisteredLongAgo(); + existing.LastPolledAt = null; + await _svc.PushStatusAsync("key", false, "Onderhoud"); + + existing.LastContactedAt = DateTimeOffset.UtcNow.AddMinutes(-4); + await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5)); + _svc.GetMasterStatus().IsAvailable.Should().BeFalse(); + } + private MasterRegistration RegisteredLongAgo() { var existing = new MasterRegistration @@ -367,8 +408,9 @@ public class MasterAvailabilityServiceTests : IDisposable [Fact] public async Task AfterAFailOpen_ASuccessfulPoll_AppliesTheMasterNotAvailableAgainAtOnce() { - RegisteredLongAgo(); + var reg = RegisteredLongAgo(); await _svc.PushStatusAsync("key", false, "Onderhoud"); + reg.LastContactedAt = DateTimeOffset.UtcNow.AddMinutes(-10); // silence since the push await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5)); _svc.GetMasterStatus().IsAvailable.Should().BeTrue(); // failed open @@ -382,8 +424,9 @@ public class MasterAvailabilityServiceTests : IDisposable [Fact] public async Task AfterAFailOpen_APushAppliesTheMasterNotAvailableAgainAtOnce() { - RegisteredLongAgo(); + var reg = RegisteredLongAgo(); await _svc.PushStatusAsync("key", false, "Onderhoud"); + reg.LastContactedAt = DateTimeOffset.UtcNow.AddMinutes(-10); // silence since the push await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5)); (await _svc.PushStatusAsync("key", false, "Onderhoud")).Should().BeTrue(); @@ -394,8 +437,9 @@ public class MasterAvailabilityServiceTests : IDisposable [Fact] public async Task AfterTheRecovery_FurtherFailedPollsDoNotFailOpenAgainWithinTheGracePeriod() { - RegisteredLongAgo(); + var reg = RegisteredLongAgo(); await _svc.PushStatusAsync("key", false, "Onderhoud"); + reg.LastContactedAt = DateTimeOffset.UtcNow.AddMinutes(-10); // silence since the push await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5)); await _svc.ApplyPolledStatusAsync(false, "Onderhoud"); // sets LastPolledAt = now @@ -419,6 +463,7 @@ public class MasterAvailabilityServiceTests : IDisposable // the other way round: a slave that failed open stays Available until the master says otherwise existing.LastPolledAt = DateTimeOffset.UtcNow.AddMinutes(-10); + existing.LastContactedAt = DateTimeOffset.UtcNow.AddMinutes(-10); // silence since the re-registration await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5)); await _svc.RegisterAsync("https://master.example.com", "key"); _svc.GetMasterStatus().IsAvailable.Should().BeTrue(); diff --git a/src/SlpModularCms.Modules.Availability/Services/MasterAvailabilityService.cs b/src/SlpModularCms.Modules.Availability/Services/MasterAvailabilityService.cs index fa6f4d2..0e415fc 100644 --- a/src/SlpModularCms.Modules.Availability/Services/MasterAvailabilityService.cs +++ b/src/SlpModularCms.Modules.Availability/Services/MasterAvailabilityService.cs @@ -102,7 +102,12 @@ public class MasterAvailabilityService : IMasterAvailabilityService var existing = await _deps.Repository.GetAsync(); if (existing is null) return; - var unreachableSince = existing.LastPolledAt ?? existing.RegisteredAt; + // The grace period runs from the last sign of life of the master: a successful poll, but also a push, + // a re-registration or the hourly integrity check (they all set LastContactedAt). Measuring from the + // last successful poll alone made a freshly pushed "not available" fall open at the next failed poll + // whenever polling had been failing for a while, although the master had just spoken to us. + var unreachableSince = new[] { existing.LastPolledAt, existing.LastContactedAt, existing.RegisteredAt } + .Where(t => t is not null).Max()!.Value; if (DateTimeOffset.UtcNow - unreachableSince < failOpenAfter) return;