Master-poll: fail-open telt vanaf het laatste contact van de master (poll, push, registratie), niet alleen vanaf de laatste geslaagde poll
Een push 'niet beschikbaar' werd bij een al langer falende poll binnen 30 s door de volgende mislukte poll teruggezet. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
+48
-3
@@ -341,6 +341,7 @@ public class MasterAvailabilityServiceTests : IDisposable
|
||||
_repo.GetAsync().Returns(existing);
|
||||
_protector.Unprotect("enc:key").Returns("key");
|
||||
await _svc.PushStatusAsync("key", false, "Down");
|
||||
existing.LastContactedAt = DateTimeOffset.UtcNow.AddMinutes(-10); // the push was a while ago: no contact since
|
||||
|
||||
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||
|
||||
@@ -349,6 +350,46 @@ public class MasterAvailabilityServiceTests : IDisposable
|
||||
status.DisableMessage.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task APushJustReceived_IsNotUndoneByTheNextFailedPoll_EvenWhenPollingHasBeenFailingForHours()
|
||||
{
|
||||
// The ASF case: polling never succeeds (LastPolledAt old), the master pushes "not available" ...
|
||||
var existing = RegisteredLongAgo();
|
||||
existing.LastPolledAt = null;
|
||||
await _svc.PushStatusAsync("key", false, "Onderhoud");
|
||||
|
||||
// ... and the very next poll (seconds later) fails.
|
||||
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||
|
||||
var status = _svc.GetMasterStatus();
|
||||
status.IsAvailable.Should().BeFalse();
|
||||
status.DisableMessage.Should().Be("Onderhoud");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WithoutAnyContactForTheGracePeriod_ThePushedStatusStillFailsOpen()
|
||||
{
|
||||
var existing = RegisteredLongAgo();
|
||||
existing.LastPolledAt = null;
|
||||
await _svc.PushStatusAsync("key", false, "Onderhoud");
|
||||
|
||||
existing.LastContactedAt = DateTimeOffset.UtcNow.AddMinutes(-5).AddSeconds(-1); // 5 minutes of silence
|
||||
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||
_svc.GetMasterStatus().IsAvailable.Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task JustBeforeTheGracePeriodEnds_TheStatusStaysUnavailable()
|
||||
{
|
||||
var existing = RegisteredLongAgo();
|
||||
existing.LastPolledAt = null;
|
||||
await _svc.PushStatusAsync("key", false, "Onderhoud");
|
||||
|
||||
existing.LastContactedAt = DateTimeOffset.UtcNow.AddMinutes(-4);
|
||||
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||
_svc.GetMasterStatus().IsAvailable.Should().BeFalse();
|
||||
}
|
||||
|
||||
private MasterRegistration RegisteredLongAgo()
|
||||
{
|
||||
var existing = new MasterRegistration
|
||||
@@ -367,8 +408,9 @@ public class MasterAvailabilityServiceTests : IDisposable
|
||||
[Fact]
|
||||
public async Task AfterAFailOpen_ASuccessfulPoll_AppliesTheMasterNotAvailableAgainAtOnce()
|
||||
{
|
||||
RegisteredLongAgo();
|
||||
var reg = RegisteredLongAgo();
|
||||
await _svc.PushStatusAsync("key", false, "Onderhoud");
|
||||
reg.LastContactedAt = DateTimeOffset.UtcNow.AddMinutes(-10); // silence since the push
|
||||
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||
_svc.GetMasterStatus().IsAvailable.Should().BeTrue(); // failed open
|
||||
|
||||
@@ -382,8 +424,9 @@ public class MasterAvailabilityServiceTests : IDisposable
|
||||
[Fact]
|
||||
public async Task AfterAFailOpen_APushAppliesTheMasterNotAvailableAgainAtOnce()
|
||||
{
|
||||
RegisteredLongAgo();
|
||||
var reg = RegisteredLongAgo();
|
||||
await _svc.PushStatusAsync("key", false, "Onderhoud");
|
||||
reg.LastContactedAt = DateTimeOffset.UtcNow.AddMinutes(-10); // silence since the push
|
||||
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||
|
||||
(await _svc.PushStatusAsync("key", false, "Onderhoud")).Should().BeTrue();
|
||||
@@ -394,8 +437,9 @@ public class MasterAvailabilityServiceTests : IDisposable
|
||||
[Fact]
|
||||
public async Task AfterTheRecovery_FurtherFailedPollsDoNotFailOpenAgainWithinTheGracePeriod()
|
||||
{
|
||||
RegisteredLongAgo();
|
||||
var reg = RegisteredLongAgo();
|
||||
await _svc.PushStatusAsync("key", false, "Onderhoud");
|
||||
reg.LastContactedAt = DateTimeOffset.UtcNow.AddMinutes(-10); // silence since the push
|
||||
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||
await _svc.ApplyPolledStatusAsync(false, "Onderhoud"); // sets LastPolledAt = now
|
||||
|
||||
@@ -419,6 +463,7 @@ public class MasterAvailabilityServiceTests : IDisposable
|
||||
|
||||
// the other way round: a slave that failed open stays Available until the master says otherwise
|
||||
existing.LastPolledAt = DateTimeOffset.UtcNow.AddMinutes(-10);
|
||||
existing.LastContactedAt = DateTimeOffset.UtcNow.AddMinutes(-10); // silence since the re-registration
|
||||
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||
await _svc.RegisterAsync("https://master.example.com", "key");
|
||||
_svc.GetMasterStatus().IsAvailable.Should().BeTrue();
|
||||
|
||||
@@ -102,7 +102,12 @@ public class MasterAvailabilityService : IMasterAvailabilityService
|
||||
var existing = await _deps.Repository.GetAsync();
|
||||
if (existing is null) return;
|
||||
|
||||
var unreachableSince = existing.LastPolledAt ?? existing.RegisteredAt;
|
||||
// The grace period runs from the last sign of life of the master: a successful poll, but also a push,
|
||||
// a re-registration or the hourly integrity check (they all set LastContactedAt). Measuring from the
|
||||
// last successful poll alone made a freshly pushed "not available" fall open at the next failed poll
|
||||
// whenever polling had been failing for a while, although the master had just spoken to us.
|
||||
var unreachableSince = new[] { existing.LastPolledAt, existing.LastContactedAt, existing.RegisteredAt }
|
||||
.Where(t => t is not null).Max()!.Value;
|
||||
if (DateTimeOffset.UtcNow - unreachableSince < failOpenAfter)
|
||||
return;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user