Master-poll: fouten zichtbaar loggen (Warning bij eerste fout, Information bij herstel); tests dat master-status na fail-open direct terugkomt

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-09 16:25:19 +01:00
co-authored by Claude Sonnet 5.5
parent a9b9ad2931
commit 6798102b6b
6 changed files with 190 additions and 3 deletions
@@ -70,10 +70,58 @@ public class MasterStatusPollingBackgroundServiceTests
await act.Should().NotThrowAsync();
}
private sealed class LogCapture : ILogger<MasterStatusPollingBackgroundService>
{
public List<(LogLevel Level, string Message)> Entries { get; } = [];
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) =>
Entries.Add((logLevel, formatter(state, exception)));
}
[Fact]
public async Task APollFailure_IsWarnedOnce_WithTheReason_AndRecoveryIsInformation()
{
var masterAvailabilityService = Substitute.For<IMasterAvailabilityService>();
masterAvailabilityService.GetPollTargetAsync().Returns(new MasterPollTarget("https://secret-master.example", "the-api-key"));
var pollClient = Substitute.For<IMasterStatusPollClient>();
pollClient.GetStatusAsync("https://secret-master.example", "the-api-key").Returns(
(PolledMasterStatus?)null, (PolledMasterStatus?)null, (PolledMasterStatus?)null, new PolledMasterStatus(false, "Onderhoud"));
pollClient.LastFailure.Returns("http_401");
var logs = new LogCapture();
var sut = CreateSut(masterAvailabilityService, pollClient, logger: logs);
for (var i = 0; i < 4; i++) await sut.ExecuteTickAsync(CancellationToken.None);
logs.Entries.Should().HaveCount(2); // not one line per failed poll
logs.Entries[0].Level.Should().Be(LogLevel.Warning);
logs.Entries[0].Message.Should().Contain("master_poll_failed").And.Contain("http_401");
logs.Entries[1].Level.Should().Be(LogLevel.Information);
logs.Entries[1].Message.Should().Contain("master_poll_recovered").And.Contain("3");
logs.Entries.Select(e => e.Message).Should().AllSatisfy(m => m.Should().NotContain("the-api-key").And.NotContain("secret-master"));
}
[Fact]
public async Task ANewFailureAfterRecovery_IsWarnedAgain()
{
var masterAvailabilityService = Substitute.For<IMasterAvailabilityService>();
masterAvailabilityService.GetPollTargetAsync().Returns(new MasterPollTarget("https://m", "k"));
var pollClient = Substitute.For<IMasterStatusPollClient>();
pollClient.GetStatusAsync("https://m", "k").Returns(
(PolledMasterStatus?)null, new PolledMasterStatus(true, null), (PolledMasterStatus?)null);
var logs = new LogCapture();
var sut = CreateSut(masterAvailabilityService, pollClient, logger: logs);
for (var i = 0; i < 3; i++) await sut.ExecuteTickAsync(CancellationToken.None);
logs.Entries.Select(e => e.Level).Should().Equal(LogLevel.Warning, LogLevel.Information, LogLevel.Warning);
}
private static MasterStatusPollingBackgroundService CreateSut(
IMasterAvailabilityService masterAvailabilityService,
IMasterStatusPollClient pollClient,
int failOpenAfterMinutes = 5)
int failOpenAfterMinutes = 5,
ILogger<MasterStatusPollingBackgroundService>? logger = null)
{
var provider = Substitute.For<IServiceProvider>();
provider.GetService(typeof(IMasterAvailabilityService)).Returns(masterAvailabilityService);
@@ -90,6 +138,6 @@ public class MasterStatusPollingBackgroundServiceTests
return new MasterStatusPollingBackgroundService(
factory,
options,
Substitute.For<ILogger<MasterStatusPollingBackgroundService>>());
logger ?? Substitute.For<ILogger<MasterStatusPollingBackgroundService>>());
}
}
@@ -349,6 +349,81 @@ public class MasterAvailabilityServiceTests : IDisposable
status.DisableMessage.Should().BeNull();
}
private MasterRegistration RegisteredLongAgo()
{
var existing = new MasterRegistration
{
Id = MasterRegistration.SingletonId,
MasterUrl = "http://master.example.com",
ApiKey = "enc:key",
RegisteredAt = DateTimeOffset.UtcNow.AddHours(-3),
LastPolledAt = DateTimeOffset.UtcNow.AddMinutes(-10)
};
_repo.GetAsync().Returns(existing);
_protector.Unprotect("enc:key").Returns("key");
return existing;
}
[Fact]
public async Task AfterAFailOpen_ASuccessfulPoll_AppliesTheMasterNotAvailableAgainAtOnce()
{
RegisteredLongAgo();
await _svc.PushStatusAsync("key", false, "Onderhoud");
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
_svc.GetMasterStatus().IsAvailable.Should().BeTrue(); // failed open
await _svc.ApplyPolledStatusAsync(false, "Onderhoud"); // the master is reachable again
var status = _svc.GetMasterStatus();
status.IsAvailable.Should().BeFalse();
status.DisableMessage.Should().Be("Onderhoud");
}
[Fact]
public async Task AfterAFailOpen_APushAppliesTheMasterNotAvailableAgainAtOnce()
{
RegisteredLongAgo();
await _svc.PushStatusAsync("key", false, "Onderhoud");
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
(await _svc.PushStatusAsync("key", false, "Onderhoud")).Should().BeTrue();
_svc.GetMasterStatus().IsAvailable.Should().BeFalse();
}
[Fact]
public async Task AfterTheRecovery_FurtherFailedPollsDoNotFailOpenAgainWithinTheGracePeriod()
{
RegisteredLongAgo();
await _svc.PushStatusAsync("key", false, "Onderhoud");
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
await _svc.ApplyPolledStatusAsync(false, "Onderhoud"); // sets LastPolledAt = now
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
_svc.GetMasterStatus().IsAvailable.Should().BeFalse();
}
[Fact]
public async Task ReRegistration_NeverChangesTheMasterStatus_InEitherDirection()
{
var existing = RegisteredLongAgo();
await _svc.PushStatusAsync("key", false, "Onderhoud");
(await _svc.RegisterAsync("https://master.example.com", "key")).Should().BeTrue(); // wrong URL corrected
existing.MasterUrl.Should().Be("https://master.example.com");
var status = _svc.GetMasterStatus();
status.IsAvailable.Should().BeFalse();
status.DisableMessage.Should().Be("Onderhoud");
// the other way round: a slave that failed open stays Available until the master says otherwise
existing.LastPolledAt = DateTimeOffset.UtcNow.AddMinutes(-10);
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
await _svc.RegisterAsync("https://master.example.com", "key");
_svc.GetMasterStatus().IsAvailable.Should().BeTrue();
}
[Fact]
public async Task RecordPollFailureAsync_DoesNothing_WhenNoRegistration()
{
@@ -50,6 +50,23 @@ public class MasterStatusPollClientTests
result.Should().BeNull();
}
[Fact]
public async Task LastFailure_NamesTheStatusOrTheExceptionType_NeverTheUrlOrKey()
{
var unauthorized = CreateSut(new FakeHttpMessageHandler(HttpStatusCode.Unauthorized));
await unauthorized.GetStatusAsync(MasterUrl, ApiKey);
unauthorized.LastFailure.Should().Be("http_401");
var refused = CreateSut(new ThrowingHttpMessageHandler());
await refused.GetStatusAsync(MasterUrl, ApiKey);
refused.LastFailure.Should().StartWith("HttpRequestException");
refused.LastFailure.Should().NotContain("Connection refused").And.NotContain(ApiKey).And.NotContain("master.test");
var ok = CreateSut(new FakeHttpMessageHandler(HttpStatusCode.OK, JsonSerializer.Serialize(new { IsAvailable = true })));
await ok.GetStatusAsync(MasterUrl, ApiKey);
ok.LastFailure.Should().BeNull();
}
[Fact]
public async Task GetStatusAsync_SetsApiKeyHeader()
{