diff --git a/src/SlpModularCms.Modules.Availability.Tests/BackgroundServices/MasterStatusPollingBackgroundServiceTests.cs b/src/SlpModularCms.Modules.Availability.Tests/BackgroundServices/MasterStatusPollingBackgroundServiceTests.cs index 5b65549..281a4b7 100644 --- a/src/SlpModularCms.Modules.Availability.Tests/BackgroundServices/MasterStatusPollingBackgroundServiceTests.cs +++ b/src/SlpModularCms.Modules.Availability.Tests/BackgroundServices/MasterStatusPollingBackgroundServiceTests.cs @@ -70,10 +70,58 @@ public class MasterStatusPollingBackgroundServiceTests await act.Should().NotThrowAsync(); } + private sealed class LogCapture : ILogger + { + public List<(LogLevel Level, string Message)> Entries { get; } = []; + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => true; + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => + Entries.Add((logLevel, formatter(state, exception))); + } + + [Fact] + public async Task APollFailure_IsWarnedOnce_WithTheReason_AndRecoveryIsInformation() + { + var masterAvailabilityService = Substitute.For(); + masterAvailabilityService.GetPollTargetAsync().Returns(new MasterPollTarget("https://secret-master.example", "the-api-key")); + var pollClient = Substitute.For(); + pollClient.GetStatusAsync("https://secret-master.example", "the-api-key").Returns( + (PolledMasterStatus?)null, (PolledMasterStatus?)null, (PolledMasterStatus?)null, new PolledMasterStatus(false, "Onderhoud")); + pollClient.LastFailure.Returns("http_401"); + var logs = new LogCapture(); + var sut = CreateSut(masterAvailabilityService, pollClient, logger: logs); + + for (var i = 0; i < 4; i++) await sut.ExecuteTickAsync(CancellationToken.None); + + logs.Entries.Should().HaveCount(2); // not one line per failed poll + logs.Entries[0].Level.Should().Be(LogLevel.Warning); + logs.Entries[0].Message.Should().Contain("master_poll_failed").And.Contain("http_401"); + logs.Entries[1].Level.Should().Be(LogLevel.Information); + logs.Entries[1].Message.Should().Contain("master_poll_recovered").And.Contain("3"); + logs.Entries.Select(e => e.Message).Should().AllSatisfy(m => m.Should().NotContain("the-api-key").And.NotContain("secret-master")); + } + + [Fact] + public async Task ANewFailureAfterRecovery_IsWarnedAgain() + { + var masterAvailabilityService = Substitute.For(); + masterAvailabilityService.GetPollTargetAsync().Returns(new MasterPollTarget("https://m", "k")); + var pollClient = Substitute.For(); + pollClient.GetStatusAsync("https://m", "k").Returns( + (PolledMasterStatus?)null, new PolledMasterStatus(true, null), (PolledMasterStatus?)null); + var logs = new LogCapture(); + var sut = CreateSut(masterAvailabilityService, pollClient, logger: logs); + + for (var i = 0; i < 3; i++) await sut.ExecuteTickAsync(CancellationToken.None); + + logs.Entries.Select(e => e.Level).Should().Equal(LogLevel.Warning, LogLevel.Information, LogLevel.Warning); + } + private static MasterStatusPollingBackgroundService CreateSut( IMasterAvailabilityService masterAvailabilityService, IMasterStatusPollClient pollClient, - int failOpenAfterMinutes = 5) + int failOpenAfterMinutes = 5, + ILogger? logger = null) { var provider = Substitute.For(); provider.GetService(typeof(IMasterAvailabilityService)).Returns(masterAvailabilityService); @@ -90,6 +138,6 @@ public class MasterStatusPollingBackgroundServiceTests return new MasterStatusPollingBackgroundService( factory, options, - Substitute.For>()); + logger ?? Substitute.For>()); } } diff --git a/src/SlpModularCms.Modules.Availability.Tests/Services/MasterAvailabilityServiceTests.cs b/src/SlpModularCms.Modules.Availability.Tests/Services/MasterAvailabilityServiceTests.cs index 789af62..1e04e84 100644 --- a/src/SlpModularCms.Modules.Availability.Tests/Services/MasterAvailabilityServiceTests.cs +++ b/src/SlpModularCms.Modules.Availability.Tests/Services/MasterAvailabilityServiceTests.cs @@ -349,6 +349,81 @@ public class MasterAvailabilityServiceTests : IDisposable status.DisableMessage.Should().BeNull(); } + private MasterRegistration RegisteredLongAgo() + { + var existing = new MasterRegistration + { + Id = MasterRegistration.SingletonId, + MasterUrl = "http://master.example.com", + ApiKey = "enc:key", + RegisteredAt = DateTimeOffset.UtcNow.AddHours(-3), + LastPolledAt = DateTimeOffset.UtcNow.AddMinutes(-10) + }; + _repo.GetAsync().Returns(existing); + _protector.Unprotect("enc:key").Returns("key"); + return existing; + } + + [Fact] + public async Task AfterAFailOpen_ASuccessfulPoll_AppliesTheMasterNotAvailableAgainAtOnce() + { + RegisteredLongAgo(); + await _svc.PushStatusAsync("key", false, "Onderhoud"); + await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5)); + _svc.GetMasterStatus().IsAvailable.Should().BeTrue(); // failed open + + await _svc.ApplyPolledStatusAsync(false, "Onderhoud"); // the master is reachable again + + var status = _svc.GetMasterStatus(); + status.IsAvailable.Should().BeFalse(); + status.DisableMessage.Should().Be("Onderhoud"); + } + + [Fact] + public async Task AfterAFailOpen_APushAppliesTheMasterNotAvailableAgainAtOnce() + { + RegisteredLongAgo(); + await _svc.PushStatusAsync("key", false, "Onderhoud"); + await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5)); + + (await _svc.PushStatusAsync("key", false, "Onderhoud")).Should().BeTrue(); + + _svc.GetMasterStatus().IsAvailable.Should().BeFalse(); + } + + [Fact] + public async Task AfterTheRecovery_FurtherFailedPollsDoNotFailOpenAgainWithinTheGracePeriod() + { + RegisteredLongAgo(); + await _svc.PushStatusAsync("key", false, "Onderhoud"); + await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5)); + await _svc.ApplyPolledStatusAsync(false, "Onderhoud"); // sets LastPolledAt = now + + await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5)); + + _svc.GetMasterStatus().IsAvailable.Should().BeFalse(); + } + + [Fact] + public async Task ReRegistration_NeverChangesTheMasterStatus_InEitherDirection() + { + var existing = RegisteredLongAgo(); + await _svc.PushStatusAsync("key", false, "Onderhoud"); + + (await _svc.RegisterAsync("https://master.example.com", "key")).Should().BeTrue(); // wrong URL corrected + + existing.MasterUrl.Should().Be("https://master.example.com"); + var status = _svc.GetMasterStatus(); + status.IsAvailable.Should().BeFalse(); + status.DisableMessage.Should().Be("Onderhoud"); + + // the other way round: a slave that failed open stays Available until the master says otherwise + existing.LastPolledAt = DateTimeOffset.UtcNow.AddMinutes(-10); + await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5)); + await _svc.RegisterAsync("https://master.example.com", "key"); + _svc.GetMasterStatus().IsAvailable.Should().BeTrue(); + } + [Fact] public async Task RecordPollFailureAsync_DoesNothing_WhenNoRegistration() { diff --git a/src/SlpModularCms.Modules.Availability.Tests/Services/MasterStatusPollClientTests.cs b/src/SlpModularCms.Modules.Availability.Tests/Services/MasterStatusPollClientTests.cs index 81b217c..e3301fd 100644 --- a/src/SlpModularCms.Modules.Availability.Tests/Services/MasterStatusPollClientTests.cs +++ b/src/SlpModularCms.Modules.Availability.Tests/Services/MasterStatusPollClientTests.cs @@ -50,6 +50,23 @@ public class MasterStatusPollClientTests result.Should().BeNull(); } + [Fact] + public async Task LastFailure_NamesTheStatusOrTheExceptionType_NeverTheUrlOrKey() + { + var unauthorized = CreateSut(new FakeHttpMessageHandler(HttpStatusCode.Unauthorized)); + await unauthorized.GetStatusAsync(MasterUrl, ApiKey); + unauthorized.LastFailure.Should().Be("http_401"); + + var refused = CreateSut(new ThrowingHttpMessageHandler()); + await refused.GetStatusAsync(MasterUrl, ApiKey); + refused.LastFailure.Should().StartWith("HttpRequestException"); + refused.LastFailure.Should().NotContain("Connection refused").And.NotContain(ApiKey).And.NotContain("master.test"); + + var ok = CreateSut(new FakeHttpMessageHandler(HttpStatusCode.OK, JsonSerializer.Serialize(new { IsAvailable = true }))); + await ok.GetStatusAsync(MasterUrl, ApiKey); + ok.LastFailure.Should().BeNull(); + } + [Fact] public async Task GetStatusAsync_SetsApiKeyHeader() { diff --git a/src/SlpModularCms.Modules.Availability/BackgroundServices/MasterStatusPollingBackgroundService.cs b/src/SlpModularCms.Modules.Availability/BackgroundServices/MasterStatusPollingBackgroundService.cs index 4dab9ff..7581e42 100644 --- a/src/SlpModularCms.Modules.Availability/BackgroundServices/MasterStatusPollingBackgroundService.cs +++ b/src/SlpModularCms.Modules.Availability/BackgroundServices/MasterStatusPollingBackgroundService.cs @@ -18,6 +18,10 @@ public class MasterStatusPollingBackgroundService( IOptions options, ILogger logger) : BackgroundService { + // Failures are reported once when they start and once when they end, not every 30 seconds. + private bool _failing; + private int _failureCount; + protected override async Task ExecuteAsync(CancellationToken stoppingToken) { // Poll once immediately on startup, so a freshly (re)started slave re-syncs right away @@ -52,9 +56,26 @@ public class MasterStatusPollingBackgroundService( if (polled is not null) { await masterAvailabilityService.ApplyPolledStatusAsync(polled.IsAvailable, polled.DisableMessage); + if (_failing) + { + logger.LogInformation( + "master_poll_recovered after {Failures} failed polls; master status applied (IsAvailable={IsAvailable}).", + _failureCount, polled.IsAvailable); + _failing = false; + _failureCount = 0; + } return; } + _failureCount++; + if (!_failing) + { + _failing = true; + logger.LogWarning( + "master_poll_failed reason={Reason}. Further failures are not logged until the master answers again.", + string.IsNullOrEmpty(pollClient.LastFailure) ? "unknown" : pollClient.LastFailure); + } + var failOpenAfter = TimeSpan.FromMinutes(options.Value.FailOpenAfterMinutes); await masterAvailabilityService.RecordPollFailureAsync(failOpenAfter); } diff --git a/src/SlpModularCms.Modules.Availability/Services/IMasterStatusPollClient.cs b/src/SlpModularCms.Modules.Availability/Services/IMasterStatusPollClient.cs index bd7187d..769c31a 100644 --- a/src/SlpModularCms.Modules.Availability/Services/IMasterStatusPollClient.cs +++ b/src/SlpModularCms.Modules.Availability/Services/IMasterStatusPollClient.cs @@ -3,6 +3,12 @@ namespace SlpModularCms.Modules.Availability.Services; public interface IMasterStatusPollClient { Task GetStatusAsync(string masterUrl, string plainApiKey); + + /// + /// Why the last returned null: an HTTP status or an exception type, never the URL, the + /// key or a message text. Null after a success. Read it on the same instance right after the call. + /// + string? LastFailure { get; } } [System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage] diff --git a/src/SlpModularCms.Modules.Availability/Services/MasterStatusPollClient.cs b/src/SlpModularCms.Modules.Availability/Services/MasterStatusPollClient.cs index 15b0d54..ed3a8b1 100644 --- a/src/SlpModularCms.Modules.Availability/Services/MasterStatusPollClient.cs +++ b/src/SlpModularCms.Modules.Availability/Services/MasterStatusPollClient.cs @@ -7,8 +7,11 @@ public class MasterStatusPollClient(HttpClient httpClient) : IMasterStatusPollCl { private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + public string? LastFailure { get; private set; } + public async Task GetStatusAsync(string masterUrl, string plainApiKey) { + LastFailure = null; try { var request = new HttpRequestMessage(HttpMethod.Get, $"{masterUrl.TrimEnd('/')}/api/v1/SlaveStatus"); @@ -16,16 +19,33 @@ public class MasterStatusPollClient(HttpClient httpClient) : IMasterStatusPollCl var response = await httpClient.SendAsync(request); if (!response.IsSuccessStatusCode) + { + LastFailure = $"http_{(int)response.StatusCode}"; return null; + } var result = await response.Content.ReadFromJsonAsync(JsonOptions); + if (result is null) LastFailure = "empty_response"; return result is null ? null : new PolledMasterStatus(result.IsAvailable, result.DisableMessage); } - catch + catch (Exception ex) { + LastFailure = Describe(ex); return null; } } + /// Exception type, plus the HTTP-level or socket error when there is one. No message text: it can quote the URL. + private static string Describe(Exception ex) + { + if (ex is TaskCanceledException or OperationCanceledException) return "timeout"; + var detail = ex switch + { + HttpRequestException h when h.HttpRequestError != HttpRequestError.Unknown => h.HttpRequestError.ToString(), + _ => (ex.InnerException as System.Net.Sockets.SocketException)?.SocketErrorCode.ToString() + }; + return detail is null ? ex.GetType().Name : $"{ex.GetType().Name}:{detail}"; + } + private record SlaveStatusResponse(bool IsAvailable, string? DisableMessage); }