Master-poll: fouten zichtbaar loggen (Warning bij eerste fout, Information bij herstel); tests dat master-status na fail-open direct terugkomt
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
+50
-2
@@ -70,10 +70,58 @@ public class MasterStatusPollingBackgroundServiceTests
|
|||||||
await act.Should().NotThrowAsync();
|
await act.Should().NotThrowAsync();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private sealed class LogCapture : ILogger<MasterStatusPollingBackgroundService>
|
||||||
|
{
|
||||||
|
public List<(LogLevel Level, string Message)> Entries { get; } = [];
|
||||||
|
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
||||||
|
public bool IsEnabled(LogLevel logLevel) => true;
|
||||||
|
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) =>
|
||||||
|
Entries.Add((logLevel, formatter(state, exception)));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task APollFailure_IsWarnedOnce_WithTheReason_AndRecoveryIsInformation()
|
||||||
|
{
|
||||||
|
var masterAvailabilityService = Substitute.For<IMasterAvailabilityService>();
|
||||||
|
masterAvailabilityService.GetPollTargetAsync().Returns(new MasterPollTarget("https://secret-master.example", "the-api-key"));
|
||||||
|
var pollClient = Substitute.For<IMasterStatusPollClient>();
|
||||||
|
pollClient.GetStatusAsync("https://secret-master.example", "the-api-key").Returns(
|
||||||
|
(PolledMasterStatus?)null, (PolledMasterStatus?)null, (PolledMasterStatus?)null, new PolledMasterStatus(false, "Onderhoud"));
|
||||||
|
pollClient.LastFailure.Returns("http_401");
|
||||||
|
var logs = new LogCapture();
|
||||||
|
var sut = CreateSut(masterAvailabilityService, pollClient, logger: logs);
|
||||||
|
|
||||||
|
for (var i = 0; i < 4; i++) await sut.ExecuteTickAsync(CancellationToken.None);
|
||||||
|
|
||||||
|
logs.Entries.Should().HaveCount(2); // not one line per failed poll
|
||||||
|
logs.Entries[0].Level.Should().Be(LogLevel.Warning);
|
||||||
|
logs.Entries[0].Message.Should().Contain("master_poll_failed").And.Contain("http_401");
|
||||||
|
logs.Entries[1].Level.Should().Be(LogLevel.Information);
|
||||||
|
logs.Entries[1].Message.Should().Contain("master_poll_recovered").And.Contain("3");
|
||||||
|
logs.Entries.Select(e => e.Message).Should().AllSatisfy(m => m.Should().NotContain("the-api-key").And.NotContain("secret-master"));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ANewFailureAfterRecovery_IsWarnedAgain()
|
||||||
|
{
|
||||||
|
var masterAvailabilityService = Substitute.For<IMasterAvailabilityService>();
|
||||||
|
masterAvailabilityService.GetPollTargetAsync().Returns(new MasterPollTarget("https://m", "k"));
|
||||||
|
var pollClient = Substitute.For<IMasterStatusPollClient>();
|
||||||
|
pollClient.GetStatusAsync("https://m", "k").Returns(
|
||||||
|
(PolledMasterStatus?)null, new PolledMasterStatus(true, null), (PolledMasterStatus?)null);
|
||||||
|
var logs = new LogCapture();
|
||||||
|
var sut = CreateSut(masterAvailabilityService, pollClient, logger: logs);
|
||||||
|
|
||||||
|
for (var i = 0; i < 3; i++) await sut.ExecuteTickAsync(CancellationToken.None);
|
||||||
|
|
||||||
|
logs.Entries.Select(e => e.Level).Should().Equal(LogLevel.Warning, LogLevel.Information, LogLevel.Warning);
|
||||||
|
}
|
||||||
|
|
||||||
private static MasterStatusPollingBackgroundService CreateSut(
|
private static MasterStatusPollingBackgroundService CreateSut(
|
||||||
IMasterAvailabilityService masterAvailabilityService,
|
IMasterAvailabilityService masterAvailabilityService,
|
||||||
IMasterStatusPollClient pollClient,
|
IMasterStatusPollClient pollClient,
|
||||||
int failOpenAfterMinutes = 5)
|
int failOpenAfterMinutes = 5,
|
||||||
|
ILogger<MasterStatusPollingBackgroundService>? logger = null)
|
||||||
{
|
{
|
||||||
var provider = Substitute.For<IServiceProvider>();
|
var provider = Substitute.For<IServiceProvider>();
|
||||||
provider.GetService(typeof(IMasterAvailabilityService)).Returns(masterAvailabilityService);
|
provider.GetService(typeof(IMasterAvailabilityService)).Returns(masterAvailabilityService);
|
||||||
@@ -90,6 +138,6 @@ public class MasterStatusPollingBackgroundServiceTests
|
|||||||
return new MasterStatusPollingBackgroundService(
|
return new MasterStatusPollingBackgroundService(
|
||||||
factory,
|
factory,
|
||||||
options,
|
options,
|
||||||
Substitute.For<ILogger<MasterStatusPollingBackgroundService>>());
|
logger ?? Substitute.For<ILogger<MasterStatusPollingBackgroundService>>());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+75
@@ -349,6 +349,81 @@ public class MasterAvailabilityServiceTests : IDisposable
|
|||||||
status.DisableMessage.Should().BeNull();
|
status.DisableMessage.Should().BeNull();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private MasterRegistration RegisteredLongAgo()
|
||||||
|
{
|
||||||
|
var existing = new MasterRegistration
|
||||||
|
{
|
||||||
|
Id = MasterRegistration.SingletonId,
|
||||||
|
MasterUrl = "http://master.example.com",
|
||||||
|
ApiKey = "enc:key",
|
||||||
|
RegisteredAt = DateTimeOffset.UtcNow.AddHours(-3),
|
||||||
|
LastPolledAt = DateTimeOffset.UtcNow.AddMinutes(-10)
|
||||||
|
};
|
||||||
|
_repo.GetAsync().Returns(existing);
|
||||||
|
_protector.Unprotect("enc:key").Returns("key");
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task AfterAFailOpen_ASuccessfulPoll_AppliesTheMasterNotAvailableAgainAtOnce()
|
||||||
|
{
|
||||||
|
RegisteredLongAgo();
|
||||||
|
await _svc.PushStatusAsync("key", false, "Onderhoud");
|
||||||
|
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||||
|
_svc.GetMasterStatus().IsAvailable.Should().BeTrue(); // failed open
|
||||||
|
|
||||||
|
await _svc.ApplyPolledStatusAsync(false, "Onderhoud"); // the master is reachable again
|
||||||
|
|
||||||
|
var status = _svc.GetMasterStatus();
|
||||||
|
status.IsAvailable.Should().BeFalse();
|
||||||
|
status.DisableMessage.Should().Be("Onderhoud");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task AfterAFailOpen_APushAppliesTheMasterNotAvailableAgainAtOnce()
|
||||||
|
{
|
||||||
|
RegisteredLongAgo();
|
||||||
|
await _svc.PushStatusAsync("key", false, "Onderhoud");
|
||||||
|
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||||
|
|
||||||
|
(await _svc.PushStatusAsync("key", false, "Onderhoud")).Should().BeTrue();
|
||||||
|
|
||||||
|
_svc.GetMasterStatus().IsAvailable.Should().BeFalse();
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task AfterTheRecovery_FurtherFailedPollsDoNotFailOpenAgainWithinTheGracePeriod()
|
||||||
|
{
|
||||||
|
RegisteredLongAgo();
|
||||||
|
await _svc.PushStatusAsync("key", false, "Onderhoud");
|
||||||
|
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||||
|
await _svc.ApplyPolledStatusAsync(false, "Onderhoud"); // sets LastPolledAt = now
|
||||||
|
|
||||||
|
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||||
|
|
||||||
|
_svc.GetMasterStatus().IsAvailable.Should().BeFalse();
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ReRegistration_NeverChangesTheMasterStatus_InEitherDirection()
|
||||||
|
{
|
||||||
|
var existing = RegisteredLongAgo();
|
||||||
|
await _svc.PushStatusAsync("key", false, "Onderhoud");
|
||||||
|
|
||||||
|
(await _svc.RegisterAsync("https://master.example.com", "key")).Should().BeTrue(); // wrong URL corrected
|
||||||
|
|
||||||
|
existing.MasterUrl.Should().Be("https://master.example.com");
|
||||||
|
var status = _svc.GetMasterStatus();
|
||||||
|
status.IsAvailable.Should().BeFalse();
|
||||||
|
status.DisableMessage.Should().Be("Onderhoud");
|
||||||
|
|
||||||
|
// the other way round: a slave that failed open stays Available until the master says otherwise
|
||||||
|
existing.LastPolledAt = DateTimeOffset.UtcNow.AddMinutes(-10);
|
||||||
|
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||||
|
await _svc.RegisterAsync("https://master.example.com", "key");
|
||||||
|
_svc.GetMasterStatus().IsAvailable.Should().BeTrue();
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task RecordPollFailureAsync_DoesNothing_WhenNoRegistration()
|
public async Task RecordPollFailureAsync_DoesNothing_WhenNoRegistration()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -50,6 +50,23 @@ public class MasterStatusPollClientTests
|
|||||||
result.Should().BeNull();
|
result.Should().BeNull();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task LastFailure_NamesTheStatusOrTheExceptionType_NeverTheUrlOrKey()
|
||||||
|
{
|
||||||
|
var unauthorized = CreateSut(new FakeHttpMessageHandler(HttpStatusCode.Unauthorized));
|
||||||
|
await unauthorized.GetStatusAsync(MasterUrl, ApiKey);
|
||||||
|
unauthorized.LastFailure.Should().Be("http_401");
|
||||||
|
|
||||||
|
var refused = CreateSut(new ThrowingHttpMessageHandler());
|
||||||
|
await refused.GetStatusAsync(MasterUrl, ApiKey);
|
||||||
|
refused.LastFailure.Should().StartWith("HttpRequestException");
|
||||||
|
refused.LastFailure.Should().NotContain("Connection refused").And.NotContain(ApiKey).And.NotContain("master.test");
|
||||||
|
|
||||||
|
var ok = CreateSut(new FakeHttpMessageHandler(HttpStatusCode.OK, JsonSerializer.Serialize(new { IsAvailable = true })));
|
||||||
|
await ok.GetStatusAsync(MasterUrl, ApiKey);
|
||||||
|
ok.LastFailure.Should().BeNull();
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task GetStatusAsync_SetsApiKeyHeader()
|
public async Task GetStatusAsync_SetsApiKeyHeader()
|
||||||
{
|
{
|
||||||
|
|||||||
+21
@@ -18,6 +18,10 @@ public class MasterStatusPollingBackgroundService(
|
|||||||
IOptions<MasterPollingOptions> options,
|
IOptions<MasterPollingOptions> options,
|
||||||
ILogger<MasterStatusPollingBackgroundService> logger) : BackgroundService
|
ILogger<MasterStatusPollingBackgroundService> logger) : BackgroundService
|
||||||
{
|
{
|
||||||
|
// Failures are reported once when they start and once when they end, not every 30 seconds.
|
||||||
|
private bool _failing;
|
||||||
|
private int _failureCount;
|
||||||
|
|
||||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
{
|
{
|
||||||
// Poll once immediately on startup, so a freshly (re)started slave re-syncs right away
|
// Poll once immediately on startup, so a freshly (re)started slave re-syncs right away
|
||||||
@@ -52,9 +56,26 @@ public class MasterStatusPollingBackgroundService(
|
|||||||
if (polled is not null)
|
if (polled is not null)
|
||||||
{
|
{
|
||||||
await masterAvailabilityService.ApplyPolledStatusAsync(polled.IsAvailable, polled.DisableMessage);
|
await masterAvailabilityService.ApplyPolledStatusAsync(polled.IsAvailable, polled.DisableMessage);
|
||||||
|
if (_failing)
|
||||||
|
{
|
||||||
|
logger.LogInformation(
|
||||||
|
"master_poll_recovered after {Failures} failed polls; master status applied (IsAvailable={IsAvailable}).",
|
||||||
|
_failureCount, polled.IsAvailable);
|
||||||
|
_failing = false;
|
||||||
|
_failureCount = 0;
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_failureCount++;
|
||||||
|
if (!_failing)
|
||||||
|
{
|
||||||
|
_failing = true;
|
||||||
|
logger.LogWarning(
|
||||||
|
"master_poll_failed reason={Reason}. Further failures are not logged until the master answers again.",
|
||||||
|
string.IsNullOrEmpty(pollClient.LastFailure) ? "unknown" : pollClient.LastFailure);
|
||||||
|
}
|
||||||
|
|
||||||
var failOpenAfter = TimeSpan.FromMinutes(options.Value.FailOpenAfterMinutes);
|
var failOpenAfter = TimeSpan.FromMinutes(options.Value.FailOpenAfterMinutes);
|
||||||
await masterAvailabilityService.RecordPollFailureAsync(failOpenAfter);
|
await masterAvailabilityService.RecordPollFailureAsync(failOpenAfter);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,12 @@ namespace SlpModularCms.Modules.Availability.Services;
|
|||||||
public interface IMasterStatusPollClient
|
public interface IMasterStatusPollClient
|
||||||
{
|
{
|
||||||
Task<PolledMasterStatus?> GetStatusAsync(string masterUrl, string plainApiKey);
|
Task<PolledMasterStatus?> GetStatusAsync(string masterUrl, string plainApiKey);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Why the last <see cref="GetStatusAsync"/> returned null: an HTTP status or an exception type, never the URL, the
|
||||||
|
/// key or a message text. Null after a success. Read it on the same instance right after the call.
|
||||||
|
/// </summary>
|
||||||
|
string? LastFailure { get; }
|
||||||
}
|
}
|
||||||
|
|
||||||
[System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage]
|
[System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage]
|
||||||
|
|||||||
@@ -7,8 +7,11 @@ public class MasterStatusPollClient(HttpClient httpClient) : IMasterStatusPollCl
|
|||||||
{
|
{
|
||||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
||||||
|
|
||||||
|
public string? LastFailure { get; private set; }
|
||||||
|
|
||||||
public async Task<PolledMasterStatus?> GetStatusAsync(string masterUrl, string plainApiKey)
|
public async Task<PolledMasterStatus?> GetStatusAsync(string masterUrl, string plainApiKey)
|
||||||
{
|
{
|
||||||
|
LastFailure = null;
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var request = new HttpRequestMessage(HttpMethod.Get, $"{masterUrl.TrimEnd('/')}/api/v1/SlaveStatus");
|
var request = new HttpRequestMessage(HttpMethod.Get, $"{masterUrl.TrimEnd('/')}/api/v1/SlaveStatus");
|
||||||
@@ -16,16 +19,33 @@ public class MasterStatusPollClient(HttpClient httpClient) : IMasterStatusPollCl
|
|||||||
|
|
||||||
var response = await httpClient.SendAsync(request);
|
var response = await httpClient.SendAsync(request);
|
||||||
if (!response.IsSuccessStatusCode)
|
if (!response.IsSuccessStatusCode)
|
||||||
|
{
|
||||||
|
LastFailure = $"http_{(int)response.StatusCode}";
|
||||||
return null;
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
var result = await response.Content.ReadFromJsonAsync<SlaveStatusResponse>(JsonOptions);
|
var result = await response.Content.ReadFromJsonAsync<SlaveStatusResponse>(JsonOptions);
|
||||||
|
if (result is null) LastFailure = "empty_response";
|
||||||
return result is null ? null : new PolledMasterStatus(result.IsAvailable, result.DisableMessage);
|
return result is null ? null : new PolledMasterStatus(result.IsAvailable, result.DisableMessage);
|
||||||
}
|
}
|
||||||
catch
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
|
LastFailure = Describe(ex);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>Exception type, plus the HTTP-level or socket error when there is one. No message text: it can quote the URL.</summary>
|
||||||
|
private static string Describe(Exception ex)
|
||||||
|
{
|
||||||
|
if (ex is TaskCanceledException or OperationCanceledException) return "timeout";
|
||||||
|
var detail = ex switch
|
||||||
|
{
|
||||||
|
HttpRequestException h when h.HttpRequestError != HttpRequestError.Unknown => h.HttpRequestError.ToString(),
|
||||||
|
_ => (ex.InnerException as System.Net.Sockets.SocketException)?.SocketErrorCode.ToString()
|
||||||
|
};
|
||||||
|
return detail is null ? ex.GetType().Name : $"{ex.GetType().Name}:{detail}";
|
||||||
|
}
|
||||||
|
|
||||||
private record SlaveStatusResponse(bool IsAvailable, string? DisableMessage);
|
private record SlaveStatusResponse(bool IsAvailable, string? DisableMessage);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user