Completes local-dev-master-slave-setup: dual-instance frontend tooling, module-capability gating, and master/slave protocol self-healing fixes
Frontend (Unit 2 completion): dual dev-server tooling (pnpm dev:slave, pnpm dev:all), per-instance browser tab titles, and a backend capability check (SystemController + useSystemCapabilities + ModuleGuard) so a Master-only page is hidden on a slave instance instead of assuming every backend has every module. Master/slave protocol fixes surfaced by actually running master and slave side by side locally: - Deactivating a CMS instance (Inactive) now releases the slave's master gate instead of leaving it stuck on its last pushed status. - The periodic integrity check now also re-pushes status to every reachable slave (previously URL-verification only) and runs once immediately on startup. - Added the originally-specified (but never implemented) slave-pull path: a slave now periodically polls its own status from the master (GET /api/v1/SlaveStatus) and fails open to Available if the master is unreachable for too long, complementing the existing push. - The slave's own Settings page can no longer "successfully" change local availability while the master controls it; it's now locked with an explanatory banner and the backend rejects the write with 409 instead of silently no-op'ing it. - CMS instance status badges now match the dashboard's color/icon styling instead of a plain grey badge. Also corrected the master-cms-module design docs to match this as-built behavior, and flagged (without a full rewrite) a larger, pre-existing divergence between its inception-stage application design and what construction actually built. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -18,6 +18,7 @@ builder.Services.AddCmsRateLimiting(builder.Configuration);
|
||||
|
||||
// 3. Add Module Services
|
||||
orchestrator.RegisterModuleServices(builder.Services);
|
||||
builder.Services.AddSingleton(orchestrator);
|
||||
|
||||
// 4. Global Controller Configuration with Conventions
|
||||
builder.Services.AddControllers(options =>
|
||||
|
||||
@@ -20,6 +20,11 @@
|
||||
"CircuitBreakerSeconds": 30,
|
||||
"StatusCacheSeconds": 1
|
||||
},
|
||||
"MasterPolling": {
|
||||
"PollIntervalSeconds": 15,
|
||||
"FailOpenAfterMinutes": 2,
|
||||
"HttpTimeoutSeconds": 5
|
||||
},
|
||||
"Cors": {
|
||||
"AllowedOrigins": [
|
||||
"http://localhost:5174",
|
||||
|
||||
@@ -20,6 +20,11 @@
|
||||
"CircuitBreakerSeconds": 30,
|
||||
"StatusCacheSeconds": 1
|
||||
},
|
||||
"MasterPolling": {
|
||||
"PollIntervalSeconds": 30,
|
||||
"FailOpenAfterMinutes": 5,
|
||||
"HttpTimeoutSeconds": 5
|
||||
},
|
||||
"Cors": {
|
||||
"AllowedOrigins": []
|
||||
},
|
||||
|
||||
@@ -18,6 +18,7 @@ builder.Services.AddCmsRateLimiting(builder.Configuration);
|
||||
|
||||
// 3. Add Module Services
|
||||
orchestrator.RegisterModuleServices(builder.Services);
|
||||
builder.Services.AddSingleton(orchestrator);
|
||||
|
||||
// 4. Global Controller Configuration with Conventions
|
||||
builder.Services.AddControllers(options =>
|
||||
|
||||
@@ -20,6 +20,11 @@
|
||||
"CircuitBreakerSeconds": 30,
|
||||
"StatusCacheSeconds": 1
|
||||
},
|
||||
"MasterPolling": {
|
||||
"PollIntervalSeconds": 15,
|
||||
"FailOpenAfterMinutes": 2,
|
||||
"HttpTimeoutSeconds": 5
|
||||
},
|
||||
"MasterModule": {
|
||||
"IntegrityCheckIntervalMinutes": 60,
|
||||
"HttpTimeoutSeconds": 10,
|
||||
|
||||
@@ -25,6 +25,11 @@
|
||||
"HttpTimeoutSeconds": 10,
|
||||
"MasterUrl": "<public-url-of-this-master-instance>"
|
||||
},
|
||||
"MasterPolling": {
|
||||
"PollIntervalSeconds": 30,
|
||||
"FailOpenAfterMinutes": 5,
|
||||
"HttpTimeoutSeconds": 5
|
||||
},
|
||||
"Cors": {
|
||||
"AllowedOrigins": []
|
||||
},
|
||||
|
||||
@@ -6,4 +6,4 @@ namespace SlpModularCms.Core.Availability;
|
||||
/// Status en optionele admin-melding van de systeembeschikbaarheid.
|
||||
/// </summary>
|
||||
[ExcludeFromCodeCoverage]
|
||||
public record AvailabilityStatusDetails(AvailabilityStatus Status, string? Message);
|
||||
public record AvailabilityStatusDetails(AvailabilityStatus Status, string? Message, bool IsMasterControlled = false);
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
namespace SlpModularCms.Core.Availability;
|
||||
|
||||
/// <summary>
|
||||
/// Thrown when an attempt is made to change the local availability status while a
|
||||
/// master CMS has taken control of this instance's gate (e.g. set it to NotAvailable).
|
||||
/// The local status change would have no visible effect since the master gate
|
||||
/// overrides it, so it's rejected outright instead of silently doing nothing.
|
||||
/// </summary>
|
||||
public class MasterControlledAvailabilityException(string? masterDisableMessage)
|
||||
: InvalidOperationException("De beschikbaarheid van dit systeem wordt beheerd door de Master-CMS en kan hier niet worden gewijzigd.")
|
||||
{
|
||||
public string? MasterDisableMessage { get; } = masterDisableMessage;
|
||||
}
|
||||
@@ -19,6 +19,12 @@ public class ModuleOrchestrator
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Names of the modules discovered on this instance (e.g. so the frontend can tell
|
||||
/// a master-only feature apart from a slave instance without that module loaded).
|
||||
/// </summary>
|
||||
public IReadOnlyList<string> ModuleNames => _modules.Select(m => m.Name).ToArray();
|
||||
|
||||
public void DiscoverModules()
|
||||
{
|
||||
_logger.LogInformation("Start module discovery...");
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
namespace SlpModularCms.Core.Hosting;
|
||||
|
||||
/// <summary>
|
||||
/// Exposes which optional modules are loaded on this instance, so clients (e.g. the
|
||||
/// frontend) can tell a master-only feature apart from an instance without that module
|
||||
/// (see the local master/slave dev setup) without guessing from a 404.
|
||||
/// </summary>
|
||||
[ApiController]
|
||||
[Route("[controller]")]
|
||||
public class SystemController : ControllerBase
|
||||
{
|
||||
private readonly ModuleOrchestrator _orchestrator;
|
||||
|
||||
public SystemController(ModuleOrchestrator orchestrator)
|
||||
{
|
||||
_orchestrator = orchestrator;
|
||||
}
|
||||
|
||||
[HttpGet("capabilities")]
|
||||
public IActionResult GetCapabilities()
|
||||
{
|
||||
return Ok(new { Modules = _orchestrator.ModuleNames });
|
||||
}
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using NSubstitute.ExceptionExtensions;
|
||||
using SlpModularCms.Modules.Availability.BackgroundServices;
|
||||
using SlpModularCms.Modules.Availability.Config;
|
||||
using SlpModularCms.Modules.Availability.Services;
|
||||
|
||||
namespace SlpModularCms.Modules.Availability.Tests.BackgroundServices;
|
||||
|
||||
public class MasterStatusPollingBackgroundServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task ExecuteTickAsync_DoesNothing_WhenNoMasterRegistered()
|
||||
{
|
||||
var masterAvailabilityService = Substitute.For<IMasterAvailabilityService>();
|
||||
masterAvailabilityService.GetPollTargetAsync().Returns((MasterPollTarget?)null);
|
||||
var pollClient = Substitute.For<IMasterStatusPollClient>();
|
||||
var sut = CreateSut(masterAvailabilityService, pollClient);
|
||||
|
||||
await sut.ExecuteTickAsync(CancellationToken.None);
|
||||
|
||||
await pollClient.DidNotReceive().GetStatusAsync(Arg.Any<string>(), Arg.Any<string>());
|
||||
await masterAvailabilityService.DidNotReceive().ApplyPolledStatusAsync(Arg.Any<bool>(), Arg.Any<string?>());
|
||||
await masterAvailabilityService.DidNotReceive().RecordPollFailureAsync(Arg.Any<TimeSpan>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteTickAsync_AppliesPolledStatus_WhenPollSucceeds()
|
||||
{
|
||||
var masterAvailabilityService = Substitute.For<IMasterAvailabilityService>();
|
||||
masterAvailabilityService.GetPollTargetAsync().Returns(new MasterPollTarget("https://master.test", "key"));
|
||||
var pollClient = Substitute.For<IMasterStatusPollClient>();
|
||||
pollClient.GetStatusAsync("https://master.test", "key").Returns(new PolledMasterStatus(false, "Onderhoud"));
|
||||
var sut = CreateSut(masterAvailabilityService, pollClient);
|
||||
|
||||
await sut.ExecuteTickAsync(CancellationToken.None);
|
||||
|
||||
await masterAvailabilityService.Received(1).ApplyPolledStatusAsync(false, "Onderhoud");
|
||||
await masterAvailabilityService.DidNotReceive().RecordPollFailureAsync(Arg.Any<TimeSpan>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteTickAsync_RecordsPollFailure_WhenPollFails()
|
||||
{
|
||||
var masterAvailabilityService = Substitute.For<IMasterAvailabilityService>();
|
||||
masterAvailabilityService.GetPollTargetAsync().Returns(new MasterPollTarget("https://master.test", "key"));
|
||||
var pollClient = Substitute.For<IMasterStatusPollClient>();
|
||||
pollClient.GetStatusAsync("https://master.test", "key").Returns((PolledMasterStatus?)null);
|
||||
var sut = CreateSut(masterAvailabilityService, pollClient, failOpenAfterMinutes: 5);
|
||||
|
||||
await sut.ExecuteTickAsync(CancellationToken.None);
|
||||
|
||||
await masterAvailabilityService.DidNotReceive().ApplyPolledStatusAsync(Arg.Any<bool>(), Arg.Any<string?>());
|
||||
await masterAvailabilityService.Received(1).RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteTickAsync_DoesNotThrow_WhenPollTargetLookupThrows()
|
||||
{
|
||||
var masterAvailabilityService = Substitute.For<IMasterAvailabilityService>();
|
||||
masterAvailabilityService.GetPollTargetAsync().Throws(new InvalidOperationException("boom"));
|
||||
var pollClient = Substitute.For<IMasterStatusPollClient>();
|
||||
var sut = CreateSut(masterAvailabilityService, pollClient);
|
||||
|
||||
var act = async () => await sut.ExecuteTickAsync(CancellationToken.None);
|
||||
|
||||
await act.Should().NotThrowAsync();
|
||||
}
|
||||
|
||||
private static MasterStatusPollingBackgroundService CreateSut(
|
||||
IMasterAvailabilityService masterAvailabilityService,
|
||||
IMasterStatusPollClient pollClient,
|
||||
int failOpenAfterMinutes = 5)
|
||||
{
|
||||
var provider = Substitute.For<IServiceProvider>();
|
||||
provider.GetService(typeof(IMasterAvailabilityService)).Returns(masterAvailabilityService);
|
||||
provider.GetService(typeof(IMasterStatusPollClient)).Returns(pollClient);
|
||||
|
||||
var scope = Substitute.For<IServiceScope>();
|
||||
scope.ServiceProvider.Returns(provider);
|
||||
|
||||
var factory = Substitute.For<IServiceScopeFactory>();
|
||||
factory.CreateAsyncScope().Returns(new AsyncServiceScope(scope));
|
||||
|
||||
var options = Options.Create(new MasterPollingOptions { FailOpenAfterMinutes = failOpenAfterMinutes });
|
||||
|
||||
return new MasterStatusPollingBackgroundService(
|
||||
factory,
|
||||
options,
|
||||
Substitute.For<ILogger<MasterStatusPollingBackgroundService>>());
|
||||
}
|
||||
}
|
||||
+50
-1
@@ -81,7 +81,10 @@ public class AvailabilityControllerTests
|
||||
var availabilityOptions = Substitute.For<IOptions<AvailabilityOptions>>();
|
||||
availabilityOptions.Value.Returns(new AvailabilityOptions { CircuitBreakerSeconds = 30 });
|
||||
|
||||
var persistentService = new PersistentAvailabilityService(context, availabilityOptions);
|
||||
var masterAvailabilityService = Substitute.For<IMasterAvailabilityService>();
|
||||
masterAvailabilityService.GetMasterStatus().Returns(new MasterGateStatus(true, null));
|
||||
|
||||
var persistentService = new PersistentAvailabilityService(context, availabilityOptions, masterAvailabilityService);
|
||||
|
||||
var controller = new AvailabilityController(persistentService);
|
||||
var httpContext = new DefaultHttpContext();
|
||||
@@ -94,4 +97,50 @@ public class AvailabilityControllerTests
|
||||
|
||||
result.Should().BeOfType<OkResult>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpdateStatus_ReturnsConflict_WhenMasterControlsTheGate()
|
||||
{
|
||||
var dbOptions = new DbContextOptionsBuilder<ApplicationDbContext>()
|
||||
.UseInMemoryDatabase(Guid.NewGuid().ToString())
|
||||
.Options;
|
||||
using var context = new ApplicationDbContext(dbOptions);
|
||||
|
||||
var availabilityOptions = Substitute.For<IOptions<AvailabilityOptions>>();
|
||||
availabilityOptions.Value.Returns(new AvailabilityOptions { CircuitBreakerSeconds = 30 });
|
||||
|
||||
var masterAvailabilityService = Substitute.For<IMasterAvailabilityService>();
|
||||
masterAvailabilityService.GetMasterStatus().Returns(new MasterGateStatus(false, "Uitgeschakeld door master"));
|
||||
|
||||
var persistentService = new PersistentAvailabilityService(context, availabilityOptions, masterAvailabilityService);
|
||||
|
||||
var controller = new AvailabilityController(persistentService);
|
||||
var httpContext = new DefaultHttpContext();
|
||||
httpContext.User = new ClaimsPrincipal(
|
||||
new ClaimsIdentity([new Claim(ClaimTypes.Name, "admin@test.com")], "TestAuth"));
|
||||
controller.ControllerContext = new ControllerContext { HttpContext = httpContext };
|
||||
|
||||
var result = await controller.UpdateStatus(
|
||||
new UpdateStatusRequest(AvailabilityStatus.Available, null));
|
||||
|
||||
var conflict = result.Should().BeOfType<ConflictObjectResult>().Subject;
|
||||
conflict.StatusCode.Should().Be(409);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetStatus_IncludesIsMasterControlled_WhenTrue()
|
||||
{
|
||||
_availabilityService.GetStatusDetailsAsync()
|
||||
.Returns(new AvailabilityStatusDetails(AvailabilityStatus.NotAvailable, "Master zegt nee", IsMasterControlled: true));
|
||||
|
||||
var result = await CreateController().GetStatus();
|
||||
|
||||
var ok = result.Should().BeOfType<OkObjectResult>().Subject;
|
||||
ok.Value.Should().BeEquivalentTo(new
|
||||
{
|
||||
Status = "NotAvailable",
|
||||
Message = "Master zegt nee",
|
||||
IsMasterControlled = true,
|
||||
}, options => options.ExcludingMissingMembers());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,8 +24,11 @@ public class PersistentAvailabilityServiceTests
|
||||
|
||||
var availabilityOptions = Substitute.For<IOptions<AvailabilityOptions>>();
|
||||
availabilityOptions.Value.Returns(new AvailabilityOptions { CircuitBreakerSeconds = 30 });
|
||||
|
||||
_service = new PersistentAvailabilityService(_context, availabilityOptions);
|
||||
|
||||
var masterAvailabilityService = Substitute.For<IMasterAvailabilityService>();
|
||||
masterAvailabilityService.GetMasterStatus().Returns(new MasterGateStatus(true, null));
|
||||
|
||||
_service = new PersistentAvailabilityService(_context, availabilityOptions, masterAvailabilityService);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -99,6 +102,24 @@ public class PersistentAvailabilityServiceTests
|
||||
details.Status.Should().Be(AvailabilityStatus.Available); // Fallback
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetStatusDetailsAsync_ShouldReturnNotAvailable_WhenMasterGateDisabled()
|
||||
{
|
||||
var availabilityOptions = Substitute.For<IOptions<AvailabilityOptions>>();
|
||||
availabilityOptions.Value.Returns(new AvailabilityOptions { CircuitBreakerSeconds = 30 });
|
||||
|
||||
var masterAvailabilityService = Substitute.For<IMasterAvailabilityService>();
|
||||
masterAvailabilityService.GetMasterStatus().Returns(new MasterGateStatus(false, "Master heeft deze slave uitgeschakeld"));
|
||||
|
||||
var service = new PersistentAvailabilityService(_context, availabilityOptions, masterAvailabilityService);
|
||||
|
||||
var details = await service.GetStatusDetailsAsync();
|
||||
|
||||
details.Status.Should().Be(AvailabilityStatus.NotAvailable);
|
||||
details.Message.Should().Be("Master heeft deze slave uitgeschakeld");
|
||||
details.IsMasterControlled.Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpdateStatusAsync_UpdatesExistingRecord_WhenCalledTwice()
|
||||
{
|
||||
@@ -112,4 +133,21 @@ public class PersistentAvailabilityServiceTests
|
||||
dbState.Status.Should().Be(AvailabilityStatus.Available);
|
||||
dbState.UpdatedBy.Should().Be("Admin2");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpdateStatusAsync_Throws_WhenMasterControlsTheGate()
|
||||
{
|
||||
var availabilityOptions = Substitute.For<IOptions<AvailabilityOptions>>();
|
||||
availabilityOptions.Value.Returns(new AvailabilityOptions { CircuitBreakerSeconds = 30 });
|
||||
|
||||
var masterAvailabilityService = Substitute.For<IMasterAvailabilityService>();
|
||||
masterAvailabilityService.GetMasterStatus().Returns(new MasterGateStatus(false, "Uitgeschakeld door master"));
|
||||
|
||||
var service = new PersistentAvailabilityService(_context, availabilityOptions, masterAvailabilityService);
|
||||
|
||||
var act = async () => await service.UpdateStatusAsync(AvailabilityStatus.Available, null, "Admin");
|
||||
|
||||
await act.Should().ThrowAsync<MasterControlledAvailabilityException>();
|
||||
(await _context.AvailabilityStates.FirstOrDefaultAsync()).Should().BeNull();
|
||||
}
|
||||
}
|
||||
|
||||
+111
@@ -237,4 +237,115 @@ public class MasterAvailabilityServiceTests : IDisposable
|
||||
status.IsAvailable.Should().BeTrue();
|
||||
status.DisableMessage.Should().BeNull();
|
||||
}
|
||||
|
||||
// --- GetPollTargetAsync ---
|
||||
|
||||
[Fact]
|
||||
public async Task GetPollTargetAsync_ReturnsMasterUrlAndPlainKey_WhenRegistered()
|
||||
{
|
||||
var existing = new MasterRegistration
|
||||
{
|
||||
Id = MasterRegistration.SingletonId,
|
||||
MasterUrl = "https://master.example.com",
|
||||
ApiKey = "enc:key",
|
||||
RegisteredAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
_repo.GetAsync().Returns(existing);
|
||||
_protector.Unprotect("enc:key").Returns("key");
|
||||
|
||||
var target = await _svc.GetPollTargetAsync();
|
||||
|
||||
target.Should().NotBeNull();
|
||||
target!.MasterUrl.Should().Be("https://master.example.com");
|
||||
target.PlainApiKey.Should().Be("key");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPollTargetAsync_ReturnsNull_WhenNoRegistration()
|
||||
{
|
||||
_repo.GetAsync().Returns((MasterRegistration?)null);
|
||||
|
||||
var target = await _svc.GetPollTargetAsync();
|
||||
|
||||
target.Should().BeNull();
|
||||
}
|
||||
|
||||
// --- ApplyPolledStatusAsync ---
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyPolledStatusAsync_UpdatesGateAndPersistsPollTimestamp()
|
||||
{
|
||||
var existing = new MasterRegistration
|
||||
{
|
||||
Id = MasterRegistration.SingletonId,
|
||||
MasterUrl = "https://master.example.com",
|
||||
ApiKey = "enc:key",
|
||||
RegisteredAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
_repo.GetAsync().Returns(existing);
|
||||
|
||||
await _svc.ApplyPolledStatusAsync(false, "Onderhoud");
|
||||
|
||||
var status = _svc.GetMasterStatus();
|
||||
status.IsAvailable.Should().BeFalse();
|
||||
status.DisableMessage.Should().Be("Onderhoud");
|
||||
existing.LastPolledAt.Should().NotBeNull();
|
||||
_repo.Received(1).Update(existing);
|
||||
await _repo.Received(1).SaveChangesAsync();
|
||||
}
|
||||
|
||||
// --- RecordPollFailureAsync ---
|
||||
|
||||
[Fact]
|
||||
public async Task RecordPollFailureAsync_DoesNotFailOpen_WhenWithinGracePeriod()
|
||||
{
|
||||
var existing = new MasterRegistration
|
||||
{
|
||||
Id = MasterRegistration.SingletonId,
|
||||
MasterUrl = "https://master.example.com",
|
||||
ApiKey = "enc:key",
|
||||
RegisteredAt = DateTimeOffset.UtcNow,
|
||||
LastPolledAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
_repo.GetAsync().Returns(existing);
|
||||
_protector.Unprotect("enc:key").Returns("key");
|
||||
await _svc.PushStatusAsync("key", false, "Down");
|
||||
|
||||
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||
|
||||
var status = _svc.GetMasterStatus();
|
||||
status.IsAvailable.Should().BeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RecordPollFailureAsync_FailsOpen_WhenMasterUnreachableTooLong()
|
||||
{
|
||||
var existing = new MasterRegistration
|
||||
{
|
||||
Id = MasterRegistration.SingletonId,
|
||||
MasterUrl = "https://master.example.com",
|
||||
ApiKey = "enc:key",
|
||||
RegisteredAt = DateTimeOffset.UtcNow.AddMinutes(-30),
|
||||
LastPolledAt = DateTimeOffset.UtcNow.AddMinutes(-10)
|
||||
};
|
||||
_repo.GetAsync().Returns(existing);
|
||||
_protector.Unprotect("enc:key").Returns("key");
|
||||
await _svc.PushStatusAsync("key", false, "Down");
|
||||
|
||||
await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||
|
||||
var status = _svc.GetMasterStatus();
|
||||
status.IsAvailable.Should().BeTrue();
|
||||
status.DisableMessage.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RecordPollFailureAsync_DoesNothing_WhenNoRegistration()
|
||||
{
|
||||
_repo.GetAsync().Returns((MasterRegistration?)null);
|
||||
|
||||
var act = async () => await _svc.RecordPollFailureAsync(TimeSpan.FromMinutes(5));
|
||||
|
||||
await act.Should().NotThrowAsync();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
using FluentAssertions;
|
||||
using SlpModularCms.Modules.Availability.Services;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace SlpModularCms.Modules.Availability.Tests.Services;
|
||||
|
||||
public class MasterStatusPollClientTests
|
||||
{
|
||||
private const string MasterUrl = "https://master.test";
|
||||
private const string ApiKey = "plain-key";
|
||||
|
||||
private static MasterStatusPollClient CreateSut(HttpMessageHandler handler)
|
||||
=> new(new HttpClient(handler));
|
||||
|
||||
[Fact]
|
||||
public async Task GetStatusAsync_ReturnsStatus_WhenResponseIsSuccess()
|
||||
{
|
||||
var json = JsonSerializer.Serialize(new { IsAvailable = false, DisableMessage = "Onderhoud" });
|
||||
var handler = new FakeHttpMessageHandler(HttpStatusCode.OK, json);
|
||||
var sut = CreateSut(handler);
|
||||
|
||||
var result = await sut.GetStatusAsync(MasterUrl, ApiKey);
|
||||
|
||||
result.Should().NotBeNull();
|
||||
result!.IsAvailable.Should().BeFalse();
|
||||
result.DisableMessage.Should().Be("Onderhoud");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetStatusAsync_ReturnsNull_WhenResponseIsFailure()
|
||||
{
|
||||
var handler = new FakeHttpMessageHandler(HttpStatusCode.Unauthorized);
|
||||
var sut = CreateSut(handler);
|
||||
|
||||
var result = await sut.GetStatusAsync(MasterUrl, ApiKey);
|
||||
|
||||
result.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetStatusAsync_ReturnsNull_WhenExceptionIsThrown()
|
||||
{
|
||||
var handler = new ThrowingHttpMessageHandler();
|
||||
var sut = CreateSut(handler);
|
||||
|
||||
var result = await sut.GetStatusAsync(MasterUrl, ApiKey);
|
||||
|
||||
result.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetStatusAsync_SetsApiKeyHeader()
|
||||
{
|
||||
string? capturedKey = null;
|
||||
var handler = new CapturingHttpMessageHandler(req =>
|
||||
{
|
||||
req.Headers.TryGetValues("X-Master-Api-Key", out var values);
|
||||
capturedKey = values?.FirstOrDefault();
|
||||
});
|
||||
var sut = CreateSut(handler);
|
||||
|
||||
await sut.GetStatusAsync(MasterUrl, ApiKey);
|
||||
|
||||
capturedKey.Should().Be(ApiKey);
|
||||
}
|
||||
|
||||
private class FakeHttpMessageHandler(HttpStatusCode statusCode, string? content = null) : HttpMessageHandler
|
||||
{
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
var response = new HttpResponseMessage(statusCode);
|
||||
if (content is not null)
|
||||
response.Content = new StringContent(content, Encoding.UTF8, "application/json");
|
||||
return Task.FromResult(response);
|
||||
}
|
||||
}
|
||||
|
||||
private class ThrowingHttpMessageHandler : HttpMessageHandler
|
||||
{
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
=> throw new HttpRequestException("Connection refused");
|
||||
}
|
||||
|
||||
private class CapturingHttpMessageHandler(Action<HttpRequestMessage> capture) : HttpMessageHandler
|
||||
{
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
capture(request);
|
||||
var response = new HttpResponseMessage(HttpStatusCode.OK);
|
||||
response.Content = new StringContent(
|
||||
JsonSerializer.Serialize(new { IsAvailable = true, DisableMessage = (string?)null }),
|
||||
Encoding.UTF8,
|
||||
"application/json");
|
||||
return Task.FromResult(response);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@ using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using SlpModularCms.Core.Availability;
|
||||
using SlpModularCms.Core.Modules;
|
||||
using SlpModularCms.Modules.Availability.BackgroundServices;
|
||||
using SlpModularCms.Modules.Availability.Config;
|
||||
using SlpModularCms.Modules.Availability.Data;
|
||||
using SlpModularCms.Modules.Availability.Middleware;
|
||||
using SlpModularCms.Modules.Availability.Repositories;
|
||||
@@ -34,6 +36,14 @@ public class AvailabilityModule : IModule
|
||||
services.AddScoped<IMasterRegistrationRepository, MasterRegistrationRepository>();
|
||||
services.AddScoped<MasterAvailabilityServiceDependencies>();
|
||||
services.AddScoped<IMasterAvailabilityService, MasterAvailabilityService>();
|
||||
|
||||
services.AddOptions<MasterPollingOptions>().BindConfiguration("MasterPolling");
|
||||
services.AddHttpClient<IMasterStatusPollClient, MasterStatusPollClient>((sp, client) =>
|
||||
{
|
||||
var pollingOptions = sp.GetRequiredService<Microsoft.Extensions.Options.IOptions<MasterPollingOptions>>();
|
||||
client.Timeout = TimeSpan.FromSeconds(pollingOptions.Value.HttpTimeoutSeconds);
|
||||
});
|
||||
services.AddHostedService<MasterStatusPollingBackgroundService>();
|
||||
}
|
||||
|
||||
public void UseModule(IApplicationBuilder app)
|
||||
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using SlpModularCms.Modules.Availability.Config;
|
||||
using SlpModularCms.Modules.Availability.Services;
|
||||
|
||||
namespace SlpModularCms.Modules.Availability.BackgroundServices;
|
||||
|
||||
/// <summary>
|
||||
/// Periodically pulls this slave's status from its registered master, so the slave's
|
||||
/// in-memory master-gate stays correct even without an explicit push from the master
|
||||
/// (e.g. after a slave restart, or if a push was missed). Fails open (Available) if the
|
||||
/// master has been unreachable for too long.
|
||||
/// </summary>
|
||||
public class MasterStatusPollingBackgroundService(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IOptions<MasterPollingOptions> options,
|
||||
ILogger<MasterStatusPollingBackgroundService> logger) : BackgroundService
|
||||
{
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
// Poll once immediately on startup, so a freshly (re)started slave re-syncs right away
|
||||
// instead of defaulting to Available until the first interval elapses.
|
||||
await ExecuteTickAsync(stoppingToken);
|
||||
|
||||
var interval = TimeSpan.FromSeconds(options.Value.PollIntervalSeconds);
|
||||
using var timer = new PeriodicTimer(interval);
|
||||
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken))
|
||||
{
|
||||
await ExecuteTickAsync(stoppingToken);
|
||||
}
|
||||
}
|
||||
|
||||
internal async Task ExecuteTickAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var masterAvailabilityService = scope.ServiceProvider.GetRequiredService<IMasterAvailabilityService>();
|
||||
var pollClient = scope.ServiceProvider.GetRequiredService<IMasterStatusPollClient>();
|
||||
|
||||
var target = await masterAvailabilityService.GetPollTargetAsync();
|
||||
if (target is null)
|
||||
{
|
||||
// No master registered yet; nothing to poll.
|
||||
return;
|
||||
}
|
||||
|
||||
var polled = await pollClient.GetStatusAsync(target.MasterUrl, target.PlainApiKey);
|
||||
if (polled is not null)
|
||||
{
|
||||
await masterAvailabilityService.ApplyPolledStatusAsync(polled.IsAvailable, polled.DisableMessage);
|
||||
return;
|
||||
}
|
||||
|
||||
var failOpenAfter = TimeSpan.FromMinutes(options.Value.FailOpenAfterMinutes);
|
||||
await masterAvailabilityService.RecordPollFailureAsync(failOpenAfter);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "Unhandled error during master status poll tick.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace SlpModularCms.Modules.Availability.Config;
|
||||
|
||||
[ExcludeFromCodeCoverage]
|
||||
public class MasterPollingOptions
|
||||
{
|
||||
/// <summary>How often the slave pulls its status from the master.</summary>
|
||||
public int PollIntervalSeconds { get; set; } = 30;
|
||||
|
||||
/// <summary>
|
||||
/// How long the master may stay unreachable before the slave gives up waiting and
|
||||
/// fails open (becomes Available again) instead of staying stuck on a stale status.
|
||||
/// </summary>
|
||||
public int FailOpenAfterMinutes { get; set; } = 5;
|
||||
|
||||
public int HttpTimeoutSeconds { get; set; } = 5;
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using SlpModularCms.Core.Availability;
|
||||
using SlpModularCms.Modules.Availability.Services;
|
||||
@@ -26,6 +27,7 @@ public class AvailabilityController : ControllerBase
|
||||
Status = details.Status.ToString(),
|
||||
CheckedAt = DateTimeOffset.UtcNow,
|
||||
Message = details.Message ?? string.Empty,
|
||||
details.IsMasterControlled,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -33,17 +35,28 @@ public class AvailabilityController : ControllerBase
|
||||
[Authorize(Policy = "OwnerOnly")]
|
||||
public async Task<IActionResult> UpdateStatus([FromBody] UpdateStatusRequest request)
|
||||
{
|
||||
if (_availabilityService is PersistentAvailabilityService persistentService)
|
||||
if (_availabilityService is not PersistentAvailabilityService persistentService)
|
||||
{
|
||||
await persistentService.UpdateStatusAsync(
|
||||
request.NewStatus,
|
||||
request.Reason,
|
||||
User.Identity?.Name);
|
||||
|
||||
return Ok();
|
||||
return BadRequest("Status update niet ondersteund door huidige service.");
|
||||
}
|
||||
|
||||
return BadRequest("Status update niet ondersteund door huidige service.");
|
||||
try
|
||||
{
|
||||
await persistentService.UpdateStatusAsync(
|
||||
request.NewStatus,
|
||||
request.Reason,
|
||||
User.Identity?.Name);
|
||||
|
||||
return Ok();
|
||||
}
|
||||
catch (MasterControlledAvailabilityException ex)
|
||||
{
|
||||
return Conflict(new ProblemDetails
|
||||
{
|
||||
Status = StatusCodes.Status409Conflict,
|
||||
Title = ex.Message,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -9,4 +9,7 @@ public class MasterRegistration
|
||||
public string ApiKey { get; set; } = string.Empty;
|
||||
public DateTimeOffset RegisteredAt { get; set; }
|
||||
public DateTimeOffset? LastContactedAt { get; set; }
|
||||
|
||||
/// <summary>Timestamp of the last time this slave successfully polled the master for its status.</summary>
|
||||
public DateTimeOffset? LastPolledAt { get; set; }
|
||||
}
|
||||
|
||||
@@ -23,12 +23,15 @@ public class AvailabilityMiddleware
|
||||
// Auth and Setup must stay open so admins can log in and the frontend
|
||||
// can determine whether the system is initialized.
|
||||
// Master endpoints bypass so master can always push status or re-register.
|
||||
// SlaveStatus bypasses so a slave can always pull the master's status, even if the
|
||||
// master instance is (for whatever reason) reporting itself as locally unavailable.
|
||||
private static readonly string[] _bypassPrefixes =
|
||||
[
|
||||
"/api/v1/Availability/status",
|
||||
"/api/v1/Auth/",
|
||||
"/api/v1/Setup/status",
|
||||
"/api/v1/master/",
|
||||
"/api/v1/SlaveStatus",
|
||||
];
|
||||
|
||||
public async Task InvokeAsync(
|
||||
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
// <auto-generated />
|
||||
using System;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Metadata;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||
using SlpModularCms.Modules.Availability.Data;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace SlpModularCms.Modules.Availability.Migrations
|
||||
{
|
||||
[DbContext(typeof(AvailabilityDbContext))]
|
||||
[Migration("20260704142458_AddLastPolledAtToMasterRegistration")]
|
||||
partial class AddLastPolledAtToMasterRegistration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
||||
{
|
||||
#pragma warning disable 612, 618
|
||||
modelBuilder
|
||||
.HasAnnotation("ProductVersion", "10.0.9")
|
||||
.HasAnnotation("Relational:MaxIdentifierLength", 128);
|
||||
|
||||
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
|
||||
|
||||
modelBuilder.Entity("SlpModularCms.Modules.Availability.Data.Entities.MasterRegistration", b =>
|
||||
{
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<string>("ApiKey")
|
||||
.IsRequired()
|
||||
.HasMaxLength(2000)
|
||||
.HasColumnType("nvarchar(2000)");
|
||||
|
||||
b.Property<DateTimeOffset?>("LastContactedAt")
|
||||
.HasColumnType("datetimeoffset");
|
||||
|
||||
b.Property<DateTimeOffset?>("LastPolledAt")
|
||||
.HasColumnType("datetimeoffset");
|
||||
|
||||
b.Property<string>("MasterUrl")
|
||||
.IsRequired()
|
||||
.HasMaxLength(500)
|
||||
.HasColumnType("nvarchar(500)");
|
||||
|
||||
b.Property<DateTimeOffset>("RegisteredAt")
|
||||
.HasColumnType("datetimeoffset");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("AvailabilityMasterRegistrations", (string)null);
|
||||
});
|
||||
#pragma warning restore 612, 618
|
||||
}
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
using System;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace SlpModularCms.Modules.Availability.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class AddLastPolledAtToMasterRegistration : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.AddColumn<DateTimeOffset>(
|
||||
name: "LastPolledAt",
|
||||
table: "AvailabilityMasterRegistrations",
|
||||
type: "datetimeoffset",
|
||||
nullable: true);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropColumn(
|
||||
name: "LastPolledAt",
|
||||
table: "AvailabilityMasterRegistrations");
|
||||
}
|
||||
}
|
||||
}
|
||||
+3
@@ -36,6 +36,9 @@ namespace SlpModularCms.Modules.Availability.Migrations
|
||||
b.Property<DateTimeOffset?>("LastContactedAt")
|
||||
.HasColumnType("datetimeoffset");
|
||||
|
||||
b.Property<DateTimeOffset?>("LastPolledAt")
|
||||
.HasColumnType("datetimeoffset");
|
||||
|
||||
b.Property<string>("MasterUrl")
|
||||
.IsRequired()
|
||||
.HasMaxLength(500)
|
||||
|
||||
@@ -6,4 +6,20 @@ public interface IMasterAvailabilityService
|
||||
Task<bool> PushStatusAsync(string apiKey, bool isAvailable, string? disableMessage);
|
||||
Task<string?> GetRegisteredUrlAsync(string apiKey);
|
||||
MasterGateStatus GetMasterStatus();
|
||||
|
||||
/// <summary>Returns the master URL and plain API key to poll, or null if no master is registered.</summary>
|
||||
Task<MasterPollTarget?> GetPollTargetAsync();
|
||||
|
||||
/// <summary>Applies a successfully polled status from the master and records the poll timestamp.</summary>
|
||||
Task ApplyPolledStatusAsync(bool isAvailable, string? disableMessage);
|
||||
|
||||
/// <summary>
|
||||
/// Called after a failed poll attempt. If the master has been unreachable for longer than
|
||||
/// <paramref name="failOpenAfter"/>, forces the gate open (Available) so a dead/unreachable
|
||||
/// master never permanently blocks the slave.
|
||||
/// </summary>
|
||||
Task RecordPollFailureAsync(TimeSpan failOpenAfter);
|
||||
}
|
||||
|
||||
[System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage]
|
||||
public record MasterPollTarget(string MasterUrl, string PlainApiKey);
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace SlpModularCms.Modules.Availability.Services;
|
||||
|
||||
public interface IMasterStatusPollClient
|
||||
{
|
||||
Task<PolledMasterStatus?> GetStatusAsync(string masterUrl, string plainApiKey);
|
||||
}
|
||||
|
||||
[System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage]
|
||||
public record PolledMasterStatus(bool IsAvailable, string? DisableMessage);
|
||||
@@ -72,6 +72,51 @@ public class MasterAvailabilityService : IMasterAvailabilityService
|
||||
return true;
|
||||
}
|
||||
|
||||
public async Task<MasterPollTarget?> GetPollTargetAsync()
|
||||
{
|
||||
var existing = await _deps.Repository.GetAsync();
|
||||
if (existing is null) return null;
|
||||
|
||||
var plainKey = _deps.KeyProtector.Unprotect(existing.ApiKey);
|
||||
if (plainKey is null) return null;
|
||||
|
||||
return new MasterPollTarget(existing.MasterUrl, plainKey);
|
||||
}
|
||||
|
||||
public async Task ApplyPolledStatusAsync(bool isAvailable, string? disableMessage)
|
||||
{
|
||||
_masterIsAvailable = isAvailable;
|
||||
_masterDisableMessage = disableMessage;
|
||||
|
||||
var existing = await _deps.Repository.GetAsync();
|
||||
if (existing is null) return;
|
||||
|
||||
existing.LastPolledAt = DateTimeOffset.UtcNow;
|
||||
existing.LastContactedAt = DateTimeOffset.UtcNow;
|
||||
_deps.Repository.Update(existing);
|
||||
await _deps.Repository.SaveChangesAsync();
|
||||
}
|
||||
|
||||
public async Task RecordPollFailureAsync(TimeSpan failOpenAfter)
|
||||
{
|
||||
var existing = await _deps.Repository.GetAsync();
|
||||
if (existing is null) return;
|
||||
|
||||
var unreachableSince = existing.LastPolledAt ?? existing.RegisteredAt;
|
||||
if (DateTimeOffset.UtcNow - unreachableSince < failOpenAfter)
|
||||
return;
|
||||
|
||||
if (_masterIsAvailable)
|
||||
return;
|
||||
|
||||
_deps.Logger.LogWarning(
|
||||
"Master unreachable since {UnreachableSince}; failing open (Available) after {FailOpenAfter}.",
|
||||
unreachableSince, failOpenAfter);
|
||||
|
||||
_masterIsAvailable = true;
|
||||
_masterDisableMessage = null;
|
||||
}
|
||||
|
||||
public async Task<string?> GetRegisteredUrlAsync(string apiKey)
|
||||
{
|
||||
var existing = await _deps.Repository.GetAsync();
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace SlpModularCms.Modules.Availability.Services;
|
||||
|
||||
public class MasterStatusPollClient(HttpClient httpClient) : IMasterStatusPollClient
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
public async Task<PolledMasterStatus?> GetStatusAsync(string masterUrl, string plainApiKey)
|
||||
{
|
||||
try
|
||||
{
|
||||
var request = new HttpRequestMessage(HttpMethod.Get, $"{masterUrl.TrimEnd('/')}/api/v1/SlaveStatus");
|
||||
request.Headers.Add("X-Master-Api-Key", plainApiKey);
|
||||
|
||||
var response = await httpClient.SendAsync(request);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
return null;
|
||||
|
||||
var result = await response.Content.ReadFromJsonAsync<SlaveStatusResponse>(JsonOptions);
|
||||
return result is null ? null : new PolledMasterStatus(result.IsAvailable, result.DisableMessage);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private record SlaveStatusResponse(bool IsAvailable, string? DisableMessage);
|
||||
}
|
||||
@@ -13,15 +13,20 @@ public class PersistentAvailabilityService : IAvailabilityService
|
||||
{
|
||||
private readonly ApplicationDbContext _context;
|
||||
private readonly AvailabilityOptions _options;
|
||||
|
||||
private readonly IMasterAvailabilityService _masterAvailabilityService;
|
||||
|
||||
// Circuit Breaker state
|
||||
private static DateTimeOffset _lastErrorTime = DateTimeOffset.MinValue;
|
||||
private static AvailabilityStatus _cachedStatus = AvailabilityStatus.Available;
|
||||
|
||||
public PersistentAvailabilityService(ApplicationDbContext context, IOptions<AvailabilityOptions> options)
|
||||
public PersistentAvailabilityService(
|
||||
ApplicationDbContext context,
|
||||
IOptions<AvailabilityOptions> options,
|
||||
IMasterAvailabilityService masterAvailabilityService)
|
||||
{
|
||||
_context = context;
|
||||
_options = options.Value;
|
||||
_masterAvailabilityService = masterAvailabilityService;
|
||||
}
|
||||
|
||||
public async Task<AvailabilityStatus> IsAvailableAsync()
|
||||
@@ -56,6 +61,12 @@ public class PersistentAvailabilityService : IAvailabilityService
|
||||
|
||||
public async Task<AvailabilityStatusDetails> GetStatusDetailsAsync()
|
||||
{
|
||||
var masterStatus = _masterAvailabilityService.GetMasterStatus();
|
||||
if (!masterStatus.IsAvailable)
|
||||
{
|
||||
return new AvailabilityStatusDetails(AvailabilityStatus.NotAvailable, masterStatus.DisableMessage, IsMasterControlled: true);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var state = await _context.AvailabilityStates.FirstOrDefaultAsync();
|
||||
@@ -77,6 +88,12 @@ public class PersistentAvailabilityService : IAvailabilityService
|
||||
/// </summary>
|
||||
public async Task UpdateStatusAsync(AvailabilityStatus newStatus, string? reason, string? updatedBy)
|
||||
{
|
||||
var masterStatus = _masterAvailabilityService.GetMasterStatus();
|
||||
if (!masterStatus.IsAvailable)
|
||||
{
|
||||
throw new MasterControlledAvailabilityException(masterStatus.DisableMessage);
|
||||
}
|
||||
|
||||
var state = await _context.AvailabilityStates.FirstOrDefaultAsync();
|
||||
|
||||
if (state == null)
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
using FluentAssertions;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using NSubstitute;
|
||||
using SlpModularCms.Modules.Master.Controllers;
|
||||
using SlpModularCms.Modules.Master.Services;
|
||||
|
||||
namespace SlpModularCms.Modules.Master.Tests.Controllers;
|
||||
|
||||
public class SlaveStatusControllerTests
|
||||
{
|
||||
private readonly ICmsInstanceService _service = Substitute.For<ICmsInstanceService>();
|
||||
private readonly SlaveStatusController _controller;
|
||||
|
||||
public SlaveStatusControllerTests()
|
||||
{
|
||||
_controller = new SlaveStatusController(_service);
|
||||
_controller.ControllerContext = new ControllerContext
|
||||
{
|
||||
HttpContext = new DefaultHttpContext()
|
||||
};
|
||||
}
|
||||
|
||||
private void SetApiKeyHeader(string? value)
|
||||
{
|
||||
if (value != null)
|
||||
_controller.HttpContext.Request.Headers["X-Master-Api-Key"] = value;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Get_Returns401_WhenHeaderMissing()
|
||||
{
|
||||
var result = await _controller.Get();
|
||||
|
||||
result.Should().BeOfType<UnauthorizedResult>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Get_Returns401_WhenKeyDoesNotMatchAnyInstance()
|
||||
{
|
||||
SetApiKeyHeader("wrong-key");
|
||||
_service.GetStatusForApiKeyAsync("wrong-key").Returns((SlaveStatusPollResponse?)null);
|
||||
|
||||
var result = await _controller.Get();
|
||||
|
||||
result.Should().BeOfType<UnauthorizedResult>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Get_Returns200_WithStatus_WhenKeyMatches()
|
||||
{
|
||||
SetApiKeyHeader("key123");
|
||||
_service.GetStatusForApiKeyAsync("key123").Returns(new SlaveStatusPollResponse(false, "Onderhoud"));
|
||||
|
||||
var result = await _controller.Get();
|
||||
|
||||
var ok = result.Should().BeOfType<OkObjectResult>().Subject;
|
||||
ok.StatusCode.Should().Be(200);
|
||||
}
|
||||
}
|
||||
@@ -140,16 +140,32 @@ public class CmsInstanceServiceTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpdateStatusAsync_DoesNotPushToSlave_WhenStatusIsInactive()
|
||||
public async Task UpdateStatusAsync_ReleasesMasterGate_WhenStatusIsInactive()
|
||||
{
|
||||
var instance = ActiveInstance();
|
||||
_repo.GetByIdAsync(instance.Id).Returns(instance);
|
||||
_protector.Unprotect("encrypted-key").Returns("plain");
|
||||
_slaveClient.PushStatusAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<bool>(), Arg.Any<string?>()).Returns(true);
|
||||
|
||||
var result = await CreateSut().UpdateStatusAsync(instance.Id, new UpdateStatusRequest(CmsInstanceStatus.Inactive, null));
|
||||
|
||||
result.Success.Should().BeTrue();
|
||||
result.SlaveContactSuccess.Should().BeTrue();
|
||||
await _slaveClient.DidNotReceive().PushStatusAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<bool>(), Arg.Any<string?>());
|
||||
await _slaveClient.Received(1).PushStatusAsync("https://slave.test", "plain", true, null);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpdateStatusAsync_ReturnsSlaveContactFalse_WhenReleasingMasterGateFails()
|
||||
{
|
||||
var instance = ActiveInstance();
|
||||
_repo.GetByIdAsync(instance.Id).Returns(instance);
|
||||
_protector.Unprotect("encrypted-key").Returns("plain");
|
||||
_slaveClient.PushStatusAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<bool>(), Arg.Any<string?>()).Returns(false);
|
||||
|
||||
var result = await CreateSut().UpdateStatusAsync(instance.Id, new UpdateStatusRequest(CmsInstanceStatus.Inactive, null));
|
||||
|
||||
result.Success.Should().BeTrue();
|
||||
result.SlaveContactSuccess.Should().BeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -180,6 +196,47 @@ public class CmsInstanceServiceTests
|
||||
result.SlaveContactSuccess.Should().BeFalse();
|
||||
}
|
||||
|
||||
// --- GetStatusForApiKeyAsync ---
|
||||
|
||||
[Fact]
|
||||
public async Task GetStatusForApiKeyAsync_ReturnsStatus_WhenKeyMatchesAnInstance()
|
||||
{
|
||||
var instance = ActiveInstance();
|
||||
instance.Status = CmsInstanceStatus.NotAvailable;
|
||||
instance.DisableMessage = "Onderhoud";
|
||||
_repo.GetActiveAsync().Returns([instance]);
|
||||
_protector.Unprotect("encrypted-key").Returns("plain");
|
||||
|
||||
var result = await CreateSut().GetStatusForApiKeyAsync("plain");
|
||||
|
||||
result.Should().NotBeNull();
|
||||
result!.IsAvailable.Should().BeFalse();
|
||||
result.DisableMessage.Should().Be("Onderhoud");
|
||||
_repo.Received(1).Update(Arg.Is<CmsInstance>(i => i.LastContactedAt.HasValue));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetStatusForApiKeyAsync_ReturnsNull_WhenNoInstanceMatchesKey()
|
||||
{
|
||||
var instance = ActiveInstance();
|
||||
_repo.GetActiveAsync().Returns([instance]);
|
||||
_protector.Unprotect("encrypted-key").Returns("some-other-key");
|
||||
|
||||
var result = await CreateSut().GetStatusForApiKeyAsync("plain");
|
||||
|
||||
result.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetStatusForApiKeyAsync_ReturnsNull_WhenNoActiveInstances()
|
||||
{
|
||||
_repo.GetActiveAsync().Returns(Array.Empty<CmsInstance>());
|
||||
|
||||
var result = await CreateSut().GetStatusForApiKeyAsync("plain");
|
||||
|
||||
result.Should().BeNull();
|
||||
}
|
||||
|
||||
// --- VerifyIntegrityAsync ---
|
||||
|
||||
[Fact]
|
||||
@@ -239,6 +296,24 @@ public class CmsInstanceServiceTests
|
||||
_repo.Received().Update(Arg.Is<CmsInstance>(i => i.LastIntegrityCheckFailedAt == null));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task VerifyIntegrityAsync_RePushesPersistedStatus_ToResyncSlaveAfterRestart()
|
||||
{
|
||||
var instance = ActiveInstance();
|
||||
instance.Status = CmsInstanceStatus.NotAvailable;
|
||||
instance.DisableMessage = "Onderhoud";
|
||||
_repo.GetActiveAsync().Returns([instance]);
|
||||
_protector.Unprotect("encrypted-key").Returns("plain");
|
||||
_slaveClient.GetRegisteredMasterUrlAsync(Arg.Any<string>(), Arg.Any<string>()).Returns("https://master.test");
|
||||
_slaveClient.PushStatusAsync(Arg.Any<string>(), Arg.Any<string>(), Arg.Any<bool>(), Arg.Any<string?>()).Returns(true);
|
||||
_httpContextAccessor.HttpContext.Returns((HttpContext?)null);
|
||||
|
||||
await CreateSut().VerifyIntegrityAsync();
|
||||
|
||||
await _slaveClient.Received(1).PushStatusAsync("https://slave.test", "plain", false, "Onderhoud");
|
||||
_repo.Received().Update(Arg.Is<CmsInstance>(i => i.LastStatusPushedAt.HasValue));
|
||||
}
|
||||
|
||||
private static CmsInstance ActiveInstance() => new()
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
|
||||
@@ -14,6 +14,10 @@ public class IntegrityCheckBackgroundService(
|
||||
{
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
// Run once immediately on startup so a freshly (re)started slave has its
|
||||
// master-gate status re-synced right away, instead of waiting up to a full interval.
|
||||
await ExecuteTickAsync(stoppingToken);
|
||||
|
||||
var interval = TimeSpan.FromMinutes(options.Value.IntegrityCheckIntervalMinutes);
|
||||
using var timer = new PeriodicTimer(interval);
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using SlpModularCms.Modules.Master.Services;
|
||||
|
||||
namespace SlpModularCms.Modules.Master.Controllers;
|
||||
|
||||
/// <summary>
|
||||
/// Lets a registered slave pull its own configured status from the master, using the
|
||||
/// same shared API key used for master-to-slave calls. This is the counterpart of the
|
||||
/// master-initiated push in CmsInstanceService.UpdateStatusAsync/VerifyIntegrityAsync,
|
||||
/// letting a slave self-heal its master-gate state (e.g. after a restart) instead of
|
||||
/// relying solely on the master successfully reaching it.
|
||||
/// </summary>
|
||||
[ApiController]
|
||||
[Route("SlaveStatus")]
|
||||
[AllowAnonymous]
|
||||
public class SlaveStatusController(ICmsInstanceService service) : ControllerBase
|
||||
{
|
||||
[HttpGet]
|
||||
public async Task<IActionResult> Get()
|
||||
{
|
||||
var apiKey = Request.Headers["X-Master-Api-Key"].FirstOrDefault();
|
||||
if (string.IsNullOrEmpty(apiKey)) return Unauthorized();
|
||||
|
||||
var result = await service.GetStatusForApiKeyAsync(apiKey);
|
||||
if (result is null) return Unauthorized();
|
||||
|
||||
return Ok(new { result.IsAvailable, result.DisableMessage });
|
||||
}
|
||||
}
|
||||
@@ -70,10 +70,28 @@ public class CmsInstanceService(MasterServiceDependencies deps) : ICmsInstanceSe
|
||||
deps.Repository.Update(instance);
|
||||
await deps.Repository.SaveChangesAsync();
|
||||
|
||||
if (request.Status == CmsInstanceStatus.Inactive)
|
||||
return new UpdateStatusResult(Success: true, SlaveContactSuccess: true);
|
||||
|
||||
var plainKey = deps.ApiKeyProtector.Unprotect(instance.ApiKey);
|
||||
|
||||
if (request.Status == CmsInstanceStatus.Inactive)
|
||||
{
|
||||
// The master no longer manages this slave, so release the master gate
|
||||
// instead of leaving it stuck on whatever status was last pushed.
|
||||
var released = await deps.SlaveClient.PushStatusAsync(instance.Url, plainKey, isAvailable: true, disableMessage: null);
|
||||
|
||||
if (released)
|
||||
{
|
||||
instance.LastStatusPushedAt = DateTimeOffset.UtcNow;
|
||||
deps.Repository.Update(instance);
|
||||
await deps.Repository.SaveChangesAsync();
|
||||
}
|
||||
else
|
||||
{
|
||||
deps.Logger.LogError("Failed to release master gate on deactivated slave {SlaveUrl}", instance.Url);
|
||||
}
|
||||
|
||||
return new UpdateStatusResult(Success: true, SlaveContactSuccess: released);
|
||||
}
|
||||
|
||||
var pushed = await deps.SlaveClient.PushStatusAsync(
|
||||
instance.Url,
|
||||
plainKey,
|
||||
@@ -94,6 +112,35 @@ public class CmsInstanceService(MasterServiceDependencies deps) : ICmsInstanceSe
|
||||
return new UpdateStatusResult(Success: true, SlaveContactSuccess: pushed);
|
||||
}
|
||||
|
||||
public async Task<SlaveStatusPollResponse?> GetStatusForApiKeyAsync(string plainApiKey)
|
||||
{
|
||||
var instances = await deps.Repository.GetActiveAsync();
|
||||
|
||||
foreach (var instance in instances)
|
||||
{
|
||||
string? candidateKey;
|
||||
try
|
||||
{
|
||||
candidateKey = deps.ApiKeyProtector.Unprotect(instance.ApiKey);
|
||||
}
|
||||
catch
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (candidateKey != plainApiKey)
|
||||
continue;
|
||||
|
||||
instance.LastContactedAt = DateTimeOffset.UtcNow;
|
||||
deps.Repository.Update(instance);
|
||||
await deps.Repository.SaveChangesAsync();
|
||||
|
||||
return new SlaveStatusPollResponse(instance.Status == CmsInstanceStatus.Available, instance.DisableMessage);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public async Task VerifyIntegrityAsync()
|
||||
{
|
||||
var masterUrl = ResolveMasterUrl();
|
||||
@@ -140,6 +187,20 @@ public class CmsInstanceService(MasterServiceDependencies deps) : ICmsInstanceSe
|
||||
instance.LastIntegrityCheckFailedAt = null;
|
||||
}
|
||||
|
||||
// The slave keeps its master-gate status in memory only, so a slave restart
|
||||
// silently resets it to "available" until the next explicit status change.
|
||||
// Re-push the master's persisted status every check to keep it in sync.
|
||||
var pushed = await deps.SlaveClient.PushStatusAsync(
|
||||
instance.Url,
|
||||
plainKey,
|
||||
isAvailable: instance.Status == CmsInstanceStatus.Available,
|
||||
disableMessage: instance.DisableMessage);
|
||||
|
||||
if (pushed)
|
||||
{
|
||||
instance.LastStatusPushedAt = DateTimeOffset.UtcNow;
|
||||
}
|
||||
|
||||
deps.Repository.Update(instance);
|
||||
await deps.Repository.SaveChangesAsync();
|
||||
}
|
||||
|
||||
@@ -8,4 +8,12 @@ public interface ICmsInstanceService
|
||||
Task<CmsInstanceDto> AddAsync(CreateCmsInstanceRequest request);
|
||||
Task<UpdateStatusResult> UpdateStatusAsync(Guid id, UpdateStatusRequest request);
|
||||
Task VerifyIntegrityAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Looks up the registered CMS instance by its plain (unprotected) API key, for a slave
|
||||
/// pulling its own status. Returns null when no instance's key matches (unauthorized).
|
||||
/// </summary>
|
||||
Task<SlaveStatusPollResponse?> GetStatusForApiKeyAsync(string plainApiKey);
|
||||
}
|
||||
|
||||
public record SlaveStatusPollResponse(bool IsAvailable, string? DisableMessage);
|
||||
|
||||
Reference in New Issue
Block a user