U6: SpaFallback-SEO: Website:NoIndex (X-Robots-Tag), robots.txt, sitemap.xml en KnownPaths (echte 404)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-11 23:44:04 +01:00
co-authored by Claude Sonnet 5.5
parent 0ef6e74671
commit a7460a0f9b
6 changed files with 330 additions and 0 deletions
@@ -0,0 +1,154 @@
using System.Net;
using FluentAssertions;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using SlpModularCms.Core.Hosting;
using SlpModularCms.Core.Hosting.Security;
using SlpModularCms.Core.Hosting.Website;
using Xunit;
namespace SlpModularCms.Core.Tests.Hosting.Website;
/// <summary>SEO behaviour of a SpaFallback site: noindex on test hosts, robots.txt, sitemap.xml, KnownPaths.</summary>
public sealed class SpaSeoTests : IDisposable
{
private readonly string _root = Path.Combine(Path.GetTempPath(), "slp-spa-" + Guid.NewGuid().ToString("N"));
private readonly List<WebApplication> _apps = [];
public SpaSeoTests()
{
var web = Path.Combine(_root, "wwwroot", "web");
Directory.CreateDirectory(web);
File.WriteAllText(Path.Combine(web, "index.html"), "<!doctype html><title>SPA</title>");
}
public void Dispose()
{
foreach (var app in _apps) app.DisposeAsync().AsTask().GetAwaiter().GetResult();
if (Directory.Exists(_root)) Directory.Delete(_root, recursive: true);
}
private async Task<HttpClient> StartAsync(Dictionary<string, string?> settings)
{
var builder = WebApplication.CreateBuilder(new WebApplicationOptions
{
EnvironmentName = Environments.Production,
ContentRootPath = _root,
WebRootPath = Path.Combine(_root, "wwwroot"),
});
builder.WebHost.UseTestServer();
builder.Logging.ClearProviders();
settings["SecurityHeaders:DefaultPolicy"] = "StrictSite";
builder.Configuration.AddInMemoryCollection(settings);
builder.Services.AddCmsSecurityHeaders(builder.Configuration);
builder.Services.AddCmsSpaSeo(builder.Configuration);
var app = builder.Build();
app.UseCmsSecurityHeaders();
app.UseCmsSpaSeo();
app.UseCmsStaticContent(WebsiteMode.SpaFallback);
app.MapGet("/api/v1/ping", () => Results.Text("pong"));
app.MapCmsSpaFallbacks(WebsiteMode.SpaFallback);
await app.StartAsync();
_apps.Add(app);
return app.GetTestClient();
}
private static Dictionary<string, string?> Config(bool noIndex, string? baseUrl = null) => new()
{
["Website:NoIndex"] = noIndex ? "true" : "false",
["Website:PublicBaseUrl"] = baseUrl,
["Website:Sitemap:Paths:0"] = "/",
["Website:Sitemap:Paths:1"] = "/privacy",
};
private static bool Has(HttpResponseMessage r, string name, string value) =>
r.Headers.TryGetValues(name, out var v) && v.Contains(value);
[Fact]
public async Task OnATestHost_EveryResponseCarriesNoindex_AndRobotsDisallowsAll()
{
var client = await StartAsync(Config(noIndex: true, baseUrl: "https://slpsoftware.nl"));
foreach (var path in new[] { "/", "/privacy", "/api/v1/ping", "/robots.txt", "/missing.js" })
{
Has(await client.GetAsync(path), "X-Robots-Tag", "noindex, nofollow").Should().BeTrue(path);
}
var robots = await client.GetAsync("/robots.txt");
robots.StatusCode.Should().Be(HttpStatusCode.OK);
(await robots.Content.ReadAsStringAsync()).Should().Be("User-agent: *\nDisallow: /\n");
(await client.GetAsync("/sitemap.xml")).StatusCode.Should().Be(HttpStatusCode.NotFound);
}
[Fact]
public async Task InProduction_RobotsAllowsAndPointsToTheSitemap_AndThereIsNoNoindex()
{
var client = await StartAsync(Config(noIndex: false, baseUrl: "https://slpsoftware.nl/"));
var page = await client.GetAsync("/");
page.Headers.Contains("X-Robots-Tag").Should().BeFalse();
var robots = await client.GetStringAsync("/robots.txt");
robots.Should().Contain("Allow: /").And.Contain("Sitemap: https://slpsoftware.nl/sitemap.xml");
robots.Should().NotContain("Disallow");
}
[Fact]
public async Task TheSitemap_ListsTheConfiguredPaths_WithAnAbsoluteBase()
{
var client = await StartAsync(Config(noIndex: false, baseUrl: "https://slpsoftware.nl"));
var response = await client.GetAsync("/sitemap.xml");
response.StatusCode.Should().Be(HttpStatusCode.OK);
response.Content.Headers.ContentType!.MediaType.Should().Be("application/xml");
var xml = await response.Content.ReadAsStringAsync();
xml.Should().Contain("<loc>https://slpsoftware.nl/</loc>")
.And.Contain("<loc>https://slpsoftware.nl/privacy</loc>")
.And.NotContain("lastmod");
}
[Fact]
public async Task WithoutABaseUrl_ThereIsNoSitemap_AndNoGeneratedRobots()
{
var client = await StartAsync(Config(noIndex: false));
(await client.GetAsync("/sitemap.xml")).StatusCode.Should().Be(HttpStatusCode.NotFound);
(await client.GetAsync("/robots.txt")).StatusCode.Should().Be(HttpStatusCode.NotFound);
}
[Fact]
public async Task APrivacyPath_ServesTheSite_AndWithoutKnownPathsEverythingIsTheSpa()
{
var client = await StartAsync(Config(noIndex: false));
var privacy = await client.GetAsync("/privacy");
privacy.StatusCode.Should().Be(HttpStatusCode.OK);
(await privacy.Content.ReadAsStringAsync()).Should().Contain("<title>SPA</title>");
(await client.GetAsync("/anything")).StatusCode.Should().Be(HttpStatusCode.OK);
}
[Fact]
public async Task WithKnownPaths_OnlyThoseAre200_TheRestIs404_AndApiStillWorks()
{
var settings = Config(noIndex: false);
settings["Website:KnownPaths:0"] = "/";
settings["Website:KnownPaths:1"] = "/privacy";
var client = await StartAsync(settings);
(await client.GetAsync("/")).StatusCode.Should().Be(HttpStatusCode.OK);
(await client.GetAsync("/privacy")).StatusCode.Should().Be(HttpStatusCode.OK);
(await client.GetAsync("/Privacy/")).StatusCode.Should().Be(HttpStatusCode.OK);
var missing = await client.GetAsync("/nope");
missing.StatusCode.Should().Be(HttpStatusCode.NotFound);
(await missing.Content.ReadAsStringAsync()).Should().Contain("<title>SPA</title>");
(await client.GetStringAsync("/api/v1/ping")).Should().Be("pong");
}
}