U6: SpaFallback-SEO: Website:NoIndex (X-Robots-Tag), robots.txt, sitemap.xml en KnownPaths (echte 404)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-11 23:44:04 +01:00
co-authored by Claude Sonnet 5.5
parent 0ef6e74671
commit a7460a0f9b
6 changed files with 330 additions and 0 deletions
@@ -9,6 +9,16 @@
}
},
"AllowedHosts": "*",
"Website": {
// Secure default: every response carries noindex and /robots.txt disallows all. A production
// instance must set Website__NoIndex=false and Website__PublicBaseUrl=https://slpsoftware.nl explicitly;
// robots.txt then allows crawling and points to /sitemap.xml.
"NoIndex": true,
"PublicBaseUrl": "",
"Sitemap": { "Paths": [ "/", "/privacy" ] },
// Only these website paths are pages (200); any other path is a real 404. /api, /admin and assets keep working.
"KnownPaths": [ "/", "/privacy" ]
},
"ConnectionStrings": {
"DefaultConnection": "Server=<production-db-host>;Port=3306;Database=SlpSoftware<Env>;Uid=<db-user>;Pwd=<db-password>"
},
@@ -0,0 +1,154 @@
using System.Net;
using FluentAssertions;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using SlpModularCms.Core.Hosting;
using SlpModularCms.Core.Hosting.Security;
using SlpModularCms.Core.Hosting.Website;
using Xunit;
namespace SlpModularCms.Core.Tests.Hosting.Website;
/// <summary>SEO behaviour of a SpaFallback site: noindex on test hosts, robots.txt, sitemap.xml, KnownPaths.</summary>
public sealed class SpaSeoTests : IDisposable
{
private readonly string _root = Path.Combine(Path.GetTempPath(), "slp-spa-" + Guid.NewGuid().ToString("N"));
private readonly List<WebApplication> _apps = [];
public SpaSeoTests()
{
var web = Path.Combine(_root, "wwwroot", "web");
Directory.CreateDirectory(web);
File.WriteAllText(Path.Combine(web, "index.html"), "<!doctype html><title>SPA</title>");
}
public void Dispose()
{
foreach (var app in _apps) app.DisposeAsync().AsTask().GetAwaiter().GetResult();
if (Directory.Exists(_root)) Directory.Delete(_root, recursive: true);
}
private async Task<HttpClient> StartAsync(Dictionary<string, string?> settings)
{
var builder = WebApplication.CreateBuilder(new WebApplicationOptions
{
EnvironmentName = Environments.Production,
ContentRootPath = _root,
WebRootPath = Path.Combine(_root, "wwwroot"),
});
builder.WebHost.UseTestServer();
builder.Logging.ClearProviders();
settings["SecurityHeaders:DefaultPolicy"] = "StrictSite";
builder.Configuration.AddInMemoryCollection(settings);
builder.Services.AddCmsSecurityHeaders(builder.Configuration);
builder.Services.AddCmsSpaSeo(builder.Configuration);
var app = builder.Build();
app.UseCmsSecurityHeaders();
app.UseCmsSpaSeo();
app.UseCmsStaticContent(WebsiteMode.SpaFallback);
app.MapGet("/api/v1/ping", () => Results.Text("pong"));
app.MapCmsSpaFallbacks(WebsiteMode.SpaFallback);
await app.StartAsync();
_apps.Add(app);
return app.GetTestClient();
}
private static Dictionary<string, string?> Config(bool noIndex, string? baseUrl = null) => new()
{
["Website:NoIndex"] = noIndex ? "true" : "false",
["Website:PublicBaseUrl"] = baseUrl,
["Website:Sitemap:Paths:0"] = "/",
["Website:Sitemap:Paths:1"] = "/privacy",
};
private static bool Has(HttpResponseMessage r, string name, string value) =>
r.Headers.TryGetValues(name, out var v) && v.Contains(value);
[Fact]
public async Task OnATestHost_EveryResponseCarriesNoindex_AndRobotsDisallowsAll()
{
var client = await StartAsync(Config(noIndex: true, baseUrl: "https://slpsoftware.nl"));
foreach (var path in new[] { "/", "/privacy", "/api/v1/ping", "/robots.txt", "/missing.js" })
{
Has(await client.GetAsync(path), "X-Robots-Tag", "noindex, nofollow").Should().BeTrue(path);
}
var robots = await client.GetAsync("/robots.txt");
robots.StatusCode.Should().Be(HttpStatusCode.OK);
(await robots.Content.ReadAsStringAsync()).Should().Be("User-agent: *\nDisallow: /\n");
(await client.GetAsync("/sitemap.xml")).StatusCode.Should().Be(HttpStatusCode.NotFound);
}
[Fact]
public async Task InProduction_RobotsAllowsAndPointsToTheSitemap_AndThereIsNoNoindex()
{
var client = await StartAsync(Config(noIndex: false, baseUrl: "https://slpsoftware.nl/"));
var page = await client.GetAsync("/");
page.Headers.Contains("X-Robots-Tag").Should().BeFalse();
var robots = await client.GetStringAsync("/robots.txt");
robots.Should().Contain("Allow: /").And.Contain("Sitemap: https://slpsoftware.nl/sitemap.xml");
robots.Should().NotContain("Disallow");
}
[Fact]
public async Task TheSitemap_ListsTheConfiguredPaths_WithAnAbsoluteBase()
{
var client = await StartAsync(Config(noIndex: false, baseUrl: "https://slpsoftware.nl"));
var response = await client.GetAsync("/sitemap.xml");
response.StatusCode.Should().Be(HttpStatusCode.OK);
response.Content.Headers.ContentType!.MediaType.Should().Be("application/xml");
var xml = await response.Content.ReadAsStringAsync();
xml.Should().Contain("<loc>https://slpsoftware.nl/</loc>")
.And.Contain("<loc>https://slpsoftware.nl/privacy</loc>")
.And.NotContain("lastmod");
}
[Fact]
public async Task WithoutABaseUrl_ThereIsNoSitemap_AndNoGeneratedRobots()
{
var client = await StartAsync(Config(noIndex: false));
(await client.GetAsync("/sitemap.xml")).StatusCode.Should().Be(HttpStatusCode.NotFound);
(await client.GetAsync("/robots.txt")).StatusCode.Should().Be(HttpStatusCode.NotFound);
}
[Fact]
public async Task APrivacyPath_ServesTheSite_AndWithoutKnownPathsEverythingIsTheSpa()
{
var client = await StartAsync(Config(noIndex: false));
var privacy = await client.GetAsync("/privacy");
privacy.StatusCode.Should().Be(HttpStatusCode.OK);
(await privacy.Content.ReadAsStringAsync()).Should().Contain("<title>SPA</title>");
(await client.GetAsync("/anything")).StatusCode.Should().Be(HttpStatusCode.OK);
}
[Fact]
public async Task WithKnownPaths_OnlyThoseAre200_TheRestIs404_AndApiStillWorks()
{
var settings = Config(noIndex: false);
settings["Website:KnownPaths:0"] = "/";
settings["Website:KnownPaths:1"] = "/privacy";
var client = await StartAsync(settings);
(await client.GetAsync("/")).StatusCode.Should().Be(HttpStatusCode.OK);
(await client.GetAsync("/privacy")).StatusCode.Should().Be(HttpStatusCode.OK);
(await client.GetAsync("/Privacy/")).StatusCode.Should().Be(HttpStatusCode.OK);
var missing = await client.GetAsync("/nope");
missing.StatusCode.Should().Be(HttpStatusCode.NotFound);
(await missing.Content.ReadAsStringAsync()).Should().Contain("<title>SPA</title>");
(await client.GetStringAsync("/api/v1/ping")).Should().Be("pong");
}
}
+11
View File
@@ -63,6 +63,10 @@ public static class CmsHost
{
builder.Services.AddCmsWebsite(builder.Configuration);
}
else
{
builder.Services.AddCmsSpaSeo(builder.Configuration);
}
// Registered BEFORE module services: modules must not configure Data Protection themselves,
// because a later registration would override this persistent key store (see
@@ -115,6 +119,13 @@ public static class CmsHost
// which is almost all of the HTML this host serves.
app.UseCmsSecurityHeaders();
// SpaFallback sites: X-Robots-Tag on test hosts and generated robots.txt / sitemap.xml. After the
// security headers (so these answers carry them) and before the static files short-circuit.
if (options.WebsiteMode == WebsiteMode.SpaFallback)
{
app.UseCmsSpaSeo();
}
app.UseRateLimiter();
// 6. Configure Pipeline
@@ -4,6 +4,8 @@ using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.FileProviders;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using SlpModularCms.Core.Hosting.Website;
namespace SlpModularCms.Core.Hosting;
@@ -136,6 +138,15 @@ public static class StaticContentExtensions
{
if (File.Exists(websiteIndex))
{
// Website:KnownPaths (optional): only those paths are real pages; the rest is a real 404, still
// with the SPA shell as body so the app can render its own not-found page.
var known = context.RequestServices.GetService<IOptions<WebsiteOptions>>()?.Value.KnownPaths;
if (known is { Count: > 0 }
&& !known.Select(SpaSeoExtensions.NormalizePath).Contains(SpaSeoExtensions.NormalizePath(context.Request.Path.Value ?? "/")))
{
context.Response.StatusCode = StatusCodes.Status404NotFound;
}
context.Response.ContentType = "text/html; charset=utf-8";
await context.Response.SendFileAsync(websiteIndex);
return;
@@ -0,0 +1,123 @@
using System.Net;
using System.Text;
using System.Xml;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using Microsoft.Net.Http.Headers;
namespace SlpModularCms.Core.Hosting.Website;
/// <summary>
/// SEO behaviour of a <see cref="WebsiteMode.SpaFallback"/> site, driven by the <c>Website</c> configuration section:
/// <c>NoIndex</c> (test hosts), generated <c>/robots.txt</c> and <c>/sitemap.xml</c>. Not part of the frontend bundle.
/// </summary>
public static class SpaSeoExtensions
{
internal const string RobotsTagValue = "noindex, nofollow";
internal const string DisallowAll = "User-agent: *\nDisallow: /\n";
public static IServiceCollection AddCmsSpaSeo(this IServiceCollection services, IConfiguration configuration)
{
services.AddOptions<WebsiteOptions>().Bind(configuration.GetSection(WebsiteOptions.SectionName));
return services;
}
/// <summary>Place directly after the security headers, before the static files short-circuit the pipeline.</summary>
public static WebApplication UseCmsSpaSeo(this WebApplication app)
{
var options = app.Services.GetRequiredService<IOptions<WebsiteOptions>>();
app.Use(async (context, next) =>
{
var website = options.Value;
if (website.NoIndex)
{
context.Response.Headers["X-Robots-Tag"] = RobotsTagValue;
}
var isRead = HttpMethods.IsGet(context.Request.Method) || HttpMethods.IsHead(context.Request.Method);
if (isRead)
{
var path = context.Request.Path.Value;
if (string.Equals(path, "/robots.txt", StringComparison.OrdinalIgnoreCase)
&& Robots(website) is { } robots)
{
await WriteAsync(context, "text/plain; charset=utf-8", robots);
return;
}
if (string.Equals(path, "/sitemap.xml", StringComparison.OrdinalIgnoreCase)
&& Sitemap(website) is { } sitemap)
{
await WriteAsync(context, "application/xml; charset=utf-8", sitemap);
return;
}
}
await next();
});
return app;
}
internal static string? Robots(WebsiteOptions o)
{
if (o.NoIndex)
{
return DisallowAll;
}
var baseUrl = BaseUrl(o);
return baseUrl is null ? null : $"User-agent: *\nAllow: /\n\nSitemap: {baseUrl}/sitemap.xml\n";
}
internal static string? Sitemap(WebsiteOptions o)
{
var baseUrl = BaseUrl(o);
if (o.NoIndex || baseUrl is null || o.Sitemap.Paths.Count == 0)
{
return null;
}
var xml = new StringBuilder("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
xml.Append("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n");
foreach (var path in o.Sitemap.Paths.Select(NormalizePath).Distinct(StringComparer.Ordinal))
{
var loc = path == "/" ? baseUrl + "/" : baseUrl + path;
xml.Append(" <url><loc>").Append(WebUtility.HtmlEncode(loc)).Append("</loc></url>\n");
}
return xml.Append("</urlset>\n").ToString();
}
/// <summary>The base URL without a trailing slash, or null when it is not a valid absolute http(s) URL.</summary>
internal static string? BaseUrl(WebsiteOptions o) =>
Uri.TryCreate(o.PublicBaseUrl?.Trim(), UriKind.Absolute, out var uri)
&& (uri.Scheme == Uri.UriSchemeHttps || uri.Scheme == Uri.UriSchemeHttp)
? uri.GetLeftPart(UriPartial.Authority)
: null;
internal static string NormalizePath(string path)
{
var p = "/" + (path ?? string.Empty).Trim().Trim('/');
return p.ToLowerInvariant();
}
private static async Task WriteAsync(HttpContext context, string contentType, string body)
{
var bytes = Encoding.UTF8.GetBytes(body);
context.Response.StatusCode = StatusCodes.Status200OK;
context.Response.ContentType = contentType;
context.Response.Headers[HeaderNames.CacheControl] = "no-cache";
context.Response.ContentLength = bytes.Length;
if (!HttpMethods.IsHead(context.Request.Method))
{
await context.Response.Body.WriteAsync(bytes);
}
}
}
@@ -19,6 +19,27 @@ public sealed class WebsiteOptions
/// </summary>
public string? LogLevel { get; set; }
/// <summary>
/// Absolute public base of the site (e.g. <c>https://slpsoftware.nl</c>), used for the sitemap and the
/// <c>Sitemap:</c> line of robots.txt. Empty = no sitemap, and no generated robots.txt on a non-NoIndex host.
/// </summary>
public string? PublicBaseUrl { get; set; }
/// <summary>Paths listed in <c>/sitemap.xml</c> (SpaFallback sites), e.g. <c>/</c> and <c>/privacy</c>.</summary>
public SitemapOptions Sitemap { get; set; } = new();
/// <summary>
/// SpaFallback sites: when non-empty, only these paths get the website (<c>index.html</c>, status 200);
/// any other website path answers 404. Empty = every path is served as the SPA (the default).
/// </summary>
public List<string> KnownPaths { get; set; } = [];
public static bool IsValidLogLevel(string? value) =>
string.IsNullOrWhiteSpace(value) || Enum.TryParse<Microsoft.Extensions.Logging.LogLevel>(value.Trim(), ignoreCase: true, out _);
}
/// <summary>Sitemap section of <see cref="WebsiteOptions"/>.</summary>
public sealed class SitemapOptions
{
public List<string> Paths { get; set; } = [];
}