U6: SpaFallback-SEO: Website:NoIndex (X-Robots-Tag), robots.txt, sitemap.xml en KnownPaths (echte 404)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-11 23:44:04 +01:00
co-authored by Claude Sonnet 5.5
parent 0ef6e74671
commit a7460a0f9b
6 changed files with 330 additions and 0 deletions
+11
View File
@@ -63,6 +63,10 @@ public static class CmsHost
{
builder.Services.AddCmsWebsite(builder.Configuration);
}
else
{
builder.Services.AddCmsSpaSeo(builder.Configuration);
}
// Registered BEFORE module services: modules must not configure Data Protection themselves,
// because a later registration would override this persistent key store (see
@@ -115,6 +119,13 @@ public static class CmsHost
// which is almost all of the HTML this host serves.
app.UseCmsSecurityHeaders();
// SpaFallback sites: X-Robots-Tag on test hosts and generated robots.txt / sitemap.xml. After the
// security headers (so these answers carry them) and before the static files short-circuit.
if (options.WebsiteMode == WebsiteMode.SpaFallback)
{
app.UseCmsSpaSeo();
}
app.UseRateLimiter();
// 6. Configure Pipeline
@@ -4,6 +4,8 @@ using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.FileProviders;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using SlpModularCms.Core.Hosting.Website;
namespace SlpModularCms.Core.Hosting;
@@ -136,6 +138,15 @@ public static class StaticContentExtensions
{
if (File.Exists(websiteIndex))
{
// Website:KnownPaths (optional): only those paths are real pages; the rest is a real 404, still
// with the SPA shell as body so the app can render its own not-found page.
var known = context.RequestServices.GetService<IOptions<WebsiteOptions>>()?.Value.KnownPaths;
if (known is { Count: > 0 }
&& !known.Select(SpaSeoExtensions.NormalizePath).Contains(SpaSeoExtensions.NormalizePath(context.Request.Path.Value ?? "/")))
{
context.Response.StatusCode = StatusCodes.Status404NotFound;
}
context.Response.ContentType = "text/html; charset=utf-8";
await context.Response.SendFileAsync(websiteIndex);
return;
@@ -0,0 +1,123 @@
using System.Net;
using System.Text;
using System.Xml;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using Microsoft.Net.Http.Headers;
namespace SlpModularCms.Core.Hosting.Website;
/// <summary>
/// SEO behaviour of a <see cref="WebsiteMode.SpaFallback"/> site, driven by the <c>Website</c> configuration section:
/// <c>NoIndex</c> (test hosts), generated <c>/robots.txt</c> and <c>/sitemap.xml</c>. Not part of the frontend bundle.
/// </summary>
public static class SpaSeoExtensions
{
internal const string RobotsTagValue = "noindex, nofollow";
internal const string DisallowAll = "User-agent: *\nDisallow: /\n";
public static IServiceCollection AddCmsSpaSeo(this IServiceCollection services, IConfiguration configuration)
{
services.AddOptions<WebsiteOptions>().Bind(configuration.GetSection(WebsiteOptions.SectionName));
return services;
}
/// <summary>Place directly after the security headers, before the static files short-circuit the pipeline.</summary>
public static WebApplication UseCmsSpaSeo(this WebApplication app)
{
var options = app.Services.GetRequiredService<IOptions<WebsiteOptions>>();
app.Use(async (context, next) =>
{
var website = options.Value;
if (website.NoIndex)
{
context.Response.Headers["X-Robots-Tag"] = RobotsTagValue;
}
var isRead = HttpMethods.IsGet(context.Request.Method) || HttpMethods.IsHead(context.Request.Method);
if (isRead)
{
var path = context.Request.Path.Value;
if (string.Equals(path, "/robots.txt", StringComparison.OrdinalIgnoreCase)
&& Robots(website) is { } robots)
{
await WriteAsync(context, "text/plain; charset=utf-8", robots);
return;
}
if (string.Equals(path, "/sitemap.xml", StringComparison.OrdinalIgnoreCase)
&& Sitemap(website) is { } sitemap)
{
await WriteAsync(context, "application/xml; charset=utf-8", sitemap);
return;
}
}
await next();
});
return app;
}
internal static string? Robots(WebsiteOptions o)
{
if (o.NoIndex)
{
return DisallowAll;
}
var baseUrl = BaseUrl(o);
return baseUrl is null ? null : $"User-agent: *\nAllow: /\n\nSitemap: {baseUrl}/sitemap.xml\n";
}
internal static string? Sitemap(WebsiteOptions o)
{
var baseUrl = BaseUrl(o);
if (o.NoIndex || baseUrl is null || o.Sitemap.Paths.Count == 0)
{
return null;
}
var xml = new StringBuilder("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
xml.Append("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n");
foreach (var path in o.Sitemap.Paths.Select(NormalizePath).Distinct(StringComparer.Ordinal))
{
var loc = path == "/" ? baseUrl + "/" : baseUrl + path;
xml.Append(" <url><loc>").Append(WebUtility.HtmlEncode(loc)).Append("</loc></url>\n");
}
return xml.Append("</urlset>\n").ToString();
}
/// <summary>The base URL without a trailing slash, or null when it is not a valid absolute http(s) URL.</summary>
internal static string? BaseUrl(WebsiteOptions o) =>
Uri.TryCreate(o.PublicBaseUrl?.Trim(), UriKind.Absolute, out var uri)
&& (uri.Scheme == Uri.UriSchemeHttps || uri.Scheme == Uri.UriSchemeHttp)
? uri.GetLeftPart(UriPartial.Authority)
: null;
internal static string NormalizePath(string path)
{
var p = "/" + (path ?? string.Empty).Trim().Trim('/');
return p.ToLowerInvariant();
}
private static async Task WriteAsync(HttpContext context, string contentType, string body)
{
var bytes = Encoding.UTF8.GetBytes(body);
context.Response.StatusCode = StatusCodes.Status200OK;
context.Response.ContentType = contentType;
context.Response.Headers[HeaderNames.CacheControl] = "no-cache";
context.Response.ContentLength = bytes.Length;
if (!HttpMethods.IsHead(context.Request.Method))
{
await context.Response.Body.WriteAsync(bytes);
}
}
}
@@ -19,6 +19,27 @@ public sealed class WebsiteOptions
/// </summary>
public string? LogLevel { get; set; }
/// <summary>
/// Absolute public base of the site (e.g. <c>https://slpsoftware.nl</c>), used for the sitemap and the
/// <c>Sitemap:</c> line of robots.txt. Empty = no sitemap, and no generated robots.txt on a non-NoIndex host.
/// </summary>
public string? PublicBaseUrl { get; set; }
/// <summary>Paths listed in <c>/sitemap.xml</c> (SpaFallback sites), e.g. <c>/</c> and <c>/privacy</c>.</summary>
public SitemapOptions Sitemap { get; set; } = new();
/// <summary>
/// SpaFallback sites: when non-empty, only these paths get the website (<c>index.html</c>, status 200);
/// any other website path answers 404. Empty = every path is served as the SPA (the default).
/// </summary>
public List<string> KnownPaths { get; set; } = [];
public static bool IsValidLogLevel(string? value) =>
string.IsNullOrWhiteSpace(value) || Enum.TryParse<Microsoft.Extensions.Logging.LogLevel>(value.Trim(), ignoreCase: true, out _);
}
/// <summary>Sitemap section of <see cref="WebsiteOptions"/>.</summary>
public sealed class SitemapOptions
{
public List<string> Paths { get; set; } = [];
}