U6: SpaFallback-SEO: Website:NoIndex (X-Robots-Tag), robots.txt, sitemap.xml en KnownPaths (echte 404)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,16 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"AllowedHosts": "*",
|
"AllowedHosts": "*",
|
||||||
|
"Website": {
|
||||||
|
// Secure default: every response carries noindex and /robots.txt disallows all. A production
|
||||||
|
// instance must set Website__NoIndex=false and Website__PublicBaseUrl=https://slpsoftware.nl explicitly;
|
||||||
|
// robots.txt then allows crawling and points to /sitemap.xml.
|
||||||
|
"NoIndex": true,
|
||||||
|
"PublicBaseUrl": "",
|
||||||
|
"Sitemap": { "Paths": [ "/", "/privacy" ] },
|
||||||
|
// Only these website paths are pages (200); any other path is a real 404. /api, /admin and assets keep working.
|
||||||
|
"KnownPaths": [ "/", "/privacy" ]
|
||||||
|
},
|
||||||
"ConnectionStrings": {
|
"ConnectionStrings": {
|
||||||
"DefaultConnection": "Server=<production-db-host>;Port=3306;Database=SlpSoftware<Env>;Uid=<db-user>;Pwd=<db-password>"
|
"DefaultConnection": "Server=<production-db-host>;Port=3306;Database=SlpSoftware<Env>;Uid=<db-user>;Pwd=<db-password>"
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -0,0 +1,154 @@
|
|||||||
|
using System.Net;
|
||||||
|
using FluentAssertions;
|
||||||
|
using Microsoft.AspNetCore.Builder;
|
||||||
|
using Microsoft.AspNetCore.Hosting;
|
||||||
|
using Microsoft.AspNetCore.Http;
|
||||||
|
using Microsoft.AspNetCore.TestHost;
|
||||||
|
using Microsoft.Extensions.Configuration;
|
||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
using Microsoft.Extensions.Hosting;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using SlpModularCms.Core.Hosting;
|
||||||
|
using SlpModularCms.Core.Hosting.Security;
|
||||||
|
using SlpModularCms.Core.Hosting.Website;
|
||||||
|
using Xunit;
|
||||||
|
|
||||||
|
namespace SlpModularCms.Core.Tests.Hosting.Website;
|
||||||
|
|
||||||
|
/// <summary>SEO behaviour of a SpaFallback site: noindex on test hosts, robots.txt, sitemap.xml, KnownPaths.</summary>
|
||||||
|
public sealed class SpaSeoTests : IDisposable
|
||||||
|
{
|
||||||
|
private readonly string _root = Path.Combine(Path.GetTempPath(), "slp-spa-" + Guid.NewGuid().ToString("N"));
|
||||||
|
private readonly List<WebApplication> _apps = [];
|
||||||
|
|
||||||
|
public SpaSeoTests()
|
||||||
|
{
|
||||||
|
var web = Path.Combine(_root, "wwwroot", "web");
|
||||||
|
Directory.CreateDirectory(web);
|
||||||
|
File.WriteAllText(Path.Combine(web, "index.html"), "<!doctype html><title>SPA</title>");
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
foreach (var app in _apps) app.DisposeAsync().AsTask().GetAwaiter().GetResult();
|
||||||
|
if (Directory.Exists(_root)) Directory.Delete(_root, recursive: true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<HttpClient> StartAsync(Dictionary<string, string?> settings)
|
||||||
|
{
|
||||||
|
var builder = WebApplication.CreateBuilder(new WebApplicationOptions
|
||||||
|
{
|
||||||
|
EnvironmentName = Environments.Production,
|
||||||
|
ContentRootPath = _root,
|
||||||
|
WebRootPath = Path.Combine(_root, "wwwroot"),
|
||||||
|
});
|
||||||
|
builder.WebHost.UseTestServer();
|
||||||
|
builder.Logging.ClearProviders();
|
||||||
|
settings["SecurityHeaders:DefaultPolicy"] = "StrictSite";
|
||||||
|
builder.Configuration.AddInMemoryCollection(settings);
|
||||||
|
builder.Services.AddCmsSecurityHeaders(builder.Configuration);
|
||||||
|
builder.Services.AddCmsSpaSeo(builder.Configuration);
|
||||||
|
|
||||||
|
var app = builder.Build();
|
||||||
|
app.UseCmsSecurityHeaders();
|
||||||
|
app.UseCmsSpaSeo();
|
||||||
|
app.UseCmsStaticContent(WebsiteMode.SpaFallback);
|
||||||
|
app.MapGet("/api/v1/ping", () => Results.Text("pong"));
|
||||||
|
app.MapCmsSpaFallbacks(WebsiteMode.SpaFallback);
|
||||||
|
await app.StartAsync();
|
||||||
|
_apps.Add(app);
|
||||||
|
return app.GetTestClient();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Dictionary<string, string?> Config(bool noIndex, string? baseUrl = null) => new()
|
||||||
|
{
|
||||||
|
["Website:NoIndex"] = noIndex ? "true" : "false",
|
||||||
|
["Website:PublicBaseUrl"] = baseUrl,
|
||||||
|
["Website:Sitemap:Paths:0"] = "/",
|
||||||
|
["Website:Sitemap:Paths:1"] = "/privacy",
|
||||||
|
};
|
||||||
|
|
||||||
|
private static bool Has(HttpResponseMessage r, string name, string value) =>
|
||||||
|
r.Headers.TryGetValues(name, out var v) && v.Contains(value);
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task OnATestHost_EveryResponseCarriesNoindex_AndRobotsDisallowsAll()
|
||||||
|
{
|
||||||
|
var client = await StartAsync(Config(noIndex: true, baseUrl: "https://slpsoftware.nl"));
|
||||||
|
|
||||||
|
foreach (var path in new[] { "/", "/privacy", "/api/v1/ping", "/robots.txt", "/missing.js" })
|
||||||
|
{
|
||||||
|
Has(await client.GetAsync(path), "X-Robots-Tag", "noindex, nofollow").Should().BeTrue(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
var robots = await client.GetAsync("/robots.txt");
|
||||||
|
robots.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||||
|
(await robots.Content.ReadAsStringAsync()).Should().Be("User-agent: *\nDisallow: /\n");
|
||||||
|
(await client.GetAsync("/sitemap.xml")).StatusCode.Should().Be(HttpStatusCode.NotFound);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task InProduction_RobotsAllowsAndPointsToTheSitemap_AndThereIsNoNoindex()
|
||||||
|
{
|
||||||
|
var client = await StartAsync(Config(noIndex: false, baseUrl: "https://slpsoftware.nl/"));
|
||||||
|
|
||||||
|
var page = await client.GetAsync("/");
|
||||||
|
page.Headers.Contains("X-Robots-Tag").Should().BeFalse();
|
||||||
|
|
||||||
|
var robots = await client.GetStringAsync("/robots.txt");
|
||||||
|
robots.Should().Contain("Allow: /").And.Contain("Sitemap: https://slpsoftware.nl/sitemap.xml");
|
||||||
|
robots.Should().NotContain("Disallow");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task TheSitemap_ListsTheConfiguredPaths_WithAnAbsoluteBase()
|
||||||
|
{
|
||||||
|
var client = await StartAsync(Config(noIndex: false, baseUrl: "https://slpsoftware.nl"));
|
||||||
|
|
||||||
|
var response = await client.GetAsync("/sitemap.xml");
|
||||||
|
|
||||||
|
response.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||||
|
response.Content.Headers.ContentType!.MediaType.Should().Be("application/xml");
|
||||||
|
var xml = await response.Content.ReadAsStringAsync();
|
||||||
|
xml.Should().Contain("<loc>https://slpsoftware.nl/</loc>")
|
||||||
|
.And.Contain("<loc>https://slpsoftware.nl/privacy</loc>")
|
||||||
|
.And.NotContain("lastmod");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task WithoutABaseUrl_ThereIsNoSitemap_AndNoGeneratedRobots()
|
||||||
|
{
|
||||||
|
var client = await StartAsync(Config(noIndex: false));
|
||||||
|
|
||||||
|
(await client.GetAsync("/sitemap.xml")).StatusCode.Should().Be(HttpStatusCode.NotFound);
|
||||||
|
(await client.GetAsync("/robots.txt")).StatusCode.Should().Be(HttpStatusCode.NotFound);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task APrivacyPath_ServesTheSite_AndWithoutKnownPathsEverythingIsTheSpa()
|
||||||
|
{
|
||||||
|
var client = await StartAsync(Config(noIndex: false));
|
||||||
|
|
||||||
|
var privacy = await client.GetAsync("/privacy");
|
||||||
|
privacy.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||||
|
(await privacy.Content.ReadAsStringAsync()).Should().Contain("<title>SPA</title>");
|
||||||
|
(await client.GetAsync("/anything")).StatusCode.Should().Be(HttpStatusCode.OK);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task WithKnownPaths_OnlyThoseAre200_TheRestIs404_AndApiStillWorks()
|
||||||
|
{
|
||||||
|
var settings = Config(noIndex: false);
|
||||||
|
settings["Website:KnownPaths:0"] = "/";
|
||||||
|
settings["Website:KnownPaths:1"] = "/privacy";
|
||||||
|
var client = await StartAsync(settings);
|
||||||
|
|
||||||
|
(await client.GetAsync("/")).StatusCode.Should().Be(HttpStatusCode.OK);
|
||||||
|
(await client.GetAsync("/privacy")).StatusCode.Should().Be(HttpStatusCode.OK);
|
||||||
|
(await client.GetAsync("/Privacy/")).StatusCode.Should().Be(HttpStatusCode.OK);
|
||||||
|
var missing = await client.GetAsync("/nope");
|
||||||
|
missing.StatusCode.Should().Be(HttpStatusCode.NotFound);
|
||||||
|
(await missing.Content.ReadAsStringAsync()).Should().Contain("<title>SPA</title>");
|
||||||
|
(await client.GetStringAsync("/api/v1/ping")).Should().Be("pong");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -63,6 +63,10 @@ public static class CmsHost
|
|||||||
{
|
{
|
||||||
builder.Services.AddCmsWebsite(builder.Configuration);
|
builder.Services.AddCmsWebsite(builder.Configuration);
|
||||||
}
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
builder.Services.AddCmsSpaSeo(builder.Configuration);
|
||||||
|
}
|
||||||
|
|
||||||
// Registered BEFORE module services: modules must not configure Data Protection themselves,
|
// Registered BEFORE module services: modules must not configure Data Protection themselves,
|
||||||
// because a later registration would override this persistent key store (see
|
// because a later registration would override this persistent key store (see
|
||||||
@@ -115,6 +119,13 @@ public static class CmsHost
|
|||||||
// which is almost all of the HTML this host serves.
|
// which is almost all of the HTML this host serves.
|
||||||
app.UseCmsSecurityHeaders();
|
app.UseCmsSecurityHeaders();
|
||||||
|
|
||||||
|
// SpaFallback sites: X-Robots-Tag on test hosts and generated robots.txt / sitemap.xml. After the
|
||||||
|
// security headers (so these answers carry them) and before the static files short-circuit.
|
||||||
|
if (options.WebsiteMode == WebsiteMode.SpaFallback)
|
||||||
|
{
|
||||||
|
app.UseCmsSpaSeo();
|
||||||
|
}
|
||||||
|
|
||||||
app.UseRateLimiter();
|
app.UseRateLimiter();
|
||||||
|
|
||||||
// 6. Configure Pipeline
|
// 6. Configure Pipeline
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ using Microsoft.AspNetCore.Http;
|
|||||||
using Microsoft.Extensions.DependencyInjection;
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
using Microsoft.Extensions.FileProviders;
|
using Microsoft.Extensions.FileProviders;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
using SlpModularCms.Core.Hosting.Website;
|
||||||
|
|
||||||
namespace SlpModularCms.Core.Hosting;
|
namespace SlpModularCms.Core.Hosting;
|
||||||
|
|
||||||
@@ -136,6 +138,15 @@ public static class StaticContentExtensions
|
|||||||
{
|
{
|
||||||
if (File.Exists(websiteIndex))
|
if (File.Exists(websiteIndex))
|
||||||
{
|
{
|
||||||
|
// Website:KnownPaths (optional): only those paths are real pages; the rest is a real 404, still
|
||||||
|
// with the SPA shell as body so the app can render its own not-found page.
|
||||||
|
var known = context.RequestServices.GetService<IOptions<WebsiteOptions>>()?.Value.KnownPaths;
|
||||||
|
if (known is { Count: > 0 }
|
||||||
|
&& !known.Select(SpaSeoExtensions.NormalizePath).Contains(SpaSeoExtensions.NormalizePath(context.Request.Path.Value ?? "/")))
|
||||||
|
{
|
||||||
|
context.Response.StatusCode = StatusCodes.Status404NotFound;
|
||||||
|
}
|
||||||
|
|
||||||
context.Response.ContentType = "text/html; charset=utf-8";
|
context.Response.ContentType = "text/html; charset=utf-8";
|
||||||
await context.Response.SendFileAsync(websiteIndex);
|
await context.Response.SendFileAsync(websiteIndex);
|
||||||
return;
|
return;
|
||||||
|
|||||||
@@ -0,0 +1,123 @@
|
|||||||
|
using System.Net;
|
||||||
|
using System.Text;
|
||||||
|
using System.Xml;
|
||||||
|
using Microsoft.AspNetCore.Builder;
|
||||||
|
using Microsoft.AspNetCore.Http;
|
||||||
|
using Microsoft.Extensions.Configuration;
|
||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
using Microsoft.Net.Http.Headers;
|
||||||
|
|
||||||
|
namespace SlpModularCms.Core.Hosting.Website;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// SEO behaviour of a <see cref="WebsiteMode.SpaFallback"/> site, driven by the <c>Website</c> configuration section:
|
||||||
|
/// <c>NoIndex</c> (test hosts), generated <c>/robots.txt</c> and <c>/sitemap.xml</c>. Not part of the frontend bundle.
|
||||||
|
/// </summary>
|
||||||
|
public static class SpaSeoExtensions
|
||||||
|
{
|
||||||
|
internal const string RobotsTagValue = "noindex, nofollow";
|
||||||
|
internal const string DisallowAll = "User-agent: *\nDisallow: /\n";
|
||||||
|
|
||||||
|
public static IServiceCollection AddCmsSpaSeo(this IServiceCollection services, IConfiguration configuration)
|
||||||
|
{
|
||||||
|
services.AddOptions<WebsiteOptions>().Bind(configuration.GetSection(WebsiteOptions.SectionName));
|
||||||
|
return services;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Place directly after the security headers, before the static files short-circuit the pipeline.</summary>
|
||||||
|
public static WebApplication UseCmsSpaSeo(this WebApplication app)
|
||||||
|
{
|
||||||
|
var options = app.Services.GetRequiredService<IOptions<WebsiteOptions>>();
|
||||||
|
|
||||||
|
app.Use(async (context, next) =>
|
||||||
|
{
|
||||||
|
var website = options.Value;
|
||||||
|
|
||||||
|
if (website.NoIndex)
|
||||||
|
{
|
||||||
|
context.Response.Headers["X-Robots-Tag"] = RobotsTagValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var isRead = HttpMethods.IsGet(context.Request.Method) || HttpMethods.IsHead(context.Request.Method);
|
||||||
|
if (isRead)
|
||||||
|
{
|
||||||
|
var path = context.Request.Path.Value;
|
||||||
|
if (string.Equals(path, "/robots.txt", StringComparison.OrdinalIgnoreCase)
|
||||||
|
&& Robots(website) is { } robots)
|
||||||
|
{
|
||||||
|
await WriteAsync(context, "text/plain; charset=utf-8", robots);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (string.Equals(path, "/sitemap.xml", StringComparison.OrdinalIgnoreCase)
|
||||||
|
&& Sitemap(website) is { } sitemap)
|
||||||
|
{
|
||||||
|
await WriteAsync(context, "application/xml; charset=utf-8", sitemap);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await next();
|
||||||
|
});
|
||||||
|
|
||||||
|
return app;
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static string? Robots(WebsiteOptions o)
|
||||||
|
{
|
||||||
|
if (o.NoIndex)
|
||||||
|
{
|
||||||
|
return DisallowAll;
|
||||||
|
}
|
||||||
|
|
||||||
|
var baseUrl = BaseUrl(o);
|
||||||
|
return baseUrl is null ? null : $"User-agent: *\nAllow: /\n\nSitemap: {baseUrl}/sitemap.xml\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static string? Sitemap(WebsiteOptions o)
|
||||||
|
{
|
||||||
|
var baseUrl = BaseUrl(o);
|
||||||
|
if (o.NoIndex || baseUrl is null || o.Sitemap.Paths.Count == 0)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var xml = new StringBuilder("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
|
||||||
|
xml.Append("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n");
|
||||||
|
foreach (var path in o.Sitemap.Paths.Select(NormalizePath).Distinct(StringComparer.Ordinal))
|
||||||
|
{
|
||||||
|
var loc = path == "/" ? baseUrl + "/" : baseUrl + path;
|
||||||
|
xml.Append(" <url><loc>").Append(WebUtility.HtmlEncode(loc)).Append("</loc></url>\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
return xml.Append("</urlset>\n").ToString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>The base URL without a trailing slash, or null when it is not a valid absolute http(s) URL.</summary>
|
||||||
|
internal static string? BaseUrl(WebsiteOptions o) =>
|
||||||
|
Uri.TryCreate(o.PublicBaseUrl?.Trim(), UriKind.Absolute, out var uri)
|
||||||
|
&& (uri.Scheme == Uri.UriSchemeHttps || uri.Scheme == Uri.UriSchemeHttp)
|
||||||
|
? uri.GetLeftPart(UriPartial.Authority)
|
||||||
|
: null;
|
||||||
|
|
||||||
|
internal static string NormalizePath(string path)
|
||||||
|
{
|
||||||
|
var p = "/" + (path ?? string.Empty).Trim().Trim('/');
|
||||||
|
return p.ToLowerInvariant();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task WriteAsync(HttpContext context, string contentType, string body)
|
||||||
|
{
|
||||||
|
var bytes = Encoding.UTF8.GetBytes(body);
|
||||||
|
context.Response.StatusCode = StatusCodes.Status200OK;
|
||||||
|
context.Response.ContentType = contentType;
|
||||||
|
context.Response.Headers[HeaderNames.CacheControl] = "no-cache";
|
||||||
|
context.Response.ContentLength = bytes.Length;
|
||||||
|
|
||||||
|
if (!HttpMethods.IsHead(context.Request.Method))
|
||||||
|
{
|
||||||
|
await context.Response.Body.WriteAsync(bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -19,6 +19,27 @@ public sealed class WebsiteOptions
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public string? LogLevel { get; set; }
|
public string? LogLevel { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Absolute public base of the site (e.g. <c>https://slpsoftware.nl</c>), used for the sitemap and the
|
||||||
|
/// <c>Sitemap:</c> line of robots.txt. Empty = no sitemap, and no generated robots.txt on a non-NoIndex host.
|
||||||
|
/// </summary>
|
||||||
|
public string? PublicBaseUrl { get; set; }
|
||||||
|
|
||||||
|
/// <summary>Paths listed in <c>/sitemap.xml</c> (SpaFallback sites), e.g. <c>/</c> and <c>/privacy</c>.</summary>
|
||||||
|
public SitemapOptions Sitemap { get; set; } = new();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// SpaFallback sites: when non-empty, only these paths get the website (<c>index.html</c>, status 200);
|
||||||
|
/// any other website path answers 404. Empty = every path is served as the SPA (the default).
|
||||||
|
/// </summary>
|
||||||
|
public List<string> KnownPaths { get; set; } = [];
|
||||||
|
|
||||||
public static bool IsValidLogLevel(string? value) =>
|
public static bool IsValidLogLevel(string? value) =>
|
||||||
string.IsNullOrWhiteSpace(value) || Enum.TryParse<Microsoft.Extensions.Logging.LogLevel>(value.Trim(), ignoreCase: true, out _);
|
string.IsNullOrWhiteSpace(value) || Enum.TryParse<Microsoft.Extensions.Logging.LogLevel>(value.Trim(), ignoreCase: true, out _);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>Sitemap section of <see cref="WebsiteOptions"/>.</summary>
|
||||||
|
public sealed class SitemapOptions
|
||||||
|
{
|
||||||
|
public List<string> Paths { get; set; } = [];
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user