diff --git a/src/SlpModularCms.Api.SlpSoftware/appsettings.json b/src/SlpModularCms.Api.SlpSoftware/appsettings.json index 7bb6fd1..ee11036 100644 --- a/src/SlpModularCms.Api.SlpSoftware/appsettings.json +++ b/src/SlpModularCms.Api.SlpSoftware/appsettings.json @@ -9,6 +9,16 @@ } }, "AllowedHosts": "*", + "Website": { + // Secure default: every response carries noindex and /robots.txt disallows all. A production + // instance must set Website__NoIndex=false and Website__PublicBaseUrl=https://slpsoftware.nl explicitly; + // robots.txt then allows crawling and points to /sitemap.xml. + "NoIndex": true, + "PublicBaseUrl": "", + "Sitemap": { "Paths": [ "/", "/privacy" ] }, + // Only these website paths are pages (200); any other path is a real 404. /api, /admin and assets keep working. + "KnownPaths": [ "/", "/privacy" ] + }, "ConnectionStrings": { "DefaultConnection": "Server=;Port=3306;Database=SlpSoftware;Uid=;Pwd=" }, diff --git a/src/SlpModularCms.Core.Tests/Hosting/Website/SpaSeoTests.cs b/src/SlpModularCms.Core.Tests/Hosting/Website/SpaSeoTests.cs new file mode 100644 index 0000000..d9832f2 --- /dev/null +++ b/src/SlpModularCms.Core.Tests/Hosting/Website/SpaSeoTests.cs @@ -0,0 +1,154 @@ +using System.Net; +using FluentAssertions; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using SlpModularCms.Core.Hosting; +using SlpModularCms.Core.Hosting.Security; +using SlpModularCms.Core.Hosting.Website; +using Xunit; + +namespace SlpModularCms.Core.Tests.Hosting.Website; + +/// SEO behaviour of a SpaFallback site: noindex on test hosts, robots.txt, sitemap.xml, KnownPaths. +public sealed class SpaSeoTests : IDisposable +{ + private readonly string _root = Path.Combine(Path.GetTempPath(), "slp-spa-" + Guid.NewGuid().ToString("N")); + private readonly List _apps = []; + + public SpaSeoTests() + { + var web = Path.Combine(_root, "wwwroot", "web"); + Directory.CreateDirectory(web); + File.WriteAllText(Path.Combine(web, "index.html"), "SPA"); + } + + public void Dispose() + { + foreach (var app in _apps) app.DisposeAsync().AsTask().GetAwaiter().GetResult(); + if (Directory.Exists(_root)) Directory.Delete(_root, recursive: true); + } + + private async Task StartAsync(Dictionary settings) + { + var builder = WebApplication.CreateBuilder(new WebApplicationOptions + { + EnvironmentName = Environments.Production, + ContentRootPath = _root, + WebRootPath = Path.Combine(_root, "wwwroot"), + }); + builder.WebHost.UseTestServer(); + builder.Logging.ClearProviders(); + settings["SecurityHeaders:DefaultPolicy"] = "StrictSite"; + builder.Configuration.AddInMemoryCollection(settings); + builder.Services.AddCmsSecurityHeaders(builder.Configuration); + builder.Services.AddCmsSpaSeo(builder.Configuration); + + var app = builder.Build(); + app.UseCmsSecurityHeaders(); + app.UseCmsSpaSeo(); + app.UseCmsStaticContent(WebsiteMode.SpaFallback); + app.MapGet("/api/v1/ping", () => Results.Text("pong")); + app.MapCmsSpaFallbacks(WebsiteMode.SpaFallback); + await app.StartAsync(); + _apps.Add(app); + return app.GetTestClient(); + } + + private static Dictionary Config(bool noIndex, string? baseUrl = null) => new() + { + ["Website:NoIndex"] = noIndex ? "true" : "false", + ["Website:PublicBaseUrl"] = baseUrl, + ["Website:Sitemap:Paths:0"] = "/", + ["Website:Sitemap:Paths:1"] = "/privacy", + }; + + private static bool Has(HttpResponseMessage r, string name, string value) => + r.Headers.TryGetValues(name, out var v) && v.Contains(value); + + [Fact] + public async Task OnATestHost_EveryResponseCarriesNoindex_AndRobotsDisallowsAll() + { + var client = await StartAsync(Config(noIndex: true, baseUrl: "https://slpsoftware.nl")); + + foreach (var path in new[] { "/", "/privacy", "/api/v1/ping", "/robots.txt", "/missing.js" }) + { + Has(await client.GetAsync(path), "X-Robots-Tag", "noindex, nofollow").Should().BeTrue(path); + } + + var robots = await client.GetAsync("/robots.txt"); + robots.StatusCode.Should().Be(HttpStatusCode.OK); + (await robots.Content.ReadAsStringAsync()).Should().Be("User-agent: *\nDisallow: /\n"); + (await client.GetAsync("/sitemap.xml")).StatusCode.Should().Be(HttpStatusCode.NotFound); + } + + [Fact] + public async Task InProduction_RobotsAllowsAndPointsToTheSitemap_AndThereIsNoNoindex() + { + var client = await StartAsync(Config(noIndex: false, baseUrl: "https://slpsoftware.nl/")); + + var page = await client.GetAsync("/"); + page.Headers.Contains("X-Robots-Tag").Should().BeFalse(); + + var robots = await client.GetStringAsync("/robots.txt"); + robots.Should().Contain("Allow: /").And.Contain("Sitemap: https://slpsoftware.nl/sitemap.xml"); + robots.Should().NotContain("Disallow"); + } + + [Fact] + public async Task TheSitemap_ListsTheConfiguredPaths_WithAnAbsoluteBase() + { + var client = await StartAsync(Config(noIndex: false, baseUrl: "https://slpsoftware.nl")); + + var response = await client.GetAsync("/sitemap.xml"); + + response.StatusCode.Should().Be(HttpStatusCode.OK); + response.Content.Headers.ContentType!.MediaType.Should().Be("application/xml"); + var xml = await response.Content.ReadAsStringAsync(); + xml.Should().Contain("https://slpsoftware.nl/") + .And.Contain("https://slpsoftware.nl/privacy") + .And.NotContain("lastmod"); + } + + [Fact] + public async Task WithoutABaseUrl_ThereIsNoSitemap_AndNoGeneratedRobots() + { + var client = await StartAsync(Config(noIndex: false)); + + (await client.GetAsync("/sitemap.xml")).StatusCode.Should().Be(HttpStatusCode.NotFound); + (await client.GetAsync("/robots.txt")).StatusCode.Should().Be(HttpStatusCode.NotFound); + } + + [Fact] + public async Task APrivacyPath_ServesTheSite_AndWithoutKnownPathsEverythingIsTheSpa() + { + var client = await StartAsync(Config(noIndex: false)); + + var privacy = await client.GetAsync("/privacy"); + privacy.StatusCode.Should().Be(HttpStatusCode.OK); + (await privacy.Content.ReadAsStringAsync()).Should().Contain("SPA"); + (await client.GetAsync("/anything")).StatusCode.Should().Be(HttpStatusCode.OK); + } + + [Fact] + public async Task WithKnownPaths_OnlyThoseAre200_TheRestIs404_AndApiStillWorks() + { + var settings = Config(noIndex: false); + settings["Website:KnownPaths:0"] = "/"; + settings["Website:KnownPaths:1"] = "/privacy"; + var client = await StartAsync(settings); + + (await client.GetAsync("/")).StatusCode.Should().Be(HttpStatusCode.OK); + (await client.GetAsync("/privacy")).StatusCode.Should().Be(HttpStatusCode.OK); + (await client.GetAsync("/Privacy/")).StatusCode.Should().Be(HttpStatusCode.OK); + var missing = await client.GetAsync("/nope"); + missing.StatusCode.Should().Be(HttpStatusCode.NotFound); + (await missing.Content.ReadAsStringAsync()).Should().Contain("SPA"); + (await client.GetStringAsync("/api/v1/ping")).Should().Be("pong"); + } +} diff --git a/src/SlpModularCms.Core/Hosting/CmsHost.cs b/src/SlpModularCms.Core/Hosting/CmsHost.cs index d2b8f7e..ca4be9c 100644 --- a/src/SlpModularCms.Core/Hosting/CmsHost.cs +++ b/src/SlpModularCms.Core/Hosting/CmsHost.cs @@ -63,6 +63,10 @@ public static class CmsHost { builder.Services.AddCmsWebsite(builder.Configuration); } + else + { + builder.Services.AddCmsSpaSeo(builder.Configuration); + } // Registered BEFORE module services: modules must not configure Data Protection themselves, // because a later registration would override this persistent key store (see @@ -115,6 +119,13 @@ public static class CmsHost // which is almost all of the HTML this host serves. app.UseCmsSecurityHeaders(); + // SpaFallback sites: X-Robots-Tag on test hosts and generated robots.txt / sitemap.xml. After the + // security headers (so these answers carry them) and before the static files short-circuit. + if (options.WebsiteMode == WebsiteMode.SpaFallback) + { + app.UseCmsSpaSeo(); + } + app.UseRateLimiter(); // 6. Configure Pipeline diff --git a/src/SlpModularCms.Core/Hosting/StaticContentExtensions.cs b/src/SlpModularCms.Core/Hosting/StaticContentExtensions.cs index 779a7a2..8152745 100644 --- a/src/SlpModularCms.Core/Hosting/StaticContentExtensions.cs +++ b/src/SlpModularCms.Core/Hosting/StaticContentExtensions.cs @@ -4,6 +4,8 @@ using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using SlpModularCms.Core.Hosting.Website; namespace SlpModularCms.Core.Hosting; @@ -136,6 +138,15 @@ public static class StaticContentExtensions { if (File.Exists(websiteIndex)) { + // Website:KnownPaths (optional): only those paths are real pages; the rest is a real 404, still + // with the SPA shell as body so the app can render its own not-found page. + var known = context.RequestServices.GetService>()?.Value.KnownPaths; + if (known is { Count: > 0 } + && !known.Select(SpaSeoExtensions.NormalizePath).Contains(SpaSeoExtensions.NormalizePath(context.Request.Path.Value ?? "/"))) + { + context.Response.StatusCode = StatusCodes.Status404NotFound; + } + context.Response.ContentType = "text/html; charset=utf-8"; await context.Response.SendFileAsync(websiteIndex); return; diff --git a/src/SlpModularCms.Core/Hosting/Website/SpaSeoExtensions.cs b/src/SlpModularCms.Core/Hosting/Website/SpaSeoExtensions.cs new file mode 100644 index 0000000..fa35233 --- /dev/null +++ b/src/SlpModularCms.Core/Hosting/Website/SpaSeoExtensions.cs @@ -0,0 +1,123 @@ +using System.Net; +using System.Text; +using System.Xml; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Microsoft.Net.Http.Headers; + +namespace SlpModularCms.Core.Hosting.Website; + +/// +/// SEO behaviour of a site, driven by the Website configuration section: +/// NoIndex (test hosts), generated /robots.txt and /sitemap.xml. Not part of the frontend bundle. +/// +public static class SpaSeoExtensions +{ + internal const string RobotsTagValue = "noindex, nofollow"; + internal const string DisallowAll = "User-agent: *\nDisallow: /\n"; + + public static IServiceCollection AddCmsSpaSeo(this IServiceCollection services, IConfiguration configuration) + { + services.AddOptions().Bind(configuration.GetSection(WebsiteOptions.SectionName)); + return services; + } + + /// Place directly after the security headers, before the static files short-circuit the pipeline. + public static WebApplication UseCmsSpaSeo(this WebApplication app) + { + var options = app.Services.GetRequiredService>(); + + app.Use(async (context, next) => + { + var website = options.Value; + + if (website.NoIndex) + { + context.Response.Headers["X-Robots-Tag"] = RobotsTagValue; + } + + var isRead = HttpMethods.IsGet(context.Request.Method) || HttpMethods.IsHead(context.Request.Method); + if (isRead) + { + var path = context.Request.Path.Value; + if (string.Equals(path, "/robots.txt", StringComparison.OrdinalIgnoreCase) + && Robots(website) is { } robots) + { + await WriteAsync(context, "text/plain; charset=utf-8", robots); + return; + } + + if (string.Equals(path, "/sitemap.xml", StringComparison.OrdinalIgnoreCase) + && Sitemap(website) is { } sitemap) + { + await WriteAsync(context, "application/xml; charset=utf-8", sitemap); + return; + } + } + + await next(); + }); + + return app; + } + + internal static string? Robots(WebsiteOptions o) + { + if (o.NoIndex) + { + return DisallowAll; + } + + var baseUrl = BaseUrl(o); + return baseUrl is null ? null : $"User-agent: *\nAllow: /\n\nSitemap: {baseUrl}/sitemap.xml\n"; + } + + internal static string? Sitemap(WebsiteOptions o) + { + var baseUrl = BaseUrl(o); + if (o.NoIndex || baseUrl is null || o.Sitemap.Paths.Count == 0) + { + return null; + } + + var xml = new StringBuilder("\n"); + xml.Append("\n"); + foreach (var path in o.Sitemap.Paths.Select(NormalizePath).Distinct(StringComparer.Ordinal)) + { + var loc = path == "/" ? baseUrl + "/" : baseUrl + path; + xml.Append(" ").Append(WebUtility.HtmlEncode(loc)).Append("\n"); + } + + return xml.Append("\n").ToString(); + } + + /// The base URL without a trailing slash, or null when it is not a valid absolute http(s) URL. + internal static string? BaseUrl(WebsiteOptions o) => + Uri.TryCreate(o.PublicBaseUrl?.Trim(), UriKind.Absolute, out var uri) + && (uri.Scheme == Uri.UriSchemeHttps || uri.Scheme == Uri.UriSchemeHttp) + ? uri.GetLeftPart(UriPartial.Authority) + : null; + + internal static string NormalizePath(string path) + { + var p = "/" + (path ?? string.Empty).Trim().Trim('/'); + return p.ToLowerInvariant(); + } + + private static async Task WriteAsync(HttpContext context, string contentType, string body) + { + var bytes = Encoding.UTF8.GetBytes(body); + context.Response.StatusCode = StatusCodes.Status200OK; + context.Response.ContentType = contentType; + context.Response.Headers[HeaderNames.CacheControl] = "no-cache"; + context.Response.ContentLength = bytes.Length; + + if (!HttpMethods.IsHead(context.Request.Method)) + { + await context.Response.Body.WriteAsync(bytes); + } + } +} diff --git a/src/SlpModularCms.Core/Hosting/Website/WebsiteOptions.cs b/src/SlpModularCms.Core/Hosting/Website/WebsiteOptions.cs index 073b533..cd421f0 100644 --- a/src/SlpModularCms.Core/Hosting/Website/WebsiteOptions.cs +++ b/src/SlpModularCms.Core/Hosting/Website/WebsiteOptions.cs @@ -19,6 +19,27 @@ public sealed class WebsiteOptions /// public string? LogLevel { get; set; } + /// + /// Absolute public base of the site (e.g. https://slpsoftware.nl), used for the sitemap and the + /// Sitemap: line of robots.txt. Empty = no sitemap, and no generated robots.txt on a non-NoIndex host. + /// + public string? PublicBaseUrl { get; set; } + + /// Paths listed in /sitemap.xml (SpaFallback sites), e.g. / and /privacy. + public SitemapOptions Sitemap { get; set; } = new(); + + /// + /// SpaFallback sites: when non-empty, only these paths get the website (index.html, status 200); + /// any other website path answers 404. Empty = every path is served as the SPA (the default). + /// + public List KnownPaths { get; set; } = []; + public static bool IsValidLogLevel(string? value) => string.IsNullOrWhiteSpace(value) || Enum.TryParse(value.Trim(), ignoreCase: true, out _); } + +/// Sitemap section of . +public sealed class SitemapOptions +{ + public List Paths { get; set; } = []; +}