Hero: varianten op de achtergrond knippen; upload en opslaan antwoorden direct

Opslaan van 8 foto's met gewijzigde focus duurde 40 s op een snelle pc en ruim 3 minuten op de Pi (proxy-timeout 60 s gaf 504, hero niet bewaard).
Upload bewaart het origineel en antwoordt; HeroRenderService knipt de set per afbeelding en schakelt over zodra de set compleet is (oude set blijft ondertussen live; afbeelding zonder set wordt niet gepubliceerd). Opslaan legt alleen de focus vast. Admin toont 'wordt verwerkt' en ververst zelf; Seed-Hero wacht tot alles klaar is. Metingen: HeroSaveTimingTests.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-16 21:14:00 +01:00
co-authored by Claude Sonnet 5.5
parent 969bb14d86
commit 852a162bc3
23 changed files with 414 additions and 46 deletions
@@ -47,6 +47,8 @@ public class HeroModule : IModule
services.AddScoped<IHeroPageModel, HeroPageModel>();
services.AddScoped<IHtmlPageTransform, HeroHtmlTransform>();
services.AddScoped<ICmsDataBlockContributor, HeroDataContributor>();
services.AddSingleton<HeroRenderSignal>();
services.AddHostedService<HeroRenderService>();
services.AddScoped<HeroMediaCleanup>();
services.AddHostedService<HeroMediaCleanupService>();
}
@@ -23,7 +23,8 @@ public record HeroImageDto(
FocusDto? WideFocus,
FocusDto? TallFocus,
string Src,
IReadOnlyList<HeroVariantDto> Variants);
IReadOnlyList<HeroVariantDto> Variants,
bool Processing = false);
public record HeroButtonDto(string Label, string Target);
@@ -1,6 +1,7 @@
using Microsoft.EntityFrameworkCore;
using SlpModularCms.Modules.Hero.Data;
using SlpModularCms.Modules.Hero.Data.Entities;
using SlpModularCms.Modules.Hero.Services;
namespace SlpModularCms.Modules.Hero.Repositories;
@@ -18,6 +19,12 @@ public interface IHeroRepository
Task<HeroImage?> GetImageAsync(Guid id, CancellationToken cancellationToken);
Task<IReadOnlyList<HeroImage>> GetImagesAsync(IReadOnlyCollection<Guid> ids, CancellationToken cancellationToken);
/// <summary>Images whose set is missing or does not match their focus, oldest first (tracked).</summary>
Task<IReadOnlyList<HeroImage>> GetImagesNeedingRenderAsync(CancellationToken cancellationToken);
/// <summary>The image as stored right now, not tracked.</summary>
Task<HeroImage?> GetImageFreshAsync(Guid id, CancellationToken cancellationToken);
Task<HeroImage?> GetImageBySha256Async(string sha256, CancellationToken cancellationToken);
void AddImage(HeroImage image);
void RemoveImage(HeroImage image);
@@ -62,6 +69,12 @@ public class HeroRepository(HeroDbContext context) : IHeroRepository
return (await context.HeroImages.ToListAsync(cancellationToken)).Where(i => wanted.Contains(i.Id)).ToList();
}
public async Task<IReadOnlyList<HeroImage>> GetImagesNeedingRenderAsync(CancellationToken cancellationToken)
=> (await context.HeroImages.ToListAsync(cancellationToken)).Where(HeroMapping.NeedsRender).OrderBy(i => i.CreatedAt).ToList();
public Task<HeroImage?> GetImageFreshAsync(Guid id, CancellationToken cancellationToken)
=> context.HeroImages.AsNoTracking().FirstOrDefaultAsync(i => i.Id == id, cancellationToken);
public Task<HeroImage?> GetImageBySha256Async(string sha256, CancellationToken cancellationToken)
=> context.HeroImages.FirstOrDefaultAsync(i => i.OriginalSha256 == sha256, cancellationToken);
@@ -12,6 +12,12 @@ namespace SlpModularCms.Modules.Hero.Services;
/// <summary>The image is still used by a slide (the administrator gets a 409 and a message).</summary>
public sealed class HeroImageInUseException() : Exception("The image is still used by a slide.");
/// <summary>Cutting the variants of one image failed; the id tells the background renderer which image to give up on after a few tries.</summary>
public sealed class HeroImageRenderException(Guid imageId, Exception inner) : Exception("Cutting the variants of a hero image failed.", inner)
{
public Guid ImageId { get; } = imageId;
}
public sealed class HeroImageNotFoundException() : Exception("The image does not exist.");
/// <summary>There is not enough room in the media directory for the image.</summary>
@@ -19,18 +25,29 @@ public sealed class MediaStorageFullException() : Exception("The media directory
public interface IHeroImageService
{
/// <summary>Admits, stores and processes an uploaded image (all variants of the default focus). The same photo twice is one image.</summary>
/// <summary>
/// Admits and stores an uploaded image and returns at once: the variants are cut afterwards by the background renderer
/// (<see cref="RenderNextPendingAsync"/>), so a slow machine never holds the request open. The same photo twice is one
/// image. Until its set exists the image has no variants and is not published.
/// </summary>
/// <exception cref="ImageRejectedException">The upload is refused; see the code.</exception>
Task<HeroImageDto> UploadAsync(Stream upload, long declaredLength, Guid callerId, CancellationToken cancellationToken);
/// <summary>
/// Applies a changed focus (general and optionally per ratio) to a tracked image: when the set changes, the variants are
/// cut again around the new point into a NEW set of files (new addresses, nothing live is touched) and the old set is
/// noted as retired. Nothing is saved here: the caller saves it together with the hero, so a changed focus only becomes
/// visible when the hero is saved. When the hero is not saved the new files are never referenced and the clean-up
/// removes them after the retention time.
/// Records a changed focus (general and optionally per ratio) on a tracked image. Nothing is cut here: the caller saves
/// the image together with the hero and then calls <see cref="NotifyPending"/>; the background renderer cuts the new set
/// into NEW files and only then switches the image to it (so the old set stays live until the new one is complete).
/// </summary>
Task StageFocusAsync(HeroImage image, HeroFocusRequest request, CancellationToken cancellationToken);
void ApplyFocus(HeroImage image, HeroFocusRequest request);
/// <summary>Wakes the background renderer: there is an image whose set is missing or does not match its focus.</summary>
void NotifyPending();
/// <summary>
/// Cuts the set of the oldest image that needs one (new upload, repair or changed focus), skipping the given ids, and
/// switches the image to it. Returns the id of the image handled, or null when nothing is pending.
/// </summary>
Task<Guid?> RenderNextPendingAsync(IReadOnlySet<Guid> skip, CancellationToken cancellationToken);
/// <summary>Removes an image nothing uses; its files go after the retention time.</summary>
Task DeleteAsync(Guid imageId, Guid callerId, CancellationToken cancellationToken);
@@ -45,7 +62,8 @@ public class HeroImageService(
IOptions<MediaOptions> mediaOptions,
ILogger<HeroImageService> logger,
TimeProvider? time = null,
IIndexHtmlCache? indexCache = null) : IHeroImageService
IIndexHtmlCache? indexCache = null,
HeroRenderSignal? signal = null) : IHeroImageService
{
private readonly TimeProvider _time = time ?? TimeProvider.System;
@@ -83,18 +101,20 @@ public class HeroImageService(
CreatedByUserId = callerId,
};
var files = await RenderSetAsync(image, original, storedFormat, cancellationToken);
// The files first, the row last: an aborted request leaves at most an original nobody refers to (the clean-up
// removes it) and a repeated upload simply does it again. The set is cut afterwards by the background renderer.
await store.PutAsync(image.OriginalKey, original, cancellationToken);
await store.PutSetAsync(HeroMapping.Area, image.SetHash, files, cancellationToken);
image.SetHash = string.Empty;
image.Variants = [];
if (existing is null)
{
repository.AddImage(image);
}
await repository.SaveChangesAsync(cancellationToken);
signal?.Notify();
logger.LogInformation("Hero image {ImageId} {Action} by user {UserId} ({Width}x{Height}, {Variants} variants)", image.Id, existing is null ? "uploaded" : "repaired", callerId, image.Width, image.Height, image.Variants.Count);
logger.LogInformation("Hero image {ImageId} {Action} by user {UserId} ({Width}x{Height}); its variants are cut in the background", image.Id, existing is null ? "uploaded" : "repaired", callerId, image.Width, image.Height);
return HeroMapping.ToImageDto(image);
}
@@ -116,21 +136,46 @@ public class HeroImageService(
return true;
}
public async Task StageFocusAsync(HeroImage image, HeroFocusRequest request, CancellationToken cancellationToken)
public void ApplyFocus(HeroImage image, HeroFocusRequest request)
{
var oldSet = image.SetHash;
var general = request.Focus.ToSetting().Normalize();
image.FocusX = general.X;
image.FocusY = general.Y;
image.Zoom = general.Zoom;
Apply(request.Wide, v => (image.WideFocusX, image.WideFocusY, image.WideZoom) = v);
Apply(request.Tall, v => (image.TallFocusX, image.TallFocusY, image.TallZoom) = v);
}
var newSet = SetHashOf(image);
if (newSet != oldSet)
public void NotifyPending() => signal?.Notify();
public async Task<Guid?> RenderNextPendingAsync(IReadOnlySet<Guid> skip, CancellationToken cancellationToken)
{
var image = (await repository.GetImagesNeedingRenderAsync(cancellationToken)).FirstOrDefault(i => !skip.Contains(i.Id));
if (image is null)
{
using var slot = await gate.EnterAsync(cancellationToken);
return null;
}
try
{
return await RenderAsync(image, cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
throw new HeroImageRenderException(image.Id, exception);
}
}
private async Task<Guid?> RenderAsync(HeroImage image, CancellationToken cancellationToken)
{
var target = HeroMapping.TargetSetHash(image);
var oldSet = image.SetHash;
Dictionary<string, byte[]> files;
List<HeroVariant> variants;
var watch = System.Diagnostics.Stopwatch.StartNew();
using (await gate.EnterAsync(cancellationToken))
{
var file = await store.OpenReadAsync(image.OriginalKey, cancellationToken)
?? throw new ImageRejectedException(ImageRejection.Unavailable, "The stored original is missing.");
using var memory = new MemoryStream();
@@ -140,12 +185,31 @@ public class HeroImageService(
}
var storedFormat = ImageFormatExtensions.FromExtension(Path.GetExtension(image.OriginalKey).TrimStart('.'));
var files = await RenderSetAsync(image, memory.ToArray(), storedFormat, cancellationToken);
await store.PutSetAsync(HeroMapping.Area, image.SetHash, files, cancellationToken);
repository.AddRetired(new HeroRetiredFile { Id = Guid.NewGuid(), Kind = "set", Key = oldSet, RetiredAt = _time.GetUtcNow() });
logger.LogInformation("Hero image {ImageId}: variants cut again for a changed focus (saved with the hero)", image.Id);
(files, variants) = await RenderSetAsync(image, memory.ToArray(), storedFormat, cancellationToken);
}
await store.PutSetAsync(HeroMapping.Area, target, files, cancellationToken);
// The focus may have been changed while the set was being cut: then this set is not the one that is wanted and the
// image stays pending (the next round cuts the right one; this set is cleaned up as an unreferenced set).
var current = await repository.GetImageFreshAsync(image.Id, cancellationToken);
if (current is null || HeroMapping.TargetSetHash(current) != target)
{
logger.LogInformation("Hero image {ImageId}: the focus changed while its variants were cut; cutting again.", image.Id);
return image.Id;
}
image.SetHash = target;
image.Variants = variants;
if (oldSet.Length > 0 && oldSet != target)
{
repository.AddRetired(new HeroRetiredFile { Id = Guid.NewGuid(), Kind = "set", Key = oldSet, RetiredAt = _time.GetUtcNow() });
}
await repository.SaveChangesAsync(cancellationToken);
indexCache?.Invalidate();
logger.LogInformation("Hero image {ImageId}: {Variants} variants cut in {Seconds:0.0} s", image.Id, variants.Count, watch.Elapsed.TotalSeconds);
return image.Id;
}
public async Task DeleteAsync(Guid imageId, Guid callerId, CancellationToken cancellationToken)
@@ -177,11 +241,8 @@ public class HeroImageService(
assign((f.X, f.Y, f.Zoom));
}
private static string SetHashOf(HeroImage image)
=> MediaPaths.SetHash(image.OriginalSha256, [("wide", image.FocusFor("wide")), ("tall", image.FocusFor("tall"))]);
/// <summary>Cuts and encodes every planned variant of the image and fills its set hash and variant list. Returns the files by name.</summary>
private async Task<Dictionary<string, byte[]>> RenderSetAsync(HeroImage image, byte[] original, ImageFormat storedFormat, CancellationToken cancellationToken)
/// <summary>Cuts and encodes every planned variant of the image at its current focus. Returns the files by name and the variant list.</summary>
private async Task<(Dictionary<string, byte[]> Files, List<HeroVariant> Variants)> RenderSetAsync(HeroImage image, byte[] original, ImageFormat storedFormat, CancellationToken cancellationToken)
{
var ratios = new[] { Ratio.Wide, Ratio.Tall }
.Select(r => new RatioPlan(r, image.FocusFor(r.Name), VariantPlanner.DefaultWidths(r)))
@@ -198,9 +259,7 @@ public class HeroImageService(
variants.Add(new HeroVariant(spec.Ratio.Name, rendered.Width, rendered.Height, spec.Format.MimeType(), name, rendered.Bytes.LongLength));
}
image.SetHash = SetHashOf(image);
image.Variants = variants;
return files;
return (files, variants);
}
private async Task EnsureRoomAsync(long incomingBytes, CancellationToken cancellationToken)
@@ -33,6 +33,13 @@ public static class HeroMapping
{
public const string Area = "hero";
/// <summary>The set hash the image should have at its current focus.</summary>
public static string TargetSetHash(HeroImage image)
=> MediaPaths.SetHash(image.OriginalSha256, [("wide", image.FocusFor("wide")), ("tall", image.FocusFor("tall"))]);
/// <summary>True when the image has no set yet or its set was cut for another focus (the work of the background renderer).</summary>
public static bool NeedsRender(HeroImage image) => image.Variants.Count == 0 || image.SetHash != TargetSetHash(image);
public static HeroVariantDto ToDto(HeroImage image, HeroVariant variant)
=> new(variant.Ratio, variant.Width, variant.Height, variant.Type, MediaPaths.Url(Area, image.SetHash, variant.FileName));
@@ -55,7 +62,8 @@ public static class HeroMapping
image.WideFocusX is null ? null : FocusDto.From(image.FocusFor("wide")),
image.TallFocusX is null ? null : FocusDto.From(image.FocusFor("tall")),
SrcFor(image),
image.Variants.Select(v => ToDto(image, v)).ToList());
image.Variants.Select(v => ToDto(image, v)).ToList(),
NeedsRender(image));
public static bool IsLive(HeroContent hero, DateTimeOffset now)
=> hero.IsActive
@@ -0,0 +1,89 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace SlpModularCms.Modules.Hero.Services;
/// <summary>Wakes the background renderer after an upload or a save. A singleton shared by the request and the worker.</summary>
public sealed class HeroRenderSignal
{
private readonly SemaphoreSlim _signal = new(0, 1);
public void Notify()
{
try
{
_signal.Release();
}
catch (SemaphoreFullException)
{
// Already signalled; the worker will look at everything that is pending.
}
}
/// <summary>Waits for a signal or for the timeout, whichever comes first.</summary>
public Task<bool> WaitAsync(TimeSpan timeout, CancellationToken cancellationToken) => _signal.WaitAsync(timeout, cancellationToken);
}
/// <summary>
/// Cuts the variants of images that have none yet (upload) or whose set no longer matches their focus (a saved focus
/// change), one image at a time, so an upload or a save never waits for the slow part. An image goes live (or switches to
/// its new set) the moment its set is complete; until then the old set, or nothing, is published. Looks again when woken
/// and at least every minute, and at start-up, so work interrupted by a restart is picked up.
/// </summary>
public sealed class HeroRenderService(IServiceScopeFactory scopes, HeroRenderSignal signal, ILogger<HeroRenderService> logger) : BackgroundService
{
private const int MaxAttempts = 3;
private readonly Dictionary<Guid, int> _failures = [];
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
try
{
// Let the start-up (migrations, health check) finish first.
await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken);
while (!stoppingToken.IsCancellationRequested)
{
await RunPassAsync(stoppingToken);
await signal.WaitAsync(TimeSpan.FromMinutes(1), stoppingToken);
}
}
catch (OperationCanceledException)
{
// Shutting down.
}
}
private async Task RunPassAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
var skip = _failures.Where(f => f.Value >= MaxAttempts).Select(f => f.Key).ToHashSet();
Guid? handled = null;
try
{
using var scope = scopes.CreateScope();
handled = await scope.ServiceProvider.GetRequiredService<IHeroImageService>().RenderNextPendingAsync(skip, stoppingToken);
}
catch (HeroImageRenderException failure)
{
var attempts = _failures[failure.ImageId] = _failures.GetValueOrDefault(failure.ImageId) + 1;
logger.LogError(failure.InnerException, "Cutting the variants of hero image {ImageId} failed (attempt {Attempt} of {Max}).", failure.ImageId, attempts, MaxAttempts);
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
continue;
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
logger.LogError(exception, "Looking for hero images to cut failed; trying again at the next wake-up.");
return;
}
if (handled is null)
{
return;
}
_failures.Remove(handled.Value);
}
}
}
@@ -115,16 +115,17 @@ public class HeroService(
}
}
// A changed focus is applied only now, together with the hero: the new variants are cut into new files first, then
// the hero and the images are saved in one go, so nothing live changes before the hero is saved.
// A changed focus is only recorded here: the new variants are cut in the background into new files and the image
// switches to them when they are complete, so saving is quick and nothing live changes half-way.
foreach (var edit in focusRequests)
{
await imageService.StageFocusAsync(images[edit.ImageId], new HeroFocusRequest(edit.Focus, edit.Wide, edit.Tall), cancellationToken);
imageService.ApplyFocus(images[edit.ImageId], new HeroFocusRequest(edit.Focus, edit.Wide, edit.Tall));
}
await repository.SaveChangesAsync(cancellationToken);
indexCache?.Invalidate();
imageService.NotifyPending();
logger.LogInformation("Hero {HeroId} {Action} by user {LastModifiedByUserId} ({Slides} slides)", hero.Id, created ? "Created" : "Updated", callerId, keep.Count);
return HeroMapping.ToAdmin(hero, now, options.Value.MaxPublishedSlides);