(bugfix): fluent migrator fixes

CHANGELOG.md

@@ -4,6 +4,34 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.3.12] - 2026-04-26
+
+### Fixed
+
+- **FluentMigrator:** Use `.ScanIn(…).For.All()` instead of `.For.Migrations()` so in-process discovery matches FluentMigrator guidance (avoids “no migrations” / incomplete runner behavior in some versions).
+- **FluentMigrator:** Throw if the engine connection string is empty when registering the runner — a null/empty `WithGlobalConnectionString` puts the processor in **connectionless/preview** mode (SQL logged, **nothing committed**), which matches reports of empty databases with no errors.
+- **Migrations:** Log host/database (no password) and count of `[Migration]` types before `MigrateUp`; after coordination DDL, verify `public.users` or `public."VersionInfo"` exists or fail with an actionable error (wrong `Database=`, permissions, or preview mode).
+- **Database bootstrap:** If the role cannot open a maintenance connection to database `postgres` (common for locked-down app users), log a warning and skip automatic `CREATE DATABASE` instead of failing the whole migration step.
+
+## [3.3.11] - 2026-04-26
+
+### Added
+
+- **Database:** FluentMigrator `RestoreUsersJwtTokensJsonIfDropped` (`20260426120000`) restores `users.JwtTokensJson` with `ADD COLUMN IF NOT EXISTS` when an older database had it removed by a prior `JwtTokensTableMigrateFromJson` revision.
+- **Helm / config:** `certsServerConfig.configuration.certsUIEngineConfiguration.autoSyncSchema` (default `true`) is rendered into server `appsettings.json` so add-only schema sync runs on every startup unless explicitly disabled.
+
+### Changed
+
+- **Startup schema policy:** Documented expand-only expectations — FluentMigrator `Up()` should add tables/columns; avoid dropping renamed or legacy columns in `Up()`. `JwtTokensTableMigrateFromJson` no longer drops `JwtTokensJson` (tokens remain in `jwt_tokens`; legacy JSON column may remain for audit).
+- **Schema sync:** `AutoSyncSchema` defaults to **true** in repo `appsettings.json`; `SchemaSyncService` desired map includes `users.IsActive`, `TwoFactorSharedKey`, and optional `JwtTokensJson` for additive repair. Still **ADD COLUMN IF NOT EXISTS** only (no DROP).
+- **ICertsEngineConfiguration / ISchemaSyncService:** Clarified that add-only sync is recommended and describes the no-DROP guarantee.
+
+## [3.3.10] - 2026-04-26
+
+### Fixed
+
+- **Database:** After FluentMigrator `MigrateUp`, `RunMigrationsService` applies idempotent `CREATE TABLE IF NOT EXISTS` / `CREATE INDEX IF NOT EXISTS` for `acme_http_challenges` and `app_runtime_leases`. If `VersionInfo` already records the migration but tables are missing (restore drift, partial apply, manual DB edits), FluentMigrator would skip `Up()` and the bootstrap lease would fail with `42P01`; this repair aligns schema with runtime needs.
+
 ## [3.3.9] - 2026-04-26
 
 ### Fixed

src/MaksIT.CertsUI.Engine/Dto/Identity/UserDto.cs

@@ -4,6 +4,7 @@ namespace MaksIT.CertsUI.Engine.Dto.Identity;
 
 /// <summary>
 /// PostgreSQL <c>users</c> row (Linq2DB). JWT sessions live in <c>jwt_tokens</c>; <see cref="JwtTokens"/> is not a mapped column.
+/// A legacy <c>JwtTokensJson</c> column may still exist on the table for expand-only migrations; it is not mapped here.
 /// </summary>
 public class UserDto : DtoDocumentBase<Guid> {
   public required string Name { get; set; }

src/MaksIT.CertsUI.Engine/Extensions/ServiceCollectionExtensions.cs

@@ -22,12 +22,16 @@ public static class ServiceCollectionExtensions {
 
     services.AddSingleton(certsEngineConfiguration);
 
-    // FluentMigrator: run migrations at startup via IRunMigrationsService (called from InitializationHostedService)
+    if (string.IsNullOrWhiteSpace(certsEngineConfiguration.ConnectionString))
+      throw new ArgumentException("Certs engine connection string is required for FluentMigrator (empty string uses connectionless/preview mode and will not create tables).", nameof(certsEngineConfiguration));
+
+    // FluentMigrator: IRunMigrationsService invoked from Program.cs before RunAsync. Use .For.All() so version metadata
+    // and migration discovery match in-process runner expectations (see FluentMigrator docs / #1062).
     services.AddFluentMigratorCore()
       .ConfigureRunner(rb => rb
         .AddPostgres()
         .WithGlobalConnectionString(certsEngineConfiguration.ConnectionString)
-        .ScanIn(typeof(BaselineCertsSchema).Assembly).For.Migrations())
+        .ScanIn(typeof(BaselineCertsSchema).Assembly).For.All())
       .AddLogging(lb => lb.AddFluentMigratorConsole());
     services.AddScoped<IRunMigrationsService, RunMigrationsService>();
     services.AddScoped<ISchemaSyncService, SchemaSyncService>();
@@ -67,7 +71,8 @@ public static class ServiceCollectionExtensions {
   #region Host initialization helpers
 
   /// <summary>
-  /// Runs FluentMigrator then optional add-only schema sync (when <see cref="ICertsEngineConfiguration.AutoSyncSchema"/> is true). Called from <c>Program.cs</c> before <c>RunAsync</c>.
+  /// Runs FluentMigrator (versioned <c>Up()</c> migrations, expand-only policy: no dropping legacy columns) then add-only
+  /// <see cref="ISchemaSyncService"/> when <see cref="ICertsEngineConfiguration.AutoSyncSchema"/> is true. Called from <c>Program.cs</c> before <c>RunAsync</c>.
   /// </summary>
   public static async Task EnsureCertsEngineMigratedAsync(this IServiceProvider serviceProvider) {
     await using var scope = serviceProvider.CreateAsyncScope();

src/MaksIT.CertsUI.Engine/FluentMigrations/20260418100000_JwtTokensTableMigrateFromJson.cs

@@ -4,7 +4,8 @@ using FluentMigrator;
 namespace MaksIT.CertsUI.Engine.FluentMigrations;
 
 /// <summary>
-/// Normalizes JWT refresh/access tokens into <c>jwt_tokens</c> (one row per token) and drops legacy <c>users.JwtTokensJson</c>.
+/// Normalizes JWT refresh/access tokens into <c>jwt_tokens</c> (one row per token). Legacy <c>users.JwtTokensJson</c> is retained
+/// (expand-only policy); the app reads <c>jwt_tokens</c> only.
 /// </summary>
 [Migration(20260418100000)]
 public class JwtTokensTableMigrateFromJson : Migration {
@@ -46,8 +47,6 @@ public class JwtTokensTableMigrateFromJson : Migration {
       ) AS elem
       WHERE (elem->>'Id') IS NOT NULL
       """);
-
-    Delete.Column("JwtTokensJson").FromTable("users");
   }
 
   public override void Down() =>

src/MaksIT.CertsUI.Engine/FluentMigrations/20260426120000_RestoreUsersJwtTokensJsonIfDropped.cs

@@ -0,0 +1,19 @@
+using FluentMigrator;
+
+namespace MaksIT.CertsUI.Engine.FluentMigrations;
+
+/// <summary>
+/// Databases that already applied <see cref="JwtTokensTableMigrateFromJson"/> when it still dropped <c>JwtTokensJson</c>
+/// get the column back (empty default). Expand-only: we never remove renamed/legacy columns in <c>Up()</c>.
+/// </summary>
+[Migration(20260426120000)]
+public class RestoreUsersJwtTokensJsonIfDropped : Migration {
+  public override void Up() {
+    Execute.Sql("""
+      ALTER TABLE "users" ADD COLUMN IF NOT EXISTS "JwtTokensJson" text NOT NULL DEFAULT '';
+      """);
+  }
+
+  public override void Down() =>
+    throw new NotSupportedException("Down is not supported for this repair migration.");
+}

src/MaksIT.CertsUI.Engine/ICertsEngineConfiguration.cs

@@ -6,7 +6,10 @@ namespace MaksIT.CertsUI.Engine;
 public interface ICertsEngineConfiguration {
   string ConnectionString { get; }
 
-  /// <summary>When true, run add-only schema sync at startup after migrations. Default false in production.</summary>
+  /// <summary>
+  /// When true (recommended), run add-only schema sync after FluentMigrator on each startup: <c>ALTER TABLE … ADD COLUMN IF NOT EXISTS</c> only,
+  /// never DROP. Disable only if you manage DDL exclusively elsewhere.
+  /// </summary>
   bool AutoSyncSchema { get; }
 
   /// <summary>Let's Encrypt production ACME directory URL (RFC 8555).</summary>

src/MaksIT.CertsUI.Engine/Infrastructure/ISchemaSyncService.cs

@@ -1,8 +1,8 @@
 namespace MaksIT.CertsUI.Engine.Infrastructure;
 
 /// <summary>
-/// Syncs the database schema to match DTOs: add missing tables and columns only (no DROP).
-/// Called from startup when <see cref="ICertsEngineConfiguration.AutoSyncSchema"/> is true.
+/// Syncs the database schema toward DTOs: <c>ADD COLUMN IF NOT EXISTS</c> only (no DROP of legacy or renamed columns).
+/// Called from startup when <see cref="ICertsEngineConfiguration.AutoSyncSchema"/> is true (recommended).
 /// </summary>
 public interface ISchemaSyncService {
   Task SyncSchemaAsync(CancellationToken cancellationToken = default);

src/MaksIT.CertsUI.Engine/Infrastructure/RunMigrationsService.cs

@@ -1,3 +1,5 @@
+using System.Reflection;
+using FluentMigrator;
 using FluentMigrator.Runner;
 using Microsoft.Extensions.Logging;
 using MaksIT.CertsUI.Engine.FluentMigrations;
@@ -6,7 +8,9 @@ using Npgsql;
 namespace MaksIT.CertsUI.Engine.Infrastructure;
 
 /// <summary>
-/// FluentMigrator runner for the Certs database: optionally creates the database, baselines legacy EF-created schemas, then migrates up.
+/// FluentMigrator runner for the Certs database: optionally creates the database, baselines legacy EF-created schemas, migrates up,
+/// then idempotent coordination-table repair. Forward <c>Up()</c> migrations should be additive (new tables/columns); avoid dropping
+/// renamed or legacy columns in <c>Up()</c> — use expand/contract and ops-driven cleanup.
 /// </summary>
 public sealed class RunMigrationsService(
   IMigrationRunner migrationRunner,
@@ -17,13 +21,82 @@ public sealed class RunMigrationsService(
   public static long BaselineVersion => BaselineCertsSchema.Version;
 
   public async Task RunAsync(CancellationToken cancellationToken = default) {
-    logger.LogInformation("Running Certs database migrations...");
+    if (string.IsNullOrWhiteSpace(config.ConnectionString))
+      throw new InvalidOperationException(
+        "Database connection string is empty. FluentMigrator would run in connectionless/preview mode and never commit DDL.");
+
+    var csb = new NpgsqlConnectionStringBuilder(config.ConnectionString);
+    logger.LogInformation(
+      "Running Certs database migrations (host={Host}, database={Database})…",
+      csb.Host ?? "(default)",
+      string.IsNullOrEmpty(csb.Database) ? "(default)" : csb.Database);
+
+    var migrationTypeCount = typeof(BaselineCertsSchema).Assembly.GetTypes()
+      .Count(t => t.GetCustomAttribute<MigrationAttribute>(inherit: false) is not null);
+    logger.LogInformation("FluentMigrator discovered {MigrationCount} migration type(s) in {Assembly}.", migrationTypeCount, typeof(BaselineCertsSchema).Assembly.GetName().Name);
+
     await EnsureDatabaseExistsAsync(cancellationToken).ConfigureAwait(false);
     await BaselineExistingEfDatabaseAsync(cancellationToken).ConfigureAwait(false);
     await Task.Run(() => migrationRunner.MigrateUp(), cancellationToken).ConfigureAwait(false);
+    await EnsureCoordinationTablesAsync(cancellationToken).ConfigureAwait(false);
+    await VerifyCoreSchemaAsync(cancellationToken).ConfigureAwait(false);
     logger.LogInformation("Certs database migrations completed.");
   }
 
+  /// <summary>Fails fast if the database is still empty after MigrateUp (misconfiguration, preview processor, wrong DB).</summary>
+  private async Task VerifyCoreSchemaAsync(CancellationToken cancellationToken) {
+    await using var conn = new NpgsqlConnection(config.ConnectionString);
+    await conn.OpenAsync(cancellationToken).ConfigureAwait(false);
+
+    await using var cmd = new NpgsqlCommand(
+      """
+      SELECT EXISTS (
+        SELECT 1 FROM information_schema.tables
+        WHERE table_schema = 'public' AND table_name = 'users')
+      OR EXISTS (
+        SELECT 1 FROM information_schema.tables
+        WHERE table_schema = 'public' AND table_name = 'VersionInfo');
+      """,
+      conn);
+
+    var any = await cmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
+    if (Equals(any, true))
+      return;
+
+    throw new InvalidOperationException(
+      "After FluentMigrator MigrateUp(), the target database still has no \"users\" or \"VersionInfo\" table in schema \"public\". " +
+      "Confirm the connection string Database= value, that the role can CREATE TABLE, and that FluentMigrator is not in preview/connectionless mode (non-empty connection string).");
+  }
+
+  /// <summary>
+  /// Idempotent DDL for HA tables from <see cref="AcmeChallengesAndRuntimeLeases"/>.
+  /// When <c>VersionInfo</c> already lists that migration but the tables are missing (restore drift, partial apply),
+  /// FluentMigrator will not re-run <c>Up()</c>; this repair keeps lease and HTTP-01 persistence working.
+  /// </summary>
+  private async Task EnsureCoordinationTablesAsync(CancellationToken cancellationToken) {
+    await using var conn = new NpgsqlConnection(config.ConnectionString);
+    await conn.OpenAsync(cancellationToken).ConfigureAwait(false);
+
+    await using var cmd = new NpgsqlCommand(
+      """
+      CREATE TABLE IF NOT EXISTS acme_http_challenges (
+        file_name text NOT NULL PRIMARY KEY,
+        token_value text NOT NULL,
+        created_at_utc timestamp with time zone NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS "IX_acme_http_challenges_created_at_utc" ON acme_http_challenges (created_at_utc);
+      CREATE TABLE IF NOT EXISTS app_runtime_leases (
+        lease_name text NOT NULL PRIMARY KEY,
+        holder_id text NOT NULL,
+        version bigint NOT NULL DEFAULT 1,
+        acquired_at_utc timestamp with time zone NOT NULL,
+        expires_at_utc timestamp with time zone NOT NULL
+      );
+      """,
+      conn);
+    await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+  }
+
   private async Task EnsureDatabaseExistsAsync(CancellationToken cancellationToken) {
     var builder = new NpgsqlConnectionStringBuilder(config.ConnectionString);
     var database = builder.Database?.Trim();
@@ -32,22 +105,32 @@ public sealed class RunMigrationsService(
     builder.Database = "postgres";
     var postgresCs = builder.ConnectionString;
 
-    await using var conn = new NpgsqlConnection(postgresCs);
-    await conn.OpenAsync(cancellationToken).ConfigureAwait(false);
+    try {
+      await using var conn = new NpgsqlConnection(postgresCs);
+      await conn.OpenAsync(cancellationToken).ConfigureAwait(false);
 
-    await using (var cmd = new NpgsqlCommand("SELECT 1 FROM pg_database WHERE datname = @dbname", conn)) {
-      cmd.Parameters.AddWithValue("dbname", database);
-      await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
-      if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
-        return;
-    }
+      await using (var cmd = new NpgsqlCommand("SELECT 1 FROM pg_database WHERE datname = @dbname", conn)) {
+        cmd.Parameters.AddWithValue("dbname", database);
+        await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
+        if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
+          return;
+      }
 
-    logger.LogInformation("Database \"{Database}\" does not exist; creating it.", database);
-    var quotedDb = $"\"{database.Replace("\"", "\"\"", StringComparison.Ordinal)}\"";
-    await using (var createCmd = new NpgsqlCommand($"CREATE DATABASE {quotedDb}", conn)) {
-      await createCmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+      logger.LogInformation("Database \"{Database}\" does not exist; creating it.", database);
+      var quotedDb = $"\"{database.Replace("\"", "\"\"", StringComparison.Ordinal)}\"";
+      await using (var createCmd = new NpgsqlCommand($"CREATE DATABASE {quotedDb}", conn)) {
+        await createCmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+      }
+
+      logger.LogInformation("Database \"{Database}\" created.", database);
+    }
+    catch (Exception ex) {
+      logger.LogWarning(
+        ex,
+        "Could not use maintenance connection to database \"postgres\" for auto-create of \"{TargetDatabase}\". " +
+        "If the target database already exists, migrations will continue; otherwise create the database manually.",
+        database);
     }
-    logger.LogInformation("Database \"{Database}\" created.", database);
   }
 
   /// <summary>

src/MaksIT.CertsUI.Engine/Infrastructure/SchemaSyncService.cs

@@ -4,7 +4,8 @@ using Npgsql;
 namespace MaksIT.CertsUI.Engine.Infrastructure;
 
 /// <summary>
-/// Add-only schema sync: adds missing columns to match DTOs (no DROP). Runs after FluentMigrator; table creation stays in migrations.
+/// Add-only schema sync on startup (after FluentMigrator): missing columns from the configured desired-schema map get
+/// <c>ALTER TABLE … ADD COLUMN IF NOT EXISTS</c>. No DROP, TRUNCATE, or column rename destructive DDL — legacy columns stay until you clean them as an ops task.
 /// </summary>
 public class SchemaSyncService(ICertsEngineConfiguration config, ILogger<SchemaSyncService> logger) : ISchemaSyncService {
   private readonly ICertsEngineConfiguration _config = config;
@@ -82,7 +83,10 @@ public class SchemaSyncService(ICertsEngineConfiguration config, ILogger<SchemaS
         ("Name", "text"),
         ("Salt", "text"),
         ("Hash", "text"),
+        ("JwtTokensJson", "text"),
         ("LastLoginUtc", "timestamp with time zone"),
+        ("IsActive", "boolean"),
+        ("TwoFactorSharedKey", "text"),
       ],
       ["jwt_tokens"] = [
         ("Id", "uuid"),
@@ -162,6 +166,11 @@ public class SchemaSyncService(ICertsEngineConfiguration config, ILogger<SchemaS
       if (column.Equals("ExpiresAtUtc", StringComparison.OrdinalIgnoreCase)) return false;
     }
 
+    if (table.Equals("users", StringComparison.OrdinalIgnoreCase)) {
+      if (column.Equals("TwoFactorSharedKey", StringComparison.OrdinalIgnoreCase)) return false;
+      if (column.Equals("JwtTokensJson", StringComparison.OrdinalIgnoreCase)) return false;
+    }
+
     return true;
   }
 

src/MaksIT.CertsUI/Configuration.cs

@@ -71,7 +71,7 @@ public class CertsUIEngineConfiguration : ICertsFlowEngineConfiguration {
   /// <summary>Npgsql connection string; optional when using legacy <c>ConnectionStrings:Certs</c>.</summary>
   public string? ConnectionString { get; set; }
 
-  /// <summary>When true, add-only schema sync after FluentMigrator. Default false.</summary>
+  /// <summary>When true, add-only schema sync after FluentMigrator (ADD COLUMN only, never DROP legacy/renamed columns). Set explicitly in appsettings/Helm (deserialization defaults missing bools to false).</summary>
   public bool AutoSyncSchema { get; set; }
 
   public required AdminUser Admin { get; set; }

src/MaksIT.CertsUI/MaksIT.CertsUI.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <Version>3.3.9</Version>
+    <Version>3.3.12</Version>
     <TargetFramework>net10.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>

src/MaksIT.CertsUI/appsettings.json

@@ -11,7 +11,7 @@
   "Configuration": {
     "CertsUIEngineConfiguration": {
       "ConnectionString": "",
-      "AutoSyncSchema": false,
+      "AutoSyncSchema": true,
 
       "Admin": {
         "Username": "",

src/helm/values.yaml

@@ -20,6 +20,8 @@ certsServerConfig:
     microsoftAspNetCore: Warning
   configuration:
     certsUIEngineConfiguration:
+      # Add-only column sync after FluentMigrator (ALTER ADD IF NOT EXISTS; never DROP). Set false to disable.
+      autoSyncSchema: true
       admin:
         username: "admin"
       jwt:
@@ -141,6 +143,7 @@ components:
           "AllowedHosts": {{ .Values.certsServerConfig.allowedHosts | toJson }},
           "Configuration": {
             "CertsUIEngineConfiguration": {
+              "AutoSyncSchema": {{ .Values.certsServerConfig.configuration.certsUIEngineConfiguration.autoSyncSchema }},
               "Admin": {
                 "Username": {{ .Values.certsServerConfig.configuration.certsUIEngineConfiguration.admin.username | toJson }},
                 "Password": ""