diff --git a/CHANGELOG.md b/CHANGELOG.md index 62fca62..28874cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,34 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.3.12] - 2026-04-26 + +### Fixed + +- **FluentMigrator:** Use `.ScanIn(…).For.All()` instead of `.For.Migrations()` so in-process discovery matches FluentMigrator guidance (avoids “no migrations” / incomplete runner behavior in some versions). +- **FluentMigrator:** Throw if the engine connection string is empty when registering the runner — a null/empty `WithGlobalConnectionString` puts the processor in **connectionless/preview** mode (SQL logged, **nothing committed**), which matches reports of empty databases with no errors. +- **Migrations:** Log host/database (no password) and count of `[Migration]` types before `MigrateUp`; after coordination DDL, verify `public.users` or `public."VersionInfo"` exists or fail with an actionable error (wrong `Database=`, permissions, or preview mode). +- **Database bootstrap:** If the role cannot open a maintenance connection to database `postgres` (common for locked-down app users), log a warning and skip automatic `CREATE DATABASE` instead of failing the whole migration step. + +## [3.3.11] - 2026-04-26 + +### Added + +- **Database:** FluentMigrator `RestoreUsersJwtTokensJsonIfDropped` (`20260426120000`) restores `users.JwtTokensJson` with `ADD COLUMN IF NOT EXISTS` when an older database had it removed by a prior `JwtTokensTableMigrateFromJson` revision. +- **Helm / config:** `certsServerConfig.configuration.certsUIEngineConfiguration.autoSyncSchema` (default `true`) is rendered into server `appsettings.json` so add-only schema sync runs on every startup unless explicitly disabled. + +### Changed + +- **Startup schema policy:** Documented expand-only expectations — FluentMigrator `Up()` should add tables/columns; avoid dropping renamed or legacy columns in `Up()`. `JwtTokensTableMigrateFromJson` no longer drops `JwtTokensJson` (tokens remain in `jwt_tokens`; legacy JSON column may remain for audit). +- **Schema sync:** `AutoSyncSchema` defaults to **true** in repo `appsettings.json`; `SchemaSyncService` desired map includes `users.IsActive`, `TwoFactorSharedKey`, and optional `JwtTokensJson` for additive repair. Still **ADD COLUMN IF NOT EXISTS** only (no DROP). +- **ICertsEngineConfiguration / ISchemaSyncService:** Clarified that add-only sync is recommended and describes the no-DROP guarantee. + +## [3.3.10] - 2026-04-26 + +### Fixed + +- **Database:** After FluentMigrator `MigrateUp`, `RunMigrationsService` applies idempotent `CREATE TABLE IF NOT EXISTS` / `CREATE INDEX IF NOT EXISTS` for `acme_http_challenges` and `app_runtime_leases`. If `VersionInfo` already records the migration but tables are missing (restore drift, partial apply, manual DB edits), FluentMigrator would skip `Up()` and the bootstrap lease would fail with `42P01`; this repair aligns schema with runtime needs. + ## [3.3.9] - 2026-04-26 ### Fixed diff --git a/src/MaksIT.CertsUI.Engine/Dto/Identity/UserDto.cs b/src/MaksIT.CertsUI.Engine/Dto/Identity/UserDto.cs index 379912e..177037c 100644 --- a/src/MaksIT.CertsUI.Engine/Dto/Identity/UserDto.cs +++ b/src/MaksIT.CertsUI.Engine/Dto/Identity/UserDto.cs @@ -4,6 +4,7 @@ namespace MaksIT.CertsUI.Engine.Dto.Identity; /// /// PostgreSQL users row (Linq2DB). JWT sessions live in jwt_tokens; is not a mapped column. +/// A legacy JwtTokensJson column may still exist on the table for expand-only migrations; it is not mapped here. /// public class UserDto : DtoDocumentBase { public required string Name { get; set; } diff --git a/src/MaksIT.CertsUI.Engine/Extensions/ServiceCollectionExtensions.cs b/src/MaksIT.CertsUI.Engine/Extensions/ServiceCollectionExtensions.cs index 9622088..5c290db 100644 --- a/src/MaksIT.CertsUI.Engine/Extensions/ServiceCollectionExtensions.cs +++ b/src/MaksIT.CertsUI.Engine/Extensions/ServiceCollectionExtensions.cs @@ -22,12 +22,16 @@ public static class ServiceCollectionExtensions { services.AddSingleton(certsEngineConfiguration); - // FluentMigrator: run migrations at startup via IRunMigrationsService (called from InitializationHostedService) + if (string.IsNullOrWhiteSpace(certsEngineConfiguration.ConnectionString)) + throw new ArgumentException("Certs engine connection string is required for FluentMigrator (empty string uses connectionless/preview mode and will not create tables).", nameof(certsEngineConfiguration)); + + // FluentMigrator: IRunMigrationsService invoked from Program.cs before RunAsync. Use .For.All() so version metadata + // and migration discovery match in-process runner expectations (see FluentMigrator docs / #1062). services.AddFluentMigratorCore() .ConfigureRunner(rb => rb .AddPostgres() .WithGlobalConnectionString(certsEngineConfiguration.ConnectionString) - .ScanIn(typeof(BaselineCertsSchema).Assembly).For.Migrations()) + .ScanIn(typeof(BaselineCertsSchema).Assembly).For.All()) .AddLogging(lb => lb.AddFluentMigratorConsole()); services.AddScoped(); services.AddScoped(); @@ -67,7 +71,8 @@ public static class ServiceCollectionExtensions { #region Host initialization helpers /// - /// Runs FluentMigrator then optional add-only schema sync (when is true). Called from Program.cs before RunAsync. + /// Runs FluentMigrator (versioned Up() migrations, expand-only policy: no dropping legacy columns) then add-only + /// when is true. Called from Program.cs before RunAsync. /// public static async Task EnsureCertsEngineMigratedAsync(this IServiceProvider serviceProvider) { await using var scope = serviceProvider.CreateAsyncScope(); diff --git a/src/MaksIT.CertsUI.Engine/FluentMigrations/20260418100000_JwtTokensTableMigrateFromJson.cs b/src/MaksIT.CertsUI.Engine/FluentMigrations/20260418100000_JwtTokensTableMigrateFromJson.cs index 43db137..1d23435 100644 --- a/src/MaksIT.CertsUI.Engine/FluentMigrations/20260418100000_JwtTokensTableMigrateFromJson.cs +++ b/src/MaksIT.CertsUI.Engine/FluentMigrations/20260418100000_JwtTokensTableMigrateFromJson.cs @@ -4,7 +4,8 @@ using FluentMigrator; namespace MaksIT.CertsUI.Engine.FluentMigrations; /// -/// Normalizes JWT refresh/access tokens into jwt_tokens (one row per token) and drops legacy users.JwtTokensJson. +/// Normalizes JWT refresh/access tokens into jwt_tokens (one row per token). Legacy users.JwtTokensJson is retained +/// (expand-only policy); the app reads jwt_tokens only. /// [Migration(20260418100000)] public class JwtTokensTableMigrateFromJson : Migration { @@ -46,8 +47,6 @@ public class JwtTokensTableMigrateFromJson : Migration { ) AS elem WHERE (elem->>'Id') IS NOT NULL """); - - Delete.Column("JwtTokensJson").FromTable("users"); } public override void Down() => diff --git a/src/MaksIT.CertsUI.Engine/FluentMigrations/20260426120000_RestoreUsersJwtTokensJsonIfDropped.cs b/src/MaksIT.CertsUI.Engine/FluentMigrations/20260426120000_RestoreUsersJwtTokensJsonIfDropped.cs new file mode 100644 index 0000000..680ca71 --- /dev/null +++ b/src/MaksIT.CertsUI.Engine/FluentMigrations/20260426120000_RestoreUsersJwtTokensJsonIfDropped.cs @@ -0,0 +1,19 @@ +using FluentMigrator; + +namespace MaksIT.CertsUI.Engine.FluentMigrations; + +/// +/// Databases that already applied when it still dropped JwtTokensJson +/// get the column back (empty default). Expand-only: we never remove renamed/legacy columns in Up(). +/// +[Migration(20260426120000)] +public class RestoreUsersJwtTokensJsonIfDropped : Migration { + public override void Up() { + Execute.Sql(""" + ALTER TABLE "users" ADD COLUMN IF NOT EXISTS "JwtTokensJson" text NOT NULL DEFAULT ''; + """); + } + + public override void Down() => + throw new NotSupportedException("Down is not supported for this repair migration."); +} diff --git a/src/MaksIT.CertsUI.Engine/ICertsEngineConfiguration.cs b/src/MaksIT.CertsUI.Engine/ICertsEngineConfiguration.cs index bcec908..fbdff84 100644 --- a/src/MaksIT.CertsUI.Engine/ICertsEngineConfiguration.cs +++ b/src/MaksIT.CertsUI.Engine/ICertsEngineConfiguration.cs @@ -6,7 +6,10 @@ namespace MaksIT.CertsUI.Engine; public interface ICertsEngineConfiguration { string ConnectionString { get; } - /// When true, run add-only schema sync at startup after migrations. Default false in production. + /// + /// When true (recommended), run add-only schema sync after FluentMigrator on each startup: ALTER TABLE … ADD COLUMN IF NOT EXISTS only, + /// never DROP. Disable only if you manage DDL exclusively elsewhere. + /// bool AutoSyncSchema { get; } /// Let's Encrypt production ACME directory URL (RFC 8555). diff --git a/src/MaksIT.CertsUI.Engine/Infrastructure/ISchemaSyncService.cs b/src/MaksIT.CertsUI.Engine/Infrastructure/ISchemaSyncService.cs index 3be63df..3c11d21 100644 --- a/src/MaksIT.CertsUI.Engine/Infrastructure/ISchemaSyncService.cs +++ b/src/MaksIT.CertsUI.Engine/Infrastructure/ISchemaSyncService.cs @@ -1,8 +1,8 @@ namespace MaksIT.CertsUI.Engine.Infrastructure; /// -/// Syncs the database schema to match DTOs: add missing tables and columns only (no DROP). -/// Called from startup when is true. +/// Syncs the database schema toward DTOs: ADD COLUMN IF NOT EXISTS only (no DROP of legacy or renamed columns). +/// Called from startup when is true (recommended). /// public interface ISchemaSyncService { Task SyncSchemaAsync(CancellationToken cancellationToken = default); diff --git a/src/MaksIT.CertsUI.Engine/Infrastructure/RunMigrationsService.cs b/src/MaksIT.CertsUI.Engine/Infrastructure/RunMigrationsService.cs index b86be89..d160e7b 100644 --- a/src/MaksIT.CertsUI.Engine/Infrastructure/RunMigrationsService.cs +++ b/src/MaksIT.CertsUI.Engine/Infrastructure/RunMigrationsService.cs @@ -1,3 +1,5 @@ +using System.Reflection; +using FluentMigrator; using FluentMigrator.Runner; using Microsoft.Extensions.Logging; using MaksIT.CertsUI.Engine.FluentMigrations; @@ -6,7 +8,9 @@ using Npgsql; namespace MaksIT.CertsUI.Engine.Infrastructure; /// -/// FluentMigrator runner for the Certs database: optionally creates the database, baselines legacy EF-created schemas, then migrates up. +/// FluentMigrator runner for the Certs database: optionally creates the database, baselines legacy EF-created schemas, migrates up, +/// then idempotent coordination-table repair. Forward Up() migrations should be additive (new tables/columns); avoid dropping +/// renamed or legacy columns in Up() — use expand/contract and ops-driven cleanup. /// public sealed class RunMigrationsService( IMigrationRunner migrationRunner, @@ -17,13 +21,82 @@ public sealed class RunMigrationsService( public static long BaselineVersion => BaselineCertsSchema.Version; public async Task RunAsync(CancellationToken cancellationToken = default) { - logger.LogInformation("Running Certs database migrations..."); + if (string.IsNullOrWhiteSpace(config.ConnectionString)) + throw new InvalidOperationException( + "Database connection string is empty. FluentMigrator would run in connectionless/preview mode and never commit DDL."); + + var csb = new NpgsqlConnectionStringBuilder(config.ConnectionString); + logger.LogInformation( + "Running Certs database migrations (host={Host}, database={Database})…", + csb.Host ?? "(default)", + string.IsNullOrEmpty(csb.Database) ? "(default)" : csb.Database); + + var migrationTypeCount = typeof(BaselineCertsSchema).Assembly.GetTypes() + .Count(t => t.GetCustomAttribute(inherit: false) is not null); + logger.LogInformation("FluentMigrator discovered {MigrationCount} migration type(s) in {Assembly}.", migrationTypeCount, typeof(BaselineCertsSchema).Assembly.GetName().Name); + await EnsureDatabaseExistsAsync(cancellationToken).ConfigureAwait(false); await BaselineExistingEfDatabaseAsync(cancellationToken).ConfigureAwait(false); await Task.Run(() => migrationRunner.MigrateUp(), cancellationToken).ConfigureAwait(false); + await EnsureCoordinationTablesAsync(cancellationToken).ConfigureAwait(false); + await VerifyCoreSchemaAsync(cancellationToken).ConfigureAwait(false); logger.LogInformation("Certs database migrations completed."); } + /// Fails fast if the database is still empty after MigrateUp (misconfiguration, preview processor, wrong DB). + private async Task VerifyCoreSchemaAsync(CancellationToken cancellationToken) { + await using var conn = new NpgsqlConnection(config.ConnectionString); + await conn.OpenAsync(cancellationToken).ConfigureAwait(false); + + await using var cmd = new NpgsqlCommand( + """ + SELECT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = 'users') + OR EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = 'VersionInfo'); + """, + conn); + + var any = await cmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + if (Equals(any, true)) + return; + + throw new InvalidOperationException( + "After FluentMigrator MigrateUp(), the target database still has no \"users\" or \"VersionInfo\" table in schema \"public\". " + + "Confirm the connection string Database= value, that the role can CREATE TABLE, and that FluentMigrator is not in preview/connectionless mode (non-empty connection string)."); + } + + /// + /// Idempotent DDL for HA tables from . + /// When VersionInfo already lists that migration but the tables are missing (restore drift, partial apply), + /// FluentMigrator will not re-run Up(); this repair keeps lease and HTTP-01 persistence working. + /// + private async Task EnsureCoordinationTablesAsync(CancellationToken cancellationToken) { + await using var conn = new NpgsqlConnection(config.ConnectionString); + await conn.OpenAsync(cancellationToken).ConfigureAwait(false); + + await using var cmd = new NpgsqlCommand( + """ + CREATE TABLE IF NOT EXISTS acme_http_challenges ( + file_name text NOT NULL PRIMARY KEY, + token_value text NOT NULL, + created_at_utc timestamp with time zone NOT NULL + ); + CREATE INDEX IF NOT EXISTS "IX_acme_http_challenges_created_at_utc" ON acme_http_challenges (created_at_utc); + CREATE TABLE IF NOT EXISTS app_runtime_leases ( + lease_name text NOT NULL PRIMARY KEY, + holder_id text NOT NULL, + version bigint NOT NULL DEFAULT 1, + acquired_at_utc timestamp with time zone NOT NULL, + expires_at_utc timestamp with time zone NOT NULL + ); + """, + conn); + await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + private async Task EnsureDatabaseExistsAsync(CancellationToken cancellationToken) { var builder = new NpgsqlConnectionStringBuilder(config.ConnectionString); var database = builder.Database?.Trim(); @@ -32,22 +105,32 @@ public sealed class RunMigrationsService( builder.Database = "postgres"; var postgresCs = builder.ConnectionString; - await using var conn = new NpgsqlConnection(postgresCs); - await conn.OpenAsync(cancellationToken).ConfigureAwait(false); + try { + await using var conn = new NpgsqlConnection(postgresCs); + await conn.OpenAsync(cancellationToken).ConfigureAwait(false); - await using (var cmd = new NpgsqlCommand("SELECT 1 FROM pg_database WHERE datname = @dbname", conn)) { - cmd.Parameters.AddWithValue("dbname", database); - await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); - if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) - return; - } + await using (var cmd = new NpgsqlCommand("SELECT 1 FROM pg_database WHERE datname = @dbname", conn)) { + cmd.Parameters.AddWithValue("dbname", database); + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + return; + } - logger.LogInformation("Database \"{Database}\" does not exist; creating it.", database); - var quotedDb = $"\"{database.Replace("\"", "\"\"", StringComparison.Ordinal)}\""; - await using (var createCmd = new NpgsqlCommand($"CREATE DATABASE {quotedDb}", conn)) { - await createCmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + logger.LogInformation("Database \"{Database}\" does not exist; creating it.", database); + var quotedDb = $"\"{database.Replace("\"", "\"\"", StringComparison.Ordinal)}\""; + await using (var createCmd = new NpgsqlCommand($"CREATE DATABASE {quotedDb}", conn)) { + await createCmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + logger.LogInformation("Database \"{Database}\" created.", database); + } + catch (Exception ex) { + logger.LogWarning( + ex, + "Could not use maintenance connection to database \"postgres\" for auto-create of \"{TargetDatabase}\". " + + "If the target database already exists, migrations will continue; otherwise create the database manually.", + database); } - logger.LogInformation("Database \"{Database}\" created.", database); } /// diff --git a/src/MaksIT.CertsUI.Engine/Infrastructure/SchemaSyncService.cs b/src/MaksIT.CertsUI.Engine/Infrastructure/SchemaSyncService.cs index 3432a33..5d46e4a 100644 --- a/src/MaksIT.CertsUI.Engine/Infrastructure/SchemaSyncService.cs +++ b/src/MaksIT.CertsUI.Engine/Infrastructure/SchemaSyncService.cs @@ -4,7 +4,8 @@ using Npgsql; namespace MaksIT.CertsUI.Engine.Infrastructure; /// -/// Add-only schema sync: adds missing columns to match DTOs (no DROP). Runs after FluentMigrator; table creation stays in migrations. +/// Add-only schema sync on startup (after FluentMigrator): missing columns from the configured desired-schema map get +/// ALTER TABLE … ADD COLUMN IF NOT EXISTS. No DROP, TRUNCATE, or column rename destructive DDL — legacy columns stay until you clean them as an ops task. /// public class SchemaSyncService(ICertsEngineConfiguration config, ILogger logger) : ISchemaSyncService { private readonly ICertsEngineConfiguration _config = config; @@ -82,7 +83,10 @@ public class SchemaSyncService(ICertsEngineConfiguration config, ILoggerNpgsql connection string; optional when using legacy ConnectionStrings:Certs. public string? ConnectionString { get; set; } - /// When true, add-only schema sync after FluentMigrator. Default false. + /// When true, add-only schema sync after FluentMigrator (ADD COLUMN only, never DROP legacy/renamed columns). Set explicitly in appsettings/Helm (deserialization defaults missing bools to false). public bool AutoSyncSchema { get; set; } public required AdminUser Admin { get; set; } diff --git a/src/MaksIT.CertsUI/MaksIT.CertsUI.csproj b/src/MaksIT.CertsUI/MaksIT.CertsUI.csproj index e446068..c5658e2 100644 --- a/src/MaksIT.CertsUI/MaksIT.CertsUI.csproj +++ b/src/MaksIT.CertsUI/MaksIT.CertsUI.csproj @@ -1,7 +1,7 @@ - 3.3.9 + 3.3.12 net10.0 enable enable diff --git a/src/MaksIT.CertsUI/appsettings.json b/src/MaksIT.CertsUI/appsettings.json index 47213ea..698b846 100644 --- a/src/MaksIT.CertsUI/appsettings.json +++ b/src/MaksIT.CertsUI/appsettings.json @@ -11,7 +11,7 @@ "Configuration": { "CertsUIEngineConfiguration": { "ConnectionString": "", - "AutoSyncSchema": false, + "AutoSyncSchema": true, "Admin": { "Username": "", diff --git a/src/helm/values.yaml b/src/helm/values.yaml index 98be717..a017e12 100644 --- a/src/helm/values.yaml +++ b/src/helm/values.yaml @@ -20,6 +20,8 @@ certsServerConfig: microsoftAspNetCore: Warning configuration: certsUIEngineConfiguration: + # Add-only column sync after FluentMigrator (ALTER ADD IF NOT EXISTS; never DROP). Set false to disable. + autoSyncSchema: true admin: username: "admin" jwt: @@ -141,6 +143,7 @@ components: "AllowedHosts": {{ .Values.certsServerConfig.allowedHosts | toJson }}, "Configuration": { "CertsUIEngineConfiguration": { + "AutoSyncSchema": {{ .Values.certsServerConfig.configuration.certsUIEngineConfiguration.autoSyncSchema }}, "Admin": { "Username": {{ .Values.certsServerConfig.configuration.certsUIEngineConfiguration.admin.username | toJson }}, "Password": ""