diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b7df57..0ad62d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ Represents the **NuGet** versions. +## v3.0.1 +- *Fixed:* The PostgreSQL `*_partition_order` index now correctly excludes the `event` column. +- *Fixed:* The PostgreSQL `NOW()` function has removed incorrect `AT TIME ZONE 'UTC'` as this is not required when the database timezone is set to UTC (which is the recommended configuration). +- *Fixed:* The JSON/YAML data correctly supports `^xxx` and the legacy `^(xxx)` replacement formats. + ## v3.0.0 All internal dependencies to [`CoreEx`](https://github.com/avanade/coreex) have been removed. This is intended to further generalize the capabilities of `DbEx`; but more importantly, break the pseudo circular dependency reference between the two repositories. - *Enhancement:* Added `net10.0` support and updated all related package dependencies to latest; now supports only `net8.0`, `net9.0` and `net10.0`. diff --git a/Common.targets b/Common.targets index c341773..99e421a 100644 --- a/Common.targets +++ b/Common.targets @@ -1,6 +1,6 @@ - 3.0.0 + 3.0.1 preview Avanade Avanade diff --git a/schema/dbex.json b/schema/dbex.json index 8f2d87c..3496c68 100644 --- a/schema/dbex.json +++ b/schema/dbex.json @@ -83,12 +83,12 @@ }, "outboxSchema": { "type": "string", - "title": "The database schema name for the outbox tables and stored procedures.", + "title": "The database schema name for the outbox tables and related objects.", "description": "Defaults to \u0027{schema}\u0027." }, "outboxName": { "type": "string", - "title": "The name of the outbox table.", + "title": "The database table base name for the outbox tables and related objects.", "description": "Defaults to \u0027Outbox\u0027." }, "tables": { diff --git a/src/DbEx.Postgres/Resources/ScriptOutbox_pgsql.hbs b/src/DbEx.Postgres/Resources/ScriptOutbox_pgsql.hbs index 71fbbdd..6492e75 100644 --- a/src/DbEx.Postgres/Resources/ScriptOutbox_pgsql.hbs +++ b/src/DbEx.Postgres/Resources/ScriptOutbox_pgsql.hbs @@ -8,7 +8,7 @@ BEGIN; CREATE TABLE "{{lookup Parameters 'Param1'}}"."{{lookup Parameters 'Param2'}}" ( "{{lookup Parameters 'Param2'}}_id" BIGSERIAL NOT NULL PRIMARY KEY, - "tenant_id" VARCHAR(255) NOT NULL, -- Optional, null indicates no tenancy. + "tenant_id" VARCHAR(255) NOT NULL, -- '(none)' indicates no tenancy. "partition_id" INTEGER NOT NULL, -- Partition number; computed in application from partition-key. "status" SMALLINT NOT NULL DEFAULT 0, -- 0=Pending, 1=Processing, 2=Done. "enqueued_utc" TIMESTAMPTZ NOT NULL, -- When the event was enqueued within application. @@ -25,14 +25,14 @@ CREATE TABLE "{{lookup Parameters 'Param1'}}"."{{lookup Parameters 'Param2'}}" ( "lease_until_utc" TIMESTAMPTZ NULL -- Leased until UTC; after which assume released due to possible application crash. ); -CREATE INDEX "ix_{{lookup Parameters 'Param1'}}_{{lookup Parameters 'Param2'}}_partition_order" ON "{{lookup Parameters 'Param1'}}"."{{lookup Parameters 'Param2'}}" ("tenant_id", "partition_id", "{{lookup Parameters 'Param2'}}_id", "status", "available_utc", "lease_until_utc", "destination", "event", "attempts"); +CREATE INDEX "ix_{{lookup Parameters 'Param1'}}_{{lookup Parameters 'Param2'}}_partition_order" ON "{{lookup Parameters 'Param1'}}"."{{lookup Parameters 'Param2'}}" ("tenant_id", "partition_id", "{{lookup Parameters 'Param2'}}_id", "status", "available_utc", "lease_until_utc", "destination", "attempts"); CREATE INDEX "ix_{{lookup Parameters 'Param1'}}_{{lookup Parameters 'Param2'}}_worker_pull" ON "{{lookup Parameters 'Param1'}}"."{{lookup Parameters 'Param2'}}" ("tenant_id", "partition_id", "status", "{{lookup Parameters 'Param2'}}_id", "available_utc"); CREATE INDEX "ix_{{lookup Parameters 'Param1'}}_{{lookup Parameters 'Param2'}}_clean_up" ON "{{lookup Parameters 'Param1'}}"."{{lookup Parameters 'Param2'}}" ("{{lookup Parameters 'Param2'}}_id", "dequeued_utc") WHERE "status" = 2; CREATE TABLE "{{lookup Parameters 'Param1'}}"."{{lookup Parameters 'Param2'}}_lease" ( - "tenant_id" VARCHAR(255) NOT NULL, -- Optional, null indicates no tenancy. + "tenant_id" VARCHAR(255) NOT NULL, -- '(none)' indicates no tenancy. "partition_id" INTEGER NOT NULL, -- Partition number; computed in application from partition-key. - "lease_id" UUID NULL, -- Unique identifier of the leasee. + "lease_id" UUID NULL, -- Unique identifier of the lessee. "lease_until_utc" TIMESTAMPTZ NULL, -- Leased until UTC; after which assume released due to possible application crash. CONSTRAINT "pk_{{lookup Parameters 'Param1'}}_{{lookup Parameters 'Param2'}}_lease" PRIMARY KEY ("tenant_id", "partition_id") diff --git a/src/DbEx.Postgres/Templates/FnOutboxBatchCancel_pgsql.hbs b/src/DbEx.Postgres/Templates/FnOutboxBatchCancel_pgsql.hbs index e4f678b..d76b30f 100644 --- a/src/DbEx.Postgres/Templates/FnOutboxBatchCancel_pgsql.hbs +++ b/src/DbEx.Postgres/Templates/FnOutboxBatchCancel_pgsql.hbs @@ -23,7 +23,7 @@ BEGIN SET LOCAL lock_timeout = '5s'; SET LOCAL transaction_isolation = 'read committed'; - _now := NOW() AT TIME ZONE 'UTC'; + _now := NOW(); -- 1) Cancel all rows in the batch. UPDATE "{{OutboxSchema}}"."{{OutboxName}}" AS o diff --git a/src/DbEx.Postgres/Templates/FnOutboxBatchClaim_pgsql.hbs b/src/DbEx.Postgres/Templates/FnOutboxBatchClaim_pgsql.hbs index f42c349..1493af2 100644 --- a/src/DbEx.Postgres/Templates/FnOutboxBatchClaim_pgsql.hbs +++ b/src/DbEx.Postgres/Templates/FnOutboxBatchClaim_pgsql.hbs @@ -39,7 +39,7 @@ BEGIN SET LOCAL lock_timeout = '5s'; SET LOCAL transaction_isolation = 'read committed'; - _now := NOW() AT TIME ZONE 'UTC'; + _now := NOW(); _effective_tenant_id := COALESCE(p_tenant_id, '(none)'); -- 1) Acquire a partition lease; exit where unsuccessful. diff --git a/src/DbEx.Postgres/Templates/FnOutboxBatchComplete_pgsql.hbs b/src/DbEx.Postgres/Templates/FnOutboxBatchComplete_pgsql.hbs index 15d4773..e934a53 100644 --- a/src/DbEx.Postgres/Templates/FnOutboxBatchComplete_pgsql.hbs +++ b/src/DbEx.Postgres/Templates/FnOutboxBatchComplete_pgsql.hbs @@ -23,7 +23,7 @@ BEGIN SET LOCAL lock_timeout = '5s'; SET LOCAL transaction_isolation = 'read committed'; - _now := NOW() AT TIME ZONE 'UTC'; + _now := NOW(); -- 1) Complete all rows in the batch. UPDATE "{{OutboxSchema}}"."{{OutboxName}}" AS o diff --git a/src/DbEx.Postgres/Templates/FnOutboxEnqueue_pgsql.hbs b/src/DbEx.Postgres/Templates/FnOutboxEnqueue_pgsql.hbs index 3bea80c..5c79fac 100644 --- a/src/DbEx.Postgres/Templates/FnOutboxEnqueue_pgsql.hbs +++ b/src/DbEx.Postgres/Templates/FnOutboxEnqueue_pgsql.hbs @@ -18,7 +18,7 @@ BEGIN * This file is automatically generated; any changes will be lost. */ - _now := NOW() AT TIME ZONE 'UTC'; + _now := NOW(); _effective_tenant_id := COALESCE(p_tenant_id, '(none)'); INSERT INTO "{{OutboxSchema}}"."{{OutboxName}}" ( diff --git a/src/DbEx.Postgres/Templates/FnOutboxLeaseAcquire_pgsql.hbs b/src/DbEx.Postgres/Templates/FnOutboxLeaseAcquire_pgsql.hbs index ad9bdc3..f6c6c29 100644 --- a/src/DbEx.Postgres/Templates/FnOutboxLeaseAcquire_pgsql.hbs +++ b/src/DbEx.Postgres/Templates/FnOutboxLeaseAcquire_pgsql.hbs @@ -24,7 +24,7 @@ BEGIN * return_code -1 = Lease not acquired; caller should backoff and retry. * * Notes: - * - The function will return -1 where lease acquisition is unsuccessful, including where another active lease exists or where a transient error occurs (e.g. lock timeout). + * - The function will return -1 where lease acquisition is unsuccessful, including where another active lease exists. * - The caller should implement an appropriate retry/backoff strategy where -1 is returned, including randomization to avoid thundering herd issues. */ @@ -32,7 +32,7 @@ BEGIN SET LOCAL lock_timeout = '5s'; SET LOCAL transaction_isolation = 'read committed'; - _now := NOW() AT TIME ZONE 'UTC'; + _now := NOW(); _until := _now + (p_lease_seconds || ' seconds')::INTERVAL; _effective_tenant_id := COALESCE(p_tenant_id, '(none)'); diff --git a/src/DbEx.Postgres/Templates/FnOutboxLeaseRelease_pgsql.hbs b/src/DbEx.Postgres/Templates/FnOutboxLeaseRelease_pgsql.hbs index 3dfdcaf..1629116 100644 --- a/src/DbEx.Postgres/Templates/FnOutboxLeaseRelease_pgsql.hbs +++ b/src/DbEx.Postgres/Templates/FnOutboxLeaseRelease_pgsql.hbs @@ -17,14 +17,14 @@ BEGIN * -1 = No rows updated (e.g. already released or invalid lease_id). * * Notes: - * - The function will return -1 where release is unsuccessful, including where the lease is already released or where a transient error occurs (e.g. lock timeout). + * - The function will return -1 where release is unsuccessful, including where the lease is already released. */ -- Set transaction parameters SET LOCAL lock_timeout = '5s'; SET LOCAL transaction_isolation = 'read committed'; - -- 1) Release lease where leasee. + -- 1) Release lease where lessee. UPDATE "{{OutboxSchema}}"."{{OutboxName}}_lease" AS ol SET "lease_id" = NULL, "lease_until_utc" = NULL diff --git a/src/DbEx.SqlServer/Resources/ScriptOutbox_sql.hbs b/src/DbEx.SqlServer/Resources/ScriptOutbox_sql.hbs index 38fd750..5bff468 100644 --- a/src/DbEx.SqlServer/Resources/ScriptOutbox_sql.hbs +++ b/src/DbEx.SqlServer/Resources/ScriptOutbox_sql.hbs @@ -8,7 +8,7 @@ BEGIN TRANSACTION CREATE TABLE [{{lookup Parameters 'Param1'}}].[{{lookup Parameters 'Param2'}}] ( [{{lookup Parameters 'Param2'}}Id] BIGINT IDENTITY (1, 1) NOT NULL PRIMARY KEY, - [TenantId] NVARCHAR(255) NOT NULL, -- Optional, null indicates no tenancy. + [TenantId] NVARCHAR(255) NOT NULL, -- '(none)' indicates no tenancy. [PartitionId] INT NOT NULL, -- Partition number; computed in application from partition-key. [Status] TINYINT NOT NULL DEFAULT 0, -- 0=Pending, 1=Processing, 2=Done. [EnqueuedUtc] DATETIME2 NOT NULL, -- When the event was enqueued within application. @@ -24,15 +24,15 @@ CREATE TABLE [{{lookup Parameters 'Param1'}}].[{{lookup Parameters 'Param2'}}] ( [LeaseId] UNIQUEIDENTIFIER NULL, -- Unique identifier of the lease. [LeaseUntilUtc] DATETIME2 NULL, -- Leased until UTC; after which assume released due to possible application crash. - INDEX [IX_{{lookup Parameters 'Param1'}}_{{lookup Parameters 'Param2'}}_PartitionOrder] ([TenantId], [PartitionId], [{{lookup Parameters 'Param2'}}Id]) INCLUDE ([Status], [AvailableUtc], [LeaseUntilUtc], [Destination], [Event], [Attempts]), + INDEX [IX_{{lookup Parameters 'Param1'}}_{{lookup Parameters 'Param2'}}_PartitionOrder] ([TenantId], [PartitionId], [{{lookup Parameters 'Param2'}}Id]) INCLUDE ([Status], [AvailableUtc], [LeaseUntilUtc], [Destination], [Attempts]), INDEX [IX_{{lookup Parameters 'Param1'}}_{{lookup Parameters 'Param2'}}_WorkerPull] ([TenantId], [PartitionId], [Status]) INCLUDE ([{{lookup Parameters 'Param2'}}Id], [AvailableUtc]), INDEX [IX_{{lookup Parameters 'Param1'}}_{{lookup Parameters 'Param2'}}_CleanUp] ([{{lookup Parameters 'Param2'}}Id]) INCLUDE ([DequeuedUtc]) WHERE [Status] = 2 ); CREATE TABLE [{{lookup Parameters 'Param1'}}].[{{lookup Parameters 'Param2'}}Lease] ( - [TenantId] NVARCHAR(255) NOT NULL, -- Optional, null indicates no tenancy. + [TenantId] NVARCHAR(255) NOT NULL, -- '(none)' indicates no tenancy. [PartitionId] INT NOT NULL, -- Partition number; computed in application from partition-key. - [LeaseId] UNIQUEIDENTIFIER NULL, -- Unique identifier of the leasee. + [LeaseId] UNIQUEIDENTIFIER NULL, -- Unique identifier of the lessee. [LeaseUntilUtc] DATETIME2 NULL -- Leased until UTC; after which assume released due to possible application crash. CONSTRAINT PK_{{lookup Parameters 'Param1'}}_{{lookup Parameters 'Param2'}}Lease PRIMARY KEY (TenantId, PartitionId) diff --git a/src/DbEx.SqlServer/Templates/SpOutboxLeaseRelease_sql.hbs b/src/DbEx.SqlServer/Templates/SpOutboxLeaseRelease_sql.hbs index 7b46cd7..51c1d87 100644 --- a/src/DbEx.SqlServer/Templates/SpOutboxLeaseRelease_sql.hbs +++ b/src/DbEx.SqlServer/Templates/SpOutboxLeaseRelease_sql.hbs @@ -23,7 +23,7 @@ BEGIN BEGIN TRY BEGIN TRAN; - -- 1) Release lease where leasee. + -- 1) Release lease where lessee. UPDATE ol SET ol.[LeaseId] = NULL, ol.[LeaseUntilUtc] = NULL diff --git a/src/DbEx/CodeGen/Config/CodeGenConfig.cs b/src/DbEx/CodeGen/Config/CodeGenConfig.cs index bf1a704..b3254ae 100644 --- a/src/DbEx/CodeGen/Config/CodeGenConfig.cs +++ b/src/DbEx/CodeGen/Config/CodeGenConfig.cs @@ -121,17 +121,17 @@ public class CodeGenConfig : ConfigRootBase, IByConventionColumnN public bool? Outbox { get; set; } /// - /// Gets or sets the database schema name used for outbox tables and stored procedures. + /// Gets or sets the database schema name used for outbox tables and related objects. /// [JsonPropertyName("outboxSchema")] - [CodeGenProperty("Outbox", Title = "The database schema name for the outbox tables and stored procedures.", Description = "Defaults to '{schema}'.")] + [CodeGenProperty("Outbox", Title = "The database schema name for the outbox tables and related objects.", Description = "Defaults to '{schema}'.")] public string? OutboxSchema { get; set; } /// - /// Gets or sets the name of the outbox table used for storing outgoing messages. + /// Gets or sets the database table base name for the outbox tables and related objects. /// [JsonPropertyName("outboxName")] - [CodeGenProperty("Outbox", Title = "The name of the outbox table.", Description = "Defaults to 'Outbox'.")] + [CodeGenProperty("Outbox", Title = "The database table base name for the outbox tables and related objects.", Description = "Defaults to 'Outbox'.")] public string? OutboxName { get; set; } #endregion diff --git a/src/DbEx/Migration/Data/DataParser.cs b/src/DbEx/Migration/Data/DataParser.cs index 2e45a77..0c6ad3f 100644 --- a/src/DbEx/Migration/Data/DataParser.cs +++ b/src/DbEx/Migration/Data/DataParser.cs @@ -303,10 +303,31 @@ private async Task ParseTableJsonAsync(List tables, DataRow? parent, throw new DataParserException(msg); } - else if (ParserArgs.ReplaceShorthandGuids && value.StartsWith('^') && int.TryParse(value[1..], out var i)) + + // Add alternate runtime format. + if (value.Length > 1 && value.StartsWith('^')) + { + var key = value[1..]; + + // Check against known values and runtime parameters. + switch (key) + { + case "user_name": return ParserArgs.UserName; + case "now": return ParserArgs.DateTimeNow; + case "guid": return Guid.NewGuid(); + default: + if (ParserArgs.Parameters.TryGetValue(key, out object? dval)) + return dval; + + break; + } + } + + // Add support for shorthand Guids in the format of ^{int} where the int is used as the first segment of the Guid and the rest are zeros. + if (ParserArgs.ReplaceShorthandGuids && value.StartsWith('^') && int.TryParse(value[1..], out var i)) return new Guid(i, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); - else - return value; + + return value; } /// diff --git a/tests/DbEx.Test.Console/Data/Data.yaml b/tests/DbEx.Test.Console/Data/Data.yaml index 6109e24..77d6a2f 100644 --- a/tests/DbEx.Test.Console/Data/Data.yaml +++ b/tests/DbEx.Test.Console/Data/Data.yaml @@ -5,10 +5,10 @@ - O: Other - Contact: - { ContactId: 1, ContactType: E, Gender: M, Name: Bob, DateOfBirth: 2001-10-22, Addresses: [ { ContactAddressId: 10, Street: "1 Main Street" } ] } - - { ContactId: 2, ContactType: I, Name: Jane, Phone: 1234, Addresses: [ { ContactAddressId: 20, ContactId: 2, Street: "1 Main Street" } ] } + - { ContactId: 2, ContactType: I, Name: ^jane_name, Phone: 1234, Addresses: [ { ContactAddressId: 20, ContactId: 2, Street: "1 Main Street" } ] } - ^Person: - { PersonId: ^88, Name: '^(DbEx.Test.Console.RuntimeValues.Name, DbEx.Test.Console)' } - - { Name: '^(DefaultName)', AddressJson: { Street: "Main St", City: "Maine" }, NicknamesJson: ["Gaz", "Baz"] } + - { Name: '^(DefaultName)', AddressJson: { Street: "Main St", City: Maine }, NicknamesJson: ["Gaz", "Baz"] } - $Gender: - X: Not specified - $Status: diff --git a/tests/DbEx.Test.Console/Migrations/008-create-test-outbox-tables.sql b/tests/DbEx.Test.Console/Migrations/008-create-test-outbox-tables.sql index c902369..bfcc025 100644 --- a/tests/DbEx.Test.Console/Migrations/008-create-test-outbox-tables.sql +++ b/tests/DbEx.Test.Console/Migrations/008-create-test-outbox-tables.sql @@ -28,7 +28,7 @@ CREATE TABLE [Test].[Outbox] ( CREATE TABLE [Test].[OutboxLease] ( [TenantId] NVARCHAR(255) NOT NULL, -- Optional, null indicates no tenancy. [PartitionId] INT NOT NULL, -- Partition number; computed in application from partition-key. - [LeaseId] UNIQUEIDENTIFIER NULL, -- Unique identifier of the leasee. + [LeaseId] UNIQUEIDENTIFIER NULL, -- Unique identifier of the lessee. [LeaseUntilUtc] DATETIME2 NULL -- Leased until UTC; after which assume released due to possible application crash. CONSTRAINT PK_Test_OutboxLease PRIMARY KEY (TenantId, PartitionId) diff --git a/tests/DbEx.Test.Console/Program.cs b/tests/DbEx.Test.Console/Program.cs index 6611891..b90f99e 100644 --- a/tests/DbEx.Test.Console/Program.cs +++ b/tests/DbEx.Test.Console/Program.cs @@ -13,6 +13,7 @@ internal static Task Main(string[] args) => SqlServerMigrationConsole c.Args.AddSchemaOrder("Test", "Outbox"); c.Args.IncludeExtendedSchemaScripts(); c.Args.DataParserArgs.Parameter("DefaultName", "Bazza") + .Parameter("jane_name", "Jane") .RefDataColumnDefault("SortOrder", i => i) .ColumnDefault("*", "*", "TenantId", _ => "test-tenant") .TableNameMappings.Add("XTest", "XContactType", "Test", "ContactType", new() { { "XNumber", "Number" } }) diff --git a/tests/DbEx.Test.Console/Schema/Stored Procedures/spOutboxLeaseRelease.g.sql b/tests/DbEx.Test.Console/Schema/Stored Procedures/spOutboxLeaseRelease.g.sql index c55ed76..5e47543 100644 --- a/tests/DbEx.Test.Console/Schema/Stored Procedures/spOutboxLeaseRelease.g.sql +++ b/tests/DbEx.Test.Console/Schema/Stored Procedures/spOutboxLeaseRelease.g.sql @@ -22,7 +22,7 @@ BEGIN BEGIN TRY BEGIN TRAN; - -- 1) Release lease where leasee. + -- 1) Release lease where lessee. UPDATE ol SET ol.[LeaseId] = NULL, ol.[LeaseUntilUtc] = NULL diff --git a/tests/DbEx.Test/SqlServerMigrationTest.cs b/tests/DbEx.Test/SqlServerMigrationTest.cs index 6712aec..79ba8d7 100644 --- a/tests/DbEx.Test/SqlServerMigrationTest.cs +++ b/tests/DbEx.Test/SqlServerMigrationTest.cs @@ -173,6 +173,7 @@ public async Task A130_MigrateAll_Console() using var m = new SqlServerMigration(a); m.Args.DataParserArgs.Parameters.Add("DefaultName", "Bazza"); + m.Args.DataParserArgs.Parameters.Add("jane_name", "Jane"); m.Args.DataParserArgs.RefDataColumnDefaults.Add("SortOrder", i => i); m.Args.DataParserArgs.ColumnDefaults.Add(new DataParserColumnDefault("*", "*", "TenantId", _ => "test-tenant")); m.Args.DataParserArgs.TableNameMappings.Add("XTest", "XContactType", "Test", "ContactType", new() { { "XNumber", "Number" } })