From 60091760a4c1bbe727e8b49647497f38bf26e19a Mon Sep 17 00:00:00 2001 From: Jesus Flores Date: Wed, 8 Jul 2026 23:13:46 -0600 Subject: [PATCH] [DMS-1255] Publish MSSQL template packages; share local CMS/DMS database --- .../EdFi.Api.Minimal.Template.MsSql.yml | 56 ++ .../EdFi.Api.Populated.Template.MsSql.yml | 87 ++++ .github/workflows/build-minimal-template.yml | 87 +++- .../workflows/build-populated-template.yml | 92 +++- build-dms.ps1 | 44 +- .../MinimalTemplateSettings.psd1 | 10 +- .../PopulatedTemplateSettings.psd1 | 10 +- .../Template-Management.psm1 | 314 +++++++++++- .../tests/Template-Management.Tests.ps1 | 482 ++++++++++++++++++ .../verify-template-restore.ps1 | 154 ++++-- eng/docker-compose/.env.example | 2 +- eng/docker-compose/.env.mssql | 6 +- eng/docker-compose/README.md | 52 +- .../bootstrap-published-dms.ps1 | 16 + eng/docker-compose/bootstrap-wrapper.psm1 | 266 +++++++++- eng/docker-compose/env-utility.psm1 | 84 ++- eng/docker-compose/load-dms-seed-data.ps1 | 40 +- eng/docker-compose/local-config.yml | 2 +- eng/docker-compose/provision-e2e-database.ps1 | 240 +++++++-- eng/docker-compose/published-config.yml | 2 +- eng/docker-compose/start-local-dms.ps1 | 84 ++- eng/docker-compose/start-published-dms.ps1 | 222 ++++++-- .../BootstrapEntryPointWorkflow.Tests.ps1 | 270 +++++++++- .../BootstrapSchemaDeploymentSafety.Tests.ps1 | 186 ++++++- .../tests/BootstrapSeedDelivery.Tests.ps1 | 128 ++++- .../DatabaseEngineEnvironmentFile.Tests.ps1 | 2 +- .../design-docs/bootstrap/bootstrap-design.md | 2 +- .../bootstrap/command-boundaries.md | 24 +- 28 files changed, 2741 insertions(+), 223 deletions(-) create mode 100644 .github/workflows/EdFi.Api.Minimal.Template.MsSql.yml create mode 100644 .github/workflows/EdFi.Api.Populated.Template.MsSql.yml diff --git a/.github/workflows/EdFi.Api.Minimal.Template.MsSql.yml b/.github/workflows/EdFi.Api.Minimal.Template.MsSql.yml new file mode 100644 index 0000000000..a78e59690e --- /dev/null +++ b/.github/workflows/EdFi.Api.Minimal.Template.MsSql.yml @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# Licensed to the Ed-Fi Alliance under one or more agreements. +# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +# See the LICENSE and NOTICES files in the project root for more information. + +name: Pkg EdFi.Api.Minimal.Template.MsSql +on: + release: + types: + - released + push: + tags: + - "v*.*.*" + workflow_dispatch: + inputs: + publish_package: + description: "Publish the built template package to the Azure Artifacts feed" + type: boolean + default: false + +permissions: + contents: write + actions: read + id-token: write + +jobs: + build-minimal-template: + name: Build Minimal Template ${{ matrix.standard_version }} + # One leg per supported Ed-Fi Data Standard version. Each leg invokes the whole reusable + # build/SBOM/provenance pipeline, so per-version outputs stay correlated. To add or drop a + # version, add or remove an `include` entry below (and its matching schema artifacts / + # template env file); see docs/DATA-STANDARD-VERSIONS.md. DS 5.2 is the default. + strategy: + fail-fast: false + matrix: + include: + - standard_version: "5.2.0" + package_name: "EdFi.Api.Minimal.Template.MsSql.5.2.0" + provenance_file: "minimal.template.intoto.jsonl" + environment_file: "./.env.template" + - standard_version: "6.1.0" + package_name: "EdFi.Api.Minimal.Template.MsSql.6.1.0" + provenance_file: "minimal.template.6.1.0.intoto.jsonl" + environment_file: "./.env.template.ds61" + uses: ./.github/workflows/build-minimal-template.yml + with: + standard_version: ${{ matrix.standard_version }} + package_name: ${{ matrix.package_name }} + manifest_file: "_manifest/spdx_2.2/manifest.spdx.json" + provenance_file: ${{ matrix.provenance_file }} + publish_package: ${{ github.event_name != 'workflow_dispatch' || inputs.publish_package == true }} + environment_file: ${{ matrix.environment_file }} + database_engine: mssql + secrets: + AZURE_ARTIFACTS_PERSONAL_ACCESS_TOKEN: ${{ secrets.AZURE_ARTIFACTS_PERSONAL_ACCESS_TOKEN }} + AZURE_ARTIFACTS_USER_NAME: ${{ secrets.AZURE_ARTIFACTS_USER_NAME }} diff --git a/.github/workflows/EdFi.Api.Populated.Template.MsSql.yml b/.github/workflows/EdFi.Api.Populated.Template.MsSql.yml new file mode 100644 index 0000000000..3d4f0dadec --- /dev/null +++ b/.github/workflows/EdFi.Api.Populated.Template.MsSql.yml @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: Apache-2.0 +# Licensed to the Ed-Fi Alliance under one or more agreements. +# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +# See the LICENSE and NOTICES files in the project root for more information. + +name: Pkg EdFi.Api.Populated.Template.MsSql +on: + release: + types: + - released + push: + tags: + - "v*.*.*" + workflow_dispatch: + inputs: + publish_package: + description: "Publish the built template package to the Azure Artifacts feed" + type: boolean + default: false + standard_version: + description: "Build only this Data Standard version (e.g. 5.2.0 or 6.1.0). Leave blank to build every version in the matrix." + type: string + default: "" + +permissions: + contents: write + actions: read + id-token: write + +jobs: + # Builds the version matrix once so a manual dispatch can narrow the run to a single Data + # Standard version (standard_version input) without spending compute on the other legs. + # Release/tag runs carry no inputs and build every leg. To add or drop a version, edit the + # `legs` array below (and its matching schema artifacts / template env file); see + # docs/DATA-STANDARD-VERSIONS.md. DS 5.2 is the default. + configure-matrix: + name: Configure matrix + runs-on: ubuntu-latest + outputs: + include: ${{ steps.build-matrix.outputs.include }} + steps: + - name: Select Data Standard legs + id: build-matrix + shell: bash + env: + REQUESTED_VERSION: ${{ github.event_name == 'workflow_dispatch' && inputs.standard_version || '' }} + run: | + legs='[ + {"standard_version":"5.2.0","package_name":"EdFi.Api.Populated.Template.MsSql.5.2.0","provenance_file":"populated.template.intoto.jsonl","environment_file":"./.env.template"}, + {"standard_version":"6.1.0","package_name":"EdFi.Api.Populated.Template.MsSql.6.1.0","provenance_file":"populated.template.6.1.0.intoto.jsonl","environment_file":"./.env.template.ds61"} + ]' + if [ -n "$REQUESTED_VERSION" ]; then + include=$(jq -c --arg v "$REQUESTED_VERSION" '[.[] | select(.standard_version == $v)]' <<< "$legs") + if [ "$(jq 'length' <<< "$include")" -eq 0 ]; then + echo "::error::standard_version '$REQUESTED_VERSION' is not a known leg." >&2 + exit 1 + fi + else + include=$(jq -c '.' <<< "$legs") + fi + echo "Selected legs: $include" + echo "include=$include" >> "$GITHUB_OUTPUT" + + build-populated-template: + name: Build Populated Template ${{ matrix.standard_version }} + needs: configure-matrix + # Each leg invokes the whole reusable build/SBOM/provenance pipeline, so per-version outputs + # stay correlated. The matrix is computed by configure-matrix (filterable on dispatch). + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.configure-matrix.outputs.include) }} + uses: ./.github/workflows/build-populated-template.yml + with: + standard_version: ${{ matrix.standard_version }} + package_name: ${{ matrix.package_name }} + manifest_file: "_manifest/spdx_2.2/manifest.spdx.json" + provenance_file: ${{ matrix.provenance_file }} + publish_package: ${{ github.event_name != 'workflow_dispatch' || inputs.publish_package == true }} + environment_file: ${{ matrix.environment_file }} + verify_restore: true + require_populated_data: true + database_engine: mssql + secrets: + AZURE_ARTIFACTS_PERSONAL_ACCESS_TOKEN: ${{ secrets.AZURE_ARTIFACTS_PERSONAL_ACCESS_TOKEN }} + AZURE_ARTIFACTS_USER_NAME: ${{ secrets.AZURE_ARTIFACTS_USER_NAME }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/build-minimal-template.yml b/.github/workflows/build-minimal-template.yml index d574802582..859084fb8d 100644 --- a/.github/workflows/build-minimal-template.yml +++ b/.github/workflows/build-minimal-template.yml @@ -40,6 +40,15 @@ on: required: false type: string default: "./.env.template" + # The built package's data dump is coupled to a specific engine image: the mssql .bak + # backup is coupled to the pinned mcr.microsoft.com/mssql/server:2022-latest image used to + # create and restore it, the same way the postgresql .sql dump is coupled to the + # PostgreSQL major version it was built and restored against. + database_engine: + description: "Database engine for the relational template datastore (postgresql or mssql)" + required: false + type: string + default: "postgresql" secrets: AZURE_ARTIFACTS_PERSONAL_ACCESS_TOKEN: required: true @@ -82,6 +91,26 @@ jobs: hash-code: ${{ steps.hash-code.outputs.hash-code }} dms-version: ${{ steps.versions.outputs.dms-v }} steps: + - name: Validate workflow inputs + env: + DATABASE_ENGINE: ${{ inputs.database_engine }} + PACKAGE_NAME: ${{ inputs.package_name }} + run: | + $databaseEngine = $env:DATABASE_ENGINE + $packageName = $env:PACKAGE_NAME + + $databaseEngines = @('postgresql', 'mssql') + if ($databaseEngine -notin $databaseEngines) { + throw "database_engine must be one of: $($databaseEngines -join ', '); got '$databaseEngine'." + } + + # Minimal template package names are engine-dependent so a build cannot silently + # publish an MSSQL dump under the PostgreSQL product name, or vice versa. + $expectedPackagePrefix = if ($databaseEngine -eq 'mssql') { 'EdFi.Api.Minimal.Template.MsSql' } else { 'EdFi.Api.Minimal.Template.PostgreSql' } + if ($packageName -notlike "$expectedPackagePrefix*") { + throw "database_engine '$databaseEngine' requires a package_name with prefix '$expectedPackagePrefix'; got '$packageName'." + } + - name: Checkout the repo uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -143,12 +172,12 @@ jobs: - name: Start DMS Docker run: | $env:LOG_LEVEL="Warning" - ./start-local-dms.ps1 -EnvironmentFile "${{ inputs.environment_file }}" -EnableConfig + ./start-local-dms.ps1 -EnvironmentFile "${{ inputs.environment_file }}" -EnableConfig -DatabaseEngine "${{ inputs.database_engine }}" working-directory: eng/docker-compose/ - name: Provision relational database run: | - ./provision-e2e-database.ps1 -EnvironmentFile "${{ inputs.environment_file }}" -Configuration Release + ./provision-e2e-database.ps1 -EnvironmentFile "${{ inputs.environment_file }}" -Configuration Release -DatabaseEngine "${{ inputs.database_engine }}" working-directory: eng/docker-compose/ # DMS fails fatally at startup when the Configuration Service has no data stores. @@ -163,9 +192,20 @@ jobs: $envValues = ReadValuesFromEnvFile "${{ inputs.environment_file }}" $postgresUser = Get-EnvValue -EnvValues $envValues -Name "POSTGRES_USER" -DefaultValue "postgres" $postgresCredential = ConvertTo-PostgresCredential -UserName $postgresUser -Secret $envValues.POSTGRES_PASSWORD + + # Add-DataStore always requires a PostgresCredential; for mssql it goes unused because + # -ConnectionString is supplied verbatim below. The Configuration Service backing store + # is always PostgreSQL regardless of database_engine, so $postgresCredential above is + # unaffected. + $dataStoreConnectionString = "" + if ("${{ inputs.database_engine }}" -eq "mssql") { + $mssqlPassword = Get-EnvValue -EnvValues $envValues -Name "MSSQL_SA_PASSWORD" -DefaultValue "abcdefgh1!" + $dataStoreConnectionString = New-DataStoreConnectionString -DatabaseEngine "mssql" -DbHost "dms-mssql" -Port 1433 -Username "sa" -Password $mssqlPassword -DatabaseName $env:RELATIONAL_DB_NAME + } + Add-CmsClient -CmsUrl "http://localhost:8081" -ClientId "data-store-admin" -ClientSecret "ValidClientSecret1234567890!Abcd" -DisplayName "Data Store Setup Administrator" $configToken = Get-CmsToken -CmsUrl "http://localhost:8081" -ClientId "data-store-admin" -ClientSecret "ValidClientSecret1234567890!Abcd" - $dataStoreId = Add-DataStore -CmsUrl "http://localhost:8081" -AccessToken $configToken -PostgresCredential $postgresCredential -PostgresDbName $env:RELATIONAL_DB_NAME -Name "Relational Template Data Store" -DataStoreType "Template" + $dataStoreId = Add-DataStore -CmsUrl "http://localhost:8081" -AccessToken $configToken -PostgresCredential $postgresCredential -PostgresDbName $env:RELATIONAL_DB_NAME -ConnectionString $dataStoreConnectionString -Name "Relational Template Data Store" -DataStoreType "Template" if (-not $dataStoreId) { throw "Failed to register the relational data store." } @@ -223,12 +263,49 @@ jobs: -ApplicationName $uniqueAppName ` -DataStoreDatabaseName $env:RELATIONAL_DB_NAME ` -DataStoreId $env:RELATIONAL_DATA_STORE_ID ` - -DumpAllUserSchemas + -DumpAllUserSchemas ` + -DatabaseEngine '${{ inputs.database_engine }}' working-directory: eng/DatabaseTemplates/ + # A loader that exits 0 does not guarantee it wrote anything: gate on descriptor content + # here, before the package is uploaded or published, so an empty bulk load cannot slip + # through as a "minimal" template. Minimal templates load only descriptor sample data + # (no sample entities), so dms.Descriptor is the content this template type owns. + - name: Gate on descriptor content before packaging + env: + DATABASE_ENGINE: ${{ inputs.database_engine }} + ENVIRONMENT_FILE: ${{ inputs.environment_file }} + run: | + Import-Module ./env-utility.psm1 -Force + $envValues = ReadValuesFromEnvFile $env:ENVIRONMENT_FILE + + if ($env:DATABASE_ENGINE -eq "mssql") { + $mssqlPassword = Get-EnvValue -EnvValues $envValues -Name "MSSQL_SA_PASSWORD" -DefaultValue "abcdefgh1!" + $query = "SET NOCOUNT ON; SELECT COUNT(*) FROM [dms].[Descriptor];" + $rawOutput = & docker exec dms-mssql /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P $mssqlPassword -d $env:RELATIONAL_DB_NAME -C -b -h -1 -W -Q $query + if ($LASTEXITCODE -ne 0) { + throw "Failed to query dms.Descriptor row count from database '$($env:RELATIONAL_DB_NAME)'." + } + $descriptorCount = [long](@($rawOutput) | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Select-Object -First 1) + } + else { + $rawOutput = & docker exec dms-postgresql psql -U postgres -d $env:RELATIONAL_DB_NAME -tA -c 'SELECT COUNT(*) FROM dms."Descriptor";' + if ($LASTEXITCODE -ne 0) { + throw "Failed to query dms.Descriptor row count from database '$($env:RELATIONAL_DB_NAME)'." + } + $descriptorCount = [long](@($rawOutput) | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Select-Object -First 1) + } + + if ($descriptorCount -lt 1) { + throw "Descriptor content gate failed: dms.Descriptor has $descriptorCount row(s) in database '$($env:RELATIONAL_DB_NAME)'; the bulk load did not populate descriptor data." + } + + Write-Host "Descriptor content gate passed: dms.Descriptor has $descriptorCount row(s)." -ForegroundColor Green + working-directory: eng/docker-compose/ + - name: Verify template package restore run: | - ./verify-template-restore.ps1 -SourceDatabaseName $env:RELATIONAL_DB_NAME + ./verify-template-restore.ps1 -SourceDatabaseName $env:RELATIONAL_DB_NAME -DatabaseEngine '${{ inputs.database_engine }}' working-directory: eng/DatabaseTemplates/ - name: Upload Bulk Load Logs diff --git a/.github/workflows/build-populated-template.yml b/.github/workflows/build-populated-template.yml index 88edd659d0..73c2a9cf4a 100644 --- a/.github/workflows/build-populated-template.yml +++ b/.github/workflows/build-populated-template.yml @@ -44,6 +44,15 @@ on: required: false type: boolean default: false + # The built package's data dump is coupled to a specific engine image: the mssql .bak + # backup is coupled to the pinned mcr.microsoft.com/mssql/server:2022-latest image used to + # create and restore it, the same way the postgresql .sql dump is coupled to the + # PostgreSQL major version it was built and restored against. + database_engine: + description: "Database engine for the relational template datastore (postgresql or mssql)" + required: false + type: string + default: "postgresql" secrets: AZURE_ARTIFACTS_PERSONAL_ACCESS_TOKEN: required: true @@ -95,6 +104,12 @@ jobs: $publishPackage = [bool]::Parse('${{ inputs.publish_package }}') $environmentFile = '${{ inputs.environment_file }}' $packageName = '${{ inputs.package_name }}' + $databaseEngine = '${{ inputs.database_engine }}' + + $databaseEngines = @('postgresql', 'mssql') + if ($databaseEngine -notin $databaseEngines) { + throw "database_engine must be one of: $($databaseEngines -join ', '); got '$databaseEngine'." + } $templateEnvFiles = @('./.env.template', './.env.template.ds61', './.env.smoke', './.env.smoke.ds61') if ($environmentFile -notin $templateEnvFiles) { @@ -103,9 +118,13 @@ jobs: # The smoke surface (Core+[TPDM]+Sample+Homograph) is an in-run smoke artifact only. It # must never publish under the product template contract, and must not reuse a product - # package name. + # package name. The smoke env files are PostgreSQL-only; MSSQL smoke coverage is out of + # scope, so combining a smoke env file with database_engine mssql is rejected outright. $smokeEnvFiles = @('./.env.smoke', './.env.smoke.ds61') if ($environmentFile -in $smokeEnvFiles) { + if ($databaseEngine -eq 'mssql') { + throw "environment_file '$environmentFile' is PostgreSQL-only and cannot be combined with database_engine 'mssql'." + } if ($publishPackage) { throw "environment_file '$environmentFile' is smoke-only and must not be published; set publish_package to false." } @@ -113,6 +132,14 @@ jobs: throw "environment_file '$environmentFile' requires a smoke package_name (prefix 'EdFi.Api.Smoke.Template.PostgreSql'); got '$packageName'." } } + else { + # Non-smoke (product) template package names are engine-dependent so a build cannot + # silently publish an MSSQL dump under the PostgreSQL product name, or vice versa. + $expectedPackagePrefix = if ($databaseEngine -eq 'mssql') { 'EdFi.Api.Populated.Template.MsSql' } else { 'EdFi.Api.Populated.Template.PostgreSql' } + if ($packageName -notlike "$expectedPackagePrefix*") { + throw "environment_file '$environmentFile' with database_engine '$databaseEngine' requires a package_name with prefix '$expectedPackagePrefix'; got '$packageName'." + } + } if ($requirePopulatedData -and -not $verifyRestore) { throw "require_populated_data requires verify_restore." @@ -179,7 +206,7 @@ jobs: - name: Start DMS infrastructure run: | $env:LOG_LEVEL="Warning" - ./start-local-dms.ps1 -EnvironmentFile "${{ inputs.environment_file }}" -EnableConfig -InfraOnly + ./start-local-dms.ps1 -EnvironmentFile "${{ inputs.environment_file }}" -EnableConfig -InfraOnly -DatabaseEngine "${{ inputs.database_engine }}" working-directory: eng/docker-compose/ # No data store is created by default; register the template data store before @@ -193,9 +220,20 @@ jobs: $envValues = ReadValuesFromEnvFile "${{ inputs.environment_file }}" $postgresUser = Get-EnvValue -EnvValues $envValues -Name "POSTGRES_USER" -DefaultValue "postgres" $postgresCredential = ConvertTo-PostgresCredential -UserName $postgresUser -Secret $envValues.POSTGRES_PASSWORD + + # Add-DataStore always requires a PostgresCredential; for mssql it goes unused because + # -ConnectionString is supplied verbatim below. The Configuration Service backing store + # is always PostgreSQL regardless of database_engine, so $postgresCredential above is + # unaffected. + $dataStoreConnectionString = "" + if ("${{ inputs.database_engine }}" -eq "mssql") { + $mssqlPassword = Get-EnvValue -EnvValues $envValues -Name "MSSQL_SA_PASSWORD" -DefaultValue "abcdefgh1!" + $dataStoreConnectionString = New-DataStoreConnectionString -DatabaseEngine "mssql" -DbHost "dms-mssql" -Port 1433 -Username "sa" -Password $mssqlPassword -DatabaseName $env:TEMPLATE_DB_NAME + } + Add-CmsClient -CmsUrl "http://localhost:8081" -ClientId "data-store-admin" -ClientSecret "ValidClientSecret1234567890!Abcd" -DisplayName "Data Store Setup Administrator" $configToken = Get-CmsToken -CmsUrl "http://localhost:8081" -ClientId "data-store-admin" -ClientSecret "ValidClientSecret1234567890!Abcd" - $dataStoreId = Add-DataStore -CmsUrl "http://localhost:8081" -AccessToken $configToken -PostgresCredential $postgresCredential -PostgresDbName $env:TEMPLATE_DB_NAME -Name "Template Data Store" -DataStoreType "Template" + $dataStoreId = Add-DataStore -CmsUrl "http://localhost:8081" -AccessToken $configToken -PostgresCredential $postgresCredential -PostgresDbName $env:TEMPLATE_DB_NAME -ConnectionString $dataStoreConnectionString -Name "Template Data Store" -DataStoreType "Template" if (-not $dataStoreId) { throw "Failed to register the template data store." } @@ -204,13 +242,13 @@ jobs: - name: Provision template database run: | - ./provision-e2e-database.ps1 -EnvironmentFile "${{ inputs.environment_file }}" -Configuration Release + ./provision-e2e-database.ps1 -EnvironmentFile "${{ inputs.environment_file }}" -Configuration Release -DatabaseEngine "${{ inputs.database_engine }}" working-directory: eng/docker-compose/ - name: Start DMS Docker run: | $env:LOG_LEVEL="Warning" - ./start-local-dms.ps1 -EnvironmentFile "${{ inputs.environment_file }}" -EnableConfig -DmsOnly + ./start-local-dms.ps1 -EnvironmentFile "${{ inputs.environment_file }}" -EnableConfig -DmsOnly -DatabaseEngine "${{ inputs.database_engine }}" working-directory: eng/docker-compose/ - name: Wait for DMS to be ready @@ -261,13 +299,53 @@ jobs: -ApplicationName $uniqueAppName ` -DataStoreDatabaseName $env:TEMPLATE_DB_NAME ` -DataStoreId $env:TEMPLATE_DATA_STORE_ID ` - -DumpAllUserSchemas + -DumpAllUserSchemas ` + -DatabaseEngine '${{ inputs.database_engine }}' working-directory: eng/DatabaseTemplates/ + # A loader that exits 0 does not guarantee it wrote anything: gate on populated content + # here, before the package is uploaded or published, so an empty bulk load cannot slip + # through as a "populated" template. This runs unconditionally (unlike the opt-in + # verify-restore step below, which validates the restored package instead). + - name: Gate on populated content before packaging + env: + DATABASE_ENGINE: ${{ inputs.database_engine }} + ENVIRONMENT_FILE: ${{ inputs.environment_file }} + run: | + Import-Module ./env-utility.psm1 -Force + $envValues = ReadValuesFromEnvFile $env:ENVIRONMENT_FILE + + if ($env:DATABASE_ENGINE -eq "mssql") { + $mssqlPassword = Get-EnvValue -EnvValues $envValues -Name "MSSQL_SA_PASSWORD" -DefaultValue "abcdefgh1!" + $query = "SET NOCOUNT ON; SELECT COUNT(*) FROM [edfi].[School];" + $rawOutput = & docker exec dms-mssql /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P $mssqlPassword -d $env:TEMPLATE_DB_NAME -C -b -h -1 -W -Q $query + if ($LASTEXITCODE -ne 0) { + throw "Failed to query edfi.School row count from database '$($env:TEMPLATE_DB_NAME)'." + } + $schoolCount = [long](@($rawOutput) | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Select-Object -First 1) + } + else { + $rawOutput = & docker exec dms-postgresql psql -U postgres -d $env:TEMPLATE_DB_NAME -tA -c 'SELECT COUNT(*) FROM edfi."School";' + if ($LASTEXITCODE -ne 0) { + throw "Failed to query edfi.School row count from database '$($env:TEMPLATE_DB_NAME)'." + } + $schoolCount = [long](@($rawOutput) | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Select-Object -First 1) + } + + if ($schoolCount -lt 1) { + throw "Populated content gate failed: edfi.School has $schoolCount row(s) in database '$($env:TEMPLATE_DB_NAME)'; the bulk load did not populate sample data." + } + + Write-Host "Populated content gate passed: edfi.School has $schoolCount row(s)." -ForegroundColor Green + working-directory: eng/docker-compose/ + - name: Verify template package restore if: ${{ inputs.verify_restore }} run: | - $verifyParams = @{ SourceDatabaseName = $env:TEMPLATE_DB_NAME } + $verifyParams = @{ + SourceDatabaseName = $env:TEMPLATE_DB_NAME + DatabaseEngine = '${{ inputs.database_engine }}' + } if ([bool]::Parse('${{ inputs.require_populated_data }}')) { $verifyParams.RequirePopulatedData = $true } diff --git a/build-dms.ps1 b/build-dms.ps1 index a0462962fe..6812bf914f 100644 --- a/build-dms.ps1 +++ b/build-dms.ps1 @@ -113,6 +113,12 @@ param( [switch] $LoadSeedData, + # For StartEnvironment only: database engine backing the stack. Forwarded to the bootstrap + # wrapper, whose own default governs when this is omitted. + [string] + [ValidateSet("postgresql", "mssql")] + $DatabaseEngine, + # Identity provider type [string] [ValidateSet("keycloak", "self-contained")] @@ -126,12 +132,18 @@ param( [string] $TestFilter, - # Optional Ed-Fi Data Standard version (e.g. "5.2", "6.1"). Forwarded to the start scripts, which - # compose the matching .env.ds overlay onto -EnvironmentFile. Omit for the default (DS 5.2). + # Optional Ed-Fi Data Standard version. Forwarded to the start scripts, which compose the + # matching .env.ds overlay onto -EnvironmentFile. Omit for the default (DS 5.2). [string] + [ValidateSet("5.2", "6.1")] $DataStandardVersion ) +# Captured here (script scope) rather than at the point of use: $PSBoundParameters inside the +# Invoke-Main script block below reflects that block's own bindings, not this script's, so the +# ContainsKey check has to run in this scope while the top-level $PSBoundParameters is populated. +$dataStandardVersionSupplied = $PSBoundParameters.ContainsKey('DataStandardVersion') + $solutionRoot = "$PSScriptRoot/src/dms" $defaultSolution = "$solutionRoot/EdFi.DataManagementService.sln" $applicationRoot = "$solutionRoot/frontend" @@ -840,8 +852,24 @@ function Start-BootstrapDockerEnvironment { [switch] $LoadSeedData, + # Forwarded to the bootstrap wrapper only when supplied, so the wrapper's own default + # ("postgresql") governs when this is omitted. Validated against + # ValidateSet("postgresql", "mssql") at the top-level parameter; left as a plain string + # here so forwarding an unset (null) value from the caller does not trip validation. + [string] + $DatabaseEngine, + + [string] + $IdentityProvider="self-contained", + + # Forwarded to the bootstrap wrapper only when the caller explicitly supplied it (see + # $dataStandardVersionSupplied), so the wrapper's own default-composition behavior governs + # when it is absent. [string] - $IdentityProvider="self-contained" + $DataStandardVersion, + + [switch] + $DataStandardVersionSupplied ) $environmentFilePath = Resolve-E2EEnvironmentFilePath -Path $EnvironmentFile @@ -869,6 +897,14 @@ function Start-BootstrapDockerEnvironment { $bootstrapArgs.LoadSeedData = $true } + if ($DatabaseEngine) { + $bootstrapArgs.DatabaseEngine = $DatabaseEngine + } + + if ($DataStandardVersionSupplied) { + $bootstrapArgs.DataStandardVersion = $DataStandardVersion + } + Invoke-WithEnvironmentFileSchemaSettings -Enabled -Action { if ($UsePublishedImage) { ./bootstrap-published-dms.ps1 @bootstrapArgs @@ -1352,7 +1388,7 @@ Invoke-Main { DockerBuild { Invoke-Step { DockerBuild } } DockerRun { Invoke-Step { DockerRun } } Run { Invoke-Step { Run } } - StartEnvironment { Invoke-Step { Start-BootstrapDockerEnvironment -UsePublishedImage:$UsePublishedImage -SkipDockerBuild:$SkipDockerBuild -LoadSeedData:$LoadSeedData -IdentityProvider $IdentityProvider } } + StartEnvironment { Invoke-Step { Start-BootstrapDockerEnvironment -UsePublishedImage:$UsePublishedImage -SkipDockerBuild:$SkipDockerBuild -LoadSeedData:$LoadSeedData -DatabaseEngine $DatabaseEngine -IdentityProvider $IdentityProvider -DataStandardVersion $DataStandardVersion -DataStandardVersionSupplied:$dataStandardVersionSupplied } } default { throw "Command '$Command' is not recognized" } } } diff --git a/eng/DatabaseTemplates/MinimalTemplateSettings.psd1 b/eng/DatabaseTemplates/MinimalTemplateSettings.psd1 index 7f42cb330e..fb742d3a8c 100644 --- a/eng/DatabaseTemplates/MinimalTemplateSettings.psd1 +++ b/eng/DatabaseTemplates/MinimalTemplateSettings.psd1 @@ -4,11 +4,11 @@ # See the LICENSE and NOTICES files in the project root for more information. @{ - DatabaseBackupName = "EdFi.Api.Minimal.Template.PostgreSql.{StandardVersion}.sql" - PackageProjectName = "EdFi.Api.Minimal.Template.PostgreSql.{StandardVersion}.csproj" - Id = "EdFi.Api.Minimal.Template.PostgreSql.{StandardVersion}" - Title = "EdFi.Api.Minimal.Template.PostgreSql.{StandardVersion}" - Description = "EdFi Dms Minimal Template Database for PostgreSql" + DatabaseBackupName = "EdFi.Api.Minimal.Template.{DatabaseEngine}.{StandardVersion}.{ArtifactExtension}" + PackageProjectName = "EdFi.Api.Minimal.Template.{DatabaseEngine}.{StandardVersion}.csproj" + Id = "EdFi.Api.Minimal.Template.{DatabaseEngine}.{StandardVersion}" + Title = "EdFi.Api.Minimal.Template.{DatabaseEngine}.{StandardVersion}" + Description = "EdFi Dms Minimal Template Database for {DatabaseEngine}" Authors = "Ed-Fi Alliance" ProjectUrl = "https://github.com/Ed-Fi-Alliance-OSS/Data-Management-Service" License = "Apache-2.0" diff --git a/eng/DatabaseTemplates/PopulatedTemplateSettings.psd1 b/eng/DatabaseTemplates/PopulatedTemplateSettings.psd1 index 690b399c92..9b430a3b86 100644 --- a/eng/DatabaseTemplates/PopulatedTemplateSettings.psd1 +++ b/eng/DatabaseTemplates/PopulatedTemplateSettings.psd1 @@ -4,11 +4,11 @@ # See the LICENSE and NOTICES files in the project root for more information. @{ - DatabaseBackupName = "EdFi.Api.Populated.Template.PostgreSql.{StandardVersion}.sql" - PackageProjectName = "EdFi.Api.Populated.Template.PostgreSql.{StandardVersion}.csproj" - Id = "EdFi.Api.Populated.Template.PostgreSql.{StandardVersion}" - Title = "EdFi.Api.Populated.Template.PostgreSql.{StandardVersion}" - Description = "EdFi Dms Populated Template Database for PostgreSql" + DatabaseBackupName = "EdFi.Api.Populated.Template.{DatabaseEngine}.{StandardVersion}.{ArtifactExtension}" + PackageProjectName = "EdFi.Api.Populated.Template.{DatabaseEngine}.{StandardVersion}.csproj" + Id = "EdFi.Api.Populated.Template.{DatabaseEngine}.{StandardVersion}" + Title = "EdFi.Api.Populated.Template.{DatabaseEngine}.{StandardVersion}" + Description = "EdFi Dms Populated Template Database for {DatabaseEngine}" Authors = "Ed-Fi Alliance" ProjectUrl = "https://github.com/Ed-Fi-Alliance-OSS/Data-Management-Service" License = "Apache-2.0" diff --git a/eng/DatabaseTemplates/Template-Management.psm1 b/eng/DatabaseTemplates/Template-Management.psm1 index c10758a611..90389708f1 100644 --- a/eng/DatabaseTemplates/Template-Management.psm1 +++ b/eng/DatabaseTemplates/Template-Management.psm1 @@ -364,38 +364,90 @@ function Get-BulkLoadFailureClassification { <# .SYNOPSIS - Dumps the specified PostgreSQL database to a file. + Dumps the specified database to a file. .DESCRIPTION - Uses Docker to execute `pg_dump` inside a running PostgreSQL container, targeting only the specified schemas. + For PostgreSQL, uses Docker to execute `pg_dump` inside a running PostgreSQL container, + targeting only the specified schemas. + + For MSSQL, runs a full `BACKUP DATABASE` inside the running SQL Server container via + `sqlcmd`, then copies the resulting `.bak` file out with `docker cp` and removes the + transient in-container backup file. A `.bak` is always a full-database artifact - SQL + Server has no per-schema equivalent to `pg_dump -n` - so `DatabaseSchemas` is ignored + on MSSQL. + +.PARAMETER DatabaseEngine + "postgresql" or "mssql". Defaults to "postgresql". .PARAMETER ContainerName - Name of the Docker container running PostgreSQL. + Name of the Docker container running the database. Defaults to "dms-postgresql" for + PostgreSQL and "dms-mssql" for MSSQL. .PARAMETER DatabaseName Name of the database to dump. .PARAMETER DatabaseSchemas - Array of schemas to include in the dump. + Array of schemas to include in the dump. PostgreSQL only; ignored on MSSQL, where a + `BACKUP DATABASE` always captures the entire database. .PARAMETER BackupDirectory Directory where the dump file will be saved. .PARAMETER BackupFileName Filename to use for the backup. + +.PARAMETER MssqlPassword + The MSSQL "sa" password. Defaults to environment variable MSSQL_SA_PASSWORD or "abcdefgh1!". #> function Invoke-DatabaseDump { [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingWriteHost', '', Justification = 'Template tooling intentionally writes operator progress to the console.')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingUsernameAndPasswordParams', '', Justification = 'The MSSQL password is handed to sqlcmd -P, which only accepts plaintext; the account is always "sa" so there is no companion username parameter, and a PSCredential adds no protection across that boundary.')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification = 'The MSSQL password is handed to sqlcmd -P, which only accepts plaintext; SecureString adds no protection across that boundary.')] param ( - [string]$ContainerName = "dms-postgresql", + [ValidateSet("postgresql", "mssql")] + [string]$DatabaseEngine = "postgresql", + + [string]$ContainerName = $(if ($DatabaseEngine -eq "mssql") { "dms-mssql" } else { "dms-postgresql" }), + [string]$DatabaseName = "edfi_datamanagementservice", [string[]]$DatabaseSchemas, [string]$BackupDirectory, - [string]$BackupFileName + [string]$BackupFileName, + + [string]$MssqlPassword = $env:MSSQL_SA_PASSWORD ?? "abcdefgh1!" ) $backupPath = Join-Path $BackupDirectory $BackupFileName + if ($DatabaseEngine -eq "mssql") { + if ($DatabaseName -notmatch "^[A-Za-z0-9_]+$") { + throw "Database name '$DatabaseName' contains unsupported characters." + } + + if ($BackupFileName -notmatch "^[A-Za-z0-9_.-]+$") { + throw "Backup file name '$BackupFileName' contains unsupported characters." + } + + $containerBackupPath = "/var/opt/mssql/data/$BackupFileName" + + $backupSql = "BACKUP DATABASE [$DatabaseName] TO DISK = N'$containerBackupPath' WITH INIT;" + & docker exec $ContainerName /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P $MssqlPassword -C -b -Q $backupSql + if ($LASTEXITCODE -ne 0) { throw "BACKUP DATABASE of '$DatabaseName' failed in container '$ContainerName'." } + + & docker cp "$($ContainerName):$containerBackupPath" $backupPath + if ($LASTEXITCODE -ne 0) { throw "Failed to copy backup '$containerBackupPath' out of container '$ContainerName'." } + + & docker exec $ContainerName rm -f $containerBackupPath + if ($LASTEXITCODE -ne 0) { throw "Failed to remove transient backup '$containerBackupPath' from container '$ContainerName' after copying it out." } + + Write-Host + Write-Host "Backup Created: " -ForegroundColor Green -NoNewline + Write-Host (Resolve-Path $backupPath) + Write-Host + + return + } + $options = @("exec", $ContainerName, "pg_dump", "-U", "postgres", $DatabaseName) foreach ($schema in $DatabaseSchemas) { @@ -567,22 +619,68 @@ function Resolve-DataStoreIdForTemplate { <# .SYNOPSIS - Discovers the non-system schemas present in a PostgreSQL database. + Discovers the non-system schemas present in a database. .DESCRIPTION - Queries pg_namespace inside the running PostgreSQL container, excluding - pg_* system schemas, information_schema, and public. Throws when the - database has no user schemas, which indicates it was never provisioned. + For PostgreSQL, queries pg_namespace inside the running PostgreSQL container, excluding + pg_* system schemas, information_schema, and public. + + For MSSQL, queries sys.schemas inside the running SQL Server container, excluding the + built-in dbo, guest, sys, and INFORMATION_SCHEMA schemas, plus the db_* fixed-role + schemas every database carries (e.g. db_datareader, db_owner). + + Throws when the database has no user schemas, which indicates it was never provisioned. + +.PARAMETER DatabaseEngine + "postgresql" or "mssql". Defaults to "postgresql". + +.PARAMETER ContainerName + Name of the Docker container running the database. Defaults to "dms-postgresql" for + PostgreSQL and "dms-mssql" for MSSQL. + +.PARAMETER DatabaseName + Name of the database to inspect. + +.PARAMETER MssqlPassword + The MSSQL "sa" password. Defaults to environment variable MSSQL_SA_PASSWORD or "abcdefgh1!". #> function Get-UserSchemaNames { [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '', Justification = 'Function intentionally returns a collection of discovered database schema names.')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingUsernameAndPasswordParams', '', Justification = 'The MSSQL password is handed to sqlcmd -P, which only accepts plaintext; the account is always "sa" so there is no companion username parameter, and a PSCredential adds no protection across that boundary.')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification = 'The MSSQL password is handed to sqlcmd -P, which only accepts plaintext; SecureString adds no protection across that boundary.')] param ( - [string]$ContainerName = "dms-postgresql", + [ValidateSet("postgresql", "mssql")] + [string]$DatabaseEngine = "postgresql", + + [string]$ContainerName = $(if ($DatabaseEngine -eq "mssql") { "dms-mssql" } else { "dms-postgresql" }), [Parameter(Mandatory = $true)] - [string]$DatabaseName + [string]$DatabaseName, + + [string]$MssqlPassword = $env:MSSQL_SA_PASSWORD ?? "abcdefgh1!" ) + if ($DatabaseEngine -eq "mssql") { + if ($DatabaseName -notmatch "^[A-Za-z0-9_]+$") { + throw "Database name '$DatabaseName' contains unsupported characters." + } + + $query = "SET NOCOUNT ON; SELECT name FROM sys.schemas WHERE name NOT IN ('dbo', 'guest', 'sys', 'INFORMATION_SCHEMA') AND name NOT LIKE 'db[_]%' ORDER BY name;" + $output = & docker exec $ContainerName /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P $MssqlPassword -d $DatabaseName -C -b -h -1 -W -Q $query + + if ($LASTEXITCODE -ne 0) { + throw "Failed to discover schemas in database '$DatabaseName'." + } + + $schemas = @($output | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | ForEach-Object { $_.Trim() }) + + if ($schemas.Count -eq 0) { + throw "No user schemas found in database '$DatabaseName'; the database does not appear to be provisioned." + } + + return $schemas + } + $query = "SELECT nspname FROM pg_namespace WHERE nspname !~ '^pg_' AND nspname NOT IN ('information_schema', 'public') ORDER BY nspname;" $output = & docker exec $ContainerName psql -U postgres -d $DatabaseName -tA -c $query @@ -604,10 +702,19 @@ function Get-UserSchemaNames { Restores a database template package into a freshly created database. .DESCRIPTION - Locates the single .nupkg in the package directory, extracts its .sql dump, - drops and recreates the target database inside the running PostgreSQL - container, and restores the dump into it. Returns the name of the restored - package file. + Locates the single .nupkg in the package directory and restores its database + artifact inside the running database container. Returns the name of the + restored package file. + + For PostgreSQL, extracts the package's .sql dump, terminates any lingering + connections to the target database, drops and recreates it, and restores + the dump into it. + + For MSSQL, extracts the package's .bak backup, copies it into the container, + puts any pre-existing target database into single-user mode and drops it, + reads every logical data/log file name via `RESTORE FILELISTONLY`, and runs + `RESTORE DATABASE ... WITH MOVE, REPLACE` with one MOVE clause per file to + relocate all of them under the target database's own name. .PARAMETER PackageDirectory Directory containing the template .nupkg (default: current directory). @@ -615,18 +722,32 @@ function Get-UserSchemaNames { .PARAMETER DatabaseName The database to drop, recreate, and restore the dump into. +.PARAMETER DatabaseEngine + "postgresql" or "mssql". Defaults to "postgresql". + .PARAMETER ContainerName - The running PostgreSQL container hosting the database. + The running database container hosting the database. Defaults to + "dms-postgresql" for PostgreSQL and "dms-mssql" for MSSQL. + +.PARAMETER MssqlPassword + The MSSQL "sa" password. Defaults to environment variable MSSQL_SA_PASSWORD or "abcdefgh1!". #> function Restore-TemplatePackage { [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingWriteHost', '', Justification = 'Template tooling intentionally writes operator progress to the console.')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingUsernameAndPasswordParams', '', Justification = 'The MSSQL password is handed to sqlcmd -P, which only accepts plaintext; the account is always "sa" so there is no companion username parameter, and a PSCredential adds no protection across that boundary.')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification = 'The MSSQL password is handed to sqlcmd -P, which only accepts plaintext; SecureString adds no protection across that boundary.')] param ( [string]$PackageDirectory = ".", [Parameter(Mandatory = $true)] [string]$DatabaseName, - [string]$ContainerName = "dms-postgresql" + [ValidateSet("postgresql", "mssql")] + [string]$DatabaseEngine = "postgresql", + + [string]$ContainerName = $(if ($DatabaseEngine -eq "mssql") { "dms-mssql" } else { "dms-postgresql" }), + + [string]$MssqlPassword = $env:MSSQL_SA_PASSWORD ?? "abcdefgh1!" ) if ($DatabaseName -notmatch "^[A-Za-z0-9_]+$") { @@ -649,12 +770,112 @@ function Restore-TemplatePackage { Copy-Item $package.FullName $zipPath Expand-Archive -Path $zipPath -DestinationPath (Join-Path $extractDirectory "contents") + if ($DatabaseEngine -eq "mssql") { + $bakFile = Get-ChildItem -Path (Join-Path $extractDirectory "contents") -Filter *.bak -Recurse | Select-Object -First 1 + + if ($null -eq $bakFile) { + throw "No .bak backup found inside package '$($package.Name)'." + } + + $containerBakPath = "/var/opt/mssql/data/template-restore-$([Guid]::NewGuid().ToString('N')).bak" + + & docker cp $bakFile.FullName "$($ContainerName):$containerBakPath" | Out-Null + if ($LASTEXITCODE -ne 0) { throw "Failed to copy the backup into container '$ContainerName'." } + + $existsQuery = "SET NOCOUNT ON; SELECT CASE WHEN DB_ID(N'$DatabaseName') IS NOT NULL THEN 1 ELSE 0 END;" + $existsOutput = & docker exec $ContainerName /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P $MssqlPassword -d master -C -b -h -1 -W -Q $existsQuery + if ($LASTEXITCODE -ne 0) { throw "Failed to check whether database '$DatabaseName' already exists." } + + $databaseExists = (@($existsOutput | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Select-Object -First 1) -eq "1") + + if ($databaseExists) { + $dropSql = "ALTER DATABASE [$DatabaseName] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [$DatabaseName];" + & docker exec $ContainerName /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P $MssqlPassword -d master -C -b -Q $dropSql | Out-Null + if ($LASTEXITCODE -ne 0) { throw "Failed to drop existing database '$DatabaseName'." } + } + + $fileListQuery = "SET NOCOUNT ON; RESTORE FILELISTONLY FROM DISK = N'$containerBakPath';" + $fileListOutput = & docker exec $ContainerName /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P $MssqlPassword -d master -C -b -h -1 -W -s "|" -Q $fileListQuery + if ($LASTEXITCODE -ne 0) { throw "Failed to read the file list from backup '$($bakFile.Name)'." } + + # Collect every data (D) and log (L) row, not just the first of each: a multi-file backup + # (secondary data file, extra filegroup, additional log) must relocate every file it lists, + # or RESTORE DATABASE fails because some file is left pointing at its original in-container + # path. Today's DMS templates are single data + single log, but this keeps a multi-file + # .bak restorable too. + $dataLogicalNames = [System.Collections.Generic.List[string]]::new() + $logLogicalNames = [System.Collections.Generic.List[string]]::new() + foreach ($line in $fileListOutput) { + if ([string]::IsNullOrWhiteSpace($line)) { continue } + $fields = $line -split '\|' + if ($fields.Count -lt 3) { continue } + $logicalName = $fields[0].Trim() + $fileType = $fields[2].Trim() + if ($fileType -eq "D") { $dataLogicalNames.Add($logicalName) } + elseif ($fileType -eq "L") { $logLogicalNames.Add($logicalName) } + } + + if ($dataLogicalNames.Count -eq 0 -or $logLogicalNames.Count -eq 0) { + throw "Could not determine the data and log logical file names from backup '$($bakFile.Name)'." + } + + # Emit one MOVE clause per file. The primary data file and first log keep today's plain + # $DatabaseName-derived names, so a single-file .bak still produces the exact same RESTORE + # command as before; every additional file gets a name suffixed with its own logical name, + # so each lands at its own deterministic path under the same target data directory. Unlike + # the MOVE...FROM side (which only needs single-quote escaping inside its N'' literal), an + # extra logical name is interpolated into a new physical path here, so it is validated + # against the same safe-character allow-list already used for $DatabaseName before use. + $moveClauses = [System.Collections.Generic.List[string]]::new() + for ($i = 0; $i -lt $dataLogicalNames.Count; $i++) { + $logicalName = $dataLogicalNames[$i] + if ($i -eq 0) { + $physicalName = "$DatabaseName.mdf" + } + else { + if ($logicalName -notmatch "^[A-Za-z0-9_]+$") { + throw "Data file logical name '$logicalName' from backup '$($bakFile.Name)' contains unsupported characters and cannot be used to derive a restore path." + } + $physicalName = "${DatabaseName}_${logicalName}.ndf" + } + $moveClauses.Add("MOVE N'$($logicalName.Replace("'", "''"))' TO N'/var/opt/mssql/data/$physicalName'") + } + for ($i = 0; $i -lt $logLogicalNames.Count; $i++) { + $logicalName = $logLogicalNames[$i] + if ($i -eq 0) { + $physicalName = "${DatabaseName}_log.ldf" + } + else { + if ($logicalName -notmatch "^[A-Za-z0-9_]+$") { + throw "Log file logical name '$logicalName' from backup '$($bakFile.Name)' contains unsupported characters and cannot be used to derive a restore path." + } + $physicalName = "${DatabaseName}_${logicalName}.ldf" + } + $moveClauses.Add("MOVE N'$($logicalName.Replace("'", "''"))' TO N'/var/opt/mssql/data/$physicalName'") + } + + $restoreSql = "RESTORE DATABASE [$DatabaseName] FROM DISK = N'$containerBakPath' WITH $($moveClauses -join ', '), REPLACE;" + & docker exec $ContainerName /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P $MssqlPassword -d master -C -b -Q $restoreSql | Out-Null + if ($LASTEXITCODE -ne 0) { throw "Restore of '$($bakFile.Name)' into '$DatabaseName' failed." } + + & docker exec $ContainerName rm -f $containerBakPath + if ($LASTEXITCODE -ne 0) { throw "Failed to remove transient backup '$containerBakPath' from container '$ContainerName' after the restore." } + + return $package.Name + } + $sqlFile = Get-ChildItem -Path (Join-Path $extractDirectory "contents") -Filter *.sql -Recurse | Select-Object -First 1 if ($null -eq $sqlFile) { throw "No .sql dump found inside package '$($package.Name)'." } + # A connected session blocks DROP DATABASE; terminate any lingering ones as defense in + # depth (restores target freshly created verification databases, so nothing should be + # connected here). + & docker exec $ContainerName psql -U postgres -d postgres -v ON_ERROR_STOP=1 -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$DatabaseName' AND pid <> pg_backend_pid();" | Out-Null + if ($LASTEXITCODE -ne 0) { throw "Failed to terminate existing connections to database '$DatabaseName'." } + & docker exec $ContainerName psql -U postgres -d postgres -v ON_ERROR_STOP=1 -c "DROP DATABASE IF EXISTS $DatabaseName;" | Out-Null if ($LASTEXITCODE -ne 0) { throw "Failed to drop existing database '$DatabaseName'." } @@ -678,7 +899,12 @@ function Restore-TemplatePackage { <# .SYNOPSIS - Builds a NuGet package containing a PostgreSQL database backup. + Builds a NuGet package containing a database backup. + +.DESCRIPTION + Substitutes "{StandardVersion}", "{DatabaseEngine}", and "{ArtifactExtension}" placeholders + in the .psd1 configuration, dumps the target database (per -DatabaseEngine), and packages + the resulting artifact (.sql for PostgreSQL, .bak for MSSQL) into a NuGet package. .PARAMETER ConfigFilePath The path to the PowerShell data file (.psd1) containing configuration settings for the package. @@ -695,12 +921,23 @@ function Restore-TemplatePackage { .PARAMETER DumpAllUserSchemas Dump every non-system schema of the target database instead of only the dms schema. + PostgreSQL only; MSSQL always backs up the entire database. + +.PARAMETER DatabaseEngine + "postgresql" or "mssql". Defaults to "postgresql". Substitutes the "{DatabaseEngine}" + placeholder with "PostgreSql" or "MsSql" and the "{ArtifactExtension}" placeholder with + "sql" or "bak" respectively. + +.PARAMETER MssqlPassword + The MSSQL "sa" password. Defaults to environment variable MSSQL_SA_PASSWORD or "abcdefgh1!". .EXAMPLE Build-TemplateNuGetPackage -ConfigFilePath "./MinimalTemplateSettings.psd1" -StandardVersion "5.3.0" -PackageVersion "1.0.0" #> function Build-TemplateNuGetPackage { [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingWriteHost', '', Justification = 'Template tooling intentionally writes operator progress to the console.')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingUsernameAndPasswordParams', '', Justification = 'The MSSQL password is handed to sqlcmd -P, which only accepts plaintext; the account is always "sa" so there is no companion username parameter, and a PSCredential adds no protection across that boundary.')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification = 'The MSSQL password is handed to sqlcmd -P, which only accepts plaintext; SecureString adds no protection across that boundary.')] param ( [Parameter(Mandatory = $true)] [string]$ConfigFilePath, @@ -713,20 +950,32 @@ function Build-TemplateNuGetPackage { [string]$DatabaseName = "edfi_datamanagementservice", - [switch]$DumpAllUserSchemas + [switch]$DumpAllUserSchemas, + + [ValidateSet("postgresql", "mssql")] + [string]$DatabaseEngine = "postgresql", + + [string]$MssqlPassword = $env:MSSQL_SA_PASSWORD ?? "abcdefgh1!" ) $config = Import-PowerShellDataFile -Path $ConfigFilePath + # Mirrors PostgreSql/.sql, the literal tokens the .psd1 files carried before these + # placeholders were introduced, so PostgreSQL output stays byte-for-byte unchanged. + $engineToken = if ($DatabaseEngine -eq "mssql") { "MsSql" } else { "PostgreSql" } + $artifactExtension = if ($DatabaseEngine -eq "mssql") { "bak" } else { "sql" } + foreach ($key in @($config.Keys)) { if ($null -ne $key -and $config[$key] -is [string]) { $config[$key] = $config[$key].Replace("{StandardVersion}", $StandardVersion) + $config[$key] = $config[$key].Replace("{DatabaseEngine}", $engineToken) + $config[$key] = $config[$key].Replace("{ArtifactExtension}", $artifactExtension) } } $databaseSchemas = if ($DumpAllUserSchemas) { - $discoveredSchemas = @(Get-UserSchemaNames -DatabaseName $DatabaseName) + $discoveredSchemas = @(Get-UserSchemaNames -DatabaseEngine $DatabaseEngine -DatabaseName $DatabaseName -MssqlPassword $MssqlPassword) Write-Host "Dumping all user schemas from '$DatabaseName': $($discoveredSchemas -join ', ')" $discoveredSchemas } @@ -734,7 +983,7 @@ function Build-TemplateNuGetPackage { @("dms") } - Invoke-DatabaseDump -DatabaseName $DatabaseName -DatabaseSchemas $databaseSchemas -BackupDirectory './' -BackupFileName $config.DatabaseBackupName + Invoke-DatabaseDump -DatabaseEngine $DatabaseEngine -DatabaseName $DatabaseName -DatabaseSchemas $databaseSchemas -BackupDirectory './' -BackupFileName $config.DatabaseBackupName -MssqlPassword $MssqlPassword New-DatabaseTemplateCsproj -Config $config -BackupDirectory './' @@ -933,10 +1182,18 @@ enum TemplateType { .PARAMETER DumpAllUserSchemas Dump every non-system schema of the target database instead of only the dms schema. + PostgreSQL only; MSSQL always backs up the entire database. .PARAMETER PostgresPassword The PostgreSQL password for database connection. Defaults to environment variable POSTGRES_PASSWORD or "abcdefgh1!". +.PARAMETER DatabaseEngine + "postgresql" or "mssql". Defaults to "postgresql". Selects the engine used when dumping + the data store database into the template package. + +.PARAMETER MssqlPassword + The MSSQL "sa" password. Defaults to environment variable MSSQL_SA_PASSWORD or "abcdefgh1!". + .EXAMPLE Build-Template -TemplateType Minimal ` -DmsUrl "http://localhost:8080" ` @@ -950,8 +1207,8 @@ enum TemplateType { -PostgresPassword "mypassword" #> function Build-Template { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingUsernameAndPasswordParams', '', Justification = 'The PostgreSQL password is handed to a PostgreSQL connection string where it must be plaintext; there is no companion username credential and a PSCredential adds no protection across that boundary.')] - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification = 'The PostgreSQL password is handed to a PostgreSQL connection string where it must be plaintext; SecureString adds no protection across that boundary.')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingUsernameAndPasswordParams', '', Justification = 'The PostgreSQL and MSSQL passwords are handed to a database connection string / sqlcmd where they must be plaintext; there is no companion username credential and a PSCredential adds no protection across that boundary.')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification = 'The PostgreSQL and MSSQL passwords are handed to a database connection string / sqlcmd where they must be plaintext; SecureString adds no protection across that boundary.')] [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingWriteHost', '', Justification = 'Template tooling intentionally writes operator progress to the console.')] param ( @@ -989,7 +1246,12 @@ function Build-Template { [switch]$DumpAllUserSchemas, - [string]$PostgresPassword = $env:POSTGRES_PASSWORD ?? "abcdefgh1!" + [string]$PostgresPassword = $env:POSTGRES_PASSWORD ?? "abcdefgh1!", + + [ValidateSet("postgresql", "mssql")] + [string]$DatabaseEngine = "postgresql", + + [string]$MssqlPassword = $env:MSSQL_SA_PASSWORD ?? "abcdefgh1!" ) if ($PSBoundParameters.ContainsKey('DataStoreId') -and -not $PSBoundParameters.ContainsKey('DataStoreDatabaseName')) { @@ -1091,7 +1353,7 @@ function Build-Template { -AllowUnresolvedReferences:$educatorPrepFiltered } - Build-TemplateNuGetPackage -ConfigFilePath $ConfigFilePath -StandardVersion $StandardVersion -PackageVersion $PackageVersion -DatabaseName $DataStoreDatabaseName -DumpAllUserSchemas:$DumpAllUserSchemas + Build-TemplateNuGetPackage -ConfigFilePath $ConfigFilePath -StandardVersion $StandardVersion -PackageVersion $PackageVersion -DatabaseName $DataStoreDatabaseName -DumpAllUserSchemas:$DumpAllUserSchemas -DatabaseEngine $DatabaseEngine -MssqlPassword $MssqlPassword } Export-ModuleMember -Function Build-Template, Get-UserSchemaNames, Restore-TemplatePackage diff --git a/eng/DatabaseTemplates/tests/Template-Management.Tests.ps1 b/eng/DatabaseTemplates/tests/Template-Management.Tests.ps1 index 646f8d1a90..29b9cbba89 100644 --- a/eng/DatabaseTemplates/tests/Template-Management.Tests.ps1 +++ b/eng/DatabaseTemplates/tests/Template-Management.Tests.ps1 @@ -352,3 +352,485 @@ Describe "Get-BulkLoadFailureClassification" { $result.UnresolvedReferenceCount | Should -Be 0 } } + +Describe "Invoke-DatabaseDump" { + BeforeAll { + # The module imports sibling modules with paths relative to its own directory, so import + # it with that directory as the working directory (matching how the template workflow + # invokes it). The dump function is not exported, so it is reached via InModuleScope. + $script:templatesDir = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot "..")) + Push-Location $script:templatesDir + try { + Import-Module (Join-Path $script:templatesDir "Template-Management.psm1") -Force + } + finally { + Pop-Location + } + } + + Context "when -DatabaseEngine is mssql" { + It "runs BACKUP DATABASE ... WITH INIT, copies the .bak out, and removes the transient in-container file" { + InModuleScope Template-Management -Parameters @{ backupDir = $TestDrive } { + param($backupDir) + $calls = [System.Collections.Generic.List[object]]::new() + Mock docker { + $calls.Add(@($args)) + if ($args[0] -eq 'cp') { + New-Item -ItemType File -Path $args[-1] -Force | Out-Null + } + $global:LASTEXITCODE = 0 + } + + Invoke-DatabaseDump -DatabaseEngine mssql -ContainerName "dms-mssql" -DatabaseName "edfi_datamanagementservice" -BackupDirectory $backupDir -BackupFileName "test.bak" -MssqlPassword "abcdefgh1!" + + $calls.Count | Should -Be 3 + ($calls[0] -join '|') | Should -Be (@( + 'exec', 'dms-mssql', '/opt/mssql-tools18/bin/sqlcmd', '-S', 'localhost', '-U', 'sa', '-P', 'abcdefgh1!', '-C', '-b', '-Q', + "BACKUP DATABASE [edfi_datamanagementservice] TO DISK = N'/var/opt/mssql/data/test.bak' WITH INIT;" + ) -join '|') + ($calls[1] -join '|') | Should -Be (@('cp', "dms-mssql:/var/opt/mssql/data/test.bak", (Join-Path $backupDir "test.bak")) -join '|') + ($calls[2] -join '|') | Should -Be (@('exec', 'dms-mssql', 'rm', '-f', '/var/opt/mssql/data/test.bak') -join '|') + } + } + + It "ignores -DatabaseSchemas because a .bak is always a full-database artifact" { + InModuleScope Template-Management -Parameters @{ backupDir = $TestDrive } { + param($backupDir) + $calls = [System.Collections.Generic.List[object]]::new() + Mock docker { + $calls.Add(@($args)) + if ($args[0] -eq 'cp') { + New-Item -ItemType File -Path $args[-1] -Force | Out-Null + } + $global:LASTEXITCODE = 0 + } + + Invoke-DatabaseDump -DatabaseEngine mssql -ContainerName "dms-mssql" -DatabaseName "edfi_datamanagementservice" -DatabaseSchemas @("dms", "edfi") -BackupDirectory $backupDir -BackupFileName "test2.bak" -MssqlPassword "abcdefgh1!" + + # BACKUP DATABASE has no per-schema equivalent to pg_dump -n, so the schema list plays no part in the command. + ($calls[0] -join '|') | Should -Be (@( + 'exec', 'dms-mssql', '/opt/mssql-tools18/bin/sqlcmd', '-S', 'localhost', '-U', 'sa', '-P', 'abcdefgh1!', '-C', '-b', '-Q', + "BACKUP DATABASE [edfi_datamanagementservice] TO DISK = N'/var/opt/mssql/data/test2.bak' WITH INIT;" + ) -join '|') + } + } + } + + Context "when -DatabaseEngine is postgresql (the default)" { + It "runs pg_dump scoped to the requested schemas, in order, and precedes the dump with a pgcrypto preamble" { + InModuleScope Template-Management -Parameters @{ backupDir = $TestDrive } { + param($backupDir) + $calls = [System.Collections.Generic.List[object]]::new() + Mock docker { + $calls.Add(@($args)) + $global:LASTEXITCODE = 0 + } + + Invoke-DatabaseDump -DatabaseEngine postgresql -ContainerName "dms-postgresql" -DatabaseName "edfi_datamanagementservice" -DatabaseSchemas @("dms", "edfi") -BackupDirectory $backupDir -BackupFileName "test.sql" + + $calls.Count | Should -Be 1 + ($calls[0] -join '|') | Should -Be (@('exec', 'dms-postgresql', 'pg_dump', '-U', 'postgres', 'edfi_datamanagementservice', '-n', 'dms', '-n', 'edfi') -join '|') + + # Schema-scoped pg_dump never emits CREATE EXTENSION, but the dumped dms.uuidv5() + # function requires pgcrypto's digest(), so the preamble must precede the dump. + $content = Get-Content -LiteralPath (Join-Path $backupDir "test.sql") -Raw + $content | Should -Match ([regex]::Escape('CREATE EXTENSION IF NOT EXISTS "pgcrypto";')) + } + } + + It "produces byte-identical pg_dump arguments whether or not -DatabaseEngine is supplied" { + InModuleScope Template-Management -Parameters @{ backupDir = $TestDrive } { + param($backupDir) + $explicitCalls = [System.Collections.Generic.List[object]]::new() + Mock docker { $explicitCalls.Add(@($args)); $global:LASTEXITCODE = 0 } + Invoke-DatabaseDump -DatabaseEngine postgresql -ContainerName "dms-postgresql" -DatabaseName "edfi_datamanagementservice" -DatabaseSchemas @("dms") -BackupDirectory $backupDir -BackupFileName "explicit.sql" + + $defaultCalls = [System.Collections.Generic.List[object]]::new() + Mock docker { $defaultCalls.Add(@($args)); $global:LASTEXITCODE = 0 } + Invoke-DatabaseDump -ContainerName "dms-postgresql" -DatabaseName "edfi_datamanagementservice" -DatabaseSchemas @("dms") -BackupDirectory $backupDir -BackupFileName "default.sql" + + $defaultCalls.Count | Should -Be $explicitCalls.Count + ($defaultCalls[0] -join '|') | Should -Be ($explicitCalls[0] -join '|') + } + } + } +} + +Describe "Restore-TemplatePackage" { + BeforeAll { + $script:templatesDir = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot "..")) + Push-Location $script:templatesDir + try { + Import-Module (Join-Path $script:templatesDir "Template-Management.psm1") -Force + } + finally { + Pop-Location + } + + # Restore-TemplatePackage only cares that the package directory contains exactly one + # .nupkg (a zip archive) with exactly one artifact file inside it, so a minimal package + # is built directly rather than driving the real dotnet pack/csproj pipeline. + function script:New-FakeTemplatePackage { + param( + [Parameter(Mandatory = $true)] + [string]$Directory, + + [Parameter(Mandatory = $true)] + [string]$ArtifactFileName, + + [string]$PackageFileName = "FakeTemplate.nupkg" + ) + + $stagingDir = Join-Path $Directory "staging" + New-Item -ItemType Directory -Path $stagingDir -Force | Out-Null + $artifactPath = Join-Path $stagingDir $ArtifactFileName + Set-Content -LiteralPath $artifactPath -Value "fake artifact content" + + $zipPath = Join-Path $Directory "package.zip" + Compress-Archive -Path $artifactPath -DestinationPath $zipPath -Force + $nupkgPath = Join-Path $Directory $PackageFileName + Copy-Item -LiteralPath $zipPath -Destination $nupkgPath -Force + + return $nupkgPath + } + } + + BeforeEach { + $script:packageDir = Join-Path $TestDrive ([Guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path $script:packageDir -Force | Out-Null + } + + Context "when -DatabaseEngine is mssql" { + It "drops an existing target database, then restores via RESTORE DATABASE ... WITH MOVE ..., REPLACE" { + New-FakeTemplatePackage -Directory $script:packageDir -ArtifactFileName "backup.bak" -PackageFileName "MyTemplate.nupkg" | Out-Null + + $calls = [System.Collections.Generic.List[object]]::new() + Mock docker -ModuleName Template-Management { + $calls.Add(@($args)) + $joined = $args -join ' ' + $global:LASTEXITCODE = 0 + if ($joined -match 'RESTORE FILELISTONLY') { + return @('MyDb|/var/opt/mssql/data/MyDb.mdf|D|PRIMARY', 'MyDb_log|/var/opt/mssql/data/MyDb_log.ldf|L|NULL') + } + if ($joined -match 'DB_ID') { + return '1' + } + } + + $result = Restore-TemplatePackage -PackageDirectory $script:packageDir -DatabaseName "testdb" -DatabaseEngine mssql -ContainerName "dms-mssql" -MssqlPassword "abcdefgh1!" + + $result | Should -Be "MyTemplate.nupkg" + $calls.Count | Should -Be 6 + + # [0] docker cp copies the extracted .bak into the container at a generated path; + # capture that path so the later RESTORE FROM DISK clause can be pinned against it. + $calls[0][0] | Should -Be 'cp' + $copyDestination = $calls[0][2] + $copyDestination | Should -Match '^dms-mssql:/var/opt/mssql/data/template-restore-.*\.bak$' + $containerBakPath = ($copyDestination -split ':', 2)[1] + + ($calls[1] -join '|') | Should -Be (@( + 'exec', 'dms-mssql', '/opt/mssql-tools18/bin/sqlcmd', '-S', 'localhost', '-U', 'sa', '-P', 'abcdefgh1!', '-d', 'master', '-C', '-b', '-h', '-1', '-W', '-Q', + "SET NOCOUNT ON; SELECT CASE WHEN DB_ID(N'testdb') IS NOT NULL THEN 1 ELSE 0 END;" + ) -join '|') + + ($calls[2] -join '|') | Should -Be (@( + 'exec', 'dms-mssql', '/opt/mssql-tools18/bin/sqlcmd', '-S', 'localhost', '-U', 'sa', '-P', 'abcdefgh1!', '-d', 'master', '-C', '-b', '-Q', + "ALTER DATABASE [testdb] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [testdb];" + ) -join '|') + + ($calls[3] -join '|') | Should -Be (@( + 'exec', 'dms-mssql', '/opt/mssql-tools18/bin/sqlcmd', '-S', 'localhost', '-U', 'sa', '-P', 'abcdefgh1!', '-d', 'master', '-C', '-b', '-h', '-1', '-W', '-s', '|', '-Q', + "SET NOCOUNT ON; RESTORE FILELISTONLY FROM DISK = N'$containerBakPath';" + ) -join '|') + + ($calls[4] -join '|') | Should -Be (@( + 'exec', 'dms-mssql', '/opt/mssql-tools18/bin/sqlcmd', '-S', 'localhost', '-U', 'sa', '-P', 'abcdefgh1!', '-d', 'master', '-C', '-b', '-Q', + "RESTORE DATABASE [testdb] FROM DISK = N'$containerBakPath' WITH MOVE N'MyDb' TO N'/var/opt/mssql/data/testdb.mdf', MOVE N'MyDb_log' TO N'/var/opt/mssql/data/testdb_log.ldf', REPLACE;" + ) -join '|') + + # The transient in-container backup is removed once the restore succeeds. + ($calls[5] -join '|') | Should -Be (@('exec', 'dms-mssql', 'rm', '-f', $containerBakPath) -join '|') + } + + It "skips the DROP DATABASE step when the target database does not already exist" { + New-FakeTemplatePackage -Directory $script:packageDir -ArtifactFileName "backup.bak" -PackageFileName "MyTemplate.nupkg" | Out-Null + + $calls = [System.Collections.Generic.List[object]]::new() + Mock docker -ModuleName Template-Management { + $calls.Add(@($args)) + $joined = $args -join ' ' + $global:LASTEXITCODE = 0 + if ($joined -match 'RESTORE FILELISTONLY') { + return @('MyDb|/var/opt/mssql/data/MyDb.mdf|D|PRIMARY', 'MyDb_log|/var/opt/mssql/data/MyDb_log.ldf|L|NULL') + } + if ($joined -match 'DB_ID') { + return '0' + } + } + + Restore-TemplatePackage -PackageDirectory $script:packageDir -DatabaseName "testdb" -DatabaseEngine mssql -ContainerName "dms-mssql" -MssqlPassword "abcdefgh1!" | Out-Null + + $calls.Count | Should -Be 5 + ($calls | Where-Object { ($_ -join ' ') -match 'SET SINGLE_USER' }).Count | Should -Be 0 + ($calls[2] -join ' ') | Should -Match 'RESTORE FILELISTONLY' + ($calls[3] -join ' ') | Should -Match 'RESTORE DATABASE \[testdb\]' + ($calls[4] -join ' ') | Should -Match '^exec dms-mssql rm -f /var/opt/mssql/data/template-restore-.*\.bak$' + } + + It "emits one MOVE clause per file for a multi-file backup (secondary data file, extra filegroup, multiple logs)" { + New-FakeTemplatePackage -Directory $script:packageDir -ArtifactFileName "backup.bak" -PackageFileName "MyTemplate.nupkg" | Out-Null + + $calls = [System.Collections.Generic.List[object]]::new() + Mock docker -ModuleName Template-Management { + $calls.Add(@($args)) + $joined = $args -join ' ' + $global:LASTEXITCODE = 0 + if ($joined -match 'RESTORE FILELISTONLY') { + return @( + 'MyDb|/var/opt/mssql/data/MyDb.mdf|D|PRIMARY', + 'MyDb2|/var/opt/mssql/data/MyDb2.ndf|D|SECONDARY', + 'MyDb_log|/var/opt/mssql/data/MyDb_log.ldf|L|NULL', + 'MyDb_log2|/var/opt/mssql/data/MyDb_log2.ldf|L|NULL' + ) + } + if ($joined -match 'DB_ID') { + return '0' + } + } + + Restore-TemplatePackage -PackageDirectory $script:packageDir -DatabaseName "testdb" -DatabaseEngine mssql -ContainerName "dms-mssql" -MssqlPassword "abcdefgh1!" | Out-Null + + $containerBakPath = ($calls[0][2] -split ':', 2)[1] + + # The primary data file and first log keep the plain $DatabaseName-derived names (matching + # the single-file case); the secondary data file and second log get names suffixed with + # their own logical name so every file lands at its own deterministic path. + ($calls[3] -join '|') | Should -Be (@( + 'exec', 'dms-mssql', '/opt/mssql-tools18/bin/sqlcmd', '-S', 'localhost', '-U', 'sa', '-P', 'abcdefgh1!', '-d', 'master', '-C', '-b', '-Q', + "RESTORE DATABASE [testdb] FROM DISK = N'$containerBakPath' WITH MOVE N'MyDb' TO N'/var/opt/mssql/data/testdb.mdf', MOVE N'MyDb2' TO N'/var/opt/mssql/data/testdb_MyDb2.ndf', MOVE N'MyDb_log' TO N'/var/opt/mssql/data/testdb_log.ldf', MOVE N'MyDb_log2' TO N'/var/opt/mssql/data/testdb_MyDb_log2.ldf', REPLACE;" + ) -join '|') + } + + It "throws when an additional file's logical name contains characters unsafe for a physical path" { + New-FakeTemplatePackage -Directory $script:packageDir -ArtifactFileName "backup.bak" -PackageFileName "MyTemplate.nupkg" | Out-Null + + Mock docker -ModuleName Template-Management { + $joined = $args -join ' ' + $global:LASTEXITCODE = 0 + if ($joined -match 'RESTORE FILELISTONLY') { + return @( + 'MyDb|/var/opt/mssql/data/MyDb.mdf|D|PRIMARY', + "MyDb'; DROP TABLE x --|/var/opt/mssql/data/MyDb2.ndf|D|SECONDARY", + 'MyDb_log|/var/opt/mssql/data/MyDb_log.ldf|L|NULL' + ) + } + if ($joined -match 'DB_ID') { + return '0' + } + } + + { Restore-TemplatePackage -PackageDirectory $script:packageDir -DatabaseName "testdb" -DatabaseEngine mssql -ContainerName "dms-mssql" -MssqlPassword "abcdefgh1!" } | + Should -Throw "*contains unsupported characters*" + } + } + + Context "when -DatabaseEngine is postgresql (the default)" { + It "drops and recreates the database, copies the dump in, and restores via psql -f" { + New-FakeTemplatePackage -Directory $script:packageDir -ArtifactFileName "dump.sql" -PackageFileName "MyPgTemplate.nupkg" | Out-Null + + $calls = [System.Collections.Generic.List[object]]::new() + Mock docker -ModuleName Template-Management { + $calls.Add(@($args)) + $global:LASTEXITCODE = 0 + } + + $result = Restore-TemplatePackage -PackageDirectory $script:packageDir -DatabaseName "testdb" -DatabaseEngine postgresql -ContainerName "dms-postgresql" + + $result | Should -Be "MyPgTemplate.nupkg" + $calls.Count | Should -Be 5 + ($calls[0] -join '|') | Should -Be (@('exec', 'dms-postgresql', 'psql', '-U', 'postgres', '-d', 'postgres', '-v', 'ON_ERROR_STOP=1', '-c', "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'testdb' AND pid <> pg_backend_pid();") -join '|') + ($calls[1] -join '|') | Should -Be (@('exec', 'dms-postgresql', 'psql', '-U', 'postgres', '-d', 'postgres', '-v', 'ON_ERROR_STOP=1', '-c', 'DROP DATABASE IF EXISTS testdb;') -join '|') + ($calls[2] -join '|') | Should -Be (@('exec', 'dms-postgresql', 'psql', '-U', 'postgres', '-d', 'postgres', '-v', 'ON_ERROR_STOP=1', '-c', 'CREATE DATABASE testdb;') -join '|') + $calls[3][0] | Should -Be 'cp' + ($calls[4] -join '|') | Should -Be (@('exec', 'dms-postgresql', 'psql', '-U', 'postgres', '-d', 'testdb', '-v', 'ON_ERROR_STOP=1', '-f', '/tmp/template-restore.sql') -join '|') + } + + It "produces byte-identical restore arguments whether or not -DatabaseEngine is supplied" { + $explicitDir = Join-Path $TestDrive ([Guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path $explicitDir -Force | Out-Null + New-FakeTemplatePackage -Directory $explicitDir -ArtifactFileName "dump.sql" -PackageFileName "Explicit.nupkg" | Out-Null + + $defaultDir = Join-Path $TestDrive ([Guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path $defaultDir -Force | Out-Null + New-FakeTemplatePackage -Directory $defaultDir -ArtifactFileName "dump.sql" -PackageFileName "Default.nupkg" | Out-Null + + $explicitCalls = [System.Collections.Generic.List[object]]::new() + Mock docker -ModuleName Template-Management { $explicitCalls.Add(@($args)); $global:LASTEXITCODE = 0 } + Restore-TemplatePackage -PackageDirectory $explicitDir -DatabaseName "paritydb" -DatabaseEngine postgresql -ContainerName "dms-postgresql" | Out-Null + + $defaultCalls = [System.Collections.Generic.List[object]]::new() + Mock docker -ModuleName Template-Management { $defaultCalls.Add(@($args)); $global:LASTEXITCODE = 0 } + Restore-TemplatePackage -PackageDirectory $defaultDir -DatabaseName "paritydb" -ContainerName "dms-postgresql" | Out-Null + + # Each call is extracted into its own randomly named temp directory, so normalize that + # GUID segment out of the local-side source path before comparing call shapes. + $normalizeGuid = { param($callArgs) @($callArgs | ForEach-Object { $_ -replace 'template-restore-[0-9a-fA-F]{32}', 'template-restore-' }) } + + $defaultCalls.Count | Should -Be $explicitCalls.Count + for ($i = 0; $i -lt $explicitCalls.Count; $i++) { + ((& $normalizeGuid $defaultCalls[$i]) -join '|') | Should -Be ((& $normalizeGuid $explicitCalls[$i]) -join '|') + } + } + } +} + +Describe "Get-UserSchemaNames" { + BeforeAll { + $script:templatesDir = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot "..")) + Push-Location $script:templatesDir + try { + Import-Module (Join-Path $script:templatesDir "Template-Management.psm1") -Force + } + finally { + Pop-Location + } + } + + Context "when -DatabaseEngine is mssql" { + It "queries sys.schemas, excluding the built-in schemas and the db_* fixed-role schemas" { + $calls = [System.Collections.Generic.List[object]]::new() + Mock docker -ModuleName Template-Management { + $calls.Add(@($args)) + $global:LASTEXITCODE = 0 + return @('dms', 'edfi') + } + + $schemas = Get-UserSchemaNames -DatabaseEngine mssql -ContainerName "dms-mssql" -DatabaseName "testdb" -MssqlPassword "abcdefgh1!" + + $schemas | Should -Be @('dms', 'edfi') + $calls.Count | Should -Be 1 + ($calls[0] -join '|') | Should -Be (@( + 'exec', 'dms-mssql', '/opt/mssql-tools18/bin/sqlcmd', '-S', 'localhost', '-U', 'sa', '-P', 'abcdefgh1!', '-d', 'testdb', '-C', '-b', '-h', '-1', '-W', '-Q', + "SET NOCOUNT ON; SELECT name FROM sys.schemas WHERE name NOT IN ('dbo', 'guest', 'sys', 'INFORMATION_SCHEMA') AND name NOT LIKE 'db[_]%' ORDER BY name;" + ) -join '|') + } + } + + Context "when -DatabaseEngine is postgresql (the default)" { + It "queries pg_namespace, excluding pg_* system schemas, information_schema, and public" { + $calls = [System.Collections.Generic.List[object]]::new() + Mock docker -ModuleName Template-Management { + $calls.Add(@($args)) + $global:LASTEXITCODE = 0 + return @('dms', 'edfi') + } + + $schemas = Get-UserSchemaNames -ContainerName "dms-postgresql" -DatabaseName "testdb" + + $schemas | Should -Be @('dms', 'edfi') + $calls.Count | Should -Be 1 + ($calls[0] -join '|') | Should -Be (@( + 'exec', 'dms-postgresql', 'psql', '-U', 'postgres', '-d', 'testdb', '-tA', '-c', + "SELECT nspname FROM pg_namespace WHERE nspname !~ '^pg_' AND nspname NOT IN ('information_schema', 'public') ORDER BY nspname;" + ) -join '|') + } + + It "issues the identical query whether or not -DatabaseEngine is supplied" { + $explicitCalls = [System.Collections.Generic.List[object]]::new() + Mock docker -ModuleName Template-Management { $explicitCalls.Add(@($args)); $global:LASTEXITCODE = 0; return @('dms') } + Get-UserSchemaNames -DatabaseEngine postgresql -ContainerName "dms-postgresql" -DatabaseName "testdb" | Out-Null + + $defaultCalls = [System.Collections.Generic.List[object]]::new() + Mock docker -ModuleName Template-Management { $defaultCalls.Add(@($args)); $global:LASTEXITCODE = 0; return @('dms') } + Get-UserSchemaNames -ContainerName "dms-postgresql" -DatabaseName "testdb" | Out-Null + + $defaultCalls.Count | Should -Be $explicitCalls.Count + ($defaultCalls[0] -join '|') | Should -Be ($explicitCalls[0] -join '|') + } + } + + It "throws when no user schemas are discovered, regardless of engine" { + Mock docker -ModuleName Template-Management { $global:LASTEXITCODE = 0; return @() } + + { Get-UserSchemaNames -ContainerName "dms-postgresql" -DatabaseName "testdb" } | Should -Throw "*does not appear to be provisioned*" + } +} + +Describe "Build-TemplateNuGetPackage package identity derivation" { + BeforeAll { + # Package identity derivation (engine token + artifact extension substitution) is internal + # to Build-TemplateNuGetPackage, so it is exercised through the real, repo-shipped .psd1 + # settings files with the heavy externals (dump, csproj authoring, dotnet pack) mocked out. + $script:templatesDir = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot "..")) + Push-Location $script:templatesDir + try { + Import-Module (Join-Path $script:templatesDir "Template-Management.psm1") -Force + } + finally { + Pop-Location + } + } + + It "derives the package Id/Title/Description/PackageProjectName and DatabaseBackupName for /" -ForEach @( + @{ TemplateType = 'Minimal'; StandardVersion = '5.2.0'; Engine = 'postgresql'; EngineToken = 'PostgreSql'; Extension = 'sql' } + @{ TemplateType = 'Minimal'; StandardVersion = '5.2.0'; Engine = 'mssql'; EngineToken = 'MsSql'; Extension = 'bak' } + @{ TemplateType = 'Minimal'; StandardVersion = '6.1.0'; Engine = 'postgresql'; EngineToken = 'PostgreSql'; Extension = 'sql' } + @{ TemplateType = 'Minimal'; StandardVersion = '6.1.0'; Engine = 'mssql'; EngineToken = 'MsSql'; Extension = 'bak' } + @{ TemplateType = 'Populated'; StandardVersion = '5.2.0'; Engine = 'postgresql'; EngineToken = 'PostgreSql'; Extension = 'sql' } + @{ TemplateType = 'Populated'; StandardVersion = '5.2.0'; Engine = 'mssql'; EngineToken = 'MsSql'; Extension = 'bak' } + @{ TemplateType = 'Populated'; StandardVersion = '6.1.0'; Engine = 'postgresql'; EngineToken = 'PostgreSql'; Extension = 'sql' } + @{ TemplateType = 'Populated'; StandardVersion = '6.1.0'; Engine = 'mssql'; EngineToken = 'MsSql'; Extension = 'bak' } + ) { + $configPath = Join-Path $script:templatesDir "${TemplateType}TemplateSettings.psd1" + + InModuleScope Template-Management -Parameters @{ + configPath = $configPath + templateType = $TemplateType + standardVersion = $StandardVersion + engine = $Engine + engineToken = $EngineToken + extension = $Extension + } { + param($configPath, $templateType, $standardVersion, $engine, $engineToken, $extension) + + Mock Invoke-DatabaseDump {} + Mock New-DatabaseTemplateCsproj {} + Mock Build-NuGetPackage {} + + Build-TemplateNuGetPackage -ConfigFilePath $configPath -StandardVersion $standardVersion -PackageVersion "1.0.0" -DatabaseEngine $engine + + $expectedId = "EdFi.Api.$templateType.Template.$engineToken.$standardVersion" + $expectedBackupName = "EdFi.Api.$templateType.Template.$engineToken.$standardVersion.$extension" + $expectedProjectName = "EdFi.Api.$templateType.Template.$engineToken.$standardVersion.csproj" + $expectedDescription = "EdFi Dms $templateType Template Database for $engineToken" + + Should -Invoke New-DatabaseTemplateCsproj -Times 1 -Exactly -ParameterFilter { + $Config.Id -eq $expectedId -and + $Config.Title -eq $expectedId -and + $Config.Description -eq $expectedDescription -and + $Config.DatabaseBackupName -eq $expectedBackupName -and + $Config.PackageProjectName -eq $expectedProjectName + } + } + } + + It "defaults -DatabaseEngine to postgresql, producing the PostgreSql token and .sql extension, when the parameter is omitted" { + InModuleScope Template-Management -Parameters @{ configPath = (Join-Path $script:templatesDir "MinimalTemplateSettings.psd1") } { + param($configPath) + Mock Invoke-DatabaseDump {} + Mock New-DatabaseTemplateCsproj {} + Mock Build-NuGetPackage {} + + Build-TemplateNuGetPackage -ConfigFilePath $configPath -StandardVersion "5.2.0" -PackageVersion "1.0.0" + + Should -Invoke New-DatabaseTemplateCsproj -Times 1 -Exactly -ParameterFilter { + $Config.Id -eq 'EdFi.Api.Minimal.Template.PostgreSql.5.2.0' -and + $Config.DatabaseBackupName -eq 'EdFi.Api.Minimal.Template.PostgreSql.5.2.0.sql' + } + + # The dump must also see the default resolved to postgresql, not left blank/unset. + Should -Invoke Invoke-DatabaseDump -Times 1 -Exactly -ParameterFilter { $DatabaseEngine -eq 'postgresql' } + } + } +} diff --git a/eng/DatabaseTemplates/verify-template-restore.ps1 b/eng/DatabaseTemplates/verify-template-restore.ps1 index 5e171fd8b6..586ae218a0 100644 --- a/eng/DatabaseTemplates/verify-template-restore.ps1 +++ b/eng/DatabaseTemplates/verify-template-restore.ps1 @@ -11,8 +11,8 @@ database that DMS can serve requests from. .DESCRIPTION - Extracts the .sql dump from the template .nupkg and restores it into a - fresh verification database inside the running PostgreSQL container. The + Extracts the database dump from the template .nupkg and restores it into a + fresh verification database inside the running database container. The restored schema set must match the source database's user schemas, and core data (dms.EffectiveSchema, dms.Document, dms.Descriptor) must be non-empty; extension schemas may be DDL-only. Serveability is then proven @@ -35,13 +35,21 @@ .PARAMETER PackageDirectory Directory containing the generated .nupkg (default: current directory). +.PARAMETER DatabaseEngine + "postgresql" or "mssql". Defaults to "postgresql". Selects the engine used + to restore and inspect the verification database. + +.PARAMETER MssqlPassword + The MSSQL "sa" password. Defaults to environment variable MSSQL_SA_PASSWORD or "abcdefgh1!". + .PARAMETER RequirePopulatedData Additionally require populated (non-descriptor) sample data to survive the restore and be serveable by DMS. #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingWriteHost', '', Justification = 'Verification entry script intentionally writes operator progress to the console.')] -[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification = 'The PostgreSQL password is read as plaintext from the environment and handed to psql / a PostgreSQL connection string, where it must be plaintext; SecureString adds no protection across that boundary.')] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingUsernameAndPasswordParams', '', Justification = 'The PostgreSQL and MSSQL passwords are handed to a database connection string / sqlcmd where they must be plaintext; there is no companion username credential and a PSCredential adds no protection across that boundary.')] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification = 'The PostgreSQL and MSSQL passwords are read as plaintext from the environment and handed to psql/sqlcmd or a database connection string, where they must be plaintext; SecureString adds no protection across that boundary.')] [CmdletBinding()] param ( [Parameter(Mandatory = $true)] @@ -51,7 +59,10 @@ param ( [string]$VerificationDatabaseName = "template_restore_verification", - [string]$ContainerName = "dms-postgresql", + [ValidateSet("postgresql", "mssql")] + [string]$DatabaseEngine = "postgresql", + + [string]$ContainerName = $(if ($DatabaseEngine -eq "mssql") { "dms-mssql" } else { "dms-postgresql" }), [string]$DmsContainerName = "ed-fi-api", @@ -61,6 +72,8 @@ param ( [string]$PostgresPassword = $env:POSTGRES_PASSWORD ?? "abcdefgh1!", + [string]$MssqlPassword = $env:MSSQL_SA_PASSWORD ?? "abcdefgh1!", + [switch]$RequirePopulatedData ) @@ -90,6 +103,21 @@ function Invoke-PsqlScalar { return [long](@($value) | Select-Object -First 1) } +function Invoke-SqlcmdScalar { + param( + [string]$DatabaseName, + [string]$Query + ) + + $value = & docker exec $ContainerName /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P $MssqlPassword -d $DatabaseName -C -b -h -1 -W -Q $Query + + if ($LASTEXITCODE -ne 0) { + throw "Query failed against database '$DatabaseName': $Query" + } + + return [long](@($value) | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Select-Object -First 1) +} + Assert-SafeDatabaseName -DatabaseName $SourceDatabaseName Assert-SafeDatabaseName -DatabaseName $VerificationDatabaseName @@ -99,14 +127,14 @@ try { Import-Module ../Dms-Management.psm1 -Force # --- Restore the package into a fresh verification database --- - $expectedSchemas = @(Get-UserSchemaNames -ContainerName $ContainerName -DatabaseName $SourceDatabaseName) + $expectedSchemas = @(Get-UserSchemaNames -DatabaseEngine $DatabaseEngine -ContainerName $ContainerName -DatabaseName $SourceDatabaseName -MssqlPassword $MssqlPassword) Write-Host "Expected schemas (from '$SourceDatabaseName'): $($expectedSchemas -join ', ')" - $packageName = Restore-TemplatePackage -PackageDirectory $PackageDirectory -DatabaseName $VerificationDatabaseName -ContainerName $ContainerName + $packageName = Restore-TemplatePackage -PackageDirectory $PackageDirectory -DatabaseName $VerificationDatabaseName -DatabaseEngine $DatabaseEngine -ContainerName $ContainerName -MssqlPassword $MssqlPassword Write-Host "Verifying template package: $packageName" -ForegroundColor Cyan # --- Structural and data assertions --- - $restoredSchemas = @(Get-UserSchemaNames -ContainerName $ContainerName -DatabaseName $VerificationDatabaseName) + $restoredSchemas = @(Get-UserSchemaNames -DatabaseEngine $DatabaseEngine -ContainerName $ContainerName -DatabaseName $VerificationDatabaseName -MssqlPassword $MssqlPassword) $missingSchemas = @($expectedSchemas | Where-Object { $_ -cnotin $restoredSchemas }) $unexpectedSchemas = @($restoredSchemas | Where-Object { $_ -cnotin $expectedSchemas }) @@ -116,14 +144,26 @@ try { Write-Host "Restored schemas match: $($restoredSchemas -join ', ')" -ForegroundColor Green - $effectiveSchemaCount = Invoke-PsqlScalar -DatabaseName $VerificationDatabaseName -Query 'SELECT COUNT(*) FROM dms."EffectiveSchema";' - if ($effectiveSchemaCount -lt 1) { throw 'Restored dms."EffectiveSchema" is empty.' } + if ($DatabaseEngine -eq "mssql") { + $effectiveSchemaCount = Invoke-SqlcmdScalar -DatabaseName $VerificationDatabaseName -Query 'SELECT COUNT(*) FROM [dms].[EffectiveSchema];' + if ($effectiveSchemaCount -lt 1) { throw 'Restored [dms].[EffectiveSchema] is empty.' } + + $documentCount = Invoke-SqlcmdScalar -DatabaseName $VerificationDatabaseName -Query 'SELECT COUNT(*) FROM [dms].[Document];' + if ($documentCount -lt 1) { throw 'Restored [dms].[Document] is empty.' } - $documentCount = Invoke-PsqlScalar -DatabaseName $VerificationDatabaseName -Query 'SELECT COUNT(*) FROM dms."Document";' - if ($documentCount -lt 1) { throw 'Restored dms."Document" is empty.' } + $descriptorCount = Invoke-SqlcmdScalar -DatabaseName $VerificationDatabaseName -Query 'SELECT COUNT(*) FROM [dms].[Descriptor];' + if ($descriptorCount -lt 1) { throw 'Restored [dms].[Descriptor] is empty.' } + } + else { + $effectiveSchemaCount = Invoke-PsqlScalar -DatabaseName $VerificationDatabaseName -Query 'SELECT COUNT(*) FROM dms."EffectiveSchema";' + if ($effectiveSchemaCount -lt 1) { throw 'Restored dms."EffectiveSchema" is empty.' } - $descriptorCount = Invoke-PsqlScalar -DatabaseName $VerificationDatabaseName -Query 'SELECT COUNT(*) FROM dms."Descriptor";' - if ($descriptorCount -lt 1) { throw 'Restored dms."Descriptor" is empty.' } + $documentCount = Invoke-PsqlScalar -DatabaseName $VerificationDatabaseName -Query 'SELECT COUNT(*) FROM dms."Document";' + if ($documentCount -lt 1) { throw 'Restored dms."Document" is empty.' } + + $descriptorCount = Invoke-PsqlScalar -DatabaseName $VerificationDatabaseName -Query 'SELECT COUNT(*) FROM dms."Descriptor";' + if ($descriptorCount -lt 1) { throw 'Restored dms."Descriptor" is empty.' } + } Write-Host "Data assertions passed: EffectiveSchema=$effectiveSchemaCount, Document=$documentCount, Descriptor=$descriptorCount" -ForegroundColor Green @@ -135,7 +175,41 @@ try { $probeSchoolId = $null if ($RequirePopulatedData) { - $populatedDocumentCountQuery = @' + if ($DatabaseEngine -eq "mssql") { + $populatedDocumentCountQuery = @' +SELECT COUNT(*) +FROM [dms].[Document] d +JOIN [dms].[ResourceKey] rk ON rk.[ResourceKeyId] = d.[ResourceKeyId] +WHERE rk.[ResourceName] NOT LIKE '%Descriptor' + AND rk.[ResourceName] NOT LIKE '%SchoolYear%'; +'@ + + $sourcePopulatedDocumentCount = Invoke-SqlcmdScalar -DatabaseName $SourceDatabaseName -Query $populatedDocumentCountQuery + if ($sourcePopulatedDocumentCount -lt 1) { + throw "Source database '$SourceDatabaseName' contains no populated (non-descriptor) documents; the populated bulk load did not land there." + } + + $restoredPopulatedDocumentCount = Invoke-SqlcmdScalar -DatabaseName $VerificationDatabaseName -Query $populatedDocumentCountQuery + if ($restoredPopulatedDocumentCount -ne $sourcePopulatedDocumentCount) { + throw "Restored populated document count ($restoredPopulatedDocumentCount) does not match the source ($sourcePopulatedDocumentCount); the dump dropped populated rows." + } + + $schoolCountQuery = 'SELECT COUNT(*) FROM [edfi].[School];' + + $sourceSchoolCount = Invoke-SqlcmdScalar -DatabaseName $SourceDatabaseName -Query $schoolCountQuery + if ($sourceSchoolCount -lt 1) { + throw "Source database '$SourceDatabaseName' contains no schools; populated sample data is incomplete." + } + + $restoredSchoolCount = Invoke-SqlcmdScalar -DatabaseName $VerificationDatabaseName -Query $schoolCountQuery + if ($restoredSchoolCount -ne $sourceSchoolCount) { + throw "Restored school count ($restoredSchoolCount) does not match the source ($sourceSchoolCount); the dump dropped school rows." + } + + $probeSchoolId = Invoke-SqlcmdScalar -DatabaseName $VerificationDatabaseName -Query 'SELECT TOP 1 [SchoolId] FROM [edfi].[School] ORDER BY [SchoolId];' + } + else { + $populatedDocumentCountQuery = @' SELECT COUNT(*) FROM dms."Document" d JOIN dms."ResourceKey" rk ON rk."ResourceKeyId" = d."ResourceKeyId" @@ -143,42 +217,58 @@ WHERE rk."ResourceName" NOT LIKE '%Descriptor' AND rk."ResourceName" NOT ILIKE '%SchoolYear%'; '@ - $sourcePopulatedDocumentCount = Invoke-PsqlScalar -DatabaseName $SourceDatabaseName -Query $populatedDocumentCountQuery - if ($sourcePopulatedDocumentCount -lt 1) { - throw "Source database '$SourceDatabaseName' contains no populated (non-descriptor) documents; the populated bulk load did not land there." - } + $sourcePopulatedDocumentCount = Invoke-PsqlScalar -DatabaseName $SourceDatabaseName -Query $populatedDocumentCountQuery + if ($sourcePopulatedDocumentCount -lt 1) { + throw "Source database '$SourceDatabaseName' contains no populated (non-descriptor) documents; the populated bulk load did not land there." + } - $restoredPopulatedDocumentCount = Invoke-PsqlScalar -DatabaseName $VerificationDatabaseName -Query $populatedDocumentCountQuery - if ($restoredPopulatedDocumentCount -ne $sourcePopulatedDocumentCount) { - throw "Restored populated document count ($restoredPopulatedDocumentCount) does not match the source ($sourcePopulatedDocumentCount); the dump dropped populated rows." - } + $restoredPopulatedDocumentCount = Invoke-PsqlScalar -DatabaseName $VerificationDatabaseName -Query $populatedDocumentCountQuery + if ($restoredPopulatedDocumentCount -ne $sourcePopulatedDocumentCount) { + throw "Restored populated document count ($restoredPopulatedDocumentCount) does not match the source ($sourcePopulatedDocumentCount); the dump dropped populated rows." + } - $schoolCountQuery = 'SELECT COUNT(*) FROM edfi."School";' + $schoolCountQuery = 'SELECT COUNT(*) FROM edfi."School";' - $sourceSchoolCount = Invoke-PsqlScalar -DatabaseName $SourceDatabaseName -Query $schoolCountQuery - if ($sourceSchoolCount -lt 1) { - throw "Source database '$SourceDatabaseName' contains no schools; populated sample data is incomplete." - } + $sourceSchoolCount = Invoke-PsqlScalar -DatabaseName $SourceDatabaseName -Query $schoolCountQuery + if ($sourceSchoolCount -lt 1) { + throw "Source database '$SourceDatabaseName' contains no schools; populated sample data is incomplete." + } - $restoredSchoolCount = Invoke-PsqlScalar -DatabaseName $VerificationDatabaseName -Query $schoolCountQuery - if ($restoredSchoolCount -ne $sourceSchoolCount) { - throw "Restored school count ($restoredSchoolCount) does not match the source ($sourceSchoolCount); the dump dropped school rows." - } + $restoredSchoolCount = Invoke-PsqlScalar -DatabaseName $VerificationDatabaseName -Query $schoolCountQuery + if ($restoredSchoolCount -ne $sourceSchoolCount) { + throw "Restored school count ($restoredSchoolCount) does not match the source ($sourceSchoolCount); the dump dropped school rows." + } - $probeSchoolId = Invoke-PsqlScalar -DatabaseName $VerificationDatabaseName -Query 'SELECT "SchoolId" FROM edfi."School" ORDER BY "SchoolId" LIMIT 1;' + $probeSchoolId = Invoke-PsqlScalar -DatabaseName $VerificationDatabaseName -Query 'SELECT "SchoolId" FROM edfi."School" ORDER BY "SchoolId" LIMIT 1;' + } Write-Host "Populated data assertions passed: PopulatedDocuments=$restoredPopulatedDocumentCount, Schools=$restoredSchoolCount" -ForegroundColor Green } # --- Serveability probe --- $cmsToken = Get-CmsToken -CmsUrl $CmsUrl + + # Add-DataStore requires a PostgresCredential regardless of target engine; for MSSQL it goes + # unused because -ConnectionString is supplied verbatim from New-DataStoreConnectionString. $postgresCredential = ConvertTo-PostgresCredential -UserName "postgres" -Secret $PostgresPassword + $dataStoreConnectionString = "" + if ($DatabaseEngine -eq "mssql") { + $dataStoreConnectionString = New-DataStoreConnectionString ` + -DatabaseEngine "mssql" ` + -DbHost $ContainerName ` + -Port 1433 ` + -Username "sa" ` + -Password $MssqlPassword ` + -DatabaseName $VerificationDatabaseName + } + $verificationDataStoreId = Add-DataStore ` -CmsUrl $CmsUrl ` -AccessToken $cmsToken ` -PostgresCredential $postgresCredential ` -PostgresDbName $VerificationDatabaseName ` + -ConnectionString $dataStoreConnectionString ` -Name "Template Restore Verification" ` -DataStoreType "Verification" diff --git a/eng/docker-compose/.env.example b/eng/docker-compose/.env.example index 0066f51ce2..97a0be2d06 100644 --- a/eng/docker-compose/.env.example +++ b/eng/docker-compose/.env.example @@ -167,7 +167,7 @@ DMS_CONFIG_DATASTORE=postgresql DMS_CONFIG_DATABASE_CONNECTION_STRING=host=dms-postgresql;port=5432;username=postgres;password=${POSTGRES_PASSWORD};database=${POSTGRES_DB_NAME}; # SQL Server datastore (alternative to PostgreSQL): # DMS_CONFIG_DATASTORE=mssql -# DMS_CONFIG_DATABASE_CONNECTION_STRING=Server=dms-mssql,1433;Database=edfi_configurationservice;User Id=sa;Password=${MSSQL_SA_PASSWORD};TrustServerCertificate=true; +# DMS_CONFIG_DATABASE_CONNECTION_STRING=Server=dms-mssql,1433;Database=${MSSQL_DB_NAME};User Id=sa;Password=${MSSQL_SA_PASSWORD};TrustServerCertificate=true; # MSSQL_SA_PASSWORD=abcdefgh1! # MSSQL_PORT=1435 DMS_CONFIG_DATABASE_ENCRYPTION_KEY=DefaultEncryptionKey32CharactersX1 diff --git a/eng/docker-compose/.env.mssql b/eng/docker-compose/.env.mssql index db57b531e7..9c8035635b 100644 --- a/eng/docker-compose/.env.mssql +++ b/eng/docker-compose/.env.mssql @@ -36,7 +36,7 @@ DATABASE_CONNECTION_STRING_ADMIN=Server=dms-mssql;Database=${MSSQL_DB_NAME};User # -------------- # Config Service # -------------- -# The CMS runs on the same SQL Server instance in its own database, deployed by CMS startup -# (DMS_CONFIG_DEPLOY_DATABASE). Connection-string form matches .env.config.mssql.e2e. +# The CMS shares the DMS datastore database on the same SQL Server instance, deployed by +# CMS startup (DMS_CONFIG_DEPLOY_DATABASE), matching the PostgreSQL shared-database default. DMS_CONFIG_DATASTORE=mssql -DMS_CONFIG_DATABASE_CONNECTION_STRING=Server=dms-mssql,1433;Database=edfi_configurationservice;User Id=sa;Password=${MSSQL_SA_PASSWORD};TrustServerCertificate=true; +DMS_CONFIG_DATABASE_CONNECTION_STRING=Server=dms-mssql,1433;Database=${MSSQL_DB_NAME};User Id=sa;Password=${MSSQL_SA_PASSWORD};TrustServerCertificate=true; diff --git a/eng/docker-compose/README.md b/eng/docker-compose/README.md index 49f0fea41b..97917d547b 100644 --- a/eng/docker-compose/README.md +++ b/eng/docker-compose/README.md @@ -138,14 +138,18 @@ image from source code. ## Running on the MSSQL backend -The local stack can run the DMS datastore on SQL Server instead of PostgreSQL using the -`-DatabaseEngine mssql` switch. Pass it to either `bootstrap-local-dms.ps1` (turnkey) or -`start-local-dms.ps1` (infrastructure) — no `-EnvironmentFile` is required: +The stack can run the DMS datastore on SQL Server instead of PostgreSQL using the +`-DatabaseEngine mssql` switch, on both the local-build and published-image workflows. Pass it to +`bootstrap-local-dms.ps1` / `bootstrap-published-dms.ps1` (turnkey) or `start-local-dms.ps1` / +`start-published-dms.ps1` (infrastructure) - no `-EnvironmentFile` is required: ```pwsh -# Turnkey: stand up SQL Server, provision the relational schema, and (optionally) load seed data +# Turnkey, local build: stand up SQL Server, provision the relational schema, and (optionally) load seed data ./bootstrap-local-dms.ps1 -DatabaseEngine mssql -EnableSwaggerUI -LoadSeedData +# Turnkey, published image: same shape, against the published DMS image +./bootstrap-published-dms.ps1 -DatabaseEngine mssql -EnableSwaggerUI -LoadSeedData + # Tear down (delete volumes) ./start-local-dms.ps1 -DatabaseEngine mssql -d -v ``` @@ -160,14 +164,21 @@ selection. Everything else (`SCHEMA_PACKAGES`, Kafka, Keycloak, identity-provide endpoints, etc.) still comes from the base environment file; pass `-EnvironmentFile` only to override those base settings, and the overlay still composes on top of it. +> [!NOTE] +> **Database topology.** Local deployments host the Configuration Service and DMS in one shared +> physical database on both engines: the CMS `dmscs` schema, and the self-contained (OpenIddict) +> identity stores, live inside the same database as the DMS datastore (`POSTGRES_DB_NAME` on +> PostgreSQL, `MSSQL_DB_NAME` on MSSQL) rather than a separate Configuration Service database. An +> opt-in separate-CMS-database mode is planned (DMS-1270). + A few things are specific to the MSSQL path: * **It is single-engine.** SQL Server hosts everything: the DMS datastore, the Configuration Service (CMS SQL Server backend), and the self-contained (OpenIddict) identity stores. - `mssql.yml` swaps in for `postgresql.yml` — both define the same `db` service that - `local-config.yml` health-gates on — and no PostgreSQL container runs. The DMS datastore - database is created by the provision phase; the CMS database (`edfi_configurationservice`) - is created by the identity init / CMS startup deploy. + `mssql.yml` swaps in for `postgresql.yml` - both define the same `db` service that + `local-config.yml` health-gates on - and no PostgreSQL container runs. The shared database is + created by the provision phase; CMS deploys its `dmscs` schema into that same database on + startup (see "Database topology" above). * **Relational backend only.** MSSQL is supported through the relational backend (`DMS_DATASTORE=mssql`). Schema is provisioned by `provision-dms-schema.ps1`, which auto-detects the SQL Server dialect from the data-store connection string and invokes @@ -176,6 +187,24 @@ A few things are specific to the MSSQL path: SQL, so Kafka, OpenSearch, and the Debezium source connector are not started on this path. * **Seed data** uses the same API-based `-LoadSeedData` (BulkLoadClient) path as PostgreSQL; it is database-engine agnostic. +* **CI publishes database-template packages for both engines.** `build-minimal-template.yml` and + `build-populated-template.yml` are reusable workflows parameterized by `database_engine` + (`postgresql` or `mssql`); the `EdFi.Api.Minimal.Template.MsSql.yml` and + `EdFi.Api.Populated.Template.MsSql.yml` caller workflows invoke them with + `database_engine: mssql` to build and publish `EdFi.Api.Minimal.Template.MsSql.*` and + `EdFi.Api.Populated.Template.MsSql.*` NuGet packages alongside the existing PostgreSQL ones. + On MSSQL the package's data dump is a native `BACKUP DATABASE` `.bak` file (restored with + `RESTORE DATABASE`); that `.bak` is coupled to the pinned + `mcr.microsoft.com/mssql/server:2022-latest` image it was built and verified against, the same + way the PostgreSQL `.sql` dump is coupled to the PostgreSQL major version it was built against. + Every published package is restore-verified in CI + (`eng/DatabaseTemplates/verify-template-restore.ps1`) before publishing. Base environment files' + `DATABASE_TEMPLATE_PACKAGE` value (e.g. `EdFi.Api.Populated.Template.PostgreSql.5.2.0`) carries + the PostgreSQL package id; composing the `.env.mssql` overlay for `-DatabaseEngine mssql` + rewrites only its engine segment to `MsSql` (`Convert-TemplatePackageToken` in + `env-utility.psm1`), so the same base `-EnvironmentFile` names the matching package id for + either engine. This section documents the CI build/publish/verify pipeline only; no local + bootstrap flow currently restores these packages. After the stack is up, run the smoke tests the same way as for PostgreSQL: @@ -314,10 +343,15 @@ Use the `bootstrap-local-dms.ps1` wrapper as above, or run the phases manually: ``` Each of the four commands above accepts `-DatabaseEngine mssql` (default `postgresql`); pass it -consistently on all four to run this manual flow against the SQL Server datastore — every phase +consistently on all four to run this manual flow against the SQL Server datastore - every phase composes the same `.env.mssql` overlay (see "Running on the MSSQL backend" above), so a mismatched flag on any one command leaves that phase reading the wrong engine. +`start-local-dms.ps1` and `start-published-dms.ps1` also accept a narrower `-DbOnly` switch that +starts only the database container and waits for it to become ready, then stops. It is mutually +exclusive with `-InfraOnly` and `-DmsOnly` and is not part of this four-command flow; it exists +for diagnostics and for other tooling to sequence a database-only startup around. + Expert mode (`-ApiSchemaPath`) and standard mode (omit `-ApiSchemaPath`) are the two schema-selection paths; there is no `-Extensions` parameter. diff --git a/eng/docker-compose/bootstrap-published-dms.ps1 b/eng/docker-compose/bootstrap-published-dms.ps1 index e12b061075..28105b802b 100644 --- a/eng/docker-compose/bootstrap-published-dms.ps1 +++ b/eng/docker-compose/bootstrap-published-dms.ps1 @@ -71,6 +71,12 @@ `start-published-dms.ps1`; when seed loading is requested, every year in the range is also passed to the seed phase via `-SchoolYear`. +.PARAMETER DatabaseEngine + Database engine for the whole stack ("postgresql" or "mssql"). Forwarded to + `start-published-dms.ps1`, which swaps mssql.yml in for postgresql.yml: SQL Server then + hosts the DMS datastore, the Configuration Service (CMS SQL Server backend), and the + self-contained OpenIddict identity stores. + .EXAMPLE pwsh ./bootstrap-published-dms.ps1 Standard mode, core only. Stages the core ApiSchema package and claims in-line (when no @@ -116,6 +122,16 @@ param( [string]$SchoolYearRange = "", + # Database engine for the whole stack. "mssql" swaps mssql.yml in for postgresql.yml: + # SQL Server hosts the DMS datastore (relational backend), the Configuration Service + # (CMS SQL Server backend), and the self-contained OpenIddict identity stores - no + # PostgreSQL container runs. Forwarded to start-published-dms.ps1. The .env.mssql overlay + # (DMS_DATASTORE=mssql, DMS_CONFIG_DATASTORE=mssql, the MSSQL_* keys, and the SQL Server + # connection strings) is composed automatically onto -EnvironmentFile, so no + # -EnvironmentFile is needed for a turnkey MSSQL deploy. + [ValidateSet("postgresql", "mssql")] + [string]$DatabaseEngine = "postgresql", + # Data standard version for the local-bootstrap package surface. The .env.bootstrap. # overlay (DS 5.2, the default: core + TPDM; DS 6.1: core only, since TPDM is folded into # core in 6.1) is composed onto -EnvironmentFile ONLY when this parameter is explicitly diff --git a/eng/docker-compose/bootstrap-wrapper.psm1 b/eng/docker-compose/bootstrap-wrapper.psm1 index 2bb81c3c5e..3a8751cd2f 100644 --- a/eng/docker-compose/bootstrap-wrapper.psm1 +++ b/eng/docker-compose/bootstrap-wrapper.psm1 @@ -137,6 +137,218 @@ function Test-WrapperManifestClaimsStaged { } } +function Get-WrapperManifestSchemaIdentity { + <# + .SYNOPSIS + Reads the staged bootstrap manifest's schema.selectionMode and schema.selectedExtensions - the + two schema-section fields that together identify which package set (if any) drove a Standard-mode + staged workspace. Returns $null when the manifest is missing, empty, malformed, unreadable, or + lacks schema.selectionMode. + + Bare JSON parse keeps this independent of bootstrap-manifest.psm1 in sandboxed Pester invocations + (mirrors Test-WrapperManifestClaimsStaged). Property presence is tested via PSObject.Properties so + the check is safe regardless of Set-StrictMode. + #> + param( + [Parameter(Mandatory)] + [string] + $ManifestPath + ) + + if (-not (Test-Path -LiteralPath $ManifestPath -PathType Leaf)) { + return $null + } + + try { + $rawManifestContent = Get-Content -LiteralPath $ManifestPath -Raw -ErrorAction Stop + if ([string]::IsNullOrWhiteSpace($rawManifestContent)) { + return $null + } + + $parsedManifest = $rawManifestContent | ConvertFrom-Json -ErrorAction Stop + if ($null -eq $parsedManifest) { + return $null + } + + $schemaProperty = $parsedManifest.PSObject.Properties['schema'] + if ($null -eq $schemaProperty -or $null -eq $schemaProperty.Value) { + return $null + } + + $selectionModeProperty = $schemaProperty.Value.PSObject.Properties['selectionMode'] + if ($null -eq $selectionModeProperty -or [string]::IsNullOrWhiteSpace([string]$selectionModeProperty.Value)) { + return $null + } + + # $null here (rather than an empty array) means the property is absent entirely, so + # Test-WrapperManifestSchemaPackagesCurrent can tell "no extensions recorded, nothing to + # compare" apart from "recorded as an explicit empty set". The assignment is split into + # its own statement (rather than the branch value of an if/else expression) because + # PowerShell flattens an empty-array branch value of a captured if/else expression to + # $null; a plain `$selectedExtensions = @(...)` assignment does not. + $selectedExtensionsProperty = $schemaProperty.Value.PSObject.Properties['selectedExtensions'] + $selectedExtensions = $null + if ($null -ne $selectedExtensionsProperty -and $null -ne $selectedExtensionsProperty.Value) { + $selectedExtensions = @($selectedExtensionsProperty.Value | ForEach-Object { [string]$_ }) + } + + return [pscustomobject]@{ + SelectionMode = [string]$selectionModeProperty.Value + SelectedExtensions = $selectedExtensions + } + } + catch { + return $null + } +} + +function Get-WrapperEffectiveSchemaExtension { + <# + .SYNOPSIS + Derives the extension identity set (lowercased project-endpoint tokens) implied by the effective + env file's SCHEMA_PACKAGES value, using the same core-vs-extension naming convention + prepare-dms-schema.ps1's SCHEMA_PACKAGES-driven staging (Invoke-SchemaPackagesModeSchemaStaging) + already relies on: EdFi.DataStandard.ApiSchema identifies the core package; every other + EdFi.DataStandard..ApiSchema entry is an extension whose endpoint token is the + segment, lowercased - the same token prepare-dms-schema.ps1 records (from the staged + package's own declared projectEndpointName) into schema.selectedExtensions, so the two sides are + directly comparable without downloading or hashing anything. + + A bare regex extraction (mirroring schema-package-utility.psm1's Get-QuotedEnvJson) keeps this + independent of that sibling module in sandboxed Pester invocations. An env file with no + SCHEMA_PACKAGES key, or an unparsable value, implies the catalog-pinned core-only default that + prepare-dms-schema.ps1 itself falls back to, so it returns an empty set rather than throwing. + #> + param( + [Parameter(Mandatory)] + [string] + $EnvironmentFile + ) + + $expectedExtensions = [System.Collections.Generic.List[string]]::new() + + if (-not (Test-Path -LiteralPath $EnvironmentFile -PathType Leaf)) { + return $expectedExtensions.ToArray() + } + + $environmentFileContent = Get-Content -LiteralPath $EnvironmentFile -Raw + $schemaPackagesMatch = [System.Text.RegularExpressions.Regex]::Match( + $environmentFileContent, + "(?ms)^[ \t]*SCHEMA_PACKAGES='(?\[.*?\])'" + ) + if (-not $schemaPackagesMatch.Success) { + return $expectedExtensions.ToArray() + } + + try { + $schemaPackages = @($schemaPackagesMatch.Groups["value"].Value | ConvertFrom-Json -ErrorAction Stop) + } + catch { + return $expectedExtensions.ToArray() + } + + $corePackageIdPattern = '^EdFi\.DataStandard\d+\.ApiSchema$' + $extensionPackageIdPattern = '^EdFi\.DataStandard\d+\.(?.+)\.ApiSchema$' + + foreach ($schemaPackage in $schemaPackages) { + $packageName = [string]$schemaPackage.name + if ([string]::IsNullOrWhiteSpace($packageName) -or $packageName -match $corePackageIdPattern) { + continue + } + + $extensionMatch = [System.Text.RegularExpressions.Regex]::Match($packageName, $extensionPackageIdPattern) + if ($extensionMatch.Success) { + $null = $expectedExtensions.Add($extensionMatch.Groups["extension"].Value.ToLowerInvariant()) + } + } + + return $expectedExtensions.ToArray() +} + +function Test-WrapperManifestSchemaPackagesCurrent { + <# + .SYNOPSIS + Returns $true when the staged bootstrap manifest's recorded schema-package identity still + matches the effective env file's SCHEMA_PACKAGES value, so the wrapper can safely reuse the + staged schema workspace instead of re-running prepare-dms-schema.ps1. + + The wrapper uses this to close a staging-reuse gap: a workspace staged for one package set (e.g. + a DS 6.1 core-only SCHEMA_PACKAGES value) could otherwise be silently served under a later + invocation whose effective env now selects a different set (e.g. DS 5.2 core + TPDM), with no + error, until something downstream (e.g. a populated template with zero rows) surfaces the + mismatch. Comparing schema.selectedExtensions - the project-endpoint tokens prepare-dms-schema.ps1 + itself records for every staged extension - against the extension identity implied by the + effective SCHEMA_PACKAGES value closes that gap without re-downloading or re-hashing anything. + + Only schema.selectionMode "Standard" (package-backed) manifests are compared: "ApiSchemaPath" + (expert filesystem) manifests are not driven by SCHEMA_PACKAGES at all, so forcing a comparison + against it would treat an intentional manual/expert override as a mismatch and discard a + hand-staged workspace that may carry custom extensions. Those are always reported as matching + (reuse as-is), consistent with the staging block's existing "reuse an already-staged workspace + as-is" contract. A Standard-mode manifest that has no schema.selectedExtensions recorded at all + (rather than an explicit, possibly-empty array) has nothing to compare either, and is likewise + reported as matching. + + A missing, empty, malformed, or unreadable manifest, or one missing schema.selectionMode, is + reported as "not matching" so the staging phase re-runs prepare-dms-schema.ps1, which then owns + the authoritative re-stage (and surfaces its own error, if any) - mirroring how + Test-WrapperManifestClaimsStaged treats a malformed manifest as "claims not staged". + + .PARAMETER ManifestPath + Path to the staged bootstrap-manifest.json. + + .PARAMETER EnvironmentFile + Path to the effective env file whose SCHEMA_PACKAGES value is the comparison target. + #> + param( + [Parameter(Mandatory)] + [string] + $ManifestPath, + + [Parameter(Mandatory)] + [string] + $EnvironmentFile + ) + + $manifestSchemaIdentity = Get-WrapperManifestSchemaIdentity -ManifestPath $ManifestPath + if ($null -eq $manifestSchemaIdentity) { + return $false + } + + if ($manifestSchemaIdentity.SelectionMode -ne "Standard") { + return $true + } + + if ($null -eq $manifestSchemaIdentity.SelectedExtensions) { + return $true + } + + try { + $stagedExtensions = [System.Collections.Generic.HashSet[string]]::new( + [System.StringComparer]::OrdinalIgnoreCase + ) + foreach ($stagedExtension in $manifestSchemaIdentity.SelectedExtensions) { + $null = $stagedExtensions.Add($stagedExtension) + } + + $expectedExtensions = @(Get-WrapperEffectiveSchemaExtension -EnvironmentFile $EnvironmentFile) + if ($stagedExtensions.Count -ne $expectedExtensions.Count) { + return $false + } + + foreach ($expectedExtension in $expectedExtensions) { + if (-not $stagedExtensions.Contains($expectedExtension)) { + return $false + } + } + + return $true + } + catch { + return $false + } +} + function Resolve-WrapperEnvironmentFilePath { <# .SYNOPSIS @@ -307,8 +519,8 @@ function Invoke-BootstrapWrapper { [string]$DmsBaseUrl, # Database engine for the DMS datastore ("postgresql" or "mssql"). Forwarded to the - # configure phase always, and to the start phases only for start-local-dms.ps1 (mssql.yml - # is a local-only tier; start-published-dms.ps1 has no -DatabaseEngine parameter). + # configure phase and to the start phase, both of which understand -DatabaseEngine + # regardless of whether the target is start-local-dms.ps1 or start-published-dms.ps1. [ValidateSet("postgresql", "mssql")] [string]$DatabaseEngine = "postgresql", @@ -332,11 +544,6 @@ function Invoke-BootstrapWrapper { $ErrorActionPreference = "Stop" - # mssql.yml is a local-only datastore tier. Only start-local-dms.ps1 understands - # -DatabaseEngine; start-published-dms.ps1 does not, so the engine is forwarded to the start - # phases only for the local start script. The configure phase always accepts it. - $startScriptSupportsDatabaseEngine = ($StartScriptName -eq "start-local-dms.ps1") - # Fail fast: IDE workflow shape parameter validation — runs before any phase invocation. # -DmsBaseUrl is only valid with -InfraOnly; reject it without -InfraOnly so a misuse # never reaches Docker or CMS state. @@ -508,11 +715,12 @@ function Invoke-BootstrapWrapper { # Schema/claims staging phase. The standard happy path needs no manual pre-staging # (bootstrap-design.md Section 9.4.1): when no workspace is staged yet, stage core-only standard # mode so a clean checkout runs `bootstrap-local-dms.ps1` with no preceding prepare step. When a - # schema workspace is already staged (a manual/expert prepare flow, or a prior run), reuse it - # as-is rather than rewriting a workspace that may still be bind-mounted into a running stack. - # There is no -Extensions parameter; extension/custom schema sets are staged via expert - # -ApiSchemaPath before invoking the wrapper. prepare-dms-schema.ps1 owns all validation and the - # rerun contract. + # schema workspace is already staged (a manual/expert prepare flow, or a prior run), it is reused + # as-is ONLY when it is still current for the effective env's SCHEMA_PACKAGES (see + # Test-WrapperManifestSchemaPackagesCurrent below) rather than unconditionally trusting a + # workspace that may still be bind-mounted into a running stack. There is no -Extensions + # parameter; extension/custom schema sets are staged via expert -ApiSchemaPath before invoking + # the wrapper. prepare-dms-schema.ps1 owns all validation and the rerun contract. # # Claims completion is staged whenever the manifest lacks the claims/seed sections: both after a # fresh schema stage above, and when a pre-existing manifest carries schema but not claims/seed @@ -528,17 +736,41 @@ function Invoke-BootstrapWrapper { $stagedManifestPath = Join-Path $PSScriptRoot ".bootstrap/bootstrap-manifest.json" $stagedManifestPresent = Test-Path -LiteralPath $stagedManifestPath -PathType Leaf + # A present manifest is reused only when its recorded schema-package identity still matches + # the effective env's SCHEMA_PACKAGES value (Test-WrapperManifestSchemaPackagesCurrent). + # Without this, a workspace staged for one Data Standard / package set could be silently + # served under a later invocation whose effective env selects a different one, with no error + # until something downstream (e.g. a populated template with zero rows) surfaces it. + $stagedManifestCurrent = $stagedManifestPresent -and + (Test-WrapperManifestSchemaPackagesCurrent -ManifestPath $stagedManifestPath -EnvironmentFile $effectiveEnvFile) + + if ($stagedManifestPresent -and -not $stagedManifestCurrent) { + $staleSchemaIdentity = Get-WrapperManifestSchemaIdentity -ManifestPath $stagedManifestPath + # Split into its own statement (rather than an if/else expression branch value): PowerShell + # flattens an empty-array branch value of a captured if/else expression to $null. + $staleExtensions = @() + if ($null -ne $staleSchemaIdentity -and $null -ne $staleSchemaIdentity.SelectedExtensions) { + $staleExtensions = @($staleSchemaIdentity.SelectedExtensions) + } + $effectiveExtensions = @(Get-WrapperEffectiveSchemaExtension -EnvironmentFile $effectiveEnvFile) + Write-Information "Staged bootstrap schema workspace does not match the effective environment: staged extensions [$($staleExtensions -join ', ')] vs effective SCHEMA_PACKAGES extensions [$($effectiveExtensions -join ', ')]. Re-running prepare-dms-schema.ps1 -EnvironmentFile $effectiveEnvFile to re-stage (or surface its divergence guidance if the existing workspace must be removed first)." -InformationAction Continue + } + # Reset the native exit-code sentinel before each prepare invocation (same pattern as the # start/configure/provision phases below). prepare-dms-*.ps1 signal failure by throwing and may # run no native command, so a stale nonzero $LASTEXITCODE left by an earlier command in the # session would otherwise make a successful staging step throw a false "failed with exit code" # before infrastructure starts. - if ((Test-Path -LiteralPath $prepareSchemaScript) -and -not $stagedManifestPresent) { + if ((Test-Path -LiteralPath $prepareSchemaScript) -and (-not $stagedManifestPresent -or -not $stagedManifestCurrent)) { $global:LASTEXITCODE = 0 # Forward the same effective env file used by the other phases so standard-mode staging # can drive itself from its SCHEMA_PACKAGES value (core plus any extensions) instead of # the catalog-pinned core-only default. This keeps the staged workspace's effective schema - # hash in sync with what the DMS container entrypoint resolves from the same env file. + # hash in sync with what the DMS container entrypoint resolves from the same env file. When + # the staged manifest is present but stale for the effective SCHEMA_PACKAGES value, + # prepare-dms-schema.ps1's own divergence check throws its remove-.bootstrap guidance on a + # true mismatch instead of silently re-staging over a workspace that may still be + # bind-mounted into a running stack. & $prepareSchemaScript -EnvironmentFile $effectiveEnvFile if ($LASTEXITCODE -is [int] -and $LASTEXITCODE -ne 0) { throw "prepare-dms-schema.ps1 failed with exit code $LASTEXITCODE." @@ -565,7 +797,7 @@ function Invoke-BootstrapWrapper { if ($EnableKafkaUI) { $startArgs.EnableKafkaUI = $true } if ($EnableSwaggerUI) { $startArgs.EnableSwaggerUI = $true } if ($AddExtensionSecurityMetadata) { $startArgs.AddExtensionSecurityMetadata = $true } - if ($startScriptSupportsDatabaseEngine) { $startArgs.DatabaseEngine = $DatabaseEngine } + $startArgs.DatabaseEngine = $DatabaseEngine $startArgs.EnvironmentFile = $effectiveEnvFile # Reset the native exit-code sentinel so the check below reflects only this start invocation and @@ -683,7 +915,7 @@ function Invoke-BootstrapWrapper { if ($EnableKafkaUI) { $healthWaitArgs.EnableKafkaUI = $true } if ($EnableSwaggerUI) { $healthWaitArgs.EnableSwaggerUI = $true } if ($AddExtensionSecurityMetadata) { $healthWaitArgs.AddExtensionSecurityMetadata = $true } - if ($startScriptSupportsDatabaseEngine) { $healthWaitArgs.DatabaseEngine = $DatabaseEngine } + $healthWaitArgs.DatabaseEngine = $DatabaseEngine & "$PSScriptRoot/$StartScriptName" @healthWaitArgs if ($LASTEXITCODE -is [int] -and $LASTEXITCODE -ne 0) { @@ -722,7 +954,7 @@ function Invoke-BootstrapWrapper { if ($EnableKafkaUI) { $dmsStartArgs.EnableKafkaUI = $true } if ($EnableSwaggerUI) { $dmsStartArgs.EnableSwaggerUI = $true } if ($AddExtensionSecurityMetadata) { $dmsStartArgs.AddExtensionSecurityMetadata = $true } - if ($startScriptSupportsDatabaseEngine) { $dmsStartArgs.DatabaseEngine = $DatabaseEngine } + $dmsStartArgs.DatabaseEngine = $DatabaseEngine & "$PSScriptRoot/$StartScriptName" @dmsStartArgs if ($LASTEXITCODE -is [int] -and $LASTEXITCODE -ne 0) { diff --git a/eng/docker-compose/env-utility.psm1 b/eng/docker-compose/env-utility.psm1 index adfd372076..5b171ffeda 100644 --- a/eng/docker-compose/env-utility.psm1 +++ b/eng/docker-compose/env-utility.psm1 @@ -650,6 +650,48 @@ function Resolve-DataStandardEnvironmentFile { -TargetPath $derivedPath } +function Convert-TemplatePackageToken { + <# + .SYNOPSIS + Rewrites the engine segment of a DATABASE_TEMPLATE_PACKAGE-shaped package id, leaving + every other segment (including the template and version) untouched. + + .DESCRIPTION + Package ids follow the shape .