From 6cce66d1151f16dc128214947e733aa6d7b7d1b5 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:05:59 +0200 Subject: [PATCH 01/11] feat(storage): cut over Azure Blob to provider-agnostic S3/MinIO storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace @azure/storage-blob with a provider-agnostic StorageService backed by S3StorageService (MinIO via @aws-sdk/client-s3), a MockStorageService for local dev, and a createStorageService() factory. blobUrl/blobName live on the base class so the URL shape stays reversible and host-independent: blobName() decodes both the legacy Azure-host and the new MinIO-host URLs to the same container-relative key, which keeps DB-persisted URLs valid across the migration. Add the GeBüV anchoring pipeline (Merkle + OpenTimestamps + WORM bucket provisioning) under storage/anchoring with the AddArchiveAnchoring migration, and record content hashes of KYC documents and EP2 settlement reports on upload as a best-effort side-booking (failures are logged, never roll back the upload). Swap the config azure.storage block for an s3 block, delete azure-storage.service.ts, and migrate all consumers (kyc-document, support-document, fiat-output-job) plus the BlobContent/Blob importers grown on develop to the new module. Restore KycDocumentService.toHostStableUrl on the storage-backed service via the shared StorageService.encodePath so served URLs stay byte-identical apart from the host. Also make the GwG compliance download (Handelsregisterauszug, Vollmacht) select KYC documents by their host-independent container-relative key instead of full-URL string equality, so pre-cutover documents whose stored Azure-host URL no longer equals the live MinIO-host URL are no longer silently dropped from the compliance ZIP. --- .../1781788436511-AddArchiveAnchoring.js | 34 + package-lock.json | 961 +++++++++++------- package.json | 7 +- scripts/kyc/kyc-storage.js | 2 +- scripts/storage/provision-bucket.ts | 135 +++ src/config/config.ts | 53 +- .../infrastructure/azure-storage.service.ts | 211 ---- .../__tests__/mock-storage.service.spec.ts | 132 +++ .../__tests__/s3-storage.service.spec.ts | 264 +++++ .../storage/__tests__/storage.factory.spec.ts | 57 ++ .../storage/__tests__/storage.service.spec.ts | 63 ++ .../__tests__/archive.scheduler.spec.ts | 58 ++ .../__tests__/archive.service.spec.ts | 345 +++++++ .../anchoring/__tests__/merkle.spec.ts | 112 ++ .../__tests__/opentimestamps.service.spec.ts | 176 ++++ .../storage/anchoring/archive-batch.entity.ts | 33 + .../anchoring/archive-batch.repository.ts | 11 + .../storage/anchoring/archive-file.entity.ts | 30 + .../anchoring/archive-file.repository.ts | 11 + .../storage/anchoring/archive.module.ts | 18 + .../storage/anchoring/archive.scheduler.ts | 46 + .../storage/anchoring/archive.service.ts | 196 ++++ .../storage/anchoring/merkle.ts | 114 +++ .../anchoring/opentimestamps.service.ts | 91 ++ .../storage/mock-storage.service.ts | 101 ++ .../storage/s3-storage.service.ts | 117 +++ .../infrastructure/storage/storage.factory.ts | 23 + .../infrastructure/storage/storage.service.ts | 59 ++ src/shared/services/process.service.ts | 2 + .../generic/gs/__tests__/gs.service.spec.ts | 17 +- .../generic/kyc/dto/kyc-file.dto.ts | 2 +- .../generic/kyc/dto/mapper/kyc-file.mapper.ts | 2 +- src/subdomains/generic/kyc/kyc.module.ts | 2 + .../services/__tests__/kyc.service.spec.ts | 6 +- .../__tests__/kyc-document.service.spec.ts | 100 ++ .../integration/kyc-document.service.ts | 41 +- .../__tests__/user-data.service.spec.ts | 4 + .../__tests__/fiat-output-job.service.spec.ts | 109 ++ .../fiat-output/fiat-output-job.service.ts | 21 +- .../fiat-output/fiat-output.module.ts | 2 + .../realunit-compliance.service.spec.ts | 6 +- .../realunit-support.controller.ts | 2 +- .../services/support-document.service.ts | 7 +- .../services/support-issue.service.ts | 2 +- .../support-issue/support-issue.controller.ts | 2 +- 45 files changed, 3168 insertions(+), 619 deletions(-) create mode 100644 migration/1781788436511-AddArchiveAnchoring.js create mode 100644 scripts/storage/provision-bucket.ts delete mode 100644 src/integration/infrastructure/azure-storage.service.ts create mode 100644 src/integration/infrastructure/storage/__tests__/mock-storage.service.spec.ts create mode 100644 src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts create mode 100644 src/integration/infrastructure/storage/__tests__/storage.factory.spec.ts create mode 100644 src/integration/infrastructure/storage/__tests__/storage.service.spec.ts create mode 100644 src/integration/infrastructure/storage/anchoring/__tests__/archive.scheduler.spec.ts create mode 100644 src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts create mode 100644 src/integration/infrastructure/storage/anchoring/__tests__/merkle.spec.ts create mode 100644 src/integration/infrastructure/storage/anchoring/__tests__/opentimestamps.service.spec.ts create mode 100644 src/integration/infrastructure/storage/anchoring/archive-batch.entity.ts create mode 100644 src/integration/infrastructure/storage/anchoring/archive-batch.repository.ts create mode 100644 src/integration/infrastructure/storage/anchoring/archive-file.entity.ts create mode 100644 src/integration/infrastructure/storage/anchoring/archive-file.repository.ts create mode 100644 src/integration/infrastructure/storage/anchoring/archive.module.ts create mode 100644 src/integration/infrastructure/storage/anchoring/archive.scheduler.ts create mode 100644 src/integration/infrastructure/storage/anchoring/archive.service.ts create mode 100644 src/integration/infrastructure/storage/anchoring/merkle.ts create mode 100644 src/integration/infrastructure/storage/anchoring/opentimestamps.service.ts create mode 100644 src/integration/infrastructure/storage/mock-storage.service.ts create mode 100644 src/integration/infrastructure/storage/s3-storage.service.ts create mode 100644 src/integration/infrastructure/storage/storage.factory.ts create mode 100644 src/integration/infrastructure/storage/storage.service.ts create mode 100644 src/subdomains/generic/kyc/services/integration/__tests__/kyc-document.service.spec.ts diff --git a/migration/1781788436511-AddArchiveAnchoring.js b/migration/1781788436511-AddArchiveAnchoring.js new file mode 100644 index 0000000000..4c40c6ccce --- /dev/null +++ b/migration/1781788436511-AddArchiveAnchoring.js @@ -0,0 +1,34 @@ +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + * @typedef {import('typeorm').QueryRunner} QueryRunner + */ + +/** + * @class + * @implements {MigrationInterface} + */ +module.exports = class AddArchiveAnchoring1781788436511 { + name = 'AddArchiveAnchoring1781788436511' + + /** + * @param {QueryRunner} queryRunner + */ + async up(queryRunner) { + await queryRunner.query(`CREATE TABLE "archive_batch" ("id" SERIAL NOT NULL, "updated" TIMESTAMP NOT NULL DEFAULT now(), "created" TIMESTAMP NOT NULL DEFAULT now(), "merkleRoot" character varying(64) NOT NULL, "otsProof" text, "bitcoinHeight" integer, "status" character varying(256) NOT NULL DEFAULT 'pendingBtc', CONSTRAINT "PK_06f898016522a01c619a214aa18" PRIMARY KEY ("id"))`); + await queryRunner.query(`CREATE TABLE "archive_file" ("id" SERIAL NOT NULL, "updated" TIMESTAMP NOT NULL DEFAULT now(), "created" TIMESTAMP NOT NULL DEFAULT now(), "bucket" character varying(256) NOT NULL, "name" character varying(256) NOT NULL, "sha256" character varying(64) NOT NULL, "leafIndex" integer, "batchId" integer, CONSTRAINT "PK_17e252452a46a911a66d67dd50d" PRIMARY KEY ("id"))`); + await queryRunner.query(`CREATE UNIQUE INDEX "IDX_e7da94bafd8e348ead75dcb000" ON "archive_file" ("bucket", "name") `); + await queryRunner.query(`CREATE INDEX "IDX_4065eeb67cf015dc7327499de9" ON "archive_file" ("batchId") `); + await queryRunner.query(`ALTER TABLE "archive_file" ADD CONSTRAINT "FK_4065eeb67cf015dc7327499de94" FOREIGN KEY ("batchId") REFERENCES "archive_batch"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); + } + + /** + * @param {QueryRunner} queryRunner + */ + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "archive_file" DROP CONSTRAINT "FK_4065eeb67cf015dc7327499de94"`); + await queryRunner.query(`DROP INDEX "public"."IDX_4065eeb67cf015dc7327499de9"`); + await queryRunner.query(`DROP INDEX "public"."IDX_e7da94bafd8e348ead75dcb000"`); + await queryRunner.query(`DROP TABLE "archive_file"`); + await queryRunner.query(`DROP TABLE "archive_batch"`); + } +} diff --git a/package-lock.json b/package-lock.json index 591560a8cb..b6f1df5dca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@arbitrum/sdk": "^3.7.3", "@arkade-os/sdk": "^0.4.36", - "@azure/storage-blob": "^12.29.1", + "@aws-sdk/client-s3": "^3.1071.0", "@blockfrost/blockfrost-js": "^6.1.0", "@buildonspark/spark-sdk": "^0.6.7", "@cardano-foundation/cardano-verify-datasignature": "^1.0.11", @@ -98,6 +98,7 @@ "node-2fa": "^2.0.3", "node-pty": "^1.1.0", "nodemailer": "^6.10.1", + "opentimestamps": "^0.4.9", "passport": "^0.6.0", "passport-jwt": "^4.0.1", "pdfkit": "^0.15.2", @@ -136,6 +137,7 @@ "@types/passport-jwt": "^3.0.13", "@types/pdfkit": "^0.13.9", "@types/supertest": "^2.0.16", + "aws-sdk-client-mock": "^4.1.0", "eslint": "^9.17.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-prettier": "^5.2.1", @@ -560,68 +562,55 @@ "node": ">=14.0.0" } }, - "node_modules/@aws-sdk/client-ses": { - "version": "3.982.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-ses/-/client-ses-3.982.0.tgz", - "integrity": "sha512-FMfZsrdevWomqEwLWaW5Jfq+8jRbROQe8sbEANVTNPYBfXvnd8TxPs/09h7TgFjtSB7hsjUv8Ja6IjeMV5HHPA==", - "dev": true, + "node_modules/@aws-sdk/checksums": { + "version": "3.1000.16", + "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.16.tgz", + "integrity": "sha512-EKnvkXSmz3IpA99tCNuI+dLFXyZyClSm8zns9sB/elvkU+MTuomAs6toJMPMBf98/fICG/urXDkzGz0/c3yyAQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/credential-provider-node": "^3.972.5", - "@aws-sdk/middleware-host-header": "^3.972.3", - "@aws-sdk/middleware-logger": "^3.972.3", - "@aws-sdk/middleware-recursion-detection": "^3.972.3", - "@aws-sdk/middleware-user-agent": "^3.972.6", - "@aws-sdk/region-config-resolver": "^3.972.3", - "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.982.0", - "@aws-sdk/util-user-agent-browser": "^3.972.3", - "@aws-sdk/util-user-agent-node": "^3.972.4", - "@smithy/config-resolver": "^4.4.6", - "@smithy/core": "^3.22.0", - "@smithy/fetch-http-handler": "^5.3.9", - "@smithy/hash-node": "^4.2.8", - "@smithy/invalid-dependency": "^4.2.8", - "@smithy/middleware-content-length": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/middleware-retry": "^4.4.29", - "@smithy/middleware-serde": "^4.2.9", - "@smithy/middleware-stack": "^4.2.8", - "@smithy/node-config-provider": "^4.3.8", - "@smithy/node-http-handler": "^4.4.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", - "@smithy/types": "^4.12.0", - "@smithy/url-parser": "^4.2.8", - "@smithy/util-base64": "^4.3.0", - "@smithy/util-body-length-browser": "^4.2.0", - "@smithy/util-body-length-node": "^4.2.1", - "@smithy/util-defaults-mode-browser": "^4.3.28", - "@smithy/util-defaults-mode-node": "^4.2.31", - "@smithy/util-endpoints": "^3.2.8", - "@smithy/util-middleware": "^4.2.8", - "@smithy/util-retry": "^4.2.8", - "@smithy/util-utf8": "^4.2.0", - "@smithy/util-waiter": "^4.2.8", + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/types": "^4.16.0", "tslib": "^2.6.2" }, "engines": { "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/client-sso": { + "node_modules/@aws-sdk/client-s3": { + "version": "3.1085.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1085.0.tgz", + "integrity": "sha512-O0xe8sR50AYkwxlvRRsV0qytEO2dtXQTQ1CF3YBBdE5xtVkbu27H0vGa1mjQi1/+fbYM80AWEIPai5jZmXyubw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/checksums": "^3.1000.16", + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/credential-provider-node": "^3.972.66", + "@aws-sdk/middleware-sdk-s3": "^3.972.62", + "@aws-sdk/signature-v4-multi-region": "^3.996.39", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/fetch-http-handler": "^5.6.4", + "@smithy/node-http-handler": "^4.9.4", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-ses": { "version": "3.982.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.982.0.tgz", - "integrity": "sha512-qJrIiivmvujdGqJ0ldSUvhN3k3N7GtPesoOI1BSt0fNXovVnMz4C/JmnkhZihU7hJhDvxJaBROLYTU+lpild4w==", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-ses/-/client-ses-3.982.0.tgz", + "integrity": "sha512-FMfZsrdevWomqEwLWaW5Jfq+8jRbROQe8sbEANVTNPYBfXvnd8TxPs/09h7TgFjtSB7hsjUv8Ja6IjeMV5HHPA==", "dev": true, "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.973.6", + "@aws-sdk/credential-provider-node": "^3.972.5", "@aws-sdk/middleware-host-header": "^3.972.3", "@aws-sdk/middleware-logger": "^3.972.3", "@aws-sdk/middleware-recursion-detection": "^3.972.3", @@ -656,6 +645,7 @@ "@smithy/util-middleware": "^4.2.8", "@smithy/util-retry": "^4.2.8", "@smithy/util-utf8": "^4.2.0", + "@smithy/util-waiter": "^4.2.8", "tslib": "^2.6.2" }, "engines": { @@ -663,41 +653,43 @@ } }, "node_modules/@aws-sdk/core": { - "version": "3.973.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.973.6.tgz", - "integrity": "sha512-pz4ZOw3BLG0NdF25HoB9ymSYyPbMiIjwQJ2aROXRhAzt+b+EOxStfFv8s5iZyP6Kiw7aYhyWxj5G3NhmkoOTKw==", - "dev": true, + "version": "3.975.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.975.1.tgz", + "integrity": "sha512-8qh/6EYb7hl/ZwVfQufhbMEZs1gQIc7GbdrIf4eprQJ7cv042+74nE6l3YDfyWNzb9iPXb8fRyYSHkNIk5eE6Q==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.1", - "@aws-sdk/xml-builder": "^3.972.4", - "@smithy/core": "^3.22.0", - "@smithy/node-config-provider": "^4.3.8", - "@smithy/property-provider": "^4.2.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/signature-v4": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", - "@smithy/types": "^4.12.0", - "@smithy/util-base64": "^4.3.0", - "@smithy/util-middleware": "^4.2.8", - "@smithy/util-utf8": "^4.2.0", + "@aws-sdk/types": "^3.974.0", + "@aws-sdk/xml-builder": "^3.972.34", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.29.2", + "@smithy/signature-v4": "^5.6.3", + "@smithy/types": "^4.16.0", + "bowser": "^2.11.0", "tslib": "^2.6.2" }, "engines": { "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/core/node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.4.tgz", - "integrity": "sha512-/8dnc7+XNMmViEom2xsNdArQxQPSgy4Z/lm6qaFPTrMFesT1bV3PsBhb19n09nmxHdrtQskYmViddUIjUQElXg==", - "dev": true, + "version": "3.972.57", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.57.tgz", + "integrity": "sha512-1RfJaF7SW1TOnvNGU7kaYjwUf5H3sfm+synGH1bHhRlqcnxCt3szebH3dmKEyY4tuGcbQ6ffzUT89cRitBV8OQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/types": "^3.973.1", - "@smithy/property-provider": "^4.2.8", - "@smithy/types": "^4.12.0", + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/types": "^4.16.0", "tslib": "^2.6.2" }, "engines": { @@ -705,21 +697,17 @@ } }, "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.972.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.6.tgz", - "integrity": "sha512-5ERWqRljiZv44AIdvIRQ3k+EAV0Sq2WeJHvXuK7gL7bovSxOf8Al7MLH7Eh3rdovH4KHFnlIty7J71mzvQBl5Q==", - "dev": true, + "version": "3.972.59", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.59.tgz", + "integrity": "sha512-sRCkpTiFnCdQvuaRVjQ6SVoHu6i7RUpurVo1c4F81HWhPvUJ7Wdp5MNtSdX1O29CNXc8em3O5m52hCjVtAD9SA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/types": "^3.973.1", - "@smithy/fetch-http-handler": "^5.3.9", - "@smithy/node-http-handler": "^4.4.8", - "@smithy/property-provider": "^4.2.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", - "@smithy/types": "^4.12.0", - "@smithy/util-stream": "^4.5.10", + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/fetch-http-handler": "^5.6.4", + "@smithy/node-http-handler": "^4.9.4", + "@smithy/types": "^4.16.0", "tslib": "^2.6.2" }, "engines": { @@ -727,25 +715,23 @@ } }, "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.4.tgz", - "integrity": "sha512-eRUg+3HaUKuXWn/lEMirdiA5HOKmEl8hEHVuszIDt2MMBUKgVX5XNGmb3XmbgU17h6DZ+RtjbxQpjhz3SbTjZg==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/credential-provider-env": "^3.972.4", - "@aws-sdk/credential-provider-http": "^3.972.6", - "@aws-sdk/credential-provider-login": "^3.972.4", - "@aws-sdk/credential-provider-process": "^3.972.4", - "@aws-sdk/credential-provider-sso": "^3.972.4", - "@aws-sdk/credential-provider-web-identity": "^3.972.4", - "@aws-sdk/nested-clients": "3.982.0", - "@aws-sdk/types": "^3.973.1", - "@smithy/credential-provider-imds": "^4.2.8", - "@smithy/property-provider": "^4.2.8", - "@smithy/shared-ini-file-loader": "^4.4.3", - "@smithy/types": "^4.12.0", + "version": "3.973.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.1.tgz", + "integrity": "sha512-6d8H6ZAh3ZPKZ6fe1nG2OWeZEZPtt9ravoD1dezPdPtsSkJRoxGAnFSHwKT3E/Te6fHE30zRzjV6TD12rvF6yQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/credential-provider-env": "^3.972.57", + "@aws-sdk/credential-provider-http": "^3.972.59", + "@aws-sdk/credential-provider-login": "^3.972.63", + "@aws-sdk/credential-provider-process": "^3.972.57", + "@aws-sdk/credential-provider-sso": "^3.973.1", + "@aws-sdk/credential-provider-web-identity": "^3.972.63", + "@aws-sdk/nested-clients": "^3.997.31", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/credential-provider-imds": "^4.4.7", + "@smithy/types": "^4.16.0", "tslib": "^2.6.2" }, "engines": { @@ -753,19 +739,16 @@ } }, "node_modules/@aws-sdk/credential-provider-login": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.4.tgz", - "integrity": "sha512-nLGjXuvWWDlQAp505xIONI7Gam0vw2p7Qu3P6on/W2q7rjJXtYjtpHbcsaOjJ/pAju3eTvEQuSuRedcRHVQIAQ==", - "dev": true, + "version": "3.972.63", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.63.tgz", + "integrity": "sha512-GREWRrMj0XnNKMaVa/Mauoaui26qBEHu71WWqXbwZOu/jFQOnPZjTf7u0KtGKC8VGa6VUs9kDWGgocrKNLS9vw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/nested-clients": "3.982.0", - "@aws-sdk/types": "^3.973.1", - "@smithy/property-provider": "^4.2.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/shared-ini-file-loader": "^4.4.3", - "@smithy/types": "^4.12.0", + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/nested-clients": "^3.997.31", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/types": "^4.16.0", "tslib": "^2.6.2" }, "engines": { @@ -773,23 +756,21 @@ } }, "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.972.5", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.5.tgz", - "integrity": "sha512-VWXKgSISQCI2GKN3zakTNHSiZ0+mux7v6YHmmbLQp/o3fvYUQJmKGcLZZzg2GFA+tGGBStplra9VFNf/WwxpYg==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/credential-provider-env": "^3.972.4", - "@aws-sdk/credential-provider-http": "^3.972.6", - "@aws-sdk/credential-provider-ini": "^3.972.4", - "@aws-sdk/credential-provider-process": "^3.972.4", - "@aws-sdk/credential-provider-sso": "^3.972.4", - "@aws-sdk/credential-provider-web-identity": "^3.972.4", - "@aws-sdk/types": "^3.973.1", - "@smithy/credential-provider-imds": "^4.2.8", - "@smithy/property-provider": "^4.2.8", - "@smithy/shared-ini-file-loader": "^4.4.3", - "@smithy/types": "^4.12.0", + "version": "3.972.66", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.66.tgz", + "integrity": "sha512-f+qjRXZpz7sgzbc4QB+6nLKfyKFgRRXzWdXbsKPv/VhVRyHsDyq4yBWC/B75BAJpFIcUeI2XR/3gdWJ677zB4A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.57", + "@aws-sdk/credential-provider-http": "^3.972.59", + "@aws-sdk/credential-provider-ini": "^3.973.1", + "@aws-sdk/credential-provider-process": "^3.972.57", + "@aws-sdk/credential-provider-sso": "^3.973.1", + "@aws-sdk/credential-provider-web-identity": "^3.972.63", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/credential-provider-imds": "^4.4.7", + "@smithy/types": "^4.16.0", "tslib": "^2.6.2" }, "engines": { @@ -797,17 +778,15 @@ } }, "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.4.tgz", - "integrity": "sha512-TCZpWUnBQN1YPk6grvd5x419OfXjHvhj5Oj44GYb84dOVChpg/+2VoEj+YVA4F4E/6huQPNnX7UYbTtxJqgihw==", - "dev": true, + "version": "3.972.57", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.57.tgz", + "integrity": "sha512-TiVQhuU0pbhIZAUZacbPHMyzrIdiH+lnx+PMY/Pu/b93dJrq3wdZwzUJ0TPpvNxaqbHsxJvQZW3/h/beLiKq7Q==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/types": "^3.973.1", - "@smithy/property-provider": "^4.2.8", - "@smithy/shared-ini-file-loader": "^4.4.3", - "@smithy/types": "^4.12.0", + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/types": "^4.16.0", "tslib": "^2.6.2" }, "engines": { @@ -815,19 +794,17 @@ } }, "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.4.tgz", - "integrity": "sha512-wzsGwv9mKlwJ3vHLyembBvGE/5nPUIwRR2I51B1cBV4Cb4ql9nIIfpmHzm050XYTY5fqTOKJQnhLj7zj89VG8g==", - "dev": true, + "version": "3.973.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.1.tgz", + "integrity": "sha512-3foTZUJ4821Ij60X7K3NJroygiZLnbBmarN+T//O2cjkISan90zElN3NBmgSlDrTQ7Gs6z/yO8V7h60QNcDZHQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-sso": "3.982.0", - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/token-providers": "3.982.0", - "@aws-sdk/types": "^3.973.1", - "@smithy/property-provider": "^4.2.8", - "@smithy/shared-ini-file-loader": "^4.4.3", - "@smithy/types": "^4.12.0", + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/nested-clients": "^3.997.31", + "@aws-sdk/token-providers": "3.1083.0", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/types": "^4.16.0", "tslib": "^2.6.2" }, "engines": { @@ -835,18 +812,16 @@ } }, "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.4.tgz", - "integrity": "sha512-hIzw2XzrG8jzsUSEatehmpkd5rWzASg5IHUfA+m01k/RtvfAML7ZJVVohuKdhAYx+wV2AThLiQJVzqn7F0khrw==", - "dev": true, + "version": "3.972.63", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.63.tgz", + "integrity": "sha512-8qZLFhM69eKcS37m459ctPR05Qimycm/74OPVioe6wNZabMT54GYhwBju0+J656RkMasNSawWQu+c8CmBe3TUQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/nested-clients": "3.982.0", - "@aws-sdk/types": "^3.973.1", - "@smithy/property-provider": "^4.2.8", - "@smithy/shared-ini-file-loader": "^4.4.3", - "@smithy/types": "^4.12.0", + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/nested-clients": "^3.997.31", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/types": "^4.16.0", "tslib": "^2.6.2" }, "engines": { @@ -901,6 +876,23 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.972.62", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.62.tgz", + "integrity": "sha512-k8JJwYXVYlOOjWnPZDThQS1xDFJgi5Dokt73qFlDtrZAbdcint5aIdjB9XgJAAQVP5OoqcefQmh1FYXiPpvsvw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/signature-v4-multi-region": "^3.996.39", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/middleware-user-agent": { "version": "3.972.6", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.6.tgz", @@ -921,49 +913,18 @@ } }, "node_modules/@aws-sdk/nested-clients": { - "version": "3.982.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.982.0.tgz", - "integrity": "sha512-VVkaH27digrJfdVrT64rjkllvOp4oRiZuuJvrylLXAKl18ujToJR7AqpDldL/LS63RVne3QWIpkygIymxFtliQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/middleware-host-header": "^3.972.3", - "@aws-sdk/middleware-logger": "^3.972.3", - "@aws-sdk/middleware-recursion-detection": "^3.972.3", - "@aws-sdk/middleware-user-agent": "^3.972.6", - "@aws-sdk/region-config-resolver": "^3.972.3", - "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.982.0", - "@aws-sdk/util-user-agent-browser": "^3.972.3", - "@aws-sdk/util-user-agent-node": "^3.972.4", - "@smithy/config-resolver": "^4.4.6", - "@smithy/core": "^3.22.0", - "@smithy/fetch-http-handler": "^5.3.9", - "@smithy/hash-node": "^4.2.8", - "@smithy/invalid-dependency": "^4.2.8", - "@smithy/middleware-content-length": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/middleware-retry": "^4.4.29", - "@smithy/middleware-serde": "^4.2.9", - "@smithy/middleware-stack": "^4.2.8", - "@smithy/node-config-provider": "^4.3.8", - "@smithy/node-http-handler": "^4.4.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", - "@smithy/types": "^4.12.0", - "@smithy/url-parser": "^4.2.8", - "@smithy/util-base64": "^4.3.0", - "@smithy/util-body-length-browser": "^4.2.0", - "@smithy/util-body-length-node": "^4.2.1", - "@smithy/util-defaults-mode-browser": "^4.3.28", - "@smithy/util-defaults-mode-node": "^4.2.31", - "@smithy/util-endpoints": "^3.2.8", - "@smithy/util-middleware": "^4.2.8", - "@smithy/util-retry": "^4.2.8", - "@smithy/util-utf8": "^4.2.0", + "version": "3.997.31", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.31.tgz", + "integrity": "sha512-BDHTpwcsZHEBNEJzOg/B1BkFYJxAXY50dau/NyVWs3d51F0WgIUGSWZot/Os+N3KpDhXeaXnz37mWffAvduREw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/signature-v4-multi-region": "^3.996.39", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/fetch-http-handler": "^5.6.4", + "@smithy/node-http-handler": "^4.9.4", + "@smithy/types": "^4.16.0", "tslib": "^2.6.2" }, "engines": { @@ -987,19 +948,32 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.39", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.39.tgz", + "integrity": "sha512-8+srXqYIF8KYMLC4FxMLEM5Ek7kUNibJu1R4m8/fUhhNYIZZz26oGtKkCr8I/HiG2fFQxBvaGgQZT4/mqRCSnA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.0", + "@smithy/signature-v4": "^5.6.3", + "@smithy/types": "^4.16.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/token-providers": { - "version": "3.982.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.982.0.tgz", - "integrity": "sha512-v3M0KYp2TVHYHNBT7jHD9lLTWAdS9CaWJ2jboRKt0WAB65bA7iUEpR+k4VqKYtpQN4+8kKSc4w+K6kUNZkHKQw==", - "dev": true, + "version": "3.1083.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1083.0.tgz", + "integrity": "sha512-s0woKnxuHrExLc5L2ArIH5BMkbonHPtt+5hSBM8oknp9M6QTuUmmAmJ2E0EdzCGONrO+8+ADPqvv6UX0nNcc7A==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/nested-clients": "3.982.0", - "@aws-sdk/types": "^3.973.1", - "@smithy/property-provider": "^4.2.8", - "@smithy/shared-ini-file-loader": "^4.4.3", - "@smithy/types": "^4.12.0", + "@aws-sdk/core": "^3.975.1", + "@aws-sdk/nested-clients": "^3.997.31", + "@aws-sdk/types": "^3.974.0", + "@smithy/core": "^3.29.2", + "@smithy/types": "^4.16.0", "tslib": "^2.6.2" }, "engines": { @@ -1007,13 +981,12 @@ } }, "node_modules/@aws-sdk/types": { - "version": "3.973.1", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.1.tgz", - "integrity": "sha512-DwHBiMNOB468JiX6+i34c+THsKHErYUdNQ3HexeXZvVn4zouLjgaS4FejiGSi2HyBuzuyHg7SuOPmjSvoU9NRg==", - "dev": true, + "version": "3.974.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.0.tgz", + "integrity": "sha512-QIBrw90CDm4O0UaIIzkU6DrFdeJzEb2Va5EPEVpyldj6sHJxB6cshhStJuhZxk3wR3PmjJlYsjPmY1kNb+KGBg==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.12.0", + "@smithy/types": "^4.16.0", "tslib": "^2.6.2" }, "engines": { @@ -1089,15 +1062,12 @@ } }, "node_modules/@aws-sdk/xml-builder": { - "version": "3.972.24", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.24.tgz", - "integrity": "sha512-V8z5YcDPfsvzrBlj0xR1vhRtocblhYbqdreCJB/voGd4Sr5zjNAeWxexbnqVtskTJe0vFb5KMqbSL++ePl+zRw==", - "dev": true, + "version": "3.972.34", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.34.tgz", + "integrity": "sha512-wHhWL1y7sN3enBA8POrPpQM5jCcmu2ozyhbRei4c8OjVcEaEs6yLucLa/pla457ggS/ysuy7bosagz3HaJkZXA==", "license": "Apache-2.0", "dependencies": { - "@nodable/entities": "2.1.0", - "@smithy/types": "^4.14.1", - "fast-xml-parser": "5.7.3", + "@smithy/types": "^4.16.0", "tslib": "^2.6.2" }, "engines": { @@ -1138,6 +1108,8 @@ "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "tslib": "^2.6.2" }, @@ -1150,6 +1122,8 @@ "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.10.1.tgz", "integrity": "sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@azure/abort-controller": "^2.1.2", "@azure/core-util": "^1.13.0", @@ -1164,6 +1138,8 @@ "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.10.1.tgz", "integrity": "sha512-Nh5PhEOeY6PrnxNPsEHRr9eimxLwgLlpmguQaHKBinFYA/RU9+kOYVOQqOrTsCL+KSxrLLl1gD8Dk5BFW/7l/w==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@azure/abort-controller": "^2.1.2", "@azure/core-auth": "^1.10.0", @@ -1182,6 +1158,8 @@ "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.3.1.tgz", "integrity": "sha512-az9BkXND3/d5VgdRRQVkiJb2gOmDU8Qcq4GvjtBmDICNiQ9udFmDk4ZpSB5Qq1OmtDJGlQAfBaS4palFsazQ5g==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@azure/abort-controller": "^2.1.2", "@azure/core-client": "^1.10.0", @@ -1196,6 +1174,8 @@ "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.7.2.tgz", "integrity": "sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@azure/abort-controller": "^2.0.0", "@azure/core-util": "^1.2.0", @@ -1211,6 +1191,8 @@ "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.6.2.tgz", "integrity": "sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "tslib": "^2.6.2" }, @@ -1223,6 +1205,8 @@ "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.22.1.tgz", "integrity": "sha512-UVZlVLfLyz6g3Hy7GNDpooMQonUygH7ghdiSASOOHy97fKj/mPLqgDX7aidOijn+sCMU+WU8NjlPlNTgnvbcGA==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@azure/abort-controller": "^2.1.2", "@azure/core-auth": "^1.10.0", @@ -1241,6 +1225,8 @@ "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.3.1.tgz", "integrity": "sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "tslib": "^2.6.2" }, @@ -1253,6 +1239,8 @@ "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.13.1.tgz", "integrity": "sha512-XPArKLzsvl0Hf0CaGyKHUyVgF7oDnhKoP85Xv6M4StF/1AhfORhZudHtOyf2s+FcbuQ9dPRAjB8J2KvRRMUK2A==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@azure/abort-controller": "^2.1.2", "@typespec/ts-http-runtime": "^0.3.0", @@ -1262,19 +1250,6 @@ "node": ">=20.0.0" } }, - "node_modules/@azure/core-xml": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.5.0.tgz", - "integrity": "sha512-D/sdlJBMJfx7gqoj66PKVmhDDaU6TKA49ptcolxdas29X7AfvLTmfAGLjAcIMBK7UZ2o4lygHIqVckOlQU3xWw==", - "license": "MIT", - "dependencies": { - "fast-xml-parser": "^5.0.7", - "tslib": "^2.8.1" - }, - "engines": { - "node": ">=20.0.0" - } - }, "node_modules/@azure/identity": { "version": "4.13.1", "resolved": "https://registry.npmjs.org/@azure/identity/-/identity-4.13.1.tgz", @@ -1370,6 +1345,8 @@ "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.3.0.tgz", "integrity": "sha512-fCqPIfOcLE+CGqGPd66c8bZpwAji98tZ4JI9i/mlTNTlsIWslCfpg48s/ypyLxZTump5sypjrKn2/kY7q8oAbA==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@typespec/ts-http-runtime": "^0.3.0", "tslib": "^2.6.2" @@ -1418,51 +1395,6 @@ "node": ">=20" } }, - "node_modules/@azure/storage-blob": { - "version": "12.29.1", - "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.29.1.tgz", - "integrity": "sha512-7ktyY0rfTM0vo7HvtK6E3UvYnI9qfd6Oz6z/+92VhGRveWng3kJwMKeUpqmW/NmwcDNbxHpSlldG+vsUnRFnBg==", - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@azure/core-auth": "^1.9.0", - "@azure/core-client": "^1.9.3", - "@azure/core-http-compat": "^2.2.0", - "@azure/core-lro": "^2.2.0", - "@azure/core-paging": "^1.6.2", - "@azure/core-rest-pipeline": "^1.19.1", - "@azure/core-tracing": "^1.2.0", - "@azure/core-util": "^1.11.0", - "@azure/core-xml": "^1.4.5", - "@azure/logger": "^1.1.4", - "@azure/storage-common": "^12.1.1", - "events": "^3.0.0", - "tslib": "^2.8.1" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@azure/storage-common": { - "version": "12.1.1", - "resolved": "https://registry.npmjs.org/@azure/storage-common/-/storage-common-12.1.1.tgz", - "integrity": "sha512-eIOH1pqFwI6UmVNnDQvmFeSg0XppuzDLFeUNO/Xht7ODAzRLgGDh7h550pSxoA+lPDxBl1+D2m/KG3jWzCUjTg==", - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@azure/core-auth": "^1.9.0", - "@azure/core-http-compat": "^2.2.0", - "@azure/core-rest-pipeline": "^1.19.1", - "@azure/core-tracing": "^1.2.0", - "@azure/core-util": "^1.11.0", - "@azure/logger": "^1.1.4", - "events": "^3.3.0", - "tslib": "^2.8.1" - }, - "engines": { - "node": ">=20.0.0" - } - }, "node_modules/@babel/code-frame": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", @@ -12870,6 +12802,17 @@ "@sinonjs/commons": "^3.0.0" } }, + "node_modules/@sinonjs/samsam": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.3.tgz", + "integrity": "sha512-hw6HbX+GyVZzmaYNh82Ecj1vdGZrqVIn/keDTg63IgAwiQPO+xCz99uG6Woqgb4tM0mUiFENKZ4cqd7IX94AXQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.1", + "type-detect": "^4.1.0" + } + }, "node_modules/@smithy/abort-controller": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.2.8.tgz", @@ -12903,21 +12846,12 @@ } }, "node_modules/@smithy/core": { - "version": "3.22.1", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.22.1.tgz", - "integrity": "sha512-x3ie6Crr58MWrm4viHqqy2Du2rHYZjwu8BekasrQx4ca+Y24dzVAwq3yErdqIbc2G3I0kLQA13PQ+/rde+u65g==", - "dev": true, + "version": "3.29.3", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.29.3.tgz", + "integrity": "sha512-L+Ys6ecjk5vwPMAKHBpPKlJ3DkqwNcnfEISXBZIsVvWG/XKXfsAP8mwIYlTeLcd2ElHdesPI8OuOmJSFAPhm6A==", "license": "Apache-2.0", "dependencies": { - "@smithy/middleware-serde": "^4.2.9", - "@smithy/protocol-http": "^5.3.8", - "@smithy/types": "^4.12.0", - "@smithy/util-base64": "^4.3.0", - "@smithy/util-body-length-browser": "^4.2.0", - "@smithy/util-middleware": "^4.2.8", - "@smithy/util-stream": "^4.5.11", - "@smithy/util-utf8": "^4.2.0", - "@smithy/uuid": "^1.1.0", + "@smithy/types": "^4.16.1", "tslib": "^2.6.2" }, "engines": { @@ -12925,16 +12859,13 @@ } }, "node_modules/@smithy/credential-provider-imds": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.8.tgz", - "integrity": "sha512-FNT0xHS1c/CPN8upqbMFP83+ul5YgdisfCfkZ86Jh2NSmnqw/AJ6x5pEogVCTVvSm7j9MopRU89bmDelxuDMYw==", - "dev": true, + "version": "4.4.8", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.8.tgz", + "integrity": "sha512-q9J7JTiXrAhB8sDp4px97uEPT7CwKH61Co78grdNQvU8QZAdiuaSRhP0tUVf2ogy36RZTrlMU1rBmDEH+cnkiA==", "license": "Apache-2.0", "dependencies": { - "@smithy/node-config-provider": "^4.3.8", - "@smithy/property-provider": "^4.2.8", - "@smithy/types": "^4.12.0", - "@smithy/url-parser": "^4.2.8", + "@smithy/core": "^3.29.3", + "@smithy/types": "^4.16.1", "tslib": "^2.6.2" }, "engines": { @@ -12942,16 +12873,13 @@ } }, "node_modules/@smithy/fetch-http-handler": { - "version": "5.3.9", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.3.9.tgz", - "integrity": "sha512-I4UhmcTYXBrct03rwzQX1Y/iqQlzVQaPxWjCjula++5EmWq9YGBrx6bbGqluGc1f0XEfhSkiY4jhLgbsJUMKRA==", - "dev": true, + "version": "5.6.5", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.5.tgz", + "integrity": "sha512-SuqeisTyPoiIPtIYru/sGxGyXzmZ+8nnFOhC+qRPglt06Ebd1yH//CDltZB2J/3WBNVhwfUaZ0EtHB3cm2X32g==", "license": "Apache-2.0", "dependencies": { - "@smithy/protocol-http": "^5.3.8", - "@smithy/querystring-builder": "^4.2.8", - "@smithy/types": "^4.12.0", - "@smithy/util-base64": "^4.3.0", + "@smithy/core": "^3.29.3", + "@smithy/types": "^4.16.1", "tslib": "^2.6.2" }, "engines": { @@ -13103,16 +13031,13 @@ } }, "node_modules/@smithy/node-http-handler": { - "version": "4.4.9", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.4.9.tgz", - "integrity": "sha512-KX5Wml5mF+luxm1szW4QDz32e3NObgJ4Fyw+irhph4I/2geXwUy4jkIMUs5ZPGflRBeR6BUkC2wqIab4Llgm3w==", - "dev": true, + "version": "4.9.5", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.5.tgz", + "integrity": "sha512-bNqdxTQTxmLbomSmlkZFz8L6B/feQ2HHzw4L2zY7Ecp2XffYAZq2uzdWDdxJHJFbEvqd+SRuluJso0P8+xPdbw==", "license": "Apache-2.0", "dependencies": { - "@smithy/abort-controller": "^4.2.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/querystring-builder": "^4.2.8", - "@smithy/types": "^4.12.0", + "@smithy/core": "^3.29.3", + "@smithy/types": "^4.16.1", "tslib": "^2.6.2" }, "engines": { @@ -13147,21 +13072,6 @@ "node": ">=18.0.0" } }, - "node_modules/@smithy/querystring-builder": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.2.8.tgz", - "integrity": "sha512-Xr83r31+DrE8CP3MqPgMJl+pQlLLmOfiEUnoyAlGzzJIrEsbKsPy1hqH0qySaQm4oWrCBlUqRt+idEgunKB+iw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "@smithy/util-uri-escape": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@smithy/querystring-parser": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.2.8.tgz", @@ -13204,19 +13114,13 @@ } }, "node_modules/@smithy/signature-v4": { - "version": "5.3.8", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.3.8.tgz", - "integrity": "sha512-6A4vdGj7qKNRF16UIcO8HhHjKW27thsxYci+5r/uVRkdcBEkOEiY8OMPuydLX4QHSrJqGHPJzPRwwVTqbLZJhg==", - "dev": true, + "version": "5.6.4", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.6.4.tgz", + "integrity": "sha512-B89bpf2t/y/wia6LZ+4JfHXYQT9PnVftsH05rgJKKIStS7r/4XSs9HOjtPoLtgcA6HCW9jVqX5DBbq7E0PAkiQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/is-array-buffer": "^4.2.0", - "@smithy/protocol-http": "^5.3.8", - "@smithy/types": "^4.12.0", - "@smithy/util-hex-encoding": "^4.2.0", - "@smithy/util-middleware": "^4.2.8", - "@smithy/util-uri-escape": "^4.2.0", - "@smithy/util-utf8": "^4.2.0", + "@smithy/core": "^3.29.3", + "@smithy/types": "^4.16.1", "tslib": "^2.6.2" }, "engines": { @@ -13243,10 +13147,9 @@ } }, "node_modules/@smithy/types": { - "version": "4.14.2", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.14.2.tgz", - "integrity": "sha512-P+otAxbV4CqBybp7EkcJCrig63yE2E7PuNVOmilVMRcx/O+QDzGULTrKsq4DV13gSfak9ObPrWaHl/9bL5YcWw==", - "dev": true, + "version": "4.16.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.16.1.tgz", + "integrity": "sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" @@ -13450,19 +13353,6 @@ "node": ">=18.0.0" } }, - "node_modules/@smithy/util-uri-escape": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-4.2.0.tgz", - "integrity": "sha512-igZpCKV9+E/Mzrpq6YacdTQ0qTiLm85gD6N/IrmyDvQFA4UnU3d5g3m8tMT/6zG/vVkWSU+VxeUyGonL62DuxA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@smithy/util-utf8": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-4.2.0.tgz", @@ -14830,6 +14720,23 @@ "@types/node": "*" } }, + "node_modules/@types/sinon": { + "version": "17.0.4", + "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-17.0.4.tgz", + "integrity": "sha512-RHnIrhfPO3+tJT0s7cFaXGZvsL4bbR3/k7z3P312qMS4JaS2Tk+KiwiLx1S0rQ56ERj00u1/BtdyVd0FY+Pdew==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/sinonjs__fake-timers": "*" + } + }, + "node_modules/@types/sinonjs__fake-timers": { + "version": "15.0.1", + "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-15.0.1.tgz", + "integrity": "sha512-Ko2tjWJq8oozHzHV+reuvS5KYIRAokHnGbDwGh/J64LntgpbuylF74ipEL24HCyRjf9FOlBiBHWBR1RlVKsI1w==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/stack-utils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", @@ -15112,6 +15019,8 @@ "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.1.tgz", "integrity": "sha512-SnbaqayTVFEA6/tYumdF0UmybY0KHyKwGPBXnyckFlrrKdhWFrL3a2HIPXHjht5ZOElKGcXfD2D63P36btb+ww==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "http-proxy-agent": "^7.0.0", "https-proxy-agent": "^7.0.0", @@ -16546,6 +16455,18 @@ "node": ">=6" } }, + "node_modules/aws-sdk-client-mock": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/aws-sdk-client-mock/-/aws-sdk-client-mock-4.1.0.tgz", + "integrity": "sha512-h/tOYTkXEsAcV3//6C1/7U4ifSpKyJvb6auveAepqqNJl6TdZaPFEtKjBQNf8UxQdDP850knB2i/whq4zlsxJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/sinon": "^17.0.3", + "sinon": "^18.0.1", + "tslib": "^2.1.0" + } + }, "node_modules/aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", @@ -17101,6 +17022,11 @@ "node": "*" } }, + "node_modules/bigi": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/bigi/-/bigi-1.4.2.tgz", + "integrity": "sha512-ddkU+dFIuEIW8lE7ZwdIAf2UPoM90eaprg5m3YXAVVTmKlqV/9BX4A2M8BOK2yOq6/VgZFVhK6QAxJebhlbhzw==" + }, "node_modules/bigint-buffer": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/bigint-buffer/-/bigint-buffer-1.1.5.tgz", @@ -17144,6 +17070,42 @@ "file-uri-to-path": "1.0.0" } }, + "node_modules/bip-schnorr": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/bip-schnorr/-/bip-schnorr-0.6.4.tgz", + "integrity": "sha512-dNKw7Lea8B0wMIN4OjEmOk/Z5qUGqoPDY0P2QttLqGk1hmDPytLWW8PR5Pb6Vxy6CprcdEgfJpOjUu+ONQveyg==", + "license": "MIT", + "dependencies": { + "bigi": "^1.4.2", + "ecurve": "^1.0.6", + "js-sha256": "^0.9.0", + "randombytes": "^2.1.0", + "safe-buffer": "^5.2.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/bip-schnorr/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/bip174": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/bip174/-/bip174-2.1.1.tgz", @@ -17321,6 +17283,34 @@ "node": ">=4.0.0" } }, + "node_modules/bitcore-lib": { + "version": "8.25.47", + "resolved": "https://registry.npmjs.org/bitcore-lib/-/bitcore-lib-8.25.47.tgz", + "integrity": "sha512-qDZr42HuP4P02I8kMGZUx/vvwuDsz8X3rQxXLfM0BtKzlQBcbSM7ycDkDN99Xc5jzpd4fxNQyyFXOmc6owUsrQ==", + "license": "MIT", + "dependencies": { + "bech32": "=2.0.0", + "bip-schnorr": "=0.6.4", + "bn.js": "=4.11.8", + "bs58": "^4.0.1", + "buffer-compare": "=1.1.1", + "elliptic": "^6.5.3", + "inherits": "=2.0.1", + "lodash": "^4.17.20" + } + }, + "node_modules/bitcore-lib/node_modules/bn.js": { + "version": "4.11.8", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz", + "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA==", + "license": "MIT" + }, + "node_modules/bitcore-lib/node_modules/inherits": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz", + "integrity": "sha512-8nWq2nLTAwd02jTqJExUYFSD/fKq6VH9Y/oG2accc/kdI0V98Bag8d5a4gi3XHz73rDWa2PvTtvcWYquKqSENA==", + "license": "ISC" + }, "node_modules/bitset": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/bitset/-/bitset-5.2.3.tgz", @@ -17551,7 +17541,6 @@ "version": "2.13.1", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.13.1.tgz", "integrity": "sha512-OHawaAbjwx6rqICCKgSG0SAnT05bzd7ppyKLVUITZpANBaaMFBAsaNkto3LoQ31tyFP5kNujE8Cdx85G9VzOkw==", - "dev": true, "license": "MIT" }, "node_modules/boxen": { @@ -17756,6 +17745,11 @@ "ieee754": "^1.2.1" } }, + "node_modules/buffer-compare": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-compare/-/buffer-compare-1.1.1.tgz", + "integrity": "sha512-O6NvNiHZMd3mlIeMDjP6t/gPG75OqGPeiRZXoMQZJ6iy9GofCls4Ijs5YkPZZwoysizLiedhticmdyx/GyHghA==" + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -17892,6 +17886,27 @@ "node": ">=10.16.0" } }, + "node_modules/bytebuffer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/bytebuffer/-/bytebuffer-5.0.1.tgz", + "integrity": "sha512-IuzSdmADppkZ6DlpycMkm8l9zeEq16fWtLvunEwFiYciR/BHo4E8/xs5piFquG+Za8OWmMqHF8zuRviz2LHvRQ==", + "license": "Apache-2.0", + "dependencies": { + "long": "~3" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/bytebuffer/node_modules/long": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/long/-/long-3.2.0.tgz", + "integrity": "sha512-ZYvPPOMqUwPoDsbJaR10iQJYnMuZhRTvHYl62ErLIEX7RgFlziSBUUvrt3OVfc47QlHHpzPZYP17g3Fv7oeJkg==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.6" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -19436,7 +19451,6 @@ "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.2.tgz", "integrity": "sha512-vtcDfH3TOjP8UekytvnHH1o1P4FcUdt4eQ1Y+Abap1tk/OB2MWQvcwS2ClCd1zuIhc3JKOx6p3kod8Vfys3E+A==", "license": "BSD-3-Clause", - "peer": true, "engines": { "node": ">=0.3.1" } @@ -19680,6 +19694,16 @@ "node": ">=8.0.0" } }, + "node_modules/ecurve": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/ecurve/-/ecurve-1.0.6.tgz", + "integrity": "sha512-/BzEjNfiSuB7jIWKcS/z8FK9jNjmEWvUV2YZ4RLSmcDtP7Lq0m6FvDuSnJpBlDpGRpfRQeTLGLBI8H+kEv0r+w==", + "license": "MIT", + "dependencies": { + "bigi": "^1.1.0", + "safe-buffer": "^5.0.1" + } + }, "node_modules/editorconfig": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-1.0.7.tgz", @@ -20752,6 +20776,7 @@ "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "devOptional": true, "license": "MIT", "engines": { "node": ">=0.8.x" @@ -21729,6 +21754,12 @@ "node": ">= 0.8" } }, + "node_modules/fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==", + "license": "ISC" + }, "node_modules/fs-extra": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", @@ -23009,6 +23040,8 @@ "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -25067,6 +25100,12 @@ "optional": true, "peer": true }, + "node_modules/js-sha256": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/js-sha256/-/js-sha256-0.9.0.tgz", + "integrity": "sha512-sga3MHh9sgQN2+pJ9VYZ+1LPwXOxuBJBA5nrR5/ofPfuiJBE2hnjsaN8se8JznOmGLN2p49Pe5U/ttafcs/apA==", + "license": "MIT" + }, "node_modules/js-sha3": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/js-sha3/-/js-sha3-0.8.0.tgz", @@ -25376,6 +25415,13 @@ "node": ">= 6" } }, + "node_modules/just-extend": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-6.2.0.tgz", + "integrity": "sha512-cYofQu2Xpom82S6qD778jBDpwvvy39s1l/hrYij2u9AMdQcGRpaBu6kY4mVhuno5kJVi1DAz4aiphA2WI1/OAw==", + "dev": true, + "license": "MIT" + }, "node_modules/jwa": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.2.tgz", @@ -27398,12 +27444,23 @@ "version": "2.30.1", "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==", - "devOptional": true, "license": "MIT", "engines": { "node": "*" } }, + "node_modules/moment-timezone": { + "version": "0.5.48", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.48.tgz", + "integrity": "sha512-f22b8LV1gbTO2ms2j2z13MuPogNoh5UzxL3nzNAYKGraILnbGc9NEE6dyiiiLv46DGRb8A4kg8UKWLjPthxBHw==", + "license": "MIT", + "dependencies": { + "moment": "^2.29.4" + }, + "engines": { + "node": "*" + } + }, "node_modules/moo": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/moo/-/moo-0.5.3.tgz", @@ -27979,6 +28036,40 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "license": "MIT" }, + "node_modules/nise": { + "version": "6.1.5", + "resolved": "https://registry.npmjs.org/nise/-/nise-6.1.5.tgz", + "integrity": "sha512-SnRDPDBjxZZoU2n0+gzzLtSvo1OZo7j6jnbXsoh3AFxEGhaFU7ZF0TmefuKERq79wxR2U+MPn7ArW+Tl+clC3A==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.1", + "@sinonjs/fake-timers": "^15.1.1", + "just-extend": "^6.2.0", + "path-to-regexp": "^8.3.0" + } + }, + "node_modules/nise/node_modules/@sinonjs/fake-timers": { + "version": "15.4.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-15.4.0.tgz", + "integrity": "sha512-DsG+8/LscQIQg68J6Ef3dv10u6nVyetYn923s3/sus5eaGfTo1of5WMZSLf0UJc9KDuKPilPH0UDJCjvNbDNCA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.1" + } + }, + "node_modules/nise/node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "dev": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/no-case": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/no-case/-/no-case-2.3.2.tgz", @@ -28415,6 +28506,34 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/opentimestamps": { + "version": "0.4.9", + "resolved": "https://registry.npmjs.org/opentimestamps/-/opentimestamps-0.4.9.tgz", + "integrity": "sha512-EWA6nFSeNsD6aSX/Ek0WrWeTT+PKoP7455zWPkyvqf7/h7QsmDDbfFEY8S3Sd9/VxeQcyVu7yW0SqkvP+Zu/fg==", + "license": "LGPL-3.0", + "dependencies": { + "bitcore-lib": "^8.14.4", + "bytebuffer": "^5.0.1", + "commander": "^2.20.3", + "fs": "0.0.1-security", + "minimatch": "^3.0.4", + "moment-timezone": "^0.5.27", + "promise": "^7.3.1", + "properties": "^1.2.1", + "randomstring": "^1.1.5", + "request": "^2.85.0", + "request-promise": "^4.2.2" + }, + "bin": { + "ots-cli.js": "ots-cli.js" + } + }, + "node_modules/opentimestamps/node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "license": "MIT" + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -29561,6 +29680,15 @@ "node": ">= 6" } }, + "node_modules/properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/properties/-/properties-1.2.1.tgz", + "integrity": "sha512-qYNxyMj1JeW54i/EWEFsM1cVwxJbtgPp8+0Wg9XjNaK6VE/c4oRi6PNu5p7w1mNXEIQIjV5Wwn8v8Gz82/QzdQ==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, "node_modules/proto-list": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", @@ -30052,6 +30180,21 @@ "safe-buffer": "^5.1.0" } }, + "node_modules/randomstring": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/randomstring/-/randomstring-1.3.1.tgz", + "integrity": "sha512-lgXZa80MUkjWdE7g2+PZ1xDLzc7/RokXVEQOv5NN2UOTChW1I8A9gha5a9xYBOqgaSoI6uJikDmCU8PyRdArRQ==", + "license": "MIT", + "dependencies": { + "randombytes": "2.1.0" + }, + "bin": { + "randomstring": "bin/randomstring" + }, + "engines": { + "node": "*" + } + }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -30257,6 +30400,40 @@ "node": ">= 6" } }, + "node_modules/request-promise": { + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.6.tgz", + "integrity": "sha512-HCHI3DJJUakkOr8fNoCc73E5nU5bqITjOYFMDrKHYOXWXrgD/SBaC7LjwuPymUprRyuF06UK7hd/lMHkmUXglQ==", + "deprecated": "request-promise has been deprecated because it extends the now deprecated request package, see https://github.com/request/request/issues/3142", + "license": "ISC", + "dependencies": { + "bluebird": "^3.5.0", + "request-promise-core": "1.1.4", + "stealthy-require": "^1.1.1", + "tough-cookie": "^2.3.3" + }, + "engines": { + "node": ">=0.10.0" + }, + "peerDependencies": { + "request": "^2.34" + } + }, + "node_modules/request-promise-core": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.4.tgz", + "integrity": "sha512-TTbAfBBRdWD7aNNOoVOBH4pN/KigV6LyapYNNlAPA8JwbovRti1E88m3sYAwsLi5ryhPKsE9APwnjFTgdUjTpw==", + "license": "ISC", + "dependencies": { + "lodash": "^4.17.19" + }, + "engines": { + "node": ">=0.10.0" + }, + "peerDependencies": { + "request": "^2.34" + } + }, "node_modules/request/node_modules/form-data": { "version": "2.5.5", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.5.tgz", @@ -30315,6 +30492,16 @@ ], "license": "MIT" }, + "node_modules/request/node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "license": "MIT", + "bin": { + "uuid": "bin/uuid" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -31795,6 +31982,35 @@ "node": ">=4" } }, + "node_modules/sinon": { + "version": "18.0.1", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-18.0.1.tgz", + "integrity": "sha512-a2N2TDY1uGviajJ6r4D1CyRAkzE9NNVlYOV1wX5xQDuAk0ONgzgRl0EjCQuRCPxOwp13ghsMwt9Gdldujs39qw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.1", + "@sinonjs/fake-timers": "11.2.2", + "@sinonjs/samsam": "^8.0.0", + "diff": "^5.2.0", + "nise": "^6.0.0", + "supports-color": "^7" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/sinon" + } + }, + "node_modules/sinon/node_modules/@sinonjs/fake-timers": { + "version": "11.2.2", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-11.2.2.tgz", + "integrity": "sha512-G2piCSxQ7oWOxwGSAyFHfPIsyeJGXYtc6mFbnFA+kRXkiEnTl8c/8jul2S329iFBnDI9HGoeWWAZvuvOkZccgw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.0" + } + }, "node_modules/sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -32270,6 +32486,15 @@ "node": ">= 0.8" } }, + "node_modules/stealthy-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", + "integrity": "sha512-ZnWpYnYugiOVEY5GkcuJK1io5V8QmNYChG62gSit9pQVGErXtrKuPC55ITaVSukmMta5qpMU7vqLt2Lnni4f/g==", + "license": "ISC", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/stop-iteration-iterator": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.1.0.tgz", diff --git a/package.json b/package.json index 323d28ce43..2a7fb35519 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "dependencies": { "@arbitrum/sdk": "^3.7.3", "@arkade-os/sdk": "^0.4.36", - "@azure/storage-blob": "^12.29.1", + "@aws-sdk/client-s3": "^3.1071.0", "@blockfrost/blockfrost-js": "^6.1.0", "@buildonspark/spark-sdk": "^0.6.7", "@cardano-foundation/cardano-verify-datasignature": "^1.0.11", @@ -116,6 +116,7 @@ "node-2fa": "^2.0.3", "node-pty": "^1.1.0", "nodemailer": "^6.10.1", + "opentimestamps": "^0.4.9", "passport": "^0.6.0", "passport-jwt": "^4.0.1", "pdfkit": "^0.15.2", @@ -154,6 +155,7 @@ "@types/passport-jwt": "^3.0.13", "@types/pdfkit": "^0.13.9", "@types/supertest": "^2.0.16", + "aws-sdk-client-mock": "^4.1.0", "eslint": "^9.17.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-prettier": "^5.2.1", @@ -197,7 +199,8 @@ "body-parser": "1.20.3", "qs": "^6.14.1", "request": { - "form-data": "2.5.5" + "form-data": "2.5.5", + "uuid": "3.4.0" }, "multer": "^2.0.0", "uuid": "^9.0.0", diff --git a/scripts/kyc/kyc-storage.js b/scripts/kyc/kyc-storage.js index ba8fa2280a..93b427c444 100644 --- a/scripts/kyc/kyc-storage.js +++ b/scripts/kyc/kyc-storage.js @@ -53,7 +53,7 @@ async function main() { console.log('========================================'); console.log(''); console.log('In local development mode, KYC files are automatically'); - console.log('loaded from scripts/kyc/dummy-files/ by the azure-storage'); + console.log('loaded from scripts/kyc/dummy-files/ by the mock storage'); console.log('service when the requested file is not in memory storage.'); console.log(''); diff --git a/scripts/storage/provision-bucket.ts b/scripts/storage/provision-bucket.ts new file mode 100644 index 0000000000..eff6021dc7 --- /dev/null +++ b/scripts/storage/provision-bucket.ts @@ -0,0 +1,135 @@ +/** + * WORM bucket provisioning for the GeBüV anchoring pipeline (Stage 3). + * + * Creates (or reconciles) an S3-compatible bucket with Object Lock enabled and a default + * COMPLIANCE-mode retention, so archived compliance objects (KYC documents, EP2 settlement + * reports) become immutable for the legally required retention period (Swiss GeBüV: 10 years + * of business records; we provision a safety margin, default 11 years). + * + * IMPORTANT — dynamic EP2 containers: + * EP2 settlement reports are written to a per-merchant container resolved at runtime + * (paymentLinksConfigObj.ep2ReportContainer in fiat-output-job.service.ts). Each such + * container is a distinct S3 bucket and MUST be provisioned with this exact same Object + * Lock + COMPLIANCE retention BEFORE its first PUT — otherwise its objects are not WORM + * protected and Object Lock cannot be retro-fitted onto an existing non-locked bucket. + * Run this script once per container name before onboarding a new EP2 merchant. + * + * Idempotent: if the bucket already exists, creation is skipped and only the Object Lock + * default-retention configuration is (re)applied. + * + * Configuration (no silent defaults for credentials — fails fast if incomplete): + * - S3 endpoint/region/credentials come from the standard S3_* env vars (via Config.s3). + * + * Run with (bucket name required, retention years optional, default 11): + * BUCKET=kyc RETENTION_YEARS=11 npx ts-node scripts/storage/provision-bucket.ts + * npx ts-node scripts/storage/provision-bucket.ts kyc 11 + */ + +import { + CreateBucketCommand, + GetBucketVersioningCommand, + HeadBucketCommand, + ObjectLockRetentionMode, + PutBucketVersioningCommand, + PutObjectLockConfigurationCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import * as dotenv from 'dotenv'; + +dotenv.config(); + +// Loaded after dotenv so Config.s3 reads the populated env. +import { Config } from '../../src/config/config'; + +function getBucketName(): string { + const bucket = process.env.BUCKET ?? process.argv[2]; + if (!bucket) throw new Error('Missing bucket name. Provide via BUCKET env var or as the first CLI argument.'); + return bucket; +} + +function getRetentionYears(): number { + const raw = process.env.RETENTION_YEARS ?? process.argv[3]; + if (raw == null) return 11; // GeBüV 10y + safety margin; explicit, not a silent credential default. + + const years = Number(raw); + if (!Number.isInteger(years) || years <= 0) + throw new Error(`Invalid RETENTION_YEARS: ${raw} (expected positive integer)`); + return years; +} + +function buildClient(): S3Client { + const { endpoint, region, accessKey, secretKey } = Config.s3; + if (!endpoint || !region || !accessKey || !secretKey) + throw new Error('Incomplete S3 config: S3_ENDPOINT, S3_REGION, S3_ACCESS_KEY and S3_SECRET_KEY are required'); + + return new S3Client({ + endpoint, + region, + forcePathStyle: true, // MinIO requires path-style addressing (matches S3StorageService) + credentials: { accessKeyId: accessKey, secretAccessKey: secretKey }, + }); +} + +async function bucketExists(client: S3Client, bucket: string): Promise { + try { + await client.send(new HeadBucketCommand({ Bucket: bucket })); + return true; + } catch (e) { + const status = e?.$metadata?.httpStatusCode; + if (status === 404 || e?.name === 'NotFound' || e?.name === 'NoSuchBucket') return false; + throw e; + } +} + +async function ensureVersioning(client: S3Client, bucket: string): Promise { + // Object Lock requires versioning; CreateBucket with ObjectLockEnabledForBucket enables it + // implicitly, but we assert/enable it explicitly so reconciling an existing bucket is safe. + const current = await client.send(new GetBucketVersioningCommand({ Bucket: bucket })); + if (current.Status === 'Enabled') { + console.log(' Versioning: already enabled'); + return; + } + + await client.send(new PutBucketVersioningCommand({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } })); + console.log(' Versioning: enabled'); +} + +async function applyObjectLock(client: S3Client, bucket: string, years: number): Promise { + await client.send( + new PutObjectLockConfigurationCommand({ + Bucket: bucket, + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { DefaultRetention: { Mode: ObjectLockRetentionMode.COMPLIANCE, Years: years } }, + }, + }), + ); + console.log(` Object Lock: default retention COMPLIANCE / ${years} year(s)`); +} + +async function main(): Promise { + const bucket = getBucketName(); + const years = getRetentionYears(); + const client = buildClient(); + + console.log(`Provisioning WORM bucket "${bucket}" (endpoint ${Config.s3.endpoint})`); + + if (await bucketExists(client, bucket)) { + console.log(' Bucket: already exists -> reconciling lock configuration only'); + } else { + await client.send(new CreateBucketCommand({ Bucket: bucket, ObjectLockEnabledForBucket: true })); + console.log(' Bucket: created with Object Lock enabled'); + } + + await ensureVersioning(client, bucket); + await applyObjectLock(client, bucket, years); + + console.log(`Done. Bucket "${bucket}" is WORM-protected (COMPLIANCE, ${years}y).`); +} + +main() + .then(() => process.exit(0)) + .catch((e) => { + console.error('Provisioning failed:', e?.message ?? e); + process.exit(1); + }); diff --git a/src/config/config.ts b/src/config/config.ts index d3fa6d4559..2765bb171f 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -312,6 +312,28 @@ export class Configuration { kycStepExpiry: 90, // days }; + // Host-independent comparison of a stored KYC step result URL against a live blob URL. + // kyc_step.result persists the full blob URL captured at upload time, while a live KycFileBlob.url + // is regenerated from the CURRENT storage backend. After the Azure Blob -> MinIO (S3) storage + // cutover the two hosts differ, so the previous full-URL string equality would silently drop every + // pre-cutover organization document (Handelsregisterauszug / Vollmacht) from the GwG compliance ZIP + // — no error, just missing files. Comparing the container-relative, decoded object key instead is + // stable across the host swap and matches BOTH legacy Azure-host and new MinIO-host values. + // Inlined here (rather than importing StorageService) to avoid a config <-> storage import cycle. + static isSameKycBlob(storedUrl: string | undefined, liveUrl: string | undefined): boolean { + if (storedUrl == null || liveUrl == null) return false; + return Configuration.kycBlobKey(storedUrl) === Configuration.kycBlobKey(liveUrl); + } + + // Reduce a KYC blob URL to its container-relative, percent-decoded object key (drops scheme/host + // and the leading `/` segment), mirroring StorageService.blobName's reversal. + private static kycBlobKey(url: string): string { + const marker = 'kyc/'; + const idx = url.indexOf(marker); + const rel = idx < 0 ? url : url.substring(idx + marker.length); + return rel.split('/').map(decodeURIComponent).join('/'); + } + fileDownloadConfig: { id: number; name: string; @@ -499,11 +521,14 @@ export class Configuration { userData.kycSteps.some( (s) => s.isCompleted && - ((s.name === KycStepName.COMMERCIAL_REGISTER && s.result === file.url) || + ((s.name === KycStepName.COMMERCIAL_REGISTER && Configuration.isSameKycBlob(s.result, file.url)) || (s.name === KycStepName.LEGAL_ENTITY && - s.getResult<{ url: string; legalEntity: LegalEntity }>().url === file.url) || + Configuration.isSameKycBlob( + s.getResult<{ url: string; legalEntity: LegalEntity }>().url, + file.url, + )) || (s.name === KycStepName.SOLE_PROPRIETORSHIP_CONFIRMATION && - s.getResult<{ url: string }>().url === file.url)), + Configuration.isSameKycBlob(s.getResult<{ url: string }>().url, file.url))), ), }, ], @@ -516,7 +541,10 @@ export class Configuration { { prefixes: (userData: UserData) => [`user/${userData.id}/Authority`], filter: (file: KycFileBlob, userData: UserData) => - userData.kycSteps.some((s) => s.name === KycStepName.AUTHORITY && s.isCompleted && s.result === file.url), + userData.kycSteps.some( + (s) => + s.name === KycStepName.AUTHORITY && s.isCompleted && Configuration.isSameKycBlob(s.result, file.url), + ), }, ], }, @@ -1252,12 +1280,17 @@ export class Configuration { tenantId: process.env.AZURE_TENANT_ID, clientId: process.env.AZURE_CLIENT_ID, clientSecret: process.env.AZURE_CLIENT_SECRET, - storage: { - url: process.env.AZURE_STORAGE_CONNECTION_STRING?.split(';') - .find((p) => p.includes('BlobEndpoint')) - ?.replace('BlobEndpoint=', ''), - connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING, - }, + }; + + // S3-compatible object storage (on-prem MinIO) — replacement for Azure Blob. + // Connection only; WORM/Object-Lock retention is provisioned on the bucket itself. + // secrets -> .env; per-env endpoint/url -> compose. + s3 = { + endpoint: process.env.S3_ENDPOINT, + region: process.env.S3_REGION, + accessKey: process.env.S3_ACCESS_KEY, + secretKey: process.env.S3_SECRET_KEY, + publicUrl: process.env.S3_PUBLIC_URL, }; alby = { diff --git a/src/integration/infrastructure/azure-storage.service.ts b/src/integration/infrastructure/azure-storage.service.ts deleted file mode 100644 index 87613cf350..0000000000 --- a/src/integration/infrastructure/azure-storage.service.ts +++ /dev/null @@ -1,211 +0,0 @@ -import { BlobGetPropertiesResponse, BlobServiceClient, ContainerClient } from '@azure/storage-blob'; -import * as fs from 'fs'; -import * as path from 'path'; -import { Config, Environment, GetConfig } from 'src/config/config'; -import { DfxLogger } from 'src/shared/services/dfx-logger'; - -export interface BlobMetaData { - contentType: string; - created: Date; - updated: Date; - metadata: Record; -} - -export interface Blob extends BlobMetaData { - name: string; - url: string; -} - -export interface BlobContent extends BlobMetaData { - data: Buffer; -} - -// In-memory storage for local development -const mockStorage = new Map }>(); - -// Dummy files directory for local development -const DUMMY_FILES_DIR = path.join(process.cwd(), 'scripts', 'kyc', 'dummy-files'); - -// Load dummy file from disk -function loadDummyFile(filename: string): Buffer { - const filePath = path.join(DUMMY_FILES_DIR, filename); - return fs.readFileSync(filePath); -} - -export class AzureStorageService { - private readonly logger = new DfxLogger(AzureStorageService); - private readonly client: ContainerClient; - private readonly isMockMode: boolean; - - constructor(private readonly container: string) { - const config = GetConfig(); - this.isMockMode = config.environment === Environment.LOC; - - if (this.isMockMode) { - this.logger.verbose(`AzureStorageService [${container}] running in MOCK mode`); - return; - } - - const connectionString = config.azure.storage.connectionString; - if (connectionString) - this.client = BlobServiceClient.fromConnectionString(connectionString).getContainerClient(container); - } - - async listBlobs(prefix?: string): Promise { - if (this.isMockMode) { - const blobs: Blob[] = []; - const keyPrefix = `${this.container}/${prefix ?? ''}`; - for (const [key, value] of mockStorage.entries()) { - if (key.startsWith(keyPrefix)) { - const name = key.replace(`${this.container}/`, ''); - blobs.push({ - name, - url: this.blobUrl(name), - contentType: value.type, - created: new Date(), - updated: new Date(), - metadata: value.metadata ?? {}, - }); - } - } - return blobs; - } - - const iterator = this.client.listBlobsFlat({ prefix, includeMetadata: true }).byPage({ maxPageSize: 100 }); - - const blobs: Blob[] = []; - - let done = false; - while (!done) { - const batch = await iterator.next(); - - const items: Blob[] = batch.value?.segment?.blobItems?.map((i) => ({ - ...this.mapProperties(i.properties), - name: i.name, - url: this.blobUrl(i.name), - metadata: i.metadata, - })); - if (items) blobs.push(...items); - - done = batch.done; - } - - return blobs; - } - - async getBlob(name: string): Promise { - if (this.isMockMode) { - const key = `${this.container}/${name}`; - const stored = mockStorage.get(key); - - // Return stored data if available, otherwise return dummy test data based on file extension - if (stored) { - return { - data: stored.data, - contentType: stored.type, - created: new Date(), - updated: new Date(), - metadata: stored.metadata ?? {}, - }; - } - - // Provide dummy data for missing files in local dev mode - const ext = name.split('.').pop()?.toLowerCase(); - const filename = name.split('/').pop() ?? name; - - // Map common KYC file names to dummy files - const dummyFileMap: Record = { - 'id_front.png': { file: 'id_front.png', type: 'image/png' }, - 'id_back.png': { file: 'id_back.png', type: 'image/png' }, - 'selfie.jpg': { file: 'selfie.jpg', type: 'image/jpeg' }, - 'passport.png': { file: 'passport.png', type: 'image/png' }, - 'residence_permit.png': { file: 'residence_permit.png', type: 'image/png' }, - 'proof_of_address.pdf': { file: 'proof_of_address.pdf', type: 'application/pdf' }, - 'bank_statement.pdf': { file: 'bank_statement.pdf', type: 'application/pdf' }, - 'source_of_funds.pdf': { file: 'source_of_funds.pdf', type: 'application/pdf' }, - 'commercial_register.pdf': { file: 'commercial_register.pdf', type: 'application/pdf' }, - 'additional_document.pdf': { file: 'additional_document.pdf', type: 'application/pdf' }, - }; - - const mapping = dummyFileMap[filename]; - if (mapping) { - return { - data: loadDummyFile(mapping.file), - contentType: mapping.type, - created: new Date(), - updated: new Date(), - metadata: {}, - }; - } - - // Fallback based on extension - const isJpg = ext === 'jpg' || ext === 'jpeg'; - const isPdf = ext === 'pdf'; - return { - data: loadDummyFile(isPdf ? 'proof_of_address.pdf' : isJpg ? 'selfie.jpg' : 'id_front.png'), - contentType: isPdf ? 'application/pdf' : isJpg ? 'image/jpeg' : 'image/png', - created: new Date(), - updated: new Date(), - metadata: {}, - }; - } - - const blobClient = this.client.getBlockBlobClient(name); - const properties = await blobClient.getProperties(); - return { - ...this.mapProperties(properties), - data: await blobClient.downloadToBuffer(), - }; - } - - async uploadBlob(name: string, data: Buffer, type: string, metadata?: Record): Promise { - if (this.isMockMode) { - const key = `${this.container}/${name}`; - mockStorage.set(key, { data, type, metadata }); - this.logger.verbose(`Mock: Uploaded ${key} (${data.length} bytes)`); - return this.blobUrl(name); - } - - await this.client - .getBlockBlobClient(name) - .uploadData(data, { blobHTTPHeaders: { blobContentType: type }, metadata: !metadata ? undefined : metadata }); - - return this.blobUrl(name); - } - - async copyBlobs(sourcePrefix: string, targetPrefix: string): Promise { - const blobs = await this.listBlobs(sourcePrefix); - - for (const blob of blobs) { - const data = await this.getBlob(blob.name); - const targetName = blob.name.replace(sourcePrefix, targetPrefix); - await this.uploadBlob(targetName, data.data, data.contentType, blob.metadata); - } - } - - // Per-segment path encoding for blob URLs: each path segment is `encodeURIComponent`-encoded while - // the `/` separators are preserved. Shared with consumers that build host-stable variants of a blob - // URL (e.g. `KycDocumentService.toHostStableUrl`) so the encoding stays byte-identical apart from the - // host. - static encodePath(path: string): string { - return path.split('/').map(encodeURIComponent).join('/'); - } - - blobUrl(name: string): string { - return `${Config.azure.storage.url}${this.container}/${AzureStorageService.encodePath(name)}`; - } - - blobName(url: string): string { - const filePath = url.split(`${this.container}/`)[1]; - return filePath.split('/').map(decodeURIComponent).join('/'); - } - - private mapProperties(properties: BlobGetPropertiesResponse): BlobMetaData { - return { - contentType: properties.contentType, - created: properties.createdOn, - updated: properties.lastModified, - metadata: properties.metadata, - }; - } -} diff --git a/src/integration/infrastructure/storage/__tests__/mock-storage.service.spec.ts b/src/integration/infrastructure/storage/__tests__/mock-storage.service.spec.ts new file mode 100644 index 0000000000..e19ae0d290 --- /dev/null +++ b/src/integration/infrastructure/storage/__tests__/mock-storage.service.spec.ts @@ -0,0 +1,132 @@ +import { Test } from '@nestjs/testing'; +import { TestUtil } from 'src/shared/utils/test.util'; +import { MockStorageService } from '../mock-storage.service'; + +const CONTAINER = 'mock-spec'; + +describe('MockStorageService', () => { + beforeAll(async () => { + await Test.createTestingModule({ + providers: [TestUtil.provideConfig({ s3: { publicUrl: 'https://files.test.local/' } })], + }).compile(); + }); + + describe('upload / get roundtrip', () => { + it('stores and reads back data, type and metadata', async () => { + const service = new MockStorageService(CONTAINER); + const data = Buffer.from('payload'); + + const url = await service.uploadBlob('user/1/note.txt', data, 'text/plain', { owner: 'u1' }); + expect(url).toBe('https://files.test.local/mock-spec/user/1/note.txt'); + + const res = await service.getBlob('user/1/note.txt'); + expect(res.data).toBe(data); + expect(res.contentType).toBe('text/plain'); + expect(res.metadata).toEqual({ owner: 'u1' }); + expect(res.created).toBeInstanceOf(Date); + expect(res.updated).toBeInstanceOf(Date); + }); + + it('defaults metadata to {} when uploaded without metadata', async () => { + const service = new MockStorageService(CONTAINER); + await service.uploadBlob('user/1/plain.txt', Buffer.from('x'), 'text/plain'); + + expect((await service.getBlob('user/1/plain.txt')).metadata).toEqual({}); + }); + }); + + describe('listBlobs', () => { + it('filters by prefix and maps stored entries', async () => { + const service = new MockStorageService('mock-spec-list'); + await service.uploadBlob('user/1/a.png', Buffer.from('a'), 'image/png', { k: 'v' }); + await service.uploadBlob('user/1/b.pdf', Buffer.from('b'), 'application/pdf'); + await service.uploadBlob('user/2/c.png', Buffer.from('c'), 'image/png'); + + const blobs = await service.listBlobs('user/1/'); + + expect(blobs.map((b) => b.name).sort()).toEqual(['user/1/a.png', 'user/1/b.pdf']); + const a = blobs.find((b) => b.name === 'user/1/a.png'); + expect(a).toMatchObject({ + url: 'https://files.test.local/mock-spec-list/user/1/a.png', + contentType: 'image/png', + metadata: { k: 'v' }, + }); + expect(a.created).toBeInstanceOf(Date); + }); + + it('returns [] when nothing matches the prefix', async () => { + const service = new MockStorageService('mock-spec-empty'); + + expect(await service.listBlobs('nope/')).toEqual([]); + }); + + it('lists all entries of the container when no prefix is given', async () => { + const service = new MockStorageService('mock-spec-all'); + await service.uploadBlob('x.png', Buffer.from('x'), 'image/png'); + await service.uploadBlob('y.png', Buffer.from('y'), 'image/png'); + + expect((await service.listBlobs()).map((b) => b.name).sort()).toEqual(['x.png', 'y.png']); + }); + }); + + describe('getBlob dummy-file fallback', () => { + const service = new MockStorageService('mock-spec-dummy'); + + it('returns a mapped dummy file by basename', async () => { + const res = await service.getBlob('user/1/id_front.png'); + expect(res.contentType).toBe('image/png'); + expect(res.data.length).toBeGreaterThan(0); + expect(res.metadata).toEqual({}); + }); + + it('returns a mapped pdf dummy file', async () => { + const res = await service.getBlob('proof_of_address.pdf'); + expect(res.contentType).toBe('application/pdf'); + expect(res.data.length).toBeGreaterThan(0); + }); + + it('falls back by .pdf extension for an unmapped name', async () => { + const res = await service.getBlob('user/1/some-random.pdf'); + expect(res.contentType).toBe('application/pdf'); + expect(res.data.length).toBeGreaterThan(0); + }); + + it('falls back by .jpg extension for an unmapped name', async () => { + const res = await service.getBlob('user/1/holiday.jpg'); + expect(res.contentType).toBe('image/jpeg'); + expect(res.data.length).toBeGreaterThan(0); + }); + + it('falls back by .jpeg extension for an unmapped name', async () => { + const res = await service.getBlob('user/1/holiday.jpeg'); + expect(res.contentType).toBe('image/jpeg'); + }); + + it('falls back to png for any other / extensionless name', async () => { + const res = await service.getBlob('user/1/whatever'); + expect(res.contentType).toBe('image/png'); + expect(res.data.length).toBeGreaterThan(0); + }); + }); + + describe('copyBlobs', () => { + it('copies matching blobs with the prefix replaced', async () => { + const service = new MockStorageService('mock-spec-copy'); + await service.uploadBlob('src/a.png', Buffer.from('a'), 'image/png', { k: 'v' }); + await service.uploadBlob('src/sub/b.pdf', Buffer.from('b'), 'application/pdf'); + + await service.copyBlobs('src/', 'dst/'); + + const copied = await service.listBlobs('dst/'); + expect(copied.map((b) => b.name).sort()).toEqual(['dst/a.png', 'dst/sub/b.pdf']); + + const copiedA = await service.getBlob('dst/a.png'); + expect(copiedA.data.equals(Buffer.from('a'))).toBe(true); + expect(copiedA.contentType).toBe('image/png'); + expect(copiedA.metadata).toEqual({ k: 'v' }); + + // source remains untouched + expect((await service.listBlobs('src/')).map((b) => b.name).sort()).toEqual(['src/a.png', 'src/sub/b.pdf']); + }); + }); +}); diff --git a/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts b/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts new file mode 100644 index 0000000000..e28ce29ca3 --- /dev/null +++ b/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts @@ -0,0 +1,264 @@ +import { + CopyObjectCommand, + GetObjectCommand, + HeadObjectCommand, + ListObjectsV2Command, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import { Test } from '@nestjs/testing'; +import { mockClient } from 'aws-sdk-client-mock'; +import { TestUtil } from 'src/shared/utils/test.util'; +import { S3StorageService } from '../s3-storage.service'; + +const validS3 = { + endpoint: 'https://s3.test.local', + region: 'us-east-1', + accessKey: 'access-key', + secretKey: 'secret-key', + publicUrl: 'https://files.test.local/', // trailing slash required +}; + +const CONTAINER = 'kyc'; + +const s3Mock = mockClient(S3Client); + +async function provideConfig(s3: Partial = validS3): Promise { + await Test.createTestingModule({ + providers: [TestUtil.provideConfig({ s3 })], + }).compile(); +} + +// Minimal stream stub matching the parts the service consumes (transformToByteArray). +function bodyStream(bytes: Uint8Array): { transformToByteArray: () => Promise } { + return { transformToByteArray: async () => bytes }; +} + +describe('S3StorageService', () => { + beforeEach(async () => { + s3Mock.reset(); + await provideConfig(); + }); + + describe('constructor', () => { + it('builds with a complete config', () => { + expect(new S3StorageService(CONTAINER)).toBeInstanceOf(S3StorageService); + }); + + it('throws when endpoint is missing', async () => { + await provideConfig({ ...validS3, endpoint: undefined }); + expect(() => new S3StorageService(CONTAINER)).toThrow('Incomplete S3 config'); + }); + + it('throws when region is missing', async () => { + await provideConfig({ ...validS3, region: undefined }); + expect(() => new S3StorageService(CONTAINER)).toThrow('Incomplete S3 config'); + }); + + it('throws when accessKey is missing', async () => { + await provideConfig({ ...validS3, accessKey: undefined }); + expect(() => new S3StorageService(CONTAINER)).toThrow('Incomplete S3 config'); + }); + + it('throws when secretKey is missing', async () => { + await provideConfig({ ...validS3, secretKey: undefined }); + expect(() => new S3StorageService(CONTAINER)).toThrow('Incomplete S3 config'); + }); + + it('throws when publicUrl is missing', async () => { + await provideConfig({ ...validS3, publicUrl: undefined }); + expect(() => new S3StorageService(CONTAINER)).toThrow('Incomplete S3 config'); + }); + + it('throws when publicUrl has no trailing slash', async () => { + await provideConfig({ ...validS3, publicUrl: 'https://files.test.local' }); + expect(() => new S3StorageService(CONTAINER)).toThrow('must end with a trailing slash'); + }); + }); + + describe('listBlobs', () => { + it('paginates ListObjectsV2 and heads each key', async () => { + s3Mock + .on(ListObjectsV2Command) + .resolvesOnce({ Contents: [{ Key: 'user/1/a.png' }], IsTruncated: true, NextContinuationToken: 'tok-1' }) + .resolvesOnce({ Contents: [{ Key: 'user/1/b.pdf' }], IsTruncated: false }); + + const created = new Date('2026-01-01T00:00:00.000Z'); + s3Mock + .on(HeadObjectCommand, { Key: 'user/1/a.png' }) + .resolves({ ContentType: 'image/png', LastModified: created, Metadata: { kind: 'id' } }); + s3Mock + .on(HeadObjectCommand, { Key: 'user/1/b.pdf' }) + .resolves({ ContentType: 'application/pdf', LastModified: created, Metadata: { kind: 'poa' } }); + + const blobs = await new S3StorageService(CONTAINER).listBlobs('user/1/'); + + expect(blobs).toEqual([ + { + name: 'user/1/a.png', + url: 'https://files.test.local/kyc/user/1/a.png', + contentType: 'image/png', + created, + updated: created, + metadata: { kind: 'id' }, + }, + { + name: 'user/1/b.pdf', + url: 'https://files.test.local/kyc/user/1/b.pdf', + contentType: 'application/pdf', + created, + updated: created, + metadata: { kind: 'poa' }, + }, + ]); + + const listCalls = s3Mock.commandCalls(ListObjectsV2Command); + expect(listCalls).toHaveLength(2); + expect(listCalls[0].args[0].input).toMatchObject({ Bucket: CONTAINER, Prefix: 'user/1/', MaxKeys: 1000 }); + expect(listCalls[0].args[0].input.ContinuationToken).toBeUndefined(); + expect(listCalls[1].args[0].input.ContinuationToken).toBe('tok-1'); + expect(s3Mock.commandCalls(HeadObjectCommand)).toHaveLength(2); + }); + + it('defaults missing metadata to an empty object', async () => { + s3Mock.on(ListObjectsV2Command).resolves({ Contents: [{ Key: 'k' }], IsTruncated: false }); + s3Mock.on(HeadObjectCommand).resolves({ ContentType: 'image/png', LastModified: new Date() }); + + const [blob] = await new S3StorageService(CONTAINER).listBlobs(); + + expect(blob.metadata).toEqual({}); + }); + + it('skips entries without a Key and returns [] for an empty listing', async () => { + s3Mock.on(ListObjectsV2Command).resolves({ Contents: [{ Key: undefined }], IsTruncated: false }); + + expect(await new S3StorageService(CONTAINER).listBlobs()).toEqual([]); + expect(s3Mock.commandCalls(HeadObjectCommand)).toHaveLength(0); + }); + + it('returns [] when Contents is absent', async () => { + s3Mock.on(ListObjectsV2Command).resolves({ IsTruncated: false }); + + expect(await new S3StorageService(CONTAINER).listBlobs()).toEqual([]); + }); + }); + + describe('getBlob', () => { + it('returns the decoded body with metadata', async () => { + const bytes = new Uint8Array([1, 2, 3, 4]); + const created = new Date('2026-02-02T00:00:00.000Z'); + s3Mock.on(GetObjectCommand, { Bucket: CONTAINER, Key: 'doc.pdf' }).resolves({ + Body: bodyStream(bytes) as never, + ContentType: 'application/pdf', + LastModified: created, + Metadata: { source: 'kyc' }, + }); + + const res = await new S3StorageService(CONTAINER).getBlob('doc.pdf'); + + expect(res.data).toBeInstanceOf(Buffer); + expect(res.data.equals(Buffer.from(bytes))).toBe(true); + expect(res.contentType).toBe('application/pdf'); + expect(res.created).toBe(created); + expect(res.updated).toBe(created); + expect(res.metadata).toEqual({ source: 'kyc' }); + }); + + it('throws when the body is empty', async () => { + s3Mock.on(GetObjectCommand).resolves({ Body: undefined }); + + await expect(new S3StorageService(CONTAINER).getBlob('missing')).rejects.toThrow( + 'Empty body for blob kyc/missing', + ); + }); + }); + + describe('uploadBlob', () => { + it('sends a PutObjectCommand and returns the blob URL', async () => { + s3Mock.on(PutObjectCommand).resolves({}); + const data = Buffer.from('hello'); + + const url = await new S3StorageService(CONTAINER).uploadBlob('user/1/file.txt', data, 'text/plain', { + owner: 'u1', + }); + + expect(url).toBe('https://files.test.local/kyc/user/1/file.txt'); + const calls = s3Mock.commandCalls(PutObjectCommand); + expect(calls).toHaveLength(1); + expect(calls[0].args[0].input).toMatchObject({ + Bucket: CONTAINER, + Key: 'user/1/file.txt', + Body: data, + ContentType: 'text/plain', + Metadata: { owner: 'u1' }, + }); + }); + + it('passes undefined metadata through', async () => { + s3Mock.on(PutObjectCommand).resolves({}); + + await new S3StorageService(CONTAINER).uploadBlob('a.bin', Buffer.from('x'), 'application/octet-stream'); + + expect(s3Mock.commandCalls(PutObjectCommand)[0].args[0].input.Metadata).toBeUndefined(); + }); + }); + + describe('copyBlobs', () => { + it('URL-encodes keys with spaces / special chars and rewrites the prefix', async () => { + const key = 'src/sub dir/file (1)+&.png'; + s3Mock.on(ListObjectsV2Command).resolves({ Contents: [{ Key: key }], IsTruncated: false }); + s3Mock.on(CopyObjectCommand).resolves({}); + + await new S3StorageService(CONTAINER).copyBlobs('src/', 'dst/'); + + const calls = s3Mock.commandCalls(CopyObjectCommand); + expect(calls).toHaveLength(1); + expect(calls[0].args[0].input).toMatchObject({ + Bucket: CONTAINER, + Key: 'dst/sub dir/file (1)+&.png', + // path separators preserved, each segment percent-encoded + CopySource: 'kyc/src/sub%20dir/file%20(1)%2B%26.png', + }); + // copy must not fan out to HeadObject + expect(s3Mock.commandCalls(HeadObjectCommand)).toHaveLength(0); + }); + + it('copies every listed key', async () => { + s3Mock.on(ListObjectsV2Command).resolves({ Contents: [{ Key: 'src/a' }, { Key: 'src/b' }], IsTruncated: false }); + s3Mock.on(CopyObjectCommand).resolves({}); + + await new S3StorageService(CONTAINER).copyBlobs('src/', 'dst/'); + + const keys = s3Mock.commandCalls(CopyObjectCommand).map((c) => c.args[0].input.Key); + expect(keys).toEqual(['dst/a', 'dst/b']); + }); + + it('does nothing for an empty source prefix', async () => { + s3Mock.on(ListObjectsV2Command).resolves({ Contents: [], IsTruncated: false }); + + await new S3StorageService(CONTAINER).copyBlobs('src/', 'dst/'); + + expect(s3Mock.commandCalls(CopyObjectCommand)).toHaveLength(0); + }); + }); + + describe('blobUrl / blobName roundtrip (inherited)', () => { + it('builds a URL and reverses it, including encoded segments', () => { + const service = new S3StorageService(CONTAINER); + const name = 'user/1/my file.png'; + + const url = service.blobUrl(name); + + expect(url).toBe('https://files.test.local/kyc/user/1/my%20file.png'); + expect(service.blobName(url)).toBe(name); + }); + + it('throws blobName for a URL outside the container', () => { + const service = new S3StorageService(CONTAINER); + + expect(() => service.blobName('https://files.test.local/other/x.png')).toThrow( + 'URL does not belong to container kyc', + ); + }); + }); +}); diff --git a/src/integration/infrastructure/storage/__tests__/storage.factory.spec.ts b/src/integration/infrastructure/storage/__tests__/storage.factory.spec.ts new file mode 100644 index 0000000000..69dfc2ab07 --- /dev/null +++ b/src/integration/infrastructure/storage/__tests__/storage.factory.spec.ts @@ -0,0 +1,57 @@ +import { Test } from '@nestjs/testing'; +import { Environment } from 'src/config/config'; +import { TestUtil } from 'src/shared/utils/test.util'; +import { MockStorageService } from '../mock-storage.service'; +import { S3StorageService } from '../s3-storage.service'; +import { createStorageService } from '../storage.factory'; + +const validS3 = { + endpoint: 'https://s3.test.local', + region: 'us-east-1', + accessKey: 'access-key', + secretKey: 'secret-key', + publicUrl: 'https://files.test.local/', +}; + +// The factory branches on `GetConfig().environment`, which reads process.env.ENVIRONMENT +// directly (a fresh Configuration), so the environment is driven via the env var here. +// The injected ConfigService still supplies the global `Config` (s3 block) used by the +// S3StorageService constructor. +async function provideConfig(environment: Environment, s3 = validS3): Promise { + process.env.ENVIRONMENT = environment; + await Test.createTestingModule({ + providers: [TestUtil.provideConfig({ environment, s3 })], + }).compile(); +} + +describe('createStorageService', () => { + const originalEnvironment = process.env.ENVIRONMENT; + + afterAll(() => { + process.env.ENVIRONMENT = originalEnvironment; + }); + + it('returns a MockStorageService for the LOC environment', async () => { + await provideConfig(Environment.LOC); + + expect(createStorageService('kyc')).toBeInstanceOf(MockStorageService); + }); + + it('returns an S3StorageService for the DEV environment', async () => { + await provideConfig(Environment.DEV); + + expect(createStorageService('kyc')).toBeInstanceOf(S3StorageService); + }); + + it('returns an S3StorageService for the PRD environment', async () => { + await provideConfig(Environment.PRD); + + expect(createStorageService('kyc')).toBeInstanceOf(S3StorageService); + }); + + it('fails fast on an incomplete S3 config in a non-LOC environment', async () => { + await provideConfig(Environment.DEV, { ...validS3, endpoint: undefined }); + + expect(() => createStorageService('kyc')).toThrow('Incomplete S3 config'); + }); +}); diff --git a/src/integration/infrastructure/storage/__tests__/storage.service.spec.ts b/src/integration/infrastructure/storage/__tests__/storage.service.spec.ts new file mode 100644 index 0000000000..4d7f4ffca2 --- /dev/null +++ b/src/integration/infrastructure/storage/__tests__/storage.service.spec.ts @@ -0,0 +1,63 @@ +import { Test } from '@nestjs/testing'; +import { TestUtil } from 'src/shared/utils/test.util'; +import { MockStorageService } from '../mock-storage.service'; + +describe('StorageService', () => { + let service: MockStorageService; + + beforeAll(async () => { + await Test.createTestingModule({ + providers: [TestUtil.provideConfig({ s3: { publicUrl: 'https://files.example.com/' } })], + }).compile(); + + service = new MockStorageService('kyc'); + }); + + describe('blobUrl / blobName round-trip', () => { + // This reversibility is the load-bearing migration invariant: URLs produced by + // uploadBlob are persisted in the DB and must decode back to the exact object key. + const keys = [ + 'user/123/Identification/id_front.png', + 'spider/9/report.pdf', + 'user/1/notes/file with spaces.pdf', + 'user/2/notes/special #&+%.png', + 'user/3/Ausweis/Gruesse_Uemlaeuet.png', + ]; + + it.each(keys)('reverses %s exactly', (key) => { + expect(service.blobName(service.blobUrl(key))).toBe(key); + }); + + it('builds the public URL from the configured base with encoded segments', () => { + expect(service.blobUrl('a b/c#d')).toBe('https://files.example.com/kyc/a%20b/c%23d'); + }); + + it('rejects a URL that does not belong to the container', () => { + expect(() => service.blobName('https://example.com/other/file.png')).toThrow(); + }); + }); + + describe('blobName is host-independent (Azure <-> MinIO cutover invariant)', () => { + // blobName() splits on `/` and is therefore prefix/host-independent: a value + // persisted while the backend was Azure Blob must still decode to the same object key when + // the same URL shape is later served from MinIO. This is what makes stored kyc_step.result + // URLs (captured on Azure) still match live regenerated URLs (MinIO) after the cutover. + const key = 'user/42/CommercialRegister/HR Auszug.pdf'; + const encoded = 'user/42/CommercialRegister/HR%20Auszug.pdf'; + + const legacyAzureUrl = `https://dfxstorageprd.blob.core.windows.net/kyc/${encoded}`; + const liveMinioUrl = `https://files.dfx.swiss/kyc/${encoded}`; + + it('decodes a legacy Azure-host URL to the container-relative key', () => { + expect(service.blobName(legacyAzureUrl)).toBe(key); + }); + + it('decodes a new MinIO-host URL to the same container-relative key', () => { + expect(service.blobName(liveMinioUrl)).toBe(key); + }); + + it('resolves both host forms to an identical key', () => { + expect(service.blobName(legacyAzureUrl)).toBe(service.blobName(liveMinioUrl)); + }); + }); +}); diff --git a/src/integration/infrastructure/storage/anchoring/__tests__/archive.scheduler.spec.ts b/src/integration/infrastructure/storage/anchoring/__tests__/archive.scheduler.spec.ts new file mode 100644 index 0000000000..66dcb10ecf --- /dev/null +++ b/src/integration/infrastructure/storage/anchoring/__tests__/archive.scheduler.spec.ts @@ -0,0 +1,58 @@ +// Stub the heavy `opentimestamps` library (pulled in transitively via ArchiveService) so its +// eager network/`request` deps never load; ArchiveService is fully mocked in this spec anyway. +jest.mock('opentimestamps', () => ({})); + +import { createMock } from '@golevelup/ts-jest'; +import { Test, TestingModule } from '@nestjs/testing'; +import { ArchiveBatch } from '../archive-batch.entity'; +import { ArchiveScheduler } from '../archive.scheduler'; +import { ArchiveService } from '../archive.service'; + +describe('ArchiveScheduler', () => { + let scheduler: ArchiveScheduler; + let archiveService: ArchiveService; + + beforeEach(async () => { + archiveService = createMock(); + + const module: TestingModule = await Test.createTestingModule({ + providers: [ArchiveScheduler, { provide: ArchiveService, useValue: archiveService }], + }).compile(); + + scheduler = module.get(ArchiveScheduler); + }); + + it('should be defined', () => { + expect(scheduler).toBeDefined(); + }); + + describe('anchorPending', () => { + it('delegates to ArchiveService.anchorPending', async () => { + (archiveService.anchorPending as jest.Mock).mockResolvedValue({ id: 1, merkleRoot: 'ab' } as ArchiveBatch); + + await scheduler.anchorPending(); + + expect(archiveService.anchorPending).toHaveBeenCalledTimes(1); + }); + + it('swallows errors so a failure never crashes the scheduler', async () => { + (archiveService.anchorPending as jest.Mock).mockRejectedValue(new Error('calendar down')); + + await expect(scheduler.anchorPending()).resolves.toBeUndefined(); + }); + }); + + describe('upgradeBatches', () => { + it('delegates to ArchiveService.upgradeBatches', async () => { + await scheduler.upgradeBatches(); + + expect(archiveService.upgradeBatches).toHaveBeenCalledTimes(1); + }); + + it('swallows errors so a failure never crashes the scheduler', async () => { + (archiveService.upgradeBatches as jest.Mock).mockRejectedValue(new Error('upgrade failed')); + + await expect(scheduler.upgradeBatches()).resolves.toBeUndefined(); + }); + }); +}); diff --git a/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts b/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts new file mode 100644 index 0000000000..0e514ccc7d --- /dev/null +++ b/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts @@ -0,0 +1,345 @@ +// Stub the heavy `opentimestamps` library so its eager network/`request` deps never load: +// this spec mocks OpenTimestampsService entirely, so the real implementation is never used. +jest.mock('opentimestamps', () => ({})); + +import { createMock } from '@golevelup/ts-jest'; +import { Test, TestingModule } from '@nestjs/testing'; +import { ArchiveBatch } from '../archive-batch.entity'; +import { ArchiveBatchRepository } from '../archive-batch.repository'; +import { ArchiveFile } from '../archive-file.entity'; +import { ArchiveFileRepository } from '../archive-file.repository'; +import { ArchiveService } from '../archive.service'; +import { sha256 } from '../merkle'; +import { OpenTimestampsService } from '../opentimestamps.service'; + +/** + * A shared in-memory store backing both fake repositories. It reproduces just enough TypeORM + * behaviour (auto-increment ids, `(bucket, name)` upsert, batch-null filtering, ordering and + * `manager.transaction`) for the round-trip the service performs, so that saves inside the + * transaction the service runs on the batch repo's manager land in the same `files`/`batches` + * arrays the assertions inspect. The Merkle math is the REAL Stage-1 module; only + * OpenTimestamps is mocked so no network is touched. + */ +function fakeStore() { + const files: ArchiveFile[] = []; + const batches: ArchiveBatch[] = []; + let fileSeq = 0; + let batchSeq = 0; + + // dispatch a single entity (or array) by type, assigning ids on first save. + const saveEntity = (entity: any): any => { + const list = Array.isArray(entity) ? entity : [entity]; + for (const item of list) { + if (item instanceof ArchiveBatch) { + if (!item.id) { + item.id = ++batchSeq; + batches.push(item); + } + } else { + if (!item.id) { + item.id = ++fileSeq; + files.push(item); + } + } + } + return entity; + }; + + const manager = { + save: async (entity: any) => saveEntity(entity), + transaction: async (run: (manager: any) => Promise) => run(manager), + }; + + const fileRepo: any = { + files, + manager, + create: (data: Partial) => Object.assign(new ArchiveFile(), data), + save: async (entity: ArchiveFile | ArchiveFile[]) => saveEntity(entity), + update: async (id: number, partial: Partial) => { + const file = files.find((f) => f.id === id); + if (file) Object.assign(file, partial); + }, + findOneBy: async (where: Partial) => + files.find((f) => f.bucket === where.bucket && f.name === where.name) ?? undefined, + // supports lookup by id or by (bucket, name). Faithful to TypeORM: the `batch` relation is + // ONLY hydrated when the caller explicitly asks for it via `relations: ['batch']`. Without + // that, `batch` is left undefined on the returned row, so a caller that drops the relation + // would see `existing.batch === undefined` here too (and the anchored-hash guard would fail + // its test) instead of silently passing on the in-memory reference. + findOne: async ({ where, relations }: any) => { + const match = files.find((f) => + where.id != null ? f.id === where.id : f.bucket === where.bucket && f.name === where.name, + ); + if (!match) return undefined; + + const wantsBatch = Array.isArray(relations) ? relations.includes('batch') : !!relations?.batch; + // Return a shallow copy so we can withhold `batch` without mutating the stored entity. + const result = Object.assign(new ArchiveFile(), match); + if (!wantsBatch) result.batch = undefined; + return result; + }, + find: async ({ where, order }: any) => { + let result = [...files]; + + if (where?.batch !== undefined) { + // TypeORM IsNull() carries `_type === 'isNull'`; otherwise we match a batch id. + const wantsNull = where.batch?._type === 'isNull' || where.batch === null; + if (wantsNull) result = result.filter((f) => f.batch == null); + else if (where.batch?.id != null) result = result.filter((f) => f.batch?.id === where.batch.id); + } + + if (order?.id === 'ASC') result.sort((a, b) => a.id - b.id); + if (order?.leafIndex === 'ASC') result.sort((a, b) => a.leafIndex - b.leafIndex); + + return result; + }, + }; + + const batchRepo: any = { + batches, + manager, + create: (data: Partial) => Object.assign(new ArchiveBatch(), data), + save: async (entity: ArchiveBatch) => saveEntity(entity), + findBy: async (where: Partial) => batches.filter((b) => b.status === where.status), + }; + + return { fileRepo, batchRepo }; +} + +describe('ArchiveService', () => { + let service: ArchiveService; + let fileRepo: any; + let batchRepo: any; + let ots: OpenTimestampsService; + + beforeEach(async () => { + ({ fileRepo, batchRepo } = fakeStore()); + + ots = createMock(); + // Deterministic, network-free fakes: the .ots bytes just wrap the root; pending forever. + (ots.stamp as jest.Mock).mockImplementation(async (digest: Buffer) => Buffer.concat([Buffer.from('OTS'), digest])); + (ots.upgrade as jest.Mock).mockImplementation(async (bytes: Buffer) => bytes); + (ots.verify as jest.Mock).mockResolvedValue({ confirmed: false, pending: true }); + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + ArchiveService, + { provide: ArchiveFileRepository, useValue: fileRepo }, + { provide: ArchiveBatchRepository, useValue: batchRepo }, + { provide: OpenTimestampsService, useValue: ots }, + ], + }).compile(); + + service = module.get(ArchiveService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('round-trip', () => { + const docs = [ + { bucket: 'archive', name: 'doc-a.pdf', data: Buffer.from('content of document A') }, + { bucket: 'archive', name: 'doc-b.pdf', data: Buffer.from('content of document B') }, + { bucket: 'archive', name: 'doc-c.pdf', data: Buffer.from('content of document C') }, + ]; + + beforeEach(async () => { + for (const doc of docs) { + await service.recordHash(doc.bucket, doc.name, sha256(doc.data).toString('hex')); + } + }); + + it('records all files unanchored', () => { + expect(fileRepo.files).toHaveLength(3); + expect(fileRepo.files.every((f: ArchiveFile) => f.batch == null)).toBe(true); + }); + + it('upserts idempotently on (bucket, name)', async () => { + await service.recordHash('archive', 'doc-a.pdf', sha256(Buffer.from('updated A')).toString('hex')); + + expect(fileRepo.files).toHaveLength(3); + const fileA = fileRepo.files.find((f: ArchiveFile) => f.name === 'doc-a.pdf'); + expect(fileA.sha256).toBe(sha256(Buffer.from('updated A')).toString('hex')); + }); + + it('anchors pending files into a batch and stamps the root', async () => { + const batch = await service.anchorPending(); + + expect(batch).toBeDefined(); + expect(batch.merkleRoot).toMatch(/^[0-9a-f]{64}$/); + expect(batch.status).toBe('pendingBtc'); + expect(batch.otsProof).toBeDefined(); + expect(ots.stamp).toHaveBeenCalledTimes(1); + + // every file got assigned to the batch with a leaf index + expect(fileRepo.files.every((f: ArchiveFile) => f.batch?.id === batch.id)).toBe(true); + expect(fileRepo.files.map((f: ArchiveFile) => f.leafIndex).sort()).toEqual([0, 1, 2]); + }); + + it('returns undefined when there is nothing to anchor', async () => { + await service.anchorPending(); + const second = await service.anchorPending(); + expect(second).toBeUndefined(); + }); + + it('verifies an unchanged, anchored document (real Merkle proof)', async () => { + await service.anchorPending(); + + const result = await service.verifyDocument('archive', 'doc-b.pdf', docs[1].data); + + expect(result.found).toBe(true); + expect(result.hashMatches).toBe(true); + expect(result.anchored).toBe(true); + expect(result.proofValid).toBe(true); + expect(result.pending).toBe(true); + expect(result.bitcoinHeight).toBeUndefined(); + }); + + it('detects tampered data via hash mismatch', async () => { + await service.anchorPending(); + + const result = await service.verifyDocument('archive', 'doc-b.pdf', Buffer.from('tampered content')); + + expect(result.found).toBe(true); + expect(result.hashMatches).toBe(false); + // the stored hash is still genuinely anchored, only the supplied bytes differ + expect(result.anchored).toBe(true); + expect(result.proofValid).toBe(true); + }); + + it('reports an unanchored file as anchored:false', async () => { + const result = await service.verifyDocument('archive', 'doc-a.pdf', docs[0].data); + + expect(result.found).toBe(true); + expect(result.hashMatches).toBe(true); + expect(result.anchored).toBe(false); + expect(result.proofValid).toBeUndefined(); + }); + + it('reports an unknown document as found:false', async () => { + const result = await service.verifyDocument('archive', 'missing.pdf', Buffer.from('whatever')); + + expect(result.found).toBe(false); + }); + + it('confirms a batch once OpenTimestamps reports a Bitcoin attestation', async () => { + await service.anchorPending(); + + (ots.verify as jest.Mock).mockResolvedValue({ confirmed: true, pending: false, bitcoin: { height: 840000 } }); + + await service.upgradeBatches(); + + expect(batchRepo.batches[0].status).toBe('confirmed'); + expect(batchRepo.batches[0].bitcoinHeight).toBe(840000); + }); + + it('loads the batch relation when recording and verifying (guards the anchored-hash check)', async () => { + // The anchored-hash guard in recordHash and the anchored-branch in verifyDocument both + // depend on `existing.batch`/`file.batch` being populated, which only happens when the + // query explicitly requests `relations: ['batch']`. Assert the option is actually passed, + // so dropping it in production (which would let an anchored leaf be silently overwritten) + // breaks this test rather than passing unnoticed. + const findOneSpy = jest.spyOn(fileRepo, 'findOne'); + + await service.recordHash('archive', 'doc-a.pdf', sha256(docs[0].data).toString('hex')); + await service.verifyDocument('archive', 'doc-a.pdf', docs[0].data); + + expect(findOneSpy).toHaveBeenCalled(); + for (const call of findOneSpy.mock.calls) { + expect(call[0]).toMatchObject({ relations: ['batch'] }); + } + }); + + it('refuses to overwrite an anchored hash with a differing hash (avoids bogus tampering)', async () => { + await service.anchorPending(); + + const anchored = fileRepo.files.find((f: ArchiveFile) => f.name === 'doc-a.pdf'); + const originalHash = anchored.sha256; + + await expect( + service.recordHash('archive', 'doc-a.pdf', sha256(Buffer.from('re-uploaded A')).toString('hex')), + ).rejects.toThrow(/Refusing to overwrite anchored hash/); + + // the anchored leaf hash is untouched + expect(anchored.sha256).toBe(originalHash); + }); + + it('is a no-op when recording the same hash on an already-anchored file', async () => { + await service.anchorPending(); + + const anchored = fileRepo.files.find((f: ArchiveFile) => f.name === 'doc-a.pdf'); + const originalHash = anchored.sha256; + + await expect(service.recordHash('archive', 'doc-a.pdf', originalHash)).resolves.toBeUndefined(); + + expect(anchored.sha256).toBe(originalHash); + expect(fileRepo.files).toHaveLength(3); + }); + + it('persists upgraded proof bytes even while verify still reports pending', async () => { + await service.anchorPending(); + + const batch = batchRepo.batches[0]; + const originalProof = batch.otsProof; + + // upgrade yields changed bytes, but the attestation is not yet on-chain + (ots.upgrade as jest.Mock).mockResolvedValueOnce(Buffer.from('UPGRADED-BUT-PENDING')); + (ots.verify as jest.Mock).mockResolvedValueOnce({ confirmed: false, pending: true }); + + await service.upgradeBatches(); + + expect(batch.otsProof).toBe(Buffer.from('UPGRADED-BUT-PENDING').toString('base64')); + expect(batch.otsProof).not.toBe(originalProof); + // still not confirmed + expect(batch.status).toBe('pendingBtc'); + expect(batch.bitcoinHeight).toBeUndefined(); + }); + + it('does not save a batch when upgrade is unchanged and still pending', async () => { + await service.anchorPending(); + + const batch = batchRepo.batches[0]; + const saveSpy = jest.spyOn(batchRepo, 'save'); + + // upgrade returns the same bytes, verify still pending => nothing to persist + await service.upgradeBatches(); + + expect(saveSpy).not.toHaveBeenCalled(); + expect(batch.status).toBe('pendingBtc'); + }); + + it('skips a pending batch that carries no OTS proof (never touches the library)', async () => { + await service.anchorPending(); + + // simulate a malformed/empty proof on the only pending batch + const batch = batchRepo.batches[0]; + batch.otsProof = null; + + const saveSpy = jest.spyOn(batchRepo, 'save'); + + await service.upgradeBatches(); + + expect(saveSpy).not.toHaveBeenCalled(); + expect(ots.upgrade).not.toHaveBeenCalled(); + expect(ots.verify).not.toHaveBeenCalled(); + expect(batch.status).toBe('pendingBtc'); + }); + + it('reports the Bitcoin height when verifying a document whose batch is confirmed on-chain', async () => { + await service.anchorPending(); + + // the proof now carries a Bitcoin attestation + (ots.verify as jest.Mock).mockResolvedValue({ confirmed: true, pending: false, bitcoin: { height: 850000 } }); + + const result = await service.verifyDocument('archive', 'doc-b.pdf', docs[1].data); + + expect(result.found).toBe(true); + expect(result.hashMatches).toBe(true); + expect(result.anchored).toBe(true); + expect(result.proofValid).toBe(true); + expect(result.pending).toBe(false); + expect(result.bitcoinHeight).toBe(850000); + }); + }); +}); diff --git a/src/integration/infrastructure/storage/anchoring/__tests__/merkle.spec.ts b/src/integration/infrastructure/storage/anchoring/__tests__/merkle.spec.ts new file mode 100644 index 0000000000..b50ab59b5e --- /dev/null +++ b/src/integration/infrastructure/storage/anchoring/__tests__/merkle.spec.ts @@ -0,0 +1,112 @@ +import { buildMerkleRoot, merkleInclusionProof, sha256, verifyMerkleProof } from '../merkle'; + +/** Deterministic leaf: sha256 of `leaf-` so tests don't depend on random data. */ +function leaf(i: number): Buffer { + return sha256(Buffer.from(`leaf-${i}`)); +} + +function leaves(count: number): Buffer[] { + return Array.from({ length: count }, (_, i) => leaf(i)); +} + +/** Reference parent hash matching the module's documented `sha256(left || right)` rule. */ +function parent(left: Buffer, right: Buffer): Buffer { + return sha256(Buffer.concat([left, right])); +} + +describe('merkle', () => { + describe('buildMerkleRoot', () => { + it('throws on zero leaves', () => { + expect(() => buildMerkleRoot([])).toThrow(); + }); + + it('returns the leaf itself for a single-leaf tree', () => { + const l = leaf(0); + expect(buildMerkleRoot([l]).equals(l)).toBe(true); + }); + + it('hashes the pair for two leaves', () => { + const [a, b] = leaves(2); + expect(buildMerkleRoot([a, b]).equals(parent(a, b))).toBe(true); + }); + + it('duplicates the last node for three leaves', () => { + const [a, b, c] = leaves(3); + // level 1: [h(a,b), h(c,c)] -> root: h( h(a,b), h(c,c) ) + const expected = parent(parent(a, b), parent(c, c)); + expect(buildMerkleRoot([a, b, c]).equals(expected)).toBe(true); + }); + + it('builds a balanced tree for four leaves', () => { + const [a, b, c, d] = leaves(4); + const expected = parent(parent(a, b), parent(c, d)); + expect(buildMerkleRoot([a, b, c, d]).equals(expected)).toBe(true); + }); + + it('is deterministic across repeated calls', () => { + const ls = leaves(4); + expect(buildMerkleRoot(ls).equals(buildMerkleRoot(ls))).toBe(true); + }); + }); + + describe('inclusion proof + verification', () => { + for (let size = 1; size <= 5; size++) { + it(`verifies every leaf in a tree of size ${size}`, () => { + const ls = leaves(size); + const root = buildMerkleRoot(ls); + + for (let index = 0; index < size; index++) { + const proof = merkleInclusionProof(ls, index); + expect(verifyMerkleProof(ls[index], proof, root)).toBe(true); + } + }); + } + + it('produces an empty proof for a single-leaf tree', () => { + const ls = leaves(1); + expect(merkleInclusionProof(ls, 0)).toEqual([]); + }); + + it('throws for an out-of-range index', () => { + const ls = leaves(3); + expect(() => merkleInclusionProof(ls, 3)).toThrow(); + expect(() => merkleInclusionProof(ls, -1)).toThrow(); + }); + + it('throws for a proof over zero leaves', () => { + expect(() => merkleInclusionProof([], 0)).toThrow(); + }); + }); + + describe('tamper detection', () => { + const ls = leaves(4); + const root = buildMerkleRoot(ls); + const index = 1; + const proof = merkleInclusionProof(ls, index); + + it('rejects a manipulated leaf', () => { + const tampered = sha256(Buffer.from('not-the-original-leaf')); + expect(verifyMerkleProof(tampered, proof, root)).toBe(false); + }); + + it('rejects a tampered sibling in the proof', () => { + const badProof = proof.map((s, i) => (i === 0 ? { ...s, sibling: sha256(Buffer.from('wrong')) } : s)); + expect(verifyMerkleProof(ls[index], badProof, root)).toBe(false); + }); + + it('rejects a flipped left/right position', () => { + const badProof = proof.map((s) => ({ ...s, right: !s.right })); + expect(verifyMerkleProof(ls[index], badProof, root)).toBe(false); + }); + + it('rejects verification against a wrong root', () => { + const wrongRoot = sha256(Buffer.from('some-other-root')); + expect(verifyMerkleProof(ls[index], proof, wrongRoot)).toBe(false); + }); + + it("rejects another leaf's proof for this leaf", () => { + const otherProof = merkleInclusionProof(ls, 2); + expect(verifyMerkleProof(ls[index], otherProof, root)).toBe(false); + }); + }); +}); diff --git a/src/integration/infrastructure/storage/anchoring/__tests__/opentimestamps.service.spec.ts b/src/integration/infrastructure/storage/anchoring/__tests__/opentimestamps.service.spec.ts new file mode 100644 index 0000000000..ef569aa55b --- /dev/null +++ b/src/integration/infrastructure/storage/anchoring/__tests__/opentimestamps.service.spec.ts @@ -0,0 +1,176 @@ +// Mock the `opentimestamps` npm library so stamp/upgrade/verify can be exercised WITHOUT any +// network access. We mock exactly the surface OpenTimestampsService touches: +// - OpenTimestamps.stamp / .upgrade / .verify (promise-returning network calls) +// - OpenTimestamps.DetachedTimestampFile.fromHash (sync constructor from a digest) +// - OpenTimestamps.DetachedTimestampFile.deserialize (sync constructor from .ots bytes) +// - OpenTimestamps.Ops.OpSHA256 (sync op constructor) +// Each fake DetachedTimestampFile carries a `serializeToBytes()` returning a recognizable byte +// array so we can assert on the bytes the service returns. + +const stampMock = jest.fn(); +const upgradeMock = jest.fn(); +const verifyMock = jest.fn(); +const fromHashMock = jest.fn(); +const deserializeMock = jest.fn(); +const opSHA256Mock = jest.fn(); + +jest.mock('opentimestamps', () => ({ + stamp: (...args: any[]) => stampMock(...args), + upgrade: (...args: any[]) => upgradeMock(...args), + verify: (...args: any[]) => verifyMock(...args), + DetachedTimestampFile: { + fromHash: (...args: any[]) => fromHashMock(...args), + deserialize: (...args: any[]) => deserializeMock(...args), + }, + Ops: { + OpSHA256: function OpSHA256(this: any) { + opSHA256Mock(); + }, + }, +})); + +import { Test, TestingModule } from '@nestjs/testing'; +import { OpenTimestampsService } from '../opentimestamps.service'; + +/** A fake DetachedTimestampFile whose serialized form is deterministic for assertions. */ +function fakeDetached(serialized: number[]) { + return { + serializeToBytes: jest.fn(() => serialized), + }; +} + +describe('OpenTimestampsService', () => { + let service: OpenTimestampsService; + + beforeEach(async () => { + jest.clearAllMocks(); + + const module: TestingModule = await Test.createTestingModule({ + providers: [OpenTimestampsService], + }).compile(); + + service = module.get(OpenTimestampsService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('stamp', () => { + it('builds a detached file from the digest, submits it and returns the serialized .ots bytes', async () => { + const digest = Buffer.alloc(32, 7); + const serialized = [1, 2, 3, 4]; + const detached = fakeDetached(serialized); + fromHashMock.mockReturnValue(detached); + stampMock.mockResolvedValue(undefined); + + const result = await service.stamp(digest); + + // a SHA-256 op was constructed and fromHash was called with that op + the raw digest + expect(opSHA256Mock).toHaveBeenCalledTimes(1); + expect(fromHashMock).toHaveBeenCalledTimes(1); + const [, passedDigest] = fromHashMock.mock.calls[0]; + expect(passedDigest).toBe(digest); + + // the library stamp was invoked on exactly that detached file + expect(stampMock).toHaveBeenCalledTimes(1); + expect(stampMock).toHaveBeenCalledWith(detached); + + // and the returned bytes are a Buffer wrapping the serialized form + expect(Buffer.isBuffer(result)).toBe(true); + expect(result).toEqual(Buffer.from(serialized)); + expect(detached.serializeToBytes).toHaveBeenCalledTimes(1); + }); + + it('propagates a calendar/network failure from the library', async () => { + fromHashMock.mockReturnValue(fakeDetached([0])); + stampMock.mockRejectedValue(new Error('calendar unreachable')); + + await expect(service.stamp(Buffer.alloc(32))).rejects.toThrow('calendar unreachable'); + }); + }); + + describe('upgrade', () => { + it('returns the re-serialized bytes when the proof changed', async () => { + const otsBytes = Buffer.from([9, 9, 9]); + const upgradedSerialized = [5, 6, 7, 8]; + const detached = fakeDetached(upgradedSerialized); + deserializeMock.mockReturnValue(detached); + upgradeMock.mockResolvedValue(true); // changed + + const result = await service.upgrade(otsBytes); + + expect(deserializeMock).toHaveBeenCalledWith(otsBytes); + expect(upgradeMock).toHaveBeenCalledWith(detached); + expect(detached.serializeToBytes).toHaveBeenCalledTimes(1); + expect(result).toEqual(Buffer.from(upgradedSerialized)); + // genuinely new bytes, not the original + expect(result.equals(otsBytes)).toBe(false); + }); + + it('returns the original bytes unchanged when nothing changed', async () => { + const otsBytes = Buffer.from([9, 9, 9]); + const detached = fakeDetached([1, 1, 1]); + deserializeMock.mockReturnValue(detached); + upgradeMock.mockResolvedValue(false); // unchanged + + const result = await service.upgrade(otsBytes); + + // exact same buffer reference is returned, serialize is never consulted + expect(result).toBe(otsBytes); + expect(detached.serializeToBytes).not.toHaveBeenCalled(); + }); + }); + + describe('verify', () => { + it('reports confirmed with the Bitcoin height when an attestation is present', async () => { + const digest = Buffer.alloc(32, 1); + const otsBytes = Buffer.from([4, 2]); + const detachedOts = fakeDetached([]); + const detachedDigest = fakeDetached([]); + deserializeMock.mockReturnValue(detachedOts); + fromHashMock.mockReturnValue(detachedDigest); + verifyMock.mockResolvedValue({ bitcoin: { height: 840123, timestamp: 1700000000 } }); + + const result = await service.verify(digest, otsBytes); + + // proof deserialized, digest re-derived, and verify run against the trusted explorers + expect(deserializeMock).toHaveBeenCalledWith(otsBytes); + expect(fromHashMock).toHaveBeenCalledTimes(1); + expect(verifyMock).toHaveBeenCalledWith(detachedOts, detachedDigest, { ignoreBitcoinNode: true }); + + expect(result).toEqual({ bitcoin: { height: 840123 }, confirmed: true, pending: false }); + }); + + it('reports pending when the library returns no attestation (undefined result)', async () => { + deserializeMock.mockReturnValue(fakeDetached([])); + fromHashMock.mockReturnValue(fakeDetached([])); + verifyMock.mockResolvedValue(undefined); + + const result = await service.verify(Buffer.alloc(32), Buffer.from([1])); + + expect(result).toEqual({ confirmed: false, pending: true }); + expect(result.bitcoin).toBeUndefined(); + }); + + it('reports pending when the result has no bitcoin attestation', async () => { + deserializeMock.mockReturnValue(fakeDetached([])); + fromHashMock.mockReturnValue(fakeDetached([])); + verifyMock.mockResolvedValue({}); + + const result = await service.verify(Buffer.alloc(32), Buffer.from([1])); + + expect(result).toEqual({ confirmed: false, pending: true }); + }); + + it('reports pending when the bitcoin field lacks a numeric height', async () => { + deserializeMock.mockReturnValue(fakeDetached([])); + fromHashMock.mockReturnValue(fakeDetached([])); + verifyMock.mockResolvedValue({ bitcoin: {} }); + + const result = await service.verify(Buffer.alloc(32), Buffer.from([1])); + + expect(result).toEqual({ confirmed: false, pending: true }); + }); + }); +}); diff --git a/src/integration/infrastructure/storage/anchoring/archive-batch.entity.ts b/src/integration/infrastructure/storage/anchoring/archive-batch.entity.ts new file mode 100644 index 0000000000..c001247f63 --- /dev/null +++ b/src/integration/infrastructure/storage/anchoring/archive-batch.entity.ts @@ -0,0 +1,33 @@ +import { IEntity } from 'src/shared/models/entity'; +import { Column, Entity, OneToMany } from 'typeorm'; +import { ArchiveFile } from './archive-file.entity'; + +/** Lifecycle of an anchoring batch on the way to a Bitcoin attestation. */ +export enum ArchiveBatchStatus { + PENDING_BTC = 'pendingBtc', + CONFIRMED = 'confirmed', +} + +/** + * A daily (or on-demand) Merkle batch over a set of {@link ArchiveFile} hashes for the + * GeBüV anchoring pipeline. The `merkleRoot` is timestamped via OpenTimestamps; the + * resulting detached `.ots` proof is persisted (base64) in `otsProof` and upgraded over + * time until it carries a Bitcoin attestation (`bitcoinHeight` set, `status` confirmed). + */ +@Entity() +export class ArchiveBatch extends IEntity { + @Column({ length: 64 }) + merkleRoot: string; + + @Column({ type: 'text', nullable: true }) + otsProof?: string; + + @Column({ type: 'int', nullable: true }) + bitcoinHeight?: number; + + @Column({ length: 256, default: ArchiveBatchStatus.PENDING_BTC }) + status: ArchiveBatchStatus; + + @OneToMany(() => ArchiveFile, (file) => file.batch) + files: ArchiveFile[]; +} diff --git a/src/integration/infrastructure/storage/anchoring/archive-batch.repository.ts b/src/integration/infrastructure/storage/anchoring/archive-batch.repository.ts new file mode 100644 index 0000000000..5ca13be275 --- /dev/null +++ b/src/integration/infrastructure/storage/anchoring/archive-batch.repository.ts @@ -0,0 +1,11 @@ +import { Injectable } from '@nestjs/common'; +import { BaseRepository } from 'src/shared/repositories/base.repository'; +import { EntityManager } from 'typeorm'; +import { ArchiveBatch } from './archive-batch.entity'; + +@Injectable() +export class ArchiveBatchRepository extends BaseRepository { + constructor(manager: EntityManager) { + super(ArchiveBatch, manager); + } +} diff --git a/src/integration/infrastructure/storage/anchoring/archive-file.entity.ts b/src/integration/infrastructure/storage/anchoring/archive-file.entity.ts new file mode 100644 index 0000000000..829a639e54 --- /dev/null +++ b/src/integration/infrastructure/storage/anchoring/archive-file.entity.ts @@ -0,0 +1,30 @@ +import { IEntity } from 'src/shared/models/entity'; +import { Column, Entity, Index, ManyToOne } from 'typeorm'; +import { ArchiveBatch } from './archive-batch.entity'; + +/** + * A single archived storage object whose content hash participates in the GeBüV anchoring + * pipeline. Each file is identified uniquely by its `(bucket, name)` location and records + * the SHA-256 of its content. Once anchored, it points to its {@link ArchiveBatch} and + * carries its `leafIndex` within that batch's Merkle tree (needed to rebuild the inclusion + * proof during verification). + */ +@Entity() +@Index((file: ArchiveFile) => [file.bucket, file.name], { unique: true }) +export class ArchiveFile extends IEntity { + @Column({ length: 256 }) + bucket: string; + + @Column({ length: 256 }) + name: string; + + @Column({ length: 64 }) + sha256: string; + + @Index() + @ManyToOne(() => ArchiveBatch, (batch) => batch.files, { nullable: true }) + batch?: ArchiveBatch; + + @Column({ type: 'int', nullable: true }) + leafIndex?: number; +} diff --git a/src/integration/infrastructure/storage/anchoring/archive-file.repository.ts b/src/integration/infrastructure/storage/anchoring/archive-file.repository.ts new file mode 100644 index 0000000000..b162ce3b20 --- /dev/null +++ b/src/integration/infrastructure/storage/anchoring/archive-file.repository.ts @@ -0,0 +1,11 @@ +import { Injectable } from '@nestjs/common'; +import { BaseRepository } from 'src/shared/repositories/base.repository'; +import { EntityManager } from 'typeorm'; +import { ArchiveFile } from './archive-file.entity'; + +@Injectable() +export class ArchiveFileRepository extends BaseRepository { + constructor(manager: EntityManager) { + super(ArchiveFile, manager); + } +} diff --git a/src/integration/infrastructure/storage/anchoring/archive.module.ts b/src/integration/infrastructure/storage/anchoring/archive.module.ts new file mode 100644 index 0000000000..04ad7ac7d2 --- /dev/null +++ b/src/integration/infrastructure/storage/anchoring/archive.module.ts @@ -0,0 +1,18 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { SharedModule } from 'src/shared/shared.module'; +import { ArchiveBatch } from './archive-batch.entity'; +import { ArchiveBatchRepository } from './archive-batch.repository'; +import { ArchiveFile } from './archive-file.entity'; +import { ArchiveFileRepository } from './archive-file.repository'; +import { ArchiveScheduler } from './archive.scheduler'; +import { ArchiveService } from './archive.service'; +import { OpenTimestampsService } from './opentimestamps.service'; + +@Module({ + imports: [SharedModule, TypeOrmModule.forFeature([ArchiveBatch, ArchiveFile])], + controllers: [], + providers: [ArchiveBatchRepository, ArchiveFileRepository, ArchiveService, OpenTimestampsService, ArchiveScheduler], + exports: [ArchiveService, OpenTimestampsService], +}) +export class ArchiveModule {} diff --git a/src/integration/infrastructure/storage/anchoring/archive.scheduler.ts b/src/integration/infrastructure/storage/anchoring/archive.scheduler.ts new file mode 100644 index 0000000000..688985f15e --- /dev/null +++ b/src/integration/infrastructure/storage/anchoring/archive.scheduler.ts @@ -0,0 +1,46 @@ +import { Injectable } from '@nestjs/common'; +import { CronExpression } from '@nestjs/schedule'; +import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { Process } from 'src/shared/services/process.service'; +import { DfxCron } from 'src/shared/utils/cron'; +import { ArchiveService } from './archive.service'; + +/** + * Stage 3 of the GeBüV anchoring pipeline: drives {@link ArchiveService} on a schedule. + * + * Uses the repo-wide `@DfxCron` decorator (see src/shared/utils/cron.ts), which the + * central DfxCronService discovers at boot and runs behind a per-method `@Lock` plus a + * `Process` guard — so a disabled process or a multi-instance deployment never double-runs + * the same job. Each job is additionally wrapped in try/catch with structured logging so a + * single failure (e.g. an OpenTimestamps calendar outage) never crashes the scheduler. + */ +@Injectable() +export class ArchiveScheduler { + private readonly logger = new DfxLogger(ArchiveScheduler); + + constructor(private readonly archiveService: ArchiveService) {} + + @DfxCron(CronExpression.EVERY_DAY_AT_2AM, { process: Process.ARCHIVE_ANCHOR, timeout: 3600 }) + async anchorPending(): Promise { + try { + const batch = await this.archiveService.anchorPending(); + + if (batch) { + this.logger.info(`Anchored batch ${batch.id} with Merkle root ${batch.merkleRoot}`); + } else { + this.logger.verbose('No unanchored archive files to anchor'); + } + } catch (e) { + this.logger.error('Failed to anchor pending archive files:', e); + } + } + + @DfxCron(CronExpression.EVERY_HOUR, { process: Process.ARCHIVE_UPGRADE, timeout: 1800 }) + async upgradeBatches(): Promise { + try { + await this.archiveService.upgradeBatches(); + } catch (e) { + this.logger.error('Failed to upgrade pending archive batches:', e); + } + } +} diff --git a/src/integration/infrastructure/storage/anchoring/archive.service.ts b/src/integration/infrastructure/storage/anchoring/archive.service.ts new file mode 100644 index 0000000000..c8d841f0ff --- /dev/null +++ b/src/integration/infrastructure/storage/anchoring/archive.service.ts @@ -0,0 +1,196 @@ +import { Injectable } from '@nestjs/common'; +import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { IsNull } from 'typeorm'; +import { ArchiveBatch, ArchiveBatchStatus } from './archive-batch.entity'; +import { ArchiveBatchRepository } from './archive-batch.repository'; +import { ArchiveFileRepository } from './archive-file.repository'; +import { buildMerkleRoot, merkleInclusionProof, sha256, verifyMerkleProof } from './merkle'; +import { OpenTimestampsService } from './opentimestamps.service'; + +/** Result of verifying an archived document against its anchored Merkle batch. */ +export interface ArchiveVerification { + /** false if no archive record exists for the given `(bucket, name)`. */ + found: boolean; + /** true if the stored SHA-256 equals the hash recomputed from the supplied data. */ + hashMatches?: boolean; + /** true once the file has been assigned to a Merkle batch. */ + anchored?: boolean; + /** true if the inclusion proof recomputes the batch's stored Merkle root. */ + proofValid?: boolean; + /** Bitcoin block height of the OpenTimestamps attestation, once anchored on-chain. */ + bitcoinHeight?: number; + /** true while the OpenTimestamps proof is still calendar-only (not yet on-chain). */ + pending?: boolean; +} + +/** + * Stage 2 of the GeBüV anchoring pipeline: it records content hashes of archived storage + * objects, batches the still-unanchored ones into a daily Merkle tree, timestamps the root + * via OpenTimestamps (Stage 1 primitives), upgrades those proofs to Bitcoin attestations, + * and verifies a given document against its anchored batch end-to-end. + * + * Leaves are the raw 32-byte SHA-256 digests of the file contents (the Merkle module does + * NOT re-hash leaves). `merkleRoot` is stored hex, `otsProof` is stored base64 of the + * serialized detached `.ots` bytes. + */ +@Injectable() +export class ArchiveService { + private readonly logger = new DfxLogger(ArchiveService); + + constructor( + private readonly archiveBatchRepo: ArchiveBatchRepository, + private readonly archiveFileRepo: ArchiveFileRepository, + private readonly ots: OpenTimestampsService, + ) {} + + /** + * Idempotently record the SHA-256 of an archived object identified by `(bucket, name)`. + * + * Only UNANCHORED records may be updated in place (their hash refreshed, kept unanchored); + * a new record is created unanchored (`batch` null). Anchoring happens later via + * {@link anchorPending}. + * + * Once a record has been assigned to a Merkle batch its leaf hash is immutable: it is part + * of a (possibly already Bitcoin-anchored) proof. Because KYC blob names are deterministic, + * a re-upload to the same `(bucket, name)` would otherwise silently overwrite the anchored + * leaf hash and make {@link verifyDocument} report bogus tampering. Therefore, for an + * already-anchored record: an identical hash is a no-op, and a differing hash is a hard + * error (the existing anchored hash is never overwritten). + */ + async recordHash(bucket: string, name: string, sha256Hex: string): Promise { + const existing = await this.archiveFileRepo.findOne({ where: { bucket, name }, relations: ['batch'] }); + + if (existing) { + if (existing.batch != null) { + if (existing.sha256 === sha256Hex) return; + + const message = + `Refusing to overwrite anchored hash for ${bucket}/${name} (file ${existing.id}, batch ` + + `${existing.batch.id}): stored ${existing.sha256} differs from new ${sha256Hex}`; + this.logger.error(message); + throw new Error(message); + } + + await this.archiveFileRepo.update(existing.id, { sha256: sha256Hex }); + return; + } + + const file = this.archiveFileRepo.create({ bucket, name, sha256: sha256Hex }); + await this.archiveFileRepo.save(file); + } + + /** + * Batch all currently unanchored files (ordered by id) into one Merkle tree, timestamp its + * root via OpenTimestamps, and persist batch + per-file assignment in a single transaction. + * + * Returns the created batch, or `undefined` if there is nothing to anchor. + */ + async anchorPending(): Promise { + const files = await this.archiveFileRepo.find({ where: { batch: IsNull() }, order: { id: 'ASC' } }); + if (files.length === 0) return undefined; + + const leaves = files.map((file) => Buffer.from(file.sha256, 'hex')); + const root = buildMerkleRoot(leaves); + + const otsBytes = await this.ots.stamp(root); + + const batch = this.archiveBatchRepo.create({ + merkleRoot: root.toString('hex'), + otsProof: otsBytes.toString('base64'), + status: ArchiveBatchStatus.PENDING_BTC, + }); + + await this.archiveBatchRepo.manager.transaction(async (manager) => { + const savedBatch = await manager.save(batch); + + files.forEach((file, index) => { + file.batch = savedBatch; + file.leafIndex = index; + }); + + await manager.save(files); + }); + + this.logger.info(`Anchored batch ${batch.id} over ${files.length} file(s), root ${batch.merkleRoot}`); + + return batch; + } + + /** + * Try to upgrade every pending batch's OpenTimestamps proof towards a Bitcoin attestation. + * + * The upgraded `.ots` bytes are persisted whenever the proof changed at all (e.g. it now + * carries additional calendar commitments but `verify` still reports pending) so that + * progress is never thrown away. `bitcoinHeight`/`status = confirmed` are set additionally + * only once `verify` reports a Bitcoin attestation. + */ + async upgradeBatches(): Promise { + const batches = await this.archiveBatchRepo.findBy({ status: ArchiveBatchStatus.PENDING_BTC }); + + for (const batch of batches) { + if (!batch.otsProof) continue; + + const rootBuffer = Buffer.from(batch.merkleRoot, 'hex'); + const originalProof = batch.otsProof; + const upgraded = await this.ots.upgrade(Buffer.from(originalProof, 'base64')); + const upgradedProof = upgraded.toString('base64'); + const result = await this.ots.verify(rootBuffer, upgraded); + + const proofChanged = upgradedProof !== originalProof; + if (!proofChanged && !result.confirmed) continue; + + // Always persist progress when the proof bytes changed; confirm only on a real attestation. + if (proofChanged) batch.otsProof = upgradedProof; + + if (result.confirmed) { + batch.bitcoinHeight = result.bitcoin.height; + batch.status = ArchiveBatchStatus.CONFIRMED; + } + + await this.archiveBatchRepo.save(batch); + + if (result.confirmed) { + this.logger.info(`Confirmed batch ${batch.id} at Bitcoin height ${batch.bitcoinHeight}`); + } else { + this.logger.info(`Upgraded pending OpenTimestamps proof for batch ${batch.id}`); + } + } + } + + /** + * Verify a supplied document against its archived, anchored Merkle batch end-to-end: + * recompute its SHA-256, compare with the stored hash, rebuild the inclusion proof against + * the batch's Merkle root, and check the OpenTimestamps attestation status. + */ + async verifyDocument(bucket: string, name: string, data: Buffer): Promise { + const file = await this.archiveFileRepo.findOne({ where: { bucket, name }, relations: ['batch'] }); + if (!file) return { found: false }; + + const computedHex = sha256(data).toString('hex'); + const hashMatches = file.sha256 === computedHex; + + const batch = file.batch; + if (!batch) return { found: true, hashMatches, anchored: false }; + + const batchFiles = await this.archiveFileRepo.find({ + where: { batch: { id: batch.id } }, + order: { leafIndex: 'ASC' }, + }); + const leaves = batchFiles.map((batchFile) => Buffer.from(batchFile.sha256, 'hex')); + + const rootBuffer = Buffer.from(batch.merkleRoot, 'hex'); + const proof = merkleInclusionProof(leaves, file.leafIndex); + const proofValid = verifyMerkleProof(Buffer.from(file.sha256, 'hex'), proof, rootBuffer); + + let bitcoinHeight: number; + let pending = true; + + if (batch.otsProof) { + const ots = await this.ots.verify(rootBuffer, Buffer.from(batch.otsProof, 'base64')); + pending = ots.pending; + if (ots.bitcoin) bitcoinHeight = ots.bitcoin.height; + } + + return { found: true, hashMatches, anchored: true, proofValid, bitcoinHeight, pending }; + } +} diff --git a/src/integration/infrastructure/storage/anchoring/merkle.ts b/src/integration/infrastructure/storage/anchoring/merkle.ts new file mode 100644 index 0000000000..5d32b8e259 --- /dev/null +++ b/src/integration/infrastructure/storage/anchoring/merkle.ts @@ -0,0 +1,114 @@ +import { createHash } from 'node:crypto'; + +/** + * Pure Merkle-tree primitives for the GeBüV anchoring pipeline. + * + * HASH / CONCATENATION RULE (load-bearing — verification depends on it): + * - All hashing is SHA-256 (`node:crypto`). + * - A parent node is `parent = sha256(left || right)`, where `||` is raw byte + * concatenation of the two 32-byte child digests (left first, then right). + * - Leaves are used as-is: they are NOT re-hashed by this module. Callers pass + * already-hashed leaves (e.g. `sha256(documentBytes)`). This keeps the module + * agnostic about leaf preimages and avoids a hidden hashing convention. + * - On a level with an ODD number of nodes, the last node is DUPLICATED and + * paired with itself (`parent = sha256(last || last)`). This is the classic + * Bitcoin-style promotion rule and is reproduced identically in proofs and + * verification so the computed root always matches. + * + * Edge cases: + * - 0 leaves: `buildMerkleRoot` throws (an empty tree has no root). + * - 1 leaf: the root IS that leaf (no hashing applied), and its inclusion proof + * is the empty path. + */ + +/** A single step of an inclusion proof: the sibling digest and whether it sits on the right. */ +export interface MerkleProofStep { + sibling: Buffer; + /** true if the sibling is the RIGHT child (i.e. the running hash is the LEFT child). */ + right: boolean; +} + +/** SHA-256 of the given bytes. */ +export function sha256(data: Buffer): Buffer { + return createHash('sha256').update(data).digest(); +} + +/** Hash a parent from its two children using the documented `sha256(left || right)` rule. */ +function hashPair(left: Buffer, right: Buffer): Buffer { + return sha256(Buffer.concat([left, right])); +} + +/** + * Build the Merkle root over `leaves`. + * + * Throws on an empty input. For a single leaf the root equals that leaf. + * On odd levels the last node is duplicated (see module rule). + */ +export function buildMerkleRoot(leaves: Buffer[]): Buffer { + if (leaves.length === 0) throw new Error('Cannot build a Merkle root from zero leaves'); + + let level = leaves; + while (level.length > 1) { + const next: Buffer[] = []; + for (let i = 0; i < level.length; i += 2) { + const left = level[i]; + // Odd node count: duplicate the last node and pair it with itself. + const right = i + 1 < level.length ? level[i + 1] : level[i]; + next.push(hashPair(left, right)); + } + level = next; + } + + return level[0]; +} + +/** + * Compute the inclusion proof for the leaf at `index` — the ordered list of sibling + * digests (with their left/right position) from the leaf up to (but excluding) the root. + * + * For a single-leaf tree the proof is empty. The duplication rule for odd levels is + * applied identically here, so a leaf that is the duplicated odd node gets a sibling + * equal to itself on the right. + */ +export function merkleInclusionProof(leaves: Buffer[], index: number): MerkleProofStep[] { + if (leaves.length === 0) throw new Error('Cannot build a proof from zero leaves'); + if (index < 0 || index >= leaves.length) throw new Error(`Leaf index ${index} out of range [0, ${leaves.length})`); + + const proof: MerkleProofStep[] = []; + + let level = leaves; + let idx = index; + while (level.length > 1) { + const isLeft = idx % 2 === 0; + // Sibling index; for the duplicated last odd node the sibling is the node itself. + const siblingIdx = isLeft ? Math.min(idx + 1, level.length - 1) : idx - 1; + + proof.push({ sibling: level[siblingIdx], right: isLeft }); + + const next: Buffer[] = []; + for (let i = 0; i < level.length; i += 2) { + const left = level[i]; + const right = i + 1 < level.length ? level[i + 1] : level[i]; + next.push(hashPair(left, right)); + } + + level = next; + idx = Math.floor(idx / 2); + } + + return proof; +} + +/** + * Recompute the root from `leaf` and its `proof` and compare it against the expected `root`. + * Returns true only on an exact byte match. + */ +export function verifyMerkleProof(leaf: Buffer, proof: MerkleProofStep[], root: Buffer): boolean { + let computed = leaf; + + for (const step of proof) { + computed = step.right ? hashPair(computed, step.sibling) : hashPair(step.sibling, computed); + } + + return computed.equals(root); +} diff --git a/src/integration/infrastructure/storage/anchoring/opentimestamps.service.ts b/src/integration/infrastructure/storage/anchoring/opentimestamps.service.ts new file mode 100644 index 0000000000..6391b5b43f --- /dev/null +++ b/src/integration/infrastructure/storage/anchoring/opentimestamps.service.ts @@ -0,0 +1,91 @@ +import { Injectable } from '@nestjs/common'; +import * as OpenTimestamps from 'opentimestamps'; + +/** Result of verifying a detached `.ots` proof against its digest. */ +export interface OtsVerifyResult { + /** Present once the timestamp is anchored in a Bitcoin block. */ + bitcoin?: { height: number }; + /** true once a Bitcoin attestation was found and verified (i.e. `bitcoin` is set). */ + confirmed: boolean; + /** + * true while the timestamp is NOT yet confirmed on-chain. This conflates "calendar-only, + * not yet anchored" and "verify could not confirm anything"; use `confirmed` to assert a + * positive Bitcoin attestation. + */ + pending: boolean; +} + +/** + * Thin async/await wrapper around the `opentimestamps` npm library for the GeBüV + * anchoring pipeline. It deliberately knows nothing about Merkle trees, storage or + * scheduling — callers feed it a single 32-byte SHA-256 digest (typically a daily + * Merkle root) and get back / consume serialized detached `.ots` proof bytes. + * + * The underlying library mixes synchronous constructors with promise-returning + * network calls; everything is normalized to `async` here. + * + * NOTE: `verify` is run with `ignoreBitcoinNode: true`, so attestation is checked + * against the public block explorers the library trusts, NOT a local Bitcoin node. + * Verifying against a trusted local node (the strongest GeBüV posture) is not possible + * from this pure service and must be wired up separately if/when a node is available. + */ +@Injectable() +export class OpenTimestampsService { + /** + * Create a detached timestamp over `digest` (an already-computed SHA-256, e.g. a + * Merkle root) by submitting it to the public OpenTimestamps calendars. + * + * Returns the serialized `.ots` bytes to be persisted. At this point the proof is + * typically still "pending" — it carries calendar commitments but no Bitcoin + * attestation yet; call `upgrade` later to complete it. + */ + async stamp(digest: Buffer): Promise { + const detached = this.detachedFromDigest(digest); + + await OpenTimestamps.stamp(detached); + + return Buffer.from(detached.serializeToBytes()); + } + + /** + * Attempt to upgrade a pending `.ots` proof to a complete Bitcoin attestation by + * asking the calendars for the now-available block path. + * + * Returns the upgraded `.ots` bytes if anything changed, otherwise the original + * bytes unchanged (still pending). + */ + async upgrade(otsBytes: Buffer): Promise { + const detached = OpenTimestamps.DetachedTimestampFile.deserialize(otsBytes); + + const changed = await OpenTimestamps.upgrade(detached); + + return changed ? Buffer.from(detached.serializeToBytes()) : otsBytes; + } + + /** + * Verify that `otsBytes` is a valid timestamp over `digest`. + * + * Returns the Bitcoin attestation height once anchored; while the proof is still + * calendar-only it reports `pending: true` with no `bitcoin` field. + */ + async verify(digest: Buffer, otsBytes: Buffer): Promise { + const detachedOts = OpenTimestamps.DetachedTimestampFile.deserialize(otsBytes); + const detached = this.detachedFromDigest(digest); + + // ignoreBitcoinNode: verify against the library's trusted explorers, not a local node. + const result = await OpenTimestamps.verify(detachedOts, detached, { ignoreBitcoinNode: true }); + + // The library returns an object keyed by chain (e.g. { bitcoin: { height, timestamp } }); + // an empty/undefined result means the proof is not yet anchored. + const bitcoin = result && result.bitcoin; + if (bitcoin && typeof bitcoin.height === 'number') + return { bitcoin: { height: bitcoin.height }, confirmed: true, pending: false }; + + return { confirmed: false, pending: true }; + } + + /** Build a DetachedTimestampFile that commits directly to an already-computed SHA-256 digest. */ + private detachedFromDigest(digest: Buffer): any { + return OpenTimestamps.DetachedTimestampFile.fromHash(new OpenTimestamps.Ops.OpSHA256(), digest); + } +} diff --git a/src/integration/infrastructure/storage/mock-storage.service.ts b/src/integration/infrastructure/storage/mock-storage.service.ts new file mode 100644 index 0000000000..cf510ea081 --- /dev/null +++ b/src/integration/infrastructure/storage/mock-storage.service.ts @@ -0,0 +1,101 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { Blob, BlobContent, StorageService } from './storage.service'; + +// In-memory storage for local development (LOC). +const mockStorage = new Map }>(); + +// Dummy KYC files for local development (used when a blob was not uploaded this process lifetime). +const DUMMY_FILES_DIR = path.join(process.cwd(), 'scripts', 'kyc', 'dummy-files'); +const DUMMY_FILE_MAP: Record = { + 'id_front.png': { file: 'id_front.png', type: 'image/png' }, + 'id_back.png': { file: 'id_back.png', type: 'image/png' }, + 'selfie.jpg': { file: 'selfie.jpg', type: 'image/jpeg' }, + 'passport.png': { file: 'passport.png', type: 'image/png' }, + 'residence_permit.png': { file: 'residence_permit.png', type: 'image/png' }, + 'proof_of_address.pdf': { file: 'proof_of_address.pdf', type: 'application/pdf' }, + 'bank_statement.pdf': { file: 'bank_statement.pdf', type: 'application/pdf' }, + 'source_of_funds.pdf': { file: 'source_of_funds.pdf', type: 'application/pdf' }, + 'commercial_register.pdf': { file: 'commercial_register.pdf', type: 'application/pdf' }, + 'additional_document.pdf': { file: 'additional_document.pdf', type: 'application/pdf' }, +}; + +function loadDummyFile(filename: string): Buffer { + return fs.readFileSync(path.join(DUMMY_FILES_DIR, filename)); +} + +export class MockStorageService extends StorageService { + constructor(container: string) { + super(container); + } + + async listBlobs(prefix?: string): Promise { + const keyPrefix = `${this.container}/${prefix ?? ''}`; + + return [...mockStorage.entries()] + .filter(([key]) => key.startsWith(keyPrefix)) + .map(([key, value]) => { + const name = key.replace(`${this.container}/`, ''); + return { + name, + url: this.blobUrl(name), + contentType: value.type, + created: new Date(), + updated: new Date(), + metadata: value.metadata ?? {}, + }; + }); + } + + async getBlob(name: string): Promise { + const stored = mockStorage.get(`${this.container}/${name}`); + if (stored) + return { + data: stored.data, + contentType: stored.type, + created: new Date(), + updated: new Date(), + metadata: stored.metadata ?? {}, + }; + + // Fallback to a dummy file (parity with the previous mock) so LOC document reads return bytes. + const fileName = name.split('/').pop() ?? name; + const mapping = DUMMY_FILE_MAP[fileName]; + if (mapping) + return { + data: loadDummyFile(mapping.file), + contentType: mapping.type, + created: new Date(), + updated: new Date(), + metadata: {}, + }; + + const ext = name.split('.').pop()?.toLowerCase(); + const isPdf = ext === 'pdf'; + const isJpg = ext === 'jpg' || ext === 'jpeg'; + return { + data: loadDummyFile(isPdf ? 'proof_of_address.pdf' : isJpg ? 'selfie.jpg' : 'id_front.png'), + contentType: isPdf ? 'application/pdf' : isJpg ? 'image/jpeg' : 'image/png', + created: new Date(), + updated: new Date(), + metadata: {}, + }; + } + + async uploadBlob(name: string, data: Buffer, type: string, metadata?: Record): Promise { + mockStorage.set(`${this.container}/${name}`, { data, type, metadata }); + return this.blobUrl(name); + } + + async copyBlobs(sourcePrefix: string, targetPrefix: string): Promise { + for (const blob of await this.listBlobs(sourcePrefix)) { + const content = await this.getBlob(blob.name); + await this.uploadBlob( + blob.name.replace(sourcePrefix, targetPrefix), + content.data, + content.contentType, + blob.metadata, + ); + } + } +} diff --git a/src/integration/infrastructure/storage/s3-storage.service.ts b/src/integration/infrastructure/storage/s3-storage.service.ts new file mode 100644 index 0000000000..f98ddf5716 --- /dev/null +++ b/src/integration/infrastructure/storage/s3-storage.service.ts @@ -0,0 +1,117 @@ +import { + CopyObjectCommand, + GetObjectCommand, + HeadObjectCommand, + ListObjectsV2Command, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import { Config } from 'src/config/config'; +import { Blob, BlobContent, BlobMetaData, StorageService } from './storage.service'; + +/** + * S3-protocol storage implementation. Talks to the configured S3-compatible + * endpoint (on-prem MinIO today; any S3 store via `Config.s3.endpoint`) — this is a + * protocol client, not the AWS cloud: no AWS account, no data leaves to AWS. + * + * Replaces AzureStorageService. The blob URL shape is kept identical so `blobName()` + * stays reversible and URLs persisted in the DB remain consistent after migration. + * + * WORM / Object-Lock is expected to be enforced server-side via the bucket's default + * retention (Compliance mode), provisioned externally at bucket setup. It is + * intentionally not applied per request here. + */ +export class S3StorageService extends StorageService { + private readonly client: S3Client; + + constructor(container: string) { + super(container); + + const { endpoint, region, accessKey, secretKey, publicUrl } = Config.s3; + if (!endpoint || !region || !accessKey || !secretKey || !publicUrl) + throw new Error('Incomplete S3 config: endpoint, region, accessKey, secretKey and publicUrl are required'); + if (!publicUrl.endsWith('/')) throw new Error('S3 publicUrl must end with a trailing slash'); + + this.client = new S3Client({ + endpoint, + region, + forcePathStyle: true, // MinIO requires path-style addressing + credentials: { accessKeyId: accessKey, secretAccessKey: secretKey }, + }); + } + + async listBlobs(prefix?: string): Promise { + // S3 listings carry no content-type / user metadata (unlike the Azure listing this + // replaces), so fetch per object. Per-prefix counts are modest (per-user KYC/support). + const keys = await this.listKeys(prefix); + return Promise.all(keys.map((key) => this.head(key))); + } + + async getBlob(name: string): Promise { + const res = await this.client.send(new GetObjectCommand({ Bucket: this.container, Key: name })); + if (!res.Body) throw new Error(`Empty body for blob ${this.container}/${name}`); + + return { data: Buffer.from(await res.Body.transformToByteArray()), ...this.toMetaData(res) }; + } + + async uploadBlob(name: string, data: Buffer, type: string, metadata?: Record): Promise { + await this.client.send( + new PutObjectCommand({ Bucket: this.container, Key: name, Body: data, ContentType: type, Metadata: metadata }), + ); + + return this.blobUrl(name); + } + + async copyBlobs(sourcePrefix: string, targetPrefix: string): Promise { + // copy needs only the keys, not metadata — avoid the per-object HeadObject fan-out. + const keys = await this.listKeys(sourcePrefix); + + for (const key of keys) { + await this.client.send( + new CopyObjectCommand({ + Bucket: this.container, + Key: key.replace(sourcePrefix, targetPrefix), + CopySource: `${this.container}/${this.encodeKey(key)}`, // key must be URL-encoded + }), + ); + } + } + + private async listKeys(prefix?: string): Promise { + const keys: string[] = []; + + let token: string | undefined; + do { + const res = await this.client.send( + new ListObjectsV2Command({ Bucket: this.container, Prefix: prefix, ContinuationToken: token, MaxKeys: 1000 }), + ); + + for (const o of res.Contents ?? []) if (o.Key) keys.push(o.Key); + + token = res.IsTruncated ? res.NextContinuationToken : undefined; + } while (token); + + return keys; + } + + private async head(name: string): Promise { + const res = await this.client.send(new HeadObjectCommand({ Bucket: this.container, Key: name })); + return { name, url: this.blobUrl(name), ...this.toMetaData(res) }; + } + + // NOTE: S3 has no creation timestamp (created == updated == LastModified) and lowercases + // user-metadata keys. contentType/timestamps are always present for objects we write + // (ContentType is always set on upload). + private toMetaData(res: { + ContentType?: string; + LastModified?: Date; + Metadata?: Record; + }): BlobMetaData { + return { + contentType: res.ContentType, + created: res.LastModified, + updated: res.LastModified, + metadata: res.Metadata ?? {}, + }; + } +} diff --git a/src/integration/infrastructure/storage/storage.factory.ts b/src/integration/infrastructure/storage/storage.factory.ts new file mode 100644 index 0000000000..af4595f3c8 --- /dev/null +++ b/src/integration/infrastructure/storage/storage.factory.ts @@ -0,0 +1,23 @@ +import { Environment, GetConfig } from 'src/config/config'; +import { MockStorageService } from './mock-storage.service'; +import { S3StorageService } from './s3-storage.service'; +import { StorageService } from './storage.service'; + +/** + * Returns the configured storage implementation for a bucket/container. + * + * Deliberately a factory function rather than a DI provider: instances are + * per-container and some containers are resolved at runtime (e.g. the per-merchant + * EP2 settlement container in fiat-output), which a singleton provider can't express. + * Drop-in replacement for `new AzureStorageService(container)` at the call sites: + * - kyc-document.service.ts (constructed at boot — eager config validation / fail-fast) + * - support-document.service.ts (constructed at boot — eager config validation / fail-fast) + * - fiat-output-job.service.ts (per-job, runtime EP2 container) + * Because the KYC/support providers are eagerly instantiated, an incomplete S3 config + * fails the application boot, not just the first storage call. + */ +export function createStorageService(container: string): StorageService { + return GetConfig().environment === Environment.LOC + ? new MockStorageService(container) + : new S3StorageService(container); +} diff --git a/src/integration/infrastructure/storage/storage.service.ts b/src/integration/infrastructure/storage/storage.service.ts new file mode 100644 index 0000000000..d4422af412 --- /dev/null +++ b/src/integration/infrastructure/storage/storage.service.ts @@ -0,0 +1,59 @@ +import { Config } from 'src/config/config'; + +export interface BlobMetaData { + contentType: string; + created: Date; + updated: Date; + metadata: Record; +} + +export interface Blob extends BlobMetaData { + name: string; + url: string; +} + +export interface BlobContent extends BlobMetaData { + data: Buffer; +} + +/** + * Provider-agnostic blob storage abstraction. + * + * The method surface is signature-compatible with the previous AzureStorageService, + * so consumers only change how the instance is obtained (see storage.factory.ts). + * + * `blobUrl`/`blobName` live here so the URL shape — and its reversibility — is + * identical across implementations and stays consistent with URLs persisted in the DB. + * The trailing-slash contract on the public URL base is enforced by the concrete + * implementation's config validation. + */ +export abstract class StorageService { + constructor(protected readonly container: string) {} + + abstract listBlobs(prefix?: string): Promise; + abstract getBlob(name: string): Promise; + abstract uploadBlob(name: string, data: Buffer, type: string, metadata?: Record): Promise; + abstract copyBlobs(sourcePrefix: string, targetPrefix: string): Promise; + + blobUrl(name: string): string { + return `${Config.s3.publicUrl}${this.container}/${this.encodeKey(name)}`; + } + + blobName(url: string): string { + const filePath = url.split(`${this.container}/`)[1]; + if (filePath == null) throw new Error(`URL does not belong to container ${this.container}: ${url}`); + return filePath.split('/').map(decodeURIComponent).join('/'); + } + + protected encodeKey(name: string): string { + return StorageService.encodePath(name); + } + + // Per-segment path encoding for blob keys/URLs: each path segment is `encodeURIComponent`-encoded + // while the `/` separators are preserved. Exposed statically so consumers that build host-stable + // variants of a blob URL (e.g. `KycDocumentService.toHostStableUrl`) encode byte-identically to + // `blobUrl`, keeping the two URLs identical apart from the host. + static encodePath(name: string): string { + return name.split('/').map(encodeURIComponent).join('/'); + } +} diff --git a/src/shared/services/process.service.ts b/src/shared/services/process.service.ts index bcd68bba68..12eb2eb0d2 100644 --- a/src/shared/services/process.service.ts +++ b/src/shared/services/process.service.ts @@ -95,6 +95,8 @@ export enum Process { TRADE_APPROVAL_DATE = 'TradeApprovalDate', SUPPORT_BOT = 'SupportBot', GUARANTEED_PRICE = 'GuaranteedPrice', + ARCHIVE_ANCHOR = 'ArchiveAnchor', + ARCHIVE_UPGRADE = 'ArchiveUpgrade', GS_DEBUG = 'GsDebug', GS_DB = 'GsDb', TRANSACTION_AML_CHECK_LOG = 'TransactionAmlCheckLog', diff --git a/src/subdomains/generic/gs/__tests__/gs.service.spec.ts b/src/subdomains/generic/gs/__tests__/gs.service.spec.ts index 0797d0d60a..6fd1e2592b 100644 --- a/src/subdomains/generic/gs/__tests__/gs.service.spec.ts +++ b/src/subdomains/generic/gs/__tests__/gs.service.spec.ts @@ -1,3 +1,7 @@ +// Stub the heavy `opentimestamps` library (pulled in transitively via KycDocumentService -> +// ArchiveService) so its eager network/`request` deps never load at jest runtime. +jest.mock('opentimestamps', () => ({})); + import { BadRequestException } from '@nestjs/common'; import { createMock } from '@golevelup/ts-jest'; import { DataSource } from 'typeorm'; @@ -28,8 +32,9 @@ import { KycFileService } from 'src/subdomains/generic/kyc/services/kyc-file.ser import { KycFileBlob } from 'src/subdomains/generic/kyc/dto/kyc-file.dto'; import { UserData } from 'src/subdomains/generic/user/models/user-data/user-data.entity'; import { AccountType } from 'src/subdomains/generic/user/models/user-data/account-type.enum'; -import { Blob } from 'src/integration/infrastructure/azure-storage.service'; -import { ConfigService } from 'src/config/config'; +import { ArchiveService } from 'src/integration/infrastructure/storage/anchoring/archive.service'; +import { Blob } from 'src/integration/infrastructure/storage/storage.service'; +import { ConfigService, Environment } from 'src/config/config'; import { DebugAggregate, DebugQueryDto, DebugWhereNode, DebugWhereOp } from '../dto/debug-query.dto'; import { plainToInstance } from 'class-transformer'; import { validate, ValidationError } from 'class-validator'; @@ -2129,9 +2134,13 @@ describe('GsService', () => { const hostStableUrl = (path: string) => `${SERVICES_HOST}/kyc/${path}`; const previousServicesUrl = process.env.SERVICES_URL; + const previousEnvironment = process.env.ENVIRONMENT; beforeEach(() => { process.env.SERVICES_URL = SERVICES_HOST; + // Pin the environment to LOC so the real KycDocumentService constructed in the round-trip test + // builds a MockStorageService via the storage factory (no S3 config required at construction). + process.env.ENVIRONMENT = Environment.LOC; new ConfigService(); }); @@ -2139,6 +2148,8 @@ describe('GsService', () => { // restore the env + Config singleton so the pinned host does not leak into other test blocks if (previousServicesUrl === undefined) delete process.env.SERVICES_URL; else process.env.SERVICES_URL = previousServicesUrl; + if (previousEnvironment === undefined) delete process.env.ENVIRONMENT; + else process.env.ENVIRONMENT = previousEnvironment; new ConfigService(); }); @@ -2231,7 +2242,7 @@ describe('GsService', () => { // Asserts both the host-stability and the path-preserving consumer invariant on the real output. it('produces a host-stable, path-preserving URL through the real KycDocumentService (round-trip)', async () => { const kycFileService = createMock(); - const realKycDocumentService = new KycDocumentService(kycFileService); + const realKycDocumentService = new KycDocumentService(kycFileService, createMock()); const userBlob = storageBlob('user/1/Identification/passport.pdf', new Date('2024-01-01')); const spiderBlob = storageBlob('spider/1/Identification/old-passport.pdf', new Date('2024-01-02')); diff --git a/src/subdomains/generic/kyc/dto/kyc-file.dto.ts b/src/subdomains/generic/kyc/dto/kyc-file.dto.ts index 8c28b89546..d7e6c5a91c 100644 --- a/src/subdomains/generic/kyc/dto/kyc-file.dto.ts +++ b/src/subdomains/generic/kyc/dto/kyc-file.dto.ts @@ -1,5 +1,5 @@ import { ApiProperty } from '@nestjs/swagger'; -import { Blob } from 'src/integration/infrastructure/azure-storage.service'; +import { Blob } from 'src/integration/infrastructure/storage/storage.service'; import { UserData } from '../../user/models/user-data/user-data.entity'; import { KycWebhookData } from '../../user/services/webhook/dto/kyc-webhook.dto'; import { KycStep } from '../entities/kyc-step.entity'; diff --git a/src/subdomains/generic/kyc/dto/mapper/kyc-file.mapper.ts b/src/subdomains/generic/kyc/dto/mapper/kyc-file.mapper.ts index aad02c6f37..e7367c6c35 100644 --- a/src/subdomains/generic/kyc/dto/mapper/kyc-file.mapper.ts +++ b/src/subdomains/generic/kyc/dto/mapper/kyc-file.mapper.ts @@ -1,4 +1,4 @@ -import { BlobContent } from 'src/integration/infrastructure/azure-storage.service'; +import { BlobContent } from 'src/integration/infrastructure/storage/storage.service'; import { KycFile } from '../../entities/kyc-file.entity'; import { KycFileDataDto } from '../kyc-file.dto'; diff --git a/src/subdomains/generic/kyc/kyc.module.ts b/src/subdomains/generic/kyc/kyc.module.ts index 8ed614b6dd..501800cd23 100644 --- a/src/subdomains/generic/kyc/kyc.module.ts +++ b/src/subdomains/generic/kyc/kyc.module.ts @@ -1,5 +1,6 @@ import { Module, forwardRef } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { ArchiveModule } from 'src/integration/infrastructure/storage/anchoring/archive.module'; import { ScorechainModule } from 'src/integration/scorechain/scorechain.module'; import { SharedModule } from 'src/shared/shared.module'; import { BuyCryptoModule } from 'src/subdomains/core/buy-crypto/buy-crypto.module'; @@ -54,6 +55,7 @@ import { TfaService } from './services/tfa.service'; KycFile, ]), SharedModule, + ArchiveModule, NotificationModule, ScorechainModule, forwardRef(() => UserModule), diff --git a/src/subdomains/generic/kyc/services/__tests__/kyc.service.spec.ts b/src/subdomains/generic/kyc/services/__tests__/kyc.service.spec.ts index 29268d6fea..9346a28519 100644 --- a/src/subdomains/generic/kyc/services/__tests__/kyc.service.spec.ts +++ b/src/subdomains/generic/kyc/services/__tests__/kyc.service.spec.ts @@ -1,7 +1,11 @@ +// Stub the heavy `opentimestamps` library (pulled in transitively via KycDocumentService -> +// ArchiveService) so its eager network/`request` deps never load at jest runtime. +jest.mock('opentimestamps', () => ({})); + import { createMock } from '@golevelup/ts-jest'; import { ForbiddenException } from '@nestjs/common'; import { Configuration, ConfigService } from 'src/config/config'; -import { BlobContent } from 'src/integration/infrastructure/azure-storage.service'; +import { BlobContent } from 'src/integration/infrastructure/storage/storage.service'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; import { UserRole } from 'src/shared/auth/user-role.enum'; import { createCustomCountry } from 'src/shared/models/country/__mocks__/country.entity.mock'; diff --git a/src/subdomains/generic/kyc/services/integration/__tests__/kyc-document.service.spec.ts b/src/subdomains/generic/kyc/services/integration/__tests__/kyc-document.service.spec.ts new file mode 100644 index 0000000000..98e3d68347 --- /dev/null +++ b/src/subdomains/generic/kyc/services/integration/__tests__/kyc-document.service.spec.ts @@ -0,0 +1,100 @@ +// Stub the heavy `opentimestamps` library (pulled in transitively via ArchiveService) so its +// eager network/`request` deps never load; ArchiveService is fully mocked in this spec. +jest.mock('opentimestamps', () => ({})); + +// Control the storage backend the service constructs in its constructor, so uploadBlob is a +// spy and no real S3/Azure/mock storage is touched. +const uploadBlobMock = jest.fn(); +jest.mock('src/integration/infrastructure/storage/storage.factory', () => ({ + createStorageService: jest.fn(() => ({ + uploadBlob: (...args: any[]) => uploadBlobMock(...args), + })), +})); + +import { createMock } from '@golevelup/ts-jest'; +import { Test, TestingModule } from '@nestjs/testing'; +import { ArchiveService } from 'src/integration/infrastructure/storage/anchoring/archive.service'; +import { sha256 } from 'src/integration/infrastructure/storage/anchoring/merkle'; +import { UserData } from 'src/subdomains/generic/user/models/user-data/user-data.entity'; +import { FileType } from '../../../dto/kyc-file.dto'; +import { KycFile } from '../../../entities/kyc-file.entity'; +import { ContentType } from '../../../enums/content-type.enum'; +import { KycFileService } from '../../kyc-file.service'; +import { KycDocumentService } from '../kyc-document.service'; + +describe('KycDocumentService - GeBüV hash recording', () => { + let service: KycDocumentService; + let kycFileService: KycFileService; + let archiveService: ArchiveService; + + const userData = { id: 42 } as UserData; + const data = Buffer.from('a kyc document payload'); + const expectedBlobName = `user/42/${FileType.IDENTIFICATION}/passport.pdf`; + const expectedHash = sha256(data).toString('hex'); + + beforeEach(async () => { + jest.clearAllMocks(); + + kycFileService = createMock(); + archiveService = createMock(); + + (kycFileService.createKycFile as jest.Mock).mockResolvedValue({ id: 7 } as KycFile); + uploadBlobMock.mockResolvedValue('https://storage/blob-url'); + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + KycDocumentService, + { provide: KycFileService, useValue: kycFileService }, + { provide: ArchiveService, useValue: archiveService }, + ], + }).compile(); + + service = module.get(KycDocumentService); + }); + + it('records the uploaded blob hash in the kyc bucket after a successful upload', async () => { + const result = await service.uploadUserFile( + userData, + FileType.IDENTIFICATION, + 'passport.pdf', + data, + ContentType.PDF, + true, + ); + + // upload happened, then recordHash with the deterministic blob name + sha256 of the data + expect(uploadBlobMock).toHaveBeenCalledTimes(1); + expect(uploadBlobMock.mock.calls[0][0]).toBe(expectedBlobName); + + expect(archiveService.recordHash).toHaveBeenCalledTimes(1); + expect(archiveService.recordHash).toHaveBeenCalledWith('kyc', expectedBlobName, expectedHash); + + expect(result).toEqual({ file: { id: 7 }, url: 'https://storage/blob-url' }); + }); + + it('does NOT roll back the upload when hash recording fails (best-effort side-booking)', async () => { + (archiveService.recordHash as jest.Mock).mockRejectedValue(new Error('archive db down')); + + const result = await service.uploadUserFile( + userData, + FileType.IDENTIFICATION, + 'passport.pdf', + data, + ContentType.PDF, + true, + ); + + // the recordHash failure was swallowed: the method still returns the uploaded file + url + expect(archiveService.recordHash).toHaveBeenCalledTimes(1); + expect(result).toEqual({ file: { id: 7 }, url: 'https://storage/blob-url' }); + }); + + it('rejects unsupported media types before any upload or hash recording', async () => { + await expect( + service.uploadUserFile(userData, FileType.IDENTIFICATION, 'note.txt', data, 'text/plain' as ContentType, true), + ).rejects.toThrow('Supported file types'); + + expect(uploadBlobMock).not.toHaveBeenCalled(); + expect(archiveService.recordHash).not.toHaveBeenCalled(); + }); +}); diff --git a/src/subdomains/generic/kyc/services/integration/kyc-document.service.ts b/src/subdomains/generic/kyc/services/integration/kyc-document.service.ts index f840c086a9..0b802b9850 100644 --- a/src/subdomains/generic/kyc/services/integration/kyc-document.service.ts +++ b/src/subdomains/generic/kyc/services/integration/kyc-document.service.ts @@ -1,6 +1,10 @@ import { Injectable, UnsupportedMediaTypeException } from '@nestjs/common'; import { Config } from 'src/config/config'; -import { AzureStorageService, BlobContent } from 'src/integration/infrastructure/azure-storage.service'; +import { ArchiveService } from 'src/integration/infrastructure/storage/anchoring/archive.service'; +import { sha256 } from 'src/integration/infrastructure/storage/anchoring/merkle'; +import { createStorageService } from 'src/integration/infrastructure/storage/storage.factory'; +import { BlobContent, StorageService } from 'src/integration/infrastructure/storage/storage.service'; +import { DfxLogger } from 'src/shared/services/dfx-logger'; import { AccountType } from 'src/subdomains/generic/user/models/user-data/account-type.enum'; import { UserData } from 'src/subdomains/generic/user/models/user-data/user-data.entity'; import { FileSubType, FileType, KycFileBlob } from '../../dto/kyc-file.dto'; @@ -14,10 +18,15 @@ const KYC_CONTAINER = 'kyc'; @Injectable() export class KycDocumentService { - private readonly storageService: AzureStorageService; + private readonly logger = new DfxLogger(KycDocumentService); - constructor(private readonly kycFileService: KycFileService) { - this.storageService = new AzureStorageService(KYC_CONTAINER); + private readonly storageService: StorageService; + + constructor( + private readonly kycFileService: KycFileService, + private readonly archiveService: ArchiveService, + ) { + this.storageService = createStorageService(KYC_CONTAINER); } async getAllUserDocuments(userDataId: number, accountType = AccountType.PERSONAL): Promise { @@ -32,12 +41,12 @@ export class KycDocumentService { // (`Config.frontend.services`) while keeping the full storage path (`/`) // intact. The raw blob URL is `//`; we swap only the // host, so a storage-backend migration stays transparent to clients. Path segments are encoded - // through the same shared helper as the raw blob URL (`AzureStorageService.encodePath`), so the + // through the same shared helper as the raw blob URL (`StorageService.encodePath`), so the // URL is a true drop-in — byte-identical apart from the host by construction. The path // (`//`) stays an intact substring, which downstream consumers rely on to extract // the file name (e.g. `url.split('//')[1]`). toHostStableUrl(path: string): string { - return `${Config.frontend.services}/${KYC_CONTAINER}/${AzureStorageService.encodePath(path)}`; + return `${Config.frontend.services}/${KYC_CONTAINER}/${StorageService.encodePath(path)}`; } async listUserFiles(userDataId: number): Promise { @@ -110,12 +119,20 @@ export class KycDocumentService { kycStep, }); - const url = await this.storageService.uploadBlob( - this.toFileId(FileCategory.USER, userData.id, type, name), - data, - contentType, - metadata, - ); + const blobName = this.toFileId(FileCategory.USER, userData.id, type, name); + + const url = await this.storageService.uploadBlob(blobName, data, contentType, metadata); + + // GeBüV anchoring (Stage 3): record the content hash of the just-uploaded KYC document + // (a retention-relevant compliance bucket) so it can later be Merkle-batched and anchored. + // This is a best-effort side-booking: the upload above has already succeeded and must not + // be rolled back if hash recording fails, so failures are logged (never silently swallowed) + // but not rethrown. + try { + await this.archiveService.recordHash(KYC_CONTAINER, blobName, sha256(data).toString('hex')); + } catch (e) { + this.logger.error(`GeBüV anchoring failed to record hash for ${KYC_CONTAINER}/${blobName}:`, e); + } return { file, url }; } diff --git a/src/subdomains/generic/user/models/user-data/__tests__/user-data.service.spec.ts b/src/subdomains/generic/user/models/user-data/__tests__/user-data.service.spec.ts index 65ab18f5ae..e347b765f9 100644 --- a/src/subdomains/generic/user/models/user-data/__tests__/user-data.service.spec.ts +++ b/src/subdomains/generic/user/models/user-data/__tests__/user-data.service.spec.ts @@ -1,3 +1,7 @@ +// Stub the heavy `opentimestamps` library (pulled in transitively via KycDocumentService -> +// ArchiveService) so its eager network/`request` deps never load at jest runtime. +jest.mock('opentimestamps', () => ({})); + import { createMock } from '@golevelup/ts-jest'; import { Test, TestingModule } from '@nestjs/testing'; import { ConflictException } from '@nestjs/common'; diff --git a/src/subdomains/supporting/fiat-output/__tests__/fiat-output-job.service.spec.ts b/src/subdomains/supporting/fiat-output/__tests__/fiat-output-job.service.spec.ts index b1aad5ac11..e18c913a47 100644 --- a/src/subdomains/supporting/fiat-output/__tests__/fiat-output-job.service.spec.ts +++ b/src/subdomains/supporting/fiat-output/__tests__/fiat-output-job.service.spec.ts @@ -1,3 +1,16 @@ +// Stub the heavy `opentimestamps` library (pulled in transitively via ArchiveService) so its +// eager network/`request` deps never load at jest runtime; ArchiveService is mocked in this spec. +jest.mock('opentimestamps', () => ({})); + +// generateReports resolves a per-merchant EP2 container at runtime via createStorageService(); +// mock the factory so uploadBlob is a spy and no real storage backend is touched. +const ep2UploadBlobMock = jest.fn(); +jest.mock('src/integration/infrastructure/storage/storage.factory', () => ({ + createStorageService: jest.fn(() => ({ + uploadBlob: (...args: any[]) => ep2UploadBlobMock(...args), + })), +})); + import { createMock } from '@golevelup/ts-jest'; import { Test, TestingModule } from '@nestjs/testing'; import { FrickPaymentState } from 'src/integration/bank/dto/frick.dto'; @@ -6,6 +19,8 @@ import { IbanService } from 'src/integration/bank/services/iban.service'; import { OlkypayService } from 'src/integration/bank/services/olkypay.service'; import { YapealService } from 'src/integration/bank/services/yapeal.service'; import { ScryptService } from 'src/integration/exchange/services/scrypt.service'; +import { ArchiveService } from 'src/integration/infrastructure/storage/anchoring/archive.service'; +import { sha256 } from 'src/integration/infrastructure/storage/anchoring/merkle'; import { createCustomAsset, createDefaultAsset } from 'src/shared/models/asset/__mocks__/asset.entity.mock'; import { AssetType } from 'src/shared/models/asset/asset.entity'; import { AssetService } from 'src/shared/models/asset/asset.service'; @@ -57,8 +72,10 @@ describe('FiatOutputJobService', () => { let virtualIbanService: VirtualIbanService; let scryptService: ScryptService; let frickPayoutService: FiatOutputFrickService; + let archiveService: ArchiveService; beforeEach(async () => { + ep2UploadBlobMock.mockReset(); fiatOutputRepo = createMock(); bankTxService = createMock(); bankTxOutgoingMatchService = createMock(); @@ -73,6 +90,7 @@ describe('FiatOutputJobService', () => { olkypayService = createMock(); virtualIbanService = createMock(); scryptService = createMock(); + archiveService = createMock(); jest.spyOn(processServiceModule, 'DisabledProcess').mockReturnValue(false); // Default mock: no virtual IBANs @@ -104,6 +122,7 @@ describe('FiatOutputJobService', () => { { provide: IbanService, useValue: createMock() }, { provide: VirtualIbanService, useValue: virtualIbanService }, { provide: ScryptService, useValue: scryptService }, + { provide: ArchiveService, useValue: archiveService }, TestUtil.provideConfig(), ], @@ -880,4 +899,94 @@ describe('FiatOutputJobService', () => { expect(fiatOutputRepo.update).not.toHaveBeenCalledWith(6, { isReadyDate: expect.any(Date) }); }); }); + + describe('generateReports - GeBüV hash recording', () => { + // A FiatOutput whose buyFiat resolves the container, route id and userData the method needs. + // The getter chain (paymentLinkPayment.link.linkConfigObj, paymentLinksConfigObj) is stubbed + // directly on plain objects so we don't have to assemble the full entity graph. + function reportableEntity() { + const buyFiat: any = { + sell: { id: 555 }, + userData: { paymentLinksConfigObj: { ep2ReportContainer: 'ep2-merchant-bucket' } }, + paymentLinkPayment: { link: { linkConfigObj: { payoutRouteId: 777 } } }, + }; + + return { + id: 1, + created: new Date('2024-03-01T10:00:00Z'), + buyFiats: [buyFiat], + } as any; + } + + beforeEach(() => { + ep2UploadBlobMock.mockResolvedValue(undefined); + (ep2ReportService.generateReport as jest.Mock).mockReturnValue(''); + }); + + it('uploads the report, then sets reportCreated and records the report hash', async () => { + jest.spyOn(fiatOutputRepo, 'find').mockResolvedValue([reportableEntity()]); + + await service['generateReports'](); + + const fileName = ep2UploadBlobMock.mock.calls[0][0]; + expect(ep2UploadBlobMock).toHaveBeenCalledTimes(1); + expect(fileName).toMatch(/^settlement_.*_777\.ep2$/); + + // reportCreated is flipped before the best-effort anchoring runs + expect(fiatOutputRepo.update).toHaveBeenCalledWith(1, { reportCreated: true }); + + // the recorded hash is the sha256 of the uploaded report buffer + const expectedHash = sha256(Buffer.from('')).toString('hex'); + expect(archiveService.recordHash).toHaveBeenCalledWith('ep2-merchant-bucket', fileName, expectedHash); + + // Load-bearing ordering: uploadBlob (WORM PUT) < update(reportCreated=true) < recordHash. + // reportCreated MUST be persisted before the best-effort recordHash, otherwise a recordHash + // failure would leave reportCreated=false and the next run would re-PUT the same fileName + // into the immutable WORM bucket and deadlock. Asserting the call order makes the test break + // if someone reorders recordHash ahead of the reportCreated update. + const uploadOrder = ep2UploadBlobMock.mock.invocationCallOrder[0]; + const updateOrder = (fiatOutputRepo.update as jest.Mock).mock.invocationCallOrder[0]; + const recordHashOrder = (archiveService.recordHash as jest.Mock).mock.invocationCallOrder[0]; + expect(uploadOrder).toBeLessThan(updateOrder); + expect(updateOrder).toBeLessThan(recordHashOrder); + }); + + it('does NOT prevent reportCreated when hash recording fails (best-effort, runs after the flag)', async () => { + jest.spyOn(fiatOutputRepo, 'find').mockResolvedValue([reportableEntity()]); + (archiveService.recordHash as jest.Mock).mockRejectedValue(new Error('archive db down')); + + await service['generateReports'](); + + // upload + reportCreated still happened despite the recordHash failure + expect(ep2UploadBlobMock).toHaveBeenCalledTimes(1); + expect(fiatOutputRepo.update).toHaveBeenCalledWith(1, { reportCreated: true }); + expect(archiveService.recordHash).toHaveBeenCalledTimes(1); + }); + + it('does not set reportCreated when the upload itself fails', async () => { + jest.spyOn(fiatOutputRepo, 'find').mockResolvedValue([reportableEntity()]); + ep2UploadBlobMock.mockRejectedValue(new Error('WORM bucket unreachable')); + + await service['generateReports'](); + + expect(fiatOutputRepo.update).not.toHaveBeenCalled(); + expect(archiveService.recordHash).not.toHaveBeenCalled(); + }); + + it('falls back to the sell route id for the file name when no payoutRouteId is configured', async () => { + // linkConfigObj has no payoutRouteId => the `routeId ?? buyFiat.sell.id` fallback kicks in + // and the file name must carry the sell id (555) instead of a payout route id. + const entity = reportableEntity(); + entity.buyFiats[0].paymentLinkPayment.link.linkConfigObj = {}; + jest.spyOn(fiatOutputRepo, 'find').mockResolvedValue([entity]); + + await service['generateReports'](); + + const fileName = ep2UploadBlobMock.mock.calls[0][0]; + expect(fileName).toMatch(/^settlement_.*_555\.ep2$/); + + const expectedHash = sha256(Buffer.from('')).toString('hex'); + expect(archiveService.recordHash).toHaveBeenCalledWith('ep2-merchant-bucket', fileName, expectedHash); + }); + }); }); diff --git a/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts b/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts index 4674b892f3..ba31c3eac2 100644 --- a/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts +++ b/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts @@ -7,7 +7,9 @@ import { Pain001Payment } from 'src/integration/bank/services/iso20022.service'; import { OlkypayService } from 'src/integration/bank/services/olkypay.service'; import { YapealService } from 'src/integration/bank/services/yapeal.service'; import { ScryptService } from 'src/integration/exchange/services/scrypt.service'; -import { AzureStorageService } from 'src/integration/infrastructure/azure-storage.service'; +import { ArchiveService } from 'src/integration/infrastructure/storage/anchoring/archive.service'; +import { sha256 } from 'src/integration/infrastructure/storage/anchoring/merkle'; +import { createStorageService } from 'src/integration/infrastructure/storage/storage.factory'; import { AssetType } from 'src/shared/models/asset/asset.entity'; import { AssetService } from 'src/shared/models/asset/asset.service'; import { Country } from 'src/shared/models/country/country.entity'; @@ -58,6 +60,7 @@ export class FiatOutputJobService { private readonly frickPayoutService: FiatOutputFrickService, private readonly virtualIbanService: VirtualIbanService, private readonly scryptService: ScryptService, + private readonly archiveService: ArchiveService, ) {} @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.FIAT_OUTPUT, timeout: 1800 }) @@ -115,10 +118,24 @@ export class FiatOutputJobService { const container = buyFiat.userData.paymentLinksConfigObj.ep2ReportContainer; const routeId = buyFiat.paymentLinkPayment.link.linkConfigObj?.payoutRouteId ?? buyFiat.sell.id; const fileName = `settlement_${Util.isoDateTime(entity.created)}_${routeId}.ep2`; + const reportBuffer = Buffer.from(report); - await new AzureStorageService(container).uploadBlob(fileName, Buffer.from(report), 'text/xml'); + await createStorageService(container).uploadBlob(fileName, reportBuffer, 'text/xml'); + // Mark the report as created as soon as the WORM upload succeeded, BEFORE the best-effort + // anchoring below. A recordHash failure must not leave reportCreated=false, otherwise the + // next run would re-PUT the same fileName into the immutable WORM bucket and block forever. await this.fiatOutputRepo.update(entity.id, { reportCreated: true }); + + // GeBüV anchoring (Stage 3): record the content hash of the just-uploaded EP2 settlement + // report (a retention-relevant compliance bucket) for later Merkle-batching and anchoring. + // Best-effort side-booking: the upload already succeeded, so a failure here is logged + // (never silently swallowed) but does not roll back the upload or the reportCreated flag. + try { + await this.archiveService.recordHash(container, fileName, sha256(reportBuffer).toString('hex')); + } catch (e) { + this.logger.error(`GeBüV anchoring failed to record hash for ${container}/${fileName}:`, e); + } } catch (e) { this.logger.error(`Failed to generate EP2 report for fiat output ${entity.id}:`, e); } diff --git a/src/subdomains/supporting/fiat-output/fiat-output.module.ts b/src/subdomains/supporting/fiat-output/fiat-output.module.ts index 17ee531e2b..4c866f518c 100644 --- a/src/subdomains/supporting/fiat-output/fiat-output.module.ts +++ b/src/subdomains/supporting/fiat-output/fiat-output.module.ts @@ -2,6 +2,7 @@ import { Module, forwardRef } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { BankIntegrationModule } from 'src/integration/bank/bank.module'; import { ExchangeModule } from 'src/integration/exchange/exchange.module'; +import { ArchiveModule } from 'src/integration/infrastructure/storage/anchoring/archive.module'; import { SharedModule } from 'src/shared/shared.module'; import { BuyCryptoRepository } from 'src/subdomains/core/buy-crypto/process/repositories/buy-crypto.repository'; import { LiquidityManagementModule } from 'src/subdomains/core/liquidity-management/liquidity-management.module'; @@ -28,6 +29,7 @@ import { FiatOutputJobService } from './fiat-output-job.service'; ExchangeModule, forwardRef(() => LiquidityManagementModule), LogModule, + ArchiveModule, ], controllers: [FiatOutputController], diff --git a/src/subdomains/supporting/realunit/__tests__/realunit-compliance.service.spec.ts b/src/subdomains/supporting/realunit/__tests__/realunit-compliance.service.spec.ts index bfe9f84440..b64e046c2a 100644 --- a/src/subdomains/supporting/realunit/__tests__/realunit-compliance.service.spec.ts +++ b/src/subdomains/supporting/realunit/__tests__/realunit-compliance.service.spec.ts @@ -1,7 +1,11 @@ +// Stub the heavy `opentimestamps` library (pulled in transitively via KycDocumentService -> +// ArchiveService) so its eager network/`request` deps never load at jest runtime. +jest.mock('opentimestamps', () => ({})); + import { NotFoundException, ServiceUnavailableException } from '@nestjs/common'; import { createMock, DeepMocked } from '@golevelup/ts-jest'; import { Configuration, ConfigService } from 'src/config/config'; -import { BlobContent } from 'src/integration/infrastructure/azure-storage.service'; +import { BlobContent } from 'src/integration/infrastructure/storage/storage.service'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; import JSZip from 'jszip'; import { Asset } from 'src/shared/models/asset/asset.entity'; diff --git a/src/subdomains/supporting/realunit/controllers/realunit-support.controller.ts b/src/subdomains/supporting/realunit/controllers/realunit-support.controller.ts index a941e013b5..7b2d697a4b 100644 --- a/src/subdomains/supporting/realunit/controllers/realunit-support.controller.ts +++ b/src/subdomains/supporting/realunit/controllers/realunit-support.controller.ts @@ -1,7 +1,7 @@ import { Body, Controller, Get, Param, Post, Put, Query, UseGuards } from '@nestjs/common'; import { AuthGuard } from '@nestjs/passport'; import { ApiBearerAuth, ApiExcludeEndpoint, ApiTags } from '@nestjs/swagger'; -import { BlobContent } from 'src/integration/infrastructure/azure-storage.service'; +import { BlobContent } from 'src/integration/infrastructure/storage/storage.service'; import { GetJwt } from 'src/shared/auth/get-jwt.decorator'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; import { RoleGuard } from 'src/shared/auth/role.guard'; diff --git a/src/subdomains/supporting/support-issue/services/support-document.service.ts b/src/subdomains/supporting/support-issue/services/support-document.service.ts index 1d33c068b7..0f971ed5e6 100644 --- a/src/subdomains/supporting/support-issue/services/support-document.service.ts +++ b/src/subdomains/supporting/support-issue/services/support-document.service.ts @@ -1,5 +1,6 @@ import { Injectable, UnsupportedMediaTypeException } from '@nestjs/common'; -import { AzureStorageService, Blob, BlobContent } from 'src/integration/infrastructure/azure-storage.service'; +import { createStorageService } from 'src/integration/infrastructure/storage/storage.factory'; +import { Blob, BlobContent, StorageService } from 'src/integration/infrastructure/storage/storage.service'; import { ContentType } from 'src/subdomains/generic/kyc/enums/content-type.enum'; export interface SupportFile extends Blob { @@ -10,10 +11,10 @@ export interface SupportFile extends Blob { @Injectable() export class SupportDocumentService { - private readonly storageService: AzureStorageService; + private readonly storageService: StorageService; constructor() { - this.storageService = new AzureStorageService('support'); + this.storageService = createStorageService('support'); } async listFilesByPrefix(prefix: string): Promise { diff --git a/src/subdomains/supporting/support-issue/services/support-issue.service.ts b/src/subdomains/supporting/support-issue/services/support-issue.service.ts index 4dc4b25ff7..9b181d422a 100644 --- a/src/subdomains/supporting/support-issue/services/support-issue.service.ts +++ b/src/subdomains/supporting/support-issue/services/support-issue.service.ts @@ -7,7 +7,7 @@ import { UnauthorizedException, } from '@nestjs/common'; import { Config } from 'src/config/config'; -import { BlobContent } from 'src/integration/infrastructure/azure-storage.service'; +import { BlobContent } from 'src/integration/infrastructure/storage/storage.service'; import { UserRole } from 'src/shared/auth/user-role.enum'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { SupportClerkAccountDto } from 'src/shared/models/setting/dto/support-clerk-account.dto'; diff --git a/src/subdomains/supporting/support-issue/support-issue.controller.ts b/src/subdomains/supporting/support-issue/support-issue.controller.ts index a44c7476e9..a36d64077c 100644 --- a/src/subdomains/supporting/support-issue/support-issue.controller.ts +++ b/src/subdomains/supporting/support-issue/support-issue.controller.ts @@ -2,7 +2,7 @@ import { Body, Controller, Get, Headers, Param, ParseIntPipe, Post, Put, Query, import { ModuleRef } from '@nestjs/core'; import { AuthGuard } from '@nestjs/passport'; import { ApiBadRequestResponse, ApiBearerAuth, ApiExcludeEndpoint, ApiTags } from '@nestjs/swagger'; -import { BlobContent } from 'src/integration/infrastructure/azure-storage.service'; +import { BlobContent } from 'src/integration/infrastructure/storage/storage.service'; import { GetJwt } from 'src/shared/auth/get-jwt.decorator'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; import { OptionalJwtAuthGuard } from 'src/shared/auth/optional.guard'; From 7f139df11001709af00885bd77f950f71cc235f9 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:06:07 +0200 Subject: [PATCH 02/11] test(compliance): prove GwG org documents survive the Azure to MinIO cutover Regression guard for the silent-data-loss blocker: when kyc_step.result holds a legacy Azure-host blob URL and the live KycFileBlob.url is regenerated from the MinIO backend, the fileDownloadConfig entries 11 (Handelsregisterauszug) and 12 (Vollmacht) must still select the document via the host-independent object key. Covers COMMERCIAL_REGISTER, LEGAL_ENTITY and AUTHORITY steps, percent-encoded file names, and the fail-closed behaviour when a URL is missing. --- .../__tests__/file-download-config.spec.ts | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 src/config/__tests__/file-download-config.spec.ts diff --git a/src/config/__tests__/file-download-config.spec.ts b/src/config/__tests__/file-download-config.spec.ts new file mode 100644 index 0000000000..fda9ab122b --- /dev/null +++ b/src/config/__tests__/file-download-config.spec.ts @@ -0,0 +1,116 @@ +import { Configuration } from 'src/config/config'; +import { KycStepName } from 'src/subdomains/generic/kyc/enums/kyc-step-name.enum'; +import { KycStep } from 'src/subdomains/generic/kyc/entities/kyc-step.entity'; +import { KycFileBlob } from 'src/subdomains/generic/kyc/dto/kyc-file.dto'; +import { UserData } from 'src/subdomains/generic/user/models/user-data/user-data.entity'; + +/** + * Regression guard for the Azure Blob -> MinIO (S3) storage cutover. + * + * kyc_step.result persists the FULL blob URL captured at upload time (Azure host before the + * cutover), while the compliance ZIP builder regenerates each KycFileBlob.url live from the + * CURRENT storage backend (MinIO host after the cutover). The GwG download config entries 11 + * (Handelsregisterauszug) and 12 (Vollmacht) select the KYC document by comparing those two + * URLs. With the previous full-URL string equality the hosts never matched after the cutover, + * so both documents were silently dropped from the compliance ZIP for every pre-cutover + * organization customer — no error, just missing files. The comparison is now host-independent + * (container-relative, decoded object key), which must match legacy Azure-host and new + * MinIO-host values alike. + */ +describe('fileDownloadConfig - host-independent KYC document selection (storage cutover)', () => { + const config = new Configuration(); + + // Same container-relative object key, served from two different storage hosts. + const key = 'user/42/CommercialRegister/hr-auszug.pdf'; + const legacyAzureUrl = `https://dfxstorageprd.blob.core.windows.net/kyc/${key}`; + const liveMinioUrl = `https://files.dfx.swiss/kyc/${key}`; + + function fileWithUrl(url: string, path: string): KycFileBlob { + return { url, path } as unknown as KycFileBlob; + } + + function step(partial: Record): KycStep { + return { isCompleted: true, ...partial } as unknown as KycStep; + } + + function filterFor(id: number): (file: KycFileBlob, userData: UserData) => boolean { + const entry = config.fileDownloadConfig.find((c) => c.id === id); + if (!entry?.files[0].filter) throw new Error(`no filter for fileDownloadConfig id ${id}`); + return entry.files[0].filter; + } + + describe('isSameKycBlob', () => { + it('treats the same object key served from Azure and MinIO hosts as equal', () => { + expect(Configuration.isSameKycBlob(legacyAzureUrl, liveMinioUrl)).toBe(true); + }); + + it('is robust to percent-encoded path segments (space in the file name)', () => { + const encodedAzure = 'https://dfxstorageprd.blob.core.windows.net/kyc/user/42/Authority/HR%20Vollmacht.pdf'; + const encodedMinio = 'https://files.dfx.swiss/kyc/user/42/Authority/HR%20Vollmacht.pdf'; + expect(Configuration.isSameKycBlob(encodedAzure, encodedMinio)).toBe(true); + }); + + it('does not match different object keys', () => { + const otherMinio = 'https://files.dfx.swiss/kyc/user/42/CommercialRegister/other.pdf'; + expect(Configuration.isSameKycBlob(legacyAzureUrl, otherMinio)).toBe(false); + }); + + it('returns false when either value is missing (fails closed, no throw)', () => { + expect(Configuration.isSameKycBlob(undefined, liveMinioUrl)).toBe(false); + expect(Configuration.isSameKycBlob(legacyAzureUrl, undefined)).toBe(false); + }); + }); + + describe('id 11 - Handelsregisterauszug', () => { + const filter = filterFor(11); + + it('selects the doc when the stored COMMERCIAL_REGISTER result is a legacy Azure URL and the live file.url is MinIO', () => { + const userData = { + kycSteps: [step({ name: KycStepName.COMMERCIAL_REGISTER, result: legacyAzureUrl })], + } as unknown as UserData; + + // Precondition: the two URLs are NOT string-equal (this is exactly what silently broke before). + expect(legacyAzureUrl).not.toEqual(liveMinioUrl); + + expect(filter(fileWithUrl(liveMinioUrl, key), userData)).toBe(true); + }); + + it('selects the doc for a LEGAL_ENTITY step whose stored getResult().url is a legacy Azure URL', () => { + const userData = { + kycSteps: [ + step({ + name: KycStepName.LEGAL_ENTITY, + getResult: () => ({ url: legacyAzureUrl, legalEntity: 'AG' }), + }), + ], + } as unknown as UserData; + + expect(filter(fileWithUrl(liveMinioUrl, key), userData)).toBe(true); + }); + + it('does not select an unrelated document (different object key)', () => { + const userData = { + kycSteps: [step({ name: KycStepName.COMMERCIAL_REGISTER, result: legacyAzureUrl })], + } as unknown as UserData; + + const otherLive = 'https://files.dfx.swiss/kyc/user/42/CommercialRegister/unrelated.pdf'; + expect(filter(fileWithUrl(otherLive, 'user/42/CommercialRegister/unrelated.pdf'), userData)).toBe(false); + }); + }); + + describe('id 12 - Vollmacht', () => { + const filter = filterFor(12); + + it('selects the doc when the stored AUTHORITY result is a legacy Azure URL and the live file.url is MinIO', () => { + const authorityKey = 'user/42/Authority/vollmacht.pdf'; + const storedAzure = `https://dfxstorageprd.blob.core.windows.net/kyc/${authorityKey}`; + const liveMinio = `https://files.dfx.swiss/kyc/${authorityKey}`; + + const userData = { + kycSteps: [step({ name: KycStepName.AUTHORITY, result: storedAzure })], + } as unknown as UserData; + + expect(filter(fileWithUrl(liveMinio, authorityKey), userData)).toBe(true); + }); + }); +}); From a85fc99f572727f8eb4978bc06727e9cca376fb1 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:25:50 +0200 Subject: [PATCH 03/11] =?UTF-8?q?feat(storage):=20fail=20closed=20on=20WOR?= =?UTF-8?q?M=20compliance-config=20gaps=20(GeB=C3=BCV)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fail-closed hardenings for the Azure->MinIO WORM cutover, so a misconfiguration surfaces as a loud error instead of silently under-protecting GeBüV-retention-relevant compliance records. - provision-bucket: reject RETENTION_YEARS below the GeBüV 10-year floor. COMPLIANCE-mode Object Lock retention is extend-only and irreversible, so a value of 1..9 would permanently under-retain WORM objects with no way to correct it. Keep the 11y default; a value >= 10 is required. - EP2 settlement sink: add StorageService.uploadWormBlob and route the per-merchant EP2 report write through it. The S3 implementation verifies the target bucket has Object Lock enabled (GetObjectLockConfiguration) before the first PUT and throws otherwise, since Object Lock cannot be retro-fitted onto an existing bucket. Verified once per container to keep the probe off the per-PUT hot path; the kyc path is unchanged. Add proving tests: the retention floor rejects 9 and accepts 10/11, and a WORM-sink write fails closed (no PUT) into a non-locked / unverifiable bucket while succeeding into a locked one. --- scripts/storage/provision-bucket.ts | 29 +++++++-- .../__tests__/provision-bucket.spec.ts | 62 ++++++++++++++++++ .../__tests__/s3-storage.service.spec.ts | 64 +++++++++++++++++++ .../storage/s3-storage.service.ts | 42 ++++++++++++ .../infrastructure/storage/storage.service.ts | 14 ++++ .../__tests__/fiat-output-job.service.spec.ts | 4 +- .../fiat-output/fiat-output-job.service.ts | 5 +- 7 files changed, 210 insertions(+), 10 deletions(-) create mode 100644 src/integration/infrastructure/storage/__tests__/provision-bucket.spec.ts diff --git a/scripts/storage/provision-bucket.ts b/scripts/storage/provision-bucket.ts index eff6021dc7..1055fb9983 100644 --- a/scripts/storage/provision-bucket.ts +++ b/scripts/storage/provision-bucket.ts @@ -47,13 +47,24 @@ function getBucketName(): string { return bucket; } -function getRetentionYears(): number { +// Swiss GeBüV requires business records to be retained for 10 years. COMPLIANCE-mode Object +// Lock retention is extend-only and irreversible, so a value provisioned too low can never be +// corrected on the objects it protects — it must therefore fail closed, never silently under-retain. +export const GEBUEV_RETENTION_FLOOR_YEARS = 10; + +export function getRetentionYears(): number { const raw = process.env.RETENTION_YEARS ?? process.argv[3]; if (raw == null) return 11; // GeBüV 10y + safety margin; explicit, not a silent credential default. const years = Number(raw); if (!Number.isInteger(years) || years <= 0) throw new Error(`Invalid RETENTION_YEARS: ${raw} (expected positive integer)`); + if (years < GEBUEV_RETENTION_FLOOR_YEARS) + throw new Error( + `RETENTION_YEARS=${years} is below the GeBüV ${GEBUEV_RETENTION_FLOOR_YEARS}-year retention floor. ` + + `COMPLIANCE-mode WORM retention is extend-only and irreversible, so refusing to provision an ` + + `under-retained bucket. Set RETENTION_YEARS to at least ${GEBUEV_RETENTION_FLOOR_YEARS} (default 11).`, + ); return years; } @@ -127,9 +138,13 @@ async function main(): Promise { console.log(`Done. Bucket "${bucket}" is WORM-protected (COMPLIANCE, ${years}y).`); } -main() - .then(() => process.exit(0)) - .catch((e) => { - console.error('Provisioning failed:', e?.message ?? e); - process.exit(1); - }); +// Only run when invoked directly as a script (npx ts-node …), not when imported (e.g. by tests +// that exercise getRetentionYears' GeBüV floor without provisioning a real bucket). +if (require.main === module) { + main() + .then(() => process.exit(0)) + .catch((e) => { + console.error('Provisioning failed:', e?.message ?? e); + process.exit(1); + }); +} diff --git a/src/integration/infrastructure/storage/__tests__/provision-bucket.spec.ts b/src/integration/infrastructure/storage/__tests__/provision-bucket.spec.ts new file mode 100644 index 0000000000..7c53d64820 --- /dev/null +++ b/src/integration/infrastructure/storage/__tests__/provision-bucket.spec.ts @@ -0,0 +1,62 @@ +import { GEBUEV_RETENTION_FLOOR_YEARS, getRetentionYears } from '../../../../../scripts/storage/provision-bucket'; + +// Proves the GeBüV retention-floor guard in the WORM bucket provisioning script: COMPLIANCE-mode +// Object Lock retention is extend-only/irreversible, so a value below the 10-year floor must fail +// closed rather than silently under-retain compliance objects. +describe('provision-bucket getRetentionYears (GeBüV retention floor)', () => { + const RETENTION_ENV = 'RETENTION_YEARS'; + let savedEnv: string | undefined; + let savedArgv: string[]; + + beforeEach(() => { + savedEnv = process.env[RETENTION_ENV]; + savedArgv = process.argv; + // Neutralize the positional CLI fallback (process.argv[3]) that getRetentionYears also reads. + process.argv = ['node', 'provision-bucket.ts']; + delete process.env[RETENTION_ENV]; + }); + + afterEach(() => { + if (savedEnv === undefined) delete process.env[RETENTION_ENV]; + else process.env[RETENTION_ENV] = savedEnv; + process.argv = savedArgv; + }); + + it('exposes a 10-year GeBüV floor', () => { + expect(GEBUEV_RETENTION_FLOOR_YEARS).toBe(10); + }); + + it('defaults to 11 years (GeBüV 10y + safety margin) when unset', () => { + expect(getRetentionYears()).toBe(11); + }); + + it('rejects RETENTION_YEARS=9 with a GeBüV-floor error and does not fall back to a value', () => { + process.env[RETENTION_ENV] = '9'; + expect(() => getRetentionYears()).toThrow(/GeBüV 10-year retention floor/); + }); + + it('rejects every year below the floor (1..9)', () => { + for (let y = 1; y <= GEBUEV_RETENTION_FLOOR_YEARS - 1; y++) { + process.env[RETENTION_ENV] = String(y); + expect(() => getRetentionYears()).toThrow(/below the GeBüV/); + } + }); + + it('accepts exactly the 10-year floor', () => { + process.env[RETENTION_ENV] = '10'; + expect(getRetentionYears()).toBe(10); + }); + + it('accepts 11 years (the default, above the floor)', () => { + process.env[RETENTION_ENV] = '11'; + expect(getRetentionYears()).toBe(11); + }); + + it('still rejects non-positive / non-integer values before the floor check', () => { + process.env[RETENTION_ENV] = '0'; + expect(() => getRetentionYears()).toThrow(/expected positive integer/); + + process.env[RETENTION_ENV] = '10.5'; + expect(() => getRetentionYears()).toThrow(/expected positive integer/); + }); +}); diff --git a/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts b/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts index e28ce29ca3..4b06c7f40d 100644 --- a/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts +++ b/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts @@ -1,6 +1,7 @@ import { CopyObjectCommand, GetObjectCommand, + GetObjectLockConfigurationCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, @@ -203,6 +204,69 @@ describe('S3StorageService', () => { }); }); + describe('uploadWormBlob (WORM/EP2 sink, fail-closed Object-Lock guard)', () => { + // Distinct container per test: the verified-container cache is a process-wide static, so reusing + // a name would let one test's verification leak into the next. + it('PUTs into a bucket whose Object Lock is enabled', async () => { + const container = 'ep2-worm-locked'; + s3Mock + .on(GetObjectLockConfigurationCommand, { Bucket: container }) + .resolves({ ObjectLockConfiguration: { ObjectLockEnabled: 'Enabled' } }); + s3Mock.on(PutObjectCommand).resolves({}); + + const url = await new S3StorageService(container).uploadWormBlob('settlement.ep2', Buffer.from(''), 'text/xml'); + + expect(url).toBe(`https://files.test.local/${container}/settlement.ep2`); + const puts = s3Mock.commandCalls(PutObjectCommand); + expect(puts).toHaveLength(1); + expect(puts[0].args[0].input).toMatchObject({ Bucket: container, Key: 'settlement.ep2', ContentType: 'text/xml' }); + }); + + it('fails closed and does NOT PUT when Object Lock is not enabled on the bucket', async () => { + const container = 'ep2-worm-unlocked'; + s3Mock.on(GetObjectLockConfigurationCommand, { Bucket: container }).resolves({ ObjectLockConfiguration: {} }); + s3Mock.on(PutObjectCommand).resolves({}); + + await expect( + new S3StorageService(container).uploadWormBlob('settlement.ep2', Buffer.from(''), 'text/xml'), + ).rejects.toThrow('Object Lock is not enabled'); + + // the mutable bucket must never receive the compliance record + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0); + }); + + it('fails closed when the lock configuration cannot be read (e.g. bucket has no Object Lock config)', async () => { + const container = 'ep2-worm-noconfig'; + const err: any = new Error('Object Lock configuration does not exist for this bucket'); + err.name = 'ObjectLockConfigurationNotFoundError'; + s3Mock.on(GetObjectLockConfigurationCommand, { Bucket: container }).rejects(err); + s3Mock.on(PutObjectCommand).resolves({}); + + await expect( + new S3StorageService(container).uploadWormBlob('settlement.ep2', Buffer.from(''), 'text/xml'), + ).rejects.toThrow('could not verify Object Lock is enabled'); + + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0); + }); + + it('verifies Object Lock once per container, then skips the probe on subsequent WORM writes', async () => { + const container = 'ep2-worm-cache'; + s3Mock + .on(GetObjectLockConfigurationCommand, { Bucket: container }) + .resolves({ ObjectLockConfiguration: { ObjectLockEnabled: 'Enabled' } }); + s3Mock.on(PutObjectCommand).resolves({}); + + const service = new S3StorageService(container); + await service.uploadWormBlob('a.ep2', Buffer.from('a'), 'text/xml'); + await service.uploadWormBlob('b.ep2', Buffer.from('b'), 'text/xml'); + // a fresh instance for the same container must also reuse the cached verification + await new S3StorageService(container).uploadWormBlob('c.ep2', Buffer.from('c'), 'text/xml'); + + expect(s3Mock.commandCalls(GetObjectLockConfigurationCommand)).toHaveLength(1); + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(3); + }); + }); + describe('copyBlobs', () => { it('URL-encodes keys with spaces / special chars and rewrites the prefix', async () => { const key = 'src/sub dir/file (1)+&.png'; diff --git a/src/integration/infrastructure/storage/s3-storage.service.ts b/src/integration/infrastructure/storage/s3-storage.service.ts index f98ddf5716..7093b4b5fd 100644 --- a/src/integration/infrastructure/storage/s3-storage.service.ts +++ b/src/integration/infrastructure/storage/s3-storage.service.ts @@ -1,6 +1,7 @@ import { CopyObjectCommand, GetObjectCommand, + GetObjectLockConfigurationCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, @@ -22,6 +23,11 @@ import { Blob, BlobContent, BlobMetaData, StorageService } from './storage.servi * intentionally not applied per request here. */ export class S3StorageService extends StorageService { + // Containers verified to have Object Lock enabled. Static so the check is amortized across the + // short-lived per-container instances the EP2 sink creates (createStorageService per report), + // keeping the GetObjectLockConfiguration probe off the per-PUT hot path (verify once per bucket). + private static readonly objectLockVerified = new Set(); + private readonly client: S3Client; constructor(container: string) { @@ -62,6 +68,42 @@ export class S3StorageService extends StorageService { return this.blobUrl(name); } + // WORM sink (GeBüV): fail closed unless the target bucket enforces Object Lock. Object Lock + // cannot be retro-fitted onto an existing bucket, so writing a compliance record into a + // non-locked (or unverifiable) bucket would leave it mutable/deletable forever — we refuse + // rather than under-protect. Verified once per container; subsequent PUTs skip the probe. + async uploadWormBlob(name: string, data: Buffer, type: string, metadata?: Record): Promise { + await this.assertObjectLockEnabled(); + return this.uploadBlob(name, data, type, metadata); + } + + private async assertObjectLockEnabled(): Promise { + if (S3StorageService.objectLockVerified.has(this.container)) return; + + let enabled: string | undefined; + try { + const res = await this.client.send(new GetObjectLockConfigurationCommand({ Bucket: this.container })); + enabled = res.ObjectLockConfiguration?.ObjectLockEnabled; + } catch (e) { + // A bucket without Object Lock returns ObjectLockConfigurationNotFoundError; any other error + // (missing bucket, transport, auth) is equally unverifiable. Either way, fail closed. + throw new Error( + `Refusing WORM write into bucket "${this.container}": could not verify Object Lock is enabled ` + + `(${e?.name ?? 'error'}: ${e?.message ?? e}). GeBüV compliance records must not be written into ` + + `an unverified bucket. Provision it first (scripts/storage/provision-bucket.ts).`, + ); + } + + if (enabled !== 'Enabled') + throw new Error( + `Refusing WORM write into bucket "${this.container}": Object Lock is not enabled. GeBüV compliance ` + + `records must be WORM-protected and Object Lock cannot be retro-fitted onto an existing bucket. ` + + `Provision it first (scripts/storage/provision-bucket.ts).`, + ); + + S3StorageService.objectLockVerified.add(this.container); + } + async copyBlobs(sourcePrefix: string, targetPrefix: string): Promise { // copy needs only the keys, not metadata — avoid the per-object HeadObject fan-out. const keys = await this.listKeys(sourcePrefix); diff --git a/src/integration/infrastructure/storage/storage.service.ts b/src/integration/infrastructure/storage/storage.service.ts index d4422af412..02b1e6948b 100644 --- a/src/integration/infrastructure/storage/storage.service.ts +++ b/src/integration/infrastructure/storage/storage.service.ts @@ -35,6 +35,20 @@ export abstract class StorageService { abstract uploadBlob(name: string, data: Buffer, type: string, metadata?: Record): Promise; abstract copyBlobs(sourcePrefix: string, targetPrefix: string): Promise; + /** + * WORM sink for GeBüV-retention-relevant compliance records (e.g. EP2 settlement reports written + * to a per-merchant container resolved at runtime). The target bucket MUST enforce Object Lock, + * which cannot be retro-fitted onto an existing non-locked bucket — so a concrete implementation + * must verify the lock is present before the first write and fail closed if it is not, rather than + * silently persisting mutable, deletable compliance records into an unprotected bucket. + * + * The base default delegates to `uploadBlob` (no server-side Object Lock in LOC/mock); the S3 + * implementation adds the fail-closed verification. + */ + async uploadWormBlob(name: string, data: Buffer, type: string, metadata?: Record): Promise { + return this.uploadBlob(name, data, type, metadata); + } + blobUrl(name: string): string { return `${Config.s3.publicUrl}${this.container}/${this.encodeKey(name)}`; } diff --git a/src/subdomains/supporting/fiat-output/__tests__/fiat-output-job.service.spec.ts b/src/subdomains/supporting/fiat-output/__tests__/fiat-output-job.service.spec.ts index e18c913a47..318a81ae72 100644 --- a/src/subdomains/supporting/fiat-output/__tests__/fiat-output-job.service.spec.ts +++ b/src/subdomains/supporting/fiat-output/__tests__/fiat-output-job.service.spec.ts @@ -3,11 +3,11 @@ jest.mock('opentimestamps', () => ({})); // generateReports resolves a per-merchant EP2 container at runtime via createStorageService(); -// mock the factory so uploadBlob is a spy and no real storage backend is touched. +// mock the factory so the WORM sink (uploadWormBlob) is a spy and no real storage backend is touched. const ep2UploadBlobMock = jest.fn(); jest.mock('src/integration/infrastructure/storage/storage.factory', () => ({ createStorageService: jest.fn(() => ({ - uploadBlob: (...args: any[]) => ep2UploadBlobMock(...args), + uploadWormBlob: (...args: any[]) => ep2UploadBlobMock(...args), })), })); diff --git a/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts b/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts index ba31c3eac2..635457dd55 100644 --- a/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts +++ b/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts @@ -120,7 +120,10 @@ export class FiatOutputJobService { const fileName = `settlement_${Util.isoDateTime(entity.created)}_${routeId}.ep2`; const reportBuffer = Buffer.from(report); - await createStorageService(container).uploadBlob(fileName, reportBuffer, 'text/xml'); + // WORM sink: uploadWormBlob fails closed if the (runtime-resolved, per-merchant) EP2 + // container is not Object-Lock protected, so a mis-provisioned bucket never silently + // accepts mutable GeBüV settlement records instead of throwing here. + await createStorageService(container).uploadWormBlob(fileName, reportBuffer, 'text/xml'); // Mark the report as created as soon as the WORM upload succeeded, BEFORE the best-effort // anchoring below. A recordHash failure must not leave reportCreated=false, otherwise the From b83347e7e06f2d8b3e62909eb886734186b92aaa Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:17:21 +0200 Subject: [PATCH 04/11] style: apply prettier formatting to s3-storage spec --- .../storage/__tests__/s3-storage.service.spec.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts b/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts index 4b06c7f40d..5b88b42723 100644 --- a/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts +++ b/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts @@ -214,12 +214,20 @@ describe('S3StorageService', () => { .resolves({ ObjectLockConfiguration: { ObjectLockEnabled: 'Enabled' } }); s3Mock.on(PutObjectCommand).resolves({}); - const url = await new S3StorageService(container).uploadWormBlob('settlement.ep2', Buffer.from(''), 'text/xml'); + const url = await new S3StorageService(container).uploadWormBlob( + 'settlement.ep2', + Buffer.from(''), + 'text/xml', + ); expect(url).toBe(`https://files.test.local/${container}/settlement.ep2`); const puts = s3Mock.commandCalls(PutObjectCommand); expect(puts).toHaveLength(1); - expect(puts[0].args[0].input).toMatchObject({ Bucket: container, Key: 'settlement.ep2', ContentType: 'text/xml' }); + expect(puts[0].args[0].input).toMatchObject({ + Bucket: container, + Key: 'settlement.ep2', + ContentType: 'text/xml', + }); }); it('fails closed and does NOT PUT when Object Lock is not enabled on the bucket', async () => { From 9c6c67d5ef654ec795adfcd7a05ce15c7abdfc46 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Wed, 15 Jul 2026 08:58:54 +0200 Subject: [PATCH 05/11] harden(anchoring): RFC 6962 Merkle tree, tamper-proof verification, persisted proofs, TOCTOU-safe recordHash - Merkle tree rebuilt per RFC 6962: 0x00 leaf / 0x01 node domain separation and the largest-power-of-two split, replacing the Bitcoin-style last-node duplication that made [a,b,c] and [a,b,c,c] hash to the same root (CVE-2012-2459). No production anchor exists yet (archive_batch is empty in every environment), so the format change is free now and would be irreversible after the first anchor. - verifyDocument derives the proof leaf from the supplied bytes, not the stored hash, so a tampered document no longer reports proofValid=true. Added a non-optional `verified` field (found && hashMatches && anchored && proofValid); `pending` stays separate. - Inclusion proofs are persisted (archive_file.merkleProof) and used verbatim; verifyDocument fails closed when a batch is set but the proof is missing, instead of live-reconstructing from sibling rows that may be pruned over the 11-year retention window. - recordHash uses a conditional IsNull() update and reloads fail-closed on race loss, closing the TOCTOU window against the nightly anchorPending run. - archive_file.bucket/name are `text`, not varchar(256): storage keys embed an unbounded, user-controlled file name; a long name made the recordHash insert throw and silently drop the document from anchoring. - merkle proof codec validates each sibling as a 32-byte hex digest before decoding. --- ...s => 1784103600000-AddArchiveAnchoring.js} | 6 +- .../__tests__/archive.service.spec.ts | 95 +++++++++- .../__tests__/merkle-proof-codec.spec.ts | 56 ++++++ .../anchoring/__tests__/merkle.spec.ts | 51 ++++-- .../storage/anchoring/archive-file.entity.ts | 16 +- .../storage/anchoring/archive.service.ts | 107 ++++++++--- .../storage/anchoring/merkle-proof-codec.ts | 40 ++++ .../storage/anchoring/merkle.ts | 171 +++++++++++------- 8 files changed, 423 insertions(+), 119 deletions(-) rename migration/{1781788436511-AddArchiveAnchoring.js => 1784103600000-AddArchiveAnchoring.js} (82%) create mode 100644 src/integration/infrastructure/storage/anchoring/__tests__/merkle-proof-codec.spec.ts create mode 100644 src/integration/infrastructure/storage/anchoring/merkle-proof-codec.ts diff --git a/migration/1781788436511-AddArchiveAnchoring.js b/migration/1784103600000-AddArchiveAnchoring.js similarity index 82% rename from migration/1781788436511-AddArchiveAnchoring.js rename to migration/1784103600000-AddArchiveAnchoring.js index 4c40c6ccce..ec5c1794d8 100644 --- a/migration/1781788436511-AddArchiveAnchoring.js +++ b/migration/1784103600000-AddArchiveAnchoring.js @@ -7,15 +7,15 @@ * @class * @implements {MigrationInterface} */ -module.exports = class AddArchiveAnchoring1781788436511 { - name = 'AddArchiveAnchoring1781788436511' +module.exports = class AddArchiveAnchoring1784103600000 { + name = 'AddArchiveAnchoring1784103600000' /** * @param {QueryRunner} queryRunner */ async up(queryRunner) { await queryRunner.query(`CREATE TABLE "archive_batch" ("id" SERIAL NOT NULL, "updated" TIMESTAMP NOT NULL DEFAULT now(), "created" TIMESTAMP NOT NULL DEFAULT now(), "merkleRoot" character varying(64) NOT NULL, "otsProof" text, "bitcoinHeight" integer, "status" character varying(256) NOT NULL DEFAULT 'pendingBtc', CONSTRAINT "PK_06f898016522a01c619a214aa18" PRIMARY KEY ("id"))`); - await queryRunner.query(`CREATE TABLE "archive_file" ("id" SERIAL NOT NULL, "updated" TIMESTAMP NOT NULL DEFAULT now(), "created" TIMESTAMP NOT NULL DEFAULT now(), "bucket" character varying(256) NOT NULL, "name" character varying(256) NOT NULL, "sha256" character varying(64) NOT NULL, "leafIndex" integer, "batchId" integer, CONSTRAINT "PK_17e252452a46a911a66d67dd50d" PRIMARY KEY ("id"))`); + await queryRunner.query(`CREATE TABLE "archive_file" ("id" SERIAL NOT NULL, "updated" TIMESTAMP NOT NULL DEFAULT now(), "created" TIMESTAMP NOT NULL DEFAULT now(), "bucket" text NOT NULL, "name" text NOT NULL, "sha256" character varying(64) NOT NULL, "leafIndex" integer, "merkleProof" text, "batchId" integer, CONSTRAINT "PK_17e252452a46a911a66d67dd50d" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE UNIQUE INDEX "IDX_e7da94bafd8e348ead75dcb000" ON "archive_file" ("bucket", "name") `); await queryRunner.query(`CREATE INDEX "IDX_4065eeb67cf015dc7327499de9" ON "archive_file" ("batchId") `); await queryRunner.query(`ALTER TABLE "archive_file" ADD CONSTRAINT "FK_4065eeb67cf015dc7327499de94" FOREIGN KEY ("batchId") REFERENCES "archive_batch"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); diff --git a/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts b/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts index 0e514ccc7d..750cc9edd1 100644 --- a/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts +++ b/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts @@ -14,11 +14,12 @@ import { OpenTimestampsService } from '../opentimestamps.service'; /** * A shared in-memory store backing both fake repositories. It reproduces just enough TypeORM - * behaviour (auto-increment ids, `(bucket, name)` upsert, batch-null filtering, ordering and - * `manager.transaction`) for the round-trip the service performs, so that saves inside the - * transaction the service runs on the batch repo's manager land in the same `files`/`batches` - * arrays the assertions inspect. The Merkle math is the REAL Stage-1 module; only - * OpenTimestamps is mocked so no network is touched. + * behaviour (auto-increment ids, `(bucket, name)` upsert, batch-null filtering, ordering, + * conditional `update` with criteria, and `manager.transaction`) for the round-trip the + * service performs, so that saves inside the transaction the service runs on the batch + * repo's manager land in the same `files`/`batches` arrays the assertions inspect. The + * Merkle math is the REAL Stage-1 module; only OpenTimestamps is mocked so no network is + * touched. */ function fakeStore() { const files: ArchiveFile[] = []; @@ -55,9 +56,23 @@ function fakeStore() { manager, create: (data: Partial) => Object.assign(new ArchiveFile(), data), save: async (entity: ArchiveFile | ArchiveFile[]) => saveEntity(entity), - update: async (id: number, partial: Partial) => { - const file = files.find((f) => f.id === id); - if (file) Object.assign(file, partial); + // TypeORM-style criteria update: evaluate conditions against current row state and + // return `{ affected }` so conditional updates (e.g. batch: IsNull()) are testable. + update: async (criteria: { id: number; batch?: any }, partial: Partial) => { + const file = files.find((f) => f.id === criteria.id); + if (!file) return { affected: 0 }; + + if (criteria.batch !== undefined) { + const wantsNull = criteria.batch?._type === 'isNull' || criteria.batch === null; + if (wantsNull) { + if (file.batch != null) return { affected: 0 }; + } else if (criteria.batch?.id != null) { + if (file.batch?.id !== criteria.batch.id) return { affected: 0 }; + } + } + + Object.assign(file, partial); + return { affected: 1 }; }, findOneBy: async (where: Partial) => files.find((f) => f.bucket === where.bucket && f.name === where.name) ?? undefined, @@ -172,9 +187,12 @@ describe('ArchiveService', () => { expect(batch.otsProof).toBeDefined(); expect(ots.stamp).toHaveBeenCalledTimes(1); - // every file got assigned to the batch with a leaf index + // every file got assigned to the batch with a leaf index and a persisted inclusion proof expect(fileRepo.files.every((f: ArchiveFile) => f.batch?.id === batch.id)).toBe(true); expect(fileRepo.files.map((f: ArchiveFile) => f.leafIndex).sort()).toEqual([0, 1, 2]); + expect( + fileRepo.files.every((f: ArchiveFile) => typeof f.merkleProof === 'string' && f.merkleProof.length > 0), + ).toBe(true); }); it('returns undefined when there is nothing to anchor', async () => { @@ -192,6 +210,7 @@ describe('ArchiveService', () => { expect(result.hashMatches).toBe(true); expect(result.anchored).toBe(true); expect(result.proofValid).toBe(true); + expect(result.verified).toBe(true); expect(result.pending).toBe(true); expect(result.bitcoinHeight).toBeUndefined(); }); @@ -205,7 +224,9 @@ describe('ArchiveService', () => { expect(result.hashMatches).toBe(false); // the stored hash is still genuinely anchored, only the supplied bytes differ expect(result.anchored).toBe(true); - expect(result.proofValid).toBe(true); + // leaf is derived from the supplied (tampered) bytes → proof must not validate + expect(result.proofValid).toBe(false); + expect(result.verified).toBe(false); }); it('reports an unanchored file as anchored:false', async () => { @@ -215,12 +236,14 @@ describe('ArchiveService', () => { expect(result.hashMatches).toBe(true); expect(result.anchored).toBe(false); expect(result.proofValid).toBeUndefined(); + expect(result.verified).toBe(false); }); it('reports an unknown document as found:false', async () => { const result = await service.verifyDocument('archive', 'missing.pdf', Buffer.from('whatever')); expect(result.found).toBe(false); + expect(result.verified).toBe(false); }); it('confirms a batch once OpenTimestamps reports a Bitcoin attestation', async () => { @@ -338,8 +361,60 @@ describe('ArchiveService', () => { expect(result.hashMatches).toBe(true); expect(result.anchored).toBe(true); expect(result.proofValid).toBe(true); + expect(result.verified).toBe(true); expect(result.pending).toBe(false); expect(result.bitcoinHeight).toBe(850000); }); + + it('throws when an anchored file is missing its persisted merkleProof', async () => { + await service.anchorPending(); + + const file = fileRepo.files.find((f: ArchiveFile) => f.name === 'doc-b.pdf'); + file.merkleProof = null; + + await expect(service.verifyDocument('archive', 'doc-b.pdf', docs[1].data)).rejects.toThrow( + /Data-integrity inconsistency.*no persisted merkleProof/, + ); + }); + + it('refuses a concurrent recordHash that races with anchorPending (TOCTOU)', async () => { + // Isolate to a single unanchored file so the interleaving is clear. + // (round-trip beforeEach already recorded three docs — clear and re-seed one.) + fileRepo.files.length = 0; + batchRepo.batches.length = 0; + await service.recordHash('archive', 'race.pdf', sha256(Buffer.from('original')).toString('hex')); + + const stored = fileRepo.files.find((f: ArchiveFile) => f.name === 'race.pdf'); + const anchoredHash = stored.sha256; + // Snapshot of the unanchored row as recordHash would have seen it at read time. + const staleSnapshot = Object.assign(new ArchiveFile(), { + id: stored.id, + bucket: stored.bucket, + name: stored.name, + sha256: stored.sha256, + batch: undefined, + leafIndex: undefined, + merkleProof: undefined, + }); + + // Between findOne (read) and the conditional update, run a full anchorPending that + // claims the row into a batch. recordHash still only sees the stale pre-anchor snapshot. + jest.spyOn(fileRepo, 'findOne').mockImplementationOnce(async () => { + await service.anchorPending(); + return staleSnapshot; + }); + + const newHash = sha256(Buffer.from('concurrent re-upload')).toString('hex'); + await expect(service.recordHash('archive', 'race.pdf', newHash)).rejects.toThrow( + /Refusing to overwrite anchored hash.*concurrent anchor/, + ); + + // Leaf committed by anchorPending must be untouched. + const after = fileRepo.files.find((f: ArchiveFile) => f.name === 'race.pdf'); + expect(after.sha256).toBe(anchoredHash); + expect(after.batch).toBeDefined(); + expect(typeof after.merkleProof).toBe('string'); + expect(after.merkleProof.length).toBeGreaterThan(0); + }); }); }); diff --git a/src/integration/infrastructure/storage/anchoring/__tests__/merkle-proof-codec.spec.ts b/src/integration/infrastructure/storage/anchoring/__tests__/merkle-proof-codec.spec.ts new file mode 100644 index 0000000000..1d496286e8 --- /dev/null +++ b/src/integration/infrastructure/storage/anchoring/__tests__/merkle-proof-codec.spec.ts @@ -0,0 +1,56 @@ +import { deserializeMerkleProof, serializeMerkleProof } from '../merkle-proof-codec'; +import { MerkleProofStep } from '../merkle'; + +describe('merkle-proof-codec', () => { + const validSibling = Buffer.alloc(32, 0xab); + const validSteps: MerkleProofStep[] = [ + { sibling: validSibling, right: true }, + { sibling: Buffer.alloc(32, 0xcd), right: false }, + ]; + + describe('serializeMerkleProof / deserializeMerkleProof roundtrip', () => { + it('returns equivalent steps after serialize then deserialize', () => { + const json = serializeMerkleProof(validSteps); + const restored = deserializeMerkleProof(json); + + expect(restored).toHaveLength(2); + expect(restored[0].right).toBe(true); + expect(restored[0].sibling.equals(validSibling)).toBe(true); + expect(restored[1].right).toBe(false); + expect(restored[1].sibling.equals(Buffer.alloc(32, 0xcd))).toBe(true); + }); + }); + + describe('deserializeMerkleProof validation', () => { + it('throws when JSON is not an array', () => { + expect(() => deserializeMerkleProof(JSON.stringify({ sibling: 'x', right: true }))).toThrow( + 'expected an array of steps', + ); + }); + + it('throws on non-hex sibling', () => { + const json = JSON.stringify([{ sibling: 'g'.repeat(64), right: true }]); + expect(() => deserializeMerkleProof(json)).toThrow('Invalid merkle proof step at index 0'); + }); + + it('throws on wrong-length hex sibling (63 chars)', () => { + const json = JSON.stringify([{ sibling: 'a'.repeat(63), right: true }]); + expect(() => deserializeMerkleProof(json)).toThrow('Invalid merkle proof step at index 0'); + }); + + it('throws on wrong-length hex sibling (65 chars)', () => { + const json = JSON.stringify([{ sibling: 'a'.repeat(65), right: true }]); + expect(() => deserializeMerkleProof(json)).toThrow('Invalid merkle proof step at index 0'); + }); + + it('throws on uppercase hex sibling (lowercase required)', () => { + const json = JSON.stringify([{ sibling: 'A'.repeat(64), right: true }]); + expect(() => deserializeMerkleProof(json)).toThrow('Invalid merkle proof step at index 0'); + }); + + it('throws on non-boolean right', () => { + const json = JSON.stringify([{ sibling: 'a'.repeat(64), right: 'yes' }]); + expect(() => deserializeMerkleProof(json)).toThrow('Invalid merkle proof step at index 0'); + }); + }); +}); diff --git a/src/integration/infrastructure/storage/anchoring/__tests__/merkle.spec.ts b/src/integration/infrastructure/storage/anchoring/__tests__/merkle.spec.ts index b50ab59b5e..813563bb9e 100644 --- a/src/integration/infrastructure/storage/anchoring/__tests__/merkle.spec.ts +++ b/src/integration/infrastructure/storage/anchoring/__tests__/merkle.spec.ts @@ -9,9 +9,16 @@ function leaves(count: number): Buffer[] { return Array.from({ length: count }, (_, i) => leaf(i)); } -/** Reference parent hash matching the module's documented `sha256(left || right)` rule. */ -function parent(left: Buffer, right: Buffer): Buffer { - return sha256(Buffer.concat([left, right])); +// Independent reference helpers — do NOT call production merkle functions. +const LEAF_PREFIX = Buffer.from([0x00]); +const NODE_PREFIX = Buffer.from([0x01]); + +function leafHash(d: Buffer): Buffer { + return sha256(Buffer.concat([LEAF_PREFIX, d])); +} + +function node(l: Buffer, r: Buffer): Buffer { + return sha256(Buffer.concat([NODE_PREFIX, l, r])); } describe('merkle', () => { @@ -20,26 +27,26 @@ describe('merkle', () => { expect(() => buildMerkleRoot([])).toThrow(); }); - it('returns the leaf itself for a single-leaf tree', () => { - const l = leaf(0); - expect(buildMerkleRoot([l]).equals(l)).toBe(true); + it('returns leafHash(d) for a single-leaf tree', () => { + const d0 = leaf(0); + expect(buildMerkleRoot([d0]).equals(leafHash(d0))).toBe(true); }); it('hashes the pair for two leaves', () => { const [a, b] = leaves(2); - expect(buildMerkleRoot([a, b]).equals(parent(a, b))).toBe(true); + expect(buildMerkleRoot([a, b]).equals(node(leafHash(a), leafHash(b)))).toBe(true); }); - it('duplicates the last node for three leaves', () => { + it('builds an unbalanced tree for three leaves (no last-node duplication)', () => { const [a, b, c] = leaves(3); - // level 1: [h(a,b), h(c,c)] -> root: h( h(a,b), h(c,c) ) - const expected = parent(parent(a, b), parent(c, c)); + // k=2: root = node(node(L0,L1), L2) — NOT node(node(L0,L1), node(L2,L2)) + const expected = node(node(leafHash(a), leafHash(b)), leafHash(c)); expect(buildMerkleRoot([a, b, c]).equals(expected)).toBe(true); }); it('builds a balanced tree for four leaves', () => { const [a, b, c, d] = leaves(4); - const expected = parent(parent(a, b), parent(c, d)); + const expected = node(node(leafHash(a), leafHash(b)), node(leafHash(c), leafHash(d))); expect(buildMerkleRoot([a, b, c, d]).equals(expected)).toBe(true); }); @@ -47,6 +54,28 @@ describe('merkle', () => { const ls = leaves(4); expect(buildMerkleRoot(ls).equals(buildMerkleRoot(ls))).toBe(true); }); + + it('CVE-2012-2459: three-leaf and four-leaf-with-duplicated-last produce different roots', () => { + const [a, b, c] = leaves(3); + const root3 = buildMerkleRoot([a, b, c]); + const root4dup = buildMerkleRoot([a, b, c, c]); + expect(root3.equals(root4dup)).toBe(false); + }); + + it('applies domain-separation prefixes (leaf and node)', () => { + const d = leaf(0); + const root1 = buildMerkleRoot([d]); + // Single leaf is leafHash(d), not the raw digest. + expect(root1.equals(d)).toBe(false); + expect(root1.equals(leafHash(d))).toBe(true); + + const [a, b] = leaves(2); + const root2 = buildMerkleRoot([a, b]); + // Two leaves use node() with 0x01 prefix, not bare sha256 of concatenated leaf hashes. + const bareNoPrefix = sha256(Buffer.concat([leafHash(a), leafHash(b)])); + expect(root2.equals(bareNoPrefix)).toBe(false); + expect(root2.equals(node(leafHash(a), leafHash(b)))).toBe(true); + }); }); describe('inclusion proof + verification', () => { diff --git a/src/integration/infrastructure/storage/anchoring/archive-file.entity.ts b/src/integration/infrastructure/storage/anchoring/archive-file.entity.ts index 829a639e54..064a1d90ec 100644 --- a/src/integration/infrastructure/storage/anchoring/archive-file.entity.ts +++ b/src/integration/infrastructure/storage/anchoring/archive-file.entity.ts @@ -6,16 +6,16 @@ import { ArchiveBatch } from './archive-batch.entity'; * A single archived storage object whose content hash participates in the GeBüV anchoring * pipeline. Each file is identified uniquely by its `(bucket, name)` location and records * the SHA-256 of its content. Once anchored, it points to its {@link ArchiveBatch} and - * carries its `leafIndex` within that batch's Merkle tree (needed to rebuild the inclusion - * proof during verification). + * carries its `leafIndex` within that batch's Merkle tree plus a persisted inclusion proof + * so verification does not depend on sibling rows remaining available. */ @Entity() @Index((file: ArchiveFile) => [file.bucket, file.name], { unique: true }) export class ArchiveFile extends IEntity { - @Column({ length: 256 }) + @Column({ type: 'text' }) bucket: string; - @Column({ length: 256 }) + @Column({ type: 'text' }) name: string; @Column({ length: 64 }) @@ -27,4 +27,12 @@ export class ArchiveFile extends IEntity { @Column({ type: 'int', nullable: true }) leafIndex?: number; + + /** + * JSON-serialized inclusion proof (`MerkleProofStep[]`, siblings hex-encoded), populated + * from `merkleInclusionProof` at anchor time. Persisted so an old file stays verifiable + * even if sibling rows in the same batch are later lost or corrupted. + */ + @Column({ type: 'text', nullable: true }) + merkleProof?: string; } diff --git a/src/integration/infrastructure/storage/anchoring/archive.service.ts b/src/integration/infrastructure/storage/anchoring/archive.service.ts index c8d841f0ff..57b9cf0a40 100644 --- a/src/integration/infrastructure/storage/anchoring/archive.service.ts +++ b/src/integration/infrastructure/storage/anchoring/archive.service.ts @@ -4,6 +4,7 @@ import { IsNull } from 'typeorm'; import { ArchiveBatch, ArchiveBatchStatus } from './archive-batch.entity'; import { ArchiveBatchRepository } from './archive-batch.repository'; import { ArchiveFileRepository } from './archive-file.repository'; +import { deserializeMerkleProof, serializeMerkleProof } from './merkle-proof-codec'; import { buildMerkleRoot, merkleInclusionProof, sha256, verifyMerkleProof } from './merkle'; import { OpenTimestampsService } from './opentimestamps.service'; @@ -17,9 +18,18 @@ export interface ArchiveVerification { anchored?: boolean; /** true if the inclusion proof recomputes the batch's stored Merkle root. */ proofValid?: boolean; + /** + * true iff found && hashMatches && anchored && proofValid (all four independently true). + * Independent of `pending`: a calendar-only (not yet Bitcoin-confirmed) proof can still + * be fully verified against its Merkle root. + */ + verified: boolean; /** Bitcoin block height of the OpenTimestamps attestation, once anchored on-chain. */ bitcoinHeight?: number; - /** true while the OpenTimestamps proof is still calendar-only (not yet on-chain). */ + /** + * true while the OpenTimestamps proof is still calendar-only (not yet on-chain). + * Independent of `verified` — pending does not mean unverified. + */ pending?: boolean; } @@ -29,9 +39,11 @@ export interface ArchiveVerification { * via OpenTimestamps (Stage 1 primitives), upgrades those proofs to Bitcoin attestations, * and verifies a given document against its anchored batch end-to-end. * - * Leaves are the raw 32-byte SHA-256 digests of the file contents (the Merkle module does - * NOT re-hash leaves). `merkleRoot` is stored hex, `otsProof` is stored base64 of the - * serialized detached `.ots` bytes. + * Leaves are the 32-byte SHA-256 digests of the file contents; the Merkle module + * domain-separates them via RFC 6962 leaf hashing (`sha256(0x00 || digest)`) before they + * enter the tree. `merkleRoot` is stored hex, `otsProof` is stored base64 of the serialized + * detached `.ots` bytes, and each file's inclusion proof is persisted as JSON on + * `archive_file.merkleProof`. */ @Injectable() export class ArchiveService { @@ -56,6 +68,10 @@ export class ArchiveService { * leaf hash and make {@link verifyDocument} report bogus tampering. Therefore, for an * already-anchored record: an identical hash is a no-op, and a differing hash is a hard * error (the existing anchored hash is never overwritten). + * + * When the row looks unanchored at read time, the update is conditional on `batch` still + * being null at write time, so a concurrent {@link anchorPending} that claims the row + * cannot be overwritten by a stale write (TOCTOU). */ async recordHash(bucket: string, name: string, sha256Hex: string): Promise { const existing = await this.archiveFileRepo.findOne({ where: { bucket, name }, relations: ['batch'] }); @@ -71,17 +87,58 @@ export class ArchiveService { throw new Error(message); } - await this.archiveFileRepo.update(existing.id, { sha256: sha256Hex }); - return; + // Conditional update: only write if still unanchored (closes TOCTOU with anchorPending). + const result = await this.archiveFileRepo.update({ id: existing.id, batch: IsNull() }, { sha256: sha256Hex }); + if (result.affected !== 0) return; + + // Lost the race: a concurrent anchorPending claimed this row between read and write. + const reloaded = await this.archiveFileRepo.findOne({ where: { id: existing.id }, relations: ['batch'] }); + if (!reloaded) { + const message = `Archive file ${existing.id} (${bucket}/${name}) disappeared during concurrent update`; + this.logger.error(message); + throw new Error(message); + } + + if (reloaded.sha256 === sha256Hex) return; + + if (reloaded.batch != null) { + const message = + `Refusing to overwrite anchored hash for ${bucket}/${name} (file ${reloaded.id}, batch ` + + `${reloaded.batch.id}): concurrent anchor claimed the row; stored ${reloaded.sha256} ` + + `differs from new ${sha256Hex}`; + this.logger.error(message); + throw new Error(message); + } + + // batch still null but conditional update matched nothing — data-integrity inconsistency. + const message = + `Unexpected data-integrity inconsistency for ${bucket}/${name} (file ${reloaded.id}): ` + + `conditional update affected 0 rows but file is still unanchored with differing hash ` + + `(stored ${reloaded.sha256}, new ${sha256Hex})`; + this.logger.error(message); + throw new Error(message); } const file = this.archiveFileRepo.create({ bucket, name, sha256: sha256Hex }); await this.archiveFileRepo.save(file); } + /** + * Names already present in the `(bucket, name)` archive index for a bucket, regardless of + * anchoring status. Used by {@link ArchiveScheduler.reconcileHashes} to diff live storage + * objects against recorded hashes and find gaps left by a failed {@link recordHash} call — + * which, unlike a normal upload failure, leaves no row at all, so a DB-only scan can never + * find it. + */ + async recordedNames(bucket: string): Promise> { + const files = await this.archiveFileRepo.find({ where: { bucket }, select: ['name'] }); + return new Set(files.map((file) => file.name)); + } + /** * Batch all currently unanchored files (ordered by id) into one Merkle tree, timestamp its - * root via OpenTimestamps, and persist batch + per-file assignment in a single transaction. + * root via OpenTimestamps, and persist batch + per-file assignment (including each file's + * inclusion proof) in a single transaction. * * Returns the created batch, or `undefined` if there is nothing to anchor. */ @@ -106,6 +163,7 @@ export class ArchiveService { files.forEach((file, index) => { file.batch = savedBatch; file.leafIndex = index; + file.merkleProof = serializeMerkleProof(merkleInclusionProof(leaves, index)); }); await manager.save(files); @@ -159,28 +217,34 @@ export class ArchiveService { /** * Verify a supplied document against its archived, anchored Merkle batch end-to-end: - * recompute its SHA-256, compare with the stored hash, rebuild the inclusion proof against - * the batch's Merkle root, and check the OpenTimestamps attestation status. + * recompute its SHA-256 from the supplied bytes, compare with the stored hash, verify the + * persisted inclusion proof against the batch's Merkle root using the supplied digest as + * the leaf, and check the OpenTimestamps attestation status. */ async verifyDocument(bucket: string, name: string, data: Buffer): Promise { const file = await this.archiveFileRepo.findOne({ where: { bucket, name }, relations: ['batch'] }); - if (!file) return { found: false }; + if (!file) return { found: false, verified: false }; - const computedHex = sha256(data).toString('hex'); + const computedDigest = sha256(data); + const computedHex = computedDigest.toString('hex'); const hashMatches = file.sha256 === computedHex; const batch = file.batch; - if (!batch) return { found: true, hashMatches, anchored: false }; - - const batchFiles = await this.archiveFileRepo.find({ - where: { batch: { id: batch.id } }, - order: { leafIndex: 'ASC' }, - }); - const leaves = batchFiles.map((batchFile) => Buffer.from(batchFile.sha256, 'hex')); + if (!batch) return { found: true, hashMatches, anchored: false, verified: false }; + + if (file.merkleProof == null || file.merkleProof === '') { + const message = + `Data-integrity inconsistency: archive file ${file.id} (${bucket}/${name}) is assigned ` + + `to batch ${batch.id} but has no persisted merkleProof`; + this.logger.error(message); + throw new Error(message); + } const rootBuffer = Buffer.from(batch.merkleRoot, 'hex'); - const proof = merkleInclusionProof(leaves, file.leafIndex); - const proofValid = verifyMerkleProof(Buffer.from(file.sha256, 'hex'), proof, rootBuffer); + const proof = deserializeMerkleProof(file.merkleProof); + // Leaf must be the digest of the SUPPLIED bytes — never the stored hash — so a + // tampered document yields proofValid: false. + const proofValid = verifyMerkleProof(computedDigest, proof, rootBuffer); let bitcoinHeight: number; let pending = true; @@ -191,6 +255,7 @@ export class ArchiveService { if (ots.bitcoin) bitcoinHeight = ots.bitcoin.height; } - return { found: true, hashMatches, anchored: true, proofValid, bitcoinHeight, pending }; + const verified = hashMatches && proofValid; + return { found: true, hashMatches, anchored: true, proofValid, verified, bitcoinHeight, pending }; } } diff --git a/src/integration/infrastructure/storage/anchoring/merkle-proof-codec.ts b/src/integration/infrastructure/storage/anchoring/merkle-proof-codec.ts new file mode 100644 index 0000000000..08c8f19f22 --- /dev/null +++ b/src/integration/infrastructure/storage/anchoring/merkle-proof-codec.ts @@ -0,0 +1,40 @@ +import { MerkleProofStep } from './merkle'; + +/** JSON shape of one persisted proof step (sibling hex-encoded). */ +interface SerializedMerkleProofStep { + sibling: string; + right: boolean; +} + +/** + * Serialize an inclusion proof for storage on `archive_file.merkleProof`. + * Siblings are hex-encoded 32-byte digests. + */ +export function serializeMerkleProof(proof: MerkleProofStep[]): string { + const serialized: SerializedMerkleProofStep[] = proof.map((step) => ({ + sibling: step.sibling.toString('hex'), + right: step.right, + })); + return JSON.stringify(serialized); +} + +/** + * Deserialize a persisted inclusion proof from `archive_file.merkleProof`. + * Throws if the JSON is not an array of steps with hex sibling digests. + */ +export function deserializeMerkleProof(json: string): MerkleProofStep[] { + const parsed: unknown = JSON.parse(json); + if (!Array.isArray(parsed)) { + throw new Error('Invalid merkle proof JSON: expected an array of steps'); + } + + return parsed.map((step: SerializedMerkleProofStep, index: number) => { + if (typeof step?.sibling !== 'string' || !/^[0-9a-f]{64}$/.test(step.sibling) || typeof step?.right !== 'boolean') { + throw new Error(`Invalid merkle proof step at index ${index}`); + } + return { + sibling: Buffer.from(step.sibling, 'hex'), + right: step.right, + }; + }); +} diff --git a/src/integration/infrastructure/storage/anchoring/merkle.ts b/src/integration/infrastructure/storage/anchoring/merkle.ts index 5d32b8e259..7a23d58a21 100644 --- a/src/integration/infrastructure/storage/anchoring/merkle.ts +++ b/src/integration/infrastructure/storage/anchoring/merkle.ts @@ -1,26 +1,37 @@ import { createHash } from 'node:crypto'; /** - * Pure Merkle-tree primitives for the GeBüV anchoring pipeline. + * Pure Merkle-tree primitives for the GeBüV anchoring pipeline, following the + * RFC 6962 Merkle Tree Hash (MTH) construction. * - * HASH / CONCATENATION RULE (load-bearing — verification depends on it): - * - All hashing is SHA-256 (`node:crypto`). - * - A parent node is `parent = sha256(left || right)`, where `||` is raw byte - * concatenation of the two 32-byte child digests (left first, then right). - * - Leaves are used as-is: they are NOT re-hashed by this module. Callers pass - * already-hashed leaves (e.g. `sha256(documentBytes)`). This keeps the module - * agnostic about leaf preimages and avoids a hidden hashing convention. - * - On a level with an ODD number of nodes, the last node is DUPLICATED and - * paired with itself (`parent = sha256(last || last)`). This is the classic - * Bitcoin-style promotion rule and is reproduced identically in proofs and - * verification so the computed root always matches. + * DOMAIN SEPARATION (load-bearing — verification depends on it): + * - LEAF_PREFIX = 0x00, NODE_PREFIX = 0x01 (single bytes). + * - `leafHash(d) = sha256(0x00 || d)` where `d` is a 32-byte document DIGEST. + * Callers (archive.service.ts) pass `archive_file.sha256` as raw digest bytes; + * the raw document bytes are NOT available inside this module. The exported + * functions therefore domain-separate each digest via `leafHash` before it + * enters the tree — leaves are never used raw. + * - `node(l, r) = sha256(0x01 || l || r)` for two 32-byte child hashes. * - * Edge cases: - * - 0 leaves: `buildMerkleRoot` throws (an empty tree has no root). - * - 1 leaf: the root IS that leaf (no hashing applied), and its inclusion proof - * is the empty path. + * TREE SHAPE (Merkle Tree Hash, MTH) for a list D[n] of digests, n >= 1: + * - n == 1: MTH(D) = leafHash(D[0]). + * - n > 1: let k be the largest power of two STRICTLY smaller than n + * (k = 1; while (k * 2 < n) k *= 2). Then + * MTH(D[0:n]) = node(MTH(D[0:k]), MTH(D[k:n])). + * There is NO duplication of a trailing odd element (avoids CVE-2012-2459). + * - n == 0: throw. + * + * Worked example (3 leaves d0,d1,d2; L_i = leafHash(d_i)): + * k=2, root = node(node(L0,L1), L2) + * — NOT node(node(L0,L1), node(L2,L2)) (the old, wrong, duplicated form). + * + * Inclusion proofs (RFC 6962 §2.1.1 audit path) use the same split rule and are + * ordered from the leaf's immediate sibling first to the root-adjacent sibling last. */ +const LEAF_PREFIX = Buffer.from([0x00]); +const NODE_PREFIX = Buffer.from([0x01]); + /** A single step of an inclusion proof: the sibling digest and whether it sits on the right. */ export interface MerkleProofStep { sibling: Buffer; @@ -33,81 +44,101 @@ export function sha256(data: Buffer): Buffer { return createHash('sha256').update(data).digest(); } -/** Hash a parent from its two children using the documented `sha256(left || right)` rule. */ -function hashPair(left: Buffer, right: Buffer): Buffer { - return sha256(Buffer.concat([left, right])); +/** Domain-separate a document digest into a leaf hash: sha256(0x00 || d). */ +function leafHash(digest: Buffer): Buffer { + return sha256(Buffer.concat([LEAF_PREFIX, digest])); +} + +/** Combine two child hashes into a parent: sha256(0x01 || left || right). */ +function node(left: Buffer, right: Buffer): Buffer { + return sha256(Buffer.concat([NODE_PREFIX, left, right])); } /** - * Build the Merkle root over `leaves`. + * Largest power of two strictly smaller than `n`. + * For n == 1 this is unused (caller handles the base case). + */ +function largestPowerOfTwoStrictlyLessThan(n: number): number { + let k = 1; + while (k * 2 < n) k *= 2; + return k; +} + +/** + * Build the Merkle root over document digests per RFC 6962 MTH. * - * Throws on an empty input. For a single leaf the root equals that leaf. - * On odd levels the last node is duplicated (see module rule). + * Throws on an empty input. Each digest is leaf-hashed before entering the tree. */ -export function buildMerkleRoot(leaves: Buffer[]): Buffer { - if (leaves.length === 0) throw new Error('Cannot build a Merkle root from zero leaves'); - - let level = leaves; - while (level.length > 1) { - const next: Buffer[] = []; - for (let i = 0; i < level.length; i += 2) { - const left = level[i]; - // Odd node count: duplicate the last node and pair it with itself. - const right = i + 1 < level.length ? level[i + 1] : level[i]; - next.push(hashPair(left, right)); - } - level = next; - } +export function buildMerkleRoot(leafDigests: Buffer[]): Buffer { + if (leafDigests.length === 0) throw new Error('Cannot build a Merkle root from zero leaves'); - return level[0]; + return mth(leafDigests); +} + +/** Recursive Merkle Tree Hash over a slice of document digests. */ +function mth(digests: Buffer[]): Buffer { + const n = digests.length; + if (n === 1) return leafHash(digests[0]); + + const k = largestPowerOfTwoStrictlyLessThan(n); + return node(mth(digests.slice(0, k)), mth(digests.slice(k))); } /** - * Compute the inclusion proof for the leaf at `index` — the ordered list of sibling - * digests (with their left/right position) from the leaf up to (but excluding) the root. + * Compute the RFC 6962 §2.1.1 inclusion (audit) path for the digest at `index`. + * + * Ordered from the leaf's immediate sibling first to the root-adjacent sibling last — + * the order `verifyMerkleProof` walks (leaf → root). * - * For a single-leaf tree the proof is empty. The duplication rule for odd levels is - * applied identically here, so a leaf that is the duplicated odd node gets a sibling - * equal to itself on the right. + * For a single-digest tree the proof is empty. Sibling subtree hashes are recomputed + * via MTH (acceptable for small daily KYC batches). */ -export function merkleInclusionProof(leaves: Buffer[], index: number): MerkleProofStep[] { - if (leaves.length === 0) throw new Error('Cannot build a proof from zero leaves'); - if (index < 0 || index >= leaves.length) throw new Error(`Leaf index ${index} out of range [0, ${leaves.length})`); - - const proof: MerkleProofStep[] = []; - - let level = leaves; - let idx = index; - while (level.length > 1) { - const isLeft = idx % 2 === 0; - // Sibling index; for the duplicated last odd node the sibling is the node itself. - const siblingIdx = isLeft ? Math.min(idx + 1, level.length - 1) : idx - 1; - - proof.push({ sibling: level[siblingIdx], right: isLeft }); - - const next: Buffer[] = []; - for (let i = 0; i < level.length; i += 2) { - const left = level[i]; - const right = i + 1 < level.length ? level[i + 1] : level[i]; - next.push(hashPair(left, right)); - } - - level = next; - idx = Math.floor(idx / 2); +export function merkleInclusionProof(leafDigests: Buffer[], index: number): MerkleProofStep[] { + if (leafDigests.length === 0) throw new Error('Cannot build a proof from zero leaves'); + if (index < 0 || index >= leafDigests.length) { + throw new Error(`Leaf index ${index} out of range [0, ${leafDigests.length})`); + } + + return auditPath(leafDigests, index); +} + +/** Recursive audit-path construction matching the MTH split rule. */ +function auditPath(digests: Buffer[], index: number): MerkleProofStep[] { + const n = digests.length; + if (n === 1) return []; + + const k = largestPowerOfTwoStrictlyLessThan(n); + + if (index < k) { + // Target in left half: recurse left, then append right-half sibling (sibling is RIGHT). + const proof = auditPath(digests.slice(0, k), index); + proof.push({ sibling: mth(digests.slice(k)), right: true }); + return proof; } + // Target in right half: recurse right, then append left-half sibling (sibling is LEFT). + const proof = auditPath(digests.slice(k), index - k); + proof.push({ sibling: mth(digests.slice(0, k)), right: false }); return proof; } /** - * Recompute the root from `leaf` and its `proof` and compare it against the expected `root`. - * Returns true only on an exact byte match. + * Recompute the root from a document DIGEST and its inclusion proof, then compare + * against the expected `root`. Returns true only on an exact byte match. + * + * `leaf` is the raw document digest (not pre-hashed). This function applies + * `leafHash` internally before walking the proof: + * computed = leafHash(leaf); + * for each step: computed = step.right + * ? node(computed, step.sibling) + * : node(step.sibling, computed); + * return computed.equals(root) */ export function verifyMerkleProof(leaf: Buffer, proof: MerkleProofStep[], root: Buffer): boolean { - let computed = leaf; + let computed = leafHash(leaf); for (const step of proof) { - computed = step.right ? hashPair(computed, step.sibling) : hashPair(step.sibling, computed); + computed = step.right ? node(computed, step.sibling) : node(step.sibling, computed); } return computed.equals(root); From 7932fe970a223de862b8e9cf3c61f448d61621dd Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Wed, 15 Jul 2026 08:59:15 +0200 Subject: [PATCH 06/11] harden(storage): WORM-verified KYC writes, anchor merge-copies, add reconcile safety net - KYC uploads go through uploadWormBlob (was uploadBlob), so a kyc bucket that does not enforce Object Lock is rejected fail-closed instead of silently accepting mutable compliance records. copyBlobs asserts Object Lock before copying, since an account-merge copy into the kyc bucket is a fresh WORM write. copyBlobs returns the destination keys so merge-copied documents get anchored the same way uploads are. - reconcileHashes cron backfills anchor gaps left by a failed best-effort recordHash. It lists keys only (no per-object HeadObject fan-out over 1.3M objects) and caps the backfill per run, logging loudly when capped, to avoid OOM and the cron timeout. Bucket enumeration is per-row fault-tolerant: one unparsable paymentLinksConfig no longer aborts the whole run, and the kyc bucket is always reconciled regardless of the EP2 lookup. - redundant per-method try/catch removed from the DfxCron jobs; the framework already logs. --- .../__tests__/mock-storage.service.spec.ts | 21 ++- .../__tests__/s3-storage.service.spec.ts | 28 +++- .../__tests__/archive.scheduler.spec.ts | 130 +++++++++++++++++- .../storage/anchoring/archive.scheduler.ts | 117 +++++++++++++--- .../storage/mock-storage.service.ts | 23 +++- .../storage/s3-storage.service.ts | 12 +- .../infrastructure/storage/storage.service.ts | 5 +- src/shared/services/process.service.ts | 1 + .../__tests__/kyc-document.service.spec.ts | 74 +++++++++- .../integration/kyc-document.service.ts | 28 +++- 10 files changed, 389 insertions(+), 50 deletions(-) diff --git a/src/integration/infrastructure/storage/__tests__/mock-storage.service.spec.ts b/src/integration/infrastructure/storage/__tests__/mock-storage.service.spec.ts index e19ae0d290..f6f5173971 100644 --- a/src/integration/infrastructure/storage/__tests__/mock-storage.service.spec.ts +++ b/src/integration/infrastructure/storage/__tests__/mock-storage.service.spec.ts @@ -69,6 +69,23 @@ describe('MockStorageService', () => { }); }); + describe('listKeys', () => { + it('filters by prefix and returns plain key strings', async () => { + const service = new MockStorageService('mock-spec-keys'); + await service.uploadBlob('user/1/a.png', Buffer.from('a'), 'image/png'); + await service.uploadBlob('user/1/b.pdf', Buffer.from('b'), 'application/pdf'); + await service.uploadBlob('user/2/c.png', Buffer.from('c'), 'image/png'); + + expect((await service.listKeys('user/1/')).sort()).toEqual(['user/1/a.png', 'user/1/b.pdf']); + }); + + it('returns [] when nothing matches the prefix', async () => { + const service = new MockStorageService('mock-spec-keys-empty'); + + expect(await service.listKeys('nope/')).toEqual([]); + }); + }); + describe('getBlob dummy-file fallback', () => { const service = new MockStorageService('mock-spec-dummy'); @@ -115,7 +132,9 @@ describe('MockStorageService', () => { await service.uploadBlob('src/a.png', Buffer.from('a'), 'image/png', { k: 'v' }); await service.uploadBlob('src/sub/b.pdf', Buffer.from('b'), 'application/pdf'); - await service.copyBlobs('src/', 'dst/'); + const targetKeys = await service.copyBlobs('src/', 'dst/'); + + expect(targetKeys.sort()).toEqual(['dst/a.png', 'dst/sub/b.pdf']); const copied = await service.listBlobs('dst/'); expect(copied.map((b) => b.name).sort()).toEqual(['dst/a.png', 'dst/sub/b.pdf']); diff --git a/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts b/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts index 5b88b42723..1ddc40654e 100644 --- a/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts +++ b/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts @@ -276,13 +276,20 @@ describe('S3StorageService', () => { }); describe('copyBlobs', () => { + beforeEach(() => { + s3Mock + .on(GetObjectLockConfigurationCommand, { Bucket: CONTAINER }) + .resolves({ ObjectLockConfiguration: { ObjectLockEnabled: 'Enabled' } }); + }); + it('URL-encodes keys with spaces / special chars and rewrites the prefix', async () => { const key = 'src/sub dir/file (1)+&.png'; s3Mock.on(ListObjectsV2Command).resolves({ Contents: [{ Key: key }], IsTruncated: false }); s3Mock.on(CopyObjectCommand).resolves({}); - await new S3StorageService(CONTAINER).copyBlobs('src/', 'dst/'); + const targetKeys = await new S3StorageService(CONTAINER).copyBlobs('src/', 'dst/'); + expect(targetKeys).toEqual(['dst/sub dir/file (1)+&.png']); const calls = s3Mock.commandCalls(CopyObjectCommand); expect(calls).toHaveLength(1); expect(calls[0].args[0].input).toMatchObject({ @@ -299,8 +306,9 @@ describe('S3StorageService', () => { s3Mock.on(ListObjectsV2Command).resolves({ Contents: [{ Key: 'src/a' }, { Key: 'src/b' }], IsTruncated: false }); s3Mock.on(CopyObjectCommand).resolves({}); - await new S3StorageService(CONTAINER).copyBlobs('src/', 'dst/'); + const targetKeys = await new S3StorageService(CONTAINER).copyBlobs('src/', 'dst/'); + expect(targetKeys).toEqual(['dst/a', 'dst/b']); const keys = s3Mock.commandCalls(CopyObjectCommand).map((c) => c.args[0].input.Key); expect(keys).toEqual(['dst/a', 'dst/b']); }); @@ -308,7 +316,21 @@ describe('S3StorageService', () => { it('does nothing for an empty source prefix', async () => { s3Mock.on(ListObjectsV2Command).resolves({ Contents: [], IsTruncated: false }); - await new S3StorageService(CONTAINER).copyBlobs('src/', 'dst/'); + const targetKeys = await new S3StorageService(CONTAINER).copyBlobs('src/', 'dst/'); + + expect(targetKeys).toEqual([]); + expect(s3Mock.commandCalls(CopyObjectCommand)).toHaveLength(0); + }); + + it('fails closed and does NOT copy when Object Lock is not enabled on the bucket', async () => { + const container = 'copy-worm-unlocked'; + s3Mock.on(GetObjectLockConfigurationCommand, { Bucket: container }).resolves({ ObjectLockConfiguration: {} }); + s3Mock.on(ListObjectsV2Command).resolves({ Contents: [{ Key: 'src/a' }], IsTruncated: false }); + s3Mock.on(CopyObjectCommand).resolves({}); + + await expect(new S3StorageService(container).copyBlobs('src/', 'dst/')).rejects.toThrow( + 'Object Lock is not enabled', + ); expect(s3Mock.commandCalls(CopyObjectCommand)).toHaveLength(0); }); diff --git a/src/integration/infrastructure/storage/anchoring/__tests__/archive.scheduler.spec.ts b/src/integration/infrastructure/storage/anchoring/__tests__/archive.scheduler.spec.ts index 66dcb10ecf..9ecb576ee8 100644 --- a/src/integration/infrastructure/storage/anchoring/__tests__/archive.scheduler.spec.ts +++ b/src/integration/infrastructure/storage/anchoring/__tests__/archive.scheduler.spec.ts @@ -2,11 +2,20 @@ // eager network/`request` deps never load; ArchiveService is fully mocked in this spec anyway. jest.mock('opentimestamps', () => ({})); +const mockCreateStorageService = jest.fn(); +jest.mock('src/integration/infrastructure/storage/storage.factory', () => ({ + createStorageService: (...args: any[]) => mockCreateStorageService(...args), +})); + import { createMock } from '@golevelup/ts-jest'; import { Test, TestingModule } from '@nestjs/testing'; +import { RepositoryFactory } from 'src/shared/repositories/repository.factory'; +import { UserData } from 'src/subdomains/generic/user/models/user-data/user-data.entity'; +import { UserDataRepository } from 'src/subdomains/generic/user/models/user-data/user-data.repository'; import { ArchiveBatch } from '../archive-batch.entity'; import { ArchiveScheduler } from '../archive.scheduler'; import { ArchiveService } from '../archive.service'; +import { sha256 } from '../merkle'; describe('ArchiveScheduler', () => { let scheduler: ArchiveScheduler; @@ -16,7 +25,11 @@ describe('ArchiveScheduler', () => { archiveService = createMock(); const module: TestingModule = await Test.createTestingModule({ - providers: [ArchiveScheduler, { provide: ArchiveService, useValue: archiveService }], + providers: [ + ArchiveScheduler, + { provide: ArchiveService, useValue: archiveService }, + { provide: RepositoryFactory, useValue: createMock() }, + ], }).compile(); scheduler = module.get(ArchiveScheduler); @@ -35,10 +48,10 @@ describe('ArchiveScheduler', () => { expect(archiveService.anchorPending).toHaveBeenCalledTimes(1); }); - it('swallows errors so a failure never crashes the scheduler', async () => { + it('propagates errors to the @DfxCron wrapper', async () => { (archiveService.anchorPending as jest.Mock).mockRejectedValue(new Error('calendar down')); - await expect(scheduler.anchorPending()).resolves.toBeUndefined(); + await expect(scheduler.anchorPending()).rejects.toThrow('calendar down'); }); }); @@ -49,10 +62,117 @@ describe('ArchiveScheduler', () => { expect(archiveService.upgradeBatches).toHaveBeenCalledTimes(1); }); - it('swallows errors so a failure never crashes the scheduler', async () => { + it('propagates errors to the @DfxCron wrapper', async () => { (archiveService.upgradeBatches as jest.Mock).mockRejectedValue(new Error('upgrade failed')); - await expect(scheduler.upgradeBatches()).resolves.toBeUndefined(); + await expect(scheduler.upgradeBatches()).rejects.toThrow('upgrade failed'); + }); + }); + + describe('reconcileHashes', () => { + let scheduler: ArchiveScheduler; + let archiveService: ArchiveService; + let repos: RepositoryFactory; + + const storageStubs: Record = {}; + + const stubStorage = (container: string, blobs: { name: string; data: Buffer }[]) => { + storageStubs[container] = { + listKeys: jest.fn().mockResolvedValue(blobs.map((b) => b.name)), + getBlob: jest.fn((name: string) => Promise.resolve({ data: blobs.find((b) => b.name === name)!.data })), + }; + }; + + beforeEach(async () => { + for (const key of Object.keys(storageStubs)) delete storageStubs[key]; + mockCreateStorageService.mockReset(); + mockCreateStorageService.mockImplementation((container: string) => storageStubs[container]); + + archiveService = createMock(); + repos = createMock(); + (repos.userData as unknown) = createMock(); + + (repos.userData.find as jest.Mock).mockResolvedValue([]); + (archiveService.recordedNames as jest.Mock).mockResolvedValue(new Set()); + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + ArchiveScheduler, + { provide: ArchiveService, useValue: archiveService }, + { provide: RepositoryFactory, useValue: repos }, + ], + }).compile(); + + scheduler = module.get(ArchiveScheduler); + }); + + it('recordHash only objects present in storage but absent from recordedNames', async () => { + const missingData = Buffer.from('missing-doc'); + const presentData = Buffer.from('already-recorded'); + + stubStorage('kyc', [ + { name: 'user/1/a.pdf', data: missingData }, + { name: 'user/1/b.pdf', data: presentData }, + ]); + (archiveService.recordedNames as jest.Mock).mockResolvedValue(new Set(['user/1/b.pdf'])); + + await scheduler.reconcileHashes(); + + expect(archiveService.recordHash).toHaveBeenCalledTimes(1); + expect(archiveService.recordHash).toHaveBeenCalledWith( + 'kyc', + 'user/1/a.pdf', + sha256(missingData).toString('hex'), + ); + expect(archiveService.recordHash).not.toHaveBeenCalledWith('kyc', 'user/1/b.pdf', expect.anything()); + }); + + it('deduplicates ep2ReportContainer values across UserData rows', async () => { + const container = 'merchant-ep2-bucket'; + const ud1 = Object.assign(new UserData(), { + id: 1, + paymentLinksConfig: JSON.stringify({ ep2ReportContainer: container }), + }); + const ud2 = Object.assign(new UserData(), { + id: 2, + paymentLinksConfig: JSON.stringify({ ep2ReportContainer: container }), + }); + (repos.userData.find as jest.Mock).mockResolvedValue([ud1, ud2]); + + stubStorage('kyc', []); + stubStorage(container, [{ name: 'settlement.ep2', data: Buffer.from('') }]); + + await scheduler.reconcileHashes(); + + const containersRequested = mockCreateStorageService.mock.calls.map((c) => c[0]); + expect(containersRequested.filter((c) => c === container)).toHaveLength(1); + expect(containersRequested).toContain('kyc'); + expect(archiveService.recordHash).toHaveBeenCalledTimes(1); + expect(archiveService.recordHash).toHaveBeenCalledWith( + container, + 'settlement.ep2', + sha256(Buffer.from('')).toString('hex'), + ); + }); + + it('continues when recordHash rejects for one missing object', async () => { + const dataA = Buffer.from('doc-a'); + const dataB = Buffer.from('doc-b'); + + stubStorage('kyc', [ + { name: 'user/1/a.pdf', data: dataA }, + { name: 'user/1/b.pdf', data: dataB }, + ]); + (archiveService.recordedNames as jest.Mock).mockResolvedValue(new Set()); + (archiveService.recordHash as jest.Mock) + .mockRejectedValueOnce(new Error('already anchored under a different hash')) + .mockResolvedValueOnce(undefined); + + await expect(scheduler.reconcileHashes()).resolves.toBeUndefined(); + + expect(archiveService.recordHash).toHaveBeenCalledTimes(2); + expect(archiveService.recordHash).toHaveBeenCalledWith('kyc', 'user/1/a.pdf', sha256(dataA).toString('hex')); + expect(archiveService.recordHash).toHaveBeenCalledWith('kyc', 'user/1/b.pdf', sha256(dataB).toString('hex')); }); }); }); diff --git a/src/integration/infrastructure/storage/anchoring/archive.scheduler.ts b/src/integration/infrastructure/storage/anchoring/archive.scheduler.ts index 688985f15e..775cdf6032 100644 --- a/src/integration/infrastructure/storage/anchoring/archive.scheduler.ts +++ b/src/integration/infrastructure/storage/anchoring/archive.scheduler.ts @@ -1,9 +1,24 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; +import { RepositoryFactory } from 'src/shared/repositories/repository.factory'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; import { DfxCron } from 'src/shared/utils/cron'; +import { IsNull, Not } from 'typeorm'; +import { createStorageService } from '../storage.factory'; import { ArchiveService } from './archive.service'; +import { sha256 } from './merkle'; + +// Mirrors KYC_CONTAINER in kyc-document.service.ts — duplicated rather than imported so this +// low-level storage/anchoring module does not depend on the KYC domain module. +const KYC_BUCKET = 'kyc'; + +// Caps how many missing objects get backfilled per reconcile run so a huge one-off gap (e.g. after +// a migration) cannot blow the job's 3600s @DfxCron timeout or exhaust memory/sockets fetching +// every object in one Promise chain. The job is idempotent and re-runs daily, so anything beyond +// the cap is simply backfilled on a later run — the gap is never silently dropped, just deferred +// (and logged loudly below). +const RECONCILE_BACKFILL_BATCH_SIZE = 5000; /** * Stage 3 of the GeBüV anchoring pipeline: drives {@link ArchiveService} on a schedule. @@ -11,36 +26,106 @@ import { ArchiveService } from './archive.service'; * Uses the repo-wide `@DfxCron` decorator (see src/shared/utils/cron.ts), which the * central DfxCronService discovers at boot and runs behind a per-method `@Lock` plus a * `Process` guard — so a disabled process or a multi-instance deployment never double-runs - * the same job. Each job is additionally wrapped in try/catch with structured logging so a - * single failure (e.g. an OpenTimestamps calendar outage) never crashes the scheduler. + * the same job. */ @Injectable() export class ArchiveScheduler { private readonly logger = new DfxLogger(ArchiveScheduler); - constructor(private readonly archiveService: ArchiveService) {} + constructor( + private readonly archiveService: ArchiveService, + private readonly repos: RepositoryFactory, + ) {} @DfxCron(CronExpression.EVERY_DAY_AT_2AM, { process: Process.ARCHIVE_ANCHOR, timeout: 3600 }) async anchorPending(): Promise { - try { - const batch = await this.archiveService.anchorPending(); + const batch = await this.archiveService.anchorPending(); - if (batch) { - this.logger.info(`Anchored batch ${batch.id} with Merkle root ${batch.merkleRoot}`); - } else { - this.logger.verbose('No unanchored archive files to anchor'); - } - } catch (e) { - this.logger.error('Failed to anchor pending archive files:', e); + if (batch) { + this.logger.info(`Anchored batch ${batch.id} with Merkle root ${batch.merkleRoot}`); + } else { + this.logger.verbose('No unanchored archive files to anchor'); } } @DfxCron(CronExpression.EVERY_HOUR, { process: Process.ARCHIVE_UPGRADE, timeout: 1800 }) async upgradeBatches(): Promise { - try { - await this.archiveService.upgradeBatches(); - } catch (e) { - this.logger.error('Failed to upgrade pending archive batches:', e); + await this.archiveService.upgradeBatches(); + } + + @DfxCron(CronExpression.EVERY_DAY_AT_1AM, { process: Process.ARCHIVE_RECONCILE, timeout: 3600 }) + async reconcileHashes(): Promise { + const buckets = await this.reconciliationBuckets(); + + for (const bucket of buckets) { + try { + await this.reconcileBucket(bucket); + } catch (e) { + this.logger.error(`Failed to reconcile archive hashes for bucket ${bucket}:`, e); + } + } + } + + // Retention-relevant buckets: the fixed KYC bucket plus every distinct EP2 settlement-report + // container currently configured on any UserData (resolved the same way + // fiat-output-job.service.ts resolves it at write time — these are per-merchant, runtime-only, + // and cannot be hardcoded). + private async reconciliationBuckets(): Promise { + const userDatas = await this.repos.userData.find({ + where: { paymentLinksConfig: Not(IsNull()) }, + select: ['id', 'paymentLinksConfig'], + }); + + const ep2Containers: string[] = []; + for (const userData of userDatas) { + try { + const container = userData.paymentLinksConfigObj.ep2ReportContainer; + if (container != null) ep2Containers.push(container); + } catch (e) { + this.logger.error( + `Skipping UserData ${userData.id} with unparsable paymentLinksConfig during reconcile bucket enumeration:`, + e, + ); + } + } + + return [...new Set([KYC_BUCKET, ...ep2Containers])]; + } + + // Diffs live storage objects against the archive index and backfills any gap left by a + // previously failed recordHash call. A per-object recordHash failure (e.g. it throws because the + // object is already anchored under a different hash — a legitimate re-upload/overwrite on a + // versioned bucket) must not abort reconciliation of the remaining objects: log loudly and keep + // going, never swallow silently. + private async reconcileBucket(bucket: string): Promise { + const storageService = createStorageService(bucket); + + const [keys, recordedNames] = await Promise.all([ + storageService.listKeys(), + this.archiveService.recordedNames(bucket), + ]); + + const missing = keys.filter((key) => !recordedNames.has(key)); + if (missing.length === 0) return; + + const batch = missing.slice(0, RECONCILE_BACKFILL_BATCH_SIZE); + if (missing.length > batch.length) { + this.logger.warn( + `Reconciling only ${batch.length} of ${missing.length} unrecorded archive object(s) in bucket ${bucket} this run; ${ + missing.length - batch.length + } deferred to the next scheduled run`, + ); + } else { + this.logger.info(`Reconciling ${batch.length} unrecorded archive object(s) in bucket ${bucket}`); + } + + for (const key of batch) { + try { + const content = await storageService.getBlob(key); + await this.archiveService.recordHash(bucket, key, sha256(content.data).toString('hex')); + } catch (e) { + this.logger.error(`Failed to reconcile archive hash for ${bucket}/${key}:`, e); + } } } } diff --git a/src/integration/infrastructure/storage/mock-storage.service.ts b/src/integration/infrastructure/storage/mock-storage.service.ts index cf510ea081..1c47006296 100644 --- a/src/integration/infrastructure/storage/mock-storage.service.ts +++ b/src/integration/infrastructure/storage/mock-storage.service.ts @@ -47,6 +47,14 @@ export class MockStorageService extends StorageService { }); } + async listKeys(prefix?: string): Promise { + const keyPrefix = `${this.container}/${prefix ?? ''}`; + + return [...mockStorage.keys()] + .filter((key) => key.startsWith(keyPrefix)) + .map((key) => key.replace(`${this.container}/`, '')); + } + async getBlob(name: string): Promise { const stored = mockStorage.get(`${this.container}/${name}`); if (stored) @@ -87,15 +95,16 @@ export class MockStorageService extends StorageService { return this.blobUrl(name); } - async copyBlobs(sourcePrefix: string, targetPrefix: string): Promise { + async copyBlobs(sourcePrefix: string, targetPrefix: string): Promise { + const targetKeys: string[] = []; + for (const blob of await this.listBlobs(sourcePrefix)) { const content = await this.getBlob(blob.name); - await this.uploadBlob( - blob.name.replace(sourcePrefix, targetPrefix), - content.data, - content.contentType, - blob.metadata, - ); + const targetKey = blob.name.replace(sourcePrefix, targetPrefix); + await this.uploadBlob(targetKey, content.data, content.contentType, blob.metadata); + targetKeys.push(targetKey); } + + return targetKeys; } } diff --git a/src/integration/infrastructure/storage/s3-storage.service.ts b/src/integration/infrastructure/storage/s3-storage.service.ts index 7093b4b5fd..0286449fd7 100644 --- a/src/integration/infrastructure/storage/s3-storage.service.ts +++ b/src/integration/infrastructure/storage/s3-storage.service.ts @@ -104,22 +104,28 @@ export class S3StorageService extends StorageService { S3StorageService.objectLockVerified.add(this.container); } - async copyBlobs(sourcePrefix: string, targetPrefix: string): Promise { + async copyBlobs(sourcePrefix: string, targetPrefix: string): Promise { + await this.assertObjectLockEnabled(); // copy needs only the keys, not metadata — avoid the per-object HeadObject fan-out. const keys = await this.listKeys(sourcePrefix); + const targetKeys: string[] = []; for (const key of keys) { + const targetKey = key.replace(sourcePrefix, targetPrefix); await this.client.send( new CopyObjectCommand({ Bucket: this.container, - Key: key.replace(sourcePrefix, targetPrefix), + Key: targetKey, CopySource: `${this.container}/${this.encodeKey(key)}`, // key must be URL-encoded }), ); + targetKeys.push(targetKey); } + + return targetKeys; } - private async listKeys(prefix?: string): Promise { + async listKeys(prefix?: string): Promise { const keys: string[] = []; let token: string | undefined; diff --git a/src/integration/infrastructure/storage/storage.service.ts b/src/integration/infrastructure/storage/storage.service.ts index 02b1e6948b..8710248999 100644 --- a/src/integration/infrastructure/storage/storage.service.ts +++ b/src/integration/infrastructure/storage/storage.service.ts @@ -31,9 +31,12 @@ export abstract class StorageService { constructor(protected readonly container: string) {} abstract listBlobs(prefix?: string): Promise; + /** Keys only, no per-object metadata fetch — cheap for large buckets (e.g. reconciliation diffing). */ + abstract listKeys(prefix?: string): Promise; abstract getBlob(name: string): Promise; abstract uploadBlob(name: string, data: Buffer, type: string, metadata?: Record): Promise; - abstract copyBlobs(sourcePrefix: string, targetPrefix: string): Promise; + /** Copies every object under `sourcePrefix` to `targetPrefix` and returns the TARGET keys of every object copied. */ + abstract copyBlobs(sourcePrefix: string, targetPrefix: string): Promise; /** * WORM sink for GeBüV-retention-relevant compliance records (e.g. EP2 settlement reports written diff --git a/src/shared/services/process.service.ts b/src/shared/services/process.service.ts index 12eb2eb0d2..5ab9544ff3 100644 --- a/src/shared/services/process.service.ts +++ b/src/shared/services/process.service.ts @@ -97,6 +97,7 @@ export enum Process { GUARANTEED_PRICE = 'GuaranteedPrice', ARCHIVE_ANCHOR = 'ArchiveAnchor', ARCHIVE_UPGRADE = 'ArchiveUpgrade', + ARCHIVE_RECONCILE = 'ArchiveReconcile', GS_DEBUG = 'GsDebug', GS_DB = 'GsDb', TRANSACTION_AML_CHECK_LOG = 'TransactionAmlCheckLog', diff --git a/src/subdomains/generic/kyc/services/integration/__tests__/kyc-document.service.spec.ts b/src/subdomains/generic/kyc/services/integration/__tests__/kyc-document.service.spec.ts index 98e3d68347..0f7ae3e701 100644 --- a/src/subdomains/generic/kyc/services/integration/__tests__/kyc-document.service.spec.ts +++ b/src/subdomains/generic/kyc/services/integration/__tests__/kyc-document.service.spec.ts @@ -2,12 +2,16 @@ // eager network/`request` deps never load; ArchiveService is fully mocked in this spec. jest.mock('opentimestamps', () => ({})); -// Control the storage backend the service constructs in its constructor, so uploadBlob is a +// Control the storage backend the service constructs in its constructor, so uploadWormBlob is a // spy and no real S3/Azure/mock storage is touched. -const uploadBlobMock = jest.fn(); +const uploadWormBlobMock = jest.fn(); +const copyBlobsMock = jest.fn(); +const getBlobMock = jest.fn(); jest.mock('src/integration/infrastructure/storage/storage.factory', () => ({ createStorageService: jest.fn(() => ({ - uploadBlob: (...args: any[]) => uploadBlobMock(...args), + uploadWormBlob: (...args: any[]) => uploadWormBlobMock(...args), + copyBlobs: (...args: any[]) => copyBlobsMock(...args), + getBlob: (...args: any[]) => getBlobMock(...args), })), })); @@ -39,7 +43,7 @@ describe('KycDocumentService - GeBüV hash recording', () => { archiveService = createMock(); (kycFileService.createKycFile as jest.Mock).mockResolvedValue({ id: 7 } as KycFile); - uploadBlobMock.mockResolvedValue('https://storage/blob-url'); + uploadWormBlobMock.mockResolvedValue('https://storage/blob-url'); const module: TestingModule = await Test.createTestingModule({ providers: [ @@ -63,8 +67,8 @@ describe('KycDocumentService - GeBüV hash recording', () => { ); // upload happened, then recordHash with the deterministic blob name + sha256 of the data - expect(uploadBlobMock).toHaveBeenCalledTimes(1); - expect(uploadBlobMock.mock.calls[0][0]).toBe(expectedBlobName); + expect(uploadWormBlobMock).toHaveBeenCalledTimes(1); + expect(uploadWormBlobMock.mock.calls[0][0]).toBe(expectedBlobName); expect(archiveService.recordHash).toHaveBeenCalledTimes(1); expect(archiveService.recordHash).toHaveBeenCalledWith('kyc', expectedBlobName, expectedHash); @@ -94,7 +98,63 @@ describe('KycDocumentService - GeBüV hash recording', () => { service.uploadUserFile(userData, FileType.IDENTIFICATION, 'note.txt', data, 'text/plain' as ContentType, true), ).rejects.toThrow('Supported file types'); - expect(uploadBlobMock).not.toHaveBeenCalled(); + expect(uploadWormBlobMock).not.toHaveBeenCalled(); expect(archiveService.recordHash).not.toHaveBeenCalled(); }); + + describe('copyFiles', () => { + it('getBlob and recordHash each copied target key with its content hash', async () => { + const copiedData = Buffer.from('copied kyc document'); + const targetKey = 'user/99/Identification/passport.pdf'; + const expectedCopyHash = sha256(copiedData).toString('hex'); + + // three prefixes: spider, spider-org, user — only the last returns a copy + copyBlobsMock.mockResolvedValueOnce([]).mockResolvedValueOnce([]).mockResolvedValueOnce([targetKey]); + getBlobMock.mockResolvedValue({ data: copiedData, contentType: 'application/pdf' }); + + await service.copyFiles(42, 99); + + expect(copyBlobsMock).toHaveBeenCalledTimes(3); + expect(copyBlobsMock).toHaveBeenCalledWith('spider/42/', 'spider/99/'); + expect(copyBlobsMock).toHaveBeenCalledWith('spider/42-organization/', 'spider/99-organization/'); + expect(copyBlobsMock).toHaveBeenCalledWith('user/42/', 'user/99/'); + + expect(getBlobMock).toHaveBeenCalledTimes(1); + expect(getBlobMock).toHaveBeenCalledWith(targetKey); + + expect(archiveService.recordHash).toHaveBeenCalledTimes(1); + expect(archiveService.recordHash).toHaveBeenCalledWith('kyc', targetKey, expectedCopyHash); + }); + + it('does not call recordHash when copyBlobs returns empty for a prefix', async () => { + copyBlobsMock.mockResolvedValue([]); + + await service.copyFiles(42, 99); + + expect(copyBlobsMock).toHaveBeenCalledTimes(3); + expect(getBlobMock).not.toHaveBeenCalled(); + expect(archiveService.recordHash).not.toHaveBeenCalled(); + }); + + it('does NOT abort remaining copies when recordHash fails for one key (best-effort)', async () => { + const dataA = Buffer.from('doc-a'); + const dataB = Buffer.from('doc-b'); + const keyA = 'user/99/Identification/a.pdf'; + const keyB = 'user/99/Identification/b.pdf'; + + copyBlobsMock.mockResolvedValueOnce([]).mockResolvedValueOnce([]).mockResolvedValueOnce([keyA, keyB]); + getBlobMock + .mockResolvedValueOnce({ data: dataA, contentType: 'application/pdf' }) + .mockResolvedValueOnce({ data: dataB, contentType: 'application/pdf' }); + (archiveService.recordHash as jest.Mock) + .mockRejectedValueOnce(new Error('archive db down')) + .mockResolvedValueOnce(undefined); + + await expect(service.copyFiles(42, 99)).resolves.toBeUndefined(); + + expect(archiveService.recordHash).toHaveBeenCalledTimes(2); + expect(archiveService.recordHash).toHaveBeenCalledWith('kyc', keyA, sha256(dataA).toString('hex')); + expect(archiveService.recordHash).toHaveBeenCalledWith('kyc', keyB, sha256(dataB).toString('hex')); + }); + }); }); diff --git a/src/subdomains/generic/kyc/services/integration/kyc-document.service.ts b/src/subdomains/generic/kyc/services/integration/kyc-document.service.ts index 0b802b9850..9df49db9c6 100644 --- a/src/subdomains/generic/kyc/services/integration/kyc-document.service.ts +++ b/src/subdomains/generic/kyc/services/integration/kyc-document.service.ts @@ -121,7 +121,7 @@ export class KycDocumentService { const blobName = this.toFileId(FileCategory.USER, userData.id, type, name); - const url = await this.storageService.uploadBlob(blobName, data, contentType, metadata); + const url = await this.storageService.uploadWormBlob(blobName, data, contentType, metadata); // GeBüV anchoring (Stage 3): record the content hash of the just-uploaded KYC document // (a retention-relevant compliance bucket) so it can later be Merkle-batched and anchored. @@ -142,12 +142,26 @@ export class KycDocumentService { } async copyFiles(sourceUserDataId: number, targetUserDataId: number): Promise { - await this.storageService.copyBlobs(`spider/${sourceUserDataId}/`, `spider/${targetUserDataId}/`); - await this.storageService.copyBlobs( - `spider/${sourceUserDataId}-organization/`, - `spider/${targetUserDataId}-organization/`, - ); - await this.storageService.copyBlobs(`user/${sourceUserDataId}/`, `user/${targetUserDataId}/`); + await this.copyAndAnchor(`spider/${sourceUserDataId}/`, `spider/${targetUserDataId}/`); + await this.copyAndAnchor(`spider/${sourceUserDataId}-organization/`, `spider/${targetUserDataId}-organization/`); + await this.copyAndAnchor(`user/${sourceUserDataId}/`, `user/${targetUserDataId}/`); + } + + // GeBüV anchoring (Stage 3): a copy made by copyBlobs is a fresh WORM write just like an + // upload, so it must be recorded the same way uploadFile does — otherwise a merged account's + // copied documents would never get a time anchor. Best-effort per file, same as uploadFile: + // failures are logged (never silently swallowed) but never abort the remaining copies. + private async copyAndAnchor(sourcePrefix: string, targetPrefix: string): Promise { + const targetKeys = await this.storageService.copyBlobs(sourcePrefix, targetPrefix); + + for (const targetKey of targetKeys) { + try { + const blob = await this.storageService.getBlob(targetKey); + await this.archiveService.recordHash(KYC_CONTAINER, targetKey, sha256(blob.data).toString('hex')); + } catch (e) { + this.logger.error(`GeBüV anchoring failed to record hash for ${KYC_CONTAINER}/${targetKey}:`, e); + } + } } // --- HELPER METHODS --- // From 7a63d8bcfbcf0cec5a247143a8f304deda22b33a Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Wed, 15 Jul 2026 08:59:28 +0200 Subject: [PATCH 07/11] harden(storage): fail-loud config, scoped provisioning creds, resilient compliance download - kycBlobKey throws on a blob URL without the container marker instead of silently returning the full URL, which would reinstate the exact host-equality bug the cutover filter fixes. To keep that fail-loud without taking down the whole GwG compliance export, the per-target filter in downloadUserData is wrapped: a single dirty step result is recorded in error_log.csv and skipped, the rest of the ZIP is still produced. - provision-bucket uses dedicated S3_ADMIN_* credentials (no fallback to the app keys, which cannot create buckets or set Object Lock), distinguishes 403 from 404 instead of guessing, and reads the Object Lock config back after applying it, failing closed on any mismatch. - .env.example documents the seven new S3_* / S3_ADMIN_* vars and drops the dead AZURE_STORAGE_CONNECTION_STRING; the still-used AZURE_* client vars stay. --- .env.example | 15 +++++- scripts/storage/provision-bucket.ts | 47 +++++++++++++++++-- .../__tests__/file-download-config.spec.ts | 13 +++++ src/config/config.ts | 12 ++++- .../models/user-data/user-data.service.ts | 15 +++++- 5 files changed, 95 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index 17e0d1d19d..833646e5f2 100644 --- a/.env.example +++ b/.env.example @@ -265,7 +265,20 @@ AZURE_CLIENT_ID= AZURE_CLIENT_SECRET= AZURE_TENANT_ID= AZURE_SUBSCRIPTION_ID= -AZURE_STORAGE_CONNECTION_STRING= + +# S3-compatible object storage (on-prem MinIO) — replacement for Azure Blob. +# endpoint/region/publicUrl are non-secret and belong in the deploy compose environment, not here; +# accessKey/secretKey are app credentials (secret -> vault in real environments). +S3_ENDPOINT= +S3_REGION= +S3_ACCESS_KEY= +S3_SECRET_KEY= +S3_PUBLIC_URL= + +# Provisioning-only S3 credentials (broader admin policy) used solely by +# scripts/storage/provision-bucket.ts — never used by the running app. Secret -> vault. +S3_ADMIN_ACCESS_KEY= +S3_ADMIN_SECRET_KEY= # OpenTelemetry trace export (OTLP/HTTP). Point this at an OTLP collector to # enable distributed tracing; leave empty to disable. e.g. http://localhost:4318 diff --git a/scripts/storage/provision-bucket.ts b/scripts/storage/provision-bucket.ts index 1055fb9983..bbf5c27ffe 100644 --- a/scripts/storage/provision-bucket.ts +++ b/scripts/storage/provision-bucket.ts @@ -18,7 +18,11 @@ * default-retention configuration is (re)applied. * * Configuration (no silent defaults for credentials — fails fast if incomplete): - * - S3 endpoint/region/credentials come from the standard S3_* env vars (via Config.s3). + * - S3 endpoint/region from Config.s3 (S3_ENDPOINT / S3_REGION — shared with the app). + * - Auth from Config.s3Admin (S3_ADMIN_ACCESS_KEY / S3_ADMIN_SECRET_KEY) — dedicated + * provisioning credentials with CreateBucket/Object-Lock rights. The app's policy- + * restricted S3_ACCESS_KEY/S3_SECRET_KEY are intentionally NOT used and NOT fallen + * back to: they cannot create buckets or apply Object Lock. * * Run with (bucket name required, retention years optional, default 11): * BUCKET=kyc RETENTION_YEARS=11 npx ts-node scripts/storage/provision-bucket.ts @@ -28,6 +32,7 @@ import { CreateBucketCommand, GetBucketVersioningCommand, + GetObjectLockConfigurationCommand, HeadBucketCommand, ObjectLockRetentionMode, PutBucketVersioningCommand, @@ -38,7 +43,7 @@ import * as dotenv from 'dotenv'; dotenv.config(); -// Loaded after dotenv so Config.s3 reads the populated env. +// Loaded after dotenv so Config.s3 / Config.s3Admin read the populated env. import { Config } from '../../src/config/config'; function getBucketName(): string { @@ -69,9 +74,12 @@ export function getRetentionYears(): number { } function buildClient(): S3Client { - const { endpoint, region, accessKey, secretKey } = Config.s3; + const { endpoint, region } = Config.s3; + const { accessKey, secretKey } = Config.s3Admin; if (!endpoint || !region || !accessKey || !secretKey) - throw new Error('Incomplete S3 config: S3_ENDPOINT, S3_REGION, S3_ACCESS_KEY and S3_SECRET_KEY are required'); + throw new Error( + 'Incomplete S3 admin config: S3_ENDPOINT, S3_REGION, S3_ADMIN_ACCESS_KEY and S3_ADMIN_SECRET_KEY are required', + ); return new S3Client({ endpoint, @@ -88,6 +96,15 @@ async function bucketExists(client: S3Client, bucket: string): Promise } catch (e) { const status = e?.$metadata?.httpStatusCode; if (status === 404 || e?.name === 'NotFound' || e?.name === 'NoSuchBucket') return false; + // HeadBucket on this deployment returns 403 (not 404) when the caller's policy lacks + // access — including for buckets that do not exist yet. Neither true nor false is safe: + // true would skip provisioning a genuinely new EP2 merchant bucket; false would blind- + // CreateBucket against a bucket that may already exist. Fail loud so the policy is fixed. + if (status === 403 || e?.name === 'AccessDenied') + throw new Error( + `Cannot verify bucket existence — provisioning credentials lack permission on "${bucket}" (403). ` + + `Check the S3_ADMIN_ACCESS_KEY/S3_ADMIN_SECRET_KEY policy.`, + ); throw e; } } @@ -118,6 +135,27 @@ async function applyObjectLock(client: S3Client, bucket: string, years: number): console.log(` Object Lock: default retention COMPLIANCE / ${years} year(s)`); } +// Object Lock cannot be retro-fitted once objects exist. Read back the applied configuration +// and refuse success unless COMPLIANCE + requested years are confirmed live on the bucket. +async function verifyObjectLock(client: S3Client, bucket: string, years: number): Promise { + const result = await client.send(new GetObjectLockConfigurationCommand({ Bucket: bucket })); + const cfg = result.ObjectLockConfiguration; + const retention = cfg?.Rule?.DefaultRetention; + + if ( + cfg?.ObjectLockEnabled !== 'Enabled' || + retention?.Mode !== ObjectLockRetentionMode.COMPLIANCE || + retention?.Years !== years + ) { + throw new Error( + `Object Lock verification failed for bucket "${bucket}": expected Enabled + COMPLIANCE / ${years} year(s), ` + + `got ObjectLockEnabled=${cfg?.ObjectLockEnabled}, Mode=${retention?.Mode}, Years=${retention?.Years}`, + ); + } + + console.log(` Object Lock verified: Enabled + COMPLIANCE / ${years} year(s)`); +} + async function main(): Promise { const bucket = getBucketName(); const years = getRetentionYears(); @@ -134,6 +172,7 @@ async function main(): Promise { await ensureVersioning(client, bucket); await applyObjectLock(client, bucket, years); + await verifyObjectLock(client, bucket, years); console.log(`Done. Bucket "${bucket}" is WORM-protected (COMPLIANCE, ${years}y).`); } diff --git a/src/config/__tests__/file-download-config.spec.ts b/src/config/__tests__/file-download-config.spec.ts index fda9ab122b..72af13861d 100644 --- a/src/config/__tests__/file-download-config.spec.ts +++ b/src/config/__tests__/file-download-config.spec.ts @@ -59,6 +59,19 @@ describe('fileDownloadConfig - host-independent KYC document selection (storage expect(Configuration.isSameKycBlob(undefined, liveMinioUrl)).toBe(false); expect(Configuration.isSameKycBlob(legacyAzureUrl, undefined)).toBe(false); }); + + // A URL without the container marker must not fall back to full-URL equality (that was the + // pre-cutover silent-drop bug). Fail loud so a malformed stored/live URL surfaces as an error + // instead of quietly excluding Handelsregisterauszug / Vollmacht from the compliance ZIP. + it('throws when a URL is missing the kyc/ container marker', () => { + const malformed = 'https://files.dfx.swiss/other-container/user/42/CommercialRegister/hr-auszug.pdf'; + expect(() => Configuration.isSameKycBlob(malformed, liveMinioUrl)).toThrow( + /Unexpected KYC blob URL format \(missing 'kyc\/' marker\)/, + ); + expect(() => Configuration.isSameKycBlob(legacyAzureUrl, malformed)).toThrow( + /Unexpected KYC blob URL format \(missing 'kyc\/' marker\)/, + ); + }); }); describe('id 11 - Handelsregisterauszug', () => { diff --git a/src/config/config.ts b/src/config/config.ts index 2765bb171f..d5a68462a8 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -327,10 +327,13 @@ export class Configuration { // Reduce a KYC blob URL to its container-relative, percent-decoded object key (drops scheme/host // and the leading `/` segment), mirroring StorageService.blobName's reversal. + // Fail loud on unexpected URL shapes: a silent full-URL fallback would reintroduce the pre-cutover + // host-equality bug for any URL missing the container marker (missing compliance docs, no error). private static kycBlobKey(url: string): string { const marker = 'kyc/'; const idx = url.indexOf(marker); - const rel = idx < 0 ? url : url.substring(idx + marker.length); + if (idx < 0) throw new Error(`Unexpected KYC blob URL format (missing 'kyc/' marker): ${url}`); + const rel = url.substring(idx + marker.length); return rel.split('/').map(decodeURIComponent).join('/'); } @@ -1293,6 +1296,13 @@ export class Configuration { publicUrl: process.env.S3_PUBLIC_URL, }; + // Provisioning-only S3 credentials (broader admin policy), distinct from the policy-restricted + // app credentials in `s3`. Used only by scripts/storage/provision-bucket.ts. secrets -> .env. + s3Admin = { + accessKey: process.env.S3_ADMIN_ACCESS_KEY, + secretKey: process.env.S3_ADMIN_SECRET_KEY, + }; + alby = { clientId: process.env.ALBY_CLIENT_ID, clientSecret: process.env.ALBY_CLIENT_SECRET, diff --git a/src/subdomains/generic/user/models/user-data/user-data.service.ts b/src/subdomains/generic/user/models/user-data/user-data.service.ts index 4c369b3a73..ad113e239a 100644 --- a/src/subdomains/generic/user/models/user-data/user-data.service.ts +++ b/src/subdomains/generic/user/models/user-data/user-data.service.ts @@ -480,7 +480,20 @@ export class UserDataService { const files = allFiles .filter((f) => prefixes(userData).some((p) => f.path.startsWith(p))) .filter((f) => !fileTypes || fileTypes.some((t) => f.contentType.startsWith(t))) - .filter((f) => !filter || filter(f, userData)); + .filter((f) => { + if (!filter) return true; + try { + return filter(f, userData); + } catch (e) { + errors.push({ + userDataId, + errorType: 'FilterError', + folder: folderName, + details: e?.message ?? String(e), + }); + return false; + } + }); if (!files.length) { if (handleFileNotFound) { From a4c70bd44aaa35e9a09fb35899f8b3d9d65b0466 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:36:58 +0200 Subject: [PATCH 08/11] fix(storage): harden anchoring pipeline and provisioning per review - provision-bucket: initialize Config via GetConfig() (was uninitialized -> crash) - s3-storage: assert Object Lock COMPLIANCE mode + retention floor, not just Enabled - anchoring: close anchorPending/recordHash TOCTOU by never overwriting sha256 on existing rows (per-file conditional update) - anchoring: recordHash fails closed (audit log + throw) on hash divergence instead of silent overwrite - fiat-output: collision-safe EP2 report filename (include entity id) - style: PascalCase enum values, typed OTS return, TypeORM object-syntax, merkleProof getter/setter --- .../1784103600000-AddArchiveAnchoring.js | 2 +- scripts/storage/provision-bucket.ts | 16 ++- .../__tests__/s3-storage.service.spec.ts | 21 +++- .../__tests__/archive.service.spec.ts | 85 +++++++++----- .../storage/anchoring/archive-batch.entity.ts | 4 +- .../storage/anchoring/archive-file.entity.ts | 14 ++- .../storage/anchoring/archive.scheduler.ts | 2 +- .../storage/anchoring/archive.service.ts | 110 +++++++++--------- .../anchoring/opentimestamps.service.ts | 12 +- .../storage/s3-storage.service.ts | 19 ++- .../storage/worm-retention.const.ts | 10 ++ .../generic/gs/__tests__/gs.service.spec.ts | 2 +- .../fiat-output/fiat-output-job.service.ts | 2 +- 13 files changed, 192 insertions(+), 107 deletions(-) create mode 100644 src/integration/infrastructure/storage/worm-retention.const.ts diff --git a/migration/1784103600000-AddArchiveAnchoring.js b/migration/1784103600000-AddArchiveAnchoring.js index ec5c1794d8..2afed71326 100644 --- a/migration/1784103600000-AddArchiveAnchoring.js +++ b/migration/1784103600000-AddArchiveAnchoring.js @@ -14,7 +14,7 @@ module.exports = class AddArchiveAnchoring1784103600000 { * @param {QueryRunner} queryRunner */ async up(queryRunner) { - await queryRunner.query(`CREATE TABLE "archive_batch" ("id" SERIAL NOT NULL, "updated" TIMESTAMP NOT NULL DEFAULT now(), "created" TIMESTAMP NOT NULL DEFAULT now(), "merkleRoot" character varying(64) NOT NULL, "otsProof" text, "bitcoinHeight" integer, "status" character varying(256) NOT NULL DEFAULT 'pendingBtc', CONSTRAINT "PK_06f898016522a01c619a214aa18" PRIMARY KEY ("id"))`); + await queryRunner.query(`CREATE TABLE "archive_batch" ("id" SERIAL NOT NULL, "updated" TIMESTAMP NOT NULL DEFAULT now(), "created" TIMESTAMP NOT NULL DEFAULT now(), "merkleRoot" character varying(64) NOT NULL, "otsProof" text, "bitcoinHeight" integer, "status" character varying(256) NOT NULL DEFAULT 'PendingBtc', CONSTRAINT "PK_06f898016522a01c619a214aa18" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE TABLE "archive_file" ("id" SERIAL NOT NULL, "updated" TIMESTAMP NOT NULL DEFAULT now(), "created" TIMESTAMP NOT NULL DEFAULT now(), "bucket" text NOT NULL, "name" text NOT NULL, "sha256" character varying(64) NOT NULL, "leafIndex" integer, "merkleProof" text, "batchId" integer, CONSTRAINT "PK_17e252452a46a911a66d67dd50d" PRIMARY KEY ("id"))`); await queryRunner.query(`CREATE UNIQUE INDEX "IDX_e7da94bafd8e348ead75dcb000" ON "archive_file" ("bucket", "name") `); await queryRunner.query(`CREATE INDEX "IDX_4065eeb67cf015dc7327499de9" ON "archive_file" ("batchId") `); diff --git a/scripts/storage/provision-bucket.ts b/scripts/storage/provision-bucket.ts index bbf5c27ffe..888af7c970 100644 --- a/scripts/storage/provision-bucket.ts +++ b/scripts/storage/provision-bucket.ts @@ -43,8 +43,15 @@ import * as dotenv from 'dotenv'; dotenv.config(); -// Loaded after dotenv so Config.s3 / Config.s3Admin read the populated env. -import { Config } from '../../src/config/config'; +// Loaded after dotenv so Config.s3 / Config.s3Admin read the populated env. `Config` (the +// module-level `export let Config`) is only ever set as a side effect of `new ConfigService()`, +// which this standalone script never instantiates — importing the raw `Config` binding would be +// permanently `undefined` here. Build a local instance directly via `GetConfig()` instead. +import { GetConfig } from '../../src/config/config'; + +const Config = GetConfig(); + +import { GEBUEV_RETENTION_FLOOR_YEARS } from '../../src/integration/infrastructure/storage/worm-retention.const'; function getBucketName(): string { const bucket = process.env.BUCKET ?? process.argv[2]; @@ -52,10 +59,7 @@ function getBucketName(): string { return bucket; } -// Swiss GeBüV requires business records to be retained for 10 years. COMPLIANCE-mode Object -// Lock retention is extend-only and irreversible, so a value provisioned too low can never be -// corrected on the objects it protects — it must therefore fail closed, never silently under-retain. -export const GEBUEV_RETENTION_FLOOR_YEARS = 10; +export { GEBUEV_RETENTION_FLOOR_YEARS }; export function getRetentionYears(): number { const raw = process.env.RETENTION_YEARS ?? process.argv[3]; diff --git a/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts b/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts index 1ddc40654e..b1b22439e3 100644 --- a/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts +++ b/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts @@ -211,7 +211,12 @@ describe('S3StorageService', () => { const container = 'ep2-worm-locked'; s3Mock .on(GetObjectLockConfigurationCommand, { Bucket: container }) - .resolves({ ObjectLockConfiguration: { ObjectLockEnabled: 'Enabled' } }); + .resolves({ + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { DefaultRetention: { Mode: 'COMPLIANCE', Years: 11 } }, + }, + }); s3Mock.on(PutObjectCommand).resolves({}); const url = await new S3StorageService(container).uploadWormBlob( @@ -261,7 +266,12 @@ describe('S3StorageService', () => { const container = 'ep2-worm-cache'; s3Mock .on(GetObjectLockConfigurationCommand, { Bucket: container }) - .resolves({ ObjectLockConfiguration: { ObjectLockEnabled: 'Enabled' } }); + .resolves({ + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { DefaultRetention: { Mode: 'COMPLIANCE', Years: 11 } }, + }, + }); s3Mock.on(PutObjectCommand).resolves({}); const service = new S3StorageService(container); @@ -279,7 +289,12 @@ describe('S3StorageService', () => { beforeEach(() => { s3Mock .on(GetObjectLockConfigurationCommand, { Bucket: CONTAINER }) - .resolves({ ObjectLockConfiguration: { ObjectLockEnabled: 'Enabled' } }); + .resolves({ + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { DefaultRetention: { Mode: 'COMPLIANCE', Years: 11 } }, + }, + }); }); it('URL-encodes keys with spaces / special chars and rewrites the prefix', async () => { diff --git a/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts b/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts index 750cc9edd1..a591dd3497 100644 --- a/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts +++ b/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts @@ -46,9 +46,37 @@ function fakeStore() { return entity; }; + // TypeORM-style criteria update: evaluate conditions against current row state and return + // `{ affected }`. Shared between the repo-level and transactional-manager-level `update` + // entrypoints (the service uses both) so a conditional update — e.g. WHERE id = ... AND + // batch IS NULL AND sha256 = — is exercised identically through either one. + const updateFile = ( + criteria: { id: number; batch?: any; sha256?: string }, + partial: Partial, + ): { affected: number } => { + const file = files.find((f) => f.id === criteria.id); + if (!file) return { affected: 0 }; + + if (criteria.batch !== undefined) { + const wantsNull = criteria.batch?._type === 'isNull' || criteria.batch === null; + if (wantsNull) { + if (file.batch != null) return { affected: 0 }; + } else if (criteria.batch?.id != null) { + if (file.batch?.id !== criteria.batch.id) return { affected: 0 }; + } + } + + if (criteria.sha256 !== undefined && file.sha256 !== criteria.sha256) return { affected: 0 }; + + Object.assign(file, partial); + return { affected: 1 }; + }; + const manager = { save: async (entity: any) => saveEntity(entity), transaction: async (run: (manager: any) => Promise) => run(manager), + update: async (_entityClass: any, criteria: { id: number; batch?: any; sha256?: string }, partial: Partial) => + updateFile(criteria, partial), }; const fileRepo: any = { @@ -56,28 +84,12 @@ function fakeStore() { manager, create: (data: Partial) => Object.assign(new ArchiveFile(), data), save: async (entity: ArchiveFile | ArchiveFile[]) => saveEntity(entity), - // TypeORM-style criteria update: evaluate conditions against current row state and - // return `{ affected }` so conditional updates (e.g. batch: IsNull()) are testable. - update: async (criteria: { id: number; batch?: any }, partial: Partial) => { - const file = files.find((f) => f.id === criteria.id); - if (!file) return { affected: 0 }; - - if (criteria.batch !== undefined) { - const wantsNull = criteria.batch?._type === 'isNull' || criteria.batch === null; - if (wantsNull) { - if (file.batch != null) return { affected: 0 }; - } else if (criteria.batch?.id != null) { - if (file.batch?.id !== criteria.batch.id) return { affected: 0 }; - } - } - - Object.assign(file, partial); - return { affected: 1 }; - }, + update: async (criteria: { id: number; batch?: any; sha256?: string }, partial: Partial) => + updateFile(criteria, partial), findOneBy: async (where: Partial) => files.find((f) => f.bucket === where.bucket && f.name === where.name) ?? undefined, // supports lookup by id or by (bucket, name). Faithful to TypeORM: the `batch` relation is - // ONLY hydrated when the caller explicitly asks for it via `relations: ['batch']`. Without + // ONLY hydrated when the caller explicitly asks for it via `relations: { batch: true }`. Without // that, `batch` is left undefined on the returned row, so a caller that drops the relation // would see `existing.batch === undefined` here too (and the anchored-hash guard would fail // its test) instead of silently passing on the in-memory reference. @@ -170,12 +182,25 @@ describe('ArchiveService', () => { expect(fileRepo.files.every((f: ArchiveFile) => f.batch == null)).toBe(true); }); - it('upserts idempotently on (bucket, name)', async () => { - await service.recordHash('archive', 'doc-a.pdf', sha256(Buffer.from('updated A')).toString('hex')); + it('is a no-op when recording the identical hash again on an unanchored file', async () => { + await service.recordHash('archive', 'doc-a.pdf', sha256(docs[0].data).toString('hex')); expect(fileRepo.files).toHaveLength(3); const fileA = fileRepo.files.find((f: ArchiveFile) => f.name === 'doc-a.pdf'); - expect(fileA.sha256).toBe(sha256(Buffer.from('updated A')).toString('hex')); + expect(fileA.sha256).toBe(sha256(docs[0].data).toString('hex')); + }); + + it('refuses to silently overwrite an unanchored file whose hash differs (auditable, no destructive overwrite)', async () => { + const fileA = fileRepo.files.find((f: ArchiveFile) => f.name === 'doc-a.pdf'); + const originalHash = fileA.sha256; + + await expect( + service.recordHash('archive', 'doc-a.pdf', sha256(Buffer.from('updated A')).toString('hex')), + ).rejects.toThrow(/Refusing to overwrite unanchored hash/); + + // the original hash must be untouched — no silent overwrite + expect(fileA.sha256).toBe(originalHash); + expect(fileRepo.files).toHaveLength(3); }); it('anchors pending files into a batch and stamps the root', async () => { @@ -183,7 +208,7 @@ describe('ArchiveService', () => { expect(batch).toBeDefined(); expect(batch.merkleRoot).toMatch(/^[0-9a-f]{64}$/); - expect(batch.status).toBe('pendingBtc'); + expect(batch.status).toBe('PendingBtc'); expect(batch.otsProof).toBeDefined(); expect(ots.stamp).toHaveBeenCalledTimes(1); @@ -253,14 +278,14 @@ describe('ArchiveService', () => { await service.upgradeBatches(); - expect(batchRepo.batches[0].status).toBe('confirmed'); + expect(batchRepo.batches[0].status).toBe('Confirmed'); expect(batchRepo.batches[0].bitcoinHeight).toBe(840000); }); it('loads the batch relation when recording and verifying (guards the anchored-hash check)', async () => { // The anchored-hash guard in recordHash and the anchored-branch in verifyDocument both // depend on `existing.batch`/`file.batch` being populated, which only happens when the - // query explicitly requests `relations: ['batch']`. Assert the option is actually passed, + // query explicitly requests `relations: { batch: true }`. Assert the option is actually passed, // so dropping it in production (which would let an anchored leaf be silently overwritten) // breaks this test rather than passing unnoticed. const findOneSpy = jest.spyOn(fileRepo, 'findOne'); @@ -270,7 +295,7 @@ describe('ArchiveService', () => { expect(findOneSpy).toHaveBeenCalled(); for (const call of findOneSpy.mock.calls) { - expect(call[0]).toMatchObject({ relations: ['batch'] }); + expect(call[0]).toMatchObject({ relations: { batch: true } }); } }); @@ -315,7 +340,7 @@ describe('ArchiveService', () => { expect(batch.otsProof).toBe(Buffer.from('UPGRADED-BUT-PENDING').toString('base64')); expect(batch.otsProof).not.toBe(originalProof); // still not confirmed - expect(batch.status).toBe('pendingBtc'); + expect(batch.status).toBe('PendingBtc'); expect(batch.bitcoinHeight).toBeUndefined(); }); @@ -329,7 +354,7 @@ describe('ArchiveService', () => { await service.upgradeBatches(); expect(saveSpy).not.toHaveBeenCalled(); - expect(batch.status).toBe('pendingBtc'); + expect(batch.status).toBe('PendingBtc'); }); it('skips a pending batch that carries no OTS proof (never touches the library)', async () => { @@ -346,7 +371,7 @@ describe('ArchiveService', () => { expect(saveSpy).not.toHaveBeenCalled(); expect(ots.upgrade).not.toHaveBeenCalled(); expect(ots.verify).not.toHaveBeenCalled(); - expect(batch.status).toBe('pendingBtc'); + expect(batch.status).toBe('PendingBtc'); }); it('reports the Bitcoin height when verifying a document whose batch is confirmed on-chain', async () => { @@ -406,7 +431,7 @@ describe('ArchiveService', () => { const newHash = sha256(Buffer.from('concurrent re-upload')).toString('hex'); await expect(service.recordHash('archive', 'race.pdf', newHash)).rejects.toThrow( - /Refusing to overwrite anchored hash.*concurrent anchor/, + /Refusing to overwrite unanchored hash/, ); // Leaf committed by anchorPending must be untouched. diff --git a/src/integration/infrastructure/storage/anchoring/archive-batch.entity.ts b/src/integration/infrastructure/storage/anchoring/archive-batch.entity.ts index c001247f63..0ea84fcea8 100644 --- a/src/integration/infrastructure/storage/anchoring/archive-batch.entity.ts +++ b/src/integration/infrastructure/storage/anchoring/archive-batch.entity.ts @@ -4,8 +4,8 @@ import { ArchiveFile } from './archive-file.entity'; /** Lifecycle of an anchoring batch on the way to a Bitcoin attestation. */ export enum ArchiveBatchStatus { - PENDING_BTC = 'pendingBtc', - CONFIRMED = 'confirmed', + PENDING_BTC = 'PendingBtc', + CONFIRMED = 'Confirmed', } /** diff --git a/src/integration/infrastructure/storage/anchoring/archive-file.entity.ts b/src/integration/infrastructure/storage/anchoring/archive-file.entity.ts index 064a1d90ec..58f64f5402 100644 --- a/src/integration/infrastructure/storage/anchoring/archive-file.entity.ts +++ b/src/integration/infrastructure/storage/anchoring/archive-file.entity.ts @@ -1,6 +1,8 @@ import { IEntity } from 'src/shared/models/entity'; import { Column, Entity, Index, ManyToOne } from 'typeorm'; import { ArchiveBatch } from './archive-batch.entity'; +import { deserializeMerkleProof, serializeMerkleProof } from './merkle-proof-codec'; +import { MerkleProofStep } from './merkle'; /** * A single archived storage object whose content hash participates in the GeBüV anchoring @@ -31,8 +33,18 @@ export class ArchiveFile extends IEntity { /** * JSON-serialized inclusion proof (`MerkleProofStep[]`, siblings hex-encoded), populated * from `merkleInclusionProof` at anchor time. Persisted so an old file stays verifiable - * even if sibling rows in the same batch are later lost or corrupted. + * even if sibling rows in the same batch are later lost or corrupted. Raw JSON string column — + * business logic should go through the typed `merkleProofSteps` getter/setter below, never + * parse/stringify this directly. */ @Column({ type: 'text', nullable: true }) merkleProof?: string; + + get merkleProofSteps(): MerkleProofStep[] | undefined { + return this.merkleProof ? deserializeMerkleProof(this.merkleProof) : undefined; + } + + set merkleProofSteps(steps: MerkleProofStep[] | undefined) { + this.merkleProof = steps ? serializeMerkleProof(steps) : undefined; + } } diff --git a/src/integration/infrastructure/storage/anchoring/archive.scheduler.ts b/src/integration/infrastructure/storage/anchoring/archive.scheduler.ts index 775cdf6032..942e8ffebf 100644 --- a/src/integration/infrastructure/storage/anchoring/archive.scheduler.ts +++ b/src/integration/infrastructure/storage/anchoring/archive.scheduler.ts @@ -73,7 +73,7 @@ export class ArchiveScheduler { private async reconciliationBuckets(): Promise { const userDatas = await this.repos.userData.find({ where: { paymentLinksConfig: Not(IsNull()) }, - select: ['id', 'paymentLinksConfig'], + select: { id: true, paymentLinksConfig: true }, }); const ep2Containers: string[] = []; diff --git a/src/integration/infrastructure/storage/anchoring/archive.service.ts b/src/integration/infrastructure/storage/anchoring/archive.service.ts index 57b9cf0a40..1682c618f4 100644 --- a/src/integration/infrastructure/storage/anchoring/archive.service.ts +++ b/src/integration/infrastructure/storage/anchoring/archive.service.ts @@ -3,8 +3,9 @@ import { DfxLogger } from 'src/shared/services/dfx-logger'; import { IsNull } from 'typeorm'; import { ArchiveBatch, ArchiveBatchStatus } from './archive-batch.entity'; import { ArchiveBatchRepository } from './archive-batch.repository'; +import { ArchiveFile } from './archive-file.entity'; import { ArchiveFileRepository } from './archive-file.repository'; -import { deserializeMerkleProof, serializeMerkleProof } from './merkle-proof-codec'; +import { serializeMerkleProof } from './merkle-proof-codec'; import { buildMerkleRoot, merkleInclusionProof, sha256, verifyMerkleProof } from './merkle'; import { OpenTimestampsService } from './opentimestamps.service'; @@ -58,28 +59,29 @@ export class ArchiveService { /** * Idempotently record the SHA-256 of an archived object identified by `(bucket, name)`. * - * Only UNANCHORED records may be updated in place (their hash refreshed, kept unanchored); - * a new record is created unanchored (`batch` null). Anchoring happens later via - * {@link anchorPending}. + * A new `(bucket, name)` is inserted as a fresh, unanchored row. An EXISTING row's `sha256` is + * NEVER mutated in place, whether or not it has been assigned to a Merkle batch: an identical + * hash is a no-op (idempotent), a differing hash is refused — logged (before → after, for audit) + * and rejected with a hard error — rather than silently applied. This is a deliberate + * "auditable mutation, no destructive overwrite" design (CONTRIBUTING.md): a re-upload/re-hash + * of the same `(bucket, name)` must never silently replace what was previously recorded, whether + * that record is already Merkle-anchored (where it would falsify {@link verifyDocument}) or not + * yet anchored (where it would erase evidence of the earlier, differing content without a + * trace). * - * Once a record has been assigned to a Merkle batch its leaf hash is immutable: it is part - * of a (possibly already Bitcoin-anchored) proof. Because KYC blob names are deterministic, - * a re-upload to the same `(bucket, name)` would otherwise silently overwrite the anchored - * leaf hash and make {@link verifyDocument} report bogus tampering. Therefore, for an - * already-anchored record: an identical hash is a no-op, and a differing hash is a hard - * error (the existing anchored hash is never overwritten). - * - * When the row looks unanchored at read time, the update is conditional on `batch` still - * being null at write time, so a concurrent {@link anchorPending} that claims the row - * cannot be overwritten by a stale write (TOCTOU). + * Because recordHash never writes to an existing row, there is nothing left for a concurrent + * {@link anchorPending} to race against here: anchorPending's own per-file conditional update + * likewise never touches `sha256` (see there). TOCTOU-safety therefore falls out of both + * functions simply never overwriting this column on an existing row, rather than out of a + * conditional-write/reload dance. */ async recordHash(bucket: string, name: string, sha256Hex: string): Promise { - const existing = await this.archiveFileRepo.findOne({ where: { bucket, name }, relations: ['batch'] }); + const existing = await this.archiveFileRepo.findOne({ where: { bucket, name }, relations: { batch: true } }); if (existing) { - if (existing.batch != null) { - if (existing.sha256 === sha256Hex) return; + if (existing.sha256 === sha256Hex) return; + if (existing.batch != null) { const message = `Refusing to overwrite anchored hash for ${bucket}/${name} (file ${existing.id}, batch ` + `${existing.batch.id}): stored ${existing.sha256} differs from new ${sha256Hex}`; @@ -87,35 +89,10 @@ export class ArchiveService { throw new Error(message); } - // Conditional update: only write if still unanchored (closes TOCTOU with anchorPending). - const result = await this.archiveFileRepo.update({ id: existing.id, batch: IsNull() }, { sha256: sha256Hex }); - if (result.affected !== 0) return; - - // Lost the race: a concurrent anchorPending claimed this row between read and write. - const reloaded = await this.archiveFileRepo.findOne({ where: { id: existing.id }, relations: ['batch'] }); - if (!reloaded) { - const message = `Archive file ${existing.id} (${bucket}/${name}) disappeared during concurrent update`; - this.logger.error(message); - throw new Error(message); - } - - if (reloaded.sha256 === sha256Hex) return; - - if (reloaded.batch != null) { - const message = - `Refusing to overwrite anchored hash for ${bucket}/${name} (file ${reloaded.id}, batch ` + - `${reloaded.batch.id}): concurrent anchor claimed the row; stored ${reloaded.sha256} ` + - `differs from new ${sha256Hex}`; - this.logger.error(message); - throw new Error(message); - } - - // batch still null but conditional update matched nothing — data-integrity inconsistency. const message = - `Unexpected data-integrity inconsistency for ${bucket}/${name} (file ${reloaded.id}): ` + - `conditional update affected 0 rows but file is still unanchored with differing hash ` + - `(stored ${reloaded.sha256}, new ${sha256Hex})`; - this.logger.error(message); + `Refusing to overwrite unanchored hash for ${bucket}/${name} (file ${existing.id}): ` + + `stored ${existing.sha256} differs from new ${sha256Hex}`; + this.logger.warn(message); throw new Error(message); } @@ -131,7 +108,7 @@ export class ArchiveService { * find it. */ async recordedNames(bucket: string): Promise> { - const files = await this.archiveFileRepo.find({ where: { bucket }, select: ['name'] }); + const files = await this.archiveFileRepo.find({ where: { bucket }, select: { name: true } }); return new Set(files.map((file) => file.name)); } @@ -140,6 +117,15 @@ export class ArchiveService { * root via OpenTimestamps, and persist batch + per-file assignment (including each file's * inclusion proof) in a single transaction. * + * Each file's assignment is written via a conditional UPDATE keyed on `id`, `batch IS NULL`, + * AND the exact `sha256` read when the files were selected above (T0). If a concurrent + * {@link recordHash} call is refused in between (or, more generally, if the row no longer + * matches its T0 snapshot for any reason), the conditional update simply affects 0 rows for + * that file: it is skipped (left `batch IS NULL`, picked up again on the next anchoring cycle) + * rather than being claimed into a batch whose committed leaf may no longer correspond to the + * row's current content. This update only ever sets `batch` / `leafIndex` / `merkleProof` — it + * never writes `sha256`, so it can never destructively race against recordHash either. + * * Returns the created batch, or `undefined` if there is nothing to anchor. */ async anchorPending(): Promise { @@ -157,19 +143,31 @@ export class ArchiveService { status: ArchiveBatchStatus.PENDING_BTC, }); + let anchoredCount = 0; + await this.archiveBatchRepo.manager.transaction(async (manager) => { const savedBatch = await manager.save(batch); - files.forEach((file, index) => { - file.batch = savedBatch; - file.leafIndex = index; - file.merkleProof = serializeMerkleProof(merkleInclusionProof(leaves, index)); - }); - - await manager.save(files); + for (const [index, file] of files.entries()) { + const result = await manager.update( + ArchiveFile, + { id: file.id, batch: IsNull(), sha256: file.sha256 }, + { batch: savedBatch, leafIndex: index, merkleProof: serializeMerkleProof(merkleInclusionProof(leaves, index)) }, + ); + + if (result.affected) { + anchoredCount++; + } else { + this.logger.warn( + `Skipped anchoring archive file ${file.id} (${file.bucket}/${file.name}) into batch ${savedBatch.id}: ` + + `its hash changed or it was reassigned between batching and commit (TOCTOU). It remains unanchored ` + + `and will be picked up by the next anchoring cycle.`, + ); + } + } }); - this.logger.info(`Anchored batch ${batch.id} over ${files.length} file(s), root ${batch.merkleRoot}`); + this.logger.info(`Anchored batch ${batch.id} over ${anchoredCount}/${files.length} file(s), root ${batch.merkleRoot}`); return batch; } @@ -222,7 +220,7 @@ export class ArchiveService { * the leaf, and check the OpenTimestamps attestation status. */ async verifyDocument(bucket: string, name: string, data: Buffer): Promise { - const file = await this.archiveFileRepo.findOne({ where: { bucket, name }, relations: ['batch'] }); + const file = await this.archiveFileRepo.findOne({ where: { bucket, name }, relations: { batch: true } }); if (!file) return { found: false, verified: false }; const computedDigest = sha256(data); @@ -241,7 +239,7 @@ export class ArchiveService { } const rootBuffer = Buffer.from(batch.merkleRoot, 'hex'); - const proof = deserializeMerkleProof(file.merkleProof); + const proof = file.merkleProofSteps; // Leaf must be the digest of the SUPPLIED bytes — never the stored hash — so a // tampered document yields proofValid: false. const proofValid = verifyMerkleProof(computedDigest, proof, rootBuffer); diff --git a/src/integration/infrastructure/storage/anchoring/opentimestamps.service.ts b/src/integration/infrastructure/storage/anchoring/opentimestamps.service.ts index 6391b5b43f..9ad1d0fae5 100644 --- a/src/integration/infrastructure/storage/anchoring/opentimestamps.service.ts +++ b/src/integration/infrastructure/storage/anchoring/opentimestamps.service.ts @@ -15,6 +15,16 @@ export interface OtsVerifyResult { pending: boolean; } +/** + * Minimal shape of the `opentimestamps` library's DetachedTimestampFile actually used here. + * The library ships no TypeScript types (no `@types/opentimestamps` dependency either), so the + * `OpenTimestamps` namespace import is untyped; this local interface replaces `any` with the + * concrete surface this service relies on. + */ +interface DetachedTimestampFile { + serializeToBytes(): Uint8Array; +} + /** * Thin async/await wrapper around the `opentimestamps` npm library for the GeBüV * anchoring pipeline. It deliberately knows nothing about Merkle trees, storage or @@ -85,7 +95,7 @@ export class OpenTimestampsService { } /** Build a DetachedTimestampFile that commits directly to an already-computed SHA-256 digest. */ - private detachedFromDigest(digest: Buffer): any { + private detachedFromDigest(digest: Buffer): DetachedTimestampFile { return OpenTimestamps.DetachedTimestampFile.fromHash(new OpenTimestamps.Ops.OpSHA256(), digest); } } diff --git a/src/integration/infrastructure/storage/s3-storage.service.ts b/src/integration/infrastructure/storage/s3-storage.service.ts index 0286449fd7..a6ec444aec 100644 --- a/src/integration/infrastructure/storage/s3-storage.service.ts +++ b/src/integration/infrastructure/storage/s3-storage.service.ts @@ -4,11 +4,13 @@ import { GetObjectLockConfigurationCommand, HeadObjectCommand, ListObjectsV2Command, + ObjectLockRetentionMode, PutObjectCommand, S3Client, } from '@aws-sdk/client-s3'; import { Config } from 'src/config/config'; import { Blob, BlobContent, BlobMetaData, StorageService } from './storage.service'; +import { GEBUEV_RETENTION_FLOOR_YEARS } from './worm-retention.const'; /** * S3-protocol storage implementation. Talks to the configured S3-compatible @@ -80,10 +82,10 @@ export class S3StorageService extends StorageService { private async assertObjectLockEnabled(): Promise { if (S3StorageService.objectLockVerified.has(this.container)) return; - let enabled: string | undefined; + let cfg: { ObjectLockEnabled?: string; Rule?: { DefaultRetention?: { Mode?: string; Years?: number } } } | undefined; try { const res = await this.client.send(new GetObjectLockConfigurationCommand({ Bucket: this.container })); - enabled = res.ObjectLockConfiguration?.ObjectLockEnabled; + cfg = res.ObjectLockConfiguration; } catch (e) { // A bucket without Object Lock returns ObjectLockConfigurationNotFoundError; any other error // (missing bucket, transport, auth) is equally unverifiable. Either way, fail closed. @@ -94,9 +96,18 @@ export class S3StorageService extends StorageService { ); } - if (enabled !== 'Enabled') + const retention = cfg?.Rule?.DefaultRetention; + const isValid = + cfg?.ObjectLockEnabled === 'Enabled' && + retention?.Mode === ObjectLockRetentionMode.COMPLIANCE && + retention?.Years != null && + retention.Years >= GEBUEV_RETENTION_FLOOR_YEARS; + + if (!isValid) throw new Error( - `Refusing WORM write into bucket "${this.container}": Object Lock is not enabled. GeBüV compliance ` + + `Refusing WORM write into bucket "${this.container}": Object Lock is not enabled with a COMPLIANCE-mode ` + + `default retention of at least ${GEBUEV_RETENTION_FLOOR_YEARS} year(s) (got ObjectLockEnabled=` + + `${cfg?.ObjectLockEnabled}, Mode=${retention?.Mode}, Years=${retention?.Years}). GeBüV compliance ` + `records must be WORM-protected and Object Lock cannot be retro-fitted onto an existing bucket. ` + `Provision it first (scripts/storage/provision-bucket.ts).`, ); diff --git a/src/integration/infrastructure/storage/worm-retention.const.ts b/src/integration/infrastructure/storage/worm-retention.const.ts new file mode 100644 index 0000000000..d075b3d328 --- /dev/null +++ b/src/integration/infrastructure/storage/worm-retention.const.ts @@ -0,0 +1,10 @@ +/** + * Swiss GeBüV requires business records to be retained for 10 years. COMPLIANCE-mode Object + * Lock retention is extend-only and irreversible, so a value provisioned too low can never be + * corrected on the objects it protects — it must therefore fail closed, never silently + * under-retain. Shared between the WORM bucket provisioning script + * (scripts/storage/provision-bucket.ts) and the runtime Object-Lock guard + * (S3StorageService.assertObjectLockEnabled) so both enforce the identical floor — never + * duplicate this number. + */ +export const GEBUEV_RETENTION_FLOOR_YEARS = 10; diff --git a/src/subdomains/generic/gs/__tests__/gs.service.spec.ts b/src/subdomains/generic/gs/__tests__/gs.service.spec.ts index 6fd1e2592b..ef003f1e4c 100644 --- a/src/subdomains/generic/gs/__tests__/gs.service.spec.ts +++ b/src/subdomains/generic/gs/__tests__/gs.service.spec.ts @@ -49,7 +49,7 @@ function asKycFileBlobs(docs: KycFileDoc[]): KycFileBlob[] { return docs as unknown as KycFileBlob[]; } -// A storage blob as returned by AzureStorageService.listBlobs (fed into the real listFilesByPrefix). +// A storage blob as returned by StorageService.listBlobs (fed into the real listFilesByPrefix). function storageBlob(name: string, created: Date): Blob { return { name, diff --git a/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts b/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts index 635457dd55..916e733f6f 100644 --- a/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts +++ b/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts @@ -117,7 +117,7 @@ export class FiatOutputJobService { const report = this.ep2ReportService.generateReport(entity); const container = buyFiat.userData.paymentLinksConfigObj.ep2ReportContainer; const routeId = buyFiat.paymentLinkPayment.link.linkConfigObj?.payoutRouteId ?? buyFiat.sell.id; - const fileName = `settlement_${Util.isoDateTime(entity.created)}_${routeId}.ep2`; + const fileName = `settlement_${Util.isoDateTime(entity.created)}_${entity.id}_${routeId}.ep2`; const reportBuffer = Buffer.from(report); // WORM sink: uploadWormBlob fails closed if the (runtime-resolved, per-merchant) EP2 From 6a500eaf88bd0018493434f9c1a36ebeeca4d17f Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:42:56 +0200 Subject: [PATCH 09/11] style: apply prettier formatting to storage/anchoring review fixes --- .../__tests__/s3-storage.service.spec.ts | 42 ++++++++----------- .../__tests__/archive.service.spec.ts | 7 +++- .../storage/anchoring/archive.service.ts | 10 ++++- .../storage/s3-storage.service.ts | 4 +- 4 files changed, 34 insertions(+), 29 deletions(-) diff --git a/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts b/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts index b1b22439e3..83e17434ff 100644 --- a/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts +++ b/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts @@ -209,14 +209,12 @@ describe('S3StorageService', () => { // a name would let one test's verification leak into the next. it('PUTs into a bucket whose Object Lock is enabled', async () => { const container = 'ep2-worm-locked'; - s3Mock - .on(GetObjectLockConfigurationCommand, { Bucket: container }) - .resolves({ - ObjectLockConfiguration: { - ObjectLockEnabled: 'Enabled', - Rule: { DefaultRetention: { Mode: 'COMPLIANCE', Years: 11 } }, - }, - }); + s3Mock.on(GetObjectLockConfigurationCommand, { Bucket: container }).resolves({ + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { DefaultRetention: { Mode: 'COMPLIANCE', Years: 11 } }, + }, + }); s3Mock.on(PutObjectCommand).resolves({}); const url = await new S3StorageService(container).uploadWormBlob( @@ -264,14 +262,12 @@ describe('S3StorageService', () => { it('verifies Object Lock once per container, then skips the probe on subsequent WORM writes', async () => { const container = 'ep2-worm-cache'; - s3Mock - .on(GetObjectLockConfigurationCommand, { Bucket: container }) - .resolves({ - ObjectLockConfiguration: { - ObjectLockEnabled: 'Enabled', - Rule: { DefaultRetention: { Mode: 'COMPLIANCE', Years: 11 } }, - }, - }); + s3Mock.on(GetObjectLockConfigurationCommand, { Bucket: container }).resolves({ + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { DefaultRetention: { Mode: 'COMPLIANCE', Years: 11 } }, + }, + }); s3Mock.on(PutObjectCommand).resolves({}); const service = new S3StorageService(container); @@ -287,14 +283,12 @@ describe('S3StorageService', () => { describe('copyBlobs', () => { beforeEach(() => { - s3Mock - .on(GetObjectLockConfigurationCommand, { Bucket: CONTAINER }) - .resolves({ - ObjectLockConfiguration: { - ObjectLockEnabled: 'Enabled', - Rule: { DefaultRetention: { Mode: 'COMPLIANCE', Years: 11 } }, - }, - }); + s3Mock.on(GetObjectLockConfigurationCommand, { Bucket: CONTAINER }).resolves({ + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { DefaultRetention: { Mode: 'COMPLIANCE', Years: 11 } }, + }, + }); }); it('URL-encodes keys with spaces / special chars and rewrites the prefix', async () => { diff --git a/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts b/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts index a591dd3497..eeb407a1c1 100644 --- a/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts +++ b/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts @@ -75,8 +75,11 @@ function fakeStore() { const manager = { save: async (entity: any) => saveEntity(entity), transaction: async (run: (manager: any) => Promise) => run(manager), - update: async (_entityClass: any, criteria: { id: number; batch?: any; sha256?: string }, partial: Partial) => - updateFile(criteria, partial), + update: async ( + _entityClass: any, + criteria: { id: number; batch?: any; sha256?: string }, + partial: Partial, + ) => updateFile(criteria, partial), }; const fileRepo: any = { diff --git a/src/integration/infrastructure/storage/anchoring/archive.service.ts b/src/integration/infrastructure/storage/anchoring/archive.service.ts index 1682c618f4..d747214d49 100644 --- a/src/integration/infrastructure/storage/anchoring/archive.service.ts +++ b/src/integration/infrastructure/storage/anchoring/archive.service.ts @@ -152,7 +152,11 @@ export class ArchiveService { const result = await manager.update( ArchiveFile, { id: file.id, batch: IsNull(), sha256: file.sha256 }, - { batch: savedBatch, leafIndex: index, merkleProof: serializeMerkleProof(merkleInclusionProof(leaves, index)) }, + { + batch: savedBatch, + leafIndex: index, + merkleProof: serializeMerkleProof(merkleInclusionProof(leaves, index)), + }, ); if (result.affected) { @@ -167,7 +171,9 @@ export class ArchiveService { } }); - this.logger.info(`Anchored batch ${batch.id} over ${anchoredCount}/${files.length} file(s), root ${batch.merkleRoot}`); + this.logger.info( + `Anchored batch ${batch.id} over ${anchoredCount}/${files.length} file(s), root ${batch.merkleRoot}`, + ); return batch; } diff --git a/src/integration/infrastructure/storage/s3-storage.service.ts b/src/integration/infrastructure/storage/s3-storage.service.ts index a6ec444aec..9c498b425c 100644 --- a/src/integration/infrastructure/storage/s3-storage.service.ts +++ b/src/integration/infrastructure/storage/s3-storage.service.ts @@ -82,7 +82,9 @@ export class S3StorageService extends StorageService { private async assertObjectLockEnabled(): Promise { if (S3StorageService.objectLockVerified.has(this.container)) return; - let cfg: { ObjectLockEnabled?: string; Rule?: { DefaultRetention?: { Mode?: string; Years?: number } } } | undefined; + let cfg: + | { ObjectLockEnabled?: string; Rule?: { DefaultRetention?: { Mode?: string; Years?: number } } } + | undefined; try { const res = await this.client.send(new GetObjectLockConfigurationCommand({ Bucket: this.container })); cfg = res.ObjectLockConfiguration; From a4b22fb861ccd9364d17ce5bf07648d9a538e481 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:15:57 +0200 Subject: [PATCH 10/11] refactor(storage): defer GeBueV anchoring pipeline to a separate PR The anchoring pipeline (Merkle/OTS/WORM-verify, Saeule 2) is decoupled from the storage cutover and moved to a dedicated hardened follow-up PR (branch feat/gebuev-anchoring). This PR now contains only the Azure Blob -> S3/MinIO storage cutover (Saeule 1). Removes the anchoring/ directory, its migration, and all ArchiveService/recordHash call sites; storage upload/copy paths unchanged. --- .../1784103600000-AddArchiveAnchoring.js | 34 -- .../__tests__/archive.scheduler.spec.ts | 178 ------- .../__tests__/archive.service.spec.ts | 448 ------------------ .../__tests__/merkle-proof-codec.spec.ts | 56 --- .../anchoring/__tests__/merkle.spec.ts | 141 ------ .../__tests__/opentimestamps.service.spec.ts | 176 ------- .../storage/anchoring/archive-batch.entity.ts | 33 -- .../anchoring/archive-batch.repository.ts | 11 - .../storage/anchoring/archive-file.entity.ts | 50 -- .../anchoring/archive-file.repository.ts | 11 - .../storage/anchoring/archive.module.ts | 18 - .../storage/anchoring/archive.scheduler.ts | 131 ----- .../storage/anchoring/archive.service.ts | 265 ----------- .../storage/anchoring/merkle-proof-codec.ts | 40 -- .../storage/anchoring/merkle.ts | 145 ------ .../anchoring/opentimestamps.service.ts | 101 ---- .../generic/gs/__tests__/gs.service.spec.ts | 7 +- src/subdomains/generic/kyc/kyc.module.ts | 2 - .../services/__tests__/kyc.service.spec.ts | 4 - .../__tests__/kyc-document.service.spec.ts | 81 +--- .../integration/kyc-document.service.ts | 47 +- .../__tests__/user-data.service.spec.ts | 4 - .../__tests__/fiat-output-job.service.spec.ts | 45 +- .../fiat-output/fiat-output-job.service.ts | 18 +- .../fiat-output/fiat-output.module.ts | 2 - .../realunit-compliance.service.spec.ts | 4 - 26 files changed, 22 insertions(+), 2030 deletions(-) delete mode 100644 migration/1784103600000-AddArchiveAnchoring.js delete mode 100644 src/integration/infrastructure/storage/anchoring/__tests__/archive.scheduler.spec.ts delete mode 100644 src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts delete mode 100644 src/integration/infrastructure/storage/anchoring/__tests__/merkle-proof-codec.spec.ts delete mode 100644 src/integration/infrastructure/storage/anchoring/__tests__/merkle.spec.ts delete mode 100644 src/integration/infrastructure/storage/anchoring/__tests__/opentimestamps.service.spec.ts delete mode 100644 src/integration/infrastructure/storage/anchoring/archive-batch.entity.ts delete mode 100644 src/integration/infrastructure/storage/anchoring/archive-batch.repository.ts delete mode 100644 src/integration/infrastructure/storage/anchoring/archive-file.entity.ts delete mode 100644 src/integration/infrastructure/storage/anchoring/archive-file.repository.ts delete mode 100644 src/integration/infrastructure/storage/anchoring/archive.module.ts delete mode 100644 src/integration/infrastructure/storage/anchoring/archive.scheduler.ts delete mode 100644 src/integration/infrastructure/storage/anchoring/archive.service.ts delete mode 100644 src/integration/infrastructure/storage/anchoring/merkle-proof-codec.ts delete mode 100644 src/integration/infrastructure/storage/anchoring/merkle.ts delete mode 100644 src/integration/infrastructure/storage/anchoring/opentimestamps.service.ts diff --git a/migration/1784103600000-AddArchiveAnchoring.js b/migration/1784103600000-AddArchiveAnchoring.js deleted file mode 100644 index 2afed71326..0000000000 --- a/migration/1784103600000-AddArchiveAnchoring.js +++ /dev/null @@ -1,34 +0,0 @@ -/** - * @typedef {import('typeorm').MigrationInterface} MigrationInterface - * @typedef {import('typeorm').QueryRunner} QueryRunner - */ - -/** - * @class - * @implements {MigrationInterface} - */ -module.exports = class AddArchiveAnchoring1784103600000 { - name = 'AddArchiveAnchoring1784103600000' - - /** - * @param {QueryRunner} queryRunner - */ - async up(queryRunner) { - await queryRunner.query(`CREATE TABLE "archive_batch" ("id" SERIAL NOT NULL, "updated" TIMESTAMP NOT NULL DEFAULT now(), "created" TIMESTAMP NOT NULL DEFAULT now(), "merkleRoot" character varying(64) NOT NULL, "otsProof" text, "bitcoinHeight" integer, "status" character varying(256) NOT NULL DEFAULT 'PendingBtc', CONSTRAINT "PK_06f898016522a01c619a214aa18" PRIMARY KEY ("id"))`); - await queryRunner.query(`CREATE TABLE "archive_file" ("id" SERIAL NOT NULL, "updated" TIMESTAMP NOT NULL DEFAULT now(), "created" TIMESTAMP NOT NULL DEFAULT now(), "bucket" text NOT NULL, "name" text NOT NULL, "sha256" character varying(64) NOT NULL, "leafIndex" integer, "merkleProof" text, "batchId" integer, CONSTRAINT "PK_17e252452a46a911a66d67dd50d" PRIMARY KEY ("id"))`); - await queryRunner.query(`CREATE UNIQUE INDEX "IDX_e7da94bafd8e348ead75dcb000" ON "archive_file" ("bucket", "name") `); - await queryRunner.query(`CREATE INDEX "IDX_4065eeb67cf015dc7327499de9" ON "archive_file" ("batchId") `); - await queryRunner.query(`ALTER TABLE "archive_file" ADD CONSTRAINT "FK_4065eeb67cf015dc7327499de94" FOREIGN KEY ("batchId") REFERENCES "archive_batch"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); - } - - /** - * @param {QueryRunner} queryRunner - */ - async down(queryRunner) { - await queryRunner.query(`ALTER TABLE "archive_file" DROP CONSTRAINT "FK_4065eeb67cf015dc7327499de94"`); - await queryRunner.query(`DROP INDEX "public"."IDX_4065eeb67cf015dc7327499de9"`); - await queryRunner.query(`DROP INDEX "public"."IDX_e7da94bafd8e348ead75dcb000"`); - await queryRunner.query(`DROP TABLE "archive_file"`); - await queryRunner.query(`DROP TABLE "archive_batch"`); - } -} diff --git a/src/integration/infrastructure/storage/anchoring/__tests__/archive.scheduler.spec.ts b/src/integration/infrastructure/storage/anchoring/__tests__/archive.scheduler.spec.ts deleted file mode 100644 index 9ecb576ee8..0000000000 --- a/src/integration/infrastructure/storage/anchoring/__tests__/archive.scheduler.spec.ts +++ /dev/null @@ -1,178 +0,0 @@ -// Stub the heavy `opentimestamps` library (pulled in transitively via ArchiveService) so its -// eager network/`request` deps never load; ArchiveService is fully mocked in this spec anyway. -jest.mock('opentimestamps', () => ({})); - -const mockCreateStorageService = jest.fn(); -jest.mock('src/integration/infrastructure/storage/storage.factory', () => ({ - createStorageService: (...args: any[]) => mockCreateStorageService(...args), -})); - -import { createMock } from '@golevelup/ts-jest'; -import { Test, TestingModule } from '@nestjs/testing'; -import { RepositoryFactory } from 'src/shared/repositories/repository.factory'; -import { UserData } from 'src/subdomains/generic/user/models/user-data/user-data.entity'; -import { UserDataRepository } from 'src/subdomains/generic/user/models/user-data/user-data.repository'; -import { ArchiveBatch } from '../archive-batch.entity'; -import { ArchiveScheduler } from '../archive.scheduler'; -import { ArchiveService } from '../archive.service'; -import { sha256 } from '../merkle'; - -describe('ArchiveScheduler', () => { - let scheduler: ArchiveScheduler; - let archiveService: ArchiveService; - - beforeEach(async () => { - archiveService = createMock(); - - const module: TestingModule = await Test.createTestingModule({ - providers: [ - ArchiveScheduler, - { provide: ArchiveService, useValue: archiveService }, - { provide: RepositoryFactory, useValue: createMock() }, - ], - }).compile(); - - scheduler = module.get(ArchiveScheduler); - }); - - it('should be defined', () => { - expect(scheduler).toBeDefined(); - }); - - describe('anchorPending', () => { - it('delegates to ArchiveService.anchorPending', async () => { - (archiveService.anchorPending as jest.Mock).mockResolvedValue({ id: 1, merkleRoot: 'ab' } as ArchiveBatch); - - await scheduler.anchorPending(); - - expect(archiveService.anchorPending).toHaveBeenCalledTimes(1); - }); - - it('propagates errors to the @DfxCron wrapper', async () => { - (archiveService.anchorPending as jest.Mock).mockRejectedValue(new Error('calendar down')); - - await expect(scheduler.anchorPending()).rejects.toThrow('calendar down'); - }); - }); - - describe('upgradeBatches', () => { - it('delegates to ArchiveService.upgradeBatches', async () => { - await scheduler.upgradeBatches(); - - expect(archiveService.upgradeBatches).toHaveBeenCalledTimes(1); - }); - - it('propagates errors to the @DfxCron wrapper', async () => { - (archiveService.upgradeBatches as jest.Mock).mockRejectedValue(new Error('upgrade failed')); - - await expect(scheduler.upgradeBatches()).rejects.toThrow('upgrade failed'); - }); - }); - - describe('reconcileHashes', () => { - let scheduler: ArchiveScheduler; - let archiveService: ArchiveService; - let repos: RepositoryFactory; - - const storageStubs: Record = {}; - - const stubStorage = (container: string, blobs: { name: string; data: Buffer }[]) => { - storageStubs[container] = { - listKeys: jest.fn().mockResolvedValue(blobs.map((b) => b.name)), - getBlob: jest.fn((name: string) => Promise.resolve({ data: blobs.find((b) => b.name === name)!.data })), - }; - }; - - beforeEach(async () => { - for (const key of Object.keys(storageStubs)) delete storageStubs[key]; - mockCreateStorageService.mockReset(); - mockCreateStorageService.mockImplementation((container: string) => storageStubs[container]); - - archiveService = createMock(); - repos = createMock(); - (repos.userData as unknown) = createMock(); - - (repos.userData.find as jest.Mock).mockResolvedValue([]); - (archiveService.recordedNames as jest.Mock).mockResolvedValue(new Set()); - - const module: TestingModule = await Test.createTestingModule({ - providers: [ - ArchiveScheduler, - { provide: ArchiveService, useValue: archiveService }, - { provide: RepositoryFactory, useValue: repos }, - ], - }).compile(); - - scheduler = module.get(ArchiveScheduler); - }); - - it('recordHash only objects present in storage but absent from recordedNames', async () => { - const missingData = Buffer.from('missing-doc'); - const presentData = Buffer.from('already-recorded'); - - stubStorage('kyc', [ - { name: 'user/1/a.pdf', data: missingData }, - { name: 'user/1/b.pdf', data: presentData }, - ]); - (archiveService.recordedNames as jest.Mock).mockResolvedValue(new Set(['user/1/b.pdf'])); - - await scheduler.reconcileHashes(); - - expect(archiveService.recordHash).toHaveBeenCalledTimes(1); - expect(archiveService.recordHash).toHaveBeenCalledWith( - 'kyc', - 'user/1/a.pdf', - sha256(missingData).toString('hex'), - ); - expect(archiveService.recordHash).not.toHaveBeenCalledWith('kyc', 'user/1/b.pdf', expect.anything()); - }); - - it('deduplicates ep2ReportContainer values across UserData rows', async () => { - const container = 'merchant-ep2-bucket'; - const ud1 = Object.assign(new UserData(), { - id: 1, - paymentLinksConfig: JSON.stringify({ ep2ReportContainer: container }), - }); - const ud2 = Object.assign(new UserData(), { - id: 2, - paymentLinksConfig: JSON.stringify({ ep2ReportContainer: container }), - }); - (repos.userData.find as jest.Mock).mockResolvedValue([ud1, ud2]); - - stubStorage('kyc', []); - stubStorage(container, [{ name: 'settlement.ep2', data: Buffer.from('') }]); - - await scheduler.reconcileHashes(); - - const containersRequested = mockCreateStorageService.mock.calls.map((c) => c[0]); - expect(containersRequested.filter((c) => c === container)).toHaveLength(1); - expect(containersRequested).toContain('kyc'); - expect(archiveService.recordHash).toHaveBeenCalledTimes(1); - expect(archiveService.recordHash).toHaveBeenCalledWith( - container, - 'settlement.ep2', - sha256(Buffer.from('')).toString('hex'), - ); - }); - - it('continues when recordHash rejects for one missing object', async () => { - const dataA = Buffer.from('doc-a'); - const dataB = Buffer.from('doc-b'); - - stubStorage('kyc', [ - { name: 'user/1/a.pdf', data: dataA }, - { name: 'user/1/b.pdf', data: dataB }, - ]); - (archiveService.recordedNames as jest.Mock).mockResolvedValue(new Set()); - (archiveService.recordHash as jest.Mock) - .mockRejectedValueOnce(new Error('already anchored under a different hash')) - .mockResolvedValueOnce(undefined); - - await expect(scheduler.reconcileHashes()).resolves.toBeUndefined(); - - expect(archiveService.recordHash).toHaveBeenCalledTimes(2); - expect(archiveService.recordHash).toHaveBeenCalledWith('kyc', 'user/1/a.pdf', sha256(dataA).toString('hex')); - expect(archiveService.recordHash).toHaveBeenCalledWith('kyc', 'user/1/b.pdf', sha256(dataB).toString('hex')); - }); - }); -}); diff --git a/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts b/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts deleted file mode 100644 index eeb407a1c1..0000000000 --- a/src/integration/infrastructure/storage/anchoring/__tests__/archive.service.spec.ts +++ /dev/null @@ -1,448 +0,0 @@ -// Stub the heavy `opentimestamps` library so its eager network/`request` deps never load: -// this spec mocks OpenTimestampsService entirely, so the real implementation is never used. -jest.mock('opentimestamps', () => ({})); - -import { createMock } from '@golevelup/ts-jest'; -import { Test, TestingModule } from '@nestjs/testing'; -import { ArchiveBatch } from '../archive-batch.entity'; -import { ArchiveBatchRepository } from '../archive-batch.repository'; -import { ArchiveFile } from '../archive-file.entity'; -import { ArchiveFileRepository } from '../archive-file.repository'; -import { ArchiveService } from '../archive.service'; -import { sha256 } from '../merkle'; -import { OpenTimestampsService } from '../opentimestamps.service'; - -/** - * A shared in-memory store backing both fake repositories. It reproduces just enough TypeORM - * behaviour (auto-increment ids, `(bucket, name)` upsert, batch-null filtering, ordering, - * conditional `update` with criteria, and `manager.transaction`) for the round-trip the - * service performs, so that saves inside the transaction the service runs on the batch - * repo's manager land in the same `files`/`batches` arrays the assertions inspect. The - * Merkle math is the REAL Stage-1 module; only OpenTimestamps is mocked so no network is - * touched. - */ -function fakeStore() { - const files: ArchiveFile[] = []; - const batches: ArchiveBatch[] = []; - let fileSeq = 0; - let batchSeq = 0; - - // dispatch a single entity (or array) by type, assigning ids on first save. - const saveEntity = (entity: any): any => { - const list = Array.isArray(entity) ? entity : [entity]; - for (const item of list) { - if (item instanceof ArchiveBatch) { - if (!item.id) { - item.id = ++batchSeq; - batches.push(item); - } - } else { - if (!item.id) { - item.id = ++fileSeq; - files.push(item); - } - } - } - return entity; - }; - - // TypeORM-style criteria update: evaluate conditions against current row state and return - // `{ affected }`. Shared between the repo-level and transactional-manager-level `update` - // entrypoints (the service uses both) so a conditional update — e.g. WHERE id = ... AND - // batch IS NULL AND sha256 = — is exercised identically through either one. - const updateFile = ( - criteria: { id: number; batch?: any; sha256?: string }, - partial: Partial, - ): { affected: number } => { - const file = files.find((f) => f.id === criteria.id); - if (!file) return { affected: 0 }; - - if (criteria.batch !== undefined) { - const wantsNull = criteria.batch?._type === 'isNull' || criteria.batch === null; - if (wantsNull) { - if (file.batch != null) return { affected: 0 }; - } else if (criteria.batch?.id != null) { - if (file.batch?.id !== criteria.batch.id) return { affected: 0 }; - } - } - - if (criteria.sha256 !== undefined && file.sha256 !== criteria.sha256) return { affected: 0 }; - - Object.assign(file, partial); - return { affected: 1 }; - }; - - const manager = { - save: async (entity: any) => saveEntity(entity), - transaction: async (run: (manager: any) => Promise) => run(manager), - update: async ( - _entityClass: any, - criteria: { id: number; batch?: any; sha256?: string }, - partial: Partial, - ) => updateFile(criteria, partial), - }; - - const fileRepo: any = { - files, - manager, - create: (data: Partial) => Object.assign(new ArchiveFile(), data), - save: async (entity: ArchiveFile | ArchiveFile[]) => saveEntity(entity), - update: async (criteria: { id: number; batch?: any; sha256?: string }, partial: Partial) => - updateFile(criteria, partial), - findOneBy: async (where: Partial) => - files.find((f) => f.bucket === where.bucket && f.name === where.name) ?? undefined, - // supports lookup by id or by (bucket, name). Faithful to TypeORM: the `batch` relation is - // ONLY hydrated when the caller explicitly asks for it via `relations: { batch: true }`. Without - // that, `batch` is left undefined on the returned row, so a caller that drops the relation - // would see `existing.batch === undefined` here too (and the anchored-hash guard would fail - // its test) instead of silently passing on the in-memory reference. - findOne: async ({ where, relations }: any) => { - const match = files.find((f) => - where.id != null ? f.id === where.id : f.bucket === where.bucket && f.name === where.name, - ); - if (!match) return undefined; - - const wantsBatch = Array.isArray(relations) ? relations.includes('batch') : !!relations?.batch; - // Return a shallow copy so we can withhold `batch` without mutating the stored entity. - const result = Object.assign(new ArchiveFile(), match); - if (!wantsBatch) result.batch = undefined; - return result; - }, - find: async ({ where, order }: any) => { - let result = [...files]; - - if (where?.batch !== undefined) { - // TypeORM IsNull() carries `_type === 'isNull'`; otherwise we match a batch id. - const wantsNull = where.batch?._type === 'isNull' || where.batch === null; - if (wantsNull) result = result.filter((f) => f.batch == null); - else if (where.batch?.id != null) result = result.filter((f) => f.batch?.id === where.batch.id); - } - - if (order?.id === 'ASC') result.sort((a, b) => a.id - b.id); - if (order?.leafIndex === 'ASC') result.sort((a, b) => a.leafIndex - b.leafIndex); - - return result; - }, - }; - - const batchRepo: any = { - batches, - manager, - create: (data: Partial) => Object.assign(new ArchiveBatch(), data), - save: async (entity: ArchiveBatch) => saveEntity(entity), - findBy: async (where: Partial) => batches.filter((b) => b.status === where.status), - }; - - return { fileRepo, batchRepo }; -} - -describe('ArchiveService', () => { - let service: ArchiveService; - let fileRepo: any; - let batchRepo: any; - let ots: OpenTimestampsService; - - beforeEach(async () => { - ({ fileRepo, batchRepo } = fakeStore()); - - ots = createMock(); - // Deterministic, network-free fakes: the .ots bytes just wrap the root; pending forever. - (ots.stamp as jest.Mock).mockImplementation(async (digest: Buffer) => Buffer.concat([Buffer.from('OTS'), digest])); - (ots.upgrade as jest.Mock).mockImplementation(async (bytes: Buffer) => bytes); - (ots.verify as jest.Mock).mockResolvedValue({ confirmed: false, pending: true }); - - const module: TestingModule = await Test.createTestingModule({ - providers: [ - ArchiveService, - { provide: ArchiveFileRepository, useValue: fileRepo }, - { provide: ArchiveBatchRepository, useValue: batchRepo }, - { provide: OpenTimestampsService, useValue: ots }, - ], - }).compile(); - - service = module.get(ArchiveService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); - - describe('round-trip', () => { - const docs = [ - { bucket: 'archive', name: 'doc-a.pdf', data: Buffer.from('content of document A') }, - { bucket: 'archive', name: 'doc-b.pdf', data: Buffer.from('content of document B') }, - { bucket: 'archive', name: 'doc-c.pdf', data: Buffer.from('content of document C') }, - ]; - - beforeEach(async () => { - for (const doc of docs) { - await service.recordHash(doc.bucket, doc.name, sha256(doc.data).toString('hex')); - } - }); - - it('records all files unanchored', () => { - expect(fileRepo.files).toHaveLength(3); - expect(fileRepo.files.every((f: ArchiveFile) => f.batch == null)).toBe(true); - }); - - it('is a no-op when recording the identical hash again on an unanchored file', async () => { - await service.recordHash('archive', 'doc-a.pdf', sha256(docs[0].data).toString('hex')); - - expect(fileRepo.files).toHaveLength(3); - const fileA = fileRepo.files.find((f: ArchiveFile) => f.name === 'doc-a.pdf'); - expect(fileA.sha256).toBe(sha256(docs[0].data).toString('hex')); - }); - - it('refuses to silently overwrite an unanchored file whose hash differs (auditable, no destructive overwrite)', async () => { - const fileA = fileRepo.files.find((f: ArchiveFile) => f.name === 'doc-a.pdf'); - const originalHash = fileA.sha256; - - await expect( - service.recordHash('archive', 'doc-a.pdf', sha256(Buffer.from('updated A')).toString('hex')), - ).rejects.toThrow(/Refusing to overwrite unanchored hash/); - - // the original hash must be untouched — no silent overwrite - expect(fileA.sha256).toBe(originalHash); - expect(fileRepo.files).toHaveLength(3); - }); - - it('anchors pending files into a batch and stamps the root', async () => { - const batch = await service.anchorPending(); - - expect(batch).toBeDefined(); - expect(batch.merkleRoot).toMatch(/^[0-9a-f]{64}$/); - expect(batch.status).toBe('PendingBtc'); - expect(batch.otsProof).toBeDefined(); - expect(ots.stamp).toHaveBeenCalledTimes(1); - - // every file got assigned to the batch with a leaf index and a persisted inclusion proof - expect(fileRepo.files.every((f: ArchiveFile) => f.batch?.id === batch.id)).toBe(true); - expect(fileRepo.files.map((f: ArchiveFile) => f.leafIndex).sort()).toEqual([0, 1, 2]); - expect( - fileRepo.files.every((f: ArchiveFile) => typeof f.merkleProof === 'string' && f.merkleProof.length > 0), - ).toBe(true); - }); - - it('returns undefined when there is nothing to anchor', async () => { - await service.anchorPending(); - const second = await service.anchorPending(); - expect(second).toBeUndefined(); - }); - - it('verifies an unchanged, anchored document (real Merkle proof)', async () => { - await service.anchorPending(); - - const result = await service.verifyDocument('archive', 'doc-b.pdf', docs[1].data); - - expect(result.found).toBe(true); - expect(result.hashMatches).toBe(true); - expect(result.anchored).toBe(true); - expect(result.proofValid).toBe(true); - expect(result.verified).toBe(true); - expect(result.pending).toBe(true); - expect(result.bitcoinHeight).toBeUndefined(); - }); - - it('detects tampered data via hash mismatch', async () => { - await service.anchorPending(); - - const result = await service.verifyDocument('archive', 'doc-b.pdf', Buffer.from('tampered content')); - - expect(result.found).toBe(true); - expect(result.hashMatches).toBe(false); - // the stored hash is still genuinely anchored, only the supplied bytes differ - expect(result.anchored).toBe(true); - // leaf is derived from the supplied (tampered) bytes → proof must not validate - expect(result.proofValid).toBe(false); - expect(result.verified).toBe(false); - }); - - it('reports an unanchored file as anchored:false', async () => { - const result = await service.verifyDocument('archive', 'doc-a.pdf', docs[0].data); - - expect(result.found).toBe(true); - expect(result.hashMatches).toBe(true); - expect(result.anchored).toBe(false); - expect(result.proofValid).toBeUndefined(); - expect(result.verified).toBe(false); - }); - - it('reports an unknown document as found:false', async () => { - const result = await service.verifyDocument('archive', 'missing.pdf', Buffer.from('whatever')); - - expect(result.found).toBe(false); - expect(result.verified).toBe(false); - }); - - it('confirms a batch once OpenTimestamps reports a Bitcoin attestation', async () => { - await service.anchorPending(); - - (ots.verify as jest.Mock).mockResolvedValue({ confirmed: true, pending: false, bitcoin: { height: 840000 } }); - - await service.upgradeBatches(); - - expect(batchRepo.batches[0].status).toBe('Confirmed'); - expect(batchRepo.batches[0].bitcoinHeight).toBe(840000); - }); - - it('loads the batch relation when recording and verifying (guards the anchored-hash check)', async () => { - // The anchored-hash guard in recordHash and the anchored-branch in verifyDocument both - // depend on `existing.batch`/`file.batch` being populated, which only happens when the - // query explicitly requests `relations: { batch: true }`. Assert the option is actually passed, - // so dropping it in production (which would let an anchored leaf be silently overwritten) - // breaks this test rather than passing unnoticed. - const findOneSpy = jest.spyOn(fileRepo, 'findOne'); - - await service.recordHash('archive', 'doc-a.pdf', sha256(docs[0].data).toString('hex')); - await service.verifyDocument('archive', 'doc-a.pdf', docs[0].data); - - expect(findOneSpy).toHaveBeenCalled(); - for (const call of findOneSpy.mock.calls) { - expect(call[0]).toMatchObject({ relations: { batch: true } }); - } - }); - - it('refuses to overwrite an anchored hash with a differing hash (avoids bogus tampering)', async () => { - await service.anchorPending(); - - const anchored = fileRepo.files.find((f: ArchiveFile) => f.name === 'doc-a.pdf'); - const originalHash = anchored.sha256; - - await expect( - service.recordHash('archive', 'doc-a.pdf', sha256(Buffer.from('re-uploaded A')).toString('hex')), - ).rejects.toThrow(/Refusing to overwrite anchored hash/); - - // the anchored leaf hash is untouched - expect(anchored.sha256).toBe(originalHash); - }); - - it('is a no-op when recording the same hash on an already-anchored file', async () => { - await service.anchorPending(); - - const anchored = fileRepo.files.find((f: ArchiveFile) => f.name === 'doc-a.pdf'); - const originalHash = anchored.sha256; - - await expect(service.recordHash('archive', 'doc-a.pdf', originalHash)).resolves.toBeUndefined(); - - expect(anchored.sha256).toBe(originalHash); - expect(fileRepo.files).toHaveLength(3); - }); - - it('persists upgraded proof bytes even while verify still reports pending', async () => { - await service.anchorPending(); - - const batch = batchRepo.batches[0]; - const originalProof = batch.otsProof; - - // upgrade yields changed bytes, but the attestation is not yet on-chain - (ots.upgrade as jest.Mock).mockResolvedValueOnce(Buffer.from('UPGRADED-BUT-PENDING')); - (ots.verify as jest.Mock).mockResolvedValueOnce({ confirmed: false, pending: true }); - - await service.upgradeBatches(); - - expect(batch.otsProof).toBe(Buffer.from('UPGRADED-BUT-PENDING').toString('base64')); - expect(batch.otsProof).not.toBe(originalProof); - // still not confirmed - expect(batch.status).toBe('PendingBtc'); - expect(batch.bitcoinHeight).toBeUndefined(); - }); - - it('does not save a batch when upgrade is unchanged and still pending', async () => { - await service.anchorPending(); - - const batch = batchRepo.batches[0]; - const saveSpy = jest.spyOn(batchRepo, 'save'); - - // upgrade returns the same bytes, verify still pending => nothing to persist - await service.upgradeBatches(); - - expect(saveSpy).not.toHaveBeenCalled(); - expect(batch.status).toBe('PendingBtc'); - }); - - it('skips a pending batch that carries no OTS proof (never touches the library)', async () => { - await service.anchorPending(); - - // simulate a malformed/empty proof on the only pending batch - const batch = batchRepo.batches[0]; - batch.otsProof = null; - - const saveSpy = jest.spyOn(batchRepo, 'save'); - - await service.upgradeBatches(); - - expect(saveSpy).not.toHaveBeenCalled(); - expect(ots.upgrade).not.toHaveBeenCalled(); - expect(ots.verify).not.toHaveBeenCalled(); - expect(batch.status).toBe('PendingBtc'); - }); - - it('reports the Bitcoin height when verifying a document whose batch is confirmed on-chain', async () => { - await service.anchorPending(); - - // the proof now carries a Bitcoin attestation - (ots.verify as jest.Mock).mockResolvedValue({ confirmed: true, pending: false, bitcoin: { height: 850000 } }); - - const result = await service.verifyDocument('archive', 'doc-b.pdf', docs[1].data); - - expect(result.found).toBe(true); - expect(result.hashMatches).toBe(true); - expect(result.anchored).toBe(true); - expect(result.proofValid).toBe(true); - expect(result.verified).toBe(true); - expect(result.pending).toBe(false); - expect(result.bitcoinHeight).toBe(850000); - }); - - it('throws when an anchored file is missing its persisted merkleProof', async () => { - await service.anchorPending(); - - const file = fileRepo.files.find((f: ArchiveFile) => f.name === 'doc-b.pdf'); - file.merkleProof = null; - - await expect(service.verifyDocument('archive', 'doc-b.pdf', docs[1].data)).rejects.toThrow( - /Data-integrity inconsistency.*no persisted merkleProof/, - ); - }); - - it('refuses a concurrent recordHash that races with anchorPending (TOCTOU)', async () => { - // Isolate to a single unanchored file so the interleaving is clear. - // (round-trip beforeEach already recorded three docs — clear and re-seed one.) - fileRepo.files.length = 0; - batchRepo.batches.length = 0; - await service.recordHash('archive', 'race.pdf', sha256(Buffer.from('original')).toString('hex')); - - const stored = fileRepo.files.find((f: ArchiveFile) => f.name === 'race.pdf'); - const anchoredHash = stored.sha256; - // Snapshot of the unanchored row as recordHash would have seen it at read time. - const staleSnapshot = Object.assign(new ArchiveFile(), { - id: stored.id, - bucket: stored.bucket, - name: stored.name, - sha256: stored.sha256, - batch: undefined, - leafIndex: undefined, - merkleProof: undefined, - }); - - // Between findOne (read) and the conditional update, run a full anchorPending that - // claims the row into a batch. recordHash still only sees the stale pre-anchor snapshot. - jest.spyOn(fileRepo, 'findOne').mockImplementationOnce(async () => { - await service.anchorPending(); - return staleSnapshot; - }); - - const newHash = sha256(Buffer.from('concurrent re-upload')).toString('hex'); - await expect(service.recordHash('archive', 'race.pdf', newHash)).rejects.toThrow( - /Refusing to overwrite unanchored hash/, - ); - - // Leaf committed by anchorPending must be untouched. - const after = fileRepo.files.find((f: ArchiveFile) => f.name === 'race.pdf'); - expect(after.sha256).toBe(anchoredHash); - expect(after.batch).toBeDefined(); - expect(typeof after.merkleProof).toBe('string'); - expect(after.merkleProof.length).toBeGreaterThan(0); - }); - }); -}); diff --git a/src/integration/infrastructure/storage/anchoring/__tests__/merkle-proof-codec.spec.ts b/src/integration/infrastructure/storage/anchoring/__tests__/merkle-proof-codec.spec.ts deleted file mode 100644 index 1d496286e8..0000000000 --- a/src/integration/infrastructure/storage/anchoring/__tests__/merkle-proof-codec.spec.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { deserializeMerkleProof, serializeMerkleProof } from '../merkle-proof-codec'; -import { MerkleProofStep } from '../merkle'; - -describe('merkle-proof-codec', () => { - const validSibling = Buffer.alloc(32, 0xab); - const validSteps: MerkleProofStep[] = [ - { sibling: validSibling, right: true }, - { sibling: Buffer.alloc(32, 0xcd), right: false }, - ]; - - describe('serializeMerkleProof / deserializeMerkleProof roundtrip', () => { - it('returns equivalent steps after serialize then deserialize', () => { - const json = serializeMerkleProof(validSteps); - const restored = deserializeMerkleProof(json); - - expect(restored).toHaveLength(2); - expect(restored[0].right).toBe(true); - expect(restored[0].sibling.equals(validSibling)).toBe(true); - expect(restored[1].right).toBe(false); - expect(restored[1].sibling.equals(Buffer.alloc(32, 0xcd))).toBe(true); - }); - }); - - describe('deserializeMerkleProof validation', () => { - it('throws when JSON is not an array', () => { - expect(() => deserializeMerkleProof(JSON.stringify({ sibling: 'x', right: true }))).toThrow( - 'expected an array of steps', - ); - }); - - it('throws on non-hex sibling', () => { - const json = JSON.stringify([{ sibling: 'g'.repeat(64), right: true }]); - expect(() => deserializeMerkleProof(json)).toThrow('Invalid merkle proof step at index 0'); - }); - - it('throws on wrong-length hex sibling (63 chars)', () => { - const json = JSON.stringify([{ sibling: 'a'.repeat(63), right: true }]); - expect(() => deserializeMerkleProof(json)).toThrow('Invalid merkle proof step at index 0'); - }); - - it('throws on wrong-length hex sibling (65 chars)', () => { - const json = JSON.stringify([{ sibling: 'a'.repeat(65), right: true }]); - expect(() => deserializeMerkleProof(json)).toThrow('Invalid merkle proof step at index 0'); - }); - - it('throws on uppercase hex sibling (lowercase required)', () => { - const json = JSON.stringify([{ sibling: 'A'.repeat(64), right: true }]); - expect(() => deserializeMerkleProof(json)).toThrow('Invalid merkle proof step at index 0'); - }); - - it('throws on non-boolean right', () => { - const json = JSON.stringify([{ sibling: 'a'.repeat(64), right: 'yes' }]); - expect(() => deserializeMerkleProof(json)).toThrow('Invalid merkle proof step at index 0'); - }); - }); -}); diff --git a/src/integration/infrastructure/storage/anchoring/__tests__/merkle.spec.ts b/src/integration/infrastructure/storage/anchoring/__tests__/merkle.spec.ts deleted file mode 100644 index 813563bb9e..0000000000 --- a/src/integration/infrastructure/storage/anchoring/__tests__/merkle.spec.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { buildMerkleRoot, merkleInclusionProof, sha256, verifyMerkleProof } from '../merkle'; - -/** Deterministic leaf: sha256 of `leaf-` so tests don't depend on random data. */ -function leaf(i: number): Buffer { - return sha256(Buffer.from(`leaf-${i}`)); -} - -function leaves(count: number): Buffer[] { - return Array.from({ length: count }, (_, i) => leaf(i)); -} - -// Independent reference helpers — do NOT call production merkle functions. -const LEAF_PREFIX = Buffer.from([0x00]); -const NODE_PREFIX = Buffer.from([0x01]); - -function leafHash(d: Buffer): Buffer { - return sha256(Buffer.concat([LEAF_PREFIX, d])); -} - -function node(l: Buffer, r: Buffer): Buffer { - return sha256(Buffer.concat([NODE_PREFIX, l, r])); -} - -describe('merkle', () => { - describe('buildMerkleRoot', () => { - it('throws on zero leaves', () => { - expect(() => buildMerkleRoot([])).toThrow(); - }); - - it('returns leafHash(d) for a single-leaf tree', () => { - const d0 = leaf(0); - expect(buildMerkleRoot([d0]).equals(leafHash(d0))).toBe(true); - }); - - it('hashes the pair for two leaves', () => { - const [a, b] = leaves(2); - expect(buildMerkleRoot([a, b]).equals(node(leafHash(a), leafHash(b)))).toBe(true); - }); - - it('builds an unbalanced tree for three leaves (no last-node duplication)', () => { - const [a, b, c] = leaves(3); - // k=2: root = node(node(L0,L1), L2) — NOT node(node(L0,L1), node(L2,L2)) - const expected = node(node(leafHash(a), leafHash(b)), leafHash(c)); - expect(buildMerkleRoot([a, b, c]).equals(expected)).toBe(true); - }); - - it('builds a balanced tree for four leaves', () => { - const [a, b, c, d] = leaves(4); - const expected = node(node(leafHash(a), leafHash(b)), node(leafHash(c), leafHash(d))); - expect(buildMerkleRoot([a, b, c, d]).equals(expected)).toBe(true); - }); - - it('is deterministic across repeated calls', () => { - const ls = leaves(4); - expect(buildMerkleRoot(ls).equals(buildMerkleRoot(ls))).toBe(true); - }); - - it('CVE-2012-2459: three-leaf and four-leaf-with-duplicated-last produce different roots', () => { - const [a, b, c] = leaves(3); - const root3 = buildMerkleRoot([a, b, c]); - const root4dup = buildMerkleRoot([a, b, c, c]); - expect(root3.equals(root4dup)).toBe(false); - }); - - it('applies domain-separation prefixes (leaf and node)', () => { - const d = leaf(0); - const root1 = buildMerkleRoot([d]); - // Single leaf is leafHash(d), not the raw digest. - expect(root1.equals(d)).toBe(false); - expect(root1.equals(leafHash(d))).toBe(true); - - const [a, b] = leaves(2); - const root2 = buildMerkleRoot([a, b]); - // Two leaves use node() with 0x01 prefix, not bare sha256 of concatenated leaf hashes. - const bareNoPrefix = sha256(Buffer.concat([leafHash(a), leafHash(b)])); - expect(root2.equals(bareNoPrefix)).toBe(false); - expect(root2.equals(node(leafHash(a), leafHash(b)))).toBe(true); - }); - }); - - describe('inclusion proof + verification', () => { - for (let size = 1; size <= 5; size++) { - it(`verifies every leaf in a tree of size ${size}`, () => { - const ls = leaves(size); - const root = buildMerkleRoot(ls); - - for (let index = 0; index < size; index++) { - const proof = merkleInclusionProof(ls, index); - expect(verifyMerkleProof(ls[index], proof, root)).toBe(true); - } - }); - } - - it('produces an empty proof for a single-leaf tree', () => { - const ls = leaves(1); - expect(merkleInclusionProof(ls, 0)).toEqual([]); - }); - - it('throws for an out-of-range index', () => { - const ls = leaves(3); - expect(() => merkleInclusionProof(ls, 3)).toThrow(); - expect(() => merkleInclusionProof(ls, -1)).toThrow(); - }); - - it('throws for a proof over zero leaves', () => { - expect(() => merkleInclusionProof([], 0)).toThrow(); - }); - }); - - describe('tamper detection', () => { - const ls = leaves(4); - const root = buildMerkleRoot(ls); - const index = 1; - const proof = merkleInclusionProof(ls, index); - - it('rejects a manipulated leaf', () => { - const tampered = sha256(Buffer.from('not-the-original-leaf')); - expect(verifyMerkleProof(tampered, proof, root)).toBe(false); - }); - - it('rejects a tampered sibling in the proof', () => { - const badProof = proof.map((s, i) => (i === 0 ? { ...s, sibling: sha256(Buffer.from('wrong')) } : s)); - expect(verifyMerkleProof(ls[index], badProof, root)).toBe(false); - }); - - it('rejects a flipped left/right position', () => { - const badProof = proof.map((s) => ({ ...s, right: !s.right })); - expect(verifyMerkleProof(ls[index], badProof, root)).toBe(false); - }); - - it('rejects verification against a wrong root', () => { - const wrongRoot = sha256(Buffer.from('some-other-root')); - expect(verifyMerkleProof(ls[index], proof, wrongRoot)).toBe(false); - }); - - it("rejects another leaf's proof for this leaf", () => { - const otherProof = merkleInclusionProof(ls, 2); - expect(verifyMerkleProof(ls[index], otherProof, root)).toBe(false); - }); - }); -}); diff --git a/src/integration/infrastructure/storage/anchoring/__tests__/opentimestamps.service.spec.ts b/src/integration/infrastructure/storage/anchoring/__tests__/opentimestamps.service.spec.ts deleted file mode 100644 index ef569aa55b..0000000000 --- a/src/integration/infrastructure/storage/anchoring/__tests__/opentimestamps.service.spec.ts +++ /dev/null @@ -1,176 +0,0 @@ -// Mock the `opentimestamps` npm library so stamp/upgrade/verify can be exercised WITHOUT any -// network access. We mock exactly the surface OpenTimestampsService touches: -// - OpenTimestamps.stamp / .upgrade / .verify (promise-returning network calls) -// - OpenTimestamps.DetachedTimestampFile.fromHash (sync constructor from a digest) -// - OpenTimestamps.DetachedTimestampFile.deserialize (sync constructor from .ots bytes) -// - OpenTimestamps.Ops.OpSHA256 (sync op constructor) -// Each fake DetachedTimestampFile carries a `serializeToBytes()` returning a recognizable byte -// array so we can assert on the bytes the service returns. - -const stampMock = jest.fn(); -const upgradeMock = jest.fn(); -const verifyMock = jest.fn(); -const fromHashMock = jest.fn(); -const deserializeMock = jest.fn(); -const opSHA256Mock = jest.fn(); - -jest.mock('opentimestamps', () => ({ - stamp: (...args: any[]) => stampMock(...args), - upgrade: (...args: any[]) => upgradeMock(...args), - verify: (...args: any[]) => verifyMock(...args), - DetachedTimestampFile: { - fromHash: (...args: any[]) => fromHashMock(...args), - deserialize: (...args: any[]) => deserializeMock(...args), - }, - Ops: { - OpSHA256: function OpSHA256(this: any) { - opSHA256Mock(); - }, - }, -})); - -import { Test, TestingModule } from '@nestjs/testing'; -import { OpenTimestampsService } from '../opentimestamps.service'; - -/** A fake DetachedTimestampFile whose serialized form is deterministic for assertions. */ -function fakeDetached(serialized: number[]) { - return { - serializeToBytes: jest.fn(() => serialized), - }; -} - -describe('OpenTimestampsService', () => { - let service: OpenTimestampsService; - - beforeEach(async () => { - jest.clearAllMocks(); - - const module: TestingModule = await Test.createTestingModule({ - providers: [OpenTimestampsService], - }).compile(); - - service = module.get(OpenTimestampsService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); - - describe('stamp', () => { - it('builds a detached file from the digest, submits it and returns the serialized .ots bytes', async () => { - const digest = Buffer.alloc(32, 7); - const serialized = [1, 2, 3, 4]; - const detached = fakeDetached(serialized); - fromHashMock.mockReturnValue(detached); - stampMock.mockResolvedValue(undefined); - - const result = await service.stamp(digest); - - // a SHA-256 op was constructed and fromHash was called with that op + the raw digest - expect(opSHA256Mock).toHaveBeenCalledTimes(1); - expect(fromHashMock).toHaveBeenCalledTimes(1); - const [, passedDigest] = fromHashMock.mock.calls[0]; - expect(passedDigest).toBe(digest); - - // the library stamp was invoked on exactly that detached file - expect(stampMock).toHaveBeenCalledTimes(1); - expect(stampMock).toHaveBeenCalledWith(detached); - - // and the returned bytes are a Buffer wrapping the serialized form - expect(Buffer.isBuffer(result)).toBe(true); - expect(result).toEqual(Buffer.from(serialized)); - expect(detached.serializeToBytes).toHaveBeenCalledTimes(1); - }); - - it('propagates a calendar/network failure from the library', async () => { - fromHashMock.mockReturnValue(fakeDetached([0])); - stampMock.mockRejectedValue(new Error('calendar unreachable')); - - await expect(service.stamp(Buffer.alloc(32))).rejects.toThrow('calendar unreachable'); - }); - }); - - describe('upgrade', () => { - it('returns the re-serialized bytes when the proof changed', async () => { - const otsBytes = Buffer.from([9, 9, 9]); - const upgradedSerialized = [5, 6, 7, 8]; - const detached = fakeDetached(upgradedSerialized); - deserializeMock.mockReturnValue(detached); - upgradeMock.mockResolvedValue(true); // changed - - const result = await service.upgrade(otsBytes); - - expect(deserializeMock).toHaveBeenCalledWith(otsBytes); - expect(upgradeMock).toHaveBeenCalledWith(detached); - expect(detached.serializeToBytes).toHaveBeenCalledTimes(1); - expect(result).toEqual(Buffer.from(upgradedSerialized)); - // genuinely new bytes, not the original - expect(result.equals(otsBytes)).toBe(false); - }); - - it('returns the original bytes unchanged when nothing changed', async () => { - const otsBytes = Buffer.from([9, 9, 9]); - const detached = fakeDetached([1, 1, 1]); - deserializeMock.mockReturnValue(detached); - upgradeMock.mockResolvedValue(false); // unchanged - - const result = await service.upgrade(otsBytes); - - // exact same buffer reference is returned, serialize is never consulted - expect(result).toBe(otsBytes); - expect(detached.serializeToBytes).not.toHaveBeenCalled(); - }); - }); - - describe('verify', () => { - it('reports confirmed with the Bitcoin height when an attestation is present', async () => { - const digest = Buffer.alloc(32, 1); - const otsBytes = Buffer.from([4, 2]); - const detachedOts = fakeDetached([]); - const detachedDigest = fakeDetached([]); - deserializeMock.mockReturnValue(detachedOts); - fromHashMock.mockReturnValue(detachedDigest); - verifyMock.mockResolvedValue({ bitcoin: { height: 840123, timestamp: 1700000000 } }); - - const result = await service.verify(digest, otsBytes); - - // proof deserialized, digest re-derived, and verify run against the trusted explorers - expect(deserializeMock).toHaveBeenCalledWith(otsBytes); - expect(fromHashMock).toHaveBeenCalledTimes(1); - expect(verifyMock).toHaveBeenCalledWith(detachedOts, detachedDigest, { ignoreBitcoinNode: true }); - - expect(result).toEqual({ bitcoin: { height: 840123 }, confirmed: true, pending: false }); - }); - - it('reports pending when the library returns no attestation (undefined result)', async () => { - deserializeMock.mockReturnValue(fakeDetached([])); - fromHashMock.mockReturnValue(fakeDetached([])); - verifyMock.mockResolvedValue(undefined); - - const result = await service.verify(Buffer.alloc(32), Buffer.from([1])); - - expect(result).toEqual({ confirmed: false, pending: true }); - expect(result.bitcoin).toBeUndefined(); - }); - - it('reports pending when the result has no bitcoin attestation', async () => { - deserializeMock.mockReturnValue(fakeDetached([])); - fromHashMock.mockReturnValue(fakeDetached([])); - verifyMock.mockResolvedValue({}); - - const result = await service.verify(Buffer.alloc(32), Buffer.from([1])); - - expect(result).toEqual({ confirmed: false, pending: true }); - }); - - it('reports pending when the bitcoin field lacks a numeric height', async () => { - deserializeMock.mockReturnValue(fakeDetached([])); - fromHashMock.mockReturnValue(fakeDetached([])); - verifyMock.mockResolvedValue({ bitcoin: {} }); - - const result = await service.verify(Buffer.alloc(32), Buffer.from([1])); - - expect(result).toEqual({ confirmed: false, pending: true }); - }); - }); -}); diff --git a/src/integration/infrastructure/storage/anchoring/archive-batch.entity.ts b/src/integration/infrastructure/storage/anchoring/archive-batch.entity.ts deleted file mode 100644 index 0ea84fcea8..0000000000 --- a/src/integration/infrastructure/storage/anchoring/archive-batch.entity.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { IEntity } from 'src/shared/models/entity'; -import { Column, Entity, OneToMany } from 'typeorm'; -import { ArchiveFile } from './archive-file.entity'; - -/** Lifecycle of an anchoring batch on the way to a Bitcoin attestation. */ -export enum ArchiveBatchStatus { - PENDING_BTC = 'PendingBtc', - CONFIRMED = 'Confirmed', -} - -/** - * A daily (or on-demand) Merkle batch over a set of {@link ArchiveFile} hashes for the - * GeBüV anchoring pipeline. The `merkleRoot` is timestamped via OpenTimestamps; the - * resulting detached `.ots` proof is persisted (base64) in `otsProof` and upgraded over - * time until it carries a Bitcoin attestation (`bitcoinHeight` set, `status` confirmed). - */ -@Entity() -export class ArchiveBatch extends IEntity { - @Column({ length: 64 }) - merkleRoot: string; - - @Column({ type: 'text', nullable: true }) - otsProof?: string; - - @Column({ type: 'int', nullable: true }) - bitcoinHeight?: number; - - @Column({ length: 256, default: ArchiveBatchStatus.PENDING_BTC }) - status: ArchiveBatchStatus; - - @OneToMany(() => ArchiveFile, (file) => file.batch) - files: ArchiveFile[]; -} diff --git a/src/integration/infrastructure/storage/anchoring/archive-batch.repository.ts b/src/integration/infrastructure/storage/anchoring/archive-batch.repository.ts deleted file mode 100644 index 5ca13be275..0000000000 --- a/src/integration/infrastructure/storage/anchoring/archive-batch.repository.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { BaseRepository } from 'src/shared/repositories/base.repository'; -import { EntityManager } from 'typeorm'; -import { ArchiveBatch } from './archive-batch.entity'; - -@Injectable() -export class ArchiveBatchRepository extends BaseRepository { - constructor(manager: EntityManager) { - super(ArchiveBatch, manager); - } -} diff --git a/src/integration/infrastructure/storage/anchoring/archive-file.entity.ts b/src/integration/infrastructure/storage/anchoring/archive-file.entity.ts deleted file mode 100644 index 58f64f5402..0000000000 --- a/src/integration/infrastructure/storage/anchoring/archive-file.entity.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { IEntity } from 'src/shared/models/entity'; -import { Column, Entity, Index, ManyToOne } from 'typeorm'; -import { ArchiveBatch } from './archive-batch.entity'; -import { deserializeMerkleProof, serializeMerkleProof } from './merkle-proof-codec'; -import { MerkleProofStep } from './merkle'; - -/** - * A single archived storage object whose content hash participates in the GeBüV anchoring - * pipeline. Each file is identified uniquely by its `(bucket, name)` location and records - * the SHA-256 of its content. Once anchored, it points to its {@link ArchiveBatch} and - * carries its `leafIndex` within that batch's Merkle tree plus a persisted inclusion proof - * so verification does not depend on sibling rows remaining available. - */ -@Entity() -@Index((file: ArchiveFile) => [file.bucket, file.name], { unique: true }) -export class ArchiveFile extends IEntity { - @Column({ type: 'text' }) - bucket: string; - - @Column({ type: 'text' }) - name: string; - - @Column({ length: 64 }) - sha256: string; - - @Index() - @ManyToOne(() => ArchiveBatch, (batch) => batch.files, { nullable: true }) - batch?: ArchiveBatch; - - @Column({ type: 'int', nullable: true }) - leafIndex?: number; - - /** - * JSON-serialized inclusion proof (`MerkleProofStep[]`, siblings hex-encoded), populated - * from `merkleInclusionProof` at anchor time. Persisted so an old file stays verifiable - * even if sibling rows in the same batch are later lost or corrupted. Raw JSON string column — - * business logic should go through the typed `merkleProofSteps` getter/setter below, never - * parse/stringify this directly. - */ - @Column({ type: 'text', nullable: true }) - merkleProof?: string; - - get merkleProofSteps(): MerkleProofStep[] | undefined { - return this.merkleProof ? deserializeMerkleProof(this.merkleProof) : undefined; - } - - set merkleProofSteps(steps: MerkleProofStep[] | undefined) { - this.merkleProof = steps ? serializeMerkleProof(steps) : undefined; - } -} diff --git a/src/integration/infrastructure/storage/anchoring/archive-file.repository.ts b/src/integration/infrastructure/storage/anchoring/archive-file.repository.ts deleted file mode 100644 index b162ce3b20..0000000000 --- a/src/integration/infrastructure/storage/anchoring/archive-file.repository.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { BaseRepository } from 'src/shared/repositories/base.repository'; -import { EntityManager } from 'typeorm'; -import { ArchiveFile } from './archive-file.entity'; - -@Injectable() -export class ArchiveFileRepository extends BaseRepository { - constructor(manager: EntityManager) { - super(ArchiveFile, manager); - } -} diff --git a/src/integration/infrastructure/storage/anchoring/archive.module.ts b/src/integration/infrastructure/storage/anchoring/archive.module.ts deleted file mode 100644 index 04ad7ac7d2..0000000000 --- a/src/integration/infrastructure/storage/anchoring/archive.module.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Module } from '@nestjs/common'; -import { TypeOrmModule } from '@nestjs/typeorm'; -import { SharedModule } from 'src/shared/shared.module'; -import { ArchiveBatch } from './archive-batch.entity'; -import { ArchiveBatchRepository } from './archive-batch.repository'; -import { ArchiveFile } from './archive-file.entity'; -import { ArchiveFileRepository } from './archive-file.repository'; -import { ArchiveScheduler } from './archive.scheduler'; -import { ArchiveService } from './archive.service'; -import { OpenTimestampsService } from './opentimestamps.service'; - -@Module({ - imports: [SharedModule, TypeOrmModule.forFeature([ArchiveBatch, ArchiveFile])], - controllers: [], - providers: [ArchiveBatchRepository, ArchiveFileRepository, ArchiveService, OpenTimestampsService, ArchiveScheduler], - exports: [ArchiveService, OpenTimestampsService], -}) -export class ArchiveModule {} diff --git a/src/integration/infrastructure/storage/anchoring/archive.scheduler.ts b/src/integration/infrastructure/storage/anchoring/archive.scheduler.ts deleted file mode 100644 index 942e8ffebf..0000000000 --- a/src/integration/infrastructure/storage/anchoring/archive.scheduler.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { CronExpression } from '@nestjs/schedule'; -import { RepositoryFactory } from 'src/shared/repositories/repository.factory'; -import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; -import { IsNull, Not } from 'typeorm'; -import { createStorageService } from '../storage.factory'; -import { ArchiveService } from './archive.service'; -import { sha256 } from './merkle'; - -// Mirrors KYC_CONTAINER in kyc-document.service.ts — duplicated rather than imported so this -// low-level storage/anchoring module does not depend on the KYC domain module. -const KYC_BUCKET = 'kyc'; - -// Caps how many missing objects get backfilled per reconcile run so a huge one-off gap (e.g. after -// a migration) cannot blow the job's 3600s @DfxCron timeout or exhaust memory/sockets fetching -// every object in one Promise chain. The job is idempotent and re-runs daily, so anything beyond -// the cap is simply backfilled on a later run — the gap is never silently dropped, just deferred -// (and logged loudly below). -const RECONCILE_BACKFILL_BATCH_SIZE = 5000; - -/** - * Stage 3 of the GeBüV anchoring pipeline: drives {@link ArchiveService} on a schedule. - * - * Uses the repo-wide `@DfxCron` decorator (see src/shared/utils/cron.ts), which the - * central DfxCronService discovers at boot and runs behind a per-method `@Lock` plus a - * `Process` guard — so a disabled process or a multi-instance deployment never double-runs - * the same job. - */ -@Injectable() -export class ArchiveScheduler { - private readonly logger = new DfxLogger(ArchiveScheduler); - - constructor( - private readonly archiveService: ArchiveService, - private readonly repos: RepositoryFactory, - ) {} - - @DfxCron(CronExpression.EVERY_DAY_AT_2AM, { process: Process.ARCHIVE_ANCHOR, timeout: 3600 }) - async anchorPending(): Promise { - const batch = await this.archiveService.anchorPending(); - - if (batch) { - this.logger.info(`Anchored batch ${batch.id} with Merkle root ${batch.merkleRoot}`); - } else { - this.logger.verbose('No unanchored archive files to anchor'); - } - } - - @DfxCron(CronExpression.EVERY_HOUR, { process: Process.ARCHIVE_UPGRADE, timeout: 1800 }) - async upgradeBatches(): Promise { - await this.archiveService.upgradeBatches(); - } - - @DfxCron(CronExpression.EVERY_DAY_AT_1AM, { process: Process.ARCHIVE_RECONCILE, timeout: 3600 }) - async reconcileHashes(): Promise { - const buckets = await this.reconciliationBuckets(); - - for (const bucket of buckets) { - try { - await this.reconcileBucket(bucket); - } catch (e) { - this.logger.error(`Failed to reconcile archive hashes for bucket ${bucket}:`, e); - } - } - } - - // Retention-relevant buckets: the fixed KYC bucket plus every distinct EP2 settlement-report - // container currently configured on any UserData (resolved the same way - // fiat-output-job.service.ts resolves it at write time — these are per-merchant, runtime-only, - // and cannot be hardcoded). - private async reconciliationBuckets(): Promise { - const userDatas = await this.repos.userData.find({ - where: { paymentLinksConfig: Not(IsNull()) }, - select: { id: true, paymentLinksConfig: true }, - }); - - const ep2Containers: string[] = []; - for (const userData of userDatas) { - try { - const container = userData.paymentLinksConfigObj.ep2ReportContainer; - if (container != null) ep2Containers.push(container); - } catch (e) { - this.logger.error( - `Skipping UserData ${userData.id} with unparsable paymentLinksConfig during reconcile bucket enumeration:`, - e, - ); - } - } - - return [...new Set([KYC_BUCKET, ...ep2Containers])]; - } - - // Diffs live storage objects against the archive index and backfills any gap left by a - // previously failed recordHash call. A per-object recordHash failure (e.g. it throws because the - // object is already anchored under a different hash — a legitimate re-upload/overwrite on a - // versioned bucket) must not abort reconciliation of the remaining objects: log loudly and keep - // going, never swallow silently. - private async reconcileBucket(bucket: string): Promise { - const storageService = createStorageService(bucket); - - const [keys, recordedNames] = await Promise.all([ - storageService.listKeys(), - this.archiveService.recordedNames(bucket), - ]); - - const missing = keys.filter((key) => !recordedNames.has(key)); - if (missing.length === 0) return; - - const batch = missing.slice(0, RECONCILE_BACKFILL_BATCH_SIZE); - if (missing.length > batch.length) { - this.logger.warn( - `Reconciling only ${batch.length} of ${missing.length} unrecorded archive object(s) in bucket ${bucket} this run; ${ - missing.length - batch.length - } deferred to the next scheduled run`, - ); - } else { - this.logger.info(`Reconciling ${batch.length} unrecorded archive object(s) in bucket ${bucket}`); - } - - for (const key of batch) { - try { - const content = await storageService.getBlob(key); - await this.archiveService.recordHash(bucket, key, sha256(content.data).toString('hex')); - } catch (e) { - this.logger.error(`Failed to reconcile archive hash for ${bucket}/${key}:`, e); - } - } - } -} diff --git a/src/integration/infrastructure/storage/anchoring/archive.service.ts b/src/integration/infrastructure/storage/anchoring/archive.service.ts deleted file mode 100644 index d747214d49..0000000000 --- a/src/integration/infrastructure/storage/anchoring/archive.service.ts +++ /dev/null @@ -1,265 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { IsNull } from 'typeorm'; -import { ArchiveBatch, ArchiveBatchStatus } from './archive-batch.entity'; -import { ArchiveBatchRepository } from './archive-batch.repository'; -import { ArchiveFile } from './archive-file.entity'; -import { ArchiveFileRepository } from './archive-file.repository'; -import { serializeMerkleProof } from './merkle-proof-codec'; -import { buildMerkleRoot, merkleInclusionProof, sha256, verifyMerkleProof } from './merkle'; -import { OpenTimestampsService } from './opentimestamps.service'; - -/** Result of verifying an archived document against its anchored Merkle batch. */ -export interface ArchiveVerification { - /** false if no archive record exists for the given `(bucket, name)`. */ - found: boolean; - /** true if the stored SHA-256 equals the hash recomputed from the supplied data. */ - hashMatches?: boolean; - /** true once the file has been assigned to a Merkle batch. */ - anchored?: boolean; - /** true if the inclusion proof recomputes the batch's stored Merkle root. */ - proofValid?: boolean; - /** - * true iff found && hashMatches && anchored && proofValid (all four independently true). - * Independent of `pending`: a calendar-only (not yet Bitcoin-confirmed) proof can still - * be fully verified against its Merkle root. - */ - verified: boolean; - /** Bitcoin block height of the OpenTimestamps attestation, once anchored on-chain. */ - bitcoinHeight?: number; - /** - * true while the OpenTimestamps proof is still calendar-only (not yet on-chain). - * Independent of `verified` — pending does not mean unverified. - */ - pending?: boolean; -} - -/** - * Stage 2 of the GeBüV anchoring pipeline: it records content hashes of archived storage - * objects, batches the still-unanchored ones into a daily Merkle tree, timestamps the root - * via OpenTimestamps (Stage 1 primitives), upgrades those proofs to Bitcoin attestations, - * and verifies a given document against its anchored batch end-to-end. - * - * Leaves are the 32-byte SHA-256 digests of the file contents; the Merkle module - * domain-separates them via RFC 6962 leaf hashing (`sha256(0x00 || digest)`) before they - * enter the tree. `merkleRoot` is stored hex, `otsProof` is stored base64 of the serialized - * detached `.ots` bytes, and each file's inclusion proof is persisted as JSON on - * `archive_file.merkleProof`. - */ -@Injectable() -export class ArchiveService { - private readonly logger = new DfxLogger(ArchiveService); - - constructor( - private readonly archiveBatchRepo: ArchiveBatchRepository, - private readonly archiveFileRepo: ArchiveFileRepository, - private readonly ots: OpenTimestampsService, - ) {} - - /** - * Idempotently record the SHA-256 of an archived object identified by `(bucket, name)`. - * - * A new `(bucket, name)` is inserted as a fresh, unanchored row. An EXISTING row's `sha256` is - * NEVER mutated in place, whether or not it has been assigned to a Merkle batch: an identical - * hash is a no-op (idempotent), a differing hash is refused — logged (before → after, for audit) - * and rejected with a hard error — rather than silently applied. This is a deliberate - * "auditable mutation, no destructive overwrite" design (CONTRIBUTING.md): a re-upload/re-hash - * of the same `(bucket, name)` must never silently replace what was previously recorded, whether - * that record is already Merkle-anchored (where it would falsify {@link verifyDocument}) or not - * yet anchored (where it would erase evidence of the earlier, differing content without a - * trace). - * - * Because recordHash never writes to an existing row, there is nothing left for a concurrent - * {@link anchorPending} to race against here: anchorPending's own per-file conditional update - * likewise never touches `sha256` (see there). TOCTOU-safety therefore falls out of both - * functions simply never overwriting this column on an existing row, rather than out of a - * conditional-write/reload dance. - */ - async recordHash(bucket: string, name: string, sha256Hex: string): Promise { - const existing = await this.archiveFileRepo.findOne({ where: { bucket, name }, relations: { batch: true } }); - - if (existing) { - if (existing.sha256 === sha256Hex) return; - - if (existing.batch != null) { - const message = - `Refusing to overwrite anchored hash for ${bucket}/${name} (file ${existing.id}, batch ` + - `${existing.batch.id}): stored ${existing.sha256} differs from new ${sha256Hex}`; - this.logger.error(message); - throw new Error(message); - } - - const message = - `Refusing to overwrite unanchored hash for ${bucket}/${name} (file ${existing.id}): ` + - `stored ${existing.sha256} differs from new ${sha256Hex}`; - this.logger.warn(message); - throw new Error(message); - } - - const file = this.archiveFileRepo.create({ bucket, name, sha256: sha256Hex }); - await this.archiveFileRepo.save(file); - } - - /** - * Names already present in the `(bucket, name)` archive index for a bucket, regardless of - * anchoring status. Used by {@link ArchiveScheduler.reconcileHashes} to diff live storage - * objects against recorded hashes and find gaps left by a failed {@link recordHash} call — - * which, unlike a normal upload failure, leaves no row at all, so a DB-only scan can never - * find it. - */ - async recordedNames(bucket: string): Promise> { - const files = await this.archiveFileRepo.find({ where: { bucket }, select: { name: true } }); - return new Set(files.map((file) => file.name)); - } - - /** - * Batch all currently unanchored files (ordered by id) into one Merkle tree, timestamp its - * root via OpenTimestamps, and persist batch + per-file assignment (including each file's - * inclusion proof) in a single transaction. - * - * Each file's assignment is written via a conditional UPDATE keyed on `id`, `batch IS NULL`, - * AND the exact `sha256` read when the files were selected above (T0). If a concurrent - * {@link recordHash} call is refused in between (or, more generally, if the row no longer - * matches its T0 snapshot for any reason), the conditional update simply affects 0 rows for - * that file: it is skipped (left `batch IS NULL`, picked up again on the next anchoring cycle) - * rather than being claimed into a batch whose committed leaf may no longer correspond to the - * row's current content. This update only ever sets `batch` / `leafIndex` / `merkleProof` — it - * never writes `sha256`, so it can never destructively race against recordHash either. - * - * Returns the created batch, or `undefined` if there is nothing to anchor. - */ - async anchorPending(): Promise { - const files = await this.archiveFileRepo.find({ where: { batch: IsNull() }, order: { id: 'ASC' } }); - if (files.length === 0) return undefined; - - const leaves = files.map((file) => Buffer.from(file.sha256, 'hex')); - const root = buildMerkleRoot(leaves); - - const otsBytes = await this.ots.stamp(root); - - const batch = this.archiveBatchRepo.create({ - merkleRoot: root.toString('hex'), - otsProof: otsBytes.toString('base64'), - status: ArchiveBatchStatus.PENDING_BTC, - }); - - let anchoredCount = 0; - - await this.archiveBatchRepo.manager.transaction(async (manager) => { - const savedBatch = await manager.save(batch); - - for (const [index, file] of files.entries()) { - const result = await manager.update( - ArchiveFile, - { id: file.id, batch: IsNull(), sha256: file.sha256 }, - { - batch: savedBatch, - leafIndex: index, - merkleProof: serializeMerkleProof(merkleInclusionProof(leaves, index)), - }, - ); - - if (result.affected) { - anchoredCount++; - } else { - this.logger.warn( - `Skipped anchoring archive file ${file.id} (${file.bucket}/${file.name}) into batch ${savedBatch.id}: ` + - `its hash changed or it was reassigned between batching and commit (TOCTOU). It remains unanchored ` + - `and will be picked up by the next anchoring cycle.`, - ); - } - } - }); - - this.logger.info( - `Anchored batch ${batch.id} over ${anchoredCount}/${files.length} file(s), root ${batch.merkleRoot}`, - ); - - return batch; - } - - /** - * Try to upgrade every pending batch's OpenTimestamps proof towards a Bitcoin attestation. - * - * The upgraded `.ots` bytes are persisted whenever the proof changed at all (e.g. it now - * carries additional calendar commitments but `verify` still reports pending) so that - * progress is never thrown away. `bitcoinHeight`/`status = confirmed` are set additionally - * only once `verify` reports a Bitcoin attestation. - */ - async upgradeBatches(): Promise { - const batches = await this.archiveBatchRepo.findBy({ status: ArchiveBatchStatus.PENDING_BTC }); - - for (const batch of batches) { - if (!batch.otsProof) continue; - - const rootBuffer = Buffer.from(batch.merkleRoot, 'hex'); - const originalProof = batch.otsProof; - const upgraded = await this.ots.upgrade(Buffer.from(originalProof, 'base64')); - const upgradedProof = upgraded.toString('base64'); - const result = await this.ots.verify(rootBuffer, upgraded); - - const proofChanged = upgradedProof !== originalProof; - if (!proofChanged && !result.confirmed) continue; - - // Always persist progress when the proof bytes changed; confirm only on a real attestation. - if (proofChanged) batch.otsProof = upgradedProof; - - if (result.confirmed) { - batch.bitcoinHeight = result.bitcoin.height; - batch.status = ArchiveBatchStatus.CONFIRMED; - } - - await this.archiveBatchRepo.save(batch); - - if (result.confirmed) { - this.logger.info(`Confirmed batch ${batch.id} at Bitcoin height ${batch.bitcoinHeight}`); - } else { - this.logger.info(`Upgraded pending OpenTimestamps proof for batch ${batch.id}`); - } - } - } - - /** - * Verify a supplied document against its archived, anchored Merkle batch end-to-end: - * recompute its SHA-256 from the supplied bytes, compare with the stored hash, verify the - * persisted inclusion proof against the batch's Merkle root using the supplied digest as - * the leaf, and check the OpenTimestamps attestation status. - */ - async verifyDocument(bucket: string, name: string, data: Buffer): Promise { - const file = await this.archiveFileRepo.findOne({ where: { bucket, name }, relations: { batch: true } }); - if (!file) return { found: false, verified: false }; - - const computedDigest = sha256(data); - const computedHex = computedDigest.toString('hex'); - const hashMatches = file.sha256 === computedHex; - - const batch = file.batch; - if (!batch) return { found: true, hashMatches, anchored: false, verified: false }; - - if (file.merkleProof == null || file.merkleProof === '') { - const message = - `Data-integrity inconsistency: archive file ${file.id} (${bucket}/${name}) is assigned ` + - `to batch ${batch.id} but has no persisted merkleProof`; - this.logger.error(message); - throw new Error(message); - } - - const rootBuffer = Buffer.from(batch.merkleRoot, 'hex'); - const proof = file.merkleProofSteps; - // Leaf must be the digest of the SUPPLIED bytes — never the stored hash — so a - // tampered document yields proofValid: false. - const proofValid = verifyMerkleProof(computedDigest, proof, rootBuffer); - - let bitcoinHeight: number; - let pending = true; - - if (batch.otsProof) { - const ots = await this.ots.verify(rootBuffer, Buffer.from(batch.otsProof, 'base64')); - pending = ots.pending; - if (ots.bitcoin) bitcoinHeight = ots.bitcoin.height; - } - - const verified = hashMatches && proofValid; - return { found: true, hashMatches, anchored: true, proofValid, verified, bitcoinHeight, pending }; - } -} diff --git a/src/integration/infrastructure/storage/anchoring/merkle-proof-codec.ts b/src/integration/infrastructure/storage/anchoring/merkle-proof-codec.ts deleted file mode 100644 index 08c8f19f22..0000000000 --- a/src/integration/infrastructure/storage/anchoring/merkle-proof-codec.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { MerkleProofStep } from './merkle'; - -/** JSON shape of one persisted proof step (sibling hex-encoded). */ -interface SerializedMerkleProofStep { - sibling: string; - right: boolean; -} - -/** - * Serialize an inclusion proof for storage on `archive_file.merkleProof`. - * Siblings are hex-encoded 32-byte digests. - */ -export function serializeMerkleProof(proof: MerkleProofStep[]): string { - const serialized: SerializedMerkleProofStep[] = proof.map((step) => ({ - sibling: step.sibling.toString('hex'), - right: step.right, - })); - return JSON.stringify(serialized); -} - -/** - * Deserialize a persisted inclusion proof from `archive_file.merkleProof`. - * Throws if the JSON is not an array of steps with hex sibling digests. - */ -export function deserializeMerkleProof(json: string): MerkleProofStep[] { - const parsed: unknown = JSON.parse(json); - if (!Array.isArray(parsed)) { - throw new Error('Invalid merkle proof JSON: expected an array of steps'); - } - - return parsed.map((step: SerializedMerkleProofStep, index: number) => { - if (typeof step?.sibling !== 'string' || !/^[0-9a-f]{64}$/.test(step.sibling) || typeof step?.right !== 'boolean') { - throw new Error(`Invalid merkle proof step at index ${index}`); - } - return { - sibling: Buffer.from(step.sibling, 'hex'), - right: step.right, - }; - }); -} diff --git a/src/integration/infrastructure/storage/anchoring/merkle.ts b/src/integration/infrastructure/storage/anchoring/merkle.ts deleted file mode 100644 index 7a23d58a21..0000000000 --- a/src/integration/infrastructure/storage/anchoring/merkle.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { createHash } from 'node:crypto'; - -/** - * Pure Merkle-tree primitives for the GeBüV anchoring pipeline, following the - * RFC 6962 Merkle Tree Hash (MTH) construction. - * - * DOMAIN SEPARATION (load-bearing — verification depends on it): - * - LEAF_PREFIX = 0x00, NODE_PREFIX = 0x01 (single bytes). - * - `leafHash(d) = sha256(0x00 || d)` where `d` is a 32-byte document DIGEST. - * Callers (archive.service.ts) pass `archive_file.sha256` as raw digest bytes; - * the raw document bytes are NOT available inside this module. The exported - * functions therefore domain-separate each digest via `leafHash` before it - * enters the tree — leaves are never used raw. - * - `node(l, r) = sha256(0x01 || l || r)` for two 32-byte child hashes. - * - * TREE SHAPE (Merkle Tree Hash, MTH) for a list D[n] of digests, n >= 1: - * - n == 1: MTH(D) = leafHash(D[0]). - * - n > 1: let k be the largest power of two STRICTLY smaller than n - * (k = 1; while (k * 2 < n) k *= 2). Then - * MTH(D[0:n]) = node(MTH(D[0:k]), MTH(D[k:n])). - * There is NO duplication of a trailing odd element (avoids CVE-2012-2459). - * - n == 0: throw. - * - * Worked example (3 leaves d0,d1,d2; L_i = leafHash(d_i)): - * k=2, root = node(node(L0,L1), L2) - * — NOT node(node(L0,L1), node(L2,L2)) (the old, wrong, duplicated form). - * - * Inclusion proofs (RFC 6962 §2.1.1 audit path) use the same split rule and are - * ordered from the leaf's immediate sibling first to the root-adjacent sibling last. - */ - -const LEAF_PREFIX = Buffer.from([0x00]); -const NODE_PREFIX = Buffer.from([0x01]); - -/** A single step of an inclusion proof: the sibling digest and whether it sits on the right. */ -export interface MerkleProofStep { - sibling: Buffer; - /** true if the sibling is the RIGHT child (i.e. the running hash is the LEFT child). */ - right: boolean; -} - -/** SHA-256 of the given bytes. */ -export function sha256(data: Buffer): Buffer { - return createHash('sha256').update(data).digest(); -} - -/** Domain-separate a document digest into a leaf hash: sha256(0x00 || d). */ -function leafHash(digest: Buffer): Buffer { - return sha256(Buffer.concat([LEAF_PREFIX, digest])); -} - -/** Combine two child hashes into a parent: sha256(0x01 || left || right). */ -function node(left: Buffer, right: Buffer): Buffer { - return sha256(Buffer.concat([NODE_PREFIX, left, right])); -} - -/** - * Largest power of two strictly smaller than `n`. - * For n == 1 this is unused (caller handles the base case). - */ -function largestPowerOfTwoStrictlyLessThan(n: number): number { - let k = 1; - while (k * 2 < n) k *= 2; - return k; -} - -/** - * Build the Merkle root over document digests per RFC 6962 MTH. - * - * Throws on an empty input. Each digest is leaf-hashed before entering the tree. - */ -export function buildMerkleRoot(leafDigests: Buffer[]): Buffer { - if (leafDigests.length === 0) throw new Error('Cannot build a Merkle root from zero leaves'); - - return mth(leafDigests); -} - -/** Recursive Merkle Tree Hash over a slice of document digests. */ -function mth(digests: Buffer[]): Buffer { - const n = digests.length; - if (n === 1) return leafHash(digests[0]); - - const k = largestPowerOfTwoStrictlyLessThan(n); - return node(mth(digests.slice(0, k)), mth(digests.slice(k))); -} - -/** - * Compute the RFC 6962 §2.1.1 inclusion (audit) path for the digest at `index`. - * - * Ordered from the leaf's immediate sibling first to the root-adjacent sibling last — - * the order `verifyMerkleProof` walks (leaf → root). - * - * For a single-digest tree the proof is empty. Sibling subtree hashes are recomputed - * via MTH (acceptable for small daily KYC batches). - */ -export function merkleInclusionProof(leafDigests: Buffer[], index: number): MerkleProofStep[] { - if (leafDigests.length === 0) throw new Error('Cannot build a proof from zero leaves'); - if (index < 0 || index >= leafDigests.length) { - throw new Error(`Leaf index ${index} out of range [0, ${leafDigests.length})`); - } - - return auditPath(leafDigests, index); -} - -/** Recursive audit-path construction matching the MTH split rule. */ -function auditPath(digests: Buffer[], index: number): MerkleProofStep[] { - const n = digests.length; - if (n === 1) return []; - - const k = largestPowerOfTwoStrictlyLessThan(n); - - if (index < k) { - // Target in left half: recurse left, then append right-half sibling (sibling is RIGHT). - const proof = auditPath(digests.slice(0, k), index); - proof.push({ sibling: mth(digests.slice(k)), right: true }); - return proof; - } - - // Target in right half: recurse right, then append left-half sibling (sibling is LEFT). - const proof = auditPath(digests.slice(k), index - k); - proof.push({ sibling: mth(digests.slice(0, k)), right: false }); - return proof; -} - -/** - * Recompute the root from a document DIGEST and its inclusion proof, then compare - * against the expected `root`. Returns true only on an exact byte match. - * - * `leaf` is the raw document digest (not pre-hashed). This function applies - * `leafHash` internally before walking the proof: - * computed = leafHash(leaf); - * for each step: computed = step.right - * ? node(computed, step.sibling) - * : node(step.sibling, computed); - * return computed.equals(root) - */ -export function verifyMerkleProof(leaf: Buffer, proof: MerkleProofStep[], root: Buffer): boolean { - let computed = leafHash(leaf); - - for (const step of proof) { - computed = step.right ? node(computed, step.sibling) : node(step.sibling, computed); - } - - return computed.equals(root); -} diff --git a/src/integration/infrastructure/storage/anchoring/opentimestamps.service.ts b/src/integration/infrastructure/storage/anchoring/opentimestamps.service.ts deleted file mode 100644 index 9ad1d0fae5..0000000000 --- a/src/integration/infrastructure/storage/anchoring/opentimestamps.service.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import * as OpenTimestamps from 'opentimestamps'; - -/** Result of verifying a detached `.ots` proof against its digest. */ -export interface OtsVerifyResult { - /** Present once the timestamp is anchored in a Bitcoin block. */ - bitcoin?: { height: number }; - /** true once a Bitcoin attestation was found and verified (i.e. `bitcoin` is set). */ - confirmed: boolean; - /** - * true while the timestamp is NOT yet confirmed on-chain. This conflates "calendar-only, - * not yet anchored" and "verify could not confirm anything"; use `confirmed` to assert a - * positive Bitcoin attestation. - */ - pending: boolean; -} - -/** - * Minimal shape of the `opentimestamps` library's DetachedTimestampFile actually used here. - * The library ships no TypeScript types (no `@types/opentimestamps` dependency either), so the - * `OpenTimestamps` namespace import is untyped; this local interface replaces `any` with the - * concrete surface this service relies on. - */ -interface DetachedTimestampFile { - serializeToBytes(): Uint8Array; -} - -/** - * Thin async/await wrapper around the `opentimestamps` npm library for the GeBüV - * anchoring pipeline. It deliberately knows nothing about Merkle trees, storage or - * scheduling — callers feed it a single 32-byte SHA-256 digest (typically a daily - * Merkle root) and get back / consume serialized detached `.ots` proof bytes. - * - * The underlying library mixes synchronous constructors with promise-returning - * network calls; everything is normalized to `async` here. - * - * NOTE: `verify` is run with `ignoreBitcoinNode: true`, so attestation is checked - * against the public block explorers the library trusts, NOT a local Bitcoin node. - * Verifying against a trusted local node (the strongest GeBüV posture) is not possible - * from this pure service and must be wired up separately if/when a node is available. - */ -@Injectable() -export class OpenTimestampsService { - /** - * Create a detached timestamp over `digest` (an already-computed SHA-256, e.g. a - * Merkle root) by submitting it to the public OpenTimestamps calendars. - * - * Returns the serialized `.ots` bytes to be persisted. At this point the proof is - * typically still "pending" — it carries calendar commitments but no Bitcoin - * attestation yet; call `upgrade` later to complete it. - */ - async stamp(digest: Buffer): Promise { - const detached = this.detachedFromDigest(digest); - - await OpenTimestamps.stamp(detached); - - return Buffer.from(detached.serializeToBytes()); - } - - /** - * Attempt to upgrade a pending `.ots` proof to a complete Bitcoin attestation by - * asking the calendars for the now-available block path. - * - * Returns the upgraded `.ots` bytes if anything changed, otherwise the original - * bytes unchanged (still pending). - */ - async upgrade(otsBytes: Buffer): Promise { - const detached = OpenTimestamps.DetachedTimestampFile.deserialize(otsBytes); - - const changed = await OpenTimestamps.upgrade(detached); - - return changed ? Buffer.from(detached.serializeToBytes()) : otsBytes; - } - - /** - * Verify that `otsBytes` is a valid timestamp over `digest`. - * - * Returns the Bitcoin attestation height once anchored; while the proof is still - * calendar-only it reports `pending: true` with no `bitcoin` field. - */ - async verify(digest: Buffer, otsBytes: Buffer): Promise { - const detachedOts = OpenTimestamps.DetachedTimestampFile.deserialize(otsBytes); - const detached = this.detachedFromDigest(digest); - - // ignoreBitcoinNode: verify against the library's trusted explorers, not a local node. - const result = await OpenTimestamps.verify(detachedOts, detached, { ignoreBitcoinNode: true }); - - // The library returns an object keyed by chain (e.g. { bitcoin: { height, timestamp } }); - // an empty/undefined result means the proof is not yet anchored. - const bitcoin = result && result.bitcoin; - if (bitcoin && typeof bitcoin.height === 'number') - return { bitcoin: { height: bitcoin.height }, confirmed: true, pending: false }; - - return { confirmed: false, pending: true }; - } - - /** Build a DetachedTimestampFile that commits directly to an already-computed SHA-256 digest. */ - private detachedFromDigest(digest: Buffer): DetachedTimestampFile { - return OpenTimestamps.DetachedTimestampFile.fromHash(new OpenTimestamps.Ops.OpSHA256(), digest); - } -} diff --git a/src/subdomains/generic/gs/__tests__/gs.service.spec.ts b/src/subdomains/generic/gs/__tests__/gs.service.spec.ts index ef003f1e4c..1bac530afa 100644 --- a/src/subdomains/generic/gs/__tests__/gs.service.spec.ts +++ b/src/subdomains/generic/gs/__tests__/gs.service.spec.ts @@ -1,7 +1,3 @@ -// Stub the heavy `opentimestamps` library (pulled in transitively via KycDocumentService -> -// ArchiveService) so its eager network/`request` deps never load at jest runtime. -jest.mock('opentimestamps', () => ({})); - import { BadRequestException } from '@nestjs/common'; import { createMock } from '@golevelup/ts-jest'; import { DataSource } from 'typeorm'; @@ -32,7 +28,6 @@ import { KycFileService } from 'src/subdomains/generic/kyc/services/kyc-file.ser import { KycFileBlob } from 'src/subdomains/generic/kyc/dto/kyc-file.dto'; import { UserData } from 'src/subdomains/generic/user/models/user-data/user-data.entity'; import { AccountType } from 'src/subdomains/generic/user/models/user-data/account-type.enum'; -import { ArchiveService } from 'src/integration/infrastructure/storage/anchoring/archive.service'; import { Blob } from 'src/integration/infrastructure/storage/storage.service'; import { ConfigService, Environment } from 'src/config/config'; import { DebugAggregate, DebugQueryDto, DebugWhereNode, DebugWhereOp } from '../dto/debug-query.dto'; @@ -2242,7 +2237,7 @@ describe('GsService', () => { // Asserts both the host-stability and the path-preserving consumer invariant on the real output. it('produces a host-stable, path-preserving URL through the real KycDocumentService (round-trip)', async () => { const kycFileService = createMock(); - const realKycDocumentService = new KycDocumentService(kycFileService, createMock()); + const realKycDocumentService = new KycDocumentService(kycFileService); const userBlob = storageBlob('user/1/Identification/passport.pdf', new Date('2024-01-01')); const spiderBlob = storageBlob('spider/1/Identification/old-passport.pdf', new Date('2024-01-02')); diff --git a/src/subdomains/generic/kyc/kyc.module.ts b/src/subdomains/generic/kyc/kyc.module.ts index 501800cd23..8ed614b6dd 100644 --- a/src/subdomains/generic/kyc/kyc.module.ts +++ b/src/subdomains/generic/kyc/kyc.module.ts @@ -1,6 +1,5 @@ import { Module, forwardRef } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; -import { ArchiveModule } from 'src/integration/infrastructure/storage/anchoring/archive.module'; import { ScorechainModule } from 'src/integration/scorechain/scorechain.module'; import { SharedModule } from 'src/shared/shared.module'; import { BuyCryptoModule } from 'src/subdomains/core/buy-crypto/buy-crypto.module'; @@ -55,7 +54,6 @@ import { TfaService } from './services/tfa.service'; KycFile, ]), SharedModule, - ArchiveModule, NotificationModule, ScorechainModule, forwardRef(() => UserModule), diff --git a/src/subdomains/generic/kyc/services/__tests__/kyc.service.spec.ts b/src/subdomains/generic/kyc/services/__tests__/kyc.service.spec.ts index 9346a28519..d959936674 100644 --- a/src/subdomains/generic/kyc/services/__tests__/kyc.service.spec.ts +++ b/src/subdomains/generic/kyc/services/__tests__/kyc.service.spec.ts @@ -1,7 +1,3 @@ -// Stub the heavy `opentimestamps` library (pulled in transitively via KycDocumentService -> -// ArchiveService) so its eager network/`request` deps never load at jest runtime. -jest.mock('opentimestamps', () => ({})); - import { createMock } from '@golevelup/ts-jest'; import { ForbiddenException } from '@nestjs/common'; import { Configuration, ConfigService } from 'src/config/config'; diff --git a/src/subdomains/generic/kyc/services/integration/__tests__/kyc-document.service.spec.ts b/src/subdomains/generic/kyc/services/integration/__tests__/kyc-document.service.spec.ts index 0f7ae3e701..77f8b25e0e 100644 --- a/src/subdomains/generic/kyc/services/integration/__tests__/kyc-document.service.spec.ts +++ b/src/subdomains/generic/kyc/services/integration/__tests__/kyc-document.service.spec.ts @@ -1,24 +1,16 @@ -// Stub the heavy `opentimestamps` library (pulled in transitively via ArchiveService) so its -// eager network/`request` deps never load; ArchiveService is fully mocked in this spec. -jest.mock('opentimestamps', () => ({})); - // Control the storage backend the service constructs in its constructor, so uploadWormBlob is a // spy and no real S3/Azure/mock storage is touched. const uploadWormBlobMock = jest.fn(); const copyBlobsMock = jest.fn(); -const getBlobMock = jest.fn(); jest.mock('src/integration/infrastructure/storage/storage.factory', () => ({ createStorageService: jest.fn(() => ({ uploadWormBlob: (...args: any[]) => uploadWormBlobMock(...args), copyBlobs: (...args: any[]) => copyBlobsMock(...args), - getBlob: (...args: any[]) => getBlobMock(...args), })), })); import { createMock } from '@golevelup/ts-jest'; import { Test, TestingModule } from '@nestjs/testing'; -import { ArchiveService } from 'src/integration/infrastructure/storage/anchoring/archive.service'; -import { sha256 } from 'src/integration/infrastructure/storage/anchoring/merkle'; import { UserData } from 'src/subdomains/generic/user/models/user-data/user-data.entity'; import { FileType } from '../../../dto/kyc-file.dto'; import { KycFile } from '../../../entities/kyc-file.entity'; @@ -26,37 +18,30 @@ import { ContentType } from '../../../enums/content-type.enum'; import { KycFileService } from '../../kyc-file.service'; import { KycDocumentService } from '../kyc-document.service'; -describe('KycDocumentService - GeBüV hash recording', () => { +describe('KycDocumentService - storage', () => { let service: KycDocumentService; let kycFileService: KycFileService; - let archiveService: ArchiveService; const userData = { id: 42 } as UserData; const data = Buffer.from('a kyc document payload'); const expectedBlobName = `user/42/${FileType.IDENTIFICATION}/passport.pdf`; - const expectedHash = sha256(data).toString('hex'); beforeEach(async () => { jest.clearAllMocks(); kycFileService = createMock(); - archiveService = createMock(); (kycFileService.createKycFile as jest.Mock).mockResolvedValue({ id: 7 } as KycFile); uploadWormBlobMock.mockResolvedValue('https://storage/blob-url'); const module: TestingModule = await Test.createTestingModule({ - providers: [ - KycDocumentService, - { provide: KycFileService, useValue: kycFileService }, - { provide: ArchiveService, useValue: archiveService }, - ], + providers: [KycDocumentService, { provide: KycFileService, useValue: kycFileService }], }).compile(); service = module.get(KycDocumentService); }); - it('records the uploaded blob hash in the kyc bucket after a successful upload', async () => { + it('uploads the blob and returns the file + url', async () => { const result = await service.uploadUserFile( userData, FileType.IDENTIFICATION, @@ -66,51 +51,26 @@ describe('KycDocumentService - GeBüV hash recording', () => { true, ); - // upload happened, then recordHash with the deterministic blob name + sha256 of the data expect(uploadWormBlobMock).toHaveBeenCalledTimes(1); expect(uploadWormBlobMock.mock.calls[0][0]).toBe(expectedBlobName); - expect(archiveService.recordHash).toHaveBeenCalledTimes(1); - expect(archiveService.recordHash).toHaveBeenCalledWith('kyc', expectedBlobName, expectedHash); - expect(result).toEqual({ file: { id: 7 }, url: 'https://storage/blob-url' }); }); - it('does NOT roll back the upload when hash recording fails (best-effort side-booking)', async () => { - (archiveService.recordHash as jest.Mock).mockRejectedValue(new Error('archive db down')); - - const result = await service.uploadUserFile( - userData, - FileType.IDENTIFICATION, - 'passport.pdf', - data, - ContentType.PDF, - true, - ); - - // the recordHash failure was swallowed: the method still returns the uploaded file + url - expect(archiveService.recordHash).toHaveBeenCalledTimes(1); - expect(result).toEqual({ file: { id: 7 }, url: 'https://storage/blob-url' }); - }); - - it('rejects unsupported media types before any upload or hash recording', async () => { + it('rejects unsupported media types before any upload', async () => { await expect( service.uploadUserFile(userData, FileType.IDENTIFICATION, 'note.txt', data, 'text/plain' as ContentType, true), ).rejects.toThrow('Supported file types'); expect(uploadWormBlobMock).not.toHaveBeenCalled(); - expect(archiveService.recordHash).not.toHaveBeenCalled(); }); describe('copyFiles', () => { - it('getBlob and recordHash each copied target key with its content hash', async () => { - const copiedData = Buffer.from('copied kyc document'); + it('copies each target key from all three prefixes', async () => { const targetKey = 'user/99/Identification/passport.pdf'; - const expectedCopyHash = sha256(copiedData).toString('hex'); // three prefixes: spider, spider-org, user — only the last returns a copy copyBlobsMock.mockResolvedValueOnce([]).mockResolvedValueOnce([]).mockResolvedValueOnce([targetKey]); - getBlobMock.mockResolvedValue({ data: copiedData, contentType: 'application/pdf' }); await service.copyFiles(42, 99); @@ -118,43 +78,14 @@ describe('KycDocumentService - GeBüV hash recording', () => { expect(copyBlobsMock).toHaveBeenCalledWith('spider/42/', 'spider/99/'); expect(copyBlobsMock).toHaveBeenCalledWith('spider/42-organization/', 'spider/99-organization/'); expect(copyBlobsMock).toHaveBeenCalledWith('user/42/', 'user/99/'); - - expect(getBlobMock).toHaveBeenCalledTimes(1); - expect(getBlobMock).toHaveBeenCalledWith(targetKey); - - expect(archiveService.recordHash).toHaveBeenCalledTimes(1); - expect(archiveService.recordHash).toHaveBeenCalledWith('kyc', targetKey, expectedCopyHash); }); - it('does not call recordHash when copyBlobs returns empty for a prefix', async () => { + it('does not fail when copyBlobs returns empty for a prefix', async () => { copyBlobsMock.mockResolvedValue([]); await service.copyFiles(42, 99); expect(copyBlobsMock).toHaveBeenCalledTimes(3); - expect(getBlobMock).not.toHaveBeenCalled(); - expect(archiveService.recordHash).not.toHaveBeenCalled(); - }); - - it('does NOT abort remaining copies when recordHash fails for one key (best-effort)', async () => { - const dataA = Buffer.from('doc-a'); - const dataB = Buffer.from('doc-b'); - const keyA = 'user/99/Identification/a.pdf'; - const keyB = 'user/99/Identification/b.pdf'; - - copyBlobsMock.mockResolvedValueOnce([]).mockResolvedValueOnce([]).mockResolvedValueOnce([keyA, keyB]); - getBlobMock - .mockResolvedValueOnce({ data: dataA, contentType: 'application/pdf' }) - .mockResolvedValueOnce({ data: dataB, contentType: 'application/pdf' }); - (archiveService.recordHash as jest.Mock) - .mockRejectedValueOnce(new Error('archive db down')) - .mockResolvedValueOnce(undefined); - - await expect(service.copyFiles(42, 99)).resolves.toBeUndefined(); - - expect(archiveService.recordHash).toHaveBeenCalledTimes(2); - expect(archiveService.recordHash).toHaveBeenCalledWith('kyc', keyA, sha256(dataA).toString('hex')); - expect(archiveService.recordHash).toHaveBeenCalledWith('kyc', keyB, sha256(dataB).toString('hex')); }); }); }); diff --git a/src/subdomains/generic/kyc/services/integration/kyc-document.service.ts b/src/subdomains/generic/kyc/services/integration/kyc-document.service.ts index 9df49db9c6..8a18cbb4f7 100644 --- a/src/subdomains/generic/kyc/services/integration/kyc-document.service.ts +++ b/src/subdomains/generic/kyc/services/integration/kyc-document.service.ts @@ -1,10 +1,7 @@ import { Injectable, UnsupportedMediaTypeException } from '@nestjs/common'; import { Config } from 'src/config/config'; -import { ArchiveService } from 'src/integration/infrastructure/storage/anchoring/archive.service'; -import { sha256 } from 'src/integration/infrastructure/storage/anchoring/merkle'; import { createStorageService } from 'src/integration/infrastructure/storage/storage.factory'; import { BlobContent, StorageService } from 'src/integration/infrastructure/storage/storage.service'; -import { DfxLogger } from 'src/shared/services/dfx-logger'; import { AccountType } from 'src/subdomains/generic/user/models/user-data/account-type.enum'; import { UserData } from 'src/subdomains/generic/user/models/user-data/user-data.entity'; import { FileSubType, FileType, KycFileBlob } from '../../dto/kyc-file.dto'; @@ -18,14 +15,9 @@ const KYC_CONTAINER = 'kyc'; @Injectable() export class KycDocumentService { - private readonly logger = new DfxLogger(KycDocumentService); - private readonly storageService: StorageService; - constructor( - private readonly kycFileService: KycFileService, - private readonly archiveService: ArchiveService, - ) { + constructor(private readonly kycFileService: KycFileService) { this.storageService = createStorageService(KYC_CONTAINER); } @@ -123,17 +115,6 @@ export class KycDocumentService { const url = await this.storageService.uploadWormBlob(blobName, data, contentType, metadata); - // GeBüV anchoring (Stage 3): record the content hash of the just-uploaded KYC document - // (a retention-relevant compliance bucket) so it can later be Merkle-batched and anchored. - // This is a best-effort side-booking: the upload above has already succeeded and must not - // be rolled back if hash recording fails, so failures are logged (never silently swallowed) - // but not rethrown. - try { - await this.archiveService.recordHash(KYC_CONTAINER, blobName, sha256(data).toString('hex')); - } catch (e) { - this.logger.error(`GeBüV anchoring failed to record hash for ${KYC_CONTAINER}/${blobName}:`, e); - } - return { file, url }; } @@ -142,26 +123,12 @@ export class KycDocumentService { } async copyFiles(sourceUserDataId: number, targetUserDataId: number): Promise { - await this.copyAndAnchor(`spider/${sourceUserDataId}/`, `spider/${targetUserDataId}/`); - await this.copyAndAnchor(`spider/${sourceUserDataId}-organization/`, `spider/${targetUserDataId}-organization/`); - await this.copyAndAnchor(`user/${sourceUserDataId}/`, `user/${targetUserDataId}/`); - } - - // GeBüV anchoring (Stage 3): a copy made by copyBlobs is a fresh WORM write just like an - // upload, so it must be recorded the same way uploadFile does — otherwise a merged account's - // copied documents would never get a time anchor. Best-effort per file, same as uploadFile: - // failures are logged (never silently swallowed) but never abort the remaining copies. - private async copyAndAnchor(sourcePrefix: string, targetPrefix: string): Promise { - const targetKeys = await this.storageService.copyBlobs(sourcePrefix, targetPrefix); - - for (const targetKey of targetKeys) { - try { - const blob = await this.storageService.getBlob(targetKey); - await this.archiveService.recordHash(KYC_CONTAINER, targetKey, sha256(blob.data).toString('hex')); - } catch (e) { - this.logger.error(`GeBüV anchoring failed to record hash for ${KYC_CONTAINER}/${targetKey}:`, e); - } - } + await this.storageService.copyBlobs(`spider/${sourceUserDataId}/`, `spider/${targetUserDataId}/`); + await this.storageService.copyBlobs( + `spider/${sourceUserDataId}-organization/`, + `spider/${targetUserDataId}-organization/`, + ); + await this.storageService.copyBlobs(`user/${sourceUserDataId}/`, `user/${targetUserDataId}/`); } // --- HELPER METHODS --- // diff --git a/src/subdomains/generic/user/models/user-data/__tests__/user-data.service.spec.ts b/src/subdomains/generic/user/models/user-data/__tests__/user-data.service.spec.ts index e347b765f9..65ab18f5ae 100644 --- a/src/subdomains/generic/user/models/user-data/__tests__/user-data.service.spec.ts +++ b/src/subdomains/generic/user/models/user-data/__tests__/user-data.service.spec.ts @@ -1,7 +1,3 @@ -// Stub the heavy `opentimestamps` library (pulled in transitively via KycDocumentService -> -// ArchiveService) so its eager network/`request` deps never load at jest runtime. -jest.mock('opentimestamps', () => ({})); - import { createMock } from '@golevelup/ts-jest'; import { Test, TestingModule } from '@nestjs/testing'; import { ConflictException } from '@nestjs/common'; diff --git a/src/subdomains/supporting/fiat-output/__tests__/fiat-output-job.service.spec.ts b/src/subdomains/supporting/fiat-output/__tests__/fiat-output-job.service.spec.ts index 318a81ae72..9ac8ca8cae 100644 --- a/src/subdomains/supporting/fiat-output/__tests__/fiat-output-job.service.spec.ts +++ b/src/subdomains/supporting/fiat-output/__tests__/fiat-output-job.service.spec.ts @@ -1,7 +1,3 @@ -// Stub the heavy `opentimestamps` library (pulled in transitively via ArchiveService) so its -// eager network/`request` deps never load at jest runtime; ArchiveService is mocked in this spec. -jest.mock('opentimestamps', () => ({})); - // generateReports resolves a per-merchant EP2 container at runtime via createStorageService(); // mock the factory so the WORM sink (uploadWormBlob) is a spy and no real storage backend is touched. const ep2UploadBlobMock = jest.fn(); @@ -19,8 +15,6 @@ import { IbanService } from 'src/integration/bank/services/iban.service'; import { OlkypayService } from 'src/integration/bank/services/olkypay.service'; import { YapealService } from 'src/integration/bank/services/yapeal.service'; import { ScryptService } from 'src/integration/exchange/services/scrypt.service'; -import { ArchiveService } from 'src/integration/infrastructure/storage/anchoring/archive.service'; -import { sha256 } from 'src/integration/infrastructure/storage/anchoring/merkle'; import { createCustomAsset, createDefaultAsset } from 'src/shared/models/asset/__mocks__/asset.entity.mock'; import { AssetType } from 'src/shared/models/asset/asset.entity'; import { AssetService } from 'src/shared/models/asset/asset.service'; @@ -72,7 +66,6 @@ describe('FiatOutputJobService', () => { let virtualIbanService: VirtualIbanService; let scryptService: ScryptService; let frickPayoutService: FiatOutputFrickService; - let archiveService: ArchiveService; beforeEach(async () => { ep2UploadBlobMock.mockReset(); @@ -90,7 +83,6 @@ describe('FiatOutputJobService', () => { olkypayService = createMock(); virtualIbanService = createMock(); scryptService = createMock(); - archiveService = createMock(); jest.spyOn(processServiceModule, 'DisabledProcess').mockReturnValue(false); // Default mock: no virtual IBANs @@ -122,7 +114,6 @@ describe('FiatOutputJobService', () => { { provide: IbanService, useValue: createMock() }, { provide: VirtualIbanService, useValue: virtualIbanService }, { provide: ScryptService, useValue: scryptService }, - { provide: ArchiveService, useValue: archiveService }, TestUtil.provideConfig(), ], @@ -900,7 +891,7 @@ describe('FiatOutputJobService', () => { }); }); - describe('generateReports - GeBüV hash recording', () => { + describe('generateReports', () => { // A FiatOutput whose buyFiat resolves the container, route id and userData the method needs. // The getter chain (paymentLinkPayment.link.linkConfigObj, paymentLinksConfigObj) is stubbed // directly on plain objects so we don't have to assemble the full entity graph. @@ -923,7 +914,7 @@ describe('FiatOutputJobService', () => { (ep2ReportService.generateReport as jest.Mock).mockReturnValue(''); }); - it('uploads the report, then sets reportCreated and records the report hash', async () => { + it('uploads the report, then sets reportCreated', async () => { jest.spyOn(fiatOutputRepo, 'find').mockResolvedValue([reportableEntity()]); await service['generateReports'](); @@ -932,35 +923,15 @@ describe('FiatOutputJobService', () => { expect(ep2UploadBlobMock).toHaveBeenCalledTimes(1); expect(fileName).toMatch(/^settlement_.*_777\.ep2$/); - // reportCreated is flipped before the best-effort anchoring runs expect(fiatOutputRepo.update).toHaveBeenCalledWith(1, { reportCreated: true }); - // the recorded hash is the sha256 of the uploaded report buffer - const expectedHash = sha256(Buffer.from('')).toString('hex'); - expect(archiveService.recordHash).toHaveBeenCalledWith('ep2-merchant-bucket', fileName, expectedHash); - - // Load-bearing ordering: uploadBlob (WORM PUT) < update(reportCreated=true) < recordHash. - // reportCreated MUST be persisted before the best-effort recordHash, otherwise a recordHash - // failure would leave reportCreated=false and the next run would re-PUT the same fileName - // into the immutable WORM bucket and deadlock. Asserting the call order makes the test break - // if someone reorders recordHash ahead of the reportCreated update. + // Load-bearing ordering: uploadBlob (WORM PUT) < update(reportCreated=true). + // reportCreated MUST be persisted after a successful upload, otherwise a later failure + // would leave reportCreated=false and the next run would re-PUT the same fileName + // into the immutable WORM bucket and deadlock. const uploadOrder = ep2UploadBlobMock.mock.invocationCallOrder[0]; const updateOrder = (fiatOutputRepo.update as jest.Mock).mock.invocationCallOrder[0]; - const recordHashOrder = (archiveService.recordHash as jest.Mock).mock.invocationCallOrder[0]; expect(uploadOrder).toBeLessThan(updateOrder); - expect(updateOrder).toBeLessThan(recordHashOrder); - }); - - it('does NOT prevent reportCreated when hash recording fails (best-effort, runs after the flag)', async () => { - jest.spyOn(fiatOutputRepo, 'find').mockResolvedValue([reportableEntity()]); - (archiveService.recordHash as jest.Mock).mockRejectedValue(new Error('archive db down')); - - await service['generateReports'](); - - // upload + reportCreated still happened despite the recordHash failure - expect(ep2UploadBlobMock).toHaveBeenCalledTimes(1); - expect(fiatOutputRepo.update).toHaveBeenCalledWith(1, { reportCreated: true }); - expect(archiveService.recordHash).toHaveBeenCalledTimes(1); }); it('does not set reportCreated when the upload itself fails', async () => { @@ -970,7 +941,6 @@ describe('FiatOutputJobService', () => { await service['generateReports'](); expect(fiatOutputRepo.update).not.toHaveBeenCalled(); - expect(archiveService.recordHash).not.toHaveBeenCalled(); }); it('falls back to the sell route id for the file name when no payoutRouteId is configured', async () => { @@ -984,9 +954,6 @@ describe('FiatOutputJobService', () => { const fileName = ep2UploadBlobMock.mock.calls[0][0]; expect(fileName).toMatch(/^settlement_.*_555\.ep2$/); - - const expectedHash = sha256(Buffer.from('')).toString('hex'); - expect(archiveService.recordHash).toHaveBeenCalledWith('ep2-merchant-bucket', fileName, expectedHash); }); }); }); diff --git a/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts b/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts index 916e733f6f..c10981cd43 100644 --- a/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts +++ b/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts @@ -7,8 +7,6 @@ import { Pain001Payment } from 'src/integration/bank/services/iso20022.service'; import { OlkypayService } from 'src/integration/bank/services/olkypay.service'; import { YapealService } from 'src/integration/bank/services/yapeal.service'; import { ScryptService } from 'src/integration/exchange/services/scrypt.service'; -import { ArchiveService } from 'src/integration/infrastructure/storage/anchoring/archive.service'; -import { sha256 } from 'src/integration/infrastructure/storage/anchoring/merkle'; import { createStorageService } from 'src/integration/infrastructure/storage/storage.factory'; import { AssetType } from 'src/shared/models/asset/asset.entity'; import { AssetService } from 'src/shared/models/asset/asset.service'; @@ -60,7 +58,6 @@ export class FiatOutputJobService { private readonly frickPayoutService: FiatOutputFrickService, private readonly virtualIbanService: VirtualIbanService, private readonly scryptService: ScryptService, - private readonly archiveService: ArchiveService, ) {} @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.FIAT_OUTPUT, timeout: 1800 }) @@ -125,20 +122,9 @@ export class FiatOutputJobService { // accepts mutable GeBüV settlement records instead of throwing here. await createStorageService(container).uploadWormBlob(fileName, reportBuffer, 'text/xml'); - // Mark the report as created as soon as the WORM upload succeeded, BEFORE the best-effort - // anchoring below. A recordHash failure must not leave reportCreated=false, otherwise the - // next run would re-PUT the same fileName into the immutable WORM bucket and block forever. + // Mark the report as created as soon as the WORM upload succeeded, so a re-run never + // re-PUTs the same fileName into the immutable WORM bucket. await this.fiatOutputRepo.update(entity.id, { reportCreated: true }); - - // GeBüV anchoring (Stage 3): record the content hash of the just-uploaded EP2 settlement - // report (a retention-relevant compliance bucket) for later Merkle-batching and anchoring. - // Best-effort side-booking: the upload already succeeded, so a failure here is logged - // (never silently swallowed) but does not roll back the upload or the reportCreated flag. - try { - await this.archiveService.recordHash(container, fileName, sha256(reportBuffer).toString('hex')); - } catch (e) { - this.logger.error(`GeBüV anchoring failed to record hash for ${container}/${fileName}:`, e); - } } catch (e) { this.logger.error(`Failed to generate EP2 report for fiat output ${entity.id}:`, e); } diff --git a/src/subdomains/supporting/fiat-output/fiat-output.module.ts b/src/subdomains/supporting/fiat-output/fiat-output.module.ts index 4c866f518c..17ee531e2b 100644 --- a/src/subdomains/supporting/fiat-output/fiat-output.module.ts +++ b/src/subdomains/supporting/fiat-output/fiat-output.module.ts @@ -2,7 +2,6 @@ import { Module, forwardRef } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { BankIntegrationModule } from 'src/integration/bank/bank.module'; import { ExchangeModule } from 'src/integration/exchange/exchange.module'; -import { ArchiveModule } from 'src/integration/infrastructure/storage/anchoring/archive.module'; import { SharedModule } from 'src/shared/shared.module'; import { BuyCryptoRepository } from 'src/subdomains/core/buy-crypto/process/repositories/buy-crypto.repository'; import { LiquidityManagementModule } from 'src/subdomains/core/liquidity-management/liquidity-management.module'; @@ -29,7 +28,6 @@ import { FiatOutputJobService } from './fiat-output-job.service'; ExchangeModule, forwardRef(() => LiquidityManagementModule), LogModule, - ArchiveModule, ], controllers: [FiatOutputController], diff --git a/src/subdomains/supporting/realunit/__tests__/realunit-compliance.service.spec.ts b/src/subdomains/supporting/realunit/__tests__/realunit-compliance.service.spec.ts index b64e046c2a..0c6578944b 100644 --- a/src/subdomains/supporting/realunit/__tests__/realunit-compliance.service.spec.ts +++ b/src/subdomains/supporting/realunit/__tests__/realunit-compliance.service.spec.ts @@ -1,7 +1,3 @@ -// Stub the heavy `opentimestamps` library (pulled in transitively via KycDocumentService -> -// ArchiveService) so its eager network/`request` deps never load at jest runtime. -jest.mock('opentimestamps', () => ({})); - import { NotFoundException, ServiceUnavailableException } from '@nestjs/common'; import { createMock, DeepMocked } from '@golevelup/ts-jest'; import { Configuration, ConfigService } from 'src/config/config'; From b5da3d60abe927782c1c3b726b1df79ae5ca67f7 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:40:06 +0200 Subject: [PATCH 11/11] harden(storage): standalone provisioning, TTL WORM-verify cache, negative WORM tests - provision-bucket: read S3 config directly from process.env (drops the src/config import chain so plain ts-node resolves, fail-loud on missing vars) - s3-storage: replace process-lifetime WORM-verify cache with a 5-min TTL so a later bucket-side retention weakening is re-detected (fail-closed on drift) - tests: WORM guard rejects GOVERNANCE mode, missing default retention, and sub-floor retention years --- scripts/storage/provision-bucket.ts | 26 ++++------ .../__tests__/s3-storage.service.spec.ts | 52 +++++++++++++++++++ .../storage/s3-storage.service.ts | 17 +++--- 3 files changed, 72 insertions(+), 23 deletions(-) diff --git a/scripts/storage/provision-bucket.ts b/scripts/storage/provision-bucket.ts index 888af7c970..0dfb680a1c 100644 --- a/scripts/storage/provision-bucket.ts +++ b/scripts/storage/provision-bucket.ts @@ -18,11 +18,11 @@ * default-retention configuration is (re)applied. * * Configuration (no silent defaults for credentials — fails fast if incomplete): - * - S3 endpoint/region from Config.s3 (S3_ENDPOINT / S3_REGION — shared with the app). - * - Auth from Config.s3Admin (S3_ADMIN_ACCESS_KEY / S3_ADMIN_SECRET_KEY) — dedicated - * provisioning credentials with CreateBucket/Object-Lock rights. The app's policy- - * restricted S3_ACCESS_KEY/S3_SECRET_KEY are intentionally NOT used and NOT fallen - * back to: they cannot create buckets or apply Object Lock. + * - S3 endpoint/region from S3_ENDPOINT / S3_REGION (shared with the app). + * - Auth from S3_ADMIN_ACCESS_KEY / S3_ADMIN_SECRET_KEY — dedicated provisioning + * credentials with CreateBucket/Object-Lock rights. The app's policy-restricted + * S3_ACCESS_KEY/S3_SECRET_KEY are intentionally NOT used and NOT fallen back to: + * they cannot create buckets or apply Object Lock. * * Run with (bucket name required, retention years optional, default 11): * BUCKET=kyc RETENTION_YEARS=11 npx ts-node scripts/storage/provision-bucket.ts @@ -43,14 +43,6 @@ import * as dotenv from 'dotenv'; dotenv.config(); -// Loaded after dotenv so Config.s3 / Config.s3Admin read the populated env. `Config` (the -// module-level `export let Config`) is only ever set as a side effect of `new ConfigService()`, -// which this standalone script never instantiates — importing the raw `Config` binding would be -// permanently `undefined` here. Build a local instance directly via `GetConfig()` instead. -import { GetConfig } from '../../src/config/config'; - -const Config = GetConfig(); - import { GEBUEV_RETENTION_FLOOR_YEARS } from '../../src/integration/infrastructure/storage/worm-retention.const'; function getBucketName(): string { @@ -78,8 +70,10 @@ export function getRetentionYears(): number { } function buildClient(): S3Client { - const { endpoint, region } = Config.s3; - const { accessKey, secretKey } = Config.s3Admin; + const endpoint = process.env.S3_ENDPOINT; + const region = process.env.S3_REGION; + const accessKey = process.env.S3_ADMIN_ACCESS_KEY; + const secretKey = process.env.S3_ADMIN_SECRET_KEY; if (!endpoint || !region || !accessKey || !secretKey) throw new Error( 'Incomplete S3 admin config: S3_ENDPOINT, S3_REGION, S3_ADMIN_ACCESS_KEY and S3_ADMIN_SECRET_KEY are required', @@ -165,7 +159,7 @@ async function main(): Promise { const years = getRetentionYears(); const client = buildClient(); - console.log(`Provisioning WORM bucket "${bucket}" (endpoint ${Config.s3.endpoint})`); + console.log(`Provisioning WORM bucket "${bucket}" (endpoint ${process.env.S3_ENDPOINT})`); if (await bucketExists(client, bucket)) { console.log(' Bucket: already exists -> reconciling lock configuration only'); diff --git a/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts b/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts index 83e17434ff..37cefb1216 100644 --- a/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts +++ b/src/integration/infrastructure/storage/__tests__/s3-storage.service.spec.ts @@ -11,6 +11,7 @@ import { Test } from '@nestjs/testing'; import { mockClient } from 'aws-sdk-client-mock'; import { TestUtil } from 'src/shared/utils/test.util'; import { S3StorageService } from '../s3-storage.service'; +import { GEBUEV_RETENTION_FLOOR_YEARS } from '../worm-retention.const'; const validS3 = { endpoint: 'https://s3.test.local', @@ -279,6 +280,57 @@ describe('S3StorageService', () => { expect(s3Mock.commandCalls(GetObjectLockConfigurationCommand)).toHaveLength(1); expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(3); }); + + it('fails closed when the default retention mode is GOVERNANCE instead of COMPLIANCE', async () => { + const container = 'ep2-worm-governance'; + s3Mock.on(GetObjectLockConfigurationCommand, { Bucket: container }).resolves({ + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { DefaultRetention: { Mode: 'GOVERNANCE', Years: 11 } }, + }, + }); + s3Mock.on(PutObjectCommand).resolves({}); + + await expect( + new S3StorageService(container).uploadWormBlob('settlement.ep2', Buffer.from(''), 'text/xml'), + ).rejects.toThrow('Object Lock is not enabled'); + + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0); + }); + + it('fails closed when Object Lock is enabled but no default retention rule is configured', async () => { + const container = 'ep2-worm-no-default-retention'; + s3Mock.on(GetObjectLockConfigurationCommand, { Bucket: container }).resolves({ + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: {}, + }, + }); + s3Mock.on(PutObjectCommand).resolves({}); + + await expect( + new S3StorageService(container).uploadWormBlob('settlement.ep2', Buffer.from(''), 'text/xml'), + ).rejects.toThrow('Object Lock is not enabled'); + + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0); + }); + + it('fails closed when the COMPLIANCE retention Years is below the GeBüV floor', async () => { + const container = 'ep2-worm-under-floor'; + s3Mock.on(GetObjectLockConfigurationCommand, { Bucket: container }).resolves({ + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { DefaultRetention: { Mode: 'COMPLIANCE', Years: GEBUEV_RETENTION_FLOOR_YEARS - 1 } }, + }, + }); + s3Mock.on(PutObjectCommand).resolves({}); + + await expect( + new S3StorageService(container).uploadWormBlob('settlement.ep2', Buffer.from(''), 'text/xml'), + ).rejects.toThrow('Object Lock is not enabled'); + + expect(s3Mock.commandCalls(PutObjectCommand)).toHaveLength(0); + }); }); describe('copyBlobs', () => { diff --git a/src/integration/infrastructure/storage/s3-storage.service.ts b/src/integration/infrastructure/storage/s3-storage.service.ts index 9c498b425c..9a9cdd9b86 100644 --- a/src/integration/infrastructure/storage/s3-storage.service.ts +++ b/src/integration/infrastructure/storage/s3-storage.service.ts @@ -25,10 +25,12 @@ import { GEBUEV_RETENTION_FLOOR_YEARS } from './worm-retention.const'; * intentionally not applied per request here. */ export class S3StorageService extends StorageService { - // Containers verified to have Object Lock enabled. Static so the check is amortized across the - // short-lived per-container instances the EP2 sink creates (createStorageService per report), - // keeping the GetObjectLockConfiguration probe off the per-PUT hot path (verify once per bucket). - private static readonly objectLockVerified = new Set(); + // Containers verified to have Object Lock enabled, with timestamp of last successful probe. + // Static so the check is amortized across the short-lived per-container instances the EP2 sink + // creates (createStorageService per report). Re-verified after TTL so a later bucket-side + // retention weakening is not masked for the rest of the process lifetime. + private static readonly objectLockVerifiedAt = new Map(); + private static readonly OBJECT_LOCK_VERIFY_TTL_MS = 5 * 60 * 1000; private readonly client: S3Client; @@ -73,14 +75,15 @@ export class S3StorageService extends StorageService { // WORM sink (GeBüV): fail closed unless the target bucket enforces Object Lock. Object Lock // cannot be retro-fitted onto an existing bucket, so writing a compliance record into a // non-locked (or unverifiable) bucket would leave it mutable/deletable forever — we refuse - // rather than under-protect. Verified once per container; subsequent PUTs skip the probe. + // rather than under-protect. Verified per container with a short TTL; hot-path PUTs skip the probe. async uploadWormBlob(name: string, data: Buffer, type: string, metadata?: Record): Promise { await this.assertObjectLockEnabled(); return this.uploadBlob(name, data, type, metadata); } private async assertObjectLockEnabled(): Promise { - if (S3StorageService.objectLockVerified.has(this.container)) return; + const verifiedAt = S3StorageService.objectLockVerifiedAt.get(this.container); + if (verifiedAt != null && Date.now() - verifiedAt < S3StorageService.OBJECT_LOCK_VERIFY_TTL_MS) return; let cfg: | { ObjectLockEnabled?: string; Rule?: { DefaultRetention?: { Mode?: string; Years?: number } } } @@ -114,7 +117,7 @@ export class S3StorageService extends StorageService { `Provision it first (scripts/storage/provision-bucket.ts).`, ); - S3StorageService.objectLockVerified.add(this.container); + S3StorageService.objectLockVerifiedAt.set(this.container, Date.now()); } async copyBlobs(sourcePrefix: string, targetPrefix: string): Promise {