Add CRC-64/NVME transactional checksum support for StageBlock

package.json

@@ -317,6 +317,7 @@
     "test:blob:in-memory": "npm run lint && cross-env AZURITE_TEST_INMEMORYPERSISTENCE=1 NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --require ts-node/register --no-timeouts --grep @loki --recursive --exit tests/blob/*.test.ts tests/blob/**/*.test.ts",
     "test:blob:sql": "npm run lint && cross-env cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 AZURITE_TEST_DB=mysql://root:my-secret-pw@127.0.0.1:3306/azurite_blob_test mocha --require ts-node/register --no-timeouts --grep @sql --recursive --exit tests/blob/*.test.ts tests/blob/**/*.test.ts",
     "test:blob:sql:ci": "npm run lint && cross-env cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 AZURITE_TEST_DB=mysql://root:my-secret-pw@127.0.0.1:13306/azurite_blob_test mocha --require ts-node/register --no-timeouts --grep @sql --recursive --exit tests/blob/*.test.ts tests/blob/**/*.test.ts",
+    "test:blob:live": "cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --require ts-node/register --no-timeouts --recursive --exit tests/blob/*.test.ts tests/blob/**/*.test.ts",
     "test:queue": "npm run lint && cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --require ts-node/register --no-timeouts --recursive --exit tests/queue/*.test.ts tests/queue/**/*.test.ts",
     "test:queue:in-memory": "npm run lint && cross-env AZURITE_TEST_INMEMORYPERSISTENCE=1 NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --require ts-node/register --no-timeouts --recursive --exit tests/queue/*.test.ts tests/queue/**/*.test.ts",
     "test:table": "npm run lint && cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --require ts-node/register --no-timeouts --recursive --exit tests/table/**/*.test.ts",

src/blob/errors/StorageErrorFactory.ts

@@ -190,6 +190,43 @@ export default class StorageErrorFactory {
     );
   }
 
+  public static getCrc64Mismatch(
+    contextID: string = DefaultID,
+    userSpecifiedCrc64: string,
+    serverCalculatedCrc64: string
+  ): StorageError {
+    return new StorageError(
+      400,
+      "Crc64Mismatch",
+      "The CRC64 value specified in the request did not match with the CRC64 value calculated by the server.",
+      contextID,
+      {
+        UserSpecifiedCrc64: userSpecifiedCrc64,
+        ServerCalculatedCrc64: serverCalculatedCrc64
+      }
+    );
+  }
+
+  public static getBothCrc64AndMd5HeaderPresent(
+    contextID: string = DefaultID
+  ): StorageError {
+    return new StorageError(
+      400,
+      "BothCrc64AndMd5HeaderPresent",
+      "Both x-ms-content-crc64 header and Content-MD5 header are present.",
+      contextID
+    );
+  }
+
+  public static getInvalidMd5(contextID: string = DefaultID): StorageError {
+    return new StorageError(
+      400,
+      "InvalidMd5",
+      "The MD5 value specified in the request is invalid. The MD5 value must be 128 bits and Base64-encoded.",
+      contextID
+    );
+  }
+
   public static getInvalidPageRange(contextID: string): StorageError {
     return new StorageError(
       416,

src/blob/handlers/AppendBlobHandler.ts

@@ -1,5 +1,7 @@
-import { convertRawHeadersToMetadata } from "../../common/utils/utils";
-import { getMD5FromStream, newEtag } from "../../common/utils/utils";
+import {
+  convertRawHeadersToMetadata,
+  newEtag
+} from "../../common/utils/utils";
 import BlobStorageContext from "../context/BlobStorageContext";
 import NotImplementedError from "../errors/NotImplementedError";
 import StorageErrorFactory from "../errors/StorageErrorFactory";
@@ -13,7 +15,7 @@ import {
   MAX_APPEND_BLOB_BLOCK_COUNT,
   MAX_APPEND_BLOB_BLOCK_SIZE
 } from "../utils/constants";
-import { getTagsFromString } from "../utils/utils";
+import { computeAndValidateTransactionalChecksums, getTagsFromString } from "../utils/utils";
 import BaseHandler from "./BaseHandler";
 
 export default class AppendBlobHandler extends BaseHandler
@@ -149,38 +151,28 @@ export default class AppendBlobHandler extends BaseHandler
       );
     }
 
-    // MD5
+    // MD5 and/or CRC64 transactional integrity validation
     const contentMD5 = blobCtx.request!.getHeader(HeaderConstants.CONTENT_MD5);
+    const contentCRC64 = options.transactionalContentCrc64;
     let contentMD5Buffer;
-    let contentMD5String;
 
     if (contentMD5 !== undefined) {
       contentMD5Buffer =
         typeof contentMD5 === "string"
           ? Buffer.from(contentMD5, "base64")
           : contentMD5;
-      contentMD5String =
-        typeof contentMD5 === "string"
-          ? contentMD5
-          : contentMD5Buffer.toString("base64");
+    }
 
-      const stream = await this.extentStore.readExtent(
-        extent,
-        blobCtx.contextId
+    // Per the Append Block REST contract, the service always computes a CRC64
+    // of the appended block and returns it in x-ms-content-crc64.
+    const stream = await this.extentStore.readExtent(extent, blobCtx.contextId);
+    const { crc64: calculatedCRC64 } =
+      await computeAndValidateTransactionalChecksums(
+        stream,
+        { md5: contentMD5, crc64: contentCRC64 },
+        context.contextId,
+        { crc64: true }
       );
-      const calculatedContentMD5Buffer = await getMD5FromStream(stream);
-      const calculatedContentMD5String = Buffer.from(
-        calculatedContentMD5Buffer
-      ).toString("base64");
-
-      if (contentMD5String !== calculatedContentMD5String) {
-        throw StorageErrorFactory.getMd5Mismatch(
-          context.contextId,
-          contentMD5String,
-          calculatedContentMD5String
-        );
-      }
-    }
 
     const originOffset = blob.properties.contentLength;
 
@@ -206,7 +198,7 @@ export default class AppendBlobHandler extends BaseHandler
       eTag: properties.etag,
       lastModified: properties.lastModified,
       contentMD5: contentMD5Buffer,
-      xMsContentCrc64: undefined,
+      xMsContentCrc64: calculatedCRC64,
       clientRequestId: options.requestId,
       version: BLOB_API_VERSION,
       date,

src/blob/handlers/BlobHandler.ts

@@ -907,6 +907,9 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler {
       date: context.startTime,
       copyId: res.copyId,
       copyStatus,
+      // Per the Copy Blob From URL REST contract, echo the source's Content-MD5
+      // back to the client when it was supplied in x-ms-source-content-md5.
+      contentMD5: options.sourceContentMD5,
       clientRequestId: options.requestId
     };
 

src/blob/handlers/BlockBlobHandler.ts

@@ -1,6 +1,5 @@
-import { convertRawHeadersToMetadata } from "../../common/utils/utils";
 import {
-  getMD5FromStream,
+  convertRawHeadersToMetadata,
   getMD5FromString,
   newEtag
 } from "../../common/utils/utils";
@@ -14,7 +13,10 @@ import { parseXML } from "../generated/utils/xml";
 import { BlobModel, BlockModel } from "../persistence/IBlobMetadataStore";
 import { BLOB_API_VERSION } from "../utils/constants";
 import BaseHandler from "./BaseHandler";
-import { getTagsFromString } from "../utils/utils";
+import {
+  computeAndValidateTransactionalChecksums,
+  getTagsFromString
+} from "../utils/utils";
 
 /**
  * BlobHandler handles Azure Storage BlockBlob related requests.
@@ -45,11 +47,19 @@ export default class BlockBlobHandler
       options.blobHTTPHeaders.blobContentType ||
       context.request!.getHeader("content-type") ||
       "application/octet-stream";
-    const contentMD5 = context.request!.getHeader("content-md5")
-      || context.request!.getHeader("x-ms-blob-content-md5")
-      ? options.blobHTTPHeaders.blobContentMD5 ||
-      context.request!.getHeader("content-md5")
-      : undefined;
+
+    // Per the Put Blob REST contract, x-ms-blob-content-md5 takes precedence
+    // over Content-MD5 for transit integrity verification on BlockBlob.
+    // Verified live. Prefer the SDK-parsed blobContentMD5 option; fall back
+    // to the raw x-ms-blob-content-md5 header (for clients that inject it
+    // directly without going through the SDK option); finally fall back to
+    // Content-MD5. Malformed values are rejected as InvalidMd5 by the
+    // unified validator below (matches real Azure for all three sources).
+    const contentMD5 =
+      options.blobHTTPHeaders.blobContentMD5
+      ?? context.request!.getHeader("x-ms-blob-content-md5")
+      ?? context.request!.getHeader("content-md5");
+    const contentCRC64 = options.transactionalContentCrc64;
 
     await this.metadataStore.checkContainerExist(
       context,
@@ -68,32 +78,19 @@ export default class BlockBlobHandler
       );
     }
 
-    // Calculate MD5 for validation
+    // MD5 is always needed (persisted as the blob's contentMD5 property);
+    // CRC64 is computed in the same pass only when the client supplied one.
     const stream = await this.extentStore.readExtent(
       persistency,
       context.contextId
     );
-    const calculatedContentMD5 = await getMD5FromStream(stream);
-    if (contentMD5 !== undefined) {
-      if (typeof contentMD5 === "string") {
-        const calculatedContentMD5String = Buffer.from(
-          calculatedContentMD5
-        ).toString("base64");
-        if (contentMD5 !== calculatedContentMD5String) {
-          throw StorageErrorFactory.getInvalidOperation(
-            context.contextId!,
-            "Provided contentMD5 doesn't match."
-          );
-        }
-      } else {
-        if (!Buffer.from(contentMD5).equals(calculatedContentMD5)) {
-          throw StorageErrorFactory.getInvalidOperation(
-            context.contextId!,
-            "Provided contentMD5 doesn't match."
-          );
-        }
-      }
-    }
+    const { md5: calculatedContentMD5 } =
+      await computeAndValidateTransactionalChecksums(
+        stream,
+        { md5: contentMD5, crc64: contentCRC64 },
+        context.contextId,
+        { md5: true }
+      );
 
     const blob: BlobModel = {
       deleted: false,
@@ -179,14 +176,15 @@ export default class BlockBlobHandler
     const blobName = blobCtx.blob!;
     const date = blobCtx.startTime!;
 
-    // stageBlock operation doesn't have blobHTTPHeaders
+    // stageBlock operation doesn't accept blob property headers per the
+    // Put Block REST contract: only Content-MD5 and x-ms-content-crc64 are
+    // honored. Verified live: real Azure silently ignores x-ms-blob-content-md5
+    // here (even malformed values), so don't use it as a fallback source.
     // https://learn.microsoft.com/en-us/rest/api/storageservices/put-block
-    // options.blobHTTPHeaders = options.blobHTTPHeaders || {};
-    const contentMD5 = context.request!.getHeader("content-md5")
-      || context.request!.getHeader("x-ms-blob-content-md5")
-      ? options.transactionalContentMD5 ||
-      context.request!.getHeader("content-md5")
-      : undefined;
+    const contentMD5 =
+      options.transactionalContentMD5 ||
+      context.request!.getHeader("content-md5");
+    const contentCRC64 = options.transactionalContentCrc64;
 
     this.validateBlockId(blockId, blobCtx);
 
@@ -208,32 +206,22 @@ export default class BlockBlobHandler
       );
     }
 
-    // Calculate MD5 for validation
+    // Per the Put Block REST contract, the service computes a CRC64 of the
+    // staged block and echoes it back in x-ms-content-crc64 unless the client
+    // supplied a Content-MD5 (Azure rejects supplying both). Compute CRC64
+    // whenever no MD5 was supplied, regardless of whether the client supplied
+    // a CRC64 themselves.
     const stream = await this.extentStore.readExtent(
       persistency,
       context.contextId
     );
-    const calculatedContentMD5 = await getMD5FromStream(stream);
-    if (contentMD5 !== undefined) {
-      if (typeof contentMD5 === "string") {
-        const calculatedContentMD5String = Buffer.from(
-          calculatedContentMD5
-        ).toString("base64");
-        if (contentMD5 !== calculatedContentMD5String) {
-          throw StorageErrorFactory.getInvalidOperation(
-            context.contextId!,
-            "Provided contentMD5 doesn't match."
-          );
-        }
-      } else {
-        if (!Buffer.from(contentMD5).equals(calculatedContentMD5)) {
-          throw StorageErrorFactory.getInvalidOperation(
-            context.contextId!,
-            "Provided contentMD5 doesn't match."
-          );
-        }
-      }
-    }
+    const { crc64: calculatedCRC64 } =
+      await computeAndValidateTransactionalChecksums(
+        stream,
+        { md5: contentMD5, crc64: contentCRC64 },
+        context.contextId,
+        { crc64: contentMD5 === undefined }
+      );
 
     const block: BlockModel = {
       accountName,
@@ -255,6 +243,7 @@ export default class BlockBlobHandler
     const response: Models.BlockBlobStageBlockResponse = {
       statusCode: 201,
       contentMD5: undefined, // TODO: Block content MD5
+      xMsContentCrc64: calculatedCRC64,
       requestId: blobCtx.contextId,
       version: BLOB_API_VERSION,
       date,

src/blob/handlers/PageBlobHandler.ts

@@ -12,8 +12,12 @@ import BlobWriteLeaseValidator from "../lease/BlobWriteLeaseValidator";
 import IBlobMetadataStore, {
   BlobModel
 } from "../persistence/IBlobMetadataStore";
-import { BLOB_API_VERSION } from "../utils/constants";
-import { deserializePageBlobRangeHeader, getTagsFromString } from "../utils/utils";
+import { BLOB_API_VERSION, HeaderConstants } from "../utils/constants";
+import {
+  computeAndValidateTransactionalChecksums,
+  deserializePageBlobRangeHeader,
+  getTagsFromString
+} from "../utils/utils";
 import BaseHandler from "./BaseHandler";
 import IPageBlobRangesManager from "./IPageBlobRangesManager";
 
@@ -236,6 +240,19 @@ export default class PageBlobHandler extends BaseHandler
       );
     }
 
+    // Transactional integrity validation. Real Azure always returns a
+    // server-computed x-ms-content-crc64 on Put Page; force CRC64 always.
+    const contentMD5 = blobCtx.request!.getHeader(HeaderConstants.CONTENT_MD5);
+    const contentCRC64 = options.transactionalContentCrc64;
+    const stream = await this.extentStore.readExtent(persistency, blobCtx.contextId);
+    const { crc64: calculatedCRC64 } =
+      await computeAndValidateTransactionalChecksums(
+        stream,
+        { md5: contentMD5, crc64: contentCRC64 },
+        context.contextId,
+        { crc64: true }
+      );
+
     const res = await this.metadataStore.uploadPages(
       context,
       blob,
@@ -251,7 +268,12 @@ export default class PageBlobHandler extends BaseHandler
       statusCode: 201,
       eTag: res.etag,
       lastModified: date,
-      contentMD5: undefined, // TODO
+      contentMD5: contentMD5 === undefined
+        ? undefined
+        : typeof contentMD5 === "string"
+        ? Buffer.from(contentMD5, "base64")
+        : contentMD5,
+      xMsContentCrc64: calculatedCRC64,
       blobSequenceNumber: res.blobSequenceNumber,
       requestId: blobCtx.contextId,
       version: BLOB_API_VERSION,

src/blob/middlewares/StrictModelMiddlewareFactory.ts

@@ -20,7 +20,6 @@ export const UnsupportedHeadersBlocker: StrictModelRequestValidator = async (
   logger: ILogger
 ): Promise<void> => {
   const UnsupportedHeaderKeys = [
-    HeaderConstants.X_MS_CONTENT_CRC64,
     HeaderConstants.X_MS_RANGE_GET_CONTENT_CRC64,
     HeaderConstants.X_MS_ENCRYPTION_KEY,
     HeaderConstants.X_MS_ENCRYPTION_KEY_SHA256,