From c71ddc68b54baa45e9f2c8368ee0e8152a6d8a80 Mon Sep 17 00:00:00 2001 From: Kun Lai Date: Wed, 20 May 2026 13:07:43 +0800 Subject: [PATCH] fix(metadata): use canonical JSON library for cross-language hash compatibility Rust and Go FlatBuffers libraries produce different binary output for the same data (vtable field ordering), causing calculate_metadata_hash to return different SHA-256 hashes across languages. Replace manual JSON construction with canonical JSON libraries that automatically sort object keys per RFC 8785: - Rust: canon-json crate (serde_json Formatter) - Go: github.com/gibson042/canonicaljson-go Add root hash fixture and cross-language verification: - gen_fixture.sh and Makefile now save root hash alongside the .fb fixture - New test TestInterop_CalculateMetadataHash_CrossLanguage verifies Go's CalculateMetadataHash matches the committed Rust-generated hash - CI now checks both .fb binary and root hash match when regenerating fixture Remove unused FlatBuffers files: - Delete metadata_hash.fbs and metadata_hash_generated.rs (Rust) - Delete metadata_hash_generated.go (Go) Update CLAUDE.md: - Add documentation sync requirement - Add pre-PR test checks (only verity tests for verity changes) --- .github/workflows/test.yml | 10 +- CLAUDE.md | 26 ++ Cargo.lock | 13 + Makefile | 9 +- cryptpilot-verity/Cargo.toml | 1 + cryptpilot-verity/build.rs | 15 +- .../src/metadata/metadata_hash.fbs | 19 -- .../src/metadata/metadata_hash_generated.rs | 299 ------------------ cryptpilot-verity/src/metadata/mod.rs | 146 +++++---- verity-go/go.mod | 2 + verity-go/go.sum | 2 + verity-go/metadata/gen_fixture.sh | 5 +- .../generated/metadata_hash_generated.go | 143 --------- verity-go/metadata/interop_test.go | 36 +++ verity-go/metadata/metadata_hash.fbs | 1 - verity-go/metadata/metadata_hash.go | 62 ++-- .../metadata/testdata/rust.metadata.fb.hash | 1 + 17 files changed, 224 insertions(+), 566 deletions(-) delete mode 100644 cryptpilot-verity/src/metadata/metadata_hash.fbs delete mode 100644 cryptpilot-verity/src/metadata/metadata_hash_generated.rs delete mode 100644 verity-go/metadata/generated/metadata_hash_generated.go delete mode 120000 verity-go/metadata/metadata_hash.fbs create mode 100644 verity-go/metadata/testdata/rust.metadata.fb.hash diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ae6b223..6296265 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -87,9 +87,17 @@ jobs: - name: Verify interop fixture is up to date run: | (cd verity-core && python3 make_testfiles.py) - cargo run -p cryptpilot-verity -- format verity-core/testfiles --hash-output - --label env=prod --force + HASH=$(cargo run -q -p cryptpilot-verity -- format verity-core/testfiles --hash-output - --label env=prod --force) diff verity-core/testfiles/cryptpilot-verity.metadata.fb verity-go/metadata/testdata/rust.metadata.fb \ || { echo "ERROR: interop fixture is stale. Run verity-go/metadata/gen_fixture.sh to regenerate."; exit 1; } + EXPECTED_HASH=$(cat verity-go/metadata/testdata/rust.metadata.fb.hash | tr -d '[:space:]') + if [ "$HASH" != "$EXPECTED_HASH" ]; then + echo "ERROR: root hash mismatch" + echo " expected: $EXPECTED_HASH" + echo " got: $HASH" + exit 1 + fi + echo "Interop fixture and root hash are up to date" - name: Run Go tests run: | cd verity-go diff --git a/CLAUDE.md b/CLAUDE.md index eb4cd47..eccf43a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,6 +4,10 @@ When modifying the Rust `cryptpilot-verity`, `verity-core`, or `verity-fuse` code, always evaluate whether the corresponding Go library (`verity-go/`) needs the same change. If the change affects core algorithms (hash computation, merkle tree, descriptor format) or metadata structures (FlatBuffers schema, serialization), apply the equivalent change to the Go code in the same commit. +## Documentation Sync + +When creating or modifying features, commands, or interfaces, always evaluate whether the corresponding documentation (README.md, CLAUDE.md, or other .md files under the project) needs to be updated. If the change introduces new commands, modifies existing behavior, adds configuration options, or changes usage examples, update the relevant documentation in the same commit. + ## Excluded Paths Never commit files under `docs/superpowers/` or `.claude/` to git. These are Claude session artifacts and should be kept local only. Add them to `.gitignore` if not already present. @@ -40,6 +44,28 @@ Fix any errors or warnings reported before proceeding with the commit. - Run `make test` in `cryptpilot-verity/` to execute the full integration test suite (format, dump, verify, open/FUSE mount, tamper detection, close). - Run `cargo test -p verity-fuse -p verity-core` for unit tests (requires `cd verity-core && python3 make_testfiles.py` first). +- Run Go tests: `cd verity-go && go test -race -v ./...` + +## Pre-Push / Pre-PR Checks + +Before pushing or creating a pull request, always run the relevant checks and ensure they pass. + +**Always run (regardless of what changed):** +```bash +cargo fmt --check +cargo build # or `make clippy` / `cargo clippy` if lints are relevant +``` + +**When modifying verity-related code** (`cryptpilot-verity`, `verity-core`, `verity-fuse`, or `verity-go`): +```bash +# Rust verity tests +cargo test -p cryptpilot-verity -p verity-core -p verity-fuse + +# Go verity tests +cd verity-go && go build ./... && go test -race -v ./... +``` + +For changes outside the verity subsystem, run the tests relevant to the affected packages only. If system dependencies are missing (e.g., `libcryptsetup`), the CI pipeline serves as the authoritative check, but `cargo fmt --check` must always pass locally. ## FUSE Dependency diff --git a/Cargo.lock b/Cargo.lock index 950e592..eb0d1fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -514,6 +514,18 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +[[package]] +name = "canon-json" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5ae9f90437d2e2efba2a6c75b8279aa6b8f2f4017e0a4aeb64a76cd9d3a2bab" +dependencies = [ + "serde", + "serde_derive", + "serde_json", + "thiserror 2.0.12", +] + [[package]] name = "cap-primitives" version = "4.0.0" @@ -1066,6 +1078,7 @@ dependencies = [ "async-trait", "async-walkdir", "base64 0.22.1", + "canon-json", "clap", "flatbuffers", "flatc", diff --git a/Makefile b/Makefile index 5a8ac34..118478f 100644 --- a/Makefile +++ b/Makefile @@ -357,7 +357,6 @@ docker-build-all: docker-build .PHONY: check-fbs check-fbs: @test -L verity-go/metadata/metadata.fbs || { echo "ERROR: metadata.fbs should be a symlink to Rust source"; exit 1; } - @test -L verity-go/metadata/metadata_hash.fbs || { echo "ERROR: metadata_hash.fbs should be a symlink to Rust source"; exit 1; } @echo "FlatBuffers schemas are shared via symlink." .PHONY: go-test @@ -372,7 +371,9 @@ check-all: clippy go-test check-fbs gen-interop-fixture: @echo "=== Generating interop fixture ===" cd verity-core && python3 make_testfiles.py - cargo run -p cryptpilot-verity -- format verity-core/testfiles --hash-output - --label env=prod --force - cp verity-core/testfiles/cryptpilot-verity.metadata.fb verity-go/metadata/testdata/rust.metadata.fb - @echo "Fixture updated: verity-go/metadata/testdata/rust.metadata.fb" + @HASH=$$(cargo run -q -p cryptpilot-verity -- format verity-core/testfiles --hash-output - --label env=prod --force); \ + cp verity-core/testfiles/cryptpilot-verity.metadata.fb verity-go/metadata/testdata/rust.metadata.fb; \ + echo "$$HASH" > verity-go/metadata/testdata/rust.metadata.fb.hash; \ + echo "Fixture updated: verity-go/metadata/testdata/rust.metadata.fb"; \ + echo "Root hash: $$HASH" diff --git a/cryptpilot-verity/Cargo.toml b/cryptpilot-verity/Cargo.toml index 3bd9111..11adfd5 100644 --- a/cryptpilot-verity/Cargo.toml +++ b/cryptpilot-verity/Cargo.toml @@ -7,6 +7,7 @@ version.workspace = true [dependencies] anyhow = {workspace = true} base64 = {workspace = true} +canon-json = "0.2.1" async-trait = {workspace = true} async-walkdir = {workspace = true} clap = {workspace = true} diff --git a/cryptpilot-verity/build.rs b/cryptpilot-verity/build.rs index 7830e8e..f1c2308 100644 --- a/cryptpilot-verity/build.rs +++ b/cryptpilot-verity/build.rs @@ -3,12 +3,10 @@ use std::path::Path; fn main() -> shadow_rs::SdResult<()> { shadow_rs::new()?; - // Compile FlatBuffers schemas + // Compile FlatBuffers schema let metadata_schema = Path::new("src/metadata/metadata.fbs"); - let hash_schema = Path::new("src/metadata/metadata_hash.fbs"); println!("cargo:rerun-if-changed={}", metadata_schema.display()); - println!("cargo:rerun-if-changed={}", hash_schema.display()); // Get flatc binary path from flatc crate let flatc_path = flatc::flatc(); @@ -17,7 +15,7 @@ fn main() -> shadow_rs::SdResult<()> { // First check with have good `flatc` flatc_cmd.check()?; - // Compile main metadata schema + // Compile schema flatc_cmd .run(flatc_rust::Args { inputs: &[metadata_schema], @@ -26,14 +24,5 @@ fn main() -> shadow_rs::SdResult<()> { }) .expect("Failed to compile metadata.fbs"); - // Compile hash schema - flatc_cmd - .run(flatc_rust::Args { - inputs: &[hash_schema], - out_dir: Path::new("src/metadata/"), - ..Default::default() - }) - .expect("Failed to compile metadata_hash.fbs"); - Ok(()) } diff --git a/cryptpilot-verity/src/metadata/metadata_hash.fbs b/cryptpilot-verity/src/metadata/metadata_hash.fbs deleted file mode 100644 index daf678c..0000000 --- a/cryptpilot-verity/src/metadata/metadata_hash.fbs +++ /dev/null @@ -1,19 +0,0 @@ -// FlatBuffers schema for metadata hash calculation -// This defines a minimal structure containing only the fields -// needed for hash calculation, avoiding redundancy. - -namespace cryptpilot.verity.hash; - -// File entry for hash calculation (only essential fields) -table FileHashEntry { - path: string; // relative path of the file - descriptor_hash: string; // hex-encoded descriptor hash -} - -// Metadata structure for hash calculation -// This contains only the minimal information needed to verify integrity -table MetadataHash { - files: [FileHashEntry]; // sorted by path for deterministic hash -} - -root_type MetadataHash; diff --git a/cryptpilot-verity/src/metadata/metadata_hash_generated.rs b/cryptpilot-verity/src/metadata/metadata_hash_generated.rs deleted file mode 100644 index 2d767e0..0000000 --- a/cryptpilot-verity/src/metadata/metadata_hash_generated.rs +++ /dev/null @@ -1,299 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify -// @generated -extern crate alloc; - - -#[allow(unused_imports, dead_code)] -pub mod cryptpilot { - -#[allow(unused_imports, dead_code)] -pub mod verity { - -#[allow(unused_imports, dead_code)] -pub mod hash { - - -pub enum FileHashEntryOffset {} -#[derive(Copy, Clone, PartialEq)] - -pub struct FileHashEntry<'a> { - pub _tab: ::flatbuffers::Table<'a>, -} - -impl<'a> ::flatbuffers::Follow<'a> for FileHashEntry<'a> { - type Inner = FileHashEntry<'a>; - #[inline] - unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { - Self { _tab: unsafe { ::flatbuffers::Table::new(buf, loc) } } - } -} - -impl<'a> FileHashEntry<'a> { - pub const VT_PATH: ::flatbuffers::VOffsetT = 4; - pub const VT_DESCRIPTOR_HASH: ::flatbuffers::VOffsetT = 6; - - #[inline] - pub unsafe fn init_from_table(table: ::flatbuffers::Table<'a>) -> Self { - FileHashEntry { _tab: table } - } - #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: ::flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut ::flatbuffers::FlatBufferBuilder<'bldr, A>, - args: &'args FileHashEntryArgs<'args> - ) -> ::flatbuffers::WIPOffset> { - let mut builder = FileHashEntryBuilder::new(_fbb); - if let Some(x) = args.descriptor_hash { builder.add_descriptor_hash(x); } - if let Some(x) = args.path { builder.add_path(x); } - builder.finish() - } - - - #[inline] - pub fn path(&self) -> Option<&'a str> { - // Safety: - // Created from valid Table for this object - // which contains a valid value in this slot - unsafe { self._tab.get::<::flatbuffers::ForwardsUOffset<&str>>(FileHashEntry::VT_PATH, None)} - } - #[inline] - pub fn descriptor_hash(&self) -> Option<&'a str> { - // Safety: - // Created from valid Table for this object - // which contains a valid value in this slot - unsafe { self._tab.get::<::flatbuffers::ForwardsUOffset<&str>>(FileHashEntry::VT_DESCRIPTOR_HASH, None)} - } -} - -impl ::flatbuffers::Verifiable for FileHashEntry<'_> { - #[inline] - fn run_verifier( - v: &mut ::flatbuffers::Verifier, pos: usize - ) -> Result<(), ::flatbuffers::InvalidFlatbuffer> { - v.visit_table(pos)? - .visit_field::<::flatbuffers::ForwardsUOffset<&str>>("path", Self::VT_PATH, false)? - .visit_field::<::flatbuffers::ForwardsUOffset<&str>>("descriptor_hash", Self::VT_DESCRIPTOR_HASH, false)? - .finish(); - Ok(()) - } -} -pub struct FileHashEntryArgs<'a> { - pub path: Option<::flatbuffers::WIPOffset<&'a str>>, - pub descriptor_hash: Option<::flatbuffers::WIPOffset<&'a str>>, -} -impl<'a> Default for FileHashEntryArgs<'a> { - #[inline] - fn default() -> Self { - FileHashEntryArgs { - path: None, - descriptor_hash: None, - } - } -} - -pub struct FileHashEntryBuilder<'a: 'b, 'b, A: ::flatbuffers::Allocator + 'a> { - fbb_: &'b mut ::flatbuffers::FlatBufferBuilder<'a, A>, - start_: ::flatbuffers::WIPOffset<::flatbuffers::TableUnfinishedWIPOffset>, -} -impl<'a: 'b, 'b, A: ::flatbuffers::Allocator + 'a> FileHashEntryBuilder<'a, 'b, A> { - #[inline] - pub fn add_path(&mut self, path: ::flatbuffers::WIPOffset<&'b str>) { - self.fbb_.push_slot_always::<::flatbuffers::WIPOffset<_>>(FileHashEntry::VT_PATH, path); - } - #[inline] - pub fn add_descriptor_hash(&mut self, descriptor_hash: ::flatbuffers::WIPOffset<&'b str>) { - self.fbb_.push_slot_always::<::flatbuffers::WIPOffset<_>>(FileHashEntry::VT_DESCRIPTOR_HASH, descriptor_hash); - } - #[inline] - pub fn new(_fbb: &'b mut ::flatbuffers::FlatBufferBuilder<'a, A>) -> FileHashEntryBuilder<'a, 'b, A> { - let start = _fbb.start_table(); - FileHashEntryBuilder { - fbb_: _fbb, - start_: start, - } - } - #[inline] - pub fn finish(self) -> ::flatbuffers::WIPOffset> { - let o = self.fbb_.end_table(self.start_); - ::flatbuffers::WIPOffset::new(o.value()) - } -} - -impl ::core::fmt::Debug for FileHashEntry<'_> { - fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { - let mut ds = f.debug_struct("FileHashEntry"); - ds.field("path", &self.path()); - ds.field("descriptor_hash", &self.descriptor_hash()); - ds.finish() - } -} -pub enum MetadataHashOffset {} -#[derive(Copy, Clone, PartialEq)] - -pub struct MetadataHash<'a> { - pub _tab: ::flatbuffers::Table<'a>, -} - -impl<'a> ::flatbuffers::Follow<'a> for MetadataHash<'a> { - type Inner = MetadataHash<'a>; - #[inline] - unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { - Self { _tab: unsafe { ::flatbuffers::Table::new(buf, loc) } } - } -} - -impl<'a> MetadataHash<'a> { - pub const VT_FILES: ::flatbuffers::VOffsetT = 4; - - #[inline] - pub unsafe fn init_from_table(table: ::flatbuffers::Table<'a>) -> Self { - MetadataHash { _tab: table } - } - #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: ::flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut ::flatbuffers::FlatBufferBuilder<'bldr, A>, - args: &'args MetadataHashArgs<'args> - ) -> ::flatbuffers::WIPOffset> { - let mut builder = MetadataHashBuilder::new(_fbb); - if let Some(x) = args.files { builder.add_files(x); } - builder.finish() - } - - - #[inline] - pub fn files(&self) -> Option<::flatbuffers::Vector<'a, ::flatbuffers::ForwardsUOffset>>> { - // Safety: - // Created from valid Table for this object - // which contains a valid value in this slot - unsafe { self._tab.get::<::flatbuffers::ForwardsUOffset<::flatbuffers::Vector<'a, ::flatbuffers::ForwardsUOffset>>>(MetadataHash::VT_FILES, None)} - } -} - -impl ::flatbuffers::Verifiable for MetadataHash<'_> { - #[inline] - fn run_verifier( - v: &mut ::flatbuffers::Verifier, pos: usize - ) -> Result<(), ::flatbuffers::InvalidFlatbuffer> { - v.visit_table(pos)? - .visit_field::<::flatbuffers::ForwardsUOffset<::flatbuffers::Vector<'_, ::flatbuffers::ForwardsUOffset>>>("files", Self::VT_FILES, false)? - .finish(); - Ok(()) - } -} -pub struct MetadataHashArgs<'a> { - pub files: Option<::flatbuffers::WIPOffset<::flatbuffers::Vector<'a, ::flatbuffers::ForwardsUOffset>>>>, -} -impl<'a> Default for MetadataHashArgs<'a> { - #[inline] - fn default() -> Self { - MetadataHashArgs { - files: None, - } - } -} - -pub struct MetadataHashBuilder<'a: 'b, 'b, A: ::flatbuffers::Allocator + 'a> { - fbb_: &'b mut ::flatbuffers::FlatBufferBuilder<'a, A>, - start_: ::flatbuffers::WIPOffset<::flatbuffers::TableUnfinishedWIPOffset>, -} -impl<'a: 'b, 'b, A: ::flatbuffers::Allocator + 'a> MetadataHashBuilder<'a, 'b, A> { - #[inline] - pub fn add_files(&mut self, files: ::flatbuffers::WIPOffset<::flatbuffers::Vector<'b , ::flatbuffers::ForwardsUOffset>>>) { - self.fbb_.push_slot_always::<::flatbuffers::WIPOffset<_>>(MetadataHash::VT_FILES, files); - } - #[inline] - pub fn new(_fbb: &'b mut ::flatbuffers::FlatBufferBuilder<'a, A>) -> MetadataHashBuilder<'a, 'b, A> { - let start = _fbb.start_table(); - MetadataHashBuilder { - fbb_: _fbb, - start_: start, - } - } - #[inline] - pub fn finish(self) -> ::flatbuffers::WIPOffset> { - let o = self.fbb_.end_table(self.start_); - ::flatbuffers::WIPOffset::new(o.value()) - } -} - -impl ::core::fmt::Debug for MetadataHash<'_> { - fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { - let mut ds = f.debug_struct("MetadataHash"); - ds.field("files", &self.files()); - ds.finish() - } -} -#[inline] -/// Verifies that a buffer of bytes contains a `MetadataHash` -/// and returns it. -/// Note that verification is still experimental and may not -/// catch every error, or be maximally performant. For the -/// previous, unchecked, behavior use -/// `root_as_metadata_hash_unchecked`. -pub fn root_as_metadata_hash(buf: &[u8]) -> Result, ::flatbuffers::InvalidFlatbuffer> { - ::flatbuffers::root::(buf) -} -#[inline] -/// Verifies that a buffer of bytes contains a size prefixed -/// `MetadataHash` and returns it. -/// Note that verification is still experimental and may not -/// catch every error, or be maximally performant. For the -/// previous, unchecked, behavior use -/// `size_prefixed_root_as_metadata_hash_unchecked`. -pub fn size_prefixed_root_as_metadata_hash(buf: &[u8]) -> Result, ::flatbuffers::InvalidFlatbuffer> { - ::flatbuffers::size_prefixed_root::(buf) -} -#[inline] -/// Verifies, with the given options, that a buffer of bytes -/// contains a `MetadataHash` and returns it. -/// Note that verification is still experimental and may not -/// catch every error, or be maximally performant. For the -/// previous, unchecked, behavior use -/// `root_as_metadata_hash_unchecked`. -pub fn root_as_metadata_hash_with_opts<'b, 'o>( - opts: &'o ::flatbuffers::VerifierOptions, - buf: &'b [u8], -) -> Result, ::flatbuffers::InvalidFlatbuffer> { - ::flatbuffers::root_with_opts::>(opts, buf) -} -#[inline] -/// Verifies, with the given verifier options, that a buffer of -/// bytes contains a size prefixed `MetadataHash` and returns -/// it. Note that verification is still experimental and may not -/// catch every error, or be maximally performant. For the -/// previous, unchecked, behavior use -/// `root_as_metadata_hash_unchecked`. -pub fn size_prefixed_root_as_metadata_hash_with_opts<'b, 'o>( - opts: &'o ::flatbuffers::VerifierOptions, - buf: &'b [u8], -) -> Result, ::flatbuffers::InvalidFlatbuffer> { - ::flatbuffers::size_prefixed_root_with_opts::>(opts, buf) -} -#[inline] -/// Assumes, without verification, that a buffer of bytes contains a MetadataHash and returns it. -/// # Safety -/// Callers must trust the given bytes do indeed contain a valid `MetadataHash`. -pub unsafe fn root_as_metadata_hash_unchecked(buf: &[u8]) -> MetadataHash<'_> { - unsafe { ::flatbuffers::root_unchecked::(buf) } -} -#[inline] -/// Assumes, without verification, that a buffer of bytes contains a size prefixed MetadataHash and returns it. -/// # Safety -/// Callers must trust the given bytes do indeed contain a valid size prefixed `MetadataHash`. -pub unsafe fn size_prefixed_root_as_metadata_hash_unchecked(buf: &[u8]) -> MetadataHash<'_> { - unsafe { ::flatbuffers::size_prefixed_root_unchecked::(buf) } -} -#[inline] -pub fn finish_metadata_hash_buffer<'a, 'b, A: ::flatbuffers::Allocator + 'a>( - fbb: &'b mut ::flatbuffers::FlatBufferBuilder<'a, A>, - root: ::flatbuffers::WIPOffset>) { - fbb.finish(root, None); -} - -#[inline] -pub fn finish_size_prefixed_metadata_hash_buffer<'a, 'b, A: ::flatbuffers::Allocator + 'a>(fbb: &'b mut ::flatbuffers::FlatBufferBuilder<'a, A>, root: ::flatbuffers::WIPOffset>) { - fbb.finish_size_prefixed(root, None); -} -} // pub mod hash -} // pub mod verity -} // pub mod cryptpilot - diff --git a/cryptpilot-verity/src/metadata/mod.rs b/cryptpilot-verity/src/metadata/mod.rs index 1ca404e..dc2a2c3 100644 --- a/cryptpilot-verity/src/metadata/mod.rs +++ b/cryptpilot-verity/src/metadata/mod.rs @@ -5,21 +5,14 @@ #[allow(warnings)] #[allow(unused_imports, dead_code)] mod metadata_generated; -#[rustfmt::skip] -#[allow(clippy::all)] -#[allow(warnings)] -#[allow(unused_imports, dead_code)] -mod metadata_hash_generated; pub use metadata_generated::cryptpilot::verity::{ FileInfo, FileInfoArgs, FsVerityDescriptor, FsVerityDescriptorArgs, KeyValue, KeyValueArgs, Metadata, MetadataArgs, }; -pub use metadata_hash_generated::cryptpilot::verity::hash::{ - FileHashEntry, FileHashEntryArgs, MetadataHash, MetadataHashArgs, -}; use anyhow::{bail, Result}; +use canon_json::CanonJsonSerialize; use flatbuffers::FlatBufferBuilder; use sha2::digest::typenum::Unsigned; use sha2::{digest::OutputSizeUser, Digest, Sha256}; @@ -247,68 +240,58 @@ pub fn deserialize_metadata(data: &[u8]) -> Result { /// This function: /// 1. Parses the full metadata /// 2. Extracts only essential fields (path, descriptor_hash) -/// 3. Serializes them to MetadataHash format +/// 3. Serializes them to canonical JSON (sorted by path, sorted keys via struct order) /// 4. Calculates SHA256 hash pub fn calculate_metadata_hash(metadata_bytes: &[u8]) -> Result { - // Parse full metadata let metadata = flatbuffers::root::(metadata_bytes) .map_err(|e| anyhow::anyhow!("Failed to parse metadata: {}", e))?; - // Convert full Metadata FlatBuffer to MetadataHash for hash calculation - let hash_bytes = { - let mut builder = FlatBufferBuilder::new(); - - let files_vector = if let Some(files) = metadata.files() { - // Build FileHashEntry vector - let mut files_offsets = Vec::with_capacity(files.len()); - for file_info in files { - let path = file_info - .path() - .ok_or_else(|| anyhow::anyhow!("Missing path in FileInfo"))?; - - let descriptor_hash = file_info - .descriptor_hash() - .ok_or_else(|| anyhow::anyhow!("Missing descriptor_hash in FileInfo"))?; - - let path_offset = builder.create_string(path); - let hash_offset = builder.create_string(descriptor_hash); - - let entry = FileHashEntry::create( - &mut builder, - &FileHashEntryArgs { - path: Some(path_offset), - descriptor_hash: Some(hash_offset), - }, - ); - files_offsets.push(entry); - } - - Some(builder.create_vector(&files_offsets)) - } else { - None - }; - - // Create MetadataHash table - let metadata_hash = MetadataHash::create( - &mut builder, - &MetadataHashArgs { - files: files_vector, - }, - ); - - builder.finish(metadata_hash, None); + // Extract path + descriptor_hash pairs into a Vec, sorted by path + let mut entries: Vec = Vec::new(); + if let Some(files) = metadata.files() { + for file_info in files { + let path = file_info + .path() + .ok_or_else(|| anyhow::anyhow!("Missing path in FileInfo"))? + .to_string(); + let descriptor_hash = file_info + .descriptor_hash() + .ok_or_else(|| anyhow::anyhow!("Missing descriptor_hash in FileInfo"))? + .to_string(); + entries.push(FileHashJsonEntry { + descriptor_hash, + path, + }); + } + } + // FlatBuffers files are already sorted by path (see serialize_metadata), + // but sort again for determinism. + entries.sort_by(|a, b| a.path.cmp(&b.path)); - // Serialize to MetadataHash format - builder.finished_data().to_vec() - }; + let doc = MetadataHashDoc { files: entries }; + let json_bytes = doc + .to_canon_json_vec() + .map_err(|e| anyhow::anyhow!("marshal canonical JSON: {}", e))?; - // Calculate SHA256 let mut hasher = sha2::Sha256::new(); - hasher.update(&hash_bytes); + hasher.update(&json_bytes); Ok(hex::encode(hasher.finalize())) } +/// Canonical JSON document for metadata hash calculation. +#[derive(serde::Serialize)] +struct MetadataHashDoc { + files: Vec, +} + +/// Single file entry for hash calculation. +#[derive(serde::Serialize)] +struct FileHashJsonEntry { + descriptor_hash: String, + path: String, +} + #[cfg(test)] mod tests { use super::*; @@ -364,4 +347,51 @@ mod tests { assert_eq!(file_infos.len(), deserialized.file_infos.len()); assert!(deserialized.labels.is_empty()); } + + #[test] + fn test_canonical_json_hash_determinism() { + let test_data = b"test file content"; + let (descriptor, merkle_tree) = calculate_fsverity_hash(test_data); + let descriptor_hash = hex::encode(descriptor.to_descriptor_hash()); + let info = FileVerityInfo { + path: "test.txt".to_string(), + descriptor, + merkle_tree, + descriptor_hash, + }; + + let file_infos = vec![info]; + let labels = BTreeMap::new(); + + let serialized = serialize_metadata(&file_infos, &labels).unwrap(); + let hash1 = calculate_metadata_hash(&serialized).unwrap(); + + // Same input → same hash + let hash2 = calculate_metadata_hash(&serialized).unwrap(); + assert_eq!(hash1, hash2); + + // Different path → different hash + let (descriptor2, merkle_tree2) = calculate_fsverity_hash(test_data); + let descriptor_hash2 = hex::encode(descriptor2.to_descriptor_hash()); + let info2 = FileVerityInfo { + path: "other.txt".to_string(), + descriptor: descriptor2, + merkle_tree: merkle_tree2, + descriptor_hash: descriptor_hash2, + }; + let serialized2 = serialize_metadata(&[info2], &labels).unwrap(); + let hash3 = calculate_metadata_hash(&serialized2).unwrap(); + assert_ne!(hash1, hash3); + } + + #[test] + fn cross_check_metadata_hash() { + // FlatBuffers generated by Go: one FileInfo with path="test.txt", descriptor_hash="abcdef1234567890" + let fb_hex = "0c0000000800080000000400080000000400000001000000100000000c000c0008000000000004000c000000080000001c00000010000000616263646566313233343536373839300000000008000000746573742e74787400000000"; + let fb_bytes = hex::decode(fb_hex).unwrap(); + let hash = calculate_metadata_hash(&fb_bytes).unwrap(); + // Expected: SHA-256 of canonical JSON: {"files":[{"descriptor_hash":"abcdef1234567890","path":"test.txt"}]} + let expected = "baa8151afa2b0a8eec0175239a0ddcf92cade8d7c364f1a26e5afa0d667335c1"; + assert_eq!(hash, expected, "Rust canonical JSON hash should match Go's"); + } } diff --git a/verity-go/go.mod b/verity-go/go.mod index efd2ce5..da5e026 100644 --- a/verity-go/go.mod +++ b/verity-go/go.mod @@ -3,3 +3,5 @@ module github.com/openanolis/cryptpilot/verity-go go 1.24 require github.com/google/flatbuffers v25.12.19+incompatible + +require github.com/gibson042/canonicaljson-go v1.0.3 diff --git a/verity-go/go.sum b/verity-go/go.sum index 799c59f..2f234a5 100644 --- a/verity-go/go.sum +++ b/verity-go/go.sum @@ -1,2 +1,4 @@ +github.com/gibson042/canonicaljson-go v1.0.3 h1:EAyF8L74AWabkyUmrvEFHEt/AGFQeD6RfwbAuf0j1bI= +github.com/gibson042/canonicaljson-go v1.0.3/go.mod h1:DsLpJTThXyGNO+KZlI85C1/KDcImpP67k/RKVjcaEqo= github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= diff --git a/verity-go/metadata/gen_fixture.sh b/verity-go/metadata/gen_fixture.sh index df63089..239011f 100755 --- a/verity-go/metadata/gen_fixture.sh +++ b/verity-go/metadata/gen_fixture.sh @@ -12,8 +12,11 @@ python3 make_testfiles.py echo "=== Running Rust format ===" cd "$REPO_ROOT" -cargo run -p cryptpilot-verity -- format verity-core/testfiles --hash-output - --label env=prod --force +HASH=$(cargo run -q -p cryptpilot-verity -- format verity-core/testfiles --hash-output - --label env=prod --force) echo "=== Copying fixture ===" cp verity-core/testfiles/cryptpilot-verity.metadata.fb verity-go/metadata/testdata/rust.metadata.fb +echo "$HASH" > verity-go/metadata/testdata/rust.metadata.fb.hash + echo "Fixture updated: verity-go/metadata/testdata/rust.metadata.fb" +echo "Root hash: $HASH" diff --git a/verity-go/metadata/generated/metadata_hash_generated.go b/verity-go/metadata/generated/metadata_hash_generated.go deleted file mode 100644 index fb4f852..0000000 --- a/verity-go/metadata/generated/metadata_hash_generated.go +++ /dev/null @@ -1,143 +0,0 @@ -// Code generated by the FlatBuffers compiler. DO NOT EDIT. - -package generated - -import ( - flatbuffers "github.com/google/flatbuffers/go" -) - -// Code generated by the FlatBuffers compiler. DO NOT EDIT. - -type FileHashEntry struct { - _tab flatbuffers.Table -} - -func GetRootAsFileHashEntry(buf []byte, offset flatbuffers.UOffsetT) *FileHashEntry { - n := flatbuffers.GetUOffsetT(buf[offset:]) - x := &FileHashEntry{} - x.Init(buf, n+offset) - return x -} - -func FinishFileHashEntryBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) { - builder.Finish(offset) -} - -func GetSizePrefixedRootAsFileHashEntry(buf []byte, offset flatbuffers.UOffsetT) *FileHashEntry { - n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:]) - x := &FileHashEntry{} - x.Init(buf, n+offset+flatbuffers.SizeUint32) - return x -} - -func FinishSizePrefixedFileHashEntryBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) { - builder.FinishSizePrefixed(offset) -} - -func (rcv *FileHashEntry) Init(buf []byte, i flatbuffers.UOffsetT) { - rcv._tab.Bytes = buf - rcv._tab.Pos = i -} - -func (rcv *FileHashEntry) Table() flatbuffers.Table { - return rcv._tab -} - -func (rcv *FileHashEntry) Path() []byte { - o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) - if o != 0 { - return rcv._tab.ByteVector(o + rcv._tab.Pos) - } - return nil -} - -func (rcv *FileHashEntry) DescriptorHash() []byte { - o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) - if o != 0 { - return rcv._tab.ByteVector(o + rcv._tab.Pos) - } - return nil -} - -func FileHashEntryStart(builder *flatbuffers.Builder) { - builder.StartObject(2) -} -func FileHashEntryAddPath(builder *flatbuffers.Builder, path flatbuffers.UOffsetT) { - builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(path), 0) -} -func FileHashEntryAddDescriptorHash(builder *flatbuffers.Builder, descriptorHash flatbuffers.UOffsetT) { - builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(descriptorHash), 0) -} -func FileHashEntryEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { - return builder.EndObject() -} - -// Code generated by the FlatBuffers compiler. DO NOT EDIT. - -type MetadataHash struct { - _tab flatbuffers.Table -} - -func GetRootAsMetadataHash(buf []byte, offset flatbuffers.UOffsetT) *MetadataHash { - n := flatbuffers.GetUOffsetT(buf[offset:]) - x := &MetadataHash{} - x.Init(buf, n+offset) - return x -} - -func FinishMetadataHashBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) { - builder.Finish(offset) -} - -func GetSizePrefixedRootAsMetadataHash(buf []byte, offset flatbuffers.UOffsetT) *MetadataHash { - n := flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:]) - x := &MetadataHash{} - x.Init(buf, n+offset+flatbuffers.SizeUint32) - return x -} - -func FinishSizePrefixedMetadataHashBuffer(builder *flatbuffers.Builder, offset flatbuffers.UOffsetT) { - builder.FinishSizePrefixed(offset) -} - -func (rcv *MetadataHash) Init(buf []byte, i flatbuffers.UOffsetT) { - rcv._tab.Bytes = buf - rcv._tab.Pos = i -} - -func (rcv *MetadataHash) Table() flatbuffers.Table { - return rcv._tab -} - -func (rcv *MetadataHash) Files(obj *FileHashEntry, j int) bool { - o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) - if o != 0 { - x := rcv._tab.Vector(o) - x += flatbuffers.UOffsetT(j) * 4 - x = rcv._tab.Indirect(x) - obj.Init(rcv._tab.Bytes, x) - return true - } - return false -} - -func (rcv *MetadataHash) FilesLength() int { - o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) - if o != 0 { - return rcv._tab.VectorLen(o) - } - return 0 -} - -func MetadataHashStart(builder *flatbuffers.Builder) { - builder.StartObject(1) -} -func MetadataHashAddFiles(builder *flatbuffers.Builder, files flatbuffers.UOffsetT) { - builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(files), 0) -} -func MetadataHashStartFilesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { - return builder.StartVector(4, numElems, 4) -} -func MetadataHashEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { - return builder.EndObject() -} diff --git a/verity-go/metadata/interop_test.go b/verity-go/metadata/interop_test.go index 8ce0b1d..c409119 100644 --- a/verity-go/metadata/interop_test.go +++ b/verity-go/metadata/interop_test.go @@ -162,3 +162,39 @@ func findTestfilesDir(t *testing.T) string { } return "" } + +// TestInterop_CalculateMetadataHash_CrossLanguage loads the Rust-generated metadata +// fixture and verifies that Go's CalculateMetadataHash produces the same root hash +// as Rust's calculate_metadata_hash. +func TestInterop_CalculateMetadataHash_CrossLanguage(t *testing.T) { + fbPath := filepath.Join(fixtureDir(), "rust.metadata.fb") + data, err := os.ReadFile(fbPath) + if err != nil { + if os.IsNotExist(err) { + t.Skip("fixture not found — run `make gen-interop-fixture` first") + } + t.Fatalf("read fixture: %v", err) + } + + // Read the expected root hash (committed alongside the fixture) + hashPath := filepath.Join(fixtureDir(), "rust.metadata.fb.hash") + expectedHashBytes, err := os.ReadFile(hashPath) + if err != nil { + if os.IsNotExist(err) { + t.Skip("root hash fixture not found — run `make gen-interop-fixture` first") + } + t.Fatalf("read root hash fixture: %v", err) + } + expectedHash := string(bytes.TrimSpace(expectedHashBytes)) + + // Calculate the hash using Go's implementation + goHash, err := CalculateMetadataHash(data) + if err != nil { + t.Fatalf("CalculateMetadataHash: %v", err) + } + + if goHash != expectedHash { + t.Errorf("root hash mismatch\nexpected (Rust): %s\ngot (Go): %s", + expectedHash, goHash) + } +} diff --git a/verity-go/metadata/metadata_hash.fbs b/verity-go/metadata/metadata_hash.fbs deleted file mode 120000 index d21b7f6..0000000 --- a/verity-go/metadata/metadata_hash.fbs +++ /dev/null @@ -1 +0,0 @@ -../../cryptpilot-verity/src/metadata/metadata_hash.fbs \ No newline at end of file diff --git a/verity-go/metadata/metadata_hash.go b/verity-go/metadata/metadata_hash.go index 74b7676..5cf88dc 100644 --- a/verity-go/metadata/metadata_hash.go +++ b/verity-go/metadata/metadata_hash.go @@ -5,47 +5,55 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "sort" + "github.com/gibson042/canonicaljson-go" "github.com/openanolis/cryptpilot/verity-go/metadata/generated" - - flatbuffers "github.com/google/flatbuffers/go" ) +// fileHashEntry is a single file entry for hash calculation. +type fileHashEntry struct { + DescriptorHash string `json:"descriptor_hash"` + Path string `json:"path"` +} + +// metadataHash is the canonical JSON structure for metadata hash calculation. +type metadataHash struct { + Files []fileHashEntry `json:"files"` +} + // CalculateMetadataHash extracts path+descriptor_hash from metadata, serializes -// to the minimal MetadataHash format, and returns its SHA-256 digest (hex-encoded). +// to canonical JSON (sorted by path, sorted keys via struct field order), and +// returns its SHA-256 digest (hex-encoded). func CalculateMetadataHash(metadataBytes []byte) (string, error) { md := generated.GetRootAsMetadata(metadataBytes, 0) - builder := flatbuffers.NewBuilder(0) - filesLen := md.FilesLength() - var filesVec flatbuffers.UOffsetT - - if filesLen > 0 { - fileOffsets := make([]flatbuffers.UOffsetT, filesLen) - var fi generated.FileInfo - for i := 0; i < filesLen; i++ { - if !md.Files(&fi, i) { - return "", &ParseError{Message: fmt.Sprintf("missing FileInfo at index %d", i)} - } - pathOff := builder.CreateString(string(fi.Path())) - hashOff := builder.CreateString(string(fi.DescriptorHash())) - generated.FileHashEntryStart(builder) - generated.FileHashEntryAddPath(builder, pathOff) - generated.FileHashEntryAddDescriptorHash(builder, hashOff) - fileOffsets[i] = generated.FileHashEntryEnd(builder) + entries := make([]fileHashEntry, filesLen) + + var fi generated.FileInfo + for i := 0; i < filesLen; i++ { + if !md.Files(&fi, i) { + return "", &ParseError{Message: fmt.Sprintf("missing FileInfo at index %d", i)} + } + entries[i] = fileHashEntry{ + DescriptorHash: string(fi.DescriptorHash()), + Path: string(fi.Path()), } - filesVec = builder.CreateVectorOfTables(fileOffsets) } + // Sort by path for deterministic output. + sort.Slice(entries, func(i, j int) bool { + return entries[i].Path < entries[j].Path + }) + + doc := metadataHash{Files: entries} - generated.MetadataHashStart(builder) - if filesLen > 0 { - generated.MetadataHashAddFiles(builder, filesVec) + jsonBytes, err := canonicaljson.Marshal(doc) + if err != nil { + return "", fmt.Errorf("marshal canonical JSON: %w", err) } - hashOff := generated.MetadataHashEnd(builder) - builder.Finish(hashOff) h := sha256.New() - h.Write(builder.FinishedBytes()) + h.Write(jsonBytes) return hex.EncodeToString(h.Sum(nil)), nil } diff --git a/verity-go/metadata/testdata/rust.metadata.fb.hash b/verity-go/metadata/testdata/rust.metadata.fb.hash new file mode 100644 index 0000000..78abb56 --- /dev/null +++ b/verity-go/metadata/testdata/rust.metadata.fb.hash @@ -0,0 +1 @@ +a9180c79c8fd70c93711d39aac2f9951cd3a4d8ded2c1d51544b70fd2ea278f2