From 5689b36d97a792bd6258fb26f50bf10035a3d7d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:54:33 +0000 Subject: [PATCH 1/2] offload: fingerprint-verified vector upgrades + whole-state pending gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make offload reachable against packed/CA offsites by fixing the two friction-log defects that together closed every gate path (F13 + F29). F13 — the pending-artifact gate was per-run. certifyPacked held the vector only when *this run's* packs were unverified, so a later run that packed nothing advanced the whole volume past an earlier still-pending pack, and once `squirrel verify` filled the fingerprints nothing re-advanced. The advance is now gated on zero pending artifacts for the whole (volume, destination) pair (CountVolumeContentsPendingFingerprint), and `squirrel verify` re-attempts the advance itself once a clean pass completes (UpgradeDestinationVectorToFingerprintVerified). F29 — the component method was never upgraded past presence+size, so the fail-closed gate refused relayed offsites forever. Capture and verify now upgrade a fully fingerprint-verified pair to a new `fingerprint-verified` verify_method (a free-text value; no schema change, no CHECK constraint). It relays over the durability pull like any other method. Decision 1b — a verified provider-fingerprint chain counts as content-verified for the gate, but only while a scheduled verify cadence (#154) keeps re-confirming it. The responder relays its effective verify cadence per destination (DurabilityComponent.VerifyEveryNs); the pull bakes a relayed fingerprint-verified component down to presence+size when that cadence is absent (older peer, or a destination the responder runs on no cadence), failing closed. The gate accepts a fingerprint-verified component when relayed (cadence proven at pull time) or when the local destination carries a live cadence; a local component without a cadence falls back to the per-object/pack scan-back, since the node re-reads its own fingerprints — the coupling binds the relayed path, where the evidence cannot be re-checked locally. Tests: both F13 sides (per-state hold, verify-then-advance) for packed and content-addressed, the capture-time method upgrade, the relayed accept / relayed-absent refuse cadence coupling, the local cadence/fallback/refuse cases, the pull-time baking, and the responder relaying its cadence. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- agent/durability.go | 17 ++-- agent/durability_test.go | 50 ++++++++++ cmd/squirrel/offload.go | 35 +++++++ offload/durability_soundness_test.go | 139 +++++++++++++++++++++++++++ offload/gate.go | 45 ++++++++- offload/offload.go | 11 ++- store/content_presence.go | 43 +++++++++ store/destination_run_ids.go | 80 +++++++++++++-- store/destination_run_ids_test.go | 4 +- sync/content_addressed.go | 39 +++++++- sync/content_addressed_test.go | 45 +++++++-- sync/durability.go | 19 +++- sync/durability_test.go | 29 ++++++ sync/handler.go | 4 + sync/packed.go | 42 +++++--- sync/packed_test.go | 79 ++++++++++++++- sync/verify_remote.go | 35 +++++++ syncproto/syncproto.go | 24 +++-- 18 files changed, 685 insertions(+), 55 deletions(-) diff --git a/agent/durability.go b/agent/durability.go index 344c7c0..1fbb027 100644 --- a/agent/durability.go +++ b/agent/durability.go @@ -57,6 +57,10 @@ func (r *peerSyncRouter) durabilityResponse(ctx context.Context, volumeName stri if err != nil { return syncproto.DurabilityResponse{}, fmt.Errorf("list push freshness: %w", err) } + // The effective verify cadence per destination, relayed so a puller can + // apply the fingerprint-verified cadence coupling against this + // responder's own re-confirmation schedule (see DurabilityComponent). + cadences := resolveVerifyCadences(r.srv.cfg.Destinations, r.srv.cfg.VerifyEvery) names := make(map[int64]string, 4) resolve := func(nodeID int64) (string, error) { if name, ok := names[nodeID]; ok { @@ -79,12 +83,13 @@ func (r *peerSyncRouter) durabilityResponse(ctx context.Context, volumeName stri return syncproto.DurabilityResponse{}, err } resp.Components = append(resp.Components, syncproto.DurabilityComponent{ - Destination: row.Destination, - OriginNode: name, - OriginRun: row.OriginRunID, - UpdatedAtNs: row.UpdatedAtNs, - VerifyMethod: row.VerifyMethod, - VerifiedAtNs: row.VerifiedAtNs.Int64, // zero when NULL (unknown) + Destination: row.Destination, + OriginNode: name, + OriginRun: row.OriginRunID, + UpdatedAtNs: row.UpdatedAtNs, + VerifyMethod: row.VerifyMethod, + VerifiedAtNs: row.VerifiedAtNs.Int64, // zero when NULL (unknown) + VerifyEveryNs: int64(cadences[row.Destination]), }) } for _, row := range fresh { diff --git a/agent/durability_test.go b/agent/durability_test.go index 0bbfbf3..9856a9c 100644 --- a/agent/durability_test.go +++ b/agent/durability_test.go @@ -7,8 +7,10 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/store" "github.com/mbertschler/squirrel/syncproto" ) @@ -93,6 +95,54 @@ func TestDurabilityEndpointListsComponents(t *testing.T) { } } +// TestDurabilityEndpointRelaysVerifyCadence: the responder relays its +// effective verify cadence per destination (a per-destination verify_every +// or the [agent] default) so a puller can apply the fingerprint-verified +// cadence coupling against the responder's own re-confirmation schedule. A +// destination with no cadence (a mirror, or one that declares none) relays +// zero, which the puller reads as fail-closed (issue #155). +func TestDurabilityEndpointRelaysVerifyCadence(t *testing.T) { + ctx := context.Background() + vol := &config.Volume{Name: "pics", Path: t.TempDir()} + srv := newTestServer(t, Config{ + Volumes: map[string]*config.Volume{vol.Name: vol}, + Destinations: map[string]*config.Destination{ + "s3archive": {Layout: config.LayoutPacked, VerifyEvery: 168 * time.Hour}, + "mirror": {Layout: config.LayoutMirror}, + }, + }) + + v, err := srv.store.CreateVolume(ctx, vol.Name, vol.Path) + if err != nil { + t.Fatalf("CreateVolume: %v", err) + } + self, err := srv.store.GetSelfNode(ctx) + if err != nil { + t.Fatalf("GetSelfNode: %v", err) + } + if err := srv.store.UpsertDestinationRunIDVerified(ctx, v.ID, "s3archive", self.ID, 7, store.VerifyMethodFingerprint, false); err != nil { + t.Fatalf("seed s3archive: %v", err) + } + if err := srv.store.UpsertDestinationRunIDVerified(ctx, v.ID, "mirror", self.ID, 3, store.VerifyMethodBlake3, false); err != nil { + t.Fatalf("seed mirror: %v", err) + } + + var resp syncproto.DurabilityResponse + if code := postDurability(t, srv, syncproto.DurabilityRequest{Volume: "pics"}, &resp); code != http.StatusOK { + t.Fatalf("status = %d, want 200", code) + } + got := map[string]int64{} + for _, c := range resp.Components { + got[c.Destination] = c.VerifyEveryNs + } + if got["s3archive"] != int64(168*time.Hour) { + t.Fatalf("s3archive verify_every_ns = %d, want %d", got["s3archive"], int64(168*time.Hour)) + } + if got["mirror"] != 0 { + t.Fatalf("mirror verify_every_ns = %d, want 0 (no cadence)", got["mirror"]) + } +} + // TestDurabilityEndpointGuards: an undeclared volume 404s, a missing // volume name 400s, and a declared volume with no store row answers // with an empty component list (a valid "nothing recorded yet"). diff --git a/cmd/squirrel/offload.go b/cmd/squirrel/offload.go index f7f1dc4..b2bee16 100644 --- a/cmd/squirrel/offload.go +++ b/cmd/squirrel/offload.go @@ -68,6 +68,7 @@ func runOffload(cmd *cobra.Command, volumeName string, paths []string, olderThan Require: vol.OffloadRequires, RequireDests: requiredDestinations(cfg, vol.OffloadRequires), MaxEvidenceAge: vol.OffloadMaxEvidenceAge, + VerifyCadenced: verifyCadencedTargets(cfg, vol.OffloadRequires), DryRun: dryRun, }) printOffloadReport(cmd.OutOrStdout(), cmd.ErrOrStderr(), rep, dryRun) @@ -94,6 +95,40 @@ func requiredDestinations(cfg *config.Config, require []string) map[string]*conf return out } +// verifyCadencedTargets marks the required targets that carry an effective +// local verify cadence — a per-destination verify_every, or the [agent] +// verify_every default applied to a content-addressed/packed destination. +// The offload gate accepts a locally-advanced fingerprint-verified +// component as content-verified only for a target in this set, so the +// provider-fingerprint evidence keeps being re-confirmed for as long as +// deletions are permitted (issue #155). A required target with no local +// destination — a peer-relayed offsite this node cannot see — is omitted; +// the responder's cadence governs it, relayed and enforced at pull time. +func verifyCadencedTargets(cfg *config.Config, require []string) map[string]bool { + var agentDefault time.Duration + if cfg.Agent != nil { + agentDefault = cfg.Agent.VerifyEvery + } + out := make(map[string]bool, len(require)) + for _, name := range require { + d, ok := cfg.Destinations[name] + if !ok { + continue + } + if d.Layout != config.LayoutContentAddressed && d.Layout != config.LayoutPacked { + continue + } + eff := d.VerifyEvery + if eff == 0 { + eff = agentDefault + } + if eff > 0 { + out[name] = true + } + } + return out +} + // olderThanDaysRE accepts the whole-day shorthand (e.g. "90d") that // time.ParseDuration lacks. var olderThanDaysRE = regexp.MustCompile(`^(\d+)d$`) diff --git a/offload/durability_soundness_test.go b/offload/durability_soundness_test.go index edb302c..74444cc 100644 --- a/offload/durability_soundness_test.go +++ b/offload/durability_soundness_test.go @@ -514,3 +514,142 @@ func TestOffloadDurableFileStillPasses(t *testing.T) { oneResult(t, rep, "a.txt", OutcomeOffloaded) oneResult(t, rep, "sub/b.txt", OutcomeOffloaded) } + +// TestOffloadFingerprintVerifiedRelayedGates is the crown-jewel relayed +// path (issue #155): a peer-asserted fingerprint-verified component gates. +// It exists only because the responder relayed a positive verify cadence +// (the pull bakes it to presence+size otherwise), so the puller — which +// holds no local fingerprint of the object — may treat the relayed +// evidence as content-verified. +func TestOffloadFingerprintVerifiedRelayedGates(t *testing.T) { + const target = "s3archive" + root := t.TempDir() + writeFile(t, filepath.Join(root, "a.txt"), "alpha") + s := setupStore(t) + ctx := context.Background() + idx := indexVolume(t, s, root) + v := testVolume(t, s) + self := selfNode(t, s) + + peer, err := s.GetOrCreateOriginNode(ctx, "nas") + if err != nil { + t.Fatalf("GetOrCreateOriginNode: %v", err) + } + if err := s.UpsertDestinationRunIDPulled(ctx, v.ID, target, self.ID, idx.RunID, store.VerifyMethodFingerprint, peer.ID, store.NowNs(), false); err != nil { + t.Fatalf("UpsertDestinationRunIDPulled: %v", err) + } + seedRelayedFreshness(t, s, v.ID, target, self.ID, idx.RunID) + + rep, err := Offload(ctx, s, root, Options{Name: volName, Paths: []string{"."}, Require: []string{target}}) + if err != nil { + t.Fatalf("Offload: %v", err) + } + oneResult(t, rep, "a.txt", OutcomeOffloaded) + mustBeGone(t, filepath.Join(root, "a.txt")) +} + +// TestOffloadFingerprintVerifiedRelayedBakedDownRefused is the fail-closed +// direction: when the responder relays no verify cadence the pull stores +// the component as presence+size (see relayedMethod), and the puller has no +// local fingerprint to back it — so the gate refuses (issue #155). +func TestOffloadFingerprintVerifiedRelayedBakedDownRefused(t *testing.T) { + const target = "s3archive" + root := t.TempDir() + writeFile(t, filepath.Join(root, "a.txt"), "alpha") + s := setupStore(t) + ctx := context.Background() + idx := indexVolume(t, s, root) + v := testVolume(t, s) + self := selfNode(t, s) + + peer, err := s.GetOrCreateOriginNode(ctx, "nas") + if err != nil { + t.Fatalf("GetOrCreateOriginNode: %v", err) + } + if err := s.UpsertDestinationRunIDPulled(ctx, v.ID, target, self.ID, idx.RunID, store.VerifyMethodPresenceSize, peer.ID, store.NowNs(), false); err != nil { + t.Fatalf("UpsertDestinationRunIDPulled: %v", err) + } + seedRelayedFreshness(t, s, v.ID, target, self.ID, idx.RunID) + + rep, err := Offload(ctx, s, root, Options{Name: volName, Paths: []string{"."}, Require: []string{target}}) + if err != nil { + t.Fatalf("Offload: %v", err) + } + res := oneResult(t, rep, "a.txt", OutcomeNotDurable) + if len(res.Reasons) != 1 || !strings.Contains(res.Reasons[0], "not content-verified") { + t.Fatalf("reasons = %v, want a not-content-verified refusal", res.Reasons) + } + mustExist(t, filepath.Join(root, "a.txt")) +} + +// TestOffloadFingerprintVerifiedLocalCadenceCoupling covers a locally- +// advanced fingerprint-verified component: it gates when the destination +// carries a verify cadence, falls back to a locally-held object fingerprint +// when it does not (the node re-reads its own objects), and refuses when +// neither holds (issue #155). +func TestOffloadFingerprintVerifiedLocalCadenceCoupling(t *testing.T) { + const target = "arch" + seed := func(t *testing.T) (*store.Store, string, store.FileRow, int64) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "a.txt"), "alpha") + s := setupStore(t) + ctx := context.Background() + idx := indexVolume(t, s, root) + v := testVolume(t, s) + self := selfNode(t, s) + if err := s.UpsertDestinationRunIDVerified(ctx, v.ID, target, self.ID, idx.RunID, store.VerifyMethodFingerprint, false); err != nil { + t.Fatalf("UpsertDestinationRunIDVerified: %v", err) + } + pushRun := recordPush(t, s, v.ID, target) + return s, root, rowAt(t, s, v.ID, "a.txt"), pushRun + } + + t.Run("cadence accepts", func(t *testing.T) { + s, root, _, _ := seed(t) + rep, err := Offload(context.Background(), s, root, Options{ + Name: volName, Paths: []string{"."}, Require: []string{target}, + VerifyCadenced: map[string]bool{target: true}, + }) + if err != nil { + t.Fatalf("Offload: %v", err) + } + oneResult(t, rep, "a.txt", OutcomeOffloaded) + mustBeGone(t, filepath.Join(root, "a.txt")) + }) + + t.Run("no cadence, no local fingerprint refuses", func(t *testing.T) { + s, root, _, _ := seed(t) + rep, err := Offload(context.Background(), s, root, Options{ + Name: volName, Paths: []string{"."}, Require: []string{target}, + }) + if err != nil { + t.Fatalf("Offload: %v", err) + } + res := oneResult(t, rep, "a.txt", OutcomeNotDurable) + if len(res.Reasons) != 1 || !strings.Contains(res.Reasons[0], "not content-verified") { + t.Fatalf("reasons = %v, want a not-content-verified refusal", res.Reasons) + } + mustExist(t, filepath.Join(root, "a.txt")) + }) + + t.Run("no cadence, local fingerprint falls back", func(t *testing.T) { + s, root, row, pushRun := seed(t) + ctx := context.Background() + if err := s.InsertRemoteObject(ctx, store.RemoteObject{ + ContentID: row.ContentID, Destination: target, UploadedRunID: pushRun, + }); err != nil { + t.Fatalf("InsertRemoteObject: %v", err) + } + if err := s.SetRemoteObjectFingerprint(ctx, row.ContentID, target, "sftp-sha256", "abc", store.NowNs()); err != nil { + t.Fatalf("SetRemoteObjectFingerprint: %v", err) + } + rep, err := Offload(ctx, s, root, Options{ + Name: volName, Paths: []string{"."}, Require: []string{target}, + }) + if err != nil { + t.Fatalf("Offload: %v", err) + } + oneResult(t, rep, "a.txt", OutcomeOffloaded) + mustBeGone(t, filepath.Join(root, "a.txt")) + }) +} diff --git a/offload/gate.go b/offload/gate.go index fd71101..c4026b6 100644 --- a/offload/gate.go +++ b/offload/gate.go @@ -55,9 +55,17 @@ type gate struct { // Zero disables the time-based policy — the opt-in default, so the // version-vector gate behaves exactly as before. maxEvidenceAge time.Duration + // cadenced marks the required targets that carry an effective local + // verify cadence (a per-destination verify_every, or the [agent] + // default). A locally-verified fingerprint-verified component gates + // only while its destination is re-confirmed on a cadence; a target + // this node cannot see (a relayed offsite) is absent from the map, and + // the relayed cadence is enforced at pull time instead. Nil is a valid + // empty set — no target is treated as cadenced. + cadenced map[string]bool } -func loadGate(ctx context.Context, s *store.Store, volumeID int64, require []string, nowNs int64, maxEvidenceAge time.Duration) (*gate, error) { +func loadGate(ctx context.Context, s *store.Store, volumeID int64, require []string, nowNs int64, maxEvidenceAge time.Duration, cadenced map[string]bool) (*gate, error) { self, err := s.GetSelfNode(ctx) if err != nil { return nil, fmt.Errorf("lookup self node: %w", err) @@ -73,6 +81,7 @@ func loadGate(ctx context.Context, s *store.Store, volumeID int64, require []str nodeNames: map[int64]string{self.ID: self.Name}, nowNs: nowNs, maxEvidenceAge: maxEvidenceAge, + cadenced: cadenced, } for _, target := range require { components, err := s.ListDestinationRunIDs(ctx, volumeID, target) @@ -249,15 +258,45 @@ func (g *gate) freshnessFailure(ctx context.Context, target string, row store.Fi // passes only once a verified scan-back fingerprint backs the gated content // — via either layout: a remote_objects row (per-hash object) or a // remote_packs row for the content's pack, each carrying a checksum and a -// verified_at_ns for this destination. Any other method (including a -// size+mtime push or an unknown/pre-v19 component) does not gate. +// verified_at_ns for this destination. +// +// A fingerprint-verified component (the upgrade of a presence+size vector +// once the whole pair carries verified fingerprints) is content-verified +// only while a scheduled verify cadence keeps that evidence fresh: +// +// - relayed (a durability pull tagged it): acceptance was decided at pull +// time, where a fingerprint-verified method survives only when the +// responder relayed a positive verify cadence for the destination +// (else it is baked down to presence+size, which then needs a local +// fingerprint this node does not hold). So a relayed fingerprint- +// verified component passes here directly. +// - local (this node advanced it): it passes while the destination has a +// live verify cadence; without one it falls back to the per-content +// scan-back check, since the node holds the fingerprints itself and +// re-reads them on every verify pass — the cadence coupling binds the +// relayed path, where the evidence cannot be re-read locally. +// +// Any other method (a size+mtime push or an unknown/pre-v19 component) +// does not gate. func (g *gate) methodVerified(ctx context.Context, target string, comp component, row store.FileRow) (bool, error) { if store.ContentVerifiedMethod(comp.method) { return true, nil } + if comp.method == store.VerifyMethodFingerprint { + if comp.source.Valid || g.cadenced[target] { + return true, nil + } + return g.contentFingerprintVerified(ctx, target, row) + } if comp.method != store.VerifyMethodPresenceSize { return false, nil } + return g.contentFingerprintVerified(ctx, target, row) +} + +// contentFingerprintVerified reports whether a verified provider +// fingerprint backs the row's content on the target (either layout). +func (g *gate) contentFingerprintVerified(ctx context.Context, target string, row store.FileRow) (bool, error) { verified, err := g.store.ContentFingerprintVerified(ctx, row.ContentID, target) if err != nil { return false, fmt.Errorf("load fingerprint for content %d on %q: %w", row.ContentID, target, err) diff --git a/offload/offload.go b/offload/offload.go index 8036d33..45b1628 100644 --- a/offload/offload.go +++ b/offload/offload.go @@ -57,6 +57,15 @@ type Options struct { // staleness policy — the opt-in default, so the version-vector gate // behaves exactly as before. MaxEvidenceAge time.Duration + // VerifyCadenced marks the required targets that carry an effective + // local verify cadence (a per-destination verify_every, or the [agent] + // default). The gate accepts a locally-advanced fingerprint-verified + // component as content-verified only while its destination is on such a + // cadence, so the provider-fingerprint evidence keeps being re-confirmed + // for as long as deletions are permitted (issue #155). A relayed + // target this node cannot see is absent — the responder's cadence is + // enforced at pull time. Nil is a valid empty set. + VerifyCadenced map[string]bool // DryRun evaluates and reports the per-file gate decisions from the // index alone: no runs row, no file reads, no deletions, no status // flips. Disk-drift checks only happen on a real run, immediately @@ -149,7 +158,7 @@ func Offload(ctx context.Context, s *store.Store, root string, opts Options) (re return Report{}, err } now := time.Now() - g, err := loadGate(ctx, s, vol.ID, opts.Require, now.UnixNano(), opts.MaxEvidenceAge) + g, err := loadGate(ctx, s, vol.ID, opts.Require, now.UnixNano(), opts.MaxEvidenceAge, opts.VerifyCadenced) if err != nil { return Report{}, err } diff --git a/store/content_presence.go b/store/content_presence.go index 9d6c5a0..e4e7035 100644 --- a/store/content_presence.go +++ b/store/content_presence.go @@ -31,6 +31,49 @@ func (s *Store) ContentPresentOnDestination(ctx context.Context, contentID int64 return present != 0, nil } +// CountVolumeContentsPendingFingerprint counts the distinct present +// contents of the volume that do NOT yet carry a verified provider +// fingerprint on the destination — the whole-state "pending artifact" +// tally the fingerprint-verified upgrade gates on. A content counts as +// pending unless it has either a remote_objects row or a member pack's +// remote_packs row on this destination with a non-NULL checksum and +// verified_at_ns, so content not yet uploaded, uploaded but not yet +// fingerprinted, or fingerprinted but not yet re-confirmed all keep the +// tally above zero. Zero means every present content is fingerprint- +// verified on the destination, so the vector may advance to +// VerifyMethodFingerprint over the whole present set. The reserved sync +// subtrees are excluded, matching PresentOriginMaxima — they never travel +// to a destination. +func (s *Store) CountVolumeContentsPendingFingerprint(ctx context.Context, volumeID int64, destination string) (int, error) { + var pending int + err := s.db.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM ( + SELECT DISTINCT f.content_id + FROM files f + JOIN folders fo ON fo.id = f.folder_id + WHERE fo.volume_id = ? AND f.status = 'present' + AND `+reservedSubtreeFilter+` + AND NOT ( + EXISTS ( + SELECT 1 FROM remote_objects ro + WHERE ro.content_id = f.content_id AND ro.destination = ? + AND ro.checksum IS NOT NULL AND ro.verified_at_ns IS NOT NULL + ) + OR EXISTS ( + SELECT 1 FROM pack_members pm + JOIN remote_packs rp ON rp.pack_id = pm.pack_id + WHERE pm.content_id = f.content_id AND rp.destination = ? + AND rp.checksum IS NOT NULL AND rp.verified_at_ns IS NOT NULL + ) + ) + ) + `, volumeID, destination, destination).Scan(&pending) + if err != nil { + return 0, fmt.Errorf("count pending fingerprints for volume %d on %q: %w", volumeID, destination, err) + } + return pending, nil +} + // ContentFingerprintVerified reports whether a verified provider // fingerprint backs the content on the destination via either layout: a // remote_objects row carrying a checksum and a verified_at_ns, or a diff --git a/store/destination_run_ids.go b/store/destination_run_ids.go index 99e9f1e..521fa22 100644 --- a/store/destination_run_ids.go +++ b/store/destination_run_ids.go @@ -31,14 +31,36 @@ const ( // content check on its own — a verified scan-back fingerprint must // back the object before such a component gates offload. VerifyMethodPresenceSize = "presence+size" + // VerifyMethodFingerprint is the upgrade of a presence+size component + // once every object and pack backing the (volume, destination) pair + // carries a verified provider fingerprint (a local BLAKE3 confirmed at + // upload, re-confirmed against the provider's checksum of the stored + // bytes). Capture and `squirrel verify` both mint it (see + // UpgradeDestinationVectorToFingerprintVerified). It relays over the + // durability pull verbatim like any other method, letting a node that + // cannot re-read the object itself (a relayed offsite) treat the + // evidence as content-verified. Because that relayed evidence is only + // as trustworthy as the responder's re-confirmation cadence, the + // offload gate accepts it as content-verified only while a scheduled + // verify cadence keeps it fresh — hence it is deliberately absent from + // ContentVerifiedMethod, which the gate treats as unconditionally + // content-verified. + VerifyMethodFingerprint = "fingerprint-verified" ) // ContentVerifiedMethod reports whether a durability component advanced -// by method carries genuine content verification — the precondition the -// offload gate applies before deleting a local copy. A presence-only or -// size+mtime method is not content-verified; an empty method (a pre-v19 -// component, or one whose provenance is unknown) is treated as -// unverified so the gate refuses rather than over-claims. +// by method carries genuine, cadence-independent content verification — +// the precondition the offload gate applies before deleting a local copy. +// A presence-only or size+mtime method is not content-verified; an empty +// method (a pre-v19 component, or one whose provenance is unknown) is +// treated as unverified so the gate refuses rather than over-claims. +// +// VerifyMethodFingerprint is deliberately excluded: its evidence counts as +// content-verified only while a scheduled verify cadence keeps it +// re-confirmed, a condition the offload gate applies itself (local: the +// configured cadence; relayed: the responder's cadence, baked in at pull +// time). Callers wanting the fingerprint-verified method to count must +// apply that cadence check themselves rather than reading it here. func ContentVerifiedMethod(method string) bool { switch method { case VerifyMethodBlake3, VerifyMethodPeer, VerifyMethodKopia: @@ -59,7 +81,7 @@ func ContentVerifiedMethod(method string) bool { // callers that accept it test for "" explicitly. func KnownVerifyMethod(method string) bool { switch method { - case VerifyMethodBlake3, VerifyMethodSizeMtime, VerifyMethodPeer, VerifyMethodKopia, VerifyMethodPresenceSize: + case VerifyMethodBlake3, VerifyMethodSizeMtime, VerifyMethodPeer, VerifyMethodKopia, VerifyMethodPresenceSize, VerifyMethodFingerprint: return true default: return false @@ -223,6 +245,52 @@ func (s *Store) AdvanceDestinationVectorTo(ctx context.Context, volumeID int64, return s.recordPushFreshness(ctx, volumeID, destination, components) } +// UpgradeDestinationVectorToFingerprintVerified re-stamps the destination's +// whole durability vector for the volume as VerifyMethodFingerprint when — +// and only when — every present content of the volume already carries a +// verified provider fingerprint on the destination (zero pending +// artifacts). It is the per-state gate that fixes the two friction-log F13 +// defects: capture and `squirrel verify` both call it, so the advance +// happens exactly when the outstanding fingerprints are complete, never +// vacuously past a still-pending pack (the advance covers the whole present +// set, so one pending artifact holds all of it) and never left stranded +// after verify fills the last fingerprint. +// +// selfNodeID is the coordinate NULL-origin content counts under, the same +// PresentOriginMaxima the push captured. It returns whether the vector was +// upgraded; a pending artifact, an empty present set, or a destination this +// volume has never successfully pushed to all return (false, nil) without +// touching the vector — the last guard keeps content that is durable on the +// destination only through *another* volume's push (objects are +// destination-global) from minting a vector component for a volume whose +// path→hash manifest never landed there. +func (s *Store) UpgradeDestinationVectorToFingerprintVerified(ctx context.Context, volumeID int64, destination string, selfNodeID int64) (bool, error) { + if _, err := s.LatestSuccessfulSyncRun(ctx, volumeID, destination); err != nil { + if IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("lookup last successful sync of %q: %w", destination, err) + } + pending, err := s.CountVolumeContentsPendingFingerprint(ctx, volumeID, destination) + if err != nil { + return false, err + } + if pending != 0 { + return false, nil + } + components, err := s.PresentOriginMaxima(ctx, volumeID, selfNodeID) + if err != nil { + return false, err + } + if len(components) == 0 { + return false, nil + } + if err := s.AdvanceDestinationVectorTo(ctx, volumeID, destination, VerifyMethodFingerprint, components); err != nil { + return false, fmt.Errorf("upgrade destination vector for %q: %w", destination, err) + } + return true, nil +} + // recordPushFreshness overwrites the destination's push-freshness maxima // to exactly the supplied snapshot — the per-origin-node maxima of the // present set this push enumerated. Distinct from the monotonic vector diff --git a/store/destination_run_ids_test.go b/store/destination_run_ids_test.go index 4046a64..78fed54 100644 --- a/store/destination_run_ids_test.go +++ b/store/destination_run_ids_test.go @@ -801,7 +801,9 @@ func TestContentVerifiedMethod(t *testing.T) { t.Fatalf("method %q should be content-verified", m) } } - for _, m := range []string{VerifyMethodPresenceSize, VerifyMethodSizeMtime, "", "bogus"} { + // fingerprint-verified is intentionally NOT unconditionally content- + // verified: its acceptance is cadence-coupled and applied by the gate. + for _, m := range []string{VerifyMethodPresenceSize, VerifyMethodSizeMtime, VerifyMethodFingerprint, "", "bogus"} { if ContentVerifiedMethod(m) { t.Fatalf("method %q must not be content-verified", m) } diff --git a/sync/content_addressed.go b/sync/content_addressed.go index bbe0ecc..316b462 100644 --- a/sync/content_addressed.go +++ b/sync/content_addressed.go @@ -182,11 +182,19 @@ func (h *contentAddressedHandler) push(ctx context.Context, rep *Report, volID, if err := h.uploadSegment(ctx, delta, runID); err != nil { return err } - // presence+size is not a content-verified method (crypt remotes - // expose no hashes): the component advances so a later scan-back - // fingerprint can upgrade it, but the offload gate holds this target - // out until a verified fingerprint backs the gated object. - if err := h.store.AdvanceDestinationVectorTo(ctx, volID, h.dest.Name, store.VerifyMethodPresenceSize, advance); err != nil { + // presence+size is not a content-verified method (crypt remotes expose + // no hashes): the component advances so the offload gate's per-object + // scan-back can back it locally. When this run's capture leaves the + // whole (volume, destination) pair with a verified fingerprint behind + // every present content, the advance is instead stamped + // fingerprint-verified — the content-verified method that also relays + // over the durability pull (see the durability-vector upgrade in + // store). A single still-pending object holds it at presence+size. + method, err := h.advanceMethod(ctx, volID, len(advance)) + if err != nil { + return err + } + if err := h.store.AdvanceDestinationVectorTo(ctx, volID, h.dest.Name, method, advance); err != nil { return fmt.Errorf("advance destination vector for %s: %w", h.dest.Name, err) } rep.Status = store.RunStatusSuccess @@ -194,6 +202,27 @@ func (h *contentAddressedHandler) push(ctx context.Context, rep *Report, volID, return nil } +// advanceMethod chooses the method the content-addressed push advances its +// durability vector with: fingerprint-verified when every present content +// of the volume already carries a verified provider fingerprint on this +// destination (the whole-state check, not just this run's uploads), and +// presence+size otherwise. advanceLen is the size of the advance snapshot; +// an empty snapshot advances nothing, so the method is immaterial and the +// pending query is skipped. +func (h *contentPusher) advanceMethod(ctx context.Context, volID int64, advanceLen int) (string, error) { + if advanceLen == 0 { + return store.VerifyMethodPresenceSize, nil + } + pending, err := h.store.CountVolumeContentsPendingFingerprint(ctx, volID, h.dest.Name) + if err != nil { + return "", err + } + if pending == 0 { + return store.VerifyMethodFingerprint, nil + } + return store.VerifyMethodPresenceSize, nil +} + // watermark resolves the run id the delta starts after: the last // successful sync of this (volume, destination), or 0 for a fresh // destination. The last success must still have its manifest segment at diff --git a/sync/content_addressed_test.go b/sync/content_addressed_test.go index fbb3f6d..eb417e6 100644 --- a/sync/content_addressed_test.go +++ b/sync/content_addressed_test.go @@ -385,14 +385,16 @@ func TestContentAddressedPushHappyPath(t *testing.T) { if vector[0].OriginNodeID != self.ID || vector[0].OriginRunID == 0 { t.Fatalf("vector component = %+v, want self node at the introduction run", vector[0]) } - // The content-addressed advance records presence+size, not a - // content-verified method: the offload gate holds this target out - // until a verified fingerprint backs the object (#109). - if vector[0].VerifyMethod != VerifyMethodPresenceSize { - t.Fatalf("verify method = %q, want %q", vector[0].VerifyMethod, VerifyMethodPresenceSize) + // Both objects were fingerprinted this run, so the whole (volume, + // destination) pair is fingerprint-verified and the advance is stamped + // fingerprint-verified — the #155 upgrade at capture. It is not + // unconditionally content-verified (ContentVerifiedMethod excludes it): + // the offload gate additionally requires a verify cadence (#155). + if vector[0].VerifyMethod != VerifyMethodFingerprint { + t.Fatalf("verify method = %q, want %q", vector[0].VerifyMethod, VerifyMethodFingerprint) } if store.ContentVerifiedMethod(vector[0].VerifyMethod) { - t.Fatalf("presence+size must not count as content-verified") + t.Fatalf("fingerprint-verified must not count as unconditionally content-verified") } // Transfers and confirmations address the crypt overlay remote. @@ -405,6 +407,37 @@ func TestContentAddressedPushHappyPath(t *testing.T) { } } +// TestContentAddressedVerifyUpgradesVector is the friction-log F13(b) side +// for the content-addressed layout: a push whose object fingerprint capture +// failed advances presence+size, and a later verify that fills the +// fingerprint upgrades the component to fingerprint-verified (issue #155). +func TestContentAddressedVerifyUpgradesVector(t *testing.T) { + f := setupContentAddressedFixture(t) + ctx := context.Background() + t.Setenv("RCLONE_FAKE_NO_HASHES", "1") + f.write(t, "a.txt", "alpha") + f.index(t) + if _, err := f.sync(t); err != nil { + t.Fatalf("sync: %v", err) + } + vec, _ := f.store.ListDestinationRunIDs(ctx, f.volumeID(t), "offsite") + if len(vec) != 1 || vec[0].VerifyMethod != VerifyMethodPresenceSize { + t.Fatalf("vector = %+v, want one presence+size component while the fingerprint is pending", vec) + } + + t.Setenv("RCLONE_FAKE_NO_HASHES", "") + if _, err := VerifyRemote(ctx, f.store, f.rcl, f.pair.Destination); err != nil { + t.Fatalf("VerifyRemote: %v", err) + } + vec, err := f.store.ListDestinationRunIDs(ctx, f.volumeID(t), "offsite") + if err != nil { + t.Fatalf("ListDestinationRunIDs: %v", err) + } + if len(vec) != 1 || vec[0].VerifyMethod != VerifyMethodFingerprint { + t.Fatalf("vector = %+v, want fingerprint-verified after verify", vec) + } +} + // TestContentAddressedUploadOnce: a second run uploads only hashes the // destination has no record of — a new path carrying already-recorded // content transfers nothing and still lands in the manifest. diff --git a/sync/durability.go b/sync/durability.go index 013f3fa..da23392 100644 --- a/sync/durability.go +++ b/sync/durability.go @@ -174,7 +174,7 @@ func pullDurability(ctx context.Context, s *store.Store, client *nodeClient, vol if err != nil { return rep, err } - err = s.UpsertDestinationRunIDPulled(ctx, volumeID, c.Destination, nodeID, c.OriginRun, c.VerifyMethod, sourceNode.ID, c.VerifiedAtNs, allowRewind) + err = s.UpsertDestinationRunIDPulled(ctx, volumeID, c.Destination, nodeID, c.OriginRun, relayedMethod(c), sourceNode.ID, c.VerifiedAtNs, allowRewind) var rewind *store.DestinationRewindError if errors.As(err, &rewind) { rep.Rewinds = append(rep.Rewinds, DurabilityRewind{ @@ -210,6 +210,23 @@ func pullDurability(ctx context.Context, s *store.Store, client *nodeClient, vol return rep, nil } +// relayedMethod is the verify method a pulled component is stored under. +// It is the responder's method verbatim, except a fingerprint-verified +// component is baked down to presence+size unless the responder relayed a +// positive verify cadence for the destination: that method is content- +// verified for the puller's offload gate (the fingerprint cannot be +// re-read locally), so it is only honoured while the responder keeps +// re-confirming it. Absence of the cadence (an older responder that never +// sends it, or a destination the responder runs on no cadence) fails the +// coupling closed — the component then needs a local fingerprint the +// relaying node does not hold, so the gate refuses (issue #155). +func relayedMethod(c syncproto.DurabilityComponent) string { + if c.VerifyMethod == store.VerifyMethodFingerprint && c.VerifyEveryNs <= 0 { + return store.VerifyMethodPresenceSize + } + return c.VerifyMethod +} + // validateComponent guards the wire-supplied component before it // touches the local vector: destination and origin names are // identities, the run id must be a positive origin-space id, and the diff --git a/sync/durability_test.go b/sync/durability_test.go index 04aa5d6..54ef682 100644 --- a/sync/durability_test.go +++ b/sync/durability_test.go @@ -5,11 +5,39 @@ import ( "fmt" "strings" "testing" + "time" "github.com/mbertschler/squirrel/store" "github.com/mbertschler/squirrel/syncproto" ) +// TestRelayedMethodCadenceCoupling: a relayed fingerprint-verified +// component is stored verbatim only when the responder relayed a positive +// verify cadence; without it the method is baked down to presence+size so +// the puller (which holds no local fingerprint) fails the offload gate +// closed. Other methods pass through regardless (issue #155). +func TestRelayedMethodCadenceCoupling(t *testing.T) { + base := syncproto.DurabilityComponent{Destination: "s3", OriginNode: "laptop", OriginRun: 5} + + c := base + c.VerifyMethod = store.VerifyMethodFingerprint + c.VerifyEveryNs = int64(168 * time.Hour) + if got := relayedMethod(c); got != store.VerifyMethodFingerprint { + t.Fatalf("relayedMethod(cadenced) = %q, want fingerprint-verified", got) + } + + c.VerifyEveryNs = 0 + if got := relayedMethod(c); got != store.VerifyMethodPresenceSize { + t.Fatalf("relayedMethod(no cadence) = %q, want presence+size (fail closed)", got) + } + + c = base + c.VerifyMethod = store.VerifyMethodBlake3 + if got := relayedMethod(c); got != store.VerifyMethodBlake3 { + t.Fatalf("relayedMethod(blake3) = %q, want blake3 unchanged", got) + } +} + // seedReceiverDurability records vector components on the receiver so // the pull tests have something to fetch. Returns the receiver's self // name (the origin-node identity its components travel under). @@ -437,6 +465,7 @@ func TestValidateComponentVerifyMethod(t *testing.T) { store.VerifyMethodPeer, store.VerifyMethodKopia, store.VerifyMethodPresenceSize, + store.VerifyMethodFingerprint, } { c := base c.VerifyMethod = method diff --git a/sync/handler.go b/sync/handler.go index ab7e21a..5ee15f3 100644 --- a/sync/handler.go +++ b/sync/handler.go @@ -35,6 +35,10 @@ const ( // (crypt remotes expose no hashes), so results carrying it stay // unverified until the provider-checksum fingerprint pass lands. VerifyMethodPresenceSize = store.VerifyMethodPresenceSize + // VerifyMethodFingerprint upgrades a presence+size component once every + // object and pack backing the (volume, destination) pair is + // fingerprint-verified. Minted by capture and by `squirrel verify`. + VerifyMethodFingerprint = store.VerifyMethodFingerprint ) // VerifyResult is the typed durability report of one handler push: how diff --git a/sync/packed.go b/sync/packed.go index 88bca90..27d652d 100644 --- a/sync/packed.go +++ b/sync/packed.go @@ -188,21 +188,32 @@ func (h *packedHandler) push(ctx context.Context, rep *Report, volID, runID int6 // certifyPacked closes the durability seam: it records a remote_packs row // per landed pack, reads each pack's scan-back fingerprint, and advances -// the destination vector only once every pack this run assembled is -// fingerprint-verified — the third leg of the three-artifact gate (packs + -// placement map + manifest segment all landed and confirmed). A pending -// capture holds the vector so the offload gate never counts unverified -// packed content as durable; `squirrel verify` fills the fingerprint later -// and a subsequent sync advances the vector. rep.Verification stays -// unverified (presence+size is not a content-verified method), so RunPair -// does not advance the vector a second time. +// the destination vector to fingerprint-verified only once the whole +// (volume, destination) pair carries a verified fingerprint behind every +// present content — packs, per-hash objects, placement map, and manifest +// segment all landed and confirmed. Gating on the whole pending set rather +// than this run's writes is the friction-log F13 fix: a run that packs +// nothing no longer advances vacuously past an earlier still-pending pack, +// and a still-pending artifact anywhere in the pair holds the whole +// advance. `squirrel verify` fills a pending fingerprint later and +// re-attempts this advance itself (see the store upgrade), so the vector no +// longer stalls until the next content-writing sync. rep.Verification stays +// unverified (presence+size is not content-verified), so RunPair does not +// advance the vector a second time. func (h *packedHandler) certifyPacked(ctx context.Context, rep *Report, volID, runID int64, writes []store.PackWrite, advance []store.OriginComponent) error { - verified := h.capturePackFingerprints(ctx, rep, runID, writes) - if verified < len(writes) { - rep.Warnings = append(rep.Warnings, fmt.Sprintf("destination %q: %d of %d pack(s) this run are not yet fingerprint-verified; the durability vector was not advanced — run `squirrel verify` to certify them", h.dest.Name, len(writes)-verified, len(writes))) + h.capturePackFingerprints(ctx, rep, runID, writes) + pending, err := h.store.CountVolumeContentsPendingFingerprint(ctx, volID, h.dest.Name) + if err != nil { + return fmt.Errorf("count pending fingerprints for %s: %w", h.dest.Name, err) + } + if pending > 0 { + rep.Warnings = append(rep.Warnings, fmt.Sprintf("destination %q: %d content(s) are not yet fingerprint-verified; the durability vector was not advanced — run `squirrel verify` to certify them", h.dest.Name, pending)) + return nil + } + if len(advance) == 0 { return nil } - if err := h.store.AdvanceDestinationVectorTo(ctx, volID, h.dest.Name, store.VerifyMethodPresenceSize, advance); err != nil { + if err := h.store.AdvanceDestinationVectorTo(ctx, volID, h.dest.Name, store.VerifyMethodFingerprint, advance); err != nil { return fmt.Errorf("advance destination vector for %s: %w", h.dest.Name, err) } return nil @@ -215,9 +226,9 @@ func (h *packedHandler) certifyPacked(ctx context.Context, rep *Report, volID, r // (never rclone's md5 slot, which is blank for a multipart object); other // backends read `rclone lsjson --hash`. A pack whose fingerprint could not // be read stays pending (checksum NULL) with a warning — never a fabricated -// value. It returns how many packs were fingerprint-verified this run. -func (h *packedHandler) capturePackFingerprints(ctx context.Context, rep *Report, runID int64, writes []store.PackWrite) int { - before := rep.Fingerprints +// value. The whole-pair pending tally (CountVolumeContentsPendingFingerprint) +// is what certifyPacked gates the vector advance on, so this returns nothing. +func (h *packedHandler) capturePackFingerprints(ctx context.Context, rep *Report, runID int64, writes []store.PackWrite) { targets := make([]captureTarget, 0, len(writes)) for _, w := range writes { pack, err := h.store.GetPackByKey(ctx, w.Pack.PackKey) @@ -241,7 +252,6 @@ func (h *packedHandler) capturePackFingerprints(ctx context.Context, rep *Report }) } h.captureScanBackFingerprints(ctx, rep, PacksDirName, targets) - return int(rep.Fingerprints - before) } // watermark resolves the run id the delta starts after: the last diff --git a/sync/packed_test.go b/sync/packed_test.go index ba285b3..847ce53 100644 --- a/sync/packed_test.go +++ b/sync/packed_test.go @@ -180,8 +180,8 @@ func TestPackedDurabilityNotCertified(t *testing.T) { if rep.Status != store.RunStatusSuccess { t.Fatalf("Status = %q, want success", rep.Status) } - // presence+size is not itself a content-verified method; the gate - // upgrades it through the pack's verified fingerprint. + // The run report stays unverified (presence+size); the durability + // vector is upgraded separately once the pack's fingerprint lands. if rep.Verification.Verified() { t.Fatalf("packed push reported verified; presence+size is not content-verified") } @@ -196,8 +196,10 @@ func TestPackedDurabilityNotCertified(t *testing.T) { if err != nil { t.Fatalf("ListDestinationRunIDs: %v", err) } - if len(vector) != 1 || vector[0].VerifyMethod != store.VerifyMethodPresenceSize { - t.Fatalf("vector = %+v, want one presence+size component (certified after fingerprint)", vector) + // With the pack fingerprinted the whole pair is fingerprint-verified, + // so certify advances the vector to fingerprint-verified (#155). + if len(vector) != 1 || vector[0].VerifyMethod != store.VerifyMethodFingerprint { + t.Fatalf("vector = %+v, want one fingerprint-verified component (certified after fingerprint)", vector) } }) @@ -233,6 +235,75 @@ func TestPackedDurabilityNotCertified(t *testing.T) { }) } +// TestPackedVerifyThenAdvance is the friction-log F13(b) fix for packed: a +// pack whose fingerprint capture failed holds the whole vector (empty), and +// a later `squirrel verify` that fills the fingerprint re-attempts the +// advance itself — upgrading the vector to fingerprint-verified rather than +// stalling until the next content-writing sync. +func TestPackedVerifyThenAdvance(t *testing.T) { + f := setupPackedFixture(t, "1MiB") + ctx := context.Background() + t.Setenv("RCLONE_FAKE_NO_HASHES", "1") + f.write(t, "small.txt", "tiny") + f.index(t) + if _, err := f.sync(t); err != nil { + t.Fatalf("sync: %v", err) + } + if vec, _ := f.store.ListDestinationRunIDs(ctx, f.volumeID(t), "offsite"); len(vec) != 0 { + t.Fatalf("vector = %+v, want empty while the pack fingerprint is pending", vec) + } + + t.Setenv("RCLONE_FAKE_NO_HASHES", "") + rep, err := VerifyRemote(ctx, f.store, f.rcl, f.pair.Destination) + if err != nil { + t.Fatalf("VerifyRemote: %v", err) + } + if rep.PacksPopulated != 1 || !rep.Clean() { + t.Fatalf("verify rep = %+v, want one pack fingerprint populated on a clean pass", rep) + } + vec, err := f.store.ListDestinationRunIDs(ctx, f.volumeID(t), "offsite") + if err != nil { + t.Fatalf("ListDestinationRunIDs: %v", err) + } + if len(vec) != 1 || vec[0].VerifyMethod != store.VerifyMethodFingerprint { + t.Fatalf("vector = %+v, want one fingerprint-verified component after verify", vec) + } +} + +// TestPackedLaterRunHeldByPendingPack is the friction-log F13(a) fix: a +// later run whose own pack is fingerprint-verified must NOT advance the +// vector while an earlier run's pack is still pending. The per-state gate +// holds the whole advance — the old per-run gate advanced vacuously past +// the earlier pending pack. +func TestPackedLaterRunHeldByPendingPack(t *testing.T) { + f := setupPackedFixture(t, "1MiB") + ctx := context.Background() + + // Run A: the pack lands but the backend exposes no checksum -> pending. + t.Setenv("RCLONE_FAKE_NO_HASHES", "1") + f.write(t, "one.txt", "first") + f.index(t) + if _, err := f.sync(t); err != nil { + t.Fatalf("run A sync: %v", err) + } + + // Run B: its own pack IS fingerprinted, but run A's pack stays pending. + t.Setenv("RCLONE_FAKE_NO_HASHES", "") + f.write(t, "two.txt", "second") + f.index(t) + if _, err := f.sync(t); err != nil { + t.Fatalf("run B sync: %v", err) + } + + vec, err := f.store.ListDestinationRunIDs(ctx, f.volumeID(t), "offsite") + if err != nil { + t.Fatalf("ListDestinationRunIDs: %v", err) + } + if len(vec) != 0 { + t.Fatalf("vector = %+v, want empty: run B must not advance past run A's still-pending pack", vec) + } +} + // TestPackedAssemblyDeterministic: the same input set produces byte-for-byte // identical pack bytes and the same pack key. func TestPackedAssemblyDeterministic(t *testing.T) { diff --git a/sync/verify_remote.go b/sync/verify_remote.go index 117d4b5..3eca4b5 100644 --- a/sync/verify_remote.go +++ b/sync/verify_remote.go @@ -123,9 +123,44 @@ func VerifyRemote(ctx context.Context, s *store.Store, rcl *Rclone, dest *config if err := recordVerifyOutcome(ctx, s, &rep, verifyErr); err != nil { return rep, err } + // A clean pass may have filled the last pending fingerprint of a + // (volume, destination) pair: re-attempt the durability-vector upgrade + // now so the vector no longer stalls until the next content-writing + // sync (friction-log F13). A dirty pass (mismatch or missing artifact) + // leaves the recorded fingerprints as found, so it must not advance + // evidence. Like RunPair's advance, an upgrade failure surfaces as the + // command's error even though the audit run already closed. + if verifyErr == nil && rep.Clean() { + if err := upgradeFingerprintVectors(ctx, s, dest.Name); err != nil { + return rep, err + } + } return rep, verifyErr } +// upgradeFingerprintVectors re-stamps the durability vector of every +// volume as fingerprint-verified where the destination now carries a +// verified fingerprint behind all its present content +// (UpgradeDestinationVectorToFingerprintVerified is a no-op for a volume +// that was never pushed to the destination or still has a pending +// artifact, so iterating every volume is safe and self-limiting). +func upgradeFingerprintVectors(ctx context.Context, s *store.Store, destination string) error { + self, err := s.GetSelfNode(ctx) + if err != nil { + return fmt.Errorf("verify %q: self node: %w", destination, err) + } + volumes, err := s.ListVolumes(ctx) + if err != nil { + return fmt.Errorf("verify %q: list volumes: %w", destination, err) + } + for _, v := range volumes { + if _, err := s.UpgradeDestinationVectorToFingerprintVerified(ctx, v.ID, destination, self.ID); err != nil { + return fmt.Errorf("verify %q: upgrade vector for volume %q: %w", destination, v.Name, err) + } + } + return nil +} + // verifyRecorded sweeps a destination's recorded content objects (the // large-file per-object sweep, shared with content-addressed) and, for a // packed destination, its recorded packs — one fingerprint check per pack diff --git a/syncproto/syncproto.go b/syncproto/syncproto.go index 4c61fa0..5809c48 100644 --- a/syncproto/syncproto.go +++ b/syncproto/syncproto.go @@ -396,13 +396,25 @@ type DurabilityResponse struct { // verification time is unknown (a pre-v23 responder, or evidence never // re-verified); the puller treats that as fail-closed. The responder's // own verification instant, never fresher than this hop's pull. +// +// VerifyEveryNs is the responder's effective verify cadence (in +// nanoseconds) for Destination — a per-destination verify_every or the +// [agent] default — relayed so the puller can hold the cadence coupling +// the offload gate requires of a fingerprint-verified component: that +// evidence counts as content-verified only while it keeps being +// re-confirmed. The puller keeps a relayed VerifyMethodFingerprint +// component content-verified only when this is positive; a zero (an older +// responder that never sends it, or a destination the responder configures +// with no cadence) fails the coupling closed. Mirrors VerifiedAtNs: relayed +// verbatim, absence is the safe direction. Immaterial for any other method. type DurabilityComponent struct { - Destination string `json:"destination"` - OriginNode string `json:"origin_node"` - OriginRun int64 `json:"origin_run"` - UpdatedAtNs int64 `json:"updated_at_ns"` - VerifyMethod string `json:"verify_method,omitempty"` - VerifiedAtNs int64 `json:"verified_at_ns,omitempty"` + Destination string `json:"destination"` + OriginNode string `json:"origin_node"` + OriginRun int64 `json:"origin_run"` + UpdatedAtNs int64 `json:"updated_at_ns"` + VerifyMethod string `json:"verify_method,omitempty"` + VerifiedAtNs int64 `json:"verified_at_ns,omitempty"` + VerifyEveryNs int64 `json:"verify_every_ns,omitempty"` } // DurabilityFreshness is one origin-space freshness coordinate: the From 11920f849c1e0ce1dcf520c51a546e40dc38f49c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 12:04:38 +0000 Subject: [PATCH 2/2] packed: correct the certifyPacked durability-gate comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comment implied the fingerprint pending/advance gate covers the placement map and manifest segment, but CountVolumeContentsPendingFingerprint only considers the volume's present file contents (verified via remote_objects / remote_packs). Reword to state exactly what the gate covers — packs and per-hash objects, the artifacts that carry content bytes — and note that the placement map and manifest segment are re-derivable metadata whose landing is confirmed upstream in push (presence+size, before the run is promoted to success), so a run whose map or segment failed to land never reaches certifyPacked. Comment-only; no behaviour change. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- sync/packed.go | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/sync/packed.go b/sync/packed.go index 27d652d..b0f0c45 100644 --- a/sync/packed.go +++ b/sync/packed.go @@ -190,16 +190,29 @@ func (h *packedHandler) push(ctx context.Context, rep *Report, volID, runID int6 // per landed pack, reads each pack's scan-back fingerprint, and advances // the destination vector to fingerprint-verified only once the whole // (volume, destination) pair carries a verified fingerprint behind every -// present content — packs, per-hash objects, placement map, and manifest -// segment all landed and confirmed. Gating on the whole pending set rather -// than this run's writes is the friction-log F13 fix: a run that packs -// nothing no longer advances vacuously past an earlier still-pending pack, -// and a still-pending artifact anywhere in the pair holds the whole -// advance. `squirrel verify` fills a pending fingerprint later and -// re-attempts this advance itself (see the store upgrade), so the vector no -// longer stalls until the next content-writing sync. rep.Verification stays -// unverified (presence+size is not content-verified), so RunPair does not -// advance the vector a second time. +// present file content. That whole-state check is +// CountVolumeContentsPendingFingerprint, which counts present contents +// lacking a verified remote_objects (per-hash object) or remote_packs (pack +// member) fingerprint on this destination — so it covers packs and per-hash +// objects, the two artifacts that carry content bytes. +// +// The placement map and manifest segment are deliberately NOT in that +// pending set: they are re-derivable squirrel-written metadata that carry +// no scan-back fingerprint. Their durability before this advance is handled +// upstream in push, which uploads and confirms both landed at their +// expected size before promoting the run to success — a run whose map or +// segment failed to land returns failed and never reaches certifyPacked, so +// the vector is untouched. +// +// Gating on the whole pending set rather than this run's writes is the +// friction-log F13 fix: a run that packs nothing no longer advances +// vacuously past an earlier still-pending pack, and a still-pending artifact +// anywhere in the pair holds the whole advance. `squirrel verify` fills a +// pending fingerprint later and re-attempts this advance itself (see the +// store upgrade), so the vector no longer stalls until the next +// content-writing sync. rep.Verification stays unverified (presence+size is +// not content-verified), so RunPair does not advance the vector a second +// time. func (h *packedHandler) certifyPacked(ctx context.Context, rep *Report, volID, runID int64, writes []store.PackWrite, advance []store.OriginComponent) error { h.capturePackFingerprints(ctx, rep, runID, writes) pending, err := h.store.CountVolumeContentsPendingFingerprint(ctx, volID, h.dest.Name)