From 61aa1f851d38ae92cadf999801722554ce1b8977 Mon Sep 17 00:00:00 2001 From: Magic <131926685+magiccodingman@users.noreply.github.com> Date: Sat, 25 Jul 2026 18:09:18 -0400 Subject: [PATCH] Fix connected IPNS version pruning --- .../IpnsVersionRetentionTests.cs | 144 ++++++++++++ .../TruthGate-Web/Properties/AssemblyInfo.cs | 3 + .../Services/IpnsUpdateWorker.cs | 125 ++++------ .../Services/IpnsVersionRetention.cs | 218 ++++++++++++++++++ 4 files changed, 410 insertions(+), 80 deletions(-) create mode 100644 TruthGate-Web/TruthGate-Web.Tests/IpnsVersionRetentionTests.cs create mode 100644 TruthGate-Web/TruthGate-Web/Properties/AssemblyInfo.cs create mode 100644 TruthGate-Web/TruthGate-Web/Services/IpnsVersionRetention.cs diff --git a/TruthGate-Web/TruthGate-Web.Tests/IpnsVersionRetentionTests.cs b/TruthGate-Web/TruthGate-Web.Tests/IpnsVersionRetentionTests.cs new file mode 100644 index 0000000..60f7445 --- /dev/null +++ b/TruthGate-Web/TruthGate-Web.Tests/IpnsVersionRetentionTests.cs @@ -0,0 +1,144 @@ +using TruthGate_Web.Services; + +namespace TruthGate_Web.Tests; + +public sealed class IpnsVersionRetentionTests +{ + [Theory] + [InlineData("site", "site-v001", 1, false)] + [InlineData("site", "site-v001-connected", 1, true)] + [InlineData("My-Site", "my-site-v042-CONNECTED", 42, true)] + public void TryParseEntryName_RecognizesPointerAndConnectedNames( + string trackedName, + string entryName, + int expectedVersion, + bool expectedConnected) + { + var parsed = IpnsVersionRetention.TryParseEntryName( + trackedName, + entryName, + out var version, + out var connected); + + Assert.True(parsed); + Assert.Equal(expectedVersion, version); + Assert.Equal(expectedConnected, connected); + } + + [Theory] + [InlineData("site", "site-v")] + [InlineData("site", "site-v001-extra")] + [InlineData("site", "other-v001")] + [InlineData("site", "site-v001-connected-extra")] + public void TryParseEntryName_RejectsUnmanagedNames(string trackedName, string entryName) + => Assert.False(IpnsVersionRetention.TryParseEntryName(trackedName, entryName, out _, out _)); + + [Fact] + public void GroupVersions_PairsPointerAndConnectedFolders() + { + var children = Children( + ("site-v001", "pointer-1"), + ("site-v001-connected", "target-1"), + ("site-v002", "pointer-2"), + ("site-v002-connected", "target-2")); + + var pairs = IpnsVersionRetention.GroupVersions("site", children); + + Assert.Equal(2, pairs.Count); + Assert.Equal("pointer-1", pairs[0].Pointer?.Cid); + Assert.Equal("target-1", pairs[0].Connected?.Cid); + Assert.Equal("pointer-2", pairs[1].Pointer?.Cid); + Assert.Equal("target-2", pairs[1].Connected?.Cid); + Assert.Equal(3, IpnsVersionRetention.ComputeNextVersion("site", children.Keys)); + } + + [Fact] + public void BuildPrunePlan_RemovesOldPointerAndConnectedPair() + { + var pairs = IpnsVersionRetention.GroupVersions( + "site", + Children( + ("site-v001", "pointer-1"), + ("site-v001-connected", "target-1"), + ("site-v002", "pointer-2"), + ("site-v002-connected", "target-2"))); + + var plan = IpnsVersionRetention.BuildPrunePlan( + pairs, + retainedVersion: 2, + new Dictionary { [1] = "target-1", [2] = "target-2" }); + + Assert.Contains("/production/pinned/site-v001", plan.PathsToRemove); + Assert.Contains("/production/pinned/site-v001-connected", plan.PathsToRemove); + Assert.DoesNotContain("/production/pinned/site-v002", plan.PathsToRemove); + Assert.Contains("pointer-1", plan.CidsToUnpin); + Assert.Contains("target-1", plan.CidsToUnpin); + Assert.DoesNotContain("pointer-2", plan.CidsToUnpin); + Assert.DoesNotContain("target-2", plan.CidsToUnpin); + } + + [Fact] + public void BuildPrunePlan_DoesNotUnpinTargetSharedWithRetainedVersion() + { + var pairs = IpnsVersionRetention.GroupVersions( + "site", + Children( + ("site-v001", "pointer-1"), + ("site-v001-connected", "shared-target"), + ("site-v002", "pointer-2"), + ("site-v002-connected", "shared-target"))); + + var plan = IpnsVersionRetention.BuildPrunePlan( + pairs, + retainedVersion: 2, + new Dictionary { [1] = "shared-target", [2] = "shared-target" }); + + Assert.Contains("pointer-1", plan.CidsToUnpin); + Assert.DoesNotContain("shared-target", plan.CidsToUnpin); + } + + [Fact] + public void BuildPrunePlan_RemovesOrphanConnectedFolderOutsideRetainedVersion() + { + var pairs = IpnsVersionRetention.GroupVersions( + "site", + Children( + ("site-v002", "pointer-2"), + ("site-v002-connected", "target-2"), + ("site-v003-connected", "orphan-target"))); + + var latest = IpnsVersionRetention.GetLatestPointerPair(pairs); + Assert.NotNull(latest); + Assert.Equal(2, latest.Version); + + var plan = IpnsVersionRetention.BuildPrunePlan( + pairs, + latest.Version, + new Dictionary { [2] = "target-2", [3] = "orphan-target" }); + + Assert.Contains("/production/pinned/site-v003-connected", plan.PathsToRemove); + Assert.Contains("orphan-target", plan.CidsToUnpin); + } + + [Theory] + [InlineData("{"tgp":1,"current":"bafy-current"}", "bafy-current")] + [InlineData("{"tgp":1,"current":"/ipfs/bafy-current"}", "bafy-current")] + [InlineData("{"tgp":2,"current":"bafy-current"}", null)] + [InlineData("not-json", null)] + public void TryReadTgpCurrentCid_ParsesVersionOnePointer(string json, string? expected) + => Assert.Equal(expected, IpnsVersionRetention.TryReadTgpCurrentCid(json)); + + [Fact] + public void TryReadLegacyTargetCid_PreservesSidecarCompatibility() + { + const string json = "{"Kind":"tgp-meta","PointerCid":"pointer","TargetCid":"/ipfs/legacy-target"}"; + Assert.Equal("legacy-target", IpnsVersionRetention.TryReadLegacyTargetCid(json)); + } + + private static Dictionary Children( + params (string Name, string Cid)[] entries) + => entries.ToDictionary( + entry => entry.Name, + entry => (entry.Cid, $"/production/pinned/{entry.Name}"), + StringComparer.OrdinalIgnoreCase); +} diff --git a/TruthGate-Web/TruthGate-Web/Properties/AssemblyInfo.cs b/TruthGate-Web/TruthGate-Web/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..6872647 --- /dev/null +++ b/TruthGate-Web/TruthGate-Web/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("TruthGate-Web.Tests")] diff --git a/TruthGate-Web/TruthGate-Web/Services/IpnsUpdateWorker.cs b/TruthGate-Web/TruthGate-Web/Services/IpnsUpdateWorker.cs index 634f914..f933114 100644 --- a/TruthGate-Web/TruthGate-Web/Services/IpnsUpdateWorker.cs +++ b/TruthGate-Web/TruthGate-Web/Services/IpnsUpdateWorker.cs @@ -1,9 +1,8 @@ -using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Options; using System.Collections.Concurrent; using System.Text; using System.Text.Json; -using System.Text.RegularExpressions; using TruthGate_Web.Endpoints; using TruthGate_Web.Models; using TruthGate_Web.Utils; @@ -43,11 +42,10 @@ public sealed class IpnsUpdateWorker : BackgroundService, IIpnsUpdateService private readonly IpnsUpdateOptions _opts; private readonly IApiKeyProvider _keys; - private static readonly Regex VersionRx = new(@"-v(?\d+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase); - private const string ManagedRoot = "/production/pinned"; private const string StagingRoot = "/production/.staging/ipns"; - private const string TgpMetaFile = "tgp.json"; // lives in each version folder + private const string TgpPointerFile = "tgp.json"; + private const string LegacyTgpMetaFile = ".tgp-meta.json"; // concurrency private readonly SemaphoreSlim _globalSlots; @@ -79,34 +77,13 @@ public IpnsUpdateWorker( _globalSlots = new SemaphoreSlim(_opts.MaxConcurrency, _opts.MaxConcurrency); } - private sealed record VersionEntry(int N, string Name, string Path, string Cid, bool IsConnected); - - private async Task<(VersionEntry? Pointer, VersionEntry? Connected)> GetLatestVersionPairAsync(string name, CancellationToken ct) + private async Task<(IpnsVersionRetention.VersionEntry? Pointer, IpnsVersionRetention.VersionEntry? Connected)> GetLatestVersionPairAsync( + string name, + CancellationToken ct) { var children = await ListMfsChildrenAsync(ManagedRoot, ct); - var prefix = $"{name}-v"; - VersionEntry? latestPointer = null; - VersionEntry? latestConnected = null; - - foreach (var kv in children) - { - if (!kv.Key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) continue; - var m = VersionRx.Match(kv.Key); - if (!m.Success || !int.TryParse(m.Groups["n"].Value, out var n)) continue; - - var isConnected = kv.Key.EndsWith("-connected", StringComparison.OrdinalIgnoreCase); - var entry = new VersionEntry(n, kv.Key, kv.Value.Path, kv.Value.Cid, isConnected); - - if (isConnected) - { - if (latestConnected is null || n > latestConnected.N) latestConnected = entry; - } - else - { - if (latestPointer is null || n > latestPointer.N) latestPointer = entry; - } - } - return (latestPointer, latestConnected); + var latest = IpnsVersionRetention.GetLatestPointerPair(name, children); + return (latest?.Pointer, latest?.Connected); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -338,13 +315,6 @@ private SemaphoreSlim GetKeyLock(string name) => _perKeyLocks.GetOrAdd(name ?? string.Empty, _ => new SemaphoreSlim(1, 1)); // ---------- TGP helpers ---------- - private sealed class TgpMeta - { - public string Kind { get; set; } = "tgp"; - public string? PointerCid { get; set; } - public string? TargetCid { get; set; } - } - private async Task TryReadTgpTargetCidAsync(string pointerCid, CancellationToken ct) { try @@ -484,16 +454,7 @@ private async Task PinRmRecursiveAsync(string cid, CancellationToken ct) private async Task ComputeNextVersionAsync(string name, CancellationToken ct) { var children = await ListMfsChildrenAsync(ManagedRoot, ct); - var prefix = $"{name}-v"; - var max = 0; - foreach (var k in children.Keys) - { - if (!k.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) continue; - var m = VersionRx.Match(k); - if (m.Success && int.TryParse(m.Groups["n"].Value, out var n)) - max = Math.Max(max, n); - } - return max + 1; + return IpnsVersionRetention.ComputeNextVersion(name, children.Keys); } private async Task FilesWriteTextAsync(string mfsPath, string text, CancellationToken ct) { @@ -513,48 +474,52 @@ private async Task FilesWriteTextAsync(string mfsPath, string text, Cancellation private async Task RemoveAllButLatestAsync(string name, CancellationToken ct) { var children = await ListMfsChildrenAsync(ManagedRoot, ct); - var prefix = $"{name}-v"; - var versions = new List<(int n, string path, string cid)>(); - foreach (var kv in children) + var pairs = IpnsVersionRetention.GroupVersions(name, children); + var latest = IpnsVersionRetention.GetLatestPointerPair(pairs); + if (latest is null) return; + + var targetCidsByVersion = new Dictionary(); + foreach (var pair in pairs) { - if (!kv.Key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) continue; - var m = VersionRx.Match(kv.Key); - if (!m.Success || !int.TryParse(m.Groups["n"].Value, out var n)) continue; - versions.Add((n, kv.Value.Path, kv.Value.Cid)); + string? targetCid = pair.Connected?.Cid; + if (string.IsNullOrWhiteSpace(targetCid) && pair.Pointer is not null) + { + var tgpJson = await FilesReadAllTextAsync($"{pair.Pointer.Path}/{TgpPointerFile}", ct); + targetCid = IpnsVersionRetention.TryReadTgpCurrentCid(tgpJson); + + if (string.IsNullOrWhiteSpace(targetCid)) + { + var legacyJson = await FilesReadAllTextAsync($"{pair.Pointer.Path}/{LegacyTgpMetaFile}", ct); + targetCid = IpnsVersionRetention.TryReadLegacyTargetCid(legacyJson); + } + } + + targetCidsByVersion[pair.Version] = targetCid; } - if (versions.Count <= 1) return; - var latest = versions.OrderByDescending(v => v.n).First(); - foreach (var v in versions.Where(v => v.n != latest.n)) + var plan = IpnsVersionRetention.BuildPrunePlan(pairs, latest.Version, targetCidsByVersion); + + foreach (var path in plan.PathsToRemove) { try { - // Read sidecar to learn any extra CIDs (TGP target) to unpin - string? metaJson = await FilesReadAllTextAsync($"{v.path}/{TgpMetaFile}", ct); - string? tgpTarget = null; - if (!string.IsNullOrWhiteSpace(metaJson)) - { - try - { - var m = JsonSerializer.Deserialize(metaJson); - if (m?.TargetCid is string s && !string.IsNullOrWhiteSpace(s)) - tgpTarget = s.Trim(); - } - catch { /* ignore */ } - } + await FilesRmRecursiveAsync(path, ct); + } + catch (Exception ex) + { + _log.LogWarning(ex, "Failed to remove old IPNS version path {Path} for {Name}.", path, name); + } + } - // Remove folder then unpin both pointer and (if present) target - await FilesRmRecursiveAsync(v.path, ct); - await PinRmRecursiveAsync(v.cid, ct); - if (!string.IsNullOrWhiteSpace(tgpTarget)) - { - try { await PinRmRecursiveAsync(tgpTarget!, ct); } - catch (Exception ex) { _log.LogWarning(ex, "Failed to unpin TGP target for {Name}-v{Version}", name, v.n); } - } + foreach (var cid in plan.CidsToUnpin) + { + try + { + await PinRmRecursiveAsync(cid, ct); } catch (Exception ex) { - _log.LogWarning(ex, "Failed to remove old version {Name}-v{Version}.", name, v.n); + _log.LogWarning(ex, "Failed to unpin old IPNS version CID {Cid} for {Name}.", cid, name); } } } diff --git a/TruthGate-Web/TruthGate-Web/Services/IpnsVersionRetention.cs b/TruthGate-Web/TruthGate-Web/Services/IpnsVersionRetention.cs new file mode 100644 index 0000000..cbca38b --- /dev/null +++ b/TruthGate-Web/TruthGate-Web/Services/IpnsVersionRetention.cs @@ -0,0 +1,218 @@ +using System.Globalization; +using System.Text.Json; + +namespace TruthGate_Web.Services; + +internal static class IpnsVersionRetention +{ + private const string ConnectedSuffix = "-connected"; + + internal sealed record VersionEntry( + int Version, + string Name, + string Path, + string Cid, + bool IsConnected); + + internal sealed class VersionPair + { + internal VersionPair(int version) => Version = version; + + internal int Version { get; } + internal VersionEntry? Pointer { get; set; } + internal VersionEntry? Connected { get; set; } + } + + internal sealed record PrunePlan( + IReadOnlyList PathsToRemove, + IReadOnlyList CidsToUnpin); + + internal static bool TryParseEntryName( + string trackedName, + string entryName, + out int version, + out bool isConnected) + { + version = 0; + isConnected = false; + + if (string.IsNullOrWhiteSpace(trackedName) || string.IsNullOrWhiteSpace(entryName)) + return false; + + var prefix = $"{trackedName}-v"; + if (!entryName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + return false; + + var versionText = entryName[prefix.Length..]; + if (versionText.EndsWith(ConnectedSuffix, StringComparison.OrdinalIgnoreCase)) + { + isConnected = true; + versionText = versionText[..^ConnectedSuffix.Length]; + } + + return versionText.Length > 0 + && int.TryParse( + versionText, + NumberStyles.None, + CultureInfo.InvariantCulture, + out version); + } + + internal static IReadOnlyList GroupVersions( + string trackedName, + IReadOnlyDictionary children) + { + var pairs = new SortedDictionary(); + + foreach (var child in children) + { + if (!TryParseEntryName(trackedName, child.Key, out var version, out var isConnected)) + continue; + + if (!pairs.TryGetValue(version, out var pair)) + { + pair = new VersionPair(version); + pairs.Add(version, pair); + } + + var entry = new VersionEntry( + version, + child.Key, + child.Value.Path, + child.Value.Cid, + isConnected); + + if (isConnected) + pair.Connected = entry; + else + pair.Pointer = entry; + } + + return pairs.Values.ToArray(); + } + + internal static VersionPair? GetLatestPointerPair( + string trackedName, + IReadOnlyDictionary children) + => GetLatestPointerPair(GroupVersions(trackedName, children)); + + internal static VersionPair? GetLatestPointerPair(IEnumerable pairs) + => pairs + .Where(pair => pair.Pointer is not null) + .OrderByDescending(pair => pair.Version) + .FirstOrDefault(); + + internal static int ComputeNextVersion(string trackedName, IEnumerable childNames) + { + var max = 0; + foreach (var childName in childNames) + { + if (TryParseEntryName(trackedName, childName, out var version, out _)) + max = Math.Max(max, version); + } + + return max + 1; + } + + internal static PrunePlan BuildPrunePlan( + IReadOnlyList pairs, + int retainedVersion, + IReadOnlyDictionary targetCidsByVersion) + { + var retained = pairs.FirstOrDefault(pair => pair.Version == retainedVersion) + ?? throw new ArgumentOutOfRangeException( + nameof(retainedVersion), + retainedVersion, + "The retained version must exist in the version set."); + + var retainedCids = new HashSet(StringComparer.OrdinalIgnoreCase); + AddCid(retainedCids, retained.Pointer?.Cid); + AddCid(retainedCids, retained.Connected?.Cid); + if (targetCidsByVersion.TryGetValue(retainedVersion, out var retainedTarget)) + AddCid(retainedCids, retainedTarget); + + var paths = new HashSet(StringComparer.OrdinalIgnoreCase); + var cids = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var pair in pairs.Where(pair => pair.Version != retainedVersion)) + { + AddPath(paths, pair.Pointer?.Path); + AddPath(paths, pair.Connected?.Path); + + AddCandidateCid(cids, retainedCids, pair.Pointer?.Cid); + AddCandidateCid(cids, retainedCids, pair.Connected?.Cid); + if (targetCidsByVersion.TryGetValue(pair.Version, out var targetCid)) + AddCandidateCid(cids, retainedCids, targetCid); + } + + return new PrunePlan(paths.ToArray(), cids.ToArray()); + } + + internal static string? TryReadTgpCurrentCid(string? json) + { + if (string.IsNullOrWhiteSpace(json)) return null; + + try + { + using var document = JsonDocument.Parse(json); + var root = document.RootElement; + if (!root.TryGetProperty("tgp", out var tgp) + || tgp.ValueKind != JsonValueKind.Number + || tgp.GetInt32() != 1 + || !root.TryGetProperty("current", out var current) + || current.ValueKind != JsonValueKind.String) + return null; + + return NormalizeCid(current.GetString()); + } + catch (JsonException) + { + return null; + } + } + + internal static string? TryReadLegacyTargetCid(string? json) + { + if (string.IsNullOrWhiteSpace(json)) return null; + + try + { + using var document = JsonDocument.Parse(json); + if (!document.RootElement.TryGetProperty("TargetCid", out var target) + || target.ValueKind != JsonValueKind.String) + return null; + + return NormalizeCid(target.GetString()); + } + catch (JsonException) + { + return null; + } + } + + private static string? NormalizeCid(string? value) + { + var cid = value?.Trim(); + if (string.IsNullOrWhiteSpace(cid)) return null; + return cid.StartsWith("/ipfs/", StringComparison.OrdinalIgnoreCase) + ? cid[6..] + : cid; + } + + private static void AddPath(ISet paths, string? path) + { + if (!string.IsNullOrWhiteSpace(path)) paths.Add(path); + } + + private static void AddCid(ISet cids, string? cid) + { + if (!string.IsNullOrWhiteSpace(cid)) cids.Add(cid.Trim()); + } + + private static void AddCandidateCid(ISet candidates, ISet retained, string? cid) + { + if (string.IsNullOrWhiteSpace(cid)) return; + cid = cid.Trim(); + if (!retained.Contains(cid)) candidates.Add(cid); + } +}