From 35098a8e853b1b563e277dc0bdc9141ac705ac42 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 22 May 2026 18:05:51 +0200 Subject: [PATCH 1/5] feat: cache SHA3-384 entries --- internal/archive/archive.go | 7 +-- internal/cache/cache.go | 83 +++++++++++++++++++++++++----------- internal/cache/cache_test.go | 59 ++++++++++++++++++------- 3 files changed, 104 insertions(+), 45 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index e24a9c24..b12c8dba 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -374,7 +374,8 @@ func (index *ubuntuIndex) distPath(suffix string) string { } func (index *ubuntuIndex) fetch(path, digest string, flags fetchFlags) (io.ReadSeekCloser, error) { - reader, err := index.archive.cache.Open(digest) + const algo = cache.SHA256 + reader, err := index.archive.cache.Open(algo, digest) if err == nil { return reader, nil } else if err != cache.MissErr { @@ -425,7 +426,7 @@ func (index *ubuntuIndex) fetch(path, digest string, flags fetchFlags) (io.ReadS body = reader } - writer := index.archive.cache.Create(digest) + writer := index.archive.cache.Create(algo, digest) defer writer.Close() _, err = io.Copy(writer, body) @@ -436,7 +437,7 @@ func (index *ubuntuIndex) fetch(path, digest string, flags fetchFlags) (io.ReadS return nil, fmt.Errorf("cannot fetch from archive: %v", err) } - return index.archive.cache.Open(writer.Digest()) + return index.archive.cache.Open(algo, writer.Digest()) } func sectionPackageInfo(section control.Section) *PackageInfo { diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 1dabc0d1..7e01168f 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -9,6 +9,8 @@ import ( "os" "path/filepath" "time" + + "golang.org/x/crypto/sha3" ) func DefaultDir(suffix string) string { @@ -89,27 +91,50 @@ func (cw *Writer) Digest() string { return cw.digest } -const digestKind = "sha256" +// hashAlgo identifies a supported hash algorithm. +type hashAlgo string + +const ( + SHA256 hashAlgo = "sha256" + SHA3384 hashAlgo = "sha3-384" +) + +var hashAlgos = []hashAlgo{SHA256, SHA3384} + +func newHash(algo hashAlgo) (hash.Hash, error) { + switch algo { + case SHA256: + return sha256.New(), nil + case SHA3384: + return sha3.New384(), nil + default: + return nil, fmt.Errorf("unsupported hash algorithm: %q", algo) + } +} var MissErr = fmt.Errorf("not cached") -func (c *Cache) filePath(digest string) string { - return filepath.Join(c.Dir, digestKind, digest) +func (c *Cache) filePath(algo hashAlgo, digest string) string { + return filepath.Join(c.Dir, string(algo), digest) } -func (c *Cache) Create(digest string) *Writer { +func (c *Cache) Create(algo hashAlgo, digest string) *Writer { if c.Dir == "" { return &Writer{err: fmt.Errorf("internal error: cache directory is unset")} } - err := os.MkdirAll(filepath.Join(c.Dir, digestKind), 0755) + h, err := newHash(algo) + if err != nil { + return &Writer{err: fmt.Errorf("cannot create cache entry: %v", err)} + } + err = os.MkdirAll(filepath.Join(c.Dir, string(algo)), 0755) if err != nil { return &Writer{err: fmt.Errorf("cannot create cache directory: %v", err)} } var file *os.File if digest == "" { - file, err = os.CreateTemp(c.filePath(""), "tmp.*") + file, err = os.CreateTemp(filepath.Join(c.Dir, string(algo)), "tmp.*") } else { - file, err = os.Create(c.filePath(digest + ".tmp")) + file, err = os.Create(c.filePath(algo, digest+".tmp")) } if err != nil { return &Writer{err: fmt.Errorf("cannot create cache file: %v", err)} @@ -117,13 +142,13 @@ func (c *Cache) Create(digest string) *Writer { return &Writer{ dir: c.Dir, digest: digest, - hash: sha256.New(), + hash: h, file: file, } } -func (c *Cache) Write(digest string, data []byte) error { - f := c.Create(digest) +func (c *Cache) Write(algo hashAlgo, digest string, data []byte) error { + f := c.Create(algo, digest) _, err1 := f.Write(data) err2 := f.Close() if err1 != nil { @@ -132,11 +157,11 @@ func (c *Cache) Write(digest string, data []byte) error { return err2 } -func (c *Cache) Open(digest string) (io.ReadSeekCloser, error) { +func (c *Cache) Open(algo hashAlgo, digest string) (io.ReadSeekCloser, error) { if c.Dir == "" || digest == "" { return nil, MissErr } - filePath := c.filePath(digest) + filePath := c.filePath(algo, digest) file, err := os.Open(filePath) if os.IsNotExist(err) { return nil, MissErr @@ -151,8 +176,8 @@ func (c *Cache) Open(digest string) (io.ReadSeekCloser, error) { return file, nil } -func (c *Cache) Read(digest string) ([]byte, error) { - file, err := c.Open(digest) +func (c *Cache) Read(algo hashAlgo, digest string) ([]byte, error) { + file, err := c.Open(algo, digest) if err != nil { return nil, err } @@ -165,22 +190,28 @@ func (c *Cache) Read(digest string) ([]byte, error) { } func (c *Cache) Expire(timeout time.Duration) error { - entries, err := os.ReadDir(filepath.Join(c.Dir, digestKind)) - if err != nil { - return fmt.Errorf("cannot list cache directory: %v", err) - } expired := time.Now().Add(-timeout) - for _, entry := range entries { - finfo, err := entry.Info() - if err != nil { - return err - } - if finfo.ModTime().After(expired) { + for _, algo := range hashAlgos { + algoDir := filepath.Join(c.Dir, string(algo)) + entries, err := os.ReadDir(algoDir) + if os.IsNotExist(err) { continue } - err = os.Remove(filepath.Join(c.Dir, digestKind, finfo.Name())) if err != nil { - return fmt.Errorf("cannot expire cache entry: %v", err) + return fmt.Errorf("cannot list cache directory: %v", err) + } + for _, entry := range entries { + finfo, err := entry.Info() + if err != nil { + return err + } + if finfo.ModTime().After(expired) { + continue + } + err = os.Remove(filepath.Join(algoDir, finfo.Name())) + if err != nil { + return fmt.Errorf("cannot expire cache entry: %v", err) + } } } return nil diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 1ad7f397..347424c2 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -45,11 +45,11 @@ func (s *S) TestDefaultDir(c *C) { func (s *S) TestCacheEmpty(c *C) { cc := cache.Cache{c.MkDir()} - _, err := cc.Open(data1Digest) + _, err := cc.Open(cache.SHA256, data1Digest) c.Assert(err, Equals, cache.MissErr) - _, err = cc.Read(data1Digest) + _, err = cc.Read(cache.SHA256, data1Digest) c.Assert(err, Equals, cache.MissErr) - _, err = cc.Read("") + _, err = cc.Read(cache.SHA256, "") c.Assert(err, Equals, cache.MissErr) } @@ -60,21 +60,21 @@ func (s *S) TestCacheReadWrite(c *C) { data2Path := filepath.Join(cc.Dir, "sha256", data2Digest) data3Path := filepath.Join(cc.Dir, "sha256", data3Digest) - err := cc.Write(data1Digest, []byte("data1")) + err := cc.Write(cache.SHA256, data1Digest, []byte("data1")) c.Assert(err, IsNil) - data1, err := cc.Read(data1Digest) + data1, err := cc.Read(cache.SHA256, data1Digest) c.Assert(err, IsNil) c.Assert(string(data1), Equals, "data1") - err = cc.Write("", []byte("data2")) + err = cc.Write(cache.SHA256, "", []byte("data2")) c.Assert(err, IsNil) - data2, err := cc.Read(data2Digest) + data2, err := cc.Read(cache.SHA256, data2Digest) c.Assert(err, IsNil) c.Assert(string(data2), Equals, "data2") - _, err = cc.Read(data3Digest) + _, err = cc.Read(cache.SHA256, data3Digest) c.Assert(err, Equals, cache.MissErr) - _, err = cc.Read("") + _, err = cc.Read(cache.SHA256, "") c.Assert(err, Equals, cache.MissErr) _, err = os.Stat(data1Path) @@ -98,7 +98,7 @@ func (s *S) TestCacheReadWrite(c *C) { func (s *S) TestCacheCreate(c *C) { cc := cache.Cache{Dir: c.MkDir()} - w := cc.Create("") + w := cc.Create(cache.SHA256, "") c.Assert(w.Digest(), Equals, "") @@ -113,7 +113,7 @@ func (s *S) TestCacheCreate(c *C) { c.Assert(w.Digest(), Equals, data1Digest) - data1, err := cc.Read(data1Digest) + data1, err := cc.Read(cache.SHA256, data1Digest) c.Assert(err, IsNil) c.Assert(string(data1), Equals, "data1") } @@ -121,7 +121,7 @@ func (s *S) TestCacheCreate(c *C) { func (s *S) TestCacheWrongDigest(c *C) { cc := cache.Cache{Dir: c.MkDir()} - w := cc.Create(data1Digest) + w := cc.Create(cache.SHA256, data1Digest) c.Assert(w.Digest(), Equals, data1Digest) @@ -130,19 +130,19 @@ func (s *S) TestCacheWrongDigest(c *C) { c.Assert(err, IsNil) c.Assert(errClose, ErrorMatches, "expected digest "+data1Digest+", got "+data2Digest) - _, err = cc.Read(data1Digest) + _, err = cc.Read(cache.SHA256, data1Digest) c.Assert(err, Equals, cache.MissErr) - _, err = cc.Read(data2Digest) + _, err = cc.Read(cache.SHA256, data2Digest) c.Assert(err, Equals, cache.MissErr) } func (s *S) TestCacheOpen(c *C) { cc := cache.Cache{Dir: c.MkDir()} - err := cc.Write(data1Digest, []byte("data1")) + err := cc.Write(cache.SHA256, data1Digest, []byte("data1")) c.Assert(err, IsNil) - f, err := cc.Open(data1Digest) + f, err := cc.Open(cache.SHA256, data1Digest) c.Assert(err, IsNil) data1, err := io.ReadAll(f) closeErr := f.Close() @@ -151,3 +151,30 @@ func (s *S) TestCacheOpen(c *C) { c.Assert(string(data1), Equals, "data1") } + +func (s *S) TestCacheSHA3384(c *C) { + cc := cache.Cache{Dir: c.MkDir()} + + w := cc.Create(cache.SHA3384, "") + _, err := w.Write([]byte("data1")) + c.Assert(err, IsNil) + err = w.Close() + c.Assert(err, IsNil) + sha3Digest := w.Digest() + + c.Assert(sha3Digest, Not(Equals), data1Digest) + + data, err := cc.Read(cache.SHA3384, sha3Digest) + c.Assert(err, IsNil) + c.Assert(string(data), Equals, "data1") + + // SHA3-384 entry must not be visible under SHA-256. + _, err = cc.Open(cache.SHA256, sha3Digest) + c.Assert(err, Equals, cache.MissErr) + + // SHA-256 entry must not be visible under SHA3-384. + err = cc.Write(cache.SHA256, data1Digest, []byte("data1")) + c.Assert(err, IsNil) + _, err = cc.Open(cache.SHA3384, data1Digest) + c.Assert(err, Equals, cache.MissErr) +} From 34eee2a982d526534326bd9adfab96f6e3903cd8 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 26 May 2026 08:11:22 +0200 Subject: [PATCH 2/5] ci: retry From 65aa18393b683ffeb08a1f75d7dfa863f0812cb3 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 1 Jun 2026 16:06:10 +0200 Subject: [PATCH 3/5] style: rename SHA3_384 Signed-off-by: Paul Mars --- internal/cache/cache.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 7e01168f..b35fed34 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -95,17 +95,17 @@ func (cw *Writer) Digest() string { type hashAlgo string const ( - SHA256 hashAlgo = "sha256" - SHA3384 hashAlgo = "sha3-384" + SHA256 hashAlgo = "sha256" + SHA3_384 hashAlgo = "sha3-384" ) -var hashAlgos = []hashAlgo{SHA256, SHA3384} +var hashAlgos = []hashAlgo{SHA256, SHA3_384} func newHash(algo hashAlgo) (hash.Hash, error) { switch algo { case SHA256: return sha256.New(), nil - case SHA3384: + case SHA3_384: return sha3.New384(), nil default: return nil, fmt.Errorf("unsupported hash algorithm: %q", algo) From 013211f0d1425dedfc950cc06b3d74fa4c8b94be Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 1 Jun 2026 16:06:42 +0200 Subject: [PATCH 4/5] test: cover Expire method Signed-off-by: Paul Mars --- internal/cache/cache_test.go | 86 +++++++++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 5 deletions(-) diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 347424c2..c96cc785 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -1,14 +1,14 @@ package cache_test import ( - . "gopkg.in/check.v1" - "io" "os" "path/filepath" "strings" "time" + . "gopkg.in/check.v1" + "github.com/canonical/chisel/internal/cache" ) @@ -155,7 +155,7 @@ func (s *S) TestCacheOpen(c *C) { func (s *S) TestCacheSHA3384(c *C) { cc := cache.Cache{Dir: c.MkDir()} - w := cc.Create(cache.SHA3384, "") + w := cc.Create(cache.SHA3_384, "") _, err := w.Write([]byte("data1")) c.Assert(err, IsNil) err = w.Close() @@ -164,7 +164,7 @@ func (s *S) TestCacheSHA3384(c *C) { c.Assert(sha3Digest, Not(Equals), data1Digest) - data, err := cc.Read(cache.SHA3384, sha3Digest) + data, err := cc.Read(cache.SHA3_384, sha3Digest) c.Assert(err, IsNil) c.Assert(string(data), Equals, "data1") @@ -175,6 +175,82 @@ func (s *S) TestCacheSHA3384(c *C) { // SHA-256 entry must not be visible under SHA3-384. err = cc.Write(cache.SHA256, data1Digest, []byte("data1")) c.Assert(err, IsNil) - _, err = cc.Open(cache.SHA3384, data1Digest) + _, err = cc.Open(cache.SHA3_384, data1Digest) c.Assert(err, Equals, cache.MissErr) } + +func (s *S) TestCacheExpireBothAlgorithms(c *C) { + cc := cache.Cache{Dir: c.MkDir()} + + // Write entries under both algorithms. + err := cc.Write(cache.SHA256, data1Digest, []byte("data1")) + c.Assert(err, IsNil) + err = cc.Write(cache.SHA256, data2Digest, []byte("data2")) + c.Assert(err, IsNil) + + w := cc.Create(cache.SHA3_384, "") + _, err = w.Write([]byte("sha3data1")) + c.Assert(err, IsNil) + err = w.Close() + c.Assert(err, IsNil) + sha3Digest1 := w.Digest() + + w = cc.Create(cache.SHA3_384, "") + _, err = w.Write([]byte("sha3data2")) + c.Assert(err, IsNil) + err = w.Close() + c.Assert(err, IsNil) + sha3Digest2 := w.Digest() + + sha256Expired := filepath.Join(cc.Dir, "sha256", data1Digest) + sha256Fresh := filepath.Join(cc.Dir, "sha256", data2Digest) + sha3Expired := filepath.Join(cc.Dir, "sha3-384", sha3Digest1) + sha3Fresh := filepath.Join(cc.Dir, "sha3-384", sha3Digest2) + + // Mark one entry per algorithm as expired. + now := time.Now() + expiredTime := now.Add(-2 * time.Hour) + err = os.Chtimes(sha256Expired, now, expiredTime) + c.Assert(err, IsNil) + err = os.Chtimes(sha3Expired, now, expiredTime) + c.Assert(err, IsNil) + + err = cc.Expire(time.Hour) + c.Assert(err, IsNil) + + // Expired entries must be removed from both algorithm directories. + _, err = os.Stat(sha256Expired) + c.Assert(os.IsNotExist(err), Equals, true) + _, err = os.Stat(sha3Expired) + c.Assert(os.IsNotExist(err), Equals, true) + + // Fresh entries must remain in both algorithm directories. + _, err = os.Stat(sha256Fresh) + c.Assert(err, IsNil) + _, err = os.Stat(sha3Fresh) + c.Assert(err, IsNil) +} + +func (s *S) TestCacheExpireMissingAlgoDir(c *C) { + cc := cache.Cache{Dir: c.MkDir()} + + // Only write under SHA-256; the sha3-384 directory won't exist. + err := cc.Write(cache.SHA256, data1Digest, []byte("data1")) + c.Assert(err, IsNil) + + // Expire should succeed despite the missing sha3-384 directory. + err = cc.Expire(time.Hour) + c.Assert(err, IsNil) + + // The fresh SHA-256 entry must still be present. + _, err = os.Stat(filepath.Join(cc.Dir, "sha256", data1Digest)) + c.Assert(err, IsNil) +} + +func (s *S) TestCacheExpireNoAlgoDirs(c *C) { + cc := cache.Cache{Dir: c.MkDir()} + + // No algorithm directories exist at all. + err := cc.Expire(time.Hour) + c.Assert(err, IsNil) +} From 7b0eb143af35a48f108d2732653e7cd1c6e4aea7 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 2 Jun 2026 11:58:57 +0200 Subject: [PATCH 5/5] fix: improve error message Signed-off-by: Paul Mars --- internal/cache/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cache/cache.go b/internal/cache/cache.go index b35fed34..10f87303 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -124,7 +124,7 @@ func (c *Cache) Create(algo hashAlgo, digest string) *Writer { } h, err := newHash(algo) if err != nil { - return &Writer{err: fmt.Errorf("cannot create cache entry: %v", err)} + return &Writer{err: fmt.Errorf("internal error: %v", err)} } err = os.MkdirAll(filepath.Join(c.Dir, string(algo)), 0755) if err != nil {