Skip to content
Open
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
a948b10
feat: add sha256
lczyk Jul 6, 2026
e1b7e40
docs: comment
lczyk Jul 6, 2026
daf5864
tests: test SHA512 cache entries
upils Jul 7, 2026
46d5658
fix: refine comments
upils Jul 7, 2026
1511187
tests: use stonking for SHA512 tests
upils Jul 7, 2026
bbdf2fe
fix: rely on ok to check digest
upils Jul 7, 2026
7501ca0
Merge remote-tracking branch 'canonical/main' into fix/issue-305-acce…
lczyk Jul 7, 2026
c973b90
test: cover by-hash over sha512
lczyk Jul 7, 2026
9fa6d53
test: propagate digest kind to packages in test helper
lczyk Jul 7, 2026
46fefb3
test: cover release publishing both digests
lczyk Jul 7, 2026
1e1a0d1
refactor: rename digestField to digestSection
lczyk Jul 7, 2026
58c0e97
test: inline makeSha256
lczyk Jul 7, 2026
82dd7e7
test: make digest kinds an archive-wide property
lczyk Jul 7, 2026
f8eb118
test: drop implicit SHA256 default for digest kinds
lczyk Jul 7, 2026
bc4b7f6
fix: check Walk return in inheritDigestKinds
lczyk Jul 7, 2026
c179489
fix: build by-hash URLs from strongest digest
lczyk Jul 8, 2026
744fae9
refactor: name digest fields, not sections
lczyk Jul 8, 2026
8c5cbc1
fix: panic on unknown digest kind in test archive
lczyk Jul 8, 2026
7a308a3
refactor: drop redundant inheritDigestKinds call
lczyk Jul 8, 2026
2ec8ab9
refactor: propagate Walk errors in test archive
lczyk Jul 8, 2026
c5ce9e3
docs: describe digestKinds field, not its writer
lczyk Jul 8, 2026
1397709
refactor: packages read digest kinds through the release
lczyk Jul 8, 2026
7cf965b
refactor: adopt packages through Walk
lczyk Jul 8, 2026
abfc91f
refactor: rename adoptPackages to wirePackages
lczyk Jul 8, 2026
0ebadf2
docs: keep findDigest comment above the table format
lczyk Jul 8, 2026
606e7a1
test: cover by-hash dirs beyond the strongest digest
lczyk Jul 8, 2026
86024cd
docs: by-hash dirs beyond the strongest are legal
lczyk Jul 8, 2026
b70e1b4
Update internal/archive/testarchive/testarchive.go
lczyk Jul 9, 2026
311052e
refactor: single strongest-first digest preference
lczyk Jul 9, 2026
0c9e9e3
test: pass DigestKinds instead of release pointer
upils Jul 15, 2026
b9da3c9
refactor: strongest kind at the release level
upils Jul 15, 2026
8caa89f
fix: respect adjustRelease intent
upils Aug 19, 2026
90c46ba
fix(test): refine DigestKinds handling
upils Aug 20, 2026
ce36ce4
fix: revert adjustRelease call move
upils Aug 20, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 48 additions & 10 deletions internal/archive/archive.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,9 @@ func (a *ubuntuArchive) Fetch(pkg string) (io.ReadSeekCloser, *PackageInfo, erro
return nil, nil, err
}
path := section.Get("Filename")
digest, digestKind := packageDigest(section)
logf("Fetching %s...", path)
reader, err := index.fetch(path, section.Get("SHA256"), fetchBulk)
reader, err := index.fetch(path, digest, digestKind, fetchBulk)
if err != nil {
return nil, nil, err
}
Expand Down Expand Up @@ -280,7 +281,9 @@ func openUbuntu(options *Options) (Archive, error) {

func (index *ubuntuIndex) fetchRelease() error {
logf("Fetching %s %s %s suite details...", index.displayName(), index.version, index.suite)
reader, err := index.fetch(index.distPath("InRelease"), "", fetchDefault)
// InRelease has no digest to check against (it is verified by its PGP
// signature below), so the digest kind here is arbitrary.
reader, err := index.fetch(index.distPath("InRelease"), "", cache.SHA256, fetchDefault)
if err != nil {
return err
}
Expand Down Expand Up @@ -328,10 +331,46 @@ func (index *ubuntuIndex) fetchRelease() error {
return nil
}

// digestField is an archive checksum field Chisel can verify. Its name doubles
// as the by-hash directory name in the archive layout.
type digestField struct {
name string
kind cache.DigestKind
}

// digestFields lists the checksum fields Chisel can verify, in order of
// preference. SHA256 is first so existing archives keep their cache keys;
// Ubuntu 26.10 and later publish SHA512-only indices, handled by the fallback.
var digestFields = []digestField{
Comment thread
lczyk marked this conversation as resolved.
{"SHA256", cache.SHA256},
{"SHA512", cache.SHA512},
}

// releaseDigest returns the checksum recorded for path in a Release "<hash>
// <size> <path>" table, along with the field it was found in.
func releaseDigest(release control.Section, path string) (digest string, field digestField) {
Comment thread
lczyk marked this conversation as resolved.
Outdated
for _, f := range digestFields {
if d, _, ok := control.ParsePathInfo(release.Get(f.name), path); ok {
return d, f
}
}
return "", digestField{}
}

func packageDigest(section control.Section) (digest string, kind cache.DigestKind) {
for _, f := range digestFields {
if d := section.Get(f.name); d != "" {
return d, f.kind
}
}
// No digest advertised; fall back to SHA256 so the package can still be
// cached and retrieved by its computed digest.
return "", cache.SHA256
}

func (index *ubuntuIndex) fetchIndex() error {
releaseDigests := index.release.Get("SHA256")
packagesPath := fmt.Sprintf("%s/binary-%s/Packages", index.component, index.arch)
packagesDigest, _, _ := control.ParsePathInfo(releaseDigests, packagesPath)
packagesDigest, field := releaseDigest(index.release, packagesPath)
if packagesDigest == "" {
return fmt.Errorf("%s is missing from %s %s component digests", packagesPath, index.suite, index.component)
}
Expand All @@ -345,10 +384,10 @@ func (index *ubuntuIndex) fetchIndex() error {
packagesGzPath := packagesPath + ".gz"
var reader io.ReadSeekCloser
if index.release.Get("Acquire-By-Hash") == "yes" {
packagesGzDigest, _, _ := control.ParsePathInfo(releaseDigests, packagesGzPath)
packagesGzDigest, _, _ := control.ParsePathInfo(index.release.Get(field.name), packagesGzPath)
if packagesGzDigest != "" {
packagesByHashPath := fmt.Sprintf("%s/binary-%s/by-hash/SHA256/%s", index.component, index.arch, packagesGzDigest)
r, err := index.fetch(index.distPath(packagesByHashPath), packagesDigest, fetchBulk|fetchGzip)
packagesByHashPath := fmt.Sprintf("%s/binary-%s/by-hash/%s/%s", index.component, index.arch, field.name, packagesGzDigest)
r, err := index.fetch(index.distPath(packagesByHashPath), packagesDigest, field.kind, fetchBulk|fetchGzip)
if err != nil && err != errNotFound {
return err
}
Expand All @@ -358,7 +397,7 @@ func (index *ubuntuIndex) fetchIndex() error {
}
}
if reader == nil {
r, err := index.fetch(index.distPath(packagesGzPath), packagesDigest, fetchBulk|fetchGzip)
r, err := index.fetch(index.distPath(packagesGzPath), packagesDigest, field.kind, fetchBulk|fetchGzip)
if err != nil {
return err
}
Expand Down Expand Up @@ -399,8 +438,7 @@ func (index *ubuntuIndex) distPath(suffix string) string {
return "dists/" + index.suite + "/" + suffix
}

func (index *ubuntuIndex) fetch(path, digest string, flags fetchFlags) (io.ReadSeekCloser, error) {
const digestKind = cache.SHA256
func (index *ubuntuIndex) fetch(path, digest string, digestKind cache.DigestKind, flags fetchFlags) (io.ReadSeekCloser, error) {
reader, err := index.archive.cache.Open(digestKind, digest)
if err == nil {
return reader, nil
Expand Down
76 changes: 76 additions & 0 deletions internal/archive/archive_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,16 @@ func (s *httpSuite) prepareArchiveAdjustRelease(suite, version, arch string, com
if adjustRelease != nil {
adjustRelease(release)
}
// Packages inherit the release's digest kind unless they set their own.
err = release.Walk(func(item testarchive.Item) error {
if p, ok := item.(*testarchive.Package); ok && p.Digest == "" {
p.Digest = release.Digest
}
return nil
})
if err != nil {
panic(err)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This wasn't here before. It's probably a good thing to add it, but please just confirm that this is okay given the context I just mentioned.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As mentioned in the other comment #306 (comment) this should not affect real archive test so this should be okay and should not risk hiding failures.

}
Comment thread
lczyk marked this conversation as resolved.
Outdated
release.Render(base.Path, s.responses)
return release
}
Expand Down Expand Up @@ -264,6 +274,32 @@ func (s *httpSuite) TestFetchPackage(c *C) {
c.Assert(read(pkg), Equals, "mypkg4 1.4 data")
}

func (s *httpSuite) TestFetchSHA512Digests(c *C) {
// Ubuntu 26.10+ publishes SHA512-only indices (no SHA256 section), so both
// the index digest and the package digest must be read from SHA512.
s.prepareArchiveAdjustRelease("stonking", "25.10", "amd64", []string{"main", "universe"},
func(release *testarchive.Release) {
release.Digest = "SHA512"
})

options := archive.Options{
Label: "ubuntu",
Version: "25.10",
Arch: "amd64",
Suites: []string{"stonking"},
Components: []string{"main", "universe"},
CacheDir: c.MkDir(),
PubKeys: []*packet.PublicKey{s.pubKey},
}

testArchive, err := archive.Open(&options)
c.Assert(err, IsNil)

pkg, _, err := testArchive.Fetch("mypkg1")
c.Assert(err, IsNil)
c.Assert(read(pkg), Equals, "mypkg1 1.1 data")
}

func (s *httpSuite) TestFetchPortsPackage(c *C) {

s.base = "http://ports.ubuntu.com/ubuntu-ports/"
Expand Down Expand Up @@ -687,6 +723,46 @@ func (s *httpSuite) TestFetchByHashSucceedsWhenNamedPathIsStale(c *C) {
c.Assert(status, Equals, 200)
}

func (s *httpSuite) TestFetchByHashSHA512(c *C) {
// Ubuntu 26.10+ advertises Acquire-By-Hash with SHA512-only indices, so
// the by-hash URL must be built under the SHA512 directory.
s.prepareArchiveAdjustRelease("stonking", "26.10", "amd64", []string{"main"}, func(release *testarchive.Release) {
release.ByHash = true
release.Digest = "SHA512"
})

// Stale content at the named Packages.gz path, so a fallback would fail
// the digest check -- only the by-hash path serves the correct bytes.
for p := range s.responses {
if strings.Contains(p, "Packages.gz") && !strings.Contains(p, "/by-hash/") {
s.responses[p] = testarchive.MakeGzip([]byte("stale Packages from previous publication"))
}
}

options := archive.Options{
Label: "ubuntu",
Version: "26.10",
Arch: "amd64",
Suites: []string{"stonking"},
Components: []string{"main"},
CacheDir: c.MkDir(),
PubKeys: []*packet.PublicKey{s.pubKey},
}

testArchive, err := archive.Open(&options)
c.Assert(err, IsNil)

pkg, _, err := testArchive.Fetch("mypkg1")
c.Assert(err, IsNil)
c.Assert(read(pkg), Equals, "mypkg1 1.1 data")

// The SHA512 by-hash request must have been attempted and succeeded;
// the named path only has stale content.
attempted, status := s.fetchRequestStatus("/by-hash/SHA512/")
c.Assert(attempted, Equals, true)
c.Assert(status, Equals, 200)
}

func (s *httpSuite) TestFetchByHashFallsBackOnNotFound(c *C) {
s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main"}, func(r *testarchive.Release) {
r.ByHash = true
Expand Down
33 changes: 27 additions & 6 deletions internal/archive/testarchive/testarchive.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"compress/gzip"
"crypto/sha256"
"crypto/sha512"
"fmt"
"path"
"strings"
Expand Down Expand Up @@ -63,6 +64,8 @@ type Package struct {
Arch string
Component string
Data []byte
// Digest names the checksum field published for this package ("SHA256" when empty).
Digest string
Comment thread
lczyk marked this conversation as resolved.
Outdated
}

func (p *Package) Path() string {
Expand All @@ -86,11 +89,11 @@ func (p *Package) Section() []byte {
Installed-Size: 10
Filename: %s
Size: %d
SHA256: %s
%s: %s
Description: Description of %s
Task: minimal

`)), p.Name, p.Arch, p.Version, p.Path(), len(content), makeSha256(content), p.Name)
`)), p.Name, p.Arch, p.Version, p.Path(), len(content), digestField(p.Digest), makeDigest(p.Digest, content), p.Name)
return []byte(section)
}

Expand All @@ -107,6 +110,8 @@ type Release struct {
Label string
Items []Item
PrivKey *packet.PrivateKey
// Digest names the checksum field published for the index table ("SHA256" when empty).
Digest string
Comment thread
lczyk marked this conversation as resolved.
Outdated
// ByHash enables the Acquire-By-Hash flag in the Release file
// and renders by-hash URLs alongside named paths.
ByHash bool
Expand All @@ -128,7 +133,7 @@ func (r *Release) Content() []byte {
digests := bytes.Buffer{}
for _, item := range r.Items {
content := item.Content()
fmt.Fprintf(&digests, " %s %d %s\n", makeSha256(content), len(content), item.Path())
fmt.Fprintf(&digests, " %s %d %s\n", makeDigest(r.Digest, content), len(content), item.Path())
}
acquireByHash := ""
if r.ByHash {
Expand All @@ -144,9 +149,9 @@ func (r *Release) Content() []byte {
Architectures: amd64 arm64 armhf i386 ppc64el riscv64 s390x
Components: main restricted universe multiverse
Description: Ubuntu %s
%sSHA256:
%s%s:
%s
`)), r.Label, r.Suite, r.Version, r.Version, acquireByHash, digests.String())
`)), r.Label, r.Suite, r.Version, r.Version, acquireByHash, digestField(r.Digest), digests.String())

var buf bytes.Buffer
writer, err := clearsign.Encode(&buf, r.PrivKey, nil)
Expand Down Expand Up @@ -175,7 +180,7 @@ func (r *Release) Render(prefix string, content map[string][]byte) error {
distItemPath := path.Join(prefix, "dists", r.Suite, itemPath)
content[distItemPath] = itemContent
if r.ByHash && itemPath != r.Path() {
byHashPath := path.Join(prefix, "dists", r.Suite, path.Dir(itemPath), "by-hash", "SHA256", makeSha256(itemContent))
byHashPath := path.Join(prefix, "dists", r.Suite, path.Dir(itemPath), "by-hash", digestField(r.Digest), makeDigest(r.Digest, itemContent))
content[byHashPath] = itemContent
}
return nil
Expand Down Expand Up @@ -216,6 +221,22 @@ func makeSha256(b []byte) string {
return fmt.Sprintf("%x", sha256.Sum256(b))
}

// digestField maps a digest kind to its Release/Packages field name, defaulting
// to SHA256.
func digestField(kind string) string {
if kind == "" {
return "SHA256"
}
return kind
}

func makeDigest(kind string, b []byte) string {
Comment thread
lczyk marked this conversation as resolved.
if kind == "SHA512" {
return fmt.Sprintf("%x", sha512.Sum512(b))
}
return makeSha256(b)
Comment thread
lczyk marked this conversation as resolved.
Outdated
}

func MakeGzip(b []byte) []byte {
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)
Expand Down
6 changes: 5 additions & 1 deletion internal/cache/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package cache

import (
"crypto/sha256"
"crypto/sha512"
"encoding/hex"
"fmt"
"hash"
Expand Down Expand Up @@ -96,9 +97,10 @@ type DigestKind string
const (
SHA256 DigestKind = "sha256"
SHA384 DigestKind = "sha384"
SHA512 DigestKind = "sha512"
)

var digestKinds = []DigestKind{SHA256, SHA384}
var digestKinds = []DigestKind{SHA256, SHA384, SHA512}

var ErrMiss = fmt.Errorf("not cached")

Expand All @@ -117,6 +119,8 @@ func (c *Cache) Create(digestKind DigestKind, digest string) *Writer {
h = sha256.New()
case SHA384:
h = sha3.New384()
case SHA512:
h = sha512.New()
default:
return &Writer{err: fmt.Errorf("internal error: unsupported digest kind: %q", digestKind)}
}
Expand Down
Loading
Loading