Test WIP

.github/workflows/ci.yml

@@ -52,7 +52,7 @@ jobs:
             driver: overlay-transient
     uses: ./.github/workflows/lima.yml
     with:
-      runner: cncf-ubuntu-2-8-x86
+      runner: cncf-ubuntu-4-16-x86
       module: storage
       distro: ${{ matrix.distro }}
       variant: ${{ matrix.driver }}
@@ -114,7 +114,7 @@ jobs:
         module: [image, image-skopeo]
     uses: ./.github/workflows/lima.yml
     with:
-      runner: cncf-ubuntu-2-8-x86
+      runner: cncf-ubuntu-4-16-x86
       module: ${{ matrix.module }}
       distro: fedora-current
       variant: ${{ matrix.variant }}
@@ -129,7 +129,7 @@ jobs:
     name: "common fedora-current"
     uses: ./.github/workflows/lima.yml
     with:
-      runner: cncf-ubuntu-2-8-x86
+      runner: cncf-ubuntu-4-16-x86
       module: common
       distro: fedora-current
       timeout: 20

storage/go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/stretchr/testify v1.11.1
 	github.com/tchap/go-patricia/v2 v2.3.3
 	github.com/ulikunitz/xz v0.5.15
-	github.com/vbatts/tar-split v0.12.1
+	github.com/vbatts/tar-split v0.12.3
 	golang.org/x/sync v0.17.0
 	golang.org/x/sys v0.37.0
 	gotest.tools/v3 v3.5.2

storage/go.sum

@@ -71,8 +71,8 @@ github.com/tchap/go-patricia/v2 v2.3.3 h1:xfNEsODumaEcCcY3gI0hYPZ/PcpVv5ju6RMAhg
 github.com/tchap/go-patricia/v2 v2.3.3/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k=
 github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
 github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
-github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo=
-github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA=
+github.com/vbatts/tar-split v0.12.3 h1:Cd46rkGXI3Td4yrVNwU8ripbxFaQbmesqhjBUUYAJSw=
+github.com/vbatts/tar-split v0.12.3/go.mod h1:sQOc6OlqGCr7HkGx/IDBeKiTIvqhmj8KffNhEXG4Nq0=
 golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
 golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

storage/layers.go

@@ -2605,7 +2605,7 @@ func applyDiff(layerOptions *LayerOptions, diff io.Reader, tarSplitFile *os.File
 	gidLog := make(map[uint32]struct{})
 	var uncompressedCounter *ioutils.WriteCounter
 
-	size, err := func() (int64, error) { // A scope for defer
+	size, err := func() (retSize int64, retErr error) { // A scope for defer
 		compressor, err := pgzip.NewWriterLevel(tarSplitWriter, pgzip.BestSpeed)
 		if err != nil {
 			return -1, err
@@ -2635,12 +2635,27 @@ func applyDiff(layerOptions *LayerOptions, diff io.Reader, tarSplitFile *os.File
 		if uncompressedDigester != nil {
 			uncompressedWriter = io.MultiWriter(uncompressedWriter, uncompressedDigester.Hash())
 		}
-		payload, err := asm.NewInputTarStream(io.TeeReader(uncompressed, uncompressedWriter), metadata, storage.NewDiscardFilePutter())
+		payload, done, err := asm.NewInputTarStreamWithDone(io.TeeReader(uncompressed, uncompressedWriter), metadata, storage.NewDiscardFilePutter())
 		if err != nil {
 			return -1, err
 		}
+		defer func() {
+			payload.Close()
+			if doneErr := <-done; doneErr != nil && retErr == nil {
+				retErr = doneErr
+			}
+		}()
 
-		return applyDriverFunc(payload)
+		size, err := applyDriverFunc(payload)
+		if err != nil {
+			return -1, err
+		}
+		// Fully consume the payload; it may contain trailing zero padding, and we need all of that
+		// recorded in tar-split (which happens when the data passes through NewInputTarStreamWithDone).
+		if _, err := io.Copy(io.Discard, payload); err != nil {
+			return -1, err
+		}
+		return size, nil
 	}()
 	if err != nil {
 		return nil, err

storage/pkg/chunked/compression_linux_test.go

@@ -36,10 +36,12 @@ func TestTarSizeFromTarSplit(t *testing.T) {
 	expectedTarSize := int64(tarball.Len())
 
 	var tarSplit bytes.Buffer
-	tsReader, err := asm.NewInputTarStream(&tarball, storage.NewJSONPacker(&tarSplit), storage.NewDiscardFilePutter())
+	tsReader, done, err := asm.NewInputTarStreamWithDone(&tarball, storage.NewJSONPacker(&tarSplit), storage.NewDiscardFilePutter())
 	require.NoError(t, err)
 	_, err = io.Copy(io.Discard, tsReader)
 	require.NoError(t, err)
+	require.NoError(t, tsReader.Close())
+	require.NoError(t, <-done)
 
 	res, err := tarSizeFromTarSplit(&tarSplit)
 	require.NoError(t, err)