diff --git a/lib/dockerregistry/manifests.go b/lib/dockerregistry/manifests.go index 4e9643c46..22826ba74 100644 --- a/lib/dockerregistry/manifests.go +++ b/lib/dockerregistry/manifests.go @@ -95,6 +95,13 @@ func (t *manifests) getDigest(path string, subtype PathSubType) ([]byte, error) return nil, &InvalidRequestError{path} } + // For tag links, we only need to return the digest. The blob will be downloaded + // when the Docker Distribution library calls Reader on the blob path. + // For revision links, we still download to verify the blob exists. + if subtype == _tags { + return []byte(digest.String()), nil + } + blob, err := t.transferer.Download(repo, digest) if err != nil { return nil, fmt.Errorf("transferer download: %w", err) @@ -147,7 +154,7 @@ func (t *manifests) verify( } } -func (t *manifests) putContent(path string, subtype PathSubType) error { +func (t *manifests) putContent(path string, subtype PathSubType, content []byte) error { switch subtype { case _tags: repo, err := GetRepo(path) @@ -158,12 +165,25 @@ func (t *manifests) putContent(path string, subtype PathSubType) error { if err != nil { return fmt.Errorf("get manifest tag: %s", err) } + var digest core.Digest if isCurrent { - return nil - } - digest, err := GetManifestDigest(path) - if err != nil { - return fmt.Errorf("get manifest digest: %s", err) + // For current/link paths, the digest is in the content, not the path + if len(content) == 0 { + return fmt.Errorf("current link content is empty") + } + // Content is the digest string (e.g., "sha256:...") + digestStr := strings.TrimSpace(string(content)) + var err error + digest, err = core.ParseSHA256Digest(digestStr) + if err != nil { + return fmt.Errorf("parse digest from content: %w", err) + } + } else { + // For index/sha256:digest/link paths, the digest is in the path + digest, err = GetManifestDigest(path) + if err != nil { + return fmt.Errorf("get manifest digest: %s", err) + } } if err := t.transferer.PutTag(fmt.Sprintf("%s:%s", repo, tag), digest); err != nil { return fmt.Errorf("post tag: %w", err) diff --git a/lib/dockerregistry/storage_driver.go b/lib/dockerregistry/storage_driver.go index c4c8087d3..66c1a6266 100644 --- a/lib/dockerregistry/storage_driver.go +++ b/lib/dockerregistry/storage_driver.go @@ -147,7 +147,8 @@ func NewReadWriteStorageDriver( config Config, cas *store.CAStore, transferer transfer.ImageTransferer, - verification func(repo string, digest core.Digest, blob store.FileReader) (SignatureVerificationDecision, error)) *KrakenStorageDriver { + verification func(repo string, digest core.Digest, blob store.FileReader) (SignatureVerificationDecision, error), +) *KrakenStorageDriver { return &KrakenStorageDriver{ config: config, transferer: transferer, @@ -162,7 +163,8 @@ func NewReadOnlyStorageDriver( config Config, bs BlobStore, transferer transfer.ImageTransferer, - verification func(repo string, digest core.Digest, blob store.FileReader) (SignatureVerificationDecision, error)) *KrakenStorageDriver { + verification func(repo string, digest core.Digest, blob store.FileReader) (SignatureVerificationDecision, error), +) *KrakenStorageDriver { return &KrakenStorageDriver{ config: config, transferer: transferer, @@ -238,7 +240,7 @@ func (d *KrakenStorageDriver) PutContent(ctx context.Context, path string, conte switch pathType { case _manifests: - err = d.manifests.putContent(path, pathSubType) + err = d.manifests.putContent(path, pathSubType, content) case _uploads: err = d.uploads.putContent(path, pathSubType, content) case _layers: diff --git a/lib/dockerregistry/storage_driver_test.go b/lib/dockerregistry/storage_driver_test.go index 34b18567a..ee8115067 100644 --- a/lib/dockerregistry/storage_driver_test.go +++ b/lib/dockerregistry/storage_driver_test.go @@ -128,7 +128,7 @@ func TestStorageDriverPutContent(t *testing.T) { {genBlobDataPath(testImage.layer1.Digest.Hex()), testImage.layer1.Content, nil}, {genManifestRevisionLinkPath(repo, testImage.manifest), nil, nil}, {genManifestTagShaLinkPath(repo, tag, testImage.manifest), nil, nil}, - {genManifestTagCurrentLinkPath(repo, tag, testImage.manifest), nil, nil}, + {genManifestTagCurrentLinkPath(repo, tag, testImage.manifest), []byte("sha256:" + testImage.manifest), nil}, } for _, tc := range testCases { diff --git a/tools/bin/puller/pull.go b/tools/bin/puller/pull.go index 0bb3d3fc7..c8605c119 100644 --- a/tools/bin/puller/pull.go +++ b/tools/bin/puller/pull.go @@ -29,6 +29,7 @@ import ( "github.com/docker/distribution/manifest/schema2" "github.com/opencontainers/go-digest" "github.com/uber/kraken/utils/closers" + "github.com/uber/kraken/utils/dockerutil" "github.com/uber/kraken/utils/errutil" "github.com/uber/kraken/utils/log" ) @@ -100,10 +101,8 @@ func pullManifest(client http.Client, source string, name string, reference stri if err != nil { return nil, err } - // Add `Accept` header to indicate schema2 is supported - req.Header.Add("Accept", schema2.MediaTypeManifest) + req.Header.Add("Accept", fmt.Sprintf("%s,%s", schema2.MediaTypeManifest, "application/vnd.oci.image.manifest.v1+json")) resp, err := client.Do(req) - if err != nil { return nil, err } @@ -117,14 +116,16 @@ func pullManifest(client http.Client, source string, name string, reference stri return nil, fmt.Errorf("server returned %v", resp.Status) } - version := resp.Header.Get("Content-Type") body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } - manifest, _, err := distribution.UnmarshalManifest(version, body) + + // Use dockerutil.ParseManifest which handles Docker v2, v2 list, and OCI formats + // This avoids issues with Content-Type headers and schema1 signature verification + manifest, _, err := dockerutil.ParseManifest(bytes.NewReader(body)) if err != nil { - return nil, err + return nil, fmt.Errorf("parse manifest: %w", err) } return manifest, nil diff --git a/utils/dockerutil/dockerutil.go b/utils/dockerutil/dockerutil.go index 8d0f386bc..7b4d7100d 100644 --- a/utils/dockerutil/dockerutil.go +++ b/utils/dockerutil/dockerutil.go @@ -14,6 +14,7 @@ package dockerutil import ( + "encoding/json" "errors" "fmt" "io" @@ -21,12 +22,15 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/manifest/manifestlist" "github.com/docker/distribution/manifest/schema2" + "github.com/opencontainers/go-digest" "github.com/uber/kraken/core" + "github.com/uber/kraken/utils/log" ) const ( _v2ManifestType = "application/vnd.docker.distribution.manifest.v2+json" _v2ManifestListType = "application/vnd.docker.distribution.manifest.list.v2+json" + _ociManifestType = "application/vnd.oci.image.manifest.v1+json" ) func ParseManifest(r io.Reader) (distribution.Manifest, core.Digest, error) { @@ -35,13 +39,20 @@ func ParseManifest(r io.Reader) (distribution.Manifest, core.Digest, error) { return nil, core.Digest{}, fmt.Errorf("read: %s", err) } + // Try Docker v2 manifest first manifest, d, err := ParseManifestV2(b) if err == nil { return manifest, d, err } - // Retry with v2 manifest list. - return ParseManifestV2List(b) + // Try Docker v2 manifest list + manifest, d, err = ParseManifestV2List(b) + if err == nil { + return manifest, d, err + } + + // Try OCI manifest v1 (same structure as Docker v2) + return ParseOCIManifest(b) } // ParseManifestV2 returns a parsed v2 manifest and its digest. @@ -88,17 +99,146 @@ func ParseManifestV2List(bytes []byte) (distribution.Manifest, core.Digest, erro // GetManifestReferences returns a list of references by a V2 manifest func GetManifestReferences(manifest distribution.Manifest) ([]core.Digest, error) { + refDescs := manifest.References() + log.Infow("GetManifestReferences called", + "num_references", len(refDescs)) + var refs []core.Digest - for _, desc := range manifest.References() { + refDigestStrings := make([]string, 0, len(refDescs)) + for _, desc := range refDescs { d, err := core.ParseSHA256Digest(string(desc.Digest)) if err != nil { return nil, fmt.Errorf("parse digest: %w", err) } refs = append(refs, d) + refDigestStrings = append(refDigestStrings, d.String()) } + + log.Infow("GetManifestReferences returning", + "num_digests", len(refs), + "digests", refDigestStrings) + return refs, nil } +// ParseOCIManifest parses an OCI image manifest v1. +// OCI manifests have the same structure as Docker v2 manifests, so we convert them. +func ParseOCIManifest(bytes []byte) (distribution.Manifest, core.Digest, error) { + // Parse the OCI manifest JSON to extract structure + var manifestJSON struct { + MediaType string `json:"mediaType"` + Config struct { + Digest string `json:"digest"` + Size int64 `json:"size"` + } `json:"config"` + Layers []struct { + Digest string `json:"digest"` + Size int64 `json:"size"` + } `json:"layers"` + } + if err := json.Unmarshal(bytes, &manifestJSON); err != nil { + return nil, core.Digest{}, fmt.Errorf("unmarshal oci manifest json: %s", err) + } + + // Verify it's an OCI manifest + if manifestJSON.MediaType != _ociManifestType { + return nil, core.Digest{}, fmt.Errorf("expected oci manifest type %s, got %s", _ociManifestType, manifestJSON.MediaType) + } + + log.Infow("Parsing OCI manifest", + "config_digest", manifestJSON.Config.Digest, + "config_size", manifestJSON.Config.Size, + "num_layers", len(manifestJSON.Layers)) + + // Calculate digest of the original manifest + d, err := core.NewDigester().FromBytes(bytes) + if err != nil { + return nil, core.Digest{}, fmt.Errorf("calculate manifest digest: %s", err) + } + + log.Infow("OCI manifest digest calculated", "digest", d.String()) + + // Parse digests from OCI format (they're already in "sha256:hex" format) + configDigest, err := core.ParseSHA256Digest(manifestJSON.Config.Digest) + if err != nil { + return nil, core.Digest{}, fmt.Errorf("parse config digest: %s", err) + } + + // Convert to Docker v2 format by creating descriptors + configDesc := distribution.Descriptor{ + Digest: digest.Digest(configDigest.String()), + MediaType: schema2.MediaTypeImageConfig, + Size: manifestJSON.Config.Size, + } + + layers := make([]distribution.Descriptor, 0, len(manifestJSON.Layers)) + layerDigests := make([]string, 0, len(manifestJSON.Layers)) + for _, layer := range manifestJSON.Layers { + layerDigest, err := core.ParseSHA256Digest(layer.Digest) + if err != nil { + return nil, core.Digest{}, fmt.Errorf("parse layer digest: %s", err) + } + layerDigests = append(layerDigests, layerDigest.String()) + layers = append(layers, distribution.Descriptor{ + Digest: digest.Digest(layerDigest.String()), + MediaType: schema2.MediaTypeLayer, + Size: layer.Size, + }) + } + + log.Infow("OCI manifest layers extracted", + "layer_digests", layerDigests, + "config_digest", configDigest.String()) + + // Create a properly initialized schema2 manifest by converting OCI JSON to Docker v2 format + // and then parsing it with the standard parser to ensure proper initialization + // This is critical - manually constructing DeserializedManifest doesn't initialize + // the internal state needed for References() to work correctly + dockerV2JSON := struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Config distribution.Descriptor `json:"config"` + Layers []distribution.Descriptor `json:"layers"` + }{ + SchemaVersion: 2, // Docker v2 schema version + MediaType: schema2.MediaTypeManifest, + Config: configDesc, + Layers: layers, + } + + dockerV2Bytes, err := json.Marshal(dockerV2JSON) + if err != nil { + return nil, core.Digest{}, fmt.Errorf("marshal docker v2 manifest: %s", err) + } + + // Parse it using the standard Docker v2 parser to ensure proper initialization + manifest, _, err := distribution.UnmarshalManifest(schema2.MediaTypeManifest, dockerV2Bytes) + if err != nil { + return nil, core.Digest{}, fmt.Errorf("unmarshal converted manifest: %s", err) + } + + deserializedManifest, ok := manifest.(*schema2.DeserializedManifest) + if !ok { + return nil, core.Digest{}, errors.New("expected schema2.DeserializedManifest after conversion") + } + + // Log what References() returns to verify it's working correctly + refs := deserializedManifest.References() + refDigests := make([]string, 0, len(refs)) + for _, ref := range refs { + refDigests = append(refDigests, ref.Digest.String()) + } + log.Infow("OCI manifest converted to Docker v2, References() returned", + "num_references", len(refs), + "reference_digests", refDigests, + "expected_config", configDigest.String(), + "expected_layers", layerDigests) + + // Return the original OCI manifest's digest (calculated from original bytes) + // This ensures we return the correct digest that matches what the registry expects + return deserializedManifest, d, nil +} + func GetSupportedManifestTypes() string { - return fmt.Sprintf("%s,%s", _v2ManifestType, _v2ManifestListType) + return fmt.Sprintf("%s,%s,%s", _v2ManifestType, _v2ManifestListType, _ociManifestType) }