feat(ocm): Add support for non-tar json component descriptor blobs

cnudie/retrieve.py

@@ -1,12 +1,10 @@
 import collections.abc
 import dataclasses
-import io
 import itertools
 import json
 import logging
 import os
 import shutil
-import tarfile
 import tempfile
 
 import cachetools
@@ -358,12 +356,12 @@ def lookup(
     return lookup
 
 
-def _raw_component_descriptor_from_oci(
+def component_descriptor_from_oci(
     component_id: ocm.ComponentIdentity,
     ocm_repos: collections.abc.Iterable[ocm.OciOcmRepository | str],
     oci_client: oc.Client,
     absent_ok: bool=False,
-) -> bytes | None:
+) -> ocm.ComponentDescriptor | None:
     for ocm_repo in ocm_repos:
         if isinstance(ocm_repo, str):
             ocm_repo = ocm.OciOcmRepository(
@@ -414,22 +412,25 @@ def _raw_component_descriptor_from_oci(
         )
 
         # by contract, there must be exactly one layer (tar w/ component-descriptor)
-        if not (layers_count := len(manifest.layers) == 1):
+        if (layers_count := len(manifest.layers)) != 1:
             logger.warning(f'XXX unexpected amount of {layers_count=}')
 
         layer_digest = manifest.layers[0].digest
         layer_mimetype = manifest.layers[0].mediaType
 
-    if not layer_mimetype in ocm.oci.component_descriptor_mimetypes:
-        logger.warning(f'{target_ref=} {layer_mimetype=} was unexpected')
-        # XXX: check for non-tar-variant
-
-    return oci_client.blob(
+    component_descriptor_blob = oci_client.blob(
         image_reference=target_ref,
         digest=layer_digest,
         stream=False, # manifests are typically small - do not bother w/ streaming
     ).content
 
+    return ocm.oci.component_descriptor_from_blob(
+        component_descriptor_blob=component_descriptor_blob,
+        layer_mimetype=layer_mimetype,
+        target_ref=target_ref,
+        component_id=component_id,
+    )
+
 
 def oci_component_descriptor_lookup(
     ocm_repository_lookup: OcmRepositoryLookup,
@@ -477,31 +478,21 @@ def lookup(
                     baseUrl=ocm_repo,
                 )
 
-            if raw := _raw_component_descriptor_from_oci(
+            if component_descriptor := component_descriptor_from_oci(
                 component_id=component_id,
                 ocm_repos=(ocm_repo,),
                 oci_client=local_oci_client,
                 absent_ok=True,
             ):
                 break
         else:
-            raw = None
+            component_descriptor = None
 
-        if not raw and absent_ok:
+        if not component_descriptor and absent_ok:
             return
-        elif not raw and not absent_ok:
+        elif not component_descriptor and not absent_ok:
             raise om.OciImageNotFoundException(component_id)
 
-        # wrap in fobj
-        blob_fobj = io.BytesIO(raw)
-        try:
-            component_descriptor = ocm.oci.component_descriptor_from_tarfileobj(
-                fileobj=blob_fobj,
-            )
-        except tarfile.ReadError as tre:
-            tre.add_note(f'{component_id=}')
-            raise tre
-
         # ensure OCM repository in which component descriptor was found is the current OCM repository
         component_descriptor.component.set_current_ocm_repo(ocm_repo)
 

cnudie/retrieve_async.py

@@ -1,11 +1,9 @@
 import collections.abc
 import dataclasses
-import io
 import itertools
 import logging
 import os
 import shutil
-import tarfile
 import tempfile
 
 import aiohttp.client_exceptions
@@ -300,12 +298,12 @@ async def lookup(
     return lookup
 
 
-async def raw_component_descriptor_from_oci(
+async def component_descriptor_from_oci(
     component_id: ocm.ComponentIdentity,
     ocm_repos: collections.abc.Iterable[ocm.OciOcmRepository | str],
     oci_client: oca.Client,
     absent_ok: bool=False,
-) -> bytes | None:
+) -> ocm.ComponentDescriptor | None:
     for ocm_repo in ocm_repos:
         if isinstance(ocm_repo, str):
             ocm_repo = ocm.OciOcmRepository(
@@ -357,21 +355,24 @@ async def raw_component_descriptor_from_oci(
         )
 
         # by contract, there must be exactly one layer (tar w/ component-descriptor)
-        if not (layers_count := len(manifest.layers) == 1):
+        if (layers_count := len(manifest.layers)) != 1:
             logger.warning(f'XXX unexpected amount of {layers_count=}')
 
         layer_digest = manifest.layers[0].digest
         layer_mimetype = manifest.layers[0].mediaType
 
-    if not layer_mimetype in ocm.oci.component_descriptor_mimetypes:
-        logger.warning(f'{target_ref=} {layer_mimetype=} was unexpected')
-        # XXX: check for non-tar-variant
-
     blob = await oci_client.blob(
         image_reference=target_ref,
         digest=layer_digest,
     )
-    return await blob.content.read()
+    component_descriptor_blob = await blob.content.read()
+
+    return ocm.oci.component_descriptor_from_blob(
+        component_descriptor_blob=component_descriptor_blob,
+        layer_mimetype=layer_mimetype,
+        target_ref=target_ref,
+        component_id=component_id,
+    )
 
 
 def oci_component_descriptor_lookup(
@@ -420,31 +421,21 @@ async def lookup(
                     baseUrl=ocm_repo,
                 )
 
-            if raw := await raw_component_descriptor_from_oci(
+            if component_descriptor := await component_descriptor_from_oci(
                 component_id=component_id,
                 ocm_repos=(ocm_repo,),
                 oci_client=local_oci_client,
                 absent_ok=True,
             ):
                 break
         else:
-            raw = None
+            component_descriptor = None
 
-        if not raw and absent_ok:
+        if not component_descriptor and absent_ok:
             return
-        elif not raw and not absent_ok:
+        elif not component_descriptor and not absent_ok:
             raise om.OciImageNotFoundException(component_id)
 
-        # wrap in fobj
-        blob_fobj = io.BytesIO(raw)
-        try:
-            component_descriptor = ocm.oci.component_descriptor_from_tarfileobj(
-                fileobj=blob_fobj,
-            )
-        except tarfile.ReadError as tre:
-            tre.add_note(f'{component_id=}')
-            raise tre
-
         # ensure OCM repository in which component descriptor was found is the current OCM repository
         component_descriptor.component.set_current_ocm_repo(ocm_repo)
 

ocm/__init__.py

@@ -712,6 +712,28 @@ def dateparse(v):
         if not _have_dacite:
             raise RuntimeError('not available without dacite')
 
+        # Normalize null arrays to empty lists for dacite
+        def normalize_nulls(obj):
+            # Null values for these keys will be replaced by empty lists, null values for other keys
+            # will be kept as-is
+            null_allowed_for = (
+                'repositoryContexts',
+                'sources',
+                'componentReferences',
+                'resources',
+            )
+
+            if isinstance(obj, dict):
+                return {
+                    k: normalize_nulls(v) if v is not None else ([] if k in null_allowed_for else v)
+                    for k, v in obj.items()
+                }
+            elif isinstance(obj, list):
+                return [normalize_nulls(item) for item in obj]
+            return obj
+
+        component_descriptor_dict = normalize_nulls(component_descriptor_dict)
+
         component_descriptor = dacite.from_dict(
             data_class=ComponentDescriptor,
             data=component_descriptor_dict,

ocm/__main__.py

@@ -333,7 +333,16 @@ def _fetch_component_descriptor(
             digest=layer_digest,
             stream=False, # manifests are typically small - do not bother w/ streaming
         ).content
-        return ocm.oci.component_descriptor_from_tarfileobj(fileobj=io.BytesIO(raw))
+
+        return ocm.oci.component_descriptor_from_blob(
+            component_descriptor_blob=raw,
+            layer_mimetype=layer_mimetype,
+            target_ref=target_ref,
+            component_id=ocm.ComponentIdentity(
+                name=name,
+                version=version,
+            ),
+        )
 
     root_component_descriptor = _fetch_component_descriptor(
         name=cname,

ocm/oci.py

@@ -7,10 +7,12 @@
 
 import dataclasses
 import io
+import json
 import logging
 import os
 import tarfile
 import typing
+
 import yaml
 
 import ocm
@@ -24,7 +26,8 @@
     'application/vnd.gardener.cloud.cnudie.component-descriptor.v2+yaml+tar'
 component_descriptor_mimetypes = (
     component_descriptor_mimetype,
-    'application/vnd.ocm.software.component-descriptor.v2+yaml+tar'
+    'application/vnd.ocm.software.component-descriptor.v2+yaml+tar',
+    'application/vnd.ocm.software.component-descriptor.v2+json',
 )
 # mimetype for component-descriptor-oci-cfg-blobs
 component_descriptor_cfg_mimetype = \
@@ -81,16 +84,34 @@ def component_descriptor_to_tarfileobj(
     return tf.fileobj
 
 
-def component_descriptor_from_tarfileobj(
-    fileobj: io.BytesIO,
-):
-    with tarfile.open(fileobj=fileobj, mode='r') as tf:
-        component_descriptor_info = tf.getmember(component_descriptor_fname)
-        raw_dict = yaml.safe_load(tf.extractfile(component_descriptor_info).read())
-
-        logger.debug(raw_dict)
-
-        if raw_dict is None:
-          raise ValueError('Component Descriptor appears to be empty')
-
-        return ocm.ComponentDescriptor.from_dict(raw_dict)
+def component_descriptor_from_blob(
+    component_descriptor_blob: bytes,
+    layer_mimetype: str,
+    target_ref: str | None=None,
+    component_id: ocm.ComponentIdentity | None=None,
+) -> ocm.ComponentDescriptor:
+    if not layer_mimetype in component_descriptor_mimetypes:
+        logger.warning(f'{target_ref=} {layer_mimetype=} was unexpected')
+
+    if '+tar' in layer_mimetype:
+        try:
+            with tarfile.open(fileobj=io.BytesIO(component_descriptor_blob), mode='r') as tf:
+                component_descriptor_info = tf.getmember(component_descriptor_fname)
+                component_descriptor_blob = tf.extractfile(component_descriptor_info).read()
+        except tarfile.ReadError as tre:
+            tre.add_note(f'{component_id=}')
+            raise tre
+
+    if '+yaml' in layer_mimetype:
+        component_descriptor_dict = yaml.safe_load(component_descriptor_blob)
+    elif '+json' in layer_mimetype:
+        component_descriptor_dict = json.loads(component_descriptor_blob)
+    else:
+        raise ValueError(f'Unsupported component descriptor {layer_mimetype=}')
+
+    logger.debug(component_descriptor_dict)
+
+    if component_descriptor_dict is None:
+        raise ValueError('Component Descriptor appears to be empty')
+
+    return ocm.ComponentDescriptor.from_dict(component_descriptor_dict)