diff --git a/cnudie/retrieve.py b/cnudie/retrieve.py index 0429f9a089..380b862e97 100644 --- a/cnudie/retrieve.py +++ b/cnudie/retrieve.py @@ -1,12 +1,10 @@ import collections.abc import dataclasses -import io import itertools import json import logging import os import shutil -import tarfile import tempfile import cachetools @@ -358,12 +356,12 @@ def lookup( return lookup -def _raw_component_descriptor_from_oci( +def component_descriptor_from_oci( component_id: ocm.ComponentIdentity, ocm_repos: collections.abc.Iterable[ocm.OciOcmRepository | str], oci_client: oc.Client, absent_ok: bool=False, -) -> bytes | None: +) -> ocm.ComponentDescriptor | None: for ocm_repo in ocm_repos: if isinstance(ocm_repo, str): ocm_repo = ocm.OciOcmRepository( @@ -414,22 +412,25 @@ def _raw_component_descriptor_from_oci( ) # by contract, there must be exactly one layer (tar w/ component-descriptor) - if not (layers_count := len(manifest.layers) == 1): + if (layers_count := len(manifest.layers)) != 1: logger.warning(f'XXX unexpected amount of {layers_count=}') layer_digest = manifest.layers[0].digest layer_mimetype = manifest.layers[0].mediaType - if not layer_mimetype in ocm.oci.component_descriptor_mimetypes: - logger.warning(f'{target_ref=} {layer_mimetype=} was unexpected') - # XXX: check for non-tar-variant - - return oci_client.blob( + component_descriptor_blob = oci_client.blob( image_reference=target_ref, digest=layer_digest, stream=False, # manifests are typically small - do not bother w/ streaming ).content + return ocm.oci.component_descriptor_from_blob( + component_descriptor_blob=component_descriptor_blob, + layer_mimetype=layer_mimetype, + target_ref=target_ref, + component_id=component_id, + ) + def oci_component_descriptor_lookup( ocm_repository_lookup: OcmRepositoryLookup, @@ -477,7 +478,7 @@ def lookup( baseUrl=ocm_repo, ) - if raw := _raw_component_descriptor_from_oci( + if component_descriptor := component_descriptor_from_oci( component_id=component_id, ocm_repos=(ocm_repo,), oci_client=local_oci_client, @@ -485,23 +486,13 @@ def lookup( ): break else: - raw = None + component_descriptor = None - if not raw and absent_ok: + if not component_descriptor and absent_ok: return - elif not raw and not absent_ok: + elif not component_descriptor and not absent_ok: raise om.OciImageNotFoundException(component_id) - # wrap in fobj - blob_fobj = io.BytesIO(raw) - try: - component_descriptor = ocm.oci.component_descriptor_from_tarfileobj( - fileobj=blob_fobj, - ) - except tarfile.ReadError as tre: - tre.add_note(f'{component_id=}') - raise tre - # ensure OCM repository in which component descriptor was found is the current OCM repository component_descriptor.component.set_current_ocm_repo(ocm_repo) diff --git a/cnudie/retrieve_async.py b/cnudie/retrieve_async.py index 9c1f94d5f2..561bbc33ce 100644 --- a/cnudie/retrieve_async.py +++ b/cnudie/retrieve_async.py @@ -1,11 +1,9 @@ import collections.abc import dataclasses -import io import itertools import logging import os import shutil -import tarfile import tempfile import aiohttp.client_exceptions @@ -300,12 +298,12 @@ async def lookup( return lookup -async def raw_component_descriptor_from_oci( +async def component_descriptor_from_oci( component_id: ocm.ComponentIdentity, ocm_repos: collections.abc.Iterable[ocm.OciOcmRepository | str], oci_client: oca.Client, absent_ok: bool=False, -) -> bytes | None: +) -> ocm.ComponentDescriptor | None: for ocm_repo in ocm_repos: if isinstance(ocm_repo, str): ocm_repo = ocm.OciOcmRepository( @@ -357,21 +355,24 @@ async def raw_component_descriptor_from_oci( ) # by contract, there must be exactly one layer (tar w/ component-descriptor) - if not (layers_count := len(manifest.layers) == 1): + if (layers_count := len(manifest.layers)) != 1: logger.warning(f'XXX unexpected amount of {layers_count=}') layer_digest = manifest.layers[0].digest layer_mimetype = manifest.layers[0].mediaType - if not layer_mimetype in ocm.oci.component_descriptor_mimetypes: - logger.warning(f'{target_ref=} {layer_mimetype=} was unexpected') - # XXX: check for non-tar-variant - blob = await oci_client.blob( image_reference=target_ref, digest=layer_digest, ) - return await blob.content.read() + component_descriptor_blob = await blob.content.read() + + return ocm.oci.component_descriptor_from_blob( + component_descriptor_blob=component_descriptor_blob, + layer_mimetype=layer_mimetype, + target_ref=target_ref, + component_id=component_id, + ) def oci_component_descriptor_lookup( @@ -420,7 +421,7 @@ async def lookup( baseUrl=ocm_repo, ) - if raw := await raw_component_descriptor_from_oci( + if component_descriptor := await component_descriptor_from_oci( component_id=component_id, ocm_repos=(ocm_repo,), oci_client=local_oci_client, @@ -428,23 +429,13 @@ async def lookup( ): break else: - raw = None + component_descriptor = None - if not raw and absent_ok: + if not component_descriptor and absent_ok: return - elif not raw and not absent_ok: + elif not component_descriptor and not absent_ok: raise om.OciImageNotFoundException(component_id) - # wrap in fobj - blob_fobj = io.BytesIO(raw) - try: - component_descriptor = ocm.oci.component_descriptor_from_tarfileobj( - fileobj=blob_fobj, - ) - except tarfile.ReadError as tre: - tre.add_note(f'{component_id=}') - raise tre - # ensure OCM repository in which component descriptor was found is the current OCM repository component_descriptor.component.set_current_ocm_repo(ocm_repo) diff --git a/ocm/__init__.py b/ocm/__init__.py index 12459e38ce..86906b5466 100644 --- a/ocm/__init__.py +++ b/ocm/__init__.py @@ -712,6 +712,28 @@ def dateparse(v): if not _have_dacite: raise RuntimeError('not available without dacite') + # Normalize null arrays to empty lists for dacite + def normalize_nulls(obj): + # Null values for these keys will be replaced by empty lists, null values for other keys + # will be kept as-is + null_allowed_for = ( + 'repositoryContexts', + 'sources', + 'componentReferences', + 'resources', + ) + + if isinstance(obj, dict): + return { + k: normalize_nulls(v) if v is not None else ([] if k in null_allowed_for else v) + for k, v in obj.items() + } + elif isinstance(obj, list): + return [normalize_nulls(item) for item in obj] + return obj + + component_descriptor_dict = normalize_nulls(component_descriptor_dict) + component_descriptor = dacite.from_dict( data_class=ComponentDescriptor, data=component_descriptor_dict, diff --git a/ocm/__main__.py b/ocm/__main__.py index 40404b32e9..2dbaad3c72 100644 --- a/ocm/__main__.py +++ b/ocm/__main__.py @@ -333,7 +333,16 @@ def _fetch_component_descriptor( digest=layer_digest, stream=False, # manifests are typically small - do not bother w/ streaming ).content - return ocm.oci.component_descriptor_from_tarfileobj(fileobj=io.BytesIO(raw)) + + return ocm.oci.component_descriptor_from_blob( + component_descriptor_blob=raw, + layer_mimetype=layer_mimetype, + target_ref=target_ref, + component_id=ocm.ComponentIdentity( + name=name, + version=version, + ), + ) root_component_descriptor = _fetch_component_descriptor( name=cname, diff --git a/ocm/oci.py b/ocm/oci.py index bcd53eead5..545e11a080 100644 --- a/ocm/oci.py +++ b/ocm/oci.py @@ -7,10 +7,12 @@ import dataclasses import io +import json import logging import os import tarfile import typing + import yaml import ocm @@ -24,7 +26,8 @@ 'application/vnd.gardener.cloud.cnudie.component-descriptor.v2+yaml+tar' component_descriptor_mimetypes = ( component_descriptor_mimetype, - 'application/vnd.ocm.software.component-descriptor.v2+yaml+tar' + 'application/vnd.ocm.software.component-descriptor.v2+yaml+tar', + 'application/vnd.ocm.software.component-descriptor.v2+json', ) # mimetype for component-descriptor-oci-cfg-blobs component_descriptor_cfg_mimetype = \ @@ -81,16 +84,34 @@ def component_descriptor_to_tarfileobj( return tf.fileobj -def component_descriptor_from_tarfileobj( - fileobj: io.BytesIO, -): - with tarfile.open(fileobj=fileobj, mode='r') as tf: - component_descriptor_info = tf.getmember(component_descriptor_fname) - raw_dict = yaml.safe_load(tf.extractfile(component_descriptor_info).read()) - - logger.debug(raw_dict) - - if raw_dict is None: - raise ValueError('Component Descriptor appears to be empty') - - return ocm.ComponentDescriptor.from_dict(raw_dict) +def component_descriptor_from_blob( + component_descriptor_blob: bytes, + layer_mimetype: str, + target_ref: str | None=None, + component_id: ocm.ComponentIdentity | None=None, +) -> ocm.ComponentDescriptor: + if not layer_mimetype in component_descriptor_mimetypes: + logger.warning(f'{target_ref=} {layer_mimetype=} was unexpected') + + if '+tar' in layer_mimetype: + try: + with tarfile.open(fileobj=io.BytesIO(component_descriptor_blob), mode='r') as tf: + component_descriptor_info = tf.getmember(component_descriptor_fname) + component_descriptor_blob = tf.extractfile(component_descriptor_info).read() + except tarfile.ReadError as tre: + tre.add_note(f'{component_id=}') + raise tre + + if '+yaml' in layer_mimetype: + component_descriptor_dict = yaml.safe_load(component_descriptor_blob) + elif '+json' in layer_mimetype: + component_descriptor_dict = json.loads(component_descriptor_blob) + else: + raise ValueError(f'Unsupported component descriptor {layer_mimetype=}') + + logger.debug(component_descriptor_dict) + + if component_descriptor_dict is None: + raise ValueError('Component Descriptor appears to be empty') + + return ocm.ComponentDescriptor.from_dict(component_descriptor_dict)