diff --git a/satpy/composites/aux_data.py b/satpy/composites/aux_data.py index 3179f826a5..3463c7368d 100644 --- a/satpy/composites/aux_data.py +++ b/satpy/composites/aux_data.py @@ -156,7 +156,7 @@ def __call__(self, *args, **kwargs): if self.area is None: raise AttributeError("Area definition needs to be configured") img.attrs["area"] = self.area - img.attrs["sensor"] = None + img.attrs["sensor"] = set() img.attrs["mode"] = "".join(img.bands.data) img.attrs.pop("modifiers", None) img.attrs.pop("calibration", None) diff --git a/satpy/composites/config_loader.py b/satpy/composites/config_loader.py index e5165ada01..6617cfc665 100644 --- a/satpy/composites/config_loader.py +++ b/satpy/composites/config_loader.py @@ -31,7 +31,7 @@ from satpy import DataID, DataQuery from satpy._config import config_search_paths, get_entry_points_config_dirs, glob_config from satpy.dataset.dataid import minimal_default_keys_config -from satpy.utils import recursive_dict_update +from satpy.utils import normalize_sensor_name, recursive_dict_update logger = logging.getLogger(__name__) @@ -268,7 +268,7 @@ def load_compositor_configs_for_sensor(sensor_name: str) -> tuple[dict[str, dict DataID key -> key properties """ - config_filename = sensor_name + ".yaml" + config_filename = normalize_sensor_name(sensor_name) + ".yaml" logger.debug("Looking for composites config file %s", config_filename) paths = get_entry_points_config_dirs("satpy.composites") composite_configs = config_search_paths( diff --git a/satpy/composites/core.py b/satpy/composites/core.py index 1b124eb88a..b568bc8819 100644 --- a/satpy/composites/core.py +++ b/satpy/composites/core.py @@ -28,7 +28,7 @@ from satpy.dataset import DataID, combine_metadata from satpy.dataset.dataid import minimal_default_keys_config -from satpy.utils import unify_chunks +from satpy.utils import get_sensors_from_attrs, unify_chunks LOG = logging.getLogger(__name__) @@ -433,20 +433,11 @@ def _concat_datasets(self, projectables, mode): return data - def _get_sensors(self, projectables): - sensor = set() + def _get_sensors(self, projectables) -> set[str]: + sensors = set() for projectable in projectables: - current_sensor = projectable.attrs.get("sensor", None) - if current_sensor: - if isinstance(current_sensor, (str, bytes)): - sensor.add(current_sensor) - else: - sensor |= current_sensor - if len(sensor) == 0: - sensor = None - elif len(sensor) == 1: - sensor = list(sensor)[0] - return sensor + sensors.update(get_sensors_from_attrs(projectable.attrs)) + return sensors def __call__( self, diff --git a/satpy/enhancements/enhancer.py b/satpy/enhancements/enhancer.py index b196c63e28..7a217bc82e 100644 --- a/satpy/enhancements/enhancer.py +++ b/satpy/enhancements/enhancer.py @@ -24,7 +24,7 @@ from satpy._config import config_search_paths, get_entry_points_config_dirs from satpy.decision_tree import DecisionTree -from satpy.utils import get_logger, recursive_dict_update +from satpy.utils import get_logger, get_sensors_from_attrs, normalize_sensor_name, recursive_dict_update LOG = get_logger(__name__) @@ -122,26 +122,23 @@ def __init__(self, enhancement_config_file=None): self.sensor_enhancement_configs = [] - def get_sensor_enhancement_config(self, sensor): + def get_sensor_enhancement_config(self, sensors: set[str]): """Get the sensor-specific config.""" - if isinstance(sensor, str): - # one single sensor - sensor = [sensor] - paths = get_entry_points_config_dirs("satpy.enhancements") - for sensor_name in sensor: - config_fn = os.path.join("enhancements", sensor_name + ".yaml") + for sensor_name in sensors: + basename = normalize_sensor_name(sensor_name) + ".yaml" + config_fn = os.path.join("enhancements", basename) config_files = config_search_paths(config_fn, search_dirs=paths) # Note: Enhancement configuration files can't overwrite individual # options, only entire sections are overwritten for config_file in config_files: yield config_file - def add_sensor_enhancements(self, sensor): + def add_sensor_enhancements(self, sensors: set[str]): """Add sensor-specific enhancements.""" # XXX: Should we just load all enhancements from the base directory? new_configs = [] - for config_file in self.get_sensor_enhancement_config(sensor): + for config_file in self.get_sensor_enhancement_config(sensors): if config_file not in self.sensor_enhancement_configs: self.sensor_enhancement_configs.append(config_file) new_configs.append(config_file) @@ -209,9 +206,9 @@ def get_enhanced_image(dataset, enhance=None, overlay=None, decorate=None, if enhancer is None or enhancer.enhancement_tree is None: LOG.debug("No enhancement being applied to dataset") else: - if dataset.attrs.get("sensor", None): - enhancer.add_sensor_enhancements(dataset.attrs["sensor"]) - + sensors = get_sensors_from_attrs(dataset.attrs) + if sensors: + enhancer.add_sensor_enhancements(sensors) enhancer.apply(img, **dataset.attrs) if overlay is not None: diff --git a/satpy/modifiers/_crefl_utils.py b/satpy/modifiers/_crefl_utils.py index b8a1d52a4b..b638bfbf2f 100644 --- a/satpy/modifiers/_crefl_utils.py +++ b/satpy/modifiers/_crefl_utils.py @@ -70,6 +70,7 @@ import xarray as xr from satpy.dataset.dataid import WavelengthRange +from satpy.utils import get_one_sensor_from_attrs, normalize_sensor_name LOG = logging.getLogger(__name__) @@ -282,7 +283,8 @@ def run_crefl(refl, :param avg_elevation: average elevation (usually pre-calculated and stored in CMGDEM.hdf) """ - runner_cls = _runner_class_for_sensor(refl.attrs["sensor"]) + sensor = get_one_sensor_from_attrs(refl.attrs) + runner_cls = _runner_class_for_sensor(sensor) runner = runner_cls(refl) corr_refl = runner(sensor_azimuth, sensor_zenith, solar_azimuth, solar_zenith, avg_elevation) return corr_refl @@ -384,7 +386,7 @@ def _run_crefl(self, mus, muv, phi, solar_zenith, sensor_zenith, height, coeffs) def _runner_class_for_sensor(sensor_name: str) -> Type[_CREFLRunner]: try: - return _SENSOR_TO_RUNNER[sensor_name] + return _SENSOR_TO_RUNNER[normalize_sensor_name(sensor_name)] except KeyError: raise NotImplementedError(f"Don't know how to apply CREFL to data from sensor {sensor_name}.") diff --git a/satpy/modifiers/atmosphere.py b/satpy/modifiers/atmosphere.py index c7144c27ca..1ed3f5d026 100644 --- a/satpy/modifiers/atmosphere.py +++ b/satpy/modifiers/atmosphere.py @@ -26,6 +26,7 @@ from satpy.modifiers import ModifierBase from satpy.modifiers._crefl import ReflectanceCorrector # noqa from satpy.modifiers.angles import compute_relative_azimuth, get_angles, get_satellite_zenith_angle +from satpy.utils import get_one_sensor_from_attrs, get_pyspectral_sensor_name logger = logging.getLogger(__name__) @@ -104,7 +105,10 @@ def __call__(self, projectables, optional_datasets=None, **info): logger.info("Removing Rayleigh scattering with atmosphere '%s' and " "aerosol type '%s' for '%s'", atmosphere, aerosol_type, vis.attrs["name"]) - corrector = Rayleigh(vis.attrs["platform_name"], vis.attrs["sensor"], + sensor = get_pyspectral_sensor_name( + get_one_sensor_from_attrs(vis.attrs) + ) + corrector = Rayleigh(vis.attrs["platform_name"], sensor, atmosphere=atmosphere, aerosol_type=aerosol_type) @@ -158,8 +162,11 @@ def __call__(self, projectables, optional_datasets=None, **info): satz = satz.data # get dask array underneath logger.info("Correction for limb cooling") + sensor = get_pyspectral_sensor_name( + get_one_sensor_from_attrs(band.attrs) + ) corrector = AtmosphericalCorrection(band.attrs["platform_name"], - band.attrs["sensor"]) + sensor) atm_corr = da.map_blocks(_call_mapped_correction, satz, band.data, corrector=corrector, diff --git a/satpy/modifiers/spectral.py b/satpy/modifiers/spectral.py index 402b5606d4..9e55972e0b 100644 --- a/satpy/modifiers/spectral.py +++ b/satpy/modifiers/spectral.py @@ -22,6 +22,7 @@ import xarray as xr from satpy.modifiers import ModifierBase +from satpy.utils import get_one_sensor_from_attrs, get_pyspectral_sensor_name try: from pyspectral.near_infrared_reflectance import Calculator @@ -131,8 +132,10 @@ def _init_reflectance_calculator(self, metadata): if not Calculator: logger.info("Couldn't load pyspectral") raise ImportError("No module named pyspectral.near_infrared_reflectance") - - reflectance_3x_calculator = Calculator(metadata["platform_name"], metadata["sensor"], metadata["name"], + sensor = get_pyspectral_sensor_name( + get_one_sensor_from_attrs(metadata) + ) + reflectance_3x_calculator = Calculator(metadata["platform_name"], sensor, metadata["name"], sunz_threshold=self.sun_zenith_threshold, masking_limit=self.masking_limit) return reflectance_3x_calculator diff --git a/satpy/readers/abi_l1b.py b/satpy/readers/abi_l1b.py index 48e82f6968..b144529ebe 100644 --- a/satpy/readers/abi_l1b.py +++ b/satpy/readers/abi_l1b.py @@ -73,7 +73,7 @@ def get_dataset(self, key, info): def _adjust_attrs(self, data, key): data.attrs.update({"platform_name": self.platform_name, - "sensor": self.sensor}) + "sensor": {self.sensor}}) # Add orbital parameters projection = self.nc["goes_imager_projection"] data.attrs["orbital_parameters"] = { diff --git a/satpy/readers/core/abi.py b/satpy/readers/core/abi.py index e50cd616c4..d51764dc38 100644 --- a/satpy/readers/core/abi.py +++ b/satpy/readers/core/abi.py @@ -111,7 +111,7 @@ def _rename_dims(nc): @property def sensor(self): """Get sensor name for current file handler.""" - return "abi" + return "ABI" def __getitem__(self, item): """Wrap `self.nc[item]` for better floating point precision. diff --git a/satpy/readers/core/file_handlers.py b/satpy/readers/core/file_handlers.py index ac6626959c..6917988aa3 100644 --- a/satpy/readers/core/file_handlers.py +++ b/satpy/readers/core/file_handlers.py @@ -155,7 +155,7 @@ def end_time(self): return self.filename_info.get("end_time", self.start_time) @property - def sensor_names(self): + def sensor_names(self) -> set: """List of sensors represented in this file.""" raise NotImplementedError diff --git a/satpy/scene.py b/satpy/scene.py index 8693cc77b4..afcdb5dbae 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -36,7 +36,7 @@ from satpy.dependency_tree import DependencyTree from satpy.node import CompositorNode, MissingDependencies, ReaderNode from satpy.readers.core.loading import load_readers -from satpy.utils import convert_remote_files_to_fsspec, get_storage_options_from_reader_kwargs +from satpy.utils import convert_remote_files_to_fsspec, get_sensors_from_attrs, get_storage_options_from_reader_kwargs LOG = logging.getLogger(__name__) @@ -197,12 +197,7 @@ def sensor_names(self) -> set[str]: def _contained_sensor_names(self) -> set[str]: sensor_names = set() for data_arr in self.values(): - if "sensor" not in data_arr.attrs: - continue - if isinstance(data_arr.attrs["sensor"], str): - sensor_names.add(data_arr.attrs["sensor"]) - elif isinstance(data_arr.attrs["sensor"], set): - sensor_names.update(data_arr.attrs["sensor"]) + sensor_names.update(get_sensors_from_attrs(data_arr.attrs)) return sensor_names @property diff --git a/satpy/utils.py b/satpy/utils.py index bdf9d77e4c..1173d0718a 100644 --- a/satpy/utils.py +++ b/satpy/utils.py @@ -940,3 +940,36 @@ def flatten_dict(d, parent_key="", sep="_"): else: items.append((new_key, v)) return dict(items) + + +def get_sensors_from_attrs(attrs: dict[str,Any]) -> set[str]: + """Get sensor names from dataset attributes.""" + return attrs.get("sensor", set()) + + +def normalize_sensor_name(sensor: str) -> str: + """Normalize sensor name for internal usage.""" + return sensor.replace("-", "").replace(" ", "_").replace("/", "-").lower() + + +def get_one_sensor_from_attrs(attrs: dict[str,Any]) -> str: + """Get a single sensor name from dataset attributes.""" + sensors = get_sensors_from_attrs(attrs) + if not sensors: + raise KeyError("No 'sensor' dataset attribute") + if len(sensors) > 1: + logger.warning(f"More than one sensor in dataset attributes, will use the first value: {sensors}") + return list(sensors)[0] + + +def get_pyspectral_sensor_name(sensor: str) -> str: + """Get sensor name expected by pyspectral.""" + return normalize_sensor_name(sensor) + + +def serialize_sensors(sensors: set[str]) -> str: + """Serialize a set of sensors.""" + return "-".join( + sensor.replace("-", "").replace(" ", "").replace("/", "").lower() + for sensor in sorted(sensors) + ) diff --git a/satpy/writers/core/base.py b/satpy/writers/core/base.py index e0a53d0f7e..f8c1c1616e 100644 --- a/satpy/writers/core/base.py +++ b/satpy/writers/core/base.py @@ -16,6 +16,7 @@ """Shared objects and base classes for writers.""" from __future__ import annotations +import contextlib import logging import os import typing @@ -23,6 +24,7 @@ from satpy.aux_download import DataDownloadMixin from satpy.plugin_base import Plugin +from satpy.utils import serialize_sensors from satpy.writers.core.compute import compute_writer_results, split_results if typing.TYPE_CHECKING: @@ -136,8 +138,8 @@ def create_filename_parser(self, base_dir): @staticmethod def _prepare_metadata_for_filename_formatting(attrs): - if isinstance(attrs.get("sensor"), set): - attrs["sensor"] = "-".join(sorted(attrs["sensor"])) + with contextlib.suppress(KeyError): + attrs["sensor"] = serialize_sensors(attrs["sensor"]) def get_filename(self, **kwargs): """Create a filename where output data will be saved. diff --git a/satpy/writers/mitiff.py b/satpy/writers/mitiff.py index 7788d4b78a..0ed78e1a6a 100644 --- a/satpy/writers/mitiff.py +++ b/satpy/writers/mitiff.py @@ -28,6 +28,7 @@ from satpy.dataset import DataID, DataQuery from satpy.enhancements.enhancer import get_enhanced_image +from satpy.utils import get_one_sensor_from_attrs from satpy.writers.core.image import ImageWriter if typing.TYPE_CHECKING: @@ -53,12 +54,9 @@ def _adjust_kwargs(dataset, kwargs): if "start_time" not in kwargs: kwargs["start_time"] = dataset.attrs["start_time"] if "sensor" not in kwargs: - kwargs["sensor"] = dataset.attrs["sensor"] - # Sensor attrs could be set. MITIFFs needing to handle sensor can only have one sensor - # Assume the first value of set as the sensor. - if isinstance(kwargs["sensor"], set): - LOG.warning("Sensor is set, will use the first value: %s", kwargs["sensor"]) - kwargs["sensor"] = (list(kwargs["sensor"]))[0] + # MITIFFs needing to handle sensor can only have one sensor + # Assume the first value of set as the sensor. + kwargs["sensor"] = get_one_sensor_from_attrs(dataset.attrs) class MITIFFWriter(ImageWriter):