From c0a07c27bae8bfd45c8bb00f5ab6aae528ad3cac Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Mon, 27 Jul 2026 19:52:42 +0200 Subject: [PATCH 01/27] unify stac harvester and stac provider functionality --- .../src/source/gdal_source/reader.rs | 1 - .../gdal_worker_process/process_impl.rs | 23 +- .../gdal_worker_process/process_pool.rs | 18 +- geoengine/sentinel.json | 438 +++++ geoengine/services/src/bin/geoengine-cli.rs | 10 +- geoengine/services/src/cli/mod.rs | 2 + .../src/cli/stac_harvester/discover.rs | 940 +++++++++++ .../src/cli/stac_harvester/harvest.rs | 1491 +++++++++++++++++ .../services/src/cli/stac_harvester/mod.rs | 47 + geoengine/services/src/cli/stac_import.rs | 14 +- .../src/datasets/external/stac/common.rs | 984 +++++++++++ .../datasets/external/stac/loading_info.rs | 92 +- .../src/datasets/external/stac/mod.rs | 1 + .../collections/code-de-minimal.json | 71 + .../collections/landsat-c2-l1-minimal.json | 55 + .../expected-mapping-code-de.json | 69 + .../expected-mapping-landsat-c2-l1.json | 44 + .../items/code-de-harvest-test.json | 200 +++ .../items/landsat-c2-l1-harvest-test.json | 82 + 19 files changed, 4547 insertions(+), 35 deletions(-) create mode 100644 geoengine/sentinel.json create mode 100644 geoengine/services/src/cli/stac_harvester/discover.rs create mode 100644 geoengine/services/src/cli/stac_harvester/harvest.rs create mode 100644 geoengine/services/src/cli/stac_harvester/mod.rs create mode 100644 geoengine/services/src/datasets/external/stac/common.rs create mode 100644 geoengine/test_data/stac_responses/collections/code-de-minimal.json create mode 100644 geoengine/test_data/stac_responses/collections/landsat-c2-l1-minimal.json create mode 100644 geoengine/test_data/stac_responses/expected-mapping-code-de.json create mode 100644 geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json create mode 100644 geoengine/test_data/stac_responses/items/code-de-harvest-test.json create mode 100644 geoengine/test_data/stac_responses/items/landsat-c2-l1-harvest-test.json diff --git a/geoengine/operators/src/source/gdal_source/reader.rs b/geoengine/operators/src/source/gdal_source/reader.rs index ebb3710f26..16075b0b26 100644 --- a/geoengine/operators/src/source/gdal_source/reader.rs +++ b/geoengine/operators/src/source/gdal_source/reader.rs @@ -144,7 +144,6 @@ mod tests { CacheHint::default(), ) }) - .map_err(Into::into) } fn tile_information_with_partition_and_shape( diff --git a/geoengine/operators/src/source/gdal_worker_process/process_impl.rs b/geoengine/operators/src/source/gdal_worker_process/process_impl.rs index 8212a44d76..3ca225a3a2 100644 --- a/geoengine/operators/src/source/gdal_worker_process/process_impl.rs +++ b/geoengine/operators/src/source/gdal_worker_process/process_impl.rs @@ -785,7 +785,7 @@ mod tests { let (sender, receiver) = ipc_channel::ipc::channel().unwrap(); - sender.send(msg.clone()).unwrap(); + sender.send(msg).unwrap(); let recv = receiver.recv().unwrap(); assert_eq!(msg, recv); } @@ -797,7 +797,7 @@ mod tests { let (sender, receiver) = ipc_channel::ipc::channel().unwrap(); - sender.send(msg.clone()).unwrap(); + sender.send(msg).unwrap(); let recv = receiver.recv().unwrap(); assert_eq!(msg, recv); } @@ -941,12 +941,12 @@ mod tests { sender.send(msg).unwrap(); let rx_result = receiver .recv() - .inspect_err(|e| panic!("IPC receive: {:?}", e)) + .inspect_err(|e| panic!("IPC receive: {e:?}")) .unwrap(); let payload = match rx_result { Ok(r) => r, - Err(e) => panic!("Error receiving from IPC process: {:?}", e), + Err(e) => panic!("Error receiving from IPC process: {e:?}"), }; let result_2: GdalIpcPayload = (&payload).try_into().unwrap(); @@ -1003,8 +1003,7 @@ mod tests { let dataset = gdc.get_or_open(&dataset_params).unwrap(); let reader_payload = - GdalHandling::load_tile_data::(dataset, &dataset_params, gdal_read_advice) - .map_err(IpcProcessError::from)?; + GdalHandling::load_tile_data::(dataset, &dataset_params, gdal_read_advice)?; let grid_and_props: GridAndProperties = reader_payload.into(); Ok(grid_and_props.into()) @@ -1144,8 +1143,10 @@ mod tests { // file not found => specific error match result { - Err(IpcProcessError::GdalError { kind, .. }) - if kind == IpcProcessGdalErrorKind::FileNotFound => {} + Err(IpcProcessError::GdalError { + kind: IpcProcessGdalErrorKind::FileNotFound, + .. + }) => {} _ => panic!("expected FileNotFound error"), } @@ -1235,8 +1236,10 @@ mod tests { // file not found => specific error match result { - Err(IpcProcessError::GdalError { kind, .. }) - if kind == IpcProcessGdalErrorKind::FileNotFound => {} + Err(IpcProcessError::GdalError { + kind: IpcProcessGdalErrorKind::FileNotFound, + .. + }) => {} _ => panic!("expected FileNotFound error"), } diff --git a/geoengine/operators/src/source/gdal_worker_process/process_pool.rs b/geoengine/operators/src/source/gdal_worker_process/process_pool.rs index 5330d3988a..27d00ded09 100644 --- a/geoengine/operators/src/source/gdal_worker_process/process_pool.rs +++ b/geoengine/operators/src/source/gdal_worker_process/process_pool.rs @@ -1033,7 +1033,7 @@ mod tests { #[test] fn score_exact_match_fresh() { let w = window([100, 100], [200, 200]); - let aff = make_affinity(42, 1, w.clone()); + let aff = make_affinity(42, 1, w); let score = aff.calculate_score(42, 1, &w, aff.timestamp); assert_approx_eq!( @@ -1058,7 +1058,7 @@ mod tests { #[test] fn score_different_band_same_dataset() { let w = window([0, 0], [7, 7]); - let aff = make_affinity(42, 1, w.clone()); + let aff = make_affinity(42, 1, w); let score = aff.calculate_score(42, 2, &w, aff.timestamp); assert_approx_eq!(f64, score, SCORE_DATASET_MATCH); @@ -1067,7 +1067,7 @@ mod tests { #[test] fn score_different_dataset() { let w = window([0, 0], [7, 7]); - let aff = make_affinity(42, 1, w.clone()); + let aff = make_affinity(42, 1, w); let score = aff.calculate_score(99, 1, &w, aff.timestamp); assert_approx_eq!(f64, score, 0.0); @@ -1076,8 +1076,10 @@ mod tests { #[test] fn score_expired_cache() { let w = window([0, 0], [7, 7]); - let mut aff = make_affinity(42, 1, w.clone()); - aff.timestamp = Instant::now() - Duration::from_secs_f64(CACHE_TTL_SECS + 1.0); + let mut aff = make_affinity(42, 1, w); + aff.timestamp = Instant::now() + .checked_sub(Duration::from_secs_f64(CACHE_TTL_SECS + 1.0)) + .unwrap(); let score = aff.calculate_score(42, 1, &w, Instant::now()); assert_approx_eq!(f64, score, 0.0); @@ -1145,8 +1147,8 @@ mod tests { GridIdx2D::new([0, 0]), GridShape2D::new([256, 256]), ), - read_window_bounds: w.clone(), - bounds_of_target: w.clone(), + read_window_bounds: w, + bounds_of_target: w, flip_y: false, }, data_type: RasterDataType::U8, @@ -1287,7 +1289,7 @@ mod tests { let dispatcher = GdalPoolDispatcher::new(pool); let w = window([0, 0], [255, 255]); - let (msg_a, _) = make_request(42, 1, w.clone()); + let (msg_a, _) = make_request(42, 1, w); let (msg_b, _) = make_request(42, 1, w); let d1 = dispatcher.clone(); diff --git a/geoengine/sentinel.json b/geoengine/sentinel.json new file mode 100644 index 0000000000..aab8411031 --- /dev/null +++ b/geoengine/sentinel.json @@ -0,0 +1,438 @@ +{ + "name": "sentinel-2-l2a from STAC", + "id": "11154a6f-ba05-422d-aafc-4812864938dc", + "description": "Auto-discovered mapping for STAC collection 'sentinel-2-l2a' at https://stac.code-de.org/v1", + "priority": 50, + "apiUrl": "https://stac.code-de.org/v1", + "collectionName": "sentinel-2-l2a", + "s3Config": null, + "timeDimension": { + "regular": { + "origin": 0, + "step": { + "granularity": "days", + "step": 1 + } + } + }, + "datasets": [ + { + "name": "sentinel-2-l2a EPSG:32632 U16 20m", + "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", + "data_type": "U16", + "resolution": { + "x": 20.0, + "y": 20.0 + }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": 699960.0, + "y": 5500020.0 + }, + "xPixelSize": 20.0, + "yPixelSize": -20.0 + }, + "gridBounds": { + "min": [ + -34998, + -224999 + ], + "max": [ + 15001, + 275001 + ] + } + }, + "state": "source" + }, + "bands": [ + { + "asset_title": "Aerosol optical thickness (AOT) - 20m", + "band_name": null + }, + { + "asset_title": "Blue (band 2) - 20m", + "band_name": null + }, + { + "asset_title": "Coastal aerosol (band 1) - 20m", + "band_name": null + }, + { + "asset_title": "Green (band 3) - 20m", + "band_name": null + }, + { + "asset_title": "NIR 2 (band 8A) - 20m", + "band_name": null + }, + { + "asset_title": "Red (band 4) - 20m", + "band_name": null + }, + { + "asset_title": "Red edge 1 (band 5) - 20m", + "band_name": null + }, + { + "asset_title": "Red edge 2 (band 6) - 20m", + "band_name": null + }, + { + "asset_title": "Red edge 3 (band 7) - 20m", + "band_name": null + }, + { + "asset_title": "Red edge 3 (band 7) - 60m", + "band_name": null + }, + { + "asset_title": "SWIR 1 (band 11) - 20m", + "band_name": null + }, + { + "asset_title": "SWIR 2 (band 12) - 20m", + "band_name": null + }, + { + "asset_title": "Water vapour (WVP) - 20m", + "band_name": null + } + ] + }, + { + "name": "sentinel-2-l2a EPSG:32632 U8 10m", + "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", + "data_type": "U8", + "resolution": { + "x": 10.0, + "y": 10.0 + }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": 699960.0, + "y": 5500020.0 + }, + "xPixelSize": 10.0, + "yPixelSize": -10.0 + }, + "gridBounds": { + "min": [ + -69996, + -449998 + ], + "max": [ + 30003, + 550002 + ] + } + }, + "state": "source" + }, + "bands": [ + { + "asset_title": "True color image", + "band_name": "True color image [B04]" + }, + { + "asset_title": "True color image [B02]", + "band_name": null + }, + { + "asset_title": "True color image [B03]", + "band_name": null + }, + { + "asset_title": "True color image [B04]", + "band_name": null + } + ] + }, + { + "name": "sentinel-2-l2a EPSG:32632 U16 10m", + "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", + "data_type": "U16", + "resolution": { + "x": 10.0, + "y": 10.0 + }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": 699960.0, + "y": 5500020.0 + }, + "xPixelSize": 10.0, + "yPixelSize": -10.0 + }, + "gridBounds": { + "min": [ + -69996, + -449998 + ], + "max": [ + 30003, + 550002 + ] + } + }, + "state": "source" + }, + "bands": [ + { + "asset_title": "Aerosol optical thickness (AOT) - 10m", + "band_name": null + }, + { + "asset_title": "Blue (band 2) - 10m", + "band_name": null + }, + { + "asset_title": "Green (band 3) - 10m", + "band_name": null + }, + { + "asset_title": "NIR 1 (band 8) - 10m", + "band_name": null + }, + { + "asset_title": "Red (band 4) - 10m", + "band_name": null + }, + { + "asset_title": "Water vapour (WVP) - 10m", + "band_name": null + } + ] + }, + { + "name": "sentinel-2-l2a EPSG:32632 U8 20m", + "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", + "data_type": "U8", + "resolution": { + "x": 20.0, + "y": 20.0 + }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": 699960.0, + "y": 5500020.0 + }, + "xPixelSize": 20.0, + "yPixelSize": -20.0 + }, + "gridBounds": { + "min": [ + -34998, + -224999 + ], + "max": [ + 15001, + 275001 + ] + } + }, + "state": "source" + }, + "bands": [ + { + "asset_title": "Cloud probability (CLD) - 20m", + "band_name": "Cloud probability (CLD) - 20m" + }, + { + "asset_title": "Scene classfication map (SCL) - 20m", + "band_name": null + }, + { + "asset_title": "Scene classification map (SCL) - 20m", + "band_name": "Scene classification map (SCL) - 20m" + }, + { + "asset_title": "Snow probability (SNW) - 20m", + "band_name": "Snow probability (SNW) - 20m" + }, + { + "asset_title": "True color image", + "band_name": "True color image [B04]" + }, + { + "asset_title": "True color image [B02]", + "band_name": null + }, + { + "asset_title": "True color image [B03]", + "band_name": null + }, + { + "asset_title": "True color image [B04]", + "band_name": null + } + ] + }, + { + "name": "sentinel-2-l2a EPSG:32632 U16 60m", + "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", + "data_type": "U16", + "resolution": { + "x": 60.0, + "y": 60.0 + }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": 699960.0, + "y": 5500020.0 + }, + "xPixelSize": 60.0, + "yPixelSize": -60.0 + }, + "gridBounds": { + "min": [ + -11666, + -74999 + ], + "max": [ + 5000, + 91667 + ] + } + }, + "state": "source" + }, + "bands": [ + { + "asset_title": "Aerosol optical thickness (AOT) - 60m", + "band_name": null + }, + { + "asset_title": "Blue (band 2) - 60m", + "band_name": null + }, + { + "asset_title": "Coastal aerosol (band 1) - 60m", + "band_name": null + }, + { + "asset_title": "Green (band 3) - 60m", + "band_name": null + }, + { + "asset_title": "NIR 2 (band 8A) - 60m", + "band_name": null + }, + { + "asset_title": "NIR 3 (band 9) - 60m", + "band_name": null + }, + { + "asset_title": "Red (band 4) - 60m", + "band_name": null + }, + { + "asset_title": "Red edge 1 (band 5) - 60m", + "band_name": null + }, + { + "asset_title": "Red edge 2 (band 6) - 60m", + "band_name": null + }, + { + "asset_title": "Red edge 3 (band 7) - 60m", + "band_name": "Red edge 3 (band 7) - 60m" + }, + { + "asset_title": "SWIR 1 (band 11) - 60m", + "band_name": null + }, + { + "asset_title": "SWIR 2 (band 12) - 60m", + "band_name": null + }, + { + "asset_title": "Water vapour (WVP) - 60m", + "band_name": null + } + ] + }, + { + "name": "sentinel-2-l2a EPSG:32632 U8 60m", + "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", + "data_type": "U8", + "resolution": { + "x": 60.0, + "y": 60.0 + }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": 699960.0, + "y": 5500020.0 + }, + "xPixelSize": 60.0, + "yPixelSize": -60.0 + }, + "gridBounds": { + "min": [ + -11666, + -74999 + ], + "max": [ + 5000, + 91667 + ] + } + }, + "state": "source" + }, + "bands": [ + { + "asset_title": "Cloud probability (CLD) - 60m", + "band_name": "Cloud probability (CLD) - 60m" + }, + { + "asset_title": "Scene classfication map (SCL) - 60m", + "band_name": null + }, + { + "asset_title": "Scene classification map (SCL) - 60m", + "band_name": "Scene classification map (SCL) - 60m" + }, + { + "asset_title": "Snow probability (SNW) - 60m", + "band_name": "Snow probability (SNW) - 60m" + }, + { + "asset_title": "True color image", + "band_name": "True color image [B04]" + }, + { + "asset_title": "True color image [B02]", + "band_name": null + }, + { + "asset_title": "True color image [B03]", + "band_name": null + }, + { + "asset_title": "True color image [B04]", + "band_name": null + } + ] + } + ] +} diff --git a/geoengine/services/src/bin/geoengine-cli.rs b/geoengine/services/src/bin/geoengine-cli.rs index cad4045b21..4e64ffe586 100644 --- a/geoengine/services/src/bin/geoengine-cli.rs +++ b/geoengine/services/src/bin/geoengine-cli.rs @@ -1,8 +1,8 @@ use clap::{Parser, Subcommand}; use geoengine_services::cli::{ - CheckSuccessfulStartup, ExpressionToolchainFile, Heartbeat, OpenAPIGenerate, StacImport, - TileImport, check_heartbeat, check_successful_startup, output_openapi_json, - output_toolchain_file, stac_import, tile_import, + CheckSuccessfulStartup, ExpressionToolchainFile, Heartbeat, OpenAPIGenerate, StacHarvester, + StacImport, TileImport, check_heartbeat, check_successful_startup, output_openapi_json, + output_toolchain_file, stac_harvester, stac_import, tile_import, }; /// CLI for Geo Engine Utilities @@ -25,6 +25,9 @@ enum Commands { #[command(name = "openapi")] OpenAPI(OpenAPIGenerate), + // Harvests STAC collections using a dataset mapping + StacHarvest(StacHarvester), + // Imports a STAC catalog as a dataset StacImport(StacImport), @@ -42,6 +45,7 @@ impl Commands { Commands::CheckSuccessfulStartup(params) => check_successful_startup(params).await, Commands::Heartbeat(params) => check_heartbeat(params).await, Commands::OpenAPI(params) => output_openapi_json(params).await, + Commands::StacHarvest(params) => stac_harvester(params).await, Commands::StacImport(params) => stac_import(params).await, Commands::TileImport(params) => tile_import(params).await, Commands::ExpressionToolchainFile(params) => output_toolchain_file(params).await, diff --git a/geoengine/services/src/cli/mod.rs b/geoengine/services/src/cli/mod.rs index e8ee25a1cf..80a45f1940 100644 --- a/geoengine/services/src/cli/mod.rs +++ b/geoengine/services/src/cli/mod.rs @@ -2,6 +2,7 @@ mod check_successful_startup; mod expression_toolchain_file; mod heartbeat; mod openapi; +mod stac_harvester; mod stac_import; mod tile_import; @@ -9,5 +10,6 @@ pub use check_successful_startup::{CheckSuccessfulStartup, check_successful_star pub use expression_toolchain_file::{ExpressionToolchainFile, output_toolchain_file}; pub use heartbeat::{Heartbeat, check_heartbeat}; pub use openapi::{OpenAPIGenerate, output_openapi_json}; +pub use stac_harvester::{StacHarvester, stac_harvester}; pub use stac_import::{StacImport, stac_import}; pub use tile_import::{TileImport, tile_import}; diff --git a/geoengine/services/src/cli/stac_harvester/discover.rs b/geoengine/services/src/cli/stac_harvester/discover.rs new file mode 100644 index 0000000000..b2aec57207 --- /dev/null +++ b/geoengine/services/src/cli/stac_harvester/discover.rs @@ -0,0 +1,940 @@ +use std::{collections::HashMap, path::PathBuf}; + +use anyhow::Context; +use clap::{Parser, ValueEnum}; +use geoengine_datatypes::{ + dataset::DataProviderId, + primitives::SpatialResolution, + raster::{GeoTransform, GridBoundingBox2D, GridIdx2D, RasterDataType}, + spatial_reference::{SpatialReference, SpatialReferenceAuthority}, + util::Identifier, +}; +use ordered_float::OrderedFloat; +use tracing::{info, warn}; + +use crate::api::model::datatypes::{Measurement, UnitlessMeasurement, UnitlessMeasurementTypeTag}; +use crate::api::model::operators::RasterBandDescriptor; +use crate::datasets::external::stac::{ + StacDataProviderDefinition, StacProviderDataset, StacProviderDatasetBand, StacProviderS3Config, + common, +}; +use geoengine_datatypes::primitives::{ + RegularTimeDimension as DtRegularTimeDimension, TimeDimension as DtTimeDimension, +}; +use geoengine_operators::engine::SpatialGridDescriptor as GeoOpSpatialGridDescriptor; + +// --------------------------------------------------------------------------- +// Discover Mapping +// --------------------------------------------------------------------------- + +/// Probe a STAC collection and sample items to auto-discover the dataset mapping. +#[derive(Debug, Parser)] +pub struct StacDiscoverMapping { + /// STAC API URL + #[arg(long)] + pub stac_url: String, + + /// STAC collection to scan + #[arg(long, default_value = "sentinel-2-l2a")] + pub stac_collection: String, + + /// S3 endpoint (if assets are hosted on S3-compatible storage) + #[arg(long)] + pub s3_endpoint: Option, + + /// S3 access key + #[arg(long)] + pub s3_access_key: Option, + + /// S3 secret key + #[arg(long)] + pub s3_secret_key: Option, + + /// Number of sample items to probe (default: 5) + #[arg(long, default_value_t = 5)] + pub sample_items: usize, + + /// Output file for the mapping JSON (default: stdout) + #[arg(long)] + pub output: Option, + + /// Filter STAC item fields to reduce response size + #[arg(long, default_value_t = false)] + pub filter_item_fields: bool, + + /// File types to import + #[arg(long, value_enum, num_args = 1.., value_delimiter = ' ', default_values_t = [ImportFileType::Cog])] + pub file_types: Vec, + + /// Filter by EPSG codes (only include datasets for these codes) + #[clap(long, value_parser, num_args = 0.., value_delimiter = ' ')] + pub epsgs: Vec, + + /// Verbose output + #[arg(long, default_value_t = false)] + pub verbose: bool, + + /// Bounding box to probe: minx miny maxx maxy (optional, defaults to UTM 32N area) + #[clap(short, long, value_parser, num_args = 1.., value_delimiter = ' ', default_value = "6.0 47.0 12.0 55.0")] + pub bbox: Option>, + + /// Use the full projected CRS extent for grid bounds instead of the first asset's shape. + /// For UTM projections, this computes grid indices covering the entire zone (easting 0-1,000,000, + /// northing 0-10,000,000) so that multi-tile datasets have correct global raster bounds. + #[arg(long, default_value_t = false)] + pub full_projection_grid: bool, + + /// Time dimension granularity (default: days) + #[arg(long, default_value = "days")] + pub time_granularity: String, + + /// Time dimension step (default: 1) + #[arg(long, default_value_t = 1)] + pub time_step: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub enum ImportFileType { + Cog, + Jp2, +} + +// --------------------------------------------------------------------------- +// Discover Mapping Implementation +// --------------------------------------------------------------------------- + +#[allow(clippy::too_many_lines)] +pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), anyhow::Error> { + let client = reqwest::Client::new(); + + info!( + "Discovering mapping for STAC collection '{}' at {}", + params.stac_collection, params.stac_url + ); + + let collection_url = format!( + "{}/collections/{}", + params.stac_url.trim_end_matches('/'), + params.stac_collection + ); + + let collection: stac::Collection = stac_api_request_parse(&client, &collection_url) + .await + .context("Failed to fetch STAC collection")?; + + // Scan collection-level item_assets for bands (partial information) + let mut dataset_bands: HashMap> = HashMap::new(); + + for (_asset_key, asset) in &collection.item_assets { + if !matches_selected_file_types_static(asset.r#type.as_deref(), ¶ms.file_types) { + continue; + } + + if let Ok(Some(bands)) = + scan_item_asset_common(&collection.version, asset, collection.summaries.as_ref()) + { + merge_dataset_bands(&mut dataset_bands, bands); + } + } + + if params.verbose { + info!( + "Found {} data type/resolution combinations from collection metadata", + dataset_bands.len() + ); + } + + // Sample items to discover EPSG codes and additional band/resolution info + let items_url = format!( + "{}/collections/{}/items", + params.stac_url.trim_end_matches('/'), + params.stac_collection + ); + + let mut query_params = Vec::new(); + if params.filter_item_fields { + query_params.push(( + "fields".to_string(), + "stac_version,stac_extensions,properties.datetime,properties.updated,properties.proj:epsg,properties.proj:code,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.eo:bands,assets.*.raster:bands,assets.*.proj:epsg,assets.*.proj:code,assets.*.proj:transform,assets.*.proj:shape,assets.*.gsd,assets.*.type".to_string(), + )); + } + if let Some(bbox) = ¶ms.bbox + && bbox.len() == 4 + { + query_params.push(( + "bbox".to_string(), + format!("{},{},{},{}", bbox[0], bbox[1], bbox[2], bbox[3]), + )); + } + + query_params.push(("limit".to_string(), format!("{}", params.sample_items))); + + let items_response: stac::ItemCollection = + stac_api_request_with_params(&client, &items_url, &query_params) + .await + .context("Failed to fetch sample items")?; + + if items_response.items.is_empty() { + anyhow::bail!("No items found in the collection. Cannot discover mapping."); + } + + info!( + "Probing {} sample item(s) to discover EPSG codes and additional bands", + items_response.items.len() + ); + + let mut discovered_datasets: HashMap = HashMap::new(); + let mut sample_band_info: HashMap> = HashMap::new(); + + for item in &items_response.items { + let item_epsg = common::epsg_code_from_item(item, common::StacExtensionMajorVersion::V2); + + for (asset_key, asset) in &item.assets { + if !matches_selected_file_types_static(asset.r#type.as_deref(), ¶ms.file_types) { + continue; + } + + let Some(geo_transform) = common::geo_transform_from_fields(&asset.additional_fields) + else { + continue; + }; + + let data_type = common::data_type_from_asset_v1_1_0(asset) + .or_else(|| data_type_from_asset_v1_0_0_fallback(asset)); + let Some(data_type) = data_type else { + continue; + }; + + let epsg = common::epsg_code_from_fields( + common::StacExtensionMajorVersion::V2, + &asset.additional_fields, + ) + .or(item_epsg); + let Some(epsg) = epsg else { + continue; + }; + + if !params.epsgs.is_empty() && !params.epsgs.contains(&epsg) { + continue; + } + + let resolution: OrderedFloat = geo_transform.x_pixel_size().abs().into(); + + let dataset_key = DatasetKey { + epsg, + data_type, + resolution, + }; + let partial_key = PartialDatasetKey { + data_type, + resolution, + }; + + let asset_title = asset.title.as_deref().unwrap_or(asset_key).to_string(); + let band_names = common::band_names_from_asset_v1_1_0(asset) + .unwrap_or_else(|_| vec![asset_title.clone()]); + + let entry = sample_band_info.entry(partial_key.clone()).or_default(); + for bn in &band_names { + if !entry.iter().any(|(t, _)| t == &asset_title) { + entry.push((asset_title.clone(), bn.clone())); + } + } + + let info_entry = + discovered_datasets + .entry(dataset_key) + .or_insert(DiscoveredDatasetInfo { + geo_transform: Some(geo_transform), + proj_shape: common::proj_shape_from_fields(&asset.additional_fields), + srs: SpatialReference::new(SpatialReferenceAuthority::Epsg, epsg), + asset_count: 0, + }); + info_entry.asset_count += 1; + } + } + + if discovered_datasets.is_empty() { + anyhow::bail!( + "No matching assets found in sample items. Check your --file-types and --epsgs filters." + ); + } + + // Build the StacDataProviderDefinition + let time_dimension = parse_time_dimension(¶ms.time_granularity, params.time_step) + .map_err(|e| anyhow::anyhow!("{e}"))?; + + let s3_config = params + .s3_endpoint + .as_ref() + .map(|endpoint| StacProviderS3Config { + endpoint: endpoint.clone(), + access_key: params.s3_access_key.clone(), + secret_key: params.s3_secret_key.clone(), + }); + + let mut datasets: Vec = Vec::new(); + + for (dataset_key, info) in &discovered_datasets { + let partial_key = PartialDatasetKey { + data_type: dataset_key.data_type, + resolution: dataset_key.resolution, + }; + + let mut bands: Vec = Vec::new(); + + // Use bands from collection-level scan + if let Some(descriptors) = dataset_bands.get(&partial_key) { + for desc in descriptors { + bands.push(StacProviderDatasetBand { + asset_title: desc.name.clone(), + band_name: None, + }); + } + } + + // Enrich with sample item band info + if let Some(sample_bands) = sample_band_info.get(&partial_key) { + for (asset_title, band_name) in sample_bands { + if !bands.iter().any(|b| b.asset_title == *asset_title) { + bands.push(StacProviderDatasetBand { + asset_title: asset_title.clone(), + band_name: Some(band_name.clone()), + }); + } + } + } + + if bands.is_empty() { + warn!("No bands found for dataset {:?}, skipping", dataset_key); + continue; + } + + bands.sort_by(|a, b| a.asset_title.cmp(&b.asset_title)); + + let spatial_grid = if params.full_projection_grid { + // Compute grid bounds covering the full projected CRS extent + if let Some(gt) = info.geo_transform { + let grid_bounds = projection_grid_bounds(gt, dataset_key.epsg) + .unwrap_or_else(|| { + // Fallback: use first asset's shape + if let Some((height, width)) = info.proj_shape { + GridBoundingBox2D::new( + GridIdx2D::new([0, 0]), + GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), + ) + .expect("fallback grid bounds should be valid") + } else { + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) + .expect("zero-size grid bounds should be valid") + } + }); + GeoOpSpatialGridDescriptor::source_from_parts(gt, grid_bounds) + } else { + GeoOpSpatialGridDescriptor::source_from_parts( + GeoTransform::new( + (0.0, 0.0).into(), + dataset_key.resolution.into_inner(), + -dataset_key.resolution.into_inner(), + ), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) + .expect("zero-size grid bounds should be valid"), + ) + } + } else if let (Some(gt), Some((height, width))) = (info.geo_transform, info.proj_shape) { + GeoOpSpatialGridDescriptor::source_from_parts( + gt, + GridBoundingBox2D::new( + GridIdx2D::new([0, 0]), + GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), + ) + .unwrap_or_else(|_| { + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) + .expect("zero-size grid bounds should be valid") + }), + ) + } else { + GeoOpSpatialGridDescriptor::source_from_parts( + GeoTransform::new( + (0.0, 0.0).into(), + dataset_key.resolution.into_inner(), + -dataset_key.resolution.into_inner(), + ), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) + .expect("zero-size grid bounds should be valid"), + ) + }; + + let dataset_name = format!( + "{} EPSG:{} {:?} {}m", + params.stac_collection, dataset_key.epsg, dataset_key.data_type, dataset_key.resolution + ); + + datasets.push(StacProviderDataset { + name: dataset_name, + description: format!( + "Auto-discovered from STAC collection '{}'", + params.stac_collection + ), + data_type: dataset_key.data_type, + resolution: SpatialResolution::new_unchecked( + dataset_key.resolution.into_inner(), + dataset_key.resolution.into_inner(), + ), + projection: info.srs, + spatial_grid, + bands, + }); + } + + let provider_def = StacDataProviderDefinition { + name: format!("{} from STAC", params.stac_collection), + id: DataProviderId::new(), + description: format!( + "Auto-discovered mapping for STAC collection '{}' at {}", + params.stac_collection, params.stac_url + ), + priority: Some(50), + api_url: params.stac_url.clone(), + collection_name: params.stac_collection.clone(), + s3_config, + time_dimension, + datasets, + }; + + let json = serde_json::to_string_pretty(&provider_def) + .context("Failed to serialize mapping to JSON")?; + + if let Some(output_path) = ¶ms.output { + std::fs::write(output_path, &json) + .with_context(|| format!("Failed to write mapping to {}", output_path.display()))?; + println!("Mapping written to {}", output_path.display()); + } else { + println!("{json}"); + } + + Ok(()) +} + +struct DiscoveredDatasetInfo { + geo_transform: Option, + proj_shape: Option<(usize, usize)>, + srs: SpatialReference, + asset_count: u32, +} + +/// Compute grid bounds that cover the full projected CRS extent for the given +/// geo-transform and EPSG code. Currently handles UTM projections (zones 32601–32660 +/// and 32701–32760) with known extents. Returns `None` for unsupported CRS types. +fn projection_grid_bounds(gt: GeoTransform, epsg: u32) -> Option { + let (min_x, max_x, min_y, max_y) = projection_extent(epsg)?; + + let ox = gt.origin_coordinate.x; + let oy = gt.origin_coordinate.y; + let ps_x = gt.x_pixel_size(); + let ps_y = gt.y_pixel_size(); + + // For north-up images ps_y < 0 and origin is top-left. + // Pixel index i = (coord - origin) / pixel_size. + let min_x_idx = ((min_x - ox) / ps_x).floor() as isize; + let max_x_idx = ((max_x - ox) / ps_x).ceil() as isize - 1; + + let (min_y_idx, max_y_idx) = if ps_y < 0.0 { + // ps_y negative: top of area (max_y) → smallest row index + let top = ((max_y - oy) / ps_y).ceil() as isize; + // bottom of area (min_y) → largest row index + let bottom = ((min_y - oy) / ps_y).floor() as isize; + (top, bottom) + } else { + let top = ((max_y - oy) / ps_y).floor() as isize; + let bottom = ((min_y - oy) / ps_y).ceil() as isize - 1; + (top, bottom) + }; + + GridBoundingBox2D::new(GridIdx2D::new([min_x_idx, min_y_idx]), GridIdx2D::new([max_x_idx, max_y_idx])).ok() +} + +/// Return the projected extent `(min_x, max_x, min_y, max_y)` for a given EPSG code. +/// Known extents for UTM zones; other CRS types return `None`. +fn projection_extent(epsg: u32) -> Option<(f64, f64, f64, f64)> { + // UTM northern hemisphere zones EPSG:32601 – 32660 + if (32601..=32660).contains(&epsg) { + return Some((0.0, 1_000_000.0, 0.0, 10_000_000.0)); + } + // UTM southern hemisphere zones EPSG:32701 – 32760 + if (32701..=32760).contains(&epsg) { + return Some((0.0, 1_000_000.0, 0.0, 10_000_000.0)); + } + None +} + +/// A key that uniquely identifies a Geo Engine dataset derived from STAC assets. +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +struct DatasetKey { + epsg: u32, + data_type: RasterDataType, + resolution: OrderedFloat, +} + +/// Partial dataset key without EPSG (used during collection scanning). +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +struct PartialDatasetKey { + data_type: RasterDataType, + resolution: OrderedFloat, +} + +// --------------------------------------------------------------------------- +// Collection scanning helpers +// --------------------------------------------------------------------------- + +fn scan_item_asset_common( + collection_version: &stac::Version, + asset: &stac::ItemAsset, + collection_summaries: Option<&serde_json::Map>, +) -> Result>>, String> { + match collection_version { + stac::Version::v1_0_0 => scan_item_asset_v1_0_0_common(asset), + stac::Version::v1_1_0 => scan_item_asset_v1_1_0_common(asset, collection_summaries), + _ => Err(format!("Unsupported STAC version: {collection_version}")), + } +} + +fn scan_item_asset_v1_0_0_common( + asset: &stac::ItemAsset, +) -> Result>>, String> { + let mut dataset_bands: HashMap> = HashMap::new(); + + let Some(raster_bands) = asset.additional_fields.get("raster:bands") else { + return Ok(None); + }; + let raster_bands: Vec = + serde_json::from_value(raster_bands.clone()) + .map_err(|e| format!("invalid raster:bands: {e}"))?; + + let band_count = raster_bands.len(); + + let eo_bands = asset + .additional_fields + .get("eo:bands") + .and_then(|v| serde_json::from_value::>(v.clone()).ok()); + + if let Some(ref eo_bands_vec) = eo_bands { + if band_count != eo_bands_vec.len() { + return Ok(None); + } + } else if band_count != 1 { + return Ok(None); + } + + for (index, raster_band) in raster_bands.into_iter().enumerate() { + let data_type = raster_band + .data_type + .ok_or_else(|| "Missing data_type in raster band".to_string())?; + let raster_data_type = common::raster_data_type_from_stac_data_type(&data_type) + .ok_or_else(|| format!("Unsupported data type: {data_type:?}"))?; + + let geo_transform = common::geo_transform_from_fields(&asset.additional_fields) + .ok_or_else(|| "Missing proj:transform".to_string())?; + let resolution: OrderedFloat = geo_transform.x_pixel_size().into(); + + let band_name = if let Some(ref eo_bands_vec) = eo_bands { + common::v1_0_0_band_name( + asset.title.as_deref(), + Some(&eo_bands_vec[index]), + band_count, + ) + } else { + common::v1_0_0_band_name(asset.title.as_deref(), None, 1) + }; + + dataset_bands + .entry(PartialDatasetKey { + data_type: raster_data_type, + resolution, + }) + .or_default() + .push(RasterBandDescriptor { + name: band_name, + measurement: Measurement::Unitless(UnitlessMeasurement { + r#type: UnitlessMeasurementTypeTag::UnitlessMeasurementTypeTag, + }), + }); + } + + Ok(Some(dataset_bands)) +} + +fn scan_item_asset_v1_1_0_common( + asset: &stac::ItemAsset, + collection_summaries: Option<&serde_json::Map>, +) -> Result>>, String> { + let mut dataset_bands: HashMap> = HashMap::new(); + + let data_type = asset + .additional_fields + .get("data_type") + .ok_or_else(|| "Missing data_type in asset additional fields".to_string())? + .as_str() + .ok_or_else(|| "data_type is not a string".to_string())?; + + let raster_data_type = common::raster_data_type_from_stac_data_type_str(data_type) + .ok_or_else(|| format!("Unsupported data_type: {data_type}"))?; + + let band_names = common::band_names_from_item_asset_v1_1_0(asset)?; + + let resolution = asset + .additional_fields + .get("gsd") + .and_then(serde_json::Value::as_f64) + .or_else(|| { + common::geo_transform_from_fields(&asset.additional_fields) + .map(|gt| gt.x_pixel_size().abs()) + }) + .or_else(|| { + collection_summaries + .and_then(|s| s.get("gsd")) + .and_then(|v| v.as_array()) + .and_then(|arr| arr.first()) + .and_then(serde_json::Value::as_f64) + }) + .ok_or_else(|| "Missing attribute `gsd` or `proj:transform`".to_string())?; + + for band_name in band_names { + dataset_bands + .entry(PartialDatasetKey { + data_type: raster_data_type, + resolution: resolution.into(), + }) + .or_default() + .push(RasterBandDescriptor { + name: band_name.clone(), + measurement: Measurement::Unitless(UnitlessMeasurement { + r#type: UnitlessMeasurementTypeTag::UnitlessMeasurementTypeTag, + }), + }); + } + + Ok(Some(dataset_bands)) +} + +fn data_type_from_asset_v1_0_0_fallback(asset: &stac::Asset) -> Option { + asset + .additional_fields + .get("raster:bands") + .and_then(|v| v.as_array()) + .and_then(|bands| bands.first()) + .and_then(|band| band.get("data_type")) + .and_then(|v| v.as_str()) + .and_then(common::raster_data_type_from_stac_data_type_str) +} + +fn matches_selected_file_types_static( + media_type: Option<&str>, + file_types: &[ImportFileType], +) -> bool { + file_types.iter().any(|file_type| match file_type { + ImportFileType::Cog => common::is_cog_media_type(media_type), + ImportFileType::Jp2 => common::is_jp2_media_type(media_type), + }) +} + +fn merge_dataset_bands( + dataset_bands: &mut HashMap>, + additions: HashMap>, +) { + for (partial_key, band_descriptors) in additions { + let existing_bands = dataset_bands.entry(partial_key).or_default(); + for descriptor in band_descriptors { + if existing_bands.iter().all(|b| b.name != descriptor.name) { + existing_bands.push(descriptor); + } + } + } +} + +fn parse_time_dimension(granularity: &str, step: u64) -> Result { + let dt_granularity = match granularity.to_lowercase().as_str() { + "days" | "day" => geoengine_datatypes::primitives::TimeGranularity::Days, + "months" | "month" => geoengine_datatypes::primitives::TimeGranularity::Months, + "years" | "year" => geoengine_datatypes::primitives::TimeGranularity::Years, + "hours" | "hour" => geoengine_datatypes::primitives::TimeGranularity::Hours, + other => return Err(format!("Unsupported time granularity: {other}")), + }; + + let step_u32: u32 = step + .try_into() + .map_err(|_| format!("step {step} exceeds u32 range"))?; + + Ok(DtTimeDimension::Regular( + DtRegularTimeDimension::new_with_epoch_origin(geoengine_datatypes::primitives::TimeStep { + granularity: dt_granularity, + step: step_u32, + }), + )) +} + +// --------------------------------------------------------------------------- +// STAC API helpers +// --------------------------------------------------------------------------- + +async fn stac_api_request_parse( + client: &reqwest::Client, + url: &str, +) -> Result { + let response = client + .get(url) + .send() + .await + .with_context(|| format!("Failed to fetch {url}"))?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("STAC API returned HTTP {status}: {body}"); + } + + response + .json() + .await + .with_context(|| format!("Failed to parse response from {url}")) +} + +async fn stac_api_request_with_params( + client: &reqwest::Client, + url: &str, + params: &[(String, String)], +) -> Result { + let response = client + .get(url) + .query(params) + .send() + .await + .with_context(|| format!("Failed to fetch {url}"))?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("STAC API returned HTTP {status}: {body}"); + } + + response + .json() + .await + .with_context(|| format!("Failed to parse response from {url}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use httptest::{Expectation, Server, all_of, matchers::request, responders}; + + const COLLECTION_PATH: &str = "/v1/collections/sentinel-2-l2a"; + const ITEMS_PATH: &str = "/v1/collections/sentinel-2-l2a/items"; + + fn stac_collection_json() -> serde_json::Value { + serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/collections/code-de-minimal.json" + )) + .expect("valid collection fixture") + } + + fn stac_items_json() -> serde_json::Value { + serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/items/code-de-harvest-test.json" + )) + .expect("valid items fixture") + } + + fn expected_mapping_json() -> serde_json::Value { + let mut mapping: serde_json::Value = serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/expected-mapping-code-de.json" + )) + .expect("valid expected mapping fixture"); + // Remove the id since it is auto-generated + mapping.as_object_mut().unwrap().remove("id"); + mapping + } + + #[tokio::test] + async fn test_discover_mapping_produces_expected_mapping() { + let stac_server = Server::run(); + + // Mock collection endpoint + stac_server.expect( + Expectation::matching(request::method_path("GET", COLLECTION_PATH)) + .times(1) + .respond_with(responders::json_encoded(stac_collection_json())), + ); + + // Mock items endpoint + stac_server.expect( + Expectation::matching(all_of![request::method("GET"), request::path(ITEMS_PATH),]) + .times(1) + .respond_with(responders::json_encoded(stac_items_json())), + ); + + let output_path = std::env::temp_dir().join("test_discover_mapping_output.json"); + + let params = StacDiscoverMapping { + stac_url: stac_server.url_str("/v1").trim_end_matches('/').to_string(), + stac_collection: "sentinel-2-l2a".to_string(), + s3_endpoint: None, + s3_access_key: None, + s3_secret_key: None, + sample_items: 2, + output: Some(output_path.clone()), + filter_item_fields: true, + file_types: vec![ImportFileType::Jp2], + epsgs: vec![], + bbox: None, + full_projection_grid: false, + verbose: false, + time_granularity: "days".to_string(), + time_step: 1, + }; + + discover_mapping(params) + .await + .expect("discover mapping should succeed"); + + assert!(output_path.exists(), "output mapping file should exist"); + + let output_content = + std::fs::read_to_string(&output_path).expect("should read output file"); + let mut output_json: serde_json::Value = + serde_json::from_str(&output_content).expect("output should be valid JSON"); + + // Normalize dynamic fields before comparison + let output_obj = output_json.as_object_mut().unwrap(); + output_obj.remove("id"); + output_obj.insert( + "apiUrl".to_string(), + serde_json::json!("https://stac.test/v1"), + ); + output_obj.insert("description".to_string(), serde_json::json!("Auto-discovered mapping for STAC collection 'sentinel-2-l2a' at https://stac.test/v1")); + + // Sort datasets by name for deterministic comparison (HashMap order) + if let Some(datasets) = output_obj + .get_mut("datasets") + .and_then(|d| d.as_array_mut()) + { + datasets.sort_by(|a, b| { + a["name"] + .as_str() + .unwrap_or("") + .cmp(b["name"].as_str().unwrap_or("")) + }); + } + + let expected = expected_mapping_json(); + + pretty_assertions::assert_eq!(expected, output_json); + + // Clean up + let _ = std::fs::remove_file(&output_path); + } + + // ----------------------------------------------------------------------- + // Landsat C2 L1 discover test + // ----------------------------------------------------------------------- + + fn landsat_collection_json() -> serde_json::Value { + serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/collections/landsat-c2-l1-minimal.json" + )) + .expect("valid Landsat collection fixture") + } + + fn landsat_items_json() -> serde_json::Value { + serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/items/landsat-c2-l1-harvest-test.json" + )) + .expect("valid Landsat items fixture") + } + + fn expected_landsat_mapping_json() -> serde_json::Value { + let mut mapping: serde_json::Value = serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/expected-mapping-landsat-c2-l1.json" + )) + .expect("valid Landsat expected mapping fixture"); + mapping.as_object_mut().unwrap().remove("id"); + mapping + } + + const LANDSAT_COLLECTION_PATH: &str = "/v1/collections/landsat-c2-l1"; + const LANDSAT_ITEMS_PATH: &str = "/v1/collections/landsat-c2-l1/items"; + + #[tokio::test] + async fn test_discover_landsat_mapping_produces_expected_mapping() { + let stac_server = Server::run(); + + stac_server.expect( + Expectation::matching(request::method_path("GET", LANDSAT_COLLECTION_PATH)) + .times(1) + .respond_with(responders::json_encoded(landsat_collection_json())), + ); + + stac_server.expect( + Expectation::matching(all_of![request::method("GET"), request::path(LANDSAT_ITEMS_PATH)]) + .times(1) + .respond_with(responders::json_encoded(landsat_items_json())), + ); + + let output_path = std::env::temp_dir().join("test_discover_landsat_mapping_output.json"); + + let params = StacDiscoverMapping { + stac_url: stac_server.url_str("/v1").trim_end_matches('/').to_string(), + stac_collection: "landsat-c2-l1".to_string(), + s3_endpoint: None, + s3_access_key: None, + s3_secret_key: None, + sample_items: 1, + output: Some(output_path.clone()), + filter_item_fields: true, + file_types: vec![ImportFileType::Cog], + epsgs: vec![], + bbox: None, + full_projection_grid: false, + verbose: false, + time_granularity: "days".to_string(), + time_step: 1, + }; + + discover_mapping(params) + .await + .expect("Landsat discover mapping should succeed"); + + assert!(output_path.exists(), "output mapping file should exist"); + + let output_content = + std::fs::read_to_string(&output_path).expect("should read output file"); + let mut output_json: serde_json::Value = + serde_json::from_str(&output_content).expect("output should be valid JSON"); + + let output_obj = output_json.as_object_mut().unwrap(); + output_obj.remove("id"); + output_obj.insert( + "apiUrl".to_string(), + serde_json::json!("https://stac.test/v1"), + ); + output_obj.insert("description".to_string(), serde_json::json!("Auto-discovered mapping for STAC collection 'landsat-c2-l1' at https://stac.test/v1")); + + if let Some(datasets) = output_obj + .get_mut("datasets") + .and_then(|d| d.as_array_mut()) + { + datasets.sort_by(|a, b| { + a["name"] + .as_str() + .unwrap_or("") + .cmp(b["name"].as_str().unwrap_or("")) + }); + } + + let expected = expected_landsat_mapping_json(); + + pretty_assertions::assert_eq!(expected, output_json); + + let _ = std::fs::remove_file(&output_path); + } +} diff --git a/geoengine/services/src/cli/stac_harvester/harvest.rs b/geoengine/services/src/cli/stac_harvester/harvest.rs new file mode 100644 index 0000000000..19fb812932 --- /dev/null +++ b/geoengine/services/src/cli/stac_harvester/harvest.rs @@ -0,0 +1,1491 @@ +use std::{ + collections::HashMap, + io::Read, + str::FromStr, + time::{Duration, Instant}, +}; + +use anyhow::Context; +use chrono::Timelike; +use futures::StreamExt; +use geoengine_datatypes::{ + dataset::NamedData, + primitives::{DateTime, TimeInstance, TimeInterval}, + raster::{GeoTransform, GridBoundingBox2D, GridIdx2D, RasterDataType}, + spatial_reference::{SpatialReference, SpatialReferenceAuthority, SpatialReferenceOption}, +}; +use tracing::{debug, error, info, warn}; + +use crate::datasets::external::stac::{StacDataProviderDefinition, StacProviderDataset, common}; +use crate::{ + api::{ + handlers::{ + datasets::AddDatasetTile, + permissions::{ + DatasetResource, DatasetResourceTypeTag, LayerCollectionResource, + LayerCollectionResourceTypeTag, LayerResource, LayerResourceTypeTag, + PermissionRequest, Resource, + }, + }, + model::{ + datatypes::{ + GdalConfigOption, GridBoundingBox2D as ApiGridBoundingBox2D, + GridIdx2D as ApiGridIdx2D, LayerId, Measurement, SpatialGridDefinition, + TimeGranularity, TimeStep, UnitlessMeasurement, UnitlessMeasurementTypeTag, + }, + operators::{ + GdalDatasetParameters, GdalMultiBand, GdalMultiBandTypeTag, RasterBandDescriptor, + RasterBandDescriptors, RasterResultDescriptor, RegularTimeDimension, + SpatialGridDescriptor, SpatialGridDescriptorState, TimeDescriptor, TimeDimension, + }, + responses::{ErrorResponse, IdResponse}, + services::{ + AddDataset, CreateDataset, DataPath, DatasetDefinition, MetaDataDefinition, + }, + }, + }, + datasets::{DatasetName, upload::VolumeName}, + layers::{ + layer::{AddLayer, AddLayerCollection, CollectionItem, LayerCollection}, + listing::LayerCollectionId, + storage::{INTERNAL_LAYER_DB_ROOT_COLLECTION_ID, INTERNAL_PROVIDER_ID}, + }, + permissions::{Permission, Role}, + workflows::workflow::Workflow, +}; +use geoengine_operators::{ + engine::{RasterOperator, TypedOperator}, + source::{MultiBandGdalSource, MultiBandGdalSourceParameters}, +}; + +// --------------------------------------------------------------------------- +// Harvest +// --------------------------------------------------------------------------- + +/// Harvest tiles from a STAC collection using a predefined dataset mapping. +#[derive(Debug, clap::Parser)] +pub struct StacHarvest { + /// Path to the `StacDataProviderDefinition` JSON file (or `-` for stdin) + #[arg(long, value_parser = parse_mapping_file)] + pub mapping: StacDataProviderDefinition, + + /// Time range start to import (optional) + #[arg(long)] + pub time_start: Option, + + /// Time range end to import (optional) + #[arg(long)] + pub time_end: Option, + + /// Bounding box to import: minx miny maxx maxy (optional) + #[clap(short, long, value_parser, num_args = 1.., value_delimiter = ' ')] + pub bbox: Option>, + + /// Import limit (page size) + #[arg(long)] + pub limit: Option, + + /// Geo Engine API URL + #[arg(long, default_value = "http://localhost:3030/api")] + pub geo_engine_url: String, + + /// Geo Engine API email + #[arg(long, default_value = "admin@localhost")] + pub geo_engine_email: String, + + /// Geo Engine API password + #[arg(long, default_value = "adminadmin")] + pub geo_engine_password: String, + + /// Volume on the server + #[arg(long, default_value = "geodata")] + pub volume_name: String, + + /// Verbose output + #[arg(long, default_value_t = false)] + pub verbose: bool, + + /// Number of pages to prefetch while processing the current page + #[arg(long, default_value_t = 2)] + pub prefetch_pages: usize, + + /// Z-index property name + #[arg(long, default_value = "updated")] + pub z_index_property_name: Option, + + /// No data value override + #[arg(long)] + pub no_data_value: Option, + + /// GDAL retry count + #[arg(long)] + pub gdal_retries: Option, + + /// Filter item fields to reduce response size + #[arg(long, default_value_t = false)] + pub filter_item_fields: bool, +} + +/// Parse a JSON file path (or `-` for stdin) into a `StacDataProviderDefinition`. +fn parse_mapping_file(s: &str) -> Result { + let json = if s == "-" { + let mut input = String::new(); + std::io::stdin() + .read_to_string(&mut input) + .map_err(|e| format!("Failed to read mapping from stdin: {e}"))?; + input + } else { + std::fs::read_to_string(s).map_err(|e| format!("Failed to read mapping from '{s}': {e}"))? + }; + serde_json::from_str(&json).map_err(|e| format!("Invalid mapping JSON: {e}")) +} + +// --------------------------------------------------------------------------- +// Harvest Implementation +// --------------------------------------------------------------------------- + +#[allow(clippy::too_many_lines)] +pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Error> { + let start_time = Instant::now(); + + let provider_def = ¶ms.mapping; + + info!( + "Harvesting STAC collection '{}' at {} with {} dataset(s)", + provider_def.collection_name, + provider_def.api_url, + provider_def.datasets.len() + ); + + let (client, session_id) = login_geo_engine( + ¶ms.geo_engine_url, + ¶ms.geo_engine_email, + ¶ms.geo_engine_password, + ) + .await?; + + let mut created_datasets: Vec<(usize, StacProviderDataset)> = Vec::new(); + + for (idx, dataset) in provider_def.datasets.iter().enumerate() { + let dataset_name = dataset_name_for_harvest(&provider_def.collection_name, dataset); + + if params.verbose { + info!("Checking dataset '{}'", dataset_name); + } + + if !dataset_exists_api(&client, ¶ms.geo_engine_url, &session_id, &dataset_name).await? { + create_dataset_api( + &client, + ¶ms.geo_engine_url, + &session_id, + &dataset_name, + dataset, + ¶ms.volume_name, + ) + .await?; + created_datasets.push((idx, dataset.clone())); + } + } + + info!( + "Created {} new dataset(s) out of {}", + created_datasets.len(), + provider_def.datasets.len() + ); + + let stac_api_url = provider_def.api_url.trim_end_matches('/').to_string(); + let items_url = format!( + "{}/collections/{}/items", + stac_api_url, provider_def.collection_name + ); + + let mut query_params: Vec<(String, String)> = Vec::new(); + + if let Some(bbox) = ¶ms.bbox + && bbox.len() == 4 + { + query_params.push(( + "bbox".to_string(), + format!("{},{},{},{}", bbox[0], bbox[1], bbox[2], bbox[3]), + )); + } + + if params.time_start.is_some() || params.time_end.is_some() { + query_params.push(( + "datetime".to_string(), + format!( + "{}/{}", + params.time_start.as_deref().unwrap_or(""), + params.time_end.as_deref().unwrap_or("") + ), + )); + } + + if let Some(limit) = params.limit { + query_params.push(("limit".to_string(), limit.to_string())); + } + + if params.filter_item_fields { + query_params.push(( + "fields".to_string(), + "stac_version,properties.datetime,properties.updated,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.proj:code,assets.*.proj:shape,assets.*.proj:transform" + .to_string(), + )); + } + + let initial_query_state = QueryState::FirstPage { + query_url: items_url.clone(), + query_params, + }; + + let page_stream = create_page_stream( + initial_query_state, + client.clone(), + params.verbose, + params.prefetch_pages, + ); + + let mut tiles_by_dataset: HashMap> = HashMap::new(); + let mut dynamic_datasets: HashMap = HashMap::new(); + let mut items_processed: u64 = 0; + let mut items_per_sec: f64; + + futures::pin_mut!(page_stream); + while let Some(result) = page_stream.next().await { + let item_collection = result?; + + for item in &item_collection.items { + process_harvest_item_dynamic( + item, + provider_def, + &mut tiles_by_dataset, + &mut dynamic_datasets, + ¶ms, + ) + .unwrap_or_else(|e| { + if params.verbose { + warn!("Skipping item {}: {}", item.id, e); + } + }); + + items_processed += 1; + } + + if params.verbose { + let elapsed = start_time.elapsed().as_secs_f64(); + items_per_sec = if elapsed > 0.0 { + items_processed as f64 / elapsed + } else { + 0.0 + }; + + if let Some(number_matched) = item_collection + .additional_fields + .get("numberMatched") + .and_then(serde_json::Value::as_u64) + { + let progress = + (items_processed as f64 / number_matched as f64 * 100.0).clamp(0.0, 100.0); + let remaining = number_matched.saturating_sub(items_processed); + let eta_secs = if items_per_sec > 0.0 { + remaining as f64 / items_per_sec + } else { + f64::INFINITY + }; + let eta_str = if eta_secs.is_finite() { + format_duration(eta_secs as u64) + } else { + "unknown".to_string() + }; + println!( + "[{progress:.1}%] Processed {items_processed}/{number_matched} items ({items_per_sec:.1} items/s, ETA: {eta_str})" + ); + } else { + println!("Processed {items_processed} items ({items_per_sec:.1} items/s)"); + } + } + } + + info!("Processed {} items total", items_processed); + + // Create any dynamic (per-EPSG) datasets that were discovered during item processing + for (dyn_dataset_name, dyn_dataset) in &dynamic_datasets { + if !dataset_exists_api( + &client, + ¶ms.geo_engine_url, + &session_id, + dyn_dataset_name, + ) + .await? + { + if params.verbose { + info!("Creating dynamic dataset '{}'", dyn_dataset_name); + } + create_dataset_api( + &client, + ¶ms.geo_engine_url, + &session_id, + dyn_dataset_name, + dyn_dataset, + ¶ms.volume_name, + ) + .await?; + } + // Also ensure tiles_by_dataset has an entry for this dataset + tiles_by_dataset + .entry(dyn_dataset_name.clone()) + .or_default(); + } + + for (dataset_name, tiles) in &tiles_by_dataset { + if tiles.is_empty() { + continue; + } + + if params.verbose { + info!("Adding {} tiles to dataset '{}'", tiles.len(), dataset_name); + } + + let batch_size = 100; + for chunk in tiles.chunks(batch_size) { + let response = retry_http( + || async { + client + .post(format!( + "{}/dataset/{}/tiles", + params.geo_engine_url, dataset_name + )) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(chunk) + .send() + .await + }, + &format!("Add tiles to dataset '{dataset_name}'"), + ) + .await + .with_context(|| format!("Failed to add tiles to dataset '{dataset_name}'"))?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + warn!("Failed to add tiles to dataset '{dataset_name}' (HTTP {status}): {body}"); + // Continue with remaining tiles; some conflicts (e.g. z-index) are expected + } + } + } + + create_harvest_layer_collections( + &client, + ¶ms.geo_engine_url, + &session_id, + provider_def, + &created_datasets, + ¶ms, + ) + .await?; + + let elapsed = start_time.elapsed(); + info!("Harvest completed in {:.2?}", elapsed); + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Item Processing (Harvest) +// --------------------------------------------------------------------------- + +/// Like `process_harvest_item`, but handles items whose EPSG code differs from +/// the mapping's projection by dynamically creating per-EPSG dataset variants. +/// Items that pass the `--epsgs` filter but have a different EPSG than the +/// mapping will be grouped into separate datasets named with their actual EPSG. +#[allow(clippy::too_many_lines)] +fn process_harvest_item_dynamic( + item: &stac::Item, + provider_def: &StacDataProviderDefinition, + tiles_by_dataset: &mut HashMap>, + dynamic_datasets: &mut HashMap, + params: &StacHarvest, +) -> Result<(), anyhow::Error> { + let Some(datetime) = item.properties.datetime else { + return Ok(()); + }; + + let date_without_time = datetime + .with_hour(0) + .and_then(|d| d.with_minute(0)) + .and_then(|d| d.with_second(0)) + .and_then(|d| d.with_nanosecond(0)) + .context("Failed to set time to zero")?; + let date_without_time: DateTime = date_without_time.into(); + let time: TimeInstance = date_without_time.into(); + + let z_index = match params.z_index_property_name.as_deref() { + Some("updated") => item + .properties + .updated + .as_deref() + .and_then(|updated| chrono::DateTime::parse_from_rfc3339(updated).ok()) + .map_or_else(|| datetime.timestamp_millis(), |dt| dt.timestamp_millis()), + _ => 0, + }; + + for dataset in &provider_def.datasets { + for (band_idx, band_def) in dataset.bands.iter().enumerate() { + let Some((_asset_key, asset)) = item + .assets + .iter() + .find(|(_, a)| a.title.as_deref() == Some(&band_def.asset_title)) + else { + continue; + }; + + // Check data type matches + if let Some(asset_dt) = data_type_from_asset_v1_1_0_fallback(asset) + && asset_dt != dataset.data_type + { + continue; + } + + // Extract the item's actual EPSG code from the asset + let item_epsg = common::epsg_code_from_fields( + common::StacExtensionMajorVersion::V2, + &asset.additional_fields, + ) + .or_else(|| { + // Also try to extract from serialized properties as fallback + let props_val = serde_json::to_value(&item.properties) + .ok() + .and_then(|v| v.as_object().cloned()) + .unwrap_or_default(); + common::epsg_code_from_fields(common::StacExtensionMajorVersion::V2, &props_val) + }); + + let Some(item_epsg) = item_epsg else { + continue; + }; + + // Determine the actual projection and dataset name for this item + let (_actual_projection, actual_dataset_name) = if dataset.projection + == SpatialReference::new(SpatialReferenceAuthority::Epsg, item_epsg) + { + // EPSG matches the mapping — use the dataset as-is + ( + dataset.projection, + dataset_name_for_harvest(&provider_def.collection_name, dataset), + ) + } else { + // EPSG differs — create a per-EPSG variant + let per_epsg_projection = + SpatialReference::new(SpatialReferenceAuthority::Epsg, item_epsg); + + // Build a modified dataset with the item's EPSG + let per_epsg_dataset = StacProviderDataset { + projection: per_epsg_projection, + ..dataset.clone() + }; + + let dyn_name = + dataset_name_for_harvest(&provider_def.collection_name, &per_epsg_dataset); + + // Register this dynamic dataset so it gets created + dynamic_datasets + .entry(dyn_name.clone()) + .or_insert_with(|| StacProviderDataset { + projection: SpatialReference::new( + SpatialReferenceAuthority::Epsg, + item_epsg, + ), + spatial_grid: dataset.spatial_grid, + ..dataset.clone() + }); + + (per_epsg_projection, dyn_name) + }; + + let Some(geo_transform) = common::geo_transform_from_fields(&asset.additional_fields) + else { + continue; + }; + + if (geo_transform.x_pixel_size().abs() - dataset.resolution.x).abs() > 1e-9 + || (geo_transform.y_pixel_size().abs() - dataset.resolution.y).abs() > 1e-9 + { + continue; + } + + let Some((height, width)) = common::proj_shape_from_fields(&asset.additional_fields) + else { + continue; + }; + + let Some(rasterband_channel) = + common::rasterband_channel_for_dataset_band(asset, band_def.band_name.as_deref()) + else { + continue; + }; + + let grid_bounds = GridBoundingBox2D::new( + GridIdx2D::new([0, 0]), + GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), + ) + .context("Failed to create grid bounds")?; + + let spatial_partition = geo_transform.grid_to_spatial_bounds(&grid_bounds); + + let Some(file_path) = common::gdal_file_path(&asset.href) else { + continue; + }; + + let gdal_config_options = common::gdal_config_options_for_file_path( + &file_path, + provider_def.s3_config.as_ref(), + ); + + let tile = AddDatasetTile { + time: TimeInterval::new(time, time + i64::from(24 * 60 * 60 * 1000)) + .context("Failed to create time interval")? + .into(), + spatial_partition: spatial_partition.into(), + band: band_idx as u32, + z_index, + params: GdalDatasetParameters { + file_path, + rasterband_channel, + geo_transform: geo_transform.into(), + width, + height, + file_not_found_handling: + crate::api::model::operators::FileNotFoundHandling::Error, + no_data_value: params.no_data_value, + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: gdal_config_options.map(|opts| { + opts.into_iter() + .map(|(k, v)| GdalConfigOption::from((k, v))) + .collect() + }), + allow_alphaband_as_mask: false, + }, + }; + + tiles_by_dataset + .entry(actual_dataset_name) + .or_default() + .push(tile); + } + } + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Dataset and Layer Creation (Harvest) +// --------------------------------------------------------------------------- + +fn dataset_name_for_harvest(collection_name: &str, dataset: &StacProviderDataset) -> String { + let cleaned_name: String = collection_name + .chars() + .map(|c| { + if geoengine_datatypes::dataset::is_invalid_name_char(c) { + '_' + } else { + c + } + }) + .collect(); + + let resolution_str = format!("{}", dataset.resolution.x); + let clean_resolution: String = resolution_str + .chars() + .map(|c| { + if geoengine_datatypes::dataset::is_invalid_name_char(c) { + '_' + } else { + c + } + }) + .collect(); + + format!( + "{}_EPSG{}_{:?}_{}", + cleaned_name, + dataset.projection.code(), + dataset.data_type, + clean_resolution + ) +} + +async fn dataset_exists_api( + client: &reqwest::Client, + geo_engine_url: &str, + session_id: &str, + dataset_name: &str, +) -> Result { + let response = retry_http( + || async { + client + .get(format!("{geo_engine_url}/dataset/{dataset_name}")) + .header("Authorization", format!("Bearer {session_id}")) + .send() + .await + }, + &format!("Check dataset existence for '{dataset_name}'"), + ) + .await?; + + if response.status().is_success() { + return Ok(true); + } + + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + if status == reqwest::StatusCode::BAD_REQUEST + && let Ok(error_response) = serde_json::from_str::(&body) + && error_response.error == "CannotLoadDataset" + { + return Ok(false); + } + + anyhow::bail!("Failed to check dataset '{dataset_name}': HTTP {status}: {body}"); +} + +async fn create_dataset_api( + client: &reqwest::Client, + geo_engine_url: &str, + session_id: &str, + dataset_name: &str, + dataset: &StacProviderDataset, + volume_name: &str, +) -> Result<(), anyhow::Error> { + let bands: Vec = dataset + .bands + .iter() + .map(|b| RasterBandDescriptor { + name: b.band_name.clone().unwrap_or_else(|| b.asset_title.clone()), + measurement: Measurement::Unitless(UnitlessMeasurement { + r#type: UnitlessMeasurementTypeTag::UnitlessMeasurementTypeTag, + }), + }) + .collect(); + + // Get GeoTransform from the spatial grid descriptor for the API + let dt_gt: GeoTransform = dataset.spatial_grid.geo_transform(); + let api_gt: crate::api::model::datatypes::GeoTransform = dt_gt.into(); + + let create_dataset_req = CreateDataset { + data_path: DataPath::Volume(VolumeName(volume_name.to_string())), + definition: DatasetDefinition { + properties: AddDataset { + name: Some( + DatasetName::from_str(dataset_name) + .map_err(|e| anyhow::anyhow!("Failed to create dataset name: {e}"))?, + ), + display_name: dataset_name.to_string(), + description: format!("{dataset_name} harvested from STAC"), + source_operator: "MultiBandGdalSource".to_string(), + symbology: None, + provenance: None, + tags: None, + }, + meta_data: MetaDataDefinition::GdalMultiBand(GdalMultiBand { + r#type: GdalMultiBandTypeTag::GdalMultiBandTypeTag, + result_descriptor: RasterResultDescriptor { + data_type: dataset.data_type.into(), + spatial_reference: SpatialReferenceOption::SpatialReference(dataset.projection) + .into(), + time: TimeDescriptor { + bounds: None, + dimension: TimeDimension::Regular(RegularTimeDimension { + origin: TimeInstance::from_millis_unchecked(0).into(), + step: TimeStep { + granularity: TimeGranularity::Days, + step: 1, + }, + }), + }, + spatial_grid: SpatialGridDescriptor { + spatial_grid: SpatialGridDefinition { + geo_transform: api_gt, + grid_bounds: ApiGridBoundingBox2D { + top_left_idx: ApiGridIdx2D { x_idx: 0, y_idx: 0 }, + bottom_right_idx: ApiGridIdx2D { x_idx: 1, y_idx: 1 }, + }, + }, + descriptor: SpatialGridDescriptorState::Source, + }, + bands: RasterBandDescriptors::new(bands).context("Invalid band descriptors")?, + }, + }), + }, + }; + + let response = retry_http( + || async { + client + .post(format!("{geo_engine_url}/dataset")) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(&create_dataset_req) + .send() + .await + }, + &format!("Create dataset '{dataset_name}'"), + ) + .await?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("Failed to create dataset '{dataset_name}': HTTP {status}: {body}"); + } + + let created_name = if let Ok(json) = response.json::().await { + json.get("datasetName") + .and_then(|v| v.as_str()) + .unwrap_or(dataset_name) + .to_string() + } else { + dataset_name.to_string() + }; + + share_dataset_api(client, geo_engine_url, session_id, &created_name).await?; + + Ok(()) +} + +async fn share_dataset_api( + client: &reqwest::Client, + geo_engine_url: &str, + session_id: &str, + dataset_name: &str, +) -> Result<(), anyhow::Error> { + let permissions = vec![ + PermissionRequest { + resource: Resource::Dataset(DatasetResource { + id: DatasetName::new(None, dataset_name.to_string()), + r#type: DatasetResourceTypeTag::DatasetResourceTypeTag, + }), + role_id: Role::registered_user_role_id(), + permission: Permission::Read, + }, + PermissionRequest { + resource: Resource::Dataset(DatasetResource { + id: DatasetName::new(None, dataset_name.to_string()), + r#type: DatasetResourceTypeTag::DatasetResourceTypeTag, + }), + role_id: Role::anonymous_role_id(), + permission: Permission::Read, + }, + ]; + + for permission in &permissions { + retry_http( + || async { + client + .put(format!("{geo_engine_url}/permissions")) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(permission) + .send() + .await + }, + &format!("Add permission for dataset '{dataset_name}'"), + ) + .await?; + } + + Ok(()) +} + +async fn create_harvest_layer_collections( + client: &reqwest::Client, + geo_engine_url: &str, + session_id: &str, + provider_def: &StacDataProviderDefinition, + created_datasets: &[(usize, StacProviderDataset)], + params: &StacHarvest, +) -> Result<(), anyhow::Error> { + let root_collection_id = create_layer_collection_api( + client, + geo_engine_url, + session_id, + &LayerCollectionId(INTERNAL_LAYER_DB_ROOT_COLLECTION_ID.to_string()), + &provider_def.collection_name, + &format!( + "{} datasets harvested from STAC", + provider_def.collection_name + ), + params, + ) + .await?; + + let temp_collection_id = create_layer_collection_api( + client, + geo_engine_url, + session_id, + &root_collection_id, + "_layers", + "All dataset layers (internal)", + params, + ) + .await?; + + for (_idx, dataset) in created_datasets { + let dataset_name = dataset_name_for_harvest(&provider_def.collection_name, dataset); + let layer_name = format!( + "EPSG:{} {:?} {}m", + dataset.projection.authority(), + dataset.data_type, + dataset.resolution.x + ); + + let add_layer = AddLayer { + name: layer_name.clone(), + description: format!("Dataset: {dataset_name}"), + workflow: Workflow::Legacy { + operator: TypedOperator::Raster( + MultiBandGdalSource { + params: MultiBandGdalSourceParameters::new(NamedData { + namespace: None, + provider: None, + name: dataset_name.clone(), + }), + } + .boxed(), + ), + }, + symbology: None, + properties: vec![], + metadata: Default::default(), + }; + + let response: IdResponse = retry_http( + || async { + client + .post(format!( + "{geo_engine_url}/layerDb/collections/{temp_collection_id}/layers" + )) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(&add_layer) + .send() + .await? + .json() + .await + }, + &format!("Create layer '{layer_name}'"), + ) + .await?; + + share_layer_api(client, geo_engine_url, session_id, &response.id).await?; + } + + Ok(()) +} + +async fn create_layer_collection_api( + client: &reqwest::Client, + geo_engine_url: &str, + session_id: &str, + parent_id: &LayerCollectionId, + name: &str, + description: &str, + params: &StacHarvest, +) -> Result { + if let Some(existing_id) = + find_child_collection_by_name(client, geo_engine_url, session_id, parent_id, name).await? + { + if params.verbose { + info!("Found existing layer collection '{name}'"); + } + return Ok(existing_id); + } + + let add_collection = AddLayerCollection { + name: name.to_string(), + description: description.to_string(), + properties: vec![], + }; + + let response: IdResponse = retry_http( + || async { + client + .post(format!( + "{geo_engine_url}/layerDb/collections/{parent_id}/collections" + )) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(&add_collection) + .send() + .await? + .json() + .await + }, + &format!("Create layer collection '{name}'"), + ) + .await?; + + share_layer_collection_api(client, geo_engine_url, session_id, &response.id).await?; + + Ok(response.id) +} + +async fn find_child_collection_by_name( + client: &reqwest::Client, + geo_engine_url: &str, + session_id: &str, + parent_id: &LayerCollectionId, + child_name: &str, +) -> Result, anyhow::Error> { + let mut offset: u32 = 0; + let limit: u32 = 20; + + loop { + let response: LayerCollection = retry_http( + || async { + client + .get(format!( + "{geo_engine_url}/layers/collections/{INTERNAL_PROVIDER_ID}/{parent_id}" + )) + .query(&[("offset", offset), ("limit", limit)]) + .header("Authorization", format!("Bearer {session_id}")) + .send() + .await? + .json() + .await + }, + &format!("List child collections of {parent_id}"), + ) + .await?; + + for item in &response.items { + if let CollectionItem::Collection(collection) = item + && collection.name == child_name + { + return Ok(Some(collection.id.collection_id.clone())); + } + } + + if response.items.len() < limit as usize { + return Ok(None); + } + + offset += limit; + } +} + +async fn share_layer_collection_api( + client: &reqwest::Client, + geo_engine_url: &str, + session_id: &str, + collection_id: &LayerCollectionId, +) -> Result<(), anyhow::Error> { + let permissions = vec![ + PermissionRequest { + resource: Resource::LayerCollection(LayerCollectionResource { + id: collection_id.clone(), + r#type: LayerCollectionResourceTypeTag::LayerCollectionResourceTypeTag, + }), + role_id: Role::registered_user_role_id(), + permission: Permission::Read, + }, + PermissionRequest { + resource: Resource::LayerCollection(LayerCollectionResource { + id: collection_id.clone(), + r#type: LayerCollectionResourceTypeTag::LayerCollectionResourceTypeTag, + }), + role_id: Role::anonymous_role_id(), + permission: Permission::Read, + }, + ]; + + for permission in &permissions { + retry_http( + || async { + client + .put(format!("{geo_engine_url}/permissions")) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(permission) + .send() + .await + }, + &format!("Share collection with role {}", permission.role_id), + ) + .await?; + } + + Ok(()) +} + +async fn share_layer_api( + client: &reqwest::Client, + geo_engine_url: &str, + session_id: &str, + layer_id: &LayerId, +) -> Result<(), anyhow::Error> { + let permissions = vec![ + PermissionRequest { + resource: Resource::Layer(LayerResource { + id: layer_id.clone(), + r#type: LayerResourceTypeTag::LayerResourceTypeTag, + }), + role_id: Role::registered_user_role_id(), + permission: Permission::Read, + }, + PermissionRequest { + resource: Resource::Layer(LayerResource { + id: layer_id.clone(), + r#type: LayerResourceTypeTag::LayerResourceTypeTag, + }), + role_id: Role::anonymous_role_id(), + permission: Permission::Read, + }, + ]; + + for permission in &permissions { + retry_http( + || async { + client + .put(format!("{geo_engine_url}/permissions")) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(permission) + .send() + .await + }, + &format!("Share layer with role {}", permission.role_id), + ) + .await?; + } + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Pagination +// --------------------------------------------------------------------------- + +const MAX_RETRIES: u32 = 10; +const INITIAL_RETRY_DELAY_MS: u64 = 1000; + +#[derive(Debug, Clone)] +enum QueryState { + FirstPage { + query_url: String, + query_params: Vec<(String, String)>, + }, + NextPage { + next_url: String, + }, + Finished, +} + +fn create_page_stream( + initial_query_state: QueryState, + client: reqwest::Client, + _verbose: bool, + prefetch_buffer: usize, +) -> impl futures::Stream> { + let page_stream = futures::stream::unfold( + (client, initial_query_state), + move |(client, state)| async move { + if matches!(state, QueryState::Finished) { + return None; + } + + debug!("Fetching page: {state:?}"); + + let result = query_item_collection_internal(&client, &state).await; + + match result { + Ok((item_collection, new_state)) => { + if item_collection.items.is_empty() { + None + } else { + Some((Ok(item_collection), (client, new_state))) + } + } + Err(e) => { + error!("Error fetching page: {e:#}"); + Some((Err(e), (client, QueryState::Finished))) + } + } + }, + ); + page_stream + .map(|result| async move { result }) + .buffered(prefetch_buffer) +} + +async fn query_item_collection_internal( + client: &reqwest::Client, + query_state: &QueryState, +) -> Result<(stac::ItemCollection, QueryState), anyhow::Error> { + match query_state { + QueryState::FirstPage { + query_url, + query_params, + } => { + let item_collection: stac::ItemCollection = retry_http( + || async { + client + .get(query_url) + .query(&query_params) + .send() + .await? + .json() + .await + }, + "Query STAC first page", + ) + .await?; + + let new_state = item_collection + .links + .iter() + .find(|link| link.rel == "next") + .map_or(QueryState::Finished, |link| QueryState::NextPage { + next_url: link.href.clone(), + }); + + Ok((item_collection, new_state)) + } + QueryState::NextPage { next_url } => { + let item_collection: stac::ItemCollection = retry_http( + || async { client.get(next_url).send().await?.json().await }, + "Query STAC next page", + ) + .await?; + + let new_state = item_collection + .links + .iter() + .find(|link| link.rel == "next") + .map_or(QueryState::Finished, |link| QueryState::NextPage { + next_url: link.href.clone(), + }); + + Ok((item_collection, new_state)) + } + QueryState::Finished => anyhow::bail!("No more pages to query"), + } +} + +// --------------------------------------------------------------------------- +// HTTP retry helper +// --------------------------------------------------------------------------- + +async fn retry_http(mut operation: F, operation_name: &str) -> Result +where + F: FnMut() -> Fut, + Fut: std::future::Future>, + E: std::fmt::Display, +{ + let mut attempt = 0; + loop { + match operation().await { + Ok(result) => return Ok(result), + Err(err) => { + attempt += 1; + if attempt >= MAX_RETRIES { + error!("{operation_name} failed after {MAX_RETRIES} attempts: {err}"); + return Err(err); + } + let delay = Duration::from_millis(INITIAL_RETRY_DELAY_MS * 2_u64.pow(attempt - 1)); + warn!( + "{operation_name} failed (attempt {attempt}/{MAX_RETRIES}): {err}. Retrying in {delay:?}..." + ); + tokio::time::sleep(delay).await; + } + } + } +} + +// --------------------------------------------------------------------------- +// Authentication helper +// --------------------------------------------------------------------------- + +async fn login_geo_engine( + geo_engine_url: &str, + geo_engine_email: &str, + geo_engine_password: &str, +) -> Result<(reqwest::Client, String), anyhow::Error> { + let client = reqwest::Client::new(); + + let response = retry_http( + || async { + client + .post(format!("{geo_engine_url}/login")) + .header("Content-Type", "application/json") + .json(&serde_json::json!({ + "email": geo_engine_email, + "password": geo_engine_password, + })) + .send() + .await + }, + "Login to Geo Engine", + ) + .await + .context("Failed to authenticate")?; + + let json = response + .json::() + .await + .context("Failed to parse auth response")?; + + let session_id = json["id"] + .as_str() + .context("No session id in response")? + .to_string(); + + Ok((client, session_id)) +} + +fn data_type_from_asset_v1_1_0_fallback(asset: &stac::Asset) -> Option { + common::data_type_from_asset_v1_1_0(asset).or_else(|| { + asset + .additional_fields + .get("data_type") + .and_then(|v| v.as_str()) + .and_then(common::raster_data_type_from_stac_data_type_str) + }) +} + +fn format_duration(secs: u64) -> String { + if secs < 60 { + format!("{secs}s") + } else if secs < 3600 { + format!("{}m{}s", secs / 60, secs % 60) + } else { + format!("{}h{}m{}s", secs / 3600, (secs % 3600) / 60, secs % 60) + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use geoengine_datatypes::primitives::SpatialResolution; + + #[test] + fn test_dataset_name_for_harvest() { + let collection = "sentinel-2-l2a"; + let dataset = StacProviderDataset { + name: "Test".to_string(), + description: String::new(), + data_type: RasterDataType::U16, + resolution: SpatialResolution::new_unchecked(10.0, 10.0), + projection: SpatialReference::new(SpatialReferenceAuthority::Epsg, 32632), + spatial_grid: geoengine_operators::engine::SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0.0, 0.0).into(), 10.0, -10.0), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])).unwrap(), + ), + bands: vec![], + }; + + let name = dataset_name_for_harvest(collection, &dataset); + assert!(name.contains("sentinel-2-l2a")); + assert!(name.contains("EPSG")); + assert!(name.contains("U16")); + } + + #[test] + fn test_process_harvest_item_produces_correct_tiles() { + let mapping: StacDataProviderDefinition = serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/expected-mapping-code-de.json" + )) + .expect("valid mapping fixture"); + + let items: stac::ItemCollection = serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/items/code-de-harvest-test.json" + )) + .expect("valid items fixture"); + + let params = StacHarvest { + mapping: mapping.clone(), + time_start: None, + time_end: None, + bbox: None, + limit: None, + geo_engine_url: String::new(), + geo_engine_email: String::new(), + geo_engine_password: String::new(), + volume_name: String::new(), + verbose: false, + prefetch_pages: 1, + z_index_property_name: Some("updated".to_string()), + no_data_value: None, + gdal_retries: None, + filter_item_fields: true, + }; + + let mut tiles_by_dataset: HashMap> = HashMap::new(); + let mut dynamic_datasets: HashMap = HashMap::new(); + + // Process the first item from the fixture + let item = &items.items[0]; + + process_harvest_item_dynamic( + item, + &mapping, + &mut tiles_by_dataset, + &mut dynamic_datasets, + ¶ms, + ) + .expect("item processing should succeed"); + + // The mapping has 2 datasets (10m and 20m), the item has assets for both + assert_eq!( + tiles_by_dataset.len(), + 2, + "tiles should be produced for 2 datasets (10m and 20m)" + ); + + // Check each dataset has valid tiles + for (dataset_name, tiles) in &tiles_by_dataset { + assert!( + !tiles.is_empty(), + "dataset {dataset_name} should have tiles" + ); + for tile in tiles { + assert!(tile.band < 10, "band index should be reasonable"); + assert!( + tile.params.width > 0 && tile.params.height > 0, + "tile dimensions should be positive" + ); + assert!( + tile.params + .file_path + .to_string_lossy() + .starts_with("/vsis3/"), + "file path should be a VSI path: {}", + tile.params.file_path.display() + ); + } + } + + // Verify no dynamic (per-EPSG) datasets since all items match the mapping EPSG + assert!( + dynamic_datasets.is_empty(), + "no dynamic datasets should be needed when EPSGs match" + ); + + // 10m dataset should have 4 tiles (B02, B03, B04, B08) + let total_tiles_10m: usize = tiles_by_dataset + .iter() + .filter(|(name, _)| name.contains("10")) + .map(|(_, tiles)| tiles.len()) + .sum(); + assert_eq!( + total_tiles_10m, 4, + "first item should produce 4 tiles for 10m bands" + ); + + // 20m dataset should have 2 tiles (B11, B12) + let total_tiles_20m: usize = tiles_by_dataset + .iter() + .filter(|(name, _)| name.contains("20")) + .map(|(_, tiles)| tiles.len()) + .sum(); + assert_eq!( + total_tiles_20m, 2, + "first item should produce 2 tiles for 20m bands" + ); + } + + #[test] + fn test_process_harvest_landsat_item_produces_correct_tiles() { + let mapping: StacDataProviderDefinition = serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/expected-mapping-landsat-c2-l1.json" + )) + .expect("valid Landsat mapping fixture"); + + let items: stac::ItemCollection = serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/items/landsat-c2-l1-harvest-test.json" + )) + .expect("valid Landsat items fixture"); + + let params = StacHarvest { + mapping: mapping.clone(), + time_start: None, + time_end: None, + bbox: None, + limit: None, + geo_engine_url: String::new(), + geo_engine_email: String::new(), + geo_engine_password: String::new(), + volume_name: String::new(), + verbose: false, + prefetch_pages: 1, + z_index_property_name: Some("updated".to_string()), + no_data_value: None, + gdal_retries: None, + filter_item_fields: true, + }; + + let mut tiles_by_dataset: HashMap> = HashMap::new(); + let mut dynamic_datasets: HashMap = HashMap::new(); + + let item = &items.items[0]; + + process_harvest_item_dynamic( + item, + &mapping, + &mut tiles_by_dataset, + &mut dynamic_datasets, + ¶ms, + ) + .expect("Landsat item processing should succeed"); + + // Mapping has 1 dataset (30m), the item has assets for it + assert_eq!( + tiles_by_dataset.len(), + 1, + "tiles should be produced for 1 dataset (30m)" + ); + + for (dataset_name, tiles) in &tiles_by_dataset { + assert!( + !tiles.is_empty(), + "dataset {dataset_name} should have tiles" + ); + for tile in tiles { + assert!(tile.band < 10, "band index should be reasonable"); + assert!( + tile.params.width > 0 && tile.params.height > 0, + "tile dimensions should be positive" + ); + assert!( + tile.params + .file_path + .to_string_lossy() + .starts_with("/vsis3/"), + "file path should be a VSI path: {}", + tile.params.file_path.display() + ); + } + } + + assert!( + dynamic_datasets.is_empty(), + "no dynamic datasets should be needed when EPSGs match" + ); + + // 30m dataset should have 4 tiles (Blue, Green, Red, NIR) + let total_tiles_30m: usize = tiles_by_dataset + .iter() + .filter(|(name, _)| name.contains("30")) + .map(|(_, tiles)| tiles.len()) + .sum(); + assert_eq!( + total_tiles_30m, 4, + "first item should produce 4 tiles for 30m bands" + ); + } +} diff --git a/geoengine/services/src/cli/stac_harvester/mod.rs b/geoengine/services/src/cli/stac_harvester/mod.rs new file mode 100644 index 0000000000..7ab60a16f6 --- /dev/null +++ b/geoengine/services/src/cli/stac_harvester/mod.rs @@ -0,0 +1,47 @@ +//! New STAC harvester CLI that separates mapping generation from tile harvesting. +//! +//! # Subcommands +//! +//! - `discover-mapping`: Probes a STAC collection and items API to generate a +//! `StacDataProviderDefinition` JSON that maps STAC assets to Geo Engine datasets. +//! - `harvest`: Reads a `StacDataProviderDefinition` and harvests tiles into Geo Engine, +//! creating datasets, tiles, and layer collections. +//! +//! The mapping JSON matches the format of `StacDataProviderDefinition` as used by the +//! STAC provider and the EDV bootstrap scripts. + +#![allow(clippy::print_stdout)] + +mod discover; +mod harvest; + +pub use discover::StacDiscoverMapping; +pub use harvest::StacHarvest; + +use clap::{Parser, Subcommand}; + +/// STAC harvester for Geo Engine +#[derive(Debug, Parser)] +pub struct StacHarvester { + #[clap(subcommand)] + pub command: StacHarvesterCommand, +} + +#[derive(Debug, Subcommand)] +#[allow(clippy::enum_variant_names)] +pub enum StacHarvesterCommand { + /// Probe a STAC API to auto-discover the dataset mapping + DiscoverMapping(Box), + /// Harvest tiles using a predefined dataset mapping + Harvest(Box), +} + +/// Run the STAC harvester +pub async fn stac_harvester(params: StacHarvester) -> Result<(), anyhow::Error> { + match params.command { + StacHarvesterCommand::DiscoverMapping(discover) => { + discover::discover_mapping(*discover).await + } + StacHarvesterCommand::Harvest(harvest) => harvest::harvest_tiles(*harvest).await, + } +} diff --git a/geoengine/services/src/cli/stac_import.rs b/geoengine/services/src/cli/stac_import.rs index c57c31902b..e7f0143cf1 100644 --- a/geoengine/services/src/cli/stac_import.rs +++ b/geoengine/services/src/cli/stac_import.rs @@ -1901,9 +1901,21 @@ impl DatasetKey { }) .collect::(); + let resolution_str = format!("{}", self.resolution); + let clean_resolution: String = resolution_str + .chars() + .map(|c| { + if geoengine_datatypes::dataset::is_invalid_name_char(c) { + '_' + } else { + c + } + }) + .collect(); + format!( "{}_EPSG{}_{:?}_{}", - cleaned_name, self.epsg, self.data_type, self.resolution + cleaned_name, self.epsg, self.data_type, clean_resolution ) } } diff --git a/geoengine/services/src/datasets/external/stac/common.rs b/geoengine/services/src/datasets/external/stac/common.rs new file mode 100644 index 0000000000..89a47f905a --- /dev/null +++ b/geoengine/services/src/datasets/external/stac/common.rs @@ -0,0 +1,984 @@ +//! Common utilities shared between the STAC provider and the STAC harvester CLI. +//! +//! This module contains functions for parsing STAC metadata (geometry, EPSG, data types, +//! band names), STAC API query helpers, and GDAL file path handling. +//! +//! All functions herein should be usable from both `loading_info.rs` (the STAC provider) +//! and `cli/stac_harvester.rs` (the new STAC harvester CLI). + +#![allow(dead_code)] + +use geoengine_datatypes::{ + raster::{GdalGeoTransform, GeoTransform, RasterDataType}, + spatial_reference::SpatialReference, +}; +use serde::Deserialize; +use std::path::PathBuf; + +use super::StacProviderS3Config; + +// --------------------------------------------------------------------------- +// STAC extension version types +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StacExtensionMajorVersion { + V1, + V2, +} + +/// Extract a `GeoTransform` from `proj:transform` in asset/collection fields. +/// +/// The STAC `proj:transform` is a 6-element array in GDAL convention: +/// `[pixel_width, rotation, origin_x, rotation, pixel_height, origin_y]`. +pub fn geo_transform_from_fields( + fields: &serde_json::Map, +) -> Option { + let proj_transform = fields.get("proj:transform")?; + let proj_transform_array = proj_transform.as_array()?; + if proj_transform_array.len() != 6 { + return None; + } + + let values: Vec = proj_transform_array + .iter() + .filter_map(serde_json::Value::as_f64) + .collect(); + if values.len() != 6 { + return None; + } + + // GDAL geo-transform: [origin_x, pixel_width, rotation, origin_y, rotation, pixel_height] + let gdal_geotransform: GdalGeoTransform = [ + values[2], // origin_x + values[0], // pixel_width + values[1], // rotation + values[5], // origin_y + values[3], // rotation + values[4], // pixel_height (negative for north-up) + ]; + Some(gdal_geotransform.into()) +} + +/// Extract `(height, width)` from `proj:shape` in asset fields. +pub fn proj_shape_from_fields( + fields: &serde_json::Map, +) -> Option<(usize, usize)> { + let proj_shape = fields.get("proj:shape")?.as_array()?; + if proj_shape.len() != 2 { + return None; + } + let height = proj_shape.first()?.as_u64()? as usize; + let width = proj_shape.get(1)?.as_u64()? as usize; + + Some((height, width)) +} + +// --------------------------------------------------------------------------- +// EPSG / projection helpers +// --------------------------------------------------------------------------- + +/// Extract EPSG code from asset fields, respecting the STAC extension version's +/// field priority (`proj:epsg` vs `proj:code`). +pub fn epsg_code_from_fields( + proj_extension_version: StacExtensionMajorVersion, + fields: &serde_json::Map, +) -> Option { + let proj_epsg = fields.get("proj:epsg").and_then(|value| { + value + .as_u64() + .map(|code| code as u32) + .or_else(|| value.as_str().and_then(|code| code.parse::().ok())) + }); + + let proj_code = fields + .get("proj:code") + .and_then(serde_json::Value::as_str) + .and_then(parse_epsg_from_proj_code); + + match proj_extension_version { + StacExtensionMajorVersion::V1 => proj_epsg.or(proj_code), + StacExtensionMajorVersion::V2 => proj_code.or(proj_epsg), + } +} + +/// Extract EPSG code from a STAC item, with multi-layered fallback between +/// additional fields and serialized properties, and fallback extension version. +pub fn epsg_code_from_item( + item: &stac::Item, + proj_extension_version: StacExtensionMajorVersion, +) -> Option { + let from_additional = + epsg_code_from_fields(proj_extension_version, &item.properties.additional_fields); + if from_additional.is_some() { + return from_additional; + } + + let properties = serde_json::to_value(item) + .ok() + .and_then(|value| value.get("properties").cloned()) + .and_then(|value| value.as_object().cloned())?; + + let from_properties = epsg_code_from_fields(proj_extension_version, &properties); + if from_properties.is_some() { + return from_properties; + } + + let fallback_version = match proj_extension_version { + StacExtensionMajorVersion::V1 => StacExtensionMajorVersion::V2, + StacExtensionMajorVersion::V2 => StacExtensionMajorVersion::V1, + }; + + epsg_code_from_fields(fallback_version, &properties) +} + +/// Parse an EPSG code from a `proj:code` string like `"EPSG:32632"` or +/// `"http://www.opengis.net/def/crs/EPSG/0/32632"`. +pub fn parse_epsg_from_proj_code(code: &str) -> Option { + if let Some(code) = code.strip_prefix("EPSG:") { + return code.parse::().ok(); + } + + // e.g. http://www.opengis.net/def/crs/EPSG/0/32632 + code.rsplit('/').next()?.parse::().ok() +} + +/// Check if the asset's `proj:code` matches the dataset's projection. +pub fn proj_code_matches_dataset( + fields: &serde_json::Map, + dataset_projection: SpatialReference, +) -> bool { + let Some(code) = fields.get("proj:code") else { + return false; + }; + + let Some(proj_code) = proj_code_as_srs_string(code) else { + return false; + }; + + proj_code == dataset_projection.to_string() +} + +/// Normalize a `proj:code` field value to an `"EPSG:nnnn"` string. +pub fn proj_code_as_srs_string(value: &serde_json::Value) -> Option { + if let Some(code_number) = value.as_u64() { + return Some(format!("EPSG:{code_number}")); + } + + let code_str = value.as_str()?.trim(); + if code_str.contains(':') { + return Some(code_str.to_ascii_uppercase()); + } + + if let Ok(code_number) = code_str.parse::() { + return Some(format!("EPSG:{code_number}")); + } + + None +} + +// --------------------------------------------------------------------------- +// Data type conversion helpers +// --------------------------------------------------------------------------- + +/// Map a `stac_extensions::raster::DataType` to a Geo Engine `RasterDataType`. +pub fn raster_data_type_from_stac_data_type( + data_type: &stac_extensions::raster::DataType, +) -> Option { + match data_type { + stac_extensions::raster::DataType::UInt8 => Some(RasterDataType::U8), + stac_extensions::raster::DataType::UInt16 => Some(RasterDataType::U16), + stac_extensions::raster::DataType::UInt32 => Some(RasterDataType::U32), + stac_extensions::raster::DataType::Int16 => Some(RasterDataType::I16), + stac_extensions::raster::DataType::Int32 => Some(RasterDataType::I32), + stac_extensions::raster::DataType::Float32 => Some(RasterDataType::F32), + stac_extensions::raster::DataType::Float64 => Some(RasterDataType::F64), + _ => None, + } +} + +/// Map a STAC data type string (e.g. `"uint16"`, `"float32"`) to a `RasterDataType`. +pub fn raster_data_type_from_stac_data_type_str(data_type_str: &str) -> Option { + match data_type_str.to_lowercase().as_str() { + "uint8" => Some(RasterDataType::U8), + "uint16" => Some(RasterDataType::U16), + "uint32" => Some(RasterDataType::U32), + "int16" => Some(RasterDataType::I16), + "int32" => Some(RasterDataType::I32), + "float32" => Some(RasterDataType::F32), + "float64" => Some(RasterDataType::F64), + _ => None, + } +} + +/// Extract data type from a STAC 1.1.0 asset (common metadata `data_type` field). +pub fn data_type_from_asset_v1_1_0(asset: &stac::Asset) -> Option { + asset + .data_type + .as_ref() + .and_then(raster_data_type_from_stac_data_type) +} + +// --------------------------------------------------------------------------- +// File path helpers +// --------------------------------------------------------------------------- + +/// Convert a STAC asset href (HTTP or S3 URL) to a GDAL VSI path. +/// +/// - `http://...` → `/vsicurl/http://...` +/// - `s3://bucket/key` → `/vsis3/bucket/key` +pub fn gdal_file_path(href: &str) -> Option { + if href.starts_with("http") { + return Some(PathBuf::from(format!("/vsicurl/{href}"))); + } + + href.strip_prefix("s3://") + .map(|s3_path| PathBuf::from(format!("/vsis3/{s3_path}"))) +} + +// --------------------------------------------------------------------------- +// Band processing helpers +// --------------------------------------------------------------------------- + +#[derive(Debug, Deserialize)] +pub struct EoBand { + pub name: String, + #[serde(default)] + pub common_name: Option, +} + +/// Map a GDAL raster band channel index for a dataset band within an asset. +/// +/// If the asset has no `bands` metadata, returns channel 1 (single-band asset). +/// If the asset has bands, matches by `band_name` against asset band names. +/// If the asset has exactly one band and no `band_name` is required, returns +/// channel 1 (single-band asset treated the same as no band metadata). +/// Returns `None` if the required band is not found. +pub fn rasterband_channel_for_dataset_band( + asset: &stac::Asset, + required_band_name: Option<&str>, +) -> Option { + if asset.bands.is_empty() || (asset.bands.len() == 1 && required_band_name.is_none()) { + if required_band_name.is_some() && asset.bands.is_empty() { + tracing::warn!( + "STAC asset with href {} does not include bands, but dataset configuration requires a band name. Skipping asset.", + asset.href + ); + return None; + } + + return Some(1); + } + + let Some(required_band_name) = required_band_name else { + tracing::warn!( + "STAC asset with href {} includes {} bands, but dataset configuration does not specify a band name. Skipping asset.", + asset.href, + asset.bands.len() + ); + return None; + }; + + let Some(asset_band_idx) = asset + .bands + .iter() + .position(|asset_band| asset_band.name.as_deref() == Some(required_band_name)) + else { + tracing::debug!( + "Skipping asset with href {} due to missing required band {}", + asset.href, + required_band_name + ); + return None; + }; + + Some(asset_band_idx + 1) +} + +/// Derive band names from a STAC 1.1.0 `Asset`, using the `bands` field. +pub fn band_names_from_asset_v1_1_0(asset: &stac::Asset) -> Result, String> { + let asset_title = asset + .title + .as_deref() + .ok_or_else(|| "Missing title in asset metadata".to_string())?; + + let bands = &asset.bands; + + if bands.is_empty() { + return Ok(vec![asset_title.to_string()]); + } + + if bands.len() == 1 { + return Ok(vec![asset_title.to_string()]); + } + + let mut names = Vec::new(); + for band in bands { + let Some(band_name) = &band.name else { + return Err("Band is missing name for multi-band asset".to_string()); + }; + names.push(format!("{asset_title} [{band_name}]")); + } + + Ok(names) +} + +/// Derive band names from a STAC 1.1.0 `ItemAsset`, using the `bands` additional field. +pub fn band_names_from_item_asset_v1_1_0(asset: &stac::ItemAsset) -> Result, String> { + let asset_title = asset + .title + .as_deref() + .ok_or_else(|| "Missing title in asset metadata".to_string())?; + + let band_names = asset + .additional_fields + .get("bands") + .and_then(serde_json::Value::as_array); + + let Some(bands) = band_names else { + return Ok(vec![asset_title.to_string()]); + }; + + if bands.is_empty() { + return Ok(vec![asset_title.to_string()]); + } + + if bands.len() == 1 { + return Ok(vec![asset_title.to_string()]); + } + + let mut names = Vec::new(); + for band in bands { + let band_name = band + .get("name") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| "Band is missing name for multi-band asset".to_string())?; + names.push(format!("{asset_title} [{band_name}]")); + } + + Ok(names) +} + +/// Normalize a label string: trim, lowercase, join whitespace-separated words with underscores. +pub fn normalize_label(value: &str) -> String { + value + .trim() + .to_lowercase() + .split_whitespace() + .collect::>() + .join("_") +} + +/// Derive a fallback band label from an asset title. +/// +/// Prefers concise acronym-like labels in parentheses, e.g. `"Scene classification map (SCL)"` → `"scl"`. +/// Falls back to a normalized version of the full title. +pub fn title_fallback_label(title: Option<&str>) -> String { + if let Some(title) = title { + // Prefer concise acronym-like labels in parentheses + if let (Some(start), Some(end)) = (title.rfind('('), title.rfind(')')) + && start < end + { + let short = title[start + 1..end].trim(); + if !short.is_empty() + && short.len() <= 32 + && short + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') + { + return short.to_lowercase(); + } + } + + let normalized = normalize_label(title); + if !normalized.is_empty() { + return normalized; + } + } + + "band".to_string() +} + +/// Map red-edge variant names for Sentinel-2 bands B05/B06/B07 from metadata. +pub fn rededge_variant_from_metadata(eo_name: &str, title: Option<&str>) -> Option<&'static str> { + let eo = eo_name.to_lowercase(); + let title_lower = title.map(str::to_lowercase).unwrap_or_default(); + + if eo.contains("b05") + || eo.contains("band_5") + || title_lower.contains("band 5") + || title_lower.contains("b05") + { + return Some("rededge1"); + } + if eo.contains("b06") + || eo.contains("band_6") + || title_lower.contains("band 6") + || title_lower.contains("b06") + { + return Some("rededge2"); + } + if eo.contains("b07") + || eo.contains("band_7") + || title_lower.contains("band 7") + || title_lower.contains("b07") + { + return Some("rededge3"); + } + + None +} + +/// Derive a stable band name from STAC 1.0.0 metadata (EO band + asset title). +pub fn v1_0_0_band_name( + title: Option<&str>, + eo_band: Option<&EoBand>, + band_count: usize, +) -> String { + let eo_name = eo_band.and_then(|band| { + let eo_name = band.name.to_lowercase(); + let common_name = band.common_name.as_ref().map(|name| name.to_lowercase()); + + match common_name.as_deref() { + // `rededge` is used for multiple Sentinel-2 bands (B05/B06/B07). + // Keep stable, unique names to avoid band collisions. + Some("rededge") => rededge_variant_from_metadata(&eo_name, title) + .map(std::string::ToString::to_string) + .or_else(|| Some(format!("rededge[{eo_name}]"))), + Some(common_name) => Some(common_name.to_string()), + None => Some(eo_name), + } + }); + + if band_count > 1 { + let asset_label = title_fallback_label(title); + let eo_name = eo_name.unwrap_or_else(|| "band".to_string()); + return format!("{asset_label}[{eo_name}]"); + } + + if let Some(eo_name) = eo_name { + return eo_name; + } + + title_fallback_label(title) +} + +// --------------------------------------------------------------------------- +// Media type helpers +// --------------------------------------------------------------------------- + +/// Check if a media type is a Cloud-Optimized `GeoTIFF`. +pub fn is_cog_media_type(media_type: Option<&str>) -> bool { + media_type == Some("image/tiff; application=geotiff; profile=cloud-optimized") +} + +/// Check if a media type is JPEG 2000. +pub fn is_jp2_media_type(media_type: Option<&str>) -> bool { + media_type == Some("image/jp2") +} + +// --------------------------------------------------------------------------- +// GDAL config options +// --------------------------------------------------------------------------- + +/// Build GDAL configuration options for `/vsis3/` paths. +/// +/// Returns the common options plus S3-specific credentials when an S3 config is provided. +pub fn gdal_config_options_for_s3( + s3_config: Option<&StacProviderS3Config>, +) -> Vec<(String, String)> { + let mut options = Vec::new(); + + if let Some(config) = s3_config { + // For old GDAL versions, the S3 endpoint may not include the protocol + options.push(("AWS_S3_ENDPOINT".to_owned(), config.endpoint.clone())); + options.push(("AWS_VIRTUAL_HOSTING".to_owned(), "FALSE".to_owned())); + + if let Some(access_key) = &config.access_key { + options.push(("AWS_ACCESS_KEY_ID".to_owned(), access_key.clone())); + } + + if let Some(secret_key) = &config.secret_key { + options.push(("AWS_SECRET_ACCESS_KEY".to_owned(), secret_key.clone())); + } + } + + options +} + +/// Build GDAL configuration options for a VSI file path, including common CURL/S3 options. +pub fn gdal_config_options_for_file_path( + file_path: &std::path::Path, + s3_config: Option<&StacProviderS3Config>, +) -> Option> { + let file_path_str = file_path.to_string_lossy(); + let is_vsi_s3 = file_path_str.starts_with("/vsis3/"); + let is_vsi_curl = file_path_str.starts_with("/vsicurl/"); + + if !is_vsi_s3 && !is_vsi_curl { + return None; + } + + let mut options = vec![ + ( + "GDAL_DISABLE_READDIR_ON_OPEN".to_owned(), + "EMPTY_DIR".to_owned(), + ), + ( + "CPL_VSIL_CURL_ALLOWED_EXTENSIONS".to_owned(), + ".tif,.tiff,.jp2".to_owned(), + ), + ]; + + if is_vsi_s3 { + options.extend(gdal_config_options_for_s3(s3_config)); + } + + Some(options) +} + +#[cfg(test)] +mod tests { + use super::*; + use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceAuthority}; + + // ----------------------------------------------------------------------- + // geo_transform_from_fields + // ----------------------------------------------------------------------- + + #[test] + fn test_geo_transform_from_fields_standard() { + let mut fields = serde_json::Map::new(); + fields.insert( + "proj:transform".to_string(), + serde_json::json!([10.0, 0.0, 399_960.0, 0.0, -10.0, 5_700_000.0]), + ); + + let gt = geo_transform_from_fields(&fields).expect("should parse transform"); + assert!((gt.origin_coordinate.x - 399_960.0).abs() < 1e-9); + assert!((gt.origin_coordinate.y - 5_700_000.0).abs() < 1e-9); + assert!((gt.x_pixel_size() - 10.0).abs() < 1e-9); + assert!((gt.y_pixel_size() - (-10.0)).abs() < 1e-9); + } + + #[test] + fn test_geo_transform_from_fields_missing() { + let fields = serde_json::Map::new(); + assert!(geo_transform_from_fields(&fields).is_none()); + } + + #[test] + fn test_geo_transform_from_fields_wrong_length() { + let mut fields = serde_json::Map::new(); + fields.insert( + "proj:transform".to_string(), + serde_json::json!([1.0, 2.0, 3.0]), + ); + assert!(geo_transform_from_fields(&fields).is_none()); + } + + // ----------------------------------------------------------------------- + // proj_shape_from_fields + // ----------------------------------------------------------------------- + + #[test] + fn test_proj_shape_from_fields_standard() { + let mut fields = serde_json::Map::new(); + fields.insert("proj:shape".to_string(), serde_json::json!([10980, 10980])); + let (height, width) = proj_shape_from_fields(&fields).expect("should parse shape"); + assert_eq!(height, 10_980); + assert_eq!(width, 10_980); + } + + #[test] + fn test_proj_shape_from_fields_missing() { + let fields = serde_json::Map::new(); + assert!(proj_shape_from_fields(&fields).is_none()); + } + + // ----------------------------------------------------------------------- + // raster_data_type_from_stac_data_type + // ----------------------------------------------------------------------- + + #[test] + fn test_raster_data_type_from_stac_data_type_all() { + use stac_extensions::raster::DataType; + assert_eq!( + raster_data_type_from_stac_data_type(&DataType::UInt8), + Some(RasterDataType::U8) + ); + assert_eq!( + raster_data_type_from_stac_data_type(&DataType::UInt16), + Some(RasterDataType::U16) + ); + assert_eq!( + raster_data_type_from_stac_data_type(&DataType::UInt32), + Some(RasterDataType::U32) + ); + assert_eq!( + raster_data_type_from_stac_data_type(&DataType::Int16), + Some(RasterDataType::I16) + ); + assert_eq!( + raster_data_type_from_stac_data_type(&DataType::Int32), + Some(RasterDataType::I32) + ); + assert_eq!( + raster_data_type_from_stac_data_type(&DataType::Float32), + Some(RasterDataType::F32) + ); + assert_eq!( + raster_data_type_from_stac_data_type(&DataType::Float64), + Some(RasterDataType::F64) + ); + } + + #[test] + fn test_raster_data_type_from_stac_data_type_unknown() { + use stac_extensions::raster::DataType; + assert_eq!(raster_data_type_from_stac_data_type(&DataType::Int8), None); + } + + // ----------------------------------------------------------------------- + // raster_data_type_from_stac_data_type_str + // ----------------------------------------------------------------------- + + #[test] + fn test_raster_data_type_from_stac_data_type_str_all() { + assert_eq!( + raster_data_type_from_stac_data_type_str("uint8"), + Some(RasterDataType::U8) + ); + assert_eq!( + raster_data_type_from_stac_data_type_str("uint16"), + Some(RasterDataType::U16) + ); + assert_eq!( + raster_data_type_from_stac_data_type_str("float32"), + Some(RasterDataType::F32) + ); + assert_eq!( + raster_data_type_from_stac_data_type_str("UINT16"), + Some(RasterDataType::U16) + ); + assert_eq!(raster_data_type_from_stac_data_type_str("unknown"), None); + } + + // ----------------------------------------------------------------------- + // gdal_file_path + // ----------------------------------------------------------------------- + + #[test] + fn test_gdal_file_path_http() { + let path = gdal_file_path("https://example.com/file.tif").expect("should parse"); + assert_eq!(path, PathBuf::from("/vsicurl/https://example.com/file.tif")); + } + + #[test] + fn test_gdal_file_path_s3() { + let path = gdal_file_path("s3://bucket/key/file.tif").expect("should parse"); + assert_eq!(path, PathBuf::from("/vsis3/bucket/key/file.tif")); + } + + #[test] + fn test_gdal_file_path_unsupported() { + assert!(gdal_file_path("/local/path.tif").is_none()); + } + + // ----------------------------------------------------------------------- + // parse_epsg_from_proj_code + // ----------------------------------------------------------------------- + + #[test] + fn test_parse_epsg_from_proj_code_epsg_prefix() { + assert_eq!(parse_epsg_from_proj_code("EPSG:32632"), Some(32632)); + } + + #[test] + fn test_parse_epsg_from_proj_code_url() { + assert_eq!( + parse_epsg_from_proj_code("http://www.opengis.net/def/crs/EPSG/0/32632"), + Some(32632) + ); + } + + #[test] + fn test_parse_epsg_from_proj_code_invalid() { + assert!(parse_epsg_from_proj_code("invalid").is_none()); + } + + // ----------------------------------------------------------------------- + // proj_code_as_srs_string + // ----------------------------------------------------------------------- + + #[test] + fn test_proj_code_as_srs_string_number() { + assert_eq!( + proj_code_as_srs_string(&serde_json::json!(32632)), + Some("EPSG:32632".to_string()) + ); + } + + #[test] + fn test_proj_code_as_srs_string_epsg_format() { + assert_eq!( + proj_code_as_srs_string(&serde_json::json!("EPSG:32632")), + Some("EPSG:32632".to_string()) + ); + } + + #[test] + fn test_proj_code_as_srs_string_lowercase() { + assert_eq!( + proj_code_as_srs_string(&serde_json::json!("epsg:32632")), + Some("EPSG:32632".to_string()) + ); + } + + // ----------------------------------------------------------------------- + // proj_code_matches_dataset + // ----------------------------------------------------------------------- + + #[test] + fn test_proj_code_matches_dataset_matching() { + let mut fields = serde_json::Map::new(); + fields.insert("proj:code".to_string(), serde_json::json!("EPSG:32632")); + let srs = SpatialReference::new(SpatialReferenceAuthority::Epsg, 32632); + assert!(proj_code_matches_dataset(&fields, srs)); + } + + #[test] + fn test_proj_code_matches_dataset_not_matching() { + let mut fields = serde_json::Map::new(); + fields.insert("proj:code".to_string(), serde_json::json!("EPSG:32633")); + let srs = SpatialReference::new(SpatialReferenceAuthority::Epsg, 32632); + assert!(!proj_code_matches_dataset(&fields, srs)); + } + + #[test] + fn test_proj_code_matches_dataset_missing() { + let fields = serde_json::Map::new(); + let srs = SpatialReference::new(SpatialReferenceAuthority::Epsg, 32632); + assert!(!proj_code_matches_dataset(&fields, srs)); + } + + // ----------------------------------------------------------------------- + // epsg_code_from_fields + // ----------------------------------------------------------------------- + + #[test] + fn test_epsg_code_from_fields_v1_epsg() { + let mut fields = serde_json::Map::new(); + fields.insert("proj:epsg".to_string(), serde_json::json!(32632)); + assert_eq!( + epsg_code_from_fields(StacExtensionMajorVersion::V1, &fields), + Some(32632) + ); + } + + #[test] + fn test_epsg_code_from_fields_v1_code() { + let mut fields = serde_json::Map::new(); + fields.insert("proj:code".to_string(), serde_json::json!("EPSG:32632")); + // V1 prefers proj:epsg over proj:code + assert_eq!( + epsg_code_from_fields(StacExtensionMajorVersion::V1, &fields), + Some(32632) + ); + } + + #[test] + fn test_epsg_code_from_fields_v2_code() { + let mut fields = serde_json::Map::new(); + fields.insert("proj:code".to_string(), serde_json::json!("EPSG:32632")); + // V2 prefers proj:code over proj:epsg + assert_eq!( + epsg_code_from_fields(StacExtensionMajorVersion::V2, &fields), + Some(32632) + ); + } + + // ----------------------------------------------------------------------- + // normalize_label / title_fallback_label + // ----------------------------------------------------------------------- + + #[test] + fn test_normalize_label() { + assert_eq!( + normalize_label("Scene classification map"), + "scene_classification_map" + ); + assert_eq!(normalize_label(" Blue band "), "blue_band"); + } + + #[test] + fn test_title_fallback_label_parentheses() { + assert_eq!( + title_fallback_label(Some("Scene classification map (SCL)")), + "scl" + ); + } + + #[test] + fn test_title_fallback_label_no_parentheses() { + assert_eq!( + title_fallback_label(Some("Blue (band 2) - 10m")), + "blue_(band_2)_-_10m" + ); + } + + #[test] + fn test_title_fallback_label_none() { + assert_eq!(title_fallback_label(None), "band"); + } + + // ----------------------------------------------------------------------- + // band_names_from_asset_v1_1_0 + // ----------------------------------------------------------------------- + + #[test] + fn test_band_names_from_asset_v1_1_0_no_bands() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif", + "title": "My Band" + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + let names = band_names_from_asset_v1_1_0(&asset).expect("should succeed"); + assert_eq!(names, vec!["My Band"]); + } + + #[test] + fn test_band_names_from_asset_v1_1_0_single_band() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif", + "title": "My Asset", + "bands": [{"name": "B01"}] + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + let names = band_names_from_asset_v1_1_0(&asset).expect("should succeed"); + assert_eq!(names, vec!["My Asset"]); + } + + #[test] + fn test_band_names_from_asset_v1_1_0_multi_band() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif", + "title": "Sentinel-2", + "bands": [{"name": "B04"}, {"name": "B03"}, {"name": "B02"}] + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + let names = band_names_from_asset_v1_1_0(&asset).expect("should succeed"); + assert_eq!( + names, + vec!["Sentinel-2 [B04]", "Sentinel-2 [B03]", "Sentinel-2 [B02]",] + ); + } + + // ----------------------------------------------------------------------- + // rasterband_channel_for_dataset_band + // ----------------------------------------------------------------------- + + #[test] + fn test_rasterband_channel_no_bands_no_required() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif" + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + assert_eq!(rasterband_channel_for_dataset_band(&asset, None), Some(1)); + } + + #[test] + fn test_rasterband_channel_no_bands_with_required() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif" + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + assert_eq!( + rasterband_channel_for_dataset_band(&asset, Some("B04")), + None + ); + } + + #[test] + fn test_rasterband_channel_with_bands_matching() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif", + "bands": [{"name": "B04"}, {"name": "B03"}] + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + assert_eq!( + rasterband_channel_for_dataset_band(&asset, Some("B04")), + Some(1) + ); + assert_eq!( + rasterband_channel_for_dataset_band(&asset, Some("B03")), + Some(2) + ); + } + + #[test] + fn test_rasterband_channel_with_bands_not_matching() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif", + "bands": [{"name": "B04"}, {"name": "B03"}] + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + assert_eq!( + rasterband_channel_for_dataset_band(&asset, Some("B08")), + None + ); + } + + // ----------------------------------------------------------------------- + // gdal_config_options + // ----------------------------------------------------------------------- + + #[test] + fn test_gdal_config_options_for_s3_empty() { + let options = gdal_config_options_for_s3(None); + assert!(options.is_empty()); + } + + #[test] + fn test_gdal_config_options_for_s3_with_config() { + let config = StacProviderS3Config { + endpoint: "eodata.example.com".to_string(), + access_key: Some("key".to_string()), + secret_key: Some("secret".to_string()), + }; + let options = gdal_config_options_for_s3(Some(&config)); + assert!(options.contains(&( + "AWS_S3_ENDPOINT".to_string(), + "eodata.example.com".to_string() + ))); + assert!(options.contains(&("AWS_ACCESS_KEY_ID".to_string(), "key".to_string()))); + assert!(options.contains(&("AWS_SECRET_ACCESS_KEY".to_string(), "secret".to_string()))); + assert!(options.contains(&("AWS_VIRTUAL_HOSTING".to_string(), "FALSE".to_string()))); + } + + // ----------------------------------------------------------------------- + // data_type_from_asset_v1_1_0 + // ----------------------------------------------------------------------- + + #[test] + fn test_data_type_from_asset_v1_1_0_uint16() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif", + "data_type": "uint16" + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + assert_eq!( + data_type_from_asset_v1_1_0(&asset), + Some(RasterDataType::U16) + ); + } + + #[test] + fn test_data_type_from_asset_v1_1_0_missing() { + let json = serde_json::json!({ + "href": "http://example.com/file.tif" + }); + let asset: stac::Asset = serde_json::from_value(json).unwrap(); + assert_eq!(data_type_from_asset_v1_1_0(&asset), None); + } +} diff --git a/geoengine/services/src/datasets/external/stac/loading_info.rs b/geoengine/services/src/datasets/external/stac/loading_info.rs index 43214e0916..56264daece 100644 --- a/geoengine/services/src/datasets/external/stac/loading_info.rs +++ b/geoengine/services/src/datasets/external/stac/loading_info.rs @@ -1,3 +1,4 @@ +use super::common; use super::{StacDataProvider, StacProviderDataset, StacProviderS3Config, cache::StacQueryCache}; use crate::error::Result; use crate::util::join_base_url_and_path; @@ -11,7 +12,7 @@ use geoengine_datatypes::primitives::{ AxisAlignedRectangle, CacheHint, RasterQueryRectangle, TimeDimension, TimeInstance, TimeInterval, TryRegularTimeFillIterExt, VectorQueryRectangle, }; -use geoengine_datatypes::raster::{GeoTransform, GridBoundingBox2D, GridIdx2D, RasterDataType}; +use geoengine_datatypes::raster::{GridBoundingBox2D, GridIdx2D}; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_operators::engine::{ MetaData, MetaDataProvider, RasterBandDescriptors, RasterResultDescriptor, TimeDescriptor, @@ -23,9 +24,7 @@ use geoengine_operators::source::{ GdalRetryOptions, MultiBandGdalLoadingInfo, MultiBandGdalLoadingInfoQueryRectangle, OgrSourceDataset, TileFile, }; -use serde_json::Value; use stac::Item; -use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::Arc; use tracing::debug; @@ -454,15 +453,16 @@ impl StacMultiBandMetaData { z_index: i64, files: &mut Vec, ) -> Result<()> { - if data_type_from_asset_v1_1_0(asset) != Some(self.dataset.data_type) { + if common::data_type_from_asset_v1_1_0(asset) != Some(self.dataset.data_type) { return Ok(()); } - if !proj_code_matches_dataset(&asset.additional_fields, self.dataset.projection) { + if !common::proj_code_matches_dataset(&asset.additional_fields, self.dataset.projection) { return Ok(()); } - let Some(geo_transform) = geo_transform_from_fields(&asset.additional_fields) else { + let Some(geo_transform) = common::geo_transform_from_fields(&asset.additional_fields) + else { tracing::warn!( "Skipping asset with href {} due to missing geo transform", asset.href @@ -470,7 +470,7 @@ impl StacMultiBandMetaData { return Ok(()); }; - let Some((height, width)) = proj_shape_from_fields(&asset.additional_fields) else { + let Some((height, width)) = common::proj_shape_from_fields(&asset.additional_fields) else { tracing::warn!( "Skipping asset with href {} due to missing projection shape", asset.href @@ -499,19 +499,21 @@ impl StacMultiBandMetaData { .map_err(|_e| geoengine_operators::error::Error::InvalidDataProviderConfig)?; let spatial_partition = geo_transform.grid_to_spatial_bounds(&grid_bounds); - let file_path = gdal_file_path(&asset.href) + let file_path = common::gdal_file_path(&asset.href) .ok_or(geoengine_operators::error::Error::InvalidDataProviderConfig)?; - let gdal_config_options = self.gdal_config_options_for_file_path(&file_path); + let gdal_config_options = + common::gdal_config_options_for_file_path(&file_path, self.s3_config.as_ref()); for (dataset_band_idx, dataset_band) in self.dataset.bands.iter().enumerate() { if dataset_band.asset_title != asset_title { continue; } - let Some(rasterband_channel) = - Self::rasterband_channel_for_dataset_band(asset, dataset_band.band_name.as_deref()) - else { + let Some(rasterband_channel) = common::rasterband_channel_for_dataset_band( + asset, + dataset_band.band_name.as_deref(), + ) else { continue; }; @@ -1142,4 +1144,70 @@ mod tests { let _result_descriptor = initialized.result_descriptor(); // If we get here, the operator initialized successfully } + + /// Test that a discover-generated mapping JSON can be used directly as a + /// `StacDataProvider`, validating the mapping format works for both the + /// harvester and the runtime provider. + #[crate::ge_context::test] + async fn mapping_from_discover_works_as_stacdataprovider(app_ctx: PostgresContext) { + // Load the discover-generated mapping JSON + let mut provider_def: crate::datasets::external::stac::StacDataProviderDefinition = + serde_json::from_str(include_str!( + "../../../../../test_data/stac_responses/expected-mapping-code-de.json" + )) + .expect("valid discover mapping fixture"); + + // Use a placeholder URL (no actual HTTP calls needed for meta_data registration) + provider_def.api_url = "https://stac.test/v1".to_owned(); + provider_def.id = DataProviderId::new(); + + let admin_session = admin_login(&app_ctx).await; + let admin_ctx = app_ctx.session_context(admin_session); + + admin_ctx + .db() + .add_layer_provider(provider_def.clone().into()) + .await + .unwrap(); + + let provider = admin_ctx + .db() + .load_layer_provider(provider_def.id) + .await + .unwrap(); + + // Verify each dataset from the discover-generated mapping can be + // resolved via meta_data (no HTTP calls needed at this stage) + for dataset in &provider_def.datasets { + let epsg_code = dataset.projection.code(); + let data_type_str = format!("{:?}", dataset.data_type).to_lowercase(); + let resolution = dataset.resolution.x as u32; + let stable_id = format!("epsg{epsg_code}_{data_type_str}_{resolution}"); + + let layer_id = geoengine_datatypes::dataset::LayerId(format!("dataset/{stable_id}")); + let data_id: DataId = ExternalDataId { + provider_id: provider_def.id, + layer_id, + } + .into(); + + let meta_result: Result< + Box< + dyn MetaData< + MultiBandGdalLoadingInfo, + RasterResultDescriptor, + MultiBandGdalLoadingInfoQueryRectangle, + >, + >, + geoengine_operators::error::Error, + > = MetaDataProvider::meta_data(provider.as_ref(), &data_id).await; + + assert!( + meta_result.is_ok(), + "meta_data should succeed for dataset '{}' (stable_id: {})", + dataset.name, + stable_id + ); + } + } } diff --git a/geoengine/services/src/datasets/external/stac/mod.rs b/geoengine/services/src/datasets/external/stac/mod.rs index f10068503e..161c32fb5b 100644 --- a/geoengine/services/src/datasets/external/stac/mod.rs +++ b/geoengine/services/src/datasets/external/stac/mod.rs @@ -14,6 +14,7 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; mod cache; +pub(crate) mod common; mod listing; mod loading_info; diff --git a/geoengine/test_data/stac_responses/collections/code-de-minimal.json b/geoengine/test_data/stac_responses/collections/code-de-minimal.json new file mode 100644 index 0000000000..5729100a9e --- /dev/null +++ b/geoengine/test_data/stac_responses/collections/code-de-minimal.json @@ -0,0 +1,71 @@ +{ + "id": "sentinel-2-l2a", + "description": "Sentinel-2 Level-2A", + "stac_version": "1.1.0", + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v2.0.0/schema.json", + "https://stac-extensions.github.io/projection/v2.0.0/schema.json", + "https://stac-extensions.github.io/raster/v2.0.0/schema.json" + ], + "links": [ + { + "rel": "items", + "type": "application/geo+json", + "href": "https://stac.code-de.org/v1/collections/sentinel-2-l2a/items" + } + ], + "summaries": { + "gsd": [10, 20], + "platform": ["sentinel-2a", "sentinel-2b"] + }, + "item_assets": { + "B02_10m": { + "type": "image/jp2", + "title": "Blue (band 2) - 10m", + "bands": [{ "name": "B02", "eo:common_name": "blue" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980] + }, + "B03_10m": { + "type": "image/jp2", + "title": "Green (band 3) - 10m", + "bands": [{ "name": "B03", "eo:common_name": "green" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980] + }, + "B04_10m": { + "type": "image/jp2", + "title": "Red (band 4) - 10m", + "bands": [{ "name": "B04", "eo:common_name": "red" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980] + }, + "B08_10m": { + "type": "image/jp2", + "title": "NIR 1 (band 8) - 10m", + "bands": [{ "name": "B08", "eo:common_name": "nir" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980] + }, + "B11_20m": { + "type": "image/jp2", + "title": "SWIR 1 (band 11) - 20m", + "bands": [{ "name": "B11", "eo:common_name": "swir16" }], + "data_type": "uint16", + "gsd": 20, + "proj:shape": [5490, 5490] + }, + "B12_20m": { + "type": "image/jp2", + "title": "SWIR 2 (band 12) - 20m", + "bands": [{ "name": "B12", "eo:common_name": "swir22" }], + "data_type": "uint16", + "gsd": 20, + "proj:shape": [5490, 5490] + } + } +} diff --git a/geoengine/test_data/stac_responses/collections/landsat-c2-l1-minimal.json b/geoengine/test_data/stac_responses/collections/landsat-c2-l1-minimal.json new file mode 100644 index 0000000000..b56a7a61a3 --- /dev/null +++ b/geoengine/test_data/stac_responses/collections/landsat-c2-l1-minimal.json @@ -0,0 +1,55 @@ +{ + "id": "landsat-c2-l1", + "description": "Landsat Collection 2 Level-1", + "stac_version": "1.1.0", + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v2.0.0/schema.json", + "https://stac-extensions.github.io/projection/v2.0.0/schema.json", + "https://stac-extensions.github.io/raster/v2.0.0/schema.json" + ], + "links": [ + { + "rel": "items", + "type": "application/geo+json", + "href": "https://stac.code-de.org/v1/collections/landsat-c2-l1/items" + } + ], + "summaries": { + "gsd": [30], + "platform": ["landsat-8", "landsat-9"] + }, + "item_assets": { + "B4_30m": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Red (band 4) - 30m", + "bands": [{ "name": "B4", "eo:common_name": "red" }], + "data_type": "uint16", + "gsd": 30, + "proj:shape": [8000, 8000] + }, + "B5_30m": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "NIR (band 5) - 30m", + "bands": [{ "name": "B5", "eo:common_name": "nir" }], + "data_type": "uint16", + "gsd": 30, + "proj:shape": [8000, 8000] + }, + "B2_30m": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Blue (band 2) - 30m", + "bands": [{ "name": "B2", "eo:common_name": "blue" }], + "data_type": "uint16", + "gsd": 30, + "proj:shape": [8000, 8000] + }, + "B3_30m": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Green (band 3) - 30m", + "bands": [{ "name": "B3", "eo:common_name": "green" }], + "data_type": "uint16", + "gsd": 30, + "proj:shape": [8000, 8000] + } + } +} diff --git a/geoengine/test_data/stac_responses/expected-mapping-code-de.json b/geoengine/test_data/stac_responses/expected-mapping-code-de.json new file mode 100644 index 0000000000..0e49bcb127 --- /dev/null +++ b/geoengine/test_data/stac_responses/expected-mapping-code-de.json @@ -0,0 +1,69 @@ +{ + "name": "sentinel-2-l2a from STAC", + "id": "00000000-0000-0000-0000-000000000000", + "description": "Auto-discovered mapping for STAC collection 'sentinel-2-l2a' at https://stac.test/v1", + "priority": 50, + "apiUrl": "https://stac.test/v1", + "collectionName": "sentinel-2-l2a", + "s3Config": null, + "timeDimension": { + "regular": { + "origin": 0, + "step": { "granularity": "days", "step": 1 } + } + }, + "datasets": [ + { + "name": "sentinel-2-l2a EPSG:32632 U16 10m", + "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", + "data_type": "U16", + "resolution": { "x": 10.0, "y": 10.0 }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { "x": 399960.0, "y": 5700000.0 }, + "xPixelSize": 10.0, + "yPixelSize": -10.0 + }, + "gridBounds": { + "min": [0, 0], + "max": [10979, 10979] + } + }, + "state": "source" + }, + "bands": [ + { "asset_title": "Blue (band 2) - 10m", "band_name": null }, + { "asset_title": "Green (band 3) - 10m", "band_name": null }, + { "asset_title": "NIR 1 (band 8) - 10m", "band_name": null }, + { "asset_title": "Red (band 4) - 10m", "band_name": null } + ] + }, + { + "name": "sentinel-2-l2a EPSG:32632 U16 20m", + "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", + "data_type": "U16", + "resolution": { "x": 20.0, "y": 20.0 }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { "x": 399960.0, "y": 5700000.0 }, + "xPixelSize": 20.0, + "yPixelSize": -20.0 + }, + "gridBounds": { + "min": [0, 0], + "max": [5489, 5489] + } + }, + "state": "source" + }, + "bands": [ + { "asset_title": "SWIR 1 (band 11) - 20m", "band_name": null }, + { "asset_title": "SWIR 2 (band 12) - 20m", "band_name": null } + ] + } + ] +} diff --git a/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json b/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json new file mode 100644 index 0000000000..05713a7a10 --- /dev/null +++ b/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json @@ -0,0 +1,44 @@ +{ + "name": "landsat-c2-l1 from STAC", + "id": "00000000-0000-0000-0000-000000000000", + "description": "Auto-discovered mapping for STAC collection 'landsat-c2-l1' at https://stac.test/v1", + "priority": 50, + "apiUrl": "https://stac.test/v1", + "collectionName": "landsat-c2-l1", + "s3Config": null, + "timeDimension": { + "regular": { + "origin": 0, + "step": { "granularity": "days", "step": 1 } + } + }, + "datasets": [ + { + "name": "landsat-c2-l1 EPSG:32632 U16 30m", + "description": "Auto-discovered from STAC collection 'landsat-c2-l1'", + "data_type": "U16", + "resolution": { "x": 30.0, "y": 30.0 }, + "projection": "EPSG:32632", + "spatial_grid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { "x": 399960.0, "y": 5800020.0 }, + "xPixelSize": 30.0, + "yPixelSize": -30.0 + }, + "gridBounds": { + "min": [0, 0], + "max": [7999, 7999] + } + }, + "state": "source" + }, + "bands": [ + { "asset_title": "Blue (band 2) - 30m", "band_name": null }, + { "asset_title": "Green (band 3) - 30m", "band_name": null }, + { "asset_title": "NIR (band 5) - 30m", "band_name": null }, + { "asset_title": "Red (band 4) - 30m", "band_name": null } + ] + } + ] +} diff --git a/geoengine/test_data/stac_responses/items/code-de-harvest-test.json b/geoengine/test_data/stac_responses/items/code-de-harvest-test.json new file mode 100644 index 0000000000..d4160b9631 --- /dev/null +++ b/geoengine/test_data/stac_responses/items/code-de-harvest-test.json @@ -0,0 +1,200 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "stac_version": "1.1.0", + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v2.0.0/schema.json", + "https://stac-extensions.github.io/projection/v2.0.0/schema.json", + "https://stac-extensions.github.io/raster/v2.0.0/schema.json" + ], + "id": "S2B_MSIL2A_20260724T153629_N0512_R068_T19QBG_20260724T190638", + "collection": "sentinel-2-l2a", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [8.7, 50.8], + [8.8, 50.8], + [8.8, 50.9], + [8.7, 50.9], + [8.7, 50.8] + ] + ] + }, + "bbox": [8.7, 50.8, 8.8, 50.9], + "properties": { + "datetime": "2026-07-24T15:36:29.024000Z", + "updated": "2026-07-24T19:29:11.371218Z" + }, + "assets": { + "B02_10m": { + "type": "image/jp2", + "title": "Blue (band 2) - 10m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBG_20260724T190638.SAFE/GRANULE/L2A_T19QBG_A049000_20260724T153625/IMG_DATA/R10m/T19QBG_20260724T153629_B02_10m.jp2", + "bands": [{ "name": "B02", "eo:common_name": "blue" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:code": "EPSG:32632", + "proj:transform": [10.0, 0.0, 399960.0, 0.0, -10.0, 5700000.0] + }, + "B03_10m": { + "type": "image/jp2", + "title": "Green (band 3) - 10m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBG_20260724T190638.SAFE/GRANULE/L2A_T19QBG_A049000_20260724T153625/IMG_DATA/R10m/T19QBG_20260724T153629_B03_10m.jp2", + "bands": [{ "name": "B03", "eo:common_name": "green" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:code": "EPSG:32632", + "proj:transform": [10.0, 0.0, 399960.0, 0.0, -10.0, 5700000.0] + }, + "B04_10m": { + "type": "image/jp2", + "title": "Red (band 4) - 10m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBG_20260724T190638.SAFE/GRANULE/L2A_T19QBG_A049000_20260724T153625/IMG_DATA/R10m/T19QBG_20260724T153629_B04_10m.jp2", + "bands": [{ "name": "B04", "eo:common_name": "red" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:code": "EPSG:32632", + "proj:transform": [10.0, 0.0, 399960.0, 0.0, -10.0, 5700000.0] + }, + "B08_10m": { + "type": "image/jp2", + "title": "NIR 1 (band 8) - 10m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBG_20260724T190638.SAFE/GRANULE/L2A_T19QBG_A049000_20260724T153625/IMG_DATA/R10m/T19QBG_20260724T153629_B08_10m.jp2", + "bands": [{ "name": "B08", "eo:common_name": "nir" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:code": "EPSG:32632", + "proj:transform": [10.0, 0.0, 399960.0, 0.0, -10.0, 5700000.0] + }, + "B11_20m": { + "type": "image/jp2", + "title": "SWIR 1 (band 11) - 20m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBG_20260724T190638.SAFE/GRANULE/L2A_T19QBG_A049000_20260724T153625/IMG_DATA/R20m/T19QBG_20260724T153629_B11_20m.jp2", + "bands": [{ "name": "B11", "eo:common_name": "swir16" }], + "data_type": "uint16", + "gsd": 20, + "proj:shape": [5490, 5490], + "proj:code": "EPSG:32632", + "proj:transform": [20.0, 0.0, 399960.0, 0.0, -20.0, 5700000.0] + }, + "B12_20m": { + "type": "image/jp2", + "title": "SWIR 2 (band 12) - 20m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBG_20260724T190638.SAFE/GRANULE/L2A_T19QBG_A049000_20260724T153625/IMG_DATA/R20m/T19QBG_20260724T153629_B12_20m.jp2", + "bands": [{ "name": "B12", "eo:common_name": "swir22" }], + "data_type": "uint16", + "gsd": 20, + "proj:shape": [5490, 5490], + "proj:code": "EPSG:32632", + "proj:transform": [20.0, 0.0, 399960.0, 0.0, -20.0, 5700000.0] + } + } + }, + { + "type": "Feature", + "stac_version": "1.1.0", + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v2.0.0/schema.json", + "https://stac-extensions.github.io/projection/v2.0.0/schema.json", + "https://stac-extensions.github.io/raster/v2.0.0/schema.json" + ], + "id": "S2B_MSIL2A_20260724T153629_N0512_R068_T19QBF_20260724T190638", + "collection": "sentinel-2-l2a", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [8.8, 50.8], + [8.9, 50.8], + [8.9, 50.9], + [8.8, 50.9], + [8.8, 50.8] + ] + ] + }, + "bbox": [8.8, 50.8, 8.9, 50.9], + "properties": { + "datetime": "2026-07-24T15:36:29.024000Z", + "updated": "2026-07-24T19:33:06.935907Z" + }, + "assets": { + "B02_10m": { + "type": "image/jp2", + "title": "Blue (band 2) - 10m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBF_20260724T190638.SAFE/GRANULE/L2A_T19QBF_A049000_20260724T153625/IMG_DATA/R10m/T19QBF_20260724T153629_B02_10m.jp2", + "bands": [{ "name": "B02", "eo:common_name": "blue" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:code": "EPSG:32632", + "proj:transform": [10.0, 0.0, 399960.0, 0.0, -10.0, 5700000.0] + }, + "B03_10m": { + "type": "image/jp2", + "title": "Green (band 3) - 10m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBF_20260724T190638.SAFE/GRANULE/L2A_T19QBF_A049000_20260724T153625/IMG_DATA/R10m/T19QBF_20260724T153629_B03_10m.jp2", + "bands": [{ "name": "B03", "eo:common_name": "green" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:code": "EPSG:32632", + "proj:transform": [10.0, 0.0, 399960.0, 0.0, -10.0, 5700000.0] + }, + "B04_10m": { + "type": "image/jp2", + "title": "Red (band 4) - 10m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBF_20260724T190638.SAFE/GRANULE/L2A_T19QBF_A049000_20260724T153625/IMG_DATA/R10m/T19QBF_20260724T153629_B04_10m.jp2", + "bands": [{ "name": "B04", "eo:common_name": "red" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:code": "EPSG:32632", + "proj:transform": [10.0, 0.0, 399960.0, 0.0, -10.0, 5700000.0] + }, + "B08_10m": { + "type": "image/jp2", + "title": "NIR 1 (band 8) - 10m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBF_20260724T190638.SAFE/GRANULE/L2A_T19QBF_A049000_20260724T153625/IMG_DATA/R10m/T19QBF_20260724T153629_B08_10m.jp2", + "bands": [{ "name": "B08", "eo:common_name": "nir" }], + "data_type": "uint16", + "gsd": 10, + "proj:shape": [10980, 10980], + "proj:code": "EPSG:32632", + "proj:transform": [10.0, 0.0, 399960.0, 0.0, -10.0, 5700000.0] + }, + "B11_20m": { + "type": "image/jp2", + "title": "SWIR 1 (band 11) - 20m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBF_20260724T190638.SAFE/GRANULE/L2A_T19QBF_A049000_20260724T153625/IMG_DATA/R20m/T19QBF_20260724T153629_B11_20m.jp2", + "bands": [{ "name": "B11", "eo:common_name": "swir16" }], + "data_type": "uint16", + "gsd": 20, + "proj:shape": [5490, 5490], + "proj:code": "EPSG:32632", + "proj:transform": [20.0, 0.0, 399960.0, 0.0, -20.0, 5700000.0] + }, + "B12_20m": { + "type": "image/jp2", + "title": "SWIR 2 (band 12) - 20m", + "href": "s3://eodata/Sentinel-2/MSI/L2A/2026/07/24/S2B_MSIL2A_20260724T153629_N0512_R068_T19QBF_20260724T190638.SAFE/GRANULE/L2A_T19QBF_A049000_20260724T153625/IMG_DATA/R20m/T19QBF_20260724T153629_B12_20m.jp2", + "bands": [{ "name": "B12", "eo:common_name": "swir22" }], + "data_type": "uint16", + "gsd": 20, + "proj:shape": [5490, 5490], + "proj:code": "EPSG:32632", + "proj:transform": [20.0, 0.0, 399960.0, 0.0, -20.0, 5700000.0] + } + } + } + ], + "links": [], + "numberReturned": 2, + "numberMatched": 2 +} diff --git a/geoengine/test_data/stac_responses/items/landsat-c2-l1-harvest-test.json b/geoengine/test_data/stac_responses/items/landsat-c2-l1-harvest-test.json new file mode 100644 index 0000000000..b22d4fcad8 --- /dev/null +++ b/geoengine/test_data/stac_responses/items/landsat-c2-l1-harvest-test.json @@ -0,0 +1,82 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "stac_version": "1.1.0", + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v2.0.0/schema.json", + "https://stac-extensions.github.io/projection/v2.0.0/schema.json", + "https://stac-extensions.github.io/raster/v2.0.0/schema.json" + ], + "id": "LC08_L1TP_192026_20260724_20260724_02_RT", + "collection": "landsat-c2-l1", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [8.5, 50.5], + [9.0, 50.5], + [9.0, 51.0], + [8.5, 51.0], + [8.5, 50.5] + ] + ] + }, + "bbox": [8.5, 50.5, 9.0, 51.0], + "properties": { + "datetime": "2026-07-24T10:32:18.024000Z", + "updated": "2026-07-24T18:15:42.137000Z" + }, + "assets": { + "B4_30m": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Red (band 4) - 30m", + "href": "s3://usgs-landsat/collection02/level-1/2026/192/026/LC08_L1TP_192026_20260724_20260724_02_RT/B4.TIF", + "bands": [{ "name": "B4", "eo:common_name": "red" }], + "data_type": "uint16", + "gsd": 30, + "proj:shape": [8000, 8000], + "proj:code": "EPSG:32632", + "proj:transform": [30.0, 0.0, 399960.0, 0.0, -30.0, 5800020.0] + }, + "B5_30m": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "NIR (band 5) - 30m", + "href": "s3://usgs-landsat/collection02/level-1/2026/192/026/LC08_L1TP_192026_20260724_20260724_02_RT/B5.TIF", + "bands": [{ "name": "B5", "eo:common_name": "nir" }], + "data_type": "uint16", + "gsd": 30, + "proj:shape": [8000, 8000], + "proj:code": "EPSG:32632", + "proj:transform": [30.0, 0.0, 399960.0, 0.0, -30.0, 5800020.0] + }, + "B2_30m": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Blue (band 2) - 30m", + "href": "s3://usgs-landsat/collection02/level-1/2026/192/026/LC08_L1TP_192026_20260724_20260724_02_RT/B2.TIF", + "bands": [{ "name": "B2", "eo:common_name": "blue" }], + "data_type": "uint16", + "gsd": 30, + "proj:shape": [8000, 8000], + "proj:code": "EPSG:32632", + "proj:transform": [30.0, 0.0, 399960.0, 0.0, -30.0, 5800020.0] + }, + "B3_30m": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Green (band 3) - 30m", + "href": "s3://usgs-landsat/collection02/level-1/2026/192/026/LC08_L1TP_192026_20260724_20260724_02_RT/B3.TIF", + "bands": [{ "name": "B3", "eo:common_name": "green" }], + "data_type": "uint16", + "gsd": 30, + "proj:shape": [8000, 8000], + "proj:code": "EPSG:32632", + "proj:transform": [30.0, 0.0, 399960.0, 0.0, -30.0, 5800020.0] + } + } + } + ], + "links": [], + "numberReturned": 1, + "numberMatched": 1 +} From 85cbed4ad8ef74b6ce0e5f5ebd704ba9b30ad3bf Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Tue, 28 Jul 2026 10:23:54 +0200 Subject: [PATCH 02/27] remove dynamic datasets --- .../src/cli/stac_harvester/harvest.rs | 135 +++--------------- 1 file changed, 23 insertions(+), 112 deletions(-) diff --git a/geoengine/services/src/cli/stac_harvester/harvest.rs b/geoengine/services/src/cli/stac_harvester/harvest.rs index 19fb812932..7c58e98de1 100644 --- a/geoengine/services/src/cli/stac_harvester/harvest.rs +++ b/geoengine/services/src/cli/stac_harvester/harvest.rs @@ -246,7 +246,6 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err ); let mut tiles_by_dataset: HashMap> = HashMap::new(); - let mut dynamic_datasets: HashMap = HashMap::new(); let mut items_processed: u64 = 0; let mut items_per_sec: f64; @@ -255,18 +254,12 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err let item_collection = result?; for item in &item_collection.items { - process_harvest_item_dynamic( - item, - provider_def, - &mut tiles_by_dataset, - &mut dynamic_datasets, - ¶ms, - ) - .unwrap_or_else(|e| { - if params.verbose { - warn!("Skipping item {}: {}", item.id, e); - } - }); + process_harvest_item(item, provider_def, &mut tiles_by_dataset, ¶ms) + .unwrap_or_else(|e| { + if params.verbose { + warn!("Skipping item {}: {}", item.id, e); + } + }); items_processed += 1; } @@ -308,35 +301,6 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err info!("Processed {} items total", items_processed); - // Create any dynamic (per-EPSG) datasets that were discovered during item processing - for (dyn_dataset_name, dyn_dataset) in &dynamic_datasets { - if !dataset_exists_api( - &client, - ¶ms.geo_engine_url, - &session_id, - dyn_dataset_name, - ) - .await? - { - if params.verbose { - info!("Creating dynamic dataset '{}'", dyn_dataset_name); - } - create_dataset_api( - &client, - ¶ms.geo_engine_url, - &session_id, - dyn_dataset_name, - dyn_dataset, - ¶ms.volume_name, - ) - .await?; - } - // Also ensure tiles_by_dataset has an entry for this dataset - tiles_by_dataset - .entry(dyn_dataset_name.clone()) - .or_default(); - } - for (dataset_name, tiles) in &tiles_by_dataset { if tiles.is_empty() { continue; @@ -395,16 +359,16 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err // Item Processing (Harvest) // --------------------------------------------------------------------------- -/// Like `process_harvest_item`, but handles items whose EPSG code differs from -/// the mapping's projection by dynamically creating per-EPSG dataset variants. -/// Items that pass the `--epsgs` filter but have a different EPSG than the -/// mapping will be grouped into separate datasets named with their actual EPSG. +/// Process a STAC item and add tiles to the appropriate dataset from the mapping. +/// +/// Only assets whose EPSG code matches the dataset's projection are included. +/// Items with a different EPSG are silently skipped — all datasets must be +/// predefined in the mapping. #[allow(clippy::too_many_lines)] -fn process_harvest_item_dynamic( +fn process_harvest_item( item: &stac::Item, provider_def: &StacDataProviderDefinition, tiles_by_dataset: &mut HashMap>, - dynamic_datasets: &mut HashMap, params: &StacHarvest, ) -> Result<(), anyhow::Error> { let Some(datetime) = item.properties.datetime else { @@ -465,43 +429,15 @@ fn process_harvest_item_dynamic( continue; }; - // Determine the actual projection and dataset name for this item - let (_actual_projection, actual_dataset_name) = if dataset.projection - == SpatialReference::new(SpatialReferenceAuthority::Epsg, item_epsg) + // Only process assets whose EPSG matches the dataset's projection + if dataset.projection + != SpatialReference::new(SpatialReferenceAuthority::Epsg, item_epsg) { - // EPSG matches the mapping — use the dataset as-is - ( - dataset.projection, - dataset_name_for_harvest(&provider_def.collection_name, dataset), - ) - } else { - // EPSG differs — create a per-EPSG variant - let per_epsg_projection = - SpatialReference::new(SpatialReferenceAuthority::Epsg, item_epsg); - - // Build a modified dataset with the item's EPSG - let per_epsg_dataset = StacProviderDataset { - projection: per_epsg_projection, - ..dataset.clone() - }; + continue; + } - let dyn_name = - dataset_name_for_harvest(&provider_def.collection_name, &per_epsg_dataset); - - // Register this dynamic dataset so it gets created - dynamic_datasets - .entry(dyn_name.clone()) - .or_insert_with(|| StacProviderDataset { - projection: SpatialReference::new( - SpatialReferenceAuthority::Epsg, - item_epsg, - ), - spatial_grid: dataset.spatial_grid, - ..dataset.clone() - }); - - (per_epsg_projection, dyn_name) - }; + let actual_dataset_name = + dataset_name_for_harvest(&provider_def.collection_name, dataset); let Some(geo_transform) = common::geo_transform_from_fields(&asset.additional_fields) else { @@ -1326,19 +1262,12 @@ mod tests { }; let mut tiles_by_dataset: HashMap> = HashMap::new(); - let mut dynamic_datasets: HashMap = HashMap::new(); // Process the first item from the fixture let item = &items.items[0]; - process_harvest_item_dynamic( - item, - &mapping, - &mut tiles_by_dataset, - &mut dynamic_datasets, - ¶ms, - ) - .expect("item processing should succeed"); + process_harvest_item(item, &mapping, &mut tiles_by_dataset, ¶ms) + .expect("item processing should succeed"); // The mapping has 2 datasets (10m and 20m), the item has assets for both assert_eq!( @@ -1370,12 +1299,6 @@ mod tests { } } - // Verify no dynamic (per-EPSG) datasets since all items match the mapping EPSG - assert!( - dynamic_datasets.is_empty(), - "no dynamic datasets should be needed when EPSGs match" - ); - // 10m dataset should have 4 tiles (B02, B03, B04, B08) let total_tiles_10m: usize = tiles_by_dataset .iter() @@ -1430,18 +1353,11 @@ mod tests { }; let mut tiles_by_dataset: HashMap> = HashMap::new(); - let mut dynamic_datasets: HashMap = HashMap::new(); let item = &items.items[0]; - process_harvest_item_dynamic( - item, - &mapping, - &mut tiles_by_dataset, - &mut dynamic_datasets, - ¶ms, - ) - .expect("Landsat item processing should succeed"); + process_harvest_item(item, &mapping, &mut tiles_by_dataset, ¶ms) + .expect("Landsat item processing should succeed"); // Mapping has 1 dataset (30m), the item has assets for it assert_eq!( @@ -1472,11 +1388,6 @@ mod tests { } } - assert!( - dynamic_datasets.is_empty(), - "no dynamic datasets should be needed when EPSGs match" - ); - // 30m dataset should have 4 tiles (Blue, Green, Red, NIR) let total_tiles_30m: usize = tiles_by_dataset .iter() From d3ddde0614f0597add4d1dd72e16d6666e192763 Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Tue, 28 Jul 2026 12:39:04 +0200 Subject: [PATCH 03/27] refactor --- .../src/cli/stac_harvester/discover.rs | 328 ++++++---- .../src/cli/stac_harvester/harvest.rs | 603 ++++++++++-------- 2 files changed, 527 insertions(+), 404 deletions(-) diff --git a/geoengine/services/src/cli/stac_harvester/discover.rs b/geoengine/services/src/cli/stac_harvester/discover.rs index b2aec57207..5d24ee9fe0 100644 --- a/geoengine/services/src/cli/stac_harvester/discover.rs +++ b/geoengine/services/src/cli/stac_harvester/discover.rs @@ -103,7 +103,6 @@ pub enum ImportFileType { // Discover Mapping Implementation // --------------------------------------------------------------------------- -#[allow(clippy::too_many_lines)] pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), anyhow::Error> { let client = reqwest::Client::new(); @@ -122,11 +121,104 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), .await .context("Failed to fetch STAC collection")?; - // Scan collection-level item_assets for bands (partial information) + let dataset_bands = scan_collection_bands(&collection, ¶ms.file_types); + + if params.verbose { + info!( + "Found {} data type/resolution combinations from collection metadata", + dataset_bands.len() + ); + } + + let items_response = fetch_sample_items(&client, ¶ms).await?; + + if items_response.items.is_empty() { + anyhow::bail!("No items found in the collection. Cannot discover mapping."); + } + + info!( + "Probing {} sample item(s) to discover EPSG codes and additional bands", + items_response.items.len() + ); + + let (discovered_datasets, sample_band_info) = + process_sample_assets(&items_response, ¶ms.file_types, ¶ms.epsgs); + + if discovered_datasets.is_empty() { + anyhow::bail!( + "No matching assets found in sample items. Check your --file-types and --epsgs filters." + ); + } + + let time_dimension = parse_time_dimension(¶ms.time_granularity, params.time_step) + .map_err(|e| anyhow::anyhow!("{e}"))?; + + let s3_config = params + .s3_endpoint + .as_ref() + .map(|endpoint| StacProviderS3Config { + endpoint: endpoint.clone(), + access_key: params.s3_access_key.clone(), + secret_key: params.s3_secret_key.clone(), + }); + + let datasets = build_datasets( + &discovered_datasets, + &dataset_bands, + &sample_band_info, + params.full_projection_grid, + ¶ms.stac_collection, + ); + + let provider_def = StacDataProviderDefinition { + name: format!("{} from STAC", params.stac_collection), + id: DataProviderId::new(), + description: format!( + "Auto-discovered mapping for STAC collection '{}' at {}", + params.stac_collection, params.stac_url + ), + priority: Some(50), + api_url: params.stac_url.clone(), + collection_name: params.stac_collection.clone(), + s3_config, + time_dimension, + datasets, + }; + + let json = serde_json::to_string_pretty(&provider_def) + .context("Failed to serialize mapping to JSON")?; + + if let Some(output_path) = ¶ms.output { + std::fs::write(output_path, &json) + .with_context(|| format!("Failed to write mapping to {}", output_path.display()))?; + println!("Mapping written to {}", output_path.display()); + } else { + println!("{json}"); + } + + Ok(()) +} + +struct DiscoveredDatasetInfo { + geo_transform: Option, + proj_shape: Option<(usize, usize)>, + srs: SpatialReference, + asset_count: u32, +} + +// --------------------------------------------------------------------------- +// Discover Helper Functions +// --------------------------------------------------------------------------- + +/// Scan the collection-level `item_assets` for band metadata (data type, resolution, band names). +fn scan_collection_bands( + collection: &stac::Collection, + file_types: &[ImportFileType], +) -> HashMap> { let mut dataset_bands: HashMap> = HashMap::new(); for (_asset_key, asset) in &collection.item_assets { - if !matches_selected_file_types_static(asset.r#type.as_deref(), ¶ms.file_types) { + if !matches_selected_file_types_static(asset.r#type.as_deref(), file_types) { continue; } @@ -137,14 +229,14 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), } } - if params.verbose { - info!( - "Found {} data type/resolution combinations from collection metadata", - dataset_bands.len() - ); - } + dataset_bands +} - // Sample items to discover EPSG codes and additional band/resolution info +/// Build query parameters and fetch sample items from the STAC items API. +async fn fetch_sample_items( + client: &reqwest::Client, + params: &StacDiscoverMapping, +) -> Result { let items_url = format!( "{}/collections/{}/items", params.stac_url.trim_end_matches('/'), @@ -169,20 +261,21 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), query_params.push(("limit".to_string(), format!("{}", params.sample_items))); - let items_response: stac::ItemCollection = - stac_api_request_with_params(&client, &items_url, &query_params) - .await - .context("Failed to fetch sample items")?; - - if items_response.items.is_empty() { - anyhow::bail!("No items found in the collection. Cannot discover mapping."); - } + stac_api_request_with_params(client, &items_url, &query_params) + .await + .context("Failed to fetch sample items") +} - info!( - "Probing {} sample item(s) to discover EPSG codes and additional bands", - items_response.items.len() - ); +/// Process sample items to discover unique datasets (by EPSG, data type, resolution) +/// and their associated bands. +type DiscoveredDatasets = HashMap; +type SampleBandInfo = HashMap>; +fn process_sample_assets( + items_response: &stac::ItemCollection, + file_types: &[ImportFileType], + epsgs: &[u32], +) -> (DiscoveredDatasets, SampleBandInfo) { let mut discovered_datasets: HashMap = HashMap::new(); let mut sample_band_info: HashMap> = HashMap::new(); @@ -190,7 +283,7 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), let item_epsg = common::epsg_code_from_item(item, common::StacExtensionMajorVersion::V2); for (asset_key, asset) in &item.assets { - if !matches_selected_file_types_static(asset.r#type.as_deref(), ¶ms.file_types) { + if !matches_selected_file_types_static(asset.r#type.as_deref(), file_types) { continue; } @@ -214,7 +307,7 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), continue; }; - if !params.epsgs.is_empty() && !params.epsgs.contains(&epsg) { + if !epsgs.is_empty() && !epsgs.contains(&epsg) { continue; } @@ -254,28 +347,21 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), } } - if discovered_datasets.is_empty() { - anyhow::bail!( - "No matching assets found in sample items. Check your --file-types and --epsgs filters." - ); - } - - // Build the StacDataProviderDefinition - let time_dimension = parse_time_dimension(¶ms.time_granularity, params.time_step) - .map_err(|e| anyhow::anyhow!("{e}"))?; - - let s3_config = params - .s3_endpoint - .as_ref() - .map(|endpoint| StacProviderS3Config { - endpoint: endpoint.clone(), - access_key: params.s3_access_key.clone(), - secret_key: params.s3_secret_key.clone(), - }); + (discovered_datasets, sample_band_info) +} +/// Build the `StacProviderDataset` list from discovered datasets, collection bands, +/// and sample band information. +fn build_datasets( + discovered_datasets: &HashMap, + dataset_bands: &HashMap>, + sample_band_info: &HashMap>, + full_projection_grid: bool, + stac_collection: &str, +) -> Vec { let mut datasets: Vec = Vec::new(); - for (dataset_key, info) in &discovered_datasets { + for (dataset_key, info) in discovered_datasets { let partial_key = PartialDatasetKey { data_type: dataset_key.data_type, resolution: dataset_key.resolution, @@ -312,70 +398,16 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), bands.sort_by(|a, b| a.asset_title.cmp(&b.asset_title)); - let spatial_grid = if params.full_projection_grid { - // Compute grid bounds covering the full projected CRS extent - if let Some(gt) = info.geo_transform { - let grid_bounds = projection_grid_bounds(gt, dataset_key.epsg) - .unwrap_or_else(|| { - // Fallback: use first asset's shape - if let Some((height, width)) = info.proj_shape { - GridBoundingBox2D::new( - GridIdx2D::new([0, 0]), - GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), - ) - .expect("fallback grid bounds should be valid") - } else { - GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) - .expect("zero-size grid bounds should be valid") - } - }); - GeoOpSpatialGridDescriptor::source_from_parts(gt, grid_bounds) - } else { - GeoOpSpatialGridDescriptor::source_from_parts( - GeoTransform::new( - (0.0, 0.0).into(), - dataset_key.resolution.into_inner(), - -dataset_key.resolution.into_inner(), - ), - GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) - .expect("zero-size grid bounds should be valid"), - ) - } - } else if let (Some(gt), Some((height, width))) = (info.geo_transform, info.proj_shape) { - GeoOpSpatialGridDescriptor::source_from_parts( - gt, - GridBoundingBox2D::new( - GridIdx2D::new([0, 0]), - GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), - ) - .unwrap_or_else(|_| { - GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) - .expect("zero-size grid bounds should be valid") - }), - ) - } else { - GeoOpSpatialGridDescriptor::source_from_parts( - GeoTransform::new( - (0.0, 0.0).into(), - dataset_key.resolution.into_inner(), - -dataset_key.resolution.into_inner(), - ), - GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) - .expect("zero-size grid bounds should be valid"), - ) - }; + let spatial_grid = build_dataset_spatial_grid(info, dataset_key, full_projection_grid); let dataset_name = format!( "{} EPSG:{} {:?} {}m", - params.stac_collection, dataset_key.epsg, dataset_key.data_type, dataset_key.resolution + stac_collection, dataset_key.epsg, dataset_key.data_type, dataset_key.resolution ); datasets.push(StacProviderDataset { name: dataset_name, - description: format!( - "Auto-discovered from STAC collection '{}'", - params.stac_collection - ), + description: format!("Auto-discovered from STAC collection '{stac_collection}'"), data_type: dataset_key.data_type, resolution: SpatialResolution::new_unchecked( dataset_key.resolution.into_inner(), @@ -387,40 +419,65 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), }); } - let provider_def = StacDataProviderDefinition { - name: format!("{} from STAC", params.stac_collection), - id: DataProviderId::new(), - description: format!( - "Auto-discovered mapping for STAC collection '{}' at {}", - params.stac_collection, params.stac_url - ), - priority: Some(50), - api_url: params.stac_url.clone(), - collection_name: params.stac_collection.clone(), - s3_config, - time_dimension, - datasets, - }; + datasets +} - let json = serde_json::to_string_pretty(&provider_def) - .context("Failed to serialize mapping to JSON")?; +/// Build the spatial grid descriptor for a discovered dataset, optionally using the +/// full projected CRS extent instead of the first asset's shape. +fn build_dataset_spatial_grid( + info: &DiscoveredDatasetInfo, + dataset_key: &DatasetKey, + full_projection_grid: bool, +) -> GeoOpSpatialGridDescriptor { + let default_grid = || { + GeoOpSpatialGridDescriptor::source_from_parts( + GeoTransform::new( + (0.0, 0.0).into(), + dataset_key.resolution.into_inner(), + -dataset_key.resolution.into_inner(), + ), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) + .expect("zero-size grid bounds should be valid"), + ) + }; - if let Some(output_path) = ¶ms.output { - std::fs::write(output_path, &json) - .with_context(|| format!("Failed to write mapping to {}", output_path.display()))?; - println!("Mapping written to {}", output_path.display()); + if full_projection_grid { + if let Some(gt) = info.geo_transform { + let grid_bounds = projection_grid_bounds(gt, dataset_key.epsg) + .unwrap_or_else(|| fallback_grid_bounds(info)); + GeoOpSpatialGridDescriptor::source_from_parts(gt, grid_bounds) + } else { + default_grid() + } + } else if let (Some(gt), Some((height, width))) = (info.geo_transform, info.proj_shape) { + GeoOpSpatialGridDescriptor::source_from_parts( + gt, + GridBoundingBox2D::new( + GridIdx2D::new([0, 0]), + GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), + ) + .unwrap_or_else(|_| { + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) + .expect("zero-size grid bounds should be valid") + }), + ) } else { - println!("{json}"); + default_grid() } - - Ok(()) } -struct DiscoveredDatasetInfo { - geo_transform: Option, - proj_shape: Option<(usize, usize)>, - srs: SpatialReference, - asset_count: u32, +/// Fallback grid bounds: use the first asset's shape, or a single-pixel grid. +fn fallback_grid_bounds(info: &DiscoveredDatasetInfo) -> GridBoundingBox2D { + if let Some((height, width)) = info.proj_shape { + GridBoundingBox2D::new( + GridIdx2D::new([0, 0]), + GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), + ) + .expect("fallback grid bounds should be valid") + } else { + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) + .expect("zero-size grid bounds should be valid") + } } /// Compute grid bounds that cover the full projected CRS extent for the given @@ -451,7 +508,11 @@ fn projection_grid_bounds(gt: GeoTransform, epsg: u32) -> Option Result { // Harvest Implementation // --------------------------------------------------------------------------- -#[allow(clippy::too_many_lines)] pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Error> { let start_time = Instant::now(); @@ -164,28 +165,7 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err ) .await?; - let mut created_datasets: Vec<(usize, StacProviderDataset)> = Vec::new(); - - for (idx, dataset) in provider_def.datasets.iter().enumerate() { - let dataset_name = dataset_name_for_harvest(&provider_def.collection_name, dataset); - - if params.verbose { - info!("Checking dataset '{}'", dataset_name); - } - - if !dataset_exists_api(&client, ¶ms.geo_engine_url, &session_id, &dataset_name).await? { - create_dataset_api( - &client, - ¶ms.geo_engine_url, - &session_id, - &dataset_name, - dataset, - ¶ms.volume_name, - ) - .await?; - created_datasets.push((idx, dataset.clone())); - } - } + let created_datasets = setup_datasets(&client, ¶ms, provider_def, &session_id).await?; info!( "Created {} new dataset(s) out of {}", @@ -199,145 +179,21 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err stac_api_url, provider_def.collection_name ); - let mut query_params: Vec<(String, String)> = Vec::new(); - - if let Some(bbox) = ¶ms.bbox - && bbox.len() == 4 - { - query_params.push(( - "bbox".to_string(), - format!("{},{},{},{}", bbox[0], bbox[1], bbox[2], bbox[3]), - )); - } - - if params.time_start.is_some() || params.time_end.is_some() { - query_params.push(( - "datetime".to_string(), - format!( - "{}/{}", - params.time_start.as_deref().unwrap_or(""), - params.time_end.as_deref().unwrap_or("") - ), - )); - } - - if let Some(limit) = params.limit { - query_params.push(("limit".to_string(), limit.to_string())); - } - - if params.filter_item_fields { - query_params.push(( - "fields".to_string(), - "stac_version,properties.datetime,properties.updated,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.proj:code,assets.*.proj:shape,assets.*.proj:transform" - .to_string(), - )); - } - - let initial_query_state = QueryState::FirstPage { - query_url: items_url.clone(), - query_params, - }; - - let page_stream = create_page_stream( - initial_query_state, - client.clone(), - params.verbose, - params.prefetch_pages, - ); - - let mut tiles_by_dataset: HashMap> = HashMap::new(); - let mut items_processed: u64 = 0; - let mut items_per_sec: f64; - - futures::pin_mut!(page_stream); - while let Some(result) = page_stream.next().await { - let item_collection = result?; + let query_params = build_stac_query_params(¶ms); - for item in &item_collection.items { - process_harvest_item(item, provider_def, &mut tiles_by_dataset, ¶ms) - .unwrap_or_else(|e| { - if params.verbose { - warn!("Skipping item {}: {}", item.id, e); - } - }); - - items_processed += 1; - } - - if params.verbose { - let elapsed = start_time.elapsed().as_secs_f64(); - items_per_sec = if elapsed > 0.0 { - items_processed as f64 / elapsed - } else { - 0.0 - }; - - if let Some(number_matched) = item_collection - .additional_fields - .get("numberMatched") - .and_then(serde_json::Value::as_u64) - { - let progress = - (items_processed as f64 / number_matched as f64 * 100.0).clamp(0.0, 100.0); - let remaining = number_matched.saturating_sub(items_processed); - let eta_secs = if items_per_sec > 0.0 { - remaining as f64 / items_per_sec - } else { - f64::INFINITY - }; - let eta_str = if eta_secs.is_finite() { - format_duration(eta_secs as u64) - } else { - "unknown".to_string() - }; - println!( - "[{progress:.1}%] Processed {items_processed}/{number_matched} items ({items_per_sec:.1} items/s, ETA: {eta_str})" - ); - } else { - println!("Processed {items_processed} items ({items_per_sec:.1} items/s)"); - } - } - } + let (tiles_by_dataset, items_processed) = process_item_stream( + &client, + &items_url, + &query_params, + provider_def, + ¶ms, + &start_time, + ) + .await?; info!("Processed {} items total", items_processed); - for (dataset_name, tiles) in &tiles_by_dataset { - if tiles.is_empty() { - continue; - } - - if params.verbose { - info!("Adding {} tiles to dataset '{}'", tiles.len(), dataset_name); - } - - let batch_size = 100; - for chunk in tiles.chunks(batch_size) { - let response = retry_http( - || async { - client - .post(format!( - "{}/dataset/{}/tiles", - params.geo_engine_url, dataset_name - )) - .header("Content-Type", "application/json") - .header("Authorization", format!("Bearer {session_id}")) - .json(chunk) - .send() - .await - }, - &format!("Add tiles to dataset '{dataset_name}'"), - ) - .await - .with_context(|| format!("Failed to add tiles to dataset '{dataset_name}'"))?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - warn!("Failed to add tiles to dataset '{dataset_name}' (HTTP {status}): {body}"); - // Continue with remaining tiles; some conflicts (e.g. z-index) are expected - } - } - } + upload_tiles_to_datasets(&client, ¶ms, &session_id, &tiles_by_dataset).await?; create_harvest_layer_collections( &client, @@ -364,7 +220,6 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err /// Only assets whose EPSG code matches the dataset's projection are included. /// Items with a different EPSG are silently skipped — all datasets must be /// predefined in the mapping. -#[allow(clippy::too_many_lines)] fn process_harvest_item( item: &stac::Item, provider_def: &StacDataProviderDefinition, @@ -396,119 +251,17 @@ fn process_harvest_item( for dataset in &provider_def.datasets { for (band_idx, band_def) in dataset.bands.iter().enumerate() { - let Some((_asset_key, asset)) = item - .assets - .iter() - .find(|(_, a)| a.title.as_deref() == Some(&band_def.asset_title)) - else { - continue; - }; - - // Check data type matches - if let Some(asset_dt) = data_type_from_asset_v1_1_0_fallback(asset) - && asset_dt != dataset.data_type - { - continue; - } - - // Extract the item's actual EPSG code from the asset - let item_epsg = common::epsg_code_from_fields( - common::StacExtensionMajorVersion::V2, - &asset.additional_fields, - ) - .or_else(|| { - // Also try to extract from serialized properties as fallback - let props_val = serde_json::to_value(&item.properties) - .ok() - .and_then(|v| v.as_object().cloned()) - .unwrap_or_default(); - common::epsg_code_from_fields(common::StacExtensionMajorVersion::V2, &props_val) - }); - - let Some(item_epsg) = item_epsg else { - continue; - }; - - // Only process assets whose EPSG matches the dataset's projection - if dataset.projection - != SpatialReference::new(SpatialReferenceAuthority::Epsg, item_epsg) - { - continue; - } - - let actual_dataset_name = - dataset_name_for_harvest(&provider_def.collection_name, dataset); - - let Some(geo_transform) = common::geo_transform_from_fields(&asset.additional_fields) - else { - continue; - }; - - if (geo_transform.x_pixel_size().abs() - dataset.resolution.x).abs() > 1e-9 - || (geo_transform.y_pixel_size().abs() - dataset.resolution.y).abs() > 1e-9 - { - continue; + if let Some((dataset_name, tile)) = try_create_tile_for_band( + item, + dataset, + band_idx, + band_def, + (time, z_index), + provider_def, + params, + ) { + tiles_by_dataset.entry(dataset_name).or_default().push(tile); } - - let Some((height, width)) = common::proj_shape_from_fields(&asset.additional_fields) - else { - continue; - }; - - let Some(rasterband_channel) = - common::rasterband_channel_for_dataset_band(asset, band_def.band_name.as_deref()) - else { - continue; - }; - - let grid_bounds = GridBoundingBox2D::new( - GridIdx2D::new([0, 0]), - GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), - ) - .context("Failed to create grid bounds")?; - - let spatial_partition = geo_transform.grid_to_spatial_bounds(&grid_bounds); - - let Some(file_path) = common::gdal_file_path(&asset.href) else { - continue; - }; - - let gdal_config_options = common::gdal_config_options_for_file_path( - &file_path, - provider_def.s3_config.as_ref(), - ); - - let tile = AddDatasetTile { - time: TimeInterval::new(time, time + i64::from(24 * 60 * 60 * 1000)) - .context("Failed to create time interval")? - .into(), - spatial_partition: spatial_partition.into(), - band: band_idx as u32, - z_index, - params: GdalDatasetParameters { - file_path, - rasterband_channel, - geo_transform: geo_transform.into(), - width, - height, - file_not_found_handling: - crate::api::model::operators::FileNotFoundHandling::Error, - no_data_value: params.no_data_value, - properties_mapping: None, - gdal_open_options: None, - gdal_config_options: gdal_config_options.map(|opts| { - opts.into_iter() - .map(|(k, v)| GdalConfigOption::from((k, v))) - .collect() - }), - allow_alphaband_as_mask: false, - }, - }; - - tiles_by_dataset - .entry(actual_dataset_name) - .or_default() - .push(tile); } } @@ -1190,6 +943,312 @@ fn data_type_from_asset_v1_1_0_fallback(asset: &stac::Asset) -> Option Result, anyhow::Error> { + let mut created_datasets: Vec<(usize, StacProviderDataset)> = Vec::new(); + + for (idx, dataset) in provider_def.datasets.iter().enumerate() { + let dataset_name = dataset_name_for_harvest(&provider_def.collection_name, dataset); + + if params.verbose { + info!("Checking dataset '{}'", dataset_name); + } + + if !dataset_exists_api(client, ¶ms.geo_engine_url, session_id, &dataset_name).await? { + create_dataset_api( + client, + ¶ms.geo_engine_url, + session_id, + &dataset_name, + dataset, + ¶ms.volume_name, + ) + .await?; + created_datasets.push((idx, dataset.clone())); + } + } + + Ok(created_datasets) +} + +/// Build the query parameters for the STAC items API request. +fn build_stac_query_params(params: &StacHarvest) -> Vec<(String, String)> { + let mut query_params: Vec<(String, String)> = Vec::new(); + + if let Some(bbox) = ¶ms.bbox + && bbox.len() == 4 + { + query_params.push(( + "bbox".to_string(), + format!("{},{},{},{}", bbox[0], bbox[1], bbox[2], bbox[3]), + )); + } + + if params.time_start.is_some() || params.time_end.is_some() { + query_params.push(( + "datetime".to_string(), + format!( + "{}/{}", + params.time_start.as_deref().unwrap_or(""), + params.time_end.as_deref().unwrap_or("") + ), + )); + } + + if let Some(limit) = params.limit { + query_params.push(("limit".to_string(), limit.to_string())); + } + + if params.filter_item_fields { + query_params.push(( + "fields".to_string(), + "stac_version,properties.datetime,properties.updated,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.proj:code,assets.*.proj:shape,assets.*.proj:transform" + .to_string(), + )); + } + + query_params +} + +/// Fetch pages of STAC items, process them, and collect tiles grouped by dataset. +async fn process_item_stream( + client: &reqwest::Client, + items_url: &str, + query_params: &[(String, String)], + provider_def: &StacDataProviderDefinition, + params: &StacHarvest, + start_time: &Instant, +) -> Result<(HashMap>, u64), anyhow::Error> { + let initial_query_state = QueryState::FirstPage { + query_url: items_url.to_string(), + query_params: query_params.to_vec(), + }; + + let page_stream = create_page_stream( + initial_query_state, + client.clone(), + params.verbose, + params.prefetch_pages, + ); + + let mut tiles_by_dataset: HashMap> = HashMap::new(); + let mut items_processed: u64 = 0; + let mut items_per_sec: f64; + + futures::pin_mut!(page_stream); + while let Some(result) = page_stream.next().await { + let item_collection = result?; + + for item in &item_collection.items { + process_harvest_item(item, provider_def, &mut tiles_by_dataset, params).unwrap_or_else( + |e| { + if params.verbose { + warn!("Skipping item {}: {}", item.id, e); + } + }, + ); + + items_processed += 1; + } + + if params.verbose { + let elapsed = start_time.elapsed().as_secs_f64(); + items_per_sec = if elapsed > 0.0 { + items_processed as f64 / elapsed + } else { + 0.0 + }; + + if let Some(number_matched) = item_collection + .additional_fields + .get("numberMatched") + .and_then(serde_json::Value::as_u64) + { + let progress = + (items_processed as f64 / number_matched as f64 * 100.0).clamp(0.0, 100.0); + let remaining = number_matched.saturating_sub(items_processed); + let eta_secs = if items_per_sec > 0.0 { + remaining as f64 / items_per_sec + } else { + f64::INFINITY + }; + let eta_str = if eta_secs.is_finite() { + format_duration(eta_secs as u64) + } else { + "unknown".to_string() + }; + println!( + "[{progress:.1}%] Processed {items_processed}/{number_matched} items ({items_per_sec:.1} items/s, ETA: {eta_str})" + ); + } else { + println!("Processed {items_processed} items ({items_per_sec:.1} items/s)"); + } + } + } + + Ok((tiles_by_dataset, items_processed)) +} + +/// Upload collected tiles to the Geo Engine server in batches. +async fn upload_tiles_to_datasets( + client: &reqwest::Client, + params: &StacHarvest, + session_id: &str, + tiles_by_dataset: &HashMap>, +) -> Result<(), anyhow::Error> { + for (dataset_name, tiles) in tiles_by_dataset { + if tiles.is_empty() { + continue; + } + + if params.verbose { + info!("Adding {} tiles to dataset '{}'", tiles.len(), dataset_name); + } + + let batch_size = 100; + for chunk in tiles.chunks(batch_size) { + let response = retry_http( + || async { + client + .post(format!( + "{}/dataset/{}/tiles", + params.geo_engine_url, dataset_name + )) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(chunk) + .send() + .await + }, + &format!("Add tiles to dataset '{dataset_name}'"), + ) + .await + .with_context(|| format!("Failed to add tiles to dataset '{dataset_name}'"))?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + warn!("Failed to add tiles to dataset '{dataset_name}' (HTTP {status}): {body}"); + // Continue with remaining tiles; some conflicts (e.g. z-index) are expected + } + } + } + + Ok(()) +} + +/// Try to create a tile for a single band of a single dataset from a STAC item asset. +/// +/// Returns `None` if no matching asset exists, the data types don't match, the EPSG code +/// doesn't match the dataset's projection, the resolution doesn't match, or any required +/// metadata is missing. +fn try_create_tile_for_band( + item: &stac::Item, + dataset: &StacProviderDataset, + band_idx: usize, + band_def: &StacProviderDatasetBand, + item_time: (TimeInstance, i64), // (time, z_index) + provider_def: &StacDataProviderDefinition, + params: &StacHarvest, +) -> Option<(String, AddDatasetTile)> { + let (time, z_index) = item_time; + let (_asset_key, asset) = item + .assets + .iter() + .find(|(_, a)| a.title.as_deref() == Some(&band_def.asset_title))?; + + // Check data type matches + if let Some(asset_dt) = data_type_from_asset_v1_1_0_fallback(asset) + && asset_dt != dataset.data_type + { + return None; + } + + // Extract the item's actual EPSG code from the asset + let item_epsg = common::epsg_code_from_fields( + common::StacExtensionMajorVersion::V2, + &asset.additional_fields, + ) + .or_else(|| { + // Also try to extract from serialized properties as fallback + let props_val = serde_json::to_value(&item.properties) + .ok() + .and_then(|v| v.as_object().cloned()) + .unwrap_or_default(); + common::epsg_code_from_fields(common::StacExtensionMajorVersion::V2, &props_val) + })?; + + // Only process assets whose EPSG matches the dataset's projection + if dataset.projection != SpatialReference::new(SpatialReferenceAuthority::Epsg, item_epsg) { + return None; + } + + let actual_dataset_name = dataset_name_for_harvest(&provider_def.collection_name, dataset); + + let geo_transform = common::geo_transform_from_fields(&asset.additional_fields)?; + + if (geo_transform.x_pixel_size().abs() - dataset.resolution.x).abs() > 1e-9 + || (geo_transform.y_pixel_size().abs() - dataset.resolution.y).abs() > 1e-9 + { + return None; + } + + let (height, width) = common::proj_shape_from_fields(&asset.additional_fields)?; + + let rasterband_channel = + common::rasterband_channel_for_dataset_band(asset, band_def.band_name.as_deref())?; + + let grid_bounds = GridBoundingBox2D::new( + GridIdx2D::new([0, 0]), + GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), + ) + .ok()?; + + let spatial_partition = geo_transform.grid_to_spatial_bounds(&grid_bounds); + + let file_path = common::gdal_file_path(&asset.href)?; + + let gdal_config_options = + common::gdal_config_options_for_file_path(&file_path, provider_def.s3_config.as_ref()); + + let tile = AddDatasetTile { + time: TimeInterval::new(time, time + i64::from(24 * 60 * 60 * 1000)) + .ok()? + .into(), + spatial_partition: spatial_partition.into(), + band: band_idx as u32, + z_index, + params: GdalDatasetParameters { + file_path, + rasterband_channel, + geo_transform: geo_transform.into(), + width, + height, + file_not_found_handling: crate::api::model::operators::FileNotFoundHandling::Error, + no_data_value: params.no_data_value, + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: gdal_config_options.map(|opts| { + opts.into_iter() + .map(|(k, v)| GdalConfigOption::from((k, v))) + .collect() + }), + allow_alphaband_as_mask: false, + }, + }; + + Some((actual_dataset_name, tile)) +} + fn format_duration(secs: u64) -> String { if secs < 60 { format!("{secs}s") From c2cf5762334aa816b363c55ad88ba5b321d3d527 Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Tue, 28 Jul 2026 12:55:10 +0200 Subject: [PATCH 04/27] refactor --- .../src/cli/stac_harvester/discover.rs | 76 +++++++++---------- .../src/cli/stac_harvester/harvest.rs | 15 +--- .../src/datasets/external/stac/common.rs | 30 ++++++++ 3 files changed, 70 insertions(+), 51 deletions(-) diff --git a/geoengine/services/src/cli/stac_harvester/discover.rs b/geoengine/services/src/cli/stac_harvester/discover.rs index 5d24ee9fe0..cb60d1d037 100644 --- a/geoengine/services/src/cli/stac_harvester/discover.rs +++ b/geoengine/services/src/cli/stac_harvester/discover.rs @@ -223,7 +223,7 @@ fn scan_collection_bands( } if let Ok(Some(bands)) = - scan_item_asset_common(&collection.version, asset, collection.summaries.as_ref()) + scan_collection_item_asset(&collection.version, asset, collection.summaries.as_ref()) { merge_dataset_bands(&mut dataset_bands, bands); } @@ -293,7 +293,7 @@ fn process_sample_assets( }; let data_type = common::data_type_from_asset_v1_1_0(asset) - .or_else(|| data_type_from_asset_v1_0_0_fallback(asset)); + .or_else(|| common::data_type_from_asset_v1_0_0_fallback(asset)); let Some(data_type) = data_type else { continue; }; @@ -436,8 +436,7 @@ fn build_dataset_spatial_grid( dataset_key.resolution.into_inner(), -dataset_key.resolution.into_inner(), ), - GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) - .expect("zero-size grid bounds should be valid"), + zero_size_grid(), ) }; @@ -450,17 +449,8 @@ fn build_dataset_spatial_grid( default_grid() } } else if let (Some(gt), Some((height, width))) = (info.geo_transform, info.proj_shape) { - GeoOpSpatialGridDescriptor::source_from_parts( - gt, - GridBoundingBox2D::new( - GridIdx2D::new([0, 0]), - GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), - ) - .unwrap_or_else(|_| { - GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) - .expect("zero-size grid bounds should be valid") - }), - ) + let grid_bounds = asset_shape_bounds(height, width).unwrap_or_else(|()| zero_size_grid()); + GeoOpSpatialGridDescriptor::source_from_parts(gt, grid_bounds) } else { default_grid() } @@ -469,17 +459,31 @@ fn build_dataset_spatial_grid( /// Fallback grid bounds: use the first asset's shape, or a single-pixel grid. fn fallback_grid_bounds(info: &DiscoveredDatasetInfo) -> GridBoundingBox2D { if let Some((height, width)) = info.proj_shape { - GridBoundingBox2D::new( - GridIdx2D::new([0, 0]), - GridIdx2D::new([(width as isize) - 1, (height as isize) - 1]), - ) - .expect("fallback grid bounds should be valid") + asset_shape_bounds(height, width).unwrap_or_else(|()| zero_size_grid()) } else { - GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) - .expect("zero-size grid bounds should be valid") + zero_size_grid() } } +/// Build grid bounds from an asset's `(height, width)` shape. +/// Returns `Err` if dimensions are zero (causing negative indices). +fn asset_shape_bounds(height: usize, width: usize) -> Result { + GridBoundingBox2D::new( + GridIdx2D::new([0, 0]), + GridIdx2D::new([ + width.saturating_sub(1) as isize, + height.saturating_sub(1) as isize, + ]), + ) + .map_err(|_| ()) +} + +/// A single-pixel grid at the origin, used as a safe fallback when no shape info is available. +fn zero_size_grid() -> GridBoundingBox2D { + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])) + .expect("zero-size grid bounds should always be valid") +} + /// Compute grid bounds that cover the full projected CRS extent for the given /// geo-transform and EPSG code. Currently handles UTM projections (zones 32601–32660 /// and 32701–32760) with known extents. Returns `None` for unsupported CRS types. @@ -548,19 +552,24 @@ struct PartialDatasetKey { // Collection scanning helpers // --------------------------------------------------------------------------- -fn scan_item_asset_common( +fn scan_collection_item_asset( collection_version: &stac::Version, asset: &stac::ItemAsset, collection_summaries: Option<&serde_json::Map>, ) -> Result>>, String> { match collection_version { - stac::Version::v1_0_0 => scan_item_asset_v1_0_0_common(asset), - stac::Version::v1_1_0 => scan_item_asset_v1_1_0_common(asset, collection_summaries), - _ => Err(format!("Unsupported STAC version: {collection_version}")), + stac::Version::v1_0_0 => scan_collection_item_asset_v1_0_0(asset), + stac::Version::v1_1_0 => scan_collection_item_asset_v1_1_0(asset, collection_summaries), + _ => { + // For unknown STAC versions, try v1.1.0 first (more common), fall back to v1.0.0 + scan_collection_item_asset_v1_1_0(asset, collection_summaries) + .or_else(|_| scan_collection_item_asset_v1_0_0(asset)) + .or(Ok(None)) + } } } -fn scan_item_asset_v1_0_0_common( +fn scan_collection_item_asset_v1_0_0( asset: &stac::ItemAsset, ) -> Result>>, String> { let mut dataset_bands: HashMap> = HashMap::new(); @@ -625,7 +634,7 @@ fn scan_item_asset_v1_0_0_common( Ok(Some(dataset_bands)) } -fn scan_item_asset_v1_1_0_common( +fn scan_collection_item_asset_v1_1_0( asset: &stac::ItemAsset, collection_summaries: Option<&serde_json::Map>, ) -> Result>>, String> { @@ -678,17 +687,6 @@ fn scan_item_asset_v1_1_0_common( Ok(Some(dataset_bands)) } -fn data_type_from_asset_v1_0_0_fallback(asset: &stac::Asset) -> Option { - asset - .additional_fields - .get("raster:bands") - .and_then(|v| v.as_array()) - .and_then(|bands| bands.first()) - .and_then(|band| band.get("data_type")) - .and_then(|v| v.as_str()) - .and_then(common::raster_data_type_from_stac_data_type_str) -} - fn matches_selected_file_types_static( media_type: Option<&str>, file_types: &[ImportFileType], diff --git a/geoengine/services/src/cli/stac_harvester/harvest.rs b/geoengine/services/src/cli/stac_harvester/harvest.rs index 112af0ad81..0c1767e578 100644 --- a/geoengine/services/src/cli/stac_harvester/harvest.rs +++ b/geoengine/services/src/cli/stac_harvester/harvest.rs @@ -11,7 +11,7 @@ use futures::StreamExt; use geoengine_datatypes::{ dataset::NamedData, primitives::{DateTime, TimeInstance, TimeInterval}, - raster::{GeoTransform, GridBoundingBox2D, GridIdx2D, RasterDataType}, + raster::{GeoTransform, GridBoundingBox2D, GridIdx2D}, spatial_reference::{SpatialReference, SpatialReferenceAuthority, SpatialReferenceOption}, }; use tracing::{debug, error, info, warn}; @@ -933,16 +933,6 @@ async fn login_geo_engine( Ok((client, session_id)) } -fn data_type_from_asset_v1_1_0_fallback(asset: &stac::Asset) -> Option { - common::data_type_from_asset_v1_1_0(asset).or_else(|| { - asset - .additional_fields - .get("data_type") - .and_then(|v| v.as_str()) - .and_then(common::raster_data_type_from_stac_data_type_str) - }) -} - // --------------------------------------------------------------------------- // Harvest Helper Functions // --------------------------------------------------------------------------- @@ -1167,7 +1157,7 @@ fn try_create_tile_for_band( .find(|(_, a)| a.title.as_deref() == Some(&band_def.asset_title))?; // Check data type matches - if let Some(asset_dt) = data_type_from_asset_v1_1_0_fallback(asset) + if let Some(asset_dt) = common::data_type_from_asset_v1_1_0_fallback(asset) && asset_dt != dataset.data_type { return None; @@ -1267,6 +1257,7 @@ fn format_duration(secs: u64) -> String { mod tests { use super::*; use geoengine_datatypes::primitives::SpatialResolution; + use geoengine_datatypes::raster::RasterDataType; #[test] fn test_dataset_name_for_harvest() { diff --git a/geoengine/services/src/datasets/external/stac/common.rs b/geoengine/services/src/datasets/external/stac/common.rs index 89a47f905a..ed175ef630 100644 --- a/geoengine/services/src/datasets/external/stac/common.rs +++ b/geoengine/services/src/datasets/external/stac/common.rs @@ -219,6 +219,36 @@ pub fn data_type_from_asset_v1_1_0(asset: &stac::Asset) -> Option Option { + data_type_from_asset_v1_1_0(asset).or_else(|| { + asset + .additional_fields + .get("data_type") + .and_then(|v| v.as_str()) + .and_then(raster_data_type_from_stac_data_type_str) + }) +} + +/// Extract data type from a STAC 1.0.0 asset, reading from `additional_fields["raster:bands"][0]["data_type"]`. +/// +/// STAC 1.0.0 stores data types inside `raster:bands[]` arrays rather than directly on the asset. +/// This function reads the first band's `data_type` as a raw string. +pub fn data_type_from_asset_v1_0_0_fallback(asset: &stac::Asset) -> Option { + asset + .additional_fields + .get("raster:bands") + .and_then(|v| v.as_array()) + .and_then(|bands| bands.first()) + .and_then(|band| band.get("data_type")) + .and_then(|v| v.as_str()) + .and_then(raster_data_type_from_stac_data_type_str) +} + // --------------------------------------------------------------------------- // File path helpers // --------------------------------------------------------------------------- From c9f8a06ddc8a929ce5b356f13ee3749e755b0ca4 Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Wed, 29 Jul 2026 16:07:08 +0200 Subject: [PATCH 05/27] add page limit --- geoengine/services/src/api/model/services.rs | 3 + .../src/cli/stac_harvester/discover.rs | 94 +++++-- .../src/cli/stac_harvester/harvest.rs | 235 +++++++++--------- .../contexts/migrations/current_schema.sql | 1 + .../migration_0028_stac_provider.sql | 1 + .../src/datasets/external/stac/listing.rs | 2 + .../datasets/external/stac/loading_info.rs | 13 +- .../src/datasets/external/stac/mod.rs | 6 +- .../provider_defs_api/stac_sentinel2.json | 3 +- .../expected-mapping-code-de.json | 43 ++-- .../expected-mapping-landsat-c2-l1.json | 29 +-- 11 files changed, 244 insertions(+), 186 deletions(-) diff --git a/geoengine/services/src/api/model/services.rs b/geoengine/services/src/api/model/services.rs index 37e63c4b9d..922fa36342 100644 --- a/geoengine/services/src/api/model/services.rs +++ b/geoengine/services/src/api/model/services.rs @@ -1122,6 +1122,7 @@ pub struct StacDataProviderDefinition { pub s3_config: Option, pub time_dimension: TimeDimension, pub datasets: Vec, + pub page_limit: i64, /// Timeout in seconds for outgoing STAC API HTTP requests. #[serde(default = "default_query_timeout")] pub query_timeout_secs: i64, @@ -1145,6 +1146,7 @@ impl From s3_config: value.s3_config.map(Into::into), time_dimension: api_time_dimension_to_datatypes(value.time_dimension), datasets: value.datasets.into_iter().map(Into::into).collect(), + page_limit: value.page_limit, query_timeout_secs: value.query_timeout_secs, } } @@ -1165,6 +1167,7 @@ impl From s3_config: value.s3_config.map(Into::into), time_dimension: datatypes_time_dimension_to_api(value.time_dimension), datasets: value.datasets.into_iter().map(Into::into).collect(), + page_limit: value.page_limit, query_timeout_secs: value.query_timeout_secs, } } diff --git a/geoengine/services/src/cli/stac_harvester/discover.rs b/geoengine/services/src/cli/stac_harvester/discover.rs index cb60d1d037..dc35b68e0e 100644 --- a/geoengine/services/src/cli/stac_harvester/discover.rs +++ b/geoengine/services/src/cli/stac_harvester/discover.rs @@ -54,6 +54,10 @@ pub struct StacDiscoverMapping { #[arg(long, default_value_t = 5)] pub sample_items: usize, + /// Page size for querying items from the STAC server (default: 100) + #[arg(long, default_value_t = 100)] + pub page_limit: usize, + /// Output file for the mapping JSON (default: stdout) #[arg(long)] pub output: Option, @@ -183,10 +187,13 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), s3_config, time_dimension, datasets, + page_limit: params.page_limit as i64, }; - let json = serde_json::to_string_pretty(&provider_def) - .context("Failed to serialize mapping to JSON")?; + let json = serde_json::to_string_pretty( + &crate::api::model::services::StacDataProviderDefinition::from(provider_def), + ) + .context("Failed to serialize mapping to JSON")?; if let Some(output_path) = ¶ms.output { std::fs::write(output_path, &json) @@ -218,7 +225,7 @@ fn scan_collection_bands( let mut dataset_bands: HashMap> = HashMap::new(); for (_asset_key, asset) in &collection.item_assets { - if !matches_selected_file_types_static(asset.r#type.as_deref(), file_types) { + if !matches_selected_file_types(asset.r#type.as_deref(), file_types) { continue; } @@ -233,11 +240,13 @@ fn scan_collection_bands( } /// Build query parameters and fetch sample items from the STAC items API. +/// Follows pagination links until `sample_items` items are collected or no +/// more pages are available. async fn fetch_sample_items( client: &reqwest::Client, params: &StacDiscoverMapping, ) -> Result { - let items_url = format!( + let base_url = format!( "{}/collections/{}/items", params.stac_url.trim_end_matches('/'), params.stac_collection @@ -259,11 +268,42 @@ async fn fetch_sample_items( )); } - query_params.push(("limit".to_string(), format!("{}", params.sample_items))); + query_params.push(("limit".to_string(), params.page_limit.to_string())); - stac_api_request_with_params(client, &items_url, &query_params) - .await - .context("Failed to fetch sample items") + let mut all_items = Vec::new(); + let mut next_url: Option = None; + + loop { + let page = if let Some(ref url) = next_url { + stac_api_request_parse::(client, url) + .await + .context("Failed to fetch sample items page")? + } else { + stac_api_request_with_params(client, &base_url, &query_params) + .await + .context("Failed to fetch sample items")? + }; + + all_items.extend(page.items); + + if all_items.len() >= params.sample_items { + all_items.truncate(params.sample_items); + break; + } + + // Follow the `next` link if available + next_url = page + .links + .iter() + .find(|link| link.rel == "next") + .map(|link| link.href.clone()); + + if next_url.is_none() { + break; + } + } + + Ok(all_items.into()) } /// Process sample items to discover unique datasets (by EPSG, data type, resolution) @@ -283,7 +323,7 @@ fn process_sample_assets( let item_epsg = common::epsg_code_from_item(item, common::StacExtensionMajorVersion::V2); for (asset_key, asset) in &item.assets { - if !matches_selected_file_types_static(asset.r#type.as_deref(), file_types) { + if !matches_selected_file_types(asset.r#type.as_deref(), file_types) { continue; } @@ -519,18 +559,26 @@ fn projection_grid_bounds(gt: GeoTransform, epsg: u32) -> Option Option<(f64, f64, f64, f64)> { - // UTM northern hemisphere zones EPSG:32601 – 32660 - if (32601..=32660).contains(&epsg) { - return Some((0.0, 1_000_000.0, 0.0, 10_000_000.0)); - } - // UTM southern hemisphere zones EPSG:32701 – 32760 - if (32701..=32760).contains(&epsg) { - return Some((0.0, 1_000_000.0, 0.0, 10_000_000.0)); - } - None + use proj::Proj; + + let proj_crs = Proj::new(&format!("EPSG:{epsg}")).ok()?; + + // Get area of use in degrees (WGS84) + let (area, _name) = proj_crs.area_of_use().ok()?; + let area = area?; + + // Project the WGS84 bounding box into the target CRS + let pipeline = Proj::new_known_crs("EPSG:4326", &format!("EPSG:{epsg}"), None).ok()?; + let bounds = pipeline + .transform_bounds(area.west, area.south, area.east, area.north, 21) + .ok()?; + + // transform_bounds returns [west, south, east, north] in the target CRS + Some((bounds[0], bounds[2], bounds[1], bounds[3])) } /// A key that uniquely identifies a Geo Engine dataset derived from STAC assets. @@ -687,7 +735,7 @@ fn scan_collection_item_asset_v1_1_0( Ok(Some(dataset_bands)) } -fn matches_selected_file_types_static( +fn matches_selected_file_types( media_type: Option<&str>, file_types: &[ImportFileType], ) -> bool { @@ -850,6 +898,7 @@ mod tests { verbose: false, time_granularity: "days".to_string(), time_step: 1, + page_limit: 100, }; discover_mapping(params) @@ -863,7 +912,7 @@ mod tests { let mut output_json: serde_json::Value = serde_json::from_str(&output_content).expect("output should be valid JSON"); - // Normalize dynamic fields before comparison + // Normalize variable fields before comparison let output_obj = output_json.as_object_mut().unwrap(); output_obj.remove("id"); output_obj.insert( @@ -960,6 +1009,7 @@ mod tests { verbose: false, time_granularity: "days".to_string(), time_step: 1, + page_limit: 100, }; discover_mapping(params) diff --git a/geoengine/services/src/cli/stac_harvester/harvest.rs b/geoengine/services/src/cli/stac_harvester/harvest.rs index 0c1767e578..9b57c259bf 100644 --- a/geoengine/services/src/cli/stac_harvester/harvest.rs +++ b/geoengine/services/src/cli/stac_harvester/harvest.rs @@ -40,7 +40,7 @@ use crate::{ RasterBandDescriptors, RasterResultDescriptor, RegularTimeDimension, SpatialGridDescriptor, SpatialGridDescriptorState, TimeDescriptor, TimeDimension, }, - responses::{ErrorResponse, IdResponse}, + responses::IdResponse, services::{ AddDataset, CreateDataset, DataPath, DatasetDefinition, MetaDataDefinition, }, @@ -59,6 +59,7 @@ use geoengine_operators::{ engine::{RasterOperator, TypedOperator}, source::{MultiBandGdalSource, MultiBandGdalSourceParameters}, }; +use geoengine_api_client::apis::configuration::Configuration as ApiConfig; // --------------------------------------------------------------------------- // Harvest @@ -83,10 +84,6 @@ pub struct StacHarvest { #[clap(short, long, value_parser, num_args = 1.., value_delimiter = ' ')] pub bbox: Option>, - /// Import limit (page size) - #[arg(long)] - pub limit: Option, - /// Geo Engine API URL #[arg(long, default_value = "http://localhost:3030/api")] pub geo_engine_url: String, @@ -139,7 +136,9 @@ fn parse_mapping_file(s: &str) -> Result { } else { std::fs::read_to_string(s).map_err(|e| format!("Failed to read mapping from '{s}': {e}"))? }; - serde_json::from_str(&json).map_err(|e| format!("Invalid mapping JSON: {e}")) + let api_def: crate::api::model::services::StacDataProviderDefinition = + serde_json::from_str(&json).map_err(|e| format!("Invalid mapping JSON: {e}"))?; + Ok(api_def.into()) } // --------------------------------------------------------------------------- @@ -158,14 +157,17 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err provider_def.datasets.len() ); - let (client, session_id) = login_geo_engine( + let api_config = create_api_config( ¶ms.geo_engine_url, ¶ms.geo_engine_email, ¶ms.geo_engine_password, ) .await?; - let created_datasets = setup_datasets(&client, ¶ms, provider_def, &session_id).await?; + // Separate reqwest client for STAC API calls (not Geo Engine) + let stac_client = reqwest::Client::new(); + + let created_datasets = setup_datasets(&api_config, ¶ms, provider_def).await?; info!( "Created {} new dataset(s) out of {}", @@ -179,10 +181,10 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err stac_api_url, provider_def.collection_name ); - let query_params = build_stac_query_params(¶ms); + let query_params = build_stac_query_params(¶ms, provider_def.page_limit as usize); let (tiles_by_dataset, items_processed) = process_item_stream( - &client, + &stac_client, &items_url, &query_params, provider_def, @@ -193,12 +195,10 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err info!("Processed {} items total", items_processed); - upload_tiles_to_datasets(&client, ¶ms, &session_id, &tiles_by_dataset).await?; + upload_tiles_to_datasets(&api_config, ¶ms, &tiles_by_dataset).await?; create_harvest_layer_collections( - &client, - ¶ms.geo_engine_url, - &session_id, + &api_config, provider_def, &created_datasets, ¶ms, @@ -306,47 +306,36 @@ fn dataset_name_for_harvest(collection_name: &str, dataset: &StacProviderDataset } async fn dataset_exists_api( - client: &reqwest::Client, - geo_engine_url: &str, - session_id: &str, + api_config: &ApiConfig, dataset_name: &str, ) -> Result { - let response = retry_http( - || async { - client - .get(format!("{geo_engine_url}/dataset/{dataset_name}")) - .header("Authorization", format!("Bearer {session_id}")) - .send() - .await - }, + let result = retry_http( + || geoengine_api_client::apis::datasets_api::get_dataset_handler(api_config, dataset_name), &format!("Check dataset existence for '{dataset_name}'"), ) - .await?; - - if response.status().is_success() { - return Ok(true); - } - - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - if status == reqwest::StatusCode::BAD_REQUEST - && let Ok(error_response) = serde_json::from_str::(&body) - && error_response.error == "CannotLoadDataset" - { - return Ok(false); + .await; + + match result { + Ok(_) => Ok(true), + Err(geoengine_api_client::apis::Error::ResponseError(resp)) + if resp.status == reqwest::StatusCode::BAD_REQUEST => + { + Ok(false) + } + Err(e) => Err(anyhow::anyhow!("Failed to check dataset '{dataset_name}': {e}")), } - - anyhow::bail!("Failed to check dataset '{dataset_name}': HTTP {status}: {body}"); } async fn create_dataset_api( - client: &reqwest::Client, - geo_engine_url: &str, - session_id: &str, + api_config: &ApiConfig, dataset_name: &str, dataset: &StacProviderDataset, volume_name: &str, ) -> Result<(), anyhow::Error> { + let geo_engine_url = &api_config.base_path; + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let client = &api_config.client; + let bands: Vec = dataset .bands .iter() @@ -358,7 +347,6 @@ async fn create_dataset_api( }) .collect(); - // Get GeoTransform from the spatial grid descriptor for the API let dt_gt: GeoTransform = dataset.spatial_grid.geo_transform(); let api_gt: crate::api::model::datatypes::GeoTransform = dt_gt.into(); @@ -438,17 +426,17 @@ async fn create_dataset_api( dataset_name.to_string() }; - share_dataset_api(client, geo_engine_url, session_id, &created_name).await?; + share_dataset_api(api_config, &created_name).await?; Ok(()) } async fn share_dataset_api( - client: &reqwest::Client, - geo_engine_url: &str, - session_id: &str, + api_config: &ApiConfig, dataset_name: &str, ) -> Result<(), anyhow::Error> { + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let permissions = vec![ PermissionRequest { resource: Resource::Dataset(DatasetResource { @@ -471,8 +459,9 @@ async fn share_dataset_api( for permission in &permissions { retry_http( || async { - client - .put(format!("{geo_engine_url}/permissions")) + api_config + .client + .put(format!("{}/permissions", api_config.base_path)) .header("Content-Type", "application/json") .header("Authorization", format!("Bearer {session_id}")) .json(permission) @@ -488,17 +477,17 @@ async fn share_dataset_api( } async fn create_harvest_layer_collections( - client: &reqwest::Client, - geo_engine_url: &str, - session_id: &str, + api_config: &ApiConfig, provider_def: &StacDataProviderDefinition, created_datasets: &[(usize, StacProviderDataset)], params: &StacHarvest, ) -> Result<(), anyhow::Error> { + let geo_engine_url = &api_config.base_path; + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let client = &api_config.client; + let root_collection_id = create_layer_collection_api( - client, - geo_engine_url, - session_id, + api_config, &LayerCollectionId(INTERNAL_LAYER_DB_ROOT_COLLECTION_ID.to_string()), &provider_def.collection_name, &format!( @@ -510,9 +499,7 @@ async fn create_harvest_layer_collections( .await?; let temp_collection_id = create_layer_collection_api( - client, - geo_engine_url, - session_id, + api_config, &root_collection_id, "_layers", "All dataset layers (internal)", @@ -567,23 +554,25 @@ async fn create_harvest_layer_collections( ) .await?; - share_layer_api(client, geo_engine_url, session_id, &response.id).await?; + share_layer_api(api_config, &response.id).await?; } Ok(()) } async fn create_layer_collection_api( - client: &reqwest::Client, - geo_engine_url: &str, - session_id: &str, + api_config: &ApiConfig, parent_id: &LayerCollectionId, name: &str, description: &str, params: &StacHarvest, ) -> Result { + let geo_engine_url = &api_config.base_path; + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let client = &api_config.client; + if let Some(existing_id) = - find_child_collection_by_name(client, geo_engine_url, session_id, parent_id, name).await? + find_child_collection_by_name(api_config, parent_id, name).await? { if params.verbose { info!("Found existing layer collection '{name}'"); @@ -615,18 +604,20 @@ async fn create_layer_collection_api( ) .await?; - share_layer_collection_api(client, geo_engine_url, session_id, &response.id).await?; + share_layer_collection_api(api_config, &response.id).await?; Ok(response.id) } async fn find_child_collection_by_name( - client: &reqwest::Client, - geo_engine_url: &str, - session_id: &str, + api_config: &ApiConfig, parent_id: &LayerCollectionId, child_name: &str, ) -> Result, anyhow::Error> { + let geo_engine_url = &api_config.base_path; + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let client = &api_config.client; + let mut offset: u32 = 0; let limit: u32 = 20; @@ -665,11 +656,13 @@ async fn find_child_collection_by_name( } async fn share_layer_collection_api( - client: &reqwest::Client, - geo_engine_url: &str, - session_id: &str, + api_config: &ApiConfig, collection_id: &LayerCollectionId, ) -> Result<(), anyhow::Error> { + let geo_engine_url = &api_config.base_path; + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let client = &api_config.client; + let permissions = vec![ PermissionRequest { resource: Resource::LayerCollection(LayerCollectionResource { @@ -709,11 +702,13 @@ async fn share_layer_collection_api( } async fn share_layer_api( - client: &reqwest::Client, - geo_engine_url: &str, - session_id: &str, + api_config: &ApiConfig, layer_id: &LayerId, ) -> Result<(), anyhow::Error> { + let geo_engine_url = &api_config.base_path; + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let client = &api_config.client; + let permissions = vec![ PermissionRequest { resource: Resource::Layer(LayerResource { @@ -896,41 +891,34 @@ where // Authentication helper // --------------------------------------------------------------------------- -async fn login_geo_engine( +async fn create_api_config( geo_engine_url: &str, geo_engine_email: &str, geo_engine_password: &str, -) -> Result<(reqwest::Client, String), anyhow::Error> { - let client = reqwest::Client::new(); +) -> Result { + use geoengine_api_client::models; - let response = retry_http( - || async { - client - .post(format!("{geo_engine_url}/login")) - .header("Content-Type", "application/json") - .json(&serde_json::json!({ - "email": geo_engine_email, - "password": geo_engine_password, - })) - .send() - .await - }, + let config = ApiConfig { + base_path: geo_engine_url.to_string(), + ..ApiConfig::default() + }; + + let credentials = models::UserCredentials::new( + geo_engine_email.to_string(), + geo_engine_password.to_string(), + ); + + let session = retry_http( + || geoengine_api_client::apis::session_api::login_handler(&config, credentials.clone()), "Login to Geo Engine", ) .await .context("Failed to authenticate")?; - let json = response - .json::() - .await - .context("Failed to parse auth response")?; - - let session_id = json["id"] - .as_str() - .context("No session id in response")? - .to_string(); - - Ok((client, session_id)) + Ok(ApiConfig { + bearer_access_token: Some(session.id.to_string()), + ..config + }) } // --------------------------------------------------------------------------- @@ -939,10 +927,9 @@ async fn login_geo_engine( /// Create datasets that don't already exist on the Geo Engine server. async fn setup_datasets( - client: &reqwest::Client, + api_config: &ApiConfig, params: &StacHarvest, provider_def: &StacDataProviderDefinition, - session_id: &str, ) -> Result, anyhow::Error> { let mut created_datasets: Vec<(usize, StacProviderDataset)> = Vec::new(); @@ -953,11 +940,9 @@ async fn setup_datasets( info!("Checking dataset '{}'", dataset_name); } - if !dataset_exists_api(client, ¶ms.geo_engine_url, session_id, &dataset_name).await? { + if !dataset_exists_api(api_config, &dataset_name).await? { create_dataset_api( - client, - ¶ms.geo_engine_url, - session_id, + api_config, &dataset_name, dataset, ¶ms.volume_name, @@ -971,7 +956,7 @@ async fn setup_datasets( } /// Build the query parameters for the STAC items API request. -fn build_stac_query_params(params: &StacHarvest) -> Vec<(String, String)> { +fn build_stac_query_params(params: &StacHarvest, page_limit: usize) -> Vec<(String, String)> { let mut query_params: Vec<(String, String)> = Vec::new(); if let Some(bbox) = ¶ms.bbox @@ -994,9 +979,7 @@ fn build_stac_query_params(params: &StacHarvest) -> Vec<(String, String)> { )); } - if let Some(limit) = params.limit { - query_params.push(("limit".to_string(), limit.to_string())); - } + query_params.push(("limit".to_string(), page_limit.to_string())); if params.filter_item_fields { query_params.push(( @@ -1090,11 +1073,14 @@ async fn process_item_stream( /// Upload collected tiles to the Geo Engine server in batches. async fn upload_tiles_to_datasets( - client: &reqwest::Client, + api_config: &ApiConfig, params: &StacHarvest, - session_id: &str, tiles_by_dataset: &HashMap>, ) -> Result<(), anyhow::Error> { + let geo_engine_url = &api_config.base_path; + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let client = &api_config.client; + for (dataset_name, tiles) in tiles_by_dataset { if tiles.is_empty() { continue; @@ -1110,8 +1096,7 @@ async fn upload_tiles_to_datasets( || async { client .post(format!( - "{}/dataset/{}/tiles", - params.geo_engine_url, dataset_name + "{geo_engine_url}/dataset/{dataset_name}/tiles", )) .header("Content-Type", "application/json") .header("Authorization", format!("Bearer {session_id}")) @@ -1283,10 +1268,12 @@ mod tests { #[test] fn test_process_harvest_item_produces_correct_tiles() { - let mapping: StacDataProviderDefinition = serde_json::from_str(include_str!( - "../../../../test_data/stac_responses/expected-mapping-code-de.json" - )) - .expect("valid mapping fixture"); + let api_mapping: crate::api::model::services::StacDataProviderDefinition = + serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/expected-mapping-code-de.json" + )) + .expect("valid mapping fixture"); + let mapping: StacDataProviderDefinition = api_mapping.into(); let items: stac::ItemCollection = serde_json::from_str(include_str!( "../../../../test_data/stac_responses/items/code-de-harvest-test.json" @@ -1298,7 +1285,6 @@ mod tests { time_start: None, time_end: None, bbox: None, - limit: None, geo_engine_url: String::new(), geo_engine_email: String::new(), geo_engine_password: String::new(), @@ -1374,10 +1360,12 @@ mod tests { #[test] fn test_process_harvest_landsat_item_produces_correct_tiles() { - let mapping: StacDataProviderDefinition = serde_json::from_str(include_str!( - "../../../../test_data/stac_responses/expected-mapping-landsat-c2-l1.json" - )) - .expect("valid Landsat mapping fixture"); + let api_mapping: crate::api::model::services::StacDataProviderDefinition = + serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/expected-mapping-landsat-c2-l1.json" + )) + .expect("valid Landsat mapping fixture"); + let mapping: StacDataProviderDefinition = api_mapping.into(); let items: stac::ItemCollection = serde_json::from_str(include_str!( "../../../../test_data/stac_responses/items/landsat-c2-l1-harvest-test.json" @@ -1389,7 +1377,6 @@ mod tests { time_start: None, time_end: None, bbox: None, - limit: None, geo_engine_url: String::new(), geo_engine_email: String::new(), geo_engine_password: String::new(), diff --git a/geoengine/services/src/contexts/migrations/current_schema.sql b/geoengine/services/src/contexts/migrations/current_schema.sql index 05296a4a2b..24d7cdfa0f 100644 --- a/geoengine/services/src/contexts/migrations/current_schema.sql +++ b/geoengine/services/src/contexts/migrations/current_schema.sql @@ -907,6 +907,7 @@ CREATE TYPE "StacDataProviderDefinition" AS ( s3_config "StacProviderS3Config", time_dimension "TimeDimension", datasets "StacProviderDataset" [], + page_limit bigint, query_timeout_secs bigint ); diff --git a/geoengine/services/src/contexts/migrations/migration_0028_stac_provider.sql b/geoengine/services/src/contexts/migrations/migration_0028_stac_provider.sql index cd9c408c63..e68600336f 100644 --- a/geoengine/services/src/contexts/migrations/migration_0028_stac_provider.sql +++ b/geoengine/services/src/contexts/migrations/migration_0028_stac_provider.sql @@ -29,6 +29,7 @@ CREATE TYPE "StacDataProviderDefinition" AS ( s3_config "StacProviderS3Config", time_dimension "TimeDimension", datasets "StacProviderDataset" [], + page_limit bigint, query_timeout_secs bigint ); diff --git a/geoengine/services/src/datasets/external/stac/listing.rs b/geoengine/services/src/datasets/external/stac/listing.rs index 8a5ddc40e6..bab0da8537 100644 --- a/geoengine/services/src/datasets/external/stac/listing.rs +++ b/geoengine/services/src/datasets/external/stac/listing.rs @@ -527,6 +527,7 @@ mod tests { ), bands: vec![], }], + 100_i64, 60, ) } @@ -593,6 +594,7 @@ mod tests { bands: vec![], }, ], + 100_i64, 60, ) } diff --git a/geoengine/services/src/datasets/external/stac/loading_info.rs b/geoengine/services/src/datasets/external/stac/loading_info.rs index 56264daece..9f4b3d9f3c 100644 --- a/geoengine/services/src/datasets/external/stac/loading_info.rs +++ b/geoengine/services/src/datasets/external/stac/loading_info.rs @@ -37,6 +37,7 @@ struct StacMultiBandMetaData { s3_config: Option, time_dimension: TimeDimension, dataset: StacProviderDataset, + page_limit: i64, client: reqwest::Client, /// Shared query-result cache from the provider. query_cache: Arc, @@ -371,7 +372,7 @@ impl StacMultiBandMetaData { .to_datetime_string_with_millis(), ), ), - ("limit".to_owned(), "100".to_owned()), + ("limit".to_owned(), self.page_limit.to_string()), ( "fields".to_owned(), "stac_version,properties.datetime,properties.updated,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.proj:code,assets.*.proj:shape,assets.*.proj:transform".to_owned(), @@ -796,6 +797,7 @@ impl s3_config: self.s3_config.clone(), time_dimension: self.time_dimension, dataset: dataset.clone(), + page_limit: self.page_limit, client: self.client.clone(), query_cache: self.query_cache.clone(), })) @@ -868,6 +870,7 @@ mod tests { }, ], }], + page_limit: 100, query_timeout_secs: 60, } } @@ -1104,6 +1107,7 @@ mod tests { ], }, ], + page_limit: 100, query_timeout_secs: 60, }; @@ -1150,13 +1154,16 @@ mod tests { /// harvester and the runtime provider. #[crate::ge_context::test] async fn mapping_from_discover_works_as_stacdataprovider(app_ctx: PostgresContext) { - // Load the discover-generated mapping JSON - let mut provider_def: crate::datasets::external::stac::StacDataProviderDefinition = + // Load the discover-generated mapping JSON via the API layer (camelCase) + let api_def: crate::api::model::services::StacDataProviderDefinition = serde_json::from_str(include_str!( "../../../../../test_data/stac_responses/expected-mapping-code-de.json" )) .expect("valid discover mapping fixture"); + let mut provider_def: crate::datasets::external::stac::StacDataProviderDefinition = + api_def.into(); + // Use a placeholder URL (no actual HTTP calls needed for meta_data registration) provider_def.api_url = "https://stac.test/v1".to_owned(); provider_def.id = DataProviderId::new(); diff --git a/geoengine/services/src/datasets/external/stac/mod.rs b/geoengine/services/src/datasets/external/stac/mod.rs index 161c32fb5b..f4dbb8cd30 100644 --- a/geoengine/services/src/datasets/external/stac/mod.rs +++ b/geoengine/services/src/datasets/external/stac/mod.rs @@ -33,7 +33,7 @@ pub struct StacDataProviderDefinition { pub s3_config: Option, pub time_dimension: TimeDimension, // TODO: should this be on dataset level? pub datasets: Vec, - // TODO: page limit(?) + pub page_limit: i64, /// Timeout in seconds for outgoing STAC API HTTP requests. #[serde(default = "default_query_timeout")] pub query_timeout_secs: i64, @@ -94,6 +94,7 @@ impl DataProviderDefinition for StacDataProviderDefinition { self.s3_config, self.time_dimension, self.datasets, + self.page_limit, self.query_timeout_secs, ))) } @@ -150,6 +151,7 @@ pub struct StacDataProvider { s3_config: Option, time_dimension: TimeDimension, datasets: Vec, + page_limit: i64, /// Shared HTTP client, reused across all requests for this provider. client: reqwest::Client, /// In-memory cache for STAC query results (tile files), keyed by dataset @@ -168,6 +170,7 @@ impl StacDataProvider { s3_config: Option, time_dimension: TimeDimension, datasets: Vec, + page_limit: i64, query_timeout_secs: i64, ) -> Self { let client = reqwest::Client::builder() @@ -183,6 +186,7 @@ impl StacDataProvider { s3_config, time_dimension, datasets, + page_limit, client, query_cache: Arc::new(StacQueryCache::default()), } diff --git a/geoengine/test_data/provider_defs_api/stac_sentinel2.json b/geoengine/test_data/provider_defs_api/stac_sentinel2.json index 6d8a2b513a..8835e3e080 100644 --- a/geoengine/test_data/provider_defs_api/stac_sentinel2.json +++ b/geoengine/test_data/provider_defs_api/stac_sentinel2.json @@ -413,5 +413,6 @@ } ] } - ] + ], + "pageLimit": 100 } diff --git a/geoengine/test_data/stac_responses/expected-mapping-code-de.json b/geoengine/test_data/stac_responses/expected-mapping-code-de.json index 0e49bcb127..a7a0fe3296 100644 --- a/geoengine/test_data/stac_responses/expected-mapping-code-de.json +++ b/geoengine/test_data/stac_responses/expected-mapping-code-de.json @@ -1,4 +1,5 @@ { + "type": "StacProviderDefinition", "name": "sentinel-2-l2a from STAC", "id": "00000000-0000-0000-0000-000000000000", "description": "Auto-discovered mapping for STAC collection 'sentinel-2-l2a' at https://stac.test/v1", @@ -7,19 +8,18 @@ "collectionName": "sentinel-2-l2a", "s3Config": null, "timeDimension": { - "regular": { - "origin": 0, - "step": { "granularity": "days", "step": 1 } - } + "type": "regular", + "origin": 0, + "step": { "granularity": "days", "step": 1 } }, "datasets": [ { "name": "sentinel-2-l2a EPSG:32632 U16 10m", "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", - "data_type": "U16", + "dataType": "U16", "resolution": { "x": 10.0, "y": 10.0 }, "projection": "EPSG:32632", - "spatial_grid": { + "spatialGrid": { "spatialGrid": { "geoTransform": { "originCoordinate": { "x": 399960.0, "y": 5700000.0 }, @@ -27,26 +27,26 @@ "yPixelSize": -10.0 }, "gridBounds": { - "min": [0, 0], - "max": [10979, 10979] + "topLeftIdx": { "yIdx": 0, "xIdx": 0 }, + "bottomRightIdx": { "yIdx": 10979, "xIdx": 10979 } } }, - "state": "source" + "descriptor": "source" }, "bands": [ - { "asset_title": "Blue (band 2) - 10m", "band_name": null }, - { "asset_title": "Green (band 3) - 10m", "band_name": null }, - { "asset_title": "NIR 1 (band 8) - 10m", "band_name": null }, - { "asset_title": "Red (band 4) - 10m", "band_name": null } + { "assetTitle": "Blue (band 2) - 10m", "bandName": null }, + { "assetTitle": "Green (band 3) - 10m", "bandName": null }, + { "assetTitle": "NIR 1 (band 8) - 10m", "bandName": null }, + { "assetTitle": "Red (band 4) - 10m", "bandName": null } ] }, { "name": "sentinel-2-l2a EPSG:32632 U16 20m", "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", - "data_type": "U16", + "dataType": "U16", "resolution": { "x": 20.0, "y": 20.0 }, "projection": "EPSG:32632", - "spatial_grid": { + "spatialGrid": { "spatialGrid": { "geoTransform": { "originCoordinate": { "x": 399960.0, "y": 5700000.0 }, @@ -54,16 +54,17 @@ "yPixelSize": -20.0 }, "gridBounds": { - "min": [0, 0], - "max": [5489, 5489] + "topLeftIdx": { "yIdx": 0, "xIdx": 0 }, + "bottomRightIdx": { "yIdx": 5489, "xIdx": 5489 } } }, - "state": "source" + "descriptor": "source" }, "bands": [ - { "asset_title": "SWIR 1 (band 11) - 20m", "band_name": null }, - { "asset_title": "SWIR 2 (band 12) - 20m", "band_name": null } + { "assetTitle": "SWIR 1 (band 11) - 20m", "bandName": null }, + { "assetTitle": "SWIR 2 (band 12) - 20m", "bandName": null } ] } - ] + ], + "pageLimit": 100 } diff --git a/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json b/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json index 05713a7a10..08d2f1a7ef 100644 --- a/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json +++ b/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json @@ -1,4 +1,5 @@ { + "type": "StacProviderDefinition", "name": "landsat-c2-l1 from STAC", "id": "00000000-0000-0000-0000-000000000000", "description": "Auto-discovered mapping for STAC collection 'landsat-c2-l1' at https://stac.test/v1", @@ -7,19 +8,18 @@ "collectionName": "landsat-c2-l1", "s3Config": null, "timeDimension": { - "regular": { - "origin": 0, - "step": { "granularity": "days", "step": 1 } - } + "type": "regular", + "origin": 0, + "step": { "granularity": "days", "step": 1 } }, "datasets": [ { "name": "landsat-c2-l1 EPSG:32632 U16 30m", "description": "Auto-discovered from STAC collection 'landsat-c2-l1'", - "data_type": "U16", + "dataType": "U16", "resolution": { "x": 30.0, "y": 30.0 }, "projection": "EPSG:32632", - "spatial_grid": { + "spatialGrid": { "spatialGrid": { "geoTransform": { "originCoordinate": { "x": 399960.0, "y": 5800020.0 }, @@ -27,18 +27,19 @@ "yPixelSize": -30.0 }, "gridBounds": { - "min": [0, 0], - "max": [7999, 7999] + "topLeftIdx": { "yIdx": 0, "xIdx": 0 }, + "bottomRightIdx": { "yIdx": 7999, "xIdx": 7999 } } }, - "state": "source" + "descriptor": "source" }, "bands": [ - { "asset_title": "Blue (band 2) - 30m", "band_name": null }, - { "asset_title": "Green (band 3) - 30m", "band_name": null }, - { "asset_title": "NIR (band 5) - 30m", "band_name": null }, - { "asset_title": "Red (band 4) - 30m", "band_name": null } + { "assetTitle": "Blue (band 2) - 30m", "bandName": null }, + { "assetTitle": "Green (band 3) - 30m", "bandName": null }, + { "assetTitle": "NIR (band 5) - 30m", "bandName": null }, + { "assetTitle": "Red (band 4) - 30m", "bandName": null } ] } - ] + ], + "pageLimit": 100 } From 28df5934cdcafdfd80c52f80f3d112ddf01711dc Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Mon, 3 Aug 2026 18:53:14 +0000 Subject: [PATCH 06/27] refactors and fixes --- geoengine/services/src/api/model/services.rs | 38 +- .../src/cli/stac_harvester/discover.rs | 328 ++++++++--- .../src/cli/stac_harvester/harvest.rs | 136 ++--- .../services/src/cli/stac_harvester/mod.rs | 43 +- .../contexts/migrations/current_schema.sql | 7 +- .../migration_0029_stac_provider_band_name.rs | 31 + ...migration_0029_stac_provider_band_name.sql | 100 ++++ .../services/src/contexts/migrations/mod.rs | 3 + .../src/datasets/external/stac/common.rs | 171 ++++-- .../datasets/external/stac/loading_info.rs | 80 ++- .../src/datasets/external/stac/mod.rs | 55 +- geoengine/services/src/datasets/upload.rs | 7 + geoengine/services/src/util/mod.rs | 1 + geoengine/services/src/util/retry.rs | 187 ++++++ .../provider_defs_api/stac_sentinel2.json | 548 +++++++++++++++--- .../expected-mapping-code-de.json | 126 +++- .../expected-mapping-landsat-c2-l1.json | 80 ++- openapi.json | 40 +- 18 files changed, 1618 insertions(+), 363 deletions(-) create mode 100644 geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.rs create mode 100644 geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql create mode 100644 geoengine/services/src/util/retry.rs diff --git a/geoengine/services/src/api/model/services.rs b/geoengine/services/src/api/model/services.rs index 922fa36342..9b4f19a240 100644 --- a/geoengine/services/src/api/model/services.rs +++ b/geoengine/services/src/api/model/services.rs @@ -989,13 +989,23 @@ impl From for StacProvide #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSchema)] #[serde(rename_all = "camelCase")] -pub struct StacProviderDatasetBand { +pub struct StacAssetBand { pub asset_title: String, pub band_name: Option, } -impl From for crate::datasets::external::stac::StacProviderDatasetBand { - fn from(value: StacProviderDatasetBand) -> Self { +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct StacProviderDatasetBand { + /// The band inside the STAC asset that this dataset band reads from + /// (addressing: which asset file + which raster channel within it). + pub asset_band: StacAssetBand, + /// The band descriptor of the resulting geo engine dataset layer. + pub band_descriptor: crate::api::model::operators::RasterBandDescriptor, +} + +impl From for crate::datasets::external::stac::StacAssetBand { + fn from(value: StacAssetBand) -> Self { Self { asset_title: value.asset_title, band_name: value.band_name, @@ -1003,8 +1013,8 @@ impl From for crate::datasets::external::stac::StacProv } } -impl From for StacProviderDatasetBand { - fn from(value: crate::datasets::external::stac::StacProviderDatasetBand) -> Self { +impl From for StacAssetBand { + fn from(value: crate::datasets::external::stac::StacAssetBand) -> Self { Self { asset_title: value.asset_title, band_name: value.band_name, @@ -1012,6 +1022,24 @@ impl From for StacProv } } +impl From for crate::datasets::external::stac::StacProviderDatasetBand { + fn from(value: StacProviderDatasetBand) -> Self { + Self { + asset_band: value.asset_band.into(), + band_descriptor: value.band_descriptor.into(), + } + } +} + +impl From for StacProviderDatasetBand { + fn from(value: crate::datasets::external::stac::StacProviderDatasetBand) -> Self { + Self { + asset_band: value.asset_band.into(), + band_descriptor: value.band_descriptor.into(), + } + } +} + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSchema)] #[serde(rename_all = "camelCase")] pub struct StacProviderDataset { diff --git a/geoengine/services/src/cli/stac_harvester/discover.rs b/geoengine/services/src/cli/stac_harvester/discover.rs index dc35b68e0e..3dc9f80304 100644 --- a/geoengine/services/src/cli/stac_harvester/discover.rs +++ b/geoengine/services/src/cli/stac_harvester/discover.rs @@ -12,12 +12,11 @@ use geoengine_datatypes::{ use ordered_float::OrderedFloat; use tracing::{info, warn}; -use crate::api::model::datatypes::{Measurement, UnitlessMeasurement, UnitlessMeasurementTypeTag}; -use crate::api::model::operators::RasterBandDescriptor; use crate::datasets::external::stac::{ - StacDataProviderDefinition, StacProviderDataset, StacProviderDatasetBand, StacProviderS3Config, - common, + StacAssetBand, StacDataProviderDefinition, StacProviderDataset, StacProviderDatasetBand, + StacProviderS3Config, common, }; +use crate::util::retry::{RetryPolicy, retry_http}; use geoengine_datatypes::primitives::{ RegularTimeDimension as DtRegularTimeDimension, TimeDimension as DtTimeDimension, }; @@ -88,6 +87,14 @@ pub struct StacDiscoverMapping { #[arg(long, default_value_t = false)] pub full_projection_grid: bool, + /// Provider id to write into the output definition. + /// + /// Discovery normally assigns a fresh random id. Pass the `id` of the + /// existing mapping file when regenerating it so that provider-scoped + /// references (e.g. `_::...` in layer bodies) stay valid. + #[arg(long)] + pub id: Option, + /// Time dimension granularity (default: days) #[arg(long, default_value = "days")] pub time_granularity: String, @@ -121,9 +128,10 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), params.stac_collection ); - let collection: stac::Collection = stac_api_request_parse(&client, &collection_url) - .await - .context("Failed to fetch STAC collection")?; + let collection: stac::Collection = + stac_api_request_parse(&client, &collection_url, &stac_request_policy()) + .await + .context("Failed to fetch STAC collection")?; let dataset_bands = scan_collection_bands(&collection, ¶ms.file_types); @@ -176,7 +184,7 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), let provider_def = StacDataProviderDefinition { name: format!("{} from STAC", params.stac_collection), - id: DataProviderId::new(), + id: params.id.unwrap_or_else(DataProviderId::new), description: format!( "Auto-discovered mapping for STAC collection '{}' at {}", params.stac_collection, params.stac_url @@ -188,13 +196,33 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), time_dimension, datasets, page_limit: params.page_limit as i64, + query_timeout_secs: 60, }; - let json = serde_json::to_string_pretty( - &crate::api::model::services::StacDataProviderDefinition::from(provider_def), + let s3_config = provider_def.s3_config.clone(); + + let mut json_value = serde_json::to_value( + crate::api::model::services::StacDataProviderDefinition::from(provider_def), ) .context("Failed to serialize mapping to JSON")?; + // The API model wraps S3 credentials in `Secret`, which serializes as + // `*****`. Keep the plain values (e.g. `__AWS_ACCESS_KEY_ID__` markers) in + // the output file instead, so they can be substituted at runtime. + if let (Some(obj), Some(s3)) = (json_value.as_object_mut(), &s3_config) + && let Some(s3_obj) = obj.get_mut("s3Config").and_then(|v| v.as_object_mut()) + { + if let Some(key) = &s3.access_key { + s3_obj.insert("accessKey".to_string(), serde_json::json!(key)); + } + if let Some(key) = &s3.secret_key { + s3_obj.insert("secretKey".to_string(), serde_json::json!(key)); + } + } + + let json = + serde_json::to_string_pretty(&json_value).context("Failed to serialize mapping to JSON")?; + if let Some(output_path) = ¶ms.output { std::fs::write(output_path, &json) .with_context(|| format!("Failed to write mapping to {}", output_path.display()))?; @@ -221,8 +249,9 @@ struct DiscoveredDatasetInfo { fn scan_collection_bands( collection: &stac::Collection, file_types: &[ImportFileType], -) -> HashMap> { - let mut dataset_bands: HashMap> = HashMap::new(); +) -> HashMap> { + let mut dataset_bands: HashMap> = + HashMap::new(); for (_asset_key, asset) in &collection.item_assets { if !matches_selected_file_types(asset.r#type.as_deref(), file_types) { @@ -275,11 +304,11 @@ async fn fetch_sample_items( loop { let page = if let Some(ref url) = next_url { - stac_api_request_parse::(client, url) + stac_api_request_parse::(client, url, &stac_request_policy()) .await .context("Failed to fetch sample items page")? } else { - stac_api_request_with_params(client, &base_url, &query_params) + stac_api_request_with_params(client, &base_url, &query_params, &stac_request_policy()) .await .context("Failed to fetch sample items")? }; @@ -364,13 +393,17 @@ fn process_sample_assets( }; let asset_title = asset.title.as_deref().unwrap_or(asset_key).to_string(); - let band_names = common::band_names_from_asset_v1_1_0(asset) - .unwrap_or_else(|_| vec![asset_title.clone()]); + let asset_info = common::band_names_from_asset_v1_1_0(asset).unwrap_or_else(|_| { + common::AssetBandInfo { + asset_title: asset_title.clone(), + band_names: vec![asset_title.clone()], + } + }); let entry = sample_band_info.entry(partial_key.clone()).or_default(); - for bn in &band_names { - if !entry.iter().any(|(t, _)| t == &asset_title) { - entry.push((asset_title.clone(), bn.clone())); + for bn in &asset_info.band_names { + if !entry.iter().any(|(t, _)| t == &asset_info.asset_title) { + entry.push((asset_info.asset_title.clone(), bn.clone())); } } @@ -394,7 +427,7 @@ fn process_sample_assets( /// and sample band information. fn build_datasets( discovered_datasets: &HashMap, - dataset_bands: &HashMap>, + dataset_bands: &HashMap>, sample_band_info: &HashMap>, full_projection_grid: bool, stac_collection: &str, @@ -408,25 +441,35 @@ fn build_datasets( }; let mut bands: Vec = Vec::new(); + // Track the *resolved* band name (what the harvest uses: + // `band_name`, falling back to the asset title). Raster band names must + // be unique, so skip any band whose resolved name is already present. + let mut seen_names: std::collections::HashSet = std::collections::HashSet::new(); // Use bands from collection-level scan if let Some(descriptors) = dataset_bands.get(&partial_key) { - for desc in descriptors { - bands.push(StacProviderDatasetBand { - asset_title: desc.name.clone(), - band_name: None, - }); + for band in descriptors { + let resolved = band + .asset_band + .band_name + .clone() + .unwrap_or_else(|| band.asset_band.asset_title.clone()); + if seen_names.insert(resolved) { + bands.push(band.clone()); + } } } // Enrich with sample item band info if let Some(sample_bands) = sample_band_info.get(&partial_key) { for (asset_title, band_name) in sample_bands { - if !bands.iter().any(|b| b.asset_title == *asset_title) { - bands.push(StacProviderDatasetBand { + // Sample bands always carry an explicit band name; skip any + // whose resolved name is already present. + if seen_names.insert(band_name.clone()) { + bands.push(StacProviderDatasetBand::new_unitless(StacAssetBand { asset_title: asset_title.clone(), band_name: Some(band_name.clone()), - }); + })); } } } @@ -436,7 +479,7 @@ fn build_datasets( continue; } - bands.sort_by(|a, b| a.asset_title.cmp(&b.asset_title)); + bands.sort_by(|a, b| a.asset_band.asset_title.cmp(&b.asset_band.asset_title)); let spatial_grid = build_dataset_spatial_grid(info, dataset_key, full_projection_grid); @@ -604,7 +647,7 @@ fn scan_collection_item_asset( collection_version: &stac::Version, asset: &stac::ItemAsset, collection_summaries: Option<&serde_json::Map>, -) -> Result>>, String> { +) -> Result>>, String> { match collection_version { stac::Version::v1_0_0 => scan_collection_item_asset_v1_0_0(asset), stac::Version::v1_1_0 => scan_collection_item_asset_v1_1_0(asset, collection_summaries), @@ -619,8 +662,9 @@ fn scan_collection_item_asset( fn scan_collection_item_asset_v1_0_0( asset: &stac::ItemAsset, -) -> Result>>, String> { - let mut dataset_bands: HashMap> = HashMap::new(); +) -> Result>>, String> { + let mut dataset_bands: HashMap> = + HashMap::new(); let Some(raster_bands) = asset.additional_fields.get("raster:bands") else { return Ok(None); @@ -644,6 +688,8 @@ fn scan_collection_item_asset_v1_0_0( return Ok(None); } + let asset_title = asset.title.clone().unwrap_or_default(); + for (index, raster_band) in raster_bands.into_iter().enumerate() { let data_type = raster_band .data_type @@ -671,12 +717,10 @@ fn scan_collection_item_asset_v1_0_0( resolution, }) .or_default() - .push(RasterBandDescriptor { - name: band_name, - measurement: Measurement::Unitless(UnitlessMeasurement { - r#type: UnitlessMeasurementTypeTag::UnitlessMeasurementTypeTag, - }), - }); + .push(StacProviderDatasetBand::new_unitless(StacAssetBand { + asset_title: asset_title.clone(), + band_name: Some(band_name), + })); } Ok(Some(dataset_bands)) @@ -685,8 +729,9 @@ fn scan_collection_item_asset_v1_0_0( fn scan_collection_item_asset_v1_1_0( asset: &stac::ItemAsset, collection_summaries: Option<&serde_json::Map>, -) -> Result>>, String> { - let mut dataset_bands: HashMap> = HashMap::new(); +) -> Result>>, String> { + let mut dataset_bands: HashMap> = + HashMap::new(); let data_type = asset .additional_fields @@ -698,7 +743,7 @@ fn scan_collection_item_asset_v1_1_0( let raster_data_type = common::raster_data_type_from_stac_data_type_str(data_type) .ok_or_else(|| format!("Unsupported data_type: {data_type}"))?; - let band_names = common::band_names_from_item_asset_v1_1_0(asset)?; + let asset_info = common::band_names_from_item_asset_v1_1_0(asset)?; let resolution = asset .additional_fields @@ -717,28 +762,23 @@ fn scan_collection_item_asset_v1_1_0( }) .ok_or_else(|| "Missing attribute `gsd` or `proj:transform`".to_string())?; - for band_name in band_names { - dataset_bands - .entry(PartialDatasetKey { - data_type: raster_data_type, - resolution: resolution.into(), - }) - .or_default() - .push(RasterBandDescriptor { - name: band_name.clone(), - measurement: Measurement::Unitless(UnitlessMeasurement { - r#type: UnitlessMeasurementTypeTag::UnitlessMeasurementTypeTag, - }), - }); + let entry = dataset_bands + .entry(PartialDatasetKey { + data_type: raster_data_type, + resolution: resolution.into(), + }) + .or_default(); + for band_name in asset_info.band_names { + entry.push(StacProviderDatasetBand::new_unitless(StacAssetBand { + asset_title: asset_info.asset_title.clone(), + band_name: Some(band_name), + })); } Ok(Some(dataset_bands)) } -fn matches_selected_file_types( - media_type: Option<&str>, - file_types: &[ImportFileType], -) -> bool { +fn matches_selected_file_types(media_type: Option<&str>, file_types: &[ImportFileType]) -> bool { file_types.iter().any(|file_type| match file_type { ImportFileType::Cog => common::is_cog_media_type(media_type), ImportFileType::Jp2 => common::is_jp2_media_type(media_type), @@ -746,14 +786,26 @@ fn matches_selected_file_types( } fn merge_dataset_bands( - dataset_bands: &mut HashMap>, - additions: HashMap>, + dataset_bands: &mut HashMap>, + additions: HashMap>, ) { - for (partial_key, band_descriptors) in additions { + for (partial_key, bands) in additions { let existing_bands = dataset_bands.entry(partial_key).or_default(); - for descriptor in band_descriptors { - if existing_bands.iter().all(|b| b.name != descriptor.name) { - existing_bands.push(descriptor); + for band in bands { + let resolved = band + .asset_band + .band_name + .clone() + .unwrap_or_else(|| band.asset_band.asset_title.clone()); + let already_present = existing_bands.iter().any(|b| { + b.asset_band + .band_name + .clone() + .unwrap_or_else(|| b.asset_band.asset_title.clone()) + == resolved + }); + if !already_present { + existing_bands.push(band); } } } @@ -784,21 +836,25 @@ fn parse_time_dimension(granularity: &str, step: u64) -> Result RetryPolicy { + RetryPolicy::new().stop_on_status(&[400, 404]) +} + async fn stac_api_request_parse( client: &reqwest::Client, url: &str, + policy: &RetryPolicy, ) -> Result { - let response = client - .get(url) - .send() - .await - .with_context(|| format!("Failed to fetch {url}"))?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - anyhow::bail!("STAC API returned HTTP {status}: {body}"); - } + let response = retry_http( + || async { client.get(url).send().await?.error_for_status() }, + &format!("Fetch {url}"), + policy, + |e| e.status().map(|s| s.as_u16()), + ) + .await + .with_context(|| format!("Failed to fetch {url}"))?; response .json() @@ -810,19 +866,23 @@ async fn stac_api_request_with_params( client: &reqwest::Client, url: &str, params: &[(String, String)], + policy: &RetryPolicy, ) -> Result { - let response = client - .get(url) - .query(params) - .send() - .await - .with_context(|| format!("Failed to fetch {url}"))?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - anyhow::bail!("STAC API returned HTTP {status}: {body}"); - } + let response = retry_http( + || async { + client + .get(url) + .query(params) + .send() + .await? + .error_for_status() + }, + &format!("Fetch {url}"), + policy, + |e| e.status().map(|s| s.as_u16()), + ) + .await + .with_context(|| format!("Failed to fetch {url}"))?; response .json() @@ -899,6 +959,7 @@ mod tests { time_granularity: "days".to_string(), time_step: 1, page_limit: 100, + id: None, }; discover_mapping(params) @@ -1010,6 +1071,7 @@ mod tests { time_granularity: "days".to_string(), time_step: 1, page_limit: 100, + id: None, }; discover_mapping(params) @@ -1049,4 +1111,96 @@ mod tests { let _ = std::fs::remove_file(&output_path); } + + // ----------------------------------------------------------------------- + // build_datasets band dedup + // ----------------------------------------------------------------------- + + fn unitless_band_descriptor(asset_title: &str, band_name: &str) -> StacProviderDatasetBand { + StacProviderDatasetBand::new_unitless(StacAssetBand { + asset_title: asset_title.to_string(), + band_name: Some(band_name.to_string()), + }) + } + + #[test] + fn test_build_datasets_dedupes_duplicate_resolved_band_names() { + let partial_key = PartialDatasetKey { + data_type: RasterDataType::U8, + resolution: OrderedFloat(10.0), + }; + let dataset_key = DatasetKey { + epsg: 32_632, + data_type: RasterDataType::U8, + resolution: OrderedFloat(10.0), + }; + + // Collection-level scan reports the true-color asset as one dataset + // band per STAC band, keeping the real asset title separate from the + // band name (no `True color image [B02]`-style synthetic titles). + let mut dataset_bands = HashMap::new(); + dataset_bands.insert( + partial_key.clone(), + vec![ + unitless_band_descriptor("True color image", "B02"), + unitless_band_descriptor("True color image", "B03"), + unitless_band_descriptor("True color image", "B04"), + ], + ); + + // Sample items report the same true-color asset with an explicit band + // name that would collide with the collection scan's `B04` band. + let mut sample_band_info = HashMap::new(); + sample_band_info.insert( + partial_key.clone(), + vec![("True color image".to_string(), "B04".to_string())], + ); + + let mut discovered_datasets = HashMap::new(); + discovered_datasets.insert( + dataset_key, + DiscoveredDatasetInfo { + geo_transform: Some(GeoTransform::new((0.0, 0.0).into(), 10.0, -10.0)), + proj_shape: Some((100, 100)), + srs: SpatialReference::new(SpatialReferenceAuthority::Epsg, 32_632), + asset_count: 1, + }, + ); + + let datasets = build_datasets( + &discovered_datasets, + &dataset_bands, + &sample_band_info, + false, + "sentinel-2-l2a", + ); + + assert_eq!(datasets.len(), 1); + + let names: Vec = datasets[0] + .bands + .iter() + .map(|b| { + b.asset_band + .band_name + .clone() + .unwrap_or_else(|| b.asset_band.asset_title.clone()) + }) + .collect(); + + // The duplicate `B04` from the sample item must be dropped so that all + // resolved band names are unique. + assert_eq!(names, vec!["B02", "B03", "B04"]); + + // All bands keep the real asset title. + assert!( + datasets[0] + .bands + .iter() + .all(|b| b.asset_band.asset_title == "True color image") + ); + + let unique: std::collections::HashSet<_> = names.iter().collect(); + assert_eq!(unique.len(), names.len()); + } } diff --git a/geoengine/services/src/cli/stac_harvester/harvest.rs b/geoengine/services/src/cli/stac_harvester/harvest.rs index 9b57c259bf..06c532fe89 100644 --- a/geoengine/services/src/cli/stac_harvester/harvest.rs +++ b/geoengine/services/src/cli/stac_harvester/harvest.rs @@ -1,9 +1,4 @@ -use std::{ - collections::HashMap, - io::Read, - str::FromStr, - time::{Duration, Instant}, -}; +use std::{collections::HashMap, io::Read, str::FromStr, time::Instant}; use anyhow::Context; use chrono::Timelike; @@ -19,6 +14,7 @@ use tracing::{debug, error, info, warn}; use crate::datasets::external::stac::{ StacDataProviderDefinition, StacProviderDataset, StacProviderDatasetBand, common, }; +use crate::util::retry::{RetryPolicy, retry_http}; use crate::{ api::{ handlers::{ @@ -32,8 +28,8 @@ use crate::{ model::{ datatypes::{ GdalConfigOption, GridBoundingBox2D as ApiGridBoundingBox2D, - GridIdx2D as ApiGridIdx2D, LayerId, Measurement, SpatialGridDefinition, - TimeGranularity, TimeStep, UnitlessMeasurement, UnitlessMeasurementTypeTag, + GridIdx2D as ApiGridIdx2D, LayerId, SpatialGridDefinition, TimeGranularity, + TimeStep, }, operators::{ GdalDatasetParameters, GdalMultiBand, GdalMultiBandTypeTag, RasterBandDescriptor, @@ -55,11 +51,11 @@ use crate::{ permissions::{Permission, Role}, workflows::workflow::Workflow, }; +use geoengine_api_client::apis::configuration::Configuration as ApiConfig; use geoengine_operators::{ engine::{RasterOperator, TypedOperator}, source::{MultiBandGdalSource, MultiBandGdalSourceParameters}, }; -use geoengine_api_client::apis::configuration::Configuration as ApiConfig; // --------------------------------------------------------------------------- // Harvest @@ -197,13 +193,7 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err upload_tiles_to_datasets(&api_config, ¶ms, &tiles_by_dataset).await?; - create_harvest_layer_collections( - &api_config, - provider_def, - &created_datasets, - ¶ms, - ) - .await?; + create_harvest_layer_collections(&api_config, provider_def, &created_datasets, ¶ms).await?; let elapsed = start_time.elapsed(); info!("Harvest completed in {:.2?}", elapsed); @@ -309,20 +299,39 @@ async fn dataset_exists_api( api_config: &ApiConfig, dataset_name: &str, ) -> Result { + // A missing dataset is signalled with HTTP 400 (`CannotLoadDataset`) and + // conventionally 404 — a definitive "doesn't exist" answer. Configure the + // retry policy to stop on those codes so they are not retried. + let policy = RetryPolicy::new().stop_on_status(&[400, 404]); + let result = retry_http( || geoengine_api_client::apis::datasets_api::get_dataset_handler(api_config, dataset_name), &format!("Check dataset existence for '{dataset_name}'"), + &policy, + apis_error_status, ) .await; match result { Ok(_) => Ok(true), Err(geoengine_api_client::apis::Error::ResponseError(resp)) - if resp.status == reqwest::StatusCode::BAD_REQUEST => + if resp.status == reqwest::StatusCode::BAD_REQUEST + || resp.status == reqwest::StatusCode::NOT_FOUND => { Ok(false) } - Err(e) => Err(anyhow::anyhow!("Failed to check dataset '{dataset_name}': {e}")), + Err(e) => Err(anyhow::anyhow!( + "Failed to check dataset '{dataset_name}': {e}" + )), + } +} + +/// Extract an HTTP status code from a `geoengine_api_client` API error, so a +/// [`RetryPolicy`] can match on it. +fn apis_error_status(e: &geoengine_api_client::apis::Error) -> Option { + match e { + geoengine_api_client::apis::Error::ResponseError(resp) => Some(resp.status.as_u16()), + _ => None, } } @@ -339,12 +348,7 @@ async fn create_dataset_api( let bands: Vec = dataset .bands .iter() - .map(|b| RasterBandDescriptor { - name: b.band_name.clone().unwrap_or_else(|| b.asset_title.clone()), - measurement: Measurement::Unitless(UnitlessMeasurement { - r#type: UnitlessMeasurementTypeTag::UnitlessMeasurementTypeTag, - }), - }) + .map(|b| b.band_descriptor.clone().into()) .collect(); let dt_gt: GeoTransform = dataset.spatial_grid.geo_transform(); @@ -408,6 +412,8 @@ async fn create_dataset_api( .await }, &format!("Create dataset '{dataset_name}'"), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; @@ -469,6 +475,8 @@ async fn share_dataset_api( .await }, &format!("Add permission for dataset '{dataset_name}'"), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; } @@ -551,6 +559,8 @@ async fn create_harvest_layer_collections( .await }, &format!("Create layer '{layer_name}'"), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; @@ -571,9 +581,7 @@ async fn create_layer_collection_api( let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); let client = &api_config.client; - if let Some(existing_id) = - find_child_collection_by_name(api_config, parent_id, name).await? - { + if let Some(existing_id) = find_child_collection_by_name(api_config, parent_id, name).await? { if params.verbose { info!("Found existing layer collection '{name}'"); } @@ -601,6 +609,8 @@ async fn create_layer_collection_api( .await }, &format!("Create layer collection '{name}'"), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; @@ -636,6 +646,8 @@ async fn find_child_collection_by_name( .await }, &format!("List child collections of {parent_id}"), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; @@ -694,6 +706,8 @@ async fn share_layer_collection_api( .await }, &format!("Share collection with role {}", permission.role_id), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; } @@ -701,10 +715,7 @@ async fn share_layer_collection_api( Ok(()) } -async fn share_layer_api( - api_config: &ApiConfig, - layer_id: &LayerId, -) -> Result<(), anyhow::Error> { +async fn share_layer_api(api_config: &ApiConfig, layer_id: &LayerId) -> Result<(), anyhow::Error> { let geo_engine_url = &api_config.base_path; let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); let client = &api_config.client; @@ -740,6 +751,8 @@ async fn share_layer_api( .await }, &format!("Share layer with role {}", permission.role_id), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; } @@ -751,9 +764,6 @@ async fn share_layer_api( // Pagination // --------------------------------------------------------------------------- -const MAX_RETRIES: u32 = 10; -const INITIAL_RETRY_DELAY_MS: u64 = 1000; - #[derive(Debug, Clone)] enum QueryState { FirstPage { @@ -823,6 +833,8 @@ async fn query_item_collection_internal( .await }, "Query STAC first page", + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; @@ -840,6 +852,8 @@ async fn query_item_collection_internal( let item_collection: stac::ItemCollection = retry_http( || async { client.get(next_url).send().await?.json().await }, "Query STAC next page", + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await?; @@ -857,36 +871,6 @@ async fn query_item_collection_internal( } } -// --------------------------------------------------------------------------- -// HTTP retry helper -// --------------------------------------------------------------------------- - -async fn retry_http(mut operation: F, operation_name: &str) -> Result -where - F: FnMut() -> Fut, - Fut: std::future::Future>, - E: std::fmt::Display, -{ - let mut attempt = 0; - loop { - match operation().await { - Ok(result) => return Ok(result), - Err(err) => { - attempt += 1; - if attempt >= MAX_RETRIES { - error!("{operation_name} failed after {MAX_RETRIES} attempts: {err}"); - return Err(err); - } - let delay = Duration::from_millis(INITIAL_RETRY_DELAY_MS * 2_u64.pow(attempt - 1)); - warn!( - "{operation_name} failed (attempt {attempt}/{MAX_RETRIES}): {err}. Retrying in {delay:?}..." - ); - tokio::time::sleep(delay).await; - } - } - } -} - // --------------------------------------------------------------------------- // Authentication helper // --------------------------------------------------------------------------- @@ -911,6 +895,8 @@ async fn create_api_config( let session = retry_http( || geoengine_api_client::apis::session_api::login_handler(&config, credentials.clone()), "Login to Geo Engine", + &RetryPolicy::new(), + apis_error_status, ) .await .context("Failed to authenticate")?; @@ -941,13 +927,7 @@ async fn setup_datasets( } if !dataset_exists_api(api_config, &dataset_name).await? { - create_dataset_api( - api_config, - &dataset_name, - dataset, - ¶ms.volume_name, - ) - .await?; + create_dataset_api(api_config, &dataset_name, dataset, ¶ms.volume_name).await?; created_datasets.push((idx, dataset.clone())); } } @@ -1095,9 +1075,7 @@ async fn upload_tiles_to_datasets( let response = retry_http( || async { client - .post(format!( - "{geo_engine_url}/dataset/{dataset_name}/tiles", - )) + .post(format!("{geo_engine_url}/dataset/{dataset_name}/tiles")) .header("Content-Type", "application/json") .header("Authorization", format!("Bearer {session_id}")) .json(chunk) @@ -1105,6 +1083,8 @@ async fn upload_tiles_to_datasets( .await }, &format!("Add tiles to dataset '{dataset_name}'"), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), ) .await .with_context(|| format!("Failed to add tiles to dataset '{dataset_name}'"))?; @@ -1139,7 +1119,7 @@ fn try_create_tile_for_band( let (_asset_key, asset) = item .assets .iter() - .find(|(_, a)| a.title.as_deref() == Some(&band_def.asset_title))?; + .find(|(_, a)| a.title.as_deref() == Some(&band_def.asset_band.asset_title))?; // Check data type matches if let Some(asset_dt) = common::data_type_from_asset_v1_1_0_fallback(asset) @@ -1179,8 +1159,10 @@ fn try_create_tile_for_band( let (height, width) = common::proj_shape_from_fields(&asset.additional_fields)?; - let rasterband_channel = - common::rasterband_channel_for_dataset_band(asset, band_def.band_name.as_deref())?; + let rasterband_channel = common::rasterband_channel_for_dataset_band( + asset, + band_def.asset_band.band_name.as_deref(), + )?; let grid_bounds = GridBoundingBox2D::new( GridIdx2D::new([0, 0]), diff --git a/geoengine/services/src/cli/stac_harvester/mod.rs b/geoengine/services/src/cli/stac_harvester/mod.rs index 7ab60a16f6..430cd0f3db 100644 --- a/geoengine/services/src/cli/stac_harvester/mod.rs +++ b/geoengine/services/src/cli/stac_harvester/mod.rs @@ -19,6 +19,11 @@ pub use discover::StacDiscoverMapping; pub use harvest::StacHarvest; use clap::{Parser, Subcommand}; +use tracing_subscriber::{ + filter::{LevelFilter, Targets}, + layer::{Layer, SubscriberExt}, + util::SubscriberInitExt, +}; /// STAC harvester for Geo Engine #[derive(Debug, Parser)] @@ -40,8 +45,44 @@ pub enum StacHarvesterCommand { pub async fn stac_harvester(params: StacHarvester) -> Result<(), anyhow::Error> { match params.command { StacHarvesterCommand::DiscoverMapping(discover) => { + init_stac_harvest_logging(discover.verbose); discover::discover_mapping(*discover).await } - StacHarvesterCommand::Harvest(harvest) => harvest::harvest_tiles(*harvest).await, + StacHarvesterCommand::Harvest(harvest) => { + init_stac_harvest_logging(harvest.verbose); + harvest::harvest_tiles(*harvest).await + } } } + +/// Initialize the tracing subscriber for STAC harvesting. +/// +/// Logs go to stderr so stdout stays clean (the `discover-mapping` command +/// emits the mapping JSON on stdout). Only the Geo Engine crates are logged — +/// dependency noise (hyper, reqwest, …) is filtered out. `--verbose` raises +/// the level to DEBUG; the default level is INFO so the progress `info!` +/// messages are visible. +fn init_stac_harvest_logging(verbose: bool) { + let geoengine_level = if verbose { + LevelFilter::DEBUG + } else { + LevelFilter::INFO + }; + + // `Targets` matches target prefixes hierarchically, so `geoengine_services` + // also covers `geoengine_services::cli::stac_harvester::harvest`. + let targets = Targets::new() + .with_target("geoengine_services", geoengine_level) + .with_target("geoengine_operators", geoengine_level) + .with_target("geoengine_datatypes", geoengine_level) + .with_target("geoengine_api_client", geoengine_level) + .with_default(LevelFilter::OFF); + + let _ = tracing_subscriber::registry() + .with( + tracing_subscriber::fmt::layer() + .with_writer(std::io::stderr) + .with_filter(targets), + ) + .try_init(); +} diff --git a/geoengine/services/src/contexts/migrations/current_schema.sql b/geoengine/services/src/contexts/migrations/current_schema.sql index 24d7cdfa0f..4f3f26d53e 100644 --- a/geoengine/services/src/contexts/migrations/current_schema.sql +++ b/geoengine/services/src/contexts/migrations/current_schema.sql @@ -882,11 +882,16 @@ CREATE TYPE "StacProviderS3Config" AS ( secret_key text ); -CREATE TYPE "StacProviderDatasetBand" AS ( +CREATE TYPE "StacAssetBand" AS ( asset_title text, band_name text ); +CREATE TYPE "StacProviderDatasetBand" AS ( + asset_band "StacAssetBand", + band_descriptor "RasterBandDescriptor" +); + CREATE TYPE "StacProviderDataset" AS ( "name" text, description text, diff --git a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.rs b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.rs new file mode 100644 index 0000000000..330a23f7d6 --- /dev/null +++ b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.rs @@ -0,0 +1,31 @@ +use super::database_migration::{DatabaseVersion, Migration}; +use crate::{ + contexts::migrations::migration_0028_stac_provider::Migration0028StacProvider, error::Result, +}; +use async_trait::async_trait; +use tokio_postgres::Transaction; + +/// This migration bundles the band addressing fields of `StacProviderDatasetBand` +/// into a nested `StacAssetBand` type and adds a `name` attribute of type +/// `RasterBandDescriptor` for the band in the resulting geo engine dataset +/// layer, independent of `asset_title`/`band_name`, which address the band +/// inside the STAC asset files. +pub struct Migration0029StacProviderBandName; + +#[async_trait] +impl Migration for Migration0029StacProviderBandName { + fn prev_version(&self) -> Option { + Some(Migration0028StacProvider.version()) + } + + fn version(&self) -> DatabaseVersion { + "0029_stac_provider_band_name".into() + } + + async fn migrate(&self, tx: &Transaction<'_>) -> Result<()> { + tx.batch_execute(include_str!("migration_0029_stac_provider_band_name.sql")) + .await?; + + Ok(()) + } +} diff --git a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql new file mode 100644 index 0000000000..55dde1e39a --- /dev/null +++ b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql @@ -0,0 +1,100 @@ +-- Bundle the band *addressing* fields into a nested `StacAssetBand` type and +-- add a `band_descriptor` (`RasterBandDescriptor`) for the band in the +-- resulting geo engine dataset layer. +-- +-- Before: "StacProviderDatasetBand" (asset_title text, band_name text) +-- After: "StacAssetBand" (asset_title text, band_name text) +-- "StacProviderDatasetBand" (asset_band "StacAssetBand", +-- band_descriptor "RasterBandDescriptor") +-- +-- `asset_title`/`band_name` *address* a band inside the STAC asset files (which +-- asset file, and which raster channel within it). `band_descriptor` is the +-- `RasterBandDescriptor` of the geo engine dataset layer band, populated with +-- the naming fallback ("use band_name, then asset_title") and a unitless +-- measurement. + +CREATE TYPE "StacAssetBand" AS ( + asset_title text, + band_name text +); + +ALTER TYPE "StacProviderDatasetBand" ADD ATTRIBUTE asset_band "StacAssetBand"; +ALTER TYPE "StacProviderDatasetBand" ADD ATTRIBUTE band_descriptor "RasterBandDescriptor"; + +-- Migrate existing stored STAC provider definitions: bundle the flat +-- `asset_title`/`band_name` attributes into the new `asset_band` attribute. +CREATE FUNCTION pg_temp.stac_migrate_bands( + bands "StacProviderDatasetBand" [] +) RETURNS "StacProviderDatasetBand" [] AS $$ +DECLARE + b "StacProviderDatasetBand"; + new_bands "StacProviderDatasetBand" [] := '{}'; +BEGIN + FOREACH b IN ARRAY bands LOOP + new_bands := array_append( + new_bands, + ROW( + NULL, + NULL, + ROW((b).asset_title, (b).band_name)::"StacAssetBand", + ROW( + COALESCE((b).band_name, (b).asset_title), + ROW(NULL, NULL)::"Measurement" + )::"RasterBandDescriptor" + )::"StacProviderDatasetBand" + ); + END LOOP; + RETURN new_bands; +END; +$$ LANGUAGE plpgsql; + +CREATE FUNCTION pg_temp.stac_migrate_provider_def( + def "StacDataProviderDefinition" +) RETURNS "StacDataProviderDefinition" AS $$ +DECLARE + d "StacProviderDataset"; + new_datasets "StacProviderDataset" [] := '{}'; +BEGIN + FOREACH d IN ARRAY (def).datasets LOOP + new_datasets := array_append( + new_datasets, + ROW( + (d).name, + (d).description, + (d).data_type, + (d).resolution, + (d).projection, + (d).spatial_grid, + pg_temp.stac_migrate_bands((d).bands) + )::"StacProviderDataset" + ); + END LOOP; + RETURN ROW( + (def).name, + (def).id, + (def).description, + (def).priority, + (def).api_url, + (def).collection_name, + (def).s3_config, + (def).time_dimension, + new_datasets, + (def).page_limit, + (def).query_timeout_secs + )::"StacDataProviderDefinition"; +END; +$$ LANGUAGE plpgsql; + +UPDATE layer_providers +SET definition = ROW( + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + pg_temp.stac_migrate_provider_def((definition).stac_data_provider_definition) +)::"DataProviderDefinition" +WHERE (definition).stac_data_provider_definition IS NOT NULL; + +-- Drop the now-redundant flat addressing attributes. +ALTER TYPE "StacProviderDatasetBand" DROP ATTRIBUTE asset_title; +ALTER TYPE "StacProviderDatasetBand" DROP ATTRIBUTE band_name; + +DROP FUNCTION pg_temp.stac_migrate_bands; +DROP FUNCTION pg_temp.stac_migrate_provider_def; diff --git a/geoengine/services/src/contexts/migrations/mod.rs b/geoengine/services/src/contexts/migrations/mod.rs index 5ef9d6773b..400213effc 100644 --- a/geoengine/services/src/contexts/migrations/mod.rs +++ b/geoengine/services/src/contexts/migrations/mod.rs @@ -13,6 +13,7 @@ pub use crate::contexts::migrations::{ migration_0025_time_descriptor::Migration0025TimeDescriptor, migration_0027_tile_z_index::Migration0027TileZIndex, migration_0028_stac_provider::Migration0028StacProvider, + migration_0029_stac_provider_band_name::Migration0029StacProviderBandName, }; pub use database_migration::{ DatabaseVersion, Migration, MigrationResult, initialize_database, migrate_database, @@ -33,6 +34,7 @@ mod migration_0025_time_descriptor; mod migration_0026_gdal_tiles; mod migration_0027_tile_z_index; mod migration_0028_stac_provider; +mod migration_0029_stac_provider_band_name; #[cfg(test)] mod schema_info; @@ -67,6 +69,7 @@ pub fn all_migrations() -> Vec> { Box::new(Migration0026GdalTiles), Box::new(Migration0027TileZIndex), Box::new(Migration0028StacProvider), + Box::new(Migration0029StacProviderBandName), ] } diff --git a/geoengine/services/src/datasets/external/stac/common.rs b/geoengine/services/src/datasets/external/stac/common.rs index ed175ef630..e537bf6244 100644 --- a/geoengine/services/src/datasets/external/stac/common.rs +++ b/geoengine/services/src/datasets/external/stac/common.rs @@ -48,6 +48,12 @@ pub fn geo_transform_from_fields( return None; } + // A `GeoTransform` requires non-zero pixel sizes. Some STAC catalogs encode + // angular/QA assets with a zero pixel height, so skip those instead of panicking. + if values[0] == 0.0 || values[4] == 0.0 { + return None; + } + // GDAL geo-transform: [origin_x, pixel_width, rotation, origin_y, rotation, pixel_height] let gdal_geotransform: GdalGeoTransform = [ values[2], // origin_x @@ -280,23 +286,36 @@ pub struct EoBand { /// Map a GDAL raster band channel index for a dataset band within an asset. /// /// If the asset has no `bands` metadata, returns channel 1 (single-band asset). -/// If the asset has bands, matches by `band_name` against asset band names. -/// If the asset has exactly one band and no `band_name` is required, returns -/// channel 1 (single-band asset treated the same as no band metadata). +/// If the asset has exactly one band, returns channel 1: a single-band asset has +/// only one GDAL raster band, and STAC servers commonly label it with a short +/// code (e.g. `B10`) while the configured band name may be the human-readable +/// asset title (e.g. `Thermal Infrared 10.9 (band 10) - 100m`), so a strict name +/// match would wrongly skip the asset. +/// If the asset has multiple bands, matches by `band_name` against asset band +/// names to select the channel. /// Returns `None` if the required band is not found. pub fn rasterband_channel_for_dataset_band( asset: &stac::Asset, required_band_name: Option<&str>, ) -> Option { - if asset.bands.is_empty() || (asset.bands.len() == 1 && required_band_name.is_none()) { - if required_band_name.is_some() && asset.bands.is_empty() { - tracing::warn!( - "STAC asset with href {} does not include bands, but dataset configuration requires a band name. Skipping asset.", - asset.href + if asset.bands.is_empty() { + // No `bands` metadata: assume a single-band raster and map to channel 1. + // STAC servers commonly omit `bands` for single-band products (e.g. + // Sentinel-2 SCL/CLD/SNW, Landsat QA bands), even when the mapping + // configures an explicit band name. Skipping here would silently lose + // the band, so proceed with channel 1 (the single-band path below does + // the same regardless of the requested name). + if required_band_name.is_some() { + tracing::debug!( + "STAC asset with href {} does not include bands, but dataset configuration requires band name {:?}. Assuming single-band raster (channel 1).", + asset.href, + required_band_name ); - return None; } + return Some(1); + } + if asset.bands.len() == 1 { return Some(1); } @@ -325,21 +344,38 @@ pub fn rasterband_channel_for_dataset_band( Some(asset_band_idx + 1) } +/// Parsed band information from a STAC 1.1.0 asset. +/// +/// Keeps the asset's display title separate from the individual band names so +/// callers can match an asset by its real STAC title and select the raster +/// channel by band name, without encoding the band name into the title (e.g. +/// `True color image [B02]`). +#[derive(Debug, Clone, PartialEq)] +pub struct AssetBandInfo { + pub asset_title: String, + pub band_names: Vec, +} + /// Derive band names from a STAC 1.1.0 `Asset`, using the `bands` field. -pub fn band_names_from_asset_v1_1_0(asset: &stac::Asset) -> Result, String> { +/// +/// For assets without `bands` metadata or with exactly one band, the single +/// band is named after the asset title. For multi-band assets the individual +/// STAC band names (e.g. `B04`) are returned, so the mapping can reference the +/// exact raster channel while keeping the real asset title. +pub fn band_names_from_asset_v1_1_0(asset: &stac::Asset) -> Result { let asset_title = asset .title .as_deref() - .ok_or_else(|| "Missing title in asset metadata".to_string())?; + .ok_or_else(|| "Missing title in asset metadata".to_string())? + .to_string(); let bands = &asset.bands; - if bands.is_empty() { - return Ok(vec![asset_title.to_string()]); - } - - if bands.len() == 1 { - return Ok(vec![asset_title.to_string()]); + if bands.is_empty() || bands.len() == 1 { + return Ok(AssetBandInfo { + asset_title: asset_title.clone(), + band_names: vec![asset_title], + }); } let mut names = Vec::new(); @@ -347,18 +383,24 @@ pub fn band_names_from_asset_v1_1_0(asset: &stac::Asset) -> Result, let Some(band_name) = &band.name else { return Err("Band is missing name for multi-band asset".to_string()); }; - names.push(format!("{asset_title} [{band_name}]")); + names.push(band_name.clone()); } - Ok(names) + Ok(AssetBandInfo { + asset_title, + band_names: names, + }) } /// Derive band names from a STAC 1.1.0 `ItemAsset`, using the `bands` additional field. -pub fn band_names_from_item_asset_v1_1_0(asset: &stac::ItemAsset) -> Result, String> { +/// +/// See [`band_names_from_asset_v1_1_0`] for the naming semantics. +pub fn band_names_from_item_asset_v1_1_0(asset: &stac::ItemAsset) -> Result { let asset_title = asset .title .as_deref() - .ok_or_else(|| "Missing title in asset metadata".to_string())?; + .ok_or_else(|| "Missing title in asset metadata".to_string())? + .to_string(); let band_names = asset .additional_fields @@ -366,15 +408,17 @@ pub fn band_names_from_item_asset_v1_1_0(asset: &stac::ItemAsset) -> Result Result geoengine_operators::util::Result { + let bands = RasterBandDescriptors::new( + self.dataset + .bands + .iter() + .map(|b| b.band_descriptor.clone()) + .collect(), + )?; + Ok(RasterResultDescriptor { data_type: self.dataset.data_type, spatial_reference: self.dataset.projection.into(), @@ -264,7 +272,7 @@ impl dimension: self.time_dimension, }, spatial_grid: self.dataset.spatial_grid, - bands: RasterBandDescriptors::new_multiple_bands(self.dataset.bands.len() as u32), + bands, }) } @@ -507,13 +515,13 @@ impl StacMultiBandMetaData { common::gdal_config_options_for_file_path(&file_path, self.s3_config.as_ref()); for (dataset_band_idx, dataset_band) in self.dataset.bands.iter().enumerate() { - if dataset_band.asset_title != asset_title { + if dataset_band.asset_band.asset_title != asset_title { continue; } let Some(rasterband_channel) = common::rasterband_channel_for_dataset_band( asset, - dataset_band.band_name.as_deref(), + dataset_band.asset_band.band_name.as_deref(), ) else { continue; }; @@ -822,7 +830,8 @@ mod tests { use geoengine_datatypes::util::Identifier; use geoengine_operators::engine::SpatialGridDescriptor; use geoengine_operators::engine::{ - MetaData, MetaDataProvider, RasterResultDescriptor, WorkflowOperatorPath, + MetaData, MetaDataProvider, RasterBandDescriptor, RasterResultDescriptor, + WorkflowOperatorPath, }; use geoengine_operators::source::{ MultiBandGdalLoadingInfo, MultiBandGdalLoadingInfoQueryRectangle, @@ -861,12 +870,18 @@ mod tests { ), bands: vec![ crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "NIR 1 (band 8) - 10m".to_owned(), - band_name: Some("B08".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "NIR 1 (band 8) - 10m".to_owned(), + band_name: Some("B08".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("B08".to_owned()), }, crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "Red (band 4) - 10m".to_owned(), - band_name: Some("B04".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "Red (band 4) - 10m".to_owned(), + band_name: Some("B04".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("B04".to_owned()), }, ], }], @@ -1060,24 +1075,39 @@ mod tests { ), bands: vec![ crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "Blue (band 2) - 10m".to_owned(), - band_name: Some("B02".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "Blue (band 2) - 10m".to_owned(), + band_name: Some("B02".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("B02".to_owned()), }, crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "Green (band 3) - 10m".to_owned(), - band_name: Some("B03".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "Green (band 3) - 10m".to_owned(), + band_name: Some("B03".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("B03".to_owned()), }, crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "Water vapour (WVP) - 10m".to_owned(), - band_name: Some("WVP".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "Water vapour (WVP) - 10m".to_owned(), + band_name: Some("WVP".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("WVP".to_owned()), }, crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "NIR 1 (band 8) - 10m".to_owned(), - band_name: Some("B08".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "NIR 1 (band 8) - 10m".to_owned(), + band_name: Some("B08".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("B08".to_owned()), }, crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "Red (band 4) - 10m".to_owned(), - band_name: Some("B04".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "Red (band 4) - 10m".to_owned(), + band_name: Some("B04".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("B04".to_owned()), }, ], }, @@ -1097,12 +1127,18 @@ mod tests { ), bands: vec![ crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "Aerosol optical thickness (AOT) - 20m".to_owned(), - band_name: Some("AOT".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "Aerosol optical thickness (AOT) - 20m".to_owned(), + band_name: Some("AOT".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("AOT".to_owned()), }, crate::datasets::external::stac::StacProviderDatasetBand { - asset_title: "Scene classification map (SCL) - 20m".to_owned(), - band_name: Some("SCL".to_owned()), + asset_band: crate::datasets::external::stac::StacAssetBand { + asset_title: "Scene classification map (SCL) - 20m".to_owned(), + band_name: Some("SCL".to_owned()), + }, + band_descriptor: RasterBandDescriptor::new_unitless("SCL".to_owned()), }, ], }, diff --git a/geoengine/services/src/datasets/external/stac/mod.rs b/geoengine/services/src/datasets/external/stac/mod.rs index f4dbb8cd30..592cdc7847 100644 --- a/geoengine/services/src/datasets/external/stac/mod.rs +++ b/geoengine/services/src/datasets/external/stac/mod.rs @@ -8,7 +8,7 @@ use geoengine_datatypes::dataset::DataProviderId; use geoengine_datatypes::primitives::{SpatialResolution, TimeDimension}; use geoengine_datatypes::raster::RasterDataType; use geoengine_datatypes::spatial_reference::SpatialReference; -use geoengine_operators::engine::SpatialGridDescriptor; +use geoengine_operators::engine::{RasterBandDescriptor, SpatialGridDescriptor}; use postgres_types::{FromSql, ToSql}; use serde::{Deserialize, Serialize}; use std::sync::Arc; @@ -72,13 +72,62 @@ pub struct StacProviderDataset { pub bands: Vec, } +/// A band inside a STAC asset. +/// +/// *Addresses* the band in the asset files of a STAC collection: +/// [`asset_title`](StacAssetBand::asset_title) selects the asset file, +/// [`band_name`](StacAssetBand::band_name) selects the raster channel within +/// it. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSql, FromSql)] -#[postgres(name = "StacProviderDatasetBand")] -pub struct StacProviderDatasetBand { +#[postgres(name = "StacAssetBand")] +pub struct StacAssetBand { + /// The title of the STAC asset in the collection. Used to *address* the + /// asset file that contains this band (matched against the STAC asset + /// `title`). pub asset_title: String, + /// The name of the band *within* the asset file. + /// + /// Matches the STAC `bands[].name` metadata (e.g. `B04`) to select the + /// raster channel inside the asset. `None` for single-band assets. pub band_name: Option, } +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSql, FromSql)] +#[postgres(name = "StacProviderDatasetBand")] +pub struct StacProviderDatasetBand { + /// The band inside the STAC asset that this dataset band reads from. + /// + /// This is the *addressing* information: which asset file + /// ([`StacAssetBand::asset_title`]) and which raster channel within it + /// ([`StacAssetBand::band_name`]). + pub asset_band: StacAssetBand, + /// The band descriptor of the resulting geo engine dataset layer. + /// + /// This is independent of [`Self::asset_band`], which *addresses* the band + /// inside the asset files. During discovery it is populated with the same + /// naming fallback (`asset_band.band_name`, then + /// `asset_band.asset_title`) and a unitless measurement. + pub band_descriptor: RasterBandDescriptor, +} + +impl StacProviderDatasetBand { + /// Create a dataset band whose resulting result-descriptor band is + /// unitless and named after the asset band using the discovery fallback + /// ([`StacAssetBand::band_name`], else [`StacAssetBand::asset_title`]). + pub fn new_unitless(asset_band: StacAssetBand) -> Self { + let band_descriptor = RasterBandDescriptor::new_unitless( + asset_band + .band_name + .clone() + .unwrap_or_else(|| asset_band.asset_title.clone()), + ); + Self { + asset_band, + band_descriptor, + } + } +} + #[async_trait] impl DataProviderDefinition for StacDataProviderDefinition { async fn initialize(self: Box, _db: D) -> crate::error::Result> { diff --git a/geoengine/services/src/datasets/upload.rs b/geoengine/services/src/datasets/upload.rs index 64987ef57f..91de9971ec 100644 --- a/geoengine/services/src/datasets/upload.rs +++ b/geoengine/services/src/datasets/upload.rs @@ -56,6 +56,13 @@ impl<'de> Deserialize<'de> for VolumeName { impl AdjustFilePath for Volume { fn adjust_file_path(&self, file_path: &Path) -> Result { + if self.name.0 == "external" { + // external data file path must not be adjusted + // TODO: remove this once we have proper volume management + // TODO: ensure the file path actually points to external data + return Ok(file_path.to_path_buf()); + } + let _file_name = file_path.file_name().ok_or(error::Error::PathIsNotAFile)?; path_with_base_path(&self.path, file_path) diff --git a/geoengine/services/src/util/mod.rs b/geoengine/services/src/util/mod.rs index 7f841913c2..e24129559d 100644 --- a/geoengine/services/src/util/mod.rs +++ b/geoengine/services/src/util/mod.rs @@ -24,6 +24,7 @@ pub mod openapi_visitors; pub mod operators; pub mod parsing; pub mod postgres; +pub mod retry; pub mod sentinel_2_utm_zones; pub mod server; // TODO: refactor to be gated by `#[cfg(test)]` diff --git a/geoengine/services/src/util/retry.rs b/geoengine/services/src/util/retry.rs new file mode 100644 index 0000000000..a71d5c4407 --- /dev/null +++ b/geoengine/services/src/util/retry.rs @@ -0,0 +1,187 @@ +//! Generic HTTP retry helper with configurable stop/retry conditions. +//! +//! [`retry_http`] retries an async operation with exponential backoff until it +//! succeeds, the configured attempt limit is reached, or the error matches a +//! *terminal* condition. Terminal conditions can be declared by HTTP status +//! code and/or by substring in the error's display message, either as "stop +//! retrying on these" or as "only retry on these" (whitelist). + +use std::time::Duration; + +use tracing::{error, warn}; + +/// Default number of attempts before giving up. +pub const DEFAULT_MAX_RETRIES: u32 = 10; +/// Default initial backoff delay in milliseconds (doubles after each attempt). +pub const DEFAULT_INITIAL_DELAY_MS: u64 = 1000; + +/// Controls when [`retry_http`] keeps retrying and when it gives up. +/// +/// Two complementary mechanisms are available, each applicable to HTTP status +/// codes and to substrings of the error's `Display` message: +/// +/// - `stop_on_*`: errors matching these are terminal and returned immediately. +/// - `retry_on_*`: if non-empty, a whitelist — only matching errors are +/// retried, everything else is terminal. +/// +/// With no conditions set, every error is retried (plain exponential backoff). +#[derive(Debug, Clone)] +pub struct RetryPolicy { + pub max_retries: u32, + pub initial_delay_ms: u64, + /// HTTP status codes on which to stop retrying immediately. + pub stop_on_status: Vec, + /// HTTP status codes to retry (whitelist; empty = retry all non-terminal errors). + pub retry_on_status: Vec, + /// Error-message substrings on which to stop retrying immediately. + pub stop_on_message_contains: Vec, + /// Error-message substrings to retry (whitelist; empty = retry all non-terminal errors). + pub retry_on_message_contains: Vec, +} + +impl Default for RetryPolicy { + fn default() -> Self { + Self { + max_retries: DEFAULT_MAX_RETRIES, + initial_delay_ms: DEFAULT_INITIAL_DELAY_MS, + stop_on_status: Vec::new(), + retry_on_status: Vec::new(), + stop_on_message_contains: Vec::new(), + retry_on_message_contains: Vec::new(), + } + } +} + +impl RetryPolicy { + pub fn new() -> Self { + Self::default() + } + + #[must_use] + pub fn max_retries(mut self, max_retries: u32) -> Self { + self.max_retries = max_retries; + self + } + + #[must_use] + pub fn initial_delay_ms(mut self, initial_delay_ms: u64) -> Self { + self.initial_delay_ms = initial_delay_ms; + self + } + + /// Abort retrying on the given HTTP status codes (e.g. `[400, 404]`). + #[must_use] + pub fn stop_on_status(mut self, codes: &[u16]) -> Self { + self.stop_on_status.extend_from_slice(codes); + self + } + + /// Only retry on the given HTTP status codes; everything else is terminal. + #[must_use] + pub fn retry_on_status(mut self, codes: &[u16]) -> Self { + self.retry_on_status.extend_from_slice(codes); + self + } + + /// Abort retrying when the error message contains any of the given substrings. + #[must_use] + pub fn stop_on_message(mut self, substrings: &[&str]) -> Self { + self.stop_on_message_contains + .extend(substrings.iter().map(ToString::to_string)); + self + } + + /// Only retry when the error message contains any of the given substrings. + #[must_use] + pub fn retry_on_message(mut self, substrings: &[&str]) -> Self { + self.retry_on_message_contains + .extend(substrings.iter().map(ToString::to_string)); + self + } +} + +/// Retry an operation with exponential backoff, honouring `policy`. +/// +/// `extract_status` maps an error to an optional HTTP status code so the policy +/// can match `stop_on_status`/`retry_on_status`; the error's `Display` output is +/// used for the message-substring conditions. +pub async fn retry_http( + mut operation: F, + operation_name: &str, + policy: &RetryPolicy, + extract_status: impl Fn(&E) -> Option, +) -> Result +where + F: FnMut() -> Fut, + Fut: std::future::Future>, + E: std::fmt::Display, +{ + let mut attempt = 0; + loop { + match operation().await { + Ok(result) => return Ok(result), + Err(err) => { + if is_terminal(&err, policy, &extract_status) { + return Err(err); + } + + attempt += 1; + if attempt >= policy.max_retries { + error!( + "{operation_name} failed after {} attempts: {err}", + policy.max_retries + ); + return Err(err); + } + + let delay = Duration::from_millis(policy.initial_delay_ms * 2_u64.pow(attempt - 1)); + warn!( + "{operation_name} failed (attempt {attempt}/{}): {err}. Retrying in {delay:?}...", + policy.max_retries + ); + tokio::time::sleep(delay).await; + } + } + } +} + +/// Decide whether `err` should abort the retry loop under `policy`. +fn is_terminal( + err: &E, + policy: &RetryPolicy, + extract_status: &impl Fn(&E) -> Option, +) -> bool { + let status = extract_status(err); + let message = err.to_string(); + + // Explicit stop conditions always win. + if let Some(status) = status + && policy.stop_on_status.contains(&status) + { + return true; + } + if policy + .stop_on_message_contains + .iter() + .any(|needle| message.contains(needle.as_str())) + { + return true; + } + + // Whitelist conditions: if configured, retry only on matches. + let status_allows_retry = if policy.retry_on_status.is_empty() { + true + } else { + status.is_some_and(|s| policy.retry_on_status.contains(&s)) + }; + let message_allows_retry = if policy.retry_on_message_contains.is_empty() { + true + } else { + policy + .retry_on_message_contains + .iter() + .any(|needle| message.contains(needle.as_str())) + }; + + !(status_allows_retry && message_allows_retry) +} diff --git a/geoengine/test_data/provider_defs_api/stac_sentinel2.json b/geoengine/test_data/provider_defs_api/stac_sentinel2.json index 8835e3e080..eb058e257c 100644 --- a/geoengine/test_data/provider_defs_api/stac_sentinel2.json +++ b/geoengine/test_data/provider_defs_api/stac_sentinel2.json @@ -55,28 +55,76 @@ }, "bands": [ { - "assetTitle": "Aerosol optical thickness (AOT) - 10m", - "bandName": "AOT" + "assetBand": { + "assetTitle": "Aerosol optical thickness (AOT) - 10m", + "bandName": "AOT" + }, + "bandDescriptor": { + "name": "AOT", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Blue (band 2) - 10m", - "bandName": "B02" + "assetBand": { + "assetTitle": "Blue (band 2) - 10m", + "bandName": "B02" + }, + "bandDescriptor": { + "name": "B02", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Green (band 3) - 10m", - "bandName": "B03" + "assetBand": { + "assetTitle": "Green (band 3) - 10m", + "bandName": "B03" + }, + "bandDescriptor": { + "name": "B03", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "NIR 1 (band 8) - 10m", - "bandName": "B08" + "assetBand": { + "assetTitle": "NIR 1 (band 8) - 10m", + "bandName": "B08" + }, + "bandDescriptor": { + "name": "B08", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red (band 4) - 10m", - "bandName": "B04" + "assetBand": { + "assetTitle": "Red (band 4) - 10m", + "bandName": "B04" + }, + "bandDescriptor": { + "name": "B04", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Water vapour (WVP) - 10m", - "bandName": "WVP" + "assetBand": { + "assetTitle": "Water vapour (WVP) - 10m", + "bandName": "WVP" + }, + "bandDescriptor": { + "name": "WVP", + "measurement": { + "type": "unitless" + } + } } ] }, @@ -114,52 +162,148 @@ }, "bands": [ { - "assetTitle": "Aerosol optical thickness (AOT) - 20m", - "bandName": "AOT" + "assetBand": { + "assetTitle": "Aerosol optical thickness (AOT) - 20m", + "bandName": "AOT" + }, + "bandDescriptor": { + "name": "AOT", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Blue (band 2) - 20m", - "bandName": "B02" + "assetBand": { + "assetTitle": "Blue (band 2) - 20m", + "bandName": "B02" + }, + "bandDescriptor": { + "name": "B02", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Coastal aerosol (band 1) - 20m", - "bandName": "B01" + "assetBand": { + "assetTitle": "Coastal aerosol (band 1) - 20m", + "bandName": "B01" + }, + "bandDescriptor": { + "name": "B01", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Green (band 3) - 20m", - "bandName": "B03" + "assetBand": { + "assetTitle": "Green (band 3) - 20m", + "bandName": "B03" + }, + "bandDescriptor": { + "name": "B03", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "NIR 2 (band 8A) - 20m", - "bandName": "B8A" + "assetBand": { + "assetTitle": "NIR 2 (band 8A) - 20m", + "bandName": "B8A" + }, + "bandDescriptor": { + "name": "B8A", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red (band 4) - 20m", - "bandName": "B04" + "assetBand": { + "assetTitle": "Red (band 4) - 20m", + "bandName": "B04" + }, + "bandDescriptor": { + "name": "B04", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red edge 1 (band 5) - 20m", - "bandName": "B05" + "assetBand": { + "assetTitle": "Red edge 1 (band 5) - 20m", + "bandName": "B05" + }, + "bandDescriptor": { + "name": "B05", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red edge 2 (band 6) - 20m", - "bandName": "B06" + "assetBand": { + "assetTitle": "Red edge 2 (band 6) - 20m", + "bandName": "B06" + }, + "bandDescriptor": { + "name": "B06", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red edge 3 (band 7) - 20m", - "bandName": "B07" + "assetBand": { + "assetTitle": "Red edge 3 (band 7) - 20m", + "bandName": "B07" + }, + "bandDescriptor": { + "name": "B07", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "SWIR 1 (band 11) - 20m", - "bandName": "B11" + "assetBand": { + "assetTitle": "SWIR 1 (band 11) - 20m", + "bandName": "B11" + }, + "bandDescriptor": { + "name": "B11", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "SWIR 2 (band 12) - 20m", - "bandName": "B12" + "assetBand": { + "assetTitle": "SWIR 2 (band 12) - 20m", + "bandName": "B12" + }, + "bandDescriptor": { + "name": "B12", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Water vapour (WVP) - 20m", - "bandName": "WVP" + "assetBand": { + "assetTitle": "Water vapour (WVP) - 20m", + "bandName": "WVP" + }, + "bandDescriptor": { + "name": "WVP", + "measurement": { + "type": "unitless" + } + } } ] }, @@ -197,56 +341,160 @@ }, "bands": [ { - "assetTitle": "Aerosol optical thickness (AOT) - 60m", - "bandName": "AOT" + "assetBand": { + "assetTitle": "Aerosol optical thickness (AOT) - 60m", + "bandName": "AOT" + }, + "bandDescriptor": { + "name": "AOT", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Blue (band 2) - 60m", - "bandName": "B02" + "assetBand": { + "assetTitle": "Blue (band 2) - 60m", + "bandName": "B02" + }, + "bandDescriptor": { + "name": "B02", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Coastal aerosol (band 1) - 60m", - "bandName": "B01" + "assetBand": { + "assetTitle": "Coastal aerosol (band 1) - 60m", + "bandName": "B01" + }, + "bandDescriptor": { + "name": "B01", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Green (band 3) - 60m", - "bandName": "B03" + "assetBand": { + "assetTitle": "Green (band 3) - 60m", + "bandName": "B03" + }, + "bandDescriptor": { + "name": "B03", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "NIR 2 (band 8A) - 60m", - "bandName": "B8A" + "assetBand": { + "assetTitle": "NIR 2 (band 8A) - 60m", + "bandName": "B8A" + }, + "bandDescriptor": { + "name": "B8A", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "NIR 3 (band 9) - 60m", - "bandName": "B09" + "assetBand": { + "assetTitle": "NIR 3 (band 9) - 60m", + "bandName": "B09" + }, + "bandDescriptor": { + "name": "B09", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red (band 4) - 60m", - "bandName": "B04" + "assetBand": { + "assetTitle": "Red (band 4) - 60m", + "bandName": "B04" + }, + "bandDescriptor": { + "name": "B04", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red edge 1 (band 5) - 60m", - "bandName": "B05" + "assetBand": { + "assetTitle": "Red edge 1 (band 5) - 60m", + "bandName": "B05" + }, + "bandDescriptor": { + "name": "B05", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red edge 2 (band 6) - 60m", - "bandName": "B06" + "assetBand": { + "assetTitle": "Red edge 2 (band 6) - 60m", + "bandName": "B06" + }, + "bandDescriptor": { + "name": "B06", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Red edge 3 (band 7) - 60m", - "bandName": "B07" + "assetBand": { + "assetTitle": "Red edge 3 (band 7) - 60m", + "bandName": "B07" + }, + "bandDescriptor": { + "name": "B07", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "SWIR 1 (band 11) - 60m", - "bandName": "B11" + "assetBand": { + "assetTitle": "SWIR 1 (band 11) - 60m", + "bandName": "B11" + }, + "bandDescriptor": { + "name": "B11", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "SWIR 2 (band 12) - 60m", - "bandName": "B12" + "assetBand": { + "assetTitle": "SWIR 2 (band 12) - 60m", + "bandName": "B12" + }, + "bandDescriptor": { + "name": "B12", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Water vapour (WVP) - 60m", - "bandName": "WVP" + "assetBand": { + "assetTitle": "Water vapour (WVP) - 60m", + "bandName": "WVP" + }, + "bandDescriptor": { + "name": "WVP", + "measurement": { + "type": "unitless" + } + } } ] }, @@ -284,16 +532,40 @@ }, "bands": [ { - "assetTitle": "True color image", - "bandName": "B02" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B02" + }, + "bandDescriptor": { + "name": "B02", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "True color image", - "bandName": "B03" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B03" + }, + "bandDescriptor": { + "name": "B03", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "True color image", - "bandName": "B04" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B04" + }, + "bandDescriptor": { + "name": "B04", + "measurement": { + "type": "unitless" + } + } } ] }, @@ -331,27 +603,75 @@ }, "bands": [ { - "assetTitle": "Cloud probability (CLD) - 20m", - "bandName": "CLD" + "assetBand": { + "assetTitle": "Cloud probability (CLD) - 20m", + "bandName": "CLD" + }, + "bandDescriptor": { + "name": "CLD", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Scene classification map (SCL) - 20m" + "assetBand": { + "assetTitle": "Scene classification map (SCL) - 20m" + }, + "bandDescriptor": { + "name": "Scene classification map (SCL) - 20m", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Snow probability (SNW) - 20m", - "bandName": "SNW" + "assetBand": { + "assetTitle": "Snow probability (SNW) - 20m", + "bandName": "SNW" + }, + "bandDescriptor": { + "name": "SNW", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "True color image", - "bandName": "B02" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B02" + }, + "bandDescriptor": { + "name": "B02", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "True color image", - "bandName": "B03" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B03" + }, + "bandDescriptor": { + "name": "B03", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "True color image", - "bandName": "B04" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B04" + }, + "bandDescriptor": { + "name": "B04", + "measurement": { + "type": "unitless" + } + } } ] }, @@ -389,27 +709,75 @@ }, "bands": [ { - "assetTitle": "Cloud probability (CLD) - 60m", - "bandName": "CLD" + "assetBand": { + "assetTitle": "Cloud probability (CLD) - 60m", + "bandName": "CLD" + }, + "bandDescriptor": { + "name": "CLD", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Scene classification map (SCL) - 60m" + "assetBand": { + "assetTitle": "Scene classification map (SCL) - 60m" + }, + "bandDescriptor": { + "name": "Scene classification map (SCL) - 60m", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "Snow probability (SNW) - 60m", - "bandName": "SNW" + "assetBand": { + "assetTitle": "Snow probability (SNW) - 60m", + "bandName": "SNW" + }, + "bandDescriptor": { + "name": "SNW", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "True color image", - "bandName": "B02" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B02" + }, + "bandDescriptor": { + "name": "B02", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "True color image", - "bandName": "B03" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B03" + }, + "bandDescriptor": { + "name": "B03", + "measurement": { + "type": "unitless" + } + } }, { - "assetTitle": "True color image", - "bandName": "B04" + "assetBand": { + "assetTitle": "True color image", + "bandName": "B04" + }, + "bandDescriptor": { + "name": "B04", + "measurement": { + "type": "unitless" + } + } } ] } diff --git a/geoengine/test_data/stac_responses/expected-mapping-code-de.json b/geoengine/test_data/stac_responses/expected-mapping-code-de.json index a7a0fe3296..daa85db195 100644 --- a/geoengine/test_data/stac_responses/expected-mapping-code-de.json +++ b/geoengine/test_data/stac_responses/expected-mapping-code-de.json @@ -10,61 +10,155 @@ "timeDimension": { "type": "regular", "origin": 0, - "step": { "granularity": "days", "step": 1 } + "step": { + "granularity": "days", + "step": 1 + } }, "datasets": [ { "name": "sentinel-2-l2a EPSG:32632 U16 10m", "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", "dataType": "U16", - "resolution": { "x": 10.0, "y": 10.0 }, + "resolution": { + "x": 10.0, + "y": 10.0 + }, "projection": "EPSG:32632", "spatialGrid": { "spatialGrid": { "geoTransform": { - "originCoordinate": { "x": 399960.0, "y": 5700000.0 }, + "originCoordinate": { + "x": 399960.0, + "y": 5700000.0 + }, "xPixelSize": 10.0, "yPixelSize": -10.0 }, "gridBounds": { - "topLeftIdx": { "yIdx": 0, "xIdx": 0 }, - "bottomRightIdx": { "yIdx": 10979, "xIdx": 10979 } + "topLeftIdx": { + "yIdx": 0, + "xIdx": 0 + }, + "bottomRightIdx": { + "yIdx": 10979, + "xIdx": 10979 + } } }, "descriptor": "source" }, "bands": [ - { "assetTitle": "Blue (band 2) - 10m", "bandName": null }, - { "assetTitle": "Green (band 3) - 10m", "bandName": null }, - { "assetTitle": "NIR 1 (band 8) - 10m", "bandName": null }, - { "assetTitle": "Red (band 4) - 10m", "bandName": null } + { + "assetBand": { + "assetTitle": "Blue (band 2) - 10m", + "bandName": "Blue (band 2) - 10m" + }, + "bandDescriptor": { + "name": "Blue (band 2) - 10m", + "measurement": { + "type": "unitless" + } + } + }, + { + "assetBand": { + "assetTitle": "Green (band 3) - 10m", + "bandName": "Green (band 3) - 10m" + }, + "bandDescriptor": { + "name": "Green (band 3) - 10m", + "measurement": { + "type": "unitless" + } + } + }, + { + "assetBand": { + "assetTitle": "NIR 1 (band 8) - 10m", + "bandName": "NIR 1 (band 8) - 10m" + }, + "bandDescriptor": { + "name": "NIR 1 (band 8) - 10m", + "measurement": { + "type": "unitless" + } + } + }, + { + "assetBand": { + "assetTitle": "Red (band 4) - 10m", + "bandName": "Red (band 4) - 10m" + }, + "bandDescriptor": { + "name": "Red (band 4) - 10m", + "measurement": { + "type": "unitless" + } + } + } ] }, { "name": "sentinel-2-l2a EPSG:32632 U16 20m", "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", "dataType": "U16", - "resolution": { "x": 20.0, "y": 20.0 }, + "resolution": { + "x": 20.0, + "y": 20.0 + }, "projection": "EPSG:32632", "spatialGrid": { "spatialGrid": { "geoTransform": { - "originCoordinate": { "x": 399960.0, "y": 5700000.0 }, + "originCoordinate": { + "x": 399960.0, + "y": 5700000.0 + }, "xPixelSize": 20.0, "yPixelSize": -20.0 }, "gridBounds": { - "topLeftIdx": { "yIdx": 0, "xIdx": 0 }, - "bottomRightIdx": { "yIdx": 5489, "xIdx": 5489 } + "topLeftIdx": { + "yIdx": 0, + "xIdx": 0 + }, + "bottomRightIdx": { + "yIdx": 5489, + "xIdx": 5489 + } } }, "descriptor": "source" }, "bands": [ - { "assetTitle": "SWIR 1 (band 11) - 20m", "bandName": null }, - { "assetTitle": "SWIR 2 (band 12) - 20m", "bandName": null } + { + "assetBand": { + "assetTitle": "SWIR 1 (band 11) - 20m", + "bandName": "SWIR 1 (band 11) - 20m" + }, + "bandDescriptor": { + "name": "SWIR 1 (band 11) - 20m", + "measurement": { + "type": "unitless" + } + } + }, + { + "assetBand": { + "assetTitle": "SWIR 2 (band 12) - 20m", + "bandName": "SWIR 2 (band 12) - 20m" + }, + "bandDescriptor": { + "name": "SWIR 2 (band 12) - 20m", + "measurement": { + "type": "unitless" + } + } + } ] } ], - "pageLimit": 100 + "pageLimit": 100, + "queryTimeoutSecs": 60 } diff --git a/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json b/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json index 08d2f1a7ef..541b6d794d 100644 --- a/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json +++ b/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json @@ -10,36 +10,96 @@ "timeDimension": { "type": "regular", "origin": 0, - "step": { "granularity": "days", "step": 1 } + "step": { + "granularity": "days", + "step": 1 + } }, "datasets": [ { "name": "landsat-c2-l1 EPSG:32632 U16 30m", "description": "Auto-discovered from STAC collection 'landsat-c2-l1'", "dataType": "U16", - "resolution": { "x": 30.0, "y": 30.0 }, + "resolution": { + "x": 30.0, + "y": 30.0 + }, "projection": "EPSG:32632", "spatialGrid": { "spatialGrid": { "geoTransform": { - "originCoordinate": { "x": 399960.0, "y": 5800020.0 }, + "originCoordinate": { + "x": 399960.0, + "y": 5800020.0 + }, "xPixelSize": 30.0, "yPixelSize": -30.0 }, "gridBounds": { - "topLeftIdx": { "yIdx": 0, "xIdx": 0 }, - "bottomRightIdx": { "yIdx": 7999, "xIdx": 7999 } + "topLeftIdx": { + "yIdx": 0, + "xIdx": 0 + }, + "bottomRightIdx": { + "yIdx": 7999, + "xIdx": 7999 + } } }, "descriptor": "source" }, "bands": [ - { "assetTitle": "Blue (band 2) - 30m", "bandName": null }, - { "assetTitle": "Green (band 3) - 30m", "bandName": null }, - { "assetTitle": "NIR (band 5) - 30m", "bandName": null }, - { "assetTitle": "Red (band 4) - 30m", "bandName": null } + { + "assetBand": { + "assetTitle": "Blue (band 2) - 30m", + "bandName": "Blue (band 2) - 30m" + }, + "bandDescriptor": { + "name": "Blue (band 2) - 30m", + "measurement": { + "type": "unitless" + } + } + }, + { + "assetBand": { + "assetTitle": "Green (band 3) - 30m", + "bandName": "Green (band 3) - 30m" + }, + "bandDescriptor": { + "name": "Green (band 3) - 30m", + "measurement": { + "type": "unitless" + } + } + }, + { + "assetBand": { + "assetTitle": "NIR (band 5) - 30m", + "bandName": "NIR (band 5) - 30m" + }, + "bandDescriptor": { + "name": "NIR (band 5) - 30m", + "measurement": { + "type": "unitless" + } + } + }, + { + "assetBand": { + "assetTitle": "Red (band 4) - 30m", + "bandName": "Red (band 4) - 30m" + }, + "bandDescriptor": { + "name": "Red (band 4) - 30m", + "measurement": { + "type": "unitless" + } + } + } ] } ], - "pageLimit": 100 + "pageLimit": 100, + "queryTimeoutSecs": 60 } diff --git a/openapi.json b/openapi.json index 92a5df68da..582359b21c 100644 --- a/openapi.json +++ b/openapi.json @@ -12231,6 +12231,23 @@ } } }, + "StacAssetBand": { + "type": "object", + "required": [ + "assetTitle" + ], + "properties": { + "assetTitle": { + "type": "string" + }, + "bandName": { + "type": [ + "string", + "null" + ] + } + } + }, "StacDataProviderDefinition": { "type": "object", "required": [ @@ -12241,7 +12258,8 @@ "apiUrl", "collectionName", "timeDimension", - "datasets" + "datasets", + "pageLimit" ], "properties": { "type": { @@ -12291,6 +12309,10 @@ "$ref": "#/components/schemas/StacProviderDataset" } }, + "pageLimit": { + "type": "integer", + "format": "int64" + }, "queryTimeoutSecs": { "type": "integer", "format": "int64", @@ -12339,17 +12361,17 @@ "StacProviderDatasetBand": { "type": "object", "required": [ - "assetTitle" + "assetBand", + "bandDescriptor" ], "properties": { - "assetTitle": { - "type": "string" + "assetBand": { + "$ref": "#/components/schemas/StacAssetBand", + "description": "The band inside the STAC asset that this dataset band reads from\n(addressing: which asset file + which raster channel within it)." }, - "bandName": { - "type": [ - "string", - "null" - ] + "bandDescriptor": { + "$ref": "#/components/schemas/RasterBandDescriptor", + "description": "The band descriptor of the resulting geo engine dataset layer.\n\nIndependent of `assetBand`, which *addresses* the band inside the asset\nfiles. Populated by discovery with the naming fallback\n(`assetBand.bandName`, then `assetBand.assetTitle`) and a unitless\nmeasurement." } } }, From bb5067e8cc0cbef7f1bbbd7667048b1fab88648b Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Mon, 3 Aug 2026 19:35:13 +0000 Subject: [PATCH 07/27] refactor migration --- geoengine/services/src/api/model/services.rs | 7 ++++++- .../src/contexts/migrations/current_schema.sql | 4 ++-- .../migrations/migration_0028_stac_provider.sql | 1 - .../migration_0029_stac_provider_band_name.rs | 7 ++++--- .../migration_0029_stac_provider_band_name.sql | 13 +++++++++++-- .../services/src/datasets/external/stac/mod.rs | 2 +- openapi.json | 8 ++++---- 7 files changed, 28 insertions(+), 14 deletions(-) diff --git a/geoengine/services/src/api/model/services.rs b/geoengine/services/src/api/model/services.rs index 9b4f19a240..479f098237 100644 --- a/geoengine/services/src/api/model/services.rs +++ b/geoengine/services/src/api/model/services.rs @@ -1001,6 +1001,11 @@ pub struct StacProviderDatasetBand { /// (addressing: which asset file + which raster channel within it). pub asset_band: StacAssetBand, /// The band descriptor of the resulting geo engine dataset layer. + /// + /// Independent of `assetBand`, which *addresses* the band inside the asset + /// files. Populated by discovery with the naming fallback + /// (`assetBand.bandName`, then `assetBand.assetTitle`) and a unitless + /// measurement. pub band_descriptor: crate::api::model::operators::RasterBandDescriptor, } @@ -1150,10 +1155,10 @@ pub struct StacDataProviderDefinition { pub s3_config: Option, pub time_dimension: TimeDimension, pub datasets: Vec, - pub page_limit: i64, /// Timeout in seconds for outgoing STAC API HTTP requests. #[serde(default = "default_query_timeout")] pub query_timeout_secs: i64, + pub page_limit: i64, } fn default_query_timeout() -> i64 { diff --git a/geoengine/services/src/contexts/migrations/current_schema.sql b/geoengine/services/src/contexts/migrations/current_schema.sql index 4f3f26d53e..2834f97c05 100644 --- a/geoengine/services/src/contexts/migrations/current_schema.sql +++ b/geoengine/services/src/contexts/migrations/current_schema.sql @@ -912,8 +912,8 @@ CREATE TYPE "StacDataProviderDefinition" AS ( s3_config "StacProviderS3Config", time_dimension "TimeDimension", datasets "StacProviderDataset" [], - page_limit bigint, - query_timeout_secs bigint + query_timeout_secs bigint, + page_limit bigint ); CREATE TYPE "DataProviderDefinition" AS ( diff --git a/geoengine/services/src/contexts/migrations/migration_0028_stac_provider.sql b/geoengine/services/src/contexts/migrations/migration_0028_stac_provider.sql index e68600336f..cd9c408c63 100644 --- a/geoengine/services/src/contexts/migrations/migration_0028_stac_provider.sql +++ b/geoengine/services/src/contexts/migrations/migration_0028_stac_provider.sql @@ -29,7 +29,6 @@ CREATE TYPE "StacDataProviderDefinition" AS ( s3_config "StacProviderS3Config", time_dimension "TimeDimension", datasets "StacProviderDataset" [], - page_limit bigint, query_timeout_secs bigint ); diff --git a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.rs b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.rs index 330a23f7d6..5479390993 100644 --- a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.rs +++ b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.rs @@ -6,10 +6,11 @@ use async_trait::async_trait; use tokio_postgres::Transaction; /// This migration bundles the band addressing fields of `StacProviderDatasetBand` -/// into a nested `StacAssetBand` type and adds a `name` attribute of type -/// `RasterBandDescriptor` for the band in the resulting geo engine dataset +/// into a nested `StacAssetBand` type and adds a `band_descriptor` attribute of +/// type `RasterBandDescriptor` for the band in the resulting geo engine dataset /// layer, independent of `asset_title`/`band_name`, which address the band -/// inside the STAC asset files. +/// inside the STAC asset files. It also takes over the `page_limit` attribute +/// of `StacDataProviderDefinition` from the released migration 0028. pub struct Migration0029StacProviderBandName; #[async_trait] diff --git a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql index 55dde1e39a..a9113bcd18 100644 --- a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql +++ b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql @@ -12,6 +12,10 @@ -- `RasterBandDescriptor` of the geo engine dataset layer band, populated with -- the naming fallback ("use band_name, then asset_title") and a unitless -- measurement. +-- +-- Additionally, this migration takes over the `page_limit` attribute of +-- `StacDataProviderDefinition` that previously lived in the released migration +-- `0028_stac_provider`, so that migration stays untouched. CREATE TYPE "StacAssetBand" AS ( asset_title text, @@ -21,6 +25,11 @@ CREATE TYPE "StacAssetBand" AS ( ALTER TYPE "StacProviderDatasetBand" ADD ATTRIBUTE asset_band "StacAssetBand"; ALTER TYPE "StacProviderDatasetBand" ADD ATTRIBUTE band_descriptor "RasterBandDescriptor"; +-- `page_limit` was moved here from migration 0028, which is already released. +-- It must be added before the data migration below, since +-- `pg_temp.stac_migrate_provider_def` reads `(def).page_limit`. +ALTER TYPE "StacDataProviderDefinition" ADD ATTRIBUTE page_limit bigint; + -- Migrate existing stored STAC provider definitions: bundle the flat -- `asset_title`/`band_name` attributes into the new `asset_band` attribute. CREATE FUNCTION pg_temp.stac_migrate_bands( @@ -79,8 +88,8 @@ BEGIN (def).s3_config, (def).time_dimension, new_datasets, - (def).page_limit, - (def).query_timeout_secs + (def).query_timeout_secs, + (def).page_limit )::"StacDataProviderDefinition"; END; $$ LANGUAGE plpgsql; diff --git a/geoengine/services/src/datasets/external/stac/mod.rs b/geoengine/services/src/datasets/external/stac/mod.rs index 592cdc7847..d6b4cce9ed 100644 --- a/geoengine/services/src/datasets/external/stac/mod.rs +++ b/geoengine/services/src/datasets/external/stac/mod.rs @@ -33,10 +33,10 @@ pub struct StacDataProviderDefinition { pub s3_config: Option, pub time_dimension: TimeDimension, // TODO: should this be on dataset level? pub datasets: Vec, - pub page_limit: i64, /// Timeout in seconds for outgoing STAC API HTTP requests. #[serde(default = "default_query_timeout")] pub query_timeout_secs: i64, + pub page_limit: i64, } fn default_query_timeout() -> i64 { diff --git a/openapi.json b/openapi.json index 582359b21c..04b2f4db92 100644 --- a/openapi.json +++ b/openapi.json @@ -12309,14 +12309,14 @@ "$ref": "#/components/schemas/StacProviderDataset" } }, - "pageLimit": { - "type": "integer", - "format": "int64" - }, "queryTimeoutSecs": { "type": "integer", "format": "int64", "description": "Timeout in seconds for outgoing STAC API HTTP requests." + }, + "pageLimit": { + "type": "integer", + "format": "int64" } } }, From 07620cb478de5bab83fd3b7ef12a232a836e990a Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Mon, 3 Aug 2026 20:36:46 +0000 Subject: [PATCH 08/27] adjust data path --- .../src/cli/stac_harvester/harvest.rs | 31 +-- .../src/datasets/external/stac/common.rs | 55 +--- .../datasets/external/stac/loading_info.rs | 247 +++--------------- 3 files changed, 60 insertions(+), 273 deletions(-) diff --git a/geoengine/services/src/cli/stac_harvester/harvest.rs b/geoengine/services/src/cli/stac_harvester/harvest.rs index 06c532fe89..9a0e2fc1ce 100644 --- a/geoengine/services/src/cli/stac_harvester/harvest.rs +++ b/geoengine/services/src/cli/stac_harvester/harvest.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, io::Read, str::FromStr, time::Instant}; +use std::{collections::HashMap, io::Read, path::PathBuf, str::FromStr, time::Instant}; use anyhow::Context; use chrono::Timelike; @@ -962,11 +962,7 @@ fn build_stac_query_params(params: &StacHarvest, page_limit: usize) -> Vec<(Stri query_params.push(("limit".to_string(), page_limit.to_string())); if params.filter_item_fields { - query_params.push(( - "fields".to_string(), - "stac_version,properties.datetime,properties.updated,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.proj:code,assets.*.proj:shape,assets.*.proj:transform" - .to_string(), - )); + query_params.push(("fields".to_string(), common::STAC_ITEM_FIELDS.to_string())); } query_params @@ -1172,7 +1168,14 @@ fn try_create_tile_for_band( let spatial_partition = geo_transform.grid_to_spatial_bounds(&grid_bounds); - let file_path = common::gdal_file_path(&asset.href)?; + let file_path = if asset.href.starts_with("http://") + || asset.href.starts_with("https://") + || asset.href.starts_with("s3://") + { + PathBuf::from(&asset.href) + } else { + return None; + }; let gdal_config_options = common::gdal_config_options_for_file_path(&file_path, provider_def.s3_config.as_ref()); @@ -1307,11 +1310,8 @@ mod tests { "tile dimensions should be positive" ); assert!( - tile.params - .file_path - .to_string_lossy() - .starts_with("/vsis3/"), - "file path should be a VSI path: {}", + tile.params.file_path.to_string_lossy().starts_with("s3://"), + "file path should be an S3 URL: {}", tile.params.file_path.display() ); } @@ -1397,11 +1397,8 @@ mod tests { "tile dimensions should be positive" ); assert!( - tile.params - .file_path - .to_string_lossy() - .starts_with("/vsis3/"), - "file path should be a VSI path: {}", + tile.params.file_path.to_string_lossy().starts_with("s3://"), + "file path should be an S3 URL: {}", tile.params.file_path.display() ); } diff --git a/geoengine/services/src/datasets/external/stac/common.rs b/geoengine/services/src/datasets/external/stac/common.rs index e537bf6244..ecc2c1cfd9 100644 --- a/geoengine/services/src/datasets/external/stac/common.rs +++ b/geoengine/services/src/datasets/external/stac/common.rs @@ -13,10 +13,13 @@ use geoengine_datatypes::{ spatial_reference::SpatialReference, }; use serde::Deserialize; -use std::path::PathBuf; use super::StacProviderS3Config; +/// STAC `fields` query parameter used to keep item responses small while including all +/// metadata needed by the provider (loading info) and the harvester (discovery/mapping). +pub const STAC_ITEM_FIELDS: &str = "stac_version,properties.datetime,properties.updated,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.proj:code,assets.*.proj:shape,assets.*.proj:transform"; + // --------------------------------------------------------------------------- // STAC extension version types // --------------------------------------------------------------------------- @@ -255,23 +258,6 @@ pub fn data_type_from_asset_v1_0_0_fallback(asset: &stac::Asset) -> Option Option { - if href.starts_with("http") { - return Some(PathBuf::from(format!("/vsicurl/{href}"))); - } - - href.strip_prefix("s3://") - .map(|s3_path| PathBuf::from(format!("/vsis3/{s3_path}"))) -} - // --------------------------------------------------------------------------- // Band processing helpers // --------------------------------------------------------------------------- @@ -558,7 +544,7 @@ pub fn is_jp2_media_type(media_type: Option<&str>) -> bool { // GDAL config options // --------------------------------------------------------------------------- -/// Build GDAL configuration options for `/vsis3/` paths. +/// Build GDAL configuration options for S3-backed file paths. /// /// Returns the common options plus S3-specific credentials when an S3 config is provided. pub fn gdal_config_options_for_s3( @@ -583,16 +569,16 @@ pub fn gdal_config_options_for_s3( options } -/// Build GDAL configuration options for a VSI file path, including common CURL/S3 options. +/// Build GDAL configuration options for a remote (HTTP or S3) file path, including common CURL/S3 options. pub fn gdal_config_options_for_file_path( file_path: &std::path::Path, s3_config: Option<&StacProviderS3Config>, ) -> Option> { let file_path_str = file_path.to_string_lossy(); - let is_vsi_s3 = file_path_str.starts_with("/vsis3/"); - let is_vsi_curl = file_path_str.starts_with("/vsicurl/"); + let is_s3 = file_path_str.starts_with("s3://"); + let is_http = file_path_str.starts_with("http://") || file_path_str.starts_with("https://"); - if !is_vsi_s3 && !is_vsi_curl { + if !is_s3 && !is_http { return None; } @@ -607,7 +593,7 @@ pub fn gdal_config_options_for_file_path( ), ]; - if is_vsi_s3 { + if is_s3 { options.extend(gdal_config_options_for_s3(s3_config)); } @@ -753,27 +739,6 @@ mod tests { assert_eq!(raster_data_type_from_stac_data_type_str("unknown"), None); } - // ----------------------------------------------------------------------- - // gdal_file_path - // ----------------------------------------------------------------------- - - #[test] - fn test_gdal_file_path_http() { - let path = gdal_file_path("https://example.com/file.tif").expect("should parse"); - assert_eq!(path, PathBuf::from("/vsicurl/https://example.com/file.tif")); - } - - #[test] - fn test_gdal_file_path_s3() { - let path = gdal_file_path("s3://bucket/key/file.tif").expect("should parse"); - assert_eq!(path, PathBuf::from("/vsis3/bucket/key/file.tif")); - } - - #[test] - fn test_gdal_file_path_unsupported() { - assert!(gdal_file_path("/local/path.tif").is_none()); - } - // ----------------------------------------------------------------------- // parse_epsg_from_proj_code // ----------------------------------------------------------------------- diff --git a/geoengine/services/src/datasets/external/stac/loading_info.rs b/geoengine/services/src/datasets/external/stac/loading_info.rs index 60ff02c250..b82b85d780 100644 --- a/geoengine/services/src/datasets/external/stac/loading_info.rs +++ b/geoengine/services/src/datasets/external/stac/loading_info.rs @@ -25,6 +25,7 @@ use geoengine_operators::source::{ OgrSourceDataset, TileFile, }; use stac::Item; +use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; use tracing::debug; @@ -356,36 +357,33 @@ impl StacMultiBandMetaData { let time_end = time_interval.end(); let query_params = vec![ - ( - "bbox".to_owned(), - format!( - "{},{},{},{}", - bbox.lower_left().x, - bbox.lower_left().y, - bbox.upper_right().x, - bbox.upper_right().y - ), - ), - ( - "datetime".to_owned(), - format!( - "{}/{}", - time_start - .as_date_time() - .ok_or(geoengine_operators::error::Error::InvalidDataProviderConfig)? - .to_datetime_string_with_millis(), - time_end - .as_date_time() - .ok_or(geoengine_operators::error::Error::InvalidDataProviderConfig)? - .to_datetime_string_with_millis(), - ), - ), - ("limit".to_owned(), self.page_limit.to_string()), - ( - "fields".to_owned(), - "stac_version,properties.datetime,properties.updated,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.proj:code,assets.*.proj:shape,assets.*.proj:transform".to_owned(), - ), - ]; + ( + "bbox".to_owned(), + format!( + "{},{},{},{}", + bbox.lower_left().x, + bbox.lower_left().y, + bbox.upper_right().x, + bbox.upper_right().y + ), + ), + ( + "datetime".to_owned(), + format!( + "{}/{}", + time_start + .as_date_time() + .ok_or(geoengine_operators::error::Error::InvalidDataProviderConfig)? + .to_datetime_string_with_millis(), + time_end + .as_date_time() + .ok_or(geoengine_operators::error::Error::InvalidDataProviderConfig)? + .to_datetime_string_with_millis(), + ), + ), + ("limit".to_owned(), self.page_limit.to_string()), + ("fields".to_owned(), common::STAC_ITEM_FIELDS.to_owned()), + ]; Ok(query_params) } @@ -508,8 +506,14 @@ impl StacMultiBandMetaData { .map_err(|_e| geoengine_operators::error::Error::InvalidDataProviderConfig)?; let spatial_partition = geo_transform.grid_to_spatial_bounds(&grid_bounds); - let file_path = common::gdal_file_path(&asset.href) - .ok_or(geoengine_operators::error::Error::InvalidDataProviderConfig)?; + let file_path = if asset.href.starts_with("http://") + || asset.href.starts_with("https://") + || asset.href.starts_with("s3://") + { + PathBuf::from(&asset.href) + } else { + return Err(geoengine_operators::error::Error::InvalidDataProviderConfig.into()); + }; let gdal_config_options = common::gdal_config_options_for_file_path(&file_path, self.s3_config.as_ref()); @@ -554,87 +558,6 @@ impl StacMultiBandMetaData { Ok(()) } - - fn rasterband_channel_for_dataset_band( - asset: &stac::Asset, - required_band_name: Option<&str>, - ) -> Option { - if asset.bands.is_empty() { - if required_band_name.is_some() { - tracing::warn!( - "STAC asset with href {} does not include bands, but dataset configuration requires a band name. Skipping asset.", - asset.href - ); - return None; - } - - return Some(1); - } - - let Some(required_band_name) = required_band_name else { - tracing::warn!( - "STAC asset with href {} includes bands, but dataset configuration does not specify a band name. Skipping asset.", - asset.href - ); - return None; - }; - - let Some(asset_band_idx) = asset - .bands - .iter() - .position(|asset_band| asset_band.name.as_deref() == Some(required_band_name)) - else { - tracing::debug!( - "Skipping asset with href {} due to missing required band {}", - asset.href, - required_band_name - ); - return None; - }; - - Some(asset_band_idx + 1) - } - - fn gdal_config_options_for_file_path(&self, file_path: &Path) -> Option> { - let file_path_str = file_path.to_string_lossy(); - let is_vsi_s3 = file_path_str.starts_with("s3://"); - let is_vsi_curl = - file_path_str.starts_with("http://") || file_path_str.starts_with("https://"); - - if !is_vsi_s3 && !is_vsi_curl { - return None; - } - - let mut options = vec![ - ( - "GDAL_DISABLE_READDIR_ON_OPEN".to_owned(), - "EMPTY_DIR".to_owned(), - ), - ( - "CPL_VSIL_CURL_ALLOWED_EXTENSIONS".to_owned(), - ".tif,.tiff,.jp2".to_owned(), - ), - ]; - - if !is_vsi_s3 { - return Some(options); - } - - if let Some(config) = self.s3_config.as_ref() { - options.push(("AWS_S3_ENDPOINT".to_owned(), config.endpoint.clone())); - options.push(("AWS_VIRTUAL_HOSTING".to_owned(), "FALSE".to_owned())); // TODO: make configurable? - - if let Some(access_key) = &config.access_key { - options.push(("AWS_ACCESS_KEY_ID".to_owned(), access_key.clone())); - } - - if let Some(secret_key) = &config.secret_key { - options.push(("AWS_SECRET_ACCESS_KEY".to_owned(), secret_key.clone())); - } - } - - Some(options) - } } fn stac_query_bbox( @@ -679,104 +602,6 @@ fn stac_query_time_interval( } } -fn gdal_file_path(href: &str) -> Option { - if href.starts_with("http://") || href.starts_with("https://") || href.starts_with("s3://") { - return Some(PathBuf::from(href)); - } - - None -} - -fn proj_shape_from_fields(fields: &serde_json::Map) -> Option<(usize, usize)> { - let proj_shape = fields.get("proj:shape")?.as_array()?; - if proj_shape.len() != 2 { - return None; - } - - let height = proj_shape.first()?.as_u64()? as usize; - let width = proj_shape.get(1)?.as_u64()? as usize; - - Some((height, width)) -} - -fn geo_transform_from_fields(fields: &serde_json::Map) -> Option { - let proj_transform = fields.get("proj:transform")?; - let proj_transform_array = proj_transform.as_array()?; - if proj_transform_array.len() != 6 { - return None; - } - - let proj_transform_values = proj_transform_array - .iter() - .map(Value::as_f64) - .collect::>>()?; - - let gdal_geotransform = [ - proj_transform_values[2], - proj_transform_values[0], - proj_transform_values[1], - proj_transform_values[5], - proj_transform_values[3], - proj_transform_values[4], - ]; - - Some(gdal_geotransform.into()) -} - -fn data_type_from_asset_v1_1_0(asset: &stac::Asset) -> Option { - asset - .data_type - .as_ref() - .and_then(raster_data_type_from_stac_data_type) -} - -fn raster_data_type_from_stac_data_type( - data_type: &stac_extensions::raster::DataType, -) -> Option { - match data_type { - stac_extensions::raster::DataType::UInt8 => Some(RasterDataType::U8), - stac_extensions::raster::DataType::UInt16 => Some(RasterDataType::U16), - stac_extensions::raster::DataType::UInt32 => Some(RasterDataType::U32), - stac_extensions::raster::DataType::Int16 => Some(RasterDataType::I16), - stac_extensions::raster::DataType::Int32 => Some(RasterDataType::I32), - stac_extensions::raster::DataType::Float32 => Some(RasterDataType::F32), - stac_extensions::raster::DataType::Float64 => Some(RasterDataType::F64), - _ => None, - } -} - -fn proj_code_matches_dataset( - fields: &serde_json::Map, - dataset_projection: SpatialReference, -) -> bool { - let Some(code) = fields.get("proj:code") else { - return false; - }; - - let Some(proj_code) = proj_code_as_srs_string(code) else { - return false; - }; - - proj_code == dataset_projection.to_string() -} - -fn proj_code_as_srs_string(value: &Value) -> Option { - if let Some(code_number) = value.as_u64() { - return Some(format!("EPSG:{code_number}")); - } - - let code_str = value.as_str()?.trim(); - if code_str.contains(':') { - return Some(code_str.to_ascii_uppercase()); - } - - if let Ok(code_number) = code_str.parse::() { - return Some(format!("EPSG:{code_number}")); - } - - None -} - #[async_trait] impl MetaDataProvider< From 11c49462b1d70688808d81f82fb6cd7639a47c3f Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Mon, 3 Aug 2026 20:52:43 +0000 Subject: [PATCH 09/27] lint --- .../migration_0029_stac_provider_band_name.sql | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql index a9113bcd18..c45e23ebd2 100644 --- a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql +++ b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql @@ -95,10 +95,13 @@ END; $$ LANGUAGE plpgsql; UPDATE layer_providers -SET definition = ROW( - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, - pg_temp.stac_migrate_provider_def((definition).stac_data_provider_definition) -)::"DataProviderDefinition" +SET + definition = ROW( + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + pg_temp.stac_migrate_provider_def( + (definition).stac_data_provider_definition + ) + )::"DataProviderDefinition" WHERE (definition).stac_data_provider_definition IS NOT NULL; -- Drop the now-redundant flat addressing attributes. From f15c2c5b6e57638ba0c333115a0142845d0b823f Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Mon, 3 Aug 2026 20:52:43 +0000 Subject: [PATCH 10/27] lint --- .../migration_0029_stac_provider_band_name.sql | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql index a9113bcd18..c82c346403 100644 --- a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql +++ b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql @@ -23,7 +23,8 @@ CREATE TYPE "StacAssetBand" AS ( ); ALTER TYPE "StacProviderDatasetBand" ADD ATTRIBUTE asset_band "StacAssetBand"; -ALTER TYPE "StacProviderDatasetBand" ADD ATTRIBUTE band_descriptor "RasterBandDescriptor"; +ALTER TYPE "StacProviderDatasetBand" +ADD ATTRIBUTE band_descriptor "RasterBandDescriptor"; -- `page_limit` was moved here from migration 0028, which is already released. -- It must be added before the data migration below, since @@ -95,10 +96,13 @@ END; $$ LANGUAGE plpgsql; UPDATE layer_providers -SET definition = ROW( - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, - pg_temp.stac_migrate_provider_def((definition).stac_data_provider_definition) -)::"DataProviderDefinition" +SET + definition = ROW( + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + pg_temp.stac_migrate_provider_def( + (definition).stac_data_provider_definition + ) + )::"DataProviderDefinition" WHERE (definition).stac_data_provider_definition IS NOT NULL; -- Drop the now-redundant flat addressing attributes. From 6da42f6bdd5862b5f417d6e32087aac68020f55f Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Tue, 4 Aug 2026 09:12:59 +0000 Subject: [PATCH 11/27] remove file --- geoengine/sentinel.json | 438 ---------------------------------------- 1 file changed, 438 deletions(-) delete mode 100644 geoengine/sentinel.json diff --git a/geoengine/sentinel.json b/geoengine/sentinel.json deleted file mode 100644 index aab8411031..0000000000 --- a/geoengine/sentinel.json +++ /dev/null @@ -1,438 +0,0 @@ -{ - "name": "sentinel-2-l2a from STAC", - "id": "11154a6f-ba05-422d-aafc-4812864938dc", - "description": "Auto-discovered mapping for STAC collection 'sentinel-2-l2a' at https://stac.code-de.org/v1", - "priority": 50, - "apiUrl": "https://stac.code-de.org/v1", - "collectionName": "sentinel-2-l2a", - "s3Config": null, - "timeDimension": { - "regular": { - "origin": 0, - "step": { - "granularity": "days", - "step": 1 - } - } - }, - "datasets": [ - { - "name": "sentinel-2-l2a EPSG:32632 U16 20m", - "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", - "data_type": "U16", - "resolution": { - "x": 20.0, - "y": 20.0 - }, - "projection": "EPSG:32632", - "spatial_grid": { - "spatialGrid": { - "geoTransform": { - "originCoordinate": { - "x": 699960.0, - "y": 5500020.0 - }, - "xPixelSize": 20.0, - "yPixelSize": -20.0 - }, - "gridBounds": { - "min": [ - -34998, - -224999 - ], - "max": [ - 15001, - 275001 - ] - } - }, - "state": "source" - }, - "bands": [ - { - "asset_title": "Aerosol optical thickness (AOT) - 20m", - "band_name": null - }, - { - "asset_title": "Blue (band 2) - 20m", - "band_name": null - }, - { - "asset_title": "Coastal aerosol (band 1) - 20m", - "band_name": null - }, - { - "asset_title": "Green (band 3) - 20m", - "band_name": null - }, - { - "asset_title": "NIR 2 (band 8A) - 20m", - "band_name": null - }, - { - "asset_title": "Red (band 4) - 20m", - "band_name": null - }, - { - "asset_title": "Red edge 1 (band 5) - 20m", - "band_name": null - }, - { - "asset_title": "Red edge 2 (band 6) - 20m", - "band_name": null - }, - { - "asset_title": "Red edge 3 (band 7) - 20m", - "band_name": null - }, - { - "asset_title": "Red edge 3 (band 7) - 60m", - "band_name": null - }, - { - "asset_title": "SWIR 1 (band 11) - 20m", - "band_name": null - }, - { - "asset_title": "SWIR 2 (band 12) - 20m", - "band_name": null - }, - { - "asset_title": "Water vapour (WVP) - 20m", - "band_name": null - } - ] - }, - { - "name": "sentinel-2-l2a EPSG:32632 U8 10m", - "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", - "data_type": "U8", - "resolution": { - "x": 10.0, - "y": 10.0 - }, - "projection": "EPSG:32632", - "spatial_grid": { - "spatialGrid": { - "geoTransform": { - "originCoordinate": { - "x": 699960.0, - "y": 5500020.0 - }, - "xPixelSize": 10.0, - "yPixelSize": -10.0 - }, - "gridBounds": { - "min": [ - -69996, - -449998 - ], - "max": [ - 30003, - 550002 - ] - } - }, - "state": "source" - }, - "bands": [ - { - "asset_title": "True color image", - "band_name": "True color image [B04]" - }, - { - "asset_title": "True color image [B02]", - "band_name": null - }, - { - "asset_title": "True color image [B03]", - "band_name": null - }, - { - "asset_title": "True color image [B04]", - "band_name": null - } - ] - }, - { - "name": "sentinel-2-l2a EPSG:32632 U16 10m", - "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", - "data_type": "U16", - "resolution": { - "x": 10.0, - "y": 10.0 - }, - "projection": "EPSG:32632", - "spatial_grid": { - "spatialGrid": { - "geoTransform": { - "originCoordinate": { - "x": 699960.0, - "y": 5500020.0 - }, - "xPixelSize": 10.0, - "yPixelSize": -10.0 - }, - "gridBounds": { - "min": [ - -69996, - -449998 - ], - "max": [ - 30003, - 550002 - ] - } - }, - "state": "source" - }, - "bands": [ - { - "asset_title": "Aerosol optical thickness (AOT) - 10m", - "band_name": null - }, - { - "asset_title": "Blue (band 2) - 10m", - "band_name": null - }, - { - "asset_title": "Green (band 3) - 10m", - "band_name": null - }, - { - "asset_title": "NIR 1 (band 8) - 10m", - "band_name": null - }, - { - "asset_title": "Red (band 4) - 10m", - "band_name": null - }, - { - "asset_title": "Water vapour (WVP) - 10m", - "band_name": null - } - ] - }, - { - "name": "sentinel-2-l2a EPSG:32632 U8 20m", - "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", - "data_type": "U8", - "resolution": { - "x": 20.0, - "y": 20.0 - }, - "projection": "EPSG:32632", - "spatial_grid": { - "spatialGrid": { - "geoTransform": { - "originCoordinate": { - "x": 699960.0, - "y": 5500020.0 - }, - "xPixelSize": 20.0, - "yPixelSize": -20.0 - }, - "gridBounds": { - "min": [ - -34998, - -224999 - ], - "max": [ - 15001, - 275001 - ] - } - }, - "state": "source" - }, - "bands": [ - { - "asset_title": "Cloud probability (CLD) - 20m", - "band_name": "Cloud probability (CLD) - 20m" - }, - { - "asset_title": "Scene classfication map (SCL) - 20m", - "band_name": null - }, - { - "asset_title": "Scene classification map (SCL) - 20m", - "band_name": "Scene classification map (SCL) - 20m" - }, - { - "asset_title": "Snow probability (SNW) - 20m", - "band_name": "Snow probability (SNW) - 20m" - }, - { - "asset_title": "True color image", - "band_name": "True color image [B04]" - }, - { - "asset_title": "True color image [B02]", - "band_name": null - }, - { - "asset_title": "True color image [B03]", - "band_name": null - }, - { - "asset_title": "True color image [B04]", - "band_name": null - } - ] - }, - { - "name": "sentinel-2-l2a EPSG:32632 U16 60m", - "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", - "data_type": "U16", - "resolution": { - "x": 60.0, - "y": 60.0 - }, - "projection": "EPSG:32632", - "spatial_grid": { - "spatialGrid": { - "geoTransform": { - "originCoordinate": { - "x": 699960.0, - "y": 5500020.0 - }, - "xPixelSize": 60.0, - "yPixelSize": -60.0 - }, - "gridBounds": { - "min": [ - -11666, - -74999 - ], - "max": [ - 5000, - 91667 - ] - } - }, - "state": "source" - }, - "bands": [ - { - "asset_title": "Aerosol optical thickness (AOT) - 60m", - "band_name": null - }, - { - "asset_title": "Blue (band 2) - 60m", - "band_name": null - }, - { - "asset_title": "Coastal aerosol (band 1) - 60m", - "band_name": null - }, - { - "asset_title": "Green (band 3) - 60m", - "band_name": null - }, - { - "asset_title": "NIR 2 (band 8A) - 60m", - "band_name": null - }, - { - "asset_title": "NIR 3 (band 9) - 60m", - "band_name": null - }, - { - "asset_title": "Red (band 4) - 60m", - "band_name": null - }, - { - "asset_title": "Red edge 1 (band 5) - 60m", - "band_name": null - }, - { - "asset_title": "Red edge 2 (band 6) - 60m", - "band_name": null - }, - { - "asset_title": "Red edge 3 (band 7) - 60m", - "band_name": "Red edge 3 (band 7) - 60m" - }, - { - "asset_title": "SWIR 1 (band 11) - 60m", - "band_name": null - }, - { - "asset_title": "SWIR 2 (band 12) - 60m", - "band_name": null - }, - { - "asset_title": "Water vapour (WVP) - 60m", - "band_name": null - } - ] - }, - { - "name": "sentinel-2-l2a EPSG:32632 U8 60m", - "description": "Auto-discovered from STAC collection 'sentinel-2-l2a'", - "data_type": "U8", - "resolution": { - "x": 60.0, - "y": 60.0 - }, - "projection": "EPSG:32632", - "spatial_grid": { - "spatialGrid": { - "geoTransform": { - "originCoordinate": { - "x": 699960.0, - "y": 5500020.0 - }, - "xPixelSize": 60.0, - "yPixelSize": -60.0 - }, - "gridBounds": { - "min": [ - -11666, - -74999 - ], - "max": [ - 5000, - 91667 - ] - } - }, - "state": "source" - }, - "bands": [ - { - "asset_title": "Cloud probability (CLD) - 60m", - "band_name": "Cloud probability (CLD) - 60m" - }, - { - "asset_title": "Scene classfication map (SCL) - 60m", - "band_name": null - }, - { - "asset_title": "Scene classification map (SCL) - 60m", - "band_name": "Scene classification map (SCL) - 60m" - }, - { - "asset_title": "Snow probability (SNW) - 60m", - "band_name": "Snow probability (SNW) - 60m" - }, - { - "asset_title": "True color image", - "band_name": "True color image [B04]" - }, - { - "asset_title": "True color image [B02]", - "band_name": null - }, - { - "asset_title": "True color image [B03]", - "band_name": null - }, - { - "asset_title": "True color image [B04]", - "band_name": null - } - ] - } - ] -} From 09a6d1e7ff594fcd31e76551b1c7fc0cf6a2495b Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Tue, 4 Aug 2026 14:58:42 +0000 Subject: [PATCH 12/27] multiple fixes --- .../src/cli/stac_harvester/harvest.rs | 162 ++++++++++++++---- .../src/datasets/external/stac/common.rs | 69 ++++++++ .../datasets/external/stac/loading_info.rs | 22 +-- 3 files changed, 199 insertions(+), 54 deletions(-) diff --git a/geoengine/services/src/cli/stac_harvester/harvest.rs b/geoengine/services/src/cli/stac_harvester/harvest.rs index 9a0e2fc1ce..42314626a8 100644 --- a/geoengine/services/src/cli/stac_harvester/harvest.rs +++ b/geoengine/services/src/cli/stac_harvester/harvest.rs @@ -1,11 +1,10 @@ use std::{collections::HashMap, io::Read, path::PathBuf, str::FromStr, time::Instant}; use anyhow::Context; -use chrono::Timelike; use futures::StreamExt; use geoengine_datatypes::{ dataset::NamedData, - primitives::{DateTime, TimeInstance, TimeInterval}, + primitives::TimeInstance, raster::{GeoTransform, GridBoundingBox2D, GridIdx2D}, spatial_reference::{SpatialReference, SpatialReferenceAuthority, SpatialReferenceOption}, }; @@ -28,13 +27,12 @@ use crate::{ model::{ datatypes::{ GdalConfigOption, GridBoundingBox2D as ApiGridBoundingBox2D, - GridIdx2D as ApiGridIdx2D, LayerId, SpatialGridDefinition, TimeGranularity, - TimeStep, + GridIdx2D as ApiGridIdx2D, LayerId, SpatialGridDefinition, }, operators::{ GdalDatasetParameters, GdalMultiBand, GdalMultiBandTypeTag, RasterBandDescriptor, - RasterBandDescriptors, RasterResultDescriptor, RegularTimeDimension, - SpatialGridDescriptor, SpatialGridDescriptorState, TimeDescriptor, TimeDimension, + RasterBandDescriptors, RasterResultDescriptor, SpatialGridDescriptor, + SpatialGridDescriptorState, TimeDescriptor, TimeDimension, }, responses::IdResponse, services::{ @@ -42,7 +40,7 @@ use crate::{ }, }, }, - datasets::{DatasetName, upload::VolumeName}, + datasets::DatasetName, layers::{ layer::{AddLayer, AddLayerCollection, CollectionItem, LayerCollection}, listing::LayerCollectionId, @@ -92,10 +90,6 @@ pub struct StacHarvest { #[arg(long, default_value = "adminadmin")] pub geo_engine_password: String, - /// Volume on the server - #[arg(long, default_value = "geodata")] - pub volume_name: String, - /// Verbose output #[arg(long, default_value_t = false)] pub verbose: bool, @@ -146,6 +140,10 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err let provider_def = ¶ms.mapping; + if provider_def.time_dimension == geoengine_datatypes::primitives::TimeDimension::Irregular { + anyhow::bail!("Harvesting does not support irregular time dimensions"); + } + info!( "Harvesting STAC collection '{}' at {} with {} dataset(s)", provider_def.collection_name, @@ -216,18 +214,22 @@ fn process_harvest_item( tiles_by_dataset: &mut HashMap>, params: &StacHarvest, ) -> Result<(), anyhow::Error> { + // Skip items whose STAC version the provider would also reject, so harvested + // datasets and provider-loaded datasets stay consistent. + if item.version != stac::Version::v1_1_0 { + warn!( + "Skipping STAC item with unsupported version: {:?}", + item.version + ); + return Ok(()); + } + let Some(datetime) = item.properties.datetime else { return Ok(()); }; - let date_without_time = datetime - .with_hour(0) - .and_then(|d| d.with_minute(0)) - .and_then(|d| d.with_second(0)) - .and_then(|d| d.with_nanosecond(0)) - .context("Failed to set time to zero")?; - let date_without_time: DateTime = date_without_time.into(); - let time: TimeInstance = date_without_time.into(); + let time: TimeInstance = TimeInstance::from_millis(datetime.timestamp_millis()) + .map_err(|e| anyhow::anyhow!("Invalid item datetime: {e}"))?; let z_index = match params.z_index_property_name.as_deref() { Some("updated") => item @@ -339,7 +341,7 @@ async fn create_dataset_api( api_config: &ApiConfig, dataset_name: &str, dataset: &StacProviderDataset, - volume_name: &str, + time_dimension: &geoengine_datatypes::primitives::TimeDimension, ) -> Result<(), anyhow::Error> { let geo_engine_url = &api_config.base_path; let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); @@ -355,7 +357,9 @@ async fn create_dataset_api( let api_gt: crate::api::model::datatypes::GeoTransform = dt_gt.into(); let create_dataset_req = CreateDataset { - data_path: DataPath::Volume(VolumeName(volume_name.to_string())), + // Tiles reference remote http(s)/s3 URLs, so datasets must be registered as + // external data. Volume/upload data paths only allow relative local paths. + data_path: DataPath::External, definition: DatasetDefinition { properties: AddDataset { name: Some( @@ -377,13 +381,16 @@ async fn create_dataset_api( .into(), time: TimeDescriptor { bounds: None, - dimension: TimeDimension::Regular(RegularTimeDimension { - origin: TimeInstance::from_millis_unchecked(0).into(), - step: TimeStep { - granularity: TimeGranularity::Days, - step: 1, - }, - }), + // Use the mapping's time dimension (granularity/step) so + // harvested datasets match the STAC provider's time handling. + dimension: match time_dimension { + geoengine_datatypes::primitives::TimeDimension::Regular(regular) => { + TimeDimension::Regular((*regular).into()) + } + geoengine_datatypes::primitives::TimeDimension::Irregular => { + TimeDimension::Irregular + } + }, }, spatial_grid: SpatialGridDescriptor { spatial_grid: SpatialGridDefinition { @@ -519,7 +526,7 @@ async fn create_harvest_layer_collections( let dataset_name = dataset_name_for_harvest(&provider_def.collection_name, dataset); let layer_name = format!( "EPSG:{} {:?} {}m", - dataset.projection.authority(), + dataset.projection.code(), dataset.data_type, dataset.resolution.x ); @@ -927,7 +934,13 @@ async fn setup_datasets( } if !dataset_exists_api(api_config, &dataset_name).await? { - create_dataset_api(api_config, &dataset_name, dataset, ¶ms.volume_name).await?; + create_dataset_api( + api_config, + &dataset_name, + dataset, + &provider_def.time_dimension, + ) + .await?; created_datasets.push((idx, dataset.clone())); } } @@ -1177,13 +1190,18 @@ fn try_create_tile_for_band( return None; }; - let gdal_config_options = - common::gdal_config_options_for_file_path(&file_path, provider_def.s3_config.as_ref()); + let gdal_config_options = common::gdal_config_options_for_file_path( + &file_path, + provider_def.s3_config.as_ref(), + params.gdal_retries, + ); + + // Snap the item timestamp to the mapping's time dimension so harvested tile + // intervals match the provider's (e.g. yearly for BioIS imperviousness data). + let time_interval = common::snap_time_interval(time, &provider_def.time_dimension)?; let tile = AddDatasetTile { - time: TimeInterval::new(time, time + i64::from(24 * 60 * 60 * 1000)) - .ok()? - .into(), + time: time_interval.into(), spatial_partition: spatial_partition.into(), band: band_idx as u32, z_index, @@ -1273,7 +1291,6 @@ mod tests { geo_engine_url: String::new(), geo_engine_email: String::new(), geo_engine_password: String::new(), - volume_name: String::new(), verbose: false, prefetch_pages: 1, z_index_property_name: Some("updated".to_string()), @@ -1362,7 +1379,6 @@ mod tests { geo_engine_url: String::new(), geo_engine_email: String::new(), geo_engine_password: String::new(), - volume_name: String::new(), verbose: false, prefetch_pages: 1, z_index_property_name: Some("updated".to_string()), @@ -1415,4 +1431,76 @@ mod tests { "first item should produce 4 tiles for 30m bands" ); } + + /// Verifies the tile-import contract: datasets are created with an `External` + /// data path (so remote http/s3 tile URLs pass `validate_tile`) and with the + /// mapping's time dimension instead of a hardcoded daily one. + #[tokio::test] + async fn test_create_dataset_api_uses_external_data_path_and_time_dimension() { + use httptest::{ + Expectation, Server, all_of, + matchers::{json_decoded, request}, + responders, + }; + + let mut server = Server::run(); + + server.expect( + Expectation::matching(all_of![ + request::method_path("POST", "/dataset"), + request::body(json_decoded(|value: &serde_json::Value| { + value["dataPath"] == serde_json::json!("external") + && value["definition"]["metaData"]["resultDescriptor"]["time"]["dimension"] + ["type"] + == serde_json::json!("regular") + && value["definition"]["metaData"]["resultDescriptor"]["time"]["dimension"] + ["step"]["granularity"] + == serde_json::json!("years") + })), + ]) + .times(1) + .respond_with(responders::status_code(200).body(r#"{"datasetName": "test_dataset"}"#)), + ); + + // create_dataset_api shares the new dataset with registered + anonymous users. + server.expect( + Expectation::matching(request::method_path("PUT", "/permissions")) + .times(2) + .respond_with(responders::status_code(200)), + ); + + let api_config = geoengine_api_client::apis::configuration::Configuration { + base_path: server.url_str("/").trim_end_matches('/').to_string(), + ..Default::default() + }; + + let dataset = StacProviderDataset { + name: "test".to_string(), + description: String::new(), + data_type: RasterDataType::U16, + resolution: SpatialResolution::new_unchecked(10.0, 10.0), + projection: SpatialReference::new(SpatialReferenceAuthority::Epsg, 32632), + spatial_grid: geoengine_operators::engine::SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0.0, 0.0).into(), 10.0, -10.0), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])).unwrap(), + ), + bands: vec![], + }; + + // A yearly time dimension, as used for BioIS imperviousness data. + let time_dimension = geoengine_datatypes::primitives::TimeDimension::Regular( + geoengine_datatypes::primitives::RegularTimeDimension::new_with_epoch_origin( + geoengine_datatypes::primitives::TimeStep { + granularity: geoengine_datatypes::primitives::TimeGranularity::Years, + step: 1, + }, + ), + ); + + create_dataset_api(&api_config, "test_dataset", &dataset, &time_dimension) + .await + .expect("create_dataset_api should succeed"); + + server.verify_and_clear(); + } } diff --git a/geoengine/services/src/datasets/external/stac/common.rs b/geoengine/services/src/datasets/external/stac/common.rs index ecc2c1cfd9..ae2cd95d48 100644 --- a/geoengine/services/src/datasets/external/stac/common.rs +++ b/geoengine/services/src/datasets/external/stac/common.rs @@ -9,6 +9,7 @@ #![allow(dead_code)] use geoengine_datatypes::{ + primitives::{TimeDimension, TimeInstance, TimeInterval}, raster::{GdalGeoTransform, GeoTransform, RasterDataType}, spatial_reference::SpatialReference, }; @@ -540,6 +541,30 @@ pub fn is_jp2_media_type(media_type: Option<&str>) -> bool { media_type == Some("image/jp2") } +// --------------------------------------------------------------------------- +// Time helpers +// --------------------------------------------------------------------------- + +/// Snap a timestamp to the previous step boundary of a regular time dimension and return +/// the interval `[start, start + step)`. +/// +/// Returns `None` for irregular dimensions or if the arithmetic fails. Both the STAC +/// provider and the STAC harvester use this so that harvested tiles and provider-loaded +/// tiles produce identical time intervals for the same item. +pub fn snap_time_interval( + time: TimeInstance, + time_dimension: &TimeDimension, +) -> Option { + match time_dimension { + TimeDimension::Regular(regular) => { + let start = regular.snap_prev(time).ok()?; + let end = (start + regular.step).ok()?; + TimeInterval::new(start, end).ok() + } + TimeDimension::Irregular => None, + } +} + // --------------------------------------------------------------------------- // GDAL config options // --------------------------------------------------------------------------- @@ -570,9 +595,13 @@ pub fn gdal_config_options_for_s3( } /// Build GDAL configuration options for a remote (HTTP or S3) file path, including common CURL/S3 options. +/// +/// When `retries` is set, adds GDAL HTTP retry options so transient failures reading +/// remote tiles are retried. pub fn gdal_config_options_for_file_path( file_path: &std::path::Path, s3_config: Option<&StacProviderS3Config>, + retries: Option, ) -> Option> { let file_path_str = file_path.to_string_lossy(); let is_s3 = file_path_str.starts_with("s3://"); @@ -593,6 +622,11 @@ pub fn gdal_config_options_for_file_path( ), ]; + if let Some(retries) = retries { + options.push(("GDAL_HTTP_MAX_RETRY".to_owned(), retries.to_string())); + options.push(("GDAL_HTTP_RETRY_DELAY".to_owned(), "5".to_owned())); + } + if is_s3 { options.extend(gdal_config_options_for_s3(s3_config)); } @@ -1038,6 +1072,41 @@ mod tests { assert!(options.contains(&("AWS_VIRTUAL_HOSTING".to_string(), "FALSE".to_string()))); } + #[test] + fn test_gdal_config_options_for_file_path_http_no_retries() { + let options = gdal_config_options_for_file_path( + std::path::Path::new("https://example.com/file.tif"), + None, + None, + ) + .expect("http path should produce options"); + assert!(!options.iter().any(|(k, _)| k == "GDAL_HTTP_MAX_RETRY")); + assert!(options.contains(&( + "GDAL_DISABLE_READDIR_ON_OPEN".to_string(), + "EMPTY_DIR".to_string() + ))); + } + + #[test] + fn test_gdal_config_options_for_file_path_with_retries() { + let options = gdal_config_options_for_file_path( + std::path::Path::new("https://example.com/file.tif"), + None, + Some(3), + ) + .expect("http path should produce options"); + assert!(options.contains(&("GDAL_HTTP_MAX_RETRY".to_string(), "3".to_string()))); + assert!(options.contains(&("GDAL_HTTP_RETRY_DELAY".to_string(), "5".to_string()))); + } + + #[test] + fn test_gdal_config_options_for_file_path_local_path_returns_none() { + assert_eq!( + gdal_config_options_for_file_path(std::path::Path::new("/data/file.tif"), None, None), + None + ); + } + // ----------------------------------------------------------------------- // data_type_from_asset_v1_1_0 // ----------------------------------------------------------------------- diff --git a/geoengine/services/src/datasets/external/stac/loading_info.rs b/geoengine/services/src/datasets/external/stac/loading_info.rs index b82b85d780..348ab29e0b 100644 --- a/geoengine/services/src/datasets/external/stac/loading_info.rs +++ b/geoengine/services/src/datasets/external/stac/loading_info.rs @@ -434,21 +434,9 @@ impl StacMultiBandMetaData { let item_time = TimeInstance::from_millis(item_datetime.timestamp_millis()) .map_err(|_e| geoengine_operators::error::Error::InvalidDataProviderConfig)?; - let time = match self.time_dimension { - TimeDimension::Regular(regular) => { - let time_start = regular - .snap_prev(item_time) - .map_err(|_e| geoengine_operators::error::Error::InvalidDataProviderConfig)?; - let time_end = (time_start + regular.step) - .map_err(|_e| geoengine_operators::error::Error::InvalidDataProviderConfig)?; - - TimeInterval::new(time_start, time_end) - .map_err(|_e| geoengine_operators::error::Error::InvalidDataProviderConfig)? - } - TimeDimension::Irregular => { - unreachable!("irregular time dimension rejected at provider initialization") - } - }; + // Shared with the STAC harvester so both produce identical intervals. + let time = common::snap_time_interval(item_time, &self.time_dimension) + .ok_or(geoengine_operators::error::Error::InvalidDataProviderConfig)?; Ok(Some((time, z_index))) } @@ -460,7 +448,7 @@ impl StacMultiBandMetaData { z_index: i64, files: &mut Vec, ) -> Result<()> { - if common::data_type_from_asset_v1_1_0(asset) != Some(self.dataset.data_type) { + if common::data_type_from_asset_v1_1_0_fallback(asset) != Some(self.dataset.data_type) { return Ok(()); } @@ -516,7 +504,7 @@ impl StacMultiBandMetaData { }; let gdal_config_options = - common::gdal_config_options_for_file_path(&file_path, self.s3_config.as_ref()); + common::gdal_config_options_for_file_path(&file_path, self.s3_config.as_ref(), None); for (dataset_band_idx, dataset_band) in self.dataset.bands.iter().enumerate() { if dataset_band.asset_band.asset_title != asset_title { From 26a5d5da3906b25be668eb5071b4870c053c7069 Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Tue, 4 Aug 2026 15:23:20 +0000 Subject: [PATCH 13/27] stac 1.0.0 harvest --- .../src/cli/stac_harvester/harvest.rs | 162 ++++++++++++++++-- .../src/datasets/external/stac/common.rs | 7 +- 2 files changed, 151 insertions(+), 18 deletions(-) diff --git a/geoengine/services/src/cli/stac_harvester/harvest.rs b/geoengine/services/src/cli/stac_harvester/harvest.rs index 42314626a8..8612d3cd4e 100644 --- a/geoengine/services/src/cli/stac_harvester/harvest.rs +++ b/geoengine/services/src/cli/stac_harvester/harvest.rs @@ -214,9 +214,11 @@ fn process_harvest_item( tiles_by_dataset: &mut HashMap>, params: &StacHarvest, ) -> Result<(), anyhow::Error> { - // Skip items whose STAC version the provider would also reject, so harvested - // datasets and provider-loaded datasets stay consistent. - if item.version != stac::Version::v1_1_0 { + // Harvest both STAC 1.0.0 and 1.1.0 items. Discovery supports both versions, so + // rejecting 1.0.0 here would silently harvest zero items from 1.0.0 collections. + // The version-aware parsing in `try_create_tile_for_band` handles the different + // metadata layouts. Skip only unknown versions. + if !matches!(item.version, stac::Version::v1_0_0 | stac::Version::v1_1_0) { warn!( "Skipping STAC item with unsupported version: {:?}", item.version @@ -1130,26 +1132,37 @@ fn try_create_tile_for_band( .iter() .find(|(_, a)| a.title.as_deref() == Some(&band_def.asset_band.asset_title))?; - // Check data type matches - if let Some(asset_dt) = common::data_type_from_asset_v1_1_0_fallback(asset) + // STAC 1.0.0 and 1.1.0 store data type and projection extension metadata + // differently, so pick the version-appropriate parsing below. + let proj_extension_version = match item.version { + stac::Version::v1_0_0 => common::StacExtensionMajorVersion::V1, + stac::Version::v1_1_0 => common::StacExtensionMajorVersion::V2, + _ => return None, + }; + + // Check data type matches. STAC 1.0.0 keeps it in `raster:bands[]`, STAC 1.1.0 + // in the asset's `data_type` field. + let asset_dt = match item.version { + stac::Version::v1_0_0 => common::data_type_from_asset_v1_0_0_fallback(asset), + stac::Version::v1_1_0 => common::data_type_from_asset_v1_1_0_fallback(asset), + _ => return None, + }; + if let Some(asset_dt) = asset_dt && asset_dt != dataset.data_type { return None; } // Extract the item's actual EPSG code from the asset - let item_epsg = common::epsg_code_from_fields( - common::StacExtensionMajorVersion::V2, - &asset.additional_fields, - ) - .or_else(|| { - // Also try to extract from serialized properties as fallback - let props_val = serde_json::to_value(&item.properties) - .ok() - .and_then(|v| v.as_object().cloned()) - .unwrap_or_default(); - common::epsg_code_from_fields(common::StacExtensionMajorVersion::V2, &props_val) - })?; + let item_epsg = common::epsg_code_from_fields(proj_extension_version, &asset.additional_fields) + .or_else(|| { + // Also try to extract from serialized properties as fallback + let props_val = serde_json::to_value(&item.properties) + .ok() + .and_then(|v| v.as_object().cloned()) + .unwrap_or_default(); + common::epsg_code_from_fields(proj_extension_version, &props_val) + })?; // Only process assets whose EPSG matches the dataset's projection if dataset.projection != SpatialReference::new(SpatialReferenceAuthority::Epsg, item_epsg) { @@ -1244,6 +1257,7 @@ fn format_duration(secs: u64) -> String { #[cfg(test)] mod tests { use super::*; + use crate::datasets::external::stac::StacAssetBand; use geoengine_datatypes::primitives::SpatialResolution; use geoengine_datatypes::raster::RasterDataType; @@ -1432,6 +1446,120 @@ mod tests { ); } + /// Verifies that STAC 1.0.0 items (e.g. from element84's STAC API) are harvested + /// rather than skipped. Their data type lives in `raster:bands[]` and the EPSG + /// code on the item properties (`proj:epsg`), so this exercises the version-aware + /// parsing in `try_create_tile_for_band`. The item fixture is a recorded response + /// from the element84 STAC API and contains no external dependencies. + #[test] + fn test_process_harvest_item_recovers_v1_0_0_item() { + use geoengine_datatypes::dataset::DataProviderId; + use geoengine_datatypes::util::Identifier; + + let mapping = StacDataProviderDefinition { + name: "element84-test".to_string(), + id: DataProviderId::new(), + description: String::new(), + priority: None, + api_url: "https://earth-search.aws.element84.com/v0".to_string(), + collection_name: "sentinel-2-l2a".to_string(), + s3_config: None, + time_dimension: geoengine_datatypes::primitives::TimeDimension::Regular( + geoengine_datatypes::primitives::RegularTimeDimension::new_with_epoch_origin( + geoengine_datatypes::primitives::TimeStep { + granularity: geoengine_datatypes::primitives::TimeGranularity::Days, + step: 1, + }, + ), + ), + datasets: vec![StacProviderDataset { + name: "test".to_string(), + description: String::new(), + data_type: RasterDataType::U16, + resolution: SpatialResolution::new_unchecked(10.0, 10.0), + projection: SpatialReference::new(SpatialReferenceAuthority::Epsg, 32632), + spatial_grid: geoengine_operators::engine::SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0.0, 0.0).into(), 10.0, -10.0), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([0, 0])).unwrap(), + ), + bands: vec![StacProviderDatasetBand::new_unitless(StacAssetBand { + asset_title: "Blue - 10m".to_string(), + band_name: Some("blue".to_string()), + })], + }], + query_timeout_secs: 60, + page_limit: 10, + }; + + let items: stac::ItemCollection = serde_json::from_str(include_str!( + "../../../../test_data/stac_responses/items/element84-marburg-minimal.json" + )) + .expect("valid element84 items fixture"); + + let params = StacHarvest { + mapping: mapping.clone(), + time_start: None, + time_end: None, + bbox: None, + geo_engine_url: String::new(), + geo_engine_email: String::new(), + geo_engine_password: String::new(), + verbose: false, + prefetch_pages: 1, + z_index_property_name: Some("updated".to_string()), + no_data_value: None, + gdal_retries: None, + filter_item_fields: true, + }; + + let mut tiles_by_dataset: HashMap> = HashMap::new(); + + let item = &items.items[0]; + + // Sanity check: this fixture is a STAC 1.0.0 response. + assert_eq!(item.version, stac::Version::v1_0_0); + + process_harvest_item(item, &mapping, &mut tiles_by_dataset, ¶ms) + .expect("1.0.0 item processing should succeed"); + + // The 1.0.0 item must be recovered (not skipped) and produce a tile. + assert_eq!( + tiles_by_dataset.len(), + 1, + "one dataset should receive tiles from the 1.0.0 item" + ); + + let (_dataset_name, tiles) = tiles_by_dataset.iter().next().expect("one dataset"); + assert_eq!(tiles.len(), 1, "blue band should produce exactly one tile"); + let tile = &tiles[0]; + + // Data type (uint16 from `raster:bands[]`) and EPSG (`proj:epsg` on the item) + // must have been recovered from the 1.0.0 metadata layout. + assert_eq!(tile.band, 0, "band index should be 0"); + assert_eq!(tile.params.rasterband_channel, 1); + assert_eq!(tile.params.width, 10_980); + assert_eq!(tile.params.height, 10_980); + + // The first matching asset is the COG GeoTIFF (https) asset. + assert!( + tile.params + .file_path + .to_string_lossy() + .starts_with("https://"), + "file path should be the COG URL: {}", + tile.params.file_path.display() + ); + + assert_eq!(tile.params.geo_transform.x_pixel_size, 10.0); + assert_eq!(tile.params.geo_transform.origin_coordinate.x, 399_960.0); + assert_eq!(tile.params.geo_transform.origin_coordinate.y, 5_700_000.0); + + // The item timestamp (2026-01-28T10:36:43Z) is snapped to the daily time + // dimension: [2026-01-28T00:00:00Z, 2026-01-29T00:00:00Z). + assert_eq!(tile.time.start.inner(), 1_769_558_400_000); + assert_eq!(tile.time.end.inner(), 1_769_644_800_000); + } + /// Verifies the tile-import contract: datasets are created with an `External` /// data path (so remote http/s3 tile URLs pass `validate_tile`) and with the /// mapping's time dimension instead of a hardcoded daily one. diff --git a/geoengine/services/src/datasets/external/stac/common.rs b/geoengine/services/src/datasets/external/stac/common.rs index ae2cd95d48..114e1ba109 100644 --- a/geoengine/services/src/datasets/external/stac/common.rs +++ b/geoengine/services/src/datasets/external/stac/common.rs @@ -19,7 +19,12 @@ use super::StacProviderS3Config; /// STAC `fields` query parameter used to keep item responses small while including all /// metadata needed by the provider (loading info) and the harvester (discovery/mapping). -pub const STAC_ITEM_FIELDS: &str = "stac_version,properties.datetime,properties.updated,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.proj:code,assets.*.proj:shape,assets.*.proj:transform"; +/// +/// Includes both the STAC 1.1.0 metadata (`assets.*.data_type`, `assets.*.bands`, +/// `assets.*.proj:code`) and the STAC 1.0.0 metadata (`assets.*.raster:bands`, +/// `properties.proj:epsg`, `assets.*.proj:epsg`) so that items of either version +/// survive the field filter. +pub const STAC_ITEM_FIELDS: &str = "stac_version,properties.datetime,properties.updated,assets.*.title,assets.*.href,assets.*.data_type,assets.*.bands,assets.*.raster:bands,assets.*.proj:code,assets.*.proj:epsg,properties.proj:epsg,assets.*.proj:shape,assets.*.proj:transform"; // --------------------------------------------------------------------------- // STAC extension version types From 7b39bf8401157422ce6189e39f22bc30025ef66d Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Mon, 17 Aug 2026 13:50:10 +0000 Subject: [PATCH 14/27] cleanup --- geoengine/services/src/api/model/services.rs | 5 -- .../src/cli/stac_harvester/discover.rs | 62 +++++++++++++++---- geoengine/services/src/cli/stac_import.rs | 19 ++++-- .../src/datasets/external/stac/common.rs | 37 ++++++----- geoengine/services/src/datasets/upload.rs | 7 --- 5 files changed, 85 insertions(+), 45 deletions(-) diff --git a/geoengine/services/src/api/model/services.rs b/geoengine/services/src/api/model/services.rs index 479f098237..47c7441bc6 100644 --- a/geoengine/services/src/api/model/services.rs +++ b/geoengine/services/src/api/model/services.rs @@ -1001,11 +1001,6 @@ pub struct StacProviderDatasetBand { /// (addressing: which asset file + which raster channel within it). pub asset_band: StacAssetBand, /// The band descriptor of the resulting geo engine dataset layer. - /// - /// Independent of `assetBand`, which *addresses* the band inside the asset - /// files. Populated by discovery with the naming fallback - /// (`assetBand.bandName`, then `assetBand.assetTitle`) and a unitless - /// measurement. pub band_descriptor: crate::api::model::operators::RasterBandDescriptor, } diff --git a/geoengine/services/src/cli/stac_harvester/discover.rs b/geoengine/services/src/cli/stac_harvester/discover.rs index 3dc9f80304..ac8808bc56 100644 --- a/geoengine/services/src/cli/stac_harvester/discover.rs +++ b/geoengine/services/src/cli/stac_harvester/discover.rs @@ -253,13 +253,13 @@ fn scan_collection_bands( let mut dataset_bands: HashMap> = HashMap::new(); - for (_asset_key, asset) in &collection.item_assets { + for (asset_key, asset) in &collection.item_assets { if !matches_selected_file_types(asset.r#type.as_deref(), file_types) { continue; } if let Ok(Some(bands)) = - scan_collection_item_asset(&collection.version, asset, collection.summaries.as_ref()) + scan_collection_item_asset(&collection.version, asset, collection.summaries.as_ref(), Some(asset_key.as_str())) { merge_dataset_bands(&mut dataset_bands, bands); } @@ -393,7 +393,7 @@ fn process_sample_assets( }; let asset_title = asset.title.as_deref().unwrap_or(asset_key).to_string(); - let asset_info = common::band_names_from_asset_v1_1_0(asset).unwrap_or_else(|_| { + let asset_info = common::band_names_from_asset_v1_1_0(asset, Some(asset_key.as_str())).unwrap_or_else(|_| { common::AssetBandInfo { asset_title: asset_title.clone(), band_names: vec![asset_title.clone()], @@ -479,13 +479,26 @@ fn build_datasets( continue; } - bands.sort_by(|a, b| a.asset_band.asset_title.cmp(&b.asset_band.asset_title)); + bands.sort_by(|a, b| { + let a_name = a + .asset_band + .band_name + .as_deref() + .unwrap_or(&a.asset_band.asset_title); + let b_name = b + .asset_band + .band_name + .as_deref() + .unwrap_or(&b.asset_band.asset_title); + a_name.cmp(b_name) + }); let spatial_grid = build_dataset_spatial_grid(info, dataset_key, full_projection_grid); + let unit_suffix = get_unit_suffix(dataset_key.epsg); let dataset_name = format!( - "{} EPSG:{} {:?} {}m", - stac_collection, dataset_key.epsg, dataset_key.data_type, dataset_key.resolution + "{} EPSG:{} {:?} {}{}", + stac_collection, dataset_key.epsg, dataset_key.data_type, dataset_key.resolution, unit_suffix ); datasets.push(StacProviderDataset { @@ -647,14 +660,15 @@ fn scan_collection_item_asset( collection_version: &stac::Version, asset: &stac::ItemAsset, collection_summaries: Option<&serde_json::Map>, + asset_key: Option<&str>, ) -> Result>>, String> { match collection_version { - stac::Version::v1_0_0 => scan_collection_item_asset_v1_0_0(asset), - stac::Version::v1_1_0 => scan_collection_item_asset_v1_1_0(asset, collection_summaries), + stac::Version::v1_0_0 => scan_collection_item_asset_v1_0_0(asset, asset_key), + stac::Version::v1_1_0 => scan_collection_item_asset_v1_1_0(asset, collection_summaries, asset_key), _ => { // For unknown STAC versions, try v1.1.0 first (more common), fall back to v1.0.0 - scan_collection_item_asset_v1_1_0(asset, collection_summaries) - .or_else(|_| scan_collection_item_asset_v1_0_0(asset)) + scan_collection_item_asset_v1_1_0(asset, collection_summaries, asset_key) + .or_else(|_| scan_collection_item_asset_v1_0_0(asset, asset_key)) .or(Ok(None)) } } @@ -662,6 +676,7 @@ fn scan_collection_item_asset( fn scan_collection_item_asset_v1_0_0( asset: &stac::ItemAsset, + asset_key: Option<&str>, ) -> Result>>, String> { let mut dataset_bands: HashMap> = HashMap::new(); @@ -703,12 +718,12 @@ fn scan_collection_item_asset_v1_0_0( let band_name = if let Some(ref eo_bands_vec) = eo_bands { common::v1_0_0_band_name( - asset.title.as_deref(), + asset_key.or_else(|| asset.title.as_deref()), Some(&eo_bands_vec[index]), band_count, ) } else { - common::v1_0_0_band_name(asset.title.as_deref(), None, 1) + common::v1_0_0_band_name(asset_key.or_else(|| asset.title.as_deref()), None, 1) }; dataset_bands @@ -729,6 +744,7 @@ fn scan_collection_item_asset_v1_0_0( fn scan_collection_item_asset_v1_1_0( asset: &stac::ItemAsset, collection_summaries: Option<&serde_json::Map>, + asset_key: Option<&str>, ) -> Result>>, String> { let mut dataset_bands: HashMap> = HashMap::new(); @@ -743,7 +759,7 @@ fn scan_collection_item_asset_v1_1_0( let raster_data_type = common::raster_data_type_from_stac_data_type_str(data_type) .ok_or_else(|| format!("Unsupported data_type: {data_type}"))?; - let asset_info = common::band_names_from_item_asset_v1_1_0(asset)?; + let asset_info = common::band_names_from_item_asset_v1_1_0(asset, asset_key)?; let resolution = asset .additional_fields @@ -811,6 +827,26 @@ fn merge_dataset_bands( } } +// --------------------------------------------------------------------------- +// Helper Functions +// --------------------------------------------------------------------------- + +/// Determine the unit suffix for a dataset based on its EPSG code. +/// Geographic CRS (like EPSG:4326) use degrees, projected CRS (like UTM) use meters. +fn get_unit_suffix(epsg: u32) -> &'static str { + match epsg { + // Geographic CRS codes (WGS84, ETRS89, and other lat/lon coordinates) + 4258 | 4267 | 4269 | 4276 | 4277 | 4278 | 4279 | 4289 | 4291 | 4308 | 4309 + | 4311 | 4312 | 4313 | 4314 | 4315 | 4316 | 4317 | 4318 | 4319 | 4322 | 4326 | 4357 + | 4359 | 4360 | 4361 | 4362 | 4363 | 4364 | 4365 | 4366 | 4367 | 4368 | 4369 | 4370 + | 4371 | 4372 | 4373 | 4374 | 4375 | 4376 | 4377 | 4378 | 4379 | 4380 | 4381 | 4382 + | 4383 | 4384 | 4385 | 4386 | 4387 | 4388 | 4389 | 4390 | 4391 | 4392 | 4393 | 4394 + | 4395 | 4396 | 4397 | 4398 | 4399 => "deg", + // Projected CRS (UTM and others) use meters + _ => "m", + } +} + fn parse_time_dimension(granularity: &str, step: u64) -> Result { let dt_granularity = match granularity.to_lowercase().as_str() { "days" | "day" => geoengine_datatypes::primitives::TimeGranularity::Days, diff --git a/geoengine/services/src/cli/stac_import.rs b/geoengine/services/src/cli/stac_import.rs index e7f0143cf1..4715e6d4a2 100644 --- a/geoengine/services/src/cli/stac_import.rs +++ b/geoengine/services/src/cli/stac_import.rs @@ -2121,7 +2121,7 @@ async fn scan_collection( raster: StacExtensionMajorVersion::V2, eo: StacExtensionMajorVersion::V2, }, - ) => scan_item_asset_v1_1_0(asset), + ) => scan_item_asset_v1_1_0(asset, Some(asset_key)), _ => Err(anyhow::anyhow!( "Unsupported STAC version or extension versions: {:?}, {stac_extension_versions:?}", collection.version @@ -2230,6 +2230,7 @@ fn scan_item_asset_v1_0_0( fn scan_item_asset_v1_1_0( asset: &stac::ItemAsset, + asset_key: Option<&str>, ) -> anyhow::Result>>> { let mut dataset_bands: HashMap> = HashMap::new(); @@ -2247,7 +2248,7 @@ fn scan_item_asset_v1_1_0( .context(format!("Unsupported data_type: {data_type}"))?; // in STAC 1.1.0 `raster:bands` and `eo:bands` are merged into common metadata `bands` - let band_names = band_names_from_item_asset_v1_1_0(asset)?; + let band_names = band_names_from_item_asset_v1_1_0(asset, asset_key)?; let resolution = asset .additional_fields @@ -2301,7 +2302,7 @@ fn band_names_from_asset_v1_1_0(asset: &stac::Asset) -> anyhow::Result anyhow::Result> { +fn band_names_from_item_asset_v1_1_0(asset: &stac::ItemAsset, asset_key: Option<&str>) -> anyhow::Result> { let asset_title = asset .title .as_deref() @@ -2313,15 +2314,21 @@ fn band_names_from_item_asset_v1_1_0(asset: &stac::ItemAsset) -> anyhow::Result< .and_then(serde_json::Value::as_array); let Some(bands) = band_names else { - return Ok(vec![asset_title.to_string()]); + // Use asset_key if provided, otherwise fall back to asset_title + let name = asset_key.unwrap_or(asset_title).to_string(); + return Ok(vec![name]); }; if bands.is_empty() { - return Ok(vec![asset_title.to_string()]); + // Use asset_key if provided, otherwise fall back to asset_title + let name = asset_key.unwrap_or(asset_title).to_string(); + return Ok(vec![name]); } if bands.len() == 1 { - return Ok(vec![asset_title.to_string()]); + // Use asset_key if provided, otherwise fall back to asset_title + let name = asset_key.unwrap_or(asset_title).to_string(); + return Ok(vec![name]); } let mut names = Vec::new(); diff --git a/geoengine/services/src/datasets/external/stac/common.rs b/geoengine/services/src/datasets/external/stac/common.rs index 114e1ba109..84ab17f26b 100644 --- a/geoengine/services/src/datasets/external/stac/common.rs +++ b/geoengine/services/src/datasets/external/stac/common.rs @@ -337,11 +337,6 @@ pub fn rasterband_channel_for_dataset_band( } /// Parsed band information from a STAC 1.1.0 asset. -/// -/// Keeps the asset's display title separate from the individual band names so -/// callers can match an asset by its real STAC title and select the raster -/// channel by band name, without encoding the band name into the title (e.g. -/// `True color image [B02]`). #[derive(Debug, Clone, PartialEq)] pub struct AssetBandInfo { pub asset_title: String, @@ -354,7 +349,10 @@ pub struct AssetBandInfo { /// band is named after the asset title. For multi-band assets the individual /// STAC band names (e.g. `B04`) are returned, so the mapping can reference the /// exact raster channel while keeping the real asset title. -pub fn band_names_from_asset_v1_1_0(asset: &stac::Asset) -> Result { +pub fn band_names_from_asset_v1_1_0( + asset: &stac::Asset, + asset_key: Option<&str>, +) -> Result { let asset_title = asset .title .as_deref() @@ -364,9 +362,11 @@ pub fn band_names_from_asset_v1_1_0(asset: &stac::Asset) -> Result Result Result { +/// Prefers the asset_key when provided (e.g., "B01", "B02" from STAC collection item_assets keys). +/// Falls back to band names from the `bands` field, or the asset title. +pub fn band_names_from_item_asset_v1_1_0( + asset: &stac::ItemAsset, + asset_key: Option<&str>, +) -> Result { let asset_title = asset .title .as_deref() @@ -400,19 +404,24 @@ pub fn band_names_from_item_asset_v1_1_0(asset: &stac::ItemAsset) -> Result Deserialize<'de> for VolumeName { impl AdjustFilePath for Volume { fn adjust_file_path(&self, file_path: &Path) -> Result { - if self.name.0 == "external" { - // external data file path must not be adjusted - // TODO: remove this once we have proper volume management - // TODO: ensure the file path actually points to external data - return Ok(file_path.to_path_buf()); - } - let _file_name = file_path.file_name().ok_or(error::Error::PathIsNotAFile)?; path_with_base_path(&self.path, file_path) From 907c97ee4ed913224dee419d261d726a59ec4552 Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Tue, 18 Aug 2026 15:33:02 +0000 Subject: [PATCH 15/27] retry --- .../datasets/external/stac/loading_info.rs | 73 ++++++----- geoengine/services/src/util/retry.rs | 117 ++++++++++++++++++ 2 files changed, 157 insertions(+), 33 deletions(-) diff --git a/geoengine/services/src/datasets/external/stac/loading_info.rs b/geoengine/services/src/datasets/external/stac/loading_info.rs index 348ab29e0b..e6a148461e 100644 --- a/geoengine/services/src/datasets/external/stac/loading_info.rs +++ b/geoengine/services/src/datasets/external/stac/loading_info.rs @@ -2,6 +2,7 @@ use super::common; use super::{StacDataProvider, StacProviderDataset, StacProviderS3Config, cache::StacQueryCache}; use crate::error::Result; use crate::util::join_base_url_and_path; +use crate::util::retry::{RetryPolicy, retry_http}; use async_trait::async_trait; use chrono::DateTime as ChronoDateTime; use geoengine_datatypes::dataset::DataId; @@ -60,6 +61,8 @@ async fn query_stac_item_collection( client: &reqwest::Client, query_state: &StacQueryState, ) -> geoengine_operators::util::Result<(stac::ItemCollection, StacQueryState)> { + let request_policy = RetryPolicy::new().stop_on_status(&[400, 404]); + match query_state { StacQueryState::FirstPage { query_url, @@ -69,23 +72,25 @@ async fn query_stac_item_collection( let request_started = std::time::Instant::now(); - let item_collection: stac::ItemCollection = client - .get(query_url.clone()) - .query(query_params) - .send() - .await - .map_err( - |e| geoengine_operators::error::Error::QueryingProcessorFailed { - source: Box::new(e), - }, - )? - .json() - .await - .map_err( - |e| geoengine_operators::error::Error::QueryingProcessorFailed { - source: Box::new(e), - }, - )?; + let item_collection: stac::ItemCollection = retry_http( + || async { + client + .get(query_url.clone()) + .query(query_params) + .send() + .await? + .error_for_status()? + .json() + .await + }, + &format!("Fetch STAC items from {query_url}"), + &request_policy, + |e| e.status().map(|s| s.as_u16()), + ) + .await + .map_err(|e| geoengine_operators::error::Error::QueryingProcessorFailed { + source: Box::new(e), + })?; debug!( "STAC response received in {:?} s", @@ -108,22 +113,24 @@ async fn query_stac_item_collection( let request_started = std::time::Instant::now(); - let item_collection: stac::ItemCollection = client - .get(next_url.clone()) - .send() - .await - .map_err( - |e| geoengine_operators::error::Error::QueryingProcessorFailed { - source: Box::new(e), - }, - )? - .json() - .await - .map_err( - |e| geoengine_operators::error::Error::QueryingProcessorFailed { - source: Box::new(e), - }, - )?; + let item_collection: stac::ItemCollection = retry_http( + || async { + client + .get(next_url.clone()) + .send() + .await? + .error_for_status()? + .json() + .await + }, + &format!("Fetch next STAC page from {next_url}"), + &request_policy, + |e| e.status().map(|s| s.as_u16()), + ) + .await + .map_err(|e| geoengine_operators::error::Error::QueryingProcessorFailed { + source: Box::new(e), + })?; debug!( "STAC response received in {:?} s", diff --git a/geoengine/services/src/util/retry.rs b/geoengine/services/src/util/retry.rs index a71d5c4407..6eb3e55370 100644 --- a/geoengine/services/src/util/retry.rs +++ b/geoengine/services/src/util/retry.rs @@ -185,3 +185,120 @@ fn is_terminal( !(status_allows_retry && message_allows_retry) } + +#[cfg(test)] +mod tests { + use super::{RetryPolicy, retry_http}; + use std::fmt; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + #[derive(Debug, Clone)] + struct TestError { + status: Option, + message: String, + } + + impl fmt::Display for TestError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Some(status) = self.status { + write!(f, "HTTP {status}: {}", self.message) + } else { + write!(f, "{}", self.message) + } + } + } + + impl std::error::Error for TestError {} + + #[tokio::test] + async fn retry_http_retries_until_success() { + let attempts = Arc::new(AtomicUsize::new(0)); + + let result: Result = retry_http( + { + let attempts = attempts.clone(); + move || { + let current = attempts.fetch_add(1, Ordering::SeqCst) + 1; + async move { + if current < 3 { + Err(TestError { + status: None, + message: "temporary network issue".to_string(), + }) + } else { + Ok("ok".to_string()) + } + } + } + }, + "transient fetch", + &RetryPolicy::new().max_retries(5).initial_delay_ms(1), + |e| e.status, + ) + .await; + + assert_eq!(result.unwrap(), "ok"); + assert_eq!(attempts.load(Ordering::SeqCst), 3); + } + + #[tokio::test] + async fn retry_http_stops_on_status_code() { + let attempts = Arc::new(AtomicUsize::new(0)); + + let result: Result = retry_http( + { + let attempts = attempts.clone(); + move || { + attempts.fetch_add(1, Ordering::SeqCst); + async { + Err(TestError { + status: Some(500), + message: "server error".to_string(), + }) + } + } + }, + "server fetch", + &RetryPolicy::new() + .max_retries(5) + .initial_delay_ms(1) + .stop_on_status(&[500]), + |e| e.status, + ) + .await; + + assert!(result.is_err()); + assert_eq!(attempts.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn retry_http_stops_on_message_contains() { + let attempts = Arc::new(AtomicUsize::new(0)); + + let result: Result = retry_http( + { + let attempts = attempts.clone(); + move || { + attempts.fetch_add(1, Ordering::SeqCst); + async { + Err(TestError { + status: Some(400), + message: "bad request".to_string(), + }) + } + } + }, + "message fetch", + &RetryPolicy::new() + .max_retries(5) + .initial_delay_ms(1) + .stop_on_message(&["bad request"]), + |e| e.status, + ) + .await; + + assert!(result.is_err()); + assert_eq!(attempts.load(Ordering::SeqCst), 1); + } +} From 3c91b5b46238693c08391987dfb52cb6491e49b9 Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Fri, 21 Aug 2026 08:52:56 +0000 Subject: [PATCH 16/27] fix tests and migration --- geoengine/services/src/api/model/services.rs | 5 ++ .../src/cli/stac_harvester/discover.rs | 38 +++++---- .../src/cli/stac_harvester/harvest.rs | 82 ++++++++++++++++--- geoengine/services/src/cli/stac_import.rs | 5 +- ...migration_0029_stac_provider_band_name.sql | 2 +- .../datasets/external/stac/loading_info.rs | 12 ++- .../src/datasets/external/stac/mod.rs | 6 ++ geoengine/services/src/util/retry.rs | 2 +- .../expected-mapping-code-de.json | 28 +++---- .../expected-mapping-landsat-c2-l1.json | 20 ++--- 10 files changed, 141 insertions(+), 59 deletions(-) diff --git a/geoengine/services/src/api/model/services.rs b/geoengine/services/src/api/model/services.rs index 060cb4223a..78b3c0d37a 100644 --- a/geoengine/services/src/api/model/services.rs +++ b/geoengine/services/src/api/model/services.rs @@ -1155,6 +1155,7 @@ pub struct StacDataProviderDefinition { /// Timeout in seconds for outgoing STAC API HTTP requests. #[serde(default = "default_query_timeout")] pub query_timeout_secs: i64, + #[serde(default = "default_page_limit")] pub page_limit: i64, } @@ -1162,6 +1163,10 @@ fn default_query_timeout() -> i64 { 60 } +fn default_page_limit() -> i64 { + 100 +} + impl From for crate::datasets::external::stac::StacDataProviderDefinition { diff --git a/geoengine/services/src/cli/stac_harvester/discover.rs b/geoengine/services/src/cli/stac_harvester/discover.rs index ac8808bc56..3f6b478efe 100644 --- a/geoengine/services/src/cli/stac_harvester/discover.rs +++ b/geoengine/services/src/cli/stac_harvester/discover.rs @@ -258,9 +258,12 @@ fn scan_collection_bands( continue; } - if let Ok(Some(bands)) = - scan_collection_item_asset(&collection.version, asset, collection.summaries.as_ref(), Some(asset_key.as_str())) - { + if let Ok(Some(bands)) = scan_collection_item_asset( + &collection.version, + asset, + collection.summaries.as_ref(), + Some(asset_key.as_str()), + ) { merge_dataset_bands(&mut dataset_bands, bands); } } @@ -393,12 +396,11 @@ fn process_sample_assets( }; let asset_title = asset.title.as_deref().unwrap_or(asset_key).to_string(); - let asset_info = common::band_names_from_asset_v1_1_0(asset, Some(asset_key.as_str())).unwrap_or_else(|_| { - common::AssetBandInfo { + let asset_info = common::band_names_from_asset_v1_1_0(asset, Some(asset_key.as_str())) + .unwrap_or_else(|_| common::AssetBandInfo { asset_title: asset_title.clone(), band_names: vec![asset_title.clone()], - } - }); + }); let entry = sample_band_info.entry(partial_key.clone()).or_default(); for bn in &asset_info.band_names { @@ -498,7 +500,11 @@ fn build_datasets( let unit_suffix = get_unit_suffix(dataset_key.epsg); let dataset_name = format!( "{} EPSG:{} {:?} {}{}", - stac_collection, dataset_key.epsg, dataset_key.data_type, dataset_key.resolution, unit_suffix + stac_collection, + dataset_key.epsg, + dataset_key.data_type, + dataset_key.resolution, + unit_suffix ); datasets.push(StacProviderDataset { @@ -664,7 +670,9 @@ fn scan_collection_item_asset( ) -> Result>>, String> { match collection_version { stac::Version::v1_0_0 => scan_collection_item_asset_v1_0_0(asset, asset_key), - stac::Version::v1_1_0 => scan_collection_item_asset_v1_1_0(asset, collection_summaries, asset_key), + stac::Version::v1_1_0 => { + scan_collection_item_asset_v1_1_0(asset, collection_summaries, asset_key) + } _ => { // For unknown STAC versions, try v1.1.0 first (more common), fall back to v1.0.0 scan_collection_item_asset_v1_1_0(asset, collection_summaries, asset_key) @@ -836,12 +844,12 @@ fn merge_dataset_bands( fn get_unit_suffix(epsg: u32) -> &'static str { match epsg { // Geographic CRS codes (WGS84, ETRS89, and other lat/lon coordinates) - 4258 | 4267 | 4269 | 4276 | 4277 | 4278 | 4279 | 4289 | 4291 | 4308 | 4309 - | 4311 | 4312 | 4313 | 4314 | 4315 | 4316 | 4317 | 4318 | 4319 | 4322 | 4326 | 4357 - | 4359 | 4360 | 4361 | 4362 | 4363 | 4364 | 4365 | 4366 | 4367 | 4368 | 4369 | 4370 - | 4371 | 4372 | 4373 | 4374 | 4375 | 4376 | 4377 | 4378 | 4379 | 4380 | 4381 | 4382 - | 4383 | 4384 | 4385 | 4386 | 4387 | 4388 | 4389 | 4390 | 4391 | 4392 | 4393 | 4394 - | 4395 | 4396 | 4397 | 4398 | 4399 => "deg", + 4258 | 4267 | 4269 | 4276 | 4277 | 4278 | 4279 | 4289 | 4291 | 4308 | 4309 | 4311 + | 4312 | 4313 | 4314 | 4315 | 4316 | 4317 | 4318 | 4319 | 4322 | 4326 | 4357 | 4359 + | 4360 | 4361 | 4362 | 4363 | 4364 | 4365 | 4366 | 4367 | 4368 | 4369 | 4370 | 4371 + | 4372 | 4373 | 4374 | 4375 | 4376 | 4377 | 4378 | 4379 | 4380 | 4381 | 4382 | 4383 + | 4384 | 4385 | 4386 | 4387 | 4388 | 4389 | 4390 | 4391 | 4392 | 4393 | 4394 | 4395 + | 4396 | 4397 | 4398 | 4399 => "deg", // Projected CRS (UTM and others) use meters _ => "m", } diff --git a/geoengine/services/src/cli/stac_harvester/harvest.rs b/geoengine/services/src/cli/stac_harvester/harvest.rs index 8612d3cd4e..84f94e854b 100644 --- a/geoengine/services/src/cli/stac_harvester/harvest.rs +++ b/geoengine/services/src/cli/stac_harvester/harvest.rs @@ -191,7 +191,8 @@ pub(super) async fn harvest_tiles(params: StacHarvest) -> Result<(), anyhow::Err upload_tiles_to_datasets(&api_config, ¶ms, &tiles_by_dataset).await?; - create_harvest_layer_collections(&api_config, provider_def, &created_datasets, ¶ms).await?; + create_harvest_layer_collections(&api_config, provider_def, &provider_def.datasets, ¶ms) + .await?; let elapsed = start_time.elapsed(); info!("Harvest completed in {:.2?}", elapsed); @@ -418,7 +419,8 @@ async fn create_dataset_api( .header("Authorization", format!("Bearer {session_id}")) .json(&create_dataset_req) .send() - .await + .await? + .error_for_status() }, &format!("Create dataset '{dataset_name}'"), &RetryPolicy::new(), @@ -426,12 +428,6 @@ async fn create_dataset_api( ) .await?; - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - anyhow::bail!("Failed to create dataset '{dataset_name}': HTTP {status}: {body}"); - } - let created_name = if let Ok(json) = response.json::().await { json.get("datasetName") .and_then(|v| v.as_str()) @@ -481,7 +477,8 @@ async fn share_dataset_api( .header("Authorization", format!("Bearer {session_id}")) .json(permission) .send() - .await + .await? + .error_for_status() }, &format!("Add permission for dataset '{dataset_name}'"), &RetryPolicy::new(), @@ -496,7 +493,7 @@ async fn share_dataset_api( async fn create_harvest_layer_collections( api_config: &ApiConfig, provider_def: &StacDataProviderDefinition, - created_datasets: &[(usize, StacProviderDataset)], + datasets: &[StacProviderDataset], params: &StacHarvest, ) -> Result<(), anyhow::Error> { let geo_engine_url = &api_config.base_path; @@ -524,7 +521,7 @@ async fn create_harvest_layer_collections( ) .await?; - for (_idx, dataset) in created_datasets { + for dataset in datasets { let dataset_name = dataset_name_for_harvest(&provider_def.collection_name, dataset); let layer_name = format!( "EPSG:{} {:?} {}m", @@ -533,6 +530,13 @@ async fn create_harvest_layer_collections( dataset.resolution.x ); + if child_layer_exists(api_config, &temp_collection_id, &layer_name).await? { + if params.verbose { + info!("Found existing layer '{layer_name}'"); + } + continue; + } + let add_layer = AddLayer { name: layer_name.clone(), description: format!("Dataset: {dataset_name}"), @@ -564,6 +568,7 @@ async fn create_harvest_layer_collections( .json(&add_layer) .send() .await? + .error_for_status()? .json() .await }, @@ -614,6 +619,7 @@ async fn create_layer_collection_api( .json(&add_collection) .send() .await? + .error_for_status()? .json() .await }, @@ -651,6 +657,7 @@ async fn find_child_collection_by_name( .header("Authorization", format!("Bearer {session_id}")) .send() .await? + .error_for_status()? .json() .await }, @@ -676,6 +683,53 @@ async fn find_child_collection_by_name( } } +async fn child_layer_exists( + api_config: &ApiConfig, + parent_id: &LayerCollectionId, + child_name: &str, +) -> Result { + let geo_engine_url = &api_config.base_path; + let session_id = api_config.bearer_access_token.as_deref().unwrap_or(""); + let client = &api_config.client; + let mut offset: u32 = 0; + let limit: u32 = 20; + + loop { + let response: LayerCollection = retry_http( + || async { + client + .get(format!( + "{geo_engine_url}/layers/collections/{INTERNAL_PROVIDER_ID}/{parent_id}" + )) + .query(&[("offset", offset), ("limit", limit)]) + .header("Authorization", format!("Bearer {session_id}")) + .send() + .await? + .error_for_status()? + .json() + .await + }, + &format!("List child layers of {parent_id}"), + &RetryPolicy::new(), + |e| e.status().map(|s| s.as_u16()), + ) + .await?; + + if response + .items + .iter() + .any(|item| matches!(item, CollectionItem::Layer(layer) if layer.name == child_name)) + { + return Ok(true); + } + + if response.items.len() < limit as usize { + return Ok(false); + } + offset += limit; + } +} + async fn share_layer_collection_api( api_config: &ApiConfig, collection_id: &LayerCollectionId, @@ -712,7 +766,8 @@ async fn share_layer_collection_api( .header("Authorization", format!("Bearer {session_id}")) .json(permission) .send() - .await + .await? + .error_for_status() }, &format!("Share collection with role {}", permission.role_id), &RetryPolicy::new(), @@ -757,7 +812,8 @@ async fn share_layer_api(api_config: &ApiConfig, layer_id: &LayerId) -> Result<( .header("Authorization", format!("Bearer {session_id}")) .json(permission) .send() - .await + .await? + .error_for_status() }, &format!("Share layer with role {}", permission.role_id), &RetryPolicy::new(), diff --git a/geoengine/services/src/cli/stac_import.rs b/geoengine/services/src/cli/stac_import.rs index 4715e6d4a2..0bdf348606 100644 --- a/geoengine/services/src/cli/stac_import.rs +++ b/geoengine/services/src/cli/stac_import.rs @@ -2302,7 +2302,10 @@ fn band_names_from_asset_v1_1_0(asset: &stac::Asset) -> anyhow::Result) -> anyhow::Result> { +fn band_names_from_item_asset_v1_1_0( + asset: &stac::ItemAsset, + asset_key: Option<&str>, +) -> anyhow::Result> { let asset_title = asset .title .as_deref() diff --git a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql index c82c346403..f7ec1a9690 100644 --- a/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql +++ b/geoengine/services/src/contexts/migrations/migration_0029_stac_provider_band_name.sql @@ -90,7 +90,7 @@ BEGIN (def).time_dimension, new_datasets, (def).query_timeout_secs, - (def).page_limit + COALESCE((def).page_limit, 100) )::"StacDataProviderDefinition"; END; $$ LANGUAGE plpgsql; diff --git a/geoengine/services/src/datasets/external/stac/loading_info.rs b/geoengine/services/src/datasets/external/stac/loading_info.rs index e6a148461e..42acdc2624 100644 --- a/geoengine/services/src/datasets/external/stac/loading_info.rs +++ b/geoengine/services/src/datasets/external/stac/loading_info.rs @@ -88,8 +88,10 @@ async fn query_stac_item_collection( |e| e.status().map(|s| s.as_u16()), ) .await - .map_err(|e| geoengine_operators::error::Error::QueryingProcessorFailed { - source: Box::new(e), + .map_err(|e| { + geoengine_operators::error::Error::QueryingProcessorFailed { + source: Box::new(e), + } })?; debug!( @@ -128,8 +130,10 @@ async fn query_stac_item_collection( |e| e.status().map(|s| s.as_u16()), ) .await - .map_err(|e| geoengine_operators::error::Error::QueryingProcessorFailed { - source: Box::new(e), + .map_err(|e| { + geoengine_operators::error::Error::QueryingProcessorFailed { + source: Box::new(e), + } })?; debug!( diff --git a/geoengine/services/src/datasets/external/stac/mod.rs b/geoengine/services/src/datasets/external/stac/mod.rs index d6b4cce9ed..a794ccfcb8 100644 --- a/geoengine/services/src/datasets/external/stac/mod.rs +++ b/geoengine/services/src/datasets/external/stac/mod.rs @@ -19,6 +19,7 @@ mod listing; mod loading_info; const DEFAULT_QUERY_TIMEOUT_SECS: i64 = 60; +const DEFAULT_PAGE_LIMIT: i64 = 100; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSql, FromSql)] #[postgres(name = "StacDataProviderDefinition")] @@ -36,6 +37,7 @@ pub struct StacDataProviderDefinition { /// Timeout in seconds for outgoing STAC API HTTP requests. #[serde(default = "default_query_timeout")] pub query_timeout_secs: i64, + #[serde(default = "default_page_limit")] pub page_limit: i64, } @@ -43,6 +45,10 @@ fn default_query_timeout() -> i64 { DEFAULT_QUERY_TIMEOUT_SECS } +fn default_page_limit() -> i64 { + DEFAULT_PAGE_LIMIT +} + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSql, FromSql)] #[postgres(name = "StacProviderS3Config")] pub struct StacProviderS3Config { diff --git a/geoengine/services/src/util/retry.rs b/geoengine/services/src/util/retry.rs index 6eb3e55370..547d35440b 100644 --- a/geoengine/services/src/util/retry.rs +++ b/geoengine/services/src/util/retry.rs @@ -190,8 +190,8 @@ fn is_terminal( mod tests { use super::{RetryPolicy, retry_http}; use std::fmt; - use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; #[derive(Debug, Clone)] struct TestError { diff --git a/geoengine/test_data/stac_responses/expected-mapping-code-de.json b/geoengine/test_data/stac_responses/expected-mapping-code-de.json index daa85db195..77e07c6f4a 100644 --- a/geoengine/test_data/stac_responses/expected-mapping-code-de.json +++ b/geoengine/test_data/stac_responses/expected-mapping-code-de.json @@ -52,10 +52,10 @@ { "assetBand": { "assetTitle": "Blue (band 2) - 10m", - "bandName": "Blue (band 2) - 10m" + "bandName": "B02_10m" }, "bandDescriptor": { - "name": "Blue (band 2) - 10m", + "name": "B02_10m", "measurement": { "type": "unitless" } @@ -64,10 +64,10 @@ { "assetBand": { "assetTitle": "Green (band 3) - 10m", - "bandName": "Green (band 3) - 10m" + "bandName": "B03_10m" }, "bandDescriptor": { - "name": "Green (band 3) - 10m", + "name": "B03_10m", "measurement": { "type": "unitless" } @@ -75,11 +75,11 @@ }, { "assetBand": { - "assetTitle": "NIR 1 (band 8) - 10m", - "bandName": "NIR 1 (band 8) - 10m" + "assetTitle": "Red (band 4) - 10m", + "bandName": "B04_10m" }, "bandDescriptor": { - "name": "NIR 1 (band 8) - 10m", + "name": "B04_10m", "measurement": { "type": "unitless" } @@ -87,11 +87,11 @@ }, { "assetBand": { - "assetTitle": "Red (band 4) - 10m", - "bandName": "Red (band 4) - 10m" + "assetTitle": "NIR 1 (band 8) - 10m", + "bandName": "B08_10m" }, "bandDescriptor": { - "name": "Red (band 4) - 10m", + "name": "B08_10m", "measurement": { "type": "unitless" } @@ -135,10 +135,10 @@ { "assetBand": { "assetTitle": "SWIR 1 (band 11) - 20m", - "bandName": "SWIR 1 (band 11) - 20m" + "bandName": "B11_20m" }, "bandDescriptor": { - "name": "SWIR 1 (band 11) - 20m", + "name": "B11_20m", "measurement": { "type": "unitless" } @@ -147,10 +147,10 @@ { "assetBand": { "assetTitle": "SWIR 2 (band 12) - 20m", - "bandName": "SWIR 2 (band 12) - 20m" + "bandName": "B12_20m" }, "bandDescriptor": { - "name": "SWIR 2 (band 12) - 20m", + "name": "B12_20m", "measurement": { "type": "unitless" } diff --git a/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json b/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json index 541b6d794d..d548dc7aa8 100644 --- a/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json +++ b/geoengine/test_data/stac_responses/expected-mapping-landsat-c2-l1.json @@ -52,10 +52,10 @@ { "assetBand": { "assetTitle": "Blue (band 2) - 30m", - "bandName": "Blue (band 2) - 30m" + "bandName": "B2_30m" }, "bandDescriptor": { - "name": "Blue (band 2) - 30m", + "name": "B2_30m", "measurement": { "type": "unitless" } @@ -64,10 +64,10 @@ { "assetBand": { "assetTitle": "Green (band 3) - 30m", - "bandName": "Green (band 3) - 30m" + "bandName": "B3_30m" }, "bandDescriptor": { - "name": "Green (band 3) - 30m", + "name": "B3_30m", "measurement": { "type": "unitless" } @@ -75,11 +75,11 @@ }, { "assetBand": { - "assetTitle": "NIR (band 5) - 30m", - "bandName": "NIR (band 5) - 30m" + "assetTitle": "Red (band 4) - 30m", + "bandName": "B4_30m" }, "bandDescriptor": { - "name": "NIR (band 5) - 30m", + "name": "B4_30m", "measurement": { "type": "unitless" } @@ -87,11 +87,11 @@ }, { "assetBand": { - "assetTitle": "Red (band 4) - 30m", - "bandName": "Red (band 4) - 30m" + "assetTitle": "NIR (band 5) - 30m", + "bandName": "B5_30m" }, "bandDescriptor": { - "name": "Red (band 4) - 30m", + "name": "B5_30m", "measurement": { "type": "unitless" } From 8e87af2b6ab245533af3bb2e157f9247e5f3b388 Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Fri, 21 Aug 2026 09:17:04 +0000 Subject: [PATCH 17/27] consolidate methods --- geoengine/services/src/api/model/operators.rs | 38 ++++---- geoengine/services/src/api/model/services.rs | 38 +------- .../src/cli/stac_harvester/discover.rs | 90 ++++++++----------- 3 files changed, 61 insertions(+), 105 deletions(-) diff --git a/geoengine/services/src/api/model/operators.rs b/geoengine/services/src/api/model/operators.rs index b9fe499965..3033a59678 100644 --- a/geoengine/services/src/api/model/operators.rs +++ b/geoengine/services/src/api/model/operators.rs @@ -86,6 +86,26 @@ pub enum TimeDimension { Irregular, } +impl From for geoengine_datatypes::primitives::TimeDimension { + fn from(value: TimeDimension) -> Self { + match value { + TimeDimension::Regular(regular) => Self::Regular(regular.into()), + TimeDimension::Irregular => Self::Irregular, + } + } +} + +impl From for TimeDimension { + fn from(value: geoengine_datatypes::primitives::TimeDimension) -> Self { + match value { + geoengine_datatypes::primitives::TimeDimension::Regular(regular) => { + Self::Regular(regular.into()) + } + geoengine_datatypes::primitives::TimeDimension::Irregular => Self::Irregular, + } + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct RegularTimeDimension { @@ -122,14 +142,7 @@ impl From for geoengine_operators::engine::TimeDescriptor { fn from(value: TimeDescriptor) -> Self { geoengine_operators::engine::TimeDescriptor::new( value.bounds.map(Into::into), - match value.dimension { - TimeDimension::Regular(d) => { - geoengine_datatypes::primitives::TimeDimension::Regular(d.into()) - } - TimeDimension::Irregular => { - geoengine_datatypes::primitives::TimeDimension::Irregular - } - }, + value.dimension.into(), ) } } @@ -138,14 +151,7 @@ impl From for TimeDescriptor { fn from(value: geoengine_operators::engine::TimeDescriptor) -> Self { Self { bounds: value.bounds.map(Into::into), - dimension: match value.dimension { - geoengine_datatypes::primitives::TimeDimension::Regular(d) => { - TimeDimension::Regular(d.into()) - } - geoengine_datatypes::primitives::TimeDimension::Irregular => { - TimeDimension::Irregular - } - }, + dimension: value.dimension.into(), } } } diff --git a/geoengine/services/src/api/model/services.rs b/geoengine/services/src/api/model/services.rs index 78b3c0d37a..9e85e6a6e5 100644 --- a/geoengine/services/src/api/model/services.rs +++ b/geoengine/services/src/api/model/services.rs @@ -6,8 +6,7 @@ use super::operators::TypedResultDescriptor; use crate::api::model::datatypes::MlModelName; use crate::api::model::operators::{ GdalMetaDataList, GdalMetaDataRegular, GdalMetaDataStatic, GdalMetadataNetCdfCf, - MlModelMetadata, MockMetaData, OgrMetaData, RegularTimeDimension, SpatialGridDescriptor, - TimeDimension, + MlModelMetadata, MockMetaData, OgrMetaData, SpatialGridDescriptor, TimeDimension, }; use crate::datasets::DatasetName; use crate::datasets::external::{GdalRetries, WildliveDataConnectorAuth}; @@ -1083,37 +1082,6 @@ impl From for StacProvider } } -#[allow(clippy::needless_pass_by_value)] -fn api_time_dimension_to_datatypes( - value: TimeDimension, -) -> geoengine_datatypes::primitives::TimeDimension { - match value { - TimeDimension::Regular(RegularTimeDimension { origin, step }) => { - geoengine_datatypes::primitives::TimeDimension::Regular( - geoengine_datatypes::primitives::RegularTimeDimension { - origin: origin.into(), - step: step.into(), - }, - ) - } - TimeDimension::Irregular => geoengine_datatypes::primitives::TimeDimension::Irregular, - } -} - -fn datatypes_time_dimension_to_api( - value: geoengine_datatypes::primitives::TimeDimension, -) -> TimeDimension { - match value { - geoengine_datatypes::primitives::TimeDimension::Regular( - geoengine_datatypes::primitives::RegularTimeDimension { origin, step }, - ) => TimeDimension::Regular(RegularTimeDimension { - origin: origin.into(), - step: step.into(), - }), - geoengine_datatypes::primitives::TimeDimension::Irregular => TimeDimension::Irregular, - } -} - #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, ToSchema)] #[serde(rename_all = "camelCase")] pub struct StacTimeStep { @@ -1179,7 +1147,7 @@ impl From api_url: value.api_url, collection_name: value.collection_name, s3_config: value.s3_config.map(Into::into), - time_dimension: api_time_dimension_to_datatypes(value.time_dimension), + time_dimension: value.time_dimension.into(), datasets: value.datasets.into_iter().map(Into::into).collect(), page_limit: value.page_limit, query_timeout_secs: value.query_timeout_secs, @@ -1200,7 +1168,7 @@ impl From api_url: value.api_url, collection_name: value.collection_name, s3_config: value.s3_config.map(Into::into), - time_dimension: datatypes_time_dimension_to_api(value.time_dimension), + time_dimension: value.time_dimension.into(), datasets: value.datasets.into_iter().map(Into::into).collect(), page_limit: value.page_limit, query_timeout_secs: value.query_timeout_secs, diff --git a/geoengine/services/src/cli/stac_harvester/discover.rs b/geoengine/services/src/cli/stac_harvester/discover.rs index 3f6b478efe..c0dcef3896 100644 --- a/geoengine/services/src/cli/stac_harvester/discover.rs +++ b/geoengine/services/src/cli/stac_harvester/discover.rs @@ -4,7 +4,7 @@ use anyhow::Context; use clap::{Parser, ValueEnum}; use geoengine_datatypes::{ dataset::DataProviderId, - primitives::SpatialResolution, + primitives::{AxisAlignedRectangle, BoundingBox2D, SpatialResolution}, raster::{GeoTransform, GridBoundingBox2D, GridIdx2D, RasterDataType}, spatial_reference::{SpatialReference, SpatialReferenceAuthority}, util::Identifier, @@ -12,14 +12,15 @@ use geoengine_datatypes::{ use ordered_float::OrderedFloat; use tracing::{info, warn}; +use crate::api::model::{ + datatypes::{TimeGranularity, TimeStep}, + operators::{RegularTimeDimension, TimeDimension}, +}; use crate::datasets::external::stac::{ StacAssetBand, StacDataProviderDefinition, StacProviderDataset, StacProviderDatasetBand, StacProviderS3Config, common, }; use crate::util::retry::{RetryPolicy, retry_http}; -use geoengine_datatypes::primitives::{ - RegularTimeDimension as DtRegularTimeDimension, TimeDimension as DtTimeDimension, -}; use geoengine_operators::engine::SpatialGridDescriptor as GeoOpSpatialGridDescriptor; // --------------------------------------------------------------------------- @@ -96,12 +97,12 @@ pub struct StacDiscoverMapping { pub id: Option, /// Time dimension granularity (default: days) - #[arg(long, default_value = "days")] - pub time_granularity: String, + #[arg(long, default_value = "days", value_parser = parse_time_granularity)] + pub time_granularity: TimeGranularity, /// Time dimension step (default: 1) #[arg(long, default_value_t = 1)] - pub time_step: u64, + pub time_step: u32, } #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] @@ -162,8 +163,15 @@ pub(super) async fn discover_mapping(params: StacDiscoverMapping) -> Result<(), ); } - let time_dimension = parse_time_dimension(¶ms.time_granularity, params.time_step) - .map_err(|e| anyhow::anyhow!("{e}"))?; + let time_dimension: geoengine_datatypes::primitives::TimeDimension = + TimeDimension::Regular(RegularTimeDimension { + origin: geoengine_datatypes::primitives::TimeInstance::from_millis_unchecked(0).into(), + step: TimeStep { + granularity: params.time_granularity, + step: params.time_step, + }, + }) + .into(); let s3_config = params .s3_endpoint @@ -625,22 +633,16 @@ fn projection_grid_bounds(gt: GeoTransform, epsg: u32) -> Option Option<(f64, f64, f64, f64)> { - use proj::Proj; - - let proj_crs = Proj::new(&format!("EPSG:{epsg}")).ok()?; - - // Get area of use in degrees (WGS84) - let (area, _name) = proj_crs.area_of_use().ok()?; - let area = area?; - - // Project the WGS84 bounding box into the target CRS - let pipeline = Proj::new_known_crs("EPSG:4326", &format!("EPSG:{epsg}"), None).ok()?; - let bounds = pipeline - .transform_bounds(area.west, area.south, area.east, area.north, 21) + let extent: BoundingBox2D = SpatialReference::new(SpatialReferenceAuthority::Epsg, epsg) + .area_of_use_projected() .ok()?; - // transform_bounds returns [west, south, east, north] in the target CRS - Some((bounds[0], bounds[2], bounds[1], bounds[3])) + Some(( + extent.lower_left().x, + extent.upper_right().x, + extent.lower_left().y, + extent.upper_right().y, + )) } /// A key that uniquely identifies a Geo Engine dataset derived from STAC assets. @@ -842,38 +844,18 @@ fn merge_dataset_bands( /// Determine the unit suffix for a dataset based on its EPSG code. /// Geographic CRS (like EPSG:4326) use degrees, projected CRS (like UTM) use meters. fn get_unit_suffix(epsg: u32) -> &'static str { - match epsg { - // Geographic CRS codes (WGS84, ETRS89, and other lat/lon coordinates) - 4258 | 4267 | 4269 | 4276 | 4277 | 4278 | 4279 | 4289 | 4291 | 4308 | 4309 | 4311 - | 4312 | 4313 | 4314 | 4315 | 4316 | 4317 | 4318 | 4319 | 4322 | 4326 | 4357 | 4359 - | 4360 | 4361 | 4362 | 4363 | 4364 | 4365 | 4366 | 4367 | 4368 | 4369 | 4370 | 4371 - | 4372 | 4373 | 4374 | 4375 | 4376 | 4377 | 4378 | 4379 | 4380 | 4381 | 4382 | 4383 - | 4384 | 4385 | 4386 | 4387 | 4388 | 4389 | 4390 | 4391 | 4392 | 4393 | 4394 | 4395 - | 4396 | 4397 | 4398 | 4399 => "deg", - // Projected CRS (UTM and others) use meters - _ => "m", + let spatial_reference = SpatialReference::new(SpatialReferenceAuthority::Epsg, epsg); + + if spatial_reference.uses_meters().unwrap_or(false) { + "m" + } else { + "deg" } } -fn parse_time_dimension(granularity: &str, step: u64) -> Result { - let dt_granularity = match granularity.to_lowercase().as_str() { - "days" | "day" => geoengine_datatypes::primitives::TimeGranularity::Days, - "months" | "month" => geoengine_datatypes::primitives::TimeGranularity::Months, - "years" | "year" => geoengine_datatypes::primitives::TimeGranularity::Years, - "hours" | "hour" => geoengine_datatypes::primitives::TimeGranularity::Hours, - other => return Err(format!("Unsupported time granularity: {other}")), - }; - - let step_u32: u32 = step - .try_into() - .map_err(|_| format!("step {step} exceeds u32 range"))?; - - Ok(DtTimeDimension::Regular( - DtRegularTimeDimension::new_with_epoch_origin(geoengine_datatypes::primitives::TimeStep { - granularity: dt_granularity, - step: step_u32, - }), - )) +fn parse_time_granularity(value: &str) -> Result { + serde_json::from_value(serde_json::Value::String(value.to_lowercase())) + .map_err(|error| error.to_string()) } // --------------------------------------------------------------------------- @@ -1000,7 +982,7 @@ mod tests { bbox: None, full_projection_grid: false, verbose: false, - time_granularity: "days".to_string(), + time_granularity: TimeGranularity::Days, time_step: 1, page_limit: 100, id: None, @@ -1112,7 +1094,7 @@ mod tests { bbox: None, full_projection_grid: false, verbose: false, - time_granularity: "days".to_string(), + time_granularity: TimeGranularity::Days, time_step: 1, page_limit: 100, id: None, From 7d98850053263327f6b09647a0cf1d4f84ee298b Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Fri, 21 Aug 2026 10:37:07 +0000 Subject: [PATCH 18/27] fix dataset bounds extension --- .../services/src/api/handlers/datasets.rs | 16 ++-- geoengine/services/src/datasets/postgres.rs | 77 +++++++++++++++---- 2 files changed, 65 insertions(+), 28 deletions(-) diff --git a/geoengine/services/src/api/handlers/datasets.rs b/geoengine/services/src/api/handlers/datasets.rs index 283e2cbc18..57c51e645f 100755 --- a/geoengine/services/src/api/handlers/datasets.rs +++ b/geoengine/services/src/api/handlers/datasets.rs @@ -5341,13 +5341,7 @@ mod tests { ) .into(), time: TimeDescriptor { - bounds: Some( - TimeInterval::new_unchecked( - TimeInstance::from_str("2014-01-01T00:00:00Z").unwrap(), - TimeInstance::from_str("2014-01-02T00:00:00Z").unwrap(), - ) - .into(), - ), + bounds: None, dimension: TimeDimension::Irregular, }, spatial_grid: SpatialGridDescriptor { @@ -5398,8 +5392,8 @@ mod tests { ) .into(), spatial_partition: SpatialPartition2D::new_unchecked( - (50., -50.).into(), - (150., -150.).into(), + (0., 0.).into(), + (100., -100.).into(), ) .into(), band: 0, @@ -5488,8 +5482,8 @@ mod tests { ) .into(), spatial_partition: SpatialPartition2D::new_unchecked( - (50., -50.).into(), - (150., -150.).into(), + (-50., 50.).into(), + (0., 0.).into(), ) .into(), band: 0, diff --git a/geoengine/services/src/datasets/postgres.rs b/geoengine/services/src/datasets/postgres.rs index 5e90ef73b3..59aebe790e 100644 --- a/geoengine/services/src/datasets/postgres.rs +++ b/geoengine/services/src/datasets/postgres.rs @@ -27,7 +27,7 @@ use geoengine_datatypes::primitives::{ TryRegularTimeFillIterExt, }; use geoengine_datatypes::primitives::{TimeInterval, VectorQueryRectangle}; -use geoengine_datatypes::raster::{GridBoundingBox2D, SpatialGridDefinition}; +use geoengine_datatypes::raster::{GridBoundingBoxExt, SpatialGridDefinition}; use geoengine_datatypes::util::Identifier; use geoengine_operators::engine::TypedResultDescriptor; use geoengine_operators::engine::{ @@ -35,7 +35,7 @@ use geoengine_operators::engine::{ }; use geoengine_operators::mock::MockDatasetDataSourceLoadingInfo; use geoengine_operators::source::{ - GdalDatasetGeoTransform, GdalDatasetParameters, GdalLoadingInfo, MultiBandGdalLoadingInfo, + GdalDatasetParameters, GdalLoadingInfo, MultiBandGdalLoadingInfo, MultiBandGdalLoadingInfoQueryRectangle, OgrSourceDataset, TileFile, }; use postgres_types::{FromSql, ToSql}; @@ -1447,6 +1447,30 @@ async fn batch_insert_tiles( Ok(()) } +fn extend_time_bounds( + time_bounds: Option, + tile_time: TimeInterval, +) -> Option { + Some(match time_bounds { + Some(time_bounds) => time_bounds.extend(&tile_time), + None => tile_time, + }) +} + +fn extend_spatial_bounds( + dataset_grid: SpatialGridDefinition, + tile_partition: SpatialPartition2D, +) -> SpatialGridDefinition { + let tile_grid = dataset_grid.spatial_bounds_to_compatible_spatial_grid(tile_partition.into()); + + SpatialGridDefinition::new( + dataset_grid.geo_transform(), + dataset_grid + .grid_bounds() + .extended(&tile_grid.grid_bounds()), + ) +} + async fn update_dataset_extents( tx: &Transaction<'_>, dataset: DatasetId, @@ -1472,22 +1496,9 @@ async fn update_dataset_extents( (row.get(0), row.get(1)); for tile in tiles { - // TODO: handle datasets with flipped y axis? - let tile_grid = SpatialGridDefinition::new( - GdalDatasetGeoTransform::from(tile.params.geo_transform).try_into()?, - GridBoundingBox2D::new_unchecked( - [0, 0], - [tile.params.height as isize, tile.params.width as isize], - ), - ); - - dataset_grid = dataset_grid - .merge(&tile_grid) - .expect("grids should be compatible because the compatibility was checked before inserting tiles"); + dataset_grid = extend_spatial_bounds(dataset_grid, tile.spatial_partition); - if let Some(time_bounds) = &mut time_bounds { - *time_bounds = time_bounds.extend(&tile.time.into()); - } + time_bounds = extend_time_bounds(time_bounds, tile.time.into()); } tx.execute( @@ -1600,6 +1611,38 @@ mod tests { }; use tokio_postgres::NoTls; + #[test] + fn it_initializes_missing_dataset_time_bounds_from_the_first_tile() { + let tile_time = TimeInterval::new_unchecked( + TimeInstance::from_millis_unchecked(1_000), + TimeInstance::from_millis_unchecked(2_000), + ); + + assert_eq!(extend_time_bounds(None, tile_time), Some(tile_time)); + } + + #[test] + fn dataset_grid_takes_precedence_when_extending_spatial_bounds() { + use geoengine_datatypes::{ + primitives::SpatialPartition2D as DatatypeSpatialPartition2D, + raster::{GeoTransform, GridBoundingBox2D, SpatialGridDefinition}, + }; + + let dataset_grid = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(0., 10., 0., -10.), + GridBoundingBox2D::new_unchecked([0, 0], [9, 9]), + ); + let original_partition = dataset_grid.spatial_partition(); + let tile_partition = + DatatypeSpatialPartition2D::new_unchecked((-15., 25.).into(), (105., -115.).into()); + + let extended = extend_spatial_bounds(dataset_grid, tile_partition.into()); + + assert_eq!(extended.geo_transform(), dataset_grid.geo_transform()); + assert!(extended.spatial_partition().contains(&original_partition)); + assert!(extended.spatial_partition().contains(&tile_partition)); + } + #[ge_context::test] async fn it_autocompletes_datasets(app_ctx: PostgresContext) { let session_a = app_ctx.create_anonymous_session().await.unwrap(); From 67faf9ecf29fd02cb2961fb84fafec228a46cfa1 Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Mon, 24 Aug 2026 14:01:59 +0000 Subject: [PATCH 19/27] simplify full projection grid --- .../src/cli/stac_harvester/discover.rs | 207 ++++++++++++++---- 1 file changed, 170 insertions(+), 37 deletions(-) diff --git a/geoengine/services/src/cli/stac_harvester/discover.rs b/geoengine/services/src/cli/stac_harvester/discover.rs index c0dcef3896..5cc2671838 100644 --- a/geoengine/services/src/cli/stac_harvester/discover.rs +++ b/geoengine/services/src/cli/stac_harvester/discover.rs @@ -594,57 +594,62 @@ fn zero_size_grid() -> GridBoundingBox2D { .expect("zero-size grid bounds should always be valid") } -/// Compute grid bounds that cover the full projected CRS extent for the given -/// geo-transform and EPSG code. Currently handles UTM projections (zones 32601–32660 -/// and 32701–32760) with known extents. Returns `None` for unsupported CRS types. +/// Compute the grid bounds (in pixel space) that cover the full extent of the +/// given CRS, snapping the extent to the pixel grid defined by `gt`'s origin and +/// pixel size. +/// +/// The extent is taken from PROJ's area of use for the CRS, projected into the +/// CRS's own coordinates, so no per-CRS coordinates are hard-coded here. fn projection_grid_bounds(gt: GeoTransform, epsg: u32) -> Option { - let (min_x, max_x, min_y, max_y) = projection_extent(epsg)?; + let extent: BoundingBox2D = SpatialReference::new(SpatialReferenceAuthority::Epsg, epsg) + .area_of_use_projected() + .ok()?; let ox = gt.origin_coordinate.x; let oy = gt.origin_coordinate.y; let ps_x = gt.x_pixel_size(); let ps_y = gt.y_pixel_size(); - // For north-up images ps_y < 0 and origin is top-left. - // Pixel index i = (coord - origin) / pixel_size. - let min_x_idx = ((min_x - ox) / ps_x).floor() as isize; - let max_x_idx = ((max_x - ox) / ps_x).ceil() as isize - 1; - - let (min_y_idx, max_y_idx) = if ps_y < 0.0 { - // ps_y negative: top of area (max_y) → smallest row index - let top = ((max_y - oy) / ps_y).ceil() as isize; - // bottom of area (min_y) → largest row index - let bottom = ((min_y - oy) / ps_y).floor() as isize; - (top, bottom) - } else { - let top = ((max_y - oy) / ps_y).floor() as isize; - let bottom = ((min_y - oy) / ps_y).ceil() as isize - 1; - (top, bottom) + // `grid_to_spatial_bounds` interprets the box's min index as the upper-left + // (top) pixel and the max index as the last pixel, so the box spans + // [min_idx, max_idx] inclusive in index space. + // + // In normalized index space a pixel with index `k` occupies `[k, k+1)`, so the + // pixel that *contains* a coordinate is `floor((coord - origin)/pixel_size)`. + // Using `floor` (rather than `round`/`ceil`) guarantees the resulting box is a + // superset of the extent for either pixel-size sign, so no per-axis special + // casing is needed. Snapping the extent's corners this way covers the whole + // projected area of use. + let corner_index = |x: f64, y: f64| -> (isize, isize) { + let idx_x = ((x - ox) / ps_x).floor() as isize; + let idx_y = ((y - oy) / ps_y).floor() as isize; + // The grid index array is ordered [y, x]. + (idx_y, idx_x) }; + let mut y_min = isize::MAX; + let mut y_max = isize::MIN; + let mut x_min = isize::MAX; + let mut x_max = isize::MIN; + for (y_idx, x_idx) in [ + corner_index(extent.lower_left().x, extent.lower_left().y), + corner_index(extent.upper_right().x, extent.upper_right().y), + corner_index(extent.lower_left().x, extent.upper_right().y), + corner_index(extent.upper_right().x, extent.lower_left().y), + ] { + y_min = y_min.min(y_idx); + y_max = y_max.max(y_idx); + x_min = x_min.min(x_idx); + x_max = x_max.max(x_idx); + } + GridBoundingBox2D::new( - GridIdx2D::new([min_x_idx, min_y_idx]), - GridIdx2D::new([max_x_idx, max_y_idx]), + GridIdx2D::new([y_min, x_min]), + GridIdx2D::new([y_max, x_max]), ) .ok() } -/// Return the projected extent `(min_x, max_x, min_y, max_y)` for a given EPSG code -/// by querying the CRS's area of use via PROJ and projecting it into the CRS's -/// own coordinate system. -fn projection_extent(epsg: u32) -> Option<(f64, f64, f64, f64)> { - let extent: BoundingBox2D = SpatialReference::new(SpatialReferenceAuthority::Epsg, epsg) - .area_of_use_projected() - .ok()?; - - Some(( - extent.lower_left().x, - extent.upper_right().x, - extent.lower_left().y, - extent.upper_right().y, - )) -} - /// A key that uniquely identifies a Geo Engine dataset derived from STAC assets. #[derive(Debug, Clone, Hash, PartialEq, Eq)] struct DatasetKey { @@ -919,8 +924,136 @@ async fn stac_api_request_with_params( #[cfg(test)] mod tests { use super::*; + use float_cmp::approx_eq; use httptest::{Expectation, Server, all_of, matchers::request, responders}; + /// Snap a projected coordinate to the pixel that contains it, using the same + /// origin/pixel-size convention as `projection_grid_bounds`. The containing + /// pixel is `floor` of the normalized index (a pixel at index `k` occupies the + /// half-open range `[k, k+1)` in index space), which is independent of the + /// pixel-size sign. + fn expect_index(coord: f64, origin: f64, pixel_size: f64) -> isize { + ((coord - origin) / pixel_size).floor() as isize + } + + #[test] + fn full_projection_grid_covers_projection_area_of_use() { + let geo_transform = GeoTransform::new_with_coordinate_x_y(363_600., 30., 4_105_800., -30.); + let extent: BoundingBox2D = SpatialReference::new(SpatialReferenceAuthority::Epsg, 32632) + .area_of_use_projected() + .expect("EPSG:32632 should have a projected area of use"); + + let bounds = projection_grid_bounds(geo_transform, 32632) + .expect("grid bounds should be computable for a valid projection"); + + // The returned pixel grid must contain the full projected area of use. + let covered = geo_transform.grid_to_spatial_bounds(&bounds); + assert!(covered.upper_left().x <= extent.lower_left().x); + assert!(covered.upper_left().y >= extent.upper_right().y); + assert!(covered.lower_right().x >= extent.upper_right().x); + assert!(covered.lower_right().y <= extent.lower_left().y); + } + + #[test] + fn full_projection_grid_epsg_32632_utm_32n_subregion() { + // EPSG:32632 (WGS 84 / UTM zone 32N) has an area of use that is a + // subregion of the full UTM zone (0..1_000_000 in x, 0..10_000_000 in y). + // The expected projected area-of-use bounds (west, east, south, north) + // are hard-coded below as the reference. + const WEST: f64 = 166_021.44; + const EAST: f64 = 833_978.56; + const SOUTH: f64 = 0.0; + const NORTH: f64 = 9_329_005.18; + const TOL: f64 = 1.0; + + let extent: BoundingBox2D = SpatialReference::new(SpatialReferenceAuthority::Epsg, 32632) + .area_of_use_projected() + .expect("EPSG:32632 should have a projected area of use"); + assert!(approx_eq!(f64, extent.lower_left().x, WEST, epsilon = TOL)); + assert!(approx_eq!(f64, extent.upper_right().x, EAST, epsilon = TOL)); + assert!(approx_eq!(f64, extent.lower_left().y, SOUTH, epsilon = TOL)); + assert!(approx_eq!( + f64, + extent.upper_right().y, + NORTH, + epsilon = TOL + )); + + // A north-up raster grid (100 m pixels) whose origin sits inside the + // subregion, slightly outside it so the grid extends in both directions. + let (ox, oy) = (166_000.0, 9_330_000.0); + let ps = 100.0; + let gt = GeoTransform::new_with_coordinate_x_y(ox, ps, oy, -ps); + + let bounds = projection_grid_bounds(gt, 32632) + .expect("grid bounds should be computable for EPSG:32632"); + + // The pixel grid must cover the whole projected area of use. + let covered = gt.grid_to_spatial_bounds(&bounds); + assert!(covered.upper_left().x <= extent.lower_left().x); + assert!(covered.upper_left().y >= extent.upper_right().y); + assert!(covered.lower_right().x >= extent.upper_right().x); + assert!(covered.lower_right().y <= extent.lower_left().y); + + // The pixel indices must be exactly derived from the reference bounds. + assert_eq!( + bounds.x_bounds(), + [expect_index(WEST, ox, ps), expect_index(EAST, ox, ps)] + ); + assert_eq!( + bounds.y_bounds(), + [expect_index(NORTH, oy, -ps), expect_index(SOUTH, oy, -ps)] + ); + } + + #[test] + fn full_projection_grid_epsg_4326_wgs84_globe() { + // EPSG:4326 (WGS 84 lat/lon) has the full globe as its area of use. + const WEST: f64 = -180.0; + const EAST: f64 = 180.0; + const SOUTH: f64 = -90.0; + const NORTH: f64 = 90.0; + const TOL: f64 = 1e-6; + + let extent: BoundingBox2D = SpatialReference::new(SpatialReferenceAuthority::Epsg, 4326) + .area_of_use_projected() + .expect("EPSG:4326 should have a projected area of use"); + assert!(approx_eq!(f64, extent.lower_left().x, WEST, epsilon = TOL)); + assert!(approx_eq!(f64, extent.upper_right().x, EAST, epsilon = TOL)); + assert!(approx_eq!(f64, extent.lower_left().y, SOUTH, epsilon = TOL)); + assert!(approx_eq!( + f64, + extent.upper_right().y, + NORTH, + epsilon = TOL + )); + + // A north-up geographic grid (1° pixels) with a top-left origin inside + // the globe, so the grid must extend west, east and south of it. + let (ox, oy) = (0.0, 60.0); + let ps = 1.0; + let gt = GeoTransform::new_with_coordinate_x_y(ox, ps, oy, -ps); + + let bounds = projection_grid_bounds(gt, 4326) + .expect("grid bounds should be computable for EPSG:4326"); + + // The pixel grid must cover the whole globe. + let covered = gt.grid_to_spatial_bounds(&bounds); + assert!(covered.upper_left().x <= extent.lower_left().x); + assert!(covered.upper_left().y >= extent.upper_right().y); + assert!(covered.lower_right().x >= extent.upper_right().x); + assert!(covered.lower_right().y <= extent.lower_left().y); + + assert_eq!( + bounds.x_bounds(), + [expect_index(WEST, ox, ps), expect_index(EAST, ox, ps)] + ); + assert_eq!( + bounds.y_bounds(), + [expect_index(NORTH, oy, -ps), expect_index(SOUTH, oy, -ps)] + ); + } + const COLLECTION_PATH: &str = "/v1/collections/sentinel-2-l2a"; const ITEMS_PATH: &str = "/v1/collections/sentinel-2-l2a/items"; From b0e8e8011f9e419955fba50ce830df14aaa4e451 Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Mon, 24 Aug 2026 14:24:42 +0000 Subject: [PATCH 20/27] port ui --- ui/package.json | 2 +- .../core/src/lib/map/map-layer.component.ts | 19 + .../src/app/app-config.service.ts | 2 +- .../src/app/main/main.component.html | 86 ++-- .../src/app/main/main.component.scss | 79 +++- .../src/app/main/main.component.ts | 440 ++++++++++++++++-- .../enhanced-data-viewer/src/assets/grey.jpg | Bin 0 -> 117808 bytes 7 files changed, 523 insertions(+), 105 deletions(-) create mode 100644 ui/projects/enhanced-data-viewer/src/assets/grey.jpg diff --git a/ui/package.json b/ui/package.json index e5517eee08..342e2f9abd 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "geoengine", - "version": "0.9.4", + "version": "0.9.2", "license": "MIT", "scripts": { "ng": "ng", diff --git a/ui/projects/core/src/lib/map/map-layer.component.ts b/ui/projects/core/src/lib/map/map-layer.component.ts index 23700e7d41..5d4110d384 100644 --- a/ui/projects/core/src/lib/map/map-layer.component.ts +++ b/ui/projects/core/src/lib/map/map-layer.component.ts @@ -447,6 +447,9 @@ export class OlOgcApiMapTileLayerComponent extends MapLayerComponent< // Show tile debug info instead of the actual layer. This is useful for debugging tile loading issues. readonly debug = input(false); + /** Emits `true` while any tile in this layer is loading, `false` when all tiles are loaded. */ + readonly loading = output(); + readonly tileSource = resource({ params: () => ({ dataConnectorId: this.dataConnectorId(), @@ -502,6 +505,22 @@ export class OlOgcApiMapTileLayerComponent extends MapLayerComponent< this.addStateListenersToOlSource(); this._mapLayer.setSource(this.source); + + // Track tile loading and emit through the loading output + let tilesPending = 0; + const onStart = (): void => { + tilesPending++; + this.loading.emit(true); + }; + const onEnd = (): void => { + tilesPending--; + if (tilesPending <= 0) { + this.loading.emit(false); + } + }; + this.source.on('tileloadstart', onStart); + this.source.on('tileloadend', onEnd); + this.source.on('tileloaderror', onEnd); }); effect(() => /* TODO: define in parent class */ { diff --git a/ui/projects/enhanced-data-viewer/src/app/app-config.service.ts b/ui/projects/enhanced-data-viewer/src/app/app-config.service.ts index ed5c568385..9d5c4d17e6 100644 --- a/ui/projects/enhanced-data-viewer/src/app/app-config.service.ts +++ b/ui/projects/enhanced-data-viewer/src/app/app-config.service.ts @@ -15,7 +15,7 @@ const APP_CONFIG_DEFAULTS = mergeDeepOverrideLists(DEFAULT_CORE_CONFIG, { DEFAULTS: { PROJECT: { NAME: 'Default', - TIME: '2026-01-01T00:00:00.000Z', + TIME: '2026-04-01T00:00:00.000Z', TIMESTEP: '1 day', PROJECTION: 'EPSG:3857', }, diff --git a/ui/projects/enhanced-data-viewer/src/app/main/main.component.html b/ui/projects/enhanced-data-viewer/src/app/main/main.component.html index 2bcfd110dc..3be41b2124 100644 --- a/ui/projects/enhanced-data-viewer/src/app/main/main.component.html +++ b/ui/projects/enhanced-data-viewer/src/app/main/main.component.html @@ -78,17 +78,17 @@

Enhanced Data Viewer

Data Source

- - Sentinel-1 - Sentinel-2 L2A - Sentinel-3 L2 - Landsat 8 + + @for (ds of dataSources; track ds.key) { + {{ ds.name }} + }

Time Selection

+ Auto select time