feat: unify stac harvester and stac provider functionality

geoengine/operators/src/source/gdal_source/loading_info.rs

@@ -48,9 +48,10 @@ impl MetaData<GdalLoadingInfo, RasterResultDescriptor, RasterQueryRectangle>
             valid.end()
         };
 
-        let known_time_after = if query.time_interval().end() <= valid.start() {
+        let query_time = query.time_interval();
+        let known_time_after = if query_time.end() <= valid.start() && !query_time.is_instant() {
             valid.start()
-        } else if query.time_interval().end() <= valid.end() {
+        } else if query_time.end() <= valid.end() {
             valid.end()
         } else {
             TimeInstance::MAX
@@ -721,6 +722,36 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_static_metadata_instant_at_validity_start() {
+        let regular_metadata = create_regular_metadata();
+        let valid = TimeInterval::new_unchecked(
+            TimeInstance::from_millis_unchecked(0),
+            TimeInstance::from_millis_unchecked(33),
+        );
+        let metadata = GdalMetaDataStatic {
+            time: Some(valid),
+            params: regular_metadata.params,
+            result_descriptor: regular_metadata.result_descriptor,
+            cache_ttl: CacheTtlSeconds::default(),
+        };
+
+        let loading_info = metadata
+            .loading_info(RasterQueryRectangle::new(
+                GridBoundingBox2D::new([-1, 0], [-1, 0]).unwrap(),
+                TimeInterval::new_instant(TimeInstance::from_millis_unchecked(0)).unwrap(),
+                BandSelection::first(),
+            ))
+            .await
+            .unwrap();
+
+        assert_eq!(
+            loading_info.start_time_of_output_stream,
+            Some(valid.start())
+        );
+        assert_eq!(loading_info.end_time_of_output_stream, Some(valid.end()));
+    }
+
     #[tokio::test]
     async fn test_regular_meta_data_result_descriptor() {
         let meta_data = create_regular_metadata();

geoengine/operators/src/source/multi_band_gdal_source/loading_info.rs

@@ -84,12 +84,9 @@ impl MultiBandGdalLoadingInfo {
             if time.intersects(&file.time)
                 && file.spatial_partition.intersects(&tile_partition)
                 && file.band == band
+                && let Some(bbox_in_tile) = file.spatial_partition.intersection(&tile_partition)
             {
-                debug_assert!(file.time == time, "file's time must match query time");
-
-                if let Some(bbox_in_tile) = file.spatial_partition.intersection(&tile_partition) {
-                    matching_files.push((file, bbox_in_tile));
-                }
+                matching_files.push((file, bbox_in_tile));
             }
         }
 

geoengine/services/src/api/handlers/datasets.rs

@@ -5341,13 +5341,7 @@ mod tests {
                         )
                         .into(),
                         time: TimeDescriptor {
-                            bounds: Some(
-                                TimeInterval::new_unchecked(
-                                    TimeInstance::from_str("2014-01-01T00:00:00Z").unwrap(),
-                                    TimeInstance::from_str("2014-01-02T00:00:00Z").unwrap(),
-                                )
-                                .into(),
-                            ),
+                            bounds: None,
                             dimension: TimeDimension::Irregular,
                         },
                         spatial_grid: SpatialGridDescriptor {
@@ -5398,8 +5392,8 @@ mod tests {
             )
             .into(),
             spatial_partition: SpatialPartition2D::new_unchecked(
-                (50., -50.).into(),
-                (150., -150.).into(),
+                (0., 0.).into(),
+                (100., -100.).into(),
             )
             .into(),
             band: 0,
@@ -5488,8 +5482,8 @@ mod tests {
             )
             .into(),
             spatial_partition: SpatialPartition2D::new_unchecked(
-                (50., -50.).into(),
-                (150., -150.).into(),
+                (-50., 50.).into(),
+                (0., 0.).into(),
             )
             .into(),
             band: 0,

geoengine/services/src/api/model/operators.rs

@@ -86,6 +86,26 @@ pub enum TimeDimension {
     Irregular,
 }
 
+impl From<TimeDimension> for geoengine_datatypes::primitives::TimeDimension {
+    fn from(value: TimeDimension) -> Self {
+        match value {
+            TimeDimension::Regular(regular) => Self::Regular(regular.into()),
+            TimeDimension::Irregular => Self::Irregular,
+        }
+    }
+}
+
+impl From<geoengine_datatypes::primitives::TimeDimension> for TimeDimension {
+    fn from(value: geoengine_datatypes::primitives::TimeDimension) -> Self {
+        match value {
+            geoengine_datatypes::primitives::TimeDimension::Regular(regular) => {
+                Self::Regular(regular.into())
+            }
+            geoengine_datatypes::primitives::TimeDimension::Irregular => Self::Irregular,
+        }
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct RegularTimeDimension {
@@ -122,14 +142,7 @@ impl From<TimeDescriptor> for geoengine_operators::engine::TimeDescriptor {
     fn from(value: TimeDescriptor) -> Self {
         geoengine_operators::engine::TimeDescriptor::new(
             value.bounds.map(Into::into),
-            match value.dimension {
-                TimeDimension::Regular(d) => {
-                    geoengine_datatypes::primitives::TimeDimension::Regular(d.into())
-                }
-                TimeDimension::Irregular => {
-                    geoengine_datatypes::primitives::TimeDimension::Irregular
-                }
-            },
+            value.dimension.into(),
         )
     }
 }
@@ -138,14 +151,7 @@ impl From<geoengine_operators::engine::TimeDescriptor> for TimeDescriptor {
     fn from(value: geoengine_operators::engine::TimeDescriptor) -> Self {
         Self {
             bounds: value.bounds.map(Into::into),
-            dimension: match value.dimension {
-                geoengine_datatypes::primitives::TimeDimension::Regular(d) => {
-                    TimeDimension::Regular(d.into())
-                }
-                geoengine_datatypes::primitives::TimeDimension::Irregular => {
-                    TimeDimension::Irregular
-                }
-            },
+            dimension: value.dimension.into(),
         }
     }
 }

geoengine/services/src/api/model/services.rs

@@ -6,8 +6,7 @@ use super::operators::TypedResultDescriptor;
 use crate::api::model::datatypes::MlModelName;
 use crate::api::model::operators::{
     GdalMetaDataList, GdalMetaDataRegular, GdalMetaDataStatic, GdalMetadataNetCdfCf,
-    MlModelMetadata, MockMetaData, OgrMetaData, RegularTimeDimension, SpatialGridDescriptor,
-    TimeDimension,
+    MlModelMetadata, MockMetaData, OgrMetaData, SpatialGridDescriptor, TimeDimension,
 };
 use crate::datasets::DatasetName;
 use crate::datasets::external::{GdalRetries, WildliveDataConnectorAuth};
@@ -991,29 +990,57 @@ impl From<crate::datasets::external::stac::StacProviderS3Config> for StacProvide
 
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSchema)]
 #[serde(rename_all = "camelCase")]
-pub struct StacProviderDatasetBand {
+pub struct StacAssetBand {
     pub asset_title: String,
     pub band_name: Option<String>,
 }
 
-impl From<StacProviderDatasetBand> for crate::datasets::external::stac::StacProviderDatasetBand {
-    fn from(value: StacProviderDatasetBand) -> Self {
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct StacProviderDatasetBand {
+    /// The band inside the STAC asset that this dataset band reads from
+    /// (addressing: which asset file + which raster channel within it).
+    pub asset_band: StacAssetBand,
+    /// The band descriptor of the resulting geo engine dataset layer.
+    pub band_descriptor: crate::api::model::operators::RasterBandDescriptor,
+}
+
+impl From<StacAssetBand> for crate::datasets::external::stac::StacAssetBand {
+    fn from(value: StacAssetBand) -> Self {
         Self {
             asset_title: value.asset_title,
             band_name: value.band_name,
         }
     }
 }
 
-impl From<crate::datasets::external::stac::StacProviderDatasetBand> for StacProviderDatasetBand {
-    fn from(value: crate::datasets::external::stac::StacProviderDatasetBand) -> Self {
+impl From<crate::datasets::external::stac::StacAssetBand> for StacAssetBand {
+    fn from(value: crate::datasets::external::stac::StacAssetBand) -> Self {
         Self {
             asset_title: value.asset_title,
             band_name: value.band_name,
         }
     }
 }
 
+impl From<StacProviderDatasetBand> for crate::datasets::external::stac::StacProviderDatasetBand {
+    fn from(value: StacProviderDatasetBand) -> Self {
+        Self {
+            asset_band: value.asset_band.into(),
+            band_descriptor: value.band_descriptor.into(),
+        }
+    }
+}
+
+impl From<crate::datasets::external::stac::StacProviderDatasetBand> for StacProviderDatasetBand {
+    fn from(value: crate::datasets::external::stac::StacProviderDatasetBand) -> Self {
+        Self {
+            asset_band: value.asset_band.into(),
+            band_descriptor: value.band_descriptor.into(),
+        }
+    }
+}
+
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, ToSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct StacProviderDataset {
@@ -1055,37 +1082,6 @@ impl From<crate::datasets::external::stac::StacProviderDataset> for StacProvider
     }
 }
 
-#[allow(clippy::needless_pass_by_value)]
-fn api_time_dimension_to_datatypes(
-    value: TimeDimension,
-) -> geoengine_datatypes::primitives::TimeDimension {
-    match value {
-        TimeDimension::Regular(RegularTimeDimension { origin, step }) => {
-            geoengine_datatypes::primitives::TimeDimension::Regular(
-                geoengine_datatypes::primitives::RegularTimeDimension {
-                    origin: origin.into(),
-                    step: step.into(),
-                },
-            )
-        }
-        TimeDimension::Irregular => geoengine_datatypes::primitives::TimeDimension::Irregular,
-    }
-}
-
-fn datatypes_time_dimension_to_api(
-    value: geoengine_datatypes::primitives::TimeDimension,
-) -> TimeDimension {
-    match value {
-        geoengine_datatypes::primitives::TimeDimension::Regular(
-            geoengine_datatypes::primitives::RegularTimeDimension { origin, step },
-        ) => TimeDimension::Regular(RegularTimeDimension {
-            origin: origin.into(),
-            step: step.into(),
-        }),
-        geoengine_datatypes::primitives::TimeDimension::Irregular => TimeDimension::Irregular,
-    }
-}
-
 #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, ToSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct StacTimeStep {
@@ -1127,12 +1123,18 @@ pub struct StacDataProviderDefinition {
     /// Timeout in seconds for outgoing STAC API HTTP requests.
     #[serde(default = "default_query_timeout")]
     pub query_timeout_secs: i64,
+    #[serde(default = "default_page_limit")]
+    pub page_limit: i64,
 }
 
 fn default_query_timeout() -> i64 {
     60
 }
 
+fn default_page_limit() -> i64 {
+    100
+}
+
 impl From<StacDataProviderDefinition>
     for crate::datasets::external::stac::StacDataProviderDefinition
 {
@@ -1145,8 +1147,9 @@ impl From<StacDataProviderDefinition>
             api_url: value.api_url,
             collection_name: value.collection_name,
             s3_config: value.s3_config.map(Into::into),
-            time_dimension: api_time_dimension_to_datatypes(value.time_dimension),
+            time_dimension: value.time_dimension.into(),
             datasets: value.datasets.into_iter().map(Into::into).collect(),
+            page_limit: value.page_limit,
             query_timeout_secs: value.query_timeout_secs,
         }
     }
@@ -1165,8 +1168,9 @@ impl From<crate::datasets::external::stac::StacDataProviderDefinition>
             api_url: value.api_url,
             collection_name: value.collection_name,
             s3_config: value.s3_config.map(Into::into),
-            time_dimension: datatypes_time_dimension_to_api(value.time_dimension),
+            time_dimension: value.time_dimension.into(),
             datasets: value.datasets.into_iter().map(Into::into).collect(),
+            page_limit: value.page_limit,
             query_timeout_secs: value.query_timeout_secs,
         }
     }

geoengine/services/src/bin/geoengine-cli.rs

@@ -1,8 +1,8 @@
 use clap::{Parser, Subcommand};
 use geoengine_services::cli::{
-    CheckSuccessfulStartup, ExpressionToolchainFile, Heartbeat, OpenAPIGenerate, StacImport,
-    TileImport, check_heartbeat, check_successful_startup, output_openapi_json,
-    output_toolchain_file, stac_import, tile_import,
+    CheckSuccessfulStartup, ExpressionToolchainFile, Heartbeat, OpenAPIGenerate, StacHarvester,
+    StacImport, TileImport, check_heartbeat, check_successful_startup, output_openapi_json,
+    output_toolchain_file, stac_harvester, stac_import, tile_import,
 };
 
 /// CLI for Geo Engine Utilities
@@ -25,6 +25,9 @@ enum Commands {
     #[command(name = "openapi")]
     OpenAPI(OpenAPIGenerate),
 
+    // Harvests STAC collections using a dataset mapping
+    StacHarvest(StacHarvester),
+
     // Imports a STAC catalog as a dataset
     StacImport(StacImport),
 
@@ -42,6 +45,7 @@ impl Commands {
             Commands::CheckSuccessfulStartup(params) => check_successful_startup(params).await,
             Commands::Heartbeat(params) => check_heartbeat(params).await,
             Commands::OpenAPI(params) => output_openapi_json(params).await,
+            Commands::StacHarvest(params) => stac_harvester(params).await,
             Commands::StacImport(params) => stac_import(params).await,
             Commands::TileImport(params) => tile_import(params).await,
             Commands::ExpressionToolchainFile(params) => output_toolchain_file(params).await,

geoengine/services/src/cli/mod.rs

@@ -2,12 +2,14 @@ mod check_successful_startup;
 mod expression_toolchain_file;
 mod heartbeat;
 mod openapi;
+mod stac_harvester;
 mod stac_import;
 mod tile_import;
 
 pub use check_successful_startup::{CheckSuccessfulStartup, check_successful_startup};
 pub use expression_toolchain_file::{ExpressionToolchainFile, output_toolchain_file};
 pub use heartbeat::{Heartbeat, check_heartbeat};
 pub use openapi::{OpenAPIGenerate, output_openapi_json};
+pub use stac_harvester::{StacHarvester, stac_harvester};
 pub use stac_import::{StacImport, stac_import};
 pub use tile_import::{TileImport, tile_import};