diff --git a/Cargo.lock b/Cargo.lock index fbb2eeb..7a10fc7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -308,7 +308,7 @@ dependencies = [ [[package]] name = "cityjson-index" version = "0.9.0" -source = "git+https://github.com/3DGI/cityjson-rs.git?branch=main#79d2ae4817e1a416108863a02e175b126e7450b6" +source = "git+https://github.com/3DGI/cityjson-rs.git?branch=main#de151a3c1015afa78690af2cacd521e76a053f63" dependencies = [ "cityjson-lib", "cityjson-types", @@ -330,7 +330,7 @@ dependencies = [ [[package]] name = "cityjson-json" version = "0.9.0" -source = "git+https://github.com/3DGI/cityjson-rs.git?branch=main#79d2ae4817e1a416108863a02e175b126e7450b6" +source = "git+https://github.com/3DGI/cityjson-rs.git?branch=main#de151a3c1015afa78690af2cacd521e76a053f63" dependencies = [ "ahash", "cityjson-types", @@ -342,7 +342,7 @@ dependencies = [ [[package]] name = "cityjson-lib" version = "0.9.0" -source = "git+https://github.com/3DGI/cityjson-rs.git?branch=main#79d2ae4817e1a416108863a02e175b126e7450b6" +source = "git+https://github.com/3DGI/cityjson-rs.git?branch=main#de151a3c1015afa78690af2cacd521e76a053f63" dependencies = [ "cityjson-json", "cityjson-types", @@ -352,7 +352,7 @@ dependencies = [ [[package]] name = "cityjson-types" version = "0.9.0" -source = "git+https://github.com/3DGI/cityjson-rs.git?branch=main#79d2ae4817e1a416108863a02e175b126e7450b6" +source = "git+https://github.com/3DGI/cityjson-rs.git?branch=main#de151a3c1015afa78690af2cacd521e76a053f63" dependencies = [ "num 0.4.3", ] @@ -1436,6 +1436,15 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libmimalloc-sys" +version = "0.1.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" +dependencies = [ + "cc", +] + [[package]] name = "libsqlite3-sys" version = "0.35.0" @@ -1513,6 +1522,15 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "mimalloc" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2593,6 +2611,7 @@ dependencies = [ "env_logger 0.10.2", "libc", "log", + "mimalloc", "morton-encoding", "num-traits", "proj-sys", diff --git a/Cargo.toml b/Cargo.toml index cbeab56..d246551 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ name = "tyler" path = "src/main.rs" [dependencies] +mimalloc = "0.1" cityjson-lib = { version = "0.9.0", features = ["json"] } cityjson-index = { version = "0.9.0" } cityjson-convert = { version = "0.1.0" } diff --git a/src/main.rs b/src/main.rs index 61bbac9..5c8fdf3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -64,6 +64,13 @@ mod parser; mod proj; mod spatial_structs; +// mimalloc returns freed memory to the OS and scales across threads without the +// per-arena retention/fragmentation that makes glibc malloc ratchet RSS up under +// tyler's heavily parallel tile conversion. (glibc MALLOC_ARENA_MAX=2 halves peak +// but serializes allocation ~6x slower; mimalloc gets the memory win at full speed.) +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fs; use std::fs::File; @@ -235,8 +242,8 @@ fn build_feature_type_lods(cli: &crate::cli::Cli) -> BTreeMap { fn build_feature_filter( cityobject_types: Option<&Vec>, feature_type_lods: &BTreeMap, -) -> cityjson_index::FeatureFilter { - cityjson_index::FeatureFilter { +) -> cityjson_index::PackageFilter { + cityjson_index::PackageFilter { cityobject_types: cityobject_types.map(|types| { types .iter() @@ -647,9 +654,15 @@ fn read_tile_feature_models( else { return Err("cjindex input mixed row references with feature ids".into()); }; - let model = city_index.get(feature_id)?.ok_or_else(|| { - format!("feature {feature_id} could not be resolved from cjindex") - })?; + // The new API resolves packages by a contained CityObject's + // external id and returns 0..N; take the first for this id. + let model = city_index + .get_packages(feature_id)? + .into_iter() + .next() + .ok_or_else(|| { + format!("feature {feature_id} could not be resolved from cjindex") + })?; models.push(model); } return Ok(models); @@ -708,7 +721,7 @@ fn deduplicate_feature_ids_by_reference( fn feature_reference_public_id(reference: &parser::FeatureReference) -> String { match reference { - parser::FeatureReference::CjIndexRef(feature) => feature.feature_id.clone(), + parser::FeatureReference::CjIndexRef(feature) => feature.model_id.clone(), parser::FeatureReference::CjIndexId(feature_id) => feature_id.clone(), } } @@ -719,19 +732,9 @@ fn feature_reference_precedes( ) -> bool { match (lhs, rhs) { (parser::FeatureReference::CjIndexRef(lhs), parser::FeatureReference::CjIndexRef(rhs)) => { - ( - lhs.source_id, - lhs.row_id, - lhs.offset, - lhs.length, - &lhs.source_path, - ) < ( - rhs.source_id, - rhs.row_id, - rhs.offset, - rhs.length, - &rhs.source_path, - ) + // record_id is the package's unique key (was row_id + source byte + // offsets, which the new ref no longer carries). + lhs.record_id < rhs.record_id } (parser::FeatureReference::CjIndexRef(_), parser::FeatureReference::CjIndexId(_)) => true, (parser::FeatureReference::CjIndexId(_), parser::FeatureReference::CjIndexRef(_)) => false, @@ -836,7 +839,7 @@ fn prepare_feature_model( model: cityjson_lib::CityModel, _feature_id: usize, _cityobject_types: Option<&Vec>, - feature_filter: &cityjson_index::FeatureFilter, + feature_filter: &cityjson_index::PackageFilter, object_attribute_types: &BTreeMap, include_parent_attributes: bool, cleanup_feature: bool, @@ -849,7 +852,10 @@ fn prepare_feature_model( apply_object_attribute_types(&mut model, object_attribute_types)?; } let filtered = feature_filter.apply(&model)?; - model = filtered.model; + // A package with no retained geometry yields `model: None`. + let Some(model) = filtered.model else { + return Ok(None); + }; let remove_empty_geometry = cleanup_feature || include_parent_attributes || !object_attribute_types.is_empty(); let model = if remove_empty_geometry { @@ -1021,7 +1027,7 @@ pub(crate) fn filter_cityjsonfeature_preserving_root_with_policy( feature_type_lods: &BTreeMap, default_highest_lod: bool, ) -> Result> { - let filter = cityjson_index::FeatureFilter { + let filter = cityjson_index::PackageFilter { cityobject_types: cityobject_types.map(|types| { types .iter() @@ -1043,7 +1049,16 @@ pub(crate) fn filter_cityjsonfeature_preserving_root_with_policy( }) .collect(), }; - Ok(filter.apply(model)?.model) + // The new API returns `model: None` when no geometry is retained; preserve + // the old contract of returning an empty model in that case. + match filter.apply(model)?.model { + Some(filtered) => Ok(filtered), + None => { + let mut empty = model.clone(); + empty.clear_cityobjects(); + Ok(empty) + } + } } fn parentless_cityobject_handle(model: &cityjson_lib::CityModel) -> Option { @@ -1991,27 +2006,21 @@ mod tests { .expect("multi type lod fixture should parse") } - fn indexed_feature_ref(feature_id: &str, source_id: i64, row_id: i64) -> parser::Feature { + fn indexed_feature_ref(feature_id: &str, _source_id: i64, row_id: i64) -> parser::Feature { parser::Feature { centroid: [0.0, 0.0], - reference: parser::FeatureReference::CjIndexRef(cityjson_index::IndexedFeatureRef { - row_id, - feature_id: feature_id.to_string(), - source_id, - source_path: PathBuf::from(format!("source-{source_id}.city.json")), - offset: row_id as u64, - length: 1, - vertices_offset: None, - vertices_length: None, - member_ranges_json: None, - bounds: cityjson_index::FeatureBounds { + reference: parser::FeatureReference::CjIndexRef(cityjson_index::IndexedPackageRef { + record_id: row_id, + model_id: feature_id.to_string(), + package_type: cityjson_index::PackageType::CityJson, + bounds: Some(cityjson_index::Bounds3D { min_x: 0.0, max_x: 1.0, min_y: 0.0, max_y: 1.0, min_z: 0.0, max_z: 1.0, - }, + }), }), bbox: [0.0, 0.0, 0.0, 1.0, 1.0, 1.0], needs_type_filter: false, @@ -2078,7 +2087,7 @@ mod tests { model.clone(), 0, None, - &cityjson_index::FeatureFilter::default(), + &cityjson_index::PackageFilter::default(), &BTreeMap::new(), false, false, @@ -2889,9 +2898,11 @@ mod tests { }; let message = error.to_string(); - assert!(message.contains("requested LoD selector matched no geometry")); - assert!(message.contains("BuildingPart requested LoD '99'")); - assert!(message.contains("available LoDs are: 1")); + // cityjson-index main reports missing explicit LoDs with this message. + assert!( + message.contains("requested LoD 99 is not available for BuildingPart"), + "unexpected error message: {message}" + ); } #[test] @@ -2932,15 +2943,13 @@ mod tests { .expect("open index"); city_index.reindex().expect("reindex ndjson dataset"); let indexed_bounds = city_index - .iter_all_bbox_pages(1) - .expect("build bbox page iterator") - .next() - .expect("bbox page should exist") - .expect("bbox page should load") + .package_ref_page_after_record_id(None, 1) + .expect("read first package ref page") .into_iter() .next() - .expect("indexed feature should exist") - .bounds; + .expect("indexed package should exist") + .bounds + .expect("indexed package should have bounds"); let feature_base_document = derive_base_document(&city_index).expect("derive base doc"); let metadata_path = dataset_dir.join("metadata.city.json"); fs::write(&metadata_path, &feature_base_document).expect("write metadata"); @@ -2957,11 +2966,10 @@ mod tests { None, ) .expect("build cjindex ndjson world"); - #[allow(clippy::float_cmp)] - { - assert_eq!(world.grid.bbox[2], indexed_bounds.min_z); - assert_eq!(world.grid.bbox[5], indexed_bounds.max_z); - } + // The grid z-extent comes from the geometry while indexed_bounds comes + // from the index; they agree to float-path precision. + assert!((world.grid.bbox[2] - indexed_bounds.min_z).abs() < 1e-6); + assert!((world.grid.bbox[5] - indexed_bounds.max_z).abs() < 1e-6); world.index_with_grid().expect("index cjindex ndjson world"); assert!(world .features diff --git a/src/parser.rs b/src/parser.rs index 681075f..8244c8e 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -41,7 +41,7 @@ thread_local! { #[derive(Serialize, Deserialize)] pub struct World { pub cityobject_types: Option>, - pub feature_filter: cityjson_index::FeatureFilter, + pub feature_filter: cityjson_index::PackageFilter, pub crs: Crs, pub features: FeatureSet, pub feature_base_document: Vec, @@ -70,16 +70,16 @@ struct ExtentStats { nr_features: usize, nr_features_ignored: usize, cityobject_types_ignored: Vec, - filter_summary: cityjson_index::FeatureFilterSummary, + filter_summary: cityjson_index::PackageFilterReport, } impl ExtentStats { fn add_selected_geometry_stats( &mut self, stats: SelectedGeometryStats, - diagnostics: &cityjson_index::FeatureFilterDiagnostics, + report: &cityjson_index::PackageFilterReport, ) { - self.filter_summary.add(diagnostics); + self.filter_summary.merge(report); if let Some(model_bbox) = stats.bbox { if let Some(current) = self.extent.as_mut() { merge_bbox(current, &model_bbox); @@ -104,7 +104,7 @@ impl ExtentStats { self.nr_features += other.nr_features; self.nr_features_ignored += other.nr_features_ignored; self.extend_ignored_types(other.cityobject_types_ignored); - merge_filter_summary(&mut self.filter_summary, other.filter_summary); + self.filter_summary.merge(&other.filter_summary); } fn extend_ignored_types(&mut self, ignored_types: Vec) { @@ -116,29 +116,6 @@ impl ExtentStats { } } -fn merge_filter_summary( - target: &mut cityjson_index::FeatureFilterSummary, - source: cityjson_index::FeatureFilterSummary, -) { - target.available_types.extend(source.available_types); - target.retained_types.extend(source.retained_types); - target.ignored_types.extend(source.ignored_types); - merge_lod_summary(&mut target.available_lods, source.available_lods); - merge_lod_summary(&mut target.retained_lods, source.retained_lods); - target.missing_lods.extend(source.missing_lods); - target.retained_feature_count += source.retained_feature_count; - target.ignored_feature_count += source.ignored_feature_count; -} - -fn merge_lod_summary( - target: &mut BTreeMap>, - source: BTreeMap>, -) { - for (cityobject_type, lods) in source { - target.entry(cityobject_type).or_default().extend(lods); - } -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct InputSource { pub dataset_root: PathBuf, @@ -200,7 +177,7 @@ impl World { feature_base_document: Vec, cellsize: u32, cityobject_types: Option>, - feature_filter: cityjson_index::FeatureFilter, + feature_filter: cityjson_index::PackageFilter, arg_minz: Option, arg_maxz: Option, ) -> Result> { @@ -291,31 +268,35 @@ impl World { fn extent_from_cjindex_features( input_source: &InputSource, cityobject_types: Option<&Vec>, - feature_filter: &cityjson_index::FeatureFilter, + feature_filter: &cityjson_index::PackageFilter, ) -> Result< ( Option, usize, usize, Vec, - cityjson_index::FeatureFilterSummary, + cityjson_index::PackageFilterReport, ), Box, > { let city_index = input_source.open_index()?; let (chunk_tx, chunk_rx) = - std::sync::mpsc::sync_channel::>(64); + std::sync::mpsc::sync_channel::>(64); // Same 2-stage pipeline as `index_with_grid`: a dedicated page-loader // thread streams owned chunks to `chunk_tx` so workers can start // processing before all pages have been read. let total = std::thread::scope(|s| -> Result { let page_loader = s.spawn(move || -> Result<(), std::io::Error> { - let pages_iter = city_index - .iter_all_feature_ref_pages(CJINDEX_PAGE_SIZE) - .map_err(|e| std::io::Error::other(e.to_string()))?; - for page_result in pages_iter { - let page = page_result.map_err(|e| std::io::Error::other(e.to_string()))?; + let mut after: Option = None; + loop { + let page = city_index + .package_ref_page_after_record_id(after, CJINDEX_PAGE_SIZE) + .map_err(|e| std::io::Error::other(e.to_string()))?; + if page.is_empty() { + break; + } + after = page.last().map(|package_ref| package_ref.record_id); for chunk in page.chunks(CJINDEX_PARALLEL_CHUNK_SIZE) { if chunk_tx.send(chunk.to_vec()).is_err() { return Ok(()); @@ -362,29 +343,44 @@ impl World { usize, usize, Vec, - cityjson_index::FeatureFilterSummary, + cityjson_index::PackageFilterReport, ), Box, > { - let Some(summary) = city_index.feature_bounds_summary()? else { - return Ok(( - None, - 0, - 0, - Vec::new(), - cityjson_index::FeatureFilterSummary::default(), - )); - }; + // The new cityjson-index has no aggregate-bounds query, so fold the whole + // dataset's package refs (keyset pagination by record_id) into an extent + + // count. Refs carry their bbox, so no geometry is read. + let mut extent: Option = None; + let mut nr_features = 0usize; + let mut after: Option = None; + loop { + let page = city_index.package_ref_page_after_record_id(after, CJINDEX_PAGE_SIZE)?; + if page.is_empty() { + break; + } + after = page.last().map(|package_ref| package_ref.record_id); + for package_ref in &page { + nr_features += 1; + if let Some(bounds) = &package_ref.bounds { + let bbox = Self::cjindex_bounds_to_world_bbox(bounds); + if let Some(current) = extent.as_mut() { + merge_bbox(current, &bbox); + } else { + extent = Some(bbox); + } + } + } + } Ok(( - Some(Self::cjindex_bounds_to_world_bbox(&summary.bounds)), - summary.feature_count, + extent, + nr_features, 0, Vec::new(), - cityjson_index::FeatureFilterSummary::default(), + cityjson_index::PackageFilterReport::default(), )) } - fn cjindex_bounds_to_world_bbox(bounds: &cityjson_index::FeatureBounds) -> Bbox { + fn cjindex_bounds_to_world_bbox(bounds: &cityjson_index::Bounds3D) -> Bbox { [ bounds.min_x, bounds.min_y, @@ -397,9 +393,9 @@ impl World { fn extent_from_cjindex_feature_refs_chunk( input_source: &InputSource, - feature_refs: &[cityjson_index::IndexedFeatureRef], + feature_refs: &[cityjson_index::IndexedPackageRef], _cityobject_types: Option<&Vec>, - feature_filter: &cityjson_index::FeatureFilter, + feature_filter: &cityjson_index::PackageFilter, ) -> Result { let filtered_features = Self::read_filtered_cjindex_features_thread_local( input_source, @@ -409,17 +405,20 @@ impl World { .map_err(|error| std::io::Error::other(error.to_string()))?; let mut chunk = ExtentStats::default(); for filtered in filtered_features { - chunk.add_selected_geometry_stats( - selected_geometry_stats(&filtered.model, None), - &filtered.diagnostics, - ); + // A package with no retained geometry yields `model: None`; treat it + // as ignored (default stats have no bbox). + let stats = match &filtered.model { + Some(model) => selected_geometry_stats(model, None), + None => SelectedGeometryStats::default(), + }; + chunk.add_selected_geometry_stats(stats, &filtered.report); } Ok(chunk) } pub(crate) fn read_cjindex_features_thread_local( input_source: &InputSource, - features: &[cityjson_index::IndexedFeatureRef], + features: &[cityjson_index::IndexedPackageRef], ) -> cityjson_lib::Result> { let index_path = &input_source.index_path; @@ -445,15 +444,17 @@ impl World { "cjindex thread-local index cache was not initialized", ))); }; - city_index.read_features(features) + city_index + .read_packages(features) + .map(|packages| packages.into_iter().map(|package| package.model).collect()) }) } fn read_filtered_cjindex_features_thread_local( input_source: &InputSource, - features: &[cityjson_index::IndexedFeatureRef], - filter: &cityjson_index::FeatureFilter, - ) -> cityjson_lib::Result> { + features: &[cityjson_index::IndexedPackageRef], + filter: &cityjson_index::PackageFilter, + ) -> cityjson_lib::Result> { let index_path = &input_source.index_path; CJINDEX_THREAD_LOCAL.with(|cell| { @@ -478,7 +479,7 @@ impl World { "cjindex thread-local index cache was not initialized", ))); }; - city_index.read_filtered_features(features, filter) + city_index.read_filtered_packages(features, filter) }) } @@ -496,11 +497,11 @@ impl World { let grid: &mut crate::spatial_structs::SquareGrid = &mut self.grid; let input_source: &InputSource = &self.input_source; let cityobject_types: Option<&Vec> = self.cityobject_types.as_ref(); - let feature_filter: &cityjson_index::FeatureFilter = &self.feature_filter; + let feature_filter: &cityjson_index::PackageFilter = &self.feature_filter; let grid_layout = grid.layout(); let (chunk_tx, chunk_rx) = - std::sync::mpsc::sync_channel::>(64); + std::sync::mpsc::sync_channel::>(64); let (result_tx, result_rx) = std::sync::mpsc::sync_channel::< Result>, std::io::Error>, >(64); @@ -519,13 +520,17 @@ impl World { // mutations to `features` and `grid`. let loader_outcome = std::thread::scope(|s| -> Result<(usize, usize), std::io::Error> { let page_loader = s.spawn(move || -> Result<(usize, usize), std::io::Error> { - let pages_iter = city_index - .iter_all_feature_ref_pages(CJINDEX_PAGE_SIZE) - .map_err(|e| std::io::Error::other(e.to_string()))?; let mut page_count = 0usize; let mut scanned_features = 0usize; - for page_result in pages_iter { - let page = page_result.map_err(|e| std::io::Error::other(e.to_string()))?; + let mut after: Option = None; + loop { + let page = city_index + .package_ref_page_after_record_id(after, CJINDEX_PAGE_SIZE) + .map_err(|e| std::io::Error::other(e.to_string()))?; + if page.is_empty() { + break; + } + after = page.last().map(|package_ref| package_ref.record_id); page_count += 1; scanned_features += page.len(); for chunk in page.chunks(CJINDEX_PARALLEL_CHUNK_SIZE) { @@ -591,8 +596,8 @@ impl World { input_source: &InputSource, grid_layout: &GridLayout, cityobject_types: Option<&Vec>, - feature_filter: &cityjson_index::FeatureFilter, - feature_refs: &[cityjson_index::IndexedFeatureRef], + feature_filter: &cityjson_index::PackageFilter, + feature_refs: &[cityjson_index::IndexedPackageRef], ) -> Result>, std::io::Error> { let filtered_features = Self::read_filtered_cjindex_features_thread_local( input_source, @@ -604,13 +609,16 @@ impl World { .iter() .cloned() .zip(filtered_features.iter()) - .map(|(feature_ref, filtered)| { - Self::index_feature_model( + .map(|(feature_ref, filtered)| match &filtered.model { + // A package with no retained geometry yields `model: None`; it + // simply isn't indexed into any grid cell. + Some(model) => Self::index_feature_model( grid_layout, cityobject_types, FeatureReference::CjIndexRef(feature_ref), - &filtered.model, - ) + model, + ), + None => Ok(None), }) .collect::, _>>() } @@ -791,7 +799,7 @@ fn default_feature_needs_type_filter() -> bool { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum FeatureReference { - CjIndexRef(cityjson_index::IndexedFeatureRef), + CjIndexRef(cityjson_index::IndexedPackageRef), CjIndexId(String), }