diff --git a/docs/compatibility/index.md b/docs/compatibility/index.md index eb09674c1..4495ac1ac 100644 --- a/docs/compatibility/index.md +++ b/docs/compatibility/index.md @@ -19,7 +19,7 @@ Detailed compatibility documents describe field-level limitations where they exi | `malloy` | — | `file_or_directory` | `.malloy` | Registered | Yes | 10 | [Detailed guide](malloy.md) | | `metricflow` | `dbt`, `dbt-semantic-layer` | `file_or_directory` | `.yml`, `.yaml` | Registered | Yes | 6 | Registry summary only | | `omni` | — | `file_or_directory` | `.yml`, `.yaml` | Registered | Yes | 4 | Registry summary only | -| `ossie` | `apache-ossie`, `osi`, `open-semantic-interchange` | `file_or_directory` | `.yml`, `.yaml`, `.json` | Registered | Yes | 4 | [Detailed guide](ossie.md) | +| `ossie` | `apache-ossie`, `osi`, `open-semantic-interchange` | `file_or_directory` | `.yml`, `.yaml`, `.json` | Registered | Yes | 5 | [Detailed guide](ossie.md) | | `rill` | — | `file_or_directory` | `.yml`, `.yaml` | Registered | Yes | 8 | Registry summary only | | `sidemantic` | `native` | `file` | `.yml`, `.yaml`, `.sql` | Registered | Yes | 1 | Registry summary only | | `snowflake` | `cortex`, `snowflake-cortex` | `file_or_directory` | `.yml`, `.yaml` | Registered | Yes | 8 | Registry summary only | diff --git a/docs/compatibility/ossie.md b/docs/compatibility/ossie.md index daf8232f3..490b2baac 100644 --- a/docs/compatibility/ossie.md +++ b/docs/compatibility/ossie.md @@ -238,3 +238,24 @@ sidemantic convert models/orders.yml \ ``` Use `--force` only when intentionally replacing an existing output file. + +## Rust forward-import status + +The experimental Rust runtime now has a strict forward Ossie import subset in +its dedicated `ossie` adapter. It supports explicit consumer profiles, +including `ossie-core` (`0.1.1` and `0.2.0.dev0`) and `dbt-1.12` compatibility +profiles. It preserves separate semantic-model scopes and uses +exact-target-then-`ANSI_SQL` expression selection for its supported runtime +targets: `ANSI_SQL`, `DUCKDB`, `POSTGRES`, `SNOWFLAKE`, `DATABRICKS`, and +`BIGQUERY`. Its import gate also checks scalar SQL structure, identifiers, +declared primary and unique keys, relationship identity and endpoints, key +arity, and target-key uniqueness. Invalid or unsupported input fails closed; +the legacy `osi` adapter remains a separate compatibility surface. + +This is a forward import subset, not full parity with the Python contract. Rust +does not yet provide the Python implementation's complete pinned JSON Schema +validation, preserved source-document and exact-byte model, ontology +preservation/reasoning boundary, permissive lowering mode, or Ossie export and +graph-synthesis path. Rust checks therefore establish strict structural import +coverage only; they are not live warehouse execution tests and do not claim +runtime coverage for every target database. diff --git a/sidemantic-rs/examples/parity_adapter.rs b/sidemantic-rs/examples/parity_adapter.rs index 69303a2ee..8a9712fcc 100644 --- a/sidemantic-rs/examples/parity_adapter.rs +++ b/sidemantic-rs/examples/parity_adapter.rs @@ -5,8 +5,9 @@ use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use sidemantic::{ build_symmetric_aggregate_sql, config::SidemanticConfig, load_from_string, Aggregation, - DimensionType, Metric, Model, QueryRewriter, RelationshipType, SemanticGraph, SemanticQuery, - SqlDialect, SqlGenerator, SymmetricAggType, TableCalculation, + DimensionType, Metric, Model, OssieConsumerProfile, OssieForwardAdapter, OssieSerialization, + OssieTarget, QueryRewriter, RelationshipType, SemanticGraph, SemanticQuery, SqlDialect, + SqlGenerator, SymmetricAggType, TableCalculation, }; #[derive(Debug, Deserialize)] @@ -90,6 +91,21 @@ enum Request { relationship: String, is_base_model: bool, }, + OssieValidate { + content: String, + serialization: String, + #[serde(default = "default_ossie_consumer_profile")] + consumer_profile: String, + }, + OssieSelectScope { + content: String, + serialization: String, + #[serde(default = "default_ossie_consumer_profile")] + consumer_profile: String, + #[serde(default = "default_ossie_target")] + target: String, + scope_id: Option, + }, } #[derive(Debug, Serialize)] @@ -117,6 +133,8 @@ struct PathStep { from_columns: Vec, to_columns: Vec, relationship: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + edge_id: Option, } fn handle(request: Request) -> sidemantic::Result { @@ -186,6 +204,7 @@ fn handle(request: Request) -> sidemantic::Result { from_columns: step.from_keys, to_columns: step.to_keys, relationship: relationship_type_name(&step.relationship_type), + edge_id: step.edge_id, }) .collect(); Ok(Response::Ok { @@ -356,9 +375,73 @@ fn handle(request: Request) -> sidemantic::Result { ) )), }), + Request::OssieValidate { + content, + serialization, + consumer_profile, + } => { + let serialization = parse_ossie_serialization(&serialization)?; + let consumer = parse_ossie_consumer(&consumer_profile)?; + let status = OssieForwardAdapter.inspect(&content, serialization, consumer); + Ok(Response::Ok { + sql: None, + path: None, + catalog: None, + value: Some( + serde_json::to_value(status).map_err(|error| { + sidemantic::SidemanticError::Validation(error.to_string()) + })?, + ), + }) + } + Request::OssieSelectScope { + content, + serialization, + consumer_profile, + target, + scope_id, + } => { + let scope = OssieForwardAdapter.select_scope( + &content, + parse_ossie_serialization(&serialization)?, + parse_ossie_consumer(&consumer_profile)?, + parse_ossie_target(&target)?, + scope_id.as_deref(), + )?; + Ok(Response::Ok { + sql: None, + path: None, + catalog: None, + value: Some( + serde_json::to_value(scope).map_err(|error| { + sidemantic::SidemanticError::Validation(error.to_string()) + })?, + ), + }) + } } } +fn default_ossie_consumer_profile() -> String { + "ossie-core".to_string() +} + +fn default_ossie_target() -> String { + "ANSI_SQL".to_string() +} + +fn parse_ossie_serialization(value: &str) -> sidemantic::Result { + OssieSerialization::parse(value).map_err(sidemantic::SidemanticError::Validation) +} + +fn parse_ossie_consumer(value: &str) -> sidemantic::Result { + OssieConsumerProfile::parse(value).map_err(sidemantic::SidemanticError::Validation) +} + +fn parse_ossie_target(value: &str) -> sidemantic::Result { + OssieTarget::parse(value).map_err(sidemantic::SidemanticError::Validation) +} + fn parse_symmetric_agg_type(agg_type: &str) -> sidemantic::Result { match agg_type { "sum" => Ok(SymmetricAggType::Sum), diff --git a/sidemantic-rs/src/adapters/cube.rs b/sidemantic-rs/src/adapters/cube.rs index d4871f34c..949bb4247 100644 --- a/sidemantic-rs/src/adapters/cube.rs +++ b/sidemantic-rs/src/adapters/cube.rs @@ -175,6 +175,8 @@ impl CubeDimension { Dimension { name: self.name, r#type: dim_type, + logical_data_type: None, + declared_is_time: None, sql, granularity: None, supported_granularities: None, @@ -224,6 +226,7 @@ impl CubeMeasure { Metric { name: self.name, extends: None, + logical_data_type: None, r#type: metric_type, agg, sql, diff --git a/sidemantic-rs/src/adapters/mod.rs b/sidemantic-rs/src/adapters/mod.rs index 957d95749..05f2b0792 100644 --- a/sidemantic-rs/src/adapters/mod.rs +++ b/sidemantic-rs/src/adapters/mod.rs @@ -13,9 +13,14 @@ use crate::error::{Result, SidemanticError}; pub mod cube; pub mod osi; +pub mod ossie; pub use cube::CubeAdapter; pub use osi::OsiAdapter; +pub use ossie::{ + OssieCatalog, OssieCompiledScope, OssieConsumerProfile, OssieDiagnostic, OssieForwardAdapter, + OssieProfile, OssieSerialization, OssieStatus, OssieTarget, +}; /// Result of parsing a single external-format document. #[derive(Debug, Default)] diff --git a/sidemantic-rs/src/adapters/osi.rs b/sidemantic-rs/src/adapters/osi.rs index 651698bed..b3f38249a 100644 --- a/sidemantic-rs/src/adapters/osi.rs +++ b/sidemantic-rs/src/adapters/osi.rs @@ -360,6 +360,8 @@ fn parse_field(field_def: &Json) -> Option { } else { DimensionType::Categorical }, + logical_data_type: None, + declared_is_time: None, sql, granularity: if is_time { Some("day".to_string()) @@ -480,6 +482,7 @@ fn add_relationship_to_model(rel_def: &Json, models: &mut [Model]) { let relationship = Relationship { name: to_model.to_string(), + edge_id: None, r#type: RelationshipType::ManyToOne, foreign_key: foreign_key_columns.first().cloned(), foreign_key_columns: Some(foreign_key_columns), diff --git a/sidemantic-rs/src/adapters/ossie.rs b/sidemantic-rs/src/adapters/ossie.rs new file mode 100644 index 000000000..202dd831f --- /dev/null +++ b/sidemantic-rs/src/adapters/ossie.rs @@ -0,0 +1,2068 @@ +//! Forward Apache Ossie importer. +//! +//! This is intentionally separate from [`super::osi`], whose permissive legacy +//! behavior is retained for compatibility. The forward adapter is strict, +//! scope-preserving, target-aware, and fail-closed. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use polyglot_sql::{DialectType, Expression, ExpressionWalk}; +use serde::Serialize; +use serde_json::{Map, Value}; + +use crate::config::schema::metric_from_sql_expression; +use crate::core::{ + Dimension, DimensionType, Metric, Model, Relationship, RelationshipType, Segment, +}; +use crate::error::{Result, SidemanticError}; + +const VALIDATION_MODE: &str = "closed_structural_subset"; +const TEMPORAL_TYPES: &[&str] = &["Date", "Time", "DateTime", "DateTimeTz"]; +const NUMERIC_TYPES: &[&str] = &["Integer", "Decimal", "Float"]; +const DATA_TYPES: &[&str] = &[ + "String", + "Integer", + "Decimal", + "Float", + "Boolean", + "Date", + "Time", + "DateTime", + "DateTimeTz", + "Opaque", +]; +const DIALECTS_0_1_1: &[&str] = &[ + "ANSI_SQL", + "SNOWFLAKE", + "MDX", + "TABLEAU", + "DATABRICKS", + "MAQL", +]; +const DIALECTS_0_2_0: &[&str] = &[ + "ANSI_SQL", + "SNOWFLAKE", + "MDX", + "TABLEAU", + "DATABRICKS", + "MAQL", + "BIGQUERY", +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DocumentKind { + Logical, + Ontology, +} + +impl DocumentKind { + fn label(self) -> &'static str { + match self { + Self::Logical => "logical", + Self::Ontology => "ontology", + } + } +} + +/// Source serialization is independent from the Ossie schema version. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OssieSerialization { + Json, + Yaml, +} + +impl OssieSerialization { + pub fn parse(value: &str) -> std::result::Result { + match value.trim().to_ascii_lowercase().as_str() { + "json" => Ok(Self::Json), + "yaml" | "yml" => Ok(Self::Yaml), + other => Err(format!("unsupported Ossie serialization {other:?}")), + } + } +} + +/// Consumer-specific compatibility contract. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OssieConsumerProfile { + OssieCore, + Dbt112, +} + +impl OssieConsumerProfile { + pub fn parse(value: &str) -> std::result::Result { + match value.trim().to_ascii_lowercase().as_str() { + "ossie-core" => Ok(Self::OssieCore), + "dbt-1.12" => Ok(Self::Dbt112), + other => Err(format!("unsupported Ossie consumer profile {other:?}")), + } + } + + fn label(self) -> &'static str { + match self { + Self::OssieCore => "ossie-core", + Self::Dbt112 => "dbt-1.12", + } + } +} + +/// Executable SQL target. Non-SQL Ossie variants are preserved but never selected. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OssieTarget { + AnsiSql, + DuckDb, + Postgres, + Snowflake, + Databricks, + BigQuery, +} + +impl OssieTarget { + pub fn parse(value: &str) -> std::result::Result { + match value.trim().to_ascii_uppercase().replace('-', "_").as_str() { + "ANSI" | "ANSI_SQL" => Ok(Self::AnsiSql), + "DUCKDB" => Ok(Self::DuckDb), + "POSTGRES" | "POSTGRESQL" => Ok(Self::Postgres), + "SNOWFLAKE" => Ok(Self::Snowflake), + "DATABRICKS" => Ok(Self::Databricks), + "BIGQUERY" | "BIG_QUERY" => Ok(Self::BigQuery), + other => Err(format!( + "unsupported executable Ossie target {other:?}; supported: ANSI_SQL, DUCKDB, POSTGRES, SNOWFLAKE, DATABRICKS, BIGQUERY" + )), + } + } + + pub fn label(self) -> &'static str { + match self { + Self::AnsiSql => "ANSI_SQL", + Self::DuckDb => "DUCKDB", + Self::Postgres => "POSTGRES", + Self::Snowflake => "SNOWFLAKE", + Self::Databricks => "DATABRICKS", + Self::BigQuery => "BIGQUERY", + } + } + + fn parser_dialect(self) -> DialectType { + match self { + Self::AnsiSql | Self::DuckDb => DialectType::DuckDB, + Self::Postgres => DialectType::PostgreSQL, + Self::Snowflake => DialectType::Snowflake, + Self::Databricks => DialectType::Databricks, + Self::BigQuery => DialectType::BigQuery, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct OssieProfile { + pub identifier: String, + pub schema_version: String, + pub consumer_profile: String, + pub validation_schema_version: String, + pub compatibility_alias_for: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct OssieDiagnostic { + pub code: String, + pub severity: &'static str, + pub message: String, + pub instance_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub scope: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub struct OssieStatus { + pub valid: bool, + pub executable: bool, + pub validation_mode: &'static str, + pub profile: Option, + pub document_kind: Option, + pub scopes: Vec, + pub diagnostics: Vec, +} + +#[derive(Debug, Clone, Serialize)] +pub struct OssieCompiledScope { + pub scope_id: String, + pub semantic_model_index: usize, + pub target_dialect: String, + pub models: Vec, + pub metrics: Vec, +} + +#[derive(Debug, Clone, Serialize)] +pub struct OssieCatalog { + pub profile: OssieProfile, + pub validation_mode: &'static str, + pub scopes: Vec, + pub diagnostics: Vec, +} + +#[derive(Debug, Clone)] +struct InspectedDocument { + root: Option, + profile: Option, + kind: Option, + scope_ids: Vec, + diagnostics: Vec, +} + +impl InspectedDocument { + fn status(&self) -> OssieStatus { + let valid = self.diagnostics.is_empty(); + OssieStatus { + valid, + executable: valid && self.kind == Some(DocumentKind::Logical), + validation_mode: VALIDATION_MODE, + profile: self.profile.clone(), + document_kind: self.kind.map(|kind| kind.label().to_string()), + scopes: self.scope_ids.clone(), + diagnostics: self.diagnostics.clone(), + } + } +} + +/// Strict forward adapter. It never dispatches through the legacy OSI adapter. +#[derive(Debug, Clone, Copy, Default)] +pub struct OssieForwardAdapter; + +impl OssieForwardAdapter { + pub fn inspect( + &self, + content: &str, + serialization: OssieSerialization, + consumer: OssieConsumerProfile, + ) -> OssieStatus { + self.inspect_document(content, serialization, consumer) + .status() + } + + pub fn parse_catalog( + &self, + content: &str, + serialization: OssieSerialization, + consumer: OssieConsumerProfile, + target: OssieTarget, + ) -> Result { + let mut inspected = self.inspect_document(content, serialization, consumer); + if !inspected.diagnostics.is_empty() { + return Err(status_error(&inspected.status())); + } + if inspected.kind != Some(DocumentKind::Logical) { + inspected.diagnostics.push(diagnostic( + "ossie.handoff.document_not_executable", + "The validated document is not a logical semantic-model document.", + "", + None, + )); + return Err(status_error(&inspected.status())); + } + + let root = inspected + .root + .as_ref() + .and_then(Value::as_object) + .ok_or_else(|| { + SidemanticError::Validation("validated Ossie root disappeared".to_string()) + })?; + let semantic_models = root + .get("semantic_model") + .and_then(Value::as_array) + .ok_or_else(|| { + SidemanticError::Validation("validated Ossie scopes disappeared".to_string()) + })?; + + let profile = inspected.profile.clone().expect("valid logical profile"); + let mut diagnostics = Vec::new(); + let mut scopes = Vec::new(); + for (index, value) in semantic_models.iter().enumerate() { + let scope_id = inspected.scope_ids[index].clone(); + if let Some(scope) = compile_scope(value, index, &scope_id, target, &mut diagnostics) { + scopes.push(scope); + } + } + sort_diagnostics(&mut diagnostics); + if !diagnostics.is_empty() { + let status = OssieStatus { + valid: false, + executable: false, + validation_mode: VALIDATION_MODE, + profile: Some(profile), + document_kind: Some("logical".to_string()), + scopes: inspected.scope_ids, + diagnostics, + }; + return Err(status_error(&status)); + } + + Ok(OssieCatalog { + profile, + validation_mode: VALIDATION_MODE, + scopes, + diagnostics, + }) + } + + pub fn select_scope( + &self, + content: &str, + serialization: OssieSerialization, + consumer: OssieConsumerProfile, + target: OssieTarget, + scope_id: Option<&str>, + ) -> Result { + let catalog = self.parse_catalog(content, serialization, consumer, target)?; + match scope_id { + Some(requested) => { + let available = catalog + .scopes + .iter() + .map(|scope| scope.scope_id.as_str()) + .collect::>() + .join(", "); + catalog + .scopes + .into_iter() + .find(|scope| scope.scope_id == requested) + .ok_or_else(|| { + SidemanticError::Validation(format!( + "Ossie scope {requested:?} not found; available: {available}" + )) + }) + } + None if catalog.scopes.len() == 1 => Ok(catalog.scopes.into_iter().next().unwrap()), + None if catalog.scopes.is_empty() => Err(SidemanticError::Validation( + "Ossie document contains no executable scopes".to_string(), + )), + None => Err(SidemanticError::Validation(format!( + "Ossie scope selection is ambiguous; select one explicitly: {}", + catalog + .scopes + .iter() + .map(|scope| scope.scope_id.as_str()) + .collect::>() + .join(", ") + ))), + } + } + + fn inspect_document( + &self, + content: &str, + serialization: OssieSerialization, + consumer: OssieConsumerProfile, + ) -> InspectedDocument { + let parsed = match serialization { + OssieSerialization::Json => { + serde_json::from_str(content).map_err(|error| error.to_string()) + } + OssieSerialization::Yaml => { + serde_yaml::from_str(content).map_err(|error| error.to_string()) + } + }; + let root: Value = match parsed { + Ok(value) => value, + Err(error) => { + return InspectedDocument { + root: None, + profile: None, + kind: None, + scope_ids: Vec::new(), + diagnostics: vec![diagnostic( + "ossie.parse.invalid_syntax", + format!("Invalid Ossie input: {error}"), + "", + None, + )], + }; + } + }; + + let Some(object) = root.as_object() else { + return InspectedDocument { + root: Some(root), + profile: None, + kind: None, + scope_ids: Vec::new(), + diagnostics: vec![diagnostic( + "ossie.schema.type", + "The Ossie document root must be an object.", + "", + None, + )], + }; + }; + + let kind = classify_document(object); + let mut diagnostics = Vec::new(); + let profile = resolve_profile(object.get("version"), consumer, &mut diagnostics); + let mut scope_ids = Vec::new(); + + match kind { + Some(DocumentKind::Logical) => { + validate_logical_root(object, profile.as_ref(), &mut diagnostics); + scope_ids = logical_scope_ids(object); + validate_semantics(object, &scope_ids, &mut diagnostics); + } + Some(DocumentKind::Ontology) => validate_ontology_root(object, &mut diagnostics), + None => diagnostics.push(diagnostic( + "ossie.schema.document_family", + "Document must contain exactly one of semantic_model or ontology/ontology_mappings.", + "", + None, + )), + } + sort_diagnostics(&mut diagnostics); + + InspectedDocument { + root: Some(root), + profile, + kind, + scope_ids, + diagnostics, + } + } +} + +fn classify_document(root: &Map) -> Option { + let logical = root.contains_key("semantic_model"); + let ontology = root.contains_key("ontology") || root.contains_key("ontology_mappings"); + match (logical, ontology) { + (true, false) => Some(DocumentKind::Logical), + (false, true) => Some(DocumentKind::Ontology), + _ => None, + } +} + +fn resolve_profile( + version: Option<&Value>, + consumer: OssieConsumerProfile, + diagnostics: &mut Vec, +) -> Option { + let Some(version) = version.and_then(Value::as_str) else { + diagnostics.push(diagnostic( + "ossie.schema.required", + "A string Ossie version is required.", + "/version", + None, + )); + return None; + }; + + let (validation_version, alias) = match (consumer, version) { + (OssieConsumerProfile::OssieCore, "0.1.1") => ("0.1.1", None), + (OssieConsumerProfile::OssieCore, "0.2.0.dev0") => ("0.2.0.dev0", None), + (OssieConsumerProfile::Dbt112, "0.1.0") => ("0.1.1", Some("0.1.1")), + (OssieConsumerProfile::Dbt112, "0.1.1") => ("0.1.1", None), + (OssieConsumerProfile::OssieCore, "0.1.0") => { + diagnostics.push(diagnostic( + "ossie.schema.profile_context_required", + "Ossie 0.1.0 is only supported as the dbt-1.12 compatibility alias.", + "/version", + None, + )); + return None; + } + _ => { + diagnostics.push(diagnostic( + "ossie.schema.profile_unsupported", + format!( + "Unsupported Ossie version {version:?} for consumer {}.", + consumer.label() + ), + "/version", + None, + )); + return None; + } + }; + + Some(OssieProfile { + identifier: format!("{}:{version}", consumer.label()), + schema_version: version.to_string(), + consumer_profile: consumer.label().to_string(), + validation_schema_version: validation_version.to_string(), + compatibility_alias_for: alias.map(str::to_string), + }) +} + +fn validate_logical_root( + root: &Map, + profile: Option<&OssieProfile>, + diagnostics: &mut Vec, +) { + let validation_version = profile.map(|profile| profile.validation_schema_version.as_str()); + let allowed_root = if validation_version == Some("0.1.1") { + &["version", "dialects", "vendors", "semantic_model"][..] + } else { + &["version", "semantic_model"][..] + }; + reject_unknown(root, allowed_root, "", diagnostics, None); + + let Some(scopes) = required_array(root, "semantic_model", "", diagnostics, None) else { + return; + }; + for (scope_index, scope) in scopes.iter().enumerate() { + let pointer = format!("/semantic_model/{scope_index}"); + let Some(scope) = require_object(scope, &pointer, diagnostics, None) else { + continue; + }; + reject_unknown( + scope, + &[ + "name", + "description", + "ai_context", + "datasets", + "relationships", + "metrics", + "custom_extensions", + ], + &pointer, + diagnostics, + None, + ); + required_string(scope, "name", &pointer, diagnostics, None); + let Some(datasets) = required_array(scope, "datasets", &pointer, diagnostics, None) else { + continue; + }; + if datasets.is_empty() { + diagnostics.push(diagnostic( + "ossie.schema.min_items", + "Semantic-model datasets must contain at least one item.", + format!("{pointer}/datasets"), + None, + )); + } + for (dataset_index, dataset) in datasets.iter().enumerate() { + validate_dataset( + dataset, + &format!("{pointer}/datasets/{dataset_index}"), + validation_version, + diagnostics, + ); + } + if let Some(relationships) = + optional_array(scope, "relationships", &pointer, diagnostics, None) + { + for (index, relationship) in relationships.iter().enumerate() { + validate_relationship_schema( + relationship, + &format!("{pointer}/relationships/{index}"), + diagnostics, + ); + } + } + if let Some(metrics) = optional_array(scope, "metrics", &pointer, diagnostics, None) { + for (index, metric) in metrics.iter().enumerate() { + validate_metric_schema( + metric, + &format!("{pointer}/metrics/{index}"), + validation_version, + diagnostics, + ); + } + } + } +} + +fn validate_dataset( + value: &Value, + pointer: &str, + version: Option<&str>, + diagnostics: &mut Vec, +) { + let Some(dataset) = require_object(value, pointer, diagnostics, None) else { + return; + }; + reject_unknown( + dataset, + &[ + "name", + "source", + "primary_key", + "unique_keys", + "description", + "ai_context", + "fields", + "custom_extensions", + ], + pointer, + diagnostics, + None, + ); + required_string(dataset, "name", pointer, diagnostics, None); + required_string(dataset, "source", pointer, diagnostics, None); + validate_string_array( + dataset.get("primary_key"), + &format!("{pointer}/primary_key"), + diagnostics, + None, + ); + validate_nested_string_array( + dataset.get("unique_keys"), + &format!("{pointer}/unique_keys"), + diagnostics, + None, + ); + if let Some(fields) = optional_array(dataset, "fields", pointer, diagnostics, None) { + for (index, field) in fields.iter().enumerate() { + validate_field_schema( + field, + &format!("{pointer}/fields/{index}"), + version, + diagnostics, + ); + } + } +} + +fn validate_field_schema( + value: &Value, + pointer: &str, + version: Option<&str>, + diagnostics: &mut Vec, +) { + let Some(field) = require_object(value, pointer, diagnostics, None) else { + return; + }; + let allowed = if version == Some("0.2.0.dev0") { + &[ + "name", + "expression", + "dimension", + "label", + "description", + "datatype", + "ai_context", + "custom_extensions", + ][..] + } else { + &[ + "name", + "expression", + "dimension", + "label", + "description", + "ai_context", + "custom_extensions", + ][..] + }; + reject_unknown(field, allowed, pointer, diagnostics, None); + required_string(field, "name", pointer, diagnostics, None); + validate_expression_schema( + field.get("expression"), + &format!("{pointer}/expression"), + version, + diagnostics, + ); + validate_data_type( + field.get("datatype"), + &format!("{pointer}/datatype"), + diagnostics, + ); + if let Some(dimension) = field.get("dimension") { + if let Some(dimension) = require_object( + dimension, + &format!("{pointer}/dimension"), + diagnostics, + None, + ) { + reject_unknown( + dimension, + &["is_time"], + &format!("{pointer}/dimension"), + diagnostics, + None, + ); + if let Some(is_time) = dimension.get("is_time") { + if !is_time.is_boolean() { + diagnostics.push(diagnostic( + "ossie.schema.type", + "dimension.is_time must be a boolean.", + format!("{pointer}/dimension/is_time"), + None, + )); + } + } + } + } +} + +fn validate_metric_schema( + value: &Value, + pointer: &str, + version: Option<&str>, + diagnostics: &mut Vec, +) { + let Some(metric) = require_object(value, pointer, diagnostics, None) else { + return; + }; + let allowed = if version == Some("0.2.0.dev0") { + &[ + "name", + "expression", + "description", + "datatype", + "ai_context", + "custom_extensions", + ][..] + } else { + &[ + "name", + "expression", + "description", + "ai_context", + "custom_extensions", + ][..] + }; + reject_unknown(metric, allowed, pointer, diagnostics, None); + required_string(metric, "name", pointer, diagnostics, None); + validate_expression_schema( + metric.get("expression"), + &format!("{pointer}/expression"), + version, + diagnostics, + ); + validate_data_type( + metric.get("datatype"), + &format!("{pointer}/datatype"), + diagnostics, + ); +} + +fn validate_relationship_schema( + value: &Value, + pointer: &str, + diagnostics: &mut Vec, +) { + let Some(relationship) = require_object(value, pointer, diagnostics, None) else { + return; + }; + reject_unknown( + relationship, + &[ + "name", + "from", + "to", + "from_columns", + "to_columns", + "ai_context", + "custom_extensions", + ], + pointer, + diagnostics, + None, + ); + for key in ["name", "from", "to"] { + required_string(relationship, key, pointer, diagnostics, None); + } + for key in ["from_columns", "to_columns"] { + let columns = required_array(relationship, key, pointer, diagnostics, None); + if columns.is_some_and(Vec::is_empty) { + diagnostics.push(diagnostic( + "ossie.schema.min_items", + format!("Relationship {key} must not be empty."), + format!("{pointer}/{key}"), + None, + )); + } + validate_string_array( + relationship.get(key), + &format!("{pointer}/{key}"), + diagnostics, + None, + ); + } +} + +fn validate_expression_schema( + value: Option<&Value>, + pointer: &str, + version: Option<&str>, + diagnostics: &mut Vec, +) { + let Some(value) = value else { + diagnostics.push(diagnostic( + "ossie.schema.required", + "An expression is required.", + pointer + .rsplit_once('/') + .map(|(parent, _)| parent) + .unwrap_or(""), + None, + )); + return; + }; + let Some(expression) = require_object(value, pointer, diagnostics, None) else { + return; + }; + reject_unknown(expression, &["dialects"], pointer, diagnostics, None); + let Some(variants) = required_array(expression, "dialects", pointer, diagnostics, None) else { + return; + }; + if variants.is_empty() { + diagnostics.push(diagnostic( + "ossie.schema.min_items", + "Expression dialects must contain at least one item.", + format!("{pointer}/dialects"), + None, + )); + } + let allowed_dialects = if version == Some("0.2.0.dev0") { + DIALECTS_0_2_0 + } else { + DIALECTS_0_1_1 + }; + let mut seen = BTreeSet::new(); + for (index, variant) in variants.iter().enumerate() { + let variant_pointer = format!("{pointer}/dialects/{index}"); + let Some(variant) = require_object(variant, &variant_pointer, diagnostics, None) else { + continue; + }; + reject_unknown( + variant, + &["dialect", "expression"], + &variant_pointer, + diagnostics, + None, + ); + let dialect = required_string(variant, "dialect", &variant_pointer, diagnostics, None); + required_string(variant, "expression", &variant_pointer, diagnostics, None); + if let Some(dialect) = dialect { + let normalized = dialect.to_ascii_uppercase(); + if !allowed_dialects.contains(&normalized.as_str()) { + diagnostics.push(diagnostic( + "ossie.schema.enum", + format!("Unsupported expression dialect {dialect:?} for this profile."), + format!("{variant_pointer}/dialect"), + None, + )); + } else if !seen.insert(normalized.clone()) { + diagnostics.push(diagnostic( + "ossie.semantic.expression.dialect_duplicate", + format!("Duplicate expression dialect {normalized:?}."), + format!("{variant_pointer}/dialect"), + None, + )); + } + } + } +} + +fn validate_data_type( + value: Option<&Value>, + pointer: &str, + diagnostics: &mut Vec, +) { + let Some(value) = value else { + return; + }; + match value.as_str() { + Some(value) if DATA_TYPES.contains(&value) => {} + Some(value) => diagnostics.push(diagnostic( + "ossie.schema.enum", + format!("Unsupported logical datatype {value:?}."), + pointer, + None, + )), + None => diagnostics.push(diagnostic( + "ossie.schema.type", + "Logical datatype must be a string.", + pointer, + None, + )), + } +} + +fn validate_ontology_root(root: &Map, diagnostics: &mut Vec) { + reject_unknown( + root, + &[ + "version", + "name", + "description", + "ai_context", + "ontology", + "ontology_mappings", + ], + "", + diagnostics, + None, + ); + if let Some(ontology) = root.get("ontology") { + if !ontology.is_array() { + diagnostics.push(diagnostic( + "ossie.schema.type", + "ontology must be an array.", + "/ontology", + None, + )); + } + } + if let Some(mappings) = root.get("ontology_mappings") { + if !mappings.is_array() { + diagnostics.push(diagnostic( + "ossie.schema.type", + "ontology_mappings must be an array.", + "/ontology_mappings", + None, + )); + } + } +} + +fn logical_scope_ids(root: &Map) -> Vec { + let Some(scopes) = root.get("semantic_model").and_then(Value::as_array) else { + return Vec::new(); + }; + let names = scopes + .iter() + .map(|scope| { + scope + .as_object() + .and_then(|scope| scope.get("name")) + .and_then(Value::as_str) + .unwrap_or("") + .to_string() + }) + .collect::>(); + let mut counts: HashMap = HashMap::new(); + for name in &names { + *counts.entry(normalize_identifier(name)).or_insert(0) += 1; + } + names + .into_iter() + .enumerate() + .map(|(index, name)| { + if counts + .get(&normalize_identifier(&name)) + .copied() + .unwrap_or(0) + > 1 + { + format!("{name}@{index}") + } else { + name + } + }) + .collect() +} + +fn validate_semantics( + root: &Map, + scope_ids: &[String], + diagnostics: &mut Vec, +) { + let Some(scopes) = root.get("semantic_model").and_then(Value::as_array) else { + return; + }; + duplicate_names( + scopes, + "/semantic_model", + "ossie.semantic.semantic_model.name_duplicate", + None, + diagnostics, + ); + for (scope_index, scope_value) in scopes.iter().enumerate() { + let Some(scope) = scope_value.as_object() else { + continue; + }; + let scope_id = scope_ids.get(scope_index).cloned(); + let scope_pointer = format!("/semantic_model/{scope_index}"); + let datasets = scope.get("datasets").and_then(Value::as_array); + let metrics = scope.get("metrics").and_then(Value::as_array); + let relationships = scope.get("relationships").and_then(Value::as_array); + if let Some(datasets) = datasets { + duplicate_names( + datasets, + &format!("{scope_pointer}/datasets"), + "ossie.semantic.dataset.name_duplicate", + scope_id.as_deref(), + diagnostics, + ); + for (dataset_index, dataset) in datasets.iter().enumerate() { + if let Some(fields) = dataset + .as_object() + .and_then(|dataset| dataset.get("fields")) + .and_then(Value::as_array) + { + duplicate_names( + fields, + &format!("{scope_pointer}/datasets/{dataset_index}/fields"), + "ossie.semantic.field.name_duplicate", + scope_id.as_deref(), + diagnostics, + ); + } + } + } + if let Some(metrics) = metrics { + duplicate_names( + metrics, + &format!("{scope_pointer}/metrics"), + "ossie.semantic.metric.name_duplicate", + scope_id.as_deref(), + diagnostics, + ); + } + if let Some(relationships) = relationships { + duplicate_names( + relationships, + &format!("{scope_pointer}/relationships"), + "ossie.semantic.relationship.name_duplicate", + scope_id.as_deref(), + diagnostics, + ); + validate_relationship_references( + datasets.map(Vec::as_slice).unwrap_or(&[]), + relationships, + scope_index, + scope_id.as_deref(), + diagnostics, + ); + } + } +} + +fn validate_relationship_references( + datasets: &[Value], + relationships: &[Value], + scope_index: usize, + scope_id: Option<&str>, + diagnostics: &mut Vec, +) { + let mut dataset_by_name = BTreeMap::new(); + for dataset in datasets { + let Some(dataset) = dataset.as_object() else { + continue; + }; + let Some(name) = dataset.get("name").and_then(Value::as_str) else { + continue; + }; + dataset_by_name.insert(normalize_identifier(name), dataset); + } + for (relationship_index, relationship) in relationships.iter().enumerate() { + let Some(relationship) = relationship.as_object() else { + continue; + }; + let pointer = format!("/semantic_model/{scope_index}/relationships/{relationship_index}"); + let from = relationship.get("from").and_then(Value::as_str); + let to = relationship.get("to").and_then(Value::as_str); + let from_dataset = + from.and_then(|name| dataset_by_name.get(&normalize_identifier(name)).copied()); + let to_dataset = + to.and_then(|name| dataset_by_name.get(&normalize_identifier(name)).copied()); + if from.is_some() && from_dataset.is_none() { + diagnostics.push(diagnostic( + "ossie.semantic.relationship.from_dataset_unknown", + "Relationship source dataset does not exist uniquely in this scope.", + format!("{pointer}/from"), + scope_id, + )); + } + if to.is_some() && to_dataset.is_none() { + diagnostics.push(diagnostic( + "ossie.semantic.relationship.to_dataset_unknown", + "Relationship target dataset does not exist uniquely in this scope.", + format!("{pointer}/to"), + scope_id, + )); + } + let from_columns = relationship.get("from_columns").and_then(Value::as_array); + let to_columns = relationship.get("to_columns").and_then(Value::as_array); + if let (Some(from_columns), Some(to_columns)) = (from_columns, to_columns) { + if from_columns.len() != to_columns.len() { + diagnostics.push(diagnostic( + "ossie.semantic.relationship.key_arity_mismatch", + "Relationship source and target key arrays must have equal length.", + &pointer, + scope_id, + )); + } + validate_key_fields( + from_dataset, + from_columns, + "from", + &format!("{pointer}/from_columns"), + scope_id, + diagnostics, + ); + let valid_target = validate_key_fields( + to_dataset, + to_columns, + "to", + &format!("{pointer}/to_columns"), + scope_id, + diagnostics, + ); + if valid_target { + if let Some(target) = to_dataset { + let target_columns = normalized_strings(to_columns); + let mut declared = Vec::new(); + if let Some(key) = target.get("primary_key").and_then(Value::as_array) { + declared.push(normalized_strings(key)); + } + if let Some(keys) = target.get("unique_keys").and_then(Value::as_array) { + for key in keys.iter().filter_map(Value::as_array) { + declared.push(normalized_strings(key)); + } + } + if !declared.contains(&target_columns) { + diagnostics.push(diagnostic( + "ossie.semantic.relationship.target_key_not_unique", + "Relationship target columns are not a declared primary or unique key.", + format!("{pointer}/to_columns"), + scope_id, + )); + } + } + } + } + } +} + +fn validate_key_fields( + dataset: Option<&Map>, + columns: &[Value], + side: &str, + pointer: &str, + scope: Option<&str>, + diagnostics: &mut Vec, +) -> bool { + let Some(dataset) = dataset else { + return false; + }; + let fields = dataset + .get("fields") + .and_then(Value::as_array) + .map(|fields| { + fields + .iter() + .filter_map(Value::as_object) + .filter_map(|field| field.get("name")) + .filter_map(Value::as_str) + .map(normalize_identifier) + .collect::>() + }) + .unwrap_or_default(); + let mut valid = true; + for (index, column) in columns.iter().enumerate() { + if let Some(column) = column.as_str() { + if !fields.contains(&normalize_identifier(column)) { + valid = false; + diagnostics.push(diagnostic( + format!("ossie.semantic.relationship.{side}_key_field_unknown"), + format!("Relationship {side} key field {column:?} is not declared."), + format!("{pointer}/{index}"), + scope, + )); + } + } + } + valid +} + +fn compile_scope( + value: &Value, + scope_index: usize, + scope_id: &str, + target: OssieTarget, + diagnostics: &mut Vec, +) -> Option { + let scope = value.as_object()?; + let datasets = scope.get("datasets")?.as_array()?; + let mut models = Vec::new(); + let mut model_index = HashMap::new(); + + for (dataset_index, dataset) in datasets.iter().enumerate() { + let dataset = dataset.as_object()?; + let name = dataset.get("name")?.as_str()?.to_string(); + let source = dataset.get("source")?.as_str()?.to_string(); + let field_names = dataset + .get("fields") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_object) + .filter_map(|field| field.get("name")) + .filter_map(Value::as_str) + .map(|name| (normalize_identifier(name), name.to_string())) + .collect::>(); + let canonical_columns = |values: &[Value]| { + values + .iter() + .filter_map(Value::as_str) + .filter_map(|name| field_names.get(&normalize_identifier(name)).cloned()) + .collect::>() + }; + let keys = dataset + .get("primary_key") + .and_then(Value::as_array) + .map(|values| canonical_columns(values)) + .unwrap_or_default(); + let unique_keys = dataset + .get("unique_keys") + .and_then(Value::as_array) + .map(|keys| { + keys.iter() + .filter_map(Value::as_array) + .map(|key| canonical_columns(key)) + .collect::>() + }) + .filter(|keys| !keys.is_empty()); + let mut dimensions = Vec::new(); + for (field_index, field) in dataset + .get("fields") + .and_then(Value::as_array) + .into_iter() + .flatten() + .enumerate() + { + let Some(field) = field.as_object() else { + continue; + }; + let Some(field_name) = field.get("name").and_then(Value::as_str) else { + continue; + }; + let pointer = format!( + "/semantic_model/{scope_index}/datasets/{dataset_index}/fields/{field_index}/expression" + ); + let Some(sql) = select_expression(field.get("expression"), target) else { + diagnostics.push(diagnostic( + "ossie.lowering.expression_unavailable", + format!( + "Field {name}.{field_name} has no {} or ANSI_SQL expression.", + target.label() + ), + pointer, + Some(scope_id), + )); + continue; + }; + if let Err(message) = validate_scalar_sql(&sql, target) { + diagnostics.push(diagnostic( + "ossie.lowering.expression_invalid", + message, + pointer, + Some(scope_id), + )); + continue; + } + let logical_data_type = field + .get("datatype") + .and_then(Value::as_str) + .map(str::to_string); + let declared_is_time = field + .get("dimension") + .and_then(Value::as_object) + .and_then(|dimension| dimension.get("is_time")) + .and_then(Value::as_bool); + let effective_is_time = declared_is_time.unwrap_or_else(|| { + logical_data_type + .as_deref() + .is_some_and(|value| TEMPORAL_TYPES.contains(&value)) + }); + let dimension_type = if effective_is_time { + DimensionType::Time + } else if logical_data_type.as_deref() == Some("Boolean") { + DimensionType::Boolean + } else if logical_data_type + .as_deref() + .is_some_and(|value| NUMERIC_TYPES.contains(&value)) + { + DimensionType::Numeric + } else { + DimensionType::Categorical + }; + dimensions.push(Dimension { + name: field_name.to_string(), + r#type: dimension_type, + logical_data_type, + declared_is_time, + sql: Some(sql), + granularity: None, + supported_granularities: None, + label: optional_string(field, "label"), + description: optional_string(field, "description"), + metadata: None, + meta: None, + format: None, + value_format_name: None, + parent: None, + window: None, + public: true, + }); + } + + let is_query = source_is_query(&source, target); + let model = Model { + name: name.clone(), + table: (!is_query).then_some(source.clone()), + sql: is_query.then_some(source), + source_uri: None, + extends: None, + primary_key: keys.first().cloned().unwrap_or_default(), + primary_key_columns: keys, + unique_keys, + dimensions, + metrics: Vec::new(), + relationships: Vec::new(), + segments: Vec::::new(), + pre_aggregations: Vec::new(), + default_time_dimension: None, + default_grain: None, + label: None, + description: optional_string(dataset, "description"), + metadata: None, + meta: None, + }; + model_index.insert(normalize_identifier(&name), models.len()); + models.push(model); + } + + for relationship in scope + .get("relationships") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + let Some(relationship) = relationship.as_object() else { + continue; + }; + let Some(edge_id) = relationship.get("name").and_then(Value::as_str) else { + continue; + }; + let Some(from) = relationship.get("from").and_then(Value::as_str) else { + continue; + }; + let Some(to) = relationship.get("to").and_then(Value::as_str) else { + continue; + }; + let Some(from_index) = model_index.get(&normalize_identifier(from)).copied() else { + continue; + }; + let Some(to_index) = model_index.get(&normalize_identifier(to)).copied() else { + continue; + }; + let from_field_names = models[from_index] + .dimensions + .iter() + .map(|field| (normalize_identifier(&field.name), field.name.clone())) + .collect::>(); + let to_field_names = models[to_index] + .dimensions + .iter() + .map(|field| (normalize_identifier(&field.name), field.name.clone())) + .collect::>(); + let from_columns = relationship + .get("from_columns") + .and_then(Value::as_array) + .map(|values| { + values + .iter() + .filter_map(Value::as_str) + .filter_map(|name| from_field_names.get(&normalize_identifier(name)).cloned()) + .collect::>() + }) + .unwrap_or_default(); + let to_columns = relationship + .get("to_columns") + .and_then(Value::as_array) + .map(|values| { + values + .iter() + .filter_map(Value::as_str) + .filter_map(|name| to_field_names.get(&normalize_identifier(name)).cloned()) + .collect::>() + }) + .unwrap_or_default(); + let target_name = models[to_index].name.clone(); + models[from_index].relationships.push(Relationship { + name: target_name, + edge_id: Some(edge_id.to_string()), + r#type: RelationshipType::ManyToOne, + foreign_key: from_columns.first().cloned(), + foreign_key_columns: Some(from_columns), + primary_key: to_columns.first().cloned(), + primary_key_columns: Some(to_columns), + through: None, + through_foreign_key: None, + through_foreign_key_columns: None, + related_foreign_key: None, + related_foreign_key_columns: None, + sql: None, + metadata: None, + }); + } + + let mut metrics = Vec::new(); + for (metric_index, metric) in scope + .get("metrics") + .and_then(Value::as_array) + .into_iter() + .flatten() + .enumerate() + { + let Some(metric) = metric.as_object() else { + continue; + }; + let Some(name) = metric.get("name").and_then(Value::as_str) else { + continue; + }; + let pointer = format!("/semantic_model/{scope_index}/metrics/{metric_index}/expression"); + let Some(sql) = select_expression(metric.get("expression"), target) else { + diagnostics.push(diagnostic( + "ossie.lowering.expression_unavailable", + format!( + "Metric {name:?} has no {} or ANSI_SQL expression.", + target.label() + ), + pointer, + Some(scope_id), + )); + continue; + }; + if let Err(message) = validate_scalar_sql(&sql, target) { + diagnostics.push(diagnostic( + "ossie.lowering.expression_invalid", + message, + pointer, + Some(scope_id), + )); + continue; + } + let mut parsed = metric_from_sql_expression( + name.to_string(), + Some(sql), + optional_string(metric, "description"), + None, + ); + parsed.logical_data_type = metric + .get("datatype") + .and_then(Value::as_str) + .map(str::to_string); + metrics.push(parsed); + } + + Some(OssieCompiledScope { + scope_id: scope_id.to_string(), + semantic_model_index: scope_index, + target_dialect: target.label().to_string(), + models, + metrics, + }) +} + +fn source_is_query(source: &str, target: OssieTarget) -> bool { + polyglot_sql::parse_one(source, target.parser_dialect()).is_ok_and(|expression| { + matches!( + expression, + Expression::Select(_) + | Expression::Union(_) + | Expression::Intersect(_) + | Expression::Except(_) + | Expression::Subquery(_) + | Expression::Values(_) + ) + }) +} + +fn select_expression(value: Option<&Value>, target: OssieTarget) -> Option { + let variants = value?.as_object()?.get("dialects")?.as_array()?; + let mut exact = None; + let mut ansi = None; + for variant in variants { + let variant = variant.as_object()?; + let dialect = variant.get("dialect")?.as_str()?.to_ascii_uppercase(); + let expression = variant.get("expression")?.as_str()?.to_string(); + if dialect == target.label() && exact.is_none() { + exact = Some(expression.clone()); + } + if dialect == "ANSI_SQL" && ansi.is_none() { + ansi = Some(expression); + } + } + exact.or(ansi) +} + +fn validate_scalar_sql(sql: &str, target: OssieTarget) -> std::result::Result<(), String> { + let wrapped = format!("SELECT {sql}"); + let parsed = polyglot_sql::parse_one(&wrapped, target.parser_dialect()) + .map_err(|error| format!("Invalid {} SQL expression: {error}", target.label()))?; + let Expression::Select(select) = parsed else { + return Err(format!( + "Expected one scalar {} SQL expression.", + target.label() + )); + }; + let has_query_clauses = select.from.is_some() + || !select.joins.is_empty() + || select.where_clause.is_some() + || select.group_by.is_some() + || select.having.is_some() + || select.qualify.is_some() + || select.order_by.is_some() + || select.limit.is_some() + || select.offset.is_some() + || select.with.is_some(); + if select.expressions.len() != 1 || has_query_clauses { + return Err(format!( + "Expected one scalar {} SQL expression without query clauses.", + target.label() + )); + } + let contains_query = select.expressions[0].contains(|expression| { + matches!( + expression, + Expression::Select(_) + | Expression::Union(_) + | Expression::Intersect(_) + | Expression::Except(_) + | Expression::Subquery(_) + | Expression::Values(_) + | Expression::Insert(_) + | Expression::Update(_) + | Expression::Delete(_) + ) + }); + if contains_query { + return Err(format!( + "Expected one scalar {} SQL expression without a nested query.", + target.label() + )); + } + Ok(()) +} + +fn duplicate_names( + values: &[Value], + pointer: &str, + code: &str, + scope: Option<&str>, + diagnostics: &mut Vec, +) { + let mut first = BTreeMap::new(); + for (index, value) in values.iter().enumerate() { + let Some(name) = value + .as_object() + .and_then(|value| value.get("name")) + .and_then(Value::as_str) + else { + continue; + }; + if let Some(first_index) = first.insert(normalize_identifier(name), index) { + diagnostics.push(diagnostic( + code, + format!("Duplicate name {name:?}; first declared at index {first_index}."), + format!("{pointer}/{index}/name"), + scope, + )); + } + } +} + +fn normalize_identifier(identifier: &str) -> String { + if identifier.len() >= 2 && identifier.starts_with('"') && identifier.ends_with('"') { + identifier[1..identifier.len() - 1].replace("\"\"", "\"") + } else { + identifier.to_uppercase() + } +} + +fn normalized_strings(values: &[Value]) -> Vec { + values + .iter() + .filter_map(Value::as_str) + .map(normalize_identifier) + .collect() +} + +fn reject_unknown( + object: &Map, + allowed: &[&str], + pointer: &str, + diagnostics: &mut Vec, + scope: Option<&str>, +) { + for key in object.keys().filter(|key| !allowed.contains(&key.as_str())) { + diagnostics.push(diagnostic( + "ossie.schema.additional_properties", + format!("Unexpected property {key:?}."), + pointer, + scope, + )); + } +} + +fn require_object<'a>( + value: &'a Value, + pointer: &str, + diagnostics: &mut Vec, + scope: Option<&str>, +) -> Option<&'a Map> { + match value.as_object() { + Some(value) => Some(value), + None => { + diagnostics.push(diagnostic( + "ossie.schema.type", + "Expected an object.", + pointer, + scope, + )); + None + } + } +} + +fn required_array<'a>( + object: &'a Map, + key: &str, + pointer: &str, + diagnostics: &mut Vec, + scope: Option<&str>, +) -> Option<&'a Vec> { + let Some(value) = object.get(key) else { + diagnostics.push(diagnostic( + "ossie.schema.required", + format!("Required property {key:?} is missing."), + pointer, + scope, + )); + return None; + }; + match value.as_array() { + Some(value) => Some(value), + None => { + diagnostics.push(diagnostic( + "ossie.schema.type", + format!("Property {key:?} must be an array."), + format!("{pointer}/{key}"), + scope, + )); + None + } + } +} + +fn optional_array<'a>( + object: &'a Map, + key: &str, + pointer: &str, + diagnostics: &mut Vec, + scope: Option<&str>, +) -> Option<&'a Vec> { + let value = object.get(key)?; + match value.as_array() { + Some(value) => Some(value), + None => { + diagnostics.push(diagnostic( + "ossie.schema.type", + format!("Property {key:?} must be an array."), + format!("{pointer}/{key}"), + scope, + )); + None + } + } +} + +fn required_string<'a>( + object: &'a Map, + key: &str, + pointer: &str, + diagnostics: &mut Vec, + scope: Option<&str>, +) -> Option<&'a str> { + let Some(value) = object.get(key) else { + diagnostics.push(diagnostic( + "ossie.schema.required", + format!("Required property {key:?} is missing."), + pointer, + scope, + )); + return None; + }; + match value.as_str().filter(|value| !value.is_empty()) { + Some(value) => Some(value), + None => { + diagnostics.push(diagnostic( + "ossie.schema.type", + format!("Property {key:?} must be a non-empty string."), + format!("{pointer}/{key}"), + scope, + )); + None + } + } +} + +fn validate_string_array( + value: Option<&Value>, + pointer: &str, + diagnostics: &mut Vec, + scope: Option<&str>, +) { + let Some(value) = value else { + return; + }; + let Some(values) = value.as_array() else { + diagnostics.push(diagnostic( + "ossie.schema.type", + "Expected an array of strings.", + pointer, + scope, + )); + return; + }; + for (index, value) in values.iter().enumerate() { + if value.as_str().filter(|value| !value.is_empty()).is_none() { + diagnostics.push(diagnostic( + "ossie.schema.type", + "Expected a non-empty string.", + format!("{pointer}/{index}"), + scope, + )); + } + } +} + +fn validate_nested_string_array( + value: Option<&Value>, + pointer: &str, + diagnostics: &mut Vec, + scope: Option<&str>, +) { + let Some(value) = value else { + return; + }; + let Some(values) = value.as_array() else { + diagnostics.push(diagnostic( + "ossie.schema.type", + "Expected an array of string arrays.", + pointer, + scope, + )); + return; + }; + for (index, value) in values.iter().enumerate() { + validate_string_array( + Some(value), + &format!("{pointer}/{index}"), + diagnostics, + scope, + ); + } +} + +fn optional_string(object: &Map, key: &str) -> Option { + object.get(key).and_then(Value::as_str).map(str::to_string) +} + +fn diagnostic( + code: impl Into, + message: impl Into, + instance_path: impl Into, + scope: Option<&str>, +) -> OssieDiagnostic { + OssieDiagnostic { + code: code.into(), + severity: "error", + message: message.into(), + instance_path: instance_path.into(), + scope: scope.map(str::to_string), + } +} + +fn sort_diagnostics(diagnostics: &mut [OssieDiagnostic]) { + diagnostics.sort_by(|left, right| { + (&left.instance_path, &left.code, &left.scope).cmp(&( + &right.instance_path, + &right.code, + &right.scope, + )) + }); +} + +fn status_error(status: &OssieStatus) -> SidemanticError { + let payload = serde_json::to_string(status).unwrap_or_else(|_| "{}".to_string()); + SidemanticError::Validation(format!("Ossie strict handoff rejected input: {payload}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + const MULTI_SCOPE: &str = r#" +version: 0.2.0.dev0 +semantic_model: + - name: commerce + datasets: + - name: orders + source: analytics.orders + primary_key: [id] + fields: + - name: id + datatype: Integer + dimension: {is_time: false} + expression: + dialects: + - {dialect: ANSI_SQL, expression: id} + - {dialect: BIGQUERY, expression: SAFE_CAST(id AS INT64)} + - {dialect: MDX, expression: "[Orders].[Id]"} + - name: operations + datasets: + - name: orders + source: operations.orders +"#; + + #[test] + fn profile_contracts_are_explicit() { + let core = OssieForwardAdapter.inspect( + "{\"version\":\"0.1.1\",\"semantic_model\":[]}", + OssieSerialization::Json, + OssieConsumerProfile::OssieCore, + ); + assert!(core.valid); + assert_eq!(core.profile.unwrap().identifier, "ossie-core:0.1.1"); + + let alias = OssieForwardAdapter.inspect( + "{\"version\":\"0.1.0\",\"semantic_model\":[]}", + OssieSerialization::Json, + OssieConsumerProfile::Dbt112, + ); + assert!(alias.valid); + let profile = alias.profile.unwrap(); + assert_eq!(profile.identifier, "dbt-1.12:0.1.0"); + assert_eq!(profile.compatibility_alias_for.as_deref(), Some("0.1.1")); + + let dbt_released = OssieForwardAdapter.inspect( + "{\"version\":\"0.1.1\",\"semantic_model\":[]}", + OssieSerialization::Json, + OssieConsumerProfile::Dbt112, + ); + assert!(dbt_released.valid, "{:?}", dbt_released.diagnostics); + assert_eq!(dbt_released.profile.unwrap().identifier, "dbt-1.12:0.1.1"); + } + + #[test] + fn executable_targets_cover_runtime_dialects() { + assert_eq!(OssieTarget::parse("duckdb"), Ok(OssieTarget::DuckDb)); + assert_eq!(OssieTarget::parse("postgresql"), Ok(OssieTarget::Postgres)); + assert_eq!(OssieTarget::parse("snowflake"), Ok(OssieTarget::Snowflake)); + assert_eq!( + OssieTarget::parse("databricks"), + Ok(OssieTarget::Databricks) + ); + assert_eq!(OssieTarget::parse("bigquery"), Ok(OssieTarget::BigQuery)); + } + + #[test] + fn expression_selection_is_exact_then_ansi_for_every_runtime_target() { + let expression = serde_json::json!({ + "dialects": [ + {"dialect": "ANSI_SQL", "expression": "ansi_value"}, + {"dialect": "SNOWFLAKE", "expression": "snowflake_value"}, + {"dialect": "DATABRICKS", "expression": "databricks_value"}, + {"dialect": "BIGQUERY", "expression": "bigquery_value"} + ] + }); + + assert_eq!( + select_expression(Some(&expression), OssieTarget::Snowflake).as_deref(), + Some("snowflake_value") + ); + assert_eq!( + select_expression(Some(&expression), OssieTarget::Databricks).as_deref(), + Some("databricks_value") + ); + assert_eq!( + select_expression(Some(&expression), OssieTarget::BigQuery).as_deref(), + Some("bigquery_value") + ); + assert_eq!( + select_expression(Some(&expression), OssieTarget::DuckDb).as_deref(), + Some("ansi_value") + ); + assert_eq!( + select_expression(Some(&expression), OssieTarget::Postgres).as_deref(), + Some("ansi_value") + ); + } + + #[test] + fn source_classification_handles_relations_and_all_query_shapes() { + for source in [ + "SELECT * FROM raw.orders", + "(SELECT * FROM raw.orders)", + "WITH recent AS (SELECT * FROM raw.orders) SELECT * FROM recent", + "VALUES (1)", + ] { + assert!(source_is_query(source, OssieTarget::DuckDb), "{source}"); + } + for source in ["analytics.orders", "\"analytics\".\"orders\""] { + assert!(!source_is_query(source, OssieTarget::DuckDb), "{source}"); + } + } + + #[test] + fn regular_identifier_references_bind_to_declared_runtime_names() { + let content = r#" +version: 0.2.0.dev0 +semantic_model: + - name: Commerce + datasets: + - name: Orders + source: analytics.orders + primary_key: [ID] + fields: + - name: Id + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} + - name: Customer_Id + expression: {dialects: [{dialect: ANSI_SQL, expression: customer_id}]} + - name: Customers + source: analytics.customers + primary_key: [id] + fields: + - name: ID + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} + relationships: + - name: Order_Customer + from: ORDERS + to: customers + from_columns: [CUSTOMER_ID] + to_columns: [Id] +"#; + let catalog = OssieForwardAdapter + .parse_catalog( + content, + OssieSerialization::Yaml, + OssieConsumerProfile::OssieCore, + OssieTarget::DuckDb, + ) + .unwrap(); + let scope = &catalog.scopes[0]; + assert_eq!(scope.models[0].primary_keys(), vec!["Id"]); + let relationship = &scope.models[0].relationships[0]; + assert_eq!(relationship.name, "Customers"); + assert_eq!(relationship.foreign_key_columns(), vec!["Customer_Id"]); + assert_eq!(relationship.primary_key_columns(), vec!["ID"]); + } + + #[test] + fn scalar_validation_rejects_query_constructs() { + for sql in [ + "id FROM orders", + "id WHERE active", + "(SELECT id FROM orders)", + "id IN (SELECT id FROM orders)", + ] { + assert!( + validate_scalar_sql(sql, OssieTarget::DuckDb).is_err(), + "{sql}" + ); + } + assert!( + validate_scalar_sql("CASE WHEN active THEN id ELSE 0 END", OssieTarget::DuckDb).is_ok() + ); + } + + #[test] + fn scopes_remain_separate_and_selection_is_explicit() { + let adapter = OssieForwardAdapter; + let catalog = adapter + .parse_catalog( + MULTI_SCOPE, + OssieSerialization::Yaml, + OssieConsumerProfile::OssieCore, + OssieTarget::BigQuery, + ) + .unwrap(); + assert_eq!( + catalog + .scopes + .iter() + .map(|scope| scope.scope_id.as_str()) + .collect::>(), + vec!["commerce", "operations"] + ); + assert_eq!( + catalog.scopes[0].models[0].dimensions[0].sql.as_deref(), + Some("SAFE_CAST(id AS INT64)") + ); + assert_eq!( + catalog.scopes[0].models[0].dimensions[0].declared_is_time, + Some(false) + ); + assert!(adapter + .select_scope( + MULTI_SCOPE, + OssieSerialization::Yaml, + OssieConsumerProfile::OssieCore, + OssieTarget::BigQuery, + None, + ) + .unwrap_err() + .to_string() + .contains("ambiguous")); + let operations = adapter + .select_scope( + MULTI_SCOPE, + OssieSerialization::Yaml, + OssieConsumerProfile::OssieCore, + OssieTarget::BigQuery, + Some("operations"), + ) + .unwrap(); + assert_eq!( + operations.models[0].table.as_deref(), + Some("operations.orders") + ); + } + + #[test] + fn ansi_fallback_ignores_non_sql_variants() { + let scope = OssieForwardAdapter + .select_scope( + MULTI_SCOPE, + OssieSerialization::Yaml, + OssieConsumerProfile::OssieCore, + OssieTarget::AnsiSql, + Some("commerce"), + ) + .unwrap(); + assert_eq!(scope.models[0].dimensions[0].sql.as_deref(), Some("id")); + } + + #[test] + fn non_sql_only_expression_fails_closed() { + let document = r#" +version: 0.2.0.dev0 +semantic_model: + - name: commerce + datasets: + - name: orders + source: analytics.orders + fields: + - name: id + expression: + dialects: + - {dialect: MDX, expression: "[Orders].[Id]"} +"#; + + let error = OssieForwardAdapter + .select_scope( + document, + OssieSerialization::Yaml, + OssieConsumerProfile::OssieCore, + OssieTarget::BigQuery, + Some("commerce"), + ) + .unwrap_err(); + assert!(error + .to_string() + .contains("ossie.lowering.expression_unavailable")); + } +} diff --git a/sidemantic-rs/src/config/loader.rs b/sidemantic-rs/src/config/loader.rs index 107d3ff1d..d3835cfe9 100644 --- a/sidemantic-rs/src/config/loader.rs +++ b/sidemantic-rs/src/config/loader.rs @@ -1178,6 +1178,7 @@ fn infer_relationships(models: &mut HashMap, skip: &HashSet, skip: &HashSet, + #[serde(default)] + pub logical_data_type: Option, + #[serde(default)] + pub declared_is_time: Option, #[serde(default, alias = "expr")] pub sql: Option, pub granularity: Option, @@ -155,6 +159,8 @@ pub struct DimensionConfig { pub struct MetricConfig { pub name: String, pub extends: Option, + #[serde(default)] + pub logical_data_type: Option, #[serde(default, rename = "type")] pub metric_type: Option, pub agg: Option, @@ -219,6 +225,8 @@ pub struct CohortInnerMetricConfig { #[serde(deny_unknown_fields)] pub struct RelationshipConfig { pub name: String, + #[serde(default)] + pub edge_id: Option, #[serde(default, rename = "type")] pub rel_type: Option, pub foreign_key: Option, @@ -618,6 +626,8 @@ impl DimensionConfig { Dimension { name: self.name, r#type: dim_type, + logical_data_type: self.logical_data_type, + declared_is_time: self.declared_is_time, sql: self.sql, granularity: self.granularity, supported_granularities: self.supported_granularities, @@ -705,6 +715,7 @@ impl MetricConfig { Metric { name: self.name, extends: self.extends, + logical_data_type: self.logical_data_type, r#type: metric_type, agg, sql, @@ -772,6 +783,7 @@ impl RelationshipConfig { Relationship { name: self.name, + edge_id: self.edge_id, r#type: rel_type, foreign_key: foreign_key_columns .as_ref() @@ -1381,6 +1393,66 @@ models: assert!(err.to_string().contains("unexpected")); } + #[test] + fn test_native_contract_preserves_optional_parity_fields_and_explicit_false() { + let yaml = r#" +models: + - name: orders + table: orders + dimensions: + - name: occurred_at + type: categorical + logical_data_type: DateTimeTz + declared_is_time: false + - name: status + type: categorical + metrics: + - name: revenue + agg: sum + sql: amount + logical_data_type: Decimal + relationships: + - name: customers + edge_id: orders_customer + type: many_to_one + foreign_key: customer_id +"#; + + let config: SidemanticConfig = serde_yaml::from_str(yaml).unwrap(); + let model = &config.models[0]; + assert_eq!( + model.dimensions[0].logical_data_type.as_deref(), + Some("DateTimeTz") + ); + assert_eq!(model.dimensions[0].declared_is_time, Some(false)); + assert_eq!(model.dimensions[1].logical_data_type, None); + assert_eq!(model.dimensions[1].declared_is_time, None); + assert_eq!( + model.metrics[0].logical_data_type.as_deref(), + Some("Decimal") + ); + assert_eq!( + model.relationships[0].edge_id.as_deref(), + Some("orders_customer") + ); + + let (models, _, _) = config.into_parts().unwrap(); + assert_eq!( + models[0].dimensions[0].logical_data_type.as_deref(), + Some("DateTimeTz") + ); + assert_eq!(models[0].dimensions[0].declared_is_time, Some(false)); + assert_eq!(models[0].dimensions[1].declared_is_time, None); + assert_eq!( + models[0].metrics[0].logical_data_type.as_deref(), + Some("Decimal") + ); + assert_eq!( + models[0].relationships[0].edge_id.as_deref(), + Some("orders_customer") + ); + } + #[test] fn test_native_contract_rejects_unknown_time_grains_with_structured_error() { let yaml = r#" diff --git a/sidemantic-rs/src/config/sql_parser.rs b/sidemantic-rs/src/config/sql_parser.rs index 41d069446..848dce9b9 100644 --- a/sidemantic-rs/src/config/sql_parser.rs +++ b/sidemantic-rs/src/config/sql_parser.rs @@ -2010,6 +2010,7 @@ fn build_relationship(props: &HashMap) -> Option { Some(Relationship { name: name.clone(), + edge_id: None, r#type: rtype, foreign_key: foreign_key_columns .as_ref() diff --git a/sidemantic-rs/src/core/graph.rs b/sidemantic-rs/src/core/graph.rs index 4d134ba96..b7b338e75 100644 --- a/sidemantic-rs/src/core/graph.rs +++ b/sidemantic-rs/src/core/graph.rs @@ -18,6 +18,8 @@ pub struct JoinStep { pub from_keys: Vec, pub to_keys: Vec, pub relationship_type: RelationshipType, + /// Stable identity of the declared relationship edge. + pub edge_id: Option, /// Custom SQL join condition (overrides FK/PK join) pub custom_condition: Option, } @@ -68,13 +70,14 @@ impl JoinPath { } } -/// Edge in the adjacency list: (target_model, from_keys, to_keys, relationship_type, custom_sql) +/// Edge in the adjacency list: target, keys, relationship type, custom SQL, edge identity. type AdjacencyEdge = ( String, Vec, Vec, RelationshipType, Option, + Option, ); /// The semantic graph holds all models and their relationships @@ -547,6 +550,7 @@ impl SemanticGraph { source_fks.clone(), RelationshipType::OneToMany, None, + rel.edge_id.clone(), )); // through -> source (many_to_one) self.adjacency @@ -558,6 +562,7 @@ impl SemanticGraph { source_pk, RelationshipType::ManyToOne, None, + rel.edge_id.clone(), )); // through -> target (many_to_one) @@ -570,6 +575,7 @@ impl SemanticGraph { target_pk.clone(), RelationshipType::ManyToOne, None, + rel.edge_id.clone(), )); // target -> through (one_to_many) self.adjacency.entry(rel.name.clone()).or_default().push(( @@ -578,6 +584,7 @@ impl SemanticGraph { target_fks, RelationshipType::OneToMany, None, + rel.edge_id.clone(), )); continue; } @@ -608,6 +615,7 @@ impl SemanticGraph { to_keys.clone(), rel.r#type.clone(), rel.sql.clone(), + rel.edge_id.clone(), )); } @@ -668,6 +676,7 @@ impl SemanticGraph { reverse_to_keys, reverse_type, reverse_sql, + rel.edge_id.clone(), )); } } @@ -697,7 +706,7 @@ impl SemanticGraph { while let Some((current, path)) = queue.pop_front() { if let Some(edges) = self.adjacency.get(¤t) { - for (target, from_keys, to_keys, rel_type, custom_sql) in edges { + for (target, from_keys, to_keys, rel_type, custom_sql, edge_id) in edges { if !visited.contains(target) { let mut new_path = path.clone(); let from_key = from_keys.first().cloned().unwrap_or_default(); @@ -710,6 +719,7 @@ impl SemanticGraph { from_keys: from_keys.clone(), to_keys: to_keys.clone(), relationship_type: rel_type.clone(), + edge_id: edge_id.clone(), custom_condition: custom_sql.clone(), }); @@ -958,6 +968,25 @@ mod tests { assert_eq!(path.steps.len(), 1); } + #[test] + fn test_edge_id_survives_direct_and_reverse_paths() { + let mut graph = SemanticGraph::new(); + let mut relationship = Relationship::many_to_one("customers"); + relationship.edge_id = Some("orders_customer".to_string()); + let orders = Model::new("orders", "order_id") + .with_table("orders") + .with_relationship(relationship); + let customers = Model::new("customers", "id").with_table("customers"); + + graph.add_model(orders).unwrap(); + graph.add_model(customers).unwrap(); + + let forward = graph.find_join_path("orders", "customers").unwrap(); + let reverse = graph.find_join_path("customers", "orders").unwrap(); + assert_eq!(forward.steps[0].edge_id.as_deref(), Some("orders_customer")); + assert_eq!(reverse.steps[0].edge_id.as_deref(), Some("orders_customer")); + } + #[test] fn test_one_to_many_omitted_key_defaults_to_id() { let mut graph = SemanticGraph::new(); @@ -1083,6 +1112,7 @@ mod tests { .with_table("orders") .with_relationship(Relationship { name: "customers".to_string(), + edge_id: None, r#type: RelationshipType::ManyToOne, foreign_key: Some("customer_id".to_string()), foreign_key_columns: None, @@ -1118,6 +1148,7 @@ mod tests { .with_table("orders") .with_relationship(Relationship { name: "products".to_string(), + edge_id: Some("orders_products".to_string()), r#type: RelationshipType::ManyToMany, foreign_key: None, foreign_key_columns: None, @@ -1149,6 +1180,7 @@ mod tests { assert_eq!(path.steps[0].from_keys, vec!["order_id".to_string()]); assert_eq!(path.steps[0].to_keys, vec!["order_id".to_string()]); assert_eq!(path.steps[0].relationship_type, RelationshipType::OneToMany); + assert_eq!(path.steps[0].edge_id.as_deref(), Some("orders_products")); // order_items -> products assert_eq!(path.steps[1].from_model, "order_items"); @@ -1158,6 +1190,14 @@ mod tests { assert_eq!(path.steps[1].from_keys, vec!["product_id".to_string()]); assert_eq!(path.steps[1].to_keys, vec!["product_id".to_string()]); assert_eq!(path.steps[1].relationship_type, RelationshipType::ManyToOne); + assert_eq!(path.steps[1].edge_id.as_deref(), Some("orders_products")); + + let reverse_path = graph.find_join_path("products", "orders").unwrap(); + assert_eq!(reverse_path.steps.len(), 2); + assert!(reverse_path + .steps + .iter() + .all(|step| step.edge_id.as_deref() == Some("orders_products"))); } #[test] @@ -1169,6 +1209,7 @@ mod tests { .with_table("orders") .with_relationship(Relationship { name: "products".to_string(), + edge_id: None, r#type: RelationshipType::ManyToMany, foreign_key: None, foreign_key_columns: None, diff --git a/sidemantic-rs/src/core/model.rs b/sidemantic-rs/src/core/model.rs index b0c116971..7c882c665 100644 --- a/sidemantic-rs/src/core/model.rs +++ b/sidemantic-rs/src/core/model.rs @@ -21,6 +21,12 @@ pub struct Dimension { pub name: String, #[serde(default)] pub r#type: DimensionType, + /// Source logical data type, independent of the effective runtime type. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logical_data_type: Option, + /// Source-declared temporal role. `None` preserves an omitted declaration. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub declared_is_time: Option, /// SQL expression (defaults to name if not provided) pub sql: Option, /// Time granularity (for time dimensions) @@ -60,6 +66,8 @@ impl Dimension { Self { name: name.into(), r#type: DimensionType::Categorical, + logical_data_type: None, + declared_is_time: None, sql: None, granularity: None, supported_granularities: None, @@ -221,6 +229,9 @@ pub struct Metric { /// Parent metric to inherit from. #[serde(default)] pub extends: Option, + /// Source logical result type, independent of aggregation/runtime type. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logical_data_type: Option, #[serde(default)] pub r#type: MetricType, /// Aggregation function (for simple metrics) @@ -347,6 +358,7 @@ impl Metric { Self { name: name.into(), extends: None, + logical_data_type: None, r#type: MetricType::Simple, agg: Some(Aggregation::Sum), sql: None, @@ -611,6 +623,9 @@ pub enum RelationshipType { pub struct Relationship { /// Target model name pub name: String, + /// Stable identity of the declared edge, independent of target model name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub edge_id: Option, #[serde(default)] pub r#type: RelationshipType, /// Foreign key column. @@ -652,6 +667,7 @@ impl Relationship { pub fn new(target: impl Into) -> Self { Self { name: target.into(), + edge_id: None, r#type: RelationshipType::ManyToOne, foreign_key: None, foreign_key_columns: None, @@ -1024,7 +1040,11 @@ impl Model { pub fn primary_keys(&self) -> Vec { if self.primary_key_columns.is_empty() { - vec![self.primary_key.clone()] + if self.primary_key.is_empty() { + Vec::new() + } else { + vec![self.primary_key.clone()] + } } else { self.primary_key_columns.clone() } @@ -1148,4 +1168,13 @@ mod tests { assert!(model.get_metric("revenue").is_some()); assert!(model.get_relationship("customers").is_some()); } + + #[test] + fn unknown_primary_key_stays_unknown() { + let mut model = Model::new("orders", "id"); + model.primary_key.clear(); + model.primary_key_columns.clear(); + + assert!(model.primary_keys().is_empty()); + } } diff --git a/sidemantic-rs/src/lib.rs b/sidemantic-rs/src/lib.rs index e46f743ee..09496a372 100644 --- a/sidemantic-rs/src/lib.rs +++ b/sidemantic-rs/src/lib.rs @@ -48,7 +48,11 @@ pub mod sql; pub mod wasm; // Re-export commonly used types -pub use adapters::{Adapter, CubeAdapter, OsiAdapter, ParsedDocument}; +pub use adapters::{ + Adapter, CubeAdapter, OsiAdapter, OssieCatalog, OssieCompiledScope, OssieConsumerProfile, + OssieDiagnostic, OssieForwardAdapter, OssieProfile, OssieSerialization, OssieStatus, + OssieTarget, ParsedDocument, +}; pub use config::{ load_from_directory, load_from_directory_with_metadata, load_from_file, load_from_string, }; @@ -66,16 +70,17 @@ pub use runtime::{ dimension_sql_expr_with_yaml, dimension_with_granularity_with_yaml, evaluate_table_calculation_expression, extract_column_references, extract_metric_dependencies_from_yaml, extract_preaggregation_patterns, find_models_for_query, - find_relationship_path_with_yaml, format_parameter_value_with_yaml, - generate_catalog_metadata_with_yaml, generate_preaggregation_definition, - generate_preaggregation_materialization_sql_with_yaml, generate_preaggregation_name, - generate_time_comparison_sql, interpolate_sql_with_parameters_with_yaml, is_relative_date, - is_sql_template, load_graph_from_directory, load_graph_with_yaml, metric_is_simple_aggregation, - metric_sql_expr, metric_to_sql, model_find_dimension_index_with_yaml, - model_find_metric_index_with_yaml, model_find_pre_aggregation_index_with_yaml, - model_find_segment_index_with_yaml, model_get_drill_down_with_yaml, - model_get_drill_up_with_yaml, model_get_hierarchy_path_with_yaml, parse_reference_with_yaml, - parse_relative_date, parse_simple_metric_aggregation, parse_sql_definitions_payload, + find_relationship_path_payload_with_yaml, find_relationship_path_with_yaml, + format_parameter_value_with_yaml, generate_catalog_metadata_with_yaml, + generate_preaggregation_definition, generate_preaggregation_materialization_sql_with_yaml, + generate_preaggregation_name, generate_time_comparison_sql, + interpolate_sql_with_parameters_with_yaml, is_relative_date, is_sql_template, + load_graph_from_directory, load_graph_with_yaml, metric_is_simple_aggregation, metric_sql_expr, + metric_to_sql, model_find_dimension_index_with_yaml, model_find_metric_index_with_yaml, + model_find_pre_aggregation_index_with_yaml, model_find_segment_index_with_yaml, + model_get_drill_down_with_yaml, model_get_drill_up_with_yaml, + model_get_hierarchy_path_with_yaml, parse_reference_with_yaml, parse_relative_date, + parse_simple_metric_aggregation, parse_sql_definitions_payload, parse_sql_graph_definitions_payload, parse_sql_model_payload, plan_preaggregation_refresh_execution, recommend_preaggregation_patterns, relationship_foreign_key_columns_with_yaml, relationship_primary_key_columns_with_yaml, @@ -89,7 +94,7 @@ pub use runtime::{ validate_query_references, validate_query_with_yaml, validate_table_calculation_payload, validate_table_formula_expression, LoadedGraphPayload, PreaggregationRefreshExecutionPlan, PreaggregationRefreshResultShape, QueryValidationContext, RelationshipPathError, - RelationshipPathStep, SidemanticRuntime, + RelationshipPathPayloadStep, RelationshipPathStep, SidemanticRuntime, }; pub use sql::{QueryRewriter, SemanticQuery, SqlGenerator}; #[cfg(feature = "wasm")] diff --git a/sidemantic-rs/src/python.rs b/sidemantic-rs/src/python.rs index 24b1251f8..e5ea8cec3 100644 --- a/sidemantic-rs/src/python.rs +++ b/sidemantic-rs/src/python.rs @@ -1,5 +1,6 @@ //! Python bindings for sidemantic-rs via PyO3. +use crate::adapters::{OssieConsumerProfile, OssieForwardAdapter, OssieSerialization, OssieTarget}; #[cfg(feature = "python-adbc")] use crate::db::{execute_with_adbc as execute_with_adbc_native, AdbcExecutionRequest, AdbcValue}; use crate::error::SidemanticError; @@ -22,6 +23,7 @@ use crate::runtime::{ extract_preaggregation_patterns as extract_preaggregation_patterns_native, find_models_for_query as find_models_for_query_native, find_models_for_query_with_yaml as find_models_for_query_with_yaml_native, + find_relationship_path_payload_with_yaml as find_relationship_path_payload_with_yaml_native, find_relationship_path_with_yaml as find_relationship_path_with_yaml_native, format_parameter_value_with_yaml as format_parameter_value_with_yaml_native, generate_catalog_metadata_with_yaml as generate_catalog_metadata_with_yaml_native, @@ -1280,6 +1282,31 @@ fn find_relationship_path_with_yaml( }) } +/// Find an edge-aware join path as a JSON object payload. +#[pyfunction] +fn find_relationship_path_payload_with_yaml( + graph_yaml: &str, + from_model: &str, + to_model: &str, +) -> PyResult { + let path = find_relationship_path_payload_with_yaml_native(graph_yaml, from_model, to_model) + .map_err(|e| match e { + RelationshipPathError::ModelNotFound(model_name) => { + PyKeyError::new_err(format!("Model {model_name} not found")) + } + RelationshipPathError::NoJoinPath { + from_model, + to_model, + } => PyValueError::new_err(format!( + "No join path found between {from_model} and {to_model}" + )), + RelationshipPathError::InvalidPayload(err) => { + PyValueError::new_err(format!("failed to parse graph payload: {err}")) + } + })?; + serde_json::to_string(&path).map_err(|e| PyValueError::new_err(e.to_string())) +} + /// Parse a qualified semantic reference using Rust graph semantics. #[pyfunction] fn parse_reference_with_yaml( @@ -1316,6 +1343,59 @@ fn find_models_for_query_with_yaml( .map_err(|e| PyValueError::new_err(e.to_string())) } +fn parse_ossie_options( + serialization: &str, + consumer_profile: &str, +) -> PyResult<(OssieSerialization, OssieConsumerProfile)> { + let serialization = OssieSerialization::parse(serialization).map_err(PyValueError::new_err)?; + let consumer = OssieConsumerProfile::parse(consumer_profile).map_err(PyValueError::new_err)?; + Ok((serialization, consumer)) +} + +/// Validate an Apache Ossie document with the strict forward handoff contract. +#[pyfunction] +#[pyo3(signature = (content, serialization, consumer_profile = "ossie-core"))] +fn ossie_validate(content: &str, serialization: &str, consumer_profile: &str) -> PyResult { + let (serialization, consumer) = parse_ossie_options(serialization, consumer_profile)?; + serde_json::to_string(&OssieForwardAdapter.inspect(content, serialization, consumer)) + .map_err(|error| PyValueError::new_err(error.to_string())) +} + +/// Compile all isolated scopes in an Apache Ossie logical document. +#[pyfunction] +#[pyo3(signature = (content, serialization, consumer_profile = "ossie-core", target = "ANSI_SQL"))] +fn ossie_parse_catalog( + content: &str, + serialization: &str, + consumer_profile: &str, + target: &str, +) -> PyResult { + let (serialization, consumer) = parse_ossie_options(serialization, consumer_profile)?; + let target = OssieTarget::parse(target).map_err(PyValueError::new_err)?; + let catalog = OssieForwardAdapter + .parse_catalog(content, serialization, consumer, target) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + serde_json::to_string(&catalog).map_err(|error| PyValueError::new_err(error.to_string())) +} + +/// Select exactly one compiled Apache Ossie semantic-model scope. +#[pyfunction] +#[pyo3(signature = (content, serialization, scope_id = None, consumer_profile = "ossie-core", target = "ANSI_SQL"))] +fn ossie_select_scope( + content: &str, + serialization: &str, + scope_id: Option<&str>, + consumer_profile: &str, + target: &str, +) -> PyResult { + let (serialization, consumer) = parse_ossie_options(serialization, consumer_profile)?; + let target = OssieTarget::parse(target).map_err(PyValueError::new_err)?; + let scope = OssieForwardAdapter + .select_scope(content, serialization, consumer, target, scope_id) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + serde_json::to_string(&scope).map_err(|error| PyValueError::new_err(error.to_string())) +} + /// Python module entrypoint. #[pymodule] fn sidemantic_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { @@ -1408,9 +1488,16 @@ fn sidemantic_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(relationship_primary_key_columns, m)?)?; m.add_function(wrap_pyfunction!(segment_get_sql, m)?)?; m.add_function(wrap_pyfunction!(find_relationship_path_with_yaml, m)?)?; + m.add_function(wrap_pyfunction!( + find_relationship_path_payload_with_yaml, + m + )?)?; m.add_function(wrap_pyfunction!(parse_reference_with_yaml, m)?)?; m.add_function(wrap_pyfunction!(find_models_for_query, m)?)?; m.add_function(wrap_pyfunction!(find_models_for_query_with_yaml, m)?)?; m.add_function(wrap_pyfunction!(generate_catalog_metadata, m)?)?; + m.add_function(wrap_pyfunction!(ossie_validate, m)?)?; + m.add_function(wrap_pyfunction!(ossie_parse_catalog, m)?)?; + m.add_function(wrap_pyfunction!(ossie_select_scope, m)?)?; Ok(()) } diff --git a/sidemantic-rs/src/runtime.rs b/sidemantic-rs/src/runtime.rs index e0f23fe92..e010353d8 100644 --- a/sidemantic-rs/src/runtime.rs +++ b/sidemantic-rs/src/runtime.rs @@ -99,6 +99,18 @@ pub struct LoadedGraphPayload { /// Tuple shape returned to Python bridge for graph path steps. pub type RelationshipPathStep = (String, String, Vec, Vec, String); +/// Edge-aware relationship path payload. The legacy tuple API remains available. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RelationshipPathPayloadStep { + pub from_model: String, + pub to_model: String, + pub from_columns: Vec, + pub to_columns: Vec, + pub relationship: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub edge_id: Option, +} + /// Relationship path discovery errors that preserve Python-compatible exception semantics. #[derive(Debug, Clone, PartialEq, Eq, Error)] pub enum RelationshipPathError { @@ -140,6 +152,8 @@ struct GraphPathModelPayload { #[derive(Debug, Deserialize)] struct GraphPathRelationshipPayload { name: String, + #[serde(default)] + edge_id: Option, #[serde(default, rename = "type")] relationship_type: Option, #[serde(default)] @@ -5566,6 +5580,28 @@ pub fn find_relationship_path_with_yaml( from_model: &str, to_model: &str, ) -> std::result::Result, RelationshipPathError> { + Ok( + find_relationship_path_payload_with_yaml(graph_yaml, from_model, to_model)? + .into_iter() + .map(|step| { + ( + step.from_model, + step.to_model, + step.from_columns, + step.to_columns, + step.relationship, + ) + }) + .collect(), + ) +} + +/// Find an edge-aware join path while preserving the legacy five-field API above. +pub fn find_relationship_path_payload_with_yaml( + graph_yaml: &str, + from_model: &str, + to_model: &str, +) -> std::result::Result, RelationshipPathError> { if from_model == to_model { return Ok(Vec::new()); } @@ -5585,7 +5621,7 @@ fn relationship_path_with_runtime( runtime: &SidemanticRuntime, from_model: &str, to_model: &str, -) -> std::result::Result, RelationshipPathError> { +) -> std::result::Result, RelationshipPathError> { let join_path = runtime .find_join_path(from_model, to_model) .map_err(|err| match err { @@ -5602,14 +5638,13 @@ fn relationship_path_with_runtime( Ok(join_path .steps .into_iter() - .map(|step| { - ( - step.from_model, - step.to_model, - step.from_keys, - step.to_keys, - relationship_type_label(&step.relationship_type).to_string(), - ) + .map(|step| RelationshipPathPayloadStep { + from_model: step.from_model, + to_model: step.to_model, + from_columns: step.from_keys, + to_columns: step.to_keys, + relationship: relationship_type_label(&step.relationship_type).to_string(), + edge_id: step.edge_id, }) .collect()) } @@ -5627,7 +5662,7 @@ fn relationship_path_with_graph( graph: &SemanticGraph, from_model: &str, to_model: &str, -) -> std::result::Result, RelationshipPathError> { +) -> std::result::Result, RelationshipPathError> { let join_path = graph .find_join_path(from_model, to_model) .map_err(|err| match err { @@ -5644,14 +5679,13 @@ fn relationship_path_with_graph( Ok(join_path .steps .into_iter() - .map(|step| { - ( - step.from_model, - step.to_model, - step.from_keys, - step.to_keys, - relationship_type_label(&step.relationship_type).to_string(), - ) + .map(|step| RelationshipPathPayloadStep { + from_model: step.from_model, + to_model: step.to_model, + from_columns: step.from_keys, + to_columns: step.to_keys, + relationship: relationship_type_label(&step.relationship_type).to_string(), + edge_id: step.edge_id, }) .collect()) } @@ -5696,6 +5730,7 @@ fn semantic_graph_from_graph_path_payload( model.relationships.push(Relationship { name: relationship_payload.name.clone(), + edge_id: relationship_payload.edge_id.clone(), r#type: normalized_type, foreign_key: foreign_key_columns .as_ref() @@ -8316,6 +8351,35 @@ models: assert_eq!(path[0].4, "many_to_one"); } + #[test] + fn test_runtime_edge_aware_path_preserves_identity_without_changing_legacy_shape() { + let graph_yaml = r#" +models: + - name: customers + table: customers + primary_key: id + - name: orders + table: orders + primary_key: order_id + relationships: + - name: customers + edge_id: orders_customer + type: many_to_one + foreign_key: customer_id + primary_key: id +"#; + + let payload = + find_relationship_path_payload_with_yaml(graph_yaml, "orders", "customers").unwrap(); + assert_eq!(payload.len(), 1); + assert_eq!(payload[0].edge_id.as_deref(), Some("orders_customer")); + + let legacy = find_relationship_path_with_yaml(graph_yaml, "orders", "customers").unwrap(); + assert_eq!(legacy.len(), 1); + assert_eq!(legacy[0].0, "orders"); + assert_eq!(legacy[0].4, "many_to_one"); + } + #[test] fn test_runtime_find_relationship_path_with_yaml_falls_back_for_compact_payload_shape() { let graph_yaml = r#" diff --git a/sidemantic-rs/src/sql/generator.rs b/sidemantic-rs/src/sql/generator.rs index 61cb0fbb8..6b89c79b2 100644 --- a/sidemantic-rs/src/sql/generator.rs +++ b/sidemantic-rs/src/sql/generator.rs @@ -5441,6 +5441,7 @@ models: .with_metric(Metric::sum("revenue", "amount")) .with_relationship(Relationship { name: "products".to_string(), + edge_id: None, r#type: RelationshipType::ManyToMany, foreign_key: None, foreign_key_columns: None, diff --git a/sidemantic-rs/tests/ossie_forward.rs b/sidemantic-rs/tests/ossie_forward.rs new file mode 100644 index 000000000..889ad0369 --- /dev/null +++ b/sidemantic-rs/tests/ossie_forward.rs @@ -0,0 +1,66 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use serde::Deserialize; +use sidemantic::{OssieConsumerProfile, OssieForwardAdapter, OssieSerialization}; + +#[derive(Debug, Deserialize)] +struct Manifest { + cases: Vec, +} + +#[derive(Debug, Deserialize)] +struct FixtureCase { + id: String, + input: String, + serialization: String, + expected: Expected, +} + +#[derive(Debug, Deserialize)] +struct Expected { + valid: bool, + #[serde(default)] + diagnostics: Vec, +} + +#[derive(Debug, Deserialize)] +struct ExpectedDiagnostic { + code: String, + instance_path: String, +} + +fn fixture_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join("tests/ossie-fixtures") +} + +#[test] +fn shared_ossie_fixture_manifest_matches_strict_rust_diagnostics() { + let root = fixture_root(); + let manifest: Manifest = + serde_yaml::from_str(&fs::read_to_string(root.join("manifest.yaml")).unwrap()).unwrap(); + + for case in manifest.cases { + let serialization = OssieSerialization::parse(&case.serialization).unwrap(); + let content = fs::read_to_string(root.join(&case.input)).unwrap(); + let status = + OssieForwardAdapter.inspect(&content, serialization, OssieConsumerProfile::OssieCore); + + assert_eq!(status.valid, case.expected.valid, "fixture {}", case.id); + for expected in case.expected.diagnostics { + assert!( + status.diagnostics.iter().any(|actual| { + actual.code == expected.code && actual.instance_path == expected.instance_path + }), + "fixture {} missing diagnostic {} at {}; actual: {:?}", + case.id, + expected.code, + expected.instance_path, + status.diagnostics + ); + } + } +} diff --git a/sidemantic/rust_bridge.py b/sidemantic/rust_bridge.py index 022f18a48..b5e797c33 100644 --- a/sidemantic/rust_bridge.py +++ b/sidemantic/rust_bridge.py @@ -466,24 +466,27 @@ def models_to_rust_yaml( } for dimension in model.dimensions: - model_data["dimensions"].append( - { - "name": dimension.name, - "type": dimension.type, - "sql": dimension.sql, - "granularity": dimension.granularity, - "supported_granularities": dimension.supported_granularities, - "description": dimension.description, - "label": dimension.label, - "metadata": dimension.metadata, - "meta": dimension.meta, - "format": dimension.format, - "value_format_name": dimension.value_format_name, - "parent": dimension.parent, - "window": dimension.window, - "public": dimension.public, - } - ) + dimension_payload = { + "name": dimension.name, + "type": dimension.type, + "sql": dimension.sql, + "granularity": dimension.granularity, + "supported_granularities": dimension.supported_granularities, + "description": dimension.description, + "label": dimension.label, + "metadata": dimension.metadata, + "meta": dimension.meta, + "format": dimension.format, + "value_format_name": dimension.value_format_name, + "parent": dimension.parent, + "window": dimension.window, + "public": dimension.public, + } + if dimension.logical_data_type is not None: + dimension_payload["logical_data_type"] = dimension.logical_data_type + if dimension.declared_is_time is not None: + dimension_payload["declared_is_time"] = dimension.declared_is_time + model_data["dimensions"].append(dimension_payload) serialized_metric_names = set() for metric in [*model.metrics, *extra_metrics_by_model.get(model.name, [])]: @@ -720,18 +723,18 @@ def parse_reference_with_rust(graph: SemanticGraph, reference: str) -> tuple[str def find_relationship_path_with_rust(graph: SemanticGraph, from_model: str, to_model: str) -> list: """Find join path between models via sidemantic-rs.""" rust_module = get_rust_module() - rust_steps = rust_module.find_relationship_path_with_yaml( - graph_to_rust_yaml(graph), - from_model, - to_model, - ) + graph_yaml = graph_to_rust_yaml(graph) + if hasattr(rust_module, "find_relationship_path_payload_with_yaml"): + rust_steps = rust_module.find_relationship_path_payload_with_yaml(graph_yaml, from_model, to_model) + else: + rust_steps = rust_module.find_relationship_path_with_yaml(graph_yaml, from_model, to_model) rust_steps = _deserialize_json_payload(rust_steps) from sidemantic.core.semantic_graph import JoinPath path = [] for step in rust_steps: - from_name, to_name, from_columns, to_columns, relationship = _normalize_relationship_path_step(step) + from_name, to_name, from_columns, to_columns, relationship, edge_id = _normalize_relationship_path_step(step) path.append( JoinPath( from_model=str(from_name), @@ -739,6 +742,7 @@ def find_relationship_path_with_rust(graph: SemanticGraph, from_model: str, to_m from_columns=[str(column) for column in from_columns], to_columns=[str(column) for column in to_columns], relationship=str(relationship), + edge_id=str(edge_id) if edge_id is not None else None, ) ) return path @@ -762,13 +766,14 @@ def _normalize_parsed_reference(parsed: object) -> tuple[str, str, str | None]: return str(model_name), str(field_name), (str(granularity) if granularity is not None else None) -def _normalize_relationship_path_step(step: object) -> tuple[str, str, list[str], list[str], str]: +def _normalize_relationship_path_step(step: object) -> tuple[str, str, list[str], list[str], str, str | None]: if isinstance(step, dict): from_name = step.get("from_model") to_name = step.get("to_model") relationship = step.get("relationship") from_columns = step.get("from_columns") to_columns = step.get("to_columns") + edge_id = step.get("edge_id") # Older payloads may expose only single-column aliases. if from_columns is None: @@ -780,6 +785,7 @@ def _normalize_relationship_path_step(step: object) -> tuple[str, str, list[str] else: try: from_name, to_name, from_columns, to_columns, relationship = step + edge_id = None except (TypeError, ValueError) as exc: raise TypeError("unexpected find_relationship_path_with_yaml step shape") from exc @@ -788,7 +794,14 @@ def _normalize_relationship_path_step(step: object) -> tuple[str, str, list[str] normalized_from_columns = _normalize_relationship_columns(from_columns) normalized_to_columns = _normalize_relationship_columns(to_columns) - return str(from_name), str(to_name), normalized_from_columns, normalized_to_columns, str(relationship) + return ( + str(from_name), + str(to_name), + normalized_from_columns, + normalized_to_columns, + str(relationship), + str(edge_id) if edge_id is not None else None, + ) def _normalize_relationship_columns(columns: object) -> list[str]: @@ -905,7 +918,7 @@ def _serialize_metric(metric, *, primary_key_columns: list[str] | None) -> dict: casts = ", '|', ".join(f"CAST({col} AS VARCHAR)" for col in primary_key_columns) metric_sql = f"CONCAT({casts})" - return { + payload = { "name": metric.name, "extends": metric.extends, "type": metric.type, @@ -947,6 +960,9 @@ def _serialize_metric(metric, *, primary_key_columns: list[str] | None) -> dict: "meta": metric.meta, "public": metric.public, } + if metric.logical_data_type is not None: + payload["logical_data_type"] = metric.logical_data_type + return payload def _serialize_parameter(parameter) -> dict: @@ -1632,7 +1648,7 @@ def _serialize_relationship(relationship, source_model, target_model) -> dict | through_foreign_key = through_foreign_key or (junction_self_fks[0] if junction_self_fks else None) related_foreign_key = related_foreign_key or (junction_related_fks[0] if junction_related_fks else None) - return { + payload = { "name": relationship.name, "type": relationship.type, "foreign_key": foreign_keys[0] if foreign_keys else None, @@ -1647,3 +1663,6 @@ def _serialize_relationship(relationship, source_model, target_model) -> dict | "sql": sql, "metadata": getattr(relationship, "metadata", None), } + if relationship.edge_id is not None: + payload["edge_id"] = relationship.edge_id + return payload diff --git a/tests/adapters/osi/test_rust_ossie_forward_parity.py b/tests/adapters/osi/test_rust_ossie_forward_parity.py new file mode 100644 index 000000000..12105e7d5 --- /dev/null +++ b/tests/adapters/osi/test_rust_ossie_forward_parity.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from sidemantic.adapters.ossie import OssieAdapter +from sidemantic.interchange.ossie import validation +from tests.rust_layer_adapter import rust_ossie_select_scope, rust_ossie_validate + +FIXTURES = Path(__file__).resolve().parents[2] / "ossie-fixtures" + + +def test_rust_and_python_match_shared_fixture_outcomes() -> None: + manifest = yaml.safe_load((FIXTURES / "manifest.yaml").read_text()) + + for case in manifest["cases"]: + source = FIXTURES / case["input"] + expected = case["expected"] + content = source.read_text() + document = json.loads(content) if case["serialization"] == "json" else yaml.safe_load(content) + rust = rust_ossie_validate(content, case["serialization"]) + python = validation.validate_ossie_schema(document, profile=case["profile"]) + + assert rust["valid"] is expected["valid"], case["id"] + assert python.valid is expected["valid"], case["id"] + rust_diagnostics = {(item["code"], item["instance_path"]) for item in rust["diagnostics"]} + python_diagnostics = {(item.code, item.json_pointer) for item in python.diagnostics} + for diagnostic in expected["diagnostics"]: + key = (diagnostic["code"], diagnostic["instance_path"]) + assert key in rust_diagnostics, case["id"] + assert key in python_diagnostics, case["id"] + assert rust_diagnostics == python_diagnostics, case["id"] + + +def test_selected_scope_shape_preserves_parity_fields_and_target_selection(tmp_path: Path) -> None: + content = """version: 0.2.0.dev0 +semantic_model: + - name: commerce + datasets: + - name: orders + source: analytics.orders + fields: + - name: created_at + datatype: DateTime + dimension: {is_time: false} + expression: + dialects: + - {dialect: ANSI_SQL, expression: created_at} + - {dialect: BIGQUERY, expression: SAFE_CAST(created_at AS DATETIME)} + - {dialect: MDX, expression: "[Orders].[Created At]"} + - name: customer_id + datatype: Integer + expression: {dialects: [{dialect: ANSI_SQL, expression: customer_id}]} + - name: customers + source: analytics.customers + primary_key: [id] + fields: + - name: id + datatype: Integer + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} + relationships: + - name: orders_customer + from: orders + to: customers + from_columns: [customer_id] + to_columns: [id] + metrics: + - name: gross_amount + datatype: Decimal + expression: + dialects: + - {dialect: ANSI_SQL, expression: SUM(amount)} + - {dialect: BIGQUERY, expression: SUM(SAFE_CAST(amount AS NUMERIC))} + - name: operations + datasets: + - name: orders + source: operations.orders +""" + source = tmp_path / "scopes.yaml" + source.write_text(content) + + rust = rust_ossie_select_scope(content, "yaml", "commerce", target="BIGQUERY") + python = OssieAdapter(scope_id="commerce", target_dialect="bigquery").parse_document(source) + python_graph = python.catalog["commerce"].graph + + assert rust["scope_id"] == "commerce" + assert [model["name"] for model in rust["models"]] == list(python_graph.models) + assert rust["models"][0]["table"] == python_graph.get_model("orders").table + rust_dimension = rust["models"][0]["dimensions"][0] + python_dimension = python_graph.get_model("orders").dimensions[0] + assert rust_dimension["sql"] == python_dimension.sql == "SAFE_CAST(created_at AS DATETIME)" + assert rust_dimension["logical_data_type"] == python_dimension.logical_data_type == "DateTime" + assert rust_dimension["declared_is_time"] is python_dimension.declared_is_time is False + assert "declared_is_time" not in rust["models"][0]["dimensions"][1] + assert rust["metrics"][0]["logical_data_type"] == "Decimal" + assert rust["metrics"][0]["agg"] == "sum" + assert rust["metrics"][0]["sql"] == "SAFE_CAST(amount AS NUMERIC)" + assert rust["models"][0]["relationships"][0]["edge_id"] == "orders_customer" + assert python_graph.get_model("orders").relationships[0].edge_id == "orders_customer" + assert rust["models"][0]["primary_key"] == "" + assert rust["models"][0]["primary_key_columns"] == [] + + with pytest.raises(ValueError, match="ambiguous"): + rust_ossie_select_scope(content, "yaml", target="BIGQUERY") diff --git a/tests/core/test_rust_bridge_yaml_serialization.py b/tests/core/test_rust_bridge_yaml_serialization.py index 6e85f0c3e..ad35f7332 100644 --- a/tests/core/test_rust_bridge_yaml_serialization.py +++ b/tests/core/test_rust_bridge_yaml_serialization.py @@ -8,7 +8,8 @@ from sidemantic.core.pre_aggregation import Index, PreAggregation, RefreshKey from sidemantic.core.relationship import Relationship from sidemantic.core.semantic_graph import SemanticGraph -from sidemantic.rust_bridge import graph_to_rust_yaml, models_to_rust_yaml +from sidemantic.rust_bridge import find_relationship_path_with_rust, graph_to_rust_yaml, models_to_rust_yaml +from tests.rust_layer_adapter import _dimension_to_rust_dict, _metric_to_rust_dict, _relationship_to_rust_dict def test_models_to_rust_yaml_preserves_extended_core_metadata(): @@ -118,6 +119,105 @@ def test_models_to_rust_yaml_does_not_invent_table_for_source_uri_model(): assert model_payload["table"] is None +def test_models_to_rust_yaml_preserves_optional_parity_fields_and_omits_none(): + model = Model( + name="orders", + table="orders", + primary_key="order_id", + dimensions=[ + Dimension( + name="occurred_at", + type="categorical", + logical_data_type="DateTimeTz", + declared_is_time=False, + ), + Dimension(name="status", type="categorical"), + ], + metrics=[Metric(name="revenue", agg="sum", sql="amount", logical_data_type="Decimal")], + relationships=[ + Relationship( + name="customers", + edge_id="orders_customer", + type="many_to_one", + foreign_key="customer_id", + ) + ], + ) + + model_payload = yaml.safe_load(models_to_rust_yaml([model]))["models"][0] + occurred_at, status = model_payload["dimensions"] + assert occurred_at["logical_data_type"] == "DateTimeTz" + assert occurred_at["declared_is_time"] is False + assert "logical_data_type" not in status + assert "declared_is_time" not in status + assert model_payload["metrics"][0]["logical_data_type"] == "Decimal" + assert model_payload["relationships"][0]["edge_id"] == "orders_customer" + + +def test_pure_rust_adapter_serializers_preserve_false_and_omit_none(): + explicit_dimension = _dimension_to_rust_dict( + Dimension( + name="occurred_at", + type="categorical", + logical_data_type="DateTimeTz", + declared_is_time=False, + ) + ) + omitted_dimension = _dimension_to_rust_dict(Dimension(name="status", type="categorical")) + metric = _metric_to_rust_dict(Metric(name="revenue", agg="sum", logical_data_type="Decimal")) + relationship = _relationship_to_rust_dict( + Relationship(name="customers", edge_id="orders_customer", type="many_to_one") + ) + + assert explicit_dimension["declared_is_time"] is False + assert explicit_dimension["logical_data_type"] == "DateTimeTz" + assert "declared_is_time" not in omitted_dimension + assert "logical_data_type" not in omitted_dimension + assert metric["logical_data_type"] == "Decimal" + assert relationship["edge_id"] == "orders_customer" + + +def test_find_relationship_path_prefers_edge_aware_payload_and_preserves_legacy_fallback(monkeypatch): + graph = SemanticGraph() + graph.add_model( + Model( + name="orders", + table="orders", + primary_key="order_id", + relationships=[ + Relationship( + name="customers", + edge_id="orders_customer", + type="many_to_one", + foreign_key="customer_id", + ) + ], + ) + ) + graph.add_model(Model(name="customers", table="customers", primary_key="id")) + + class EdgeAwareRustModule: + @staticmethod + def find_relationship_path_payload_with_yaml(_yaml, _from_model, _to_model): + return ( + '[{"from_model":"orders","to_model":"customers","from_columns":["customer_id"],' + '"to_columns":["id"],"relationship":"many_to_one","edge_id":"orders_customer"}]' + ) + + monkeypatch.setattr("sidemantic.rust_bridge.get_rust_module", lambda: EdgeAwareRustModule()) + path = find_relationship_path_with_rust(graph, "orders", "customers") + assert path[0].edge_id == "orders_customer" + + class LegacyRustModule: + @staticmethod + def find_relationship_path_with_yaml(_yaml, _from_model, _to_model): + return [("orders", "customers", ["customer_id"], ["id"], "many_to_one")] + + monkeypatch.setattr("sidemantic.rust_bridge.get_rust_module", lambda: LegacyRustModule()) + legacy_path = find_relationship_path_with_rust(graph, "orders", "customers") + assert legacy_path[0].edge_id is None + + def test_graph_to_rust_yaml_assigns_complex_metrics_by_entity_dimension(): graph = SemanticGraph() graph.add_model( diff --git a/tests/rust_layer_adapter.py b/tests/rust_layer_adapter.py index 972e4b04e..0c509cd0a 100644 --- a/tests/rust_layer_adapter.py +++ b/tests/rust_layer_adapter.py @@ -269,6 +269,7 @@ def _rust_join_path(self, from_model: str, to_model: str) -> list[RustJoinPath]: from_columns=step["from_columns"], to_columns=step["to_columns"], relationship=step["relationship"], + edge_id=step.get("edge_id"), ) for step in response["path"] ] @@ -410,6 +411,7 @@ def find_relationship_path(self, from_model: str, to_model: str) -> list[RustJoi from_columns=step["from_columns"], to_columns=step["to_columns"], relationship=step["relationship"], + edge_id=step.get("edge_id"), ) for step in response["path"] ] @@ -428,6 +430,7 @@ class RustJoinPath: from_columns: list[str] to_columns: list[str] relationship: str + edge_id: str | None = None @property def from_entity(self) -> str: @@ -613,6 +616,46 @@ def rust_needs_symmetric_aggregate(relationship: str, is_base_model: bool) -> bo return bool(response["value"]) +def rust_ossie_validate( + content: str, + serialization: str, + consumer_profile: str = "ossie-core", +) -> dict[str, Any]: + response = _rust_request( + { + "action": "ossie_validate", + "content": content, + "serialization": serialization, + "consumer_profile": consumer_profile, + } + ) + if response["status"] == "error": + raise ValueError(response["error"]) + return response["value"] + + +def rust_ossie_select_scope( + content: str, + serialization: str, + scope_id: str | None = None, + consumer_profile: str = "ossie-core", + target: str = "ANSI_SQL", +) -> dict[str, Any]: + response = _rust_request( + { + "action": "ossie_select_scope", + "content": content, + "serialization": serialization, + "consumer_profile": consumer_profile, + "target": target, + "scope_id": scope_id, + } + ) + if response["status"] == "error": + raise ValueError(response["error"]) + return response["value"] + + def _model_to_rust_dict(model: Model) -> dict[str, Any]: return _drop_none( { @@ -639,7 +682,7 @@ def _model_to_rust_dict(model: Model) -> dict[str, Any]: def _dimension_to_rust_dict(dimension) -> dict[str, Any]: - return _drop_none( + payload = _drop_none( { "name": dimension.name, "type": dimension.type, @@ -657,10 +700,15 @@ def _dimension_to_rust_dict(dimension) -> dict[str, Any]: "public": dimension.public, } ) + if dimension.logical_data_type is not None: + payload["logical_data_type"] = dimension.logical_data_type + if dimension.declared_is_time is not None: + payload["declared_is_time"] = dimension.declared_is_time + return payload def _metric_to_rust_dict(metric: Metric) -> dict[str, Any]: - return _drop_none( + payload = _drop_none( { "name": metric.name, "type": metric.type, @@ -703,6 +751,9 @@ def _metric_to_rust_dict(metric: Metric) -> dict[str, Any]: "public": metric.public, } ) + if metric.logical_data_type is not None: + payload["logical_data_type"] = metric.logical_data_type + return payload def _relationship_to_rust_dict(relationship) -> dict[str, Any]: @@ -717,6 +768,8 @@ def _relationship_to_rust_dict(relationship) -> dict[str, Any]: "sql": getattr(relationship, "sql", None), "metadata": relationship.metadata, } + if relationship.edge_id is not None: + payload["edge_id"] = relationship.edge_id if isinstance(relationship.foreign_key, list): payload["foreign_key_columns"] = relationship.foreign_key else: @@ -860,7 +913,11 @@ def _ensure_rust_adapter_binary() -> None: text=True, capture_output=True, cwd=ROOT, - env={**os.environ, "CARGO_TARGET_DIR": str(RUST_TARGET_DIR)}, + env={ + **os.environ, + "CARGO_BUILD_RUSTC_WRAPPER": "", + "CARGO_TARGET_DIR": str(RUST_TARGET_DIR), + }, check=False, ) if result.returncode != 0: