Fix/entity name validation v2

ingestion/src/metadata/ingestion/models/custom_basemodel_validation.py

@@ -111,6 +111,8 @@ def _initialize_transformable_entities():
 
 
 def revert_separators(value):
+    if not isinstance(value, str):  # Add this check
+        return value
     return (
         value.replace(RESERVED_COLON_KEYWORD, "::")
         .replace(RESERVED_ARROW_KEYWORD, ">")
@@ -119,6 +121,8 @@ def revert_separators(value):
 
 
 def replace_separators(value):
+    if not isinstance(value, str):
+        return value
     return (
         value.replace("::", RESERVED_COLON_KEYWORD)
         .replace(">", RESERVED_ARROW_KEYWORD)

ingestion/src/metadata/ingestion/source/database/athena/metadata.py

@@ -27,6 +27,7 @@
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
 from metadata.generated.schema.entity.data.table import (
     Column,
+    ColumnName,
     PartitionColumnDetails,
     PartitionIntervalTypes,
     Table,
@@ -202,7 +203,7 @@ def get_table_partition_details(
             partition_details = TablePartition(
                 columns=[
                     PartitionColumnDetails(
-                        columnName=col["name"],
+                        columnName=ColumnName(root=col["name"]),
                         intervalType=ATHENA_INTERVAL_TYPE_MAP.get(
                             col.get("projection_type", str(col["type"])),
                             PartitionIntervalTypes.COLUMN_VALUE,

ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py

@@ -36,6 +36,7 @@
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
 from metadata.generated.schema.entity.data.storedProcedure import StoredProcedureCode
 from metadata.generated.schema.entity.data.table import (
+    ColumnName,
     ConstraintType,
     PartitionColumnDetails,
     PartitionIntervalTypes,
@@ -203,9 +204,7 @@ def _build_formatted_table_id(table):
     return f"{table.table_id}"
 
 
-BigQueryDialect._build_formatted_table_id = (  # pylint: disable=protected-access
-    _build_formatted_table_id
-)
+BigQueryDialect._build_formatted_table_id = _build_formatted_table_id  # pylint: disable=protected-access
 BigQueryDialect.get_pk_constraint = get_pk_constraint
 BigQueryDialect.get_foreign_keys = get_foreign_keys
 
@@ -974,9 +973,11 @@ def get_table_partition_details(
                 return True, TablePartition(
                     columns=[
                         PartitionColumnDetails(
-                            columnName=self._get_partition_column_name(
-                                columns=columns,
-                                partition_field_name=field,
+                            columnName=ColumnName(
+                                root=self._get_partition_column_name(
+                                    columns=columns,
+                                    partition_field_name=field,
+                                )
                             ),
                             interval=str(partition_details._properties.get("mode")),
                             intervalType=PartitionIntervalTypes.OTHER,
@@ -990,9 +991,11 @@ def get_table_partition_details(
                     table_partition = TablePartition(
                         columns=[
                             PartitionColumnDetails(
-                                columnName=self._get_partition_column_name(
-                                    columns=columns,
-                                    partition_field_name=table.time_partitioning.field,
+                                columnName=ColumnName(
+                                    root=self._get_partition_column_name(
+                                        columns=columns,
+                                        partition_field_name=table.time_partitioning.field,
+                                    )
                                 ),
                                 interval=str(table.time_partitioning.type_),
                                 intervalType=PartitionIntervalTypes.TIME_UNIT,
@@ -1003,8 +1006,8 @@ def get_table_partition_details(
                 return True, TablePartition(
                     columns=[
                         PartitionColumnDetails(
-                            columnName=(
-                                "_PARTITIONTIME" if table.time_partitioning.type_ == "HOUR" else "_PARTITIONDATE"
+                            columnName=ColumnName(
+                                root=("_PARTITIONTIME" if table.time_partitioning.type_ == "HOUR" else "_PARTITIONDATE")
                             ),
                             interval=str(table.time_partitioning.type_),
                             intervalType=PartitionIntervalTypes.INGESTION_TIME,
@@ -1013,16 +1016,18 @@ def get_table_partition_details(
                 )
             if table.range_partitioning:
                 table_partition = PartitionColumnDetails(
-                    columnName=self._get_partition_column_name(
-                        columns=columns,
-                        partition_field_name=table.range_partitioning.field,
+                    columnName=ColumnName(
+                        root=self._get_partition_column_name(
+                            columns=columns,
+                            partition_field_name=table.range_partitioning.field,
+                        )
                     ),
                     intervalType=PartitionIntervalTypes.INTEGER_RANGE,
                     interval=None,
                 )
                 if hasattr(table.range_partitioning, "range_") and hasattr(table.range_partitioning.range_, "interval"):
                     table_partition.interval = table.range_partitioning.range_.interval
-                table_partition.columnName = table.range_partitioning.field
+                table_partition.columnName = ColumnName(root=table.range_partitioning.field)
                 return True, TablePartition(columns=[table_partition])
             if (
                 hasattr(table, "_properties")
@@ -1032,9 +1037,11 @@ def get_table_partition_details(
                 return True, TablePartition(
                     columns=[
                         PartitionColumnDetails(
-                            columnName=self._get_partition_column_name(
-                                columns=columns,
-                                partition_field_name=field.get("field"),
+                            columnName=ColumnName(
+                                root=self._get_partition_column_name(
+                                    columns=columns,
+                                    partition_field_name=field.get("field"),
+                                )
                             ),
                             intervalType=PartitionIntervalTypes.OTHER,
                         )

ingestion/src/metadata/ingestion/source/database/burstiq/metadata.py

@@ -27,6 +27,7 @@
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
 from metadata.generated.schema.entity.data.table import (
     Column,
+    ColumnName,
     Constraint,
     ConstraintType,
     Table,
@@ -356,7 +357,7 @@
         # Regular types - no array element type
         return (type_mapping.get(burstiq_type, "VARCHAR"), None)
 
     def get_table_constraints(self, dictionary: BurstIQDictionary) -> Optional[List[TableConstraint]]:  # noqa: UP006, UP045
         """
         Get all table constraints (primary key, unique, and foreign key) from BurstIQ dictionary
 
@@ -372,7 +373,7 @@
             table_constraints.append(
                 TableConstraint(
                     constraintType=ConstraintType.PRIMARY_KEY,
-                    columns=dictionary.get_primary_key_columns(),
+                    columns=[ColumnName(root=column) for column in dictionary.get_primary_key_columns()],
                 )
             )
 
@@ -381,7 +382,7 @@
                 table_constraints.append(  # noqa: PERF401
                     TableConstraint(
                         constraintType=ConstraintType.UNIQUE,
-                        columns=index.attributes,
+                        columns=[ColumnName(root=attribute) for attribute in index.attributes],
                     )
                 )
 
@@ -407,7 +408,7 @@
                         table_constraints.append(
                             TableConstraint(
                                 constraintType=ConstraintType.FOREIGN_KEY,
-                                columns=[attribute.name],
+                                columns=[ColumnName(root=attribute.name)],
                                 referredColumns=[FullyQualifiedEntityName(col_fqn)],
                             )
                         )

ingestion/src/metadata/ingestion/source/database/clickhouse/metadata.py

@@ -79,18 +79,14 @@ def execute(self, query, params=None):
 
 ClickHouseDialect.get_unique_constraints = get_unique_constraints
 ClickHouseDialect.get_pk_constraint = get_pk_constraint
-ClickHouseDialect._get_column_type = (  # pylint: disable=protected-access
-    _get_column_type
-)
+ClickHouseDialect._get_column_type = _get_column_type  # pylint: disable=protected-access
 RequestsTransport.execute = execute
 ClickHouseDialect.get_view_definition = get_view_definition
 ClickHouseDialect.get_view_names = get_view_names
 ClickHouseDialect.get_table_comment = get_table_comment
 ClickHouseDialect.get_all_view_definitions = get_all_view_definitions
 ClickHouseDialect.get_all_table_comments = get_all_table_comments
-ClickHouseDialect._get_column_info = (  # pylint: disable=protected-access
-    _get_column_info
-)
+ClickHouseDialect._get_column_info = _get_column_info  # pylint: disable=protected-access
 Inspector.get_mview_names = get_mview_names
 ClickHouseDialect.get_mview_names = get_mview_names_dialect
 Inspector.get_all_table_ddls = get_all_table_ddls

ingestion/src/metadata/ingestion/source/database/database_service.py

@@ -37,6 +37,7 @@
 from metadata.generated.schema.entity.data.storedProcedure import StoredProcedure
 from metadata.generated.schema.entity.data.table import (
     Column,
+    ColumnName,
     DataModel,
     Table,
     TableConstraint,
@@ -341,7 +342,11 @@ def normalize_table_constraints(
                 column_name_map[col_name.lower()] = col_name
         for constraint in table_constraints:
             if constraint.columns:
-                constraint.columns = [column_name_map.get(c.lower(), c) for c in constraint.columns]
+                normalized_columns = []
+                for column in constraint.columns:
+                    column_name = model_str(column)
+                    normalized_columns.append(ColumnName(root=column_name_map.get(column_name.lower(), column_name)))
+                constraint.columns = normalized_columns
         return table_constraints
 
     def update_table_constraints(

ingestion/src/metadata/ingestion/source/database/redshift/metadata.py

@@ -128,9 +128,7 @@
 RedshiftDialect.get_view_definition = get_view_definition
 RedshiftDialect.get_temp_table_names = get_temp_table_names
 RedshiftDialect._get_redshift_columns = get_redshift_columns
-RedshiftDialect._get_all_relation_info = (  # pylint: disable=protected-access
-    _get_all_relation_info
-)
+RedshiftDialect._get_all_relation_info = _get_all_relation_info  # pylint: disable=protected-access
 Inspector.get_all_table_ddls = get_all_table_ddls
 Inspector.get_table_ddl = get_table_ddl
 

ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py

@@ -37,6 +37,7 @@
     StoredProcedureType,
 )
 from metadata.generated.schema.entity.data.table import (
+    ColumnName,
     PartitionColumnDetails,
     PartitionIntervalTypes,
     Table,
@@ -166,7 +167,7 @@
 logger = ingestion_logger()
 
 # pylint: disable=protected-access
-SnowflakeDialect._json_deserializer = json.loads
+SnowflakeDialect._json_deserializer = json.loads  # type: ignore[reportAttributeAccessIssue]
 SnowflakeDialect.get_table_names = get_table_names
 SnowflakeDialect.get_view_names = get_view_names
 SnowflakeDialect.get_stream_names = get_stream_names
@@ -502,7 +503,7 @@
             partition_details = TablePartition(
                 columns=[
                     PartitionColumnDetails(
-                        columnName=column,
+                        columnName=ColumnName(root=column),
                         intervalType=PartitionIntervalTypes.COLUMN_VALUE,
                         interval=None,
                     )
@@ -514,7 +515,7 @@
             return True, partition_details
         return False, None
 
     def yield_tag(self, schema_name: str) -> Iterable[Either[OMetaTagAndClassification]]:
         """
         Yield tags for tables/columns and schemas.
         """
@@ -1049,7 +1050,7 @@
                 return True
         return False
 
     def get_schema_tag_labels(self, schema_name: str) -> Optional[List[TagLabel]]:  # noqa: UP006, UP045
         """
         Return tags for schema entity including:
         1. Snowflake schema-level tags
@@ -1088,7 +1089,7 @@
 
         return schema_tags if schema_tags else None
 
     def get_tag_labels(self, table_name: str) -> Optional[List[TagLabel]]:  # noqa: UP006, UP045
         """
         Override to include inherited tags from both schema and database levels.
         This method combines:

ingestion/src/metadata/ingestion/source/database/sql_column_handler.py

@@ -27,6 +27,7 @@
     TableConstraint,
     TableType,
 )
+from metadata.ingestion.ometa.utils import model_str
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
 from metadata.ingestion.source.database.json_schema_extractor import (
     infer_json_schema_from_sample,
@@ -89,7 +90,7 @@
             return []
         if not table_columns:
             return []
-        column_names_lower = {col.name.root.lower() for col in table_columns}
+        column_names_lower = {model_str(col.name).lower() for col in table_columns}
         valid = []
         for constraint in table_constraints:
             if constraint is None:
@@ -99,13 +100,13 @@
                     "Filtering out table constraint %s: missing or empty columns",
                     constraint.constraintType.name,
                 )
-            elif all(c.lower() in column_names_lower for c in constraint.columns):
+            elif all(model_str(c).lower() in column_names_lower for c in constraint.columns):
                 valid.append(constraint)
             else:
                 logger.warning(
                     "Filtering out table constraint %s: references columns %s not found in processed columns",
                     constraint.constraintType.name,
-                    [c for c in constraint.columns if c.lower() not in column_names_lower],
+                    [c for c in constraint.columns if model_str(c).lower() not in column_names_lower],
                 )
         return valid
 
@@ -132,7 +133,7 @@
             data_type_display = f"array<{arr_data_type}>"
         return data_type_display
 
     def _process_col_type(self, column: dict, schema: str) -> Tuple:  # noqa: UP006
         data_type_display = None
         arr_data_type = None
         parsed_string = None
@@ -149,8 +150,8 @@
             # if SQLAlchemy does not provide any further information
             if col_type == "ARRAY" and getattr(column["type"], "item_type", None):
                 arr_data_type = ColumnTypeParser.get_column_type(column["type"].item_type)
             if col_type == "ARRAY" and re.match(r"(?:\w*)(?:\()(\w*)(?:.*)", str(column["type"])):
                 arr_data_type = re.match(r"(?:\w*)(?:[(]*)(\w*)(?:.*)", str(column["type"])).groups()
                 if isinstance(arr_data_type, (list, tuple)):
                     arr_data_type = ColumnTypeParser.get_column_type(arr_data_type[0])
                 data_type_display = column["type"]

ingestion/src/metadata/ingestion/source/database/timescale/metadata.py

@@ -20,6 +20,7 @@
 from sqlalchemy.engine import Inspector
 
 from metadata.generated.schema.entity.data.table import (
+    ColumnName,
     PartitionColumnDetails,
     PartitionIntervalTypes,
     Table,
@@ -187,7 +188,7 @@ def _build_hypertable_partition(self, hypertable: HypertableInfo) -> Optional[Ta
         return TablePartition(
             columns=[
                 PartitionColumnDetails(
-                    columnName=hypertable.column_name,
+                    columnName=ColumnName(root=hypertable.column_name),
                     intervalType=interval_type,
                     interval=interval,
                 )

ingestion/src/metadata/sampler/partition.py

@@ -25,6 +25,7 @@
 from metadata.generated.schema.entity.services.databaseService import (
     DatabaseServiceType,
 )
+from metadata.ingestion.ometa.utils import model_str
 from metadata.sampler.models import TableConfig
 from metadata.utils.logger import sampler_logger
 
@@ -57,10 +58,10 @@
     for column_partition in column_partitions:
         if column_partition.intervalType == PartitionIntervalTypes.INJECTED:
             if table_profiler_config is None or profiler_partitioning_config is None:
-                raise RuntimeError(error_msg.format(column_name=column_partition.columnName))
+                raise RuntimeError(error_msg.format(column_name=model_str(column_partition.columnName)))
 
-            if profiler_partitioning_config.partitionColumnName != column_partition.columnName:
-                raise RuntimeError(error_msg.format(column_name=column_partition.columnName))
+            if profiler_partitioning_config.partitionColumnName != model_str(column_partition.columnName):
+                raise RuntimeError(error_msg.format(column_name=model_str(column_partition.columnName)))
 
 
 def get_partition_details(
@@ -104,7 +105,7 @@
     return None
 
 
 def _handle_bigquery_partition(entity: Table, table_partition: TablePartition) -> Optional[PartitionProfilerConfig]:  # noqa: UP045
     """Bigquery specific logic for partitions"""
     if table_partition:
         column_partitions: Optional[List[PartitionColumnDetails]] = entity.tablePartition.columns  # noqa: UP006, UP045
@@ -116,7 +117,7 @@
         if partition.intervalType == PartitionIntervalTypes.TIME_UNIT:
             return PartitionProfilerConfig(
                 enablePartitioning=True,
-                partitionColumnName=partition.columnName,
+                partitionColumnName=model_str(partition.columnName),
                 partitionIntervalUnit=PartitionIntervalUnit.DAY if partition.interval != "HOUR" else partition.interval,
                 partitionInterval=1,
                 partitionIntervalType=partition.intervalType.value,
@@ -138,7 +139,7 @@
         if partition.intervalType == PartitionIntervalTypes.INTEGER_RANGE:
             return PartitionProfilerConfig(
                 enablePartitioning=True,
-                partitionColumnName=partition.columnName,
+                partitionColumnName=model_str(partition.columnName),
                 partitionIntervalUnit=None,
                 partitionInterval=None,
                 partitionIntervalType=partition.intervalType.value,

ingestion/src/metadata/sampler/sqlalchemy/trino/sampler.py

@@ -30,7 +30,7 @@ def __init__(self, *args, **kwargs):
         # pylint: disable=import-outside-toplevel
         from trino.sqlalchemy.dialect import TrinoDialect  # noqa: PLC0415
 
-        TrinoDialect._json_deserializer = None
+        TrinoDialect._json_deserializer = None  # type: ignore[reportAttributeAccessIssue]
 
         super().__init__(*args, **kwargs)
 

openmetadata-integration-tests/src/test/java/org/openmetadata/it/bootstrap/TestSuiteBootstrap.java

@@ -28,8 +28,11 @@
 import jakarta.validation.Validator;
 import java.io.IOException;
 import java.lang.reflect.Field;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.Map;
+import java.util.TimeZone;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.hc.client5.http.auth.AuthScope;
 import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
@@ -162,6 +165,9 @@ public void launcherSessionOpened(LauncherSession session) {
     cacheProvider = System.getProperty("cacheProvider", "none");
 
     LOG.info("=== TestSuiteBootstrap: Starting test infrastructure ===");
+    System.setProperty("user.timezone", "UTC");
+    TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
+    LOG.info("Test JVM timezone set to {}", TimeZone.getDefault().getID());
     LOG.info("Database type: {}", databaseType);
     LOG.info("Search type: {}", searchType);
     LOG.info("RDF enabled: {}", rdfEnabled);
@@ -505,8 +511,10 @@ private void startApplication() throws Exception {
     config.setDataSourceFactory(dataSourceFactory);
 
     String projectRoot = System.getProperty("user.dir");
-    if (projectRoot.endsWith("openmetadata-integration-tests")) {
-      projectRoot = projectRoot.substring(0, projectRoot.lastIndexOf("/"));
+    Path projectRootPath = Paths.get(projectRoot);
+    if (projectRootPath.endsWith("openmetadata-integration-tests")
+        && projectRootPath.getParent() != null) {
+      projectRoot = projectRootPath.getParent().toString();
     }
     String flyWayMigrationScriptsLocation =
         projectRoot + "/bootstrap/sql/migrations/flyway/" + DATABASE_CONTAINER.getDriverClassName();