From 7635005220b525e0fcde253041a75d4f07a042d3 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 9 Jul 2026 14:21:12 +0530 Subject: [PATCH 01/22] feat(transformations): add OneLake external data source data classes Add read/write/list/usability data classes for Fabric OneLake external data sources, public re-exports, IAM capability helper, and unit tests for serialization round-trips. --- cognite/client/data_classes/__init__.py | 28 +- cognite/client/data_classes/capabilities.py | 16 + .../transformations/external_data.py | 509 ++++++++++++++++++ .../test_transformations/__init__.py | 0 .../test_external_data.py | 111 ++++ 5 files changed, 660 insertions(+), 4 deletions(-) create mode 100644 cognite/client/data_classes/transformations/external_data.py create mode 100644 tests/tests_unit/test_data_classes/test_transformations/__init__.py create mode 100644 tests/tests_unit/test_data_classes/test_transformations/test_external_data.py diff --git a/cognite/client/data_classes/__init__.py b/cognite/client/data_classes/__init__.py index fb0d6ea238..8a722ff9f0 100644 --- a/cognite/client/data_classes/__init__.py +++ b/cognite/client/data_classes/__init__.py @@ -29,6 +29,7 @@ EntityMatchingPredictionResult, JobStatus, ) +from cognite.client.data_classes.data_modeling.records import RecordId from cognite.client.data_classes.data_sets import ( DataSet, DataSetFilter, @@ -161,7 +162,6 @@ LabelFilter, ) from cognite.client.data_classes.limits import Limit, LimitList -from cognite.client.data_classes.metering import MeteringData, MeteringDataList, MeteringDataPoint from cognite.client.data_classes.raw import ( Database, DatabaseList, @@ -251,6 +251,18 @@ TransformationBlockedInfo, TransformationDestination, ) +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSource, + ExternalDataSourceList, + ExternalDataSourceUsability, + ExternalDataSourceWrite, + ExternalDataSourceWriteList, + OneLakeCredentialsRead, + OneLakeCredentialsWrite, + OneLakeDataSourceSettingsRead, + OneLakeDataSourceSettingsWrite, + OneLakeLocationDescription, +) from cognite.client.data_classes.transformations.jobs import ( TransformationJob, TransformationJobFilter, @@ -384,6 +396,11 @@ "EventUpdate", "EventWrite", "EventWriteList", + "ExternalDataSource", + "ExternalDataSourceList", + "ExternalDataSourceUsability", + "ExternalDataSourceWrite", + "ExternalDataSourceWriteList", "ExtractionPipeline", "ExtractionPipelineConfig", "ExtractionPipelineConfigRevision", @@ -455,11 +472,14 @@ "LatestDatapointQuery", "Limit", "LimitList", - "MeteringData", - "MeteringDataList", - "MeteringDataPoint", "OidcCredentials", + "OneLakeCredentialsRead", + "OneLakeCredentialsWrite", + "OneLakeDataSourceSettingsRead", + "OneLakeDataSourceSettingsWrite", + "OneLakeLocationDescription", "RawTable", + "RecordId", "Relationship", "RelationshipFilter", "RelationshipList", diff --git a/cognite/client/data_classes/capabilities.py b/cognite/client/data_classes/capabilities.py index 35536f3a19..529a89a1aa 100644 --- a/cognite/client/data_classes/capabilities.py +++ b/cognite/client/data_classes/capabilities.py @@ -1104,6 +1104,22 @@ class Scope: DataSet = DataSetScope +@dataclass +class TransformationsExternalDataSourcesAcl(Capability): + _capability_name = "transformationsExternalDataSourcesAcl" + actions: Sequence[Action] + scope: AllScope | DataSetScope + + class Action(Capability.Action): # type: ignore [misc] + Read = "READ" + Write = "WRITE" + Use = "USE" + + class Scope: + All = AllScope + DataSet = DataSetScope + + @dataclass class TypesAcl(Capability): _capability_name = "typesAcl" diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py new file mode 100644 index 0000000000..c70997ffd5 --- /dev/null +++ b/cognite/client/data_classes/transformations/external_data.py @@ -0,0 +1,509 @@ +from __future__ import annotations + +import warnings +from abc import ABC +from typing import Any, ClassVar + +from typing_extensions import Self + +from cognite.client.data_classes._base import ( + CogniteResource, + CogniteResourceList, + ExternalIDTransformerMixin, + WriteableCogniteResource, + WriteableCogniteResourceList, +) + +__all__ = [ + "ExternalDataSource", + "ExternalDataSourceCore", + "ExternalDataSourceList", + "ExternalDataSourceUsability", + "ExternalDataSourceWrite", + "ExternalDataSourceWriteList", + "OneLakeCredentialsRead", + "OneLakeCredentialsWrite", + "OneLakeDataSourceSettingsRead", + "OneLakeDataSourceSettingsWrite", + "OneLakeLocationDescription", +] + + +class OneLakeLocationDescription(CogniteResource): + """Location of a Fabric OneLake lakehouse. + + Args: + workspace_name (str): Fabric workspace GUID or name. + container_name (str): Fabric lakehouse GUID or name. + """ + + def __init__(self, workspace_name: str, container_name: str) -> None: + self.workspace_name = workspace_name + self.container_name = container_name + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + workspace_name=resource["workspaceName"], + container_name=resource["containerName"], + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + if camel_case: + return {"workspaceName": self.workspace_name, "containerName": self.container_name} + return {"workspace_name": self.workspace_name, "container_name": self.container_name} + + +class OneLakeCredentialsRead(CogniteResource): + """Read-only view of Azure credentials for Fabric OneLake (clientSecret is never returned by the API). + + Args: + client_id (str): Azure application (client) ID. + tenant_id (str): Azure tenant (directory) ID. + """ + + def __init__(self, client_id: str, tenant_id: str) -> None: + self.client_id = client_id + self.tenant_id = tenant_id + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + client_id=resource["clientId"], + tenant_id=resource["tenantId"], + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + if camel_case: + return {"clientId": self.client_id, "tenantId": self.tenant_id} + return {"client_id": self.client_id, "tenant_id": self.tenant_id} + + +class OneLakeCredentialsWrite(CogniteResource): + """Azure credentials for writing to Fabric OneLake. + + Args: + client_id (str): Azure application (client) ID. + tenant_id (str): Azure tenant (directory) ID. + client_secret (str | None): Azure client secret. Required for upsert; None when reconstructed + from a read model via as_write() since the API never returns the secret. + """ + + def __init__(self, client_id: str, tenant_id: str, client_secret: str | None = None) -> None: + self.client_id = client_id + self.tenant_id = tenant_id + self.client_secret = client_secret + + def __repr__(self) -> str: + secret_display = "***" if self.client_secret is not None else None + return ( + f"OneLakeCredentialsWrite(client_id={self.client_id!r}, tenant_id={self.tenant_id!r}," + f" client_secret={secret_display!r})" + ) + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + client_id=resource["clientId"], + tenant_id=resource["tenantId"], + client_secret=resource.get("clientSecret"), + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result: dict[str, Any] + if camel_case: + result = {"clientId": self.client_id, "tenantId": self.tenant_id} + if self.client_secret is not None: + result["clientSecret"] = self.client_secret + else: + result = {"client_id": self.client_id, "tenant_id": self.tenant_id} + if self.client_secret is not None: + result["client_secret"] = self.client_secret + return result + + +class OneLakeDataSourceSettingsRead(CogniteResource): + """Settings for a Fabric OneLake external data source (read model — no client secret). + + Args: + credentials (OneLakeCredentialsRead | None): Azure credentials (client ID and tenant ID only). + location_description (OneLakeLocationDescription | None): Fabric workspace and lakehouse identifiers. + """ + + def __init__( + self, + credentials: OneLakeCredentialsRead | None = None, + location_description: OneLakeLocationDescription | None = None, + ) -> None: + self.credentials = credentials + self.location_description = location_description + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + credentials = None + if (creds_raw := resource.get("credentials")) is not None: + credentials = OneLakeCredentialsRead._load(creds_raw) + location_description = None + if (loc_raw := resource.get("locationDescription")) is not None: + location_description = OneLakeLocationDescription._load(loc_raw) + return cls(credentials=credentials, location_description=location_description) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result: dict[str, Any] = {} + if self.credentials is not None: + result["credentials"] = self.credentials.dump(camel_case=camel_case) + if self.location_description is not None: + key = "locationDescription" if camel_case else "location_description" + result[key] = self.location_description.dump(camel_case=camel_case) + return result + + +class OneLakeDataSourceSettingsWrite(CogniteResource): + """Settings for writing a Fabric OneLake external data source (includes client secret). + + Args: + credentials (OneLakeCredentialsWrite | None): Azure credentials including client secret. + location_description (OneLakeLocationDescription | None): Fabric workspace and lakehouse identifiers. + """ + + def __init__( + self, + credentials: OneLakeCredentialsWrite | None = None, + location_description: OneLakeLocationDescription | None = None, + ) -> None: + self.credentials = credentials + self.location_description = location_description + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + credentials = None + if (creds_raw := resource.get("credentials")) is not None: + credentials = OneLakeCredentialsWrite._load(creds_raw) + location_description = None + if (loc_raw := resource.get("locationDescription")) is not None: + location_description = OneLakeLocationDescription._load(loc_raw) + return cls(credentials=credentials, location_description=location_description) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result: dict[str, Any] = {} + if self.credentials is not None: + result["credentials"] = self.credentials.dump(camel_case=camel_case) + if self.location_description is not None: + key = "locationDescription" if camel_case else "location_description" + result[key] = self.location_description.dump(camel_case=camel_case) + return result + + +class ExternalDataSourceCore(WriteableCogniteResource["ExternalDataSourceWrite"], ABC): + """Shared base for ExternalDataSource (read) and ExternalDataSourceWrite (write). + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + external_id (str): External ID of the data source. Must be unique within the project. + name (str | None): Human-readable name for the data source. + data_set_id (int | None): ID of the data set that owns this resource (for ACL scoping). + """ + + _FORMAT: ClassVar[str] = "one_lake" + + def __init__( + self, + external_id: str, + name: str | None = None, + data_set_id: int | None = None, + ) -> None: + self.external_id = external_id + self.name = name + self.data_set_id = data_set_id + + +class ExternalDataSource(ExternalDataSourceCore): + """A Fabric OneLake external data source (read model — returned by list). + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + The ``clientSecret`` field is **never** returned by the API. + + Args: + external_id (str): External ID of the data source. + name (str | None): Human-readable name. + data_set_id (int | None): Data set ID for ACL scoping. + settings (OneLakeDataSourceSettingsRead | None): Connection settings (no client secret). + format (str | None): Backend format identifier (always ``"one_lake"`` for OneLake sources). + created_time (int | None): Time the resource was created (milliseconds since epoch). + last_updated_time (int | None): Time the resource was last updated (milliseconds since epoch). + """ + + def __init__( + self, + external_id: str, + name: str | None = None, + data_set_id: int | None = None, + settings: OneLakeDataSourceSettingsRead | None = None, + format: str | None = None, + created_time: int | None = None, + last_updated_time: int | None = None, + ) -> None: + super().__init__(external_id=external_id, name=name, data_set_id=data_set_id) + self.settings = settings + self.format = format + self.created_time = created_time + self.last_updated_time = last_updated_time + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + fmt = resource.get("format") + if fmt is not None and fmt != cls._FORMAT: + warnings.warn( + f"Unknown external data source format: {fmt!r}. This version of the SDK may not fully support it.", + UserWarning, + stacklevel=2, + ) + settings = None + if (settings_raw := resource.get("settings")) is not None: + settings = OneLakeDataSourceSettingsRead._load(settings_raw) + return cls( + external_id=resource["externalId"], + name=resource.get("name"), + data_set_id=resource.get("dataSetId"), + settings=settings, + format=fmt, + created_time=resource.get("createdTime"), + last_updated_time=resource.get("lastUpdatedTime"), + ) + + def as_write(self) -> ExternalDataSourceWrite: + """Return this source as an ExternalDataSourceWrite. + + Note: The ``client_secret`` cannot be reconstructed from the read model (the API never returns it). + The returned write object will have ``client_secret=None`` on its credentials. + """ + settings_write: OneLakeDataSourceSettingsWrite | None = None + if self.settings is not None: + creds_write: OneLakeCredentialsWrite | None = None + if self.settings.credentials is not None: + creds_write = OneLakeCredentialsWrite( + client_id=self.settings.credentials.client_id, + tenant_id=self.settings.credentials.tenant_id, + client_secret=None, + ) + settings_write = OneLakeDataSourceSettingsWrite( + credentials=creds_write, + location_description=self.settings.location_description, + ) + return ExternalDataSourceWrite( + external_id=self.external_id, + name=self.name, + data_set_id=self.data_set_id, + settings=settings_write, + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result: dict[str, Any] = {} + if camel_case: + result["externalId"] = self.external_id + if self.name is not None: + result["name"] = self.name + if self.data_set_id is not None: + result["dataSetId"] = self.data_set_id + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=True) + if self.format is not None: + result["format"] = self.format + if self.created_time is not None: + result["createdTime"] = self.created_time + if self.last_updated_time is not None: + result["lastUpdatedTime"] = self.last_updated_time + else: + result["external_id"] = self.external_id + if self.name is not None: + result["name"] = self.name + if self.data_set_id is not None: + result["data_set_id"] = self.data_set_id + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=False) + if self.format is not None: + result["format"] = self.format + if self.created_time is not None: + result["created_time"] = self.created_time + if self.last_updated_time is not None: + result["last_updated_time"] = self.last_updated_time + return result + + +class ExternalDataSourceWrite(ExternalDataSourceCore): + """A Fabric OneLake external data source (write model — used for upsert). + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + The ``format`` field is always ``"one_lake"`` and is injected automatically on serialization. + + Args: + external_id (str): External ID of the data source. + name (str | None): Human-readable name. + data_set_id (int | None): Data set ID for ACL scoping. + settings (OneLakeDataSourceSettingsWrite | None): Connection settings including client secret. + """ + + def __init__( + self, + external_id: str, + name: str | None = None, + data_set_id: int | None = None, + settings: OneLakeDataSourceSettingsWrite | None = None, + ) -> None: + super().__init__(external_id=external_id, name=name, data_set_id=data_set_id) + self.settings = settings + + @classmethod + def onelake( + cls, + external_id: str, + client_id: str, + tenant_id: str, + client_secret: str, + workspace_name: str, + container_name: str, + name: str | None = None, + data_set_id: int | None = None, + ) -> ExternalDataSourceWrite: + """Create an ExternalDataSourceWrite for a Fabric OneLake source. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + external_id (str): External ID for the data source. Must be unique. + client_id (str): Azure application (client) ID. + tenant_id (str): Azure tenant (directory) ID. + client_secret (str): Azure client secret. + workspace_name (str): Fabric workspace GUID or name. + container_name (str): Fabric lakehouse GUID or name. + name (str | None): Human-readable name. + data_set_id (int | None): Data set ID for ACL scoping. + + Returns: + ExternalDataSourceWrite: Ready to pass to ``client.transformations.external_data_sources.upsert()``. + + Examples: + + Register a Fabric OneLake source: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.transformations.external_data import ( + ... ExternalDataSourceWrite, + ... ) + >>> client = CogniteClient() + >>> source = ExternalDataSourceWrite.onelake( + ... external_id="fabric-lakehouse-prod", + ... name="Production lakehouse", + ... client_id="", + ... tenant_id="", + ... client_secret="", + ... workspace_name="", + ... container_name="", + ... data_set_id=123456, + ... ) + >>> client.transformations.external_data_sources.upsert(source) + """ + return cls( + external_id=external_id, + name=name, + data_set_id=data_set_id, + settings=OneLakeDataSourceSettingsWrite( + credentials=OneLakeCredentialsWrite( + client_id=client_id, + tenant_id=tenant_id, + client_secret=client_secret, + ), + location_description=OneLakeLocationDescription( + workspace_name=workspace_name, + container_name=container_name, + ), + ), + ) + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + settings = None + if (settings_raw := resource.get("settings")) is not None: + settings = OneLakeDataSourceSettingsWrite._load(settings_raw) + return cls( + external_id=resource["externalId"], + name=resource.get("name"), + data_set_id=resource.get("dataSetId"), + settings=settings, + ) + + def as_write(self) -> ExternalDataSourceWrite: + return self + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result: dict[str, Any] = {} + if camel_case: + result["externalId"] = self.external_id + result["format"] = self._FORMAT + if self.name is not None: + result["name"] = self.name + if self.data_set_id is not None: + result["dataSetId"] = self.data_set_id + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=True) + else: + result["external_id"] = self.external_id + result["format"] = self._FORMAT + if self.name is not None: + result["name"] = self.name + if self.data_set_id is not None: + result["data_set_id"] = self.data_set_id + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=False) + return result + + +class ExternalDataSourceList( + WriteableCogniteResourceList[ExternalDataSourceWrite, ExternalDataSource], ExternalIDTransformerMixin +): + """A list of ExternalDataSource (read model) objects.""" + + _RESOURCE = ExternalDataSource + + def as_write(self) -> ExternalDataSourceWriteList: + """Return all sources in their write format (client_secret will be None on each).""" + return ExternalDataSourceWriteList([item.as_write() for item in self.data]) + + +class ExternalDataSourceWriteList(CogniteResourceList[ExternalDataSourceWrite], ExternalIDTransformerMixin): + """A list of ExternalDataSourceWrite objects.""" + + _RESOURCE = ExternalDataSourceWrite + + +class ExternalDataSourceUsability(CogniteResource): + """Result of verifying a Fabric OneLake external data source's usability. + + Args: + external_id (str | None): External ID of the verified data source. + usable_version (str | None): UUID indicating the data source is accessible and credentials are valid. + ``None`` if the source cannot be accessed. + """ + + def __init__(self, external_id: str | None = None, usable_version: str | None = None) -> None: + self.external_id = external_id + self.usable_version = usable_version + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + external_id=resource.get("externalId"), + usable_version=resource.get("usableVersion"), + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + if camel_case: + return {"externalId": self.external_id, "usableVersion": self.usable_version} + return {"external_id": self.external_id, "usable_version": self.usable_version} diff --git a/tests/tests_unit/test_data_classes/test_transformations/__init__.py b/tests/tests_unit/test_data_classes/test_transformations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py new file mode 100644 index 0000000000..38f66cb3cf --- /dev/null +++ b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import pytest + +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSource, + ExternalDataSourceUsability, + ExternalDataSourceWrite, + OneLakeCredentialsWrite, +) + + +def test_onelake_factory_produces_valid_structure() -> None: + source = ExternalDataSourceWrite.onelake( + external_id="x", + client_id="cid", + tenant_id="tid", + client_secret="sec", + workspace_name="ws", + container_name="cn", + ) + dumped = source.dump(camel_case=True) + + assert dumped["format"] == "one_lake" + assert dumped["externalId"] == "x" + assert dumped["settings"]["credentials"]["clientId"] == "cid" + assert dumped["settings"]["credentials"]["clientSecret"] == "sec" + assert dumped["settings"]["locationDescription"]["workspaceName"] == "ws" + + +def test_write_dump_always_includes_format() -> None: + source = ExternalDataSourceWrite(external_id="x") + dumped = source.dump(camel_case=True) + + assert "format" in dumped + assert dumped["format"] == "one_lake" + + +def test_read_load_parses_settings() -> None: + raw = { + "externalId": "x", + "format": "one_lake", + "settings": { + "credentials": {"clientId": "cid", "tenantId": "tid"}, + "locationDescription": {"workspaceName": "ws", "containerName": "cn"}, + }, + } + source = ExternalDataSource._load(raw) + + assert source.external_id == "x" + assert source.format == "one_lake" + assert source.settings is not None + assert source.settings.credentials is not None + assert source.settings.credentials.client_id == "cid" + assert source.settings.credentials.tenant_id == "tid" + assert source.settings.location_description is not None + assert source.settings.location_description.workspace_name == "ws" + assert source.settings.location_description.container_name == "cn" + # Read model never carries client_secret + assert not hasattr(source.settings.credentials, "client_secret") + + +def test_read_load_unknown_format_warns_not_raises() -> None: + raw = {"externalId": "x", "format": "delta_sharing", "settings": {}} + + with pytest.warns(UserWarning, match="Unknown external data source format"): + source = ExternalDataSource._load(raw) + + # Must not raise — the object is returned despite the unknown format + assert source.external_id == "x" + assert source.format == "delta_sharing" + + +def test_as_write_returns_write_with_none_secret() -> None: + raw = { + "externalId": "x", + "format": "one_lake", + "settings": { + "credentials": {"clientId": "cid", "tenantId": "tid"}, + "locationDescription": {"workspaceName": "ws", "containerName": "cn"}, + }, + } + read_source = ExternalDataSource._load(raw) + write_source = read_source.as_write() + + assert isinstance(write_source, ExternalDataSourceWrite) + assert write_source.settings is not None + assert write_source.settings.credentials is not None + assert write_source.settings.credentials.client_secret is None + + +def test_credentials_write_repr_masks_secret() -> None: + creds = OneLakeCredentialsWrite("cid", "tid", "actual-secret") + result = repr(creds) + + assert "actual-secret" not in result + assert "***" in result + + +def test_usability_load_with_version() -> None: + usability = ExternalDataSourceUsability._load({"externalId": "x", "usableVersion": "some-uuid"}) + + assert usability.external_id == "x" + assert usability.usable_version == "some-uuid" + + +def test_usability_load_with_null_version() -> None: + usability = ExternalDataSourceUsability._load({"externalId": "x", "usableVersion": None}) + + assert usability.external_id == "x" + assert usability.usable_version is None From bebeafba06998a8319eba8fc5b9f06f312aac530 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 9 Jul 2026 14:21:12 +0530 Subject: [PATCH 02/22] fix(transformations): restore MeteringData exports in __init__.py The external data source exports accidentally dropped MeteringData from the public data_classes surface and added an unrelated RecordId export. --- cognite/client/data_classes/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cognite/client/data_classes/__init__.py b/cognite/client/data_classes/__init__.py index 8a722ff9f0..d6f62ed916 100644 --- a/cognite/client/data_classes/__init__.py +++ b/cognite/client/data_classes/__init__.py @@ -29,7 +29,6 @@ EntityMatchingPredictionResult, JobStatus, ) -from cognite.client.data_classes.data_modeling.records import RecordId from cognite.client.data_classes.data_sets import ( DataSet, DataSetFilter, @@ -162,6 +161,7 @@ LabelFilter, ) from cognite.client.data_classes.limits import Limit, LimitList +from cognite.client.data_classes.metering import MeteringData, MeteringDataList, MeteringDataPoint from cognite.client.data_classes.raw import ( Database, DatabaseList, @@ -472,6 +472,9 @@ "LatestDatapointQuery", "Limit", "LimitList", + "MeteringData", + "MeteringDataList", + "MeteringDataPoint", "OidcCredentials", "OneLakeCredentialsRead", "OneLakeCredentialsWrite", @@ -479,7 +482,6 @@ "OneLakeDataSourceSettingsWrite", "OneLakeLocationDescription", "RawTable", - "RecordId", "Relationship", "RelationshipFilter", "RelationshipList", From ad1e459feb6de6ee8ce641c3998cc23a7728a88e Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 9 Jul 2026 14:21:13 +0530 Subject: [PATCH 03/22] fix(ci): restore streamlit pyodide cryptography workaround Replace the date-expired authlib/cryptography shim with a Pyodide version check so stlite's 0.26.2 job keeps working. --- scripts/test-pyodide.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/test-pyodide.js b/scripts/test-pyodide.js index 4dc068252d..ba523bd4d7 100644 --- a/scripts/test-pyodide.js +++ b/scripts/test-pyodide.js @@ -38,15 +38,15 @@ server.listen(PORT, () => { await pyodide.loadPackage("micropip"); const micropip = pyodide.pyimport("micropip"); - // TEMPORARY WORKAROUND (added 2026-04-19, auto-disables 2026-05-03): - // authlib 1.7+ requires cryptography>=45.0.1, which has no pure-Python wheel; - // the streamlit-pinned Pyodide 0.26.2 only ships cryptography 43.x, so - // micropip resolution fails. Preload Pyodide's built cryptography and cap - // authlib below 1.7 to satisfy the transitive requirement. Revisit once - // stlite bumps to a Pyodide release that ships cryptography>=45.0.1 - // (Pyodide 0.29.0 already does). After the expiry date, the workaround is - // skipped — if it's still needed the install will fail loudly. - if (new Date() < new Date("2026-07-04")) { + // authlib 1.7+ requires cryptography>=45.0.1, which has no pure-Python wheel. + // Older Pyodide releases (e.g. stlite's 0.26.2) only ship cryptography 43.x, + // so micropip resolution fails. On those runtimes, preload Pyodide's bundled + // cryptography and cap authlib below 1.7 to satisfy the transitive requirement. + // Pyodide >= 0.29 ships cryptography>=45.0.1, so the workaround is skipped there. + const [pyMajor, pyMinor] = pyodide.version.split(".").map(Number); + const needsCryptographyWorkaround = pyMajor === 0 && pyMinor < 29; + if (needsCryptographyWorkaround) { + console.log(`Applying cryptography workaround for Pyodide ${pyodide.version}`); await pyodide.loadPackage(["cryptography", "ssl"]); await micropip.install("authlib<1.7"); } From fb8e853aa3b28a0bcf5af59b8c2ea176e1b43694 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 9 Jul 2026 15:04:12 +0530 Subject: [PATCH 04/22] refactor(transformations): simplify external data dump methods Drop redundant custom dump implementations on simple OneLake credential and usability classes, use super().dump() on source models, and remove manual load/dump unit tests covered by test_base.py per style guide. --- .../transformations/external_data.py | 86 ++----------------- .../test_external_data.py | 40 --------- 2 files changed, 9 insertions(+), 117 deletions(-) diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index c70997ffd5..3bd224df88 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -48,11 +48,6 @@ def _load(cls, resource: dict[str, Any]) -> Self: container_name=resource["containerName"], ) - def dump(self, camel_case: bool = True) -> dict[str, Any]: - if camel_case: - return {"workspaceName": self.workspace_name, "containerName": self.container_name} - return {"workspace_name": self.workspace_name, "container_name": self.container_name} - class OneLakeCredentialsRead(CogniteResource): """Read-only view of Azure credentials for Fabric OneLake (clientSecret is never returned by the API). @@ -73,11 +68,6 @@ def _load(cls, resource: dict[str, Any]) -> Self: tenant_id=resource["tenantId"], ) - def dump(self, camel_case: bool = True) -> dict[str, Any]: - if camel_case: - return {"clientId": self.client_id, "tenantId": self.tenant_id} - return {"client_id": self.client_id, "tenant_id": self.tenant_id} - class OneLakeCredentialsWrite(CogniteResource): """Azure credentials for writing to Fabric OneLake. @@ -109,18 +99,6 @@ def _load(cls, resource: dict[str, Any]) -> Self: client_secret=resource.get("clientSecret"), ) - def dump(self, camel_case: bool = True) -> dict[str, Any]: - result: dict[str, Any] - if camel_case: - result = {"clientId": self.client_id, "tenantId": self.tenant_id} - if self.client_secret is not None: - result["clientSecret"] = self.client_secret - else: - result = {"client_id": self.client_id, "tenant_id": self.tenant_id} - if self.client_secret is not None: - result["client_secret"] = self.client_secret - return result - class OneLakeDataSourceSettingsRead(CogniteResource): """Settings for a Fabric OneLake external data source (read model — no client secret). @@ -149,7 +127,7 @@ def _load(cls, resource: dict[str, Any]) -> Self: return cls(credentials=credentials, location_description=location_description) def dump(self, camel_case: bool = True) -> dict[str, Any]: - result: dict[str, Any] = {} + result = super().dump(camel_case=camel_case) if self.credentials is not None: result["credentials"] = self.credentials.dump(camel_case=camel_case) if self.location_description is not None: @@ -185,7 +163,7 @@ def _load(cls, resource: dict[str, Any]) -> Self: return cls(credentials=credentials, location_description=location_description) def dump(self, camel_case: bool = True) -> dict[str, Any]: - result: dict[str, Any] = {} + result = super().dump(camel_case=camel_case) if self.credentials is not None: result["credentials"] = self.credentials.dump(camel_case=camel_case) if self.location_description is not None: @@ -302,35 +280,9 @@ def as_write(self) -> ExternalDataSourceWrite: ) def dump(self, camel_case: bool = True) -> dict[str, Any]: - result: dict[str, Any] = {} - if camel_case: - result["externalId"] = self.external_id - if self.name is not None: - result["name"] = self.name - if self.data_set_id is not None: - result["dataSetId"] = self.data_set_id - if self.settings is not None: - result["settings"] = self.settings.dump(camel_case=True) - if self.format is not None: - result["format"] = self.format - if self.created_time is not None: - result["createdTime"] = self.created_time - if self.last_updated_time is not None: - result["lastUpdatedTime"] = self.last_updated_time - else: - result["external_id"] = self.external_id - if self.name is not None: - result["name"] = self.name - if self.data_set_id is not None: - result["data_set_id"] = self.data_set_id - if self.settings is not None: - result["settings"] = self.settings.dump(camel_case=False) - if self.format is not None: - result["format"] = self.format - if self.created_time is not None: - result["created_time"] = self.created_time - if self.last_updated_time is not None: - result["last_updated_time"] = self.last_updated_time + result = super().dump(camel_case=camel_case) + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=camel_case) return result @@ -443,25 +395,10 @@ def as_write(self) -> ExternalDataSourceWrite: return self def dump(self, camel_case: bool = True) -> dict[str, Any]: - result: dict[str, Any] = {} - if camel_case: - result["externalId"] = self.external_id - result["format"] = self._FORMAT - if self.name is not None: - result["name"] = self.name - if self.data_set_id is not None: - result["dataSetId"] = self.data_set_id - if self.settings is not None: - result["settings"] = self.settings.dump(camel_case=True) - else: - result["external_id"] = self.external_id - result["format"] = self._FORMAT - if self.name is not None: - result["name"] = self.name - if self.data_set_id is not None: - result["data_set_id"] = self.data_set_id - if self.settings is not None: - result["settings"] = self.settings.dump(camel_case=False) + result = super().dump(camel_case=camel_case) + result["format"] = self._FORMAT + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=camel_case) return result @@ -502,8 +439,3 @@ def _load(cls, resource: dict[str, Any]) -> Self: external_id=resource.get("externalId"), usable_version=resource.get("usableVersion"), ) - - def dump(self, camel_case: bool = True) -> dict[str, Any]: - if camel_case: - return {"externalId": self.external_id, "usableVersion": self.usable_version} - return {"external_id": self.external_id, "usable_version": self.usable_version} diff --git a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py index 38f66cb3cf..877a4a1cdf 100644 --- a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py +++ b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py @@ -4,7 +4,6 @@ from cognite.client.data_classes.transformations.external_data import ( ExternalDataSource, - ExternalDataSourceUsability, ExternalDataSourceWrite, OneLakeCredentialsWrite, ) @@ -36,37 +35,12 @@ def test_write_dump_always_includes_format() -> None: assert dumped["format"] == "one_lake" -def test_read_load_parses_settings() -> None: - raw = { - "externalId": "x", - "format": "one_lake", - "settings": { - "credentials": {"clientId": "cid", "tenantId": "tid"}, - "locationDescription": {"workspaceName": "ws", "containerName": "cn"}, - }, - } - source = ExternalDataSource._load(raw) - - assert source.external_id == "x" - assert source.format == "one_lake" - assert source.settings is not None - assert source.settings.credentials is not None - assert source.settings.credentials.client_id == "cid" - assert source.settings.credentials.tenant_id == "tid" - assert source.settings.location_description is not None - assert source.settings.location_description.workspace_name == "ws" - assert source.settings.location_description.container_name == "cn" - # Read model never carries client_secret - assert not hasattr(source.settings.credentials, "client_secret") - - def test_read_load_unknown_format_warns_not_raises() -> None: raw = {"externalId": "x", "format": "delta_sharing", "settings": {}} with pytest.warns(UserWarning, match="Unknown external data source format"): source = ExternalDataSource._load(raw) - # Must not raise — the object is returned despite the unknown format assert source.external_id == "x" assert source.format == "delta_sharing" @@ -95,17 +69,3 @@ def test_credentials_write_repr_masks_secret() -> None: assert "actual-secret" not in result assert "***" in result - - -def test_usability_load_with_version() -> None: - usability = ExternalDataSourceUsability._load({"externalId": "x", "usableVersion": "some-uuid"}) - - assert usability.external_id == "x" - assert usability.usable_version == "some-uuid" - - -def test_usability_load_with_null_version() -> None: - usability = ExternalDataSourceUsability._load({"externalId": "x", "usableVersion": None}) - - assert usability.external_id == "x" - assert usability.usable_version is None From 66dcea8daff6ffc92669eb8cecf91dcbe58c8378 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 9 Jul 2026 15:25:16 +0530 Subject: [PATCH 05/22] docs(transformations): clarify OneLake Write models and fix pyodide parse *Write classes are CDF upsert models, not lakehouse write APIs. Use parseInt for Pyodide version components per Gemini review. --- .../transformations/external_data.py | 54 +++++++++++-------- scripts/test-pyodide.js | 2 +- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index 3bd224df88..72a4d8220c 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -50,7 +50,7 @@ def _load(cls, resource: dict[str, Any]) -> Self: class OneLakeCredentialsRead(CogniteResource): - """Read-only view of Azure credentials for Fabric OneLake (clientSecret is never returned by the API). + """Response model for Azure credentials returned by list/get (``clientSecret`` is never included). Args: client_id (str): Azure application (client) ID. @@ -70,13 +70,17 @@ def _load(cls, resource: dict[str, Any]) -> Self: class OneLakeCredentialsWrite(CogniteResource): - """Azure credentials for writing to Fabric OneLake. + """Upsert model for Azure credentials when registering a OneLake external data source in CDF. + + ``Write`` follows the SDK read/upsert naming convention — it does **not** mean writing data into + OneLake. Transforms only read from OneLake via ``ext_onelake()``; this class supplies + ``client_secret`` for ``external_data_sources.upsert()``. Args: client_id (str): Azure application (client) ID. tenant_id (str): Azure tenant (directory) ID. - client_secret (str | None): Azure client secret. Required for upsert; None when reconstructed - from a read model via as_write() since the API never returns the secret. + client_secret (str | None): Azure client secret. Required for upsert; ``None`` when + reconstructed from a read model via ``as_write()`` because the API never returns it. """ def __init__(self, client_id: str, tenant_id: str, client_secret: str | None = None) -> None: @@ -101,7 +105,7 @@ def _load(cls, resource: dict[str, Any]) -> Self: class OneLakeDataSourceSettingsRead(CogniteResource): - """Settings for a Fabric OneLake external data source (read model — no client secret). + """Response model for OneLake connection settings (no ``client_secret``). Args: credentials (OneLakeCredentialsRead | None): Azure credentials (client ID and tenant ID only). @@ -137,10 +141,10 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]: class OneLakeDataSourceSettingsWrite(CogniteResource): - """Settings for writing a Fabric OneLake external data source (includes client secret). + """Upsert model for OneLake connection settings registered in CDF (includes ``client_secret``). Args: - credentials (OneLakeCredentialsWrite | None): Azure credentials including client secret. + credentials (OneLakeCredentialsWrite | None): Azure credentials for ``upsert()``. location_description (OneLakeLocationDescription | None): Fabric workspace and lakehouse identifiers. """ @@ -173,10 +177,11 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]: class ExternalDataSourceCore(WriteableCogniteResource["ExternalDataSourceWrite"], ABC): - """Shared base for ExternalDataSource (read) and ExternalDataSourceWrite (write). + """Shared base for ``ExternalDataSource`` (API read model) and ``ExternalDataSourceWrite`` (API upsert model). - OneLake external data sources are **read-only** from a transform perspective — transforms can - read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + OneLake external data sources are **read-only from a transform perspective** — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing transform output to OneLake + is not supported. Args: external_id (str): External ID of the data source. Must be unique within the project. @@ -198,10 +203,11 @@ def __init__( class ExternalDataSource(ExternalDataSourceCore): - """A Fabric OneLake external data source (read model — returned by list). + """A Fabric OneLake external data source (API read model — returned by list/get). - OneLake external data sources are **read-only** from a transform perspective — transforms can - read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + OneLake external data sources are **read-only from a transform perspective** — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing transform output to OneLake + is not supported. The ``clientSecret`` field is **never** returned by the API. @@ -254,10 +260,11 @@ def _load(cls, resource: dict[str, Any]) -> Self: ) def as_write(self) -> ExternalDataSourceWrite: - """Return this source as an ExternalDataSourceWrite. + """Return an upsert model for updating this source in CDF. - Note: The ``client_secret`` cannot be reconstructed from the read model (the API never returns it). - The returned write object will have ``client_secret=None`` on its credentials. + ``client_secret`` cannot be reconstructed from the read model (the API never returns it). + The returned object has ``client_secret=None`` on its credentials — supply a new secret on + ``upsert()`` when updating credentials. """ settings_write: OneLakeDataSourceSettingsWrite | None = None if self.settings is not None: @@ -287,10 +294,11 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]: class ExternalDataSourceWrite(ExternalDataSourceCore): - """A Fabric OneLake external data source (write model — used for upsert). + """Upsert model for a Fabric OneLake external data source (``external_data_sources.upsert()``). - OneLake external data sources are **read-only** from a transform perspective — transforms can - read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + OneLake external data sources are **read-only from a transform perspective** — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing transform output to OneLake + is not supported. The ``format`` field is always ``"one_lake"`` and is injected automatically on serialization. @@ -323,10 +331,10 @@ def onelake( name: str | None = None, data_set_id: int | None = None, ) -> ExternalDataSourceWrite: - """Create an ExternalDataSourceWrite for a Fabric OneLake source. + """Create an ``ExternalDataSourceWrite`` for registering a Fabric OneLake source in CDF. - OneLake external data sources are **read-only** from a transform perspective — transforms can - read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + Registers Azure credentials and lakehouse location so transforms can **read** via + ``ext_onelake()``. Does not write data into OneLake. Args: external_id (str): External ID for the data source. Must be unique. @@ -410,7 +418,7 @@ class ExternalDataSourceList( _RESOURCE = ExternalDataSource def as_write(self) -> ExternalDataSourceWriteList: - """Return all sources in their write format (client_secret will be None on each).""" + """Return upsert models for each source (``client_secret`` will be ``None`` on each).""" return ExternalDataSourceWriteList([item.as_write() for item in self.data]) diff --git a/scripts/test-pyodide.js b/scripts/test-pyodide.js index ba523bd4d7..320aeabe93 100644 --- a/scripts/test-pyodide.js +++ b/scripts/test-pyodide.js @@ -43,7 +43,7 @@ server.listen(PORT, () => { // so micropip resolution fails. On those runtimes, preload Pyodide's bundled // cryptography and cap authlib below 1.7 to satisfy the transitive requirement. // Pyodide >= 0.29 ships cryptography>=45.0.1, so the workaround is skipped there. - const [pyMajor, pyMinor] = pyodide.version.split(".").map(Number); + const [pyMajor, pyMinor] = pyodide.version.split(".").map((val) => parseInt(val, 10)); const needsCryptographyWorkaround = pyMajor === 0 && pyMinor < 29; if (needsCryptographyWorkaround) { console.log(`Applying cryptography workaround for Pyodide ${pyodide.version}`); From 2ec6da33e8ed98f568f095d169f41957aa3a827f Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Wed, 15 Jul 2026 14:12:24 +0530 Subject: [PATCH 06/22] refactor(transformations): drop Read suffix on OneLake read models Rename OneLakeCredentialsRead and OneLakeDataSourceSettingsRead to match SDK convention where read models have no suffix and write models use Write. Co-authored-by: Cursor --- cognite/client/data_classes/__init__.py | 8 ++++---- .../transformations/external_data.py | 20 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cognite/client/data_classes/__init__.py b/cognite/client/data_classes/__init__.py index d6f62ed916..9f9b298a10 100644 --- a/cognite/client/data_classes/__init__.py +++ b/cognite/client/data_classes/__init__.py @@ -257,9 +257,9 @@ ExternalDataSourceUsability, ExternalDataSourceWrite, ExternalDataSourceWriteList, - OneLakeCredentialsRead, + OneLakeCredentials, OneLakeCredentialsWrite, - OneLakeDataSourceSettingsRead, + OneLakeDataSourceSettings, OneLakeDataSourceSettingsWrite, OneLakeLocationDescription, ) @@ -476,9 +476,9 @@ "MeteringDataList", "MeteringDataPoint", "OidcCredentials", - "OneLakeCredentialsRead", + "OneLakeCredentials", "OneLakeCredentialsWrite", - "OneLakeDataSourceSettingsRead", + "OneLakeDataSourceSettings", "OneLakeDataSourceSettingsWrite", "OneLakeLocationDescription", "RawTable", diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index 72a4d8220c..087424ccaf 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -21,9 +21,9 @@ "ExternalDataSourceUsability", "ExternalDataSourceWrite", "ExternalDataSourceWriteList", - "OneLakeCredentialsRead", + "OneLakeCredentials", "OneLakeCredentialsWrite", - "OneLakeDataSourceSettingsRead", + "OneLakeDataSourceSettings", "OneLakeDataSourceSettingsWrite", "OneLakeLocationDescription", ] @@ -49,7 +49,7 @@ def _load(cls, resource: dict[str, Any]) -> Self: ) -class OneLakeCredentialsRead(CogniteResource): +class OneLakeCredentials(CogniteResource): """Response model for Azure credentials returned by list/get (``clientSecret`` is never included). Args: @@ -104,17 +104,17 @@ def _load(cls, resource: dict[str, Any]) -> Self: ) -class OneLakeDataSourceSettingsRead(CogniteResource): +class OneLakeDataSourceSettings(CogniteResource): """Response model for OneLake connection settings (no ``client_secret``). Args: - credentials (OneLakeCredentialsRead | None): Azure credentials (client ID and tenant ID only). + credentials (OneLakeCredentials | None): Azure credentials (client ID and tenant ID only). location_description (OneLakeLocationDescription | None): Fabric workspace and lakehouse identifiers. """ def __init__( self, - credentials: OneLakeCredentialsRead | None = None, + credentials: OneLakeCredentials | None = None, location_description: OneLakeLocationDescription | None = None, ) -> None: self.credentials = credentials @@ -124,7 +124,7 @@ def __init__( def _load(cls, resource: dict[str, Any]) -> Self: credentials = None if (creds_raw := resource.get("credentials")) is not None: - credentials = OneLakeCredentialsRead._load(creds_raw) + credentials = OneLakeCredentials._load(creds_raw) location_description = None if (loc_raw := resource.get("locationDescription")) is not None: location_description = OneLakeLocationDescription._load(loc_raw) @@ -215,7 +215,7 @@ class ExternalDataSource(ExternalDataSourceCore): external_id (str): External ID of the data source. name (str | None): Human-readable name. data_set_id (int | None): Data set ID for ACL scoping. - settings (OneLakeDataSourceSettingsRead | None): Connection settings (no client secret). + settings (OneLakeDataSourceSettings | None): Connection settings (no client secret). format (str | None): Backend format identifier (always ``"one_lake"`` for OneLake sources). created_time (int | None): Time the resource was created (milliseconds since epoch). last_updated_time (int | None): Time the resource was last updated (milliseconds since epoch). @@ -226,7 +226,7 @@ def __init__( external_id: str, name: str | None = None, data_set_id: int | None = None, - settings: OneLakeDataSourceSettingsRead | None = None, + settings: OneLakeDataSourceSettings | None = None, format: str | None = None, created_time: int | None = None, last_updated_time: int | None = None, @@ -248,7 +248,7 @@ def _load(cls, resource: dict[str, Any]) -> Self: ) settings = None if (settings_raw := resource.get("settings")) is not None: - settings = OneLakeDataSourceSettingsRead._load(settings_raw) + settings = OneLakeDataSourceSettings._load(settings_raw) return cls( external_id=resource["externalId"], name=resource.get("name"), From 3192b42f2e3423abbaca2a7a676bc50ff802eccd Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Wed, 15 Jul 2026 16:21:31 +0530 Subject: [PATCH 07/22] Update cognite/client/data_classes/transformations/external_data.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Håkon V. Treider --- cognite/client/data_classes/transformations/external_data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index 087424ccaf..312da373fd 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -79,8 +79,7 @@ class OneLakeCredentialsWrite(CogniteResource): Args: client_id (str): Azure application (client) ID. tenant_id (str): Azure tenant (directory) ID. - client_secret (str | None): Azure client secret. Required for upsert; ``None`` when - reconstructed from a read model via ``as_write()`` because the API never returns it. + client_secret (str): Azure client secret """ def __init__(self, client_id: str, tenant_id: str, client_secret: str | None = None) -> None: From 21384d74f253abd1821e4e9a45df40bb94e3d1bd Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Wed, 15 Jul 2026 16:21:43 +0530 Subject: [PATCH 08/22] Update cognite/client/data_classes/transformations/external_data.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Håkon V. Treider --- .../client/data_classes/transformations/external_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index 312da373fd..badba7a23d 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -72,9 +72,9 @@ def _load(cls, resource: dict[str, Any]) -> Self: class OneLakeCredentialsWrite(CogniteResource): """Upsert model for Azure credentials when registering a OneLake external data source in CDF. - ``Write`` follows the SDK read/upsert naming convention — it does **not** mean writing data into - OneLake. Transforms only read from OneLake via ``ext_onelake()``; this class supplies - ``client_secret`` for ``external_data_sources.upsert()``. + Note: + ``Write`` does **not** mean writing data into OneLake, it merely follows the naming convention + in the SDK for create/update-style classes. Args: client_id (str): Azure application (client) ID. From 64eed27ef66d86dabf8a5b5010db783745e96f37 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Wed, 15 Jul 2026 16:22:00 +0530 Subject: [PATCH 09/22] Update cognite/client/data_classes/transformations/external_data.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Håkon V. Treider --- cognite/client/data_classes/transformations/external_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index badba7a23d..6248d8db53 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -99,7 +99,7 @@ def _load(cls, resource: dict[str, Any]) -> Self: return cls( client_id=resource["clientId"], tenant_id=resource["tenantId"], - client_secret=resource.get("clientSecret"), + client_secret=resource["clientSecret"], ) From 2d6ead6a81fb2d8cc2c246253b885449dfdac8c8 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Wed, 15 Jul 2026 16:26:05 +0530 Subject: [PATCH 10/22] refactor(transformations): address haakonvt review on external data models Use _load_if, tighten client_secret handling, redact str/repr, and require client_secret when converting read credentials via as_write(). Co-authored-by: Cursor --- .../transformations/external_data.py | 74 +++++++++---------- .../test_external_data.py | 31 +++++++- 2 files changed, 63 insertions(+), 42 deletions(-) diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index 6248d8db53..d9cd20ce40 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -82,18 +82,20 @@ class OneLakeCredentialsWrite(CogniteResource): client_secret (str): Azure client secret """ - def __init__(self, client_id: str, tenant_id: str, client_secret: str | None = None) -> None: + def __init__(self, client_id: str, tenant_id: str, client_secret: str) -> None: self.client_id = client_id self.tenant_id = tenant_id self.client_secret = client_secret def __repr__(self) -> str: - secret_display = "***" if self.client_secret is not None else None return ( f"OneLakeCredentialsWrite(client_id={self.client_id!r}, tenant_id={self.tenant_id!r}," - f" client_secret={secret_display!r})" + f" client_secret=)" ) + def __str__(self) -> str: + return self.__repr__() + @classmethod def _load(cls, resource: dict[str, Any]) -> Self: return cls( @@ -104,7 +106,7 @@ def _load(cls, resource: dict[str, Any]) -> Self: class OneLakeDataSourceSettings(CogniteResource): - """Response model for OneLake connection settings (no ``client_secret``). + """Response model for OneLake connection settings. Args: credentials (OneLakeCredentials | None): Azure credentials (client ID and tenant ID only). @@ -121,13 +123,10 @@ def __init__( @classmethod def _load(cls, resource: dict[str, Any]) -> Self: - credentials = None - if (creds_raw := resource.get("credentials")) is not None: - credentials = OneLakeCredentials._load(creds_raw) - location_description = None - if (loc_raw := resource.get("locationDescription")) is not None: - location_description = OneLakeLocationDescription._load(loc_raw) - return cls(credentials=credentials, location_description=location_description) + return cls( + credentials=OneLakeCredentials._load_if(resource.get("credentials")), + location_description=OneLakeLocationDescription._load_if(resource.get("locationDescription")), + ) def dump(self, camel_case: bool = True) -> dict[str, Any]: result = super().dump(camel_case=camel_case) @@ -140,7 +139,7 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]: class OneLakeDataSourceSettingsWrite(CogniteResource): - """Upsert model for OneLake connection settings registered in CDF (includes ``client_secret``). + """Upsert model for OneLake connection settings registered in CDF. Args: credentials (OneLakeCredentialsWrite | None): Azure credentials for ``upsert()``. @@ -157,13 +156,10 @@ def __init__( @classmethod def _load(cls, resource: dict[str, Any]) -> Self: - credentials = None - if (creds_raw := resource.get("credentials")) is not None: - credentials = OneLakeCredentialsWrite._load(creds_raw) - location_description = None - if (loc_raw := resource.get("locationDescription")) is not None: - location_description = OneLakeLocationDescription._load(loc_raw) - return cls(credentials=credentials, location_description=location_description) + return cls( + credentials=OneLakeCredentialsWrite._load_if(resource.get("credentials")), + location_description=OneLakeLocationDescription._load_if(resource.get("locationDescription")), + ) def dump(self, camel_case: bool = True) -> dict[str, Any]: result = super().dump(camel_case=camel_case) @@ -208,13 +204,11 @@ class ExternalDataSource(ExternalDataSourceCore): read data from OneLake tables via ``ext_onelake()`` SQL, but writing transform output to OneLake is not supported. - The ``clientSecret`` field is **never** returned by the API. - Args: external_id (str): External ID of the data source. name (str | None): Human-readable name. data_set_id (int | None): Data set ID for ACL scoping. - settings (OneLakeDataSourceSettings | None): Connection settings (no client secret). + settings (OneLakeDataSourceSettings | None): Connection settings. format (str | None): Backend format identifier (always ``"one_lake"`` for OneLake sources). created_time (int | None): Time the resource was created (milliseconds since epoch). last_updated_time (int | None): Time the resource was last updated (milliseconds since epoch). @@ -245,9 +239,7 @@ def _load(cls, resource: dict[str, Any]) -> Self: UserWarning, stacklevel=2, ) - settings = None - if (settings_raw := resource.get("settings")) is not None: - settings = OneLakeDataSourceSettings._load(settings_raw) + settings = OneLakeDataSourceSettings._load_if(resource.get("settings")) return cls( external_id=resource["externalId"], name=resource.get("name"), @@ -258,21 +250,27 @@ def _load(cls, resource: dict[str, Any]) -> Self: last_updated_time=resource.get("lastUpdatedTime"), ) - def as_write(self) -> ExternalDataSourceWrite: + def as_write(self, client_secret: str | None = None) -> ExternalDataSourceWrite: """Return an upsert model for updating this source in CDF. - ``client_secret`` cannot be reconstructed from the read model (the API never returns it). - The returned object has ``client_secret=None`` on its credentials — supply a new secret on - ``upsert()`` when updating credentials. + Args: + client_secret (str | None): Required when the read model includes credentials, because the API + does not return ``client_secret``. Omit when only metadata (name, data set, location) changes. """ settings_write: OneLakeDataSourceSettingsWrite | None = None if self.settings is not None: creds_write: OneLakeCredentialsWrite | None = None if self.settings.credentials is not None: + if client_secret is None: + raise ValueError( + "client_secret is required to convert credentials to a write model because the API " + "does not return it. Pass client_secret to as_write(), or use " + "ExternalDataSourceWrite.onelake()." + ) creds_write = OneLakeCredentialsWrite( client_id=self.settings.credentials.client_id, tenant_id=self.settings.credentials.tenant_id, - client_secret=None, + client_secret=client_secret, ) settings_write = OneLakeDataSourceSettingsWrite( credentials=creds_write, @@ -299,8 +297,6 @@ class ExternalDataSourceWrite(ExternalDataSourceCore): read data from OneLake tables via ``ext_onelake()`` SQL, but writing transform output to OneLake is not supported. - The ``format`` field is always ``"one_lake"`` and is injected automatically on serialization. - Args: external_id (str): External ID of the data source. name (str | None): Human-readable name. @@ -388,9 +384,7 @@ def onelake( @classmethod def _load(cls, resource: dict[str, Any]) -> Self: - settings = None - if (settings_raw := resource.get("settings")) is not None: - settings = OneLakeDataSourceSettingsWrite._load(settings_raw) + settings = OneLakeDataSourceSettingsWrite._load_if(resource.get("settings")) return cls( external_id=resource["externalId"], name=resource.get("name"), @@ -416,9 +410,13 @@ class ExternalDataSourceList( _RESOURCE = ExternalDataSource - def as_write(self) -> ExternalDataSourceWriteList: - """Return upsert models for each source (``client_secret`` will be ``None`` on each).""" - return ExternalDataSourceWriteList([item.as_write() for item in self.data]) + def as_write(self, client_secret: str | None = None) -> ExternalDataSourceWriteList: + """Return upsert models for each source. + + Args: + client_secret (str | None): Passed through to :meth:`ExternalDataSource.as_write` for each item. + """ + return ExternalDataSourceWriteList([item.as_write(client_secret=client_secret) for item in self.data]) class ExternalDataSourceWriteList(CogniteResourceList[ExternalDataSourceWrite], ExternalIDTransformerMixin): diff --git a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py index 877a4a1cdf..022688cc6b 100644 --- a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py +++ b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py @@ -45,7 +45,7 @@ def test_read_load_unknown_format_warns_not_raises() -> None: assert source.format == "delta_sharing" -def test_as_write_returns_write_with_none_secret() -> None: +def test_as_write_raises_without_client_secret() -> None: raw = { "externalId": "x", "format": "one_lake", @@ -55,12 +55,27 @@ def test_as_write_returns_write_with_none_secret() -> None: }, } read_source = ExternalDataSource._load(raw) - write_source = read_source.as_write() + + with pytest.raises(ValueError, match="client_secret is required"): + read_source.as_write() + + +def test_as_write_with_client_secret() -> None: + raw = { + "externalId": "x", + "format": "one_lake", + "settings": { + "credentials": {"clientId": "cid", "tenantId": "tid"}, + "locationDescription": {"workspaceName": "ws", "containerName": "cn"}, + }, + } + read_source = ExternalDataSource._load(raw) + write_source = read_source.as_write(client_secret="new-secret") assert isinstance(write_source, ExternalDataSourceWrite) assert write_source.settings is not None assert write_source.settings.credentials is not None - assert write_source.settings.credentials.client_secret is None + assert write_source.settings.credentials.client_secret == "new-secret" def test_credentials_write_repr_masks_secret() -> None: @@ -68,4 +83,12 @@ def test_credentials_write_repr_masks_secret() -> None: result = repr(creds) assert "actual-secret" not in result - assert "***" in result + assert "" in result + + +def test_credentials_write_str_masks_secret() -> None: + creds = OneLakeCredentialsWrite("cid", "tid", "actual-secret") + result = str(creds) + + assert "actual-secret" not in result + assert "" in result From a05483c374826526e2b2da1e327e27b27fae82f6 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Wed, 15 Jul 2026 16:36:41 +0530 Subject: [PATCH 11/22] fix(transformations): satisfy docstring lint for as_write methods Add Returns sections and single-line Args formatting expected by pydoclint and the custom docstring checker. Co-authored-by: Cursor --- .../client/data_classes/transformations/external_data.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index d9cd20ce40..ce3bde78a1 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -254,8 +254,9 @@ def as_write(self, client_secret: str | None = None) -> ExternalDataSourceWrite: """Return an upsert model for updating this source in CDF. Args: - client_secret (str | None): Required when the read model includes credentials, because the API - does not return ``client_secret``. Omit when only metadata (name, data set, location) changes. + client_secret (str | None): Required when the read model includes credentials, because the API does not return ``client_secret``. Omit when only metadata (name, data set, location) changes. + Returns: + ExternalDataSourceWrite: Upsert model for this source. """ settings_write: OneLakeDataSourceSettingsWrite | None = None if self.settings is not None: @@ -415,6 +416,8 @@ def as_write(self, client_secret: str | None = None) -> ExternalDataSourceWriteL Args: client_secret (str | None): Passed through to :meth:`ExternalDataSource.as_write` for each item. + Returns: + ExternalDataSourceWriteList: Upsert models for each source in the list. """ return ExternalDataSourceWriteList([item.as_write(client_secret=client_secret) for item in self.data]) From d84bb6d3769a42d0fcf7e0b3e10fb5b767d1dec5 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Wed, 15 Jul 2026 16:54:47 +0530 Subject: [PATCH 12/22] test(transformations): pass client_secret in base as_write tests ExternalDataSource.as_write requires client_secret when the read model includes credentials; update generic writable tests accordingly. Co-authored-by: Cursor --- tests/tests_unit/test_base.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index e9d94a8384..9ead3e5c68 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -46,6 +46,7 @@ from cognite.client.data_classes.postgres_gateway import TableList, User, UserCreated, UserCreatedList, UserList from cognite.client.data_classes.sequences import SequenceUpdate from cognite.client.data_classes.time_series import TimeSeries, TimeSeriesList +from cognite.client.data_classes.transformations.external_data import ExternalDataSource from cognite.client.data_classes.workflows import UnknownWorkflowTaskParameters, WorkflowTaskOutput from cognite.client.exceptions import CogniteMissingClientError from cognite.client.testing import CogniteClientMock @@ -264,7 +265,11 @@ def test_writable_as_write( instance.instance_id = None assert instance.external_id is not None - write_format = instance.as_write() + if cognite_writable_cls is ExternalDataSource: + # Read model never includes client_secret; pass a placeholder when fake data has credentials. + write_format = instance.as_write(client_secret="placeholder-secret") + else: + write_format = instance.as_write() assert isinstance(write_format, CogniteResource) @pytest.mark.dsl @@ -308,7 +313,10 @@ def test_writable_list_as_write( item.instance_id = None assert item.external_id is not None - write_format = resource_list.as_write() + if resource_cls is ExternalDataSource: + write_format = resource_list.as_write(client_secret="placeholder-secret") + else: + write_format = resource_list.as_write() assert isinstance(write_format, CogniteResourceList) @pytest.mark.dsl From 27abac5d56cd6cc6b8ddb45e8c0cad3d30af9980 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Wed, 15 Jul 2026 16:57:29 +0530 Subject: [PATCH 13/22] test(transformations): fix mypy on ExternalDataSource as_write tests Use isinstance narrowing so client_secret is only passed on the concrete ExternalDataSource types, not WriteableCogniteResource. Co-authored-by: Cursor --- tests/tests_unit/test_base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index 9ead3e5c68..b99a127845 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -46,7 +46,7 @@ from cognite.client.data_classes.postgres_gateway import TableList, User, UserCreated, UserCreatedList, UserList from cognite.client.data_classes.sequences import SequenceUpdate from cognite.client.data_classes.time_series import TimeSeries, TimeSeriesList -from cognite.client.data_classes.transformations.external_data import ExternalDataSource +from cognite.client.data_classes.transformations.external_data import ExternalDataSource, ExternalDataSourceList from cognite.client.data_classes.workflows import UnknownWorkflowTaskParameters, WorkflowTaskOutput from cognite.client.exceptions import CogniteMissingClientError from cognite.client.testing import CogniteClientMock @@ -265,7 +265,7 @@ def test_writable_as_write( instance.instance_id = None assert instance.external_id is not None - if cognite_writable_cls is ExternalDataSource: + if isinstance(instance, ExternalDataSource): # Read model never includes client_secret; pass a placeholder when fake data has credentials. write_format = instance.as_write(client_secret="placeholder-secret") else: @@ -313,8 +313,8 @@ def test_writable_list_as_write( item.instance_id = None assert item.external_id is not None - if resource_cls is ExternalDataSource: - write_format = resource_list.as_write(client_secret="placeholder-secret") + if isinstance(resource_list, ExternalDataSourceList): + write_format: CogniteResourceList = resource_list.as_write(client_secret="placeholder-secret") else: write_format = resource_list.as_write() assert isinstance(write_format, CogniteResourceList) From b69a0696dced8aafcc95845e6e85d77f5de82268 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 16 Jul 2026 13:47:38 +0530 Subject: [PATCH 14/22] refactor(transformations): address haakonvt round-2 on external data Add OneLakeExternalDataSource(Write) subclasses, export from transformations package only, register ACL in capability tests, and trim redundant dump tests per SDK conventions. Co-authored-by: Cursor --- cognite/client/data_classes/__init__.py | 22 -- .../data_classes/transformations/__init__.py | 36 +++ .../transformations/external_data.py | 209 ++++++++++-------- tests/tests_unit/test_base.py | 16 +- .../test_data_classes/test_capabilities.py | 7 + .../test_external_data.py | 27 ++- 6 files changed, 190 insertions(+), 127 deletions(-) diff --git a/cognite/client/data_classes/__init__.py b/cognite/client/data_classes/__init__.py index 9f9b298a10..fb0d6ea238 100644 --- a/cognite/client/data_classes/__init__.py +++ b/cognite/client/data_classes/__init__.py @@ -251,18 +251,6 @@ TransformationBlockedInfo, TransformationDestination, ) -from cognite.client.data_classes.transformations.external_data import ( - ExternalDataSource, - ExternalDataSourceList, - ExternalDataSourceUsability, - ExternalDataSourceWrite, - ExternalDataSourceWriteList, - OneLakeCredentials, - OneLakeCredentialsWrite, - OneLakeDataSourceSettings, - OneLakeDataSourceSettingsWrite, - OneLakeLocationDescription, -) from cognite.client.data_classes.transformations.jobs import ( TransformationJob, TransformationJobFilter, @@ -396,11 +384,6 @@ "EventUpdate", "EventWrite", "EventWriteList", - "ExternalDataSource", - "ExternalDataSourceList", - "ExternalDataSourceUsability", - "ExternalDataSourceWrite", - "ExternalDataSourceWriteList", "ExtractionPipeline", "ExtractionPipelineConfig", "ExtractionPipelineConfigRevision", @@ -476,11 +459,6 @@ "MeteringDataList", "MeteringDataPoint", "OidcCredentials", - "OneLakeCredentials", - "OneLakeCredentialsWrite", - "OneLakeDataSourceSettings", - "OneLakeDataSourceSettingsWrite", - "OneLakeLocationDescription", "RawTable", "Relationship", "RelationshipFilter", diff --git a/cognite/client/data_classes/transformations/__init__.py b/cognite/client/data_classes/transformations/__init__.py index a55f9483b0..dc521829c8 100644 --- a/cognite/client/data_classes/transformations/__init__.py +++ b/cognite/client/data_classes/transformations/__init__.py @@ -26,6 +26,42 @@ TransformationBlockedInfo, TransformationDestination, ) +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSource as ExternalDataSource, +) +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSourceList as ExternalDataSourceList, +) +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSourceUsability as ExternalDataSourceUsability, +) +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSourceWrite as ExternalDataSourceWrite, +) +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSourceWriteList as ExternalDataSourceWriteList, +) +from cognite.client.data_classes.transformations.external_data import ( + OneLakeCredentials as OneLakeCredentials, +) +from cognite.client.data_classes.transformations.external_data import ( + OneLakeCredentialsWrite as OneLakeCredentialsWrite, +) +from cognite.client.data_classes.transformations.external_data import ( + OneLakeDataSourceSettings as OneLakeDataSourceSettings, +) +from cognite.client.data_classes.transformations.external_data import ( + OneLakeDataSourceSettingsWrite as OneLakeDataSourceSettingsWrite, +) +from cognite.client.data_classes.transformations.external_data import ( + OneLakeExternalDataSource as OneLakeExternalDataSource, +) +from cognite.client.data_classes.transformations.external_data import ( + OneLakeExternalDataSourceWrite as OneLakeExternalDataSourceWrite, +) +from cognite.client.data_classes.transformations.external_data import ( + OneLakeLocationDescription as OneLakeLocationDescription, +) from cognite.client.data_classes.transformations.jobs import TransformationJob, TransformationJobList from cognite.client.data_classes.transformations.schedules import TransformationSchedule from cognite.client.data_classes.transformations.schema import TransformationSchemaColumnList diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index ce3bde78a1..f38ec0a95d 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -2,7 +2,7 @@ import warnings from abc import ABC -from typing import Any, ClassVar +from typing import Any, ClassVar, cast from typing_extensions import Self @@ -14,20 +14,6 @@ WriteableCogniteResourceList, ) -__all__ = [ - "ExternalDataSource", - "ExternalDataSourceCore", - "ExternalDataSourceList", - "ExternalDataSourceUsability", - "ExternalDataSourceWrite", - "ExternalDataSourceWriteList", - "OneLakeCredentials", - "OneLakeCredentialsWrite", - "OneLakeDataSourceSettings", - "OneLakeDataSourceSettingsWrite", - "OneLakeLocationDescription", -] - class OneLakeLocationDescription(CogniteResource): """Location of a Fabric OneLake lakehouse. @@ -94,7 +80,7 @@ def __repr__(self) -> str: ) def __str__(self) -> str: - return self.__repr__() + return repr(self) @classmethod def _load(cls, resource: dict[str, Any]) -> Self: @@ -198,18 +184,16 @@ def __init__( class ExternalDataSource(ExternalDataSourceCore): - """A Fabric OneLake external data source (API read model — returned by list/get). + """An external data source (API read model — returned by list/get). - OneLake external data sources are **read-only from a transform perspective** — transforms can - read data from OneLake tables via ``ext_onelake()`` SQL, but writing transform output to OneLake - is not supported. + Use :class:`OneLakeExternalDataSource` for Fabric OneLake sources. Args: external_id (str): External ID of the data source. name (str | None): Human-readable name. data_set_id (int | None): Data set ID for ACL scoping. settings (OneLakeDataSourceSettings | None): Connection settings. - format (str | None): Backend format identifier (always ``"one_lake"`` for OneLake sources). + format (str | None): Backend format identifier (``"one_lake"`` for OneLake sources). created_time (int | None): Time the resource was created (milliseconds since epoch). last_updated_time (int | None): Time the resource was last updated (milliseconds since epoch). """ @@ -233,12 +217,13 @@ def __init__( @classmethod def _load(cls, resource: dict[str, Any]) -> Self: fmt = resource.get("format") - if fmt is not None and fmt != cls._FORMAT: - warnings.warn( - f"Unknown external data source format: {fmt!r}. This version of the SDK may not fully support it.", - UserWarning, - stacklevel=2, - ) + if fmt is None or fmt == cls._FORMAT: + return cast(Self, OneLakeExternalDataSource._load(resource)) + warnings.warn( + f"Unknown external data source format: {fmt!r}. This version of the SDK may not fully support it.", + UserWarning, + stacklevel=2, + ) settings = OneLakeDataSourceSettings._load_if(resource.get("settings")) return cls( external_id=resource["externalId"], @@ -265,8 +250,8 @@ def as_write(self, client_secret: str | None = None) -> ExternalDataSourceWrite: if client_secret is None: raise ValueError( "client_secret is required to convert credentials to a write model because the API " - "does not return it. Pass client_secret to as_write(), or use " - "ExternalDataSourceWrite.onelake()." + "does not return it. Pass client_secret to as_write(), or construct " + "OneLakeExternalDataSourceWrite with credentials." ) creds_write = OneLakeCredentialsWrite( client_id=self.settings.credentials.client_id, @@ -291,13 +276,42 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]: return result -class ExternalDataSourceWrite(ExternalDataSourceCore): - """Upsert model for a Fabric OneLake external data source (``external_data_sources.upsert()``). +class OneLakeExternalDataSource(ExternalDataSource): + """A Fabric OneLake external data source (API read model). OneLake external data sources are **read-only from a transform perspective** — transforms can read data from OneLake tables via ``ext_onelake()`` SQL, but writing transform output to OneLake is not supported. + Args: + external_id (str): External ID of the data source. + name (str | None): Human-readable name. + data_set_id (int | None): Data set ID for ACL scoping. + settings (OneLakeDataSourceSettings | None): Connection settings. + format (str | None): Backend format identifier (always ``"one_lake"``). + created_time (int | None): Time the resource was created (milliseconds since epoch). + last_updated_time (int | None): Time the resource was last updated (milliseconds since epoch). + """ + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + settings = OneLakeDataSourceSettings._load_if(resource.get("settings")) + return cls( + external_id=resource["externalId"], + name=resource.get("name"), + data_set_id=resource.get("dataSetId"), + settings=settings, + format=resource.get("format"), + created_time=resource.get("createdTime"), + last_updated_time=resource.get("lastUpdatedTime"), + ) + + +class ExternalDataSourceWrite(ExternalDataSourceCore): + """Upsert model for an external data source (``external_data_sources.upsert()``). + + Use :class:`OneLakeExternalDataSourceWrite` to register a Fabric OneLake source. + Args: external_id (str): External ID of the data source. name (str | None): Human-readable name. @@ -316,8 +330,66 @@ def __init__( self.settings = settings @classmethod - def onelake( - cls, + def _load(cls, resource: dict[str, Any]) -> Self: + settings = OneLakeDataSourceSettingsWrite._load_if(resource.get("settings")) + return cls( + external_id=resource["externalId"], + name=resource.get("name"), + data_set_id=resource.get("dataSetId"), + settings=settings, + ) + + def as_write(self) -> ExternalDataSourceWrite: + return self + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result = super().dump(camel_case=camel_case) + result["format"] = self._FORMAT + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=camel_case) + return result + + +class OneLakeExternalDataSourceWrite(ExternalDataSourceWrite): + """Upsert model for registering a Fabric OneLake external data source in CDF. + + Registers Azure credentials and lakehouse location so transforms can **read** via + ``ext_onelake()``. Does not write data into OneLake. + + Args: + external_id (str): External ID for the data source. Must be unique. + client_id (str): Azure application (client) ID. + tenant_id (str): Azure tenant (directory) ID. + client_secret (str): Azure client secret. + workspace_name (str): Fabric workspace GUID or name. + container_name (str): Fabric lakehouse GUID or name. + name (str | None): Human-readable name. + data_set_id (int | None): Data set ID for ACL scoping. + + Examples: + + Register a Fabric OneLake source: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.transformations import ( + ... OneLakeExternalDataSourceWrite, + ... ) + >>> client = CogniteClient() + >>> source = OneLakeExternalDataSourceWrite( + ... external_id="fabric-lakehouse-prod", + ... name="Production lakehouse", + ... client_id="", + ... tenant_id="", + ... client_secret="", + ... workspace_name="", + ... container_name="", + ... data_set_id=123456, + ... ) + >>> client.transformations.external_data_sources.upsert(source) + """ + + def __init__( + self, external_id: str, client_id: str, tenant_id: str, @@ -326,47 +398,8 @@ def onelake( container_name: str, name: str | None = None, data_set_id: int | None = None, - ) -> ExternalDataSourceWrite: - """Create an ``ExternalDataSourceWrite`` for registering a Fabric OneLake source in CDF. - - Registers Azure credentials and lakehouse location so transforms can **read** via - ``ext_onelake()``. Does not write data into OneLake. - - Args: - external_id (str): External ID for the data source. Must be unique. - client_id (str): Azure application (client) ID. - tenant_id (str): Azure tenant (directory) ID. - client_secret (str): Azure client secret. - workspace_name (str): Fabric workspace GUID or name. - container_name (str): Fabric lakehouse GUID or name. - name (str | None): Human-readable name. - data_set_id (int | None): Data set ID for ACL scoping. - - Returns: - ExternalDataSourceWrite: Ready to pass to ``client.transformations.external_data_sources.upsert()``. - - Examples: - - Register a Fabric OneLake source: - - >>> from cognite.client import CogniteClient - >>> from cognite.client.data_classes.transformations.external_data import ( - ... ExternalDataSourceWrite, - ... ) - >>> client = CogniteClient() - >>> source = ExternalDataSourceWrite.onelake( - ... external_id="fabric-lakehouse-prod", - ... name="Production lakehouse", - ... client_id="", - ... tenant_id="", - ... client_secret="", - ... workspace_name="", - ... container_name="", - ... data_set_id=123456, - ... ) - >>> client.transformations.external_data_sources.upsert(source) - """ - return cls( + ) -> None: + super().__init__( external_id=external_id, name=name, data_set_id=data_set_id, @@ -386,22 +419,18 @@ def onelake( @classmethod def _load(cls, resource: dict[str, Any]) -> Self: settings = OneLakeDataSourceSettingsWrite._load_if(resource.get("settings")) - return cls( - external_id=resource["externalId"], - name=resource.get("name"), - data_set_id=resource.get("dataSetId"), - settings=settings, - ) - - def as_write(self) -> ExternalDataSourceWrite: - return self - - def dump(self, camel_case: bool = True) -> dict[str, Any]: - result = super().dump(camel_case=camel_case) - result["format"] = self._FORMAT - if self.settings is not None: - result["settings"] = self.settings.dump(camel_case=camel_case) - return result + if settings is not None and settings.credentials is not None and settings.location_description is not None: + return cls( + external_id=resource["externalId"], + client_id=settings.credentials.client_id, + tenant_id=settings.credentials.tenant_id, + client_secret=settings.credentials.client_secret, + workspace_name=settings.location_description.workspace_name, + container_name=settings.location_description.container_name, + name=resource.get("name"), + data_set_id=resource.get("dataSetId"), + ) + return super()._load(resource) class ExternalDataSourceList( diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index b99a127845..15449c14eb 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -266,10 +266,10 @@ def test_writable_as_write( assert instance.external_id is not None if isinstance(instance, ExternalDataSource): - # Read model never includes client_secret; pass a placeholder when fake data has credentials. - write_format = instance.as_write(client_secret="placeholder-secret") - else: - write_format = instance.as_write() + if instance.settings is not None and instance.settings.credentials is not None: + instance.settings.credentials = None + + write_format = instance.as_write() assert isinstance(write_format, CogniteResource) @pytest.mark.dsl @@ -314,9 +314,11 @@ def test_writable_list_as_write( assert item.external_id is not None if isinstance(resource_list, ExternalDataSourceList): - write_format: CogniteResourceList = resource_list.as_write(client_secret="placeholder-secret") - else: - write_format = resource_list.as_write() + for item in resource_list: + if item.settings is not None and item.settings.credentials is not None: + item.settings.credentials = None + + write_format: CogniteResourceList = resource_list.as_write() assert isinstance(write_format, CogniteResourceList) @pytest.mark.dsl diff --git a/tests/tests_unit/test_data_classes/test_capabilities.py b/tests/tests_unit/test_data_classes/test_capabilities.py index f004308b9c..3eb42b9416 100644 --- a/tests/tests_unit/test_data_classes/test_capabilities.py +++ b/tests/tests_unit/test_data_classes/test_capabilities.py @@ -164,6 +164,13 @@ def all_acls() -> Iterator[dict[str, Any]]: {"timeSeriesAcl": {"actions": ["WRITE", "READ"], "scope": {"assetRootIdScope": {"rootIds": ["58"]}}}}, {"transformationsAcl": {"actions": ["READ", "WRITE"], "scope": {"all": {}}}}, {"transformationsAcl": {"actions": ["READ", "WRITE"], "scope": {"datasetScope": {"ids": ["94"]}}}}, + {"transformationsExternalDataSourcesAcl": {"actions": ["READ", "WRITE", "USE"], "scope": {"all": {}}}}, + { + "transformationsExternalDataSourcesAcl": { + "actions": ["READ", "WRITE", "USE"], + "scope": {"datasetScope": {"ids": ["94"]}}, + } + }, {"visionModelAcl": {"actions": ["READ", "WRITE"], "scope": {"all": {}}}}, {"wellsAcl": {"actions": ["READ", "WRITE"], "scope": {"all": {}}}}, {"workflowOrchestrationAcl": {"actions": ["READ", "WRITE"], "scope": {"all": {}}}}, diff --git a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py index 022688cc6b..bee8174721 100644 --- a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py +++ b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py @@ -6,11 +6,13 @@ ExternalDataSource, ExternalDataSourceWrite, OneLakeCredentialsWrite, + OneLakeExternalDataSource, + OneLakeExternalDataSourceWrite, ) -def test_onelake_factory_produces_valid_structure() -> None: - source = ExternalDataSourceWrite.onelake( +def test_onelake_write_init_structure() -> None: + source = OneLakeExternalDataSourceWrite( external_id="x", client_id="cid", tenant_id="tid", @@ -18,13 +20,14 @@ def test_onelake_factory_produces_valid_structure() -> None: workspace_name="ws", container_name="cn", ) - dumped = source.dump(camel_case=True) - assert dumped["format"] == "one_lake" - assert dumped["externalId"] == "x" - assert dumped["settings"]["credentials"]["clientId"] == "cid" - assert dumped["settings"]["credentials"]["clientSecret"] == "sec" - assert dumped["settings"]["locationDescription"]["workspaceName"] == "ws" + assert source.external_id == "x" + assert source.settings is not None + assert source.settings.credentials is not None + assert source.settings.credentials.client_id == "cid" + assert source.settings.credentials.client_secret == "sec" + assert source.settings.location_description is not None + assert source.settings.location_description.workspace_name == "ws" def test_write_dump_always_includes_format() -> None: @@ -35,6 +38,13 @@ def test_write_dump_always_includes_format() -> None: assert dumped["format"] == "one_lake" +def test_one_lake_load_returns_subclass() -> None: + raw = {"externalId": "x", "format": "one_lake", "settings": {}} + source = ExternalDataSource._load(raw) + + assert isinstance(source, OneLakeExternalDataSource) + + def test_read_load_unknown_format_warns_not_raises() -> None: raw = {"externalId": "x", "format": "delta_sharing", "settings": {}} @@ -43,6 +53,7 @@ def test_read_load_unknown_format_warns_not_raises() -> None: assert source.external_id == "x" assert source.format == "delta_sharing" + assert not isinstance(source, OneLakeExternalDataSource) def test_as_write_raises_without_client_secret() -> None: From 5487e7b574f4310b9b2d06f60a1ea2ca20245c02 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 16 Jul 2026 13:53:13 +0530 Subject: [PATCH 15/22] style(transformations): collapse external data re-exports Single import block with noqa F401; avoids ruff isort splitting as-aliases. Co-authored-by: Cursor --- .../data_classes/transformations/__init__.py | 48 +++++-------------- 1 file changed, 13 insertions(+), 35 deletions(-) diff --git a/cognite/client/data_classes/transformations/__init__.py b/cognite/client/data_classes/transformations/__init__.py index dc521829c8..28c5ac2a3e 100644 --- a/cognite/client/data_classes/transformations/__init__.py +++ b/cognite/client/data_classes/transformations/__init__.py @@ -26,41 +26,19 @@ TransformationBlockedInfo, TransformationDestination, ) -from cognite.client.data_classes.transformations.external_data import ( - ExternalDataSource as ExternalDataSource, -) -from cognite.client.data_classes.transformations.external_data import ( - ExternalDataSourceList as ExternalDataSourceList, -) -from cognite.client.data_classes.transformations.external_data import ( - ExternalDataSourceUsability as ExternalDataSourceUsability, -) -from cognite.client.data_classes.transformations.external_data import ( - ExternalDataSourceWrite as ExternalDataSourceWrite, -) -from cognite.client.data_classes.transformations.external_data import ( - ExternalDataSourceWriteList as ExternalDataSourceWriteList, -) -from cognite.client.data_classes.transformations.external_data import ( - OneLakeCredentials as OneLakeCredentials, -) -from cognite.client.data_classes.transformations.external_data import ( - OneLakeCredentialsWrite as OneLakeCredentialsWrite, -) -from cognite.client.data_classes.transformations.external_data import ( - OneLakeDataSourceSettings as OneLakeDataSourceSettings, -) -from cognite.client.data_classes.transformations.external_data import ( - OneLakeDataSourceSettingsWrite as OneLakeDataSourceSettingsWrite, -) -from cognite.client.data_classes.transformations.external_data import ( - OneLakeExternalDataSource as OneLakeExternalDataSource, -) -from cognite.client.data_classes.transformations.external_data import ( - OneLakeExternalDataSourceWrite as OneLakeExternalDataSourceWrite, -) -from cognite.client.data_classes.transformations.external_data import ( - OneLakeLocationDescription as OneLakeLocationDescription, +from cognite.client.data_classes.transformations.external_data import ( # noqa: F401 + ExternalDataSource, + ExternalDataSourceList, + ExternalDataSourceUsability, + ExternalDataSourceWrite, + ExternalDataSourceWriteList, + OneLakeCredentials, + OneLakeCredentialsWrite, + OneLakeDataSourceSettings, + OneLakeDataSourceSettingsWrite, + OneLakeExternalDataSource, + OneLakeExternalDataSourceWrite, + OneLakeLocationDescription, ) from cognite.client.data_classes.transformations.jobs import TransformationJob, TransformationJobList from cognite.client.data_classes.transformations.schedules import TransformationSchedule From 9a80848e7294f52323fc8616ad38a4137ecb8764 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 16 Jul 2026 14:17:03 +0530 Subject: [PATCH 16/22] refactor(transformations): add UnknownExternalDataSource and drop Core Return UnknownExternalDataSource for unsupported formats, require format on load per jetfire contract, and flatten ExternalDataSourceCore away. Co-authored-by: Cursor --- .../data_classes/transformations/__init__.py | 1 + .../transformations/external_data.py | 133 +++++++++++------- tests/tests_unit/test_base.py | 11 +- .../test_external_data.py | 25 +++- 4 files changed, 111 insertions(+), 59 deletions(-) diff --git a/cognite/client/data_classes/transformations/__init__.py b/cognite/client/data_classes/transformations/__init__.py index 28c5ac2a3e..c19c6a8a62 100644 --- a/cognite/client/data_classes/transformations/__init__.py +++ b/cognite/client/data_classes/transformations/__init__.py @@ -39,6 +39,7 @@ OneLakeExternalDataSource, OneLakeExternalDataSourceWrite, OneLakeLocationDescription, + UnknownExternalDataSource, ) from cognite.client.data_classes.transformations.jobs import TransformationJob, TransformationJobList from cognite.client.data_classes.transformations.schedules import TransformationSchedule diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index f38ec0a95d..5a4459e956 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -1,7 +1,5 @@ from __future__ import annotations -import warnings -from abc import ABC from typing import Any, ClassVar, cast from typing_extensions import Self @@ -13,6 +11,9 @@ WriteableCogniteResource, WriteableCogniteResourceList, ) +from cognite.client.utils._text import convert_all_keys_recursive + +ONE_LAKE_FORMAT = "one_lake" class OneLakeLocationDescription(CogniteResource): @@ -157,43 +158,17 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]: return result -class ExternalDataSourceCore(WriteableCogniteResource["ExternalDataSourceWrite"], ABC): - """Shared base for ``ExternalDataSource`` (API read model) and ``ExternalDataSourceWrite`` (API upsert model). - - OneLake external data sources are **read-only from a transform perspective** — transforms can - read data from OneLake tables via ``ext_onelake()`` SQL, but writing transform output to OneLake - is not supported. - - Args: - external_id (str): External ID of the data source. Must be unique within the project. - name (str | None): Human-readable name for the data source. - data_set_id (int | None): ID of the data set that owns this resource (for ACL scoping). - """ - - _FORMAT: ClassVar[str] = "one_lake" - - def __init__( - self, - external_id: str, - name: str | None = None, - data_set_id: int | None = None, - ) -> None: - self.external_id = external_id - self.name = name - self.data_set_id = data_set_id - - -class ExternalDataSource(ExternalDataSourceCore): +class ExternalDataSource(WriteableCogniteResource["ExternalDataSourceWrite"]): """An external data source (API read model — returned by list/get). Use :class:`OneLakeExternalDataSource` for Fabric OneLake sources. Args: external_id (str): External ID of the data source. + format (str): Backend format identifier (``"one_lake"`` for OneLake sources). name (str | None): Human-readable name. data_set_id (int | None): Data set ID for ACL scoping. settings (OneLakeDataSourceSettings | None): Connection settings. - format (str | None): Backend format identifier (``"one_lake"`` for OneLake sources). created_time (int | None): Time the resource was created (milliseconds since epoch). last_updated_time (int | None): Time the resource was last updated (milliseconds since epoch). """ @@ -201,14 +176,16 @@ class ExternalDataSource(ExternalDataSourceCore): def __init__( self, external_id: str, + format: str, name: str | None = None, data_set_id: int | None = None, settings: OneLakeDataSourceSettings | None = None, - format: str | None = None, created_time: int | None = None, last_updated_time: int | None = None, ) -> None: - super().__init__(external_id=external_id, name=name, data_set_id=data_set_id) + self.external_id = external_id + self.name = name + self.data_set_id = data_set_id self.settings = settings self.format = format self.created_time = created_time @@ -216,24 +193,10 @@ def __init__( @classmethod def _load(cls, resource: dict[str, Any]) -> Self: - fmt = resource.get("format") - if fmt is None or fmt == cls._FORMAT: + fmt = resource["format"] + if fmt == ONE_LAKE_FORMAT: return cast(Self, OneLakeExternalDataSource._load(resource)) - warnings.warn( - f"Unknown external data source format: {fmt!r}. This version of the SDK may not fully support it.", - UserWarning, - stacklevel=2, - ) - settings = OneLakeDataSourceSettings._load_if(resource.get("settings")) - return cls( - external_id=resource["externalId"], - name=resource.get("name"), - data_set_id=resource.get("dataSetId"), - settings=settings, - format=fmt, - created_time=resource.get("createdTime"), - last_updated_time=resource.get("lastUpdatedTime"), - ) + return cast(Self, UnknownExternalDataSource._load(resource)) def as_write(self, client_secret: str | None = None) -> ExternalDataSourceWrite: """Return an upsert model for updating this source in CDF. @@ -285,10 +248,10 @@ class OneLakeExternalDataSource(ExternalDataSource): Args: external_id (str): External ID of the data source. + format (str): Backend format identifier (always ``"one_lake"``). name (str | None): Human-readable name. data_set_id (int | None): Data set ID for ACL scoping. settings (OneLakeDataSourceSettings | None): Connection settings. - format (str | None): Backend format identifier (always ``"one_lake"``). created_time (int | None): Time the resource was created (milliseconds since epoch). last_updated_time (int | None): Time the resource was last updated (milliseconds since epoch). """ @@ -298,16 +261,76 @@ def _load(cls, resource: dict[str, Any]) -> Self: settings = OneLakeDataSourceSettings._load_if(resource.get("settings")) return cls( external_id=resource["externalId"], + format=resource["format"], + name=resource.get("name"), + data_set_id=resource.get("dataSetId"), + settings=settings, + created_time=resource.get("createdTime"), + last_updated_time=resource.get("lastUpdatedTime"), + ) + + +class UnknownExternalDataSource(ExternalDataSource): + """An external data source with a format not supported by this SDK version. + + Args: + external_id (str): External ID of the data source. + format (str): Backend format identifier returned by the API. + name (str | None): Human-readable name. + data_set_id (int | None): Data set ID for ACL scoping. + settings (OneLakeDataSourceSettings | None): Connection settings, if present in the payload. + created_time (int | None): Time the resource was created (milliseconds since epoch). + last_updated_time (int | None): Time the resource was last updated (milliseconds since epoch). + """ + + def __init__( + self, + external_id: str, + format: str, + name: str | None = None, + data_set_id: int | None = None, + settings: OneLakeDataSourceSettings | None = None, + created_time: int | None = None, + last_updated_time: int | None = None, + ) -> None: + super().__init__( + external_id=external_id, + format=format, + name=name, + data_set_id=data_set_id, + settings=settings, + created_time=created_time, + last_updated_time=last_updated_time, + ) + self._raw: dict[str, Any] | None = None + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + settings = OneLakeDataSourceSettings._load_if(resource.get("settings")) + instance = cls( + external_id=resource["externalId"], + format=resource["format"], name=resource.get("name"), data_set_id=resource.get("dataSetId"), settings=settings, - format=resource.get("format"), created_time=resource.get("createdTime"), last_updated_time=resource.get("lastUpdatedTime"), ) + instance._raw = resource + return instance + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + if self._raw is not None: + return convert_all_keys_recursive(self._raw, camel_case=camel_case) + return super().dump(camel_case=camel_case) + + def as_write(self, client_secret: str | None = None) -> ExternalDataSourceWrite: + raise ValueError( + f"Cannot convert unknown external data source format {self.format!r} to a write model in this SDK version." + ) -class ExternalDataSourceWrite(ExternalDataSourceCore): +class ExternalDataSourceWrite(WriteableCogniteResource["ExternalDataSourceWrite"]): """Upsert model for an external data source (``external_data_sources.upsert()``). Use :class:`OneLakeExternalDataSourceWrite` to register a Fabric OneLake source. @@ -319,6 +342,8 @@ class ExternalDataSourceWrite(ExternalDataSourceCore): settings (OneLakeDataSourceSettingsWrite | None): Connection settings including client secret. """ + _FORMAT: ClassVar[str] = ONE_LAKE_FORMAT + def __init__( self, external_id: str, @@ -326,7 +351,9 @@ def __init__( data_set_id: int | None = None, settings: OneLakeDataSourceSettingsWrite | None = None, ) -> None: - super().__init__(external_id=external_id, name=name, data_set_id=data_set_id) + self.external_id = external_id + self.name = name + self.data_set_id = data_set_id self.settings = settings @classmethod diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index 15449c14eb..7d1428f2fe 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -46,7 +46,11 @@ from cognite.client.data_classes.postgres_gateway import TableList, User, UserCreated, UserCreatedList, UserList from cognite.client.data_classes.sequences import SequenceUpdate from cognite.client.data_classes.time_series import TimeSeries, TimeSeriesList -from cognite.client.data_classes.transformations.external_data import ExternalDataSource, ExternalDataSourceList +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSource, + ExternalDataSourceList, + UnknownExternalDataSource, +) from cognite.client.data_classes.workflows import UnknownWorkflowTaskParameters, WorkflowTaskOutput from cognite.client.exceptions import CogniteMissingClientError from cognite.client.testing import CogniteClientMock @@ -265,6 +269,11 @@ def test_writable_as_write( instance.instance_id = None assert instance.external_id is not None + if isinstance(instance, UnknownExternalDataSource): + with pytest.raises(ValueError, match="unknown external data source format"): + instance.as_write() + return + if isinstance(instance, ExternalDataSource): if instance.settings is not None and instance.settings.credentials is not None: instance.settings.credentials = None diff --git a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py index bee8174721..6a6d9dd570 100644 --- a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py +++ b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py @@ -8,6 +8,7 @@ OneLakeCredentialsWrite, OneLakeExternalDataSource, OneLakeExternalDataSourceWrite, + UnknownExternalDataSource, ) @@ -43,17 +44,31 @@ def test_one_lake_load_returns_subclass() -> None: source = ExternalDataSource._load(raw) assert isinstance(source, OneLakeExternalDataSource) + assert source.format == "one_lake" -def test_read_load_unknown_format_warns_not_raises() -> None: +def test_read_load_unknown_format_returns_unknown_subclass() -> None: raw = {"externalId": "x", "format": "delta_sharing", "settings": {}} + source = ExternalDataSource._load(raw) - with pytest.warns(UserWarning, match="Unknown external data source format"): - source = ExternalDataSource._load(raw) - + assert isinstance(source, UnknownExternalDataSource) assert source.external_id == "x" assert source.format == "delta_sharing" - assert not isinstance(source, OneLakeExternalDataSource) + assert source.dump(camel_case=True) == raw + + +def test_read_load_requires_format() -> None: + raw = {"externalId": "x", "settings": {}} + + with pytest.raises(KeyError, match="format"): + ExternalDataSource._load(raw) + + +def test_unknown_as_write_raises() -> None: + source = UnknownExternalDataSource(external_id="x", format="delta_sharing") + + with pytest.raises(ValueError, match="unknown external data source format"): + source.as_write() def test_as_write_raises_without_client_secret() -> None: From f7976e8a2db032241ea379edfb74a59ea941c566 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 16 Jul 2026 14:30:59 +0530 Subject: [PATCH 17/22] refactor(transformations): move OneLake settings off base read model ExternalDataSource is now format-agnostic; settings and as_write() live on OneLakeExternalDataSource. UnknownExternalDataSource has no typed settings. Write base drops settings; OneLakeExternalDataSourceWrite owns format-specific dump/upsert shape. Co-authored-by: Cursor --- .../transformations/external_data.py | 177 +++++++++++------- tests/tests_unit/test_base.py | 18 +- .../test_external_data.py | 24 ++- 3 files changed, 142 insertions(+), 77 deletions(-) diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index 5a4459e956..b747deb686 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -161,14 +161,13 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]: class ExternalDataSource(WriteableCogniteResource["ExternalDataSourceWrite"]): """An external data source (API read model — returned by list/get). - Use :class:`OneLakeExternalDataSource` for Fabric OneLake sources. + Format-specific subclasses (e.g. :class:`OneLakeExternalDataSource`) hold typed settings. Args: external_id (str): External ID of the data source. format (str): Backend format identifier (``"one_lake"`` for OneLake sources). name (str | None): Human-readable name. data_set_id (int | None): Data set ID for ACL scoping. - settings (OneLakeDataSourceSettings | None): Connection settings. created_time (int | None): Time the resource was created (milliseconds since epoch). last_updated_time (int | None): Time the resource was last updated (milliseconds since epoch). """ @@ -179,14 +178,12 @@ def __init__( format: str, name: str | None = None, data_set_id: int | None = None, - settings: OneLakeDataSourceSettings | None = None, created_time: int | None = None, last_updated_time: int | None = None, ) -> None: self.external_id = external_id self.name = name self.data_set_id = data_set_id - self.settings = settings self.format = format self.created_time = created_time self.last_updated_time = last_updated_time @@ -202,42 +199,15 @@ def as_write(self, client_secret: str | None = None) -> ExternalDataSourceWrite: """Return an upsert model for updating this source in CDF. Args: - client_secret (str | None): Required when the read model includes credentials, because the API does not return ``client_secret``. Omit when only metadata (name, data set, location) changes. + client_secret (str | None): Passed through on format-specific subclasses. Returns: ExternalDataSourceWrite: Upsert model for this source. """ - settings_write: OneLakeDataSourceSettingsWrite | None = None - if self.settings is not None: - creds_write: OneLakeCredentialsWrite | None = None - if self.settings.credentials is not None: - if client_secret is None: - raise ValueError( - "client_secret is required to convert credentials to a write model because the API " - "does not return it. Pass client_secret to as_write(), or construct " - "OneLakeExternalDataSourceWrite with credentials." - ) - creds_write = OneLakeCredentialsWrite( - client_id=self.settings.credentials.client_id, - tenant_id=self.settings.credentials.tenant_id, - client_secret=client_secret, - ) - settings_write = OneLakeDataSourceSettingsWrite( - credentials=creds_write, - location_description=self.settings.location_description, - ) - return ExternalDataSourceWrite( - external_id=self.external_id, - name=self.name, - data_set_id=self.data_set_id, - settings=settings_write, + raise NotImplementedError( + "as_write() is only supported on format-specific external data source subclasses; " + "list/get responses are loaded as OneLakeExternalDataSource or UnknownExternalDataSource." ) - def dump(self, camel_case: bool = True) -> dict[str, Any]: - result = super().dump(camel_case=camel_case) - if self.settings is not None: - result["settings"] = self.settings.dump(camel_case=camel_case) - return result - class OneLakeExternalDataSource(ExternalDataSource): """A Fabric OneLake external data source (API read model). @@ -256,6 +226,26 @@ class OneLakeExternalDataSource(ExternalDataSource): last_updated_time (int | None): Time the resource was last updated (milliseconds since epoch). """ + def __init__( + self, + external_id: str, + format: str, + name: str | None = None, + data_set_id: int | None = None, + settings: OneLakeDataSourceSettings | None = None, + created_time: int | None = None, + last_updated_time: int | None = None, + ) -> None: + super().__init__( + external_id=external_id, + format=format, + name=name, + data_set_id=data_set_id, + created_time=created_time, + last_updated_time=last_updated_time, + ) + self.settings = settings + @classmethod def _load(cls, resource: dict[str, Any]) -> Self: settings = OneLakeDataSourceSettings._load_if(resource.get("settings")) @@ -269,6 +259,46 @@ def _load(cls, resource: dict[str, Any]) -> Self: last_updated_time=resource.get("lastUpdatedTime"), ) + def as_write(self, client_secret: str | None = None) -> OneLakeExternalDataSourceWrite: + """Return an upsert model for updating this source in CDF. + + Args: + client_secret (str | None): Required when the read model includes credentials, because the API does not return ``client_secret``. Omit when only metadata (name, data set, location) changes. + Returns: + OneLakeExternalDataSourceWrite: Upsert model for this source. + """ + settings_write: OneLakeDataSourceSettingsWrite | None = None + if self.settings is not None: + creds_write: OneLakeCredentialsWrite | None = None + if self.settings.credentials is not None: + if client_secret is None: + raise ValueError( + "client_secret is required to convert credentials to a write model because the API " + "does not return it. Pass client_secret to as_write(), or construct " + "OneLakeExternalDataSourceWrite with credentials." + ) + creds_write = OneLakeCredentialsWrite( + client_id=self.settings.credentials.client_id, + tenant_id=self.settings.credentials.tenant_id, + client_secret=client_secret, + ) + settings_write = OneLakeDataSourceSettingsWrite( + credentials=creds_write, + location_description=self.settings.location_description, + ) + return OneLakeExternalDataSourceWrite.with_settings( + external_id=self.external_id, + name=self.name, + data_set_id=self.data_set_id, + settings=settings_write, + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result = super().dump(camel_case=camel_case) + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=camel_case) + return result + class UnknownExternalDataSource(ExternalDataSource): """An external data source with a format not supported by this SDK version. @@ -278,7 +308,6 @@ class UnknownExternalDataSource(ExternalDataSource): format (str): Backend format identifier returned by the API. name (str | None): Human-readable name. data_set_id (int | None): Data set ID for ACL scoping. - settings (OneLakeDataSourceSettings | None): Connection settings, if present in the payload. created_time (int | None): Time the resource was created (milliseconds since epoch). last_updated_time (int | None): Time the resource was last updated (milliseconds since epoch). """ @@ -289,7 +318,6 @@ def __init__( format: str, name: str | None = None, data_set_id: int | None = None, - settings: OneLakeDataSourceSettings | None = None, created_time: int | None = None, last_updated_time: int | None = None, ) -> None: @@ -298,7 +326,6 @@ def __init__( format=format, name=name, data_set_id=data_set_id, - settings=settings, created_time=created_time, last_updated_time=last_updated_time, ) @@ -306,13 +333,11 @@ def __init__( @classmethod def _load(cls, resource: dict[str, Any]) -> Self: - settings = OneLakeDataSourceSettings._load_if(resource.get("settings")) instance = cls( external_id=resource["externalId"], format=resource["format"], name=resource.get("name"), data_set_id=resource.get("dataSetId"), - settings=settings, created_time=resource.get("createdTime"), last_updated_time=resource.get("lastUpdatedTime"), ) @@ -339,43 +364,31 @@ class ExternalDataSourceWrite(WriteableCogniteResource["ExternalDataSourceWrite" external_id (str): External ID of the data source. name (str | None): Human-readable name. data_set_id (int | None): Data set ID for ACL scoping. - settings (OneLakeDataSourceSettingsWrite | None): Connection settings including client secret. """ - _FORMAT: ClassVar[str] = ONE_LAKE_FORMAT - def __init__( self, external_id: str, name: str | None = None, data_set_id: int | None = None, - settings: OneLakeDataSourceSettingsWrite | None = None, ) -> None: self.external_id = external_id self.name = name self.data_set_id = data_set_id - self.settings = settings @classmethod def _load(cls, resource: dict[str, Any]) -> Self: - settings = OneLakeDataSourceSettingsWrite._load_if(resource.get("settings")) + if resource.get("format") == ONE_LAKE_FORMAT: + return cast(Self, OneLakeExternalDataSourceWrite._load(resource)) return cls( external_id=resource["externalId"], name=resource.get("name"), data_set_id=resource.get("dataSetId"), - settings=settings, ) def as_write(self) -> ExternalDataSourceWrite: return self - def dump(self, camel_case: bool = True) -> dict[str, Any]: - result = super().dump(camel_case=camel_case) - result["format"] = self._FORMAT - if self.settings is not None: - result["settings"] = self.settings.dump(camel_case=camel_case) - return result - class OneLakeExternalDataSourceWrite(ExternalDataSourceWrite): """Upsert model for registering a Fabric OneLake external data source in CDF. @@ -415,6 +428,8 @@ class OneLakeExternalDataSourceWrite(ExternalDataSourceWrite): >>> client.transformations.external_data_sources.upsert(source) """ + _FORMAT: ClassVar[str] = ONE_LAKE_FORMAT + def __init__( self, external_id: str, @@ -426,23 +441,33 @@ def __init__( name: str | None = None, data_set_id: int | None = None, ) -> None: - super().__init__( - external_id=external_id, - name=name, - data_set_id=data_set_id, - settings=OneLakeDataSourceSettingsWrite( - credentials=OneLakeCredentialsWrite( - client_id=client_id, - tenant_id=tenant_id, - client_secret=client_secret, - ), - location_description=OneLakeLocationDescription( - workspace_name=workspace_name, - container_name=container_name, - ), + super().__init__(external_id=external_id, name=name, data_set_id=data_set_id) + self.settings = OneLakeDataSourceSettingsWrite( + credentials=OneLakeCredentialsWrite( + client_id=client_id, + tenant_id=tenant_id, + client_secret=client_secret, + ), + location_description=OneLakeLocationDescription( + workspace_name=workspace_name, + container_name=container_name, ), ) + @classmethod + def with_settings( + cls, + external_id: str, + name: str | None = None, + data_set_id: int | None = None, + settings: OneLakeDataSourceSettingsWrite | None = None, + ) -> OneLakeExternalDataSourceWrite: + """Build a write model from metadata and/or partial OneLake settings (used by ``as_write()``).""" + instance = cls.__new__(cls) + ExternalDataSourceWrite.__init__(instance, external_id, name, data_set_id) + instance.settings = settings + return instance + @classmethod def _load(cls, resource: dict[str, Any]) -> Self: settings = OneLakeDataSourceSettingsWrite._load_if(resource.get("settings")) @@ -457,7 +482,19 @@ def _load(cls, resource: dict[str, Any]) -> Self: name=resource.get("name"), data_set_id=resource.get("dataSetId"), ) - return super()._load(resource) + return cls.with_settings( + external_id=resource["externalId"], + name=resource.get("name"), + data_set_id=resource.get("dataSetId"), + settings=settings, + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result = super().dump(camel_case=camel_case) + result["format"] = self._FORMAT + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=camel_case) + return result class ExternalDataSourceList( @@ -471,7 +508,7 @@ def as_write(self, client_secret: str | None = None) -> ExternalDataSourceWriteL """Return upsert models for each source. Args: - client_secret (str | None): Passed through to :meth:`ExternalDataSource.as_write` for each item. + client_secret (str | None): Passed through to :meth:`OneLakeExternalDataSource.as_write` for each item. Returns: ExternalDataSourceWriteList: Upsert models for each source in the list. """ diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index 7d1428f2fe..0b3044eb35 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -49,6 +49,7 @@ from cognite.client.data_classes.transformations.external_data import ( ExternalDataSource, ExternalDataSourceList, + OneLakeExternalDataSource, UnknownExternalDataSource, ) from cognite.client.data_classes.workflows import UnknownWorkflowTaskParameters, WorkflowTaskOutput @@ -274,10 +275,15 @@ def test_writable_as_write( instance.as_write() return - if isinstance(instance, ExternalDataSource): + if isinstance(instance, OneLakeExternalDataSource): if instance.settings is not None and instance.settings.credentials is not None: instance.settings.credentials = None + if type(instance) is ExternalDataSource: + with pytest.raises(NotImplementedError, match="format-specific"): + instance.as_write() + return + write_format = instance.as_write() assert isinstance(write_format, CogniteResource) @@ -307,7 +313,10 @@ def test_writable_list_as_write( resource_cls = writable_list_cls._RESOURCE instance_generator = FakeCogniteResourceGenerator(seed=52, async_client=cognite_async_mock_client_placeholder) # TODO(doctrino): Why not have gen. create the list directly? - resource_list = writable_list_cls([instance_generator.create_instance(resource_cls)]) + if writable_list_cls is ExternalDataSourceList: + resource_list = ExternalDataSourceList([instance_generator.create_instance(OneLakeExternalDataSource)]) + else: + resource_list = writable_list_cls([instance_generator.create_instance(resource_cls)]) if resource_cls is FunctionSchedule: # FunctionSchedulesList.as_write() calls FunctionSchedule.as_write() on each item, @@ -324,8 +333,9 @@ def test_writable_list_as_write( if isinstance(resource_list, ExternalDataSourceList): for item in resource_list: - if item.settings is not None and item.settings.credentials is not None: - item.settings.credentials = None + if isinstance(item, OneLakeExternalDataSource) and item.settings is not None: + if item.settings.credentials is not None: + item.settings.credentials = None write_format: CogniteResourceList = resource_list.as_write() assert isinstance(write_format, CogniteResourceList) diff --git a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py index 6a6d9dd570..94a36faab0 100644 --- a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py +++ b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py @@ -4,7 +4,6 @@ from cognite.client.data_classes.transformations.external_data import ( ExternalDataSource, - ExternalDataSourceWrite, OneLakeCredentialsWrite, OneLakeExternalDataSource, OneLakeExternalDataSourceWrite, @@ -32,11 +31,27 @@ def test_onelake_write_init_structure() -> None: def test_write_dump_always_includes_format() -> None: - source = ExternalDataSourceWrite(external_id="x") + source = OneLakeExternalDataSourceWrite( + external_id="x", + client_id="cid", + tenant_id="tid", + client_secret="sec", + workspace_name="ws", + container_name="cn", + ) dumped = source.dump(camel_case=True) assert "format" in dumped assert dumped["format"] == "one_lake" + assert dumped["settings"]["credentials"]["clientId"] == "cid" + + +def test_base_write_has_no_settings_field() -> None: + source = OneLakeExternalDataSourceWrite.with_settings(external_id="x", settings=None) + dumped = source.dump(camel_case=True) + + assert dumped["externalId"] == "x" + assert "settings" not in dumped or dumped.get("settings") is None def test_one_lake_load_returns_subclass() -> None: @@ -54,6 +69,7 @@ def test_read_load_unknown_format_returns_unknown_subclass() -> None: assert isinstance(source, UnknownExternalDataSource) assert source.external_id == "x" assert source.format == "delta_sharing" + assert not hasattr(source, "settings") or getattr(source, "settings", None) is None assert source.dump(camel_case=True) == raw @@ -81,6 +97,7 @@ def test_as_write_raises_without_client_secret() -> None: }, } read_source = ExternalDataSource._load(raw) + assert isinstance(read_source, OneLakeExternalDataSource) with pytest.raises(ValueError, match="client_secret is required"): read_source.as_write() @@ -96,9 +113,10 @@ def test_as_write_with_client_secret() -> None: }, } read_source = ExternalDataSource._load(raw) + assert isinstance(read_source, OneLakeExternalDataSource) write_source = read_source.as_write(client_secret="new-secret") - assert isinstance(write_source, ExternalDataSourceWrite) + assert isinstance(write_source, OneLakeExternalDataSourceWrite) assert write_source.settings is not None assert write_source.settings.credentials is not None assert write_source.settings.credentials.client_secret == "new-secret" From 45c96c8a91d8375acde8222d7b70279b82261fe9 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 16 Jul 2026 14:38:13 +0530 Subject: [PATCH 18/22] fix(transformations): satisfy mypy on external data refactor Annotate optional settings on OneLakeExternalDataSourceWrite, cast _load dispatch return, and narrow test_base list/as_write helpers for mypy. Co-authored-by: Cursor --- .../data_classes/transformations/external_data.py | 14 +++++++++----- tests/tests_unit/test_base.py | 6 ++++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index b747deb686..bf75076dad 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -429,6 +429,7 @@ class OneLakeExternalDataSourceWrite(ExternalDataSourceWrite): """ _FORMAT: ClassVar[str] = ONE_LAKE_FORMAT + settings: OneLakeDataSourceSettingsWrite | None def __init__( self, @@ -482,11 +483,14 @@ def _load(cls, resource: dict[str, Any]) -> Self: name=resource.get("name"), data_set_id=resource.get("dataSetId"), ) - return cls.with_settings( - external_id=resource["externalId"], - name=resource.get("name"), - data_set_id=resource.get("dataSetId"), - settings=settings, + return cast( + Self, + cls.with_settings( + external_id=resource["externalId"], + name=resource.get("name"), + data_set_id=resource.get("dataSetId"), + settings=settings, + ), ) def dump(self, camel_case: bool = True) -> dict[str, Any]: diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index 0b3044eb35..6f020048dc 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -313,6 +313,7 @@ def test_writable_list_as_write( resource_cls = writable_list_cls._RESOURCE instance_generator = FakeCogniteResourceGenerator(seed=52, async_client=cognite_async_mock_client_placeholder) # TODO(doctrino): Why not have gen. create the list directly? + resource_list: WriteableCogniteResourceList[Any, Any] if writable_list_cls is ExternalDataSourceList: resource_list = ExternalDataSourceList([instance_generator.create_instance(OneLakeExternalDataSource)]) else: @@ -328,8 +329,9 @@ def test_writable_list_as_write( # Files and time series with instance ID can not be created through "old APIs". Doing as_write with # instance_id set raises an error, so we clear it here: for item in resource_list: - item.instance_id = None - assert item.external_id is not None + if isinstance(item, (FileMetadata, TimeSeries)): + item.instance_id = None + assert item.external_id is not None if isinstance(resource_list, ExternalDataSourceList): for item in resource_list: From 2ef4b6a9e774dd32dd69f1eeee2bc1eebd4732fc Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 16 Jul 2026 15:14:14 +0530 Subject: [PATCH 19/22] fix(transformations): return ExternalDataSource from _load dispatch Use explicit base return type instead of Self when delegating to format subclasses, satisfying mypy on polymorphic _load. --- .../client/data_classes/transformations/external_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index bf75076dad..ec0f3c7e11 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -189,11 +189,11 @@ def __init__( self.last_updated_time = last_updated_time @classmethod - def _load(cls, resource: dict[str, Any]) -> Self: + def _load(cls, resource: dict[str, Any]) -> ExternalDataSource: fmt = resource["format"] if fmt == ONE_LAKE_FORMAT: - return cast(Self, OneLakeExternalDataSource._load(resource)) - return cast(Self, UnknownExternalDataSource._load(resource)) + return OneLakeExternalDataSource._load(resource) + return UnknownExternalDataSource._load(resource) def as_write(self, client_secret: str | None = None) -> ExternalDataSourceWrite: """Return an upsert model for updating this source in CDF. From 4e3151f7b5f819f20b23c552eb590b140a6585ed Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 16 Jul 2026 15:15:16 +0530 Subject: [PATCH 20/22] fix(transformations): annotate UnknownExternalDataSource.as_write NoReturn as_write always raises ValueError for unknown formats; NoReturn makes the contract explicit to mypy. --- cognite/client/data_classes/transformations/external_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index ec0f3c7e11..33a1746d4f 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, ClassVar, cast +from typing import Any, ClassVar, NoReturn, cast from typing_extensions import Self @@ -349,7 +349,7 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]: return convert_all_keys_recursive(self._raw, camel_case=camel_case) return super().dump(camel_case=camel_case) - def as_write(self, client_secret: str | None = None) -> ExternalDataSourceWrite: + def as_write(self, client_secret: str | None = None) -> NoReturn: raise ValueError( f"Cannot convert unknown external data source format {self.format!r} to a write model in this SDK version." ) From f7888df0e1995b997982021d9ba37706136ebac6 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 16 Jul 2026 15:24:57 +0530 Subject: [PATCH 21/22] fix(transformations): abstract read base, drop with_settings Make ExternalDataSource an ABC so test_base skips it. Fold partial settings into OneLakeExternalDataSourceWrite.__init__ and remove the __new__/with_settings factory. Revert unrelated FileMetadataList loop change; mypy ignore documents the list-item union only. --- .../transformations/external_data.py | 94 +++++++++---------- tests/tests_unit/test_base.py | 11 +-- .../test_external_data.py | 2 +- 3 files changed, 48 insertions(+), 59 deletions(-) diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index 33a1746d4f..60cd4bdacb 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -1,5 +1,6 @@ from __future__ import annotations +from abc import ABC, abstractmethod from typing import Any, ClassVar, NoReturn, cast from typing_extensions import Self @@ -158,7 +159,7 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]: return result -class ExternalDataSource(WriteableCogniteResource["ExternalDataSourceWrite"]): +class ExternalDataSource(WriteableCogniteResource["ExternalDataSourceWrite"], ABC): """An external data source (API read model — returned by list/get). Format-specific subclasses (e.g. :class:`OneLakeExternalDataSource`) hold typed settings. @@ -195,6 +196,7 @@ def _load(cls, resource: dict[str, Any]) -> ExternalDataSource: return OneLakeExternalDataSource._load(resource) return UnknownExternalDataSource._load(resource) + @abstractmethod def as_write(self, client_secret: str | None = None) -> ExternalDataSourceWrite: """Return an upsert model for updating this source in CDF. @@ -203,10 +205,6 @@ def as_write(self, client_secret: str | None = None) -> ExternalDataSourceWrite: Returns: ExternalDataSourceWrite: Upsert model for this source. """ - raise NotImplementedError( - "as_write() is only supported on format-specific external data source subclasses; " - "list/get responses are loaded as OneLakeExternalDataSource or UnknownExternalDataSource." - ) class OneLakeExternalDataSource(ExternalDataSource): @@ -286,7 +284,7 @@ def as_write(self, client_secret: str | None = None) -> OneLakeExternalDataSourc credentials=creds_write, location_description=self.settings.location_description, ) - return OneLakeExternalDataSourceWrite.with_settings( + return OneLakeExternalDataSourceWrite( external_id=self.external_id, name=self.name, data_set_id=self.data_set_id, @@ -398,13 +396,15 @@ class OneLakeExternalDataSourceWrite(ExternalDataSourceWrite): Args: external_id (str): External ID for the data source. Must be unique. - client_id (str): Azure application (client) ID. - tenant_id (str): Azure tenant (directory) ID. - client_secret (str): Azure client secret. - workspace_name (str): Fabric workspace GUID or name. - container_name (str): Fabric lakehouse GUID or name. + client_id (str | None): Azure application (client) ID. + tenant_id (str | None): Azure tenant (directory) ID. + client_secret (str | None): Azure client secret. + workspace_name (str | None): Fabric workspace GUID or name. + container_name (str | None): Fabric lakehouse GUID or name. name (str | None): Human-readable name. data_set_id (int | None): Data set ID for ACL scoping. + settings (OneLakeDataSourceSettingsWrite | None): Pre-built settings tree; used by + ``as_write()`` for partial upserts when credential/location fields are omitted. Examples: @@ -434,40 +434,39 @@ class OneLakeExternalDataSourceWrite(ExternalDataSourceWrite): def __init__( self, external_id: str, - client_id: str, - tenant_id: str, - client_secret: str, - workspace_name: str, - container_name: str, + client_id: str | None = None, + tenant_id: str | None = None, + client_secret: str | None = None, + workspace_name: str | None = None, + container_name: str | None = None, name: str | None = None, data_set_id: int | None = None, + *, + settings: OneLakeDataSourceSettingsWrite | None = None, ) -> None: super().__init__(external_id=external_id, name=name, data_set_id=data_set_id) - self.settings = OneLakeDataSourceSettingsWrite( - credentials=OneLakeCredentialsWrite( - client_id=client_id, - tenant_id=tenant_id, - client_secret=client_secret, - ), - location_description=OneLakeLocationDescription( - workspace_name=workspace_name, - container_name=container_name, - ), - ) - - @classmethod - def with_settings( - cls, - external_id: str, - name: str | None = None, - data_set_id: int | None = None, - settings: OneLakeDataSourceSettingsWrite | None = None, - ) -> OneLakeExternalDataSourceWrite: - """Build a write model from metadata and/or partial OneLake settings (used by ``as_write()``).""" - instance = cls.__new__(cls) - ExternalDataSourceWrite.__init__(instance, external_id, name, data_set_id) - instance.settings = settings - return instance + if settings is not None: + self.settings = settings + elif ( + client_id is not None + and tenant_id is not None + and client_secret is not None + and workspace_name is not None + and container_name is not None + ): + self.settings = OneLakeDataSourceSettingsWrite( + credentials=OneLakeCredentialsWrite( + client_id=client_id, + tenant_id=tenant_id, + client_secret=client_secret, + ), + location_description=OneLakeLocationDescription( + workspace_name=workspace_name, + container_name=container_name, + ), + ) + else: + self.settings = None @classmethod def _load(cls, resource: dict[str, Any]) -> Self: @@ -483,14 +482,11 @@ def _load(cls, resource: dict[str, Any]) -> Self: name=resource.get("name"), data_set_id=resource.get("dataSetId"), ) - return cast( - Self, - cls.with_settings( - external_id=resource["externalId"], - name=resource.get("name"), - data_set_id=resource.get("dataSetId"), - settings=settings, - ), + return cls( + external_id=resource["externalId"], + name=resource.get("name"), + data_set_id=resource.get("dataSetId"), + settings=settings, ) def dump(self, camel_case: bool = True) -> dict[str, Any]: diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index 6f020048dc..5c55894e84 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -47,7 +47,6 @@ from cognite.client.data_classes.sequences import SequenceUpdate from cognite.client.data_classes.time_series import TimeSeries, TimeSeriesList from cognite.client.data_classes.transformations.external_data import ( - ExternalDataSource, ExternalDataSourceList, OneLakeExternalDataSource, UnknownExternalDataSource, @@ -279,11 +278,6 @@ def test_writable_as_write( if instance.settings is not None and instance.settings.credentials is not None: instance.settings.credentials = None - if type(instance) is ExternalDataSource: - with pytest.raises(NotImplementedError, match="format-specific"): - instance.as_write() - return - write_format = instance.as_write() assert isinstance(write_format, CogniteResource) @@ -329,9 +323,8 @@ def test_writable_list_as_write( # Files and time series with instance ID can not be created through "old APIs". Doing as_write with # instance_id set raises an error, so we clear it here: for item in resource_list: - if isinstance(item, (FileMetadata, TimeSeries)): - item.instance_id = None - assert item.external_id is not None + item.instance_id = None # type: ignore[union-attr] + assert item.external_id is not None if isinstance(resource_list, ExternalDataSourceList): for item in resource_list: diff --git a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py index 94a36faab0..fdde47f572 100644 --- a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py +++ b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py @@ -47,7 +47,7 @@ def test_write_dump_always_includes_format() -> None: def test_base_write_has_no_settings_field() -> None: - source = OneLakeExternalDataSourceWrite.with_settings(external_id="x", settings=None) + source = OneLakeExternalDataSourceWrite(external_id="x", settings=None) dumped = source.dump(camel_case=True) assert dumped["externalId"] == "x" From 41b1cba71c4ef4ef3be26bf740edce5196a9bdab Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 16 Jul 2026 16:27:42 +0530 Subject: [PATCH 22/22] refactor(transformations): settings-only OneLakeExternalDataSourceWrite Drop flat credential/location kwargs from __init__; callers pass OneLakeDataSourceSettingsWrite, matching the read model and API shape. --- .../transformations/external_data.py | 74 +++++-------------- .../test_external_data.py | 27 +++---- 2 files changed, 30 insertions(+), 71 deletions(-) diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index 60cd4bdacb..4e32b6b3ef 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -273,7 +273,7 @@ def as_write(self, client_secret: str | None = None) -> OneLakeExternalDataSourc raise ValueError( "client_secret is required to convert credentials to a write model because the API " "does not return it. Pass client_secret to as_write(), or construct " - "OneLakeExternalDataSourceWrite with credentials." + "OneLakeExternalDataSourceWrite with settings containing credentials." ) creds_write = OneLakeCredentialsWrite( client_id=self.settings.credentials.client_id, @@ -396,15 +396,9 @@ class OneLakeExternalDataSourceWrite(ExternalDataSourceWrite): Args: external_id (str): External ID for the data source. Must be unique. - client_id (str | None): Azure application (client) ID. - tenant_id (str | None): Azure tenant (directory) ID. - client_secret (str | None): Azure client secret. - workspace_name (str | None): Fabric workspace GUID or name. - container_name (str | None): Fabric lakehouse GUID or name. + settings (OneLakeDataSourceSettingsWrite | None): OneLake credentials and location. name (str | None): Human-readable name. data_set_id (int | None): Data set ID for ACL scoping. - settings (OneLakeDataSourceSettingsWrite | None): Pre-built settings tree; used by - ``as_write()`` for partial upserts when credential/location fields are omitted. Examples: @@ -412,18 +406,27 @@ class OneLakeExternalDataSourceWrite(ExternalDataSourceWrite): >>> from cognite.client import CogniteClient >>> from cognite.client.data_classes.transformations import ( + ... OneLakeCredentialsWrite, + ... OneLakeDataSourceSettingsWrite, ... OneLakeExternalDataSourceWrite, + ... OneLakeLocationDescription, ... ) >>> client = CogniteClient() >>> source = OneLakeExternalDataSourceWrite( ... external_id="fabric-lakehouse-prod", ... name="Production lakehouse", - ... client_id="", - ... tenant_id="", - ... client_secret="", - ... workspace_name="", - ... container_name="", ... data_set_id=123456, + ... settings=OneLakeDataSourceSettingsWrite( + ... credentials=OneLakeCredentialsWrite( + ... client_id="", + ... tenant_id="", + ... client_secret="", + ... ), + ... location_description=OneLakeLocationDescription( + ... workspace_name="", + ... container_name="", + ... ), + ... ), ... ) >>> client.transformations.external_data_sources.upsert(source) """ @@ -434,59 +437,20 @@ class OneLakeExternalDataSourceWrite(ExternalDataSourceWrite): def __init__( self, external_id: str, - client_id: str | None = None, - tenant_id: str | None = None, - client_secret: str | None = None, - workspace_name: str | None = None, - container_name: str | None = None, + settings: OneLakeDataSourceSettingsWrite | None = None, name: str | None = None, data_set_id: int | None = None, - *, - settings: OneLakeDataSourceSettingsWrite | None = None, ) -> None: super().__init__(external_id=external_id, name=name, data_set_id=data_set_id) - if settings is not None: - self.settings = settings - elif ( - client_id is not None - and tenant_id is not None - and client_secret is not None - and workspace_name is not None - and container_name is not None - ): - self.settings = OneLakeDataSourceSettingsWrite( - credentials=OneLakeCredentialsWrite( - client_id=client_id, - tenant_id=tenant_id, - client_secret=client_secret, - ), - location_description=OneLakeLocationDescription( - workspace_name=workspace_name, - container_name=container_name, - ), - ) - else: - self.settings = None + self.settings = settings @classmethod def _load(cls, resource: dict[str, Any]) -> Self: - settings = OneLakeDataSourceSettingsWrite._load_if(resource.get("settings")) - if settings is not None and settings.credentials is not None and settings.location_description is not None: - return cls( - external_id=resource["externalId"], - client_id=settings.credentials.client_id, - tenant_id=settings.credentials.tenant_id, - client_secret=settings.credentials.client_secret, - workspace_name=settings.location_description.workspace_name, - container_name=settings.location_description.container_name, - name=resource.get("name"), - data_set_id=resource.get("dataSetId"), - ) return cls( external_id=resource["externalId"], name=resource.get("name"), data_set_id=resource.get("dataSetId"), - settings=settings, + settings=OneLakeDataSourceSettingsWrite._load_if(resource.get("settings")), ) def dump(self, camel_case: bool = True) -> dict[str, Any]: diff --git a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py index fdde47f572..dcfef3ec3e 100644 --- a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py +++ b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py @@ -5,22 +5,24 @@ from cognite.client.data_classes.transformations.external_data import ( ExternalDataSource, OneLakeCredentialsWrite, + OneLakeDataSourceSettingsWrite, OneLakeExternalDataSource, OneLakeExternalDataSourceWrite, + OneLakeLocationDescription, UnknownExternalDataSource, ) -def test_onelake_write_init_structure() -> None: - source = OneLakeExternalDataSourceWrite( - external_id="x", - client_id="cid", - tenant_id="tid", - client_secret="sec", - workspace_name="ws", - container_name="cn", +def _onelake_write_settings() -> OneLakeDataSourceSettingsWrite: + return OneLakeDataSourceSettingsWrite( + credentials=OneLakeCredentialsWrite("cid", "tid", "sec"), + location_description=OneLakeLocationDescription("ws", "cn"), ) + +def test_onelake_write_init_structure() -> None: + source = OneLakeExternalDataSourceWrite(external_id="x", settings=_onelake_write_settings()) + assert source.external_id == "x" assert source.settings is not None assert source.settings.credentials is not None @@ -31,14 +33,7 @@ def test_onelake_write_init_structure() -> None: def test_write_dump_always_includes_format() -> None: - source = OneLakeExternalDataSourceWrite( - external_id="x", - client_id="cid", - tenant_id="tid", - client_secret="sec", - workspace_name="ws", - container_name="cn", - ) + source = OneLakeExternalDataSourceWrite(external_id="x", settings=_onelake_write_settings()) dumped = source.dump(camel_case=True) assert "format" in dumped