feat(transformations): add Fabric OneLake external data sources API

cognite/client/_api/transformations/__init__.py

@@ -3,6 +3,7 @@
 from collections.abc import AsyncIterator, Sequence
 from typing import TYPE_CHECKING, Any, Literal, overload
 
+from cognite.client._api.transformations.external_data import TransformationExternalDataAPI
 from cognite.client._api.transformations.jobs import TransformationJobsAPI
 from cognite.client._api.transformations.notifications import TransformationNotificationsAPI
 from cognite.client._api.transformations.schedules import TransformationSchedulesAPI
@@ -36,6 +37,7 @@ def __init__(self, config: ClientConfig, api_version: str | None, cognite_client
         self.schedules = TransformationSchedulesAPI(config, api_version, cognite_client)
         self.schema = TransformationSchemaAPI(config, api_version, cognite_client)
         self.notifications = TransformationNotificationsAPI(config, api_version, cognite_client)
+        self.external_data_sources = TransformationExternalDataAPI(config, api_version, cognite_client)
 
     @overload
     def __call__(

cognite/client/_api/transformations/external_data.py

@@ -0,0 +1,160 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from cognite.client._api_client import APIClient
+from cognite.client.data_classes.transformations.external_data import (
+    ExternalDataSource,
+    ExternalDataSourceList,
+    ExternalDataSourceUsability,
+    ExternalDataSourceWrite,
+)
+from cognite.client.utils._identifier import IdentifierSequence
+from cognite.client.utils.useful_types import SequenceNotStr
+
+
+class TransformationExternalDataAPI(APIClient):
+    """Manage Fabric OneLake external data sources for transformations.
+
+    OneLake external data sources are **read-only** from a transform perspective — transforms can
+    read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported.
+    """
+
+    _RESOURCE_PATH = "/transformations/external_data"
+    _CREATE_LIMIT = 1000
+    _DELETE_LIMIT = 1000
+
+    async def list(self, limit: int | None = None) -> ExternalDataSourceList:
+        """List Fabric OneLake external data sources.
+
+        OneLake external data sources are **read-only** from a transform perspective — transforms can
+        read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported.
+
+        Args:
+            limit (int | None): Maximum number of results to return. Use ``None`` or ``-1`` to return all.
+                Defaults to returning all.
+
+        Returns:
+            ExternalDataSourceList: All registered OneLake external data sources.
+
+        Examples:
+
+            List all registered external data sources:
+
+                >>> from cognite.client import CogniteClient
+                >>> client = CogniteClient()
+                >>> sources = client.transformations.external_data_sources.list()
+        """
+        return await self._list(
+            method="GET",
+            list_cls=ExternalDataSourceList,
+            resource_cls=ExternalDataSource,
+            limit=limit,
+        )
+
+    async def upsert(
+        self,
+        source: ExternalDataSourceWrite | Sequence[ExternalDataSourceWrite],
+    ) -> ExternalDataSource | ExternalDataSourceList:
+        """Create or update (upsert) Fabric OneLake external data sources.
+
+        An upsert creates the source if it doesn't exist, or overwrites it entirely if it does.
+        Uniqueness is determined by ``externalId``.
+
+        OneLake external data sources are **read-only** from a transform perspective — transforms can
+        read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported.
+
+        Args:
+            source (ExternalDataSourceWrite | Sequence[ExternalDataSourceWrite]): Single source or list of sources to upsert.
+
+        Returns:
+            ExternalDataSource | ExternalDataSourceList: The upserted source(s).
+
+        Examples:
+
+            Register a Fabric OneLake source:
+
+                >>> from cognite.client import CogniteClient
+                >>> from cognite.client.data_classes.transformations.external_data import (
+                ...     ExternalDataSourceWrite,
+                ... )
+                >>> client = CogniteClient()
+                >>> source = ExternalDataSourceWrite.onelake(
+                ...     external_id="fabric-lakehouse-prod",
+                ...     name="Production lakehouse",
+                ...     client_id="<azure-app-id>",
+                ...     tenant_id="<azure-tenant-uuid>",
+                ...     client_secret="<secret>",
+                ...     workspace_name="<fabric-workspace-guid>",
+                ...     container_name="<fabric-lakehouse-guid>",
+                ...     data_set_id=123456,
+                ... )
+                >>> res = client.transformations.external_data_sources.upsert(source)
+        """
+        return await self._create_multiple(
+            items=source,
+            list_cls=ExternalDataSourceList,
+            resource_cls=ExternalDataSource,
+            input_resource_cls=ExternalDataSourceWrite,
+        )
+
+    async def delete(self, external_id: str | SequenceNotStr[str]) -> None:
+        """Delete Fabric OneLake external data sources.
+
+        Args:
+            external_id (str | SequenceNotStr[str]): External ID or list of external IDs to delete.
+
+        Examples:
+
+            Delete a source by external ID:
+
+                >>> from cognite.client import CogniteClient
+                >>> client = CogniteClient()
+                >>> client.transformations.external_data_sources.delete("fabric-lakehouse-prod")
+
+            Delete multiple sources:
+
+                >>> client.transformations.external_data_sources.delete(
+                ...     ["fabric-lakehouse-prod", "fabric-lakehouse-staging"]
+                ... )
+        """
+        await self._delete_multiple(
+            identifiers=IdentifierSequence.load(external_ids=external_id),
+            wrap_ids=True,
+        )
+
+    async def verify_usability(self, external_id: str) -> ExternalDataSourceUsability:
+        """Verify that a Fabric OneLake external data source is usable.
+
+        Checks that the source exists and that the configured Azure credentials can access the specified
+        Fabric lakehouse. Returns a ``usable_version`` UUID if the source is accessible.
+
+        OneLake external data sources are **read-only** from a transform perspective — transforms can
+        read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported.
+
+        Args:
+            external_id (str): External ID of the source to verify.
+
+        Returns:
+            ExternalDataSourceUsability: Contains ``usable_version`` (a UUID) if the source is accessible,
+                or ``None`` if the credentials are invalid or the source cannot be reached.
+
+        Examples:
+
+            Verify a source before running a transformation:
+
+                >>> from cognite.client import CogniteClient
+                >>> client = CogniteClient()
+                >>> result = client.transformations.external_data_sources.verify_usability(
+                ...     "fabric-lakehouse-prod"
+                ... )
+                >>> assert result.usable_version is not None, (
+                ...     "Source not configured or credentials invalid"
+                ... )
+        """
+        res = await self._post(
+            url_path=self._RESOURCE_PATH + "/usability",
+            json={"externalId": external_id},
+            semaphore=self._get_semaphore("write"),
+        )
+        return ExternalDataSourceUsability._load(res.json())

cognite/client/_cognite_client.py

@@ -97,6 +97,7 @@
         ThreeDRevisionsAPI,
     )
     from cognite.client._api.transformations import (  # type: ignore[attr-defined]
+        TransformationExternalDataAPI,
         TransformationJobsAPI,
         TransformationNotificationsAPI,
         TransformationSchedulesAPI,
@@ -425,6 +426,7 @@ def _make_accessors_for_building_docs() -> None:
     AsyncCogniteClient.transformations.notifications = TransformationNotificationsAPI  # type: ignore
     AsyncCogniteClient.transformations.jobs = TransformationJobsAPI  # type: ignore
     AsyncCogniteClient.transformations.schema = TransformationSchemaAPI  # type: ignore
+    AsyncCogniteClient.transformations.external_data_sources = TransformationExternalDataAPI  # type: ignore
     AsyncCogniteClient.diagrams = DiagramsAPI  # type: ignore
     AsyncCogniteClient.annotations = AnnotationsAPI  # type: ignore
     AsyncCogniteClient.functions = FunctionsAPI  # type: ignore

cognite/client/_sync_api/transformations/__init__.py

@@ -1,6 +1,6 @@
 """
 ===============================================================================
-8bca411648596701cde79f2c9e6f2a88
+e545d766fa2f6a8b3d5946accd494c89
 This file is auto-generated from the Async API modules, - do not edit manually!
 ===============================================================================
 """
@@ -12,6 +12,7 @@
 
 from cognite.client import AsyncCogniteClient
 from cognite.client._constants import DEFAULT_LIMIT_READ
+from cognite.client._sync_api.transformations.external_data import SyncTransformationExternalDataAPI
 from cognite.client._sync_api.transformations.jobs import SyncTransformationJobsAPI
 from cognite.client._sync_api.transformations.notifications import SyncTransformationNotificationsAPI
 from cognite.client._sync_api.transformations.schedules import SyncTransformationSchedulesAPI
@@ -41,6 +42,7 @@ def __init__(self, async_client: AsyncCogniteClient) -> None:
         self.schedules = SyncTransformationSchedulesAPI(async_client)
         self.schema = SyncTransformationSchemaAPI(async_client)
         self.notifications = SyncTransformationNotificationsAPI(async_client)
+        self.external_data_sources = SyncTransformationExternalDataAPI(async_client)
 
     @overload
     def __call__(

cognite/client/_sync_api/transformations/external_data.py

@@ -0,0 +1,150 @@
+"""
+===============================================================================
+53ec7cc4c0a0655facc509b38fedaff9
+This file is auto-generated from the Async API modules, - do not edit manually!
+===============================================================================
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from cognite.client import AsyncCogniteClient
+from cognite.client._sync_api_client import SyncAPIClient
+from cognite.client.data_classes.transformations.external_data import (
+    ExternalDataSource,
+    ExternalDataSourceList,
+    ExternalDataSourceUsability,
+    ExternalDataSourceWrite,
+)
+from cognite.client.utils._async_helpers import run_sync
+from cognite.client.utils.useful_types import SequenceNotStr
+
+
+class SyncTransformationExternalDataAPI(SyncAPIClient):
+    """Auto-generated, do not modify manually."""
+
+    def __init__(self, async_client: AsyncCogniteClient) -> None:
+        self.__async_client = async_client
+
+    def list(self, limit: int | None = None) -> ExternalDataSourceList:
+        """
+        List Fabric OneLake external data sources.
+
+        OneLake external data sources are **read-only** from a transform perspective — transforms can
+        read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported.
+
+        Args:
+            limit (int | None): Maximum number of results to return. Use ``None`` or ``-1`` to return all.
+                Defaults to returning all.
+
+        Returns:
+            ExternalDataSourceList: All registered OneLake external data sources.
+
+        Examples:
+
+            List all registered external data sources:
+
+                >>> from cognite.client import CogniteClient
+                >>> client = CogniteClient()
+                >>> sources = client.transformations.external_data_sources.list()
+        """
+        return run_sync(self.__async_client.transformations.external_data_sources.list(limit=limit))
+
+    def upsert(
+        self, source: ExternalDataSourceWrite | Sequence[ExternalDataSourceWrite]
+    ) -> ExternalDataSource | ExternalDataSourceList:
+        """
+        Create or update (upsert) Fabric OneLake external data sources.
+
+        An upsert creates the source if it doesn't exist, or overwrites it entirely if it does.
+        Uniqueness is determined by ``externalId``.
+
+        OneLake external data sources are **read-only** from a transform perspective — transforms can
+        read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported.
+
+        Args:
+            source (ExternalDataSourceWrite | Sequence[ExternalDataSourceWrite]): Single source or list of sources to upsert.
+
+        Returns:
+            ExternalDataSource | ExternalDataSourceList: The upserted source(s).
+
+        Examples:
+
+            Register a Fabric OneLake source:
+
+                >>> from cognite.client import CogniteClient
+                >>> from cognite.client.data_classes.transformations.external_data import (
+                ...     ExternalDataSourceWrite,
+                ... )
+                >>> client = CogniteClient()
+                >>> source = ExternalDataSourceWrite.onelake(
+                ...     external_id="fabric-lakehouse-prod",
+                ...     name="Production lakehouse",
+                ...     client_id="<azure-app-id>",
+                ...     tenant_id="<azure-tenant-uuid>",
+                ...     client_secret="<secret>",
+                ...     workspace_name="<fabric-workspace-guid>",
+                ...     container_name="<fabric-lakehouse-guid>",
+                ...     data_set_id=123456,
+                ... )
+                >>> res = client.transformations.external_data_sources.upsert(source)
+        """
+        return run_sync(self.__async_client.transformations.external_data_sources.upsert(source=source))
+
+    def delete(self, external_id: str | SequenceNotStr[str]) -> None:
+        """
+        Delete Fabric OneLake external data sources.
+
+        Args:
+            external_id (str | SequenceNotStr[str]): External ID or list of external IDs to delete.
+
+        Examples:
+
+            Delete a source by external ID:
+
+                >>> from cognite.client import CogniteClient
+                >>> client = CogniteClient()
+                >>> client.transformations.external_data_sources.delete("fabric-lakehouse-prod")
+
+            Delete multiple sources:
+
+                >>> client.transformations.external_data_sources.delete(
+                ...     ["fabric-lakehouse-prod", "fabric-lakehouse-staging"]
+                ... )
+        """
+        return run_sync(self.__async_client.transformations.external_data_sources.delete(external_id=external_id))
+
+    def verify_usability(self, external_id: str) -> ExternalDataSourceUsability:
+        """
+        Verify that a Fabric OneLake external data source is usable.
+
+        Checks that the source exists and that the configured Azure credentials can access the specified
+        Fabric lakehouse. Returns a ``usable_version`` UUID if the source is accessible.
+
+        OneLake external data sources are **read-only** from a transform perspective — transforms can
+        read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported.
+
+        Args:
+            external_id (str): External ID of the source to verify.
+
+        Returns:
+            ExternalDataSourceUsability: Contains ``usable_version`` (a UUID) if the source is accessible,
+                or ``None`` if the credentials are invalid or the source cannot be reached.
+
+        Examples:
+
+            Verify a source before running a transformation:
+
+                >>> from cognite.client import CogniteClient
+                >>> client = CogniteClient()
+                >>> result = client.transformations.external_data_sources.verify_usability(
+                ...     "fabric-lakehouse-prod"
+                ... )
+                >>> assert result.usable_version is not None, (
+                ...     "Source not configured or credentials invalid"
+                ... )
+        """
+        return run_sync(
+            self.__async_client.transformations.external_data_sources.verify_usability(external_id=external_id)
+        )