diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command deleted file mode 100644 index dbd3afb9..00000000 --- a/.buildkite/hooks/pre-command +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -set -e - -# Export secrets for Docker containers. -# Restrict exposing secrets only to the steps that need them -export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") -export CI_DATABRICKS_SQL_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_SQL_DBT_HTTP_PATH" --project="dbt-package-testing-363917") -export CI_DATABRICKS_SQL_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_SQL_DBT_TOKEN" --project="dbt-package-testing-363917") -export CI_SQLSERVER_DBT_SERVER=$(gcloud secrets versions access latest --secret="CI_SQLSERVER_DBT_SERVER" --project="dbt-package-testing-363917") -export CI_SQLSERVER_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SQLSERVER_DBT_DATABASE" --project="dbt-package-testing-363917") -export CI_SQLSERVER_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SQLSERVER_DBT_USER" --project="dbt-package-testing-363917") -export CI_SQLSERVER_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SQLSERVER_DBT_PASS" --project="dbt-package-testing-363917") \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index 0efc4ef4..00000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,104 +0,0 @@ -steps: - - label: ":postgres: Run Tests - Postgres" - key: "run-dbt-postgres" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_POSTGRES_DBT_DBNAME" - - "CI_POSTGRES_DBT_HOST" - - "CI_POSTGRES_DBT_PASS" - - "CI_POSTGRES_DBT_USER" - commands: | - bash .buildkite/scripts/run_models.sh postgres - - - label: ":snowflake-db: Run Tests - Snowflake" - key: "run_dbt_snowflake" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_SNOWFLAKE_DBT_ACCOUNT" - - "CI_SNOWFLAKE_DBT_DATABASE" - - "CI_SNOWFLAKE_DBT_PASS" - - "CI_SNOWFLAKE_DBT_ROLE" - - "CI_SNOWFLAKE_DBT_USER" - - "CI_SNOWFLAKE_DBT_WAREHOUSE" - commands: | - bash .buildkite/scripts/run_models.sh snowflake - - - label: ":gcloud: Run Tests - BigQuery" - key: "run_dbt_bigquery" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "GCLOUD_SERVICE_KEY" - commands: | - bash .buildkite/scripts/run_models.sh bigquery - - - label: ":amazon-redshift: Run Tests - Redshift" - key: "run_dbt_redshift" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_REDSHIFT_DBT_DBNAME" - - "CI_REDSHIFT_DBT_HOST" - - "CI_REDSHIFT_DBT_PASS" - - "CI_REDSHIFT_DBT_USER" - commands: | - bash .buildkite/scripts/run_models.sh redshift - - - label: ":databricks: Run Tests - Databricks" - key: "run_dbt_databricks" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_DATABRICKS_DBT_HOST" - - "CI_DATABRICKS_DBT_HTTP_PATH" - - "CI_DATABRICKS_DBT_TOKEN" - - "CI_DATABRICKS_DBT_CATALOG" - commands: | - bash .buildkite/scripts/run_models.sh databricks - - - label: ":databricks: :database: Run Tests - Databricks SQL Warehouse" - key: "run_dbt_databricks_sql" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_DATABRICKS_DBT_HOST" - - "CI_DATABRICKS_SQL_DBT_HTTP_PATH" - - "CI_DATABRICKS_SQL_DBT_TOKEN" - - "CI_DATABRICKS_DBT_CATALOG" - commands: | - bash .buildkite/scripts/run_models.sh databricks-sql - - - label: ":azure: Run Tests - SQLServer" - key: "run_dbt_sqlserver" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_SQLSERVER_DBT_SERVER" - - "CI_SQLSERVER_DBT_DATABASE" - - "CI_SQLSERVER_DBT_USER" - - "CI_SQLSERVER_DBT_PASS" - commands: | - bash .buildkite/scripts/run_models.sh sqlserver \ No newline at end of file diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh deleted file mode 100644 index 0f2bda34..00000000 --- a/.buildkite/scripts/run_models.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -set -euo pipefail - -apt-get update -apt-get install libsasl2-dev - -python3 -m venv venv -. venv/bin/activate -pip install --upgrade pip setuptools -if [ "$1" == "sqlserver" ]; then - pip install -r integration_tests/requirements_sqlserver.txt - - curl -sSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /usr/share/keyrings/microsoft-prod.gpg - curl -sSL https://packages.microsoft.com/config/debian/12/prod.list > /etc/apt/sources.list.d/mssql-release.list - - apt-get update - ACCEPT_EULA=Y apt-get install -y msodbcsql18 - ACCEPT_EULA=Y apt-get install -y mssql-tools18 - echo 'export PATH="$PATH:/opt/mssql-tools18/bin"' >> ~/.bashrc - source ~/.bashrc - apt-get -y install unixodbc-dev - apt-get update - - pip uninstall -y pyodbc - pip install --no-cache-dir --no-binary :all: pyodbc==4.0.39 - - # odbcinst -j - -else - pip install -r integration_tests/requirements.txt -fi -mkdir -p ~/.dbt -cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml - -db=$1 -echo `pwd` -cd integration_tests -dbt deps -if [ "$db" = "databricks-sql" ]; then -dbt seed --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" --full-refresh -dbt compile --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" --full-refresh -dbt run --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" --full-refresh -dbt run --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" -dbt test --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" -dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__usage_pricing: true, fivetran_platform_using_connection: false}' --target "$db" --full-refresh -dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__usage_pricing: true, fivetran_platform_using_connection: false}' --target "$db" -dbt test --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" -dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true, fivetran_platform_using_transformations: true}' --target "$db" --full-refresh -dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true, fivetran_platform_using_transformations: true}' --target "$db" -dbt test --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" -dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false, fivetran_platform_using_transformations: false}' --target "$db" --full-refresh -dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false, fivetran_platform_using_transformations: false}' --target "$db" -dbt test --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" -else -dbt seed --target "$db" --full-refresh -dbt compile --target "$db" --full-refresh -dbt run --target "$db" --full-refresh -dbt run --target "$db" -dbt test --target "$db" -if [ "$db" = "bigquery" ] || [ "$db" = "redshift" ] || [ "$db" = "postgres" ]; then -dbt run --vars '{fivetran_platform_log_identifier: log_json_data}' --target "$db" --full-refresh -dbt test --target "$db" -fi -dbt run --vars '{fivetran_platform__usage_pricing: true, fivetran_platform_using_connection: false}' --target "$db" --full-refresh -dbt run --vars '{fivetran_platform__usage_pricing: true, fivetran_platform_using_connection: false}' --target "$db" -dbt test --target "$db" -dbt run --vars '{fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true, fivetran_platform_using_transformations: true}' --target "$db" --full-refresh -dbt run --vars '{fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true, fivetran_platform_using_transformations: true}' --target "$db" -dbt test --target "$db" -dbt run --vars '{fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false, fivetran_platform_using_transformations: false}' --target "$db" --full-refresh -dbt run --vars '{fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false, fivetran_platform_using_transformations: false}' --target "$db" -dbt test --target "$db" -fi -if [ "$1" != "sqlserver" ]; then -dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" -fi diff --git a/.quickstart/quickstart.yml b/.quickstart/quickstart.yml index 366f8e50..6422e342 100644 --- a/.quickstart/quickstart.yml +++ b/.quickstart/quickstart.yml @@ -10,6 +10,8 @@ table_variables: - user fivetran_platform_using_connection: - connection + fivetran_platform_using_connector_sdk_log: + - connector_sdk_log destination_configurations: databricks: @@ -24,5 +26,6 @@ public_models: [ "fivetran_platform__connection_status", "fivetran_platform__usage_history", "fivetran_platform__audit_table", - "fivetran_platform__mar_table_history" + "fivetran_platform__mar_table_history", + "fivetran_platform__errors_and_warnings" ] diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f2c1a05..161eae9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,18 @@ # dbt_fivetran_log v2.6.0 -[PR #XXX](https://github.com/fivetran/dbt_fivetran_log/pull/XXX) includes the following updates: +[PR #196](https://github.com/fivetran/dbt_fivetran_log/pull/196) includes the following updates: +## Schema/Data Change +**2 total changes • 0 possible breaking changes** + +| Data Model(s) | Change type | Old | New | Notes | +| ------------- | ----------- | --- | --- | ----- | +| fivetran_platform__errors_and_warnings | New model | | | Unions error and warning events from the `log` (standard connections) and `connector_sdk_log` (Connector SDK connections) tables, enriched with `connection_name`. | +| stg_fivetran_platform__connector_sdk_log | New model | | | Staging model for the new `connector_sdk_log` source table. | + +## Feature Update +- Adds the `fivetran_platform_using_connector_sdk_log` variable (default `true`) to enable or disable the `connector_sdk_log` source and its downstream logic. Set this to `false` if your destination does not contain the `connector_sdk_log` table. + +# dbt_fivetran_log v2.5.2 [PR #183](https://github.com/fivetran/dbt_fivetran_log/pull/183) includes the following updates: ## Under the Hood diff --git a/DECISIONLOG.md b/DECISIONLOG.md index bac237fe..512171bf 100644 --- a/DECISIONLOG.md +++ b/DECISIONLOG.md @@ -25,3 +25,16 @@ vars: Some records in the `fivetran_platform__mar_table_history` model may lack an associated `connection_id`. This can occur for a few reasons. For example, the record may originate from a deleted connection or from HVR sources that do not populate this field. Previously, we excluded these records under the assumption that they were erroneous. However, we've found cases where these rows provide valuable context, particularly for tracking metadata activity from legacy or nonstandard sources and now include them in the model. While this change will increase the number of rows returned, the added visibility supports more complete auditing and analysis. + +## Severity values in `fivetran_platform__errors_and_warnings` +The `fivetran_platform__errors_and_warnings` model unions error and warning events from two sources: the `log` table (standard connections) and the `connector_sdk_log` table (Connector SDK connections). Both sources report `WARNING` and `SEVERE`, while `ERROR` is reported only by the `connector_sdk_log` source. We pass each source's `severity_level` through as its raw value rather than normalizing or remapping it, so each event reflects exactly what its source reported. The `connector_type` column (`standard_connector` or `connector_sdk`) identifies which source a row came from. + +We exclude events that are not attributable to a connection (where `connection_id` is null). The `log` table records some warning- and error-level events that are not connector events — for example, the dbt run output of a transformation job — and these carry no `connection_id`. The goal is for customers to monitor the severity of warnings and errors raised by their connectors, not warnings raised by the transformation jobs recorded in the log, so we require a non-null `connection_id` in both sources. + +## Connector SDK log source +Not all customers have the `connector_sdk_log` source table, as it is only populated for accounts using the Fivetran Connector SDK. This source and its downstream logic are controlled by the `fivetran_platform_using_connector_sdk_log` variable, which defaults to `true`. If your destination does not contain the `connector_sdk_log` table, set this variable to `false` in your root `dbt_project.yml` to disable the source and exclude Connector SDK events from `fivetran_platform__errors_and_warnings`: + +```yml +vars: + fivetran_platform_using_connector_sdk_log: false ## Default is true. Set to false if the connector_sdk_log source table is not present in your destination. +``` diff --git a/README.md b/README.md index 19765008..276ca5c0 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ By default, this package materializes the following final tables: | [fivetran_platform__schema_changelog](https://fivetran.github.io/dbt_fivetran_log/#!/model/model.fivetran_log.fivetran_platform__schema_changelog) | Documents all schema changes made to your connections including table alterations, table creations, schema creations, and configuration changes with detailed metadata about each event to track data structure evolution and troubleshoot schema-related issues. | | [fivetran_platform__audit_table](https://fivetran.github.io/dbt_fivetran_log/#!/model/model.fivetran_log.fivetran_platform__audit_table) | Replaces the deprecated [`_fivetran_audit` table](https://fivetran.com/docs/getting-started/system-columns-and-tables#audittables) and tracks each table receiving data during connection syncs with comprehensive timestamps for connection and table-level sync progress plus detailed counts of records inserted, replaced, updated, and deleted to monitor data processing and sync performance. | | [fivetran_platform__audit_user_activity](https://fivetran.github.io/dbt_fivetran_log/#!/model/model.fivetran_log.fivetran_platform__audit_user_activity) | Records all user-triggered actions within your Fivetran account to provide a comprehensive audit trail that helps you trace user activities to specific [log events](https://fivetran.com/docs/logs#logeventlist) such as schema changes, sync frequency updates, manual syncs, connection failures, and other operational events for compliance and troubleshooting purposes. | +| [fivetran_platform__errors_and_warnings](https://fivetran.github.io/dbt_fivetran_log/#!/model/model.fivetran_log.fivetran_platform__errors_and_warnings) | Consolidates error and warning events from both standard connections and Connector SDK connections into a single feed, enriched with connection details, so you can monitor and triage the severity of issues across your data pipelines. Excludes `INFO`-level events. | ¹ Each Quickstart transformation job run materializes these models if all components of this data model are enabled. This count includes all staging, intermediate, and final models materialized as `view`, `table`, or `incremental`. @@ -103,12 +104,13 @@ vars: ``` ### Disable Models for Non Existent Sources -If you do not leverage Fivetran RBAC, then you will not have the `user` or `destination_membership` source tables. The `user` and `destination_membership` are enabled by default. Therefore in order to switch the default configurations, you must add the following variable(s) to your root `dbt_project.yml` file for the respective source tables you wish to disable: +If you do not leverage Fivetran RBAC, then you will not have the `user` or `destination_membership` source tables. The `user` and `destination_membership` are enabled by default. Similarly, the `connector_sdk_log` table is only present if you use the Fivetran Connector SDK. Therefore in order to switch the default configurations, you must add the following variable(s) to your root `dbt_project.yml` file for the respective source tables you wish to disable: ```yml vars: fivetran_platform_using_destination_membership: false # Default is true. This will disable only the destination membership logic fivetran_platform_using_user: false # Default is true. This will disable only the user logic + fivetran_platform_using_connector_sdk_log: false # Default is true. This will disable the connector_sdk_log source and exclude Connector SDK events from fivetran_platform__errors_and_warnings ``` #### Leveraging `CONNECTION` vs `CONNECTOR` diff --git a/dbt_project.yml b/dbt_project.yml index f00e3a8c..1d1ea1ce 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -21,6 +21,7 @@ vars: destination: "{{ source('fivetran_platform', 'destination') }}" destination_membership: "{{ source('fivetran_platform', 'destination_membership') }}" log: "{{ source('fivetran_platform', 'log') }}" + connector_sdk_log: "{{ source('fivetran_platform', 'connector_sdk_log') }}" user: "{{ source('fivetran_platform', 'user') }}" usage_cost: "{{ source('fivetran_platform', 'usage_cost') }}" transformation_runs: "{{ source('fivetran_platform','transformation_runs') }}" diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml deleted file mode 100644 index 616ba7e1..00000000 --- a/integration_tests/ci/sample.profiles.yml +++ /dev/null @@ -1,68 +0,0 @@ - -# HEY! This file is used in the dbt package integrations tests with Buildkite. -# You should __NEVER__ check credentials into version control. Thanks for reading :) - -integration_tests: - target: redshift - outputs: - redshift: - type: redshift - host: "{{ env_var('CI_REDSHIFT_DBT_HOST') }}" - user: "{{ env_var('CI_REDSHIFT_DBT_USER') }}" - pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" - dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" - port: 5439 - schema: fivetran_platform_integration_tests - threads: 8 - bigquery: - type: bigquery - method: service-account-json - project: 'dbt-package-testing' - schema: fivetran_platform_integration_tests - threads: 8 - keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" - snowflake: - type: snowflake - account: "{{ env_var('CI_SNOWFLAKE_DBT_ACCOUNT') }}" - user: "{{ env_var('CI_SNOWFLAKE_DBT_USER') }}" - password: "{{ env_var('CI_SNOWFLAKE_DBT_PASS') }}" - role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" - database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" - warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: fivetran_platform_integration_tests - threads: 8 - postgres: - type: postgres - host: "{{ env_var('CI_POSTGRES_DBT_HOST') }}" - user: "{{ env_var('CI_POSTGRES_DBT_USER') }}" - pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" - dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" - port: 5432 - schema: fivetran_platform_integration_tests - threads: 8 - databricks: - catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" - host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" - http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: fivetran_platform_integration_tests - threads: 1 - token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" - type: databricks - databricks-sql: - catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" - host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" - http_path: "{{ env_var('CI_DATABRICKS_SQL_DBT_HTTP_PATH') }}" - schema: fivetran_platform_sqlw_tests - threads: 1 - token: "{{ env_var('CI_DATABRICKS_SQL_DBT_TOKEN') }}" - type: databricks - sqlserver: - type: sqlserver - driver: 'ODBC Driver 18 for SQL Server' - server: "{{ env_var('CI_SQLSERVER_DBT_SERVER') }}" - port: 1433 - database: "{{ env_var('CI_SQLSERVER_DBT_DATABASE') }}" - schema: fivetran_platform_integration_tests - user: "{{ env_var('CI_SQLSERVER_DBT_USER') }}" - password: "{{ env_var('CI_SQLSERVER_DBT_PASS') }}" - threads: 8 diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml new file mode 100644 index 00000000..bc9bc8f9 --- /dev/null +++ b/integration_tests/ci/test_scenarios.yml @@ -0,0 +1,46 @@ +# Only set if not using default adapter ranges +# dbt_adapter_versions: +# snowflake: ">=1.3.0,<2.0.0" +# bigquery: ">=1.3.0,<2.0.0" +# postgres: ">=1.3.0,<2.0.0" +# redshift: ">=1.3.0,<2.0.0" +# databricks: ">=1.6.0,<2.0.0" + +schema_variable_name: "fivetran_platform_schema" + +include_databricks_sql: true +include_sqlserver: true +include_dbt_compile: true + +test_scenarios: + - name: "Default" + vars: {} + include_incremental: false + + - name: "log json data" + vars: + fivetran_platform_log_identifier: "log_json_data" + include_incremental: false + warehouses: ["bigquery", "redshift", "postgres"] + + - name: "usage pricing enabled" + vars: + fivetran_platform__usage_pricing: true + fivetran_platform_using_connection: false + include_incremental: false + + - name: "usage pricing with transformations" + vars: + fivetran_platform__credits_pricing: false + fivetran_platform__usage_pricing: true + fivetran_platform_using_transformations: true + include_incremental: false + + - name: "minimal features" + vars: + fivetran_platform__usage_pricing: false + fivetran_platform_using_destination_membership: false + fivetran_platform_using_user: false + fivetran_platform_using_transformations: false + fivetran_platform_using_connector_sdk_log: false + include_incremental: false \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index f21bd622..e72da667 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -20,6 +20,7 @@ vars: fivetran_platform_destination_identifier: "destination" fivetran_platform_destination_membership_identifier: "destination_membership" fivetran_platform_log_identifier: "log" + fivetran_platform_connector_sdk_log_identifier: "connector_sdk_log" fivetran_platform_user_identifier: "user" fivetran_platform_transformation_runs_identifier: "transformation_runs" @@ -28,55 +29,73 @@ models: relation: "{{ false if target.type in ('sqlserver') else true }}" columns: "{{ false if target.type in ('sqlserver','databricks') else true }}" fivetran_log: - +schema: "{{ 'fivetran_platform_sqlw_tests' if target.name == 'databricks-sql' else 'fivetran_platform' }}" - # +schema: "fivetran_platform_{{ var('directed_schema','dev') }}" + +schema: "fivetran_platform_{{ var('directed_schema','dev') }}" seeds: +docs: show: False fivetran_log_integration_tests: - +column_types: - _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" account: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" created_at: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" incremental_mar: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" measured_date: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" incremental_rows: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" connection: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" signed_up: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" _fivetran_deleted: "{{ 'bit' if target.type == 'sqlserver' else 'boolean' }}" connector: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" signed_up: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" _fivetran_deleted: "{{ 'bit' if target.type == 'sqlserver' else 'boolean' }}" credits_used: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" credits_consumed: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" destination: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" created_at: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" id: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' if target.type != 'sqlserver' else 'varchar(256)' }}" destination_membership: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" activated_at: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" joined_at: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" log: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" time_stamp: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" message_data: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' if target.type != 'sqlserver' else 'varchar(256)' }}" transformation_id: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' if target.type != 'sqlserver' else 'varchar(256)' }}" + connector_sdk_log: + +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" + event_time: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" + message: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' if target.type != 'sqlserver' else 'varchar(256)' }}" log_json_data: +enabled: "{{ target.type in ('bigquery', 'redshift', 'postgres') }}" # @docs-ignore # +enabled: false # @docs-include +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" time_stamp: timestamp message_data: "{{ 'super' if target.type == 'redshift' else 'json' }}" transformation_id: "{{ 'string' if target.type == 'bigquery' else 'varchar' }}" + transformation_runs: + +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" + usage_cost: + +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" user: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" created_at: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" clean-targets: # directories to be removed by `dbt clean` diff --git a/integration_tests/seeds/connector_sdk_log.csv b/integration_tests/seeds/connector_sdk_log.csv new file mode 100644 index 00000000..e6b7f807 --- /dev/null +++ b/integration_tests/seeds/connector_sdk_log.csv @@ -0,0 +1,5 @@ +id,event_time,_fivetran_synced,connection_id,level,message,message_origin,sync_id +aGktalz7uFZbVYKR7zlDeUd4nSM=,2024-08-13 13:34:03.343,2024-08-16 21:11:07.065,legislative_lazy,INFO,Sync started,connector_sdk,sdk789 +bf7EVPLOIMhuOs45jlXWN3lutVEc=,2024-08-13 13:35:05.555,2024-08-16 21:48:51.493,legislative_lazy,WARNING,Rate limit approaching for endpoint /v1/records,connector_sdk,sdk789 +cfJktL6p9gQE83WaCPdevLs9FXU=,2024-08-13 14:27:00.504,2024-08-16 20:30:53.959,paint_wedges,SEVERE,Failed to fetch data: connection timed out,connector_sdk,sdk790 +df7UkkDoGwEVnG7M2tFQFy7FSoDM=,2024-08-13 14:30:52.433,2024-08-16 20:35:53.957,paint_wedges,ERROR,Schema detection failed for table orders,connector_sdk,sdk790 diff --git a/integration_tests/seeds/log.csv b/integration_tests/seeds/log.csv index 848d294b..b4d2675f 100644 --- a/integration_tests/seeds/log.csv +++ b/integration_tests/seeds/log.csv @@ -34,3 +34,5 @@ D7UqnKYn6OT04HkUcPNjXA95qqp=,2024-11-18 16:26:29.719,2024-11-19 20:30:53.878,aft h7UqnKYn6OT04HkUcPNjXA95qlp=,2024-11-19 16:26:29.719,2024-11-19 20:30:53.878,aft_gleeful,INFO,"{""totalQueries"":5,""total_rows"":4810}",extract_summary,,456abc D7UqnKYn6OT04HkUcPNjXA95ttI=,2025-12-13 14:26:29.719,2025-12-13 20:30:53.878,this_connection,INFO,"{""table"":""media_insights""}",write_to_table_start,123,456abc D7UqnKYn6OT04HkUcPNjXA95ttI=,2025-12-15 14:26:29.719,2025-12-15 20:30:53.878,this_connection,INFO,"{""table"":""media_insights""}",write_to_table_start,123,456abc +WaRnAfTgLeEfUl0000000000000=,2024-11-20 10:00:00.000,2024-11-20 12:00:00.000,aft_gleeful,WARNING,"{""type"":""table_excluded_by_system"",""message"":""schema.table has been excluded by system. Reason: Not queryable""}",warning,,syncaft1 +SeVeReAfTgLeEfUl000000000000=,2024-11-20 11:00:00.000,2024-11-20 12:30:00.000,aft_gleeful,SEVERE,"{""message"":""Sync failed due to an authentication error""}",sync_failed,,syncaft2 diff --git a/integration_tests/tests/consistency/consistency__errors_and_warnings.sql b/integration_tests/tests/consistency/consistency__errors_and_warnings.sql new file mode 100644 index 00000000..1ae57203 --- /dev/null +++ b/integration_tests/tests/consistency/consistency__errors_and_warnings.sql @@ -0,0 +1,43 @@ + +{{ config( + tags="fivetran_validations", + enabled=var('fivetran_validation_tests_enabled', false) +) }} + +with prod as ( + select + connection_id, + connector_type, + severity_level, + count(*) as total_records + from {{ target.schema }}_fivetran_platform_prod.fivetran_platform__errors_and_warnings + group by 1, 2, 3 +), + +dev as ( + select + connection_id, + connector_type, + severity_level, + count(*) as total_records + from {{ target.schema }}_fivetran_platform_dev.fivetran_platform__errors_and_warnings + group by 1, 2, 3 +), + +final as ( + select + prod.connection_id, + prod.connector_type, + prod.severity_level, + prod.total_records as prod_total, + dev.total_records as dev_total + from prod + left join dev + on dev.connection_id = prod.connection_id + and dev.connector_type = prod.connector_type + and dev.severity_level = prod.severity_level +) + +select * +from final +where prod_total != dev_total diff --git a/integration_tests/tests/integrity/row_count__errors_and_warnings.sql b/integration_tests/tests/integrity/row_count__errors_and_warnings.sql new file mode 100644 index 00000000..4b340413 --- /dev/null +++ b/integration_tests/tests/integrity/row_count__errors_and_warnings.sql @@ -0,0 +1,42 @@ + +{{ config( + tags="fivetran_validations", + enabled=var('fivetran_validation_tests_enabled', false) +) }} + +-- Verifies the end model preserves one row per source error/warning event (no fan-out or loss from the connection join). +with end_model as ( + select + connector_type, + count(*) as row_count + from {{ ref('fivetran_platform__errors_and_warnings') }} + group by 1 +), + +staging_model as ( + select + 'standard_connector' as connector_type, + count(*) as row_count + from {{ ref('stg_fivetran_platform__log') }} + where lower(event_type) in ('warning', 'error', 'severe') + and connection_id is not null + + {% if var('fivetran_platform_using_connector_sdk_log', true) %} + union all + select + 'connector_sdk' as connector_type, + count(*) as row_count + from {{ ref('stg_fivetran_platform__connector_sdk_log') }} + where lower(level) in ('warning', 'error', 'severe') + and connection_id is not null + {% endif %} +) + +select + end_model.connector_type, + end_model.row_count as end_model_row_count, + staging_model.row_count as staging_model_row_count +from end_model +left join staging_model + on end_model.connector_type = staging_model.connector_type +where end_model.row_count != staging_model.row_count diff --git a/macros/staging/get_connector_sdk_log_columns.sql b/macros/staging/get_connector_sdk_log_columns.sql new file mode 100644 index 00000000..9eefb4d5 --- /dev/null +++ b/macros/staging/get_connector_sdk_log_columns.sql @@ -0,0 +1,16 @@ +{% macro get_connector_sdk_log_columns() %} + +{% set columns = [ + {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()}, + {"name": "connection_id", "datatype": dbt.type_string()}, + {"name": "event_time", "datatype": dbt.type_timestamp()}, + {"name": "id", "datatype": dbt.type_string()}, + {"name": "level", "datatype": dbt.type_string()}, + {"name": "message", "datatype": dbt.type_string()}, + {"name": "message_origin", "datatype": dbt.type_string()}, + {"name": "sync_id", "datatype": dbt.type_string()} +] %} + +{{ return(columns) }} + +{% endmacro %} diff --git a/models/fivetran_platform.yml b/models/fivetran_platform.yml index f1d1b07f..4f0bf5cf 100644 --- a/models/fivetran_platform.yml +++ b/models/fivetran_platform.yml @@ -234,4 +234,28 @@ models: - name: message_data description: The details of the event in a JSON-like string format. - name: log_id - description: Unique ID of the log record. \ No newline at end of file + description: Unique ID of the log record. + + - name: fivetran_platform__errors_and_warnings + description: > + Table of connector error and warning events across both standard Fivetran connections (from the `log` table) and + Connector SDK connections (from the `connector_sdk_log` table). + columns: + - name: connection_id + description: Foreign key referencing the connection that emitted the event. + - name: connection_name + description: Name of the connection that emitted the event. May be null if the connection is no longer present in the `connection` table. + - name: event_time + description: Timestamp when the event was logged. + - name: severity_level + description: > + Raw severity of the event as reported by the source, passed through without normalization. Both sources + report `WARNING` and `SEVERE`; `ERROR` is reported only by the `connector_sdk_log` source. + - name: message + description: The details of the event. For standard connections this is the JSON-like `message_data` string; for Connector SDK connections this is the plain-text log message. + - name: connector_type + description: > + Indicates which source the event originated from. Possible values are `standard_connector` (from the `log` table) + and `connector_sdk` (from the `connector_sdk_log` table). + - name: sync_id + description: Identifies which sync the event is associated with. \ No newline at end of file diff --git a/models/fivetran_platform__errors_and_warnings.sql b/models/fivetran_platform__errors_and_warnings.sql new file mode 100644 index 00000000..2f9b2e20 --- /dev/null +++ b/models/fivetran_platform__errors_and_warnings.sql @@ -0,0 +1,67 @@ +with log_events as ( + + select + connection_id, + sync_id, + created_at as event_time, + event_type as severity_level, + message_data as message, + 'standard_connector' as connector_type + from {{ ref('stg_fivetran_platform__log') }} + -- limit to connector error and warning severities. excludes INFO, and excludes events not attributable to a + -- connection (connection_id is null), such as transformation job dbt run logs, which are not connector events. + where lower(event_type) in ('warning', 'error', 'severe') + and connection_id is not null +), + +{% if var('fivetran_platform_using_connector_sdk_log', true) -%} +connector_sdk_events as ( + + select + connection_id, + sync_id, + created_at as event_time, + level as severity_level, + message, + 'connector_sdk' as connector_type + from {{ ref('stg_fivetran_platform__connector_sdk_log') }} + where lower(level) in ('warning', 'error', 'severe') + and connection_id is not null +), +{%- endif %} + +unioned as ( + + select * from log_events + + {% if var('fivetran_platform_using_connector_sdk_log', true) -%} + union all + select * from connector_sdk_events + {%- endif %} +), + +connection as ( + + select + connection_id, + connection_name + from {{ ref('stg_fivetran_platform__connection') }} +), + +final as ( + + select + unioned.connection_id, + connection.connection_name, + unioned.event_time, + unioned.severity_level, + unioned.message, + unioned.connector_type, + unioned.sync_id + from unioned + left join connection + on unioned.connection_id = connection.connection_id +) + +select * +from final diff --git a/models/staging/src_fivetran_platform.yml b/models/staging/src_fivetran_platform.yml index 7f1a24ae..3c2f5e93 100644 --- a/models/staging/src_fivetran_platform.yml +++ b/models/staging/src_fivetran_platform.yml @@ -180,6 +180,32 @@ sources: - name: _fivetran_synced description: Timestamp when the record was last synced. + - name: connector_sdk_log + identifier: "{{ var('fivetran_platform_connector_sdk_log_identifier', 'connector_sdk_log')}}" + description: > + Table of logged events emitted by Connector SDK connections. Only populated for accounts + that use the Fivetran Connector SDK. + config: + enabled: "{{ var('fivetran_platform_using_connector_sdk_log', true) }}" + freshness: null + columns: + - name: id + description: Unique ID of the Connector SDK log record. + - name: event_time + description: Timestamp when the event was logged. + - name: connection_id + description: Foreign key referencing the connection that emitted the log event. + - name: level + description: Severity level of the log event. Possible values include `INFO`, `WARNING`, `SEVERE`, and `ERROR`. + - name: message + description: The details of the logged event. + - name: message_origin + description: The origin of the log message within the Connector SDK connection. + - name: sync_id + description: Identifies which sync the event is associated with. + - name: _fivetran_synced + description: Timestamp when the record was last synced. + - name: account identifier: "{{ var('fivetran_platform_account_identifier', 'account')}}" description: Table of information about this Fivetran account. diff --git a/models/staging/stg_fivetran_platform.yml b/models/staging/stg_fivetran_platform.yml index 73231e21..ccb59364 100644 --- a/models/staging/stg_fivetran_platform.yml +++ b/models/staging/stg_fivetran_platform.yml @@ -110,6 +110,26 @@ models: - name: transformation_id description: Foreign key referencing the transformation if the event is related to a transformation run. + - name: stg_fivetran_platform__connector_sdk_log + description: > + Table of logged events emitted by Connector SDK connections. Only populated for accounts + that use the Fivetran Connector SDK. + columns: + - name: connector_sdk_log_id + description: Unique ID of the Connector SDK log record. + - name: sync_id + description: Identifies which sync the event is associated with. + - name: connection_id + description: Foreign key referencing the connection that emitted the log event. + - name: created_at + description: Timestamp when the event was logged. + - name: level + description: Severity level of the log event. Possible values include `INFO`, `WARNING`, `SEVERE`, and `ERROR`. + - name: message + description: The details of the logged event. + - name: message_origin + description: The origin of the log message within the Connector SDK connection. + - name: stg_fivetran_platform__account description: Table of information about this Fivetran account. columns: diff --git a/models/staging/stg_fivetran_platform__connector_sdk_log.sql b/models/staging/stg_fivetran_platform__connector_sdk_log.sql new file mode 100644 index 00000000..63f6d7e7 --- /dev/null +++ b/models/staging/stg_fivetran_platform__connector_sdk_log.sql @@ -0,0 +1,34 @@ +{{ config(enabled=var('fivetran_platform_using_connector_sdk_log', true)) }} + +with base as ( + + select * + from {{ var('connector_sdk_log') }} +), + +fields as ( + select + {{ + fivetran_utils.fill_staging_columns( + source_columns=adapter.get_columns_in_relation(source('fivetran_platform', 'connector_sdk_log')), + staging_columns=get_connector_sdk_log_columns() + ) + }} + from base +), + +final as ( + + select + id as connector_sdk_log_id, + sync_id, + connection_id, + cast(event_time as {{ dbt.type_timestamp() }}) as created_at, + level, + message, + message_origin + from fields +) + +select * +from final