diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command deleted file mode 100644 index dbd3afb9..00000000 --- a/.buildkite/hooks/pre-command +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -set -e - -# Export secrets for Docker containers. -# Restrict exposing secrets only to the steps that need them -export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") -export CI_DATABRICKS_SQL_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_SQL_DBT_HTTP_PATH" --project="dbt-package-testing-363917") -export CI_DATABRICKS_SQL_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_SQL_DBT_TOKEN" --project="dbt-package-testing-363917") -export CI_SQLSERVER_DBT_SERVER=$(gcloud secrets versions access latest --secret="CI_SQLSERVER_DBT_SERVER" --project="dbt-package-testing-363917") -export CI_SQLSERVER_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SQLSERVER_DBT_DATABASE" --project="dbt-package-testing-363917") -export CI_SQLSERVER_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SQLSERVER_DBT_USER" --project="dbt-package-testing-363917") -export CI_SQLSERVER_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SQLSERVER_DBT_PASS" --project="dbt-package-testing-363917") \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index 0efc4ef4..00000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,104 +0,0 @@ -steps: - - label: ":postgres: Run Tests - Postgres" - key: "run-dbt-postgres" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_POSTGRES_DBT_DBNAME" - - "CI_POSTGRES_DBT_HOST" - - "CI_POSTGRES_DBT_PASS" - - "CI_POSTGRES_DBT_USER" - commands: | - bash .buildkite/scripts/run_models.sh postgres - - - label: ":snowflake-db: Run Tests - Snowflake" - key: "run_dbt_snowflake" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_SNOWFLAKE_DBT_ACCOUNT" - - "CI_SNOWFLAKE_DBT_DATABASE" - - "CI_SNOWFLAKE_DBT_PASS" - - "CI_SNOWFLAKE_DBT_ROLE" - - "CI_SNOWFLAKE_DBT_USER" - - "CI_SNOWFLAKE_DBT_WAREHOUSE" - commands: | - bash .buildkite/scripts/run_models.sh snowflake - - - label: ":gcloud: Run Tests - BigQuery" - key: "run_dbt_bigquery" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "GCLOUD_SERVICE_KEY" - commands: | - bash .buildkite/scripts/run_models.sh bigquery - - - label: ":amazon-redshift: Run Tests - Redshift" - key: "run_dbt_redshift" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_REDSHIFT_DBT_DBNAME" - - "CI_REDSHIFT_DBT_HOST" - - "CI_REDSHIFT_DBT_PASS" - - "CI_REDSHIFT_DBT_USER" - commands: | - bash .buildkite/scripts/run_models.sh redshift - - - label: ":databricks: Run Tests - Databricks" - key: "run_dbt_databricks" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_DATABRICKS_DBT_HOST" - - "CI_DATABRICKS_DBT_HTTP_PATH" - - "CI_DATABRICKS_DBT_TOKEN" - - "CI_DATABRICKS_DBT_CATALOG" - commands: | - bash .buildkite/scripts/run_models.sh databricks - - - label: ":databricks: :database: Run Tests - Databricks SQL Warehouse" - key: "run_dbt_databricks_sql" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_DATABRICKS_DBT_HOST" - - "CI_DATABRICKS_SQL_DBT_HTTP_PATH" - - "CI_DATABRICKS_SQL_DBT_TOKEN" - - "CI_DATABRICKS_DBT_CATALOG" - commands: | - bash .buildkite/scripts/run_models.sh databricks-sql - - - label: ":azure: Run Tests - SQLServer" - key: "run_dbt_sqlserver" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_SQLSERVER_DBT_SERVER" - - "CI_SQLSERVER_DBT_DATABASE" - - "CI_SQLSERVER_DBT_USER" - - "CI_SQLSERVER_DBT_PASS" - commands: | - bash .buildkite/scripts/run_models.sh sqlserver \ No newline at end of file diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh deleted file mode 100644 index 0f2bda34..00000000 --- a/.buildkite/scripts/run_models.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -set -euo pipefail - -apt-get update -apt-get install libsasl2-dev - -python3 -m venv venv -. venv/bin/activate -pip install --upgrade pip setuptools -if [ "$1" == "sqlserver" ]; then - pip install -r integration_tests/requirements_sqlserver.txt - - curl -sSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /usr/share/keyrings/microsoft-prod.gpg - curl -sSL https://packages.microsoft.com/config/debian/12/prod.list > /etc/apt/sources.list.d/mssql-release.list - - apt-get update - ACCEPT_EULA=Y apt-get install -y msodbcsql18 - ACCEPT_EULA=Y apt-get install -y mssql-tools18 - echo 'export PATH="$PATH:/opt/mssql-tools18/bin"' >> ~/.bashrc - source ~/.bashrc - apt-get -y install unixodbc-dev - apt-get update - - pip uninstall -y pyodbc - pip install --no-cache-dir --no-binary :all: pyodbc==4.0.39 - - # odbcinst -j - -else - pip install -r integration_tests/requirements.txt -fi -mkdir -p ~/.dbt -cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml - -db=$1 -echo `pwd` -cd integration_tests -dbt deps -if [ "$db" = "databricks-sql" ]; then -dbt seed --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" --full-refresh -dbt compile --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" --full-refresh -dbt run --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" --full-refresh -dbt run --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" -dbt test --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" -dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__usage_pricing: true, fivetran_platform_using_connection: false}' --target "$db" --full-refresh -dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__usage_pricing: true, fivetran_platform_using_connection: false}' --target "$db" -dbt test --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" -dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true, fivetran_platform_using_transformations: true}' --target "$db" --full-refresh -dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true, fivetran_platform_using_transformations: true}' --target "$db" -dbt test --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" -dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false, fivetran_platform_using_transformations: false}' --target "$db" --full-refresh -dbt run --vars '{fivetran_platform_schema: sqlw_tests_4, fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false, fivetran_platform_using_transformations: false}' --target "$db" -dbt test --vars '{fivetran_platform_schema: sqlw_tests_4}' --target "$db" -else -dbt seed --target "$db" --full-refresh -dbt compile --target "$db" --full-refresh -dbt run --target "$db" --full-refresh -dbt run --target "$db" -dbt test --target "$db" -if [ "$db" = "bigquery" ] || [ "$db" = "redshift" ] || [ "$db" = "postgres" ]; then -dbt run --vars '{fivetran_platform_log_identifier: log_json_data}' --target "$db" --full-refresh -dbt test --target "$db" -fi -dbt run --vars '{fivetran_platform__usage_pricing: true, fivetran_platform_using_connection: false}' --target "$db" --full-refresh -dbt run --vars '{fivetran_platform__usage_pricing: true, fivetran_platform_using_connection: false}' --target "$db" -dbt test --target "$db" -dbt run --vars '{fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true, fivetran_platform_using_transformations: true}' --target "$db" --full-refresh -dbt run --vars '{fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true, fivetran_platform_using_transformations: true}' --target "$db" -dbt test --target "$db" -dbt run --vars '{fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false, fivetran_platform_using_transformations: false}' --target "$db" --full-refresh -dbt run --vars '{fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false, fivetran_platform_using_transformations: false}' --target "$db" -dbt test --target "$db" -fi -if [ "$1" != "sqlserver" ]; then -dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" -fi diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml deleted file mode 100644 index 616ba7e1..00000000 --- a/integration_tests/ci/sample.profiles.yml +++ /dev/null @@ -1,68 +0,0 @@ - -# HEY! This file is used in the dbt package integrations tests with Buildkite. -# You should __NEVER__ check credentials into version control. Thanks for reading :) - -integration_tests: - target: redshift - outputs: - redshift: - type: redshift - host: "{{ env_var('CI_REDSHIFT_DBT_HOST') }}" - user: "{{ env_var('CI_REDSHIFT_DBT_USER') }}" - pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" - dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" - port: 5439 - schema: fivetran_platform_integration_tests - threads: 8 - bigquery: - type: bigquery - method: service-account-json - project: 'dbt-package-testing' - schema: fivetran_platform_integration_tests - threads: 8 - keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" - snowflake: - type: snowflake - account: "{{ env_var('CI_SNOWFLAKE_DBT_ACCOUNT') }}" - user: "{{ env_var('CI_SNOWFLAKE_DBT_USER') }}" - password: "{{ env_var('CI_SNOWFLAKE_DBT_PASS') }}" - role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" - database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" - warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: fivetran_platform_integration_tests - threads: 8 - postgres: - type: postgres - host: "{{ env_var('CI_POSTGRES_DBT_HOST') }}" - user: "{{ env_var('CI_POSTGRES_DBT_USER') }}" - pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" - dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" - port: 5432 - schema: fivetran_platform_integration_tests - threads: 8 - databricks: - catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" - host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" - http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: fivetran_platform_integration_tests - threads: 1 - token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" - type: databricks - databricks-sql: - catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" - host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" - http_path: "{{ env_var('CI_DATABRICKS_SQL_DBT_HTTP_PATH') }}" - schema: fivetran_platform_sqlw_tests - threads: 1 - token: "{{ env_var('CI_DATABRICKS_SQL_DBT_TOKEN') }}" - type: databricks - sqlserver: - type: sqlserver - driver: 'ODBC Driver 18 for SQL Server' - server: "{{ env_var('CI_SQLSERVER_DBT_SERVER') }}" - port: 1433 - database: "{{ env_var('CI_SQLSERVER_DBT_DATABASE') }}" - schema: fivetran_platform_integration_tests - user: "{{ env_var('CI_SQLSERVER_DBT_USER') }}" - password: "{{ env_var('CI_SQLSERVER_DBT_PASS') }}" - threads: 8 diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml new file mode 100644 index 00000000..8a00c3a1 --- /dev/null +++ b/integration_tests/ci/test_scenarios.yml @@ -0,0 +1,36 @@ +schema_variable_name: "fivetran_platform_schema" + +include_databricks_sql: true +include_sqlserver: true + +test_scenarios: + - name: "Default" + vars: {} + include_incremental: true + + - name: "log json data" + vars: + fivetran_platform_log_identifier: "log_json_data" + include_incremental: false + warehouses: ["bigquery", "redshift", "postgres"] + + - name: "usage pricing enabled" + vars: + fivetran_platform__usage_pricing: true + fivetran_platform_using_connection: false + include_incremental: true + + - name: "usage pricing with transformations" + vars: + fivetran_platform__credits_pricing: false + fivetran_platform__usage_pricing: true + fivetran_platform_using_transformations: true + include_incremental: true + + - name: "minimal features" + vars: + fivetran_platform__usage_pricing: false + fivetran_platform_using_destination_membership: false + fivetran_platform_using_user: false + fivetran_platform_using_transformations: false + include_incremental: true \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 60185ace..bfbcb1f6 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -35,36 +35,42 @@ seeds: +docs: show: False fivetran_log_integration_tests: - +column_types: - _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" account: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" created_at: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" incremental_mar: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" measured_date: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" incremental_rows: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" connection: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" signed_up: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" _fivetran_deleted: "{{ 'bit' if target.type == 'sqlserver' else 'boolean' }}" connector: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" signed_up: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" _fivetran_deleted: "{{ 'bit' if target.type == 'sqlserver' else 'boolean' }}" credits_used: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" credits_consumed: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" destination: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" created_at: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" id: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' if target.type != 'sqlserver' else 'varchar(256)' }}" destination_membership: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" activated_at: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" joined_at: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" log: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" time_stamp: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" message_data: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' if target.type != 'sqlserver' else 'varchar(256)' }}" transformation_id: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' if target.type != 'sqlserver' else 'varchar(256)' }}" @@ -72,11 +78,19 @@ seeds: +enabled: "{{ target.type in ('bigquery', 'redshift', 'postgres') }}" # @docs-ignore # +enabled: false # @docs-include +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" time_stamp: timestamp message_data: "{{ 'super' if target.type == 'redshift' else 'json' }}" transformation_id: "{{ 'string' if target.type == 'bigquery' else 'varchar' }}" + transformation_runs: + +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" + usage_cost: + +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" user: +column_types: + _fivetran_synced: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" created_at: "{{ 'datetime2' if target.type == 'sqlserver' else 'timestamp' }}" clean-targets: # directories to be removed by `dbt clean` @@ -86,4 +100,4 @@ clean-targets: # directories to be removed by `dbt clean` flags: send_anonymous_usage_stats: False - use_colors: True + use_colors: True \ No newline at end of file diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 8c55c238..e6157201 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -5,4 +5,3 @@ dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0 dbt-databricks>=1.6.0,<2.0.0 -certifi==2025.1.31 \ No newline at end of file