v1.6.0 - Add fields and persist passthrough cols

.quickstart/quickstart.yml

@@ -88,31 +88,41 @@ supported_vars:
     description: "Additional `CLASSIFICATION` columns to include in `netsuite2__income_statement`"
     display_name: "Additional Class Columns"
 
+  customers_pass_through_columns:
+    type: dictionary
+    description: "Additional `CUSTOMER` columns to include in `netsuite2__transaction_details`"
+    display_name: "Additional Customer Columns"
+
   departments_pass_through_columns:
     type: dictionary
     description: "Additional `DEPARTMENT` columns to include in `netsuite2__transaction_details` and `netsuite2__income_statement`"
     display_name: "Additional Department Columns"
 
-  transactions_pass_through_columns:
+  items_pass_through_columns:
     type: dictionary
-    description: "Additional `TRANSACTION` columns to include in `netsuite2__transaction_details`"
-    display_name: "Additional Transaction Columns"
-
-  transaction_lines_pass_through_columns:
-    type: dictionary
-    description: "Additional `TRANSACTION_LINE` columns to include in `netsuite2__transaction_details`"
-    display_name: "Additional Transaction Line Columns"
+    description: "Additional `ITEM` columns to include in `netsuite2__transaction_details`"
+    display_name: "Additional Item Columns"
 
   locations_pass_through_columns:
     type: dictionary
     description: "Additional `LOCATION` columns to include in `netsuite2__transaction_details`"
     display_name: "Additional Location Columns"
-
+    
   subsidiaries_pass_through_columns:
     type: dictionary
     description: "Additional `SUBSIDIARY` columns to include in `netsuite2__transaction_details`"
     display_name: "Additional Subsidiary Columns"
 
+  transactions_pass_through_columns:
+    type: dictionary
+    description: "Additional `TRANSACTION` columns to include in `netsuite2__transaction_details`"
+    display_name: "Additional Transaction Columns"
+
+  transaction_lines_pass_through_columns:
+    type: dictionary
+    description: "Additional `TRANSACTION_LINE` columns to include in `netsuite2__transaction_details`"
+    display_name: "Additional Transaction Line Columns"
+
   balance_sheet_transaction_detail_columns:
     type: list
     description: "Additional columns from `netsuite2__transaction_details` to include in `netsuite2__balance_sheet`"

CHANGELOG.md

@@ -1,3 +1,32 @@
+# dbt_netsuite v1.6.0-a1
+[PR #200](https://github.com/fivetran/dbt_netsuite/pull/200) includes the following updates:
+
+## Schema/Data Change (--full-refresh required after upgrading)
+**2 total changes • 0 possible breaking changes**
+
+| Data Model(s) | Change type | Old | New | Notes |
+| ------------- | ----------- | --- | --- | ----- |
+| `netsuite2__transaction_details`<br>`netsuite2__entity_subsidiary_relationships` | Filter on `VENDOR` data | Soft-deleted records excluded | Soft-deleted records included |  |
+| `netsuite2__transaction_details` | New field |  | `amount_linked` | The amount applied against another transaction (e.g. a payment applied to an invoice). Sourced from `amountlinked` in the `TRANSACTION_ACCOUNTING_LINE` table. |
+
+<details><summary> <i>dbt Core/Additional Details</i> </summary>
+
+**5 total changes • 2 possible breaking changes**
+
+| Data Model(s) | Change type | Old | New | Notes |
+| ------------- | ----------- | --- | --- | ----- |
+| `stg_netsuite2__customers` | New fields |  | `is_inactive`, `comments`, `url`, `created_at` | Additional customer attributes now available in the staging model. Sourced from `isinactive`, `comments`, `url`, and `datecreated` in the `CUSTOMER` table. Remove from `customers_pass_through_columns` if currently included. |
+| `stg_netsuite2__entities` | New fields |  | `date_created`, `_fivetran_deleted` | Remove from `entities_pass_through_columns` if currently included. |
+| `stg_netsuite2__transaction_accounting_lines` | New field |  | `amount_linked` | Sourced from `amountlinked` in the `TRANSACTION_ACCOUNTING_LINE` table. |
+| `stg_netsuite2__transaction_lines` | New field | | `_fivetran_deleted` |  |
+| `stg_netsuite2__transactions` | New field | | `_fivetran_deleted` | Soft-deleted records are still filtered out. |
+
+</details>
+<br>
+
+## Feature Update
+- Adds **Additional Customer Columns** (`customers_pass_through_columns`) and **Additional Item Columns** (`items_pass_through_columns`) to Quickstart, persisting chosen custom columns from the `CUSTOMER` and `ITEM` source tables. These fields are brought into `netsuite2__transaction_details`.
+
 # dbt_netsuite v1.5.1
 [PR #199](https://github.com/fivetran/dbt_netsuite/pull/199) includes the following updates:
 

README.md

@@ -126,7 +126,7 @@ Include the following netsuite package version in your `packages.yml` file:
 ```yaml
 packages:
   - package: fivetran/netsuite
-    version: [">=1.5.0", "<1.6.0"]
+    version: 1.6.0-a1
 ```
 
 #### Databricks dispatch configuration
@@ -272,40 +272,40 @@ vars:
     accounts_pass_through_columns: # Included in all end models
         - name: "new_custom_field"
           alias: "custom_field"
-    classes_pass_through_columns: # Included in income_statement models
+    classes_pass_through_columns: # Included in income_statement model
         - name: "this_field"
+    customers_pass_through_columns: # Included in transaction_details model
+        - name: "customer_custom_field"
+          alias: "customer_field"
+    items_pass_through_columns: # Included in transaction_details model
+        - name: "items_custom_field"
     departments_pass_through_columns: # Included in income_statement and transaction_details models
         - name: "unique_string_field"
           alias: "field_id"
           transform_sql: "cast(field_id as string)"
-    transactions_pass_through_columns: # Included in transaction_details models
+    transactions_pass_through_columns: # Included in transaction_details model
         - name: "that_field"
-    transaction_lines_pass_through_columns: # Included in transaction_details models
+    transaction_lines_pass_through_columns: # Included in transaction_details model
         - name: "other_id"
           alias: "another_id"
           transform_sql: "cast(another_id as int64)"
-    locations_pass_through_columns: # Included in transaction_details models
+    locations_pass_through_columns: # Included in transaction_details model
         - name: "location_custom_field"
-    subsidiaries_pass_through_columns: # Included in transaction_details models
+    subsidiaries_pass_through_columns: # Included in transaction_details model
         - name: "sub_field"
           alias: "subsidiary_field"
-    customers_pass_through_columns: # Not included in end models; only in stg customer models
-        - name: "customer_custom_field"
-          alias: "customer_field"
-    consolidated_exchange_rates_pass_through_columns: # Not included in end models; only in stg consolidated_exchange_rates models
+    consolidated_exchange_rates_pass_through_columns: # Not included in end models; only in stg consolidated_exchange_rates model
         - name: "consolidate_this_field"
     entities_pass_through_columns: # Not included in end models; only in stg entities model
         - name: "entity_custom_field"
           alias: "entity_field"
-    accounting_periods_pass_through_columns: # Not included in end models; only in stg accounting_periods models
+    accounting_periods_pass_through_columns: # Not included in end models; only in stg accounting_periods model
         - name: "custom_field"
           transform_sql: "cast(custom_field as string)"
-    vendors_pass_through_columns: # Not included in end models; only in stg vendors models
+    vendors_pass_through_columns: # Not included in end models; only in stg vendors model
         - name: "vendors_custom_field"
           alias: "vendors_field"
-    items_pass_through_columns: # Not included in end models; only in stg items models
-        - name: "items_custom_field"
-    nexuses_pass_through_columns: # Not included in end models; only in stg items models
+    nexuses_pass_through_columns: # Not included in end models; only in stg nexuses model
         - name: "items_custom_field"
 ```
 

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'netsuite'
-version: '1.5.1'
+version: '1.6.0'
 require-dbt-version: [">=1.3.0", "<3.0.0"]
 
 models:

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'netsuite_integration_tests'
-version: '1.5.1'
+version: '1.6.0'
 profile: 'integration_tests'
 config-version: 2
 

integration_tests/seeds/netsuite2_transaction_data.csv

@@ -1,5 +1,5 @@
-_fivetran_synced,_fivetran_active,id,transactionnumber,type,memo,trandate,status,createddate,duedate,closedate,currency,entity,postingperiod,posting,isreversal,reversal,reversaldate,reversaldefer
-2022-05-27 13:41:13,TRUE,5914,,VendBill,,2017-05-07 0:00:00,A,2017-05-07 10:34:43,2017-05-05 0:00:00,,1,1570,,T,F,325251,2023-08-01 0:00:00,F
-2022-05-27 13:41:13,TRUE,3,,VendBill,,2015-09-10 0:00:00,B,2017-05-12 3:41:25,2015-10-09 0:00:00,2015-10-05 0:00:00,1,36,251,T,F,325252,2023-08-02 0:00:00,F
-2022-05-27 13:41:13,TRUE,302,,VendBill,ZZZZZ,2016-03-20 0:00:00,B,2017-04-25 3:41:25,2016-04-19 0:00:00,2017-05-07 0:00:00,1,5,261,T,F,325323,2023-08-03 0:00:00,F
-2022-05-26 13:41:13,FALSE,302,,VendBill,ZZZZZ,2016-03-20 0:00:00,B,2017-04-25 3:41:25,2016-04-19 0:00:00,2017-05-07 0:00:00,1,5,261,T,F,325323,2023-08-03 0:00:00,F
+_fivetran_synced,_fivetran_active,_fivetran_deleted,id,transactionnumber,type,memo,trandate,status,createddate,duedate,closedate,currency,entity,postingperiod,posting,isreversal,reversal,reversaldate,reversaldefer
+2022-05-27 13:41:13,TRUE,FALSE,5914,,VendBill,,2017-05-07 0:00:00,A,2017-05-07 10:34:43,2017-05-05 0:00:00,,1,1570,,T,F,325251,2023-08-01 0:00:00,F
+2022-05-27 13:41:13,TRUE,FALSE,3,,VendBill,,2015-09-10 0:00:00,B,2017-05-12 3:41:25,2015-10-09 0:00:00,2015-10-05 0:00:00,1,36,251,T,F,325252,2023-08-02 0:00:00,F
+2022-05-27 13:41:13,TRUE,FALSE,302,,VendBill,ZZZZZ,2016-03-20 0:00:00,B,2017-04-25 3:41:25,2016-04-19 0:00:00,2017-05-07 0:00:00,1,5,261,T,F,325323,2023-08-03 0:00:00,F
+2022-05-26 13:41:13,FALSE,FALSE,302,,VendBill,ZZZZZ,2016-03-20 0:00:00,B,2017-04-25 3:41:25,2016-04-19 0:00:00,2017-05-07 0:00:00,1,5,261,T,F,325323,2023-08-03 0:00:00,F

macros/staging/get_customers_columns.sql

@@ -28,17 +28,21 @@
     {"name": "externalid", "datatype": dbt.type_string()},
     {"name": "parent", "datatype": dbt.type_int()},
     {"name": "isperson", "datatype": dbt.type_string()},
+    {"name": "isinactive", "datatype": dbt.type_string()},
     {"name": "altname", "datatype": dbt.type_string()},
     {"name": "companyname", "datatype": dbt.type_string()},
     {"name": "firstname", "datatype": dbt.type_string()},
     {"name": "lastname", "datatype": dbt.type_string()},
     {"name": "email", "datatype": dbt.type_string()},
     {"name": "phone", "datatype": dbt.type_string()},
+    {"name": "comments", "datatype": dbt.type_string()},
+    {"name": "url", "datatype": dbt.type_string()},
     {"name": "defaultbillingaddress", "datatype": dbt.type_int()},
     {"name": "defaultshippingaddress", "datatype": dbt.type_int()},
     {"name": "receivablesaccount", "datatype": dbt.type_int()},
     {"name": "currency", "datatype": dbt.type_int()},
-    {"name": "firstorderdate", "datatype": dbt.type_timestamp()}
+    {"name": "firstorderdate", "datatype": dbt.type_timestamp()},
+    {"name": "datecreated", "datatype": dbt.type_timestamp()}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('customers_pass_through_columns')) }}

macros/staging/get_entity_columns.sql

@@ -6,6 +6,7 @@
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
     {"name": "contact", "datatype": dbt.type_int()},
     {"name": "customer", "datatype": dbt.type_int()}, 
+    {"name": "datecreated", "datatype": dbt.type_timestamp()},
     {"name": "employee", "datatype": dbt.type_int()},
     {"name": "entitytitle", "datatype": dbt.type_string()},
     {"name": "id", "datatype": dbt.type_int()},

macros/staging/get_transactionaccountingline_columns.sql

@@ -34,6 +34,7 @@
     {"name": "account", "datatype": dbt.type_int()},
     {"name": "accountingbook", "datatype": dbt.type_int()},
     {"name": "amount", "datatype": dbt.type_float()},
+    {"name": "amountlinked", "datatype": dbt.type_float()},
     {"name": "amountpaid", "datatype": dbt.type_float()},
     {"name": "amountunpaid", "datatype": dbt.type_float()},
     {"name": "credit", "datatype": dbt.type_float()},

models/netsuite2.yml

@@ -384,6 +384,8 @@ models:
       description: Original transaction line amount, without sign adjustments. Reflects the raw amount entered in the source system.
     - name: transaction_line_amount_raw
       description: Original net amount of the transaction line, without sign adjustments. Matches the amount entered in the transaction currency.
+    - name: amount_linked
+      description: The amount of the transaction line that has been applied against another transaction (e.g. a payment applied to an invoice).
     - name: accounting_book_id
       description: The unique identifier of the accounting book.
     - name: accounting_book_name

models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql

@@ -35,6 +35,7 @@ joined as (
 
         transaction_accounting_lines.exchange_rate,
         transaction_accounting_lines.amount,
+        transaction_accounting_lines.amount_linked,
         transaction_accounting_lines.credit_amount,
         transaction_accounting_lines.debit_amount,
         transaction_accounting_lines.paid_amount,
@@ -46,7 +47,7 @@ joined as (
         on transaction_lines.transaction_line_id = transaction_accounting_lines.transaction_line_id
         and transaction_lines.transaction_id = transaction_accounting_lines.transaction_id
         and transaction_lines.source_relation = transaction_accounting_lines.source_relation
-        
+
     {% if multibook_accounting_enabled %}
     left join accounting_books
         on accounting_books.accounting_book_id = transaction_accounting_lines.accounting_book_id
@@ -61,6 +62,7 @@ joined as (
         accounting_books.accounting_book_name,
         transaction_accounting_lines.exchange_rate,
         transaction_accounting_lines.amount,
+        transaction_accounting_lines.amount_linked,
         transaction_accounting_lines.credit_amount,
         transaction_accounting_lines.debit_amount,
         transaction_accounting_lines.paid_amount,

models/netsuite2/netsuite2__transaction_details.sql

@@ -3,6 +3,8 @@
 {%- set using_exchange_rate = var('netsuite2__using_exchange_rate', true) -%}
 {%- set using_vendor_categories = var('netsuite2__using_vendor_categories', true) -%}
 {%- set accounts_pass_through_columns = var('accounts_pass_through_columns', []) -%}
+{%- set customers_pass_through_columns = var('customers_pass_through_columns', []) -%}
+{%- set items_pass_through_columns = var('items_pass_through_columns', []) -%}
 {%- set departments_pass_through_columns = var('departments_pass_through_columns', []) -%}
 {%- set locations_pass_through_columns = var('locations_pass_through_columns', []) -%}
 {%- set subsidiaries_pass_through_columns = var('subsidiaries_pass_through_columns', []) -%}
@@ -228,12 +230,27 @@ transaction_details as (
       when lower(transactions.transaction_type) in ('custinvc', 'custcred') then customers__transactions.customer_external_id
       else customers__transaction_lines.customer_external_id
         end as customer_external_id,
+
+    {% if customers_pass_through_columns != [] %}
+      {% for col in customers_pass_through_columns%}
+
+        case 
+            when lower(transactions.transaction_type) in ('custinvc', 'custcred') then customers__transactions.{{ col.alias if col.alias else col.name }}
+            else customers__transaction_lines.{{ col.alias if col.alias else col.name }}
+              end as {{ col.alias if col.alias else col.name }},
+      {% endfor %}
+    {% endif %}
+
     classes.class_id,
     classes.full_name as class_full_name,
     transaction_lines.item_id,
     items.name as item_name,
     items.type_name as item_type_name,
-    items.sales_description,
+    items.sales_description
+
+    -- The below script allows for items table pass through columns.
+    {{ netsuite.persist_pass_through_columns(items_pass_through_columns, identifier='items') }},
+
     locations.location_id,
     locations.name as location_name,
     locations.city as location_city,
@@ -303,7 +320,8 @@ transaction_details as (
 
     transactions_with_converted_amounts.converted_amount_using_transaction_accounting_period as converted_amount_raw,
     transaction_lines.amount as transaction_amount_raw,
-    transaction_lines.netamount as transaction_line_amount_raw
+    transaction_lines.netamount as transaction_line_amount_raw,
+    transaction_lines.amount_linked
   from transaction_lines
 
   join transactions

models/netsuite2/staging/src_netsuite2.yml

@@ -283,6 +283,8 @@ sources:
             description: Reference to the parent customer.
           - name: isperson
             description: Boolean indicating if the customer is an individual person. Represented as "T" or "F" for true and false respectively.
+          - name: isinactive
+            description: Boolean indicating if the customer record is inactive. Represented as "T" or "F" for true and false respectively.
           - name: companyname
             description: "{{ doc('company_name') }}"
           - name: firstname
@@ -293,6 +295,10 @@ sources:
             description: Customers email address.
           - name: phone
             description: Phone number of the customer.
+          - name: comments
+            description: Internal comments or notes on the customer record.
+          - name: url
+            description: Website URL associated with the customer.
           - name: defaultbillingaddress
             description: Reference to the associated billing address.
           - name: defaultshippingaddress
@@ -303,8 +309,12 @@ sources:
             description: "{{ doc('currency_id') }}"
           - name: firstorderdate
             description: Timestamp of when the first order was created.
+          - name: datecreated
+            description: The date and time when the customer record was created.
           - name: altname
             description: "{{ doc('alt_name') }}"
+          - name: _fivetran_deleted
+            description: "{{ doc('_fivetran_deleted') }}"
 
       - name: customer_subsidiary_relationship
         identifier: "{{ var('netsuite2_customer_subsidiary_relationship_identifier', 'customersubsidiaryrelationship') }}"
@@ -417,6 +427,10 @@ sources:
             description: The entity type (Contact, CustJob, Job, etc).
           - name: vendor
             description: The unique identifier of the vendor associated with the entity.
+          - name: datecreated
+            description: The date and time when the entity record was created.
+          - name: _fivetran_deleted
+            description: "{{ doc('_fivetran_deleted') }}"
 
       - name: entity_address
         identifier: "{{ var('netsuite2_entity_address_identifier', 'entityaddress') }}"
@@ -637,6 +651,8 @@ sources:
             description: Total amount paid.
           - name: amountunpaid
             description: Total amount unpaid.
+          - name: amountlinked
+            description: Total amount linked to other transaction lines.
 
       - name: transaction_line
         identifier: "{{ var('netsuite2_transaction_line_identifier', 'transactionline') }}"
@@ -682,6 +698,8 @@ sources:
             description: Boolean indicating if the transaction line is eliminated. Represented as "T" or "F" for true and false respectively.
           - name: netamount
             description: Net amount of transaction line.
+          - name: _fivetran_deleted
+            description: "{{ doc('_fivetran_deleted') }}"
 
       - name: transaction
         identifier: "{{ var('netsuite2_transaction_identifier', 'transaction') }}"
@@ -727,6 +745,8 @@ sources:
             description: If the transaction is a reversal, date of the reversal entry.
           - name: reversaldefer
             description: Boolean indicating if the transaction is a reversal and if it's been deferred.
+          - name: _fivetran_deleted
+            description: "{{ doc('_fivetran_deleted') }}"
 
       - name: vendor_category
         identifier: "{{ var('netsuite2_vendor_category_identifier', 'vendorcategory') }}"