Adds bulk-import option for "import-records" task

.github/scripts/check_exported_data.py

@@ -10,6 +10,14 @@
 - The file contains the expected plugin configuration
 - The file contains the expected plugin database records
 
+It can also optionally check the presence / absence of several categories of
+data which 'export-records' can include or exclude via --include-x /
+--exclude-x flags (email logs, API tokens, SSO app/token data, user sessions,
+and non-empty group/user permissions) - pass e.g. '--check-email include' or
+'--check-email exclude' to assert that category was (or was not) found in the
+exported data. Any '--check-x' option which is *not* passed is simply not
+checked at all (not even implicitly assumed absent) - so existing invocations
+which don't pass any of them keep working unchanged.
 """
 
 PLUGIN_KEY = 'dummy_app_plugin'
@@ -19,10 +27,112 @@
 import json
 import os
 
+
+def check_category(
+    data: list[dict], label: str, model_names: list[str], expect: str | None
+):
+    """Check that all of the given model names are present / absent as expected.
+
+    Arguments:
+        data: The loaded (parsed) exported data file.
+        label: A human-readable label for this category, for error messages.
+        model_names: The Django model labels (e.g. 'common.emailmessage')
+            which make up this category.
+        expect: 'include' - at least one entry for *each* model name must be
+            present. 'exclude' - *no* entry for *any* model name may be
+            present. None - this category was not requested to be checked;
+            do nothing.
+    """
+    if expect is None:
+        return
+
+    expect_present = expect == 'include'
+
+    counts = dict.fromkeys(model_names, 0)
+
+    for entry in data:
+        model = entry.get('model', None)
+        if model in counts:
+            counts[model] += 1
+
+    if expect_present:
+        for model, count in counts.items():
+            if count == 0:
+                print(f"Error: Expected '{label}' data ('{model}') was not found")
+                exit(1)
+        print(f"Found expected '{label}' data ({counts})")
+    else:
+        for model, count in counts.items():
+            if count > 0:
+                print(
+                    f"Error: '{label}' data ('{model}') was found, but should have been excluded ({count} record(s))"
+                )
+                exit(1)
+        print(f"Confirmed '{label}' data was correctly excluded")
+
+
+def check_permissions(data: list[dict], expect: str | None):
+    """Check that auth.group / auth.user permission fields are stripped or preserved as expected.
+
+    Arguments:
+        data: The loaded (parsed) exported data file.
+        expect: 'include' - at least one auth.group / auth.user entry must
+            have non-empty permissions. 'exclude' - all such entries must have
+            empty permissions. None - not checked; do nothing.
+    """
+    if expect is None:
+        return
+
+    expect_present = expect == 'include'
+
+    group_perms = [
+        entry['fields'].get('permissions', [])
+        for entry in data
+        if entry.get('model') == 'auth.group'
+    ]
+    user_perms = [
+        entry['fields'].get('user_permissions', [])
+        for entry in data
+        if entry.get('model') == 'auth.user'
+    ]
+
+    any_group_perms = any(group_perms)
+    any_user_perms = any(user_perms)
+
+    if expect_present:
+        if not any_group_perms and not any_user_perms:
+            print(
+                'Error: Expected at least one auth.group / auth.user entry with '
+                'non-empty permissions, but all were empty'
+            )
+            exit(1)
+        print('Found expected non-empty group/user permissions')
+    else:
+        if any_group_perms or any_user_perms:
+            print(
+                'Error: Found non-empty group/user permissions, but they should '
+                'have been stripped'
+            )
+            exit(1)
+        print('Confirmed group/user permissions were correctly stripped')
+
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Check exported data file')
     parser.add_argument('datafile', help='Path to the exported data file (JSON)')
 
+    # Plugin data is checked unconditionally below (it always has been) - this
+    # just controls which direction is expected, mirroring export-records' own
+    # --exclude-plugins flag (plugin data is included by default).
+    parser.add_argument('--exclude-plugins', action='store_true')
+
+    # The remaining categories are only checked when explicitly requested -
+    # pass 'include' or 'exclude' to assert that direction, or omit the flag
+    # entirely to skip checking that category (the default, for backwards
+    # compatibility with existing invocations that don't pass any of these).
+    for flag in ('email', 'tokens', 'sso', 'session', 'permissions'):
+        parser.add_argument(f'--check-{flag}', choices=['include', 'exclude'])
+
     args = parser.parse_args()
 
     if not os.path.isfile(args.datafile):
@@ -84,18 +194,45 @@
         )
         exit(1)
 
-    if not found_plugin_config:
-        print(f'Error: No plugin configuration found for plugin "{PLUGIN_KEY}"')
-        exit(1)
-
-    # Check the extracted plugin records
-    expected_keys = ['alpha', 'beta', 'gamma', 'delta']
-
-    for key in expected_keys:
-        if key not in plugin_data_records:
-            print(
-                f'Error: Expected plugin record with key "{key}" not found in exported data'
-            )
+    # Plugin data is included by default (export-records only excludes it when
+    # given --exclude-plugins), so preserve that as the default expectation here
+    if not args.exclude_plugins:
+        if not found_plugin_config:
+            print(f'Error: No plugin configuration found for plugin "{PLUGIN_KEY}"')
             exit(1)
 
+        # Check the extracted plugin records
+        expected_keys = ['alpha', 'beta', 'gamma', 'delta']
+
+        for key in expected_keys:
+            if key not in plugin_data_records:
+                print(
+                    f'Error: Expected plugin record with key "{key}" not found in exported data'
+                )
+                exit(1)
+    elif found_plugin_config or plugin_data_records:
+        print('Error: Plugin data was found, but should have been excluded')
+        exit(1)
+    else:
+        print('Confirmed plugin data was correctly excluded')
+
+    # Content-excludes checks - only run for '--check-x' flags that were actually passed
+    check_category(
+        data, 'email', ['common.emailmessage', 'common.emailthread'], args.check_email
+    )
+    check_category(data, 'tokens', ['users.apitoken'], args.check_tokens)
+    check_category(
+        data,
+        'sso',
+        ['socialaccount.socialapp', 'socialaccount.socialtoken'],
+        args.check_sso,
+    )
+    check_category(
+        data,
+        'session',
+        ['sessions.session', 'usersessions.usersession'],
+        args.check_session,
+    )
+    check_permissions(data, args.check_permissions)
+
     print('All checks passed successfully!')

.github/scripts/seed_content_excludes_data.py

@@ -0,0 +1,112 @@
+"""Script to seed test data for the 'content-excludes' export CI job.
+
+'export-records' can optionally include/exclude several categories of data
+(email logs, API tokens, SSO app/token data, user sessions, and group/user
+permissions) via --include-x / --exclude-x flags. Toggling one of those flags
+only proves anything if the source database actually contains a row in that
+category to begin with - otherwise "the export doesn't contain it" is true
+regardless of whether the flag/exclusion logic works at all.
+
+This script creates exactly one row in each such category, so the
+import_export.yaml workflow's content-excludes job can meaningfully assert
+both "included when asked for" and "excluded by default".
+
+Intended to be run from 'src/backend/InvenTree', e.g.:
+    cd src/backend/InvenTree && python ../../../.github/scripts/seed_content_excludes_data.py
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.getcwd())
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'InvenTree.settings')
+
+import django
+
+django.setup()
+
+from django.contrib.auth import get_user_model
+from django.contrib.auth.models import Group, Permission
+from django.contrib.sessions.backends.db import SessionStore
+
+from allauth.socialaccount.models import SocialAccount, SocialApp, SocialToken
+from allauth.usersessions.models import UserSession
+
+from common.models import EmailMessage, Priority
+from users.models import ApiToken
+
+User = get_user_model()
+
+
+def main():
+    """Seed one row of test data in each optional export/import category."""
+    user = User.objects.filter(is_superuser=True).first()
+
+    if user is None:
+        print('Error: no superuser found - run `invoke dev.setup-test` first')
+        sys.exit(1)
+
+    # Ensure at least one group has a non-empty permission set, so toggling
+    # --include-permissions has something real to include/strip. Doesn't need
+    # to be a group the superuser belongs to - InvenTree's RuleSet groups
+    # already have permissions assigned regardless of membership.
+    group = Group.objects.first()
+
+    if group is None:
+        print('Error: no groups found - run `invoke dev.setup-test` first')
+        sys.exit(1)
+
+    if not group.permissions.exists():
+        group.permissions.add(Permission.objects.first())
+        print(f"- Added a permission to group '{group.name}' (was empty)")
+    else:
+        print(f"- Group '{group.name}' already has permissions")
+
+    # Email log entry (thread is auto-created by EmailMessage.save() if omitted)
+    EmailMessage.objects.get_or_create(
+        subject='CI content-excludes test email',
+        defaults={
+            'body': 'CI content-excludes test email body',
+            'to': 'ci-recipient@example.com',
+            'sender': 'ci-sender@example.com',
+            'priority': Priority.NORMAL,
+        },
+    )
+    print('- Created email log entry')
+
+    # API token
+    ApiToken.objects.get_or_create(user=user, name='ci-content-excludes-token')
+    print('- Created API token')
+
+    # SSO application + linked account + token
+    app, _ = SocialApp.objects.get_or_create(
+        provider='google',
+        name='CI Content-Excludes Test App',
+        defaults={'client_id': 'ci-test-client-id'},
+    )
+    account, _ = SocialAccount.objects.get_or_create(
+        user=user, provider='google', uid='ci-test-external-uid'
+    )
+    SocialToken.objects.get_or_create(
+        app=app, account=account, defaults={'token': 'ci-test-token-value'}
+    )
+    print('- Created SSO application, account and token')
+
+    # A real, properly-encoded session (avoids writing an undecodable session_data blob)
+    store = SessionStore()
+    store['ci_content_excludes_test'] = True
+    store.create()
+    print('- Created session entry')
+
+    # allauth user-session record (tracked separately from the raw Session table)
+    UserSession.objects.get_or_create(
+        session_key='ci-content-excludes-user-session',
+        defaults={'user': user, 'ip': '127.0.0.1', 'user_agent': 'ci-test-agent'},
+    )
+    print('- Created user session entry')
+
+    print('Content-excludes seed data created successfully')
+
+
+if __name__ == '__main__':
+    main()

.github/workflows/import_export.yaml

@@ -58,6 +58,7 @@
             server:
               - .github/workflows/import_export.yaml
               - .github/scripts/check_exported_data.py
+              - .github/scripts/seed_content_excludes_data.py
               - 'src/backend/**'
               - 'tasks.py'
   test:
@@ -112,9 +113,78 @@
           test -f /home/runner/work/InvenTree/test_inventree_db.sqlite3 || (echo "Sqlite database not created" && exit 1)
       - name: Import Sqlite Dataset
         run: |
+          # Run two imports back-to-back to ensure that the import process is idempotent
+          invoke import-records -c -f ${{ env.DATA_FILE }} --strict
           invoke import-records -c -f ${{ env.DATA_FILE }} --strict
           cd src/backend/InvenTree && python manage.py check_dummy_data
-      - name: Export Sqlite Dataset
-        run: |
           invoke export-records -o -f ${{ env.DATA_FILE }}
           python .github/scripts/check_exported_data.py ${{ env.DATA_FILE }}
+      - name: Bulk Import Sqlite Dataset
+        run: |
+          # Ensure that the 'bulk' import process works as expected
+          invoke import-records -c -f ${{ env.DATA_FILE }} --strict --bulk
+          cd src/backend/InvenTree && python manage.py check_dummy_data
+          invoke export-records -o -f ${{ env.DATA_FILE }} --prettify
+          python .github/scripts/check_exported_data.py ${{ env.DATA_FILE }}
+
+  content-excludes:
+    # Ensure that 'export-records' correctly includes / excludes each optional
+    # category of data (email logs, API tokens, SSO app/token data, user
+    # sessions, and group/user permissions) according to its --include-x /
+    # --exclude-x flags. Separate from the 'test' job above since it exercises
+    # a different axis of behaviour (export content, not the import/export
+    # round-trip) and doesn't need the Sqlite half at all.
+    runs-on: ubuntu-latest
+    needs: paths-filter
+    if: needs.paths-filter.outputs.server == 'true' || contains(github.event.pull_request.labels.*.name, 'full-run')
+
+    services:
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_USER: inventree
+          POSTGRES_PASSWORD: password
+        ports:
+          - 5432:5432
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Environment Setup
+        uses: ./.github/actions/setup
+        with:
+          apt-dependency: gettext poppler-utils libpq-dev
+          pip-dependency: psycopg
+          update: true
+          static: false
+      - name: Setup Postgres Database
+        run: |
+          invoke migrate
+          invoke dev.setup-test -i
+      - name: Create Plugin Data
+        run: |
+          pip install -U inventree-dummy-app-plugin==0.1.0
+          invoke migrate
+          cd src/backend/InvenTree && python manage.py create_dummy_data
+      - name: Seed Content-Excludes Test Data
+        run: |
+          # Creates one row in each optional export category (email log, API
+          # token, SSO app/token, session, group permissions), so that toggling
+          # the corresponding flag below has real data to prove it actually works
+          cd src/backend/InvenTree
+          python ../../../.github/scripts/seed_content_excludes_data.py
+      - name: Export - All Optional Categories Included
+        run: |
+          invoke export-records -o -f ${{ env.DATA_FILE }} --include-email --include-permissions --include-tokens --include-sso --include-session
+          python .github/scripts/check_exported_data.py ${{ env.DATA_FILE }} \
+            --check-email include --check-tokens include --check-sso include \
+            --check-session include --check-permissions include
+      - name: Export - All Optional Categories Excluded
+        run: |
+          invoke export-records -o -f ${{ env.DATA_FILE }} --exclude-plugins
+          python .github/scripts/check_exported_data.py ${{ env.DATA_FILE }} --exclude-plugins \
+            --check-email exclude --check-tokens exclude --check-sso exclude \
+            --check-session exclude --check-permissions exclude