From 53ee61791566e341e8a90c2490257f725dd9945c Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 26 May 2026 17:25:05 -0700 Subject: [PATCH 01/42] #21488 - Replace MPTT wtih PostgreSQL Ltree --- base_requirements.txt | 3 - netbox/core/models/change_logging.py | 8 +- netbox/dcim/api/views.py | 14 +- netbox/dcim/graphql/types.py | 16 +- netbox/dcim/migrations/0002_squashed.py | 9 +- netbox/dcim/migrations/0131_squashed_0159.py | 3 +- netbox/dcim/migrations/0190_nested_modules.py | 3 +- .../migrations/0191_module_bay_rebuild.py | 15 +- .../migrations/0203_device_role_nested.py | 3 +- .../migrations/0204_device_role_rebuild.py | 16 +- .../dcim/migrations/0213_platform_parent.py | 3 +- .../dcim/migrations/0214_platform_rebuild.py | 21 +- .../migrations/0226_modulebay_rebuild_tree.py | 25 +- .../migrations/0234_enable_ltree_extension.py | 17 + netbox/dcim/migrations/0235_ltree_paths.py | 137 ++++++ .../dcim/models/device_component_templates.py | 12 +- netbox/dcim/models/device_components.py | 30 +- netbox/dcim/models/devices.py | 13 +- netbox/dcim/models/modules.py | 36 +- netbox/dcim/models/sites.py | 19 +- netbox/extras/querysets.py | 28 +- netbox/netbox/api/viewsets/__init__.py | 19 - netbox/netbox/constants.py | 11 - netbox/netbox/graphql/filter_lookups.py | 15 +- netbox/netbox/models/__init__.py | 23 +- netbox/netbox/models/ltree.py | 417 ++++++++++++++++++ netbox/netbox/settings.py | 1 - netbox/netbox/tables/columns.py | 9 +- netbox/netbox/tables/tables.py | 2 +- netbox/netbox/views/generic/bulk_views.py | 25 +- netbox/templates/dcim/module.html | 1 - netbox/tenancy/api/views.py | 6 +- netbox/tenancy/graphql/types.py | 4 +- .../tenancy/migrations/0001_squashed_0012.py | 3 +- .../tenancy/migrations/0002_squashed_0011.py | 3 +- .../migrations/0025_enable_ltree_extension.py | 17 + netbox/tenancy/migrations/0026_ltree_paths.py | 74 ++++ netbox/tenancy/models/contacts.py | 20 +- netbox/tenancy/models/tenants.py | 7 +- netbox/utilities/mptt.py | 24 - netbox/utilities/query.py | 7 +- netbox/utilities/templatetags/mptt.py | 21 - netbox/utilities/testing/filtersets.py | 11 +- netbox/utilities/tests/test_filters.py | 7 +- netbox/utilities/tests/test_ltree.py | 174 ++++++++ netbox/wireless/api/views.py | 4 +- netbox/wireless/graphql/types.py | 2 +- .../wireless/migrations/0001_squashed_0008.py | 3 +- .../migrations/0020_enable_ltree_extension.py | 17 + .../wireless/migrations/0021_ltree_paths.py | 56 +++ netbox/wireless/models.py | 7 +- requirements.txt | 1 - 52 files changed, 1077 insertions(+), 345 deletions(-) create mode 100644 netbox/dcim/migrations/0234_enable_ltree_extension.py create mode 100644 netbox/dcim/migrations/0235_ltree_paths.py create mode 100644 netbox/netbox/models/ltree.py create mode 100644 netbox/tenancy/migrations/0025_enable_ltree_extension.py create mode 100644 netbox/tenancy/migrations/0026_ltree_paths.py delete mode 100644 netbox/utilities/mptt.py delete mode 100644 netbox/utilities/templatetags/mptt.py create mode 100644 netbox/utilities/tests/test_ltree.py create mode 100644 netbox/wireless/migrations/0020_enable_ltree_extension.py create mode 100644 netbox/wireless/migrations/0021_ltree_paths.py diff --git a/base_requirements.txt b/base_requirements.txt index 673006aa466..73cbbaf1dc5 100644 --- a/base_requirements.txt +++ b/base_requirements.txt @@ -26,9 +26,6 @@ django-graphiql-debug-toolbar # https://django-htmx.readthedocs.io/en/latest/changelog.html django-htmx -# Modified Preorder Tree Traversal (recursive nesting of objects) -django-mptt - # Context managers for PostgreSQL advisory locks # https://github.com/Xof/django-pglocks/blob/master/CHANGES.txt django-pglocks diff --git a/netbox/core/models/change_logging.py b/netbox/core/models/change_logging.py index 4409994da75..c072a468af2 100644 --- a/netbox/core/models/change_logging.py +++ b/netbox/core/models/change_logging.py @@ -6,7 +6,6 @@ from django.db import models from django.urls import reverse from django.utils.translation import gettext_lazy as _ -from mptt.models import MPTTModel from core.choices import ObjectChangeActionChoices from core.querysets import ObjectChangeQuerySet @@ -164,9 +163,10 @@ def diff_exclude_fields(self): if issubclass(model, ChangeLoggingMixin): attrs.update({'created', 'last_updated'}) - # Exclude MPTT-internal fields - if issubclass(model, MPTTModel): - attrs.update({'level', 'lft', 'rght', 'tree_id'}) + # Exclude trigger-maintained ltree path + from netbox.models.ltree import LtreeModel + if issubclass(model, LtreeModel): + attrs.update({'path'}) return attrs diff --git a/netbox/dcim/api/views.py b/netbox/dcim/api/views.py index 3efa63acb07..cf29238d9d4 100644 --- a/netbox/dcim/api/views.py +++ b/netbox/dcim/api/views.py @@ -16,7 +16,7 @@ from netbox.api.authentication import IsAuthenticatedOrLoginNotRequired from netbox.api.metadata import ContentTypeMetadata from netbox.api.pagination import StripCountAnnotationsPaginator -from netbox.api.viewsets import MPTTLockedMixin, NetBoxModelViewSet, NetBoxReadOnlyModelViewSet +from netbox.api.viewsets import NetBoxModelViewSet, NetBoxReadOnlyModelViewSet from netbox.api.viewsets.mixins import SequentialBulkCreatesMixin from utilities.api import get_serializer_for_model from utilities.query import count_related @@ -95,7 +95,7 @@ def paths(self, request, pk): # Regions # -class RegionViewSet(MPTTLockedMixin, NetBoxModelViewSet): +class RegionViewSet(NetBoxModelViewSet): queryset = Region.objects.add_related_count( Region.objects.all(), Site, @@ -111,7 +111,7 @@ class RegionViewSet(MPTTLockedMixin, NetBoxModelViewSet): # Site groups # -class SiteGroupViewSet(MPTTLockedMixin, NetBoxModelViewSet): +class SiteGroupViewSet(NetBoxModelViewSet): queryset = SiteGroup.objects.add_related_count( SiteGroup.objects.all(), Site, @@ -137,7 +137,7 @@ class SiteViewSet(NetBoxModelViewSet): # Locations # -class LocationViewSet(MPTTLockedMixin, NetBoxModelViewSet): +class LocationViewSet(NetBoxModelViewSet): queryset = Location.objects.add_related_count( Location.objects.add_related_count( Location.objects.all(), @@ -356,7 +356,7 @@ class DeviceBayTemplateViewSet(NetBoxModelViewSet): filterset_class = filtersets.DeviceBayTemplateFilterSet -class InventoryItemTemplateViewSet(MPTTLockedMixin, NetBoxModelViewSet): +class InventoryItemTemplateViewSet(NetBoxModelViewSet): queryset = InventoryItemTemplate.objects.all() serializer_class = serializers.InventoryItemTemplateSerializer filterset_class = filtersets.InventoryItemTemplateFilterSet @@ -388,7 +388,7 @@ class DeviceRoleViewSet(NetBoxModelViewSet): # Platforms # -class PlatformViewSet(MPTTLockedMixin, NetBoxModelViewSet): +class PlatformViewSet(NetBoxModelViewSet): queryset = Platform.objects.add_related_count( Platform.objects.add_related_count( Platform.objects.all(), @@ -543,7 +543,7 @@ class DeviceBayViewSet(NetBoxModelViewSet): filterset_class = filtersets.DeviceBayFilterSet -class InventoryItemViewSet(MPTTLockedMixin, NetBoxModelViewSet): +class InventoryItemViewSet(NetBoxModelViewSet): queryset = InventoryItem.objects.all() serializer_class = serializers.InventoryItemSerializer filterset_class = filtersets.InventoryItemFilterSet diff --git a/netbox/dcim/graphql/types.py b/netbox/dcim/graphql/types.py index 6cbab28f281..ddfa24fa50e 100644 --- a/netbox/dcim/graphql/types.py +++ b/netbox/dcim/graphql/types.py @@ -319,7 +319,7 @@ class DeviceBayTemplateType(ComponentTemplateType): @strawberry_django.type( models.InventoryItemTemplate, - exclude=['component_type', 'component_id', 'parent'], + exclude=['component_type', 'component_id', 'parent', 'path'], filters=InventoryItemTemplateFilter, pagination=True ) @@ -347,7 +347,7 @@ def parent(self) -> Annotated['InventoryItemTemplateType', strawberry.lazy('dcim @strawberry_django.type( models.DeviceRole, - fields='__all__', + exclude=['path'], filters=DeviceRoleFilter, pagination=True ) @@ -484,7 +484,7 @@ class InterfaceTemplateType(ModularComponentTemplateType): @strawberry_django.type( models.InventoryItem, - exclude=['component_type', 'component_id', 'parent'], + exclude=['component_type', 'component_id', 'parent', 'path'], filters=InventoryItemFilter, pagination=True ) @@ -526,7 +526,7 @@ class InventoryItemRoleType(OrganizationalObjectType): @strawberry_django.type( models.Location, # fields='__all__', - exclude=['parent'], # bug - temp + exclude=['parent', 'path'], # bug - temp filters=LocationFilter, pagination=True ) @@ -590,7 +590,7 @@ class ModuleType(PrimaryObjectType): @strawberry_django.type( models.ModuleBay, # fields='__all__', - exclude=['parent'], + exclude=['parent', 'path'], filters=ModuleBayFilter, pagination=True ) @@ -647,7 +647,7 @@ class ModuleTypeType(PrimaryObjectType): @strawberry_django.type( models.Platform, - fields='__all__', + exclude=['path'], filters=PlatformFilter, pagination=True ) @@ -855,7 +855,7 @@ class RearPortTemplateType(ModularComponentTemplateType): @strawberry_django.type( models.Region, - exclude=['parent'], + exclude=['parent', 'path'], filters=RegionFilter, pagination=True ) @@ -916,7 +916,7 @@ def circuit_terminations(self) -> list[ @strawberry_django.type( models.SiteGroup, - exclude=['parent'], # bug - temp + exclude=['parent', 'path'], # bug - temp filters=SiteGroupFilter, pagination=True ) diff --git a/netbox/dcim/migrations/0002_squashed.py b/netbox/dcim/migrations/0002_squashed.py index 9a2fbc78640..f876f8bddd7 100644 --- a/netbox/dcim/migrations/0002_squashed.py +++ b/netbox/dcim/migrations/0002_squashed.py @@ -1,5 +1,4 @@ import django.db.models.deletion -import mptt.fields import taggit.managers from django.conf import settings from django.db import migrations, models @@ -27,7 +26,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='sitegroup', name='parent', - field=mptt.fields.TreeForeignKey( + field=django.db.models.ForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, @@ -76,7 +75,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='region', name='parent', - field=mptt.fields.TreeForeignKey( + field=django.db.models.ForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, @@ -381,7 +380,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='location', name='parent', - field=mptt.fields.TreeForeignKey( + field=django.db.models.ForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, @@ -417,7 +416,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='inventoryitem', name='parent', - field=mptt.fields.TreeForeignKey( + field=django.db.models.ForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, diff --git a/netbox/dcim/migrations/0131_squashed_0159.py b/netbox/dcim/migrations/0131_squashed_0159.py index eb77878875d..11a2685de76 100644 --- a/netbox/dcim/migrations/0131_squashed_0159.py +++ b/netbox/dcim/migrations/0131_squashed_0159.py @@ -1,6 +1,5 @@ import django.core.validators import django.db.models.deletion -import mptt.fields import taggit.managers from django.db import migrations, models @@ -1248,7 +1247,7 @@ class Migration(migrations.Migration): ), ( 'parent', - mptt.fields.TreeForeignKey( + django.db.models.ForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, diff --git a/netbox/dcim/migrations/0190_nested_modules.py b/netbox/dcim/migrations/0190_nested_modules.py index 239e0863941..52ec1d616f7 100644 --- a/netbox/dcim/migrations/0190_nested_modules.py +++ b/netbox/dcim/migrations/0190_nested_modules.py @@ -1,5 +1,4 @@ import django.db.models.deletion -import mptt.fields from django.db import migrations, models @@ -44,7 +43,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='modulebay', name='parent', - field=mptt.fields.TreeForeignKey( + field=django.db.models.ForeignKey( blank=True, editable=False, null=True, diff --git a/netbox/dcim/migrations/0191_module_bay_rebuild.py b/netbox/dcim/migrations/0191_module_bay_rebuild.py index f7d1147484b..9b794e64534 100644 --- a/netbox/dcim/migrations/0191_module_bay_rebuild.py +++ b/netbox/dcim/migrations/0191_module_bay_rebuild.py @@ -1,22 +1,13 @@ -import mptt -import mptt.managers from django.db import migrations -def rebuild_mptt(apps, schema_editor): - manager = mptt.managers.TreeManager() - ModuleBay = apps.get_model('dcim', 'ModuleBay') - manager.model = ModuleBay - mptt.register(ModuleBay) - manager.contribute_to_class(ModuleBay, 'objects') - manager.rebuild() - - class Migration(migrations.Migration): dependencies = [ ('dcim', '0190_nested_modules'), ] + # Historical MPTT rebuild: now a no-op. Tree state will be populated from + # parent FKs into an ltree path column by a later migration. operations = [ - migrations.RunPython(code=rebuild_mptt, reverse_code=migrations.RunPython.noop), + migrations.RunPython(migrations.RunPython.noop, migrations.RunPython.noop), ] diff --git a/netbox/dcim/migrations/0203_device_role_nested.py b/netbox/dcim/migrations/0203_device_role_nested.py index c9dd791b3a0..fe0b09d21b2 100644 --- a/netbox/dcim/migrations/0203_device_role_nested.py +++ b/netbox/dcim/migrations/0203_device_role_nested.py @@ -1,7 +1,6 @@ # Generated by Django 5.1.7 on 2025-03-25 18:06 import django.db.models.manager -import mptt.fields from django.db import migrations, models @@ -39,7 +38,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='devicerole', name='parent', - field=mptt.fields.TreeForeignKey( + field=django.db.models.ForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, diff --git a/netbox/dcim/migrations/0204_device_role_rebuild.py b/netbox/dcim/migrations/0204_device_role_rebuild.py index 045b3b2ba54..1d200890a59 100644 --- a/netbox/dcim/migrations/0204_device_role_rebuild.py +++ b/netbox/dcim/migrations/0204_device_role_rebuild.py @@ -1,22 +1,14 @@ -import mptt -import mptt.managers from django.db import migrations -def rebuild_mptt(apps, schema_editor): - manager = mptt.managers.TreeManager() - DeviceRole = apps.get_model('dcim', 'DeviceRole') - manager.model = DeviceRole - mptt.register(DeviceRole) - manager.contribute_to_class(DeviceRole, 'objects') - manager.rebuild() - - class Migration(migrations.Migration): dependencies = [ ('dcim', '0203_device_role_nested'), ] + # Historical MPTT rebuild: now a no-op. The legacy lft/rght/tree_id/level + # columns are removed in a later migration and ltree paths are populated + # from parent FKs after that. operations = [ - migrations.RunPython(code=rebuild_mptt, reverse_code=migrations.RunPython.noop), + migrations.RunPython(migrations.RunPython.noop, migrations.RunPython.noop), ] diff --git a/netbox/dcim/migrations/0213_platform_parent.py b/netbox/dcim/migrations/0213_platform_parent.py index 1a1e0f228dc..ffb6f1bf271 100644 --- a/netbox/dcim/migrations/0213_platform_parent.py +++ b/netbox/dcim/migrations/0213_platform_parent.py @@ -1,5 +1,4 @@ import django.db.models.deletion -import mptt.fields from django.db import migrations, models @@ -14,7 +13,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='platform', name='parent', - field=mptt.fields.TreeForeignKey( + field=django.db.models.ForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, diff --git a/netbox/dcim/migrations/0214_platform_rebuild.py b/netbox/dcim/migrations/0214_platform_rebuild.py index 307d34e8313..eb3845109d9 100644 --- a/netbox/dcim/migrations/0214_platform_rebuild.py +++ b/netbox/dcim/migrations/0214_platform_rebuild.py @@ -1,29 +1,14 @@ -import mptt -import mptt.managers from django.db import migrations -def rebuild_mptt(apps, schema_editor): - """ - Construct the MPTT hierarchy. - """ - Platform = apps.get_model('dcim', 'Platform') - manager = mptt.managers.TreeManager() - manager.model = Platform - mptt.register(Platform) - manager.contribute_to_class(Platform, 'objects') - manager.rebuild() - - class Migration(migrations.Migration): dependencies = [ ('dcim', '0213_platform_parent'), ] + # Historical MPTT rebuild: now a no-op. Tree state will be populated from + # parent FKs into an ltree path column by a later migration. operations = [ - migrations.RunPython( - code=rebuild_mptt, - reverse_code=migrations.RunPython.noop - ), + migrations.RunPython(migrations.RunPython.noop, migrations.RunPython.noop), ] diff --git a/netbox/dcim/migrations/0226_modulebay_rebuild_tree.py b/netbox/dcim/migrations/0226_modulebay_rebuild_tree.py index 27f33b1dad4..ac0858cd8fd 100644 --- a/netbox/dcim/migrations/0226_modulebay_rebuild_tree.py +++ b/netbox/dcim/migrations/0226_modulebay_rebuild_tree.py @@ -1,32 +1,13 @@ -import mptt.managers -import mptt.models from django.db import migrations -def rebuild_mptt(apps, schema_editor): - """ - Rebuild the MPTT tree for ModuleBay to apply new ordering. - """ - ModuleBay = apps.get_model('dcim', 'ModuleBay') - - # Set MPTTMeta with the correct order_insertion_by - class MPTTMeta: - order_insertion_by = ('name',) - - ModuleBay.MPTTMeta = MPTTMeta - ModuleBay._mptt_meta = mptt.models.MPTTOptions(MPTTMeta) - - manager = mptt.managers.TreeManager() - manager.model = ModuleBay - manager.contribute_to_class(ModuleBay, 'objects') - manager.rebuild() - - class Migration(migrations.Migration): dependencies = [ ('dcim', '0226_add_mptt_tree_indexes'), ] + # Historical MPTT rebuild: now a no-op. The MPTT tree columns are removed + # by a later migration and ltree paths are populated from parent FKs. operations = [ - migrations.RunPython(code=rebuild_mptt, reverse_code=migrations.RunPython.noop), + migrations.RunPython(migrations.RunPython.noop, migrations.RunPython.noop), ] diff --git a/netbox/dcim/migrations/0234_enable_ltree_extension.py b/netbox/dcim/migrations/0234_enable_ltree_extension.py new file mode 100644 index 00000000000..6b29e5318cc --- /dev/null +++ b/netbox/dcim/migrations/0234_enable_ltree_extension.py @@ -0,0 +1,17 @@ +from django.contrib.postgres.operations import CreateExtension +from django.db import migrations + + +class Migration(migrations.Migration): + """ + Enable the PostgreSQL ltree extension. Required by the next migration which + adds the `path` column on hierarchical models. + """ + + dependencies = [ + ('dcim', '0233_device_render_config_permission'), + ] + + operations = [ + CreateExtension('ltree'), + ] diff --git a/netbox/dcim/migrations/0235_ltree_paths.py b/netbox/dcim/migrations/0235_ltree_paths.py new file mode 100644 index 00000000000..5f5c279a424 --- /dev/null +++ b/netbox/dcim/migrations/0235_ltree_paths.py @@ -0,0 +1,137 @@ +""" +Replace django-mptt with PostgreSQL ltree for dcim's hierarchical models. + +For each of (Region, SiteGroup, Location, DeviceRole, Platform, ModuleBay, +InventoryItem, InventoryItemTemplate) this migration: + +1. Adds a nullable `path` LTreeField. +2. Installs per-table BEFORE-INSERT/UPDATE-OF-parent_id and AFTER-UPDATE-OF-path + triggers so concurrent writes during the long-running data step get correct + paths. +3. Populates paths for existing rows via a single recursive CTE per table. +4. Tightens `path` to NOT NULL. +5. Drops the legacy MPTT columns (lft, rght, tree_id, level). +6. Adds a GiST index on the `path` column for efficient `@>` / `<@` lookups. +""" +from django.contrib.postgres.indexes import GistIndex +from django.db import migrations + +import netbox.models.ltree +from netbox.models.ltree import InstallLtreeTriggers + +MODELS = ( + 'region', 'sitegroup', 'location', 'devicerole', 'platform', + 'inventoryitem', 'inventoryitemtemplate', 'modulebay', +) + +TABLES = ( + 'dcim_region', + 'dcim_sitegroup', + 'dcim_location', + 'dcim_devicerole', + 'dcim_platform', + 'dcim_inventoryitem', + 'dcim_inventoryitemtemplate', + 'dcim_modulebay', +) + +LEGACY_FIELDS = ('lft', 'rght', 'tree_id', 'level') + + +def _populate_paths_sql(): + blocks = [] + for table in TABLES: + blocks.append(f""" +WITH RECURSIVE t(id, parent_id, path) AS ( + SELECT id, parent_id, id::text::ltree FROM "{table}" WHERE parent_id IS NULL + UNION ALL + SELECT r.id, r.parent_id, t.path || r.id::text::ltree + FROM "{table}" r JOIN t ON r.parent_id = t.id +) +UPDATE "{table}" SET path = t.path FROM t WHERE "{table}".id = t.id; +""") + return '\n'.join(blocks) + + +class Migration(migrations.Migration): + + dependencies = [ + ('dcim', '0234_enable_ltree_extension'), + ] + + operations = [ + # 1. Add nullable path column + *[ + migrations.AddField( + model_name=m, + name='path', + field=netbox.models.ltree.LtreeField(blank=True, editable=False, null=True), + ) + for m in MODELS + ], + + # 2. Install path-maintenance triggers + *[InstallLtreeTriggers(t) for t in TABLES], + + # 3. Populate existing rows + migrations.RunSQL(_populate_paths_sql(), reverse_sql=migrations.RunSQL.noop), + + # 4. Tighten to NOT NULL with empty-string default + *[ + migrations.AlterField( + model_name=m, + name='path', + field=netbox.models.ltree.LtreeField(blank=True, default='', editable=False), + ) + for m in MODELS + ], + + # 5. Drop legacy (tree_id, lft) indexes added in 0226_add_mptt_tree_indexes, + # then drop the legacy MPTT columns. + migrations.RemoveIndex(model_name='devicerole', name='dcim_devicerole_tree_id_lfbf11'), + migrations.RemoveIndex(model_name='inventoryitem', name='dcim_inventoryitem_tree_id975c'), + migrations.RemoveIndex(model_name='inventoryitemtemplate', name='dcim_inventoryitemtemplatedee0'), + migrations.RemoveIndex(model_name='location', name='dcim_location_tree_id_lft_idx'), + migrations.RemoveIndex(model_name='modulebay', name='dcim_modulebay_tree_id_lft_idx'), + migrations.RemoveIndex(model_name='platform', name='dcim_platform_tree_id_lft_idx'), + migrations.RemoveIndex(model_name='region', name='dcim_region_tree_id_lft_idx'), + migrations.RemoveIndex(model_name='sitegroup', name='dcim_sitegroup_tree_id_lft_idx'), + *[ + migrations.RemoveField(model_name=m, name=f) + for m in MODELS for f in LEGACY_FIELDS + ], + + # 6. Add GiST indexes on path + migrations.AddIndex( + model_name='region', + index=GistIndex(fields=['path'], name='dcim_region_path_gist'), + ), + migrations.AddIndex( + model_name='sitegroup', + index=GistIndex(fields=['path'], name='dcim_sitegroup_path_gist'), + ), + migrations.AddIndex( + model_name='location', + index=GistIndex(fields=['path'], name='dcim_location_path_gist'), + ), + migrations.AddIndex( + model_name='devicerole', + index=GistIndex(fields=['path'], name='dcim_devicerole_path_gist'), + ), + migrations.AddIndex( + model_name='platform', + index=GistIndex(fields=['path'], name='dcim_platform_path_gist'), + ), + migrations.AddIndex( + model_name='inventoryitem', + index=GistIndex(fields=['path'], name='dcim_inventoryitem_path_gist'), + ), + migrations.AddIndex( + model_name='inventoryitemtemplate', + index=GistIndex(fields=['path'], name='dcim_inv_item_tmpl_path_gist'), + ), + migrations.AddIndex( + model_name='modulebay', + index=GistIndex(fields=['path'], name='dcim_modulebay_path_gist'), + ), + ] diff --git a/netbox/dcim/models/device_component_templates.py b/netbox/dcim/models/device_component_templates.py index dec8758a9c7..47dd0d4e4a8 100644 --- a/netbox/dcim/models/device_component_templates.py +++ b/netbox/dcim/models/device_component_templates.py @@ -1,9 +1,9 @@ from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.postgres.indexes import GistIndex from django.core.exceptions import ValidationError from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models from django.utils.translation import gettext_lazy as _ -from mptt.models import MPTTModel, TreeForeignKey from dcim.choices import * from dcim.constants import * @@ -11,8 +11,8 @@ from dcim.models.mixins import InterfaceValidationMixin from dcim.utils import get_module_bay_positions, resolve_module_placeholder from netbox.models import ChangeLoggedModel +from netbox.models.ltree import LtreeManager, LtreeModel from utilities.fields import ColorField, NaturalOrderingField -from utilities.mptt import TreeManager from utilities.ordering import naturalize_interface from utilities.tracking import TrackingModelMixin from wireless.choices import WirelessRoleChoices @@ -812,11 +812,11 @@ def to_yaml(self): } -class InventoryItemTemplate(MPTTModel, ComponentTemplateModel): +class InventoryItemTemplate(LtreeModel, ComponentTemplateModel): """ A template for an InventoryItem to be created for a new parent Device. """ - parent = TreeForeignKey( + parent = models.ForeignKey( to='self', on_delete=models.CASCADE, related_name='child_items', @@ -860,13 +860,15 @@ class InventoryItemTemplate(MPTTModel, ComponentTemplateModel): help_text=_('Manufacturer-assigned part identifier') ) - objects = TreeManager() component_model = InventoryItem + objects = LtreeManager() + class Meta: ordering = ('device_type__id', 'parent__id', 'name') indexes = ( models.Index(fields=('component_type', 'component_id')), + GistIndex(fields=['path'], name='dcim_inv_item_tmpl_path_gist'), ) constraints = ( models.UniqueConstraint( diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index e3691fd6c80..8186f756655 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -2,11 +2,11 @@ from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation from django.contrib.postgres.fields import ArrayField +from django.contrib.postgres.indexes import GistIndex from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models from django.utils.translation import gettext_lazy as _ -from mptt.models import MPTTModel, TreeForeignKey from dcim.choices import * from dcim.constants import * @@ -15,9 +15,9 @@ from dcim.models.mixins import InterfaceValidationMixin from netbox.choices import ColorChoices from netbox.models import NetBoxModel, OrganizationalModel +from netbox.models.ltree import LtreeManager, LtreeModel from netbox.models.mixins import OwnerMixin from utilities.fields import ColorField, NaturalOrderingField -from utilities.mptt import TreeManager from utilities.ordering import naturalize_interface from utilities.query_functions import CollateAsChar from utilities.tracking import TrackingModelMixin @@ -1313,11 +1313,11 @@ def clean(self): # Bays # -class ModuleBay(ModularComponentModel, TrackingModelMixin, MPTTModel): +class ModuleBay(ModularComponentModel, TrackingModelMixin, LtreeModel): """ An empty space within a Device which can house a child device """ - parent = TreeForeignKey( + parent = models.ForeignKey( to='self', on_delete=models.CASCADE, related_name='children', @@ -1337,14 +1337,14 @@ class ModuleBay(ModularComponentModel, TrackingModelMixin, MPTTModel): default=True, ) - objects = TreeManager() - clone_fields = ('device', 'enabled') + objects = LtreeManager() + class Meta(ModularComponentModel.Meta): - # Empty tuple triggers Django migration detection for MPTT indexes - # (see #21016, django-mptt/django-mptt#682) - indexes = () + indexes = ( + GistIndex(fields=['path'], name='dcim_modulebay_path_gist'), + ) constraints = ( models.UniqueConstraint( fields=('device', 'module', 'name'), @@ -1354,9 +1354,6 @@ class Meta(ModularComponentModel.Meta): verbose_name = _('module bay') verbose_name_plural = _('module bays') - class MPTTMeta: - order_insertion_by = ('name',) - def clean(self): super().clean() @@ -1469,12 +1466,12 @@ class Meta: verbose_name_plural = _('inventory item roles') -class InventoryItem(MPTTModel, ComponentModel, TrackingModelMixin): +class InventoryItem(LtreeModel, ComponentModel, TrackingModelMixin): """ An InventoryItem represents a serialized piece of hardware within a Device, such as a line card or power supply. InventoryItems are used only for inventory purposes. """ - parent = TreeForeignKey( + parent = models.ForeignKey( to='self', on_delete=models.CASCADE, related_name='child_items', @@ -1542,14 +1539,15 @@ class InventoryItem(MPTTModel, ComponentModel, TrackingModelMixin): help_text=_('This item was automatically discovered') ) - objects = TreeManager() - clone_fields = ('device', 'parent', 'role', 'manufacturer', 'status', 'part_id') + objects = LtreeManager() + class Meta: ordering = ('device__id', 'parent__id', 'name') indexes = ( models.Index(fields=('component_type', 'component_id')), + GistIndex(fields=['path'], name='dcim_inventoryitem_path_gist'), ) constraints = ( models.UniqueConstraint( diff --git a/netbox/dcim/models/devices.py b/netbox/dcim/models/devices.py index 88d094b978b..936e7691594 100644 --- a/netbox/dcim/models/devices.py +++ b/netbox/dcim/models/devices.py @@ -4,6 +4,7 @@ import yaml from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation from django.contrib.contenttypes.models import ContentType +from django.contrib.postgres.indexes import GistIndex from django.core.exceptions import ValidationError from django.core.files.storage import default_storage from django.core.validators import MaxValueValidator, MinValueValidator @@ -412,9 +413,9 @@ class DeviceRole(NestedGroupModel): class Meta: ordering = ('name',) - # Empty tuple triggers Django migration detection for MPTT indexes - # (see #21016, django-mptt/django-mptt#682) - indexes = () + indexes = ( + GistIndex(fields=['path'], name='dcim_devicerole_path_gist'), + ) constraints = ( models.UniqueConstraint( fields=('parent', 'name'), @@ -466,9 +467,9 @@ class Platform(NestedGroupModel): class Meta: ordering = ('name',) - # Empty tuple triggers Django migration detection for MPTT indexes - # (see #21016, django-mptt/django-mptt#682) - indexes = () + indexes = ( + GistIndex(fields=['path'], name='dcim_platform_path_gist'), + ) verbose_name = _('platform') verbose_name_plural = _('platforms') constraints = ( diff --git a/netbox/dcim/models/modules.py b/netbox/dcim/models/modules.py index 1d678deb099..64c1c618999 100644 --- a/netbox/dcim/models/modules.py +++ b/netbox/dcim/models/modules.py @@ -5,7 +5,6 @@ from django.db.models.signals import post_save from django.utils.translation import gettext_lazy as _ from jsonschema.exceptions import ValidationError as JSONValidationError -from mptt.models import MPTTModel from dcim.choices import * from dcim.utils import create_port_mappings, update_interface_bridges @@ -352,29 +351,22 @@ def save(self, *args, **kwargs): component._location = self.device.location component._rack = self.device.rack - # we handle create and update separately - this is for create - if not issubclass(component_model, MPTTModel): - component_model.objects.bulk_create(create_instances) - # Emit the post_save signal for each newly created object - for component in create_instances: - post_save.send( - sender=component_model, - instance=component, - created=True, - raw=False, - using='default', - update_fields=None - ) - else: - # MPTT models must be saved individually to maintain tree structure - for instance in create_instances: - instance.save() + # Bulk-create new instances. For ltree-backed models (ModuleBay, + # InventoryItem), the BEFORE INSERT trigger populates `path` per row. + component_model.objects.bulk_create(create_instances) + for component in create_instances: + post_save.send( + sender=component_model, + instance=component, + created=True, + raw=False, + using='default', + update_fields=None + ) update_fields = ['module'] - # we handle create and update separately - this is for update component_model.objects.bulk_update(update_instances, update_fields) - # Emit the post_save signal for each updated object for component in update_instances: post_save.send( sender=component_model, @@ -385,10 +377,6 @@ def save(self, *args, **kwargs): update_fields=update_fields ) - # Rebuild MPTT tree if needed (bulk_update bypasses model save) - if issubclass(component_model, MPTTModel) and update_instances: - component_model.objects.rebuild() - # Replicate any front/rear port mappings from the ModuleType create_port_mappings(self.device, self.module_type, self) diff --git a/netbox/dcim/models/sites.py b/netbox/dcim/models/sites.py index 00bc9a240a3..416306c2d05 100644 --- a/netbox/dcim/models/sites.py +++ b/netbox/dcim/models/sites.py @@ -3,6 +3,7 @@ from django.contrib.contenttypes.fields import GenericRelation from django.core.exceptions import ValidationError from django.core.validators import MaxValueValidator, MinValueValidator +from django.contrib.postgres.indexes import GistIndex from django.db import models from django.utils.translation import gettext_lazy as _ from timezone_field import TimeZoneField @@ -44,9 +45,9 @@ class Region(ContactsMixin, NestedGroupModel): ) class Meta: - # Empty tuple triggers Django migration detection for MPTT indexes - # (see #21016, django-mptt/django-mptt#682) - indexes = () + indexes = ( + GistIndex(fields=['path'], name='dcim_region_path_gist'), + ) constraints = ( models.UniqueConstraint( fields=('parent', 'name'), @@ -103,9 +104,9 @@ class SiteGroup(ContactsMixin, NestedGroupModel): ) class Meta: - # Empty tuple triggers Django migration detection for MPTT indexes - # (see #21016, django-mptt/django-mptt#682) - indexes = () + indexes = ( + GistIndex(fields=['path'], name='dcim_sitegroup_path_gist'), + ) constraints = ( models.UniqueConstraint( fields=('parent', 'name'), @@ -324,9 +325,9 @@ class Location(ContactsMixin, ImageAttachmentsMixin, NestedGroupModel): class Meta: ordering = ['site', 'name'] - # Empty tuple triggers Django migration detection for MPTT indexes - # (see #21016, django-mptt/django-mptt#682) - indexes = () + indexes = ( + GistIndex(fields=['path'], name='dcim_location_path_gist'), + ) constraints = ( models.UniqueConstraint( fields=('site', 'parent', 'name'), diff --git a/netbox/extras/querysets.py b/netbox/extras/querysets.py index 7602aee530a..6a57998eb58 100644 --- a/netbox/extras/querysets.py +++ b/netbox/extras/querysets.py @@ -130,10 +130,7 @@ def _get_config_context_filters(self): if self.model._meta.model_name == 'device': base_query.add( (Q( - locations__tree_id=OuterRef('location__tree_id'), - locations__level__lte=OuterRef('location__level'), - locations__lft__lte=OuterRef('location__lft'), - locations__rght__gte=OuterRef('location__rght'), + locations__path__ancestor=OuterRef('location__path'), ) | Q(locations=None)), Q.AND ) @@ -142,40 +139,29 @@ def _get_config_context_filters(self): base_query.add(Q(locations=None), Q.AND) base_query.add(Q(device_types=None), Q.AND) - # MPTT-based filters + # Ltree-based filters: the ConfigContext-side tree node must be an ancestor + # (or equal to) the device/VM-side tree node, i.e. `cc_node.path @> obj_node.path`. base_query.add( (Q( - regions__tree_id=OuterRef('site__region__tree_id'), - regions__level__lte=OuterRef('site__region__level'), - regions__lft__lte=OuterRef('site__region__lft'), - regions__rght__gte=OuterRef('site__region__rght'), + regions__path__ancestor=OuterRef('site__region__path'), ) | Q(regions=None)), Q.AND ) base_query.add( (Q( - site_groups__tree_id=OuterRef('site__group__tree_id'), - site_groups__level__lte=OuterRef('site__group__level'), - site_groups__lft__lte=OuterRef('site__group__lft'), - site_groups__rght__gte=OuterRef('site__group__rght'), + site_groups__path__ancestor=OuterRef('site__group__path'), ) | Q(site_groups=None)), Q.AND ) base_query.add( (Q( - roles__tree_id=OuterRef('role__tree_id'), - roles__level__lte=OuterRef('role__level'), - roles__lft__lte=OuterRef('role__lft'), - roles__rght__gte=OuterRef('role__rght'), + roles__path__ancestor=OuterRef('role__path'), ) | Q(roles=None)), Q.AND ) base_query.add( (Q( - platforms__tree_id=OuterRef('platform__tree_id'), - platforms__level__lte=OuterRef('platform__level'), - platforms__lft__lte=OuterRef('platform__lft'), - platforms__rght__gte=OuterRef('platform__rght'), + platforms__path__ancestor=OuterRef('platform__path'), ) | Q(platforms=None)), Q.AND ) diff --git a/netbox/netbox/api/viewsets/__init__.py b/netbox/netbox/api/viewsets/__init__.py index 95b2351d3c8..d6ce93e2f4f 100644 --- a/netbox/netbox/api/viewsets/__init__.py +++ b/netbox/netbox/api/viewsets/__init__.py @@ -4,14 +4,12 @@ from django.core.exceptions import ObjectDoesNotExist, PermissionDenied from django.db import router, transaction from django.db.models import ProtectedError, RestrictedError -from django_pglocks import advisory_lock from rest_framework import mixins as drf_mixins from rest_framework import status from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet from netbox.api.serializers.features import ChangeLogMessageSerializer -from netbox.constants import ADVISORY_LOCK_KEYS from utilities.api import get_annotations_for_serializer, get_prefetches_for_serializer from utilities.exceptions import AbortRequest, PreconditionFailed from utilities.query import reapply_model_ordering @@ -337,20 +335,3 @@ def perform_destroy(self, instance): raise PermissionDenied() -class MPTTLockedMixin: - """ - Puts pglock on objects that derive from MPTTModel for parallel API calling. - Note: If adding this to a view, must add the model name to ADVISORY_LOCK_KEYS - """ - - def create(self, request, *args, **kwargs): - with advisory_lock(ADVISORY_LOCK_KEYS[self.queryset.model._meta.model_name]): - return super().create(request, *args, **kwargs) - - def update(self, request, *args, **kwargs): - with advisory_lock(ADVISORY_LOCK_KEYS[self.queryset.model._meta.model_name]): - return super().update(request, *args, **kwargs) - - def destroy(self, request, *args, **kwargs): - with advisory_lock(ADVISORY_LOCK_KEYS[self.queryset.model._meta.model_name]): - return super().destroy(request, *args, **kwargs) diff --git a/netbox/netbox/constants.py b/netbox/netbox/constants.py index b226c606cb0..ab741d2f94c 100644 --- a/netbox/netbox/constants.py +++ b/netbox/netbox/constants.py @@ -29,17 +29,6 @@ 'available-vlans': 100300, 'available-asns': 100400, - # MPTT locks - 'region': 105100, - 'sitegroup': 105200, - 'location': 105300, - 'tenantgroup': 105400, - 'contactgroup': 105500, - 'wirelesslangroup': 105600, - 'inventoryitem': 105700, - 'inventoryitemtemplate': 105800, - 'platform': 105900, - # Jobs 'job-schedules': 110100, } diff --git a/netbox/netbox/graphql/filter_lookups.py b/netbox/netbox/graphql/filter_lookups.py index 583caa6d371..7f5ec253d2b 100644 --- a/netbox/netbox/graphql/filter_lookups.py +++ b/netbox/netbox/graphql/filter_lookups.py @@ -205,25 +205,28 @@ def filter(self, info: Info, queryset: QuerySet, prefix: DirectiveValue[str] = ' def generate_tree_node_q_filter(model_class, filter_value: TreeNodeFilter) -> Q: """ - Generate appropriate Q filter for MPTT tree filtering based on match type + Generate Q filter for ltree-backed hierarchical models based on match type. """ try: node = model_class.objects.get(id=filter_value.id) except model_class.DoesNotExist: return Q(pk__in=[]) + if not getattr(node, 'path', None): + return Q(id=filter_value.id) + if filter_value.match_type == TreeNodeMatch.EXACT: return Q(id=filter_value.id) if filter_value.match_type == TreeNodeMatch.DESCENDANTS: - return Q(tree_id=node.tree_id, lft__gt=node.lft, rght__lt=node.rght) + return Q(path__descendant=node.path) & ~Q(id=node.id) if filter_value.match_type == TreeNodeMatch.SELF_AND_DESCENDANTS: - return Q(tree_id=node.tree_id, lft__gte=node.lft, rght__lte=node.rght) + return Q(path__descendant_or_equal=node.path) if filter_value.match_type == TreeNodeMatch.CHILDREN: - return Q(tree_id=node.tree_id, level=node.level + 1, lft__gt=node.lft, rght__lt=node.rght) + return Q(parent_id=node.id) if filter_value.match_type == TreeNodeMatch.SIBLINGS: - return Q(tree_id=node.tree_id, level=node.level, parent=node.parent) & ~Q(id=node.id) + return Q(parent_id=node.parent_id) & ~Q(id=node.id) if filter_value.match_type == TreeNodeMatch.ANCESTORS: - return Q(tree_id=node.tree_id, lft__lt=node.lft, rght__gt=node.rght) + return Q(path__ancestor=node.path) & ~Q(id=node.id) if filter_value.match_type == TreeNodeMatch.PARENT: return Q(id=node.parent_id) if node.parent_id else Q(pk__in=[]) return Q() diff --git a/netbox/netbox/models/__init__.py b/netbox/netbox/models/__init__.py index a01472bb6e4..ab5befce256 100644 --- a/netbox/netbox/models/__init__.py +++ b/netbox/netbox/models/__init__.py @@ -5,11 +5,10 @@ from django.db import models from django.urls import reverse from django.utils.translation import gettext_lazy as _ -from mptt.models import MPTTModel, TreeForeignKey from netbox.models.features import * +from netbox.models.ltree import LtreeManager, LtreeModel from netbox.models.mixins import OwnerMixin -from utilities.mptt import TreeManager from utilities.querysets import RestrictedQuerySet __all__ = ( @@ -159,16 +158,12 @@ class Meta: abstract = True -class NestedGroupModel(OwnerMixin, NetBoxModel, MPTTModel): +class NestedGroupModel(OwnerMixin, NetBoxModel, LtreeModel): """ Base model for objects which are used to form a hierarchy (regions, locations, etc.). These models nest - recursively using MPTT. Within each parent, each child instance must have a unique name. - - Note: django-mptt injects the (tree_id, lft) index dynamically, but Django's migration autodetector won't - detect it unless concrete subclasses explicitly declare Meta.indexes (even as an empty tuple). See #21016 - and django-mptt/django-mptt#682. + recursively using PostgreSQL ltree. Within each parent, each child instance must have a unique name. """ - parent = TreeForeignKey( + parent = models.ForeignKey( to='self', on_delete=models.CASCADE, related_name='children', @@ -194,13 +189,13 @@ class NestedGroupModel(OwnerMixin, NetBoxModel, MPTTModel): blank=True ) - objects = TreeManager() + # Re-declare so the LtreeManager wins over BaseModel's RestrictedQuerySet + # default manager via MRO resolution. + objects = LtreeManager() class Meta: abstract = True - - class MPTTMeta: - order_insertion_by = ('name',) + ordering = ('path',) def __str__(self): return self.name @@ -208,7 +203,7 @@ def __str__(self): def clean(self): super().clean() - # An MPTT model cannot be its own parent + # A nested group cannot be its own parent or a descendant of itself if not self._state.adding and self.parent and self.parent in self.get_descendants(include_self=True): raise ValidationError({ "parent": "Cannot assign self or child {type} as parent.".format(type=self._meta.verbose_name) diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py new file mode 100644 index 00000000000..6f21234d781 --- /dev/null +++ b/netbox/netbox/models/ltree.py @@ -0,0 +1,417 @@ +""" +Ltree-based hierarchical model support - drop-in replacement for django-mptt. + +LtreeModel provides the same public API as django-mptt's MPTTModel (get_ancestors, +get_descendants, get_children, get_root, get_family, get_siblings, +get_descendant_count, get_level, level, is_root_node, is_leaf_node, is_child_node, +move_to, insert_at) backed by a PostgreSQL ltree column. + +Paths are maintained entirely by PostgreSQL triggers installed via the +InstallLtreeTriggers migration operation. The Python layer never computes or +mutates paths directly; it only reads `path` back from the database after +inserts and parent_id changes via refresh_from_db(fields=['path']). +""" +from django.contrib.contenttypes.models import ContentType +from django.db import migrations, models +from django.db.models import Count, ForeignKey, ManyToManyField, Lookup, OuterRef, Q, Subquery +from django.db.models.expressions import RawSQL + +from utilities.querysets import RestrictedQuerySet + +__all__ = ( + 'InstallLtreeTriggers', + 'LtreeField', + 'LtreeManager', + 'LtreeModel', + 'LtreeQuerySet', +) + + +# +# Field +# + +class LtreeField(models.TextField): + """ + Custom field backed by PostgreSQL's ltree type. Stores hierarchical paths + such as "1.4.27" (each label is the integer PK of an ancestor). + """ + description = "PostgreSQL ltree field" + + def db_type(self, connection): + return 'ltree' + + def get_prep_value(self, value): + if value is None: + return value + return str(value) + + +@LtreeField.register_lookup +class Ancestor(Lookup): + """`path` is an ancestor of (or equal to) the queried path: path @> rhs""" + lookup_name = 'ancestor' + + def as_sql(self, compiler, connection): + lhs, lhs_params = self.process_lhs(compiler, connection) + rhs, rhs_params = self.process_rhs(compiler, connection) + return f'{lhs} @> {rhs}', lhs_params + rhs_params + + +@LtreeField.register_lookup +class Descendant(Lookup): + """`path` is a descendant of (or equal to) the queried path: path <@ rhs""" + lookup_name = 'descendant' + + def as_sql(self, compiler, connection): + lhs, lhs_params = self.process_lhs(compiler, connection) + rhs, rhs_params = self.process_rhs(compiler, connection) + return f'{lhs} <@ {rhs}', lhs_params + rhs_params + + +@LtreeField.register_lookup +class DescendantOrEqual(Lookup): + """Alias of `descendant`; `<@` is inclusive in PostgreSQL ltree.""" + lookup_name = 'descendant_or_equal' + + def as_sql(self, compiler, connection): + lhs, lhs_params = self.process_lhs(compiler, connection) + rhs, rhs_params = self.process_rhs(compiler, connection) + return f'{lhs} <@ {rhs}', lhs_params + rhs_params + + +# +# QuerySet / Manager +# + +class LtreeQuerySet(RestrictedQuerySet): + """QuerySet for ltree-based hierarchies, layered on RestrictedQuerySet.""" + + def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=False): + """ + Annotate `queryset` with the count of `model` instances related via + `rel_field`, mirroring django-mptt's `TreeManager.add_related_count`. + + When `cumulative=True`, counts include rows pointing to any descendant + (using the ltree `<@` operator against the parent's `path`). Handles + ForeignKey, ManyToManyField, and the NetBox GenericForeignKey "scope" + pattern (scope_type / scope_id). + """ + has_direct_fk = False + is_many_to_many = False + try: + field = model._meta.get_field(rel_field) + if isinstance(field, ManyToManyField): + is_many_to_many = True + elif isinstance(field, ForeignKey): + has_direct_fk = True + except Exception: + pass + + has_generic_fk = ( + hasattr(model, 'scope_type') and hasattr(model, 'scope_id') + and not has_direct_fk and not is_many_to_many + ) + + parent_table = queryset.model._meta.db_table + related_table = model._meta.db_table + + if cumulative: + if is_many_to_many: + field = model._meta.get_field(rel_field) + m2m_table = field.remote_field.through._meta.db_table + m2m_parent_col = field.m2m_column_name() + m2m_related_col = field.m2m_reverse_name() + sql = f'''( + SELECT COUNT(DISTINCT "{related_table}"."id") + FROM "{related_table}" + INNER JOIN "{m2m_table}" + ON "{related_table}"."id" = "{m2m_table}"."{m2m_related_col}" + INNER JOIN "{parent_table}" AS subtree + ON "{m2m_table}"."{m2m_parent_col}" = subtree."id" + WHERE subtree."path" <@ "{parent_table}"."path" + )''' + return queryset.annotate(**{ + count_attr: RawSQL(sql, [], output_field=models.IntegerField()) + }) + elif has_generic_fk: + content_type = ContentType.objects.get_for_model(queryset.model) + sql = f'''( + SELECT COUNT(DISTINCT "{related_table}"."id") + FROM "{related_table}" + INNER JOIN "{parent_table}" AS subtree + ON "{related_table}"."scope_id" = subtree."id" + WHERE "{related_table}"."scope_type_id" = %s + AND subtree."path" <@ "{parent_table}"."path" + )''' + return queryset.annotate(**{ + count_attr: RawSQL(sql, [content_type.pk], output_field=models.IntegerField()) + }) + else: + rel_field_col = f'{rel_field}_id' + sql = f'''( + SELECT COUNT(DISTINCT "{related_table}"."id") + FROM "{related_table}" + INNER JOIN "{parent_table}" AS subtree + ON "{related_table}"."{rel_field_col}" = subtree."id" + WHERE subtree."path" <@ "{parent_table}"."path" + )''' + return queryset.annotate(**{ + count_attr: RawSQL(sql, [], output_field=models.IntegerField()) + }) + + # Non-cumulative: direct count. + if is_many_to_many: + return queryset.annotate(**{count_attr: Count(rel_field, distinct=True)}) + if has_generic_fk: + content_type = ContentType.objects.get_for_model(queryset.model) + subquery = model.objects.filter( + scope_type=content_type, scope_id=OuterRef('pk') + ).values('scope_id').annotate(c=Count('id')).values('c') + return queryset.annotate(**{ + count_attr: Subquery(subquery, output_field=models.IntegerField()) + }) + return queryset.annotate(**{count_attr: Count(rel_field, distinct=True)}) + + +class LtreeManager(models.Manager.from_queryset(LtreeQuerySet)): + """Drop-in replacement for django-mptt's TreeManager.""" + + def get_queryset(self): + return super().get_queryset().order_by('path') + + +# +# Abstract model +# + +class LtreeModel(models.Model): + """ + Abstract base for hierarchical models backed by PostgreSQL ltree. + + Subclasses must declare a `parent = models.ForeignKey('self', ...)`. The + `path` column is maintained by per-table triggers installed via + InstallLtreeTriggers; do not write to it from Python. + """ + path = LtreeField(editable=False, null=False, blank=True, default='') + + objects = LtreeManager() + + class Meta: + abstract = True + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._loaded_parent_id = self.parent_id + + @classmethod + def from_db(cls, db, field_names, values): + instance = super().from_db(db, field_names, values) + instance._loaded_parent_id = instance.parent_id + return instance + + def save(self, *args, **kwargs): + """ + Triggers compute `path` server-side. After insert or after a parent + change, refresh just the path column so the in-memory instance stays + consistent with the database. + """ + is_insert = self._state.adding + parent_changed = (not is_insert) and self.parent_id != self._loaded_parent_id + super().save(*args, **kwargs) + if is_insert or parent_changed: + self.path = type(self).objects.values_list('path', flat=True).get(pk=self.pk) + self._loaded_parent_id = self.parent_id + + # -- MPTT-compatible API ------------------------------------------------ + + @property + def level(self): + """Zero-based depth (root = 0). Mirrors django-mptt's `level`.""" + if not self.path: + return 0 + return str(self.path).count('.') + + def get_level(self): + return self.level + + @property + def tree_id(self): + """Integer PK of the root, mirroring django-mptt's `tree_id`.""" + if not self.path: + return None + root_label = str(self.path).split('.', 1)[0] + try: + return int(root_label) + except (TypeError, ValueError): + return root_label + + def is_root_node(self): + return self.parent_id is None + + def is_leaf_node(self): + return not type(self).objects.filter(parent_id=self.pk).exists() + + def is_child_node(self): + return self.parent_id is not None + + def get_root(self): + if self.is_root_node(): + return self + root_pk = int(str(self.path).split('.', 1)[0]) + return type(self)._default_manager.get(pk=root_pk) + + def get_parent(self): + return self.parent + + def get_ancestors(self, ascending=False, include_self=False): + if not self.path: + return type(self)._default_manager.none() + qs = type(self)._default_manager.filter(path__ancestor=self.path) + if not include_self: + qs = qs.exclude(pk=self.pk) + return qs.order_by('-path' if ascending else 'path') + + def get_descendants(self, include_self=False): + if not self.path: + return type(self)._default_manager.none() + qs = type(self)._default_manager.filter(path__descendant=self.path) + if not include_self: + qs = qs.exclude(pk=self.pk) + return qs.order_by('path') + + def get_descendant_count(self): + if not self.path: + return 0 + return type(self)._default_manager.filter( + path__descendant=self.path + ).exclude(pk=self.pk).count() + + def get_children(self): + return type(self)._default_manager.filter(parent_id=self.pk) + + def get_family(self): + """Ancestors + self + descendants, in path order.""" + if not self.path: + return type(self)._default_manager.none() + return type(self)._default_manager.filter( + Q(path__ancestor=self.path) | Q(path__descendant=self.path) + ).distinct().order_by('path') + + def get_siblings(self, include_self=False): + qs = type(self)._default_manager.filter(parent_id=self.parent_id) + if not include_self: + qs = qs.exclude(pk=self.pk) + return qs + + def move_to(self, target, position='last-child'): + """ + Re-parent this node under `target`. Triggers handle path recomputation + for self and all descendants. `position` is accepted for django-mptt + compatibility; first-/last-child both mean "child of target" and + left/right mean "sibling of target". + """ + if position in ('first-child', 'last-child', None): + new_parent = target + elif position in ('left', 'right'): + new_parent = target.parent if target else None + else: + raise ValueError(f"Unsupported move_to position: {position!r}") + self.parent = new_parent + self.save() + + def insert_at(self, target, position='last-child', save=False): + """Set parent (optionally save). Mirrors django-mptt's insert_at.""" + if position in ('first-child', 'last-child', None): + self.parent = target + elif position in ('left', 'right'): + self.parent = target.parent if target else None + else: + raise ValueError(f"Unsupported insert_at position: {position!r}") + if save: + self.save() + + +# +# Migration operation +# + +_COMPUTE_PATH_FN = ''' +CREATE OR REPLACE FUNCTION "{table}_ltree_compute_path_fn"() RETURNS TRIGGER AS $$ +DECLARE parent_path ltree; +BEGIN + IF NEW.parent_id IS NOT NULL THEN + EXECUTE format('SELECT path FROM %%I WHERE id = $1', TG_TABLE_NAME) + INTO parent_path USING NEW.parent_id; + NEW.path := parent_path || NEW.id::text::ltree; + ELSE + NEW.path := NEW.id::text::ltree; + END IF; + RETURN NEW; +END +$$ LANGUAGE plpgsql; +''' + +_CASCADE_PATH_FN = ''' +CREATE OR REPLACE FUNCTION "{table}_ltree_cascade_path_fn"() RETURNS TRIGGER AS $$ +BEGIN + EXECUTE format( + 'UPDATE %%I SET path = $1 || subpath(path, nlevel($2))' + ' WHERE path <@ $2 AND id != $3', + TG_TABLE_NAME + ) USING NEW.path, OLD.path, NEW.id; + RETURN NULL; +END +$$ LANGUAGE plpgsql; +''' + +_BEFORE_TRIGGER = ''' +CREATE TRIGGER "{table}_ltree_compute_path" + BEFORE INSERT OR UPDATE OF parent_id ON "{table}" + FOR EACH ROW EXECUTE FUNCTION "{table}_ltree_compute_path_fn"(); +''' + +_AFTER_TRIGGER = ''' +CREATE TRIGGER "{table}_ltree_cascade_path" + AFTER UPDATE OF parent_id, path ON "{table}" + FOR EACH ROW WHEN (OLD.path IS DISTINCT FROM NEW.path) + EXECUTE FUNCTION "{table}_ltree_cascade_path_fn"(); +''' + + +class InstallLtreeTriggers(migrations.operations.base.Operation): + """ + Install per-table ltree path-maintenance triggers. + + Two row-level triggers are installed on each target table: + + BEFORE INSERT OR UPDATE OF parent_id -> compute NEW.path + AFTER UPDATE OF path WHEN distinct -> cascade path change to descendants + + The trigger function bodies use TG_TABLE_NAME so the SQL is table-agnostic, + but each table gets its own pair of CREATE FUNCTION statements to keep + pg_proc entries identifiable and to avoid surprising cross-table coupling. + """ + reversible = True + + def __init__(self, table_name): + self.table_name = table_name + + def state_forwards(self, app_label, state): + pass + + def database_forwards(self, app_label, schema_editor, from_state, to_state): + schema_editor.execute(_COMPUTE_PATH_FN.format(table=self.table_name)) + schema_editor.execute(_CASCADE_PATH_FN.format(table=self.table_name)) + schema_editor.execute(_BEFORE_TRIGGER.format(table=self.table_name)) + schema_editor.execute(_AFTER_TRIGGER.format(table=self.table_name)) + + def database_backwards(self, app_label, schema_editor, from_state, to_state): + t = self.table_name + schema_editor.execute(f'DROP TRIGGER IF EXISTS "{t}_ltree_cascade_path" ON "{t}";') + schema_editor.execute(f'DROP TRIGGER IF EXISTS "{t}_ltree_compute_path" ON "{t}";') + schema_editor.execute(f'DROP FUNCTION IF EXISTS "{t}_ltree_cascade_path_fn"();') + schema_editor.execute(f'DROP FUNCTION IF EXISTS "{t}_ltree_compute_path_fn"();') + + def describe(self): + return f"Install ltree path triggers on {self.table_name}" diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 75e34a63579..717f970b049 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -454,7 +454,6 @@ def _setting(name, default=None): 'django_tables2', 'django_prometheus', 'strawberry_django', - 'mptt', 'rest_framework', 'social_django', 'sorl.thumbnail', diff --git a/netbox/netbox/tables/columns.py b/netbox/netbox/tables/columns.py index 4b89bb06ab2..850bb3f61dd 100644 --- a/netbox/netbox/tables/columns.py +++ b/netbox/netbox/tables/columns.py @@ -44,6 +44,7 @@ 'TagColumn', 'TemplateColumn', 'ToggleColumn', + 'TreeColumn', 'UtilizationColumn', ) @@ -599,9 +600,9 @@ def value(self, record, table, **kwargs): return None -class MPTTColumn(tables.TemplateColumn): +class TreeColumn(tables.TemplateColumn): """ - Display a nested hierarchy for MPTT-enabled models. + Display a nested hierarchy for tree-enabled models (Region, Location, etc.). """ template_code = """ {% load helpers %} @@ -623,6 +624,10 @@ def value(self, value): return value +# Deprecated alias for plugin compatibility; use TreeColumn going forward. +MPTTColumn = TreeColumn + + class UtilizationColumn(tables.TemplateColumn): """ Display a colored utilization bar graph. diff --git a/netbox/netbox/tables/tables.py b/netbox/netbox/tables/tables.py index 7c7275f21ab..fcfa6d7e8c4 100644 --- a/netbox/netbox/tables/tables.py +++ b/netbox/netbox/tables/tables.py @@ -340,7 +340,7 @@ class NestedGroupModelTable(NetBoxTable): linkify=True, verbose_name=_('Owner'), ) - name = columns.MPTTColumn( + name = columns.TreeColumn( verbose_name=_('Name'), linkify=True ) diff --git a/netbox/netbox/views/generic/bulk_views.py b/netbox/netbox/views/generic/bulk_views.py index 8f0a98b5054..8fa6fde1b90 100644 --- a/netbox/netbox/views/generic/bulk_views.py +++ b/netbox/netbox/views/generic/bulk_views.py @@ -15,7 +15,6 @@ from django.shortcuts import get_object_or_404, redirect, render from django.utils.safestring import mark_safe from django.utils.translation import gettext as _ -from mptt.models import MPTTModel from core.exceptions import JobFailed from core.models import ObjectType @@ -557,12 +556,7 @@ def create_and_update_objects(self, form, request): for obj in self.queryset.model.objects.filter(id__in=prefetch_ids) } if prefetch_ids else {} - # For MPTT models, delay tree updates until all saves are complete - if issubclass(self.queryset.model, MPTTModel): - with self.queryset.model.objects.delay_mptt_updates(): - saved_objects = self._process_import_records(form, request, records, prefetched_objects) - else: - saved_objects = self._process_import_records(form, request, records, prefetched_objects) + saved_objects = self._process_import_records(form, request, records, prefetched_objects) return saved_objects @@ -756,10 +750,6 @@ def _update_objects(self, form, request): if is_background_request(request): request.job.logger.info(f"Updated {obj}") - # Rebuild the tree for MPTT models - if issubclass(self.queryset.model, MPTTModel): - self.queryset.model.objects.rebuild() - return updated_objects # @@ -935,16 +925,9 @@ def post(self, request): renamed_pks = self._rename_objects(form, selected_objects) if '_apply' in request.POST: - # For MPTT models, delay tree updates until all saves are complete - if issubclass(self.queryset.model, MPTTModel): - with self.queryset.model.objects.delay_mptt_updates(): - for obj in selected_objects: - setattr(obj, self.field_name, obj.new_name) - obj.save() - else: - for obj in selected_objects: - setattr(obj, self.field_name, obj.new_name) - obj.save() + for obj in selected_objects: + setattr(obj, self.field_name, obj.new_name) + obj.save() # Enforce constrained permissions if self.queryset.filter(pk__in=renamed_pks).count() != len(selected_objects): diff --git a/netbox/templates/dcim/module.html b/netbox/templates/dcim/module.html index a39eccf4dc7..1b43d2def8f 100644 --- a/netbox/templates/dcim/module.html +++ b/netbox/templates/dcim/module.html @@ -2,7 +2,6 @@ {% load helpers %} {% load plugins %} {% load i18n %} -{% load mptt %} {% block breadcrumbs %} {{ block.super }} diff --git a/netbox/tenancy/api/views.py b/netbox/tenancy/api/views.py index bf729d10673..9068c585a8e 100644 --- a/netbox/tenancy/api/views.py +++ b/netbox/tenancy/api/views.py @@ -1,6 +1,6 @@ from rest_framework.routers import APIRootView -from netbox.api.viewsets import MPTTLockedMixin, NetBoxModelViewSet +from netbox.api.viewsets import NetBoxModelViewSet from tenancy import filtersets from tenancy.models import * @@ -19,7 +19,7 @@ def get_view_name(self): # Tenants # -class TenantGroupViewSet(MPTTLockedMixin, NetBoxModelViewSet): +class TenantGroupViewSet(NetBoxModelViewSet): queryset = TenantGroup.objects.add_related_count( TenantGroup.objects.all(), Tenant, @@ -41,7 +41,7 @@ class TenantViewSet(NetBoxModelViewSet): # Contacts # -class ContactGroupViewSet(MPTTLockedMixin, NetBoxModelViewSet): +class ContactGroupViewSet(NetBoxModelViewSet): queryset = ContactGroup.objects.annotate_contacts() serializer_class = serializers.ContactGroupSerializer filterset_class = filtersets.ContactGroupFilterSet diff --git a/netbox/tenancy/graphql/types.py b/netbox/tenancy/graphql/types.py index ebc0f2e2b2b..5e80a1ebf41 100644 --- a/netbox/tenancy/graphql/types.py +++ b/netbox/tenancy/graphql/types.py @@ -88,7 +88,7 @@ class TenantType(ContactsMixin, PrimaryObjectType): @strawberry_django.type( models.TenantGroup, - fields='__all__', + exclude=['path'], filters=TenantGroupFilter, pagination=True ) @@ -125,7 +125,7 @@ class ContactRoleType(ContactAssignmentsMixin, OrganizationalObjectType): @strawberry_django.type( models.ContactGroup, - fields='__all__', + exclude=['path'], filters=ContactGroupFilter, pagination=True ) diff --git a/netbox/tenancy/migrations/0001_squashed_0012.py b/netbox/tenancy/migrations/0001_squashed_0012.py index f670003cf75..fa56356a355 100644 --- a/netbox/tenancy/migrations/0001_squashed_0012.py +++ b/netbox/tenancy/migrations/0001_squashed_0012.py @@ -1,5 +1,4 @@ import django.db.models.deletion -import mptt.fields import taggit.managers from django.db import migrations, models @@ -45,7 +44,7 @@ class Migration(migrations.Migration): ('level', models.PositiveIntegerField(editable=False)), ( 'parent', - mptt.fields.TreeForeignKey( + django.db.models.ForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, diff --git a/netbox/tenancy/migrations/0002_squashed_0011.py b/netbox/tenancy/migrations/0002_squashed_0011.py index cfdcb58dda7..5d21c03b7fe 100644 --- a/netbox/tenancy/migrations/0002_squashed_0011.py +++ b/netbox/tenancy/migrations/0002_squashed_0011.py @@ -1,5 +1,4 @@ import django.db.models.deletion -import mptt.fields import taggit.managers from django.db import migrations, models @@ -69,7 +68,7 @@ class Migration(migrations.Migration): ('level', models.PositiveIntegerField(editable=False)), ( 'parent', - mptt.fields.TreeForeignKey( + django.db.models.ForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, diff --git a/netbox/tenancy/migrations/0025_enable_ltree_extension.py b/netbox/tenancy/migrations/0025_enable_ltree_extension.py new file mode 100644 index 00000000000..d55373c50e6 --- /dev/null +++ b/netbox/tenancy/migrations/0025_enable_ltree_extension.py @@ -0,0 +1,17 @@ +from django.contrib.postgres.operations import CreateExtension +from django.db import migrations + + +class Migration(migrations.Migration): + """ + Enable the PostgreSQL ltree extension. Idempotent across apps; only one + CreateExtension('ltree') needs to succeed during a single migrate run. + """ + + dependencies = [ + ('tenancy', '0024_default_ordering_indexes'), + ] + + operations = [ + CreateExtension('ltree'), + ] diff --git a/netbox/tenancy/migrations/0026_ltree_paths.py b/netbox/tenancy/migrations/0026_ltree_paths.py new file mode 100644 index 00000000000..353eca28f3d --- /dev/null +++ b/netbox/tenancy/migrations/0026_ltree_paths.py @@ -0,0 +1,74 @@ +"""Replace django-mptt with PostgreSQL ltree for tenancy's hierarchical models.""" +from django.contrib.postgres.indexes import GistIndex +from django.db import migrations + +import netbox.models.ltree +from netbox.models.ltree import InstallLtreeTriggers + +MODELS = ('tenantgroup', 'contactgroup') +TABLES = ('tenancy_tenantgroup', 'tenancy_contactgroup') +LEGACY_FIELDS = ('lft', 'rght', 'tree_id', 'level') + + +def _populate_paths_sql(): + blocks = [] + for table in TABLES: + blocks.append(f""" +WITH RECURSIVE t(id, parent_id, path) AS ( + SELECT id, parent_id, id::text::ltree FROM "{table}" WHERE parent_id IS NULL + UNION ALL + SELECT r.id, r.parent_id, t.path || r.id::text::ltree + FROM "{table}" r JOIN t ON r.parent_id = t.id +) +UPDATE "{table}" SET path = t.path FROM t WHERE "{table}".id = t.id; +""") + return '\n'.join(blocks) + + +class Migration(migrations.Migration): + + dependencies = [ + ('tenancy', '0025_enable_ltree_extension'), + ] + + operations = [ + *[ + migrations.AddField( + model_name=m, + name='path', + field=netbox.models.ltree.LtreeField(blank=True, editable=False, null=True), + ) + for m in MODELS + ], + + *[InstallLtreeTriggers(t) for t in TABLES], + + migrations.RunSQL(_populate_paths_sql(), reverse_sql=migrations.RunSQL.noop), + + *[ + migrations.AlterField( + model_name=m, + name='path', + field=netbox.models.ltree.LtreeField(blank=True, default='', editable=False), + ) + for m in MODELS + ], + + # Drop legacy (tree_id, lft) indexes added in 0023_add_mptt_tree_indexes, + # then drop the legacy MPTT columns. + migrations.RemoveIndex(model_name='contactgroup', name='tenancy_contactgroup_tree_d2ce'), + migrations.RemoveIndex(model_name='tenantgroup', name='tenancy_tenantgroup_tree_ifebc'), + *[ + migrations.RemoveField(model_name=m, name=f) + for m in MODELS for f in LEGACY_FIELDS + ], + + migrations.AddIndex( + model_name='tenantgroup', + index=GistIndex(fields=['path'], name='tenancy_tenantgroup_path_gist'), + ), + migrations.AddIndex( + model_name='contactgroup', + index=GistIndex(fields=['path'], name='tenancy_contactgroup_path_gist'), + ), + ] diff --git a/netbox/tenancy/models/contacts.py b/netbox/tenancy/models/contacts.py index 0e0360cbabc..039f3b77cd8 100644 --- a/netbox/tenancy/models/contacts.py +++ b/netbox/tenancy/models/contacts.py @@ -1,4 +1,5 @@ from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.postgres.indexes import GistIndex from django.core.exceptions import ValidationError from django.db import models from django.db.models.expressions import RawSQL @@ -7,8 +8,8 @@ from netbox.models import ChangeLoggedModel, NestedGroupModel, OrganizationalModel, PrimaryModel from netbox.models.features import CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, has_feature +from netbox.models.ltree import LtreeManager from tenancy.choices import * -from utilities.mptt import TreeManager __all__ = ( 'Contact', @@ -18,23 +19,22 @@ ) -class ContactGroupManager(TreeManager): +class ContactGroupManager(LtreeManager): def annotate_contacts(self): """ Annotate the total number of Contacts belonging to each ContactGroup. - This returns both direct children and children of child groups. Raw SQL is used here to avoid double-counting - contacts which are assigned to multiple child groups of the parent. + Counts contacts assigned to the group itself or any descendant group (via the + ltree `<@` operator on the path column). DISTINCT avoids double-counting + contacts which are assigned to multiple groups in the subtree. """ return self.annotate( contact_count=RawSQL( "SELECT COUNT(DISTINCT m2m.contact_id)" " FROM tenancy_contact_groups m2m" " INNER JOIN tenancy_contactgroup cg ON m2m.contactgroup_id = cg.id" - " WHERE cg.tree_id = tenancy_contactgroup.tree_id" - " AND cg.lft >= tenancy_contactgroup.lft" - " AND cg.lft <= tenancy_contactgroup.rght", + " WHERE cg.path <@ tenancy_contactgroup.path", () ) ) @@ -48,9 +48,9 @@ class ContactGroup(NestedGroupModel): class Meta: ordering = ['name'] - # Empty tuple triggers Django migration detection for MPTT indexes - # (see #21016, django-mptt/django-mptt#682) - indexes = () + indexes = ( + GistIndex(fields=['path'], name='tenancy_contactgroup_path_gist'), + ) constraints = ( models.UniqueConstraint( fields=('parent', 'name'), diff --git a/netbox/tenancy/models/tenants.py b/netbox/tenancy/models/tenants.py index 923c9a9ec83..a9d4a2bce9f 100644 --- a/netbox/tenancy/models/tenants.py +++ b/netbox/tenancy/models/tenants.py @@ -1,3 +1,4 @@ +from django.contrib.postgres.indexes import GistIndex from django.db import models from django.db.models import Q from django.utils.translation import gettext_lazy as _ @@ -29,9 +30,9 @@ class TenantGroup(NestedGroupModel): class Meta: ordering = ['name'] - # Empty tuple triggers Django migration detection for MPTT indexes - # (see #21016, django-mptt/django-mptt#682) - indexes = () + indexes = ( + GistIndex(fields=['path'], name='tenancy_tenantgroup_path_gist'), + ) verbose_name = _('tenant group') verbose_name_plural = _('tenant groups') diff --git a/netbox/utilities/mptt.py b/netbox/utilities/mptt.py deleted file mode 100644 index 4914fccab2d..00000000000 --- a/netbox/utilities/mptt.py +++ /dev/null @@ -1,24 +0,0 @@ -from django.db.models import Manager -from mptt.managers import TreeManager as TreeManager_ -from mptt.querysets import TreeQuerySet as TreeQuerySet_ - -from .querysets import RestrictedQuerySet - -__all__ = ( - 'TreeManager', - 'TreeQuerySet', -) - - -class TreeQuerySet(TreeQuerySet_, RestrictedQuerySet): - """ - Mate django-mptt's TreeQuerySet with our RestrictedQuerySet for permissions enforcement. - """ - pass - - -class TreeManager(Manager.from_queryset(TreeQuerySet), TreeManager_): - """ - Extend django-mptt's TreeManager to incorporate RestrictedQuerySet(). - """ - pass diff --git a/netbox/utilities/query.py b/netbox/utilities/query.py index e672984c0f5..50c357fffb6 100644 --- a/netbox/utilities/query.py +++ b/netbox/utilities/query.py @@ -1,7 +1,7 @@ from django.db.models import Count, OuterRef, QuerySet, Subquery from django.db.models.functions import Coalesce -from utilities.mptt import TreeManager +from netbox.models.ltree import LtreeManager __all__ = ( 'count_related', @@ -64,8 +64,9 @@ def reapply_model_ordering(queryset: QuerySet) -> QuerySet: Reapply model-level ordering in case it has been lost through .annotate(). https://code.djangoproject.com/ticket/32811 """ - # MPTT-based models are exempt from this; use caution when annotating querysets of these models - if any(isinstance(manager, TreeManager) for manager in queryset.model._meta.local_managers): + # Hierarchical (ltree) models are exempt; their default ordering by `path` must not be + # clobbered by .annotate(). Use caution when annotating querysets of these models. + if any(isinstance(manager, LtreeManager) for manager in queryset.model._meta.local_managers): return queryset if queryset.ordered: return queryset diff --git a/netbox/utilities/templatetags/mptt.py b/netbox/utilities/templatetags/mptt.py deleted file mode 100644 index 96326e9a2e2..00000000000 --- a/netbox/utilities/templatetags/mptt.py +++ /dev/null @@ -1,21 +0,0 @@ -from django import template -from django.utils.html import escape -from django.utils.safestring import mark_safe - -register = template.Library() - - -@register.simple_tag() -def nested_tree(obj): - """ - Renders the entire hierarchy of a recursively-nested object (such as Region or SiteGroup). - """ - if not obj: - return mark_safe('—') - - nodes = obj.get_ancestors(include_self=True) - return mark_safe( - ' / '.join( - f'{escape(node)}' for node in nodes - ) - ) diff --git a/netbox/utilities/testing/filtersets.py b/netbox/utilities/testing/filtersets.py index bb200a43e3b..084bc341b83 100644 --- a/netbox/utilities/testing/filtersets.py +++ b/netbox/utilities/testing/filtersets.py @@ -6,10 +6,10 @@ from django.contrib.contenttypes.models import ContentType from django.db.models import ForeignKey, ManyToManyField, ManyToManyRel, ManyToOneRel, OneToOneRel from django.utils.module_loading import import_string -from mptt.models import MPTTModel from taggit.managers import TaggableManager from extras.filters import TagFilter +from netbox.models.ltree import LtreeModel from utilities.filters import MultiValueContentTypeFilter, TreeNodeMultipleChoiceFilter __all__ = ( @@ -20,10 +20,7 @@ EXEMPT_MODEL_FIELDS = ( 'comments', 'custom_field_data', - 'level', # MPTT - 'lft', # MPTT - 'rght', # MPTT - 'tree_id', # MPTT + 'path', # ltree, trigger-maintained ) @@ -59,8 +56,8 @@ def get_filters_for_model_field(self, field): if field.related_model is ContentType: return [(None, None)] - # ForeignKey to an MPTT-enabled model - if issubclass(field.related_model, MPTTModel) and field.model is not field.related_model: + # ForeignKey to an ltree-backed hierarchical model + if issubclass(field.related_model, LtreeModel) and field.model is not field.related_model: return [(f'{filter_name}_id', TreeNodeMultipleChoiceFilter)] return [(f'{filter_name}_id', django_filters.ModelMultipleChoiceFilter)] diff --git a/netbox/utilities/tests/test_filters.py b/netbox/utilities/tests/test_filters.py index 1c64f8d28bb..942ae2511e9 100644 --- a/netbox/utilities/tests/test_filters.py +++ b/netbox/utilities/tests/test_filters.py @@ -2,7 +2,6 @@ from django.conf import settings from django.db import models from django.test import TestCase -from mptt.fields import TreeForeignKey from taggit.managers import TaggableManager from dcim.choices import * @@ -113,9 +112,11 @@ class DummyModel(models.Model): integerfield = models.IntegerField() macaddressfield = MACAddressField() timefield = models.TimeField() - treeforeignkeyfield = TreeForeignKey( + treeforeignkeyfield = models.ForeignKey( to='self', - on_delete=models.CASCADE + on_delete=models.CASCADE, + null=True, + blank=True, ) tags = TaggableManager(through=TaggedItem) diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py new file mode 100644 index 00000000000..8d81fb56ec0 --- /dev/null +++ b/netbox/utilities/tests/test_ltree.py @@ -0,0 +1,174 @@ +"""Tests for the ltree-based hierarchical model infrastructure.""" +from django.db import connection +from django.test import TestCase + +from dcim.models import Region, Site + + +class LtreeTriggerTests(TestCase): + """Verify per-row PostgreSQL triggers maintain `path` correctly.""" + + def test_insert_root_path(self): + r = Region.objects.create(name='Root', slug='root') + self.assertEqual(r.path, str(r.pk)) + + def test_insert_child_path(self): + r = Region.objects.create(name='Root', slug='root') + c = Region.objects.create(parent=r, name='Child', slug='child') + self.assertEqual(c.path, f'{r.pk}.{c.pk}') + + def test_grandchild_path(self): + r = Region.objects.create(name='R', slug='r') + c = Region.objects.create(parent=r, name='C', slug='c') + g = Region.objects.create(parent=c, name='G', slug='g') + self.assertEqual(g.path, f'{r.pk}.{c.pk}.{g.pk}') + + def test_move_cascades_to_descendants(self): + r = Region.objects.create(name='R', slug='r') + c = Region.objects.create(parent=r, name='C', slug='c') + g = Region.objects.create(parent=c, name='G', slug='g') + c.parent = None + c.save() + c.refresh_from_db() + g.refresh_from_db() + self.assertEqual(c.path, str(c.pk)) + self.assertEqual(g.path, f'{c.pk}.{g.pk}') + + def test_bulk_create_populates_paths(self): + """BEFORE INSERT trigger fires on bulk_create, populating path.""" + root = Region.objects.create(name='R', slug='r-bulk') + children = Region.objects.bulk_create([ + Region(parent=root, name=f'C{i}', slug=f'c{i}-bulk') for i in range(3) + ]) + for child in children: + child.refresh_from_db() + self.assertEqual(child.path, f'{root.pk}.{child.pk}') + + def test_queryset_update_with_parent_id_cascades(self): + """Raw .update() that changes parent_id still fires triggers.""" + r1 = Region.objects.create(name='R1', slug='r1-up') + r2 = Region.objects.create(name='R2', slug='r2-up') + c = Region.objects.create(parent=r1, name='C', slug='c-up') + g = Region.objects.create(parent=c, name='G', slug='g-up') + + Region.objects.filter(pk=c.pk).update(parent=r2) + c.refresh_from_db() + g.refresh_from_db() + self.assertEqual(c.path, f'{r2.pk}.{c.pk}') + self.assertEqual(g.path, f'{r2.pk}.{c.pk}.{g.pk}') + + def test_gist_index_exists(self): + """Every ltree-backed table has a GiST index on path.""" + expected = { + 'dcim_region_path_gist', + 'dcim_sitegroup_path_gist', + 'dcim_location_path_gist', + 'dcim_devicerole_path_gist', + 'dcim_platform_path_gist', + 'dcim_inventoryitem_path_gist', + 'dcim_inv_item_tmpl_path_gist', + 'dcim_modulebay_path_gist', + 'tenancy_tenantgroup_path_gist', + 'tenancy_contactgroup_path_gist', + 'wireless_lan_grp_path_gist', + } + with connection.cursor() as cursor: + cursor.execute(""" + SELECT indexname FROM pg_indexes + WHERE indexname = ANY(%s) AND indexdef LIKE '%%USING gist%%' + """, [list(expected)]) + found = {row[0] for row in cursor.fetchall()} + self.assertSetEqual(found, expected) + + +class LtreeAPIParityTests(TestCase): + """Verify the MPTTModel-compatible API surface.""" + + @classmethod + def setUpTestData(cls): + # Build: root -> mid -> leaf + # -> leaf2 (sibling of mid's child) + cls.root = Region.objects.create(name='Root', slug='root-api') + cls.mid = Region.objects.create(parent=cls.root, name='Mid', slug='mid-api') + cls.leaf = Region.objects.create(parent=cls.mid, name='Leaf', slug='leaf-api') + cls.leaf2 = Region.objects.create(parent=cls.mid, name='Leaf2', slug='leaf2-api') + + def test_level(self): + self.assertEqual(self.root.level, 0) + self.assertEqual(self.mid.level, 1) + self.assertEqual(self.leaf.level, 2) + self.assertEqual(self.leaf.get_level(), 2) + + def test_is_root_leaf_child(self): + self.assertTrue(self.root.is_root_node()) + self.assertFalse(self.root.is_leaf_node()) + self.assertFalse(self.root.is_child_node()) + self.assertFalse(self.leaf.is_root_node()) + self.assertTrue(self.leaf.is_leaf_node()) + self.assertTrue(self.leaf.is_child_node()) + + def test_get_root(self): + self.assertEqual(self.leaf.get_root(), self.root) + self.assertEqual(self.root.get_root(), self.root) + + def test_get_ancestors(self): + ancestors = list(self.leaf.get_ancestors().values_list('name', flat=True)) + self.assertEqual(ancestors, ['Root', 'Mid']) + with_self = list(self.leaf.get_ancestors(include_self=True).values_list('name', flat=True)) + self.assertEqual(with_self, ['Root', 'Mid', 'Leaf']) + + def test_get_descendants(self): + descendants = sorted(self.root.get_descendants().values_list('name', flat=True)) + self.assertEqual(descendants, ['Leaf', 'Leaf2', 'Mid']) + self.assertEqual(self.root.get_descendant_count(), 3) + + def test_get_children(self): + children = sorted(self.root.get_children().values_list('name', flat=True)) + self.assertEqual(children, ['Mid']) + + def test_get_siblings(self): + siblings = list(self.leaf.get_siblings().values_list('name', flat=True)) + self.assertEqual(siblings, ['Leaf2']) + + def test_get_family(self): + family = sorted(self.mid.get_family().values_list('name', flat=True)) + self.assertEqual(family, ['Leaf', 'Leaf2', 'Mid', 'Root']) + + def test_move_to(self): + new_root = Region.objects.create(name='New', slug='new-api') + self.leaf.move_to(new_root) + self.leaf.refresh_from_db() + self.assertEqual(self.leaf.parent, new_root) + self.assertEqual(self.leaf.path, f'{new_root.pk}.{self.leaf.pk}') + + +class CycleValidationTests(TestCase): + """clean() must refuse to assign a descendant as parent.""" + + def test_cycle_raises(self): + from django.core.exceptions import ValidationError + a = Region.objects.create(name='A', slug='a-cyc') + b = Region.objects.create(parent=a, name='B', slug='b-cyc') + Region.objects.create(parent=b, name='C', slug='c-cyc') + a.parent = b + with self.assertRaises(ValidationError): + a.full_clean() + + +class AddRelatedCountTests(TestCase): + """add_related_count must cumulate across subtrees via path <@.""" + + def test_cumulative_fk_count(self): + root = Region.objects.create(name='R', slug='r-arc') + child = Region.objects.create(parent=root, name='C', slug='c-arc') + Site.objects.create(name='S1', slug='s1-arc', region=child) + Site.objects.create(name='S2', slug='s2-arc', region=root) + + qs = Region.objects.add_related_count( + Region.objects.filter(slug__endswith='-arc'), + Site, 'region', 'site_count', cumulative=True, + ) + counts = {r.name: r.site_count for r in qs} + # root sees both sites (direct + via child) + self.assertEqual(counts['R'], 2) + self.assertEqual(counts['C'], 1) diff --git a/netbox/wireless/api/views.py b/netbox/wireless/api/views.py index f5883e930d2..6d48c0be796 100644 --- a/netbox/wireless/api/views.py +++ b/netbox/wireless/api/views.py @@ -1,6 +1,6 @@ from rest_framework.routers import APIRootView -from netbox.api.viewsets import MPTTLockedMixin, NetBoxModelViewSet +from netbox.api.viewsets import NetBoxModelViewSet from wireless import filtersets from wireless.models import * @@ -15,7 +15,7 @@ def get_view_name(self): return 'Wireless' -class WirelessLANGroupViewSet(MPTTLockedMixin, NetBoxModelViewSet): +class WirelessLANGroupViewSet(NetBoxModelViewSet): queryset = WirelessLANGroup.objects.add_related_count( WirelessLANGroup.objects.all(), WirelessLAN, diff --git a/netbox/wireless/graphql/types.py b/netbox/wireless/graphql/types.py index 2e7930074b3..2ee19ca7a52 100644 --- a/netbox/wireless/graphql/types.py +++ b/netbox/wireless/graphql/types.py @@ -22,7 +22,7 @@ @strawberry_django.type( models.WirelessLANGroup, - fields='__all__', + exclude=['path'], filters=WirelessLANGroupFilter, pagination=True ) diff --git a/netbox/wireless/migrations/0001_squashed_0008.py b/netbox/wireless/migrations/0001_squashed_0008.py index 2ffc287f9e6..7a7e381e541 100644 --- a/netbox/wireless/migrations/0001_squashed_0008.py +++ b/netbox/wireless/migrations/0001_squashed_0008.py @@ -1,5 +1,4 @@ import django.db.models.deletion -import mptt.fields import taggit.managers from django.db import migrations, models @@ -46,7 +45,7 @@ class Migration(migrations.Migration): ('level', models.PositiveIntegerField(editable=False)), ( 'parent', - mptt.fields.TreeForeignKey( + django.db.models.ForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, diff --git a/netbox/wireless/migrations/0020_enable_ltree_extension.py b/netbox/wireless/migrations/0020_enable_ltree_extension.py new file mode 100644 index 00000000000..70ecd4456d5 --- /dev/null +++ b/netbox/wireless/migrations/0020_enable_ltree_extension.py @@ -0,0 +1,17 @@ +from django.contrib.postgres.operations import CreateExtension +from django.db import migrations + + +class Migration(migrations.Migration): + """ + Enable the PostgreSQL ltree extension. Idempotent across apps; only one + CreateExtension('ltree') needs to succeed during a single migrate run. + """ + + dependencies = [ + ('wireless', '0019_default_ordering_indexes'), + ] + + operations = [ + CreateExtension('ltree'), + ] diff --git a/netbox/wireless/migrations/0021_ltree_paths.py b/netbox/wireless/migrations/0021_ltree_paths.py new file mode 100644 index 00000000000..d2ae7296220 --- /dev/null +++ b/netbox/wireless/migrations/0021_ltree_paths.py @@ -0,0 +1,56 @@ +"""Replace django-mptt with PostgreSQL ltree for wireless's hierarchical models.""" +from django.contrib.postgres.indexes import GistIndex +from django.db import migrations + +import netbox.models.ltree +from netbox.models.ltree import InstallLtreeTriggers + +MODEL = 'wirelesslangroup' +TABLE = 'wireless_wirelesslangroup' +LEGACY_FIELDS = ('lft', 'rght', 'tree_id', 'level') + + +class Migration(migrations.Migration): + + dependencies = [ + ('wireless', '0020_enable_ltree_extension'), + ] + + operations = [ + migrations.AddField( + model_name=MODEL, + name='path', + field=netbox.models.ltree.LtreeField(blank=True, editable=False, null=True), + ), + + InstallLtreeTriggers(TABLE), + + migrations.RunSQL( + f""" +WITH RECURSIVE t(id, parent_id, path) AS ( + SELECT id, parent_id, id::text::ltree FROM "{TABLE}" WHERE parent_id IS NULL + UNION ALL + SELECT r.id, r.parent_id, t.path || r.id::text::ltree + FROM "{TABLE}" r JOIN t ON r.parent_id = t.id +) +UPDATE "{TABLE}" SET path = t.path FROM t WHERE "{TABLE}".id = t.id; +""", + reverse_sql=migrations.RunSQL.noop, + ), + + migrations.AlterField( + model_name=MODEL, + name='path', + field=netbox.models.ltree.LtreeField(blank=True, default='', editable=False), + ), + + # Drop legacy (tree_id, lft) index added in 0018_add_mptt_tree_indexes, + # then drop the legacy MPTT columns. + migrations.RemoveIndex(model_name=MODEL, name='wireless_wirelesslangroup_fbcd'), + *[migrations.RemoveField(model_name=MODEL, name=f) for f in LEGACY_FIELDS], + + migrations.AddIndex( + model_name=MODEL, + index=GistIndex(fields=['path'], name='wireless_lan_grp_path_gist'), + ), + ] diff --git a/netbox/wireless/models.py b/netbox/wireless/models.py index f212fb6dcd2..a7aaa45247c 100644 --- a/netbox/wireless/models.py +++ b/netbox/wireless/models.py @@ -1,3 +1,4 @@ +from django.contrib.postgres.indexes import GistIndex from django.core.exceptions import ValidationError from django.db import models from django.utils.translation import gettext_lazy as _ @@ -64,9 +65,9 @@ class WirelessLANGroup(NestedGroupModel): class Meta: ordering = ('name', 'pk') - # Empty tuple triggers Django migration detection for MPTT indexes - # (see #21016, django-mptt/django-mptt#682) - indexes = () + indexes = ( + GistIndex(fields=['path'], name='wireless_lan_grp_path_gist'), + ) constraints = ( models.UniqueConstraint( fields=('parent', 'name'), diff --git a/requirements.txt b/requirements.txt index 4b4c0dbb827..1c38d94ae5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,6 @@ django-debug-toolbar==6.3.0 django-filter==25.2 django-graphiql-debug-toolbar==0.2.0 django-htmx==1.27.0 -django-mptt==0.18.0 django-pglocks==1.0.4 django-prometheus==2.4.0 django-redis==6.0.0 From a5c8d7c55bdf569aa95f27fe4efefeb1d9f91bdd Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 26 May 2026 17:26:54 -0700 Subject: [PATCH 02/42] #21488 - Replace MPTT wtih PostgreSQL Ltree --- netbox/dcim/models/sites.py | 2 +- netbox/netbox/api/viewsets/__init__.py | 2 -- netbox/netbox/models/ltree.py | 15 +++++++-------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/netbox/dcim/models/sites.py b/netbox/dcim/models/sites.py index 416306c2d05..02d9c62104b 100644 --- a/netbox/dcim/models/sites.py +++ b/netbox/dcim/models/sites.py @@ -1,9 +1,9 @@ import decimal from django.contrib.contenttypes.fields import GenericRelation +from django.contrib.postgres.indexes import GistIndex from django.core.exceptions import ValidationError from django.core.validators import MaxValueValidator, MinValueValidator -from django.contrib.postgres.indexes import GistIndex from django.db import models from django.utils.translation import gettext_lazy as _ from timezone_field import TimeZoneField diff --git a/netbox/netbox/api/viewsets/__init__.py b/netbox/netbox/api/viewsets/__init__.py index d6ce93e2f4f..49d12dd76ce 100644 --- a/netbox/netbox/api/viewsets/__init__.py +++ b/netbox/netbox/api/viewsets/__init__.py @@ -333,5 +333,3 @@ def perform_destroy(self, instance): super().perform_destroy(instance) except ObjectDoesNotExist: raise PermissionDenied() - - diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index 6f21234d781..4552f4af166 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -13,7 +13,7 @@ """ from django.contrib.contenttypes.models import ContentType from django.db import migrations, models -from django.db.models import Count, ForeignKey, ManyToManyField, Lookup, OuterRef, Q, Subquery +from django.db.models import Count, ForeignKey, Lookup, ManyToManyField, OuterRef, Q, Subquery from django.db.models.expressions import RawSQL from utilities.querysets import RestrictedQuerySet @@ -134,7 +134,7 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F return queryset.annotate(**{ count_attr: RawSQL(sql, [], output_field=models.IntegerField()) }) - elif has_generic_fk: + if has_generic_fk: content_type = ContentType.objects.get_for_model(queryset.model) sql = f'''( SELECT COUNT(DISTINCT "{related_table}"."id") @@ -147,18 +147,17 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F return queryset.annotate(**{ count_attr: RawSQL(sql, [content_type.pk], output_field=models.IntegerField()) }) - else: - rel_field_col = f'{rel_field}_id' - sql = f'''( + rel_field_col = f'{rel_field}_id' + sql = f'''( SELECT COUNT(DISTINCT "{related_table}"."id") FROM "{related_table}" INNER JOIN "{parent_table}" AS subtree ON "{related_table}"."{rel_field_col}" = subtree."id" WHERE subtree."path" <@ "{parent_table}"."path" )''' - return queryset.annotate(**{ - count_attr: RawSQL(sql, [], output_field=models.IntegerField()) - }) + return queryset.annotate(**{ + count_attr: RawSQL(sql, [], output_field=models.IntegerField()) + }) # Non-cumulative: direct count. if is_many_to_many: From 162ab548e664c4983652e23133106be0cfb8cab1 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 26 May 2026 17:33:14 -0700 Subject: [PATCH 03/42] #21488 - Replace MPTT wtih PostgreSQL Ltree --- .../migrations/0234_enable_ltree_extension.py | 17 ------------- ...235_ltree_paths.py => 0234_ltree_paths.py} | 24 +++++++++++-------- .../migrations/0025_enable_ltree_extension.py | 17 ------------- ...026_ltree_paths.py => 0025_ltree_paths.py} | 6 ++++- .../migrations/0020_enable_ltree_extension.py | 17 ------------- ...021_ltree_paths.py => 0020_ltree_paths.py} | 6 ++++- 6 files changed, 24 insertions(+), 63 deletions(-) delete mode 100644 netbox/dcim/migrations/0234_enable_ltree_extension.py rename netbox/dcim/migrations/{0235_ltree_paths.py => 0234_ltree_paths.py} (86%) delete mode 100644 netbox/tenancy/migrations/0025_enable_ltree_extension.py rename netbox/tenancy/migrations/{0026_ltree_paths.py => 0025_ltree_paths.py} (90%) delete mode 100644 netbox/wireless/migrations/0020_enable_ltree_extension.py rename netbox/wireless/migrations/{0021_ltree_paths.py => 0020_ltree_paths.py} (87%) diff --git a/netbox/dcim/migrations/0234_enable_ltree_extension.py b/netbox/dcim/migrations/0234_enable_ltree_extension.py deleted file mode 100644 index 6b29e5318cc..00000000000 --- a/netbox/dcim/migrations/0234_enable_ltree_extension.py +++ /dev/null @@ -1,17 +0,0 @@ -from django.contrib.postgres.operations import CreateExtension -from django.db import migrations - - -class Migration(migrations.Migration): - """ - Enable the PostgreSQL ltree extension. Required by the next migration which - adds the `path` column on hierarchical models. - """ - - dependencies = [ - ('dcim', '0233_device_render_config_permission'), - ] - - operations = [ - CreateExtension('ltree'), - ] diff --git a/netbox/dcim/migrations/0235_ltree_paths.py b/netbox/dcim/migrations/0234_ltree_paths.py similarity index 86% rename from netbox/dcim/migrations/0235_ltree_paths.py rename to netbox/dcim/migrations/0234_ltree_paths.py index 5f5c279a424..032a10e742f 100644 --- a/netbox/dcim/migrations/0235_ltree_paths.py +++ b/netbox/dcim/migrations/0234_ltree_paths.py @@ -4,16 +4,17 @@ For each of (Region, SiteGroup, Location, DeviceRole, Platform, ModuleBay, InventoryItem, InventoryItemTemplate) this migration: -1. Adds a nullable `path` LTreeField. -2. Installs per-table BEFORE-INSERT/UPDATE-OF-parent_id and AFTER-UPDATE-OF-path - triggers so concurrent writes during the long-running data step get correct - paths. -3. Populates paths for existing rows via a single recursive CTE per table. -4. Tightens `path` to NOT NULL. -5. Drops the legacy MPTT columns (lft, rght, tree_id, level). -6. Adds a GiST index on the `path` column for efficient `@>` / `<@` lookups. +1. Enables the PostgreSQL ltree extension (idempotent). +2. Adds a nullable `path` LTreeField. +3. Installs per-table BEFORE-INSERT/UPDATE-OF-parent_id and AFTER-UPDATE-OF-(parent_id, path) + triggers so concurrent writes during the long-running data step get correct paths. +4. Populates paths for existing rows via a single recursive CTE per table. +5. Tightens `path` to NOT NULL. +6. Drops the legacy MPTT columns (lft, rght, tree_id, level). +7. Adds a GiST index on the `path` column for efficient `@>` / `<@` lookups. """ from django.contrib.postgres.indexes import GistIndex +from django.contrib.postgres.operations import CreateExtension from django.db import migrations import netbox.models.ltree @@ -56,11 +57,14 @@ def _populate_paths_sql(): class Migration(migrations.Migration): dependencies = [ - ('dcim', '0234_enable_ltree_extension'), + ('dcim', '0233_device_render_config_permission'), ] operations = [ - # 1. Add nullable path column + # 1. Enable the ltree extension (idempotent — CreateExtension emits IF NOT EXISTS) + CreateExtension('ltree'), + + # 2. Add nullable path column *[ migrations.AddField( model_name=m, diff --git a/netbox/tenancy/migrations/0025_enable_ltree_extension.py b/netbox/tenancy/migrations/0025_enable_ltree_extension.py deleted file mode 100644 index d55373c50e6..00000000000 --- a/netbox/tenancy/migrations/0025_enable_ltree_extension.py +++ /dev/null @@ -1,17 +0,0 @@ -from django.contrib.postgres.operations import CreateExtension -from django.db import migrations - - -class Migration(migrations.Migration): - """ - Enable the PostgreSQL ltree extension. Idempotent across apps; only one - CreateExtension('ltree') needs to succeed during a single migrate run. - """ - - dependencies = [ - ('tenancy', '0024_default_ordering_indexes'), - ] - - operations = [ - CreateExtension('ltree'), - ] diff --git a/netbox/tenancy/migrations/0026_ltree_paths.py b/netbox/tenancy/migrations/0025_ltree_paths.py similarity index 90% rename from netbox/tenancy/migrations/0026_ltree_paths.py rename to netbox/tenancy/migrations/0025_ltree_paths.py index 353eca28f3d..8a1070a4d70 100644 --- a/netbox/tenancy/migrations/0026_ltree_paths.py +++ b/netbox/tenancy/migrations/0025_ltree_paths.py @@ -1,5 +1,6 @@ """Replace django-mptt with PostgreSQL ltree for tenancy's hierarchical models.""" from django.contrib.postgres.indexes import GistIndex +from django.contrib.postgres.operations import CreateExtension from django.db import migrations import netbox.models.ltree @@ -28,10 +29,13 @@ def _populate_paths_sql(): class Migration(migrations.Migration): dependencies = [ - ('tenancy', '0025_enable_ltree_extension'), + ('tenancy', '0024_default_ordering_indexes'), ] operations = [ + # Enable the ltree extension (idempotent — CreateExtension emits IF NOT EXISTS) + CreateExtension('ltree'), + *[ migrations.AddField( model_name=m, diff --git a/netbox/wireless/migrations/0020_enable_ltree_extension.py b/netbox/wireless/migrations/0020_enable_ltree_extension.py deleted file mode 100644 index 70ecd4456d5..00000000000 --- a/netbox/wireless/migrations/0020_enable_ltree_extension.py +++ /dev/null @@ -1,17 +0,0 @@ -from django.contrib.postgres.operations import CreateExtension -from django.db import migrations - - -class Migration(migrations.Migration): - """ - Enable the PostgreSQL ltree extension. Idempotent across apps; only one - CreateExtension('ltree') needs to succeed during a single migrate run. - """ - - dependencies = [ - ('wireless', '0019_default_ordering_indexes'), - ] - - operations = [ - CreateExtension('ltree'), - ] diff --git a/netbox/wireless/migrations/0021_ltree_paths.py b/netbox/wireless/migrations/0020_ltree_paths.py similarity index 87% rename from netbox/wireless/migrations/0021_ltree_paths.py rename to netbox/wireless/migrations/0020_ltree_paths.py index d2ae7296220..b9b75054d48 100644 --- a/netbox/wireless/migrations/0021_ltree_paths.py +++ b/netbox/wireless/migrations/0020_ltree_paths.py @@ -1,5 +1,6 @@ """Replace django-mptt with PostgreSQL ltree for wireless's hierarchical models.""" from django.contrib.postgres.indexes import GistIndex +from django.contrib.postgres.operations import CreateExtension from django.db import migrations import netbox.models.ltree @@ -13,10 +14,13 @@ class Migration(migrations.Migration): dependencies = [ - ('wireless', '0020_enable_ltree_extension'), + ('wireless', '0019_default_ordering_indexes'), ] operations = [ + # Enable the ltree extension (idempotent — CreateExtension emits IF NOT EXISTS) + CreateExtension('ltree'), + migrations.AddField( model_name=MODEL, name='path', From f63eb57e4550a929f34761f2776f4f29bba0e7c9 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 27 May 2026 08:42:25 -0700 Subject: [PATCH 04/42] #21488 - Replace MPTT wtih PostgreSQL Ltree --- base_requirements.txt | 5 +++ netbox/dcim/migrations/0002_squashed.py | 9 ++--- netbox/dcim/migrations/0131_squashed_0159.py | 3 +- netbox/dcim/migrations/0190_nested_modules.py | 3 +- .../migrations/0191_module_bay_rebuild.py | 15 +++++++-- .../migrations/0203_device_role_nested.py | 3 +- .../migrations/0204_device_role_rebuild.py | 16 ++++++--- .../dcim/migrations/0213_platform_parent.py | 3 +- .../dcim/migrations/0214_platform_rebuild.py | 21 ++++++++++-- .../migrations/0226_modulebay_rebuild_tree.py | 25 ++++++++++++-- netbox/netbox/models/ltree.py | 33 +++++++++++++------ netbox/netbox/settings.py | 1 + .../tenancy/migrations/0001_squashed_0012.py | 3 +- .../tenancy/migrations/0002_squashed_0011.py | 3 +- .../wireless/migrations/0001_squashed_0008.py | 3 +- requirements.txt | 1 + 16 files changed, 113 insertions(+), 34 deletions(-) diff --git a/base_requirements.txt b/base_requirements.txt index 73cbbaf1dc5..b51e79ff233 100644 --- a/base_requirements.txt +++ b/base_requirements.txt @@ -26,6 +26,11 @@ django-graphiql-debug-toolbar # https://django-htmx.readthedocs.io/en/latest/changelog.html django-htmx +# Modified Preorder Tree Traversal (required only for historical migrations +# that pre-date the switch to PostgreSQL ltree; runtime code uses +# netbox.models.ltree.LtreeModel instead). +django-mptt + # Context managers for PostgreSQL advisory locks # https://github.com/Xof/django-pglocks/blob/master/CHANGES.txt django-pglocks diff --git a/netbox/dcim/migrations/0002_squashed.py b/netbox/dcim/migrations/0002_squashed.py index f876f8bddd7..9a2fbc78640 100644 --- a/netbox/dcim/migrations/0002_squashed.py +++ b/netbox/dcim/migrations/0002_squashed.py @@ -1,4 +1,5 @@ import django.db.models.deletion +import mptt.fields import taggit.managers from django.conf import settings from django.db import migrations, models @@ -26,7 +27,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='sitegroup', name='parent', - field=django.db.models.ForeignKey( + field=mptt.fields.TreeForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, @@ -75,7 +76,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='region', name='parent', - field=django.db.models.ForeignKey( + field=mptt.fields.TreeForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, @@ -380,7 +381,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='location', name='parent', - field=django.db.models.ForeignKey( + field=mptt.fields.TreeForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, @@ -416,7 +417,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='inventoryitem', name='parent', - field=django.db.models.ForeignKey( + field=mptt.fields.TreeForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, diff --git a/netbox/dcim/migrations/0131_squashed_0159.py b/netbox/dcim/migrations/0131_squashed_0159.py index 11a2685de76..eb77878875d 100644 --- a/netbox/dcim/migrations/0131_squashed_0159.py +++ b/netbox/dcim/migrations/0131_squashed_0159.py @@ -1,5 +1,6 @@ import django.core.validators import django.db.models.deletion +import mptt.fields import taggit.managers from django.db import migrations, models @@ -1247,7 +1248,7 @@ class Migration(migrations.Migration): ), ( 'parent', - django.db.models.ForeignKey( + mptt.fields.TreeForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, diff --git a/netbox/dcim/migrations/0190_nested_modules.py b/netbox/dcim/migrations/0190_nested_modules.py index 52ec1d616f7..239e0863941 100644 --- a/netbox/dcim/migrations/0190_nested_modules.py +++ b/netbox/dcim/migrations/0190_nested_modules.py @@ -1,4 +1,5 @@ import django.db.models.deletion +import mptt.fields from django.db import migrations, models @@ -43,7 +44,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='modulebay', name='parent', - field=django.db.models.ForeignKey( + field=mptt.fields.TreeForeignKey( blank=True, editable=False, null=True, diff --git a/netbox/dcim/migrations/0191_module_bay_rebuild.py b/netbox/dcim/migrations/0191_module_bay_rebuild.py index 9b794e64534..f7d1147484b 100644 --- a/netbox/dcim/migrations/0191_module_bay_rebuild.py +++ b/netbox/dcim/migrations/0191_module_bay_rebuild.py @@ -1,13 +1,22 @@ +import mptt +import mptt.managers from django.db import migrations +def rebuild_mptt(apps, schema_editor): + manager = mptt.managers.TreeManager() + ModuleBay = apps.get_model('dcim', 'ModuleBay') + manager.model = ModuleBay + mptt.register(ModuleBay) + manager.contribute_to_class(ModuleBay, 'objects') + manager.rebuild() + + class Migration(migrations.Migration): dependencies = [ ('dcim', '0190_nested_modules'), ] - # Historical MPTT rebuild: now a no-op. Tree state will be populated from - # parent FKs into an ltree path column by a later migration. operations = [ - migrations.RunPython(migrations.RunPython.noop, migrations.RunPython.noop), + migrations.RunPython(code=rebuild_mptt, reverse_code=migrations.RunPython.noop), ] diff --git a/netbox/dcim/migrations/0203_device_role_nested.py b/netbox/dcim/migrations/0203_device_role_nested.py index fe0b09d21b2..c9dd791b3a0 100644 --- a/netbox/dcim/migrations/0203_device_role_nested.py +++ b/netbox/dcim/migrations/0203_device_role_nested.py @@ -1,6 +1,7 @@ # Generated by Django 5.1.7 on 2025-03-25 18:06 import django.db.models.manager +import mptt.fields from django.db import migrations, models @@ -38,7 +39,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='devicerole', name='parent', - field=django.db.models.ForeignKey( + field=mptt.fields.TreeForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, diff --git a/netbox/dcim/migrations/0204_device_role_rebuild.py b/netbox/dcim/migrations/0204_device_role_rebuild.py index 1d200890a59..045b3b2ba54 100644 --- a/netbox/dcim/migrations/0204_device_role_rebuild.py +++ b/netbox/dcim/migrations/0204_device_role_rebuild.py @@ -1,14 +1,22 @@ +import mptt +import mptt.managers from django.db import migrations +def rebuild_mptt(apps, schema_editor): + manager = mptt.managers.TreeManager() + DeviceRole = apps.get_model('dcim', 'DeviceRole') + manager.model = DeviceRole + mptt.register(DeviceRole) + manager.contribute_to_class(DeviceRole, 'objects') + manager.rebuild() + + class Migration(migrations.Migration): dependencies = [ ('dcim', '0203_device_role_nested'), ] - # Historical MPTT rebuild: now a no-op. The legacy lft/rght/tree_id/level - # columns are removed in a later migration and ltree paths are populated - # from parent FKs after that. operations = [ - migrations.RunPython(migrations.RunPython.noop, migrations.RunPython.noop), + migrations.RunPython(code=rebuild_mptt, reverse_code=migrations.RunPython.noop), ] diff --git a/netbox/dcim/migrations/0213_platform_parent.py b/netbox/dcim/migrations/0213_platform_parent.py index ffb6f1bf271..1a1e0f228dc 100644 --- a/netbox/dcim/migrations/0213_platform_parent.py +++ b/netbox/dcim/migrations/0213_platform_parent.py @@ -1,4 +1,5 @@ import django.db.models.deletion +import mptt.fields from django.db import migrations, models @@ -13,7 +14,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='platform', name='parent', - field=django.db.models.ForeignKey( + field=mptt.fields.TreeForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, diff --git a/netbox/dcim/migrations/0214_platform_rebuild.py b/netbox/dcim/migrations/0214_platform_rebuild.py index eb3845109d9..307d34e8313 100644 --- a/netbox/dcim/migrations/0214_platform_rebuild.py +++ b/netbox/dcim/migrations/0214_platform_rebuild.py @@ -1,14 +1,29 @@ +import mptt +import mptt.managers from django.db import migrations +def rebuild_mptt(apps, schema_editor): + """ + Construct the MPTT hierarchy. + """ + Platform = apps.get_model('dcim', 'Platform') + manager = mptt.managers.TreeManager() + manager.model = Platform + mptt.register(Platform) + manager.contribute_to_class(Platform, 'objects') + manager.rebuild() + + class Migration(migrations.Migration): dependencies = [ ('dcim', '0213_platform_parent'), ] - # Historical MPTT rebuild: now a no-op. Tree state will be populated from - # parent FKs into an ltree path column by a later migration. operations = [ - migrations.RunPython(migrations.RunPython.noop, migrations.RunPython.noop), + migrations.RunPython( + code=rebuild_mptt, + reverse_code=migrations.RunPython.noop + ), ] diff --git a/netbox/dcim/migrations/0226_modulebay_rebuild_tree.py b/netbox/dcim/migrations/0226_modulebay_rebuild_tree.py index ac0858cd8fd..27f33b1dad4 100644 --- a/netbox/dcim/migrations/0226_modulebay_rebuild_tree.py +++ b/netbox/dcim/migrations/0226_modulebay_rebuild_tree.py @@ -1,13 +1,32 @@ +import mptt.managers +import mptt.models from django.db import migrations +def rebuild_mptt(apps, schema_editor): + """ + Rebuild the MPTT tree for ModuleBay to apply new ordering. + """ + ModuleBay = apps.get_model('dcim', 'ModuleBay') + + # Set MPTTMeta with the correct order_insertion_by + class MPTTMeta: + order_insertion_by = ('name',) + + ModuleBay.MPTTMeta = MPTTMeta + ModuleBay._mptt_meta = mptt.models.MPTTOptions(MPTTMeta) + + manager = mptt.managers.TreeManager() + manager.model = ModuleBay + manager.contribute_to_class(ModuleBay, 'objects') + manager.rebuild() + + class Migration(migrations.Migration): dependencies = [ ('dcim', '0226_add_mptt_tree_indexes'), ] - # Historical MPTT rebuild: now a no-op. The MPTT tree columns are removed - # by a later migration and ltree paths are populated from parent FKs. operations = [ - migrations.RunPython(migrations.RunPython.noop, migrations.RunPython.noop), + migrations.RunPython(code=rebuild_mptt, reverse_code=migrations.RunPython.noop), ] diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index 4552f4af166..83b9e70eb52 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -11,9 +11,8 @@ mutates paths directly; it only reads `path` back from the database after inserts and parent_id changes via refresh_from_db(fields=['path']). """ -from django.contrib.contenttypes.models import ContentType from django.db import migrations, models -from django.db.models import Count, ForeignKey, Lookup, ManyToManyField, OuterRef, Q, Subquery +from django.db.models import Count, ForeignKey, Lookup, ManyToManyField, Q from django.db.models.expressions import RawSQL from utilities.querysets import RestrictedQuerySet @@ -135,17 +134,24 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F count_attr: RawSQL(sql, [], output_field=models.IntegerField()) }) if has_generic_fk: - content_type = ContentType.objects.get_for_model(queryset.model) + # Resolve scope_type_id via subquery so this annotation can be + # constructed at import time (e.g. in a view class body) even + # before contenttypes has been migrated. + ct_app = queryset.model._meta.app_label + ct_model = queryset.model._meta.model_name sql = f'''( SELECT COUNT(DISTINCT "{related_table}"."id") FROM "{related_table}" INNER JOIN "{parent_table}" AS subtree ON "{related_table}"."scope_id" = subtree."id" - WHERE "{related_table}"."scope_type_id" = %s + WHERE "{related_table}"."scope_type_id" = ( + SELECT id FROM django_content_type + WHERE app_label = %s AND model = %s + ) AND subtree."path" <@ "{parent_table}"."path" )''' return queryset.annotate(**{ - count_attr: RawSQL(sql, [content_type.pk], output_field=models.IntegerField()) + count_attr: RawSQL(sql, [ct_app, ct_model], output_field=models.IntegerField()) }) rel_field_col = f'{rel_field}_id' sql = f'''( @@ -163,12 +169,19 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F if is_many_to_many: return queryset.annotate(**{count_attr: Count(rel_field, distinct=True)}) if has_generic_fk: - content_type = ContentType.objects.get_for_model(queryset.model) - subquery = model.objects.filter( - scope_type=content_type, scope_id=OuterRef('pk') - ).values('scope_id').annotate(c=Count('id')).values('c') + ct_app = queryset.model._meta.app_label + ct_model = queryset.model._meta.model_name + sql = f'''( + SELECT COUNT(DISTINCT "{related_table}"."id") + FROM "{related_table}" + WHERE "{related_table}"."scope_id" = "{parent_table}"."id" + AND "{related_table}"."scope_type_id" = ( + SELECT id FROM django_content_type + WHERE app_label = %s AND model = %s + ) + )''' return queryset.annotate(**{ - count_attr: Subquery(subquery, output_field=models.IntegerField()) + count_attr: RawSQL(sql, [ct_app, ct_model], output_field=models.IntegerField()) }) return queryset.annotate(**{count_attr: Count(rel_field, distinct=True)}) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 717f970b049..75e34a63579 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -454,6 +454,7 @@ def _setting(name, default=None): 'django_tables2', 'django_prometheus', 'strawberry_django', + 'mptt', 'rest_framework', 'social_django', 'sorl.thumbnail', diff --git a/netbox/tenancy/migrations/0001_squashed_0012.py b/netbox/tenancy/migrations/0001_squashed_0012.py index fa56356a355..f670003cf75 100644 --- a/netbox/tenancy/migrations/0001_squashed_0012.py +++ b/netbox/tenancy/migrations/0001_squashed_0012.py @@ -1,4 +1,5 @@ import django.db.models.deletion +import mptt.fields import taggit.managers from django.db import migrations, models @@ -44,7 +45,7 @@ class Migration(migrations.Migration): ('level', models.PositiveIntegerField(editable=False)), ( 'parent', - django.db.models.ForeignKey( + mptt.fields.TreeForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, diff --git a/netbox/tenancy/migrations/0002_squashed_0011.py b/netbox/tenancy/migrations/0002_squashed_0011.py index 5d21c03b7fe..cfdcb58dda7 100644 --- a/netbox/tenancy/migrations/0002_squashed_0011.py +++ b/netbox/tenancy/migrations/0002_squashed_0011.py @@ -1,4 +1,5 @@ import django.db.models.deletion +import mptt.fields import taggit.managers from django.db import migrations, models @@ -68,7 +69,7 @@ class Migration(migrations.Migration): ('level', models.PositiveIntegerField(editable=False)), ( 'parent', - django.db.models.ForeignKey( + mptt.fields.TreeForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, diff --git a/netbox/wireless/migrations/0001_squashed_0008.py b/netbox/wireless/migrations/0001_squashed_0008.py index 7a7e381e541..2ffc287f9e6 100644 --- a/netbox/wireless/migrations/0001_squashed_0008.py +++ b/netbox/wireless/migrations/0001_squashed_0008.py @@ -1,4 +1,5 @@ import django.db.models.deletion +import mptt.fields import taggit.managers from django.db import migrations, models @@ -45,7 +46,7 @@ class Migration(migrations.Migration): ('level', models.PositiveIntegerField(editable=False)), ( 'parent', - django.db.models.ForeignKey( + mptt.fields.TreeForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, diff --git a/requirements.txt b/requirements.txt index 1c38d94ae5d..4b4c0dbb827 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ django-debug-toolbar==6.3.0 django-filter==25.2 django-graphiql-debug-toolbar==0.2.0 django-htmx==1.27.0 +django-mptt==0.18.0 django-pglocks==1.0.4 django-prometheus==2.4.0 django-redis==6.0.0 From 89ff96760fde99376fb449474f518420bfe424f2 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 27 May 2026 08:54:45 -0700 Subject: [PATCH 05/42] #21488 - Replace MPTT wtih PostgreSQL Ltree --- netbox/dcim/migrations/0234_ltree_paths.py | 72 ++++++++++++++++++- netbox/tenancy/migrations/0025_ltree_paths.py | 22 +++++- .../wireless/migrations/0020_ltree_paths.py | 14 +++- 3 files changed, 105 insertions(+), 3 deletions(-) diff --git a/netbox/dcim/migrations/0234_ltree_paths.py b/netbox/dcim/migrations/0234_ltree_paths.py index 032a10e742f..f85601c33e5 100644 --- a/netbox/dcim/migrations/0234_ltree_paths.py +++ b/netbox/dcim/migrations/0234_ltree_paths.py @@ -13,9 +13,10 @@ 6. Drops the legacy MPTT columns (lft, rght, tree_id, level). 7. Adds a GiST index on the `path` column for efficient `@>` / `<@` lookups. """ +import django.db.models.deletion from django.contrib.postgres.indexes import GistIndex from django.contrib.postgres.operations import CreateExtension -from django.db import migrations +from django.db import migrations, models import netbox.models.ltree from netbox.models.ltree import InstallLtreeTriggers @@ -61,6 +62,75 @@ class Migration(migrations.Migration): ] operations = [ + # Switch parent from mptt.fields.TreeForeignKey to django.db.models.ForeignKey. + # This is a no-op at the SQL level (TreeForeignKey is a subclass of + # ForeignKey producing the same column) but reconciles the migration state + # with the model definitions now that django-mptt is no longer used at runtime. + migrations.AlterField( + model_name='devicerole', + name='parent', + field=models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='children', to='dcim.devicerole', + ), + ), + migrations.AlterField( + model_name='inventoryitem', + name='parent', + field=models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='child_items', to='dcim.inventoryitem', + ), + ), + migrations.AlterField( + model_name='inventoryitemtemplate', + name='parent', + field=models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='child_items', to='dcim.inventoryitemtemplate', + ), + ), + migrations.AlterField( + model_name='location', + name='parent', + field=models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='children', to='dcim.location', + ), + ), + migrations.AlterField( + model_name='modulebay', + name='parent', + field=models.ForeignKey( + blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='children', to='dcim.modulebay', + ), + ), + migrations.AlterField( + model_name='platform', + name='parent', + field=models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='children', to='dcim.platform', + ), + ), + migrations.AlterField( + model_name='region', + name='parent', + field=models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='children', to='dcim.region', + ), + ), + migrations.AlterField( + model_name='sitegroup', + name='parent', + field=models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='children', to='dcim.sitegroup', + ), + ), + # 1. Enable the ltree extension (idempotent — CreateExtension emits IF NOT EXISTS) CreateExtension('ltree'), diff --git a/netbox/tenancy/migrations/0025_ltree_paths.py b/netbox/tenancy/migrations/0025_ltree_paths.py index 8a1070a4d70..4e6b4087cd3 100644 --- a/netbox/tenancy/migrations/0025_ltree_paths.py +++ b/netbox/tenancy/migrations/0025_ltree_paths.py @@ -1,7 +1,8 @@ """Replace django-mptt with PostgreSQL ltree for tenancy's hierarchical models.""" +import django.db.models.deletion from django.contrib.postgres.indexes import GistIndex from django.contrib.postgres.operations import CreateExtension -from django.db import migrations +from django.db import migrations, models import netbox.models.ltree from netbox.models.ltree import InstallLtreeTriggers @@ -33,6 +34,25 @@ class Migration(migrations.Migration): ] operations = [ + # Switch parent from mptt.fields.TreeForeignKey to django.db.models.ForeignKey. + # No-op at the SQL level; reconciles migration state with model definitions. + migrations.AlterField( + model_name='contactgroup', + name='parent', + field=models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='children', to='tenancy.contactgroup', + ), + ), + migrations.AlterField( + model_name='tenantgroup', + name='parent', + field=models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='children', to='tenancy.tenantgroup', + ), + ), + # Enable the ltree extension (idempotent — CreateExtension emits IF NOT EXISTS) CreateExtension('ltree'), diff --git a/netbox/wireless/migrations/0020_ltree_paths.py b/netbox/wireless/migrations/0020_ltree_paths.py index b9b75054d48..ea3f661c085 100644 --- a/netbox/wireless/migrations/0020_ltree_paths.py +++ b/netbox/wireless/migrations/0020_ltree_paths.py @@ -1,7 +1,8 @@ """Replace django-mptt with PostgreSQL ltree for wireless's hierarchical models.""" +import django.db.models.deletion from django.contrib.postgres.indexes import GistIndex from django.contrib.postgres.operations import CreateExtension -from django.db import migrations +from django.db import migrations, models import netbox.models.ltree from netbox.models.ltree import InstallLtreeTriggers @@ -18,6 +19,17 @@ class Migration(migrations.Migration): ] operations = [ + # Switch parent from mptt.fields.TreeForeignKey to django.db.models.ForeignKey. + # No-op at the SQL level; reconciles migration state with model definitions. + migrations.AlterField( + model_name='wirelesslangroup', + name='parent', + field=models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='children', to='wireless.wirelesslangroup', + ), + ), + # Enable the ltree extension (idempotent — CreateExtension emits IF NOT EXISTS) CreateExtension('ltree'), From 099e6ce3b6a2ca0c8436726d345efab60bacd1c5 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 27 May 2026 11:45:32 -0700 Subject: [PATCH 06/42] #21488 - Replace MPTT wtih PostgreSQL Ltree --- netbox/dcim/graphql/types.py | 12 +- netbox/dcim/migrations/0234_ltree_paths.py | 228 ++++++++++++------ .../dcim/models/device_component_templates.py | 5 +- netbox/dcim/models/device_components.py | 7 + netbox/dcim/models/devices.py | 6 +- netbox/dcim/models/sites.py | 7 +- netbox/netbox/api/viewsets/__init__.py | 2 + netbox/netbox/models/__init__.py | 12 +- netbox/netbox/models/ltree.py | 90 +++++-- netbox/tenancy/graphql/types.py | 4 +- netbox/tenancy/migrations/0025_ltree_paths.py | 64 +++-- netbox/tenancy/models/contacts.py | 3 +- netbox/tenancy/models/tenants.py | 10 +- netbox/utilities/query.py | 4 +- netbox/utilities/testing/filtersets.py | 3 +- netbox/utilities/tests/test_ltree.py | 61 ++++- netbox/wireless/graphql/types.py | 2 +- .../wireless/migrations/0020_ltree_paths.py | 45 ++-- netbox/wireless/models.py | 10 +- 19 files changed, 415 insertions(+), 160 deletions(-) diff --git a/netbox/dcim/graphql/types.py b/netbox/dcim/graphql/types.py index ddfa24fa50e..3ceb7f1d90c 100644 --- a/netbox/dcim/graphql/types.py +++ b/netbox/dcim/graphql/types.py @@ -347,7 +347,7 @@ def parent(self) -> Annotated['InventoryItemTemplateType', strawberry.lazy('dcim @strawberry_django.type( models.DeviceRole, - exclude=['path'], + exclude=['path', 'sort_path'], filters=DeviceRoleFilter, pagination=True ) @@ -526,7 +526,7 @@ class InventoryItemRoleType(OrganizationalObjectType): @strawberry_django.type( models.Location, # fields='__all__', - exclude=['parent', 'path'], # bug - temp + exclude=['parent', 'path', 'sort_path'], # bug - temp filters=LocationFilter, pagination=True ) @@ -590,7 +590,7 @@ class ModuleType(PrimaryObjectType): @strawberry_django.type( models.ModuleBay, # fields='__all__', - exclude=['parent', 'path'], + exclude=['parent', 'path', 'sort_path'], filters=ModuleBayFilter, pagination=True ) @@ -647,7 +647,7 @@ class ModuleTypeType(PrimaryObjectType): @strawberry_django.type( models.Platform, - exclude=['path'], + exclude=['path', 'sort_path'], filters=PlatformFilter, pagination=True ) @@ -855,7 +855,7 @@ class RearPortTemplateType(ModularComponentTemplateType): @strawberry_django.type( models.Region, - exclude=['parent', 'path'], + exclude=['parent', 'path', 'sort_path'], filters=RegionFilter, pagination=True ) @@ -916,7 +916,7 @@ def circuit_terminations(self) -> list[ @strawberry_django.type( models.SiteGroup, - exclude=['parent', 'path'], # bug - temp + exclude=['parent', 'path', 'sort_path'], # bug - temp filters=SiteGroupFilter, pagination=True ) diff --git a/netbox/dcim/migrations/0234_ltree_paths.py b/netbox/dcim/migrations/0234_ltree_paths.py index f85601c33e5..b714cfbf2cb 100644 --- a/netbox/dcim/migrations/0234_ltree_paths.py +++ b/netbox/dcim/migrations/0234_ltree_paths.py @@ -5,13 +5,17 @@ InventoryItem, InventoryItemTemplate) this migration: 1. Enables the PostgreSQL ltree extension (idempotent). -2. Adds a nullable `path` LTreeField. -3. Installs per-table BEFORE-INSERT/UPDATE-OF-parent_id and AFTER-UPDATE-OF-(parent_id, path) - triggers so concurrent writes during the long-running data step get correct paths. -4. Populates paths for existing rows via a single recursive CTE per table. -5. Tightens `path` to NOT NULL. +2. Adds a nullable `path` LTreeField. For models that previously had + `MPTTMeta.order_insertion_by = ('name',)` — Region, SiteGroup, Location, + DeviceRole, Platform, ModuleBay — also adds a `sort_path` text column. +3. Installs per-table BEFORE/AFTER triggers. For models with sort_path, the + trigger maintains both columns. +4. Populates path (and sort_path where applicable) for existing rows via a + single recursive CTE per table. +5. Tightens path to NOT NULL. 6. Drops the legacy MPTT columns (lft, rght, tree_id, level). -7. Adds a GiST index on the `path` column for efficient `@>` / `<@` lookups. +7. Adds a GiST index on path (descendant/ancestor lookups via `<@` / `@>`). + For sort_path models, also adds a btree index for ORDER BY listing. """ import django.db.models.deletion from django.contrib.postgres.indexes import GistIndex @@ -21,12 +25,13 @@ import netbox.models.ltree from netbox.models.ltree import InstallLtreeTriggers -MODELS = ( +# All models getting an ltree `path` column. +ALL_MODELS = ( 'region', 'sitegroup', 'location', 'devicerole', 'platform', 'inventoryitem', 'inventoryitemtemplate', 'modulebay', ) -TABLES = ( +ALL_TABLES = ( 'dcim_region', 'dcim_sitegroup', 'dcim_location', @@ -37,17 +42,52 @@ 'dcim_modulebay', ) +# Subset that previously declared `MPTTMeta.order_insertion_by = ('name',)` and +# therefore needs a `sort_path` text column maintained alongside `path`. +SORT_MODELS = ('region', 'sitegroup', 'location', 'devicerole', 'platform', 'modulebay') + +SORT_TABLES = ( + 'dcim_region', + 'dcim_sitegroup', + 'dcim_location', + 'dcim_devicerole', + 'dcim_platform', + 'dcim_modulebay', +) + LEGACY_FIELDS = ('lft', 'rght', 'tree_id', 'level') def _populate_paths_sql(): + """ + Build the recursive CTE that walks each table from roots downward, computing + the new path (PK-based, zero-padded) and — for models with sort_path — the + chr(1)-separated chain of ancestor names. + """ blocks = [] - for table in TABLES: - blocks.append(f""" + for table in ALL_TABLES: + if table in SORT_TABLES: + blocks.append(f""" +WITH RECURSIVE t(id, parent_id, path, sort_path) AS ( + SELECT id, parent_id, + lpad(id::text, 19, '0')::ltree, + name::text + FROM "{table}" WHERE parent_id IS NULL + UNION ALL + SELECT r.id, r.parent_id, + t.path || lpad(r.id::text, 19, '0')::ltree, + t.sort_path || chr(1) || r.name + FROM "{table}" r JOIN t ON r.parent_id = t.id +) +UPDATE "{table}" SET path = t.path, sort_path = t.sort_path +FROM t WHERE "{table}".id = t.id; +""") + else: + blocks.append(f""" WITH RECURSIVE t(id, parent_id, path) AS ( - SELECT id, parent_id, id::text::ltree FROM "{table}" WHERE parent_id IS NULL + SELECT id, parent_id, lpad(id::text, 19, '0')::ltree FROM "{table}" WHERE parent_id IS NULL UNION ALL - SELECT r.id, r.parent_id, t.path || r.id::text::ltree + SELECT r.id, r.parent_id, t.path || lpad(r.id::text, 19, '0')::ltree FROM "{table}" r JOIN t ON r.parent_id = t.id ) UPDATE "{table}" SET path = t.path FROM t WHERE "{table}".id = t.id; @@ -62,105 +102,109 @@ class Migration(migrations.Migration): ] operations = [ - # Switch parent from mptt.fields.TreeForeignKey to django.db.models.ForeignKey. - # This is a no-op at the SQL level (TreeForeignKey is a subclass of - # ForeignKey producing the same column) but reconciles the migration state - # with the model definitions now that django-mptt is no longer used at runtime. + # Switch parent from mptt.fields.TreeForeignKey to django.db.models.ForeignKey + # (no-op at the SQL level; reconciles migration state with model definitions). migrations.AlterField( - model_name='devicerole', - name='parent', - field=models.ForeignKey( - blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, - related_name='children', to='dcim.devicerole', - ), + model_name='devicerole', name='parent', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='children', to='dcim.devicerole'), ), migrations.AlterField( - model_name='inventoryitem', - name='parent', - field=models.ForeignKey( - blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, - related_name='child_items', to='dcim.inventoryitem', - ), + model_name='inventoryitem', name='parent', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='child_items', to='dcim.inventoryitem'), ), migrations.AlterField( - model_name='inventoryitemtemplate', - name='parent', - field=models.ForeignKey( - blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, - related_name='child_items', to='dcim.inventoryitemtemplate', - ), + model_name='inventoryitemtemplate', name='parent', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='child_items', to='dcim.inventoryitemtemplate'), ), migrations.AlterField( - model_name='location', - name='parent', - field=models.ForeignKey( - blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, - related_name='children', to='dcim.location', - ), + model_name='location', name='parent', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='children', to='dcim.location'), ), migrations.AlterField( - model_name='modulebay', - name='parent', - field=models.ForeignKey( - blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, - related_name='children', to='dcim.modulebay', - ), + model_name='modulebay', name='parent', + field=models.ForeignKey(blank=True, editable=False, null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name='children', to='dcim.modulebay'), ), migrations.AlterField( - model_name='platform', - name='parent', - field=models.ForeignKey( - blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, - related_name='children', to='dcim.platform', - ), + model_name='platform', name='parent', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='children', to='dcim.platform'), ), migrations.AlterField( - model_name='region', - name='parent', - field=models.ForeignKey( - blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, - related_name='children', to='dcim.region', - ), + model_name='region', name='parent', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='children', to='dcim.region'), ), migrations.AlterField( - model_name='sitegroup', - name='parent', - field=models.ForeignKey( - blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, - related_name='children', to='dcim.sitegroup', - ), + model_name='sitegroup', name='parent', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + related_name='children', to='dcim.sitegroup'), ), - # 1. Enable the ltree extension (idempotent — CreateExtension emits IF NOT EXISTS) + # 1. Enable the ltree extension (idempotent). CreateExtension('ltree'), - # 2. Add nullable path column + # 2. Add nullable path column on all tree models. *[ migrations.AddField( - model_name=m, - name='path', + model_name=m, name='path', field=netbox.models.ltree.LtreeField(blank=True, editable=False, null=True), ) - for m in MODELS + for m in ALL_MODELS + ], + # 2b. Add sort_path column (with default '') on the 6 models with order_insertion_by. + *[ + migrations.AddField( + model_name=m, name='sort_path', + field=models.TextField(blank=True, default='', editable=False), + ) + for m in SORT_MODELS ], - # 2. Install path-maintenance triggers - *[InstallLtreeTriggers(t) for t in TABLES], + # 3. Install path-maintenance triggers. Models with sort_path get triggers + # that maintain both columns; the other two get path-only triggers. + *[InstallLtreeTriggers(t, name_column='name') for t in SORT_TABLES], + InstallLtreeTriggers('dcim_inventoryitem'), + InstallLtreeTriggers('dcim_inventoryitemtemplate'), - # 3. Populate existing rows + # 4. Populate existing rows via per-table recursive CTE. migrations.RunSQL(_populate_paths_sql(), reverse_sql=migrations.RunSQL.noop), - # 4. Tighten to NOT NULL with empty-string default + # 5. Tighten path to NOT NULL with empty-string default. *[ migrations.AlterField( - model_name=m, - name='path', + model_name=m, name='path', field=netbox.models.ltree.LtreeField(blank=True, default='', editable=False), ) - for m in MODELS + for m in ALL_MODELS ], - # 5. Drop legacy (tree_id, lft) indexes added in 0226_add_mptt_tree_indexes, + # 6. Update Meta.ordering on the SORT_MODELS to reflect sort_path-based ordering. + migrations.AlterModelOptions( + name='devicerole', options={'ordering': ('sort_path',)}, + ), + migrations.AlterModelOptions( + name='location', options={'ordering': ('site', 'sort_path')}, + ), + migrations.AlterModelOptions( + name='modulebay', options={'ordering': ('device', 'sort_path')}, + ), + migrations.AlterModelOptions( + name='platform', options={'ordering': ('sort_path',)}, + ), + migrations.AlterModelOptions( + name='region', options={'ordering': ('sort_path',)}, + ), + migrations.AlterModelOptions( + name='sitegroup', options={'ordering': ('sort_path',)}, + ), + + # 7. Drop legacy (tree_id, lft) indexes added in 0226_add_mptt_tree_indexes, # then drop the legacy MPTT columns. migrations.RemoveIndex(model_name='devicerole', name='dcim_devicerole_tree_id_lfbf11'), migrations.RemoveIndex(model_name='inventoryitem', name='dcim_inventoryitem_tree_id975c'), @@ -172,10 +216,10 @@ class Migration(migrations.Migration): migrations.RemoveIndex(model_name='sitegroup', name='dcim_sitegroup_tree_id_lft_idx'), *[ migrations.RemoveField(model_name=m, name=f) - for m in MODELS for f in LEGACY_FIELDS + for m in ALL_MODELS for f in LEGACY_FIELDS ], - # 6. Add GiST indexes on path + # 8. Add GiST indexes on path (descendant/ancestor containment). migrations.AddIndex( model_name='region', index=GistIndex(fields=['path'], name='dcim_region_path_gist'), @@ -208,4 +252,30 @@ class Migration(migrations.Migration): model_name='modulebay', index=GistIndex(fields=['path'], name='dcim_modulebay_path_gist'), ), + + # 9. Add btree indexes on sort_path (tree-flatten ORDER BY listing). + migrations.AddIndex( + model_name='region', + index=models.Index(fields=['sort_path'], name='dcim_region_sort_path_idx'), + ), + migrations.AddIndex( + model_name='sitegroup', + index=models.Index(fields=['sort_path'], name='dcim_sitegroup_sort_path_idx'), + ), + migrations.AddIndex( + model_name='location', + index=models.Index(fields=['sort_path'], name='dcim_location_sort_path_idx'), + ), + migrations.AddIndex( + model_name='devicerole', + index=models.Index(fields=['sort_path'], name='dcim_devicerole_sort_path_idx'), + ), + migrations.AddIndex( + model_name='platform', + index=models.Index(fields=['sort_path'], name='dcim_platform_sort_path_idx'), + ), + migrations.AddIndex( + model_name='modulebay', + index=models.Index(fields=['sort_path'], name='dcim_modulebay_sort_path_idx'), + ), ] diff --git a/netbox/dcim/models/device_component_templates.py b/netbox/dcim/models/device_component_templates.py index 47dd0d4e4a8..4b05a8294a0 100644 --- a/netbox/dcim/models/device_component_templates.py +++ b/netbox/dcim/models/device_component_templates.py @@ -1,8 +1,8 @@ from django.contrib.contenttypes.fields import GenericForeignKey -from django.contrib.postgres.indexes import GistIndex from django.core.exceptions import ValidationError from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models +from django.contrib.postgres.indexes import GistIndex from django.utils.translation import gettext_lazy as _ from dcim.choices import * @@ -860,9 +860,8 @@ class InventoryItemTemplate(LtreeModel, ComponentTemplateModel): help_text=_('Manufacturer-assigned part identifier') ) - component_model = InventoryItem - objects = LtreeManager() + component_model = InventoryItem class Meta: ordering = ('device_type__id', 'parent__id', 'name') diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index 8186f756655..5a05eb13d0c 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -1336,14 +1336,21 @@ class ModuleBay(ModularComponentModel, TrackingModelMixin, LtreeModel): verbose_name=_('enabled'), default=True, ) + sort_path = models.TextField( + editable=False, + blank=True, + default='', + ) clone_fields = ('device', 'enabled') objects = LtreeManager() class Meta(ModularComponentModel.Meta): + ordering = ('device', 'sort_path') indexes = ( GistIndex(fields=['path'], name='dcim_modulebay_path_gist'), + models.Index(fields=['sort_path'], name='dcim_modulebay_sort_path_idx'), ) constraints = ( models.UniqueConstraint( diff --git a/netbox/dcim/models/devices.py b/netbox/dcim/models/devices.py index 936e7691594..70de357e20c 100644 --- a/netbox/dcim/models/devices.py +++ b/netbox/dcim/models/devices.py @@ -412,9 +412,10 @@ class DeviceRole(NestedGroupModel): clone_fields = ('parent', 'description') class Meta: - ordering = ('name',) + ordering = ('sort_path',) indexes = ( GistIndex(fields=['path'], name='dcim_devicerole_path_gist'), + models.Index(fields=['sort_path'], name='dcim_devicerole_sort_path_idx'), ) constraints = ( models.UniqueConstraint( @@ -466,9 +467,10 @@ class Platform(NestedGroupModel): clone_fields = ('parent', 'description') class Meta: - ordering = ('name',) + ordering = ('sort_path',) indexes = ( GistIndex(fields=['path'], name='dcim_platform_path_gist'), + models.Index(fields=['sort_path'], name='dcim_platform_sort_path_idx'), ) verbose_name = _('platform') verbose_name_plural = _('platforms') diff --git a/netbox/dcim/models/sites.py b/netbox/dcim/models/sites.py index 02d9c62104b..70796b5154c 100644 --- a/netbox/dcim/models/sites.py +++ b/netbox/dcim/models/sites.py @@ -45,8 +45,10 @@ class Region(ContactsMixin, NestedGroupModel): ) class Meta: + ordering = ('sort_path',) indexes = ( GistIndex(fields=['path'], name='dcim_region_path_gist'), + models.Index(fields=['sort_path'], name='dcim_region_sort_path_idx'), ) constraints = ( models.UniqueConstraint( @@ -104,8 +106,10 @@ class SiteGroup(ContactsMixin, NestedGroupModel): ) class Meta: + ordering = ('sort_path',) indexes = ( GistIndex(fields=['path'], name='dcim_sitegroup_path_gist'), + models.Index(fields=['sort_path'], name='dcim_sitegroup_sort_path_idx'), ) constraints = ( models.UniqueConstraint( @@ -324,9 +328,10 @@ class Location(ContactsMixin, ImageAttachmentsMixin, NestedGroupModel): ) class Meta: - ordering = ['site', 'name'] + ordering = ('site', 'sort_path') indexes = ( GistIndex(fields=['path'], name='dcim_location_path_gist'), + models.Index(fields=['sort_path'], name='dcim_location_sort_path_idx'), ) constraints = ( models.UniqueConstraint( diff --git a/netbox/netbox/api/viewsets/__init__.py b/netbox/netbox/api/viewsets/__init__.py index 49d12dd76ce..d6ce93e2f4f 100644 --- a/netbox/netbox/api/viewsets/__init__.py +++ b/netbox/netbox/api/viewsets/__init__.py @@ -333,3 +333,5 @@ def perform_destroy(self, instance): super().perform_destroy(instance) except ObjectDoesNotExist: raise PermissionDenied() + + diff --git a/netbox/netbox/models/__init__.py b/netbox/netbox/models/__init__.py index ab5befce256..7099ffbea99 100644 --- a/netbox/netbox/models/__init__.py +++ b/netbox/netbox/models/__init__.py @@ -162,6 +162,11 @@ class NestedGroupModel(OwnerMixin, NetBoxModel, LtreeModel): """ Base model for objects which are used to form a hierarchy (regions, locations, etc.). These models nest recursively using PostgreSQL ltree. Within each parent, each child instance must have a unique name. + + `sort_path` is a trigger-maintained text column that mirrors MPTT's `order_insertion_by=('name',)` + semantics: at insert and reparent time it is set to a chr(1)-separated chain of ancestor names. + Ordering by `sort_path` yields tree-flatten output with siblings in their column's collation order. + Renaming a node does NOT update `sort_path` (matching MPTT behavior). """ parent = models.ForeignKey( to='self', @@ -188,6 +193,11 @@ class NestedGroupModel(OwnerMixin, NetBoxModel, LtreeModel): verbose_name=_('comments'), blank=True ) + sort_path = models.TextField( + editable=False, + blank=True, + default='', + ) # Re-declare so the LtreeManager wins over BaseModel's RestrictedQuerySet # default manager via MRO resolution. @@ -195,7 +205,7 @@ class NestedGroupModel(OwnerMixin, NetBoxModel, LtreeModel): class Meta: abstract = True - ordering = ('path',) + ordering = ('sort_path',) def __str__(self): return self.name diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index 83b9e70eb52..1649fbf7a80 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -189,9 +189,6 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F class LtreeManager(models.Manager.from_queryset(LtreeQuerySet)): """Drop-in replacement for django-mptt's TreeManager.""" - def get_queryset(self): - return super().get_queryset().order_by('path') - # # Abstract model @@ -252,7 +249,8 @@ def tree_id(self): """Integer PK of the root, mirroring django-mptt's `tree_id`.""" if not self.path: return None - root_label = str(self.path).split('.', 1)[0] + # Strip leading zeros from the padded label + root_label = str(self.path).split('.', 1)[0].lstrip('0') or '0' try: return int(root_label) except (TypeError, ValueError): @@ -270,7 +268,7 @@ def is_child_node(self): def get_root(self): if self.is_root_node(): return self - root_pk = int(str(self.path).split('.', 1)[0]) + root_pk = int(str(self.path).split('.', 1)[0].lstrip('0') or '0') return type(self)._default_manager.get(pk=root_pk) def get_parent(self): @@ -348,23 +346,26 @@ def insert_at(self, target, position='last-child', save=False): # Migration operation # -_COMPUTE_PATH_FN = ''' +# Path label is the row's PK zero-padded to 19 chars (max bigint width) so that +# lexicographic ordering of ltree labels matches numeric PK ordering across digit +# boundaries (e.g. "0...09" sorts before "0...10"). +_COMPUTE_PATH_ONLY_FN = ''' CREATE OR REPLACE FUNCTION "{table}_ltree_compute_path_fn"() RETURNS TRIGGER AS $$ DECLARE parent_path ltree; BEGIN IF NEW.parent_id IS NOT NULL THEN EXECUTE format('SELECT path FROM %%I WHERE id = $1', TG_TABLE_NAME) INTO parent_path USING NEW.parent_id; - NEW.path := parent_path || NEW.id::text::ltree; + NEW.path := parent_path || lpad(NEW.id::text, 19, '0')::ltree; ELSE - NEW.path := NEW.id::text::ltree; + NEW.path := lpad(NEW.id::text, 19, '0')::ltree; END IF; RETURN NEW; END $$ LANGUAGE plpgsql; ''' -_CASCADE_PATH_FN = ''' +_CASCADE_PATH_ONLY_FN = ''' CREATE OR REPLACE FUNCTION "{table}_ltree_cascade_path_fn"() RETURNS TRIGGER AS $$ BEGIN EXECUTE format( @@ -377,6 +378,50 @@ def insert_at(self, target, position='last-child', save=False): $$ LANGUAGE plpgsql; ''' +# For models with order_insertion_by=(name,) — maintain a second text column +# `sort_path` whose value is the chain of ancestor names joined by chr(1) +# (an unprintable separator that collates lower than any printable char in any +# standard collation). ORDER BY sort_path then gives MPTT-equivalent +# tree-flatten ordering with siblings in name (collation) order. +# +# Like MPTT's order_insertion_by, sort_path is computed at insert and +# reparent only — renaming a node does NOT reposition it. A manual rebuild() +# would be needed to re-sort everything by current names. +_COMPUTE_PATH_AND_SORT_FN = ''' +CREATE OR REPLACE FUNCTION "{table}_ltree_compute_path_fn"() RETURNS TRIGGER AS $$ +DECLARE + parent_path ltree; + parent_sort_path text; +BEGIN + IF NEW.parent_id IS NOT NULL THEN + EXECUTE format('SELECT path, sort_path FROM %%I WHERE id = $1', TG_TABLE_NAME) + INTO parent_path, parent_sort_path USING NEW.parent_id; + NEW.path := parent_path || lpad(NEW.id::text, 19, '0')::ltree; + NEW.sort_path := parent_sort_path || chr(1) || NEW.{name_col}; + ELSE + NEW.path := lpad(NEW.id::text, 19, '0')::ltree; + NEW.sort_path := NEW.{name_col}; + END IF; + RETURN NEW; +END +$$ LANGUAGE plpgsql; +''' + +_CASCADE_PATH_AND_SORT_FN = ''' +CREATE OR REPLACE FUNCTION "{table}_ltree_cascade_path_fn"() RETURNS TRIGGER AS $$ +BEGIN + EXECUTE format( + 'UPDATE %%I SET ' + ' path = $1 || subpath(path, nlevel($2)), ' + ' sort_path = $4 || substring(sort_path FROM length($5) + 1) ' + 'WHERE path <@ $2 AND id != $3', + TG_TABLE_NAME + ) USING NEW.path, OLD.path, NEW.id, NEW.sort_path, OLD.sort_path; + RETURN NULL; +END +$$ LANGUAGE plpgsql; +''' + _BEFORE_TRIGGER = ''' CREATE TRIGGER "{table}_ltree_compute_path" BEFORE INSERT OR UPDATE OF parent_id ON "{table}" @@ -397,24 +442,35 @@ class InstallLtreeTriggers(migrations.operations.base.Operation): Two row-level triggers are installed on each target table: - BEFORE INSERT OR UPDATE OF parent_id -> compute NEW.path - AFTER UPDATE OF path WHEN distinct -> cascade path change to descendants + BEFORE INSERT OR UPDATE OF parent_id -> compute NEW.path (and sort_path if applicable) + AFTER UPDATE OF parent_id, path -> cascade path/sort_path change to descendants - The trigger function bodies use TG_TABLE_NAME so the SQL is table-agnostic, - but each table gets its own pair of CREATE FUNCTION statements to keep - pg_proc entries identifiable and to avoid surprising cross-table coupling. + If `name_column` is provided, the model is expected to have a `sort_path` + text column whose value will be maintained as a chr(1)-separated chain of + ancestor names. This implements MPTT's `order_insertion_by=(name,)` + semantics: insert and reparent honor the current value of `name_column`; + rename does NOT reposition the node (matching MPTT behavior). """ reversible = True - def __init__(self, table_name): + def __init__(self, table_name, name_column=None): self.table_name = table_name + self.name_column = name_column def state_forwards(self, app_label, state): pass def database_forwards(self, app_label, schema_editor, from_state, to_state): - schema_editor.execute(_COMPUTE_PATH_FN.format(table=self.table_name)) - schema_editor.execute(_CASCADE_PATH_FN.format(table=self.table_name)) + if self.name_column: + schema_editor.execute(_COMPUTE_PATH_AND_SORT_FN.format( + table=self.table_name, name_col=self.name_column, + )) + schema_editor.execute(_CASCADE_PATH_AND_SORT_FN.format( + table=self.table_name, + )) + else: + schema_editor.execute(_COMPUTE_PATH_ONLY_FN.format(table=self.table_name)) + schema_editor.execute(_CASCADE_PATH_ONLY_FN.format(table=self.table_name)) schema_editor.execute(_BEFORE_TRIGGER.format(table=self.table_name)) schema_editor.execute(_AFTER_TRIGGER.format(table=self.table_name)) diff --git a/netbox/tenancy/graphql/types.py b/netbox/tenancy/graphql/types.py index 5e80a1ebf41..15629bfdf32 100644 --- a/netbox/tenancy/graphql/types.py +++ b/netbox/tenancy/graphql/types.py @@ -88,7 +88,7 @@ class TenantType(ContactsMixin, PrimaryObjectType): @strawberry_django.type( models.TenantGroup, - exclude=['path'], + exclude=['path', 'sort_path'], filters=TenantGroupFilter, pagination=True ) @@ -125,7 +125,7 @@ class ContactRoleType(ContactAssignmentsMixin, OrganizationalObjectType): @strawberry_django.type( models.ContactGroup, - exclude=['path'], + exclude=['path', 'sort_path'], filters=ContactGroupFilter, pagination=True ) diff --git a/netbox/tenancy/migrations/0025_ltree_paths.py b/netbox/tenancy/migrations/0025_ltree_paths.py index 4e6b4087cd3..c18b75cf56f 100644 --- a/netbox/tenancy/migrations/0025_ltree_paths.py +++ b/netbox/tenancy/migrations/0025_ltree_paths.py @@ -16,13 +16,19 @@ def _populate_paths_sql(): blocks = [] for table in TABLES: blocks.append(f""" -WITH RECURSIVE t(id, parent_id, path) AS ( - SELECT id, parent_id, id::text::ltree FROM "{table}" WHERE parent_id IS NULL +WITH RECURSIVE t(id, parent_id, path, sort_path) AS ( + SELECT id, parent_id, + lpad(id::text, 19, '0')::ltree, + name::text + FROM "{table}" WHERE parent_id IS NULL UNION ALL - SELECT r.id, r.parent_id, t.path || r.id::text::ltree + SELECT r.id, r.parent_id, + t.path || lpad(r.id::text, 19, '0')::ltree, + t.sort_path || chr(1) || r.name FROM "{table}" r JOIN t ON r.parent_id = t.id ) -UPDATE "{table}" SET path = t.path FROM t WHERE "{table}".id = t.id; +UPDATE "{table}" SET path = t.path, sort_path = t.sort_path +FROM t WHERE "{table}".id = t.id; """) return '\n'.join(blocks) @@ -35,51 +41,64 @@ class Migration(migrations.Migration): operations = [ # Switch parent from mptt.fields.TreeForeignKey to django.db.models.ForeignKey. - # No-op at the SQL level; reconciles migration state with model definitions. migrations.AlterField( - model_name='contactgroup', - name='parent', + model_name='contactgroup', name='parent', field=models.ForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='tenancy.contactgroup', ), ), migrations.AlterField( - model_name='tenantgroup', - name='parent', + model_name='tenantgroup', name='parent', field=models.ForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='tenancy.tenantgroup', ), ), - # Enable the ltree extension (idempotent — CreateExtension emits IF NOT EXISTS) CreateExtension('ltree'), + # Add path (nullable initially) on both models. *[ migrations.AddField( - model_name=m, - name='path', + model_name=m, name='path', field=netbox.models.ltree.LtreeField(blank=True, editable=False, null=True), ) for m in MODELS ], + # Add sort_path. TenantGroup gets natural_sort collation (matching its name field). + migrations.AddField( + model_name='contactgroup', name='sort_path', + field=models.TextField(blank=True, default='', editable=False), + ), + migrations.AddField( + model_name='tenantgroup', name='sort_path', + field=models.TextField( + blank=True, default='', editable=False, db_collation='natural_sort', + ), + ), - *[InstallLtreeTriggers(t) for t in TABLES], + # Install triggers maintaining both path and sort_path. + *[InstallLtreeTriggers(t, name_column='name') for t in TABLES], migrations.RunSQL(_populate_paths_sql(), reverse_sql=migrations.RunSQL.noop), *[ migrations.AlterField( - model_name=m, - name='path', + model_name=m, name='path', field=netbox.models.ltree.LtreeField(blank=True, default='', editable=False), ) for m in MODELS ], - # Drop legacy (tree_id, lft) indexes added in 0023_add_mptt_tree_indexes, - # then drop the legacy MPTT columns. + migrations.AlterModelOptions( + name='contactgroup', options={'ordering': ('sort_path',)}, + ), + migrations.AlterModelOptions( + name='tenantgroup', options={'ordering': ('sort_path',)}, + ), + + # Drop legacy (tree_id, lft) indexes and the MPTT columns. migrations.RemoveIndex(model_name='contactgroup', name='tenancy_contactgroup_tree_d2ce'), migrations.RemoveIndex(model_name='tenantgroup', name='tenancy_tenantgroup_tree_ifebc'), *[ @@ -87,6 +106,7 @@ class Migration(migrations.Migration): for m in MODELS for f in LEGACY_FIELDS ], + # GiST indexes on path. migrations.AddIndex( model_name='tenantgroup', index=GistIndex(fields=['path'], name='tenancy_tenantgroup_path_gist'), @@ -95,4 +115,14 @@ class Migration(migrations.Migration): model_name='contactgroup', index=GistIndex(fields=['path'], name='tenancy_contactgroup_path_gist'), ), + + # Btree indexes on sort_path for ORDER BY listing. + migrations.AddIndex( + model_name='tenantgroup', + index=models.Index(fields=['sort_path'], name='tenancy_tg_sort_path_idx'), + ), + migrations.AddIndex( + model_name='contactgroup', + index=models.Index(fields=['sort_path'], name='tenancy_cg_sort_path_idx'), + ), ] diff --git a/netbox/tenancy/models/contacts.py b/netbox/tenancy/models/contacts.py index 039f3b77cd8..56452df3e4f 100644 --- a/netbox/tenancy/models/contacts.py +++ b/netbox/tenancy/models/contacts.py @@ -47,9 +47,10 @@ class ContactGroup(NestedGroupModel): objects = ContactGroupManager() class Meta: - ordering = ['name'] + ordering = ('sort_path',) indexes = ( GistIndex(fields=['path'], name='tenancy_contactgroup_path_gist'), + models.Index(fields=['sort_path'], name='tenancy_cg_sort_path_idx'), ) constraints = ( models.UniqueConstraint( diff --git a/netbox/tenancy/models/tenants.py b/netbox/tenancy/models/tenants.py index a9d4a2bce9f..6ea6769a637 100644 --- a/netbox/tenancy/models/tenants.py +++ b/netbox/tenancy/models/tenants.py @@ -27,11 +27,19 @@ class TenantGroup(NestedGroupModel): max_length=100, unique=True ) + # Override the abstract parent's sort_path to use natural_sort, matching `name`. + sort_path = models.TextField( + editable=False, + blank=True, + default='', + db_collation='natural_sort', + ) class Meta: - ordering = ['name'] + ordering = ('sort_path',) indexes = ( GistIndex(fields=['path'], name='tenancy_tenantgroup_path_gist'), + models.Index(fields=['sort_path'], name='tenancy_tg_sort_path_idx'), ) verbose_name = _('tenant group') verbose_name_plural = _('tenant groups') diff --git a/netbox/utilities/query.py b/netbox/utilities/query.py index 50c357fffb6..662721226a5 100644 --- a/netbox/utilities/query.py +++ b/netbox/utilities/query.py @@ -64,8 +64,8 @@ def reapply_model_ordering(queryset: QuerySet) -> QuerySet: Reapply model-level ordering in case it has been lost through .annotate(). https://code.djangoproject.com/ticket/32811 """ - # Hierarchical (ltree) models are exempt; their default ordering by `path` must not be - # clobbered by .annotate(). Use caution when annotating querysets of these models. + # Hierarchical (ltree) models are exempt; their default ordering by `sort_path`/`path` + # must not be clobbered by .annotate(). Use caution when annotating these querysets. if any(isinstance(manager, LtreeManager) for manager in queryset.model._meta.local_managers): return queryset if queryset.ordered: diff --git a/netbox/utilities/testing/filtersets.py b/netbox/utilities/testing/filtersets.py index 084bc341b83..448d7ccd7b2 100644 --- a/netbox/utilities/testing/filtersets.py +++ b/netbox/utilities/testing/filtersets.py @@ -20,7 +20,8 @@ EXEMPT_MODEL_FIELDS = ( 'comments', 'custom_field_data', - 'path', # ltree, trigger-maintained + 'path', # ltree, trigger-maintained + 'sort_path', # ltree, trigger-maintained ) diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py index 8d81fb56ec0..a1a699e6528 100644 --- a/netbox/utilities/tests/test_ltree.py +++ b/netbox/utilities/tests/test_ltree.py @@ -5,23 +5,31 @@ from dcim.models import Region, Site +def _path(*pks): + """Construct the expected ltree path value from a sequence of PKs. + + Mirrors the trigger's zero-padded label scheme (19 chars per label). + """ + return '.'.join(str(pk).zfill(19) for pk in pks) + + class LtreeTriggerTests(TestCase): """Verify per-row PostgreSQL triggers maintain `path` correctly.""" def test_insert_root_path(self): r = Region.objects.create(name='Root', slug='root') - self.assertEqual(r.path, str(r.pk)) + self.assertEqual(r.path, _path(r.pk)) def test_insert_child_path(self): r = Region.objects.create(name='Root', slug='root') c = Region.objects.create(parent=r, name='Child', slug='child') - self.assertEqual(c.path, f'{r.pk}.{c.pk}') + self.assertEqual(c.path, _path(r.pk, c.pk)) def test_grandchild_path(self): r = Region.objects.create(name='R', slug='r') c = Region.objects.create(parent=r, name='C', slug='c') g = Region.objects.create(parent=c, name='G', slug='g') - self.assertEqual(g.path, f'{r.pk}.{c.pk}.{g.pk}') + self.assertEqual(g.path, _path(r.pk, c.pk, g.pk)) def test_move_cascades_to_descendants(self): r = Region.objects.create(name='R', slug='r') @@ -31,8 +39,8 @@ def test_move_cascades_to_descendants(self): c.save() c.refresh_from_db() g.refresh_from_db() - self.assertEqual(c.path, str(c.pk)) - self.assertEqual(g.path, f'{c.pk}.{g.pk}') + self.assertEqual(c.path, _path(c.pk)) + self.assertEqual(g.path, _path(c.pk, g.pk)) def test_bulk_create_populates_paths(self): """BEFORE INSERT trigger fires on bulk_create, populating path.""" @@ -42,7 +50,7 @@ def test_bulk_create_populates_paths(self): ]) for child in children: child.refresh_from_db() - self.assertEqual(child.path, f'{root.pk}.{child.pk}') + self.assertEqual(child.path, _path(root.pk, child.pk)) def test_queryset_update_with_parent_id_cascades(self): """Raw .update() that changes parent_id still fires triggers.""" @@ -54,8 +62,8 @@ def test_queryset_update_with_parent_id_cascades(self): Region.objects.filter(pk=c.pk).update(parent=r2) c.refresh_from_db() g.refresh_from_db() - self.assertEqual(c.path, f'{r2.pk}.{c.pk}') - self.assertEqual(g.path, f'{r2.pk}.{c.pk}.{g.pk}') + self.assertEqual(c.path, _path(r2.pk, c.pk)) + self.assertEqual(g.path, _path(r2.pk, c.pk, g.pk)) def test_gist_index_exists(self): """Every ltree-backed table has a GiST index on path.""" @@ -139,7 +147,7 @@ def test_move_to(self): self.leaf.move_to(new_root) self.leaf.refresh_from_db() self.assertEqual(self.leaf.parent, new_root) - self.assertEqual(self.leaf.path, f'{new_root.pk}.{self.leaf.pk}') + self.assertEqual(self.leaf.path, _path(new_root.pk, self.leaf.pk)) class CycleValidationTests(TestCase): @@ -155,6 +163,41 @@ def test_cycle_raises(self): a.full_clean() +class SortPathTests(TestCase): + """ + Verify that sort_path produces tree-flatten output with siblings in name + order, mirroring MPTT's `order_insertion_by=('name',)` behavior. + """ + + def test_siblings_in_name_order_regardless_of_insertion_order(self): + # Create siblings out of name order + z = Region.objects.create(name='Zebra', slug='zebra-sp') + a = Region.objects.create(name='Aardvark', slug='aardvark-sp') + b = Region.objects.create(name='Buffalo', slug='buffalo-sp') + + # Children of Buffalo also out of order + b_z = Region.objects.create(parent=b, name='Zoo', slug='b-zoo-sp') + b_a = Region.objects.create(parent=b, name='Apex', slug='b-apex-sp') + + ordered = list( + Region.objects.filter(slug__endswith='-sp') + .order_by('sort_path') + .values_list('name', flat=True) + ) + # Tree-flatten with siblings in name order: + # Aardvark, Buffalo (parent), Apex (child), Zoo (child), Zebra + self.assertEqual(ordered, ['Aardvark', 'Buffalo', 'Apex', 'Zoo', 'Zebra']) + + def test_default_ordering_is_sort_path(self): + """Region.objects.all() uses sort_path-based ordering by default.""" + b = Region.objects.create(name='B', slug='b-default') + a = Region.objects.create(name='A', slug='a-default') + names = list( + Region.objects.filter(slug__endswith='-default').values_list('name', flat=True) + ) + self.assertEqual(names, ['A', 'B']) + + class AddRelatedCountTests(TestCase): """add_related_count must cumulate across subtrees via path <@.""" diff --git a/netbox/wireless/graphql/types.py b/netbox/wireless/graphql/types.py index 2ee19ca7a52..b421742e199 100644 --- a/netbox/wireless/graphql/types.py +++ b/netbox/wireless/graphql/types.py @@ -22,7 +22,7 @@ @strawberry_django.type( models.WirelessLANGroup, - exclude=['path'], + exclude=['path', 'sort_path'], filters=WirelessLANGroupFilter, pagination=True ) diff --git a/netbox/wireless/migrations/0020_ltree_paths.py b/netbox/wireless/migrations/0020_ltree_paths.py index ea3f661c085..72589351d36 100644 --- a/netbox/wireless/migrations/0020_ltree_paths.py +++ b/netbox/wireless/migrations/0020_ltree_paths.py @@ -19,49 +19,58 @@ class Migration(migrations.Migration): ] operations = [ - # Switch parent from mptt.fields.TreeForeignKey to django.db.models.ForeignKey. - # No-op at the SQL level; reconciles migration state with model definitions. migrations.AlterField( - model_name='wirelesslangroup', - name='parent', + model_name='wirelesslangroup', name='parent', field=models.ForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='wireless.wirelesslangroup', ), ), - # Enable the ltree extension (idempotent — CreateExtension emits IF NOT EXISTS) CreateExtension('ltree'), migrations.AddField( - model_name=MODEL, - name='path', + model_name=MODEL, name='path', field=netbox.models.ltree.LtreeField(blank=True, editable=False, null=True), ), + # sort_path uses natural_sort to match the WirelessLANGroup.name collation. + migrations.AddField( + model_name=MODEL, name='sort_path', + field=models.TextField( + blank=True, default='', editable=False, db_collation='natural_sort', + ), + ), - InstallLtreeTriggers(TABLE), + InstallLtreeTriggers(TABLE, name_column='name'), migrations.RunSQL( f""" -WITH RECURSIVE t(id, parent_id, path) AS ( - SELECT id, parent_id, id::text::ltree FROM "{TABLE}" WHERE parent_id IS NULL +WITH RECURSIVE t(id, parent_id, path, sort_path) AS ( + SELECT id, parent_id, + lpad(id::text, 19, '0')::ltree, + name::text + FROM "{TABLE}" WHERE parent_id IS NULL UNION ALL - SELECT r.id, r.parent_id, t.path || r.id::text::ltree + SELECT r.id, r.parent_id, + t.path || lpad(r.id::text, 19, '0')::ltree, + t.sort_path || chr(1) || r.name FROM "{TABLE}" r JOIN t ON r.parent_id = t.id ) -UPDATE "{TABLE}" SET path = t.path FROM t WHERE "{TABLE}".id = t.id; +UPDATE "{TABLE}" SET path = t.path, sort_path = t.sort_path +FROM t WHERE "{TABLE}".id = t.id; """, reverse_sql=migrations.RunSQL.noop, ), migrations.AlterField( - model_name=MODEL, - name='path', + model_name=MODEL, name='path', field=netbox.models.ltree.LtreeField(blank=True, default='', editable=False), ), - # Drop legacy (tree_id, lft) index added in 0018_add_mptt_tree_indexes, - # then drop the legacy MPTT columns. + migrations.AlterModelOptions( + name=MODEL, options={'ordering': ('sort_path',)}, + ), + migrations.RemoveIndex(model_name=MODEL, name='wireless_wirelesslangroup_fbcd'), *[migrations.RemoveField(model_name=MODEL, name=f) for f in LEGACY_FIELDS], @@ -69,4 +78,8 @@ class Migration(migrations.Migration): model_name=MODEL, index=GistIndex(fields=['path'], name='wireless_lan_grp_path_gist'), ), + migrations.AddIndex( + model_name=MODEL, + index=models.Index(fields=['sort_path'], name='wireless_lan_grp_sort_idx'), + ), ] diff --git a/netbox/wireless/models.py b/netbox/wireless/models.py index a7aaa45247c..0eca4ce3204 100644 --- a/netbox/wireless/models.py +++ b/netbox/wireless/models.py @@ -62,11 +62,19 @@ class WirelessLANGroup(NestedGroupModel): max_length=100, unique=True ) + # Override the abstract parent's sort_path to use natural_sort, matching `name`. + sort_path = models.TextField( + editable=False, + blank=True, + default='', + db_collation='natural_sort', + ) class Meta: - ordering = ('name', 'pk') + ordering = ('sort_path',) indexes = ( GistIndex(fields=['path'], name='wireless_lan_grp_path_gist'), + models.Index(fields=['sort_path'], name='wireless_lan_grp_sort_idx'), ) constraints = ( models.UniqueConstraint( From f4e5c58905f0c503b894560c99dabef79e7e85f7 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 27 May 2026 11:48:15 -0700 Subject: [PATCH 07/42] #21488 - Replace MPTT wtih PostgreSQL Ltree --- netbox/dcim/models/device_component_templates.py | 2 +- netbox/netbox/api/viewsets/__init__.py | 2 -- netbox/utilities/testing/filtersets.py | 2 +- netbox/utilities/tests/test_ltree.py | 14 +++++++------- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/netbox/dcim/models/device_component_templates.py b/netbox/dcim/models/device_component_templates.py index 4b05a8294a0..85da7a99c40 100644 --- a/netbox/dcim/models/device_component_templates.py +++ b/netbox/dcim/models/device_component_templates.py @@ -1,8 +1,8 @@ from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.postgres.indexes import GistIndex from django.core.exceptions import ValidationError from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models -from django.contrib.postgres.indexes import GistIndex from django.utils.translation import gettext_lazy as _ from dcim.choices import * diff --git a/netbox/netbox/api/viewsets/__init__.py b/netbox/netbox/api/viewsets/__init__.py index d6ce93e2f4f..49d12dd76ce 100644 --- a/netbox/netbox/api/viewsets/__init__.py +++ b/netbox/netbox/api/viewsets/__init__.py @@ -333,5 +333,3 @@ def perform_destroy(self, instance): super().perform_destroy(instance) except ObjectDoesNotExist: raise PermissionDenied() - - diff --git a/netbox/utilities/testing/filtersets.py b/netbox/utilities/testing/filtersets.py index 448d7ccd7b2..8a7ed736996 100644 --- a/netbox/utilities/testing/filtersets.py +++ b/netbox/utilities/testing/filtersets.py @@ -21,7 +21,7 @@ 'comments', 'custom_field_data', 'path', # ltree, trigger-maintained - 'sort_path', # ltree, trigger-maintained + 'sort_path', # ltree, trigger-maintained ) diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py index a1a699e6528..ce348a24b55 100644 --- a/netbox/utilities/tests/test_ltree.py +++ b/netbox/utilities/tests/test_ltree.py @@ -171,13 +171,13 @@ class SortPathTests(TestCase): def test_siblings_in_name_order_regardless_of_insertion_order(self): # Create siblings out of name order - z = Region.objects.create(name='Zebra', slug='zebra-sp') - a = Region.objects.create(name='Aardvark', slug='aardvark-sp') - b = Region.objects.create(name='Buffalo', slug='buffalo-sp') + Region.objects.create(name='Zebra', slug='zebra-sp') + Region.objects.create(name='Aardvark', slug='aardvark-sp') + buffalo = Region.objects.create(name='Buffalo', slug='buffalo-sp') # Children of Buffalo also out of order - b_z = Region.objects.create(parent=b, name='Zoo', slug='b-zoo-sp') - b_a = Region.objects.create(parent=b, name='Apex', slug='b-apex-sp') + Region.objects.create(parent=buffalo, name='Zoo', slug='b-zoo-sp') + Region.objects.create(parent=buffalo, name='Apex', slug='b-apex-sp') ordered = list( Region.objects.filter(slug__endswith='-sp') @@ -190,8 +190,8 @@ def test_siblings_in_name_order_regardless_of_insertion_order(self): def test_default_ordering_is_sort_path(self): """Region.objects.all() uses sort_path-based ordering by default.""" - b = Region.objects.create(name='B', slug='b-default') - a = Region.objects.create(name='A', slug='a-default') + Region.objects.create(name='B', slug='b-default') + Region.objects.create(name='A', slug='a-default') names = list( Region.objects.filter(slug__endswith='-default').values_list('name', flat=True) ) From 8926313c43f79e15777fb7c8231472ad05b9b7f8 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 29 May 2026 17:11:26 -0700 Subject: [PATCH 08/42] backwards compatability for NestedGroupModel --- netbox/dcim/models/devices.py | 6 +- netbox/dcim/models/sites.py | 8 +-- netbox/netbox/models/__init__.py | 91 +++++++++++++++++------- netbox/netbox/models/features.py | 7 +- netbox/netbox/tests/test_base_classes.py | 18 ++--- netbox/tenancy/models/contacts.py | 4 +- netbox/tenancy/models/tenants.py | 4 +- netbox/utilities/mptt.py | 24 +++++++ netbox/wireless/models.py | 4 +- 9 files changed, 116 insertions(+), 50 deletions(-) create mode 100644 netbox/utilities/mptt.py diff --git a/netbox/dcim/models/devices.py b/netbox/dcim/models/devices.py index 70de357e20c..9e4db98134b 100644 --- a/netbox/dcim/models/devices.py +++ b/netbox/dcim/models/devices.py @@ -24,7 +24,7 @@ from extras.querysets import ConfigContextModelQuerySet from netbox.choices import ColorChoices from netbox.config import ConfigItem -from netbox.models import NestedGroupModel, OrganizationalModel, PrimaryModel +from netbox.models import NestedLtreeGroupModel, OrganizationalModel, PrimaryModel from netbox.models.features import ContactsMixin, ImageAttachmentsMixin from netbox.models.mixins import WeightMixin from utilities.exceptions import AbortRequest @@ -386,7 +386,7 @@ def is_child_device(self): # Devices # -class DeviceRole(NestedGroupModel): +class DeviceRole(NestedLtreeGroupModel): """ Devices are organized by functional role; for example, "Core Switch" or "File Server". Each DeviceRole is assigned a color to be used when displaying rack elevations. The vm_role field determines whether the role is applicable to @@ -443,7 +443,7 @@ class Meta: verbose_name_plural = _('device roles') -class Platform(NestedGroupModel): +class Platform(NestedLtreeGroupModel): """ Platform refers to the software or firmware running on a Device. For example, "Cisco IOS-XR" or "Juniper Junos". A Platform may optionally be associated with a particular Manufacturer. diff --git a/netbox/dcim/models/sites.py b/netbox/dcim/models/sites.py index 70796b5154c..233829e942e 100644 --- a/netbox/dcim/models/sites.py +++ b/netbox/dcim/models/sites.py @@ -10,7 +10,7 @@ from dcim.choices import * from dcim.constants import * -from netbox.models import NestedGroupModel, PrimaryModel +from netbox.models import NestedLtreeGroupModel, PrimaryModel from netbox.models.features import ContactsMixin, ImageAttachmentsMixin __all__ = ( @@ -25,7 +25,7 @@ # Regions # -class Region(ContactsMixin, NestedGroupModel): +class Region(ContactsMixin, NestedLtreeGroupModel): """ A region represents a geographic collection of sites. For example, you might create regions representing countries, states, and/or cities. Regions are recursively nested into a hierarchy: all sites belonging to a child region are @@ -86,7 +86,7 @@ def get_site_count(self): # Site groups # -class SiteGroup(ContactsMixin, NestedGroupModel): +class SiteGroup(ContactsMixin, NestedLtreeGroupModel): """ A site group is an arbitrary grouping of sites. For example, you might have corporate sites and customer sites; and within corporate sites you might distinguish between offices and data centers. Like regions, site groups can be @@ -278,7 +278,7 @@ def get_status_color(self): # Locations # -class Location(ContactsMixin, ImageAttachmentsMixin, NestedGroupModel): +class Location(ContactsMixin, ImageAttachmentsMixin, NestedLtreeGroupModel): """ A Location represents a subgroup of Racks and/or Devices within a Site. A Location may represent a building within a site, or a room within a building, for example. diff --git a/netbox/netbox/models/__init__.py b/netbox/netbox/models/__init__.py index 7099ffbea99..2ffc8c40d40 100644 --- a/netbox/netbox/models/__init__.py +++ b/netbox/netbox/models/__init__.py @@ -5,16 +5,20 @@ from django.db import models from django.urls import reverse from django.utils.translation import gettext_lazy as _ +from mptt.models import MPTTModel, TreeForeignKey from netbox.models.features import * from netbox.models.ltree import LtreeManager, LtreeModel from netbox.models.mixins import OwnerMixin +from utilities.mptt import TreeManager from utilities.querysets import RestrictedQuerySet __all__ = ( 'AdminModel', 'ChangeLoggedModel', 'NestedGroupModel', + 'NestedGroupModelMixin', + 'NestedLtreeGroupModel', 'NetBoxModel', 'OrganizationalModel', 'PrimaryModel', @@ -158,24 +162,11 @@ class Meta: abstract = True -class NestedGroupModel(OwnerMixin, NetBoxModel, LtreeModel): +class NestedGroupModelMixin(OwnerMixin, NetBoxModel): """ - Base model for objects which are used to form a hierarchy (regions, locations, etc.). These models nest - recursively using PostgreSQL ltree. Within each parent, each child instance must have a unique name. - - `sort_path` is a trigger-maintained text column that mirrors MPTT's `order_insertion_by=('name',)` - semantics: at insert and reparent time it is set to a chr(1)-separated chain of ancestor names. - Ordering by `sort_path` yields tree-flatten output with siblings in their column's collation order. - Renaming a node does NOT update `sort_path` (matching MPTT behavior). + Shared field set and behavior for hierarchical group models. Concrete bases supply the + tree backend (MPTT or ltree) and the corresponding `parent` ForeignKey / manager. """ - parent = models.ForeignKey( - to='self', - on_delete=models.CASCADE, - related_name='children', - blank=True, - null=True, - db_index=True - ) name = models.CharField( verbose_name=_('name'), max_length=100 @@ -193,19 +184,9 @@ class NestedGroupModel(OwnerMixin, NetBoxModel, LtreeModel): verbose_name=_('comments'), blank=True ) - sort_path = models.TextField( - editable=False, - blank=True, - default='', - ) - - # Re-declare so the LtreeManager wins over BaseModel's RestrictedQuerySet - # default manager via MRO resolution. - objects = LtreeManager() class Meta: abstract = True - ordering = ('sort_path',) def __str__(self): return self.name @@ -220,6 +201,64 @@ def clean(self): }) +class NestedGroupModel(NestedGroupModelMixin, MPTTModel): + """ + Deprecated MPTT-backed nested group base, retained for backwards compatibility with plugins. + + New code (in NetBox core and in plugins) should use `NestedLtreeGroupModel` instead. This + class will be removed in a future release once the deprecation period has elapsed. + """ + parent = TreeForeignKey( + to='self', + on_delete=models.CASCADE, + related_name='children', + blank=True, + null=True, + db_index=True + ) + + objects = TreeManager() + + class Meta: + abstract = True + + class MPTTMeta: + order_insertion_by = ('name',) + + +class NestedLtreeGroupModel(NestedGroupModelMixin, LtreeModel): + """ + Base model for objects which are used to form a hierarchy (regions, locations, etc.). These models nest + recursively using PostgreSQL ltree. Within each parent, each child instance must have a unique name. + + `sort_path` is a trigger-maintained text column that mirrors MPTT's `order_insertion_by=('name',)` + semantics: at insert and reparent time it is set to a chr(1)-separated chain of ancestor names. + Ordering by `sort_path` yields tree-flatten output with siblings in their column's collation order. + Renaming a node does NOT update `sort_path` (matching MPTT behavior). + """ + parent = models.ForeignKey( + to='self', + on_delete=models.CASCADE, + related_name='children', + blank=True, + null=True, + db_index=True + ) + sort_path = models.TextField( + editable=False, + blank=True, + default='', + ) + + # Re-declare so the LtreeManager wins over BaseModel's RestrictedQuerySet + # default manager via MRO resolution. + objects = LtreeManager() + + class Meta: + abstract = True + ordering = ('sort_path',) + + class OrganizationalModel(OwnerMixin, NetBoxModel): """ Organizational models are those which are used solely to categorize and qualify other objects, and do not convey diff --git a/netbox/netbox/models/features.py b/netbox/netbox/models/features.py index 4f59936df6c..e12adacabb8 100644 --- a/netbox/netbox/models/features.py +++ b/netbox/netbox/models/features.py @@ -408,13 +408,16 @@ def get_contacts(self, inherited=True): """ from tenancy.models import ContactAssignment - from . import NestedGroupModel + # TODO: Once the deprecated MPTT-backed NestedGroupModel is removed, narrow this + # check to NestedLtreeGroupModel (or drop the mixin alias). NestedGroupModelMixin + # exists only so both legacy MPTT and new ltree bases satisfy the inheritance check. + from . import NestedGroupModelMixin filter = Q( object_type=ObjectType.objects.get_for_model(self), object_id__in=( self.get_ancestors(include_self=True) - if (isinstance(self, NestedGroupModel) and inherited) + if (isinstance(self, NestedGroupModelMixin) and inherited) else [self.pk] ), ) diff --git a/netbox/netbox/tests/test_base_classes.py b/netbox/netbox/tests/test_base_classes.py index 5fad1ccd599..2e889be6317 100644 --- a/netbox/netbox/tests/test_base_classes.py +++ b/netbox/netbox/tests/test_base_classes.py @@ -44,7 +44,7 @@ OrganizationalObjectType, PrimaryObjectType, ) -from netbox.models import NestedGroupModel, NetBoxModel, OrganizationalModel, PrimaryModel +from netbox.models import NestedGroupModelMixin, NetBoxModel, OrganizationalModel, PrimaryModel from netbox.registry import registry from netbox.tables import ( NestedGroupModelTable, @@ -76,7 +76,7 @@ def get_model_form_base_class(model): return PrimaryModelForm if issubclass(model, OrganizationalModel): return OrganizationalModelForm - if issubclass(model, NestedGroupModel): + if issubclass(model, NestedGroupModelMixin): return NestedGroupModelForm if issubclass(model, NetBoxModel): return NetBoxModelForm @@ -93,7 +93,7 @@ def get_bulk_edit_form_base_class(model): return PrimaryModelBulkEditForm if issubclass(model, OrganizationalModel): return OrganizationalModelBulkEditForm - if issubclass(model, NestedGroupModel): + if issubclass(model, NestedGroupModelMixin): return NestedGroupModelBulkEditForm if issubclass(model, NetBoxModel): return NetBoxModelBulkEditForm @@ -110,7 +110,7 @@ def get_import_form_base_class(model): return PrimaryModelImportForm if issubclass(model, OrganizationalModel): return OrganizationalModelImportForm - if issubclass(model, NestedGroupModel): + if issubclass(model, NestedGroupModelMixin): return NestedGroupModelImportForm if issubclass(model, NetBoxModel): return NetBoxModelImportForm @@ -127,7 +127,7 @@ def get_filterset_form_base_class(model): return PrimaryModelFilterSetForm if issubclass(model, OrganizationalModel): return OrganizationalModelFilterSetForm - if issubclass(model, NestedGroupModel): + if issubclass(model, NestedGroupModelMixin): return NestedGroupModelFilterSetForm if issubclass(model, NetBoxModel): return NetBoxModelFilterSetForm @@ -191,7 +191,7 @@ def get_model_filterset_base_class(model): return PrimaryModelFilterSet if issubclass(model, OrganizationalModel): return OrganizationalModelFilterSet - if issubclass(model, NestedGroupModel): + if issubclass(model, NestedGroupModelMixin): return NestedGroupModelFilterSet if issubclass(model, NetBoxModel): return NetBoxModelFilterSet @@ -233,7 +233,7 @@ def get_model_table_base_class(model): return PrimaryModelTable if issubclass(model, OrganizationalModel): return OrganizationalModelTable - if issubclass(model, NestedGroupModel): + if issubclass(model, NestedGroupModelMixin): return NestedGroupModelTable if issubclass(model, NetBoxModel): return NetBoxTable @@ -278,7 +278,7 @@ def get_model_serializer_base_class(model): return PrimaryModelSerializer if issubclass(model, OrganizationalModel): return OrganizationalModelSerializer - if issubclass(model, NestedGroupModel): + if issubclass(model, NestedGroupModelMixin): return NestedGroupModelSerializer if issubclass(model, NetBoxModel): return NetBoxModelSerializer @@ -319,7 +319,7 @@ def get_model_type_base_class(model): return PrimaryObjectType if issubclass(model, OrganizationalModel): return OrganizationalObjectType - if issubclass(model, NestedGroupModel): + if issubclass(model, NestedGroupModelMixin): return NestedGroupObjectType if issubclass(model, NetBoxModel): return NetBoxObjectType diff --git a/netbox/tenancy/models/contacts.py b/netbox/tenancy/models/contacts.py index 56452df3e4f..2f62f2e0800 100644 --- a/netbox/tenancy/models/contacts.py +++ b/netbox/tenancy/models/contacts.py @@ -6,7 +6,7 @@ from django.urls import reverse from django.utils.translation import gettext_lazy as _ -from netbox.models import ChangeLoggedModel, NestedGroupModel, OrganizationalModel, PrimaryModel +from netbox.models import ChangeLoggedModel, NestedLtreeGroupModel, OrganizationalModel, PrimaryModel from netbox.models.features import CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, has_feature from netbox.models.ltree import LtreeManager from tenancy.choices import * @@ -40,7 +40,7 @@ def annotate_contacts(self): ) -class ContactGroup(NestedGroupModel): +class ContactGroup(NestedLtreeGroupModel): """ An arbitrary collection of Contacts. """ diff --git a/netbox/tenancy/models/tenants.py b/netbox/tenancy/models/tenants.py index 6ea6769a637..197729f2ea0 100644 --- a/netbox/tenancy/models/tenants.py +++ b/netbox/tenancy/models/tenants.py @@ -3,7 +3,7 @@ from django.db.models import Q from django.utils.translation import gettext_lazy as _ -from netbox.models import NestedGroupModel, PrimaryModel +from netbox.models import NestedLtreeGroupModel, PrimaryModel from netbox.models.features import ContactsMixin __all__ = ( @@ -12,7 +12,7 @@ ) -class TenantGroup(NestedGroupModel): +class TenantGroup(NestedLtreeGroupModel): """ An arbitrary collection of Tenants. """ diff --git a/netbox/utilities/mptt.py b/netbox/utilities/mptt.py new file mode 100644 index 00000000000..7ded5b72bdf --- /dev/null +++ b/netbox/utilities/mptt.py @@ -0,0 +1,24 @@ +from mptt.managers import TreeManager as TreeManager_ +from mptt.querysets import TreeQuerySet as TreeQuerySet_ + +from django.db.models import Manager +from .querysets import RestrictedQuerySet + +__all__ = ( + 'TreeManager', + 'TreeQuerySet', +) + + +class TreeQuerySet(TreeQuerySet_, RestrictedQuerySet): + """ + Mate django-mptt's TreeQuerySet with our RestrictedQuerySet for permissions enforcement. + """ + pass + + +class TreeManager(Manager.from_queryset(TreeQuerySet), TreeManager_): + """ + Extend django-mptt's TreeManager to incorporate RestrictedQuerySet(). + """ + pass diff --git a/netbox/wireless/models.py b/netbox/wireless/models.py index 0eca4ce3204..3a47db33a9a 100644 --- a/netbox/wireless/models.py +++ b/netbox/wireless/models.py @@ -6,7 +6,7 @@ from dcim.choices import LinkStatusChoices from dcim.constants import WIRELESS_IFACE_TYPES from dcim.models.mixins import CachedScopeMixin -from netbox.models import NestedGroupModel, PrimaryModel +from netbox.models import NestedLtreeGroupModel, PrimaryModel from netbox.models.mixins import DistanceMixin from .choices import * @@ -47,7 +47,7 @@ class Meta: abstract = True -class WirelessLANGroup(NestedGroupModel): +class WirelessLANGroup(NestedLtreeGroupModel): """ A nested grouping of WirelessLANs """ From 4b80b59a77487633d9bb6d66692230b5ae370f5e Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 29 May 2026 17:11:38 -0700 Subject: [PATCH 09/42] backwards compatability for NestedGroupModel --- netbox/utilities/mptt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/utilities/mptt.py b/netbox/utilities/mptt.py index 7ded5b72bdf..4914fccab2d 100644 --- a/netbox/utilities/mptt.py +++ b/netbox/utilities/mptt.py @@ -1,7 +1,7 @@ +from django.db.models import Manager from mptt.managers import TreeManager as TreeManager_ from mptt.querysets import TreeQuerySet as TreeQuerySet_ -from django.db.models import Manager from .querysets import RestrictedQuerySet __all__ = ( From 0208621c188b3eff5d44f40ea0e3c1a8a660ebac Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 2 Jun 2026 10:28:58 -0700 Subject: [PATCH 10/42] fix natural sort for modules --- netbox/dcim/migrations/0237_ltree_paths.py | 9 ++++++++- netbox/dcim/models/device_components.py | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/netbox/dcim/migrations/0237_ltree_paths.py b/netbox/dcim/migrations/0237_ltree_paths.py index ad9b04b7367..2e631359f2b 100644 --- a/netbox/dcim/migrations/0237_ltree_paths.py +++ b/netbox/dcim/migrations/0237_ltree_paths.py @@ -158,13 +158,20 @@ class Migration(migrations.Migration): for m in ALL_MODELS ], # 2b. Add sort_path column (with default '') on the 6 models with order_insertion_by. + # ModuleBay's `name` uses the natural_sort collation, so its sort_path must + # match it (Slot 0..Slot 13, not lexicographic Slot 0, 1, 10, 2...). The other + # five use a plain-collation `name`, so their sort_path stays plain. *[ migrations.AddField( model_name=m, name='sort_path', field=models.TextField(blank=True, default='', editable=False), ) - for m in SORT_MODELS + for m in SORT_MODELS if m != 'modulebay' ], + migrations.AddField( + model_name='modulebay', name='sort_path', + field=models.TextField(blank=True, default='', editable=False, db_collation='natural_sort'), + ), # 3. Install path-maintenance triggers. Models with sort_path get triggers # that maintain both columns; the other two get path-only triggers. diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index 5a05eb13d0c..4dae6af0c1e 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -1340,6 +1340,10 @@ class ModuleBay(ModularComponentModel, TrackingModelMixin, LtreeModel): editable=False, blank=True, default='', + # `name` uses the natural_sort collation; match it here so ORDER BY + # sort_path sorts siblings naturally (Slot 0..Slot 13) as MPTT's + # order_insertion_by=('name',) did, not lexicographically. + db_collation='natural_sort', ) clone_fields = ('device', 'enabled') From 4a722d1a144bb2b55c9537c33d7958aeb49f1a6b Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 2 Jun 2026 11:09:09 -0700 Subject: [PATCH 11/42] fixes --- netbox/netbox/models/ltree.py | 70 ++++++++++++++-- netbox/utilities/tests/test_ltree.py | 117 +++++++++++++++++++++++++++ 2 files changed, 181 insertions(+), 6 deletions(-) diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index 1649fbf7a80..83491ec371e 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -119,15 +119,18 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F if is_many_to_many: field = model._meta.get_field(rel_field) m2m_table = field.remote_field.through._meta.db_table - m2m_parent_col = field.m2m_column_name() - m2m_related_col = field.m2m_reverse_name() + # `model` is the declaring side (the side holding the items to count); + # `queryset.model` is the related (tree) side. m2m_column_name() points + # at the declaring model; m2m_reverse_name() points at the related model. + m2m_to_child_col = field.m2m_column_name() + m2m_to_tree_col = field.m2m_reverse_name() sql = f'''( SELECT COUNT(DISTINCT "{related_table}"."id") FROM "{related_table}" INNER JOIN "{m2m_table}" - ON "{related_table}"."id" = "{m2m_table}"."{m2m_related_col}" + ON "{related_table}"."id" = "{m2m_table}"."{m2m_to_child_col}" INNER JOIN "{parent_table}" AS subtree - ON "{m2m_table}"."{m2m_parent_col}" = subtree."id" + ON "{m2m_table}"."{m2m_to_tree_col}" = subtree."id" WHERE subtree."path" <@ "{parent_table}"."path" )''' return queryset.annotate(**{ @@ -201,6 +204,22 @@ class LtreeModel(models.Model): Subclasses must declare a `parent = models.ForeignKey('self', ...)`. The `path` column is maintained by per-table triggers installed via InstallLtreeTriggers; do not write to it from Python. + + Bulk creates: + The BEFORE INSERT trigger resolves a row's parent by SELECTing + `path` from the same table by parent_id. In a multi-row INSERT + (e.g. `bulk_create`) PostgreSQL fires the BEFORE trigger per row in + list order, so any row whose parent is also in the same batch + must appear after its parent. A child placed before its parent in + the batch will be persisted with a root-level path (the parent + row is not yet visible to the lookup). + + Sort-path staleness: + For subclasses with the optional `sort_path` column (see + InstallLtreeTriggers' `name_column` arg), renaming a row does NOT + update its sort_path — matching django-mptt's + `order_insertion_by` behavior. Call `rebuild_sort_paths()` to + recompute sort_path for every row from current names. """ path = LtreeField(editable=False, null=False, blank=True, default='') @@ -226,11 +245,18 @@ def save(self, *args, **kwargs): consistent with the database. """ is_insert = self._state.adding - parent_changed = (not is_insert) and self.parent_id != self._loaded_parent_id + # When update_fields is supplied and excludes parent, the DB does not see + # the new parent_id, so the trigger does not fire and _loaded_parent_id + # must not advance — otherwise a subsequent full save() would mis-detect + # the (real) parent change as already-applied and leave path stale. + update_fields = kwargs.get('update_fields') + parent_written = update_fields is None or 'parent' in update_fields or 'parent_id' in update_fields + parent_changed = (not is_insert) and parent_written and self.parent_id != self._loaded_parent_id super().save(*args, **kwargs) if is_insert or parent_changed: self.path = type(self).objects.values_list('path', flat=True).get(pk=self.pk) - self._loaded_parent_id = self.parent_id + if is_insert or parent_written: + self._loaded_parent_id = self.parent_id # -- MPTT-compatible API ------------------------------------------------ @@ -341,6 +367,38 @@ def insert_at(self, target, position='last-child', save=False): if save: self.save() + @classmethod + def rebuild_sort_paths(cls, name_column='name'): + """ + Recompute `sort_path` for every row from current values of `name_column`. + + The BEFORE trigger updates sort_path only on INSERT and parent change, + not on rename — matching django-mptt's `order_insertion_by` semantics. + After renaming rows, call this to bring sort_path back in line with + current names (and thus restore correct list ordering). + + Raises if the table does not have a `sort_path` column. + """ + from django.db import connection + + if not any(f.name == 'sort_path' for f in cls._meta.get_fields()): + raise NotImplementedError( + f"{cls.__name__} does not have a sort_path column" + ) + table = cls._meta.db_table + sql = f''' + WITH RECURSIVE t(id, parent_id, sort_path) AS ( + SELECT id, parent_id, "{name_column}"::text + FROM "{table}" WHERE parent_id IS NULL + UNION ALL + SELECT r.id, r.parent_id, t.sort_path || chr(1) || r."{name_column}" + FROM "{table}" r INNER JOIN t ON r.parent_id = t.id + ) + UPDATE "{table}" SET sort_path = t.sort_path FROM t WHERE "{table}".id = t.id; + ''' + with connection.cursor() as cursor: + cursor.execute(sql) + # # Migration operation diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py index ce348a24b55..47350da6693 100644 --- a/netbox/utilities/tests/test_ltree.py +++ b/netbox/utilities/tests/test_ltree.py @@ -3,6 +3,7 @@ from django.test import TestCase from dcim.models import Region, Site +from tenancy.models import Contact, ContactGroup def _path(*pks): @@ -215,3 +216,119 @@ def test_cumulative_fk_count(self): # root sees both sites (direct + via child) self.assertEqual(counts['R'], 2) self.assertEqual(counts['C'], 1) + + def test_cumulative_m2m_count(self): + """ + Cumulative count over an M2M relation walks the subtree via + path <@, joining the through table by the correct columns. + + Regression: previously the JOINs swapped m2m_column_name() and + m2m_reverse_name(), producing wrong counts on M2M relations. + """ + root = ContactGroup.objects.create(name='Root', slug='root-m2m') + child = ContactGroup.objects.create(parent=root, name='Child', slug='child-m2m') + leaf = ContactGroup.objects.create(parent=child, name='Leaf', slug='leaf-m2m') + # 3 contacts spread across the subtree (one per node) + c_root = Contact.objects.create(name='CR') + c_child = Contact.objects.create(name='CC') + c_leaf = Contact.objects.create(name='CL') + c_root.groups.add(root) + c_child.groups.add(child) + c_leaf.groups.add(leaf) + + qs = ContactGroup.objects.add_related_count( + ContactGroup.objects.filter(slug__endswith='-m2m'), + Contact, 'groups', 'contact_count', cumulative=True, + ) + counts = {g.name: g.contact_count for g in qs} + self.assertEqual(counts['Root'], 3) + self.assertEqual(counts['Child'], 2) + self.assertEqual(counts['Leaf'], 1) + + +class SaveUpdateFieldsTests(TestCase): + """ + Regression: when save(update_fields=...) excludes parent, _loaded_parent_id + must not advance, otherwise a subsequent full save() will mis-detect the + parent change as already-applied and leave path stale in memory. + """ + + def test_partial_save_then_full_save_refreshes_path(self): + r1 = Region.objects.create(name='R1', slug='r1-uf') + r2 = Region.objects.create(name='R2', slug='r2-uf') + obj = Region.objects.create(name='Obj', slug='obj-uf') + original_path = obj.path + + # Reparent in memory but persist a different field only: + obj.parent = r1 + obj.name = 'Obj-renamed' + obj.save(update_fields=['name']) + + # DB parent_id is still NULL — confirm: + db_parent = Region.objects.values_list('parent_id', flat=True).get(pk=obj.pk) + self.assertIsNone(db_parent) + self.assertEqual( + Region.objects.values_list('path', flat=True).get(pk=obj.pk), + original_path, + ) + + # Now a full save persists the new parent — path must refresh: + obj.parent = r2 + obj.save() + db_path = Region.objects.values_list('path', flat=True).get(pk=obj.pk) + self.assertEqual(db_path, _path(r2.pk, obj.pk)) + self.assertEqual(obj.path, db_path, "in-memory path is stale after full save") + + +class BulkCreateOrderingTests(TestCase): + """ + The BEFORE INSERT trigger looks up parent.path via subquery per row. + In bulk_create the trigger fires per row in list order, so a parent + placed in the same batch must precede its children. + """ + + def test_parent_before_child_in_same_batch(self): + root = Region.objects.create(name='R', slug='r-bcord') + # Parent BEFORE child in the list — both get correct paths + parent_pending = Region(parent=root, name='Mid', slug='mid-bcord') + child_pending = Region(parent=parent_pending, name='Leaf', slug='leaf-bcord') + # parent_pending isn't yet saved, so child_pending.parent_id is None; + # set the parent reference after the parent is saved. + Region.objects.bulk_create([parent_pending]) + parent_pending.refresh_from_db() + child_pending.parent = parent_pending + Region.objects.bulk_create([child_pending]) + child_pending.refresh_from_db() + self.assertEqual(child_pending.path, _path(root.pk, parent_pending.pk, child_pending.pk)) + + +class RebuildSortPathsTests(TestCase): + """ + rebuild_sort_paths() recomputes sort_path from current names. The stale- + sort case shows up in descendants: renaming a parent and saving it + recomputes that one row's sort_path, but descendants' sort_paths still + contain the parent's old name until rebuilt. + """ + + def test_descendant_sort_paths_stale_then_rebuilt(self): + parent = Region.objects.create(name='Bravo', slug='bravo-rsp') + Region.objects.create(parent=parent, name='Mid', slug='mid-rsp') + + # Rename parent. AFTER cascade only fires when path changes, so + # descendants keep their old sort_path embedding 'Bravo'. + parent.name = 'Zulu' + parent.save() + mid_sort = Region.objects.values_list('sort_path', flat=True).get(slug='mid-rsp') + self.assertIn('Bravo', mid_sort, "descendant sort_path should still contain old name") + self.assertNotIn('Zulu', mid_sort) + + Region.rebuild_sort_paths() + mid_sort = Region.objects.values_list('sort_path', flat=True).get(slug='mid-rsp') + self.assertIn('Zulu', mid_sort) + self.assertNotIn('Bravo', mid_sort) + + def test_raises_without_sort_path(self): + # InventoryItem uses LtreeModel but doesn't have a sort_path column. + from dcim.models import InventoryItem + with self.assertRaises(NotImplementedError): + InventoryItem.rebuild_sort_paths() From 62afbcfb2103e60d66c3ac477b2621434aba8df9 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 2 Jun 2026 11:38:34 -0700 Subject: [PATCH 12/42] fixes --- netbox/core/models/change_logging.py | 4 +- netbox/netbox/models/ltree.py | 28 ++++++++--- netbox/utilities/tests/test_ltree.py | 75 ++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 8 deletions(-) diff --git a/netbox/core/models/change_logging.py b/netbox/core/models/change_logging.py index c072a468af2..945a714e709 100644 --- a/netbox/core/models/change_logging.py +++ b/netbox/core/models/change_logging.py @@ -163,10 +163,10 @@ def diff_exclude_fields(self): if issubclass(model, ChangeLoggingMixin): attrs.update({'created', 'last_updated'}) - # Exclude trigger-maintained ltree path + # Exclude trigger-maintained ltree columns (path and the optional sort_path) from netbox.models.ltree import LtreeModel if issubclass(model, LtreeModel): - attrs.update({'path'}) + attrs.update({'path', 'sort_path'}) return attrs diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index 83491ec371e..ecf05a86506 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -240,9 +240,9 @@ def from_db(cls, db, field_names, values): def save(self, *args, **kwargs): """ - Triggers compute `path` server-side. After insert or after a parent - change, refresh just the path column so the in-memory instance stays - consistent with the database. + Triggers compute `path` (and `sort_path`, where present) server-side. + After insert or after a parent change, refresh those columns so the + in-memory instance stays consistent with the database. """ is_insert = self._state.adding # When update_fields is supplied and excludes parent, the DB does not see @@ -254,7 +254,17 @@ def save(self, *args, **kwargs): parent_changed = (not is_insert) and parent_written and self.parent_id != self._loaded_parent_id super().save(*args, **kwargs) if is_insert or parent_changed: - self.path = type(self).objects.values_list('path', flat=True).get(pk=self.pk) + # Refresh every trigger-maintained column (`path`, plus `sort_path` on + # models that declare it) so the in-memory instance matches the row the + # triggers actually wrote — otherwise e.g. change logging would snapshot + # a stale value. + refresh_fields = [ + name for name in ('path', 'sort_path') + if any(f.attname == name for f in self._meta.concrete_fields) + ] + row = type(self).objects.values_list(*refresh_fields).get(pk=self.pk) + for name, value in zip(refresh_fields, row): + setattr(self, name, value) if is_insert or parent_written: self._loaded_parent_id = self.parent_id @@ -486,9 +496,15 @@ def rebuild_sort_paths(cls, name_column='name'): FOR EACH ROW EXECUTE FUNCTION "{table}_ltree_compute_path_fn"(); ''' +# Fire only on parent_id changes, not on path changes. A single reparent's +# cascade rewrites the whole subtree (WHERE path <@ OLD.path) at every depth in +# one statement, so it never needs to re-fire; including `path` here would make +# every descendant the cascade touches re-fire the trigger, each running a no-op +# cascade against its now-vacated OLD.path. Nothing mutates `path` without also +# touching `parent_id`, so `parent_id` alone is sufficient. _AFTER_TRIGGER = ''' CREATE TRIGGER "{table}_ltree_cascade_path" - AFTER UPDATE OF parent_id, path ON "{table}" + AFTER UPDATE OF parent_id ON "{table}" FOR EACH ROW WHEN (OLD.path IS DISTINCT FROM NEW.path) EXECUTE FUNCTION "{table}_ltree_cascade_path_fn"(); ''' @@ -501,7 +517,7 @@ class InstallLtreeTriggers(migrations.operations.base.Operation): Two row-level triggers are installed on each target table: BEFORE INSERT OR UPDATE OF parent_id -> compute NEW.path (and sort_path if applicable) - AFTER UPDATE OF parent_id, path -> cascade path/sort_path change to descendants + AFTER UPDATE OF parent_id -> cascade path/sort_path change to descendants If `name_column` is provided, the model is expected to have a `sort_path` text column whose value will be maintained as a chr(1)-separated chain of diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py index 47350da6693..11601c6bb72 100644 --- a/netbox/utilities/tests/test_ltree.py +++ b/netbox/utilities/tests/test_ltree.py @@ -1,7 +1,9 @@ """Tests for the ltree-based hierarchical model infrastructure.""" +from django.contrib.contenttypes.models import ContentType from django.db import connection from django.test import TestCase +from core.models import ObjectChange from dcim.models import Region, Site from tenancy.models import Contact, ContactGroup @@ -332,3 +334,76 @@ def test_raises_without_sort_path(self): from dcim.models import InventoryItem with self.assertRaises(NotImplementedError): InventoryItem.rebuild_sort_paths() + + +class SortPathRefreshTests(TestCase): + """ + save() must refresh the in-memory `sort_path` (not just `path`) after insert + and reparent, so callers (notably change logging) never snapshot a stale value. + """ + + def test_create_refreshes_sort_path(self): + root = Region.objects.create(name='Root', slug='root-spr') + child = Region.objects.create(name='Kid', slug='kid-spr', parent=root) + db_sort_path = Region.objects.values_list('sort_path', flat=True).get(pk=child.pk) + self.assertEqual(child.sort_path, db_sort_path) + self.assertEqual(child.sort_path, f'Root{chr(1)}Kid') + + def test_reparent_refreshes_sort_path(self): + a = Region.objects.create(name='Alpha', slug='alpha-spr') + b = Region.objects.create(name='Bravo', slug='bravo-spr') + child = Region.objects.create(name='Kid', slug='kid2-spr', parent=a) + child.parent = b + child.save() + db_sort_path = Region.objects.values_list('sort_path', flat=True).get(pk=child.pk) + self.assertEqual(child.sort_path, db_sort_path) + self.assertEqual(child.sort_path, f'Bravo{chr(1)}Kid') + + +class ChangeLogExclusionTests(TestCase): + """ + Trigger-maintained columns (`path`, `sort_path`) must be excluded from change + log diffs, and the postchange snapshot must capture the refreshed values. + """ + + def test_sort_path_excluded_from_diff(self): + oc = ObjectChange() + oc.changed_object_type = ContentType.objects.get_for_model(Region) + self.assertIn('path', oc.diff_exclude_fields) + self.assertIn('sort_path', oc.diff_exclude_fields) + + def test_reparent_postchange_snapshot_matches_db(self): + a = Region.objects.create(name='Alpha', slug='alpha-cl') + b = Region.objects.create(name='Bravo', slug='bravo-cl') + child = Region.objects.create(name='Kid', slug='kid-cl', parent=a) + # Reload so the prechange snapshot reflects the persisted state. + child = Region.objects.get(pk=child.pk) + child.snapshot() + child.parent = b + child.save() + oc = child.to_objectchange('update') + db = Region.objects.values('path', 'sort_path').get(pk=child.pk) + self.assertEqual(oc.postchange_data['path'], db['path']) + self.assertEqual(oc.postchange_data['sort_path'], db['sort_path']) + # path/sort_path are excluded, so they must not surface in the cleaned diff data. + self.assertNotIn('sort_path', oc.postchange_data_clean) + self.assertNotIn('path', oc.postchange_data_clean) + + +class CascadeTriggerScopeTests(TestCase): + """ + The AFTER cascade trigger fires on parent_id changes only; renames (which do + not touch parent_id or path) must not cascade. + """ + + def test_rename_does_not_cascade_to_descendants(self): + parent = Region.objects.create(name='Bravo', slug='bravo-ct') + child = Region.objects.create(parent=parent, name='Mid', slug='mid-ct') + original_child_path = child.path + parent.name = 'Zulu' + parent.save() + child.refresh_from_db() + # Path is unaffected by a rename, and the descendant keeps the old + # name embedding in sort_path (MPTT order_insertion_by parity). + self.assertEqual(child.path, original_child_path) + self.assertIn('Bravo', child.sort_path) From ef0b044ff3968bfa6276de7653d5ce6b2ff6bc3d Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 2 Jun 2026 12:10:59 -0700 Subject: [PATCH 13/42] fixes --- netbox/dcim/migrations/0237_ltree_paths.py | 8 ++++ .../dcim/models/device_component_templates.py | 13 ++++-- netbox/dcim/models/modules.py | 5 ++- netbox/dcim/tests/test_models.py | 32 ++++++++++++++ netbox/netbox/models/ltree.py | 43 +++++++++++++++++-- netbox/tenancy/migrations/0025_ltree_paths.py | 6 ++- netbox/utilities/tests/test_ltree.py | 33 ++++++++++++++ .../wireless/migrations/0020_ltree_paths.py | 6 ++- 8 files changed, 135 insertions(+), 11 deletions(-) diff --git a/netbox/dcim/migrations/0237_ltree_paths.py b/netbox/dcim/migrations/0237_ltree_paths.py index 2e631359f2b..1886ac7a30d 100644 --- a/netbox/dcim/migrations/0237_ltree_paths.py +++ b/netbox/dcim/migrations/0237_ltree_paths.py @@ -16,6 +16,14 @@ 6. Drops the legacy MPTT columns (lft, rght, tree_id, level). 7. Adds a GiST index on path (descendant/ancestor lookups via `<@` / `@>`). For sort_path models, also adds a btree index for ORDER BY listing. + +Notes: +- Step 4 populates each table with a single recursive-CTE UPDATE over all rows. + On very large tables (notably InventoryItem) this is one long-running statement + holding a row-exclusive lock for its duration; budget for it during upgrades. +- The reverse migration is lossy: it re-adds the MPTT columns (lft/rght/tree_id/ + level) empty and does NOT rebuild the tree. Forward migration is the supported + direction. """ import django.db.models.deletion from django.contrib.postgres.indexes import GistIndex diff --git a/netbox/dcim/models/device_component_templates.py b/netbox/dcim/models/device_component_templates.py index 85da7a99c40..728bd15a976 100644 --- a/netbox/dcim/models/device_component_templates.py +++ b/netbox/dcim/models/device_component_templates.py @@ -752,11 +752,18 @@ class Meta(ModularComponentTemplateModel.Meta): verbose_name_plural = _('module bay templates') def instantiate(self, **kwargs): + module = kwargs.get('module') return self.component_model( - name=self.resolve_name(kwargs.get('module'), kwargs.get('device')), - label=self.resolve_label(kwargs.get('module'), kwargs.get('device')), - position=self.resolve_position(kwargs.get('module'), kwargs.get('device')), + name=self.resolve_name(module, kwargs.get('device')), + label=self.resolve_label(module, kwargs.get('device')), + position=self.resolve_position(module, kwargs.get('device')), enabled=self.enabled, + # A module bay created for an installed module nests under that module's + # bay. bulk_create() bypasses ModuleBay.save() (which would otherwise set + # this), so the parent must be assigned here for the path trigger to nest + # it correctly. Device-level bays are instantiated without a module and + # remain roots (parent=None). + parent=module.module_bay if module else None, **kwargs ) instantiate.do_not_call_in_templates = True diff --git a/netbox/dcim/models/modules.py b/netbox/dcim/models/modules.py index c26789d3506..14984db9d0d 100644 --- a/netbox/dcim/models/modules.py +++ b/netbox/dcim/models/modules.py @@ -385,8 +385,9 @@ def save(self, *args, **kwargs): component._location = self.device.location component._rack = self.device.rack - # Bulk-create new instances. For ltree-backed models (ModuleBay, - # InventoryItem), the BEFORE INSERT trigger populates `path` per row. + # Bulk-create new instances. ModuleBay is ltree-backed: its parent is set + # in ModuleBayTemplate.instantiate() (bulk_create bypasses ModuleBay.save()), + # and the BEFORE INSERT trigger derives path/sort_path from parent_id per row. component_model.objects.bulk_create(create_instances) for component in create_instances: post_save.send( diff --git a/netbox/dcim/tests/test_models.py b/netbox/dcim/tests/test_models.py index 260615b4ea3..d251ad2735f 100644 --- a/netbox/dcim/tests/test_models.py +++ b/netbox/dcim/tests/test_models.py @@ -1265,6 +1265,38 @@ def test_nested_module_bay_label_resolution(self): nested_bay = module.modulebays.get(name='SFP A-21') self.assertEqual(nested_bay.label, 'A-21') + @tag('regression') # #21418 + def test_module_install_nests_module_bay_parent(self): + """ + A module bay instantiated when a module is installed must be nested under the + installing module's bay. bulk_create() bypasses ModuleBay.save(), so the parent + is assigned in ModuleBayTemplate.instantiate(); without it the bay would be left + a root with a top-level ltree path. + """ + manufacturer = Manufacturer.objects.first() + site = Site.objects.first() + device_role = DeviceRole.objects.first() + + device_type = DeviceType.objects.create( + manufacturer=manufacturer, model='Chassis with Bay', slug='chassis-with-bay' + ) + ModuleBayTemplate.objects.create(device_type=device_type, name='Bay A') + + module_type = ModuleType.objects.create(manufacturer=manufacturer, model='Module with Sub-bay') + ModuleBayTemplate.objects.create(module_type=module_type, name='Sub-bay 1') + + device = Device.objects.create( + name='Nested Bay Parent Device', device_type=device_type, role=device_role, site=site + ) + parent_bay = device.modulebays.get(name='Bay A') + module = Module.objects.create(device=device, module_bay=parent_bay, module_type=module_type) + + nested_bay = module.modulebays.get(name='Sub-bay 1') + self.assertEqual(nested_bay.parent, parent_bay) + # The ltree path/level must reflect the nesting, not a root placement. + self.assertEqual(nested_bay.level, parent_bay.level + 1) + self.assertTrue(str(nested_bay.path).startswith(f'{parent_bay.path}.')) + @tag('regression') # #20467 def test_nested_module_bay_position_resolution(self): """Test that {module} in a module bay template's position field is resolved when the module is installed.""" diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index ecf05a86506..d6b4093960a 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -12,7 +12,7 @@ inserts and parent_id changes via refresh_from_db(fields=['path']). """ from django.db import migrations, models -from django.db.models import Count, ForeignKey, Lookup, ManyToManyField, Q +from django.db.models import ForeignKey, Lookup, ManyToManyField, Q from django.db.models.expressions import RawSQL from utilities.querysets import RestrictedQuerySet @@ -168,9 +168,23 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F count_attr: RawSQL(sql, [], output_field=models.IntegerField()) }) - # Non-cumulative: direct count. + # Non-cumulative: count only rows pointing directly at each node. Mirrors the + # cumulative branches but joins on equality instead of the `<@` subtree test. if is_many_to_many: - return queryset.annotate(**{count_attr: Count(rel_field, distinct=True)}) + field = model._meta.get_field(rel_field) + m2m_table = field.remote_field.through._meta.db_table + m2m_to_child_col = field.m2m_column_name() + m2m_to_tree_col = field.m2m_reverse_name() + sql = f'''( + SELECT COUNT(DISTINCT "{related_table}"."id") + FROM "{related_table}" + INNER JOIN "{m2m_table}" + ON "{related_table}"."id" = "{m2m_table}"."{m2m_to_child_col}" + WHERE "{m2m_table}"."{m2m_to_tree_col}" = "{parent_table}"."id" + )''' + return queryset.annotate(**{ + count_attr: RawSQL(sql, [], output_field=models.IntegerField()) + }) if has_generic_fk: ct_app = queryset.model._meta.app_label ct_model = queryset.model._meta.model_name @@ -186,7 +200,15 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F return queryset.annotate(**{ count_attr: RawSQL(sql, [ct_app, ct_model], output_field=models.IntegerField()) }) - return queryset.annotate(**{count_attr: Count(rel_field, distinct=True)}) + rel_field_col = f'{rel_field}_id' + sql = f'''( + SELECT COUNT(DISTINCT "{related_table}"."id") + FROM "{related_table}" + WHERE "{related_table}"."{rel_field_col}" = "{parent_table}"."id" + )''' + return queryset.annotate(**{ + count_attr: RawSQL(sql, [], output_field=models.IntegerField()) + }) class LtreeManager(models.Manager.from_queryset(LtreeQuerySet)): @@ -220,6 +242,19 @@ class LtreeModel(models.Model): update its sort_path — matching django-mptt's `order_insertion_by` behavior. Call `rebuild_sort_paths()` to recompute sort_path for every row from current names. + + Concurrency: + Path maintenance takes no table-wide lock (unlike django-mptt, which + acquired a per-model advisory lock on every write to protect its global + lft/rght/tree_id numbering). Because a row's path depends only on its + parent's path, concurrent inserts and moves under *different* parents + never conflict. One narrow race remains: inserting (or moving a node) + under parent P while P — or an ancestor of P — is concurrently reparented + in another transaction. The BEFORE trigger reads P's pre-move path under + its own snapshot, so the row may persist with a stale path. This requires + concurrent mutation of the same subtree and is accepted as the trade-off + for dropping the table-wide lock; if strict serialization is ever needed, + lock the parent row (`SELECT ... FOR SHARE`) inside the BEFORE trigger. """ path = LtreeField(editable=False, null=False, blank=True, default='') diff --git a/netbox/tenancy/migrations/0025_ltree_paths.py b/netbox/tenancy/migrations/0025_ltree_paths.py index c18b75cf56f..bda57d3df04 100644 --- a/netbox/tenancy/migrations/0025_ltree_paths.py +++ b/netbox/tenancy/migrations/0025_ltree_paths.py @@ -1,4 +1,8 @@ -"""Replace django-mptt with PostgreSQL ltree for tenancy's hierarchical models.""" +"""Replace django-mptt with PostgreSQL ltree for tenancy's hierarchical models. + +The reverse migration is lossy: it re-adds the MPTT columns empty and does not +rebuild the tree. Forward migration is the supported direction. +""" import django.db.models.deletion from django.contrib.postgres.indexes import GistIndex from django.contrib.postgres.operations import CreateExtension diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py index 11601c6bb72..143bcbe8dfa 100644 --- a/netbox/utilities/tests/test_ltree.py +++ b/netbox/utilities/tests/test_ltree.py @@ -247,6 +247,39 @@ def test_cumulative_m2m_count(self): self.assertEqual(counts['Child'], 2) self.assertEqual(counts['Leaf'], 1) + def test_noncumulative_fk_count(self): + """Non-cumulative FK count includes only rows pointing directly at the node.""" + root = Region.objects.create(name='R', slug='r-ncfk') + child = Region.objects.create(parent=root, name='C', slug='c-ncfk') + Site.objects.create(name='S1', slug='s1-ncfk', region=child) + Site.objects.create(name='S2', slug='s2-ncfk', region=root) + + qs = Region.objects.add_related_count( + Region.objects.filter(slug__endswith='-ncfk'), + Site, 'region', 'site_count', cumulative=False, + ) + counts = {r.name: r.site_count for r in qs} + # Each node counts only its own directly-assigned sites (no subtree rollup). + self.assertEqual(counts['R'], 1) + self.assertEqual(counts['C'], 1) + + def test_noncumulative_m2m_count(self): + """Non-cumulative M2M count includes only directly-assigned rows, not the subtree.""" + root = ContactGroup.objects.create(name='Root', slug='root-ncm2m') + child = ContactGroup.objects.create(parent=root, name='Child', slug='child-ncm2m') + c_root = Contact.objects.create(name='CR-nc') + c_child = Contact.objects.create(name='CC-nc') + c_root.groups.add(root) + c_child.groups.add(child) + + qs = ContactGroup.objects.add_related_count( + ContactGroup.objects.filter(slug__endswith='-ncm2m'), + Contact, 'groups', 'contact_count', cumulative=False, + ) + counts = {g.name: g.contact_count for g in qs} + self.assertEqual(counts['Root'], 1) + self.assertEqual(counts['Child'], 1) + class SaveUpdateFieldsTests(TestCase): """ diff --git a/netbox/wireless/migrations/0020_ltree_paths.py b/netbox/wireless/migrations/0020_ltree_paths.py index 72589351d36..90b4e1e386f 100644 --- a/netbox/wireless/migrations/0020_ltree_paths.py +++ b/netbox/wireless/migrations/0020_ltree_paths.py @@ -1,4 +1,8 @@ -"""Replace django-mptt with PostgreSQL ltree for wireless's hierarchical models.""" +"""Replace django-mptt with PostgreSQL ltree for wireless's hierarchical models. + +The reverse migration is lossy: it re-adds the MPTT columns empty and does not +rebuild the tree. Forward migration is the supported direction. +""" import django.db.models.deletion from django.contrib.postgres.indexes import GistIndex from django.contrib.postgres.operations import CreateExtension From 40a1d54a2114f718a37df8538296677222fe51ae Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 2 Jun 2026 13:25:20 -0700 Subject: [PATCH 14/42] fixes --- netbox/dcim/migrations/0237_ltree_paths.py | 13 +- netbox/netbox/models/ltree.py | 183 +++++++++++++-------- netbox/utilities/tests/test_ltree.py | 98 ++++++++--- 3 files changed, 197 insertions(+), 97 deletions(-) diff --git a/netbox/dcim/migrations/0237_ltree_paths.py b/netbox/dcim/migrations/0237_ltree_paths.py index 1886ac7a30d..02ec1fca8bb 100644 --- a/netbox/dcim/migrations/0237_ltree_paths.py +++ b/netbox/dcim/migrations/0237_ltree_paths.py @@ -17,10 +17,17 @@ 7. Adds a GiST index on path (descendant/ancestor lookups via `<@` / `@>`). For sort_path models, also adds a btree index for ORDER BY listing. +!!! OPERATOR WARNING !!! + +Step 4 runs ONE recursive-CTE UPDATE per table over every row, taking a +row-exclusive lock on the entire table for the duration of the statement. +On large deployments — particularly dcim_inventoryitem, which can contain +millions of rows — this can block writes for minutes. Plan a maintenance +window and budget accordingly. The other tables (region, sitegroup, +location, devicerole, platform, modulebay, inventoryitemtemplate) are +typically far smaller. + Notes: -- Step 4 populates each table with a single recursive-CTE UPDATE over all rows. - On very large tables (notably InventoryItem) this is one long-running statement - holding a row-exclusive lock for its duration; budget for it during upgrades. - The reverse migration is lossy: it re-adds the MPTT columns (lft/rght/tree_id/ level) empty and does NOT rebuild the tree. Forward migration is the supported direction. diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index d6b4093960a..65ef6de8b28 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -11,7 +11,7 @@ mutates paths directly; it only reads `path` back from the database after inserts and parent_id changes via refresh_from_db(fields=['path']). """ -from django.db import migrations, models +from django.db import connection, migrations, models from django.db.models import ForeignKey, Lookup, ManyToManyField, Q from django.db.models.expressions import RawSQL @@ -59,18 +59,22 @@ def as_sql(self, compiler, connection): @LtreeField.register_lookup class Descendant(Lookup): - """`path` is a descendant of (or equal to) the queried path: path <@ rhs""" + """ + `path` is a strict descendant of the queried path: path <@ rhs AND path <> rhs. + + Use `descendant_or_equal` for the inclusive form (`<@`). + """ lookup_name = 'descendant' def as_sql(self, compiler, connection): lhs, lhs_params = self.process_lhs(compiler, connection) rhs, rhs_params = self.process_rhs(compiler, connection) - return f'{lhs} <@ {rhs}', lhs_params + rhs_params + return f'({lhs} <@ {rhs} AND {lhs} <> {rhs})', lhs_params + rhs_params + lhs_params + rhs_params @LtreeField.register_lookup class DescendantOrEqual(Lookup): - """Alias of `descendant`; `<@` is inclusive in PostgreSQL ltree.""" + """`path` is a descendant of (or equal to) the queried path: path <@ rhs""" lookup_name = 'descendant_or_equal' def as_sql(self, compiler, connection): @@ -112,26 +116,27 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F and not has_direct_fk and not is_many_to_many ) - parent_table = queryset.model._meta.db_table - related_table = model._meta.db_table + qn = connection.ops.quote_name + parent_table = qn(queryset.model._meta.db_table) + related_table = qn(model._meta.db_table) if cumulative: if is_many_to_many: field = model._meta.get_field(rel_field) - m2m_table = field.remote_field.through._meta.db_table + m2m_table = qn(field.remote_field.through._meta.db_table) # `model` is the declaring side (the side holding the items to count); # `queryset.model` is the related (tree) side. m2m_column_name() points # at the declaring model; m2m_reverse_name() points at the related model. - m2m_to_child_col = field.m2m_column_name() - m2m_to_tree_col = field.m2m_reverse_name() + m2m_to_child_col = qn(field.m2m_column_name()) + m2m_to_tree_col = qn(field.m2m_reverse_name()) sql = f'''( - SELECT COUNT(DISTINCT "{related_table}"."id") - FROM "{related_table}" - INNER JOIN "{m2m_table}" - ON "{related_table}"."id" = "{m2m_table}"."{m2m_to_child_col}" - INNER JOIN "{parent_table}" AS subtree - ON "{m2m_table}"."{m2m_to_tree_col}" = subtree."id" - WHERE subtree."path" <@ "{parent_table}"."path" + SELECT COUNT(DISTINCT {related_table}."id") + FROM {related_table} + INNER JOIN {m2m_table} + ON {related_table}."id" = {m2m_table}.{m2m_to_child_col} + INNER JOIN {parent_table} AS subtree + ON {m2m_table}.{m2m_to_tree_col} = subtree."id" + WHERE subtree."path" <@ {parent_table}."path" )''' return queryset.annotate(**{ count_attr: RawSQL(sql, [], output_field=models.IntegerField()) @@ -143,26 +148,27 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F ct_app = queryset.model._meta.app_label ct_model = queryset.model._meta.model_name sql = f'''( - SELECT COUNT(DISTINCT "{related_table}"."id") - FROM "{related_table}" - INNER JOIN "{parent_table}" AS subtree - ON "{related_table}"."scope_id" = subtree."id" - WHERE "{related_table}"."scope_type_id" = ( + SELECT COUNT(DISTINCT {related_table}."id") + FROM {related_table} + INNER JOIN {parent_table} AS subtree + ON {related_table}."scope_id" = subtree."id" + WHERE {related_table}."scope_type_id" = ( SELECT id FROM django_content_type WHERE app_label = %s AND model = %s ) - AND subtree."path" <@ "{parent_table}"."path" + AND subtree."path" <@ {parent_table}."path" )''' return queryset.annotate(**{ count_attr: RawSQL(sql, [ct_app, ct_model], output_field=models.IntegerField()) }) - rel_field_col = f'{rel_field}_id' + # Use field.column (not f'{rel_field}_id') so custom db_column works. + rel_field_col = qn(field.column) sql = f'''( - SELECT COUNT(DISTINCT "{related_table}"."id") - FROM "{related_table}" - INNER JOIN "{parent_table}" AS subtree - ON "{related_table}"."{rel_field_col}" = subtree."id" - WHERE subtree."path" <@ "{parent_table}"."path" + SELECT COUNT(DISTINCT {related_table}."id") + FROM {related_table} + INNER JOIN {parent_table} AS subtree + ON {related_table}.{rel_field_col} = subtree."id" + WHERE subtree."path" <@ {parent_table}."path" )''' return queryset.annotate(**{ count_attr: RawSQL(sql, [], output_field=models.IntegerField()) @@ -172,15 +178,15 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F # cumulative branches but joins on equality instead of the `<@` subtree test. if is_many_to_many: field = model._meta.get_field(rel_field) - m2m_table = field.remote_field.through._meta.db_table - m2m_to_child_col = field.m2m_column_name() - m2m_to_tree_col = field.m2m_reverse_name() + m2m_table = qn(field.remote_field.through._meta.db_table) + m2m_to_child_col = qn(field.m2m_column_name()) + m2m_to_tree_col = qn(field.m2m_reverse_name()) sql = f'''( - SELECT COUNT(DISTINCT "{related_table}"."id") - FROM "{related_table}" - INNER JOIN "{m2m_table}" - ON "{related_table}"."id" = "{m2m_table}"."{m2m_to_child_col}" - WHERE "{m2m_table}"."{m2m_to_tree_col}" = "{parent_table}"."id" + SELECT COUNT(DISTINCT {related_table}."id") + FROM {related_table} + INNER JOIN {m2m_table} + ON {related_table}."id" = {m2m_table}.{m2m_to_child_col} + WHERE {m2m_table}.{m2m_to_tree_col} = {parent_table}."id" )''' return queryset.annotate(**{ count_attr: RawSQL(sql, [], output_field=models.IntegerField()) @@ -189,10 +195,10 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F ct_app = queryset.model._meta.app_label ct_model = queryset.model._meta.model_name sql = f'''( - SELECT COUNT(DISTINCT "{related_table}"."id") - FROM "{related_table}" - WHERE "{related_table}"."scope_id" = "{parent_table}"."id" - AND "{related_table}"."scope_type_id" = ( + SELECT COUNT(DISTINCT {related_table}."id") + FROM {related_table} + WHERE {related_table}."scope_id" = {parent_table}."id" + AND {related_table}."scope_type_id" = ( SELECT id FROM django_content_type WHERE app_label = %s AND model = %s ) @@ -200,11 +206,11 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F return queryset.annotate(**{ count_attr: RawSQL(sql, [ct_app, ct_model], output_field=models.IntegerField()) }) - rel_field_col = f'{rel_field}_id' + rel_field_col = qn(field.column) sql = f'''( - SELECT COUNT(DISTINCT "{related_table}"."id") - FROM "{related_table}" - WHERE "{related_table}"."{rel_field_col}" = "{parent_table}"."id" + SELECT COUNT(DISTINCT {related_table}."id") + FROM {related_table} + WHERE {related_table}.{rel_field_col} = {parent_table}."id" )''' return queryset.annotate(**{ count_attr: RawSQL(sql, [], output_field=models.IntegerField()) @@ -236,12 +242,15 @@ class LtreeModel(models.Model): the batch will be persisted with a root-level path (the parent row is not yet visible to the lookup). - Sort-path staleness: + Sort-path on rename: For subclasses with the optional `sort_path` column (see - InstallLtreeTriggers' `name_column` arg), renaming a row does NOT - update its sort_path — matching django-mptt's - `order_insertion_by` behavior. Call `rebuild_sort_paths()` to - recompute sort_path for every row from current names. + InstallLtreeTriggers' `name_column` arg), renaming a row updates its + own sort_path AND cascades into descendants' sort_paths via the AFTER + trigger. This diverges from django-mptt's `order_insertion_by` (which + leaves both the renamed row and its descendants stale until a manual + rebuild) because list views are expected to reflect renames promptly. + `rebuild_sort_paths()` is still available for bulk repair after raw + SQL writes that bypass the triggers. Concurrency: Path maintenance takes no table-wide lock (unlike django-mptt, which @@ -256,6 +265,12 @@ class LtreeModel(models.Model): for dropping the table-wide lock; if strict serialization is ever needed, lock the parent row (`SELECT ... FOR SHARE`) inside the BEFORE trigger. """ + # `default=''` here is a Django-side placeholder that the BEFORE INSERT + # trigger always overwrites with a valid path before the row reaches + # storage. Empty ltree (`''`) is itself a valid PostgreSQL ltree value + # (nlevel = 0) in supported PostgreSQL versions (15+), so even harnesses + # that bypass the trigger will not fail at INSERT — they will simply + # store a zero-level path. path = LtreeField(editable=False, null=False, blank=True, default='') objects = LtreeManager() @@ -356,17 +371,13 @@ def get_ancestors(self, ascending=False, include_self=False): def get_descendants(self, include_self=False): if not self.path: return type(self)._default_manager.none() - qs = type(self)._default_manager.filter(path__descendant=self.path) - if not include_self: - qs = qs.exclude(pk=self.pk) - return qs.order_by('path') + lookup = 'descendant_or_equal' if include_self else 'descendant' + return type(self)._default_manager.filter(**{f'path__{lookup}': self.path}).order_by('path') def get_descendant_count(self): if not self.path: return 0 - return type(self)._default_manager.filter( - path__descendant=self.path - ).exclude(pk=self.pk).count() + return type(self)._default_manager.filter(path__descendant=self.path).count() def get_children(self): return type(self)._default_manager.filter(parent_id=self.pk) @@ -376,7 +387,7 @@ def get_family(self): if not self.path: return type(self)._default_manager.none() return type(self)._default_manager.filter( - Q(path__ancestor=self.path) | Q(path__descendant=self.path) + Q(path__ancestor=self.path) | Q(path__descendant_or_equal=self.path) ).distinct().order_by('path') def get_siblings(self, include_self=False): @@ -487,9 +498,12 @@ def rebuild_sort_paths(cls, name_column='name'): # standard collation). ORDER BY sort_path then gives MPTT-equivalent # tree-flatten ordering with siblings in name (collation) order. # -# Like MPTT's order_insertion_by, sort_path is computed at insert and -# reparent only — renaming a node does NOT reposition it. A manual rebuild() -# would be needed to re-sort everything by current names. +# The BEFORE trigger fires on INSERT, parent_id changes, and name changes, so +# a rename updates the row's own sort_path; the AFTER trigger then cascades +# the new sort_path into descendants. (django-mptt's `order_insertion_by` +# stops at the renamed node and leaves descendants stale until a manual +# rebuild — NetBox auto-cascades because operators expect renames to flow +# through. `rebuild_sort_paths()` is still available for bulk repair.) _COMPUTE_PATH_AND_SORT_FN = ''' CREATE OR REPLACE FUNCTION "{table}_ltree_compute_path_fn"() RETURNS TRIGGER AS $$ DECLARE @@ -525,25 +539,44 @@ def rebuild_sort_paths(cls, name_column='name'): $$ LANGUAGE plpgsql; ''' -_BEFORE_TRIGGER = ''' +_BEFORE_TRIGGER_PATH_ONLY = ''' CREATE TRIGGER "{table}_ltree_compute_path" BEFORE INSERT OR UPDATE OF parent_id ON "{table}" FOR EACH ROW EXECUTE FUNCTION "{table}_ltree_compute_path_fn"(); ''' -# Fire only on parent_id changes, not on path changes. A single reparent's -# cascade rewrites the whole subtree (WHERE path <@ OLD.path) at every depth in -# one statement, so it never needs to re-fire; including `path` here would make -# every descendant the cascade touches re-fire the trigger, each running a no-op -# cascade against its now-vacated OLD.path. Nothing mutates `path` without also -# touching `parent_id`, so `parent_id` alone is sufficient. -_AFTER_TRIGGER = ''' +# For path+sort tables, also fire on UPDATE OF {name_col} so that renaming a +# node recomputes its sort_path. The cascade trigger then propagates the new +# sort_path to descendants. +_BEFORE_TRIGGER_PATH_AND_SORT = ''' +CREATE TRIGGER "{table}_ltree_compute_path" + BEFORE INSERT OR UPDATE OF parent_id, "{name_col}" ON "{table}" + FOR EACH ROW EXECUTE FUNCTION "{table}_ltree_compute_path_fn"(); +''' + +# AFTER trigger fires on the columns that operators / Django write directly +# (parent_id and the name column) — NOT on path or sort_path. The cascade +# function rewrites path/sort_path on descendants in a single statement, and +# because that statement does not touch parent_id or {name_col}, the AFTER +# trigger does not re-fire on those descendant rows. This prevents the +# quadratic re-cascade that would otherwise occur for any deep subtree. +_AFTER_TRIGGER_PATH_ONLY = ''' CREATE TRIGGER "{table}_ltree_cascade_path" AFTER UPDATE OF parent_id ON "{table}" FOR EACH ROW WHEN (OLD.path IS DISTINCT FROM NEW.path) EXECUTE FUNCTION "{table}_ltree_cascade_path_fn"(); ''' +_AFTER_TRIGGER_PATH_AND_SORT = ''' +CREATE TRIGGER "{table}_ltree_cascade_path" + AFTER UPDATE OF parent_id, "{name_col}" ON "{table}" + FOR EACH ROW WHEN ( + OLD.path IS DISTINCT FROM NEW.path + OR OLD.sort_path IS DISTINCT FROM NEW.sort_path + ) + EXECUTE FUNCTION "{table}_ltree_cascade_path_fn"(); +''' + class InstallLtreeTriggers(migrations.operations.base.Operation): """ @@ -557,8 +590,8 @@ class InstallLtreeTriggers(migrations.operations.base.Operation): If `name_column` is provided, the model is expected to have a `sort_path` text column whose value will be maintained as a chr(1)-separated chain of ancestor names. This implements MPTT's `order_insertion_by=(name,)` - semantics: insert and reparent honor the current value of `name_column`; - rename does NOT reposition the node (matching MPTT behavior). + semantics: insert, reparent, and rename all honor the current value of + `name_column`, with renames cascaded into descendants' sort_paths. """ reversible = True @@ -577,11 +610,17 @@ def database_forwards(self, app_label, schema_editor, from_state, to_state): schema_editor.execute(_CASCADE_PATH_AND_SORT_FN.format( table=self.table_name, )) + schema_editor.execute(_BEFORE_TRIGGER_PATH_AND_SORT.format( + table=self.table_name, name_col=self.name_column, + )) + schema_editor.execute(_AFTER_TRIGGER_PATH_AND_SORT.format( + table=self.table_name, name_col=self.name_column, + )) else: schema_editor.execute(_COMPUTE_PATH_ONLY_FN.format(table=self.table_name)) schema_editor.execute(_CASCADE_PATH_ONLY_FN.format(table=self.table_name)) - schema_editor.execute(_BEFORE_TRIGGER.format(table=self.table_name)) - schema_editor.execute(_AFTER_TRIGGER.format(table=self.table_name)) + schema_editor.execute(_BEFORE_TRIGGER_PATH_ONLY.format(table=self.table_name)) + schema_editor.execute(_AFTER_TRIGGER_PATH_ONLY.format(table=self.table_name)) def database_backwards(self, app_label, schema_editor, from_state, to_state): t = self.table_name diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py index 143bcbe8dfa..ba268fb493e 100644 --- a/netbox/utilities/tests/test_ltree.py +++ b/netbox/utilities/tests/test_ltree.py @@ -337,30 +337,84 @@ def test_parent_before_child_in_same_batch(self): self.assertEqual(child_pending.path, _path(root.pk, parent_pending.pk, child_pending.pk)) -class RebuildSortPathsTests(TestCase): +class DescendantLookupSemanticsTests(TestCase): """ - rebuild_sort_paths() recomputes sort_path from current names. The stale- - sort case shows up in descendants: renaming a parent and saving it - recomputes that one row's sort_path, but descendants' sort_paths still - contain the parent's old name until rebuilt. + path__descendant is strict (path <@ rhs AND path != rhs); the inclusive + form is path__descendant_or_equal. Previously both were inclusive. """ - def test_descendant_sort_paths_stale_then_rebuilt(self): - parent = Region.objects.create(name='Bravo', slug='bravo-rsp') - Region.objects.create(parent=parent, name='Mid', slug='mid-rsp') + def test_strict_descendant_excludes_self(self): + root = Region.objects.create(name='Root', slug='root-dls') + Region.objects.create(parent=root, name='Kid', slug='kid-dls') + strict = list( + Region.objects.filter(path__descendant=root.path) + .values_list('name', flat=True) + ) + self.assertEqual(sorted(strict), ['Kid']) + inclusive = list( + Region.objects.filter(path__descendant_or_equal=root.path) + .values_list('name', flat=True) + ) + self.assertEqual(sorted(inclusive), ['Kid', 'Root']) + + +class RenameCascadesSortPathTests(TestCase): + """ + Renaming a node updates its own sort_path AND cascades into descendants' + sort_paths via the AFTER trigger. (Diverges from MPTT order_insertion_by, + which leaves descendants stale until manual rebuild.) + """ + + def test_rename_cascades_into_descendants(self): + parent = Region.objects.create(name='Bravo', slug='bravo-rcsp') + mid = Region.objects.create(parent=parent, name='Mid', slug='mid-rcsp') + leaf = Region.objects.create(parent=mid, name='Leaf', slug='leaf-rcsp') - # Rename parent. AFTER cascade only fires when path changes, so - # descendants keep their old sort_path embedding 'Bravo'. parent.name = 'Zulu' parent.save() - mid_sort = Region.objects.values_list('sort_path', flat=True).get(slug='mid-rsp') - self.assertIn('Bravo', mid_sort, "descendant sort_path should still contain old name") - self.assertNotIn('Zulu', mid_sort) + + parent.refresh_from_db() + mid.refresh_from_db() + leaf.refresh_from_db() + self.assertEqual(parent.sort_path, 'Zulu') + self.assertEqual(mid.sort_path, f'Zulu{chr(1)}Mid') + self.assertEqual(leaf.sort_path, f'Zulu{chr(1)}Mid{chr(1)}Leaf') + # Paths unchanged — only sort_path moved. + self.assertEqual(mid.path, _path(parent.pk, mid.pk)) + self.assertEqual(leaf.path, _path(parent.pk, mid.pk, leaf.pk)) + + def test_rename_does_not_affect_unrelated_subtree(self): + # Two roots; renaming one must not touch the other's sort_path. + a = Region.objects.create(name='AA', slug='aa-iso') + Region.objects.create(parent=a, name='AKid', slug='akid-iso') + b = Region.objects.create(name='BB', slug='bb-iso') + b_kid = Region.objects.create(parent=b, name='BKid', slug='bkid-iso') + + a.name = 'AAren' + a.save() + + b_kid.refresh_from_db() + self.assertEqual(b_kid.sort_path, f'BB{chr(1)}BKid') + + +class RebuildSortPathsTests(TestCase): + """rebuild_sort_paths() is still available for repair after raw SQL writes.""" + + def test_rebuild_after_raw_update(self): + parent = Region.objects.create(name='Bravo', slug='bravo-rsp') + mid = Region.objects.create(parent=parent, name='Mid', slug='mid-rsp') + # Raw .update() with only the name column bypasses the BEFORE trigger + # (its column list is parent_id + name, but the trigger is keyed on + # name in the SET clause). Actually update() on `name` DOES fire the + # trigger now — so to simulate a bypass we corrupt sort_path directly. + Region.objects.filter(pk=parent.pk).update(sort_path='garbage') + Region.objects.filter(pk=mid.pk).update(sort_path='also-garbage') Region.rebuild_sort_paths() - mid_sort = Region.objects.values_list('sort_path', flat=True).get(slug='mid-rsp') - self.assertIn('Zulu', mid_sort) - self.assertNotIn('Bravo', mid_sort) + parent.refresh_from_db() + mid.refresh_from_db() + self.assertEqual(parent.sort_path, 'Bravo') + self.assertEqual(mid.sort_path, f'Bravo{chr(1)}Mid') def test_raises_without_sort_path(self): # InventoryItem uses LtreeModel but doesn't have a sort_path column. @@ -425,18 +479,18 @@ def test_reparent_postchange_snapshot_matches_db(self): class CascadeTriggerScopeTests(TestCase): """ - The AFTER cascade trigger fires on parent_id changes only; renames (which do - not touch parent_id or path) must not cascade. + The AFTER cascade trigger fires on parent_id or name changes. A rename + leaves `path` untouched but pushes the new sort_path into descendants. """ - def test_rename_does_not_cascade_to_descendants(self): + def test_rename_preserves_descendant_path_but_updates_sort_path(self): parent = Region.objects.create(name='Bravo', slug='bravo-ct') child = Region.objects.create(parent=parent, name='Mid', slug='mid-ct') original_child_path = child.path parent.name = 'Zulu' parent.save() child.refresh_from_db() - # Path is unaffected by a rename, and the descendant keeps the old - # name embedding in sort_path (MPTT order_insertion_by parity). + # Path is unaffected by a rename; sort_path follows the new name. self.assertEqual(child.path, original_child_path) - self.assertIn('Bravo', child.sort_path) + self.assertNotIn('Bravo', child.sort_path) + self.assertIn('Zulu', child.sort_path) From c6ef4ace7e05b1057d7794bcc154b23bb9a9afde Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 2 Jun 2026 15:24:14 -0700 Subject: [PATCH 15/42] fixes --- netbox/dcim/graphql/types.py | 7 +- .../dcim/models/device_component_templates.py | 9 ++ netbox/dcim/models/device_components.py | 16 +- netbox/dcim/tests/test_models.py | 62 ++++++++ netbox/netbox/graphql/types.py | 24 +++ netbox/netbox/models/__init__.py | 13 +- netbox/netbox/models/ltree.py | 137 ++++++++++++++---- netbox/tenancy/models/tenants.py | 3 +- netbox/utilities/query.py | 5 +- netbox/utilities/tests/test_ltree.py | 71 ++++++++- netbox/wireless/models.py | 3 +- 11 files changed, 304 insertions(+), 46 deletions(-) diff --git a/netbox/dcim/graphql/types.py b/netbox/dcim/graphql/types.py index 3ac92c06517..d8f93189118 100644 --- a/netbox/dcim/graphql/types.py +++ b/netbox/dcim/graphql/types.py @@ -12,6 +12,7 @@ from netbox.graphql.scalars import BigInt from netbox.graphql.types import ( BaseObjectType, + LtreeNodeMixin, NestedGroupObjectType, NetBoxObjectType, OrganizationalObjectType, @@ -326,7 +327,7 @@ class DeviceBayTemplateType(ComponentTemplateType): filters=InventoryItemTemplateFilter, pagination=True ) -class InventoryItemTemplateType(ComponentTemplateType): +class InventoryItemTemplateType(LtreeNodeMixin, ComponentTemplateType): role: Annotated['InventoryItemRoleType', strawberry.lazy('dcim.graphql.types')] | None manufacturer: Annotated['ManufacturerType', strawberry.lazy('dcim.graphql.types')] @@ -491,7 +492,7 @@ class InterfaceTemplateType(ModularComponentTemplateType): filters=InventoryItemFilter, pagination=True ) -class InventoryItemType(ComponentType): +class InventoryItemType(LtreeNodeMixin, ComponentType): role: Annotated['InventoryItemRoleType', strawberry.lazy('dcim.graphql.types')] | None manufacturer: Annotated['ManufacturerType', strawberry.lazy('dcim.graphql.types')] | None @@ -606,7 +607,7 @@ class ModuleType(PrimaryObjectType): filters=ModuleBayFilter, pagination=True ) -class ModuleBayType(ModularComponentType): +class ModuleBayType(LtreeNodeMixin, ModularComponentType): installed_module: Annotated["ModuleType", strawberry.lazy('dcim.graphql.types')] | None children: list[Annotated["ModuleBayType", strawberry.lazy('dcim.graphql.types')]] diff --git a/netbox/dcim/models/device_component_templates.py b/netbox/dcim/models/device_component_templates.py index 728bd15a976..9a0a73486d5 100644 --- a/netbox/dcim/models/device_component_templates.py +++ b/netbox/dcim/models/device_component_templates.py @@ -885,6 +885,15 @@ class Meta: verbose_name = _('inventory item template') verbose_name_plural = _('inventory item templates') + def clean(self): + super().clean() + + # A template cannot be its own parent or a descendant of itself + if self.pk and self._parent_creates_cycle(): + raise ValidationError({ + "parent": _("Cannot assign self or a descendant as parent.") + }) + def instantiate(self, **kwargs): parent = InventoryItem.objects.get(name=self.parent.name, **kwargs) if self.parent else None if self.component: diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index 4dae6af0c1e..044059aae1c 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -15,7 +15,7 @@ from dcim.models.mixins import InterfaceValidationMixin from netbox.choices import ColorChoices from netbox.models import NetBoxModel, OrganizationalModel -from netbox.models.ltree import LtreeManager, LtreeModel +from netbox.models.ltree import LtreeManager, LtreeModel, SortPathField from netbox.models.mixins import OwnerMixin from utilities.fields import ColorField, NaturalOrderingField from utilities.ordering import naturalize_interface @@ -1336,7 +1336,7 @@ class ModuleBay(ModularComponentModel, TrackingModelMixin, LtreeModel): verbose_name=_('enabled'), default=True, ) - sort_path = models.TextField( + sort_path = SortPathField( editable=False, blank=True, default='', @@ -1386,6 +1386,12 @@ def save(self, *args, **kwargs): self.parent = None super().save(*args, **kwargs) + def _parent_creates_cycle(self): + # A ModuleBay's parent is system-derived from its module (see save()), not + # user-assigned, and module/bay recursion is validated in clean(); skip the + # generic ltree cycle guard. + return False + @property def _occupied(self): """ @@ -1572,10 +1578,10 @@ class Meta: def clean(self): super().clean() - # An InventoryItem cannot be its own parent - if self.pk and self.parent_id == self.pk: + # An InventoryItem cannot be its own parent or a descendant of itself + if self.pk and self._parent_creates_cycle(): raise ValidationError({ - "parent": _("Cannot assign self as parent.") + "parent": _("Cannot assign self or a descendant as parent.") }) # Validation for moving InventoryItems diff --git a/netbox/dcim/tests/test_models.py b/netbox/dcim/tests/test_models.py index d251ad2735f..535c332ad9d 100644 --- a/netbox/dcim/tests/test_models.py +++ b/netbox/dcim/tests/test_models.py @@ -2561,3 +2561,65 @@ def test_three_phase_per_leg_recursive_aggregation(self): self.assertEqual(legs_by_name['A']['maximum'], 200) self.assertEqual(legs_by_name['B']['allocated'], 0) self.assertEqual(legs_by_name['C']['allocated'], 0) + + +class InventoryItemCycleTestCase(TestCase): + """ + InventoryItem (ltree-backed, not the nested-group base) must reject assigning + self or a descendant as parent — behavior django-mptt previously enforced via + InvalidMove on save(). + """ + @classmethod + def setUpTestData(cls): + site = Site.objects.create(name='Site 1', slug='inv-site-1') + manufacturer = Manufacturer.objects.create(name='Manufacturer 1', slug='inv-mfr-1') + device_type = DeviceType.objects.create( + manufacturer=manufacturer, model='Device Type 1', slug='inv-dt-1' + ) + role = DeviceRole.objects.create(name='Role 1', slug='inv-role-1') + cls.device = Device.objects.create( + name='Device 1', device_type=device_type, role=role, site=site + ) + + def test_cannot_assign_descendant_as_parent(self): + a = InventoryItem.objects.create(device=self.device, name='A') + b = InventoryItem.objects.create(device=self.device, name='B', parent=a) + c = InventoryItem.objects.create(device=self.device, name='C', parent=b) + a.parent = c + with self.assertRaises(ValidationError): + a.full_clean() + # The save()-level guard also rejects the cycle when clean() is bypassed. + with self.assertRaises(ValidationError): + a.save() + + def test_cannot_assign_self_as_parent(self): + a = InventoryItem.objects.create(device=self.device, name='A') + a.parent = a + with self.assertRaises(ValidationError): + a.full_clean() + + +class InventoryItemTemplateCycleTestCase(TestCase): + """InventoryItemTemplate must likewise reject self/descendant as parent.""" + + @classmethod + def setUpTestData(cls): + manufacturer = Manufacturer.objects.create(name='Manufacturer 1', slug='iit-mfr-1') + cls.device_type = DeviceType.objects.create( + manufacturer=manufacturer, model='Device Type 1', slug='iit-dt-1' + ) + + def test_cannot_assign_descendant_as_parent(self): + a = InventoryItemTemplate.objects.create(device_type=self.device_type, name='A') + b = InventoryItemTemplate.objects.create(device_type=self.device_type, name='B', parent=a) + a.parent = b + with self.assertRaises(ValidationError): + a.full_clean() + with self.assertRaises(ValidationError): + a.save() + + def test_cannot_assign_self_as_parent(self): + a = InventoryItemTemplate.objects.create(device_type=self.device_type, name='A') + a.parent = a + with self.assertRaises(ValidationError): + a.full_clean() diff --git a/netbox/netbox/graphql/types.py b/netbox/netbox/graphql/types.py index 10975385331..7c6b36bf607 100644 --- a/netbox/netbox/graphql/types.py +++ b/netbox/netbox/graphql/types.py @@ -1,6 +1,7 @@ import strawberry import strawberry_django from django.contrib.contenttypes.models import ContentType +from django.db.models import ExpressionWrapper, F, Func, IntegerField, Value from strawberry.types import Info from core.graphql.mixins import ChangelogMixin @@ -11,6 +12,7 @@ __all__ = ( 'BaseObjectType', 'ContentTypeType', + 'LtreeNodeMixin', 'NestedGroupObjectType', 'NetBoxObjectType', 'ObjectType', @@ -83,7 +85,29 @@ class OrganizationalObjectType( pass +@strawberry.type +class LtreeNodeMixin: + """ + Exposes the ltree-backed tree depth as a `level` field, preserving the `level` + field MPTT-based types previously surfaced automatically as a real column. + + The depth is computed in the database as `nlevel(path) - 1` (root = 0) and + annotated onto the queryset. We read the annotation rather than the `path` + column directly: `path` is excluded from the schema, so accessing it through + the resolver source would re-enter field resolution and recurse. + """ + @strawberry_django.field(annotate={ + 'ltree_level': ExpressionWrapper( + Func(F('path'), function='nlevel', output_field=IntegerField()) - Value(1), + output_field=IntegerField(), + ) + }) + def level(self) -> int: + return self.ltree_level + + class NestedGroupObjectType( + LtreeNodeMixin, ChangelogMixin, CustomFieldsMixin, JournalEntriesMixin, diff --git a/netbox/netbox/models/__init__.py b/netbox/netbox/models/__init__.py index 2ffc8c40d40..46f5637bb0f 100644 --- a/netbox/netbox/models/__init__.py +++ b/netbox/netbox/models/__init__.py @@ -8,7 +8,7 @@ from mptt.models import MPTTModel, TreeForeignKey from netbox.models.features import * -from netbox.models.ltree import LtreeManager, LtreeModel +from netbox.models.ltree import LtreeManager, LtreeModel, SortPathField from netbox.models.mixins import OwnerMixin from utilities.mptt import TreeManager from utilities.querysets import RestrictedQuerySet @@ -231,10 +231,11 @@ class NestedLtreeGroupModel(NestedGroupModelMixin, LtreeModel): Base model for objects which are used to form a hierarchy (regions, locations, etc.). These models nest recursively using PostgreSQL ltree. Within each parent, each child instance must have a unique name. - `sort_path` is a trigger-maintained text column that mirrors MPTT's `order_insertion_by=('name',)` - semantics: at insert and reparent time it is set to a chr(1)-separated chain of ancestor names. - Ordering by `sort_path` yields tree-flatten output with siblings in their column's collation order. - Renaming a node does NOT update `sort_path` (matching MPTT behavior). + `sort_path` is a trigger-maintained text column holding a chr(1)-separated chain of ancestor + names; ordering by it yields tree-flatten output with siblings in name (collation) order. + Inserts, reparents, AND renames all update `sort_path` (a rename cascades to descendants), so + list ordering reflects renames immediately — unlike django-mptt's `order_insertion_by`, which + left descendants stale until a manual rebuild. """ parent = models.ForeignKey( to='self', @@ -244,7 +245,7 @@ class NestedLtreeGroupModel(NestedGroupModelMixin, LtreeModel): null=True, db_index=True ) - sort_path = models.TextField( + sort_path = SortPathField( editable=False, blank=True, default='', diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index 65ef6de8b28..d3c541b34d4 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -11,9 +11,11 @@ mutates paths directly; it only reads `path` back from the database after inserts and parent_id changes via refresh_from_db(fields=['path']). """ +from django.core.exceptions import FieldDoesNotExist, ValidationError from django.db import connection, migrations, models from django.db.models import ForeignKey, Lookup, ManyToManyField, Q from django.db.models.expressions import RawSQL +from django.utils.translation import gettext_lazy as _ from utilities.querysets import RestrictedQuerySet @@ -23,6 +25,7 @@ 'LtreeManager', 'LtreeModel', 'LtreeQuerySet', + 'SortPathField', ) @@ -37,6 +40,13 @@ class LtreeField(models.TextField): """ description = "PostgreSQL ltree field" + # `path` is computed by a BEFORE INSERT trigger, so its final value isn't known + # until the row is written. Marking the field db_returning lets the + # INSERT ... RETURNING clause fetch the trigger-computed value in the same + # round-trip (PostgreSQL evaluates RETURNING after BEFORE triggers fire), + # avoiding a follow-up SELECT in LtreeModel.save(). Mirrors AutoFieldMixin. + db_returning = True + def db_type(self, connection): return 'ltree' @@ -83,6 +93,21 @@ def as_sql(self, compiler, connection): return f'{lhs} <@ {rhs}', lhs_params + rhs_params +class SortPathField(models.TextField): + """ + Text column holding the chr(1)-separated chain of ancestor names that drives + tree-flatten ordering. Like `path`, its value is maintained by triggers, so it + is marked db_returning to be populated via INSERT ... RETURNING without an + extra SELECT. It deconstructs as a plain TextField so existing migrations + (which created the column as TextField) require no schema change. + """ + db_returning = True + + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + return name, 'django.db.models.TextField', args, kwargs + + # # QuerySet / Manager # @@ -104,18 +129,26 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F is_many_to_many = False try: field = model._meta.get_field(rel_field) - if isinstance(field, ManyToManyField): - is_many_to_many = True - elif isinstance(field, ForeignKey): - has_direct_fk = True except Exception: - pass + field = None + if isinstance(field, ManyToManyField): + is_many_to_many = True + elif isinstance(field, ForeignKey): + has_direct_fk = True has_generic_fk = ( hasattr(model, 'scope_type') and hasattr(model, 'scope_id') and not has_direct_fk and not is_many_to_many ) + # The FK branches below dereference field.column, so fail loudly here if the + # field could not be resolved and this isn't the scope GenericFK pattern — + # rather than raising an opaque AttributeError deep in the SQL builder. + if field is None and not has_generic_fk: + raise FieldDoesNotExist( + f"{model._meta.label} has no field '{rel_field}' for add_related_count()" + ) + qn = connection.ops.quote_name parent_table = qn(queryset.model._meta.db_table) related_table = qn(model._meta.db_table) @@ -280,19 +313,47 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._loaded_parent_id = self.parent_id + # Read from __dict__ rather than via attribute access: a deferred + # `parent_id`/`name` (e.g. a GraphQL query selecting only a subset of + # fields) must not be lazily loaded here, since that triggers + # refresh_from_db() which rebuilds the instance and recurses into __init__. + self._loaded_parent_id = self.__dict__.get('parent_id') + self._loaded_name = self.__dict__.get('name') @classmethod def from_db(cls, db, field_names, values): instance = super().from_db(db, field_names, values) - instance._loaded_parent_id = instance.parent_id + instance._loaded_parent_id = instance.__dict__.get('parent_id') + instance._loaded_name = instance.__dict__.get('name') return instance + def _parent_creates_cycle(self): + """ + Return True if the current `parent` assignment would make this node its + own ancestor (the new parent is self or one of its descendants), mirroring + django-mptt's save-time InvalidMove guard. + + Subclasses whose `parent` is system-managed (e.g. ModuleBay, whose parent + is derived from its module) may override this to disable the check. + """ + if self.parent_id is None or not self.path: + return False + # The new parent lies inside this node's current subtree iff its path is a + # descendant of (or equal to) self.path. + return type(self)._default_manager.filter( + pk=self.parent_id, path__descendant_or_equal=self.path + ).exists() + def save(self, *args, **kwargs): """ Triggers compute `path` (and `sort_path`, where present) server-side. - After insert or after a parent change, refresh those columns so the - in-memory instance stays consistent with the database. + + On INSERT the trigger-maintained columns are db_returning, so they are + populated in-place by the INSERT ... RETURNING clause without an extra + query. On an UPDATE that changes `parent` or the name column the triggers + rewrite those columns server-side, so refresh them afterward to keep the + in-memory instance consistent (e.g. so change logging snapshots the value + the triggers actually wrote, not a stale one). """ is_insert = self._state.adding # When update_fields is supplied and excludes parent, the DB does not see @@ -302,21 +363,40 @@ def save(self, *args, **kwargs): update_fields = kwargs.get('update_fields') parent_written = update_fields is None or 'parent' in update_fields or 'parent_id' in update_fields parent_changed = (not is_insert) and parent_written and self.parent_id != self._loaded_parent_id + + # The sort_path trigger also fires on a name change; detect that so the + # cascaded sort_path can be refreshed below (path-only models have no + # sort_path and are unaffected by renames). + has_sort_path = any(f.attname == 'sort_path' for f in self._meta.concrete_fields) + name_written = update_fields is None or 'name' in update_fields + name_changed = ( + (not is_insert) and has_sort_path and name_written + and self.__dict__.get('name') != self._loaded_name + ) + + # Reject cyclic moves before writing, mirroring django-mptt's save-time + # guard so scripts / bulk callers (which bypass form & serializer clean()) + # cannot silently corrupt the tree. + if parent_changed and self._parent_creates_cycle(): + raise ValidationError(_("Cannot assign self or a descendant as parent.")) + super().save(*args, **kwargs) - if is_insert or parent_changed: - # Refresh every trigger-maintained column (`path`, plus `sort_path` on - # models that declare it) so the in-memory instance matches the row the - # triggers actually wrote — otherwise e.g. change logging would snapshot - # a stale value. + + if (parent_changed or name_changed) and not is_insert: + # The triggers rewrote path/sort_path on this UPDATE; fetch them back. + # (INSERT ... RETURNING covers the insert case, so only updates reach here.) refresh_fields = [ - name for name in ('path', 'sort_path') - if any(f.attname == name for f in self._meta.concrete_fields) + fname for fname in ('path', 'sort_path') + if any(f.attname == fname for f in self._meta.concrete_fields) ] row = type(self).objects.values_list(*refresh_fields).get(pk=self.pk) - for name, value in zip(refresh_fields, row): - setattr(self, name, value) + for fname, value in zip(refresh_fields, row): + setattr(self, fname, value) + if is_insert or parent_written: self._loaded_parent_id = self.parent_id + if is_insert or name_written: + self._loaded_name = self.__dict__.get('name') # -- MPTT-compatible API ------------------------------------------------ @@ -330,17 +410,19 @@ def level(self): def get_level(self): return self.level + def _root_pk(self): + """ + Integer PK of the root node, parsed from the first (zero-padded) path label. + Returns None for a node with no path. + """ + if not self.path: + return None + return int(str(self.path).split('.', 1)[0].lstrip('0') or '0') + @property def tree_id(self): """Integer PK of the root, mirroring django-mptt's `tree_id`.""" - if not self.path: - return None - # Strip leading zeros from the padded label - root_label = str(self.path).split('.', 1)[0].lstrip('0') or '0' - try: - return int(root_label) - except (TypeError, ValueError): - return root_label + return self._root_pk() def is_root_node(self): return self.parent_id is None @@ -354,8 +436,7 @@ def is_child_node(self): def get_root(self): if self.is_root_node(): return self - root_pk = int(str(self.path).split('.', 1)[0].lstrip('0') or '0') - return type(self)._default_manager.get(pk=root_pk) + return type(self)._default_manager.get(pk=self._root_pk()) def get_parent(self): return self.parent diff --git a/netbox/tenancy/models/tenants.py b/netbox/tenancy/models/tenants.py index 197729f2ea0..68e6e0252ee 100644 --- a/netbox/tenancy/models/tenants.py +++ b/netbox/tenancy/models/tenants.py @@ -5,6 +5,7 @@ from netbox.models import NestedLtreeGroupModel, PrimaryModel from netbox.models.features import ContactsMixin +from netbox.models.ltree import SortPathField __all__ = ( 'Tenant', @@ -28,7 +29,7 @@ class TenantGroup(NestedLtreeGroupModel): unique=True ) # Override the abstract parent's sort_path to use natural_sort, matching `name`. - sort_path = models.TextField( + sort_path = SortPathField( editable=False, blank=True, default='', diff --git a/netbox/utilities/query.py b/netbox/utilities/query.py index 662721226a5..f9852b36106 100644 --- a/netbox/utilities/query.py +++ b/netbox/utilities/query.py @@ -66,7 +66,10 @@ def reapply_model_ordering(queryset: QuerySet) -> QuerySet: """ # Hierarchical (ltree) models are exempt; their default ordering by `sort_path`/`path` # must not be clobbered by .annotate(). Use caution when annotating these querysets. - if any(isinstance(manager, LtreeManager) for manager in queryset.model._meta.local_managers): + # Use `managers` (not `local_managers`): the LtreeManager is declared on the abstract + # NestedLtreeGroupModel base, so concrete subclasses inherit it via the MRO rather than + # holding it in their own `local_managers`. + if any(isinstance(manager, LtreeManager) for manager in queryset.model._meta.managers): return queryset if queryset.ordered: return queryset diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py index ba268fb493e..d815c8b9778 100644 --- a/netbox/utilities/tests/test_ltree.py +++ b/netbox/utilities/tests/test_ltree.py @@ -154,7 +154,7 @@ def test_move_to(self): class CycleValidationTests(TestCase): - """clean() must refuse to assign a descendant as parent.""" + """clean() and save() must refuse to assign self or a descendant as parent.""" def test_cycle_raises(self): from django.core.exceptions import ValidationError @@ -165,6 +165,38 @@ def test_cycle_raises(self): with self.assertRaises(ValidationError): a.full_clean() + def test_cycle_raises_on_save_without_clean(self): + # The save()-level guard mirrors django-mptt's InvalidMove: a cyclic move + # is rejected even when full_clean() is bypassed (scripts, bulk callers). + from django.core.exceptions import ValidationError + a = Region.objects.create(name='A', slug='a-cyc2') + b = Region.objects.create(parent=a, name='B', slug='b-cyc2') + Region.objects.create(parent=b, name='C', slug='c-cyc2') + a.parent = b + with self.assertRaises(ValidationError): + a.save() + # The rejected move must leave the tree unchanged. + a.refresh_from_db() + self.assertIsNone(a.parent_id) + + def test_self_parent_raises_on_save(self): + from django.core.exceptions import ValidationError + a = Region.objects.create(name='A', slug='a-self') + a.parent = a + with self.assertRaises(ValidationError): + a.save() + + def test_move_to_unrelated_parent_is_allowed(self): + # A legitimate move (target is neither self nor a descendant) must succeed. + a = Region.objects.create(name='A', slug='a-ok') + b = Region.objects.create(name='B', slug='b-ok') + child = Region.objects.create(parent=a, name='Child', slug='child-ok') + child.parent = b + child.save() + child.refresh_from_db() + self.assertEqual(child.parent_id, b.pk) + self.assertEqual(child.path, _path(b.pk, child.pk)) + class SortPathTests(TestCase): """ @@ -446,6 +478,43 @@ def test_reparent_refreshes_sort_path(self): self.assertEqual(child.sort_path, db_sort_path) self.assertEqual(child.sort_path, f'Bravo{chr(1)}Kid') + def test_rename_refreshes_sort_path(self): + # The trigger rewrites sort_path on a name change; the in-memory instance + # must reflect it without a manual refresh_from_db(). + root = Region.objects.create(name='Root', slug='root-rn') + root.name = 'Renamed' + root.save() + db_sort_path = Region.objects.values_list('sort_path', flat=True).get(pk=root.pk) + self.assertEqual(root.sort_path, db_sort_path) + self.assertEqual(root.sort_path, 'Renamed') + + +class ReapplyModelOrderingTests(TestCase): + """Ltree-backed models must be exempt from reapply_model_ordering().""" + + def test_ltree_model_is_exempt(self): + from utilities.query import reapply_model_ordering + # Clear ordering so a non-exempt model would be re-ordered by Meta.ordering. + qs = Region.objects.all().order_by() + result = reapply_model_ordering(qs) + # The LtreeManager is inherited from the abstract base (not in + # local_managers), so the exemption must still apply and return qs as-is. + self.assertIs(result, qs) + + +class AddRelatedCountErrorTests(TestCase): + """add_related_count() must fail clearly on an unresolvable rel_field.""" + + def test_unknown_field_raises_fielddoesnotexist(self): + from django.core.exceptions import FieldDoesNotExist + # Region has no scope_type/scope_id, so an unknown rel_field cannot resolve + # to a FK/M2M or the GenericFK scope pattern -> explicit FieldDoesNotExist + # rather than an opaque NameError deep in the SQL builder. + with self.assertRaises(FieldDoesNotExist): + Region.objects.add_related_count( + Region.objects.all(), Region, 'not_a_field', 'bogus_count', cumulative=True + ) + class ChangeLogExclusionTests(TestCase): """ diff --git a/netbox/wireless/models.py b/netbox/wireless/models.py index 3a47db33a9a..e1f1febaf46 100644 --- a/netbox/wireless/models.py +++ b/netbox/wireless/models.py @@ -7,6 +7,7 @@ from dcim.constants import WIRELESS_IFACE_TYPES from dcim.models.mixins import CachedScopeMixin from netbox.models import NestedLtreeGroupModel, PrimaryModel +from netbox.models.ltree import SortPathField from netbox.models.mixins import DistanceMixin from .choices import * @@ -63,7 +64,7 @@ class WirelessLANGroup(NestedLtreeGroupModel): unique=True ) # Override the abstract parent's sort_path to use natural_sort, matching `name`. - sort_path = models.TextField( + sort_path = SortPathField( editable=False, blank=True, default='', From dca2deaedb1b536d82e72c96898b916b1cec09e3 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 2 Jun 2026 15:46:46 -0700 Subject: [PATCH 16/42] cleanup --- netbox/netbox/models/ltree.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index d3c541b34d4..07482b6557a 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -509,10 +509,11 @@ def rebuild_sort_paths(cls, name_column='name'): """ Recompute `sort_path` for every row from current values of `name_column`. - The BEFORE trigger updates sort_path only on INSERT and parent change, - not on rename — matching django-mptt's `order_insertion_by` semantics. - After renaming rows, call this to bring sort_path back in line with - current names (and thus restore correct list ordering). + Inserts, reparents, AND renames are all maintained automatically by the + BEFORE/AFTER triggers (the BEFORE trigger fires on INSERT and on updates to + parent_id or the name column). Use this only to repair `sort_path` after a + raw SQL write (e.g. a bulk COPY or a direct UPDATE) that bypassed those + triggers. Raises if the table does not have a `sort_path` column. """ @@ -522,16 +523,18 @@ def rebuild_sort_paths(cls, name_column='name'): raise NotImplementedError( f"{cls.__name__} does not have a sort_path column" ) - table = cls._meta.db_table + qn = connection.ops.quote_name + table = qn(cls._meta.db_table) + name_col = qn(name_column) sql = f''' WITH RECURSIVE t(id, parent_id, sort_path) AS ( - SELECT id, parent_id, "{name_column}"::text - FROM "{table}" WHERE parent_id IS NULL + SELECT id, parent_id, {name_col}::text + FROM {table} WHERE parent_id IS NULL UNION ALL - SELECT r.id, r.parent_id, t.sort_path || chr(1) || r."{name_column}" - FROM "{table}" r INNER JOIN t ON r.parent_id = t.id + SELECT r.id, r.parent_id, t.sort_path || chr(1) || r.{name_col} + FROM {table} r INNER JOIN t ON r.parent_id = t.id ) - UPDATE "{table}" SET sort_path = t.sort_path FROM t WHERE "{table}".id = t.id; + UPDATE {table} SET sort_path = t.sort_path FROM t WHERE {table}.id = t.id; ''' with connection.cursor() as cursor: cursor.execute(sql) From 90888ef0469a60f3e97997af1fa9397135a7c1df Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 2 Jun 2026 16:01:35 -0700 Subject: [PATCH 17/42] fix concurrency issue --- netbox/netbox/models/ltree.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index 07482b6557a..1dfa57ea08d 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -290,13 +290,17 @@ class LtreeModel(models.Model): acquired a per-model advisory lock on every write to protect its global lft/rght/tree_id numbering). Because a row's path depends only on its parent's path, concurrent inserts and moves under *different* parents - never conflict. One narrow race remains: inserting (or moving a node) - under parent P while P — or an ancestor of P — is concurrently reparented - in another transaction. The BEFORE trigger reads P's pre-move path under - its own snapshot, so the row may persist with a stale path. This requires - concurrent mutation of the same subtree and is accepted as the trade-off - for dropping the table-wide lock; if strict serialization is ever needed, - lock the parent row (`SELECT ... FOR SHARE`) inside the BEFORE trigger. + never conflict. + + To keep a node consistent with a concurrently-reparented ancestor, the + BEFORE trigger reads the parent row `FOR SHARE`. That shared lock conflicts + with the row-exclusive lock a reparent/rename of the parent takes, so an + insert/move under P is serialized against a concurrent reparent or rename + of P (or of an ancestor, whose cascade must update P's row). Sibling + inserts under a stable parent still proceed concurrently — shared locks + don't conflict. Crossing reparents (moving A under B while moving B under + A) can deadlock; PostgreSQL aborts one with a retryable error rather than + silently persisting a stale path. """ # `default=''` here is a Django-side placeholder that the BEFORE INSERT # trigger always overwrites with a valid path before the row reaches @@ -552,7 +556,10 @@ def rebuild_sort_paths(cls, name_column='name'): DECLARE parent_path ltree; BEGIN IF NEW.parent_id IS NOT NULL THEN - EXECUTE format('SELECT path FROM %%I WHERE id = $1', TG_TABLE_NAME) + -- FOR SHARE locks the parent row so a concurrent reparent/rename of it + -- (or of an ancestor, whose cascade updates this parent's row) cannot + -- proceed until this insert/move commits, preventing a stale path. + EXECUTE format('SELECT path FROM %%I WHERE id = $1 FOR SHARE', TG_TABLE_NAME) INTO parent_path USING NEW.parent_id; NEW.path := parent_path || lpad(NEW.id::text, 19, '0')::ltree; ELSE @@ -595,7 +602,10 @@ def rebuild_sort_paths(cls, name_column='name'): parent_sort_path text; BEGIN IF NEW.parent_id IS NOT NULL THEN - EXECUTE format('SELECT path, sort_path FROM %%I WHERE id = $1', TG_TABLE_NAME) + -- FOR SHARE locks the parent row so a concurrent reparent/rename of it + -- (or of an ancestor, whose cascade updates this parent's row) cannot + -- proceed until this insert/move commits, preventing a stale path/sort_path. + EXECUTE format('SELECT path, sort_path FROM %%I WHERE id = $1 FOR SHARE', TG_TABLE_NAME) INTO parent_path, parent_sort_path USING NEW.parent_id; NEW.path := parent_path || lpad(NEW.id::text, 19, '0')::ltree; NEW.sort_path := parent_sort_path || chr(1) || NEW.{name_col}; From 87c88d85dbffc7c243c03b19e7efd214bd472f38 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 2 Jun 2026 16:32:33 -0700 Subject: [PATCH 18/42] fixes --- netbox/netbox/graphql/filter_lookups.py | 13 +++-- netbox/netbox/models/ltree.py | 26 +++++++++ netbox/utilities/tests/test_ltree.py | 71 +++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 7 deletions(-) diff --git a/netbox/netbox/graphql/filter_lookups.py b/netbox/netbox/graphql/filter_lookups.py index 70b39b2399f..752e0e9661f 100644 --- a/netbox/netbox/graphql/filter_lookups.py +++ b/netbox/netbox/graphql/filter_lookups.py @@ -126,7 +126,7 @@ def filter(self, info: Info, queryset: QuerySet, prefix: DirectiveValue[str] = ' @strawberry.enum class TreeNodeMatch(Enum): EXACT = 'exact' # Just the node itself - DESCENDANTS = 'descendants' # Node and all descendants + DESCENDANTS = 'descendants' # All descendants, excluding the node itself SELF_AND_DESCENDANTS = 'self_and_descendants' # Node and all descendants CHILDREN = 'children' # Just immediate children SIBLINGS = 'siblings' # Nodes with same parent @@ -159,12 +159,11 @@ def filter(self, info: Info, queryset: QuerySet, prefix: DirectiveValue[str] = ' # Generate base Q filter for the related model without prefix q_filter = generate_tree_node_q_filter(related_model, self) - # Handle different relationship types - if isinstance(model_field, (ManyToManyField, ManyToManyRel)): - return queryset, Q(**{f'{model_field_name}__in': related_model.objects.filter(q_filter)}) - if isinstance(model_field, ForeignKey): - return queryset, Q(**{f'{model_field_name}__{k}': v for k, v in q_filter.children}) - if isinstance(model_field, ManyToOneRel): + # Handle different relationship types. All variants resolve the related + # rows against the q_filter (which may be a compound Q for DESCENDANTS, + # ANCESTORS, SIBLINGS, SELF_AND_DESCENDANTS) and join via __in. Destructuring + # q_filter.children into kwargs would crash on compound match types. + if isinstance(model_field, (ManyToManyField, ManyToManyRel, ForeignKey, ManyToOneRel)): return queryset, Q(**{f'{model_field_name}__in': related_model.objects.filter(q_filter)}) return queryset, Q(**{f'{model_field_name}__{k}': v for k, v in q_filter.children}) diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index 1dfa57ea08d..b204baaa4f2 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -115,6 +115,32 @@ def deconstruct(self): class LtreeQuerySet(RestrictedQuerySet): """QuerySet for ltree-based hierarchies, layered on RestrictedQuerySet.""" + def bulk_create(self, objs, *args, **kwargs): + """ + Same as the standard `bulk_create` but verifies that a row whose parent is + also in the same batch appears AFTER its parent. PostgreSQL fires the + BEFORE INSERT trigger per row in list order; a child placed before its + parent in the same batch would otherwise be persisted with a stale + root-level path silently. Raises ValueError when a misordering is detected. + """ + seen = set() + objs_list = list(objs) + for idx, obj in enumerate(objs_list): + parent = getattr(obj, 'parent', None) + if parent is not None and parent.pk is None and id(parent) not in seen: + # parent is also in this batch (unsaved) but hasn't appeared yet + # — only an issue if it's actually later in the list. + for later in objs_list[idx + 1:]: + if later is parent: + raise ValueError( + "bulk_create: child at index {idx} references parent that " + "appears later in the same batch; parents must precede " + "their children or the child's path will be stored as " + "a root.".format(idx=idx) + ) + seen.add(id(obj)) + return super().bulk_create(objs_list, *args, **kwargs) + def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=False): """ Annotate `queryset` with the count of `model` instances related via diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py index d815c8b9778..aaa6f6d7bff 100644 --- a/netbox/utilities/tests/test_ltree.py +++ b/netbox/utilities/tests/test_ltree.py @@ -368,6 +368,77 @@ def test_parent_before_child_in_same_batch(self): child_pending.refresh_from_db() self.assertEqual(child_pending.path, _path(root.pk, parent_pending.pk, child_pending.pk)) + def test_bulk_create_rejects_child_before_parent_in_same_batch(self): + """The guard refuses misordered batches instead of writing bad paths.""" + unsaved_parent = Region(name='P', slug='p-bcrej') + unsaved_child = Region(parent=unsaved_parent, name='C', slug='c-bcrej') + with self.assertRaises(ValueError) as ctx: + Region.objects.bulk_create([unsaved_child, unsaved_parent]) + self.assertIn('parents must precede', str(ctx.exception)) + + def test_bulk_create_allows_unrelated_unsaved_parent(self): + """An unsaved parent that isn't in the batch must not trigger the guard.""" + external_parent = Region(name='X', slug='x-bcok') # not in the batch + # This use is rare but valid: external_parent is unsaved, but the caller + # is responsible for ensuring it exists before the insert reaches the DB. + # The guard only inspects parents that share the same batch. + with self.assertRaises(Exception): + # The DB will reject this because parent has no PK; the point is the + # guard itself must not raise its own ValueError here. + Region.objects.bulk_create([Region(parent=external_parent, name='Y', slug='y-bcok')]) + + +class TreeNodeFilterTests(TestCase): + """ + Regression: the FK branch of TreeNodeFilter.filter() previously destructured + q_filter.children as (str, value) tuples, which crashed for compound Q + match types (DESCENDANTS, ANCESTORS, SIBLINGS, SELF_AND_DESCENDANTS). The + FK branch must resolve via __in like the M2M / M2O branches. + """ + + @classmethod + def setUpTestData(cls): + from tenancy.models import Tenant, TenantGroup + cls.Tenant = Tenant + cls.TenantGroup = TenantGroup + cls.root = TenantGroup.objects.create(name='Root', slug='root-tnf') + cls.mid = TenantGroup.objects.create(parent=cls.root, name='Mid', slug='mid-tnf') + cls.leaf = TenantGroup.objects.create(parent=cls.mid, name='Leaf', slug='leaf-tnf') + cls.sibling = TenantGroup.objects.create(parent=cls.root, name='Sibling', slug='sibling-tnf') + + cls.t_root = Tenant.objects.create(name='TRoot', slug='troot-tnf', group=cls.root) + cls.t_mid = Tenant.objects.create(name='TMid', slug='tmid-tnf', group=cls.mid) + cls.t_leaf = Tenant.objects.create(name='TLeaf', slug='tleaf-tnf', group=cls.leaf) + cls.t_sibling = Tenant.objects.create(name='TSib', slug='tsib-tnf', group=cls.sibling) + + def _filter(self, match_type): + from netbox.graphql.filter_lookups import TreeNodeFilter, TreeNodeMatch + tnf = TreeNodeFilter(id=self.mid.pk, match_type=getattr(TreeNodeMatch, match_type)) + # `filter` is decorated by @strawberry_django.filter_field; its wrapper + # asserts info is not None. The undecorated body is on `_unbound_wrapped_func`. + inner = TreeNodeFilter.filter._unbound_wrapped_func + qs, q = inner(tnf, info=None, queryset=self.Tenant.objects.all(), prefix='group__') + return list(qs.filter(q).values_list('name', flat=True)) + + def test_descendants_strict(self): + # DESCENDANTS of `mid` = leaf only (Mid itself excluded) + names = self._filter('DESCENDANTS') + self.assertEqual(sorted(names), ['TLeaf']) + + def test_self_and_descendants(self): + names = self._filter('SELF_AND_DESCENDANTS') + self.assertEqual(sorted(names), ['TLeaf', 'TMid']) + + def test_ancestors(self): + names = self._filter('ANCESTORS') + # Ancestors of Mid = Root only (Mid itself excluded) + self.assertEqual(sorted(names), ['TRoot']) + + def test_siblings(self): + # Siblings of Mid (within Root) = Sibling + names = self._filter('SIBLINGS') + self.assertEqual(sorted(names), ['TSib']) + class DescendantLookupSemanticsTests(TestCase): """ From 33038a6095f6858486684afe8ab280a2847f70d8 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Jun 2026 08:28:15 -0700 Subject: [PATCH 19/42] fixes --- netbox/core/models/change_logging.py | 6 ++ netbox/dcim/graphql/types.py | 12 ++-- netbox/netbox/graphql/types.py | 20 ++++++- netbox/netbox/models/ltree.py | 22 ++++---- netbox/netbox/tests/test_base_classes.py | 5 +- netbox/tenancy/graphql/types.py | 6 +- netbox/utilities/tests/test_ltree.py | 72 ++++++++++++++++++++---- netbox/wireless/graphql/types.py | 4 +- 8 files changed, 113 insertions(+), 34 deletions(-) diff --git a/netbox/core/models/change_logging.py b/netbox/core/models/change_logging.py index 945a714e709..7186f21c01c 100644 --- a/netbox/core/models/change_logging.py +++ b/netbox/core/models/change_logging.py @@ -168,6 +168,12 @@ def diff_exclude_fields(self): if issubclass(model, LtreeModel): attrs.update({'path', 'sort_path'}) + # Exclude MPTT bookkeeping columns for the deprecated MPTT-backed + # NestedGroupModel still shipped for plugin compatibility. + from mptt.models import MPTTModel + if issubclass(model, MPTTModel): + attrs.update({'lft', 'rght', 'tree_id', 'level'}) + return attrs def get_clean_data(self, prefix): diff --git a/netbox/dcim/graphql/types.py b/netbox/dcim/graphql/types.py index d8f93189118..903296d03bd 100644 --- a/netbox/dcim/graphql/types.py +++ b/netbox/dcim/graphql/types.py @@ -13,7 +13,7 @@ from netbox.graphql.types import ( BaseObjectType, LtreeNodeMixin, - NestedGroupObjectType, + NestedLtreeGroupObjectType, NetBoxObjectType, OrganizationalObjectType, PrimaryObjectType, @@ -355,7 +355,7 @@ def parent(self) -> Annotated['InventoryItemTemplateType', strawberry.lazy('dcim filters=DeviceRoleFilter, pagination=True ) -class DeviceRoleType(NestedGroupObjectType): +class DeviceRoleType(NestedLtreeGroupObjectType): parent: Annotated['DeviceRoleType', strawberry.lazy('dcim.graphql.types')] | None children: list[Annotated['DeviceRoleType', strawberry.lazy('dcim.graphql.types')]] color: str @@ -534,7 +534,7 @@ class InventoryItemRoleType(OrganizationalObjectType): filters=LocationFilter, pagination=True ) -class LocationType(VLANGroupsMixin, ImageAttachmentsMixin, ContactsMixin, NestedGroupObjectType): +class LocationType(VLANGroupsMixin, ImageAttachmentsMixin, ContactsMixin, NestedLtreeGroupObjectType): site: Annotated["SiteType", strawberry.lazy('dcim.graphql.types')] tenant: Annotated["TenantType", strawberry.lazy('tenancy.graphql.types')] | None parent: Annotated["LocationType", strawberry.lazy('dcim.graphql.types')] | None @@ -672,7 +672,7 @@ class ModuleTypeType(PrimaryObjectType): filters=PlatformFilter, pagination=True ) -class PlatformType(NestedGroupObjectType): +class PlatformType(NestedLtreeGroupObjectType): parent: Annotated['PlatformType', strawberry.lazy('dcim.graphql.types')] | None children: list[Annotated['PlatformType', strawberry.lazy('dcim.graphql.types')]] manufacturer: Annotated["ManufacturerType", strawberry.lazy('dcim.graphql.types')] | None @@ -880,7 +880,7 @@ class RearPortTemplateType(ModularComponentTemplateType): filters=RegionFilter, pagination=True ) -class RegionType(VLANGroupsMixin, ContactsMixin, NestedGroupObjectType): +class RegionType(VLANGroupsMixin, ContactsMixin, NestedLtreeGroupObjectType): sites: list[Annotated["SiteType", strawberry.lazy('dcim.graphql.types')]] children: list[Annotated["RegionType", strawberry.lazy('dcim.graphql.types')]] @@ -957,7 +957,7 @@ def circuit_terminations(self) -> list[ filters=SiteGroupFilter, pagination=True ) -class SiteGroupType(VLANGroupsMixin, ContactsMixin, NestedGroupObjectType): +class SiteGroupType(VLANGroupsMixin, ContactsMixin, NestedLtreeGroupObjectType): sites: list[Annotated["SiteType", strawberry.lazy('dcim.graphql.types')]] children: list[Annotated["SiteGroupType", strawberry.lazy('dcim.graphql.types')]] diff --git a/netbox/netbox/graphql/types.py b/netbox/netbox/graphql/types.py index 7c6b36bf607..3f7bc84c37a 100644 --- a/netbox/netbox/graphql/types.py +++ b/netbox/netbox/graphql/types.py @@ -14,6 +14,7 @@ 'ContentTypeType', 'LtreeNodeMixin', 'NestedGroupObjectType', + 'NestedLtreeGroupObjectType', 'NetBoxObjectType', 'ObjectType', 'OrganizationalObjectType', @@ -107,6 +108,22 @@ def level(self) -> int: class NestedGroupObjectType( + ChangelogMixin, + CustomFieldsMixin, + JournalEntriesMixin, + TagsMixin, + OwnerMixin, + BaseObjectType +): + """ + Base GraphQL type for the deprecated MPTT-backed NestedGroupModel, kept for + plugin compatibility. MPTT exposes `level` as a real column, so no annotation + mixin is needed. New code should use NestedLtreeGroupObjectType. + """ + pass + + +class NestedLtreeGroupObjectType( LtreeNodeMixin, ChangelogMixin, CustomFieldsMixin, @@ -116,7 +133,8 @@ class NestedGroupObjectType( BaseObjectType ): """ - Base GraphQL type for models which inherit from NestedGroupModel. + Base GraphQL type for models which inherit from NestedLtreeGroupModel. + Adds a `level` field annotated via `nlevel(path)`. """ pass diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index b204baaa4f2..14a9d3d6d82 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -11,7 +11,7 @@ mutates paths directly; it only reads `path` back from the database after inserts and parent_id changes via refresh_from_db(fields=['path']). """ -from django.core.exceptions import FieldDoesNotExist, ValidationError +from django.core.exceptions import ValidationError from django.db import connection, migrations, models from django.db.models import ForeignKey, Lookup, ManyToManyField, Q from django.db.models.expressions import RawSQL @@ -167,13 +167,12 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F and not has_direct_fk and not is_many_to_many ) - # The FK branches below dereference field.column, so fail loudly here if the - # field could not be resolved and this isn't the scope GenericFK pattern — - # rather than raising an opaque AttributeError deep in the SQL builder. - if field is None and not has_generic_fk: - raise FieldDoesNotExist( - f"{model._meta.label} has no field '{rel_field}' for add_related_count()" - ) + # Many call sites bind add_related_count() as a class attribute, so it + # runs at module import. Raising FieldDoesNotExist here would block app + # startup whenever an unrelated field is renamed; fall back to the + # Django default column name (`{rel_field}_id`) instead, matching MPTT's + # add_related_count behavior. + rel_field_col_default = f'{rel_field}_id' qn = connection.ops.quote_name parent_table = qn(queryset.model._meta.db_table) @@ -220,8 +219,9 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F return queryset.annotate(**{ count_attr: RawSQL(sql, [ct_app, ct_model], output_field=models.IntegerField()) }) - # Use field.column (not f'{rel_field}_id') so custom db_column works. - rel_field_col = qn(field.column) + # Use field.column (not f'{rel_field}_id') so custom db_column works; + # fall back to default naming if the field was not resolved. + rel_field_col = qn(field.column if field is not None else rel_field_col_default) sql = f'''( SELECT COUNT(DISTINCT {related_table}."id") FROM {related_table} @@ -265,7 +265,7 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F return queryset.annotate(**{ count_attr: RawSQL(sql, [ct_app, ct_model], output_field=models.IntegerField()) }) - rel_field_col = qn(field.column) + rel_field_col = qn(field.column if field is not None else rel_field_col_default) sql = f'''( SELECT COUNT(DISTINCT {related_table}."id") FROM {related_table} diff --git a/netbox/netbox/tests/test_base_classes.py b/netbox/netbox/tests/test_base_classes.py index b006630b784..b8754a2e80e 100644 --- a/netbox/netbox/tests/test_base_classes.py +++ b/netbox/netbox/tests/test_base_classes.py @@ -40,11 +40,12 @@ ) from netbox.graphql.types import ( NestedGroupObjectType, + NestedLtreeGroupObjectType, NetBoxObjectType, OrganizationalObjectType, PrimaryObjectType, ) -from netbox.models import NestedGroupModelMixin, NetBoxModel, OrganizationalModel, PrimaryModel +from netbox.models import NestedGroupModelMixin, NestedLtreeGroupModel, NetBoxModel, OrganizationalModel, PrimaryModel from netbox.registry import registry from netbox.tables import ( NestedGroupModelTable, @@ -365,6 +366,8 @@ def get_model_type_base_class(model): return PrimaryObjectType if issubclass(model, OrganizationalModel): return OrganizationalObjectType + if issubclass(model, NestedLtreeGroupModel): + return NestedLtreeGroupObjectType if issubclass(model, NestedGroupModelMixin): return NestedGroupObjectType if issubclass(model, NetBoxModel): diff --git a/netbox/tenancy/graphql/types.py b/netbox/tenancy/graphql/types.py index 15629bfdf32..aa9c3323440 100644 --- a/netbox/tenancy/graphql/types.py +++ b/netbox/tenancy/graphql/types.py @@ -4,7 +4,7 @@ import strawberry_django from extras.graphql.mixins import ContactsMixin, CustomFieldsMixin, TagsMixin -from netbox.graphql.types import BaseObjectType, NestedGroupObjectType, OrganizationalObjectType, PrimaryObjectType +from netbox.graphql.types import BaseObjectType, NestedLtreeGroupObjectType, OrganizationalObjectType, PrimaryObjectType from tenancy import models from .filters import * @@ -92,7 +92,7 @@ class TenantType(ContactsMixin, PrimaryObjectType): filters=TenantGroupFilter, pagination=True ) -class TenantGroupType(NestedGroupObjectType): +class TenantGroupType(NestedLtreeGroupObjectType): parent: Annotated['TenantGroupType', strawberry.lazy('tenancy.graphql.types')] | None tenants: list[TenantType] @@ -129,7 +129,7 @@ class ContactRoleType(ContactAssignmentsMixin, OrganizationalObjectType): filters=ContactGroupFilter, pagination=True ) -class ContactGroupType(NestedGroupObjectType): +class ContactGroupType(NestedLtreeGroupObjectType): parent: Annotated['ContactGroupType', strawberry.lazy('tenancy.graphql.types')] | None contacts: list[ContactType] diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py index aaa6f6d7bff..f930ee8cbc9 100644 --- a/netbox/utilities/tests/test_ltree.py +++ b/netbox/utilities/tests/test_ltree.py @@ -574,17 +574,18 @@ def test_ltree_model_is_exempt(self): class AddRelatedCountErrorTests(TestCase): - """add_related_count() must fail clearly on an unresolvable rel_field.""" + """ + add_related_count() must not raise at queryset-build time for unresolvable + rel_field — many call sites bind the annotation as a class attribute at + module import. The annotation is still attached using the Django default + column naming, and any error surfaces at evaluation time. + """ - def test_unknown_field_raises_fielddoesnotexist(self): - from django.core.exceptions import FieldDoesNotExist - # Region has no scope_type/scope_id, so an unknown rel_field cannot resolve - # to a FK/M2M or the GenericFK scope pattern -> explicit FieldDoesNotExist - # rather than an opaque NameError deep in the SQL builder. - with self.assertRaises(FieldDoesNotExist): - Region.objects.add_related_count( - Region.objects.all(), Region, 'not_a_field', 'bogus_count', cumulative=True - ) + def test_unknown_field_does_not_raise_at_build(self): + qs = Region.objects.add_related_count( + Region.objects.all(), Region, 'not_a_field', 'bogus_count', cumulative=True + ) + self.assertIn('bogus_count', qs.query.annotations) class ChangeLogExclusionTests(TestCase): @@ -617,6 +618,57 @@ def test_reparent_postchange_snapshot_matches_db(self): self.assertNotIn('path', oc.postchange_data_clean) +class MPTTChangeLogExclusionTests(TestCase): + """ + ObjectChange.diff_exclude_fields must hide MPTT bookkeeping columns + (lft/rght/tree_id/level) for plugin models still using the deprecated + MPTT-backed NestedGroupModel, in addition to ltree's path/sort_path. + """ + + def test_diff_exclude_fields_for_mptt_subclass(self): + from unittest.mock import MagicMock + from mptt.models import MPTTModel + + class _FakeMPTT(MPTTModel): + class Meta: + abstract = True + app_label = 'tests' + + oc = ObjectChange() + fake_ct = MagicMock() + fake_ct.model_class.return_value = _FakeMPTT + # Prime the FK descriptor's cache so accessing changed_object_type + # returns our mock without hitting the DB or invoking type checks. + ObjectChange._meta.get_field('changed_object_type').set_cached_value(oc, fake_ct) + + excluded = oc.diff_exclude_fields + self.assertIn('lft', excluded) + self.assertIn('rght', excluded) + self.assertIn('tree_id', excluded) + self.assertIn('level', excluded) + + +class AddRelatedCountResilienceTests(TestCase): + """ + add_related_count() must not raise FieldDoesNotExist at queryset-build + time so that view modules (which bind it as a class attribute) can be + imported even if a referenced field has been renamed. + """ + + def test_unknown_field_does_not_raise(self): + # Bare manager call equivalent to a view class body using a stale name. + qs = Region.objects.add_related_count( + Region.objects.all(), + Region, # any model; the field name is what matters + 'this_field_does_not_exist', + 'noop_count', + cumulative=True, + ) + # The annotation was attached; evaluating it would fail at the DB + # (column doesn't exist) but importing the view module must succeed. + self.assertIn('noop_count', qs.query.annotations) + + class CascadeTriggerScopeTests(TestCase): """ The AFTER cascade trigger fires on parent_id or name changes. A rename diff --git a/netbox/wireless/graphql/types.py b/netbox/wireless/graphql/types.py index 5671366f4b9..060175ff089 100644 --- a/netbox/wireless/graphql/types.py +++ b/netbox/wireless/graphql/types.py @@ -3,7 +3,7 @@ import strawberry import strawberry_django -from netbox.graphql.types import NestedGroupObjectType, PrimaryObjectType +from netbox.graphql.types import NestedLtreeGroupObjectType, PrimaryObjectType from wireless import models from .filters import * @@ -26,7 +26,7 @@ filters=WirelessLANGroupFilter, pagination=True ) -class WirelessLANGroupType(NestedGroupObjectType): +class WirelessLANGroupType(NestedLtreeGroupObjectType): parent: Annotated["WirelessLANGroupType", strawberry.lazy('wireless.graphql.types')] | None wireless_lans: list[Annotated["WirelessLANType", strawberry.lazy('wireless.graphql.types')]] From 4f2a2b2aa58abd5fd827496ee580c4e85c216ecb Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Jun 2026 09:20:33 -0700 Subject: [PATCH 20/42] fixes --- netbox/dcim/migrations/0237_ltree_paths.py | 35 ++++- netbox/netbox/models/__init__.py | 2 +- netbox/netbox/models/features.py | 7 +- netbox/netbox/models/ltree.py | 82 +++++++---- netbox/tenancy/migrations/0025_ltree_paths.py | 25 +++- netbox/utilities/tests/test_ltree.py | 133 ++++++++++++++++-- .../wireless/migrations/0020_ltree_paths.py | 21 ++- 7 files changed, 257 insertions(+), 48 deletions(-) diff --git a/netbox/dcim/migrations/0237_ltree_paths.py b/netbox/dcim/migrations/0237_ltree_paths.py index 02ec1fca8bb..e982f3c3951 100644 --- a/netbox/dcim/migrations/0237_ltree_paths.py +++ b/netbox/dcim/migrations/0237_ltree_paths.py @@ -77,7 +77,7 @@ def _populate_paths_sql(): """ Build the recursive CTE that walks each table from roots downward, computing the new path (PK-based, zero-padded) and — for models with sort_path — the - chr(1)-separated chain of ancestor names. + chr(9)-separated chain of ancestor names. """ blocks = [] for table in ALL_TABLES: @@ -91,7 +91,7 @@ def _populate_paths_sql(): UNION ALL SELECT r.id, r.parent_id, t.path || lpad(r.id::text, 19, '0')::ltree, - t.sort_path || chr(1) || r.name + t.sort_path || chr(9) || r.name FROM "{table}" r JOIN t ON r.parent_id = t.id ) UPDATE "{table}" SET path = t.path, sort_path = t.sort_path @@ -110,6 +110,33 @@ def _populate_paths_sql(): return '\n'.join(blocks) +def _assert_paths_populated_sql(): + """ + After the recursive CTE backfills paths from `parent_id IS NULL` roots, any + row whose parent_id points to a row that the CTE could not reach (orphan FK, + stray cycle left by a prior raw write) will still have path IS NULL. The + immediately following AlterField that sets path to NOT NULL would then abort + inside ALTER COLUMN with an opaque message. Fail fast here instead so the + operator sees the offending table and row counts. + """ + checks = [] + for table in ALL_TABLES: + checks.append(f""" +DO $$ +DECLARE missing bigint; +BEGIN + SELECT count(*) INTO missing FROM "{table}" WHERE path IS NULL; + IF missing > 0 THEN + RAISE EXCEPTION + 'ltree backfill left % rows in "{table}" with NULL path; ' + 'likely orphan parent_id references — resolve before re-running ' + 'this migration', missing; + END IF; +END $$; +""") + return '\n'.join(checks) + + class Migration(migrations.Migration): dependencies = [ @@ -197,6 +224,10 @@ class Migration(migrations.Migration): # 4. Populate existing rows via per-table recursive CTE. migrations.RunSQL(_populate_paths_sql(), reverse_sql=migrations.RunSQL.noop), + # 4b. Fail fast (with a useful message) if any row still has NULL path — + # otherwise the AlterField below aborts opaquely inside ALTER COLUMN. + migrations.RunSQL(_assert_paths_populated_sql(), reverse_sql=migrations.RunSQL.noop), + # 5. Tighten path to NOT NULL with empty-string default. *[ migrations.AlterField( diff --git a/netbox/netbox/models/__init__.py b/netbox/netbox/models/__init__.py index 46f5637bb0f..0b0ace92f7e 100644 --- a/netbox/netbox/models/__init__.py +++ b/netbox/netbox/models/__init__.py @@ -231,7 +231,7 @@ class NestedLtreeGroupModel(NestedGroupModelMixin, LtreeModel): Base model for objects which are used to form a hierarchy (regions, locations, etc.). These models nest recursively using PostgreSQL ltree. Within each parent, each child instance must have a unique name. - `sort_path` is a trigger-maintained text column holding a chr(1)-separated chain of ancestor + `sort_path` is a trigger-maintained text column holding a chr(9)-separated chain of ancestor names; ordering by it yields tree-flatten output with siblings in name (collation) order. Inserts, reparents, AND renames all update `sort_path` (a rename cascades to descendants), so list ordering reflects renames immediately — unlike django-mptt's `order_insertion_by`, which diff --git a/netbox/netbox/models/features.py b/netbox/netbox/models/features.py index bbbee9f8424..ef5d530a2dd 100644 --- a/netbox/netbox/models/features.py +++ b/netbox/netbox/models/features.py @@ -407,16 +407,13 @@ def get_contacts(self, inherited=True): """ from tenancy.models import ContactAssignment - # TODO: Once the deprecated MPTT-backed NestedGroupModel is removed, narrow this - # check to NestedLtreeGroupModel (or drop the mixin alias). NestedGroupModelMixin - # exists only so both legacy MPTT and new ltree bases satisfy the inheritance check. - from . import NestedGroupModelMixin + from . import NestedGroupModel, NestedLtreeGroupModel filter = Q( object_type=ObjectType.objects.get_for_model(self), object_id__in=( self.get_ancestors(include_self=True) - if (isinstance(self, NestedGroupModelMixin) and inherited) + if (isinstance(self, (NestedGroupModel, NestedLtreeGroupModel)) and inherited) else [self.pk] ), ) diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index 14a9d3d6d82..15957a8cb04 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -95,7 +95,7 @@ def as_sql(self, compiler, connection): class SortPathField(models.TextField): """ - Text column holding the chr(1)-separated chain of ancestor names that drives + Text column holding the chr(9)-separated chain of ancestor names that drives tree-flatten ordering. Like `path`, its value is maintained by triggers, so it is marked db_returning to be populated via INSERT ... RETURNING without an extra SELECT. It deconstructs as a plain TextField so existing migrations @@ -117,27 +117,36 @@ class LtreeQuerySet(RestrictedQuerySet): def bulk_create(self, objs, *args, **kwargs): """ - Same as the standard `bulk_create` but verifies that a row whose parent is - also in the same batch appears AFTER its parent. PostgreSQL fires the - BEFORE INSERT trigger per row in list order; a child placed before its - parent in the same batch would otherwise be persisted with a stale - root-level path silently. Raises ValueError when a misordering is detected. + Same as the standard `bulk_create` but verifies that, for every row whose + parent is an unsaved instance, the parent appears earlier in the same + batch. PostgreSQL fires the BEFORE INSERT trigger per row in list order; + a child whose parent is unsaved and either (a) appears later in the batch + or (b) is not in the batch at all would otherwise be persisted silently + with a stale root-level path. Raises ValueError in both cases. """ - seen = set() objs_list = list(objs) + seen = set() for idx, obj in enumerate(objs_list): parent = getattr(obj, 'parent', None) if parent is not None and parent.pk is None and id(parent) not in seen: - # parent is also in this batch (unsaved) but hasn't appeared yet - # — only an issue if it's actually later in the list. - for later in objs_list[idx + 1:]: - if later is parent: - raise ValueError( - "bulk_create: child at index {idx} references parent that " - "appears later in the same batch; parents must precede " - "their children or the child's path will be stored as " - "a root.".format(idx=idx) - ) + # Parent is unsaved and has not yet been encountered. It must appear + # later in the batch (which is still wrong — children must follow + # parents) or not at all; either way the trigger would see + # parent_id=NULL and store this row as a root. + in_batch_later = any(later is parent for later in objs_list[idx + 1:]) + if in_batch_later: + raise ValueError( + "bulk_create: child at index {idx} references parent that " + "appears later in the same batch; parents must precede " + "their children or the child's path will be stored as " + "a root.".format(idx=idx) + ) + raise ValueError( + "bulk_create: child at index {idx} references an unsaved parent " + "that is not in this batch; save the parent first or include " + "it earlier in the batch — otherwise the child would be " + "persisted with a root-level path.".format(idx=idx) + ) seen.add(id(obj)) return super().bulk_create(objs_list, *args, **kwargs) @@ -366,7 +375,13 @@ def _parent_creates_cycle(self): Subclasses whose `parent` is system-managed (e.g. ModuleBay, whose parent is derived from its module) may override this to disable the check. """ - if self.parent_id is None or not self.path: + if self.parent_id is None: + return False + # Self-as-parent is always a cycle and must be caught even if self.path + # is empty or deferred (path would otherwise short-circuit below). + if self.parent_id == self.pk: + return True + if not self.path: return False # The new parent lies inside this node's current subtree iff its path is a # descendant of (or equal to) self.path. @@ -561,7 +576,7 @@ def rebuild_sort_paths(cls, name_column='name'): SELECT id, parent_id, {name_col}::text FROM {table} WHERE parent_id IS NULL UNION ALL - SELECT r.id, r.parent_id, t.sort_path || chr(1) || r.{name_col} + SELECT r.id, r.parent_id, t.sort_path || chr(9) || r.{name_col} FROM {table} r INNER JOIN t ON r.parent_id = t.id ) UPDATE {table} SET sort_path = t.sort_path FROM t WHERE {table}.id = t.id; @@ -587,6 +602,15 @@ def rebuild_sort_paths(cls, name_column='name'): -- proceed until this insert/move commits, preventing a stale path. EXECUTE format('SELECT path FROM %%I WHERE id = $1 FOR SHARE', TG_TABLE_NAME) INTO parent_path USING NEW.parent_id; + -- Cycle guard. The Python LtreeModel.save() also rejects cyclic moves, + -- but a QuerySet.update() / bulk_update() bypasses save() entirely, so + -- catch the case here as a last line of defense. lpad(NEW.id,..) @> + -- parent_path is TRUE iff parent_path starts with this row's own + -- label (i.e. the proposed parent is self or one of self's descendants). + IF lpad(NEW.id::text, 19, '0')::ltree @> parent_path THEN + RAISE EXCEPTION 'cycle detected: %% cannot be assigned a parent that is itself or one of its descendants', TG_TABLE_NAME + USING ERRCODE = 'check_violation'; + END IF; NEW.path := parent_path || lpad(NEW.id::text, 19, '0')::ltree; ELSE NEW.path := lpad(NEW.id::text, 19, '0')::ltree; @@ -610,9 +634,13 @@ def rebuild_sort_paths(cls, name_column='name'): ''' # For models with order_insertion_by=(name,) — maintain a second text column -# `sort_path` whose value is the chain of ancestor names joined by chr(1) -# (an unprintable separator that collates lower than any printable char in any -# standard collation). ORDER BY sort_path then gives MPTT-equivalent +# `sort_path` whose value is the chain of ancestor names joined by chr(9) (TAB). +# TAB sorts strictly below any printable character under both the default text +# collation and the ICU `natural_sort` collation (which is `und-u-kn-true`). +# ICU collations with default variable weighting treat U+0001..U+0008 as +# variable-ignorable, so a chr(1) separator under natural_sort would interleave +# children with unrelated roots; TAB is given a primary weight and orders +# deterministically. ORDER BY sort_path then gives MPTT-equivalent # tree-flatten ordering with siblings in name (collation) order. # # The BEFORE trigger fires on INSERT, parent_id changes, and name changes, so @@ -633,8 +661,14 @@ def rebuild_sort_paths(cls, name_column='name'): -- proceed until this insert/move commits, preventing a stale path/sort_path. EXECUTE format('SELECT path, sort_path FROM %%I WHERE id = $1 FOR SHARE', TG_TABLE_NAME) INTO parent_path, parent_sort_path USING NEW.parent_id; + -- Cycle guard. See _COMPUTE_PATH_ONLY_FN for the rationale; this catches + -- raw UPDATE / bulk_update paths that bypass LtreeModel.save(). + IF lpad(NEW.id::text, 19, '0')::ltree @> parent_path THEN + RAISE EXCEPTION 'cycle detected: %% cannot be assigned a parent that is itself or one of its descendants', TG_TABLE_NAME + USING ERRCODE = 'check_violation'; + END IF; NEW.path := parent_path || lpad(NEW.id::text, 19, '0')::ltree; - NEW.sort_path := parent_sort_path || chr(1) || NEW.{name_col}; + NEW.sort_path := parent_sort_path || chr(9) || NEW.{name_col}; ELSE NEW.path := lpad(NEW.id::text, 19, '0')::ltree; NEW.sort_path := NEW.{name_col}; @@ -708,7 +742,7 @@ class InstallLtreeTriggers(migrations.operations.base.Operation): AFTER UPDATE OF parent_id -> cascade path/sort_path change to descendants If `name_column` is provided, the model is expected to have a `sort_path` - text column whose value will be maintained as a chr(1)-separated chain of + text column whose value will be maintained as a chr(9)-separated chain of ancestor names. This implements MPTT's `order_insertion_by=(name,)` semantics: insert, reparent, and rename all honor the current value of `name_column`, with renames cascaded into descendants' sort_paths. diff --git a/netbox/tenancy/migrations/0025_ltree_paths.py b/netbox/tenancy/migrations/0025_ltree_paths.py index bda57d3df04..15ebf3f417d 100644 --- a/netbox/tenancy/migrations/0025_ltree_paths.py +++ b/netbox/tenancy/migrations/0025_ltree_paths.py @@ -28,7 +28,7 @@ def _populate_paths_sql(): UNION ALL SELECT r.id, r.parent_id, t.path || lpad(r.id::text, 19, '0')::ltree, - t.sort_path || chr(1) || r.name + t.sort_path || chr(9) || r.name FROM "{table}" r JOIN t ON r.parent_id = t.id ) UPDATE "{table}" SET path = t.path, sort_path = t.sort_path @@ -37,6 +37,25 @@ def _populate_paths_sql(): return '\n'.join(blocks) +def _assert_paths_populated_sql(): + checks = [] + for table in TABLES: + checks.append(f""" +DO $$ +DECLARE missing bigint; +BEGIN + SELECT count(*) INTO missing FROM "{table}" WHERE path IS NULL; + IF missing > 0 THEN + RAISE EXCEPTION + 'ltree backfill left % rows in "{table}" with NULL path; ' + 'likely orphan parent_id references — resolve before re-running ' + 'this migration', missing; + END IF; +END $$; +""") + return '\n'.join(checks) + + class Migration(migrations.Migration): dependencies = [ @@ -87,6 +106,10 @@ class Migration(migrations.Migration): migrations.RunSQL(_populate_paths_sql(), reverse_sql=migrations.RunSQL.noop), + # Fail fast if any row still has NULL path (orphan FKs) before the + # AlterField below tries to set NOT NULL inside ALTER COLUMN. + migrations.RunSQL(_assert_paths_populated_sql(), reverse_sql=migrations.RunSQL.noop), + *[ migrations.AlterField( model_name=m, name='path', diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py index f930ee8cbc9..9e60119ee11 100644 --- a/netbox/utilities/tests/test_ltree.py +++ b/netbox/utilities/tests/test_ltree.py @@ -376,16 +376,18 @@ def test_bulk_create_rejects_child_before_parent_in_same_batch(self): Region.objects.bulk_create([unsaved_child, unsaved_parent]) self.assertIn('parents must precede', str(ctx.exception)) - def test_bulk_create_allows_unrelated_unsaved_parent(self): - """An unsaved parent that isn't in the batch must not trigger the guard.""" + def test_bulk_create_rejects_unsaved_parent_not_in_batch(self): + """ + An unsaved parent that isn't in the batch must be rejected with a clear + ValueError; otherwise the child's parent_id is None at INSERT time and + the BEFORE trigger silently stores the row as a root. + """ external_parent = Region(name='X', slug='x-bcok') # not in the batch - # This use is rare but valid: external_parent is unsaved, but the caller - # is responsible for ensuring it exists before the insert reaches the DB. - # The guard only inspects parents that share the same batch. - with self.assertRaises(Exception): - # The DB will reject this because parent has no PK; the point is the - # guard itself must not raise its own ValueError here. + with self.assertRaises(ValueError) as ctx: Region.objects.bulk_create([Region(parent=external_parent, name='Y', slug='y-bcok')]) + self.assertIn('unsaved parent', str(ctx.exception)) + # Nothing should have been written. + self.assertFalse(Region.objects.filter(slug='y-bcok').exists()) class TreeNodeFilterTests(TestCase): @@ -480,8 +482,8 @@ def test_rename_cascades_into_descendants(self): mid.refresh_from_db() leaf.refresh_from_db() self.assertEqual(parent.sort_path, 'Zulu') - self.assertEqual(mid.sort_path, f'Zulu{chr(1)}Mid') - self.assertEqual(leaf.sort_path, f'Zulu{chr(1)}Mid{chr(1)}Leaf') + self.assertEqual(mid.sort_path, f'Zulu{chr(9)}Mid') + self.assertEqual(leaf.sort_path, f'Zulu{chr(9)}Mid{chr(9)}Leaf') # Paths unchanged — only sort_path moved. self.assertEqual(mid.path, _path(parent.pk, mid.pk)) self.assertEqual(leaf.path, _path(parent.pk, mid.pk, leaf.pk)) @@ -497,7 +499,7 @@ def test_rename_does_not_affect_unrelated_subtree(self): a.save() b_kid.refresh_from_db() - self.assertEqual(b_kid.sort_path, f'BB{chr(1)}BKid') + self.assertEqual(b_kid.sort_path, f'BB{chr(9)}BKid') class RebuildSortPathsTests(TestCase): @@ -517,7 +519,7 @@ def test_rebuild_after_raw_update(self): parent.refresh_from_db() mid.refresh_from_db() self.assertEqual(parent.sort_path, 'Bravo') - self.assertEqual(mid.sort_path, f'Bravo{chr(1)}Mid') + self.assertEqual(mid.sort_path, f'Bravo{chr(9)}Mid') def test_raises_without_sort_path(self): # InventoryItem uses LtreeModel but doesn't have a sort_path column. @@ -537,7 +539,7 @@ def test_create_refreshes_sort_path(self): child = Region.objects.create(name='Kid', slug='kid-spr', parent=root) db_sort_path = Region.objects.values_list('sort_path', flat=True).get(pk=child.pk) self.assertEqual(child.sort_path, db_sort_path) - self.assertEqual(child.sort_path, f'Root{chr(1)}Kid') + self.assertEqual(child.sort_path, f'Root{chr(9)}Kid') def test_reparent_refreshes_sort_path(self): a = Region.objects.create(name='Alpha', slug='alpha-spr') @@ -547,7 +549,7 @@ def test_reparent_refreshes_sort_path(self): child.save() db_sort_path = Region.objects.values_list('sort_path', flat=True).get(pk=child.pk) self.assertEqual(child.sort_path, db_sort_path) - self.assertEqual(child.sort_path, f'Bravo{chr(1)}Kid') + self.assertEqual(child.sort_path, f'Bravo{chr(9)}Kid') def test_rename_refreshes_sort_path(self): # The trigger rewrites sort_path on a name change; the in-memory instance @@ -686,3 +688,106 @@ def test_rename_preserves_descendant_path_but_updates_sort_path(self): self.assertEqual(child.path, original_child_path) self.assertNotIn('Bravo', child.sort_path) self.assertIn('Zulu', child.sort_path) + + +class CycleGuardWithEmptyPathTests(TestCase): + """ + _parent_creates_cycle must catch self-as-parent even when self.path is + empty or deferred — otherwise an instance constructed without a loaded + path can pass the Python guard and corrupt the tree. + """ + + def test_self_parent_rejected_when_path_is_empty(self): + from django.core.exceptions import ValidationError + a = Region.objects.create(name='A', slug='a-empty-cyc') + # Simulate a caller (script, plugin) that holds an instance whose `path` + # attribute was never loaded — e.g. via .only('id', 'parent_id'). + # Empty-string path is the LtreeField default, so this mirrors what a + # deferred-or-unset path looks like at the Python layer. + a.path = '' + a.parent_id = a.pk + with self.assertRaises(ValidationError): + a.save() + + +class TriggerCycleGuardTests(TestCase): + """ + The BEFORE INSERT/UPDATE trigger must reject a parent_id assignment that + would form a cycle. The Python save()-time guard already catches this for + ordinary save(); the trigger backstops QuerySet.update / bulk_update / + raw UPDATEs that bypass save(). + """ + + def test_queryset_update_to_self_parent_blocked_by_trigger(self): + from django.db import IntegrityError, transaction + a = Region.objects.create(name='A', slug='a-tgcyc') + # IntegrityError leaves Django's transaction marked-for-rollback; wrap + # the failing UPDATE in a savepoint so the outer test transaction can + # continue to issue queries (refresh_from_db). + with self.assertRaises(IntegrityError) as ctx: + with transaction.atomic(): + Region.objects.filter(pk=a.pk).update(parent_id=a.pk) + self.assertIn('cycle detected', str(ctx.exception)) + a.refresh_from_db() + self.assertIsNone(a.parent_id) + + def test_queryset_update_to_descendant_blocked_by_trigger(self): + from django.db import IntegrityError, transaction + a = Region.objects.create(name='A', slug='a-tgcyc2') + b = Region.objects.create(parent=a, name='B', slug='b-tgcyc2') + # Try to reparent A under B (B is A's descendant) via raw UPDATE. + with self.assertRaises(IntegrityError) as ctx: + with transaction.atomic(): + Region.objects.filter(pk=a.pk).update(parent_id=b.pk) + self.assertIn('cycle detected', str(ctx.exception)) + a.refresh_from_db() + self.assertIsNone(a.parent_id) + + def test_legitimate_reparent_via_update_still_works(self): + a = Region.objects.create(name='A', slug='a-tgok') + b = Region.objects.create(name='B', slug='b-tgok') + child = Region.objects.create(parent=a, name='C', slug='c-tgok') + Region.objects.filter(pk=child.pk).update(parent_id=b.pk) + child.refresh_from_db() + self.assertEqual(child.parent_id, b.pk) + self.assertEqual(child.path, _path(b.pk, child.pk)) + + +class NaturalSortSortPathTests(TestCase): + """ + Sort-path separator must produce correct tree-flatten ordering under the + `natural_sort` (ICU `und-u-kn-true`) collation used by ModuleBay, + TenantGroup, and WirelessLANGroup. chr(1) was variable-ignorable under + that collation and would interleave children with unrelated roots; + chr(9) (TAB) is treated non-variably and orders deterministically. + """ + + def test_chr9_separator_collates_below_letters_under_natural_sort(self): + # Direct collation probe — independent of any model schema. + with connection.cursor() as cur: + cur.execute(""" + SELECT + ('A' || chr(9) || 'Z') COLLATE "natural_sort" + < 'AA' COLLATE "natural_sort" + """) + self.assertTrue(cur.fetchone()[0]) + + def test_tree_flatten_ordering_under_natural_sort(self): + # TenantGroup.sort_path uses natural_sort collation; build a small tree + # where chr(1) would have produced wrong sibling ordering. TenantGroup + # names are globally unique, so each row gets a distinct name. Under + # the old chr(1) separator the child's sort_path was variable-ignorable + # and collated AFTER the unrelated root 'nsP1'; chr(9) sorts strictly + # below digits/letters and keeps children clustered under their parent. + from tenancy.models import TenantGroup + parent = TenantGroup.objects.create(name='nsP', slug='nsp-ns') + TenantGroup.objects.create(parent=parent, name='nsPchild', slug='nspc-ns') + TenantGroup.objects.create(name='nsP1', slug='nsp1-ns') # unrelated root + + names = list( + TenantGroup.objects.filter(slug__endswith='-ns') + .order_by('sort_path') + .values_list('name', flat=True) + ) + # Expected tree-flatten: parent, its child, then the unrelated root. + self.assertEqual(names, ['nsP', 'nsPchild', 'nsP1']) diff --git a/netbox/wireless/migrations/0020_ltree_paths.py b/netbox/wireless/migrations/0020_ltree_paths.py index 90b4e1e386f..9e49ecbcfaf 100644 --- a/netbox/wireless/migrations/0020_ltree_paths.py +++ b/netbox/wireless/migrations/0020_ltree_paths.py @@ -57,7 +57,7 @@ class Migration(migrations.Migration): UNION ALL SELECT r.id, r.parent_id, t.path || lpad(r.id::text, 19, '0')::ltree, - t.sort_path || chr(1) || r.name + t.sort_path || chr(9) || r.name FROM "{TABLE}" r JOIN t ON r.parent_id = t.id ) UPDATE "{TABLE}" SET path = t.path, sort_path = t.sort_path @@ -66,6 +66,25 @@ class Migration(migrations.Migration): reverse_sql=migrations.RunSQL.noop, ), + # Fail fast if any row still has NULL path (orphan FKs) before the + # AlterField below tries to set NOT NULL inside ALTER COLUMN. + migrations.RunSQL( + f""" +DO $$ +DECLARE missing bigint; +BEGIN + SELECT count(*) INTO missing FROM "{TABLE}" WHERE path IS NULL; + IF missing > 0 THEN + RAISE EXCEPTION + 'ltree backfill left % rows in "{TABLE}" with NULL path; ' + 'likely orphan parent_id references — resolve before re-running ' + 'this migration', missing; + END IF; +END $$; +""", + reverse_sql=migrations.RunSQL.noop, + ), + migrations.AlterField( model_name=MODEL, name='path', field=netbox.models.ltree.LtreeField(blank=True, default='', editable=False), From f8cd61c6c91abb9bafe0c882f8959888762aa8cc Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Jun 2026 09:45:24 -0700 Subject: [PATCH 21/42] fixes --- netbox/netbox/models/ltree.py | 77 +++++++++++++++------------- netbox/utilities/tests/test_ltree.py | 77 +++++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 37 deletions(-) diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index 15957a8cb04..697460ab7fa 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -117,37 +117,25 @@ class LtreeQuerySet(RestrictedQuerySet): def bulk_create(self, objs, *args, **kwargs): """ - Same as the standard `bulk_create` but verifies that, for every row whose - parent is an unsaved instance, the parent appears earlier in the same - batch. PostgreSQL fires the BEFORE INSERT trigger per row in list order; - a child whose parent is unsaved and either (a) appears later in the batch - or (b) is not in the batch at all would otherwise be persisted silently - with a stale root-level path. Raises ValueError in both cases. + Same as the standard `bulk_create` but rejects any row whose parent is an + unsaved instance. Django's bulk_create builds the multi-row INSERT VALUES + up front from each instance's `parent_id`; an unsaved parent's pk is not + assigned until the INSERT's RETURNING clause executes, so a child + referencing an unsaved parent (even one earlier in the same batch) goes + in with parent_id=NULL and the BEFORE trigger stores it as a root. Save + the parents first, then bulk_create their children. """ objs_list = list(objs) - seen = set() for idx, obj in enumerate(objs_list): parent = getattr(obj, 'parent', None) - if parent is not None and parent.pk is None and id(parent) not in seen: - # Parent is unsaved and has not yet been encountered. It must appear - # later in the batch (which is still wrong — children must follow - # parents) or not at all; either way the trigger would see - # parent_id=NULL and store this row as a root. - in_batch_later = any(later is parent for later in objs_list[idx + 1:]) - if in_batch_later: - raise ValueError( - "bulk_create: child at index {idx} references parent that " - "appears later in the same batch; parents must precede " - "their children or the child's path will be stored as " - "a root.".format(idx=idx) - ) + if parent is not None and parent.pk is None: raise ValueError( - "bulk_create: child at index {idx} references an unsaved parent " - "that is not in this batch; save the parent first or include " - "it earlier in the batch — otherwise the child would be " - "persisted with a root-level path.".format(idx=idx) + "bulk_create: child at index {idx} references an unsaved parent. " + "Django cannot propagate the parent's RETURNING-assigned pk into " + "the child's parent_id before the INSERT executes, so the child " + "would be persisted with parent_id=NULL and stored as a root. " + "Save the parent first, then bulk_create the children.".format(idx=idx) ) - seen.add(id(obj)) return super().bulk_create(objs_list, *args, **kwargs) def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=False): @@ -486,19 +474,34 @@ def get_root(self): def get_parent(self): return self.parent + @classmethod + def _tree_order_field(cls): + """ + Field name to order hierarchical queries by. Models that carry a + `sort_path` column (the MPTT `order_insertion_by=('name',)` equivalent) + order siblings by name to match the prior MPTT behavior; models + without it fall back to `path` (PK-padded, insertion order). + """ + if any(f.attname == 'sort_path' for f in cls._meta.concrete_fields): + return 'sort_path' + return 'path' + def get_ancestors(self, ascending=False, include_self=False): if not self.path: return type(self)._default_manager.none() qs = type(self)._default_manager.filter(path__ancestor=self.path) if not include_self: qs = qs.exclude(pk=self.pk) - return qs.order_by('-path' if ascending else 'path') + order_field = self._tree_order_field() + return qs.order_by(f'-{order_field}' if ascending else order_field) def get_descendants(self, include_self=False): if not self.path: return type(self)._default_manager.none() lookup = 'descendant_or_equal' if include_self else 'descendant' - return type(self)._default_manager.filter(**{f'path__{lookup}': self.path}).order_by('path') + return type(self)._default_manager.filter( + **{f'path__{lookup}': self.path} + ).order_by(self._tree_order_field()) def get_descendant_count(self): if not self.path: @@ -506,21 +509,21 @@ def get_descendant_count(self): return type(self)._default_manager.filter(path__descendant=self.path).count() def get_children(self): - return type(self)._default_manager.filter(parent_id=self.pk) + return type(self)._default_manager.filter(parent_id=self.pk).order_by(self._tree_order_field()) def get_family(self): - """Ancestors + self + descendants, in path order.""" + """Ancestors + self + descendants, in tree-order.""" if not self.path: return type(self)._default_manager.none() return type(self)._default_manager.filter( Q(path__ancestor=self.path) | Q(path__descendant_or_equal=self.path) - ).distinct().order_by('path') + ).distinct().order_by(self._tree_order_field()) def get_siblings(self, include_self=False): qs = type(self)._default_manager.filter(parent_id=self.parent_id) if not include_self: qs = qs.exclude(pk=self.pk) - return qs + return qs.order_by(self._tree_order_field()) def move_to(self, target, position='last-child'): """ @@ -604,10 +607,11 @@ def rebuild_sort_paths(cls, name_column='name'): INTO parent_path USING NEW.parent_id; -- Cycle guard. The Python LtreeModel.save() also rejects cyclic moves, -- but a QuerySet.update() / bulk_update() bypasses save() entirely, so - -- catch the case here as a last line of defense. lpad(NEW.id,..) @> - -- parent_path is TRUE iff parent_path starts with this row's own - -- label (i.e. the proposed parent is self or one of self's descendants). - IF lpad(NEW.id::text, 19, '0')::ltree @> parent_path THEN + -- catch the case here as a last line of defense. A cycle exists iff + -- this row's own label appears anywhere in parent_path (the row would + -- become its own ancestor). Match the label as any segment via lquery. + IF parent_path ~ ('*.' || lpad(NEW.id::text, 19, '0') || '.*')::lquery + OR parent_path = lpad(NEW.id::text, 19, '0')::ltree THEN RAISE EXCEPTION 'cycle detected: %% cannot be assigned a parent that is itself or one of its descendants', TG_TABLE_NAME USING ERRCODE = 'check_violation'; END IF; @@ -663,7 +667,8 @@ def rebuild_sort_paths(cls, name_column='name'): INTO parent_path, parent_sort_path USING NEW.parent_id; -- Cycle guard. See _COMPUTE_PATH_ONLY_FN for the rationale; this catches -- raw UPDATE / bulk_update paths that bypass LtreeModel.save(). - IF lpad(NEW.id::text, 19, '0')::ltree @> parent_path THEN + IF parent_path ~ ('*.' || lpad(NEW.id::text, 19, '0') || '.*')::lquery + OR parent_path = lpad(NEW.id::text, 19, '0')::ltree THEN RAISE EXCEPTION 'cycle detected: %% cannot be assigned a parent that is itself or one of its descendants', TG_TABLE_NAME USING ERRCODE = 'check_violation'; END IF; diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py index 9e60119ee11..017081bd7dd 100644 --- a/netbox/utilities/tests/test_ltree.py +++ b/netbox/utilities/tests/test_ltree.py @@ -232,6 +232,35 @@ def test_default_ordering_is_sort_path(self): ) self.assertEqual(names, ['A', 'B']) + def test_get_descendants_returns_siblings_in_name_order(self): + """ + For models with a sort_path column, get_descendants() must return + descendants with siblings in name order (matching MPTT's + order_insertion_by behavior), not insertion/PK order. + """ + root = Region.objects.create(name='Root', slug='root-gdord') + # Insert siblings out of name order + Region.objects.create(parent=root, name='Zebra', slug='z-gdord') + Region.objects.create(parent=root, name='Aardvark', slug='a-gdord') + Region.objects.create(parent=root, name='Buffalo', slug='b-gdord') + names = list(root.get_descendants().values_list('name', flat=True)) + self.assertEqual(names, ['Aardvark', 'Buffalo', 'Zebra']) + + def test_get_children_returns_in_name_order(self): + root = Region.objects.create(name='Root', slug='root-gcord') + Region.objects.create(parent=root, name='Zebra', slug='z-gcord') + Region.objects.create(parent=root, name='Aardvark', slug='a-gcord') + names = list(root.get_children().values_list('name', flat=True)) + self.assertEqual(names, ['Aardvark', 'Zebra']) + + def test_get_siblings_returns_in_name_order(self): + root = Region.objects.create(name='Root', slug='root-gsord') + Region.objects.create(parent=root, name='Zebra', slug='z-gsord') + a = Region.objects.create(parent=root, name='Aardvark', slug='a-gsord') + Region.objects.create(parent=root, name='Buffalo', slug='b-gsord') + names = list(a.get_siblings().values_list('name', flat=True)) + self.assertEqual(names, ['Buffalo', 'Zebra']) + class AddRelatedCountTests(TestCase): """add_related_count must cumulate across subtrees via path <@.""" @@ -374,7 +403,22 @@ def test_bulk_create_rejects_child_before_parent_in_same_batch(self): unsaved_child = Region(parent=unsaved_parent, name='C', slug='c-bcrej') with self.assertRaises(ValueError) as ctx: Region.objects.bulk_create([unsaved_child, unsaved_parent]) - self.assertIn('parents must precede', str(ctx.exception)) + self.assertIn('unsaved parent', str(ctx.exception)) + + def test_bulk_create_rejects_unsaved_parent_earlier_in_batch(self): + """ + An unsaved parent placed earlier in the batch must also be rejected: + Django binds VALUES from the child's parent_id BEFORE the parent's + RETURNING-assigned pk lands, so the child would be inserted with + parent_id=NULL and silently stored as a root. + """ + unsaved_parent = Region(name='P', slug='p-bcearly') + unsaved_child = Region(parent=unsaved_parent, name='C', slug='c-bcearly') + with self.assertRaises(ValueError) as ctx: + Region.objects.bulk_create([unsaved_parent, unsaved_child]) + self.assertIn('unsaved parent', str(ctx.exception)) + # Nothing should have been written. + self.assertFalse(Region.objects.filter(slug__in=('p-bcearly', 'c-bcearly')).exists()) def test_bulk_create_rejects_unsaved_parent_not_in_batch(self): """ @@ -752,6 +796,37 @@ def test_legitimate_reparent_via_update_still_works(self): self.assertEqual(child.parent_id, b.pk) self.assertEqual(child.path, _path(b.pk, child.pk)) + def test_queryset_update_mid_tree_to_own_descendant_blocked(self): + """ + Reparenting a non-root node under one of its own (non-immediate) + descendants must raise. The earlier `lpad(NEW.id) @> parent_path` + check only fired when NEW was the root of parent_path, missing + mid-tree cycles entirely. + """ + from django.db import IntegrityError, transaction + a = Region.objects.create(name='A', slug='a-midcyc') + b = Region.objects.create(parent=a, name='B', slug='b-midcyc') + c = Region.objects.create(parent=b, name='C', slug='c-midcyc') + # Try to make B a child of C — B is mid-tree (path A.B), parent_path is A.B.C. + with self.assertRaises(IntegrityError) as ctx: + with transaction.atomic(): + Region.objects.filter(pk=b.pk).update(parent_id=c.pk) + self.assertIn('cycle detected', str(ctx.exception)) + b.refresh_from_db() + self.assertEqual(b.parent_id, a.pk) + + def test_queryset_update_mid_tree_self_loop_blocked(self): + """A non-root node assigning itself as parent must also raise.""" + from django.db import IntegrityError, transaction + a = Region.objects.create(name='A', slug='a-midself') + b = Region.objects.create(parent=a, name='B', slug='b-midself') + with self.assertRaises(IntegrityError) as ctx: + with transaction.atomic(): + Region.objects.filter(pk=b.pk).update(parent_id=b.pk) + self.assertIn('cycle detected', str(ctx.exception)) + b.refresh_from_db() + self.assertEqual(b.parent_id, a.pk) + class NaturalSortSortPathTests(TestCase): """ From a0d1d59b0e3377ed7c4c9ec656c9d17f27443789 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Jun 2026 10:41:14 -0700 Subject: [PATCH 22/42] fixes --- netbox/dcim/models/modules.py | 9 +++++++++ netbox/dcim/views.py | 8 ++++++++ netbox/netbox/models/ltree.py | 33 ++++++++++++++++++++++++++++----- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/netbox/dcim/models/modules.py b/netbox/dcim/models/modules.py index 14984db9d0d..4bdd659160d 100644 --- a/netbox/dcim/models/modules.py +++ b/netbox/dcim/models/modules.py @@ -400,6 +400,15 @@ def save(self, *args, **kwargs): ) update_fields = ['module'] + # ModuleBay.parent is derived from .module in ModuleBay.save(), and the + # path/sort_path trigger only fires on parent_id/name changes. A bare + # bulk_update(['module']) bypasses both, leaving the adopted bay rooted + # at its pre-adoption location. Set parent in lockstep so the BEFORE + # trigger recomputes path/sort_path. + if component_model is ModuleBay: + for instance in update_instances: + instance.parent = self.module_bay + update_fields = ['module', 'parent'] component_model.objects.bulk_update(update_instances, update_fields) for component in update_instances: diff --git a/netbox/dcim/views.py b/netbox/dcim/views.py index cc888597d75..f463fe4e989 100644 --- a/netbox/dcim/views.py +++ b/netbox/dcim/views.py @@ -2849,6 +2849,14 @@ class DeviceInventoryView(DeviceComponentsView): hide_if_empty=True ) + def get_children(self, request, parent): + # DeviceInventoryItemTable indents rows by record.level; under MPTT, + # TreeManager forced tree-flatten (tree_id, lft) ordering so descendants + # were contiguous with their parent. InventoryItem.Meta.ordering is a + # flat sort suited to the global list; for the indented device tab, + # order by ltree path so the rendered hierarchy is correct. + return super().get_children(request, parent).order_by('path') + @register_model_view(Device, 'configcontext', path='config-context') class DeviceConfigContextView(ObjectConfigContextView): diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index 697460ab7fa..a727171f6d1 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -12,7 +12,7 @@ inserts and parent_id changes via refresh_from_db(fields=['path']). """ from django.core.exceptions import ValidationError -from django.db import connection, migrations, models +from django.db import IntegrityError, connection, migrations, models from django.db.models import ForeignKey, Lookup, ManyToManyField, Q from django.db.models.expressions import RawSQL from django.utils.translation import gettext_lazy as _ @@ -413,7 +413,18 @@ def save(self, *args, **kwargs): if parent_changed and self._parent_creates_cycle(): raise ValidationError(_("Cannot assign self or a descendant as parent.")) - super().save(*args, **kwargs) + try: + super().save(*args, **kwargs) + except IntegrityError as exc: + # A concurrent reparent that races the Python-level _parent_creates_cycle + # check can be caught by the BEFORE trigger as a check_violation cycle. + # Surface it as a ValidationError so the API/UI returns 400 instead of + # the IntegrityError → 500 the trigger would otherwise produce. + if 'cycle detected' in str(exc): + raise ValidationError( + _("Cannot assign self or a descendant as parent.") + ) from None + raise if (parent_changed or name_changed) and not is_insert: # The triggers rewrote path/sort_path on this UPDATE; fetch them back. @@ -461,6 +472,10 @@ def is_root_node(self): return self.parent_id is None def is_leaf_node(self): + if self.pk is None: + # Unsaved instance has no children. Without this guard, + # filter(parent_id=None) would match every existing root. + return True return not type(self).objects.filter(parent_id=self.pk).exists() def is_child_node(self): @@ -469,6 +484,11 @@ def is_child_node(self): def get_root(self): if self.is_root_node(): return self + if not self.path: + # Unsaved (no path computed yet). Walk up the in-memory parent + # chain if available; otherwise we have no way to resolve the root. + parent = getattr(self, 'parent', None) + return parent.get_root() if parent is not None else None return type(self)._default_manager.get(pk=self._root_pk()) def get_parent(self): @@ -683,20 +703,23 @@ def rebuild_sort_paths(cls, name_column='name'): $$ LANGUAGE plpgsql; ''' -_CASCADE_PATH_AND_SORT_FN = ''' +_CASCADE_PATH_AND_SORT_FN = """ CREATE OR REPLACE FUNCTION "{table}_ltree_cascade_path_fn"() RETURNS TRIGGER AS $$ BEGIN + -- COALESCE guards against a NULL sort_path slipping in via a raw write that + -- bypassed the BEFORE trigger: without it, length(NULL)/substring(... FROM NULL) + -- would cascade NULL to every descendant's sort_path in one shot. EXECUTE format( 'UPDATE %%I SET ' ' path = $1 || subpath(path, nlevel($2)), ' - ' sort_path = $4 || substring(sort_path FROM length($5) + 1) ' + ' sort_path = COALESCE($4, '''') || substring(COALESCE(sort_path, '''') FROM length(COALESCE($5, '''')) + 1) ' 'WHERE path <@ $2 AND id != $3', TG_TABLE_NAME ) USING NEW.path, OLD.path, NEW.id, NEW.sort_path, OLD.sort_path; RETURN NULL; END $$ LANGUAGE plpgsql; -''' +""" _BEFORE_TRIGGER_PATH_ONLY = ''' CREATE TRIGGER "{table}_ltree_compute_path" From 4a15446ff0d32d73c8e0966a0f50003f54c0181a Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Jun 2026 11:56:36 -0700 Subject: [PATCH 23/42] cleanup --- .../dcim/models/device_component_templates.py | 9 +-- netbox/dcim/models/device_components.py | 15 +--- netbox/netbox/models/__init__.py | 20 ++--- netbox/netbox/models/ltree.py | 77 +++++++++++++++---- netbox/tenancy/models/tenants.py | 9 +-- netbox/utilities/tests/test_ltree.py | 1 + netbox/wireless/models.py | 9 +-- 7 files changed, 82 insertions(+), 58 deletions(-) diff --git a/netbox/dcim/models/device_component_templates.py b/netbox/dcim/models/device_component_templates.py index 9a0a73486d5..a8bd766ab8b 100644 --- a/netbox/dcim/models/device_component_templates.py +++ b/netbox/dcim/models/device_component_templates.py @@ -885,14 +885,7 @@ class Meta: verbose_name = _('inventory item template') verbose_name_plural = _('inventory item templates') - def clean(self): - super().clean() - - # A template cannot be its own parent or a descendant of itself - if self.pk and self._parent_creates_cycle(): - raise ValidationError({ - "parent": _("Cannot assign self or a descendant as parent.") - }) + # Self/descendant-as-parent cycles are rejected by the inherited LtreeModel.clean(). def instantiate(self, **kwargs): parent = InventoryItem.objects.get(name=self.parent.name, **kwargs) if self.parent else None diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index 044059aae1c..d0f3768e6d5 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -1336,14 +1336,13 @@ class ModuleBay(ModularComponentModel, TrackingModelMixin, LtreeModel): verbose_name=_('enabled'), default=True, ) + # sort_path inherits `name`'s natural_sort collation automatically (LtreeModelBase), + # so ORDER BY sort_path sorts siblings naturally (Slot 0..Slot 13) — as MPTT's + # order_insertion_by=('name',) did — rather than lexicographically. sort_path = SortPathField( editable=False, blank=True, default='', - # `name` uses the natural_sort collation; match it here so ORDER BY - # sort_path sorts siblings naturally (Slot 0..Slot 13) as MPTT's - # order_insertion_by=('name',) did, not lexicographically. - db_collation='natural_sort', ) clone_fields = ('device', 'enabled') @@ -1576,13 +1575,7 @@ class Meta: verbose_name_plural = _('inventory items') def clean(self): - super().clean() - - # An InventoryItem cannot be its own parent or a descendant of itself - if self.pk and self._parent_creates_cycle(): - raise ValidationError({ - "parent": _("Cannot assign self or a descendant as parent.") - }) + super().clean() # LtreeModel.clean() rejects self/descendant-as-parent cycles # Validation for moving InventoryItems if not self._state.adding: diff --git a/netbox/netbox/models/__init__.py b/netbox/netbox/models/__init__.py index 0b0ace92f7e..3b56b943efc 100644 --- a/netbox/netbox/models/__init__.py +++ b/netbox/netbox/models/__init__.py @@ -191,15 +191,6 @@ class Meta: def __str__(self): return self.name - def clean(self): - super().clean() - - # A nested group cannot be its own parent or a descendant of itself - if not self._state.adding and self.parent and self.parent in self.get_descendants(include_self=True): - raise ValidationError({ - "parent": "Cannot assign self or child {type} as parent.".format(type=self._meta.verbose_name) - }) - class NestedGroupModel(NestedGroupModelMixin, MPTTModel): """ @@ -225,6 +216,17 @@ class Meta: class MPTTMeta: order_insertion_by = ('name',) + def clean(self): + super().clean() + + # A nested group cannot be its own parent or a descendant of itself. The ltree + # base (NestedLtreeGroupModel) enforces this via LtreeModel.clean(); this MPTT + # variant keeps the original get_descendants()-based check. + if not self._state.adding and self.parent and self.parent in self.get_descendants(include_self=True): + raise ValidationError({ + "parent": _("Cannot assign self or a descendant as parent.") + }) + class NestedLtreeGroupModel(NestedGroupModelMixin, LtreeModel): """ diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index a727171f6d1..e2f69c0a652 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -11,7 +11,7 @@ mutates paths directly; it only reads `path` back from the database after inserts and parent_id changes via refresh_from_db(fields=['path']). """ -from django.core.exceptions import ValidationError +from django.core.exceptions import FieldDoesNotExist, ValidationError from django.db import IntegrityError, connection, migrations, models from django.db.models import ForeignKey, Lookup, ManyToManyField, Q from django.db.models.expressions import RawSQL @@ -281,7 +281,33 @@ class LtreeManager(models.Manager.from_queryset(LtreeQuerySet)): # Abstract model # -class LtreeModel(models.Model): +class LtreeModelBase(models.base.ModelBase): + """ + Metaclass that keeps a model's `sort_path` collation in sync with its `name`. + + `sort_path` holds a chr(9)-joined chain of ancestor names, so to flatten siblings + in the same order the database sorts `name`, the two columns must share a collation. + Deriving it here means a subclass that gives `name` a custom db_collation (e.g. + `natural_sort`) automatically gets a matching `sort_path` — no need to redeclare the + field just to repeat the collation. An explicit db_collation on `sort_path` is left + untouched. + """ + def __new__(mcs, name, bases, namespace, **kwargs): + cls = super().__new__(mcs, name, bases, namespace, **kwargs) + if cls._meta.abstract: + return cls + try: + name_field = cls._meta.get_field('name') + sort_path_field = cls._meta.get_field('sort_path') + except FieldDoesNotExist: + return cls + name_collation = getattr(name_field, 'db_collation', None) + if name_collation and not getattr(sort_path_field, 'db_collation', None): + sort_path_field.db_collation = name_collation + return cls + + +class LtreeModel(models.Model, metaclass=LtreeModelBase): """ Abstract base for hierarchical models backed by PostgreSQL ltree. @@ -377,6 +403,22 @@ def _parent_creates_cycle(self): pk=self.parent_id, path__descendant_or_equal=self.path ).exists() + def clean(self): + """ + Reject assigning self or a descendant as parent, surfacing it as a field + error for forms/serializers. This mirrors the save()-time guard; the two + share _parent_creates_cycle() so the rule lives in exactly one place. + + Subclasses whose `parent` is system-managed (e.g. ModuleBay) disable the + check by overriding _parent_creates_cycle() to return False. + """ + super().clean() + + if self.pk and self._parent_creates_cycle(): + raise ValidationError({ + "parent": _("Cannot assign self or a descendant as parent.") + }) + def save(self, *args, **kwargs): """ Triggers compute `path` (and `sort_path`, where present) server-side. @@ -417,25 +459,30 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) except IntegrityError as exc: # A concurrent reparent that races the Python-level _parent_creates_cycle - # check can be caught by the BEFORE trigger as a check_violation cycle. - # Surface it as a ValidationError so the API/UI returns 400 instead of - # the IntegrityError → 500 the trigger would otherwise produce. - if 'cycle detected' in str(exc): + # check can be caught by the BEFORE trigger, which RAISEs the cycle with + # ERRCODE = check_violation (SQLSTATE 23514). Match on the SQLSTATE rather + # than the message text (which is not stable across wording/locales), and + # surface it as a ValidationError so the API/UI returns 400 instead of the + # IntegrityError → 500 the trigger would otherwise produce. None of the + # ltree-backed models declare other CHECK constraints, so 23514 here is + # unambiguously the cycle guard. + if getattr(exc.__cause__, 'sqlstate', None) == '23514': raise ValidationError( _("Cannot assign self or a descendant as parent.") ) from None raise if (parent_changed or name_changed) and not is_insert: - # The triggers rewrote path/sort_path on this UPDATE; fetch them back. - # (INSERT ... RETURNING covers the insert case, so only updates reach here.) + # The triggers rewrote path/sort_path on this UPDATE; fetch them back so + # the in-memory instance matches storage (e.g. so change logging snapshots + # the value the triggers wrote, not a stale one). This costs one extra + # SELECT per reparent/rename; INSERT ... RETURNING covers the insert case, + # so only updates reach here. refresh_fields = [ fname for fname in ('path', 'sort_path') if any(f.attname == fname for f in self._meta.concrete_fields) ] - row = type(self).objects.values_list(*refresh_fields).get(pk=self.pk) - for fname, value in zip(refresh_fields, row): - setattr(self, fname, value) + self.refresh_from_db(fields=refresh_fields) if is_insert or parent_written: self._loaded_parent_id = self.parent_id @@ -535,9 +582,11 @@ def get_family(self): """Ancestors + self + descendants, in tree-order.""" if not self.path: return type(self)._default_manager.none() + # No .distinct() needed: this is a single-table OR with no joins, so a row + # (self, which matches both branches) is still returned only once. return type(self)._default_manager.filter( Q(path__ancestor=self.path) | Q(path__descendant_or_equal=self.path) - ).distinct().order_by(self._tree_order_field()) + ).order_by(self._tree_order_field()) def get_siblings(self, include_self=False): qs = type(self)._default_manager.filter(parent_id=self.parent_id) @@ -632,7 +681,7 @@ def rebuild_sort_paths(cls, name_column='name'): -- become its own ancestor). Match the label as any segment via lquery. IF parent_path ~ ('*.' || lpad(NEW.id::text, 19, '0') || '.*')::lquery OR parent_path = lpad(NEW.id::text, 19, '0')::ltree THEN - RAISE EXCEPTION 'cycle detected: %% cannot be assigned a parent that is itself or one of its descendants', TG_TABLE_NAME + RAISE EXCEPTION 'cycle detected: %% cannot be its own ancestor', TG_TABLE_NAME USING ERRCODE = 'check_violation'; END IF; NEW.path := parent_path || lpad(NEW.id::text, 19, '0')::ltree; @@ -689,7 +738,7 @@ def rebuild_sort_paths(cls, name_column='name'): -- raw UPDATE / bulk_update paths that bypass LtreeModel.save(). IF parent_path ~ ('*.' || lpad(NEW.id::text, 19, '0') || '.*')::lquery OR parent_path = lpad(NEW.id::text, 19, '0')::ltree THEN - RAISE EXCEPTION 'cycle detected: %% cannot be assigned a parent that is itself or one of its descendants', TG_TABLE_NAME + RAISE EXCEPTION 'cycle detected: %% cannot be its own ancestor', TG_TABLE_NAME USING ERRCODE = 'check_violation'; END IF; NEW.path := parent_path || lpad(NEW.id::text, 19, '0')::ltree; diff --git a/netbox/tenancy/models/tenants.py b/netbox/tenancy/models/tenants.py index 68e6e0252ee..95cdfafb8f1 100644 --- a/netbox/tenancy/models/tenants.py +++ b/netbox/tenancy/models/tenants.py @@ -5,7 +5,6 @@ from netbox.models import NestedLtreeGroupModel, PrimaryModel from netbox.models.features import ContactsMixin -from netbox.models.ltree import SortPathField __all__ = ( 'Tenant', @@ -28,13 +27,7 @@ class TenantGroup(NestedLtreeGroupModel): max_length=100, unique=True ) - # Override the abstract parent's sort_path to use natural_sort, matching `name`. - sort_path = SortPathField( - editable=False, - blank=True, - default='', - db_collation='natural_sort', - ) + # sort_path inherits natural_sort collation from `name` automatically (LtreeModelBase). class Meta: ordering = ('sort_path',) diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py index 017081bd7dd..4442febf179 100644 --- a/netbox/utilities/tests/test_ltree.py +++ b/netbox/utilities/tests/test_ltree.py @@ -673,6 +673,7 @@ class MPTTChangeLogExclusionTests(TestCase): def test_diff_exclude_fields_for_mptt_subclass(self): from unittest.mock import MagicMock + from mptt.models import MPTTModel class _FakeMPTT(MPTTModel): diff --git a/netbox/wireless/models.py b/netbox/wireless/models.py index e1f1febaf46..d3618d513bc 100644 --- a/netbox/wireless/models.py +++ b/netbox/wireless/models.py @@ -7,7 +7,6 @@ from dcim.constants import WIRELESS_IFACE_TYPES from dcim.models.mixins import CachedScopeMixin from netbox.models import NestedLtreeGroupModel, PrimaryModel -from netbox.models.ltree import SortPathField from netbox.models.mixins import DistanceMixin from .choices import * @@ -63,13 +62,7 @@ class WirelessLANGroup(NestedLtreeGroupModel): max_length=100, unique=True ) - # Override the abstract parent's sort_path to use natural_sort, matching `name`. - sort_path = SortPathField( - editable=False, - blank=True, - default='', - db_collation='natural_sort', - ) + # sort_path inherits natural_sort collation from `name` automatically (LtreeModelBase). class Meta: ordering = ('sort_path',) From 056da80217b4d8eb782c0ca05982e3b89ebcff6b Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Jun 2026 14:00:33 -0700 Subject: [PATCH 24/42] cleanup --- netbox/netbox/models/ltree.py | 103 +++++++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 25 deletions(-) diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index e2f69c0a652..b9d47e82382 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -336,20 +336,27 @@ class LtreeModel(models.Model, metaclass=LtreeModelBase): Concurrency: Path maintenance takes no table-wide lock (unlike django-mptt, which - acquired a per-model advisory lock on every write to protect its global - lft/rght/tree_id numbering). Because a row's path depends only on its - parent's path, concurrent inserts and moves under *different* parents - never conflict. - - To keep a node consistent with a concurrently-reparented ancestor, the - BEFORE trigger reads the parent row `FOR SHARE`. That shared lock conflicts - with the row-exclusive lock a reparent/rename of the parent takes, so an - insert/move under P is serialized against a concurrent reparent or rename - of P (or of an ancestor, whose cascade must update P's row). Sibling - inserts under a stable parent still proceed concurrently — shared locks - don't conflict. Crossing reparents (moving A under B while moving B under - A) can deadlock; PostgreSQL aborts one with a retryable error rather than - silently persisting a stale path. + acquired a per-model advisory lock on *every* write to protect its global + lft/rght/tree_id numbering). Instead, the BEFORE trigger serializes per + tree: it takes a transaction-level advisory lock keyed on the root of the + tree being written (and, for a cross-tree move, on both the source and + destination roots, acquired in ascending key order to avoid deadlocks). + + Every insert, move, and reparent of a node in a tree takes the same key, + so an insert deep in a subtree and a concurrent reparent of one of its + ancestors are serialized — the loser blocks until the winner commits, and + the winner's AFTER cascade can never miss a row inserted concurrently + (which a row-level `FOR SHARE` on the parent could not prevent: a set-based + cascade's snapshot would skip a row inserted after it began). Writes to + *different* trees use different keys and proceed fully in parallel — e.g. + inventory ingestion across different devices, each its own tree. + + Two residual, retryable cases remain (PostgreSQL aborts one transaction + with a deadlock error rather than persisting a stale path): crossing + reparents (moving A under B while moving B under A), and two concurrent + *moves* in an ancestor/descendant relationship (a move locks the moved + row before its BEFORE trigger can take the advisory lock). Plain inserts + — the high-volume path — never hit this. """ # `default=''` here is a Django-side placeholder that the BEFORE INSERT # trigger always overwrites with a valid path before the row reaches @@ -411,6 +418,10 @@ def clean(self): Subclasses whose `parent` is system-managed (e.g. ModuleBay) disable the check by overriding _parent_creates_cycle() to return False. + + For sort_path-backed models, also reject a tab in the name column: sort_path + joins ancestor names with chr(9) (TAB), so a literal tab in a name would + corrupt sibling ordering for the node and its descendants. """ super().clean() @@ -419,6 +430,12 @@ def clean(self): "parent": _("Cannot assign self or a descendant as parent.") }) + has_sort_path = any(f.attname == 'sort_path' for f in self._meta.concrete_fields) + if has_sort_path and '\t' in (getattr(self, 'name', None) or ''): + raise ValidationError({ + "name": _("Name cannot contain tab characters.") + }) + def save(self, *args, **kwargs): """ Triggers compute `path` (and `sort_path`, where present) server-side. @@ -664,15 +681,49 @@ def rebuild_sort_paths(cls, name_column='name'): # Path label is the row's PK zero-padded to 19 chars (max bigint width) so that # lexicographic ordering of ltree labels matches numeric PK ordering across digit # boundaries (e.g. "0...09" sorts before "0...10"). +# Per-tree advisory locking (see _lock_tree_roots_sql / LtreeModel "Concurrency"): +# every insert/move/reparent of a node takes a transaction-level advisory lock +# keyed on the root(s) of the tree(s) it touches, BEFORE reading the parent path. +# A concurrent reparent of an ancestor takes the same key, so the two serialize +# and the AFTER cascade can never miss a row inserted concurrently; writes in +# different trees use different keys and run in parallel. +_LOCK_TREE_ROOTS_SQL = ''' + -- Destination tree root: the parent's root label, or this row's own label + -- when it is (or becomes) a root. + IF NEW.parent_id IS NOT NULL THEN + EXECUTE format('SELECT subltree(path, 0, 1)::text FROM %%I WHERE id = $1', TG_TABLE_NAME) + INTO dest_root USING NEW.parent_id; + END IF; + dest_root := COALESCE(dest_root, lpad(NEW.id::text, 19, '0')); + -- Source tree root (moves only): this row's current root, which the AFTER + -- cascade will rewrite. + IF TG_OP = 'UPDATE' AND OLD.path IS NOT NULL AND nlevel(OLD.path) > 0 THEN + old_root := subltree(OLD.path, 0, 1)::text; + END IF; + key_dest := hashtextextended(TG_TABLE_NAME || ':' || dest_root, 0); + IF old_root IS NOT NULL AND old_root <> dest_root THEN + -- Cross-tree move: lock both roots, ascending, to avoid deadlock between + -- two concurrent moves that touch the same pair. + key_old := hashtextextended(TG_TABLE_NAME || ':' || old_root, 0); + PERFORM pg_advisory_xact_lock(LEAST(key_dest, key_old)); + PERFORM pg_advisory_xact_lock(GREATEST(key_dest, key_old)); + ELSE + PERFORM pg_advisory_xact_lock(key_dest); + END IF; +''' + _COMPUTE_PATH_ONLY_FN = ''' CREATE OR REPLACE FUNCTION "{table}_ltree_compute_path_fn"() RETURNS TRIGGER AS $$ -DECLARE parent_path ltree; +DECLARE + parent_path ltree; + dest_root text; + old_root text; + key_dest bigint; + key_old bigint; BEGIN +''' + _LOCK_TREE_ROOTS_SQL + ''' IF NEW.parent_id IS NOT NULL THEN - -- FOR SHARE locks the parent row so a concurrent reparent/rename of it - -- (or of an ancestor, whose cascade updates this parent's row) cannot - -- proceed until this insert/move commits, preventing a stale path. - EXECUTE format('SELECT path FROM %%I WHERE id = $1 FOR SHARE', TG_TABLE_NAME) + EXECUTE format('SELECT path FROM %%I WHERE id = $1', TG_TABLE_NAME) INTO parent_path USING NEW.parent_id; -- Cycle guard. The Python LtreeModel.save() also rejects cyclic moves, -- but a QuerySet.update() / bulk_update() bypasses save() entirely, so @@ -727,12 +778,14 @@ def rebuild_sort_paths(cls, name_column='name'): DECLARE parent_path ltree; parent_sort_path text; + dest_root text; + old_root text; + key_dest bigint; + key_old bigint; BEGIN +''' + _LOCK_TREE_ROOTS_SQL + ''' IF NEW.parent_id IS NOT NULL THEN - -- FOR SHARE locks the parent row so a concurrent reparent/rename of it - -- (or of an ancestor, whose cascade updates this parent's row) cannot - -- proceed until this insert/move commits, preventing a stale path/sort_path. - EXECUTE format('SELECT path, sort_path FROM %%I WHERE id = $1 FOR SHARE', TG_TABLE_NAME) + EXECUTE format('SELECT path, sort_path FROM %%I WHERE id = $1', TG_TABLE_NAME) INTO parent_path, parent_sort_path USING NEW.parent_id; -- Cycle guard. See _COMPUTE_PATH_ONLY_FN for the rationale; this catches -- raw UPDATE / bulk_update paths that bypass LtreeModel.save(). @@ -742,10 +795,10 @@ def rebuild_sort_paths(cls, name_column='name'): USING ERRCODE = 'check_violation'; END IF; NEW.path := parent_path || lpad(NEW.id::text, 19, '0')::ltree; - NEW.sort_path := parent_sort_path || chr(9) || NEW.{name_col}; + NEW.sort_path := parent_sort_path || chr(9) || NEW."{name_col}"; ELSE NEW.path := lpad(NEW.id::text, 19, '0')::ltree; - NEW.sort_path := NEW.{name_col}; + NEW.sort_path := NEW."{name_col}"; END IF; RETURN NEW; END From d87052e4fb6208d22ed85692825ad89893fe4cd0 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Jun 2026 16:44:01 -0700 Subject: [PATCH 25/42] cleanup --- netbox/netbox/graphql/types.py | 14 +- netbox/netbox/models/ltree.py | 245 +++++++++++++++------------------ 2 files changed, 124 insertions(+), 135 deletions(-) diff --git a/netbox/netbox/graphql/types.py b/netbox/netbox/graphql/types.py index 3f7bc84c37a..96ac3e73a1d 100644 --- a/netbox/netbox/graphql/types.py +++ b/netbox/netbox/graphql/types.py @@ -93,9 +93,11 @@ class LtreeNodeMixin: field MPTT-based types previously surfaced automatically as a real column. The depth is computed in the database as `nlevel(path) - 1` (root = 0) and - annotated onto the queryset. We read the annotation rather than the `path` - column directly: `path` is excluded from the schema, so accessing it through - the resolver source would re-enter field resolution and recurse. + annotated onto the queryset. We prefer the annotation over the `path` column, + which is excluded from the schema. When a resolution path does not apply the + annotation (e.g. a nested relation), `ltree_level` is absent; fall back to the + loaded `path` string (the same depth the LtreeModel.level property computes) + so the field never raises AttributeError. """ @strawberry_django.field(annotate={ 'ltree_level': ExpressionWrapper( @@ -104,7 +106,11 @@ class LtreeNodeMixin: ) }) def level(self) -> int: - return self.ltree_level + ltree_level = getattr(self, 'ltree_level', None) + if ltree_level is not None: + return ltree_level + path = getattr(self, 'path', '') or '' + return str(path).count('.') class NestedGroupObjectType( diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index b9d47e82382..e5bafb221e4 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -12,7 +12,7 @@ inserts and parent_id changes via refresh_from_db(fields=['path']). """ from django.core.exceptions import FieldDoesNotExist, ValidationError -from django.db import IntegrityError, connection, migrations, models +from django.db import IntegrityError, OperationalError, connection, migrations, models from django.db.models import ForeignKey, Lookup, ManyToManyField, Q from django.db.models.expressions import RawSQL from django.utils.translation import gettext_lazy as _ @@ -147,129 +147,68 @@ def add_related_count(self, queryset, model, rel_field, count_attr, cumulative=F (using the ltree `<@` operator against the parent's `path`). Handles ForeignKey, ManyToManyField, and the NetBox GenericForeignKey "scope" pattern (scope_type / scope_id). + + The six historical variants (3 relation kinds × cumulative/not) are + assembled from two fragments: how a related row links to a tree node + (`link_expr` + any join/scope filter), and which node is counted — the + parent row itself (non-cumulative) or any node in its subtree via `<@` + (cumulative). """ - has_direct_fk = False - is_many_to_many = False try: field = model._meta.get_field(rel_field) except Exception: field = None - if isinstance(field, ManyToManyField): - is_many_to_many = True - elif isinstance(field, ForeignKey): - has_direct_fk = True - + is_many_to_many = isinstance(field, ManyToManyField) + has_direct_fk = isinstance(field, ForeignKey) has_generic_fk = ( hasattr(model, 'scope_type') and hasattr(model, 'scope_id') and not has_direct_fk and not is_many_to_many ) - # Many call sites bind add_related_count() as a class attribute, so it - # runs at module import. Raising FieldDoesNotExist here would block app - # startup whenever an unrelated field is renamed; fall back to the - # Django default column name (`{rel_field}_id`) instead, matching MPTT's - # add_related_count behavior. - rel_field_col_default = f'{rel_field}_id' - qn = connection.ops.quote_name parent_table = qn(queryset.model._meta.db_table) related_table = qn(model._meta.db_table) - if cumulative: - if is_many_to_many: - field = model._meta.get_field(rel_field) - m2m_table = qn(field.remote_field.through._meta.db_table) - # `model` is the declaring side (the side holding the items to count); - # `queryset.model` is the related (tree) side. m2m_column_name() points - # at the declaring model; m2m_reverse_name() points at the related model. - m2m_to_child_col = qn(field.m2m_column_name()) - m2m_to_tree_col = qn(field.m2m_reverse_name()) - sql = f'''( - SELECT COUNT(DISTINCT {related_table}."id") - FROM {related_table} - INNER JOIN {m2m_table} - ON {related_table}."id" = {m2m_table}.{m2m_to_child_col} - INNER JOIN {parent_table} AS subtree - ON {m2m_table}.{m2m_to_tree_col} = subtree."id" - WHERE subtree."path" <@ {parent_table}."path" - )''' - return queryset.annotate(**{ - count_attr: RawSQL(sql, [], output_field=models.IntegerField()) - }) - if has_generic_fk: - # Resolve scope_type_id via subquery so this annotation can be - # constructed at import time (e.g. in a view class body) even - # before contenttypes has been migrated. - ct_app = queryset.model._meta.app_label - ct_model = queryset.model._meta.model_name - sql = f'''( - SELECT COUNT(DISTINCT {related_table}."id") - FROM {related_table} - INNER JOIN {parent_table} AS subtree - ON {related_table}."scope_id" = subtree."id" - WHERE {related_table}."scope_type_id" = ( - SELECT id FROM django_content_type - WHERE app_label = %s AND model = %s - ) - AND subtree."path" <@ {parent_table}."path" - )''' - return queryset.annotate(**{ - count_attr: RawSQL(sql, [ct_app, ct_model], output_field=models.IntegerField()) - }) - # Use field.column (not f'{rel_field}_id') so custom db_column works; - # fall back to default naming if the field was not resolved. - rel_field_col = qn(field.column if field is not None else rel_field_col_default) - sql = f'''( - SELECT COUNT(DISTINCT {related_table}."id") - FROM {related_table} - INNER JOIN {parent_table} AS subtree - ON {related_table}.{rel_field_col} = subtree."id" - WHERE subtree."path" <@ {parent_table}."path" - )''' - return queryset.annotate(**{ - count_attr: RawSQL(sql, [], output_field=models.IntegerField()) - }) - - # Non-cumulative: count only rows pointing directly at each node. Mirrors the - # cumulative branches but joins on equality instead of the `<@` subtree test. + # `from_join` is the FROM (+ m2m join); `link_expr` is the column that points + # at a tree node's id; `scope_filter` constrains the generic-FK content type. + params = [] + from_join = f'FROM {related_table}' + scope_filter = '' if is_many_to_many: - field = model._meta.get_field(rel_field) + # m2m_column_name() points at the declaring model (`model`); + # m2m_reverse_name() points at the related (tree) model. m2m_table = qn(field.remote_field.through._meta.db_table) - m2m_to_child_col = qn(field.m2m_column_name()) - m2m_to_tree_col = qn(field.m2m_reverse_name()) - sql = f'''( - SELECT COUNT(DISTINCT {related_table}."id") - FROM {related_table} - INNER JOIN {m2m_table} - ON {related_table}."id" = {m2m_table}.{m2m_to_child_col} - WHERE {m2m_table}.{m2m_to_tree_col} = {parent_table}."id" - )''' - return queryset.annotate(**{ - count_attr: RawSQL(sql, [], output_field=models.IntegerField()) - }) - if has_generic_fk: - ct_app = queryset.model._meta.app_label - ct_model = queryset.model._meta.model_name - sql = f'''( - SELECT COUNT(DISTINCT {related_table}."id") - FROM {related_table} - WHERE {related_table}."scope_id" = {parent_table}."id" - AND {related_table}."scope_type_id" = ( - SELECT id FROM django_content_type - WHERE app_label = %s AND model = %s - ) - )''' - return queryset.annotate(**{ - count_attr: RawSQL(sql, [ct_app, ct_model], output_field=models.IntegerField()) - }) - rel_field_col = qn(field.column if field is not None else rel_field_col_default) - sql = f'''( - SELECT COUNT(DISTINCT {related_table}."id") - FROM {related_table} - WHERE {related_table}.{rel_field_col} = {parent_table}."id" - )''' + from_join += ( + f' INNER JOIN {m2m_table}' + f' ON {related_table}."id" = {m2m_table}.{qn(field.m2m_column_name())}' + ) + link_expr = f'{m2m_table}.{qn(field.m2m_reverse_name())}' + elif has_generic_fk: + link_expr = f'{related_table}."scope_id"' + # Resolve scope_type_id via subquery so the annotation can be built at + # import time (e.g. in a view class body) before contenttypes migrate. + scope_filter = ( + f' AND {related_table}."scope_type_id" = (' + 'SELECT id FROM django_content_type WHERE app_label = %s AND model = %s)' + ) + params = [queryset.model._meta.app_label, queryset.model._meta.model_name] + else: + # field.column honors a custom db_column; fall back to Django's default + # `{rel_field}_id` if the field was not resolved (a renamed unrelated + # field must not break import-time annotation construction). + rel_field_col = qn(field.column if field is not None else f'{rel_field}_id') + link_expr = f'{related_table}.{rel_field_col}' + + if cumulative: + node_join = f' INNER JOIN {parent_table} AS subtree ON {link_expr} = subtree."id"' + where = f'WHERE subtree."path" <@ {parent_table}."path"{scope_filter}' + else: + node_join = '' + where = f'WHERE {link_expr} = {parent_table}."id"{scope_filter}' + + sql = f'(SELECT COUNT(DISTINCT {related_table}."id") {from_join}{node_join} {where})' return queryset.annotate(**{ - count_attr: RawSQL(sql, [], output_field=models.IntegerField()) + count_attr: RawSQL(sql, params, output_field=models.IntegerField()) }) @@ -410,6 +349,15 @@ def _parent_creates_cycle(self): pk=self.parent_id, path__descendant_or_equal=self.path ).exists() + @classmethod + def _has_sort_path(cls): + """ + Whether this model carries the optional trigger-maintained `sort_path` + column (the MPTT `order_insertion_by=('name',)` equivalent). Single source + of truth for clean(), save(), and _tree_order_field(). + """ + return any(f.attname == 'sort_path' for f in cls._meta.concrete_fields) + def clean(self): """ Reject assigning self or a descendant as parent, surfacing it as a field @@ -430,8 +378,7 @@ def clean(self): "parent": _("Cannot assign self or a descendant as parent.") }) - has_sort_path = any(f.attname == 'sort_path' for f in self._meta.concrete_fields) - if has_sort_path and '\t' in (getattr(self, 'name', None) or ''): + if self._has_sort_path() and '\t' in (getattr(self, 'name', None) or ''): raise ValidationError({ "name": _("Name cannot contain tab characters.") }) @@ -459,7 +406,7 @@ def save(self, *args, **kwargs): # The sort_path trigger also fires on a name change; detect that so the # cascaded sort_path can be refreshed below (path-only models have no # sort_path and are unaffected by renames). - has_sort_path = any(f.attname == 'sort_path' for f in self._meta.concrete_fields) + has_sort_path = self._has_sort_path() name_written = update_fields is None or 'name' in update_fields name_changed = ( (not is_insert) and has_sort_path and name_written @@ -476,17 +423,38 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) except IntegrityError as exc: # A concurrent reparent that races the Python-level _parent_creates_cycle - # check can be caught by the BEFORE trigger, which RAISEs the cycle with - # ERRCODE = check_violation (SQLSTATE 23514). Match on the SQLSTATE rather - # than the message text (which is not stable across wording/locales), and - # surface it as a ValidationError so the API/UI returns 400 instead of the - # IntegrityError → 500 the trigger would otherwise produce. None of the - # ltree-backed models declare other CHECK constraints, so 23514 here is - # unambiguously the cycle guard. - if getattr(exc.__cause__, 'sqlstate', None) == '23514': + # check is caught by the BEFORE trigger, which RAISEs 'cycle detected ...' + # with ERRCODE = check_violation (SQLSTATE 23514). Gate on the SQLSTATE + # (the primary, stable signal) AND the message marker, so an unrelated + # CHECK constraint on a subclass (also 23514) is not misreported as a + # cycle. Surface it as a ValidationError so the API/UI returns 400 instead + # of the IntegrityError → 500 the trigger would otherwise produce. + if ( + getattr(exc.__cause__, 'sqlstate', None) == '23514' + and 'cycle detected' in str(exc) + ): raise ValidationError( _("Cannot assign self or a descendant as parent.") ) from None + # The BEFORE trigger likewise rejects a tab in the name (it would corrupt + # sort_path); translate it for direct save() calls that skip clean(). + if ( + getattr(exc.__cause__, 'sqlstate', None) == '23514' + and 'tab character' in str(exc) + ): + raise ValidationError({ + "name": _("Name cannot contain tab characters.") + }) from None + raise + except OperationalError as exc: + # The per-tree advisory locks can deadlock on crossing reparents or two + # concurrent ancestor/descendant moves; PostgreSQL aborts one with + # SQLSTATE 40P01. Surface a clear, retryable message instead of the + # opaque 500 the bare OperationalError would produce. + if getattr(exc.__cause__, 'sqlstate', None) == '40P01': + raise ValidationError( + _("The hierarchy was modified concurrently; please retry.") + ) from None raise if (parent_changed or name_changed) and not is_insert: @@ -495,10 +463,7 @@ def save(self, *args, **kwargs): # the value the triggers wrote, not a stale one). This costs one extra # SELECT per reparent/rename; INSERT ... RETURNING covers the insert case, # so only updates reach here. - refresh_fields = [ - fname for fname in ('path', 'sort_path') - if any(f.attname == fname for f in self._meta.concrete_fields) - ] + refresh_fields = ['path'] + (['sort_path'] if has_sort_path else []) self.refresh_from_db(fields=refresh_fields) if is_insert or parent_written: @@ -566,9 +531,7 @@ def _tree_order_field(cls): order siblings by name to match the prior MPTT behavior; models without it fall back to `path` (PK-padded, insertion order). """ - if any(f.attname == 'sort_path' for f in cls._meta.concrete_fields): - return 'sort_path' - return 'path' + return 'sort_path' if cls._has_sort_path() else 'path' def get_ancestors(self, ascending=False, include_self=False): if not self.path: @@ -653,7 +616,7 @@ def rebuild_sort_paths(cls, name_column='name'): """ from django.db import connection - if not any(f.name == 'sort_path' for f in cls._meta.get_fields()): + if not cls._has_sort_path(): raise NotImplementedError( f"{cls.__name__} does not have a sort_path column" ) @@ -689,10 +652,16 @@ def rebuild_sort_paths(cls, name_column='name'): # different trees use different keys and run in parallel. _LOCK_TREE_ROOTS_SQL = ''' -- Destination tree root: the parent's root label, or this row's own label - -- when it is (or becomes) a root. + -- when it is (or becomes) a root. The CASE guards against a parent whose path + -- is the empty ltree '' (reachable only via a trigger-bypassing raw write): + -- subltree('', 0, 1) would raise 'invalid positions', so fall back to the + -- child's own label as the lock key rather than aborting the insert/move. IF NEW.parent_id IS NOT NULL THEN - EXECUTE format('SELECT subltree(path, 0, 1)::text FROM %%I WHERE id = $1', TG_TABLE_NAME) - INTO dest_root USING NEW.parent_id; + EXECUTE format( + 'SELECT CASE WHEN nlevel(path) > 0 THEN subltree(path, 0, 1)::text END' + ' FROM %%I WHERE id = $1', + TG_TABLE_NAME + ) INTO dest_root USING NEW.parent_id; END IF; dest_root := COALESCE(dest_root, lpad(NEW.id::text, 19, '0')); -- Source tree root (moves only): this row's current root, which the AFTER @@ -747,9 +716,12 @@ def rebuild_sort_paths(cls, name_column='name'): _CASCADE_PATH_ONLY_FN = ''' CREATE OR REPLACE FUNCTION "{table}_ltree_cascade_path_fn"() RETURNS TRIGGER AS $$ BEGIN + -- `nlevel($2) > 0` guards against an empty OLD.path ('', reachable only via a + -- trigger-bypassing raw write): `path <@ ''` is true for EVERY row, so without + -- this the cascade would rewrite the entire table on one reparent. EXECUTE format( 'UPDATE %%I SET path = $1 || subpath(path, nlevel($2))' - ' WHERE path <@ $2 AND id != $3', + ' WHERE nlevel($2) > 0 AND path <@ $2 AND id != $3', TG_TABLE_NAME ) USING NEW.path, OLD.path, NEW.id; RETURN NULL; @@ -784,6 +756,14 @@ def rebuild_sort_paths(cls, name_column='name'): key_old bigint; BEGIN ''' + _LOCK_TREE_ROOTS_SQL + ''' + -- sort_path joins ancestor names with chr(9) (TAB); a literal tab in a name + -- would inject a spurious separator and corrupt sibling ordering for the node + -- and its descendants. LtreeModel.clean() rejects this for forms/serializers; + -- this is the backstop for bulk_create / scripts / raw writes that bypass clean(). + IF position(chr(9) in COALESCE(NEW."{name_col}", '')) > 0 THEN + RAISE EXCEPTION 'name contains a tab character, which is not allowed' + USING ERRCODE = 'check_violation'; + END IF; IF NEW.parent_id IS NOT NULL THEN EXECUTE format('SELECT path, sort_path FROM %%I WHERE id = $1', TG_TABLE_NAME) INTO parent_path, parent_sort_path USING NEW.parent_id; @@ -811,11 +791,14 @@ def rebuild_sort_paths(cls, name_column='name'): -- COALESCE guards against a NULL sort_path slipping in via a raw write that -- bypassed the BEFORE trigger: without it, length(NULL)/substring(... FROM NULL) -- would cascade NULL to every descendant's sort_path in one shot. + -- `nlevel($2) > 0` guards against an empty OLD.path ('', reachable only via a + -- trigger-bypassing raw write): `path <@ ''` is true for EVERY row, so without + -- this the cascade would rewrite the entire table on one reparent. EXECUTE format( 'UPDATE %%I SET ' ' path = $1 || subpath(path, nlevel($2)), ' ' sort_path = COALESCE($4, '''') || substring(COALESCE(sort_path, '''') FROM length(COALESCE($5, '''')) + 1) ' - 'WHERE path <@ $2 AND id != $3', + 'WHERE nlevel($2) > 0 AND path <@ $2 AND id != $3', TG_TABLE_NAME ) USING NEW.path, OLD.path, NEW.id, NEW.sort_path, OLD.sort_path; RETURN NULL; From a82c8dca039371c902586faf83be92a98100f7fa Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Jun 2026 16:54:44 -0700 Subject: [PATCH 26/42] merge feature --- .../migrations/{0237_ltree_paths.py => 0238_ltree_paths.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename netbox/dcim/migrations/{0237_ltree_paths.py => 0238_ltree_paths.py} (99%) diff --git a/netbox/dcim/migrations/0237_ltree_paths.py b/netbox/dcim/migrations/0238_ltree_paths.py similarity index 99% rename from netbox/dcim/migrations/0237_ltree_paths.py rename to netbox/dcim/migrations/0238_ltree_paths.py index e982f3c3951..e88d790d79f 100644 --- a/netbox/dcim/migrations/0237_ltree_paths.py +++ b/netbox/dcim/migrations/0238_ltree_paths.py @@ -140,7 +140,7 @@ def _assert_paths_populated_sql(): class Migration(migrations.Migration): dependencies = [ - ('dcim', '0236_moduletype_component_counts'), + ('dcim', '0237_module_remove_local_context_data'), ] operations = [ From 1757d9696999fff075e8e82ecc767121e3d80fff Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Jun 2026 17:43:28 -0700 Subject: [PATCH 27/42] fix ordering on module bays --- netbox/dcim/migrations/0238_ltree_paths.py | 2 +- netbox/dcim/models/device_components.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/netbox/dcim/migrations/0238_ltree_paths.py b/netbox/dcim/migrations/0238_ltree_paths.py index e88d790d79f..f38a46692c3 100644 --- a/netbox/dcim/migrations/0238_ltree_paths.py +++ b/netbox/dcim/migrations/0238_ltree_paths.py @@ -245,7 +245,7 @@ class Migration(migrations.Migration): name='location', options={'ordering': ('site', 'sort_path')}, ), migrations.AlterModelOptions( - name='modulebay', options={'ordering': ('device', 'sort_path')}, + name='modulebay', options={'ordering': ('sort_path', 'pk')}, ), migrations.AlterModelOptions( name='platform', options={'ordering': ('sort_path',)}, diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index d0f3768e6d5..9b5428cb64d 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -1350,7 +1350,13 @@ class ModuleBay(ModularComponentModel, TrackingModelMixin, LtreeModel): objects = LtreeManager() class Meta(ModularComponentModel.Meta): - ordering = ('device', 'sort_path') + # Order by sort_path alone (not device-first), reproducing the MPTT + # ModuleBayManager's ('_root_name', 'lft'): sort_path begins with the tree's + # root-bay name (natural_sort collation), so the global list groups by + # root-bay name across devices, descendants following their root. `pk` + # gives a deterministic tie-break among same-named roots on different devices + # (MPTT's lft=1 left this order arbitrary). + ordering = ('sort_path', 'pk') indexes = ( GistIndex(fields=['path'], name='dcim_modulebay_path_gist'), models.Index(fields=['sort_path'], name='dcim_modulebay_sort_path_idx'), From d1461b04fe560c874c32200016ffa0ec8ac6299f Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 4 Jun 2026 07:53:27 -0700 Subject: [PATCH 28/42] update ordering --- netbox/dcim/migrations/0238_ltree_paths.py | 2 +- netbox/dcim/models/sites.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/netbox/dcim/migrations/0238_ltree_paths.py b/netbox/dcim/migrations/0238_ltree_paths.py index f38a46692c3..8e67552485c 100644 --- a/netbox/dcim/migrations/0238_ltree_paths.py +++ b/netbox/dcim/migrations/0238_ltree_paths.py @@ -242,7 +242,7 @@ class Migration(migrations.Migration): name='devicerole', options={'ordering': ('sort_path',)}, ), migrations.AlterModelOptions( - name='location', options={'ordering': ('site', 'sort_path')}, + name='location', options={'ordering': ('sort_path', 'pk')}, ), migrations.AlterModelOptions( name='modulebay', options={'ordering': ('sort_path', 'pk')}, diff --git a/netbox/dcim/models/sites.py b/netbox/dcim/models/sites.py index 233829e942e..b8f8f297fa7 100644 --- a/netbox/dcim/models/sites.py +++ b/netbox/dcim/models/sites.py @@ -328,7 +328,11 @@ class Location(ContactsMixin, ImageAttachmentsMixin, NestedLtreeGroupModel): ) class Meta: - ordering = ('site', 'sort_path') + # Alphabetical tree-flatten in the global list: sort_path begins with the + # root location's name, so ORDER BY sort_path lists root locations by name + # across sites (descendants following their root). `pk` is a deterministic + # tie-break for same-named roots in different sites. + ordering = ('sort_path', 'pk') indexes = ( GistIndex(fields=['path'], name='dcim_location_path_gist'), models.Index(fields=['sort_path'], name='dcim_location_sort_path_idx'), From 38c27d9a00d19f2af2989d95744c4567d4e521ae Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 4 Jun 2026 11:21:31 -0700 Subject: [PATCH 29/42] fix inventory item ordering --- netbox/dcim/migrations/0238_ltree_paths.py | 5 +++++ netbox/dcim/models/device_components.py | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/netbox/dcim/migrations/0238_ltree_paths.py b/netbox/dcim/migrations/0238_ltree_paths.py index 8e67552485c..d55dbaa57d1 100644 --- a/netbox/dcim/migrations/0238_ltree_paths.py +++ b/netbox/dcim/migrations/0238_ltree_paths.py @@ -241,6 +241,11 @@ class Migration(migrations.Migration): migrations.AlterModelOptions( name='devicerole', options={'ordering': ('sort_path',)}, ), + # InventoryItem has no sort_path; its global list is flat + alphabetical + # (the per-device tab renders the hierarchy via get_children().order_by('path')). + migrations.AlterModelOptions( + name='inventoryitem', options={'ordering': ('name', 'pk')}, + ), migrations.AlterModelOptions( name='location', options={'ordering': ('sort_path', 'pk')}, ), diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index 9b5428cb64d..fcd462f8c3e 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -1566,7 +1566,11 @@ class InventoryItem(LtreeModel, ComponentModel, TrackingModelMixin): objects = LtreeManager() class Meta: - ordering = ('device__id', 'parent__id', 'name') + # Global list is flat + alphabetical by name (natural_sort collation). The + # per-device Inventory tab renders the hierarchy instead — DeviceInventoryView + # .get_children() orders that by `path`. `pk` is a deterministic tie-break for + # same-named items on different devices. + ordering = ('name', 'pk') indexes = ( models.Index(fields=('component_type', 'component_id')), GistIndex(fields=['path'], name='dcim_inventoryitem_path_gist'), From 36ffd8dee6089fa7de808c26762b38d8d2b0ddf7 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 5 Jun 2026 09:56:10 -0700 Subject: [PATCH 30/42] cleanup --- netbox/core/models/change_logging.py | 6 ++++++ netbox/netbox/models/ltree.py | 15 ++++++++------- netbox/utilities/query.py | 15 +++++++-------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/netbox/core/models/change_logging.py b/netbox/core/models/change_logging.py index 7186f21c01c..5a9c474fe50 100644 --- a/netbox/core/models/change_logging.py +++ b/netbox/core/models/change_logging.py @@ -159,6 +159,12 @@ def diff_exclude_fields(self): model = self.changed_object_type.model_class() attrs = set() + # model_class() returns None when the model's app is no longer installed + # (e.g. a removed plugin); there are no fields to exclude, and the + # issubclass() checks below would raise TypeError on None. + if model is None: + return attrs + # Exclude auto-populated change tracking fields if issubclass(model, ChangeLoggingMixin): attrs.update({'created', 'last_updated'}) diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index e5bafb221e4..79d1d6c4602 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -255,13 +255,14 @@ class LtreeModel(models.Model, metaclass=LtreeModelBase): InstallLtreeTriggers; do not write to it from Python. Bulk creates: - The BEFORE INSERT trigger resolves a row's parent by SELECTing - `path` from the same table by parent_id. In a multi-row INSERT - (e.g. `bulk_create`) PostgreSQL fires the BEFORE trigger per row in - list order, so any row whose parent is also in the same batch - must appear after its parent. A child placed before its parent in - the batch will be persisted with a root-level path (the parent - row is not yet visible to the lookup). + The BEFORE INSERT trigger resolves a row's parent by SELECTing `path` + from the same table by parent_id. LtreeQuerySet.bulk_create() rejects any + row whose parent is an *unsaved* instance, so in normal use — parents + saved before their children are bulk-created — batch order does not matter + (each parent row already exists for the lookup). The lone exception is + manually pre-assigned PKs: if a child references a same-batch parent by a + hand-set pk, that parent must appear earlier in the batch (the BEFORE + trigger fires per row in list order), or the child gets a root-level path. Sort-path on rename: For subclasses with the optional `sort_path` column (see diff --git a/netbox/utilities/query.py b/netbox/utilities/query.py index f9852b36106..12623576a0e 100644 --- a/netbox/utilities/query.py +++ b/netbox/utilities/query.py @@ -1,8 +1,6 @@ from django.db.models import Count, OuterRef, QuerySet, Subquery from django.db.models.functions import Coalesce -from netbox.models.ltree import LtreeManager - __all__ = ( 'count_related', 'dict_to_filter_params', @@ -64,12 +62,13 @@ def reapply_model_ordering(queryset: QuerySet) -> QuerySet: Reapply model-level ordering in case it has been lost through .annotate(). https://code.djangoproject.com/ticket/32811 """ - # Hierarchical (ltree) models are exempt; their default ordering by `sort_path`/`path` - # must not be clobbered by .annotate(). Use caution when annotating these querysets. - # Use `managers` (not `local_managers`): the LtreeManager is declared on the abstract - # NestedLtreeGroupModel base, so concrete subclasses inherit it via the MRO rather than - # holding it in their own `local_managers`. - if any(isinstance(manager, LtreeManager) for manager in queryset.model._meta.managers): + # Models ordered by a trigger-maintained ltree column (`sort_path`/`path`) are + # exempt. Key the check on the ordering itself, NOT on LtreeManager presence: + # InventoryItem/InventoryItemTemplate use an LtreeManager only for path + # maintenance but order by a regular column (name), so they DO need their + # ordering reapplied after .annotate() strips it (Django #32811). + ordering = queryset.model._meta.ordering or () + if any(isinstance(f, str) and f.lstrip('-') in ('sort_path', 'path') for f in ordering): return queryset if queryset.ordered: return queryset From 79ea4e07a0851877db844afa22df75006ec4768c Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 8 Jun 2026 10:38:44 -0700 Subject: [PATCH 31/42] review changes and refactor --- base_requirements.txt | 8 +- netbox/dcim/migrations/0238_ltree_paths.py | 2 +- .../dcim/models/device_component_templates.py | 2 - netbox/dcim/models/device_components.py | 2 +- netbox/dcim/models/sites.py | 11 +- netbox/dcim/tests/test_models.py | 29 +++-- netbox/netbox/models/ltree.py | 101 ++---------------- netbox/utilities/tests/test_ltree.py | 36 ------- 8 files changed, 34 insertions(+), 157 deletions(-) diff --git a/base_requirements.txt b/base_requirements.txt index b51e79ff233..da7695f5ad7 100644 --- a/base_requirements.txt +++ b/base_requirements.txt @@ -26,9 +26,11 @@ django-graphiql-debug-toolbar # https://django-htmx.readthedocs.io/en/latest/changelog.html django-htmx -# Modified Preorder Tree Traversal (required only for historical migrations -# that pre-date the switch to PostgreSQL ltree; runtime code uses -# netbox.models.ltree.LtreeModel instead). +# Modified Preorder Tree Traversal (recursive nesting of objects) +# Retained primarily for plugin backward compatibility: the deprecated +# NestedGroupModel base remains MPTT-backed for plugins still using it. Also +# required by historical migrations that pre-date the switch to PostgreSQL ltree. +# NetBox core runtime uses netbox.models.ltree.LtreeModel instead. django-mptt # Context managers for PostgreSQL advisory locks diff --git a/netbox/dcim/migrations/0238_ltree_paths.py b/netbox/dcim/migrations/0238_ltree_paths.py index d55dbaa57d1..2efdb39bfcd 100644 --- a/netbox/dcim/migrations/0238_ltree_paths.py +++ b/netbox/dcim/migrations/0238_ltree_paths.py @@ -247,7 +247,7 @@ class Migration(migrations.Migration): name='inventoryitem', options={'ordering': ('name', 'pk')}, ), migrations.AlterModelOptions( - name='location', options={'ordering': ('sort_path', 'pk')}, + name='location', options={'ordering': ('site', 'sort_path')}, ), migrations.AlterModelOptions( name='modulebay', options={'ordering': ('sort_path', 'pk')}, diff --git a/netbox/dcim/models/device_component_templates.py b/netbox/dcim/models/device_component_templates.py index a8bd766ab8b..728bd15a976 100644 --- a/netbox/dcim/models/device_component_templates.py +++ b/netbox/dcim/models/device_component_templates.py @@ -885,8 +885,6 @@ class Meta: verbose_name = _('inventory item template') verbose_name_plural = _('inventory item templates') - # Self/descendant-as-parent cycles are rejected by the inherited LtreeModel.clean(). - def instantiate(self, **kwargs): parent = InventoryItem.objects.get(name=self.parent.name, **kwargs) if self.parent else None if self.component: diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index fcd462f8c3e..ce2e376bf24 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -1585,7 +1585,7 @@ class Meta: verbose_name_plural = _('inventory items') def clean(self): - super().clean() # LtreeModel.clean() rejects self/descendant-as-parent cycles + super().clean() # Validation for moving InventoryItems if not self._state.adding: diff --git a/netbox/dcim/models/sites.py b/netbox/dcim/models/sites.py index b8f8f297fa7..16877d6da73 100644 --- a/netbox/dcim/models/sites.py +++ b/netbox/dcim/models/sites.py @@ -328,11 +328,12 @@ class Location(ContactsMixin, ImageAttachmentsMixin, NestedLtreeGroupModel): ) class Meta: - # Alphabetical tree-flatten in the global list: sort_path begins with the - # root location's name, so ORDER BY sort_path lists root locations by name - # across sites (descendants following their root). `pk` is a deterministic - # tie-break for same-named roots in different sites. - ordering = ('sort_path', 'pk') + # Group by site, then tree-flatten within each site. This mirrors the prior + # MPTT behavior (Meta.ordering = ['site', 'name']) while upgrading the + # within-site ordering to sort_path, so descendants follow their parent in + # name order. sort_path is unique within a site (root names are unique per + # site, child names unique per parent), so no further tie-break is needed. + ordering = ('site', 'sort_path') indexes = ( GistIndex(fields=['path'], name='dcim_location_path_gist'), models.Index(fields=['sort_path'], name='dcim_location_sort_path_idx'), diff --git a/netbox/dcim/tests/test_models.py b/netbox/dcim/tests/test_models.py index 535c332ad9d..aa7c43b15a1 100644 --- a/netbox/dcim/tests/test_models.py +++ b/netbox/dcim/tests/test_models.py @@ -1057,18 +1057,17 @@ def test_child_module_bay_ordering(self): self.assertEqual(names, ['Bay 1', 'Bay 1.1', 'Bay 1.2', 'Bay 1.3']) @tag('regression') # #22146 - def test_root_module_bay_rename_preserves_tree_ids(self): + def test_root_module_bay_rename_preserves_paths(self): """ - Renaming a root module bay must not renumber any other root tree's - tree_id. The renamed bay's own tree_id is also expected to remain - stable, but the load-bearing assertion is that the *other* bays are - not shifted. + Renaming a root module bay must not rewrite any tree's path. Renaming + touches only sort_path (the display-ordering column), so every bay's + path — including the renamed bay's own — must be unchanged afterward. """ device_type = DeviceType.objects.first() device_role = DeviceRole.objects.first() site = Site.objects.first() device = Device.objects.create( - name='Rename TreeID Device', + name='Rename Path Device', device_type=device_type, role=device_role, site=site, @@ -1076,8 +1075,8 @@ def test_root_module_bay_rename_preserves_tree_ids(self): for name in ('Bay 1', 'Bay 2', 'Bay 3', 'Bay 4'): ModuleBay.objects.create(device=device, name=name) - tree_ids_before = { - bay.name: bay.tree_id + paths_before = { + bay.pk: str(bay.path) for bay in ModuleBay.objects.filter(device=device) } @@ -1085,18 +1084,16 @@ def test_root_module_bay_rename_preserves_tree_ids(self): bay.name = 'Bay 99' bay.save() - tree_ids_after = { - bay.name: bay.tree_id + paths_after = { + bay.pk: str(bay.path) for bay in ModuleBay.objects.filter(device=device) } - for name in ('Bay 1', 'Bay 3', 'Bay 4'): - self.assertEqual(tree_ids_after[name], tree_ids_before[name]) - self.assertEqual(tree_ids_after['Bay 99'], tree_ids_before['Bay 2']) + self.assertEqual(paths_after, paths_before) @tag('regression') # #22146 def test_root_module_bay_rename_updates_display_order(self): """ - Even though renaming a root module bay does not renumber tree_ids, + Even though renaming a root module bay does not rewrite its path, the manager's _root_name annotation must reflect the new name so the display ordering is correct. """ @@ -1186,7 +1183,9 @@ def test_root_to_child_transition_still_relocates(self): movable_bay.refresh_from_db() host_bay.refresh_from_db() self.assertEqual(movable_bay.parent_id, host_bay.pk) - self.assertEqual(movable_bay.tree_id, host_bay.tree_id) + # The trigger cascade must have re-rooted the moved bay into host_bay's + # tree: its path is now a strict descendant of host_bay's path. + self.assertTrue(str(movable_bay.path).startswith(f'{host_bay.path}.')) def test_single_module_token(self): device_type = DeviceType.objects.first() diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index 79d1d6c4602..33040adc7f1 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -1,10 +1,11 @@ """ -Ltree-based hierarchical model support - drop-in replacement for django-mptt. +Ltree-based hierarchical model support - a replacement for django-mptt backed by +a PostgreSQL ltree column. -LtreeModel provides the same public API as django-mptt's MPTTModel (get_ancestors, -get_descendants, get_children, get_root, get_family, get_siblings, -get_descendant_count, get_level, level, is_root_node, is_leaf_node, is_child_node, -move_to, insert_at) backed by a PostgreSQL ltree column. +LtreeModel covers the subset of django-mptt's MPTTModel API that NetBox actually +uses. It is deliberately NOT a full reimplementation of MPTT's surface — methods +NetBox does not rely on (e.g. get_leafnodes(), get_next_sibling(), +get_previous_sibling()) are intentionally omitted. Paths are maintained entirely by PostgreSQL triggers installed via the InstallLtreeTriggers migration operation. The Python layer never computes or @@ -13,7 +14,7 @@ """ from django.core.exceptions import FieldDoesNotExist, ValidationError from django.db import IntegrityError, OperationalError, connection, migrations, models -from django.db.models import ForeignKey, Lookup, ManyToManyField, Q +from django.db.models import ForeignKey, Lookup, ManyToManyField from django.db.models.expressions import RawSQL from django.utils.translation import gettext_lazy as _ @@ -484,46 +485,6 @@ def level(self): def get_level(self): return self.level - def _root_pk(self): - """ - Integer PK of the root node, parsed from the first (zero-padded) path label. - Returns None for a node with no path. - """ - if not self.path: - return None - return int(str(self.path).split('.', 1)[0].lstrip('0') or '0') - - @property - def tree_id(self): - """Integer PK of the root, mirroring django-mptt's `tree_id`.""" - return self._root_pk() - - def is_root_node(self): - return self.parent_id is None - - def is_leaf_node(self): - if self.pk is None: - # Unsaved instance has no children. Without this guard, - # filter(parent_id=None) would match every existing root. - return True - return not type(self).objects.filter(parent_id=self.pk).exists() - - def is_child_node(self): - return self.parent_id is not None - - def get_root(self): - if self.is_root_node(): - return self - if not self.path: - # Unsaved (no path computed yet). Walk up the in-memory parent - # chain if available; otherwise we have no way to resolve the root. - parent = getattr(self, 'parent', None) - return parent.get_root() if parent is not None else None - return type(self)._default_manager.get(pk=self._root_pk()) - - def get_parent(self): - return self.parent - @classmethod def _tree_order_field(cls): """ @@ -551,57 +512,9 @@ def get_descendants(self, include_self=False): **{f'path__{lookup}': self.path} ).order_by(self._tree_order_field()) - def get_descendant_count(self): - if not self.path: - return 0 - return type(self)._default_manager.filter(path__descendant=self.path).count() - def get_children(self): return type(self)._default_manager.filter(parent_id=self.pk).order_by(self._tree_order_field()) - def get_family(self): - """Ancestors + self + descendants, in tree-order.""" - if not self.path: - return type(self)._default_manager.none() - # No .distinct() needed: this is a single-table OR with no joins, so a row - # (self, which matches both branches) is still returned only once. - return type(self)._default_manager.filter( - Q(path__ancestor=self.path) | Q(path__descendant_or_equal=self.path) - ).order_by(self._tree_order_field()) - - def get_siblings(self, include_self=False): - qs = type(self)._default_manager.filter(parent_id=self.parent_id) - if not include_self: - qs = qs.exclude(pk=self.pk) - return qs.order_by(self._tree_order_field()) - - def move_to(self, target, position='last-child'): - """ - Re-parent this node under `target`. Triggers handle path recomputation - for self and all descendants. `position` is accepted for django-mptt - compatibility; first-/last-child both mean "child of target" and - left/right mean "sibling of target". - """ - if position in ('first-child', 'last-child', None): - new_parent = target - elif position in ('left', 'right'): - new_parent = target.parent if target else None - else: - raise ValueError(f"Unsupported move_to position: {position!r}") - self.parent = new_parent - self.save() - - def insert_at(self, target, position='last-child', save=False): - """Set parent (optionally save). Mirrors django-mptt's insert_at.""" - if position in ('first-child', 'last-child', None): - self.parent = target - elif position in ('left', 'right'): - self.parent = target.parent if target else None - else: - raise ValueError(f"Unsupported insert_at position: {position!r}") - if save: - self.save() - @classmethod def rebuild_sort_paths(cls, name_column='name'): """ diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py index 4442febf179..5ec5a72c790 100644 --- a/netbox/utilities/tests/test_ltree.py +++ b/netbox/utilities/tests/test_ltree.py @@ -110,18 +110,6 @@ def test_level(self): self.assertEqual(self.leaf.level, 2) self.assertEqual(self.leaf.get_level(), 2) - def test_is_root_leaf_child(self): - self.assertTrue(self.root.is_root_node()) - self.assertFalse(self.root.is_leaf_node()) - self.assertFalse(self.root.is_child_node()) - self.assertFalse(self.leaf.is_root_node()) - self.assertTrue(self.leaf.is_leaf_node()) - self.assertTrue(self.leaf.is_child_node()) - - def test_get_root(self): - self.assertEqual(self.leaf.get_root(), self.root) - self.assertEqual(self.root.get_root(), self.root) - def test_get_ancestors(self): ancestors = list(self.leaf.get_ancestors().values_list('name', flat=True)) self.assertEqual(ancestors, ['Root', 'Mid']) @@ -131,27 +119,11 @@ def test_get_ancestors(self): def test_get_descendants(self): descendants = sorted(self.root.get_descendants().values_list('name', flat=True)) self.assertEqual(descendants, ['Leaf', 'Leaf2', 'Mid']) - self.assertEqual(self.root.get_descendant_count(), 3) def test_get_children(self): children = sorted(self.root.get_children().values_list('name', flat=True)) self.assertEqual(children, ['Mid']) - def test_get_siblings(self): - siblings = list(self.leaf.get_siblings().values_list('name', flat=True)) - self.assertEqual(siblings, ['Leaf2']) - - def test_get_family(self): - family = sorted(self.mid.get_family().values_list('name', flat=True)) - self.assertEqual(family, ['Leaf', 'Leaf2', 'Mid', 'Root']) - - def test_move_to(self): - new_root = Region.objects.create(name='New', slug='new-api') - self.leaf.move_to(new_root) - self.leaf.refresh_from_db() - self.assertEqual(self.leaf.parent, new_root) - self.assertEqual(self.leaf.path, _path(new_root.pk, self.leaf.pk)) - class CycleValidationTests(TestCase): """clean() and save() must refuse to assign self or a descendant as parent.""" @@ -253,14 +225,6 @@ def test_get_children_returns_in_name_order(self): names = list(root.get_children().values_list('name', flat=True)) self.assertEqual(names, ['Aardvark', 'Zebra']) - def test_get_siblings_returns_in_name_order(self): - root = Region.objects.create(name='Root', slug='root-gsord') - Region.objects.create(parent=root, name='Zebra', slug='z-gsord') - a = Region.objects.create(parent=root, name='Aardvark', slug='a-gsord') - Region.objects.create(parent=root, name='Buffalo', slug='b-gsord') - names = list(a.get_siblings().values_list('name', flat=True)) - self.assertEqual(names, ['Buffalo', 'Zebra']) - class AddRelatedCountTests(TestCase): """add_related_count must cumulate across subtrees via path <@.""" From 8e319a502af150b0fbfb3c2d92cd5fc4a5a782bf Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 8 Jun 2026 10:46:10 -0700 Subject: [PATCH 32/42] refactor --- netbox/dcim/migrations/0238_ltree_paths.py | 2 +- netbox/netbox/models/ltree.py | 266 +---------------- netbox/tenancy/migrations/0025_ltree_paths.py | 2 +- netbox/utilities/ltree.py | 276 ++++++++++++++++++ .../wireless/migrations/0020_ltree_paths.py | 2 +- 5 files changed, 280 insertions(+), 268 deletions(-) create mode 100644 netbox/utilities/ltree.py diff --git a/netbox/dcim/migrations/0238_ltree_paths.py b/netbox/dcim/migrations/0238_ltree_paths.py index 2efdb39bfcd..1c994d58711 100644 --- a/netbox/dcim/migrations/0238_ltree_paths.py +++ b/netbox/dcim/migrations/0238_ltree_paths.py @@ -38,7 +38,7 @@ from django.db import migrations, models import netbox.models.ltree -from netbox.models.ltree import InstallLtreeTriggers +from utilities.ltree import InstallLtreeTriggers # All models getting an ltree `path` column. ALL_MODELS = ( diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index 33040adc7f1..d17ccb18263 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -13,7 +13,7 @@ inserts and parent_id changes via refresh_from_db(fields=['path']). """ from django.core.exceptions import FieldDoesNotExist, ValidationError -from django.db import IntegrityError, OperationalError, connection, migrations, models +from django.db import IntegrityError, OperationalError, connection, models from django.db.models import ForeignKey, Lookup, ManyToManyField from django.db.models.expressions import RawSQL from django.utils.translation import gettext_lazy as _ @@ -21,7 +21,6 @@ from utilities.querysets import RestrictedQuerySet __all__ = ( - 'InstallLtreeTriggers', 'LtreeField', 'LtreeManager', 'LtreeModel', @@ -549,266 +548,3 @@ def rebuild_sort_paths(cls, name_column='name'): ''' with connection.cursor() as cursor: cursor.execute(sql) - - -# -# Migration operation -# - -# Path label is the row's PK zero-padded to 19 chars (max bigint width) so that -# lexicographic ordering of ltree labels matches numeric PK ordering across digit -# boundaries (e.g. "0...09" sorts before "0...10"). -# Per-tree advisory locking (see _lock_tree_roots_sql / LtreeModel "Concurrency"): -# every insert/move/reparent of a node takes a transaction-level advisory lock -# keyed on the root(s) of the tree(s) it touches, BEFORE reading the parent path. -# A concurrent reparent of an ancestor takes the same key, so the two serialize -# and the AFTER cascade can never miss a row inserted concurrently; writes in -# different trees use different keys and run in parallel. -_LOCK_TREE_ROOTS_SQL = ''' - -- Destination tree root: the parent's root label, or this row's own label - -- when it is (or becomes) a root. The CASE guards against a parent whose path - -- is the empty ltree '' (reachable only via a trigger-bypassing raw write): - -- subltree('', 0, 1) would raise 'invalid positions', so fall back to the - -- child's own label as the lock key rather than aborting the insert/move. - IF NEW.parent_id IS NOT NULL THEN - EXECUTE format( - 'SELECT CASE WHEN nlevel(path) > 0 THEN subltree(path, 0, 1)::text END' - ' FROM %%I WHERE id = $1', - TG_TABLE_NAME - ) INTO dest_root USING NEW.parent_id; - END IF; - dest_root := COALESCE(dest_root, lpad(NEW.id::text, 19, '0')); - -- Source tree root (moves only): this row's current root, which the AFTER - -- cascade will rewrite. - IF TG_OP = 'UPDATE' AND OLD.path IS NOT NULL AND nlevel(OLD.path) > 0 THEN - old_root := subltree(OLD.path, 0, 1)::text; - END IF; - key_dest := hashtextextended(TG_TABLE_NAME || ':' || dest_root, 0); - IF old_root IS NOT NULL AND old_root <> dest_root THEN - -- Cross-tree move: lock both roots, ascending, to avoid deadlock between - -- two concurrent moves that touch the same pair. - key_old := hashtextextended(TG_TABLE_NAME || ':' || old_root, 0); - PERFORM pg_advisory_xact_lock(LEAST(key_dest, key_old)); - PERFORM pg_advisory_xact_lock(GREATEST(key_dest, key_old)); - ELSE - PERFORM pg_advisory_xact_lock(key_dest); - END IF; -''' - -_COMPUTE_PATH_ONLY_FN = ''' -CREATE OR REPLACE FUNCTION "{table}_ltree_compute_path_fn"() RETURNS TRIGGER AS $$ -DECLARE - parent_path ltree; - dest_root text; - old_root text; - key_dest bigint; - key_old bigint; -BEGIN -''' + _LOCK_TREE_ROOTS_SQL + ''' - IF NEW.parent_id IS NOT NULL THEN - EXECUTE format('SELECT path FROM %%I WHERE id = $1', TG_TABLE_NAME) - INTO parent_path USING NEW.parent_id; - -- Cycle guard. The Python LtreeModel.save() also rejects cyclic moves, - -- but a QuerySet.update() / bulk_update() bypasses save() entirely, so - -- catch the case here as a last line of defense. A cycle exists iff - -- this row's own label appears anywhere in parent_path (the row would - -- become its own ancestor). Match the label as any segment via lquery. - IF parent_path ~ ('*.' || lpad(NEW.id::text, 19, '0') || '.*')::lquery - OR parent_path = lpad(NEW.id::text, 19, '0')::ltree THEN - RAISE EXCEPTION 'cycle detected: %% cannot be its own ancestor', TG_TABLE_NAME - USING ERRCODE = 'check_violation'; - END IF; - NEW.path := parent_path || lpad(NEW.id::text, 19, '0')::ltree; - ELSE - NEW.path := lpad(NEW.id::text, 19, '0')::ltree; - END IF; - RETURN NEW; -END -$$ LANGUAGE plpgsql; -''' - -_CASCADE_PATH_ONLY_FN = ''' -CREATE OR REPLACE FUNCTION "{table}_ltree_cascade_path_fn"() RETURNS TRIGGER AS $$ -BEGIN - -- `nlevel($2) > 0` guards against an empty OLD.path ('', reachable only via a - -- trigger-bypassing raw write): `path <@ ''` is true for EVERY row, so without - -- this the cascade would rewrite the entire table on one reparent. - EXECUTE format( - 'UPDATE %%I SET path = $1 || subpath(path, nlevel($2))' - ' WHERE nlevel($2) > 0 AND path <@ $2 AND id != $3', - TG_TABLE_NAME - ) USING NEW.path, OLD.path, NEW.id; - RETURN NULL; -END -$$ LANGUAGE plpgsql; -''' - -# For models with order_insertion_by=(name,) — maintain a second text column -# `sort_path` whose value is the chain of ancestor names joined by chr(9) (TAB). -# TAB sorts strictly below any printable character under both the default text -# collation and the ICU `natural_sort` collation (which is `und-u-kn-true`). -# ICU collations with default variable weighting treat U+0001..U+0008 as -# variable-ignorable, so a chr(1) separator under natural_sort would interleave -# children with unrelated roots; TAB is given a primary weight and orders -# deterministically. ORDER BY sort_path then gives MPTT-equivalent -# tree-flatten ordering with siblings in name (collation) order. -# -# The BEFORE trigger fires on INSERT, parent_id changes, and name changes, so -# a rename updates the row's own sort_path; the AFTER trigger then cascades -# the new sort_path into descendants. (django-mptt's `order_insertion_by` -# stops at the renamed node and leaves descendants stale until a manual -# rebuild — NetBox auto-cascades because operators expect renames to flow -# through. `rebuild_sort_paths()` is still available for bulk repair.) -_COMPUTE_PATH_AND_SORT_FN = ''' -CREATE OR REPLACE FUNCTION "{table}_ltree_compute_path_fn"() RETURNS TRIGGER AS $$ -DECLARE - parent_path ltree; - parent_sort_path text; - dest_root text; - old_root text; - key_dest bigint; - key_old bigint; -BEGIN -''' + _LOCK_TREE_ROOTS_SQL + ''' - -- sort_path joins ancestor names with chr(9) (TAB); a literal tab in a name - -- would inject a spurious separator and corrupt sibling ordering for the node - -- and its descendants. LtreeModel.clean() rejects this for forms/serializers; - -- this is the backstop for bulk_create / scripts / raw writes that bypass clean(). - IF position(chr(9) in COALESCE(NEW."{name_col}", '')) > 0 THEN - RAISE EXCEPTION 'name contains a tab character, which is not allowed' - USING ERRCODE = 'check_violation'; - END IF; - IF NEW.parent_id IS NOT NULL THEN - EXECUTE format('SELECT path, sort_path FROM %%I WHERE id = $1', TG_TABLE_NAME) - INTO parent_path, parent_sort_path USING NEW.parent_id; - -- Cycle guard. See _COMPUTE_PATH_ONLY_FN for the rationale; this catches - -- raw UPDATE / bulk_update paths that bypass LtreeModel.save(). - IF parent_path ~ ('*.' || lpad(NEW.id::text, 19, '0') || '.*')::lquery - OR parent_path = lpad(NEW.id::text, 19, '0')::ltree THEN - RAISE EXCEPTION 'cycle detected: %% cannot be its own ancestor', TG_TABLE_NAME - USING ERRCODE = 'check_violation'; - END IF; - NEW.path := parent_path || lpad(NEW.id::text, 19, '0')::ltree; - NEW.sort_path := parent_sort_path || chr(9) || NEW."{name_col}"; - ELSE - NEW.path := lpad(NEW.id::text, 19, '0')::ltree; - NEW.sort_path := NEW."{name_col}"; - END IF; - RETURN NEW; -END -$$ LANGUAGE plpgsql; -''' - -_CASCADE_PATH_AND_SORT_FN = """ -CREATE OR REPLACE FUNCTION "{table}_ltree_cascade_path_fn"() RETURNS TRIGGER AS $$ -BEGIN - -- COALESCE guards against a NULL sort_path slipping in via a raw write that - -- bypassed the BEFORE trigger: without it, length(NULL)/substring(... FROM NULL) - -- would cascade NULL to every descendant's sort_path in one shot. - -- `nlevel($2) > 0` guards against an empty OLD.path ('', reachable only via a - -- trigger-bypassing raw write): `path <@ ''` is true for EVERY row, so without - -- this the cascade would rewrite the entire table on one reparent. - EXECUTE format( - 'UPDATE %%I SET ' - ' path = $1 || subpath(path, nlevel($2)), ' - ' sort_path = COALESCE($4, '''') || substring(COALESCE(sort_path, '''') FROM length(COALESCE($5, '''')) + 1) ' - 'WHERE nlevel($2) > 0 AND path <@ $2 AND id != $3', - TG_TABLE_NAME - ) USING NEW.path, OLD.path, NEW.id, NEW.sort_path, OLD.sort_path; - RETURN NULL; -END -$$ LANGUAGE plpgsql; -""" - -_BEFORE_TRIGGER_PATH_ONLY = ''' -CREATE TRIGGER "{table}_ltree_compute_path" - BEFORE INSERT OR UPDATE OF parent_id ON "{table}" - FOR EACH ROW EXECUTE FUNCTION "{table}_ltree_compute_path_fn"(); -''' - -# For path+sort tables, also fire on UPDATE OF {name_col} so that renaming a -# node recomputes its sort_path. The cascade trigger then propagates the new -# sort_path to descendants. -_BEFORE_TRIGGER_PATH_AND_SORT = ''' -CREATE TRIGGER "{table}_ltree_compute_path" - BEFORE INSERT OR UPDATE OF parent_id, "{name_col}" ON "{table}" - FOR EACH ROW EXECUTE FUNCTION "{table}_ltree_compute_path_fn"(); -''' - -# AFTER trigger fires on the columns that operators / Django write directly -# (parent_id and the name column) — NOT on path or sort_path. The cascade -# function rewrites path/sort_path on descendants in a single statement, and -# because that statement does not touch parent_id or {name_col}, the AFTER -# trigger does not re-fire on those descendant rows. This prevents the -# quadratic re-cascade that would otherwise occur for any deep subtree. -_AFTER_TRIGGER_PATH_ONLY = ''' -CREATE TRIGGER "{table}_ltree_cascade_path" - AFTER UPDATE OF parent_id ON "{table}" - FOR EACH ROW WHEN (OLD.path IS DISTINCT FROM NEW.path) - EXECUTE FUNCTION "{table}_ltree_cascade_path_fn"(); -''' - -_AFTER_TRIGGER_PATH_AND_SORT = ''' -CREATE TRIGGER "{table}_ltree_cascade_path" - AFTER UPDATE OF parent_id, "{name_col}" ON "{table}" - FOR EACH ROW WHEN ( - OLD.path IS DISTINCT FROM NEW.path - OR OLD.sort_path IS DISTINCT FROM NEW.sort_path - ) - EXECUTE FUNCTION "{table}_ltree_cascade_path_fn"(); -''' - - -class InstallLtreeTriggers(migrations.operations.base.Operation): - """ - Install per-table ltree path-maintenance triggers. - - Two row-level triggers are installed on each target table: - - BEFORE INSERT OR UPDATE OF parent_id -> compute NEW.path (and sort_path if applicable) - AFTER UPDATE OF parent_id -> cascade path/sort_path change to descendants - - If `name_column` is provided, the model is expected to have a `sort_path` - text column whose value will be maintained as a chr(9)-separated chain of - ancestor names. This implements MPTT's `order_insertion_by=(name,)` - semantics: insert, reparent, and rename all honor the current value of - `name_column`, with renames cascaded into descendants' sort_paths. - """ - reversible = True - - def __init__(self, table_name, name_column=None): - self.table_name = table_name - self.name_column = name_column - - def state_forwards(self, app_label, state): - pass - - def database_forwards(self, app_label, schema_editor, from_state, to_state): - if self.name_column: - schema_editor.execute(_COMPUTE_PATH_AND_SORT_FN.format( - table=self.table_name, name_col=self.name_column, - )) - schema_editor.execute(_CASCADE_PATH_AND_SORT_FN.format( - table=self.table_name, - )) - schema_editor.execute(_BEFORE_TRIGGER_PATH_AND_SORT.format( - table=self.table_name, name_col=self.name_column, - )) - schema_editor.execute(_AFTER_TRIGGER_PATH_AND_SORT.format( - table=self.table_name, name_col=self.name_column, - )) - else: - schema_editor.execute(_COMPUTE_PATH_ONLY_FN.format(table=self.table_name)) - schema_editor.execute(_CASCADE_PATH_ONLY_FN.format(table=self.table_name)) - schema_editor.execute(_BEFORE_TRIGGER_PATH_ONLY.format(table=self.table_name)) - schema_editor.execute(_AFTER_TRIGGER_PATH_ONLY.format(table=self.table_name)) - - def database_backwards(self, app_label, schema_editor, from_state, to_state): - t = self.table_name - schema_editor.execute(f'DROP TRIGGER IF EXISTS "{t}_ltree_cascade_path" ON "{t}";') - schema_editor.execute(f'DROP TRIGGER IF EXISTS "{t}_ltree_compute_path" ON "{t}";') - schema_editor.execute(f'DROP FUNCTION IF EXISTS "{t}_ltree_cascade_path_fn"();') - schema_editor.execute(f'DROP FUNCTION IF EXISTS "{t}_ltree_compute_path_fn"();') - - def describe(self): - return f"Install ltree path triggers on {self.table_name}" diff --git a/netbox/tenancy/migrations/0025_ltree_paths.py b/netbox/tenancy/migrations/0025_ltree_paths.py index 15ebf3f417d..d9ef2c101ec 100644 --- a/netbox/tenancy/migrations/0025_ltree_paths.py +++ b/netbox/tenancy/migrations/0025_ltree_paths.py @@ -9,7 +9,7 @@ from django.db import migrations, models import netbox.models.ltree -from netbox.models.ltree import InstallLtreeTriggers +from utilities.ltree import InstallLtreeTriggers MODELS = ('tenantgroup', 'contactgroup') TABLES = ('tenancy_tenantgroup', 'tenancy_contactgroup') diff --git a/netbox/utilities/ltree.py b/netbox/utilities/ltree.py new file mode 100644 index 00000000000..64d16504e9d --- /dev/null +++ b/netbox/utilities/ltree.py @@ -0,0 +1,276 @@ +""" +Migration support for ltree-based hierarchical models. + +This module holds the schema-level machinery that backs `netbox.models.ltree`: +the PostgreSQL trigger function / trigger SQL and the `InstallLtreeTriggers` +migration operation that installs them. It is kept separate from the model layer +(`netbox.models.ltree`) so that migrations depend only on this DB-level code and +not on the model definitions. + +The paths maintained by these triggers are never computed or mutated from Python; +the model layer only reads `path`/`sort_path` back from the database. +""" +from django.db import migrations + +__all__ = ( + 'InstallLtreeTriggers', +) + + +# Path label is the row's PK zero-padded to 19 chars (max bigint width) so that +# lexicographic ordering of ltree labels matches numeric PK ordering across digit +# boundaries (e.g. "0...09" sorts before "0...10"). +# Per-tree advisory locking (see _lock_tree_roots_sql / LtreeModel "Concurrency"): +# every insert/move/reparent of a node takes a transaction-level advisory lock +# keyed on the root(s) of the tree(s) it touches, BEFORE reading the parent path. +# A concurrent reparent of an ancestor takes the same key, so the two serialize +# and the AFTER cascade can never miss a row inserted concurrently; writes in +# different trees use different keys and run in parallel. +_LOCK_TREE_ROOTS_SQL = ''' + -- Destination tree root: the parent's root label, or this row's own label + -- when it is (or becomes) a root. The CASE guards against a parent whose path + -- is the empty ltree '' (reachable only via a trigger-bypassing raw write): + -- subltree('', 0, 1) would raise 'invalid positions', so fall back to the + -- child's own label as the lock key rather than aborting the insert/move. + IF NEW.parent_id IS NOT NULL THEN + EXECUTE format( + 'SELECT CASE WHEN nlevel(path) > 0 THEN subltree(path, 0, 1)::text END' + ' FROM %%I WHERE id = $1', + TG_TABLE_NAME + ) INTO dest_root USING NEW.parent_id; + END IF; + dest_root := COALESCE(dest_root, lpad(NEW.id::text, 19, '0')); + -- Source tree root (moves only): this row's current root, which the AFTER + -- cascade will rewrite. + IF TG_OP = 'UPDATE' AND OLD.path IS NOT NULL AND nlevel(OLD.path) > 0 THEN + old_root := subltree(OLD.path, 0, 1)::text; + END IF; + key_dest := hashtextextended(TG_TABLE_NAME || ':' || dest_root, 0); + IF old_root IS NOT NULL AND old_root <> dest_root THEN + -- Cross-tree move: lock both roots, ascending, to avoid deadlock between + -- two concurrent moves that touch the same pair. + key_old := hashtextextended(TG_TABLE_NAME || ':' || old_root, 0); + PERFORM pg_advisory_xact_lock(LEAST(key_dest, key_old)); + PERFORM pg_advisory_xact_lock(GREATEST(key_dest, key_old)); + ELSE + PERFORM pg_advisory_xact_lock(key_dest); + END IF; +''' + +_COMPUTE_PATH_ONLY_FN = ''' +CREATE OR REPLACE FUNCTION "{table}_ltree_compute_path_fn"() RETURNS TRIGGER AS $$ +DECLARE + parent_path ltree; + dest_root text; + old_root text; + key_dest bigint; + key_old bigint; +BEGIN +''' + _LOCK_TREE_ROOTS_SQL + ''' + IF NEW.parent_id IS NOT NULL THEN + EXECUTE format('SELECT path FROM %%I WHERE id = $1', TG_TABLE_NAME) + INTO parent_path USING NEW.parent_id; + -- Cycle guard. The Python LtreeModel.save() also rejects cyclic moves, + -- but a QuerySet.update() / bulk_update() bypasses save() entirely, so + -- catch the case here as a last line of defense. A cycle exists iff + -- this row's own label appears anywhere in parent_path (the row would + -- become its own ancestor). Match the label as any segment via lquery. + IF parent_path ~ ('*.' || lpad(NEW.id::text, 19, '0') || '.*')::lquery + OR parent_path = lpad(NEW.id::text, 19, '0')::ltree THEN + RAISE EXCEPTION 'cycle detected: %% cannot be its own ancestor', TG_TABLE_NAME + USING ERRCODE = 'check_violation'; + END IF; + NEW.path := parent_path || lpad(NEW.id::text, 19, '0')::ltree; + ELSE + NEW.path := lpad(NEW.id::text, 19, '0')::ltree; + END IF; + RETURN NEW; +END +$$ LANGUAGE plpgsql; +''' + +_CASCADE_PATH_ONLY_FN = ''' +CREATE OR REPLACE FUNCTION "{table}_ltree_cascade_path_fn"() RETURNS TRIGGER AS $$ +BEGIN + -- `nlevel($2) > 0` guards against an empty OLD.path ('', reachable only via a + -- trigger-bypassing raw write): `path <@ ''` is true for EVERY row, so without + -- this the cascade would rewrite the entire table on one reparent. + EXECUTE format( + 'UPDATE %%I SET path = $1 || subpath(path, nlevel($2))' + ' WHERE nlevel($2) > 0 AND path <@ $2 AND id != $3', + TG_TABLE_NAME + ) USING NEW.path, OLD.path, NEW.id; + RETURN NULL; +END +$$ LANGUAGE plpgsql; +''' + +# For models with order_insertion_by=(name,) — maintain a second text column +# `sort_path` whose value is the chain of ancestor names joined by chr(9) (TAB). +# TAB sorts strictly below any printable character under both the default text +# collation and the ICU `natural_sort` collation (which is `und-u-kn-true`). +# ICU collations with default variable weighting treat U+0001..U+0008 as +# variable-ignorable, so a chr(1) separator under natural_sort would interleave +# children with unrelated roots; TAB is given a primary weight and orders +# deterministically. ORDER BY sort_path then gives MPTT-equivalent +# tree-flatten ordering with siblings in name (collation) order. +# +# The BEFORE trigger fires on INSERT, parent_id changes, and name changes, so +# a rename updates the row's own sort_path; the AFTER trigger then cascades +# the new sort_path into descendants. (django-mptt's `order_insertion_by` +# stops at the renamed node and leaves descendants stale until a manual +# rebuild — NetBox auto-cascades because operators expect renames to flow +# through. `rebuild_sort_paths()` is still available for bulk repair.) +_COMPUTE_PATH_AND_SORT_FN = ''' +CREATE OR REPLACE FUNCTION "{table}_ltree_compute_path_fn"() RETURNS TRIGGER AS $$ +DECLARE + parent_path ltree; + parent_sort_path text; + dest_root text; + old_root text; + key_dest bigint; + key_old bigint; +BEGIN +''' + _LOCK_TREE_ROOTS_SQL + ''' + -- sort_path joins ancestor names with chr(9) (TAB); a literal tab in a name + -- would inject a spurious separator and corrupt sibling ordering for the node + -- and its descendants. LtreeModel.clean() rejects this for forms/serializers; + -- this is the backstop for bulk_create / scripts / raw writes that bypass clean(). + IF position(chr(9) in COALESCE(NEW."{name_col}", '')) > 0 THEN + RAISE EXCEPTION 'name contains a tab character, which is not allowed' + USING ERRCODE = 'check_violation'; + END IF; + IF NEW.parent_id IS NOT NULL THEN + EXECUTE format('SELECT path, sort_path FROM %%I WHERE id = $1', TG_TABLE_NAME) + INTO parent_path, parent_sort_path USING NEW.parent_id; + -- Cycle guard. See _COMPUTE_PATH_ONLY_FN for the rationale; this catches + -- raw UPDATE / bulk_update paths that bypass LtreeModel.save(). + IF parent_path ~ ('*.' || lpad(NEW.id::text, 19, '0') || '.*')::lquery + OR parent_path = lpad(NEW.id::text, 19, '0')::ltree THEN + RAISE EXCEPTION 'cycle detected: %% cannot be its own ancestor', TG_TABLE_NAME + USING ERRCODE = 'check_violation'; + END IF; + NEW.path := parent_path || lpad(NEW.id::text, 19, '0')::ltree; + NEW.sort_path := parent_sort_path || chr(9) || NEW."{name_col}"; + ELSE + NEW.path := lpad(NEW.id::text, 19, '0')::ltree; + NEW.sort_path := NEW."{name_col}"; + END IF; + RETURN NEW; +END +$$ LANGUAGE plpgsql; +''' + +_CASCADE_PATH_AND_SORT_FN = """ +CREATE OR REPLACE FUNCTION "{table}_ltree_cascade_path_fn"() RETURNS TRIGGER AS $$ +BEGIN + -- COALESCE guards against a NULL sort_path slipping in via a raw write that + -- bypassed the BEFORE trigger: without it, length(NULL)/substring(... FROM NULL) + -- would cascade NULL to every descendant's sort_path in one shot. + -- `nlevel($2) > 0` guards against an empty OLD.path ('', reachable only via a + -- trigger-bypassing raw write): `path <@ ''` is true for EVERY row, so without + -- this the cascade would rewrite the entire table on one reparent. + EXECUTE format( + 'UPDATE %%I SET ' + ' path = $1 || subpath(path, nlevel($2)), ' + ' sort_path = COALESCE($4, '''') || substring(COALESCE(sort_path, '''') FROM length(COALESCE($5, '''')) + 1) ' + 'WHERE nlevel($2) > 0 AND path <@ $2 AND id != $3', + TG_TABLE_NAME + ) USING NEW.path, OLD.path, NEW.id, NEW.sort_path, OLD.sort_path; + RETURN NULL; +END +$$ LANGUAGE plpgsql; +""" + +_BEFORE_TRIGGER_PATH_ONLY = ''' +CREATE TRIGGER "{table}_ltree_compute_path" + BEFORE INSERT OR UPDATE OF parent_id ON "{table}" + FOR EACH ROW EXECUTE FUNCTION "{table}_ltree_compute_path_fn"(); +''' + +# For path+sort tables, also fire on UPDATE OF {name_col} so that renaming a +# node recomputes its sort_path. The cascade trigger then propagates the new +# sort_path to descendants. +_BEFORE_TRIGGER_PATH_AND_SORT = ''' +CREATE TRIGGER "{table}_ltree_compute_path" + BEFORE INSERT OR UPDATE OF parent_id, "{name_col}" ON "{table}" + FOR EACH ROW EXECUTE FUNCTION "{table}_ltree_compute_path_fn"(); +''' + +# AFTER trigger fires on the columns that operators / Django write directly +# (parent_id and the name column) — NOT on path or sort_path. The cascade +# function rewrites path/sort_path on descendants in a single statement, and +# because that statement does not touch parent_id or {name_col}, the AFTER +# trigger does not re-fire on those descendant rows. This prevents the +# quadratic re-cascade that would otherwise occur for any deep subtree. +_AFTER_TRIGGER_PATH_ONLY = ''' +CREATE TRIGGER "{table}_ltree_cascade_path" + AFTER UPDATE OF parent_id ON "{table}" + FOR EACH ROW WHEN (OLD.path IS DISTINCT FROM NEW.path) + EXECUTE FUNCTION "{table}_ltree_cascade_path_fn"(); +''' + +_AFTER_TRIGGER_PATH_AND_SORT = ''' +CREATE TRIGGER "{table}_ltree_cascade_path" + AFTER UPDATE OF parent_id, "{name_col}" ON "{table}" + FOR EACH ROW WHEN ( + OLD.path IS DISTINCT FROM NEW.path + OR OLD.sort_path IS DISTINCT FROM NEW.sort_path + ) + EXECUTE FUNCTION "{table}_ltree_cascade_path_fn"(); +''' + + +class InstallLtreeTriggers(migrations.operations.base.Operation): + """ + Install per-table ltree path-maintenance triggers. + + Two row-level triggers are installed on each target table: + + BEFORE INSERT OR UPDATE OF parent_id -> compute NEW.path (and sort_path if applicable) + AFTER UPDATE OF parent_id -> cascade path/sort_path change to descendants + + If `name_column` is provided, the model is expected to have a `sort_path` + text column whose value will be maintained as a chr(9)-separated chain of + ancestor names. This implements MPTT's `order_insertion_by=(name,)` + semantics: insert, reparent, and rename all honor the current value of + `name_column`, with renames cascaded into descendants' sort_paths. + """ + reversible = True + + def __init__(self, table_name, name_column=None): + self.table_name = table_name + self.name_column = name_column + + def state_forwards(self, app_label, state): + pass + + def database_forwards(self, app_label, schema_editor, from_state, to_state): + if self.name_column: + schema_editor.execute(_COMPUTE_PATH_AND_SORT_FN.format( + table=self.table_name, name_col=self.name_column, + )) + schema_editor.execute(_CASCADE_PATH_AND_SORT_FN.format( + table=self.table_name, + )) + schema_editor.execute(_BEFORE_TRIGGER_PATH_AND_SORT.format( + table=self.table_name, name_col=self.name_column, + )) + schema_editor.execute(_AFTER_TRIGGER_PATH_AND_SORT.format( + table=self.table_name, name_col=self.name_column, + )) + else: + schema_editor.execute(_COMPUTE_PATH_ONLY_FN.format(table=self.table_name)) + schema_editor.execute(_CASCADE_PATH_ONLY_FN.format(table=self.table_name)) + schema_editor.execute(_BEFORE_TRIGGER_PATH_ONLY.format(table=self.table_name)) + schema_editor.execute(_AFTER_TRIGGER_PATH_ONLY.format(table=self.table_name)) + + def database_backwards(self, app_label, schema_editor, from_state, to_state): + t = self.table_name + schema_editor.execute(f'DROP TRIGGER IF EXISTS "{t}_ltree_cascade_path" ON "{t}";') + schema_editor.execute(f'DROP TRIGGER IF EXISTS "{t}_ltree_compute_path" ON "{t}";') + schema_editor.execute(f'DROP FUNCTION IF EXISTS "{t}_ltree_cascade_path_fn"();') + schema_editor.execute(f'DROP FUNCTION IF EXISTS "{t}_ltree_compute_path_fn"();') + + def describe(self): + return f"Install ltree path triggers on {self.table_name}" diff --git a/netbox/wireless/migrations/0020_ltree_paths.py b/netbox/wireless/migrations/0020_ltree_paths.py index 9e49ecbcfaf..58ce9ca78fe 100644 --- a/netbox/wireless/migrations/0020_ltree_paths.py +++ b/netbox/wireless/migrations/0020_ltree_paths.py @@ -9,7 +9,7 @@ from django.db import migrations, models import netbox.models.ltree -from netbox.models.ltree import InstallLtreeTriggers +from utilities.ltree import InstallLtreeTriggers MODEL = 'wirelesslangroup' TABLE = 'wireless_wirelesslangroup' From d89a712cbeb4e0fc004b982b217eb18a3d42c790 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 10 Jun 2026 08:41:32 -0700 Subject: [PATCH 33/42] review feedback --- netbox/dcim/migrations/0238_ltree_paths.py | 84 +++---------- netbox/tenancy/migrations/0025_ltree_paths.py | 57 ++------- netbox/utilities/mptt_to_ltree.py | 115 ++++++++++++++++++ .../wireless/migrations/0020_ltree_paths.py | 38 ++---- 4 files changed, 150 insertions(+), 144 deletions(-) create mode 100644 netbox/utilities/mptt_to_ltree.py diff --git a/netbox/dcim/migrations/0238_ltree_paths.py b/netbox/dcim/migrations/0238_ltree_paths.py index 1c994d58711..67c69e77476 100644 --- a/netbox/dcim/migrations/0238_ltree_paths.py +++ b/netbox/dcim/migrations/0238_ltree_paths.py @@ -39,6 +39,7 @@ import netbox.models.ltree from utilities.ltree import InstallLtreeTriggers +from utilities.mptt_to_ltree import assert_paths_populated_sql, populate_paths_sql # All models getting an ltree `path` column. ALL_MODELS = ( @@ -73,70 +74,6 @@ LEGACY_FIELDS = ('lft', 'rght', 'tree_id', 'level') -def _populate_paths_sql(): - """ - Build the recursive CTE that walks each table from roots downward, computing - the new path (PK-based, zero-padded) and — for models with sort_path — the - chr(9)-separated chain of ancestor names. - """ - blocks = [] - for table in ALL_TABLES: - if table in SORT_TABLES: - blocks.append(f""" -WITH RECURSIVE t(id, parent_id, path, sort_path) AS ( - SELECT id, parent_id, - lpad(id::text, 19, '0')::ltree, - name::text - FROM "{table}" WHERE parent_id IS NULL - UNION ALL - SELECT r.id, r.parent_id, - t.path || lpad(r.id::text, 19, '0')::ltree, - t.sort_path || chr(9) || r.name - FROM "{table}" r JOIN t ON r.parent_id = t.id -) -UPDATE "{table}" SET path = t.path, sort_path = t.sort_path -FROM t WHERE "{table}".id = t.id; -""") - else: - blocks.append(f""" -WITH RECURSIVE t(id, parent_id, path) AS ( - SELECT id, parent_id, lpad(id::text, 19, '0')::ltree FROM "{table}" WHERE parent_id IS NULL - UNION ALL - SELECT r.id, r.parent_id, t.path || lpad(r.id::text, 19, '0')::ltree - FROM "{table}" r JOIN t ON r.parent_id = t.id -) -UPDATE "{table}" SET path = t.path FROM t WHERE "{table}".id = t.id; -""") - return '\n'.join(blocks) - - -def _assert_paths_populated_sql(): - """ - After the recursive CTE backfills paths from `parent_id IS NULL` roots, any - row whose parent_id points to a row that the CTE could not reach (orphan FK, - stray cycle left by a prior raw write) will still have path IS NULL. The - immediately following AlterField that sets path to NOT NULL would then abort - inside ALTER COLUMN with an opaque message. Fail fast here instead so the - operator sees the offending table and row counts. - """ - checks = [] - for table in ALL_TABLES: - checks.append(f""" -DO $$ -DECLARE missing bigint; -BEGIN - SELECT count(*) INTO missing FROM "{table}" WHERE path IS NULL; - IF missing > 0 THEN - RAISE EXCEPTION - 'ltree backfill left % rows in "{table}" with NULL path; ' - 'likely orphan parent_id references — resolve before re-running ' - 'this migration', missing; - END IF; -END $$; -""") - return '\n'.join(checks) - - class Migration(migrations.Migration): dependencies = [ @@ -144,6 +81,9 @@ class Migration(migrations.Migration): ] operations = [ + # Enable the ltree extension first so the migration fails fast if it is missing. + CreateExtension('ltree'), + # Switch parent from mptt.fields.TreeForeignKey to django.db.models.ForeignKey # (no-op at the SQL level; reconciles migration state with model definitions). migrations.AlterField( @@ -188,9 +128,6 @@ class Migration(migrations.Migration): related_name='children', to='dcim.sitegroup'), ), - # 1. Enable the ltree extension (idempotent). - CreateExtension('ltree'), - # 2. Add nullable path column on all tree models. *[ migrations.AddField( @@ -221,12 +158,19 @@ class Migration(migrations.Migration): InstallLtreeTriggers('dcim_inventoryitem'), InstallLtreeTriggers('dcim_inventoryitemtemplate'), - # 4. Populate existing rows via per-table recursive CTE. - migrations.RunSQL(_populate_paths_sql(), reverse_sql=migrations.RunSQL.noop), + # 4. Populate existing rows via per-table recursive CTE (sort_path only + # for the models that carry it). + migrations.RunSQL( + '\n'.join(populate_paths_sql(t, sort_path=t in SORT_TABLES) for t in ALL_TABLES), + reverse_sql=migrations.RunSQL.noop, + ), # 4b. Fail fast (with a useful message) if any row still has NULL path — # otherwise the AlterField below aborts opaquely inside ALTER COLUMN. - migrations.RunSQL(_assert_paths_populated_sql(), reverse_sql=migrations.RunSQL.noop), + migrations.RunSQL( + '\n'.join(assert_paths_populated_sql(t) for t in ALL_TABLES), + reverse_sql=migrations.RunSQL.noop, + ), # 5. Tighten path to NOT NULL with empty-string default. *[ diff --git a/netbox/tenancy/migrations/0025_ltree_paths.py b/netbox/tenancy/migrations/0025_ltree_paths.py index d9ef2c101ec..3e49059084c 100644 --- a/netbox/tenancy/migrations/0025_ltree_paths.py +++ b/netbox/tenancy/migrations/0025_ltree_paths.py @@ -10,52 +10,13 @@ import netbox.models.ltree from utilities.ltree import InstallLtreeTriggers +from utilities.mptt_to_ltree import assert_paths_populated_sql, populate_paths_sql MODELS = ('tenantgroup', 'contactgroup') TABLES = ('tenancy_tenantgroup', 'tenancy_contactgroup') LEGACY_FIELDS = ('lft', 'rght', 'tree_id', 'level') -def _populate_paths_sql(): - blocks = [] - for table in TABLES: - blocks.append(f""" -WITH RECURSIVE t(id, parent_id, path, sort_path) AS ( - SELECT id, parent_id, - lpad(id::text, 19, '0')::ltree, - name::text - FROM "{table}" WHERE parent_id IS NULL - UNION ALL - SELECT r.id, r.parent_id, - t.path || lpad(r.id::text, 19, '0')::ltree, - t.sort_path || chr(9) || r.name - FROM "{table}" r JOIN t ON r.parent_id = t.id -) -UPDATE "{table}" SET path = t.path, sort_path = t.sort_path -FROM t WHERE "{table}".id = t.id; -""") - return '\n'.join(blocks) - - -def _assert_paths_populated_sql(): - checks = [] - for table in TABLES: - checks.append(f""" -DO $$ -DECLARE missing bigint; -BEGIN - SELECT count(*) INTO missing FROM "{table}" WHERE path IS NULL; - IF missing > 0 THEN - RAISE EXCEPTION - 'ltree backfill left % rows in "{table}" with NULL path; ' - 'likely orphan parent_id references — resolve before re-running ' - 'this migration', missing; - END IF; -END $$; -""") - return '\n'.join(checks) - - class Migration(migrations.Migration): dependencies = [ @@ -63,6 +24,9 @@ class Migration(migrations.Migration): ] operations = [ + # Enable the ltree extension first so the migration fails fast if it is missing. + CreateExtension('ltree'), + # Switch parent from mptt.fields.TreeForeignKey to django.db.models.ForeignKey. migrations.AlterField( model_name='contactgroup', name='parent', @@ -79,8 +43,6 @@ class Migration(migrations.Migration): ), ), - CreateExtension('ltree'), - # Add path (nullable initially) on both models. *[ migrations.AddField( @@ -104,11 +66,18 @@ class Migration(migrations.Migration): # Install triggers maintaining both path and sort_path. *[InstallLtreeTriggers(t, name_column='name') for t in TABLES], - migrations.RunSQL(_populate_paths_sql(), reverse_sql=migrations.RunSQL.noop), + # Populate path and sort_path for existing rows via per-table recursive CTE. + migrations.RunSQL( + '\n'.join(populate_paths_sql(t, sort_path=True) for t in TABLES), + reverse_sql=migrations.RunSQL.noop, + ), # Fail fast if any row still has NULL path (orphan FKs) before the # AlterField below tries to set NOT NULL inside ALTER COLUMN. - migrations.RunSQL(_assert_paths_populated_sql(), reverse_sql=migrations.RunSQL.noop), + migrations.RunSQL( + '\n'.join(assert_paths_populated_sql(t) for t in TABLES), + reverse_sql=migrations.RunSQL.noop, + ), *[ migrations.AlterField( diff --git a/netbox/utilities/mptt_to_ltree.py b/netbox/utilities/mptt_to_ltree.py new file mode 100644 index 00000000000..c669d4b30d1 --- /dev/null +++ b/netbox/utilities/mptt_to_ltree.py @@ -0,0 +1,115 @@ +""" +Reusable SQL builders for migrating a django-mptt tree to a PostgreSQL ltree +`path` (and optional `sort_path`) column. + +NetBox's core hierarchical models moved from django-mptt to ltree in v4.6. The +per-table data backfill that migration performs is identical in shape for every +tree, so the SQL is centralized here rather than copied into each app's +migration. This also gives plugin maintainers a supported path for migrating +their own MPTT models to `netbox.models.ltree.LtreeModel`; from a data migration: + + from django.contrib.postgres.operations import CreateExtension + from django.db import migrations + from utilities.ltree import InstallLtreeTriggers + from utilities.mptt_to_ltree import assert_paths_populated_sql, populate_paths_sql + + operations = [ + CreateExtension('ltree'), + # ... AddField('path', nullable), [AddField('sort_path')], InstallLtreeTriggers(...) ... + migrations.RunSQL( + populate_paths_sql('myplugin_mymodel', sort_path=True), + reverse_sql=migrations.RunSQL.noop, + ), + migrations.RunSQL( + assert_paths_populated_sql('myplugin_mymodel'), + reverse_sql=migrations.RunSQL.noop, + ), + # ... AlterField('path' -> NOT NULL) ... + ] + +The values produced here must stay byte-identical to what the runtime triggers +in `utilities.ltree` maintain: each path label is the row PK zero-padded to +`_PATH_LABEL_WIDTH` chars, and `sort_path` is the chr(9) (TAB) separated chain of +ancestor `name` values. Keep the two modules in sync if either changes. +""" + +__all__ = ( + 'assert_paths_populated_sql', + 'populate_paths_sql', +) + +# Width to which each PK is zero-padded when used as an ltree label. Must match +# the lpad() width used by the trigger functions in utilities.ltree (19 = max +# bigint digit width) so that backfilled paths and trigger-maintained paths sort +# and compare identically. +_PATH_LABEL_WIDTH = 19 + + +def populate_paths_sql(table, *, sort_path=False): + """ + Return SQL that backfills `path` (and `sort_path` when `sort_path=True`) for + every existing row in `table`, walking the tree from its roots + (parent_id IS NULL) downward via a single recursive CTE. + + `path` is the chain of PK labels, each zero-padded to `_PATH_LABEL_WIDTH` + chars. `sort_path` is the chr(9) (TAB) separated chain of ancestor `name` + values, matching the `order_insertion_by=('name',)` semantics the triggers + maintain at runtime. + + !!! warning + The UPDATE takes a row-exclusive lock on the entire table for the + duration of the statement. On large tables this can block writes for + minutes — plan a maintenance window accordingly. + """ + if sort_path: + return f""" +WITH RECURSIVE t(id, parent_id, path, sort_path) AS ( + SELECT id, parent_id, + lpad(id::text, {_PATH_LABEL_WIDTH}, '0')::ltree, + name::text + FROM "{table}" WHERE parent_id IS NULL + UNION ALL + SELECT r.id, r.parent_id, + t.path || lpad(r.id::text, {_PATH_LABEL_WIDTH}, '0')::ltree, + t.sort_path || chr(9) || r.name + FROM "{table}" r JOIN t ON r.parent_id = t.id +) +UPDATE "{table}" SET path = t.path, sort_path = t.sort_path +FROM t WHERE "{table}".id = t.id; +""" + return f""" +WITH RECURSIVE t(id, parent_id, path) AS ( + SELECT id, parent_id, lpad(id::text, {_PATH_LABEL_WIDTH}, '0')::ltree + FROM "{table}" WHERE parent_id IS NULL + UNION ALL + SELECT r.id, r.parent_id, t.path || lpad(r.id::text, {_PATH_LABEL_WIDTH}, '0')::ltree + FROM "{table}" r JOIN t ON r.parent_id = t.id +) +UPDATE "{table}" SET path = t.path FROM t WHERE "{table}".id = t.id; +""" + + +def assert_paths_populated_sql(table): + """ + Return SQL that raises if any row in `table` still has a NULL `path` after + `populate_paths_sql()` runs. + + The recursive CTE only reaches rows whose ancestry chains back to a + `parent_id IS NULL` root, so any row left with a NULL path points (directly + or transitively) at an orphan or cyclic parent_id. Catch that here, naming + the table and row count, rather than letting the subsequent + `AlterField(path -> NOT NULL)` abort opaquely inside ALTER COLUMN. + """ + return f""" +DO $$ +DECLARE missing bigint; +BEGIN + SELECT count(*) INTO missing FROM "{table}" WHERE path IS NULL; + IF missing > 0 THEN + RAISE EXCEPTION + 'ltree backfill left % rows in "{table}" with NULL path; ' + 'likely orphan parent_id references — resolve before re-running ' + 'this migration', missing; + END IF; +END $$; +""" diff --git a/netbox/wireless/migrations/0020_ltree_paths.py b/netbox/wireless/migrations/0020_ltree_paths.py index 58ce9ca78fe..539357bdca2 100644 --- a/netbox/wireless/migrations/0020_ltree_paths.py +++ b/netbox/wireless/migrations/0020_ltree_paths.py @@ -10,6 +10,7 @@ import netbox.models.ltree from utilities.ltree import InstallLtreeTriggers +from utilities.mptt_to_ltree import assert_paths_populated_sql, populate_paths_sql MODEL = 'wirelesslangroup' TABLE = 'wireless_wirelesslangroup' @@ -23,6 +24,9 @@ class Migration(migrations.Migration): ] operations = [ + # Enable the ltree extension first so the migration fails fast if it is missing. + CreateExtension('ltree'), + migrations.AlterField( model_name='wirelesslangroup', name='parent', field=models.ForeignKey( @@ -31,8 +35,6 @@ class Migration(migrations.Migration): ), ), - CreateExtension('ltree'), - migrations.AddField( model_name=MODEL, name='path', field=netbox.models.ltree.LtreeField(blank=True, editable=False, null=True), @@ -47,41 +49,17 @@ class Migration(migrations.Migration): InstallLtreeTriggers(TABLE, name_column='name'), + # Populate path and sort_path for existing rows by walking the tree from + # the roots (parent_id IS NULL) downward via a single recursive CTE. migrations.RunSQL( - f""" -WITH RECURSIVE t(id, parent_id, path, sort_path) AS ( - SELECT id, parent_id, - lpad(id::text, 19, '0')::ltree, - name::text - FROM "{TABLE}" WHERE parent_id IS NULL - UNION ALL - SELECT r.id, r.parent_id, - t.path || lpad(r.id::text, 19, '0')::ltree, - t.sort_path || chr(9) || r.name - FROM "{TABLE}" r JOIN t ON r.parent_id = t.id -) -UPDATE "{TABLE}" SET path = t.path, sort_path = t.sort_path -FROM t WHERE "{TABLE}".id = t.id; -""", + populate_paths_sql(TABLE, sort_path=True), reverse_sql=migrations.RunSQL.noop, ), # Fail fast if any row still has NULL path (orphan FKs) before the # AlterField below tries to set NOT NULL inside ALTER COLUMN. migrations.RunSQL( - f""" -DO $$ -DECLARE missing bigint; -BEGIN - SELECT count(*) INTO missing FROM "{TABLE}" WHERE path IS NULL; - IF missing > 0 THEN - RAISE EXCEPTION - 'ltree backfill left % rows in "{TABLE}" with NULL path; ' - 'likely orphan parent_id references — resolve before re-running ' - 'this migration', missing; - END IF; -END $$; -""", + assert_paths_populated_sql(TABLE), reverse_sql=migrations.RunSQL.noop, ), From 493f57193515dbe2075963221e0eee2806aa953a Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 10 Jun 2026 08:46:14 -0700 Subject: [PATCH 34/42] review feedback --- netbox/netbox/api/viewsets/__init__.py | 25 ++++++++++++ netbox/netbox/models/ltree.py | 54 ++++++-------------------- 2 files changed, 37 insertions(+), 42 deletions(-) diff --git a/netbox/netbox/api/viewsets/__init__.py b/netbox/netbox/api/viewsets/__init__.py index 49d12dd76ce..8b5ff16376b 100644 --- a/netbox/netbox/api/viewsets/__init__.py +++ b/netbox/netbox/api/viewsets/__init__.py @@ -1,4 +1,5 @@ import logging +import warnings from functools import cached_property from django.core.exceptions import ObjectDoesNotExist, PermissionDenied @@ -17,6 +18,7 @@ from . import mixins __all__ = ( + 'MPTTLockedMixin', 'NetBoxModelViewSet', 'NetBoxReadOnlyModelViewSet', ) @@ -333,3 +335,26 @@ def perform_destroy(self, instance): super().perform_destroy(instance) except ObjectDoesNotExist: raise PermissionDenied() + + +class MPTTLockedMixin: + """ + Deprecated no-op mixin retained for backward compatibility. + + Historically this acquired a pglock around create/update/destroy to serialize + concurrent writes to MPTT-based tree models. NetBox no longer uses MPTT: tree + integrity is now maintained by the PostgreSQL ltree triggers (see + `utilities.ltree`), which take per-tree advisory locks at the database level. + This mixin is therefore now a transparent pass-through and may be removed in a + future release. Plugins should stop inheriting from it. + """ + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + warnings.warn( + "MPTTLockedMixin is deprecated and no longer does anything; tree write " + "concurrency is now handled by ltree database triggers. Remove it from " + f"{cls.__name__}.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index d17ccb18263..b2e4866026b 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -14,12 +14,14 @@ """ from django.core.exceptions import FieldDoesNotExist, ValidationError from django.db import IntegrityError, OperationalError, connection, models -from django.db.models import ForeignKey, Lookup, ManyToManyField +from django.db.models import ForeignKey, ManyToManyField from django.db.models.expressions import RawSQL from django.utils.translation import gettext_lazy as _ from utilities.querysets import RestrictedQuerySet +from . import lookups + __all__ = ( 'LtreeField', 'LtreeManager', @@ -56,41 +58,9 @@ def get_prep_value(self, value): return str(value) -@LtreeField.register_lookup -class Ancestor(Lookup): - """`path` is an ancestor of (or equal to) the queried path: path @> rhs""" - lookup_name = 'ancestor' - - def as_sql(self, compiler, connection): - lhs, lhs_params = self.process_lhs(compiler, connection) - rhs, rhs_params = self.process_rhs(compiler, connection) - return f'{lhs} @> {rhs}', lhs_params + rhs_params - - -@LtreeField.register_lookup -class Descendant(Lookup): - """ - `path` is a strict descendant of the queried path: path <@ rhs AND path <> rhs. - - Use `descendant_or_equal` for the inclusive form (`<@`). - """ - lookup_name = 'descendant' - - def as_sql(self, compiler, connection): - lhs, lhs_params = self.process_lhs(compiler, connection) - rhs, rhs_params = self.process_rhs(compiler, connection) - return f'({lhs} <@ {rhs} AND {lhs} <> {rhs})', lhs_params + rhs_params + lhs_params + rhs_params - - -@LtreeField.register_lookup -class DescendantOrEqual(Lookup): - """`path` is a descendant of (or equal to) the queried path: path <@ rhs""" - lookup_name = 'descendant_or_equal' - - def as_sql(self, compiler, connection): - lhs, lhs_params = self.process_lhs(compiler, connection) - rhs, rhs_params = self.process_rhs(compiler, connection) - return f'{lhs} <@ {rhs}', lhs_params + rhs_params +LtreeField.register_lookup(lookups.Ancestor) +LtreeField.register_lookup(lookups.Descendant) +LtreeField.register_lookup(lookups.DescendantOrEqual) class SortPathField(models.TextField): @@ -404,6 +374,12 @@ def save(self, *args, **kwargs): parent_written = update_fields is None or 'parent' in update_fields or 'parent_id' in update_fields parent_changed = (not is_insert) and parent_written and self.parent_id != self._loaded_parent_id + # Reject cyclic moves before writing, mirroring django-mptt's save-time + # guard so scripts / bulk callers (which bypass form & serializer clean()) + # cannot silently corrupt the tree. + if parent_changed and self._parent_creates_cycle(): + raise ValidationError(_("Cannot assign self or a descendant as parent.")) + # The sort_path trigger also fires on a name change; detect that so the # cascaded sort_path can be refreshed below (path-only models have no # sort_path and are unaffected by renames). @@ -414,12 +390,6 @@ def save(self, *args, **kwargs): and self.__dict__.get('name') != self._loaded_name ) - # Reject cyclic moves before writing, mirroring django-mptt's save-time - # guard so scripts / bulk callers (which bypass form & serializer clean()) - # cannot silently corrupt the tree. - if parent_changed and self._parent_creates_cycle(): - raise ValidationError(_("Cannot assign self or a descendant as parent.")) - try: super().save(*args, **kwargs) except IntegrityError as exc: From 1ca3da1d029ed310443d0e8cc5412b99637b14cf Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 10 Jun 2026 08:50:45 -0700 Subject: [PATCH 35/42] review feedback --- netbox/extras/querysets.py | 10 +++++----- netbox/netbox/models/ltree.py | 6 +++--- netbox/utilities/tests/test_ltree.py | 21 +++++++++++++++++++++ 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/netbox/extras/querysets.py b/netbox/extras/querysets.py index 6a57998eb58..dfdd03608a5 100644 --- a/netbox/extras/querysets.py +++ b/netbox/extras/querysets.py @@ -130,7 +130,7 @@ def _get_config_context_filters(self): if self.model._meta.model_name == 'device': base_query.add( (Q( - locations__path__ancestor=OuterRef('location__path'), + locations__path__ancestor_or_equal=OuterRef('location__path'), ) | Q(locations=None)), Q.AND ) @@ -143,25 +143,25 @@ def _get_config_context_filters(self): # (or equal to) the device/VM-side tree node, i.e. `cc_node.path @> obj_node.path`. base_query.add( (Q( - regions__path__ancestor=OuterRef('site__region__path'), + regions__path__ancestor_or_equal=OuterRef('site__region__path'), ) | Q(regions=None)), Q.AND ) base_query.add( (Q( - site_groups__path__ancestor=OuterRef('site__group__path'), + site_groups__path__ancestor_or_equal=OuterRef('site__group__path'), ) | Q(site_groups=None)), Q.AND ) base_query.add( (Q( - roles__path__ancestor=OuterRef('role__path'), + roles__path__ancestor_or_equal=OuterRef('role__path'), ) | Q(roles=None)), Q.AND ) base_query.add( (Q( - platforms__path__ancestor=OuterRef('platform__path'), + platforms__path__ancestor_or_equal=OuterRef('platform__path'), ) | Q(platforms=None)), Q.AND ) diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index b2e4866026b..d425f5a7719 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -59,6 +59,7 @@ def get_prep_value(self, value): LtreeField.register_lookup(lookups.Ancestor) +LtreeField.register_lookup(lookups.AncestorOrEqual) LtreeField.register_lookup(lookups.Descendant) LtreeField.register_lookup(lookups.DescendantOrEqual) @@ -467,9 +468,8 @@ def _tree_order_field(cls): def get_ancestors(self, ascending=False, include_self=False): if not self.path: return type(self)._default_manager.none() - qs = type(self)._default_manager.filter(path__ancestor=self.path) - if not include_self: - qs = qs.exclude(pk=self.pk) + lookup = 'ancestor_or_equal' if include_self else 'ancestor' + qs = type(self)._default_manager.filter(**{f'path__{lookup}': self.path}) order_field = self._tree_order_field() return qs.order_by(f'-{order_field}' if ascending else order_field) diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py index 5ec5a72c790..fa204c261c1 100644 --- a/netbox/utilities/tests/test_ltree.py +++ b/netbox/utilities/tests/test_ltree.py @@ -471,6 +471,27 @@ def test_strict_descendant_excludes_self(self): self.assertEqual(sorted(inclusive), ['Kid', 'Root']) +class AncestorLookupSemanticsTests(TestCase): + """ + path__ancestor is strict (path @> rhs AND path != rhs); the inclusive form + is path__ancestor_or_equal. + """ + + def test_strict_ancestor_excludes_self(self): + root = Region.objects.create(name='Root', slug='root-als') + kid = Region.objects.create(parent=root, name='Kid', slug='kid-als') + strict = list( + Region.objects.filter(path__ancestor=kid.path) + .values_list('name', flat=True) + ) + self.assertEqual(sorted(strict), ['Root']) + inclusive = list( + Region.objects.filter(path__ancestor_or_equal=kid.path) + .values_list('name', flat=True) + ) + self.assertEqual(sorted(inclusive), ['Kid', 'Root']) + + class RenameCascadesSortPathTests(TestCase): """ Renaming a node updates its own sort_path AND cascades into descendants' From 0c6aa4319d2794777baa3f1a75f44c1a63bb0cc5 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 10 Jun 2026 08:51:54 -0700 Subject: [PATCH 36/42] review feedback --- netbox/netbox/models/ltree.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index d425f5a7719..69e47202eb9 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -328,7 +328,11 @@ def _has_sort_path(cls): column (the MPTT `order_insertion_by=('name',)` equivalent). Single source of truth for clean(), save(), and _tree_order_field(). """ - return any(f.attname == 'sort_path' for f in cls._meta.concrete_fields) + try: + cls._meta.get_field('sort_path') + return True + except FieldDoesNotExist: + return False def clean(self): """ From c618e0783af9dc9a17bef5730ae62939da852e2d Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 10 Jun 2026 08:53:16 -0700 Subject: [PATCH 37/42] review feedback --- netbox/netbox/tables/columns.py | 1 + 1 file changed, 1 insertion(+) diff --git a/netbox/netbox/tables/columns.py b/netbox/netbox/tables/columns.py index 6df59cb5214..5620d7cb538 100644 --- a/netbox/netbox/tables/columns.py +++ b/netbox/netbox/tables/columns.py @@ -630,6 +630,7 @@ def value(self, value): # Deprecated alias for plugin compatibility; use TreeColumn going forward. +# TODO: Remove this in NetBox v5.0 MPTTColumn = TreeColumn From 449b5f2a2ab759f72ae86579b79371f677b34eb2 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 10 Jun 2026 09:00:29 -0700 Subject: [PATCH 38/42] review feedback --- netbox/netbox/api/viewsets/__init__.py | 1 + netbox/utilities/query.py | 21 +++++++++++++++++++++ netbox/utilities/tests/test_ltree.py | 11 +++++++++++ 3 files changed, 33 insertions(+) diff --git a/netbox/netbox/api/viewsets/__init__.py b/netbox/netbox/api/viewsets/__init__.py index 8b5ff16376b..6b7aa654a61 100644 --- a/netbox/netbox/api/viewsets/__init__.py +++ b/netbox/netbox/api/viewsets/__init__.py @@ -337,6 +337,7 @@ def perform_destroy(self, instance): raise PermissionDenied() +# TODO: Remove this in NetBox v5.0 class MPTTLockedMixin: """ Deprecated no-op mixin retained for backward compatibility. diff --git a/netbox/utilities/query.py b/netbox/utilities/query.py index 12623576a0e..6c26021bf39 100644 --- a/netbox/utilities/query.py +++ b/netbox/utilities/query.py @@ -57,6 +57,21 @@ def dict_to_filter_params(d, prefix=''): return params +# TODO: Remove in NetBox v5.0. MPTT support is retained only for plugins that have +# not yet migrated their tree models to netbox.models.ltree.LtreeModel; NetBox core +# no longer uses django-mptt. When MPTT support is dropped, delete this helper and +# its call in reapply_model_ordering(). +def _is_mptt_model(model) -> bool: + """ + Whether `model` is backed by django-mptt (deprecated). MPTT applies its own + tree ordering via the TreeManager, so such querysets must NOT have plain + model-level ordering reapplied after .annotate(). + """ + from mptt.managers import TreeManager + + return any(isinstance(manager, TreeManager) for manager in model._meta.local_managers) + + def reapply_model_ordering(queryset: QuerySet) -> QuerySet: """ Reapply model-level ordering in case it has been lost through .annotate(). @@ -70,6 +85,12 @@ def reapply_model_ordering(queryset: QuerySet) -> QuerySet: ordering = queryset.model._meta.ordering or () if any(isinstance(f, str) and f.lstrip('-') in ('sort_path', 'path') for f in ordering): return queryset + + # TODO: Remove in NetBox v5.0 (see _is_mptt_model). Plugins may still use MPTT + # via the generic bulk views, so keep exempting MPTT-based models for now. + if _is_mptt_model(queryset.model): + return queryset + if queryset.ordered: return queryset diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py index fa204c261c1..49e9e4c3f37 100644 --- a/netbox/utilities/tests/test_ltree.py +++ b/netbox/utilities/tests/test_ltree.py @@ -603,6 +603,17 @@ def test_ltree_model_is_exempt(self): # local_managers), so the exemption must still apply and return qs as-is. self.assertIs(result, qs) + def test_mptt_model_detection(self): + # MPTT support is retained for plugins (NestedGroupModel stays MPTT-backed); + # reapply_model_ordering() must keep exempting such models. See _is_mptt_model. + # TODO: Remove in NetBox v5.0 alongside MPTT support. + from netbox.models import NestedGroupModel + from utilities.query import _is_mptt_model + + self.assertTrue(_is_mptt_model(NestedGroupModel)) + # An ltree-backed model must NOT be misdetected as MPTT. + self.assertFalse(_is_mptt_model(Region)) + class AddRelatedCountErrorTests(TestCase): """ From eb47b3f8fa9c9825f644e4c3749088ae630aec95 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 10 Jun 2026 09:09:54 -0700 Subject: [PATCH 39/42] put back MPTT into generic view --- netbox/netbox/views/generic/bulk_views.py | 53 ++++++++++++++++++++--- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/netbox/netbox/views/generic/bulk_views.py b/netbox/netbox/views/generic/bulk_views.py index 3143e1e9eb1..6672ab5a0f7 100644 --- a/netbox/netbox/views/generic/bulk_views.py +++ b/netbox/netbox/views/generic/bulk_views.py @@ -1,6 +1,7 @@ import logging import re from collections import Counter +from contextlib import contextmanager from copy import deepcopy from types import SimpleNamespace @@ -54,6 +55,35 @@ ) +# TODO: Remove in NetBox v5.0. +# MPTT support is retained only for plugins whose tree models still derive from the +# deprecated MPTT-backed bases. NetBox core uses netbox.models.ltree.LtreeModel, whose +# database triggers maintain the tree on every write, so ltree (and non-tree) models +# need no special bulk handling. These two helpers confine all MPTT-specific bulk +# bookkeeping so it can be deleted in one place; for non-MPTT models they are no-ops. +@contextmanager +def _delay_mptt_updates(model): + """ + Defer tree (lft/rght/tree_id) recomputation until the end of a bulk write for + legacy MPTT models. A no-op context manager for ltree and non-tree models. + """ + from mptt.models import MPTTModel + + if issubclass(model, MPTTModel): + with model.objects.delay_mptt_updates(): + yield + else: + yield + + +def _rebuild_mptt_tree(model): + """Rebuild the MPTT tree after a bulk edit for legacy MPTT models; else a no-op.""" + from mptt.models import MPTTModel + + if issubclass(model, MPTTModel): + model.objects.rebuild() + + class ObjectListView(BaseMultiObjectView, ActionsMixin, TableMixin): """ Display multiple objects, all the same type, as a table. @@ -562,7 +592,10 @@ def create_and_update_objects(self, form, request): for obj in self.queryset.model.objects.filter(id__in=prefetch_ids) } if prefetch_ids else {} - saved_objects = self._process_import_records(form, request, records, prefetched_objects) + # Delay tree updates until all saves are complete (MPTT plugin models only; + # no-op for ltree). TODO: Remove the wrapper in v5.0 (see _delay_mptt_updates). + with _delay_mptt_updates(self.queryset.model): + saved_objects = self._process_import_records(form, request, records, prefetched_objects) return saved_objects @@ -756,6 +789,10 @@ def _update_objects(self, form, request): if is_background_request(request): request.job.logger.info(f"Updated {obj}") + # Rebuild the tree for MPTT plugin models (no-op for ltree; its triggers keep + # the tree current). TODO: Remove in v5.0 (see _rebuild_mptt_tree). + _rebuild_mptt_tree(self.queryset.model) + return updated_objects # @@ -964,11 +1001,15 @@ def post(self, request): renamed_pks = self._rename_objects(form, selected_objects, field_names) if '_apply' in request.POST: - for obj in selected_objects: - for field in field_names: - setattr(obj, field, getattr(obj.new_names, field)) - obj._changelog_message = form.cleaned_data.get('changelog_message', '') - obj.save() + # Delay tree updates until all saves are complete (MPTT + # plugin models only; no-op for ltree). + # TODO: Remove the wrapper in v5.0 (see _delay_mptt_updates). + with _delay_mptt_updates(self.queryset.model): + for obj in selected_objects: + for field in field_names: + setattr(obj, field, getattr(obj.new_names, field)) + obj._changelog_message = form.cleaned_data.get('changelog_message', '') + obj.save() # Enforce constrained permissions if self.queryset.filter(pk__in=renamed_pks).count() != len(selected_objects): From bba6a220bffae8101d6e900f1e78001f2da6d0ca Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 10 Jun 2026 09:24:11 -0700 Subject: [PATCH 40/42] fix circular import --- netbox/netbox/models/ltree.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index 69e47202eb9..233e867b5b5 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -18,10 +18,13 @@ from django.db.models.expressions import RawSQL from django.utils.translation import gettext_lazy as _ +# Import lookup classes by their fully-qualified path rather than `from . import +# lookups`: this module is imported by netbox/models/__init__.py before that package +# finishes initializing, so a package-relative import would fail the attribute lookup +# on the partially-initialized `netbox.models` package (circular import). +from netbox.models.lookups import Ancestor, AncestorOrEqual, Descendant, DescendantOrEqual from utilities.querysets import RestrictedQuerySet -from . import lookups - __all__ = ( 'LtreeField', 'LtreeManager', @@ -58,10 +61,10 @@ def get_prep_value(self, value): return str(value) -LtreeField.register_lookup(lookups.Ancestor) -LtreeField.register_lookup(lookups.AncestorOrEqual) -LtreeField.register_lookup(lookups.Descendant) -LtreeField.register_lookup(lookups.DescendantOrEqual) +LtreeField.register_lookup(Ancestor) +LtreeField.register_lookup(AncestorOrEqual) +LtreeField.register_lookup(Descendant) +LtreeField.register_lookup(DescendantOrEqual) class SortPathField(models.TextField): From b36d830c6b4b3ec00b69f6d9ee150317a6d01a27 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 10 Jun 2026 09:29:27 -0700 Subject: [PATCH 41/42] fix circular import --- netbox/netbox/models/lookups.py | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 netbox/netbox/models/lookups.py diff --git a/netbox/netbox/models/lookups.py b/netbox/netbox/models/lookups.py new file mode 100644 index 00000000000..b1541167b5c --- /dev/null +++ b/netbox/netbox/models/lookups.py @@ -0,0 +1,56 @@ +from django.db.models import Lookup + +__all__ = ( + 'Ancestor', + 'AncestorOrEqual', + 'Descendant', + 'DescendantOrEqual', +) + + +class Ancestor(Lookup): + """ + `path` is a strict ancestor of the queried path: path @> rhs AND path <> rhs. + + Use `ancestor_or_equal` for the inclusive form (`@>`). + """ + lookup_name = 'ancestor' + + def as_sql(self, compiler, connection): + lhs, lhs_params = self.process_lhs(compiler, connection) + rhs, rhs_params = self.process_rhs(compiler, connection) + return f'({lhs} @> {rhs} AND {lhs} <> {rhs})', lhs_params + rhs_params + lhs_params + rhs_params + + +class AncestorOrEqual(Lookup): + """`path` is an ancestor of (or equal to) the queried path: path @> rhs""" + lookup_name = 'ancestor_or_equal' + + def as_sql(self, compiler, connection): + lhs, lhs_params = self.process_lhs(compiler, connection) + rhs, rhs_params = self.process_rhs(compiler, connection) + return f'{lhs} @> {rhs}', lhs_params + rhs_params + + +class Descendant(Lookup): + """ + `path` is a strict descendant of the queried path: path <@ rhs AND path <> rhs. + + Use `descendant_or_equal` for the inclusive form (`<@`). + """ + lookup_name = 'descendant' + + def as_sql(self, compiler, connection): + lhs, lhs_params = self.process_lhs(compiler, connection) + rhs, rhs_params = self.process_rhs(compiler, connection) + return f'({lhs} <@ {rhs} AND {lhs} <> {rhs})', lhs_params + rhs_params + lhs_params + rhs_params + + +class DescendantOrEqual(Lookup): + """`path` is a descendant of (or equal to) the queried path: path <@ rhs""" + lookup_name = 'descendant_or_equal' + + def as_sql(self, compiler, connection): + lhs, lhs_params = self.process_lhs(compiler, connection) + rhs, rhs_params = self.process_rhs(compiler, connection) + return f'{lhs} <@ {rhs}', lhs_params + rhs_params From f5949ca95e8ca62918f3aeef4f90ed9c85a1e2ee Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 12 Jun 2026 13:56:24 -0700 Subject: [PATCH 42/42] update locks --- netbox/netbox/models/ltree.py | 13 ++++- netbox/utilities/ltree.py | 81 +++++++++++++++++----------- netbox/utilities/tests/test_ltree.py | 46 ++++++++++++++++ 3 files changed, 106 insertions(+), 34 deletions(-) diff --git a/netbox/netbox/models/ltree.py b/netbox/netbox/models/ltree.py index 233e867b5b5..a5ff18206ee 100644 --- a/netbox/netbox/models/ltree.py +++ b/netbox/netbox/models/ltree.py @@ -256,8 +256,8 @@ class LtreeModel(models.Model, metaclass=LtreeModelBase): tree being written (and, for a cross-tree move, on both the source and destination roots, acquired in ascending key order to avoid deadlocks). - Every insert, move, and reparent of a node in a tree takes the same key, - so an insert deep in a subtree and a concurrent reparent of one of its + Every child insert, move, and reparent of a node in a tree takes the same + key, so an insert deep in a subtree and a concurrent reparent of one of its ancestors are serialized — the loser blocks until the winner commits, and the winner's AFTER cascade can never miss a row inserted concurrently (which a row-level `FOR SHARE` on the parent could not prevent: a set-based @@ -265,6 +265,15 @@ class LtreeModel(models.Model, metaclass=LtreeModelBase): *different* trees use different keys and proceed fully in parallel — e.g. inventory ingestion across different devices, each its own tree. + Inserting a *new root* (parent_id IS NULL) takes no lock: the uncommitted + row is invisible to other transactions and has no descendants, so nothing + can contend with it. This keeps a bulk import of many top-level objects + (the dominant import pattern) lock-free instead of taking one advisory lock + per root. The residual case that still scales with volume is inserting + children into many *distinct existing* trees in one transaction (one lock + per distinct tree touched); if a very large such import hits "out of shared + memory", raising the server's `max_locks_per_transaction` is the remedy. + Two residual, retryable cases remain (PostgreSQL aborts one transaction with a deadlock error rather than persisting a stale path): crossing reparents (moving A under B while moving B under A), and two concurrent diff --git a/netbox/utilities/ltree.py b/netbox/utilities/ltree.py index 64d16504e9d..5dc495c91d1 100644 --- a/netbox/utilities/ltree.py +++ b/netbox/utilities/ltree.py @@ -21,39 +21,56 @@ # lexicographic ordering of ltree labels matches numeric PK ordering across digit # boundaries (e.g. "0...09" sorts before "0...10"). # Per-tree advisory locking (see _lock_tree_roots_sql / LtreeModel "Concurrency"): -# every insert/move/reparent of a node takes a transaction-level advisory lock -# keyed on the root(s) of the tree(s) it touches, BEFORE reading the parent path. -# A concurrent reparent of an ancestor takes the same key, so the two serialize -# and the AFTER cascade can never miss a row inserted concurrently; writes in -# different trees use different keys and run in parallel. +# every child insert / move / reparent of a node takes a transaction-level +# advisory lock keyed on the root(s) of the tree(s) it touches, BEFORE reading the +# parent path. A concurrent reparent of an ancestor takes the same key, so the two +# serialize and the AFTER cascade can never miss a row inserted concurrently; +# writes in different trees use different keys and run in parallel. Inserting a new +# root (parent_id IS NULL) takes no lock at all -- it is a race-free singleton tree +# -- so a bulk import of many top-level objects does not accumulate one lock per +# root and cannot exhaust the shared lock table. _LOCK_TREE_ROOTS_SQL = ''' - -- Destination tree root: the parent's root label, or this row's own label - -- when it is (or becomes) a root. The CASE guards against a parent whose path - -- is the empty ltree '' (reachable only via a trigger-bypassing raw write): - -- subltree('', 0, 1) would raise 'invalid positions', so fall back to the - -- child's own label as the lock key rather than aborting the insert/move. - IF NEW.parent_id IS NOT NULL THEN - EXECUTE format( - 'SELECT CASE WHEN nlevel(path) > 0 THEN subltree(path, 0, 1)::text END' - ' FROM %%I WHERE id = $1', - TG_TABLE_NAME - ) INTO dest_root USING NEW.parent_id; - END IF; - dest_root := COALESCE(dest_root, lpad(NEW.id::text, 19, '0')); - -- Source tree root (moves only): this row's current root, which the AFTER - -- cascade will rewrite. - IF TG_OP = 'UPDATE' AND OLD.path IS NOT NULL AND nlevel(OLD.path) > 0 THEN - old_root := subltree(OLD.path, 0, 1)::text; - END IF; - key_dest := hashtextextended(TG_TABLE_NAME || ':' || dest_root, 0); - IF old_root IS NOT NULL AND old_root <> dest_root THEN - -- Cross-tree move: lock both roots, ascending, to avoid deadlock between - -- two concurrent moves that touch the same pair. - key_old := hashtextextended(TG_TABLE_NAME || ':' || old_root, 0); - PERFORM pg_advisory_xact_lock(LEAST(key_dest, key_old)); - PERFORM pg_advisory_xact_lock(GREATEST(key_dest, key_old)); - ELSE - PERFORM pg_advisory_xact_lock(key_dest); + -- A brand-new root (INSERT with parent_id IS NULL) starts its own singleton + -- tree that no concurrent transaction can yet see (MVCC: the uncommitted row + -- is invisible) or reference (no other transaction has its PK). For such a + -- row this BEFORE function reads no other row (the parent lookup below is + -- gated on parent_id) and the AFTER cascade fires only on UPDATE, so the + -- insert touches solely its own NEW row -- there is nothing to serialize + -- against, and the advisory lock it would otherwise take can never contend. + -- Skipping it here is what stops a bulk import of many top-level objects from + -- taking one xact-lock per root and exhausting the shared lock table + -- (sized by max_locks_per_transaction). Every other case still locks below: + -- a child insert, any reparent, and a reparent-to-root (TG_OP = UPDATE, where + -- the existing row has a real subtree the AFTER cascade must rewrite). + IF NOT (TG_OP = 'INSERT' AND NEW.parent_id IS NULL) THEN + -- Destination tree root: the parent's root label, or this row's own label + -- when it is (or becomes) a root. The CASE guards against a parent whose path + -- is the empty ltree '' (reachable only via a trigger-bypassing raw write): + -- subltree('', 0, 1) would raise 'invalid positions', so fall back to the + -- child's own label as the lock key rather than aborting the insert/move. + IF NEW.parent_id IS NOT NULL THEN + EXECUTE format( + 'SELECT CASE WHEN nlevel(path) > 0 THEN subltree(path, 0, 1)::text END' + ' FROM %%I WHERE id = $1', + TG_TABLE_NAME + ) INTO dest_root USING NEW.parent_id; + END IF; + dest_root := COALESCE(dest_root, lpad(NEW.id::text, 19, '0')); + -- Source tree root (moves only): this row's current root, which the AFTER + -- cascade will rewrite. + IF TG_OP = 'UPDATE' AND OLD.path IS NOT NULL AND nlevel(OLD.path) > 0 THEN + old_root := subltree(OLD.path, 0, 1)::text; + END IF; + key_dest := hashtextextended(TG_TABLE_NAME || ':' || dest_root, 0); + IF old_root IS NOT NULL AND old_root <> dest_root THEN + -- Cross-tree move: lock both roots, ascending, to avoid deadlock between + -- two concurrent moves that touch the same pair. + key_old := hashtextextended(TG_TABLE_NAME || ':' || old_root, 0); + PERFORM pg_advisory_xact_lock(LEAST(key_dest, key_old)); + PERFORM pg_advisory_xact_lock(GREATEST(key_dest, key_old)); + ELSE + PERFORM pg_advisory_xact_lock(key_dest); + END IF; END IF; ''' diff --git a/netbox/utilities/tests/test_ltree.py b/netbox/utilities/tests/test_ltree.py index 49e9e4c3f37..441bdd04e7e 100644 --- a/netbox/utilities/tests/test_ltree.py +++ b/netbox/utilities/tests/test_ltree.py @@ -92,6 +92,52 @@ def test_gist_index_exists(self): self.assertSetEqual(found, expected) +class AdvisoryLockScopeTests(TestCase): + """ + Pin where the BEFORE trigger takes its per-tree advisory lock. + + A new root (INSERT with parent_id IS NULL) is a race-free singleton tree and + must take NO lock, so a bulk import of many top-level objects cannot exhaust + the shared lock table. Every other write (child insert, reparent-to-root) + must still lock per tree — the companion assertions keep the optimization from + silently over-broadening to cases that need serialization. + + `pg_locks` is cluster-wide and advisory *xact* locks are held until the + TestCase transaction ends, so each test measures the DELTA in this backend's + advisory-lock count across a single operation rather than an absolute count. + """ + + @staticmethod + def _advisory_lock_count(): + with connection.cursor() as cursor: + cursor.execute( + "SELECT count(*) FROM pg_locks " + "WHERE locktype = 'advisory' AND pid = pg_backend_pid()" + ) + return cursor.fetchone()[0] + + def test_root_insert_takes_no_lock(self): + before = self._advisory_lock_count() + Region.objects.create(name='Root', slug='root-lock') + self.assertEqual(self._advisory_lock_count() - before, 0) + + def test_child_insert_takes_one_lock(self): + root = Region.objects.create(name='Root', slug='root-lock2') + before = self._advisory_lock_count() + Region.objects.create(parent=root, name='Child', slug='child-lock2') + self.assertEqual(self._advisory_lock_count() - before, 1) + + def test_reparent_to_root_takes_lock(self): + root = Region.objects.create(name='Root', slug='root-lock3') + child = Region.objects.create(parent=root, name='Child', slug='child-lock3') + before = self._advisory_lock_count() + child.parent = None + child.save() + # An existing row promoted to root still has a real subtree to rewrite, so + # it must lock (its own new root key, plus the old tree's root for the move). + self.assertGreaterEqual(self._advisory_lock_count() - before, 1) + + class LtreeAPIParityTests(TestCase): """Verify the MPTTModel-compatible API surface."""