diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 8f341d2a1f..44298fec3f 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -2,6 +2,7 @@ ## Enhancements +* [#3799](https://github.com/netbox-community/netbox/issues/3799) - Greatly improve performance when ordering device components * [#4100](https://github.com/netbox-community/netbox/issues/4100) - Add device filter to component list views * [#4113](https://github.com/netbox-community/netbox/issues/4113) - Add bulk edit functionality for device type components * [#4116](https://github.com/netbox-community/netbox/issues/4116) - Enable bulk edit and delete functions for device component list views diff --git a/netbox/dcim/managers.py b/netbox/dcim/managers.py index e1124b84ee..5027196460 100644 --- a/netbox/dcim/managers.py +++ b/netbox/dcim/managers.py @@ -1,18 +1,7 @@ from django.db.models import Manager, QuerySet -from django.db.models.expressions import RawSQL from .constants import NONCONNECTABLE_IFACE_TYPES -# Regular expressions for parsing Interface names -TYPE_RE = r"SUBSTRING({} FROM '^([^0-9\.:]+)')" -SLOT_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9]+)?(\d{{1,9}})/') AS integer), NULL)" -SUBSLOT_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9\.:]+)?\d{{1,9}}/(\d{{1,9}})') AS integer), NULL)" -POSITION_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9]+)?(?:\d{{1,9}}/){{2}}(\d{{1,9}})') AS integer), NULL)" -SUBPOSITION_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9]+)?(?:\d{{1,9}}/){{3}}(\d{{1,9}})') AS integer), NULL)" -ID_RE = r"CAST(SUBSTRING({} FROM '^(?:[^0-9\.:]+)?(\d{{1,9}})([^/]|$)') AS integer)" -CHANNEL_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^.*:(\d{{1,9}})(\.\d{{1,9}})?$') AS integer), 0)" -VC_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^.*\.(\d{{1,9}})$') AS integer), 0)" - class InterfaceQuerySet(QuerySet): @@ -27,47 +16,4 @@ def connectable(self): class InterfaceManager(Manager): def get_queryset(self): - """ - Naturally order interfaces by their type and numeric position. To order interfaces naturally, the `name` field - is split into eight distinct components: leading text (type), slot, subslot, position, subposition, ID, channel, - and virtual circuit: - - {type}{slot or ID}/{subslot}/{position}/{subposition}:{channel}.{vc} - - Components absent from the interface name are coalesced to zero or null. For example, an interface named - GigabitEthernet1/2/3 would be parsed as follows: - - type = 'GigabitEthernet' - slot = 1 - subslot = 2 - position = 3 - subposition = None - id = None - channel = 0 - vc = 0 - - The original `name` field is considered in its entirety to serve as a fallback in the event interfaces do not - match any of the prescribed fields. - - The `id` field is included to enforce deterministic ordering of interfaces in similar vein of other device - components. - """ - - sql_col = '{}.name'.format(self.model._meta.db_table) - ordering = [ - '_slot', '_subslot', '_position', '_subposition', '_type', '_id', '_channel', '_vc', 'name', 'pk' - - ] - - fields = { - '_type': RawSQL(TYPE_RE.format(sql_col), []), - '_id': RawSQL(ID_RE.format(sql_col), []), - '_slot': RawSQL(SLOT_RE.format(sql_col), []), - '_subslot': RawSQL(SUBSLOT_RE.format(sql_col), []), - '_position': RawSQL(POSITION_RE.format(sql_col), []), - '_subposition': RawSQL(SUBPOSITION_RE.format(sql_col), []), - '_channel': RawSQL(CHANNEL_RE.format(sql_col), []), - '_vc': RawSQL(VC_RE.format(sql_col), []), - } - - return InterfaceQuerySet(self.model, using=self._db).annotate(**fields).order_by(*ordering) + return InterfaceQuerySet(self.model, using=self._db) diff --git a/netbox/dcim/migrations/0093_device_component_ordering.py b/netbox/dcim/migrations/0093_device_component_ordering.py new file mode 100644 index 0000000000..017241c8ba --- /dev/null +++ b/netbox/dcim/migrations/0093_device_component_ordering.py @@ -0,0 +1,147 @@ +from django.db import migrations +import utilities.fields +import utilities.ordering + + +def _update_model_names(model): + # Update each unique field value in bulk + for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name)) + + +def naturalize_consoleports(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'ConsolePort')) + + +def naturalize_consoleserverports(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'ConsoleServerPort')) + + +def naturalize_powerports(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'PowerPort')) + + +def naturalize_poweroutlets(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'PowerOutlet')) + + +def naturalize_frontports(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'FrontPort')) + + +def naturalize_rearports(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'RearPort')) + + +def naturalize_devicebays(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'DeviceBay')) + + +class Migration(migrations.Migration): + + dependencies = [ + ('dcim', '0092_fix_rack_outer_unit'), + ] + + operations = [ + migrations.AlterModelOptions( + name='consoleport', + options={'ordering': ('device', '_name')}, + ), + migrations.AlterModelOptions( + name='consoleserverport', + options={'ordering': ('device', '_name')}, + ), + migrations.AlterModelOptions( + name='devicebay', + options={'ordering': ('device', '_name')}, + ), + migrations.AlterModelOptions( + name='frontport', + options={'ordering': ('device', '_name')}, + ), + migrations.AlterModelOptions( + name='inventoryitem', + options={'ordering': ('device__id', 'parent__id', '_name')}, + ), + migrations.AlterModelOptions( + name='poweroutlet', + options={'ordering': ('device', '_name')}, + ), + migrations.AlterModelOptions( + name='powerport', + options={'ordering': ('device', '_name')}, + ), + migrations.AlterModelOptions( + name='rearport', + options={'ordering': ('device', '_name')}, + ), + migrations.AddField( + model_name='consoleport', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='consoleserverport', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='devicebay', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='frontport', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='inventoryitem', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='poweroutlet', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='powerport', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='rearport', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.RunPython( + code=naturalize_consoleports, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_consoleserverports, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_powerports, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_poweroutlets, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_frontports, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_rearports, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_devicebays, + reverse_code=migrations.RunPython.noop + ), + ] diff --git a/netbox/dcim/migrations/0094_device_component_template_ordering.py b/netbox/dcim/migrations/0094_device_component_template_ordering.py new file mode 100644 index 0000000000..fc39f76b2d --- /dev/null +++ b/netbox/dcim/migrations/0094_device_component_template_ordering.py @@ -0,0 +1,138 @@ +from django.db import migrations +import utilities.fields +import utilities.ordering + + +def _update_model_names(model): + # Update each unique field value in bulk + for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name)) + + +def naturalize_consoleporttemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'ConsolePortTemplate')) + + +def naturalize_consoleserverporttemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'ConsoleServerPortTemplate')) + + +def naturalize_powerporttemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'PowerPortTemplate')) + + +def naturalize_poweroutlettemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'PowerOutletTemplate')) + + +def naturalize_frontporttemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'FrontPortTemplate')) + + +def naturalize_rearporttemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'RearPortTemplate')) + + +def naturalize_devicebaytemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'DeviceBayTemplate')) + + +class Migration(migrations.Migration): + + dependencies = [ + ('dcim', '0093_device_component_ordering'), + ] + + operations = [ + migrations.AlterModelOptions( + name='consoleporttemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AlterModelOptions( + name='consoleserverporttemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AlterModelOptions( + name='devicebaytemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AlterModelOptions( + name='frontporttemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AlterModelOptions( + name='poweroutlettemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AlterModelOptions( + name='powerporttemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AlterModelOptions( + name='rearporttemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AddField( + model_name='consoleporttemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='consoleserverporttemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='devicebaytemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='frontporttemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='poweroutlettemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='powerporttemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='rearporttemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.RunPython( + code=naturalize_consoleporttemplates, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_consoleserverporttemplates, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_powerporttemplates, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_poweroutlettemplates, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_frontporttemplates, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_rearporttemplates, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_devicebaytemplates, + reverse_code=migrations.RunPython.noop + ), + ] diff --git a/netbox/dcim/migrations/0095_primary_model_ordering.py b/netbox/dcim/migrations/0095_primary_model_ordering.py new file mode 100644 index 0000000000..9cef0a581f --- /dev/null +++ b/netbox/dcim/migrations/0095_primary_model_ordering.py @@ -0,0 +1,70 @@ +from django.db import migrations +import utilities.fields +import utilities.ordering + + +def _update_model_names(model): + # Update each unique field value in bulk + for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize(name)) + + +def naturalize_sites(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'Site')) + + +def naturalize_racks(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'Rack')) + + +def naturalize_devices(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'Device')) + + +class Migration(migrations.Migration): + + dependencies = [ + ('dcim', '0094_device_component_template_ordering'), + ] + + operations = [ + migrations.AlterModelOptions( + name='device', + options={'ordering': ('_name', 'pk'), 'permissions': (('napalm_read', 'Read-only access to devices via NAPALM'), ('napalm_write', 'Read/write access to devices via NAPALM'))}, + ), + migrations.AlterModelOptions( + name='rack', + options={'ordering': ('site', 'group', '_name', 'pk')}, + ), + migrations.AlterModelOptions( + name='site', + options={'ordering': ('_name',)}, + ), + migrations.AddField( + model_name='device', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize, null=True), + ), + migrations.AddField( + model_name='rack', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.AddField( + model_name='site', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize), + ), + migrations.RunPython( + code=naturalize_sites, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_racks, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_devices, + reverse_code=migrations.RunPython.noop + ), + ] diff --git a/netbox/dcim/migrations/0096_interface_ordering.py b/netbox/dcim/migrations/0096_interface_ordering.py new file mode 100644 index 0000000000..2840664624 --- /dev/null +++ b/netbox/dcim/migrations/0096_interface_ordering.py @@ -0,0 +1,53 @@ +from django.db import migrations +import utilities.fields +import utilities.ordering + + +def _update_model_names(model): + # Update each unique field value in bulk + for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize_interface(name)) + + +def naturalize_interfacetemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'InterfaceTemplate')) + + +def naturalize_interfaces(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'Interface')) + + +class Migration(migrations.Migration): + + dependencies = [ + ('dcim', '0095_primary_model_ordering'), + ] + + operations = [ + migrations.AlterModelOptions( + name='interface', + options={'ordering': ('device', '_name')}, + ), + migrations.AlterModelOptions( + name='interfacetemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AddField( + model_name='interface', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize_interface), + ), + migrations.AddField( + model_name='interfacetemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize_interface), + ), + migrations.RunPython( + code=naturalize_interfacetemplates, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_interfaces, + reverse_code=migrations.RunPython.noop + ), + ] diff --git a/netbox/dcim/models/__init__.py b/netbox/dcim/models/__init__.py index 2a4988ba4a..c31f4c713f 100644 --- a/netbox/dcim/models/__init__.py +++ b/netbox/dcim/models/__init__.py @@ -22,8 +22,7 @@ from dcim.constants import * from dcim.fields import ASNField from extras.models import ConfigContextModel, CustomFieldModel, TaggedItem -from utilities.fields import ColorField -from utilities.managers import NaturalOrderingManager +from utilities.fields import ColorField, NaturalOrderingField from utilities.models import ChangeLoggedModel from utilities.utils import foreground_color, to_meters from .device_component_templates import ( @@ -134,6 +133,11 @@ class Site(ChangeLoggedModel, CustomFieldModel): max_length=50, unique=True ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) slug = models.SlugField( unique=True ) @@ -215,8 +219,6 @@ class Site(ChangeLoggedModel, CustomFieldModel): images = GenericRelation( to='extras.ImageAttachment' ) - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = [ @@ -235,7 +237,7 @@ class Site(ChangeLoggedModel, CustomFieldModel): } class Meta: - ordering = ['name'] + ordering = ('_name',) def __str__(self): return self.name @@ -516,6 +518,11 @@ class Rack(ChangeLoggedModel, CustomFieldModel, RackElevationHelperMixin): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) facility_id = models.CharField( max_length=50, blank=True, @@ -612,8 +619,6 @@ class Rack(ChangeLoggedModel, CustomFieldModel, RackElevationHelperMixin): images = GenericRelation( to='extras.ImageAttachment' ) - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = [ @@ -634,12 +639,12 @@ class Rack(ChangeLoggedModel, CustomFieldModel, RackElevationHelperMixin): } class Meta: - ordering = ('site', 'group', 'name', 'pk') # (site, group, name) may be non-unique - unique_together = [ + ordering = ('site', 'group', '_name', 'pk') # (site, group, name) may be non-unique + unique_together = ( # Name and facility_id must be unique *only* within a RackGroup - ['group', 'name'], - ['group', 'facility_id'], - ] + ('group', 'name'), + ('group', 'facility_id'), + ) def __str__(self): return self.display_name or super().__str__() @@ -1313,6 +1318,12 @@ class Device(ChangeLoggedModel, ConfigContextModel, CustomFieldModel): blank=True, null=True ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True, + null=True + ) serial = models.CharField( max_length=50, blank=True, @@ -1407,8 +1418,6 @@ class Device(ChangeLoggedModel, ConfigContextModel, CustomFieldModel): images = GenericRelation( to='extras.ImageAttachment' ) - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = [ @@ -1430,12 +1439,12 @@ class Device(ChangeLoggedModel, ConfigContextModel, CustomFieldModel): } class Meta: - ordering = ('name', 'pk') # Name may be NULL - unique_together = [ - ['site', 'tenant', 'name'], # See validate_unique below - ['rack', 'position', 'face'], - ['virtual_chassis', 'vc_position'], - ] + ordering = ('_name', 'pk') # Name may be null + unique_together = ( + ('site', 'tenant', 'name'), # See validate_unique below + ('rack', 'position', 'face'), + ('virtual_chassis', 'vc_position'), + ) permissions = ( ('napalm_read', 'Read-only access to devices via NAPALM'), ('napalm_write', 'Read/write access to devices via NAPALM'), diff --git a/netbox/dcim/models/device_component_templates.py b/netbox/dcim/models/device_component_templates.py index 2aa46d0ea7..ab4a078cf2 100644 --- a/netbox/dcim/models/device_component_templates.py +++ b/netbox/dcim/models/device_component_templates.py @@ -4,9 +4,9 @@ from dcim.choices import * from dcim.constants import * -from dcim.managers import InterfaceManager from extras.models import ObjectChange -from utilities.managers import NaturalOrderingManager +from utilities.fields import NaturalOrderingField +from utilities.ordering import naturalize_interface from utilities.utils import serialize_object from .device_components import ( ConsolePort, ConsoleServerPort, DeviceBay, FrontPort, Interface, PowerOutlet, PowerPort, RearPort, @@ -58,17 +58,20 @@ class ConsolePortTemplate(ComponentTemplateModel): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=ConsolePortTypeChoices, blank=True ) - objects = NaturalOrderingManager() - class Meta: - ordering = ['device_type', 'name'] - unique_together = ['device_type', 'name'] + ordering = ('device_type', '_name') + unique_together = ('device_type', 'name') def __str__(self): return self.name @@ -93,17 +96,20 @@ class ConsoleServerPortTemplate(ComponentTemplateModel): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=ConsolePortTypeChoices, blank=True ) - objects = NaturalOrderingManager() - class Meta: - ordering = ['device_type', 'name'] - unique_together = ['device_type', 'name'] + ordering = ('device_type', '_name') + unique_together = ('device_type', 'name') def __str__(self): return self.name @@ -128,6 +134,11 @@ class PowerPortTemplate(ComponentTemplateModel): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=PowerPortTypeChoices, @@ -146,11 +157,9 @@ class PowerPortTemplate(ComponentTemplateModel): help_text="Allocated power draw (watts)" ) - objects = NaturalOrderingManager() - class Meta: - ordering = ['device_type', 'name'] - unique_together = ['device_type', 'name'] + ordering = ('device_type', '_name') + unique_together = ('device_type', 'name') def __str__(self): return self.name @@ -176,6 +185,11 @@ class PowerOutletTemplate(ComponentTemplateModel): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=PowerOutletTypeChoices, @@ -195,11 +209,9 @@ class PowerOutletTemplate(ComponentTemplateModel): help_text="Phase (for three-phase feeds)" ) - objects = NaturalOrderingManager() - class Meta: - ordering = ['device_type', 'name'] - unique_together = ['device_type', 'name'] + ordering = ('device_type', '_name') + unique_together = ('device_type', 'name') def __str__(self): return self.name @@ -237,6 +249,12 @@ class InterfaceTemplate(ComponentTemplateModel): name = models.CharField( max_length=64 ) + _name = NaturalOrderingField( + target_field='name', + naturalize_function=naturalize_interface, + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=InterfaceTypeChoices @@ -246,11 +264,9 @@ class InterfaceTemplate(ComponentTemplateModel): verbose_name='Management only' ) - objects = InterfaceManager() - class Meta: - ordering = ['device_type', 'name'] - unique_together = ['device_type', 'name'] + ordering = ('device_type', '_name') + unique_together = ('device_type', 'name') def __str__(self): return self.name @@ -276,6 +292,11 @@ class FrontPortTemplate(ComponentTemplateModel): name = models.CharField( max_length=64 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=PortTypeChoices @@ -290,14 +311,12 @@ class FrontPortTemplate(ComponentTemplateModel): validators=[MinValueValidator(1), MaxValueValidator(64)] ) - objects = NaturalOrderingManager() - class Meta: - ordering = ['device_type', 'name'] - unique_together = [ - ['device_type', 'name'], - ['rear_port', 'rear_port_position'], - ] + ordering = ('device_type', '_name') + unique_together = ( + ('device_type', 'name'), + ('rear_port', 'rear_port_position'), + ) def __str__(self): return self.name @@ -344,6 +363,11 @@ class RearPortTemplate(ComponentTemplateModel): name = models.CharField( max_length=64 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=PortTypeChoices @@ -353,11 +377,9 @@ class RearPortTemplate(ComponentTemplateModel): validators=[MinValueValidator(1), MaxValueValidator(64)] ) - objects = NaturalOrderingManager() - class Meta: - ordering = ['device_type', 'name'] - unique_together = ['device_type', 'name'] + ordering = ('device_type', '_name') + unique_together = ('device_type', 'name') def __str__(self): return self.name @@ -383,12 +405,15 @@ class DeviceBayTemplate(ComponentTemplateModel): name = models.CharField( max_length=50 ) - - objects = NaturalOrderingManager() + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) class Meta: - ordering = ['device_type', 'name'] - unique_together = ['device_type', 'name'] + ordering = ('device_type', '_name') + unique_together = ('device_type', 'name') def __str__(self): return self.name diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index 431419a7a3..a41eda5761 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -10,9 +10,9 @@ from dcim.constants import * from dcim.exceptions import LoopDetected from dcim.fields import MACAddressField -from dcim.managers import InterfaceManager from extras.models import ObjectChange, TaggedItem -from utilities.managers import NaturalOrderingManager +from utilities.fields import NaturalOrderingField +from utilities.ordering import naturalize_interface from utilities.utils import serialize_object from virtualization.choices import VMInterfaceTypeChoices @@ -181,6 +181,11 @@ class ConsolePort(CableTermination, ComponentModel): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=ConsolePortTypeChoices, @@ -197,15 +202,13 @@ class ConsolePort(CableTermination, ComponentModel): choices=CONNECTION_STATUS_CHOICES, blank=True ) - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = ['device', 'name', 'type', 'description'] class Meta: - ordering = ['device', 'name'] - unique_together = ['device', 'name'] + ordering = ('device', '_name') + unique_together = ('device', 'name') def __str__(self): return self.name @@ -238,6 +241,11 @@ class ConsoleServerPort(CableTermination, ComponentModel): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=ConsolePortTypeChoices, @@ -247,14 +255,13 @@ class ConsoleServerPort(CableTermination, ComponentModel): choices=CONNECTION_STATUS_CHOICES, blank=True ) - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = ['device', 'name', 'type', 'description'] class Meta: - unique_together = ['device', 'name'] + ordering = ('device', '_name') + unique_together = ('device', 'name') def __str__(self): return self.name @@ -287,6 +294,11 @@ class PowerPort(CableTermination, ComponentModel): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=PowerPortTypeChoices, @@ -322,15 +334,13 @@ class PowerPort(CableTermination, ComponentModel): choices=CONNECTION_STATUS_CHOICES, blank=True ) - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = ['device', 'name', 'type', 'maximum_draw', 'allocated_draw', 'description'] class Meta: - ordering = ['device', 'name'] - unique_together = ['device', 'name'] + ordering = ('device', '_name') + unique_together = ('device', 'name') def __str__(self): return self.name @@ -433,6 +443,11 @@ class PowerOutlet(CableTermination, ComponentModel): name = models.CharField( max_length=50 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=PowerOutletTypeChoices, @@ -455,14 +470,13 @@ class PowerOutlet(CableTermination, ComponentModel): choices=CONNECTION_STATUS_CHOICES, blank=True ) - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = ['device', 'name', 'type', 'power_port', 'feed_leg', 'description'] class Meta: - unique_together = ['device', 'name'] + ordering = ('device', '_name') + unique_together = ('device', 'name') def __str__(self): return self.name @@ -515,6 +529,12 @@ class Interface(CableTermination, ComponentModel): name = models.CharField( max_length=64 ) + _name = NaturalOrderingField( + target_field='name', + naturalize_function=naturalize_interface, + max_length=100, + blank=True + ) _connected_interface = models.OneToOneField( to='self', on_delete=models.SET_NULL, @@ -583,8 +603,6 @@ class Interface(CableTermination, ComponentModel): blank=True, verbose_name='Tagged VLANs' ) - - objects = InterfaceManager() tags = TaggableManager(through=TaggedItem) csv_headers = [ @@ -593,8 +611,9 @@ class Interface(CableTermination, ComponentModel): ] class Meta: - ordering = ['device', 'name'] - unique_together = ['device', 'name'] + # TODO: ordering and unique_together should include virtual_machine + ordering = ('device', '_name') + unique_together = ('device', 'name') def __str__(self): return self.name @@ -761,6 +780,11 @@ class FrontPort(CableTermination, ComponentModel): name = models.CharField( max_length=64 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=PortTypeChoices @@ -774,20 +798,17 @@ class FrontPort(CableTermination, ComponentModel): default=1, validators=[MinValueValidator(1), MaxValueValidator(64)] ) - - is_path_endpoint = False - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = ['device', 'name', 'type', 'rear_port', 'rear_port_position', 'description'] + is_path_endpoint = False class Meta: - ordering = ['device', 'name'] - unique_together = [ - ['device', 'name'], - ['rear_port', 'rear_port_position'], - ] + ordering = ('device', '_name') + unique_together = ( + ('device', 'name'), + ('rear_port', 'rear_port_position'), + ) def __str__(self): return self.name @@ -831,6 +852,11 @@ class RearPort(CableTermination, ComponentModel): name = models.CharField( max_length=64 ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=PortTypeChoices @@ -839,17 +865,14 @@ class RearPort(CableTermination, ComponentModel): default=1, validators=[MinValueValidator(1), MaxValueValidator(64)] ) - - is_path_endpoint = False - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = ['device', 'name', 'type', 'positions', 'description'] + is_path_endpoint = False class Meta: - ordering = ['device', 'name'] - unique_together = ['device', 'name'] + ordering = ('device', '_name') + unique_together = ('device', 'name') def __str__(self): return self.name @@ -881,6 +904,11 @@ class DeviceBay(ComponentModel): max_length=50, verbose_name='Name' ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) installed_device = models.OneToOneField( to='dcim.Device', on_delete=models.SET_NULL, @@ -888,15 +916,13 @@ class DeviceBay(ComponentModel): blank=True, null=True ) - - objects = NaturalOrderingManager() tags = TaggableManager(through=TaggedItem) csv_headers = ['device', 'name', 'installed_device', 'description'] class Meta: - ordering = ['device', 'name'] - unique_together = ['device', 'name'] + ordering = ('device', '_name') + unique_together = ('device', 'name') def __str__(self): return '{} - {}'.format(self.device.name, self.name) @@ -960,6 +986,11 @@ class InventoryItem(ComponentModel): max_length=50, verbose_name='Name' ) + _name = NaturalOrderingField( + target_field='name', + max_length=100, + blank=True + ) manufacturer = models.ForeignKey( to='dcim.Manufacturer', on_delete=models.PROTECT, @@ -997,8 +1028,8 @@ class InventoryItem(ComponentModel): ] class Meta: - ordering = ['device__id', 'parent__id', 'name'] - unique_together = ['device', 'parent', 'name'] + ordering = ('device__id', 'parent__id', '_name') + unique_together = ('device', 'parent', 'name') def __str__(self): return self.name diff --git a/netbox/dcim/tables.py b/netbox/dcim/tables.py index 1b3076c6cc..473d465bd0 100644 --- a/netbox/dcim/tables.py +++ b/netbox/dcim/tables.py @@ -229,7 +229,7 @@ class Meta(BaseTable.Meta): class SiteTable(BaseTable): pk = ToggleColumn() - name = tables.LinkColumn(order_by=('_nat1', '_nat2', '_nat3')) + name = tables.LinkColumn(order_by=('_name',)) status = tables.TemplateColumn(template_code=STATUS_LABEL, verbose_name='Status') region = tables.TemplateColumn(template_code=SITE_REGION_LINK) tenant = tables.TemplateColumn(template_code=COL_TENANT) @@ -291,7 +291,7 @@ class Meta(BaseTable.Meta): class RackTable(BaseTable): pk = ToggleColumn() - name = tables.LinkColumn(order_by=('_nat1', '_nat2', '_nat3')) + name = tables.LinkColumn(order_by=('_name',)) site = tables.LinkColumn('dcim:site', args=[Accessor('site.slug')]) group = tables.Column(accessor=Accessor('group.name'), verbose_name='Group') tenant = tables.TemplateColumn(template_code=COL_TENANT) @@ -409,6 +409,7 @@ class Meta(BaseTable.Meta): class ConsolePortTemplateTable(BaseTable): pk = ToggleColumn() + name = tables.Column(order_by=('_name',)) actions = tables.TemplateColumn( template_code=get_component_template_actions('consoleporttemplate'), attrs={'td': {'class': 'text-right noprint'}}, @@ -432,6 +433,7 @@ class Meta(BaseTable.Meta): class ConsoleServerPortTemplateTable(BaseTable): pk = ToggleColumn() + name = tables.Column(order_by=('_name',)) actions = tables.TemplateColumn( template_code=get_component_template_actions('consoleserverporttemplate'), attrs={'td': {'class': 'text-right noprint'}}, @@ -455,6 +457,7 @@ class Meta(BaseTable.Meta): class PowerPortTemplateTable(BaseTable): pk = ToggleColumn() + name = tables.Column(order_by=('_name',)) actions = tables.TemplateColumn( template_code=get_component_template_actions('powerporttemplate'), attrs={'td': {'class': 'text-right noprint'}}, @@ -478,6 +481,7 @@ class Meta(BaseTable.Meta): class PowerOutletTemplateTable(BaseTable): pk = ToggleColumn() + name = tables.Column(order_by=('_name',)) actions = tables.TemplateColumn( template_code=get_component_template_actions('poweroutlettemplate'), attrs={'td': {'class': 'text-right noprint'}}, @@ -526,6 +530,7 @@ class Meta(BaseTable.Meta): class FrontPortTemplateTable(BaseTable): pk = ToggleColumn() + name = tables.Column(order_by=('_name',)) rear_port_position = tables.Column( verbose_name='Position' ) @@ -552,6 +557,7 @@ class Meta(BaseTable.Meta): class RearPortTemplateTable(BaseTable): pk = ToggleColumn() + name = tables.Column(order_by=('_name',)) actions = tables.TemplateColumn( template_code=get_component_template_actions('rearporttemplate'), attrs={'td': {'class': 'text-right noprint'}}, @@ -575,6 +581,7 @@ class Meta(BaseTable.Meta): class DeviceBayTemplateTable(BaseTable): pk = ToggleColumn() + name = tables.Column(order_by=('_name',)) actions = tables.TemplateColumn( template_code=get_component_template_actions('devicebaytemplate'), attrs={'td': {'class': 'text-right noprint'}}, @@ -654,7 +661,7 @@ class Meta(BaseTable.Meta): class DeviceTable(BaseTable): pk = ToggleColumn() name = tables.TemplateColumn( - order_by=('_nat1', '_nat2', '_nat3'), + order_by=('_name',), template_code=DEVICE_LINK ) status = tables.TemplateColumn(template_code=STATUS_LABEL, verbose_name='Status') @@ -704,6 +711,7 @@ class Meta(BaseTable.Meta): class DeviceComponentDetailTable(BaseTable): pk = ToggleColumn() + name = tables.Column(order_by=('_name',)) cable = tables.LinkColumn() class Meta(BaseTable.Meta): @@ -713,6 +721,7 @@ class Meta(BaseTable.Meta): class ConsolePortTable(BaseTable): + name = tables.Column(order_by=('_name',)) class Meta(BaseTable.Meta): model = ConsolePort @@ -727,6 +736,7 @@ class Meta(DeviceComponentDetailTable.Meta, ConsolePortTable.Meta): class ConsoleServerPortTable(BaseTable): + name = tables.Column(order_by=('_name',)) class Meta(BaseTable.Meta): model = ConsoleServerPort @@ -741,6 +751,7 @@ class Meta(DeviceComponentDetailTable.Meta, ConsoleServerPortTable.Meta): class PowerPortTable(BaseTable): + name = tables.Column(order_by=('_name',)) class Meta(BaseTable.Meta): model = PowerPort @@ -755,6 +766,7 @@ class Meta(DeviceComponentDetailTable.Meta, PowerPortTable.Meta): class PowerOutletTable(BaseTable): + name = tables.Column(order_by=('_name',)) class Meta(BaseTable.Meta): model = PowerOutlet @@ -786,6 +798,7 @@ class Meta(InterfaceTable.Meta): class FrontPortTable(BaseTable): + name = tables.Column(order_by=('_name',)) class Meta(BaseTable.Meta): model = FrontPort @@ -801,6 +814,7 @@ class Meta(DeviceComponentDetailTable.Meta, FrontPortTable.Meta): class RearPortTable(BaseTable): + name = tables.Column(order_by=('_name',)) class Meta(BaseTable.Meta): model = RearPort @@ -816,6 +830,7 @@ class Meta(DeviceComponentDetailTable.Meta, RearPortTable.Meta): class DeviceBayTable(BaseTable): + name = tables.Column(order_by=('_name',)) class Meta(BaseTable.Meta): model = DeviceBay diff --git a/netbox/utilities/fields.py b/netbox/utilities/fields.py index d2165aea65..6181a7ca19 100644 --- a/netbox/utilities/fields.py +++ b/netbox/utilities/fields.py @@ -1,6 +1,7 @@ from django.core.validators import RegexValidator from django.db import models +from utilities.ordering import naturalize from .forms import ColorSelect ColorValidator = RegexValidator( @@ -35,3 +36,35 @@ def __init__(self, *args, **kwargs): def formfield(self, **kwargs): kwargs['widget'] = ColorSelect return super().formfield(**kwargs) + + +class NaturalOrderingField(models.CharField): + """ + A field which stores a naturalized representation of its target field, to be used for ordering its parent model. + + :param target_field: Name of the field of the parent model to be naturalized + :param naturalize_function: The function used to generate a naturalized value (optional) + """ + description = "Stores a representation of its target field suitable for natural ordering" + + def __init__(self, target_field, naturalize_function=naturalize, *args, **kwargs): + self.target_field = target_field + self.naturalize_function = naturalize_function + super().__init__(*args, **kwargs) + + def pre_save(self, model_instance, add): + """ + Generate a naturalized value from the target field + """ + value = getattr(model_instance, self.target_field) + return self.naturalize_function(value, max_length=self.max_length) + + def deconstruct(self): + kwargs = super().deconstruct()[3] # Pass kwargs from CharField + kwargs['naturalize_function'] = self.naturalize_function + return ( + self.name, + 'utilities.fields.NaturalOrderingField', + ['target_field'], + kwargs, + ) diff --git a/netbox/utilities/managers.py b/netbox/utilities/managers.py deleted file mode 100644 index ad646a78e5..0000000000 --- a/netbox/utilities/managers.py +++ /dev/null @@ -1,45 +0,0 @@ -from django.db.models import Manager -from django.db.models.expressions import RawSQL - -NAT1 = r"CAST(SUBSTRING({}.{} FROM '^(\d{{1,9}})') AS integer)" -NAT2 = r"SUBSTRING({}.{} FROM '^\d*(.*?)\d*$')" -NAT3 = r"CAST(SUBSTRING({}.{} FROM '(\d{{1,9}})$') AS integer)" - - -class NaturalOrderingManager(Manager): - """ - Order objects naturally by a designated field (defaults to 'name'). Leading and/or trailing digits of values within - this field will be cast as independent integers and sorted accordingly. For example, "Foo2" will be ordered before - "Foo10", even though the digit 1 is normally ordered before the digit 2. - """ - natural_order_field = 'name' - - def get_queryset(self): - - queryset = super().get_queryset() - - db_table = self.model._meta.db_table - db_field = self.natural_order_field - - # Append the three subfields derived from the designated natural ordering field - queryset = ( - queryset.annotate(_nat1=RawSQL(NAT1.format(db_table, db_field), ())) - .annotate(_nat2=RawSQL(NAT2.format(db_table, db_field), ())) - .annotate(_nat3=RawSQL(NAT3.format(db_table, db_field), ())) - ) - - # Replace any instance of the designated natural ordering field with its three subfields - ordering = [] - for field in self.model._meta.ordering: - if field == self.natural_order_field: - ordering.append('_nat1') - ordering.append('_nat2') - ordering.append('_nat3') - else: - ordering.append(field) - - # Default to using the _nat indexes if Meta.ordering is empty - if not ordering: - ordering = ('_nat1', '_nat2', '_nat3') - - return queryset.order_by(*ordering) diff --git a/netbox/utilities/ordering.py b/netbox/utilities/ordering.py new file mode 100644 index 0000000000..d459e6f6c2 --- /dev/null +++ b/netbox/utilities/ordering.py @@ -0,0 +1,80 @@ +import re + +INTERFACE_NAME_REGEX = r'(^(?P[^\d\.:]+)?)' \ + r'((?P\d+)/)?' \ + r'((?P\d+)/)?' \ + r'((?P\d+)/)?' \ + r'((?P\d+)/)?' \ + r'((?P\d+))?' \ + r'(:(?P\d+))?' \ + r'(.(?P\d+)$)?' + + +def naturalize(value, max_length=None, integer_places=8): + """ + Take an alphanumeric string and prepend all integers to `integer_places` places to ensure the strings + are ordered naturally. For example: + + site9router21 + site10router4 + site10router19 + + becomes: + + site00000009router00000021 + site00000010router00000004 + site00000010router00000019 + + :param value: The value to be naturalized + :param max_length: The maximum length of the returned string. Characters beyond this length will be stripped. + :param integer_places: The number of places to which each integer will be expanded. (Default: 8) + """ + if not value: + return value + output = [] + for segment in re.split(r'(\d+)', value): + if segment.isdigit(): + output.append(segment.rjust(integer_places, '0')) + elif segment: + output.append(segment) + ret = ''.join(output) + + return ret[:max_length] if max_length else ret + + +def naturalize_interface(value, max_length=None): + """ + Similar in nature to naturalize(), but takes into account a particular naming format adapted from the old + InterfaceManager. + + :param value: The value to be naturalized + :param max_length: The maximum length of the returned string. Characters beyond this length will be stripped. + """ + output = [] + match = re.search(INTERFACE_NAME_REGEX, value) + if match is None: + return value + + # First, we order by slot/position, padding each to four digits. If a field is not present, + # set it to 9999 to ensure it is ordered last. + for part_name in ('slot', 'subslot', 'position', 'subposition'): + part = match.group(part_name) + if part is not None: + output.append(part.rjust(4, '0')) + else: + output.append('9999') + + # Append the type, if any. + if match.group('type') is not None: + output.append(match.group('type')) + + # Finally, append any remaining fields, left-padding to eight digits each. + for part_name in ('id', 'channel', 'vc'): + part = match.group(part_name) + if part is not None: + output.append(part.rjust(6, '0')) + else: + output.append('000000') + + ret = ''.join(output) + return ret[:max_length] if max_length else ret