Merge pull request #3885 from hSaria/568-csv-import-cf

Fixes #568: CSV import/export of custom fields

Author: jeremystretch

docs/release-notes/version-2.7.md

@@ -10,6 +10,7 @@
 
 ## Enhancements
 
+* [#568](https://github.com/netbox-community/netbox/issues/568) - Allow custom fields to be imported and exported using CSV
 * [#3310](https://github.com/netbox-community/netbox/issues/3310) - Pre-select site/rack for B side when creating a new cable
 * [#3338](https://github.com/netbox-community/netbox/issues/3338) - Include circuit terminations in API representation of circuits
 * [#3509](https://github.com/netbox-community/netbox/issues/3509) - Add IP address variables for custom scripts

netbox/circuits/forms.py

@@ -46,7 +46,7 @@ class Meta:
         }
 
 
-class ProviderCSVForm(forms.ModelForm):
+class ProviderCSVForm(CustomFieldForm):
     slug = SlugField()
 
     class Meta:
@@ -188,7 +188,7 @@ class Meta:
         }
 
 
-class CircuitCSVForm(forms.ModelForm):
+class CircuitCSVForm(CustomFieldForm):
     provider = forms.ModelChoiceField(
         queryset=Provider.objects.all(),
         to_field_name='name',

netbox/dcim/forms.py

@@ -263,7 +263,7 @@ class Meta:
         }
 
 
-class SiteCSVForm(forms.ModelForm):
+class SiteCSVForm(CustomFieldForm):
     status = CSVChoiceField(
         choices=SiteStatusChoices,
         required=False,
@@ -504,7 +504,7 @@ class Meta:
         }
 
 
-class RackCSVForm(forms.ModelForm):
+class RackCSVForm(CustomFieldForm):
     site = forms.ModelChoiceField(
         queryset=Site.objects.all(),
         to_field_name='name',
@@ -1724,7 +1724,7 @@ def __init__(self, *args, **kwargs):
             self.initial['rack'] = self.instance.parent_bay.device.rack_id
 
 
-class BaseDeviceCSVForm(forms.ModelForm):
+class BaseDeviceCSVForm(CustomFieldForm):
     device_role = forms.ModelChoiceField(
         queryset=DeviceRole.objects.all(),
         to_field_name='name',
@@ -4286,7 +4286,7 @@ def __init__(self, *args, **kwargs):
             self.initial['site'] = self.instance.power_panel.site
 
 
-class PowerFeedCSVForm(forms.ModelForm):
+class PowerFeedCSVForm(CustomFieldForm):
     site = forms.ModelChoiceField(
         queryset=Site.objects.all(),
         to_field_name='name',

netbox/extras/forms.py

@@ -10,8 +10,8 @@
 from tenancy.models import Tenant, TenantGroup
 from utilities.forms import (
     add_blank_choice, APISelectMultiple, BootstrapMixin, BulkEditForm, BulkEditNullBooleanSelect, ColorSelect,
-    CommentField, ContentTypeSelect, DatePicker, DateTimePicker, FilterChoiceField, LaxURLField, JSONField,
-    SlugField, StaticSelect2, BOOLEAN_WITH_BLANK_CHOICES,
+    CustomFieldChoiceField, CommentField, ContentTypeSelect, DatePicker, DateTimePicker, FilterChoiceField,
+    LaxURLField, JSONField, SlugField, StaticSelect2, BOOLEAN_WITH_BLANK_CHOICES,
 )
 from .choices import *
 from .models import ConfigContext, CustomField, CustomFieldValue, ImageAttachment, ObjectChange, Tag
@@ -61,7 +61,7 @@ def get_custom_fields_for_model(content_type, filterable_only=False, bulk_edit=F
 
         # Select
         elif cf.type == CustomFieldTypeChoices.TYPE_SELECT:
-            choices = [(cfc.pk, cfc) for cfc in cf.choices.all()]
+            choices = [(cfc.pk, cfc.value) for cfc in cf.choices.all()]
             if not cf.required or bulk_edit or filterable_only:
                 choices = [(None, '---------')] + choices
             # Check for a default choice
@@ -71,7 +71,7 @@ def get_custom_fields_for_model(content_type, filterable_only=False, bulk_edit=F
                     default_choice = cf.choices.get(value=initial).pk
                 except ObjectDoesNotExist:
                     pass
-            field = forms.TypedChoiceField(
+            field = CustomFieldChoiceField(
                 choices=choices, coerce=int, required=cf.required, initial=default_choice, widget=StaticSelect2()
             )
 

netbox/extras/tests/test_customfields.py

@@ -1,14 +1,14 @@
 from datetime import date
 
 from django.contrib.contenttypes.models import ContentType
-from django.test import TestCase
+from django.test import Client, TestCase
 from django.urls import reverse
 from rest_framework import status
 
 from dcim.models import Site
 from extras.choices import *
 from extras.models import CustomField, CustomFieldValue, CustomFieldChoice
-from utilities.testing import APITestCase
+from utilities.testing import APITestCase, create_test_user
 from virtualization.models import VirtualMachine
 
 
@@ -364,3 +364,62 @@ def test_list_cfc(self):
         self.assertEqual(self.cf_choice_1.pk, response.data[self.cf_1.name][self.cf_choice_1.value])
         self.assertEqual(self.cf_choice_2.pk, response.data[self.cf_1.name][self.cf_choice_2.value])
         self.assertEqual(self.cf_choice_3.pk, response.data[self.cf_2.name][self.cf_choice_3.value])
+
+
+class CustomFieldCSV(TestCase):
+    def setUp(self):
+        super().setUp()
+
+        user = create_test_user(
+            permissions=[
+                'dcim.view_site',
+                'dcim.add_site',
+            ]
+        )
+        self.client = Client()
+        self.client.force_login(user)
+
+        obj_type = ContentType.objects.get_for_model(Site)
+
+        self.cf_text = CustomField.objects.create(name="text", type=CustomFieldTypeChoices.TYPE_TEXT)
+        self.cf_text.obj_type.set([obj_type])
+        self.cf_text.save()
+
+        self.cf_choice = CustomField.objects.create(name="choice", type=CustomFieldTypeChoices.TYPE_SELECT)
+        self.cf_choice.obj_type.set([obj_type])
+        self.cf_choice.save()
+
+        self.cf_choice_1 = CustomFieldChoice.objects.create(field=self.cf_choice, value="cf_field_1")
+        self.cf_choice_2 = CustomFieldChoice.objects.create(field=self.cf_choice, value="cf_field_2")
+        self.cf_choice_3 = CustomFieldChoice.objects.create(field=self.cf_choice, value="cf_field_3")
+
+    def test_import(self):
+        """
+        Import a site with custom fields
+        """
+        csv_data = (
+            "name,slug,cf_text,cf_choice",
+            "Site 1,site-1,something,cf_field_1",
+        )
+
+        response = self.client.post(reverse('dcim:site_import'), {'csv': '\n'.join(csv_data)})
+        self.assertEqual(response.status_code, 200)
+
+        site1_custom_fields = Site.objects.get(name='Site 1').get_custom_fields()
+        self.assertEqual(len(site1_custom_fields), 2)
+        self.assertEqual(site1_custom_fields[self.cf_text], 'something')
+        self.assertEqual(site1_custom_fields[self.cf_choice], self.cf_choice_1)
+
+    def test_import_invalid_choice(self):
+        """
+        Import a site with an invalid choice
+        """
+        csv_data = (
+            "name,slug,cf_choice",
+            "Site 2,site-2,cf_field_4",
+        )
+
+        response = self.client.post(reverse('dcim:site_import'), {'csv': '\n'.join(csv_data)})
+        self.assertEqual(response.status_code, 200)
+
+        self.assertFalse(len(Site.objects.filter(name="Site 2")), 0)

netbox/ipam/forms.py

@@ -49,7 +49,7 @@ class Meta:
         }
 
 
-class VRFCSVForm(forms.ModelForm):
+class VRFCSVForm(CustomFieldForm):
     tenant = forms.ModelChoiceField(
         queryset=Tenant.objects.all(),
         required=False,
@@ -166,7 +166,7 @@ class Meta:
         }
 
 
-class AggregateCSVForm(forms.ModelForm):
+class AggregateCSVForm(CustomFieldForm):
     rir = forms.ModelChoiceField(
         queryset=RIR.objects.all(),
         to_field_name='name',
@@ -341,7 +341,7 @@ def __init__(self, *args, **kwargs):
         self.fields['vrf'].empty_label = 'Global'
 
 
-class PrefixCSVForm(forms.ModelForm):
+class PrefixCSVForm(CustomFieldForm):
     vrf = FlexibleModelChoiceField(
         queryset=VRF.objects.all(),
         to_field_name='rd',
@@ -771,7 +771,7 @@ def __init__(self, *args, **kwargs):
         self.fields['vrf'].empty_label = 'Global'
 
 
-class IPAddressCSVForm(forms.ModelForm):
+class IPAddressCSVForm(CustomFieldForm):
     vrf = FlexibleModelChoiceField(
         queryset=VRF.objects.all(),
         to_field_name='rd',
@@ -1135,7 +1135,7 @@ class Meta:
         }
 
 
-class VLANCSVForm(forms.ModelForm):
+class VLANCSVForm(CustomFieldForm):
     site = forms.ModelChoiceField(
         queryset=Site.objects.all(),
         required=False,

netbox/secrets/forms.py

@@ -116,7 +116,7 @@ def clean(self):
             })
 
 
-class SecretCSVForm(forms.ModelForm):
+class SecretCSVForm(CustomFieldForm):
     device = FlexibleModelChoiceField(
         queryset=Device.objects.all(),
         to_field_name='name',

netbox/tenancy/forms.py

@@ -57,7 +57,7 @@ class Meta:
         }
 
 
-class TenantCSVForm(forms.ModelForm):
+class TenantCSVForm(CustomFieldForm):
     slug = SlugField()
     group = forms.ModelChoiceField(
         queryset=TenantGroup.objects.all(),

netbox/utilities/forms.py

@@ -442,6 +442,23 @@ def clean(self, value):
         return self.choice_values[value]
 
 
+class CustomFieldChoiceField(forms.TypedChoiceField):
+    """
+    Accept human-friendly label as input, and return the database value. If the label is not matched, the normal,
+    value-based input is assumed.
+    """
+
+    def __init__(self, choices, *args, **kwargs):
+        super().__init__(choices=choices, *args, **kwargs)
+        self.choice_values = {label: value for value, label in unpack_grouped_choices(choices)}
+
+    def clean(self, value):
+        # Check if the value is actually a label
+        if value in self.choice_values:
+            return self.choice_values[value]
+        return super().clean(value)
+
+
 class ExpandableNameField(forms.CharField):
     """
     A field which allows for numeric range expansion

netbox/utilities/views.py

@@ -88,15 +88,27 @@ def queryset_to_csv(self):
         Export the queryset of objects as comma-separated value (CSV), using the model's to_csv() method.
         """
         csv_data = []
+        custom_fields = []
 
         # Start with the column headers
-        headers = ','.join(self.queryset.model.csv_headers)
-        csv_data.append(headers)
+        headers = self.queryset.model.csv_headers.copy()
+
+        # Add custom field headers, if any
+        if hasattr(self.queryset.model, 'get_custom_fields'):
+            for custom_field in self.queryset.model().get_custom_fields():
+                headers.append(custom_field.name)
+                custom_fields.append(custom_field.name)
+
+        csv_data.append(','.join(headers))
 
         # Iterate through the queryset appending each object
         for obj in self.queryset:
-            data = csv_format(obj.to_csv())
-            csv_data.append(data)
+            data = obj.to_csv()
+
+            for custom_field in custom_fields:
+                data += (obj.cf.get(custom_field, ''),)
+
+            csv_data.append(csv_format(data))
 
         return '\n'.join(csv_data)
 

netbox/virtualization/forms.py

@@ -98,7 +98,7 @@ class Meta:
         }
 
 
-class ClusterCSVForm(forms.ModelForm):
+class ClusterCSVForm(CustomFieldForm):
     type = forms.ModelChoiceField(
         queryset=ClusterType.objects.all(),
         to_field_name='name',
@@ -430,7 +430,7 @@ def __init__(self, *args, **kwargs):
             self.fields['primary_ip6'].widget.attrs['readonly'] = True
 
 
-class VirtualMachineCSVForm(forms.ModelForm):
+class VirtualMachineCSVForm(CustomFieldForm):
     status = CSVChoiceField(
         choices=VirtualMachineStatusChoices,
         required=False,