Inventory: Start setting up integration tests (netbox-community#56)

Author: DouglasHeriot

.travis.yml

@@ -27,15 +27,16 @@ jobs:
         - docker-compose pull
         - docker-compose up -d
         - cd ..
-        - pip install -U pip
+        # setuptools <46.1.3 has issues with ansible-test executables https://github.com/ansible/ansible/issues/68963
+        - pip install -U pip "setuptools>=46.1.3"
         # coverage 5.0 is not compatible with ansible-test https://github.com/ansible/ansible/issues/65907
         - pip install pytest==4.6.5 pytest-mock pytest-xdist jinja2 PyYAML black==19.10b0 "coverage<5"
-        - pip install pynetbox cryptography codecov jmespath ansible
+        - pip install pynetbox cryptography codecov jmespath jsondiff pycodestyle ansible
 
-    # Latest release of Netbox and Ansible
     - name: "Python 3.6 - Netbox 2.7 - Latest PyPi Ansible"
       python: 3.6
-      env: PYTHON_VER=3.6 VERSION=v2.7
+      env:
+      - PYTHON_VER=3.6 VERSION=v2.7
       install:
         - cd ..
         # Setup netbox container for integration testing
@@ -44,16 +45,32 @@ jobs:
         - docker-compose pull
         - docker-compose up -d
         - cd ..
-        - pip install -U pip
+        - pip install -U pip "setuptools>=46.1.3"
         - pip install pytest==4.6.5 pytest-mock pytest-xdist jinja2 PyYAML black==19.10b0 "coverage<5"
-        - pip install pynetbox cryptography codecov jmespath ansible
+        - pip install pynetbox cryptography codecov jmespath jsondiff pycodestyle ansible
+
+    - name: "Python 3.8 - Netbox 2.8 - Latest PyPi Ansible"
+      python: 3.8
+      env:
+      - PYTHON_VER=3.8 VERSION=v2.8
+      install:
+        - cd ..
+        # Setup netbox container for integration testing
+        - git clone https://github.com/netbox-community/netbox-docker.git
+        - cd netbox-docker
+        - docker-compose pull
+        - docker-compose up -d
+        - cd ..
+        - pip install -U pip "setuptools>=46.1.3"
+        - pip install pytest==4.6.5 pytest-mock pytest-xdist jinja2 PyYAML black==19.10b0 "coverage<5"
+        - pip install pynetbox cryptography codecov jmespath jsondiff pycodestyle ansible
 
     # Latest development versions of Netbox and Ansible, newest Python
     # This may be broken sometimes by changes in the netbox & ansible projects
     # Failures will be allowed in this build
-    - name: "Python 3.8 - Netbox develop-2.8 - Ansible Devel"
+    - name: "Python 3.8 - Netbox develop branch (snapshot) - Ansible Devel"
       python: 3.8
-      env: PYTHON_VER=3.8 VERSION=develop-2.8
+      env: PYTHON_VER=3.8 VERSION=snapshot
       install:
         - cd ..
         # Setup netbox container for integration testing
@@ -62,16 +79,16 @@ jobs:
         - docker-compose pull
         - docker-compose up -d
         - cd ..
-        - pip install -U pip
+        - pip install -U pip "setuptools>=46.1.3"
         - pip install pytest==4.6.5 pytest-mock pytest-xdist jinja2 PyYAML black==19.10b0 "coverage<5"
-        - pip install pynetbox cryptography jmespath
+        - pip install pynetbox cryptography jmespath jsondiff pycodestyle
         - git clone https://github.com/ansible/ansible.git
         - cd ansible
         - source hacking/env-setup
         - cd  ..
   allow_failures:
     # When testing against dev netbox and dev ansible, allow failures
-    - env: PYTHON_VER=3.8 VERSION=develop-2.8
+    - env: PYTHON_VER=3.8 VERSION=snapshot
 
 before_script:
   - mkdir -p ~/ansible_collections/$COLLECTION_NAMESPACE
@@ -80,28 +97,44 @@ before_script:
   - ansible-galaxy collection build .
   - ansible-galaxy collection install $COLLECTION_NAMESPACE-$COLLECTION_NAME-$COLLECTION_VERSION.tar.gz -p /home/travis/.ansible/collections
 
+  # Run all further tests from within the installed directory
+  # Required to resolve imports of other collections
+  - cd /home/travis/.ansible/collections/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME
+
+  # Set runme.sh execute permissions stripped by ansible-galaxy. Should be fixed in Ansible 2.10
+  # https://github.com/ansible/ansible/issues/68415
+  - chmod +x tests/integration/targets/inventory/runme.sh
+  - chmod +x tests/integration/targets/inventory/compare_inventory_json.py
+
 script:
-  # Perform unit tests on collection from within the installed directory, not the source directory
-  # Required for imports of other collections (ie. ansible.netcommon) to work correctly
-  - (cd /home/travis/.ansible/collections/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME && ansible-test units --coverage --python $PYTHON_VER -v)
 
   # Check python syntax
   - black . --check --diff
 
+  # Sanity tests
+  - ansible-test sanity -v --python $PYTHON_VER
+
+  # Unit tests, with code coverage
+  - ansible-test units -vvvv --coverage --python $PYTHON_VER
+
   # Wait for Netbox containers to be running
   - timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:32768)" != "200" ]]; do echo "waiting for Netbox"; sleep 5; done' || false
 
   # Prepare data in Netbox
   - python tests/integration/netbox-deploy.py
 
   # Run regression and integration tests
+
+  # Run the inventory test first, in case any of the other tests modify the data.
+  # -vvvv is too much for integration tests
+  # Only run on Netbox > 2.6 - test data is different with API 2.6, haven't yet worked out a difference
+  - if [ "$VERSION" != "v2.6" ] ; then ansible-test integration -vv --coverage --python $PYTHON_VER inventory; fi
+
   - ansible-playbook tests/integration/regression-tests.yml -vvvv
   - ansible-playbook tests/integration/$VERSION/main.yml -vvvv
-  - ansible-inventory -vvvv -i tests/integration/test-inventory.yml --list
-  - ansible-inventory -vvvv -i tests/integration/test-inventory-plurals.yml --list
 
   # Report code coverage
-  - (cd /home/travis/.ansible/collections/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME && ansible-test coverage report)
+  - ansible-test coverage report
 
 deploy:
   provider: script

tests/integration/integration.cfg

@@ -0,0 +1,5 @@
+[defaults]
+force_valid_group_names = always
+
+[inventory]
+enable_plugins = netbox.netbox.nb_inventory, auto, host_list, yaml, ini, toml, script

tests/integration/targets/inventory/aliases

@@ -0,0 +1 @@
+# https://docs.ansible.com/ansible/devel/dev_guide/testing/sanity/integration-aliases.html

tests/integration/targets/inventory/compare_inventory_json.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+
+# Inspired by community.aws collection script_inventory_ec2 test
+# https://github.com/ansible-collections/community.aws/blob/master/tests/integration/targets/script_inventory_ec2/inventory_diff.py
+
+import sys
+import json
+from jsondiff import diff
+from typing import Iterable
+
+# Netbox includes created and last_updated times on objects.
+# These end up in the interfaces objects that are included verbatim from the Netbox API.
+# Ignore these when performing diffs as they will be different for each test run
+ignored_keys = set(["created", "last_updated"])
+
+
+# Assume the object will not be recursive, as it originally came from JSON
+def remove_ignored_keys(obj):
+
+    if isinstance(obj, dict):
+        keys_to_remove = ignored_keys.intersection(obj.keys())
+        for key in keys_to_remove:
+            del obj[key]
+
+        for (key, value) in obj.items():
+            remove_ignored_keys(value)
+
+    elif isinstance(obj, list):
+        for item in obj:
+            remove_ignored_keys(item)
+
+
+def main():
+    if len(sys.argv) != 3:
+        print(
+            "Error: Must have exactly 2 file to compare passed as arguments. Received %i."
+            % (len(sys.argv) - 1),
+            file=sys.stderr,
+        )
+        sys.exit(2)
+
+    a = sys.argv[1]
+    b = sys.argv[2]
+
+    with open(a, "r") as f:
+        adata = json.loads(f.read())
+    with open(b, "r") as f:
+        bdata = json.loads(f.read())
+
+    # Remove keys that we don't want to diff
+    # ie. times that will change on each run of tests
+    remove_ignored_keys(adata)
+    remove_ignored_keys(bdata)
+
+    # Perform the diff
+    # syntax='symmetric' will produce output that prints both the before and after as "$insert" and "$delete"
+    # marshal=True removes any special types, allowing to be dumped as json
+    result = diff(adata, bdata, marshal=True, syntax="symmetric")
+
+    if result:
+        # Dictionary is not empty - print differences
+        print(json.dumps(result, sort_keys=True, indent=4))
+        sys.exit(1)
+    else:
+        # Success, no differences
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()