diff --git a/tests/profile/test_profile_against_externals.py b/tests/profile/test_profile_against_externals.py
new file mode 100644
index 0000000000..e7159264be
--- /dev/null
+++ b/tests/profile/test_profile_against_externals.py
@@ -0,0 +1,61 @@
+""" Profiles basic -jX functionality """
+# Copyright (c) 2020 Frank Harrison <doublethefish@gmail.com>
+
+# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+# For details: https://github.com/PyCQA/pylint/blob/master/COPYING
+
+# pylint: disable=protected-access,missing-function-docstring,no-self-use
+
+import os
+import pprint
+import shutil
+import tempfile
+
+import pytest
+
+from pylint.lint import Run
+from pylint.testutils import TestReporter as Reporter
+
+
+def _get_py_files(scanpath):
+    assert os.path.exists(scanpath), "Dir not found %s" % scanpath
+
+    filepaths = []
+    for dirpath, dirnames, filenames in os.walk(scanpath):
+        dirnames[:] = [dirname for dirname in dirnames if dirname != "__pycache__"]
+        filepaths.extend(
+            [
+                os.path.join(dirpath, filename)
+                for filename in filenames
+                if filename.endswith(".py")
+            ]
+        )
+    return filepaths
+
+
+@pytest.mark.skipif(
+    not os.environ.get("PYTEST_PROFILE_EXTERNAL", False),
+    reason="PYTEST_PROFILE_EXTERNAL, not set, assuming not a profile run",
+)
+@pytest.mark.parametrize(
+    "name,git_repo", [("numpy", "https://github.com/numpy/numpy.git")]
+)
+def test_run(tmp_path, name, git_repo):
+    """ Runs pylint against external sources """
+    checkoutdir = tmp_path / name
+    checkoutdir.mkdir()
+    os.system(
+        "git clone --depth=1 {git_repo} {checkoutdir}".format(
+            git_repo=git_repo, checkoutdir=str(checkoutdir)
+        )
+    )
+    filepaths = _get_py_files(scanpath=str(checkoutdir))
+    print("Have %d files" % len(filepaths))
+
+    runner = Run(filepaths, reporter=Reporter(), do_exit=False)
+
+    print(
+        "Had %d files with %d messages"
+        % (len(filepaths), len(runner.linter.reporter.messages))
+    )
+    pprint.pprint(runner.linter.reporter.messages)
diff --git a/tox.ini b/tox.ini
index 9ea7949837..bb8cf99943 100644
--- a/tox.ini
+++ b/tox.ini
@@ -157,4 +157,21 @@ commands =
                          --benchmark-group-by="group" \
                          {posargs:}
 
+[testenv:profile_against_external]
+deps =
+   https://github.com/PyCQA/astroid/tarball/master#egg=astroid-master-2.0
+   gprof2dot
+   mccabe
+   pytest
+   pytest-profiling
+   pytest-xdist
+
+setenv =
+  PYTEST_PROFILE_EXTERNAL = 1
+
+commands =
+    python -Wi -m pytest --exitfirst \
+                         --profile-svg \
+                         {toxinidir}/tests/profile/test_profile_against_externals.py
+
 changedir = {toxworkdir}