[bootstrap] Fix building setuptools

.github/workflows/unit_tests.yml

@@ -156,7 +156,7 @@ jobs:
           EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
           EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
           EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-          EB_BOOTSTRAP_EXPECTED="20200820.01 d490d229a18bd5eaa717bb8d5684d754729143d5e995e35a40c84d03ffb1de50"
+          EB_BOOTSTRAP_EXPECTED="20200914.01 8b89f828d034f9f0a175a1551f71bc3034db487e781e2f41a0f14b747a72b0ee"
           test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
 
           # test bootstrap script

.travis.yml

@@ -89,7 +89,7 @@ script:
     - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
     - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
     - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-    - EB_BOOTSTRAP_EXPECTED="20200820.01 d490d229a18bd5eaa717bb8d5684d754729143d5e995e35a40c84d03ffb1de50"
+    - EB_BOOTSTRAP_EXPECTED="20200914.01 8b89f828d034f9f0a175a1551f71bc3034db487e781e2f41a0f14b747a72b0ee"
     - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
     # test bootstrap script
     - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap

easybuild/scripts/bootstrap_eb.py

@@ -62,7 +62,7 @@
     import urllib.request as std_urllib
 
 
-EB_BOOTSTRAP_VERSION = '20200820.01'
+EB_BOOTSTRAP_VERSION = '20200914.01'
 
 # argparse preferrred, optparse deprecated >=2.7
 HAVE_ARGPARSE = False
@@ -334,32 +334,33 @@ def check_setuptools():
     #       So, we'll check things by running commands through os.system rather than importing setuptools directly.
     cmd_tmpl = "%s -c '%%s' > %s 2>&1" % (sys.executable, outfile)
 
-    # check setuptools version
-    try:
-        os.system(cmd_tmpl % "import setuptools; print(setuptools.__version__)")
-        setuptools_ver = LooseVersion(open(outfile).read().strip())
-        debug("Found setuptools version %s" % setuptools_ver)
-
-        min_setuptools_ver = '0.6c11'
-        if setuptools_ver < LooseVersion(min_setuptools_ver):
-            debug("Minimal setuptools version %s not satisfied, found '%s'" % (min_setuptools_ver, setuptools_ver))
-            res = False
-    except Exception as err:
-        debug("Failed to check setuptools version: %s" % err)
-        res = False
-
     os.system(cmd_tmpl % "from setuptools.command import easy_install; print(easy_install.__file__)")
     out = open(outfile).read().strip()
-    debug("Location of setuptools' easy_install module: %s" % out)
     if 'setuptools/command/easy_install' not in out:
         debug("Module 'setuptools.command.easy_install not found")
         res = False
+    else:
+        debug("Location of setuptools' easy_install module: %s" % out)
 
-    if res is None:
-        os.system(cmd_tmpl % "import setuptools; print(setuptools.__file__)")
-        setuptools_loc = open(outfile).read().strip()
-        res = os.path.dirname(os.path.dirname(setuptools_loc))
-        debug("Location of setuptools installation: %s" % res)
+        # check setuptools version
+        try:
+            os.system(cmd_tmpl % "import setuptools; print(setuptools.__version__)")
+            setuptools_ver = LooseVersion(open(outfile).read().strip())
+            debug("Found setuptools version %s" % setuptools_ver)
+
+            min_setuptools_ver = '0.6c11'
+            if setuptools_ver < LooseVersion(min_setuptools_ver):
+                debug("Minimal setuptools version %s not satisfied, found '%s'" % (min_setuptools_ver, setuptools_ver))
+                res = False
+        except Exception as err:
+            debug("Failed to check setuptools version: %s" % err)
+            res = False
+
+        if res is None:
+            os.system(cmd_tmpl % "import setuptools; print(setuptools.__file__)")
+            setuptools_loc = open(outfile).read().strip()
+            res = os.path.dirname(os.path.dirname(setuptools_loc))
+            debug("Location of setuptools installation: %s" % res)
 
     try:
         os.remove(outfile)
@@ -375,6 +376,10 @@ def run_easy_install(args):
     debug("Active setuptools installation: %s" % setuptools.__file__)
     from setuptools.command import easy_install
 
+    # PyPi no longer supports http but setuptools 0.6.x easy_install defaults to that
+    if LooseVersion(setuptools.__version__) < LooseVersion('0.7.0'):
+        args.insert(0, '--index-url=https://pypi.python.org/simple')
+
     orig_stdout, orig_stderr = mock_stdout_stderr()
     try:
         easy_install.main(args)
@@ -432,6 +437,9 @@ def stage0(tmpdir):
         # silence distribute_setup.py completely by setting high log level threshold
         txt = re.sub(r'([^\n]*)(# extracting the tarball[^\n]*)', r'\1log.set_verbosity(1000)\n\1\2', txt)
 
+    # PyPi no longer supports http
+    txt = txt.replace('http://', 'https://')
+
     # write distribute_setup.py to file (with correct header)
     distribute_setup = os.path.join(tmpdir, 'distribute_setup.py')
     f = open(distribute_setup, "w")
@@ -748,6 +756,10 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
     sources_tmpl = "%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s"
     if eb_looseversion < LooseVersion('4.0'):
         sources_tmpl = "%(vsc-install)s%(vsc-base)s" + sources_tmpl
+        templates['toolchain'] = EASYBUILD_EASYCONFIG_TOOLCHAIN_PRE4
+    else:
+        templates['toolchain'] = EASYBUILD_EASYCONFIG_TOOLCHAIN
+
 
     templates.update({
         'source_urls': '\n'.join(["'%s'," % x for x in pkg_urls]),
@@ -825,7 +837,7 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath):
     eb_spec = {
         'name': 'EasyBuild',
         'hidden': False,
-        'toolchain': {'name': 'dummy', 'version': 'dummy'},
+        'toolchain': templates['toolchain'],
         'version': templates['version'],
         'versionprefix': '',
         'versionsuffix': '',
@@ -992,7 +1004,7 @@ def main():
 written in Python that allows you to install software in a structured,
 repeatable and robust way.\"\"\"
 
-toolchain = {'name': 'dummy', 'version': 'dummy'}
+toolchain = %(toolchain)s
 
 source_urls = [%(source_urls)s]
 sources = [%(sources)s]
@@ -1010,6 +1022,10 @@ def main():
 
 moduleclass = 'tools'
 """
+# Toolchain to be used for EasyBuild < 4.0
+EASYBUILD_EASYCONFIG_TOOLCHAIN_PRE4 = {'name': 'dummy', 'version': 'dummy'}
+# Toolchain to be used for EasyBuild >= 4.0 (new default)
+EASYBUILD_EASYCONFIG_TOOLCHAIN = {'name': 'system', 'version': ''}
 
 # check Python version
 loose_pyver = LooseVersion(python_version())
@@ -1151,8 +1167,10 @@ def main():
 7lwukTHLpfpLDH2GT+yCCf8D2cp1xw==
 """
 if IS_PY3:
-    DISTRIBUTE_SETUP_PY = DISTRIBUTE_SETUP_PY.encode('ascii')
+    DISTRIBUTE_SETUP_PY = DISTRIBUTE_SETUP_PY.encode()
 DISTRIBUTE_SETUP_PY = codecs.decode(codecs.decode(DISTRIBUTE_SETUP_PY, "base64"), "zlib")
+if IS_PY3:
+    DISTRIBUTE_SETUP_PY = DISTRIBUTE_SETUP_PY.decode()
 
 # run main function as body of script
 main()