diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 855c3254f4..39112ff8f8 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -156,7 +156,7 @@ jobs: EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - EB_BOOTSTRAP_EXPECTED="20200820.01 d490d229a18bd5eaa717bb8d5684d754729143d5e995e35a40c84d03ffb1de50" + EB_BOOTSTRAP_EXPECTED="20200914.01 8b89f828d034f9f0a175a1551f71bc3034db487e781e2f41a0f14b747a72b0ee" test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script diff --git a/.travis.yml b/.travis.yml index 396fa68c5e..6b9f463fa5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,7 +89,7 @@ script: - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - - EB_BOOTSTRAP_EXPECTED="20200820.01 d490d229a18bd5eaa717bb8d5684d754729143d5e995e35a40c84d03ffb1de50" + - EB_BOOTSTRAP_EXPECTED="20200914.01 8b89f828d034f9f0a175a1551f71bc3034db487e781e2f41a0f14b747a72b0ee" - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) # test bootstrap script - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index 104790a048..2954bad851 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -62,7 +62,7 @@ import urllib.request as std_urllib -EB_BOOTSTRAP_VERSION = '20200820.01' +EB_BOOTSTRAP_VERSION = '20200914.01' # argparse preferrred, optparse deprecated >=2.7 HAVE_ARGPARSE = False @@ -334,32 +334,33 @@ def check_setuptools(): # So, we'll check things by running commands through os.system rather than importing setuptools directly. cmd_tmpl = "%s -c '%%s' > %s 2>&1" % (sys.executable, outfile) - # check setuptools version - try: - os.system(cmd_tmpl % "import setuptools; print(setuptools.__version__)") - setuptools_ver = LooseVersion(open(outfile).read().strip()) - debug("Found setuptools version %s" % setuptools_ver) - - min_setuptools_ver = '0.6c11' - if setuptools_ver < LooseVersion(min_setuptools_ver): - debug("Minimal setuptools version %s not satisfied, found '%s'" % (min_setuptools_ver, setuptools_ver)) - res = False - except Exception as err: - debug("Failed to check setuptools version: %s" % err) - res = False - os.system(cmd_tmpl % "from setuptools.command import easy_install; print(easy_install.__file__)") out = open(outfile).read().strip() - debug("Location of setuptools' easy_install module: %s" % out) if 'setuptools/command/easy_install' not in out: debug("Module 'setuptools.command.easy_install not found") res = False + else: + debug("Location of setuptools' easy_install module: %s" % out) - if res is None: - os.system(cmd_tmpl % "import setuptools; print(setuptools.__file__)") - setuptools_loc = open(outfile).read().strip() - res = os.path.dirname(os.path.dirname(setuptools_loc)) - debug("Location of setuptools installation: %s" % res) + # check setuptools version + try: + os.system(cmd_tmpl % "import setuptools; print(setuptools.__version__)") + setuptools_ver = LooseVersion(open(outfile).read().strip()) + debug("Found setuptools version %s" % setuptools_ver) + + min_setuptools_ver = '0.6c11' + if setuptools_ver < LooseVersion(min_setuptools_ver): + debug("Minimal setuptools version %s not satisfied, found '%s'" % (min_setuptools_ver, setuptools_ver)) + res = False + except Exception as err: + debug("Failed to check setuptools version: %s" % err) + res = False + + if res is None: + os.system(cmd_tmpl % "import setuptools; print(setuptools.__file__)") + setuptools_loc = open(outfile).read().strip() + res = os.path.dirname(os.path.dirname(setuptools_loc)) + debug("Location of setuptools installation: %s" % res) try: os.remove(outfile) @@ -375,6 +376,10 @@ def run_easy_install(args): debug("Active setuptools installation: %s" % setuptools.__file__) from setuptools.command import easy_install + # PyPi no longer supports http but setuptools 0.6.x easy_install defaults to that + if LooseVersion(setuptools.__version__) < LooseVersion('0.7.0'): + args.insert(0, '--index-url=https://pypi.python.org/simple') + orig_stdout, orig_stderr = mock_stdout_stderr() try: easy_install.main(args) @@ -432,6 +437,9 @@ def stage0(tmpdir): # silence distribute_setup.py completely by setting high log level threshold txt = re.sub(r'([^\n]*)(# extracting the tarball[^\n]*)', r'\1log.set_verbosity(1000)\n\1\2', txt) + # PyPi no longer supports http + txt = txt.replace('http://', 'https://') + # write distribute_setup.py to file (with correct header) distribute_setup = os.path.join(tmpdir, 'distribute_setup.py') f = open(distribute_setup, "w") @@ -748,6 +756,10 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): sources_tmpl = "%(easybuild-framework)s%(easybuild-easyblocks)s%(easybuild-easyconfigs)s" if eb_looseversion < LooseVersion('4.0'): sources_tmpl = "%(vsc-install)s%(vsc-base)s" + sources_tmpl + templates['toolchain'] = EASYBUILD_EASYCONFIG_TOOLCHAIN_PRE4 + else: + templates['toolchain'] = EASYBUILD_EASYCONFIG_TOOLCHAIN + templates.update({ 'source_urls': '\n'.join(["'%s'," % x for x in pkg_urls]), @@ -825,7 +837,7 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): eb_spec = { 'name': 'EasyBuild', 'hidden': False, - 'toolchain': {'name': 'dummy', 'version': 'dummy'}, + 'toolchain': templates['toolchain'], 'version': templates['version'], 'versionprefix': '', 'versionsuffix': '', @@ -992,7 +1004,7 @@ def main(): written in Python that allows you to install software in a structured, repeatable and robust way.\"\"\" -toolchain = {'name': 'dummy', 'version': 'dummy'} +toolchain = %(toolchain)s source_urls = [%(source_urls)s] sources = [%(sources)s] @@ -1010,6 +1022,10 @@ def main(): moduleclass = 'tools' """ +# Toolchain to be used for EasyBuild < 4.0 +EASYBUILD_EASYCONFIG_TOOLCHAIN_PRE4 = {'name': 'dummy', 'version': 'dummy'} +# Toolchain to be used for EasyBuild >= 4.0 (new default) +EASYBUILD_EASYCONFIG_TOOLCHAIN = {'name': 'system', 'version': ''} # check Python version loose_pyver = LooseVersion(python_version()) @@ -1151,8 +1167,10 @@ def main(): 7lwukTHLpfpLDH2GT+yCCf8D2cp1xw== """ if IS_PY3: - DISTRIBUTE_SETUP_PY = DISTRIBUTE_SETUP_PY.encode('ascii') + DISTRIBUTE_SETUP_PY = DISTRIBUTE_SETUP_PY.encode() DISTRIBUTE_SETUP_PY = codecs.decode(codecs.decode(DISTRIBUTE_SETUP_PY, "base64"), "zlib") +if IS_PY3: + DISTRIBUTE_SETUP_PY = DISTRIBUTE_SETUP_PY.decode() # run main function as body of script main()