From f299564ca7a0d29276c1ae3122b0669a55784aac Mon Sep 17 00:00:00 2001 From: rafie Date: Thu, 2 Jan 2020 15:38:21 +0200 Subject: [PATCH 01/10] Amazon Linux support --- get_deps.sh | 4 ++-- opt/readies/paella/platform.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/get_deps.sh b/get_deps.sh index b5d3d013a..807390239 100755 --- a/get_deps.sh +++ b/get_deps.sh @@ -120,7 +120,7 @@ if [[ $WITH_TF != 0 ]]; then LIBTF_ARCHIVE=libtensorflow-${TF_BUILD}-${TF_OS}-${TF_ARCH}-${TF_VERSION}.tar.gz - [[ ! -f $LIBTF_ARCHIVE || $FORCE == 1 ]] && wget --quiet $LIBTF_URL_BASE/$LIBTF_ARCHIVE + [[ ! -f $LIBTF_ARCHIVE || $FORCE == 1 ]] && wget -q $LIBTF_URL_BASE/$LIBTF_ARCHIVE rm -rf $LIBTENSORFLOW.x mkdir $LIBTENSORFLOW.x @@ -169,7 +169,7 @@ if [[ $WITH_TFLITE != 0 ]]; then LIBTFLITE_ARCHIVE=libtensorflowlite-${TFLITE_OS}-${TFLITE_ARCH}-${TFLITE_VERSION}.tar.gz - [[ ! -f $LIBTFLITE_ARCHIVE || $FORCE == 1 ]] && wget $LIBTF_URL_BASE/$LIBTFLITE_ARCHIVE + [[ ! -f $LIBTFLITE_ARCHIVE || $FORCE == 1 ]] && wget -q $LIBTF_URL_BASE/$LIBTFLITE_ARCHIVE rm -rf $LIBTFLITE.x mkdir $LIBTFLITE.x diff --git a/opt/readies/paella/platform.py b/opt/readies/paella/platform.py index 7f18bd147..6a94c4e7d 100755 --- a/opt/readies/paella/platform.py +++ b/opt/readies/paella/platform.py @@ -55,6 +55,9 @@ def __init__(self, strict=False): distname = 'redhat' elif distname.startswith('suse'): distname = 'suse' + elif distname.startswith('amzn'): + distname = 'amzn' + os.nick = 'amzn' + str(os_release.version()) else: if strict: assert(False), "Cannot determine distribution" @@ -94,7 +97,7 @@ def is_debian_compat(self): return self.dist == 'debian' or self.dist == 'ubuntu' def is_redhat_compat(self): - return self.dist == 'redhat' or self.dist == 'centos' + return self.dist == 'redhat' or self.dist == 'centos' or self.dist == 'amzn' def is_container(self): with open('/proc/1/cgroups', 'r') as conf: @@ -144,6 +147,8 @@ def invoke(self): self.suse() elif dist == 'arch': self.arch() + elif dist == 'amzn': + self.amzn() else: assert(False), "Cannot determine installer" elif os == 'macosx': @@ -201,3 +206,6 @@ def bsd_compat(self): def freebsd(self): pass + + def amzn(self): + pass From 3dd50c4530663ebf2a86db080128077e05a54a9e Mon Sep 17 00:00:00 2001 From: rafie Date: Thu, 2 Jan 2020 15:58:59 +0200 Subject: [PATCH 02/10] Fix 1 --- opt/readies/paella/platform.py | 2 +- opt/readies/paella/setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/opt/readies/paella/platform.py b/opt/readies/paella/platform.py index 6a94c4e7d..44a395bc9 100755 --- a/opt/readies/paella/platform.py +++ b/opt/readies/paella/platform.py @@ -57,7 +57,7 @@ def __init__(self, strict=False): distname = 'suse' elif distname.startswith('amzn'): distname = 'amzn' - os.nick = 'amzn' + str(os_release.version()) + self.osnick = 'amzn' + str(os_release.version()) else: if strict: assert(False), "Cannot determine distribution" diff --git a/opt/readies/paella/setup.py b/opt/readies/paella/setup.py index 3bd8abcec..0a849625d 100755 --- a/opt/readies/paella/setup.py +++ b/opt/readies/paella/setup.py @@ -120,9 +120,9 @@ def install(self, packs, group=False, _try=False): if self.os == 'linux': if self.dist == 'fedora': self.dnf_install(packs, group=group, _try=_try) - elif self.dist == 'ubuntu' or self.dist == 'debian': + elif self.platform.is_debian_compat(): self.apt_install(packs, group=group, _try=_try) - elif self.dist == 'centos' or self.dist == 'redhat': + elif self.platform.is_redhat_compat(): self.yum_install(packs, group=group, _try=_try) elif self.dist == 'suse': self.zypper_install(packs, group=group, _try=_try) From b3d8bb1c9d3cec34aab246406603dcc26a2dceb7 Mon Sep 17 00:00:00 2001 From: rafie Date: Thu, 2 Jan 2020 16:05:16 +0200 Subject: [PATCH 03/10] Fix 2 --- opt/system-setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/opt/system-setup.py b/opt/system-setup.py index e7c754b0f..5bef34e2a 100755 --- a/opt/system-setup.py +++ b/opt/system-setup.py @@ -34,7 +34,10 @@ def redhat_compat(self): self.group_install("'Development Tools'") self.install("redhat-lsb-core") - self.install("epel-release") + if not self.dist == "amzn": + self.install("epel-release") + else + self.run("amazon-linux-extras install epel") self.install("python36 python36-pip") self.install("python36-psutil") From 1568ec8974e2b8fd565925fda0798bb333f04555 Mon Sep 17 00:00:00 2001 From: rafie Date: Thu, 2 Jan 2020 16:05:52 +0200 Subject: [PATCH 04/10] Fix 3 --- opt/system-setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opt/system-setup.py b/opt/system-setup.py index 5bef34e2a..f856de893 100755 --- a/opt/system-setup.py +++ b/opt/system-setup.py @@ -36,7 +36,7 @@ def redhat_compat(self): if not self.dist == "amzn": self.install("epel-release") - else + else: self.run("amazon-linux-extras install epel") self.install("python36 python36-pip") self.install("python36-psutil") From 1224ca9648f9cf2d49291c114f010c1e4b8eee5b Mon Sep 17 00:00:00 2001 From: rafie Date: Thu, 2 Jan 2020 16:48:40 +0200 Subject: [PATCH 05/10] Fix 4 --- opt/readies/paella/setup.py | 80 +++++++++++++++++++------------------ opt/system-setup.py | 6 ++- 2 files changed, 45 insertions(+), 41 deletions(-) diff --git a/opt/readies/paella/setup.py b/opt/readies/paella/setup.py index 0a849625d..e7113cc98 100755 --- a/opt/readies/paella/setup.py +++ b/opt/readies/paella/setup.py @@ -61,16 +61,13 @@ def __init__(self, nop=False): self.dist = self.platform.dist self.ver = self.platform.os_ver - if self.has_command("python3"): - self.python = "python3" - elif self.has_command("python"): - self.python = "python" - elif self.has_command("python2"): - self.python = "python2" + self.python = sys.executable if self.os == 'macosx': # this is required because osx pip installed are done with --user - os.environ["PATH"] = os.environ["PATH"] + ':' + '$HOME/Library/Python/2.7/bin' + os.environ["PATH"] = os.environ["PATH"] + ':' + os.environ["HOME"] + '/Library/Python/2.7/bin' + # this prevents brew updating before each install + os.environ["HOMEBREW_NO_AUTO_UPDATE"] = "1" if self.platform.is_debian_compat(): # prevents apt-get from interactively prompting @@ -118,7 +115,7 @@ def brew_install(self, packs, group=False, _try=False): def install(self, packs, group=False, _try=False): if self.os == 'linux': - if self.dist == 'fedora': + if self.dist == 'fedora': # also include centos 8 self.dnf_install(packs, group=group, _try=_try) elif self.platform.is_debian_compat(): self.apt_install(packs, group=group, _try=_try) @@ -135,48 +132,48 @@ def install(self, packs, group=False, _try=False): else: Assert(False), "Cannot determine installer" - def group_install(self, packs): - self.install(packs, group=True) + def group_install(self, packs, _try=False): + self.install(packs, group=True, _try=_try) #------------------------------------------------------------------------------------------ - def yum_add_repo(self, repourl, repo=""): + def yum_add_repo(self, repourl, repo="", _try=False): if not self.has_command("yum-config-manager"): self.install("yum-utils") - self.run("yum-config-manager -y --add-repo {}".format(repourl)) + self.run("yum-config-manager -y --add-repo {}".format(repourl), _try=_try) - def apt_add_repo(self, repourl, repo=""): + def apt_add_repo(self, repourl, repo="", _try=False): if not self.has_command("yum-config-manager"): self.install("software-properties-common") - self.run("add-apt-repository -y {}".format(repourl)) - self.run("apt-get -qq update") + self.run("add-apt-repository -y {}".format(repourl), _try=_try) + self.run("apt-get -qq update", _try=_try) - def dnf_add_repo(self, repourl, repo=""): + def dnf_add_repo(self, repourl, repo="", _try=False): if self.run("dnf config-manager 2>/dev/null", _try=True): - self.install("dnf-plugins-core") - self.run("dnf config-manager -y --add-repo {}".format(repourl)) + self.install("dnf-plugins-core", _try=_try) + self.run("dnf config-manager -y --add-repo {}".format(repourl), _try=_try) - def zypper_add_repo(self, repourl, repo=""): + def zypper_add_repo(self, repourl, repo="", _try=False): pass - def pacman_add_repo(self, repourl, repo=""): + def pacman_add_repo(self, repourl, repo="", _try=False): pass - def brew_add_repo(self, repourl, repo=""): + def brew_add_repo(self, repourl, repo="", _try=False): pass - def add_repo(self, repourl, repo=""): + def add_repo(self, repourl, repo="", _try=False): if self.os == 'linux': if self.dist == 'fedora': - self.dnf_add_repo(repourl, repo=repo) + self.dnf_add_repo(repourl, repo=repo, _try=_try) elif self.dist == 'ubuntu' or self.dist == 'debian': - self.apt_add_repo(repourl, repo=repo) + self.apt_add_repo(repourl, repo=repo, _try=_try) elif self.dist == 'centos' or self.dist == 'redhat': - self.yum_add_repo(repourl, repo=repo) + self.yum_add_repo(repourl, repo=repo, _try=_try) elif self.dist == 'suse': - self.zypper_add_repo(repourl, repo=repo) + self.zypper_add_repo(repourl, repo=repo, _try=_try) elif self.dist == 'arch': - self.pacman_add_repo(repourl, repo=repo) + self.pacman_add_repo(repourl, repo=repo, _try=_try) else: Assert(False), "Cannot determine installer" elif self.os == 'macosx': @@ -196,20 +193,25 @@ def pip3_install(self, cmd, _try=False): pip_user = '' if self.os == 'macosx': pip_user = '--user ' - self.run("pip3 install --disable-pip-version-check " + pip_user + cmd, output_on_error=True, _try=_try) + self.run(self.python + " -m pip install --disable-pip-version-check " + pip_user + cmd, output_on_error=True, _try=_try) - def setup_pip(self): + def setup_pip(self, _try=False): get_pip = "set -e; wget -q https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py" - if not self.has_command("pip3"): - self.install("python3-distutils", _try=True) + if self.run(self.python + " -m pip --version", _try=True) != 0: + if sys.version_info.major == 3: + self.install("python3-distutils") self.install_downloaders() - self.run(get_pip + "; " + self.python + " /tmp/get-pip.py", output_on_error=True) + self.run(get_pip + "; " + self.python + " /tmp/get-pip.py", output_on_error=True, _try=_try) - def install_downloaders(self): + def install_downloaders(self, _try=False): if self.os == 'linux': - self.install("ca-certificates") - self.install("curl wget") - - def install_git_lfs_on_linux(self): - self.run("curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash") - self.install("git-lfs") + self.install("ca-certificates", _try=_try) + self.install("curl wget", _try=_try) + + def install_git_lfs_on_linux(self, _try=False): + cmd = "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.{}.sh | bash" + if self.platform.is_redhat_compat(): + self.run(cmd.format('rpm'), _try=_try) + elif self.platform.is_debian_compat(): + self.run(cmd.format('deb'), _try=_try) + self.install("git-lfs", _try=_try) diff --git a/opt/system-setup.py b/opt/system-setup.py index f856de893..31fc9418e 100755 --- a/opt/system-setup.py +++ b/opt/system-setup.py @@ -36,10 +36,12 @@ def redhat_compat(self): if not self.dist == "amzn": self.install("epel-release") + self.install("python36 python36-pip") + self.install("python36-psutil") else: self.run("amazon-linux-extras install epel") - self.install("python36 python36-pip") - self.install("python36-psutil") + self.install("python3 python3-devel") + self.pip_install("psutil") self.install_git_lfs_on_linux() From ffc8d65f8c04cc14e20fc1845ca609f3a4fd5f10 Mon Sep 17 00:00:00 2001 From: rafie Date: Thu, 2 Jan 2020 16:51:04 +0200 Subject: [PATCH 06/10] Fix 5 --- opt/readies/paella/setup.py | 2 +- opt/system-setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opt/readies/paella/setup.py b/opt/readies/paella/setup.py index e7113cc98..8a414d9b0 100755 --- a/opt/readies/paella/setup.py +++ b/opt/readies/paella/setup.py @@ -187,7 +187,7 @@ def pip_install(self, cmd, _try=False): pip_user = '' if self.os == 'macosx': pip_user = '--user ' - self.run("pip install --disable-pip-version-check " + pip_user + cmd, output_on_error=True, _try=_try) + self.run(self.python + " -m pip install --disable-pip-version-check " + pip_user + cmd, output_on_error=True, _try=_try) def pip3_install(self, cmd, _try=False): pip_user = '' diff --git a/opt/system-setup.py b/opt/system-setup.py index 31fc9418e..4dabcc659 100755 --- a/opt/system-setup.py +++ b/opt/system-setup.py @@ -39,7 +39,7 @@ def redhat_compat(self): self.install("python36 python36-pip") self.install("python36-psutil") else: - self.run("amazon-linux-extras install epel") + self.run("amazon-linux-extras install epel", output_on_error=True) self.install("python3 python3-devel") self.pip_install("psutil") From 2b2f0bc1b16e78503eae888f2bda35f433582762 Mon Sep 17 00:00:00 2001 From: rafie Date: Thu, 2 Jan 2020 18:30:40 +0200 Subject: [PATCH 07/10] Fix 6 --- opt/readies/paella/setup.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/opt/readies/paella/setup.py b/opt/readies/paella/setup.py index 8a414d9b0..67207fbf3 100755 --- a/opt/readies/paella/setup.py +++ b/opt/readies/paella/setup.py @@ -209,9 +209,12 @@ def install_downloaders(self, _try=False): self.install("curl wget", _try=_try) def install_git_lfs_on_linux(self, _try=False): - cmd = "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.{}.sh | bash" - if self.platform.is_redhat_compat(): - self.run(cmd.format('rpm'), _try=_try) - elif self.platform.is_debian_compat(): - self.run(cmd.format('deb'), _try=_try) - self.install("git-lfs", _try=_try) + self.run("set -e; wget -q https://github.com/git-lfs/git-lfs/releases/download/v2.9.2/git-lfs-linux-amd64-v2.9.2.tar.gz -O /tmp/git-lfs.tar.gz") + self.run("cd /tmp; tar xzf git-lfs.tar.gz; ./install.sh; git lfs install") + +# cmd = "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.{}.sh | bash" +# if self.platform.is_redhat_compat(): +# self.run(cmd.format('rpm'), _try=_try) +# elif self.platform.is_debian_compat(): +# self.run(cmd.format('deb'), _try=_try) +# self.install("git-lfs", _try=_try) From f9e385d84debcd1d6b74a34f4cf45cd9741d3bd9 Mon Sep 17 00:00:00 2001 From: rafie Date: Fri, 3 Jan 2020 14:26:01 +0200 Subject: [PATCH 08/10] Fix 7 --- opt/readies/paella/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opt/readies/paella/setup.py b/opt/readies/paella/setup.py index 67207fbf3..4c3f787d0 100755 --- a/opt/readies/paella/setup.py +++ b/opt/readies/paella/setup.py @@ -210,7 +210,7 @@ def install_downloaders(self, _try=False): def install_git_lfs_on_linux(self, _try=False): self.run("set -e; wget -q https://github.com/git-lfs/git-lfs/releases/download/v2.9.2/git-lfs-linux-amd64-v2.9.2.tar.gz -O /tmp/git-lfs.tar.gz") - self.run("cd /tmp; tar xzf git-lfs.tar.gz; ./install.sh; git lfs install") + self.run("cd /tmp; tar xzf git-lfs.tar.gz; ./install.sh") # cmd = "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.{}.sh | bash" # if self.platform.is_redhat_compat(): From eee143e0393fa1a8d4f6c1ae04027ceab6441e41 Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 21 Jan 2020 17:00:38 +0200 Subject: [PATCH 09/10] Added CircleCI macOS build + automation fixes --- .circleci/config.yml | 22 ++++++++++++++++++---- opt/readies/bin/platform | 9 ++++++++- opt/readies/paella/platform.py | 9 ++++++++- opt/readies/paella/setup.py | 5 +++-- opt/system-setup.py | 4 ---- 5 files changed, 37 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a2d1919b6..9067d5c9d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -159,10 +159,13 @@ workflows: filters: tags: only: /.*/ - # - build-macos: - # filters: - # tags: - # only: /.*/ + - build-macos: + filters: + branches: + # only: /.*/ + ignore: /.*/ + tags: + only: /^v[0-9].*/ #- build-multiarch-docker: # filters: # tags: @@ -198,3 +201,14 @@ workflows: filters: branches: only: master + + nightly: + triggers: + - schedule: + cron: "20 17 * * *" + filters: + branches: + only: + - master + jobs: + - build-macos diff --git a/opt/readies/bin/platform b/opt/readies/bin/platform index 31af5f5bd..679b78f34 100755 --- a/opt/readies/bin/platform +++ b/opt/readies/bin/platform @@ -12,6 +12,7 @@ parser = argparse.ArgumentParser(description='Report platform characteristics.') parser.add_argument('--os', action="store_true", help='Operating system') parser.add_argument('--version', action="store_true", help='OS/Distribution version') parser.add_argument('--dist', action="store_true", help='Linux distribution (if applicable)') +parser.add_argument('--osnick', action="store_true", help='OS/distribution/version is a single word') parser.add_argument('--arch', action="store_true", help='CPU Architecture') parser.add_argument('--kernel', action="store_true", help='Kernel version (if applicable)') parser.add_argument('--glibc', action="store_true", help='GLIBC version (if applicable)') @@ -26,6 +27,8 @@ except: ret = "" if args.os: ret += " " + platform.os +if args.osnick: + ret += " " + platform.osnick if args.dist: ret += " " + platform.dist if args.version: @@ -41,5 +44,9 @@ if ret == "": dist = platform.dist if dist != "": os = dist + " " + os - ret = os + " " + platform.os_ver + " " + platform.arch + if platform.osnick != "": + nick = " (" + platform.osnick + ")" + else: + nick = "" + ret = os + " " + platform.os_ver + nick + " " + platform.arch print(ret.strip()) diff --git a/opt/readies/paella/platform.py b/opt/readies/paella/platform.py index 44a395bc9..ebb4dde30 100755 --- a/opt/readies/paella/platform.py +++ b/opt/readies/paella/platform.py @@ -24,7 +24,10 @@ def version(self): return self.defs["VERSION_ID"] def osnick(self): - return self.defs["VERSION_CODENAME"] + try: + return self.defs["VERSION_CODENAME"] + except: + return "" #------------------------------------------------------------------------------------------ @@ -42,6 +45,9 @@ def __init__(self, strict=False): os_release = Platform.OSRelease() distname = os_release.distname() self.os_ver = self.full_os_ver = os_release.version() + self.osnick = os_release.osnick() + if self.osnick == "": + self.osnick = distname + str(self.os_ver) except: if strict: assert(False), "Cannot determine distribution" @@ -69,6 +75,7 @@ def __init__(self, strict=False): self.full_os_ver = mac_ver[0] # e.g. 10.14, but also 10.5.8 self.os_ver = '.'.join(self.full_os_ver.split('.')[:2]) # major.minor # self.arch = mac_ver[2] # e.g. x64_64 + self.osnick = self.os + str(self.full_os_ver.split('.')[1]) elif self.os == 'windows': self.dist = self.os self.os_ver = platform.release() diff --git a/opt/readies/paella/setup.py b/opt/readies/paella/setup.py index 4c3f787d0..995556369 100755 --- a/opt/readies/paella/setup.py +++ b/opt/readies/paella/setup.py @@ -197,11 +197,12 @@ def pip3_install(self, cmd, _try=False): def setup_pip(self, _try=False): get_pip = "set -e; wget -q https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py" - if self.run(self.python + " -m pip --version", _try=True) != 0: + if self.run(self.python + " -m pip --version", _try=True, output_on_error=False) != 0: if sys.version_info.major == 3: self.install("python3-distutils") self.install_downloaders() - self.run(get_pip + "; " + self.python + " /tmp/get-pip.py", output_on_error=True, _try=_try) + pip_user = ' --user' if self.os == 'macosx' else '' + self.run(get_pip + "; " + self.python + " /tmp/get-pip.py" + pip_user + " 'pip==19.3.1'", output_on_error=True, _try=_try) def install_downloaders(self, _try=False): if self.os == 'linux': diff --git a/opt/system-setup.py b/opt/system-setup.py index 4dabcc659..79fcb8e62 100755 --- a/opt/system-setup.py +++ b/opt/system-setup.py @@ -58,10 +58,6 @@ def macosx(self): self.install("git-lfs") self.install("redis") - def install_git_lfs_on_linux(self): - self.run("curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash") - self.install("git-lfs") - def common_last(self): if not self.has_command("RLTest"): self.pip3_install("git+https://github.com/RedisLabsModules/RLTest.git@master") From f99e7a5f64e9db4735b31840b53ee7081efff416 Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 21 Jan 2020 17:12:44 +0200 Subject: [PATCH 10/10] Fix 8 --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9067d5c9d..16ecddfa0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,12 +68,12 @@ jobs: steps: - ci_steps: platform: debian - # build-macos: - # macos: - # xcode: 10.2.1 - # steps: - # - ci_steps: - # platform: macosx + build-macos: + macos: + xcode: 10.2.1 + steps: + - ci_steps: + platform: macosx build-multiarch-docker: machine: enabled: true