diff --git a/.circleci/config.yml b/.circleci/config.yml index a2d1919b6..16ecddfa0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,12 +68,12 @@ jobs: steps: - ci_steps: platform: debian - # build-macos: - # macos: - # xcode: 10.2.1 - # steps: - # - ci_steps: - # platform: macosx + build-macos: + macos: + xcode: 10.2.1 + steps: + - ci_steps: + platform: macosx build-multiarch-docker: machine: enabled: true @@ -159,10 +159,13 @@ workflows: filters: tags: only: /.*/ - # - build-macos: - # filters: - # tags: - # only: /.*/ + - build-macos: + filters: + branches: + # only: /.*/ + ignore: /.*/ + tags: + only: /^v[0-9].*/ #- build-multiarch-docker: # filters: # tags: @@ -198,3 +201,14 @@ workflows: filters: branches: only: master + + nightly: + triggers: + - schedule: + cron: "20 17 * * *" + filters: + branches: + only: + - master + jobs: + - build-macos diff --git a/get_deps.sh b/get_deps.sh index b5d3d013a..807390239 100755 --- a/get_deps.sh +++ b/get_deps.sh @@ -120,7 +120,7 @@ if [[ $WITH_TF != 0 ]]; then LIBTF_ARCHIVE=libtensorflow-${TF_BUILD}-${TF_OS}-${TF_ARCH}-${TF_VERSION}.tar.gz - [[ ! -f $LIBTF_ARCHIVE || $FORCE == 1 ]] && wget --quiet $LIBTF_URL_BASE/$LIBTF_ARCHIVE + [[ ! -f $LIBTF_ARCHIVE || $FORCE == 1 ]] && wget -q $LIBTF_URL_BASE/$LIBTF_ARCHIVE rm -rf $LIBTENSORFLOW.x mkdir $LIBTENSORFLOW.x @@ -169,7 +169,7 @@ if [[ $WITH_TFLITE != 0 ]]; then LIBTFLITE_ARCHIVE=libtensorflowlite-${TFLITE_OS}-${TFLITE_ARCH}-${TFLITE_VERSION}.tar.gz - [[ ! -f $LIBTFLITE_ARCHIVE || $FORCE == 1 ]] && wget $LIBTF_URL_BASE/$LIBTFLITE_ARCHIVE + [[ ! -f $LIBTFLITE_ARCHIVE || $FORCE == 1 ]] && wget -q $LIBTF_URL_BASE/$LIBTFLITE_ARCHIVE rm -rf $LIBTFLITE.x mkdir $LIBTFLITE.x diff --git a/opt/readies/bin/platform b/opt/readies/bin/platform index 31af5f5bd..679b78f34 100755 --- a/opt/readies/bin/platform +++ b/opt/readies/bin/platform @@ -12,6 +12,7 @@ parser = argparse.ArgumentParser(description='Report platform characteristics.') parser.add_argument('--os', action="store_true", help='Operating system') parser.add_argument('--version', action="store_true", help='OS/Distribution version') parser.add_argument('--dist', action="store_true", help='Linux distribution (if applicable)') +parser.add_argument('--osnick', action="store_true", help='OS/distribution/version is a single word') parser.add_argument('--arch', action="store_true", help='CPU Architecture') parser.add_argument('--kernel', action="store_true", help='Kernel version (if applicable)') parser.add_argument('--glibc', action="store_true", help='GLIBC version (if applicable)') @@ -26,6 +27,8 @@ except: ret = "" if args.os: ret += " " + platform.os +if args.osnick: + ret += " " + platform.osnick if args.dist: ret += " " + platform.dist if args.version: @@ -41,5 +44,9 @@ if ret == "": dist = platform.dist if dist != "": os = dist + " " + os - ret = os + " " + platform.os_ver + " " + platform.arch + if platform.osnick != "": + nick = " (" + platform.osnick + ")" + else: + nick = "" + ret = os + " " + platform.os_ver + nick + " " + platform.arch print(ret.strip()) diff --git a/opt/readies/paella/platform.py b/opt/readies/paella/platform.py index 7f18bd147..ebb4dde30 100755 --- a/opt/readies/paella/platform.py +++ b/opt/readies/paella/platform.py @@ -24,7 +24,10 @@ def version(self): return self.defs["VERSION_ID"] def osnick(self): - return self.defs["VERSION_CODENAME"] + try: + return self.defs["VERSION_CODENAME"] + except: + return "" #------------------------------------------------------------------------------------------ @@ -42,6 +45,9 @@ def __init__(self, strict=False): os_release = Platform.OSRelease() distname = os_release.distname() self.os_ver = self.full_os_ver = os_release.version() + self.osnick = os_release.osnick() + if self.osnick == "": + self.osnick = distname + str(self.os_ver) except: if strict: assert(False), "Cannot determine distribution" @@ -55,6 +61,9 @@ def __init__(self, strict=False): distname = 'redhat' elif distname.startswith('suse'): distname = 'suse' + elif distname.startswith('amzn'): + distname = 'amzn' + self.osnick = 'amzn' + str(os_release.version()) else: if strict: assert(False), "Cannot determine distribution" @@ -66,6 +75,7 @@ def __init__(self, strict=False): self.full_os_ver = mac_ver[0] # e.g. 10.14, but also 10.5.8 self.os_ver = '.'.join(self.full_os_ver.split('.')[:2]) # major.minor # self.arch = mac_ver[2] # e.g. x64_64 + self.osnick = self.os + str(self.full_os_ver.split('.')[1]) elif self.os == 'windows': self.dist = self.os self.os_ver = platform.release() @@ -94,7 +104,7 @@ def is_debian_compat(self): return self.dist == 'debian' or self.dist == 'ubuntu' def is_redhat_compat(self): - return self.dist == 'redhat' or self.dist == 'centos' + return self.dist == 'redhat' or self.dist == 'centos' or self.dist == 'amzn' def is_container(self): with open('/proc/1/cgroups', 'r') as conf: @@ -144,6 +154,8 @@ def invoke(self): self.suse() elif dist == 'arch': self.arch() + elif dist == 'amzn': + self.amzn() else: assert(False), "Cannot determine installer" elif os == 'macosx': @@ -201,3 +213,6 @@ def bsd_compat(self): def freebsd(self): pass + + def amzn(self): + pass diff --git a/opt/readies/paella/setup.py b/opt/readies/paella/setup.py index 3bd8abcec..995556369 100755 --- a/opt/readies/paella/setup.py +++ b/opt/readies/paella/setup.py @@ -61,16 +61,13 @@ def __init__(self, nop=False): self.dist = self.platform.dist self.ver = self.platform.os_ver - if self.has_command("python3"): - self.python = "python3" - elif self.has_command("python"): - self.python = "python" - elif self.has_command("python2"): - self.python = "python2" + self.python = sys.executable if self.os == 'macosx': # this is required because osx pip installed are done with --user - os.environ["PATH"] = os.environ["PATH"] + ':' + '$HOME/Library/Python/2.7/bin' + os.environ["PATH"] = os.environ["PATH"] + ':' + os.environ["HOME"] + '/Library/Python/2.7/bin' + # this prevents brew updating before each install + os.environ["HOMEBREW_NO_AUTO_UPDATE"] = "1" if self.platform.is_debian_compat(): # prevents apt-get from interactively prompting @@ -118,11 +115,11 @@ def brew_install(self, packs, group=False, _try=False): def install(self, packs, group=False, _try=False): if self.os == 'linux': - if self.dist == 'fedora': + if self.dist == 'fedora': # also include centos 8 self.dnf_install(packs, group=group, _try=_try) - elif self.dist == 'ubuntu' or self.dist == 'debian': + elif self.platform.is_debian_compat(): self.apt_install(packs, group=group, _try=_try) - elif self.dist == 'centos' or self.dist == 'redhat': + elif self.platform.is_redhat_compat(): self.yum_install(packs, group=group, _try=_try) elif self.dist == 'suse': self.zypper_install(packs, group=group, _try=_try) @@ -135,48 +132,48 @@ def install(self, packs, group=False, _try=False): else: Assert(False), "Cannot determine installer" - def group_install(self, packs): - self.install(packs, group=True) + def group_install(self, packs, _try=False): + self.install(packs, group=True, _try=_try) #------------------------------------------------------------------------------------------ - def yum_add_repo(self, repourl, repo=""): + def yum_add_repo(self, repourl, repo="", _try=False): if not self.has_command("yum-config-manager"): self.install("yum-utils") - self.run("yum-config-manager -y --add-repo {}".format(repourl)) + self.run("yum-config-manager -y --add-repo {}".format(repourl), _try=_try) - def apt_add_repo(self, repourl, repo=""): + def apt_add_repo(self, repourl, repo="", _try=False): if not self.has_command("yum-config-manager"): self.install("software-properties-common") - self.run("add-apt-repository -y {}".format(repourl)) - self.run("apt-get -qq update") + self.run("add-apt-repository -y {}".format(repourl), _try=_try) + self.run("apt-get -qq update", _try=_try) - def dnf_add_repo(self, repourl, repo=""): + def dnf_add_repo(self, repourl, repo="", _try=False): if self.run("dnf config-manager 2>/dev/null", _try=True): - self.install("dnf-plugins-core") - self.run("dnf config-manager -y --add-repo {}".format(repourl)) + self.install("dnf-plugins-core", _try=_try) + self.run("dnf config-manager -y --add-repo {}".format(repourl), _try=_try) - def zypper_add_repo(self, repourl, repo=""): + def zypper_add_repo(self, repourl, repo="", _try=False): pass - def pacman_add_repo(self, repourl, repo=""): + def pacman_add_repo(self, repourl, repo="", _try=False): pass - def brew_add_repo(self, repourl, repo=""): + def brew_add_repo(self, repourl, repo="", _try=False): pass - def add_repo(self, repourl, repo=""): + def add_repo(self, repourl, repo="", _try=False): if self.os == 'linux': if self.dist == 'fedora': - self.dnf_add_repo(repourl, repo=repo) + self.dnf_add_repo(repourl, repo=repo, _try=_try) elif self.dist == 'ubuntu' or self.dist == 'debian': - self.apt_add_repo(repourl, repo=repo) + self.apt_add_repo(repourl, repo=repo, _try=_try) elif self.dist == 'centos' or self.dist == 'redhat': - self.yum_add_repo(repourl, repo=repo) + self.yum_add_repo(repourl, repo=repo, _try=_try) elif self.dist == 'suse': - self.zypper_add_repo(repourl, repo=repo) + self.zypper_add_repo(repourl, repo=repo, _try=_try) elif self.dist == 'arch': - self.pacman_add_repo(repourl, repo=repo) + self.pacman_add_repo(repourl, repo=repo, _try=_try) else: Assert(False), "Cannot determine installer" elif self.os == 'macosx': @@ -190,26 +187,35 @@ def pip_install(self, cmd, _try=False): pip_user = '' if self.os == 'macosx': pip_user = '--user ' - self.run("pip install --disable-pip-version-check " + pip_user + cmd, output_on_error=True, _try=_try) + self.run(self.python + " -m pip install --disable-pip-version-check " + pip_user + cmd, output_on_error=True, _try=_try) def pip3_install(self, cmd, _try=False): pip_user = '' if self.os == 'macosx': pip_user = '--user ' - self.run("pip3 install --disable-pip-version-check " + pip_user + cmd, output_on_error=True, _try=_try) + self.run(self.python + " -m pip install --disable-pip-version-check " + pip_user + cmd, output_on_error=True, _try=_try) - def setup_pip(self): + def setup_pip(self, _try=False): get_pip = "set -e; wget -q https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py" - if not self.has_command("pip3"): - self.install("python3-distutils", _try=True) + if self.run(self.python + " -m pip --version", _try=True, output_on_error=False) != 0: + if sys.version_info.major == 3: + self.install("python3-distutils") self.install_downloaders() - self.run(get_pip + "; " + self.python + " /tmp/get-pip.py", output_on_error=True) + pip_user = ' --user' if self.os == 'macosx' else '' + self.run(get_pip + "; " + self.python + " /tmp/get-pip.py" + pip_user + " 'pip==19.3.1'", output_on_error=True, _try=_try) - def install_downloaders(self): + def install_downloaders(self, _try=False): if self.os == 'linux': - self.install("ca-certificates") - self.install("curl wget") - - def install_git_lfs_on_linux(self): - self.run("curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash") - self.install("git-lfs") + self.install("ca-certificates", _try=_try) + self.install("curl wget", _try=_try) + + def install_git_lfs_on_linux(self, _try=False): + self.run("set -e; wget -q https://github.com/git-lfs/git-lfs/releases/download/v2.9.2/git-lfs-linux-amd64-v2.9.2.tar.gz -O /tmp/git-lfs.tar.gz") + self.run("cd /tmp; tar xzf git-lfs.tar.gz; ./install.sh") + +# cmd = "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.{}.sh | bash" +# if self.platform.is_redhat_compat(): +# self.run(cmd.format('rpm'), _try=_try) +# elif self.platform.is_debian_compat(): +# self.run(cmd.format('deb'), _try=_try) +# self.install("git-lfs", _try=_try) diff --git a/opt/system-setup.py b/opt/system-setup.py index e7c754b0f..79fcb8e62 100755 --- a/opt/system-setup.py +++ b/opt/system-setup.py @@ -34,9 +34,14 @@ def redhat_compat(self): self.group_install("'Development Tools'") self.install("redhat-lsb-core") - self.install("epel-release") - self.install("python36 python36-pip") - self.install("python36-psutil") + if not self.dist == "amzn": + self.install("epel-release") + self.install("python36 python36-pip") + self.install("python36-psutil") + else: + self.run("amazon-linux-extras install epel", output_on_error=True) + self.install("python3 python3-devel") + self.pip_install("psutil") self.install_git_lfs_on_linux() @@ -53,10 +58,6 @@ def macosx(self): self.install("git-lfs") self.install("redis") - def install_git_lfs_on_linux(self): - self.run("curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash") - self.install("git-lfs") - def common_last(self): if not self.has_command("RLTest"): self.pip3_install("git+https://github.com/RedisLabsModules/RLTest.git@master")