diff --git a/.circleci/config.yml b/.circleci/config.yml index b9caa0c65..7da263b68 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,18 @@ version: 2.1 commands: - setup_steps: + setup-executor: + steps: + - run: + name: Setup executor + command: | + apt-get -qq update + apt-get -q install -y git openssh-client curl ca-certificates make tar gzip + bash <(curl -fsSL https://get.docker.com) + - setup_remote_docker: + docker_layer_caching: true + + build-steps: parameters: platform: type: string @@ -9,7 +20,7 @@ commands: - checkout - run: name: Submodule checkout - command: git submodule update --init --recursive + command: git submodule update --init --recursive - restore_cache: keys: - v1-dependencies-{{ checksum "get_deps.sh" }} @@ -19,50 +30,59 @@ commands: command: | ./opt/readies/bin/getpy3 BREW_NO_UPDATE=1 ./opt/system-setup.py - # git clone git://github.com/antirez/redis.git --branch 6.0.1; (cd redis; make malloc=libc -j $(nproc); make install); redis-server --version + # ./opt/readies/bin/getredis -v 6 --force ./get_deps.sh cpu - save_cache: paths: - deps key: build-dependencies-{{ checksum "get_deps.sh" }} - - run: - name: Set up workspace - command: | - mkdir -p ~/workspace - chown `whoami` ~/workspace - - test_steps: - parameters: - platform: - type: string - steps: - - checkout - - run: - name: Submodule checkout - command: git submodule update --init --recursive - run: name: Build command: make -C opt all SHOW=1 - run: name: Test command: | - mkdir -p ~/workspace/tests make -C opt test SHOW=1 no_output_timeout: 20m - run: name: Package - command: make -C opt pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=~/workspace/packages SHOW=1 + command: make -C opt pack SHOW=1 - persist_to_workspace: - root: ~/workspace + root: bin/ paths: - - 'packages/release/*.zip' - - 'packages/release/*.tgz' - - 'packages/branch/*.zip' - - 'packages/branch/*.tgz' + - artifacts/*.zip + - artifacts/*.tgz - store_artifacts: path: test/logs - deploy: + platform-build-steps: + parameters: + platform: + type: string + steps: + - setup-executor + - checkout + - run: + name: Setup + command: | + git submodule update --init --recursive + ./opt/readies/bin/getpy3 + - run: + name: Build for platform + command: | + docker login -u redisfab -p $DOCKER_REDISFAB_PWD + cd opt/build/docker + #@@ make build publish $(./version-params) CPU=1 OSNICK=<> X64=1 ARTIFACTS=1 TEST=1 VERBOSE=1 + make build publish $(./version-params) CPU=1 OSNICK=<> X64=1 ARTIFACTS=1 VERBOSE=1 + make build publish $(./version-params) GPU=1 OSNICK=<> X64=1 ARTIFACTS=1 VERBOSE=1 + no_output_timeout: 40m + - persist_to_workspace: + root: bin/ + paths: + - artifacts/*.zip + - artifacts/*.tgz + + deploy-steps: parameters: from: type: string @@ -70,18 +90,26 @@ commands: - run: name: Deploy to S3 command: | - aws s3 cp <>/ s3://redismodules/$PACKAGE_NAME/ --acl public-read --recursive --exclude "*" --include "*.zip" --include "*.tgz" + aws s3 cp artifacts/ s3://redismodules/$PACKAGE_NAME/ --acl public-read --recursive --exclude "*" --include "*.zip" --include "*.tgz" jobs: - build: + build-debian: docker: - image: redisfab/rmbuilder:6.0.1-x64-buster steps: - - setup_steps: - platform: debian - - test_steps: + - build-steps: platform: debian + platform-build: + parameters: + platform: + type: string + docker: + - image: debian:buster + steps: + - platform-build-steps: + platform: <> + coverage: docker: - image: redisfab/rmbuilder:6.0.1-x64-buster @@ -99,13 +127,8 @@ jobs: command: | ./opt/readies/bin/getpy3 ./opt/system-setup.py - # git clone git://github.com/antirez/redis.git --branch 6.0.1; cd redis; make malloc=libc -j $(nproc); make install; redis-server --version + # ./opt/readies/bin/getredis -v 6 --force ./get_deps.sh cpu - - run: - name: Set up workspace - command: | - mkdir -p ~/workspace - chown `whoami` ~/workspace - run: name: Build for coverage command: | @@ -119,21 +142,19 @@ jobs: build-macos: macos: - xcode: 11.1.0 + xcode: 11.3.0 steps: - run: name: Fix macOS Python installation command: | brew reinstall -f python2 - - setup_steps: - platform: macosx - - test_steps: + - build-steps: platform: macosx build-multiarch-docker: machine: enabled: true - image: ubuntu-1604:201903-01 + image: cimg/base:2020.01 steps: - checkout - run: @@ -155,10 +176,11 @@ jobs: name: Build command: | sudo docker login -u redisfab -p $DOCKER_REDISFAB_PWD - make -C opt/build/docker build - sudo make -C opt/build/docker publish + cd opt/build/docker + make build + sudo make publish - build-gpu: + build-and-test-gpu: machine: enabled: true docker_layer_caching: true @@ -176,13 +198,13 @@ jobs: - run: name: Test command: | - mkdir -p ~/workspace/tests - docker run --gpus all -v $HOME/workspace/tests:/build/test/logs -it --rm redisai-gpu:latest-x64-bionic-test + mkdir -p $HOME/tests + docker run --gpus all -v $HOME/tests:/build/test/logs -it --rm redisai-gpu:latest-x64-bionic-test no_output_timeout: 30m - store_artifacts: - path: ~/workspace/tests + path: tests - deploy_package: + deploy-artifacts: parameters: package: type: string @@ -190,68 +212,106 @@ jobs: - image: redisfab/rmbuilder:6.0.1-x64-buster steps: - attach_workspace: - at: ~/workspace - - deploy: - from: ~/workspace/packages/<> - - store_artifacts: - path: ~/workspace/packages/<> + at: workspace + - run: + name: Deploy to S3 + command: | + cd workspace + aws s3 cp artifacts/ s3://redismodules/$PACKAGE_NAME/ --acl public-read --recursive --exclude "*" --include "*.zip" --include "*.tgz" + + +on-any-branch: &on-any-branch + filters: + branches: + only: /.*/ + tags: + only: /.*/ + +never: &never + filters: + branches: + ignore: /.*/ + tags: + ignore: /.*/ + +on-master: &on-master + filters: + branches: + only: master + +on-version-tags: &on-version-tags + filters: + branches: + ignore: /.*/ + tags: + only: /^v[0-9].*/ + +on-master-and-version-tags: &on-master-and-version-tags + filters: + branches: + only: + - master + tags: + only: /^v[0-9].*/ + + +platform-build-defs: &platform-build-defs + requires: + - build-debian + #@@ <<: *on-master-and-version-tags + <<: *on-any-branch + +after-platform-builds: &after-platform-builds + requires: + - build-centos7 + - build-bionic + - build-xenial + workflows: version: 2 build_and_package: jobs: - - build: - filters: - tags: - only: /.*/ + - build-debian: + <<: *on-any-branch + - platform-build: + name: build-centos7 + platform: centos7 + <<: *platform-build-defs + - platform-build: + name: build-bionic + platform: bionic + <<: *platform-build-defs + - platform-build: + name: build-xenial + platform: xenial + <<: *platform-build-defs - coverage: - filters: - branches: - only: /.*/ - tags: - only: /.*/ - # - build-macos: - # filters: - # branches: - # ignore: /.*/ - # tags: - # only: /^v[0-9].*/ - #- build-multiarch-docker: - # filters: - # tags: - # only: /^v[0-9].*/ - # branches: - # ignore: /.*/ - - build-gpu: - filters: - tags: - only: /.*/ - - deploy_package: - name: deploy_branch + <<: *on-any-branch + - build-and-test-gpu: + <<: *on-any-branch + - build-macos: + <<: *never + # <<: *on-version-tags + - build-multiarch-docker: + <<: *never + # <<: *on-version-tags + - deploy-artifacts: + name: deploy-branch package: branch - requires: - - build - filters: - branches: - only: master - - deploy_package: - name: deploy_release + <<: *after-platform-builds + <<: *on-any-branch + - deploy-artifacts: + name: deploy-release package: release - requires: - - build - filters: - branches: - ignore: /.*/ - tags: - only: '/^v[0-9].*/' + <<: *after-platform-builds + <<: *on-version-tags - #nightly: - # triggers: - # - schedule: - # cron: "20 17 * * *" - # filters: - # branches: - # only: - # - master - # jobs: - # - build-macos + nightly: + triggers: + - schedule: + cron: "20 17 * * *" + <<: *on-master + jobs: + - build-macos: + <<: *never diff --git a/Dockerfile b/Dockerfile index dbc7362fd..7153708bf 100755 --- a/Dockerfile +++ b/Dockerfile @@ -11,10 +11,20 @@ ARG OS=debian:buster-slim # ARCH=x64|arm64v8|arm32v7 ARG ARCH=x64 +ARG PACK=0 +ARG TEST=0 + #---------------------------------------------------------------------------------------------- FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis FROM ${OS} AS builder +ARG OSNICK +ARG OS +ARG ARCH +ARG REDIS_VER + +RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH} [with Redis ${REDIS_VER}]" + WORKDIR /build COPY --from=redis /usr/local/ /usr/local/ @@ -30,18 +40,29 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_ ARG BUILD_ARGS="" ADD ./ /build -RUN make -C opt build $BUILD_ARGS SHOW=1 +RUN set -e ;\ + . ./opt/readies/bin/sourced ./profile.d ;\ + make -C opt build $BUILD_ARGS SHOW=1 -ARG PACK=0 -ARG TEST=0 +ARG PACK +ARG TEST +RUN mkdir -p bin/artifacts RUN if [ "$PACK" = "1" ]; then make -C opt pack; fi -RUN if [ "$TEST" = "1" ]; then make -C opt test $BUILD_ARGS NO_LFS=1; fi + +RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test $BUILD_ARGS NO_LFS=1; fi #---------------------------------------------------------------------------------------------- FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} -RUN set -e; apt-get -qq update; apt-get -q install -y libgomp1 +ARG OSNICK +ARG OS +ARG ARCH +ARG REDIS_VER +ARG PACK + +RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi +RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi ENV REDIS_MODULES /usr/lib/redis/modules ENV LD_LIBRARY_PATH $REDIS_MODULES @@ -50,6 +71,8 @@ RUN mkdir -p $REDIS_MODULES/ COPY --from=builder /build/install-cpu/ $REDIS_MODULES/ +COPY --from=builder /build/bin/artifacts/ /var/opt/redislabs/artifacts + WORKDIR /data EXPOSE 6379 CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so"] diff --git a/Dockerfile.arm b/Dockerfile.arm index 64f9fbde3..dd08fc937 100755 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -11,12 +11,24 @@ ARG OS=debian:buster-slim # ARCH=arm64v8|arm32v7 ARG ARCH=arm64v8 +ARG PACK=0 +ARG TEST=0 + #---------------------------------------------------------------------------------------------- -FROM redisfab/redis-xbuild:${REDIS_VER}-${ARCH}-${OSNICK} AS builder +FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis +FROM redisfab/xbuild:${ARCH}-${OS} AS builder + +ARG OSNICK +ARG OS +ARG ARCH +ARG REDIS_VER RUN [ "cross-build-start" ] +RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH}" + WORKDIR /build +COPY --from=redis /usr/local/ /usr/local/ COPY ./opt/ opt/ COPY ./test/test_requirements.txt test/ @@ -30,13 +42,15 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_ ARG BUILD_ARGS="" ADD ./ /build -RUN make -C opt build $BUILD_ARGS SHOW=1 +RUN set -e ;\ + . ./opt/readies/bin/sourced ./profile.d ;\ + make -C opt build $BUILD_ARGS SHOW=1 -ARG PACK=0 -ARG TEST=0 +ARG PACK +ARG TEST RUN if [ "$PACK" = "1" ]; then make -C opt pack; fi -RUN if [ "$TEST" = "1" ]; then make -C opt test $BUILD_ARGS NO_LFS=1; fi +RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test $BUILD_ARGS NO_LFS=1; fi RUN [ "cross-build-end" ] @@ -45,7 +59,14 @@ FROM redisfab/redis-xbuild:${REDIS_VER}-${ARCH}-${OSNICK} RUN [ "cross-build-start" ] -RUN set -e; apt-get -qq update; apt-get -q install -y libgomp1 +ARG OSNICK +ARG OS +ARG ARCH +ARG REDIS_VER +ARG PACK + +RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi +RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi ENV REDIS_MODULES /usr/lib/redis/modules ENV LD_LIBRARY_PATH $REDIS_MODULES diff --git a/Dockerfile.gpu b/Dockerfile.gpu index 7e8d73b27..87179c48b 100644 --- a/Dockerfile.gpu +++ b/Dockerfile.gpu @@ -8,11 +8,25 @@ ARG OSNICK=bionic # ARCH=x64|arm64v8|arm32v7 ARG ARCH=x64 +# OS=ubuntu18.04|ubuntu16.04|centos7 +ARG OS=ubuntu18.04 + +ARG CUDA_VER=10.1-cudnn7 + +ARG PACK=0 +ARG TEST=0 + #---------------------------------------------------------------------------------------------- FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis -FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04 AS builder +FROM nvidia/cuda:${CUDA_VER}-devel-${OS} AS builder + +ARG OSNICK +ARG OS +ARG REDIS_VER +ARG CUDA_VER + +RUN echo "Building for ${OSNICK} (${OS}) [with Redis ${REDIS_VER}]" -# ENV DEPS "git ca-certificates wget unzip cmake libgomp1 patchelf coreutils tcl libjemalloc-dev" ENV NVIDIA_VISIBLE_DEVICES all ENV NVIDIA_DRIVER_CAPABILITIES compute,utility @@ -31,24 +45,34 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_ ARG BUILD_ARGS="" ADD ./ /build -RUN make -C opt build GPU=1 $BUILD_ARGS SHOW=1 +RUN set -e ;\ + . ./opt/readies/bin/sourced ./profile.d ;\ + make -C opt build GPU=1 $BUILD_ARGS SHOW=1 -ARG PACK=0 -ARG TEST=0 +ARG PACK +ARG TEST +RUN mkdir -p bin/artifacts RUN if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi -RUN if [ "$TEST" = "1" ]; then TEST="" make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1; fi + +RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1; fi #---------------------------------------------------------------------------------------------- -FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04 +FROM nvidia/cuda:${CUDA_VER}-runtime-${OS} -RUN set -e; apt-get -qq update; apt-get -q install -y libgomp1 +ARG OS + +RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi +RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi ENV REDIS_MODULES /usr/lib/redis/modules RUN mkdir -p $REDIS_MODULES/ COPY --from=redis /usr/local/ /usr/local/ COPY --from=builder /build/install-gpu/ $REDIS_MODULES/ + +COPY --from=builder /build/bin/artifacts/ /var/opt/redislabs/artifacts + # COPY --from=builder /usr/local/etc/redis/redis.conf /usr/local/etc/redis/ WORKDIR /data diff --git a/Dockerfile.gpu-test b/Dockerfile.gpu-test index 20dcf0a35..943a03bd0 100644 --- a/Dockerfile.gpu-test +++ b/Dockerfile.gpu-test @@ -8,9 +8,15 @@ ARG OSNICK=bionic # ARCH=x64|arm64v8|arm32v7 ARG ARCH=x64 +# OS=ubuntu18.04|ubuntu16.04|centos7 +ARG OS=ubuntu18.04 + +ARG CUDA_VER=10.1-cudnn7 + +ARG PACK=1 #---------------------------------------------------------------------------------------------- FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis -FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04 AS builder +FROM nvidia/cuda:${CUDA_VER}-devel-${OS} AS builder SHELL ["/bin/bash", "-c"] @@ -43,10 +49,11 @@ RUN set -e ;\ ARG BUILD_ARGS="" ADD ./ /build RUN set -e ;\ - . venv/bin/activate ;\ - make -C opt build GPU=1 $BUILD_ARGS SHOW=1 + . venv/bin/activate ;\ + . ./opt/readies/bin/sourced ./profile.d ;\ + make -C opt build GPU=1 $BUILD_ARGS SHOW=1 -ARG PACK=1 +ARG PACK RUN set -e ;\ if [[ $PACK == 1 ]]; then \ diff --git a/opt/Makefile b/opt/Makefile index 47bfc81ff..ae638ab3f 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -47,7 +47,6 @@ make cov-upload # upload coverage data to codecov.io (requires CODECOV_TOKEN) make pack # create installation packages PACK_DEPS=0 # do not pack dependencies - INTO=dir # place artifacts in `dir` BRANCH=name # use `name` as branch name make deploy # copy packages to S3 make release # release a version @@ -178,10 +177,11 @@ fetch deps: pack: $(INSTALLED_TARGET) $(SHOW)find $(INSTALL_DIR) -name "*.so" -exec chmod +x {} \; + $(SHOW)mkdir -p $(ROOT)/bin/artifacts ifneq ($(PACK_DEPS),0) - $(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) BRANCH=$(BRANCH) INTO=$(INTO) DEPS=1 ./pack.sh + $(SHOW)DEVICE=$(DEVICE) BINDIR=$(ROOT)/bin/artifacts INSTALL_DIR=$(INSTALL_DIR) BRANCH=$(BRANCH) DEPS=1 ./pack.sh else - $(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) BRANCH=$(BRANCH) INTO=$(INTO) DEPS=0 ./pack.sh + $(SHOW)DEVICE=$(DEVICE) BINDIR=$(ROOT)/bin/artifacts INSTALL_DIR=$(INSTALL_DIR) BRANCH=$(BRANCH) DEPS=0 ./pack.sh endif #---------------------------------------------------------------------------------------------- diff --git a/opt/build/docker/Makefile b/opt/build/docker/Makefile index 37401ee30..afff92f26 100755 --- a/opt/build/docker/Makefile +++ b/opt/build/docker/Makefile @@ -1,22 +1,15 @@ -.NOTPARALLEL: - ROOT=../../.. +include $(ROOT)/opt/readies/mk/main -ifeq ($(VERSION),) -VERSION:=$(patsubst v%,%,$(shell git describe --tags `git rev-list --tags --max-count=1`)) -endif -ifeq ($(VERSION),) -$(error Cannot determine version. Aborting.) -endif - -# OSNICK=buster|stretch|bionic +# OSNICK=buster|stretch|bionic|xenial|centos7 OSNICK ?= buster REPO=redisfab -STEM=$(REPO)/redisai-cpu -BUILD_OPT=--rm -# --squash +STEM=$(REPO)/redisai + +ART_DIR=$(ROOT)/bin/artifacts +ART_INT_DIR=/var/opt/redislabs/artifacts FETCH_ARGS.arm64v8=WITH_ORT=0 WITH_PT=0 BUILD_ARGS.arm64v8=WITH_ORT=0 WITH_PT=0 @@ -24,95 +17,25 @@ BUILD_ARGS.arm64v8=WITH_ORT=0 WITH_PT=0 FETCH_ARGS.arm32v7=WITH_ORT=0 WITH_PT=0 BUILD_ARGS.arm32v7=WITH_ORT=0 WITH_PT=0 -PACK ?= 1 -TEST ?= 1 - -#---------------------------------------------------------------------------------------------- - -define targets # (1=OP, 2=op) -$(1)_TARGETS := -$(1)_TARGETS += $(if $(findstring $(X64),1),$(2)_x64) -$(1)_TARGETS += $(if $(findstring $(ARM7),1),$(2)_arm32v7) -$(1)_TARGETS += $(if $(findstring $(ARM8),1),$(2)_arm64v8) - -$(1)_TARGETS += $$(if $$(strip $$($(1)_TARGETS)),,$(2)_x64 $(2)_arm32v7 $(2)_arm64v8) -endef - -$(eval $(call targets,BUILD,build)) -$(eval $(call targets,PUSH,push)) - -#---------------------------------------------------------------------------------------------- - -define build_x64 # (1=arch) -build_$(1): - docker build $(BUILD_OPT) -t $(STEM)-$(OSNICK):$(VERSION)-x64 -f $(ROOT)/Dockerfile \ - --build-arg DEPS_ARGS="$(FETCH_ARGS.x64)" \ - --build-arg BUILD_ARGS="$(BUILD_ARGS.x64)" \ - --build-arg PACK="$(PACK)" \ - --build-arg TEST="$(TEST)" \ - $(ROOT) +ifeq ($(CPU),1) +VAR=cpu -.PHONY: build_$(1) -endef +else ifeq ($(GPU),1) +VAR=gpu +DOCKERFILE_STEM ?= $(ROOT)/Dockerfile.gpu -define build_arm # (1=arch) -build_$(1): - docker build $(BUILD_OPT) -t $(STEM)-$(OSNICK):$(VERSION)-$(1) -f $(ROOT)/Dockerfile.arm \ - --build-arg ARCH=$(1) \ - --build-arg DEPS_ARGS="$(FETCH_ARGS.$(1))" \ - --build-arg BUILD_ARGS="$(BUILD_ARGS.$(1))" \ - --build-arg PACK="$(PACK)" \ - --build-arg TEST="$(TEST)" \ - $(ROOT) +OS.custom:=1 -.PHONY: build_$(1) -endef +OS.centos7=centos7 +OS.bionic=ubuntu18.04 +OS.xenial=ubuntu16.04 -define push # (1=arch) -push_$(1): - docker push $(STEM)-$(OSNICK):$(VERSION)-$(1) - -.PHONY: push_$(1) -endef - -define create_manifest # (1=version) -docker manifest create -a $(STEM)-$(OSNICK):$(1) \ - -a $(STEM)-$(OSNICK):$(VERSION)-x64 \ - -a $(STEM)-$(OSNICK):$(VERSION)-arm64v8 \ - -a $(STEM)-$(OSNICK):$(VERSION)-arm32v7 -docker manifest annotate $(STEM)-$(OSNICK):$(1) $(STEM)-$(OSNICK):$(VERSION)-arm32v7 --os linux --arch arm --variant v7 -docker manifest annotate $(STEM)-$(OSNICK):$(1) $(STEM)-$(OSNICK):$(VERSION)-arm64v8 --os linux --arch arm64 --variant v8 -docker manifest push -p $(STEM)-$(OSNICK):$(1) -endef - -#---------------------------------------------------------------------------------------------- - -all: build publish - -$(eval $(call build_x64,x64)) -$(eval $(call build_arm,arm64v8)) -$(eval $(call build_arm,arm32v7)) - -$(eval $(call push,x64)) -$(eval $(call push,arm64v8)) -$(eval $(call push,arm32v7)) - -build: $(BUILD_TARGETS) - -ifneq ($(PUSH),0) -publish: $(PUSH_TARGETS) else -publish: -endif - $(call create_manifest,$(VERSION)) - $(call create_manifest,latest) - -show: - @echo "$(STEM)-$(OSNICK):" -ifeq ($(INSPECT),1) - @docker manifest inspect $(STEM)-$(OSNICK):$(VERSION) | jq -else - @curl -s -X "GET" https://cloud.docker.com/v2/repositories/$(STEM)-$(OSNICK)/tags/ | jq +VAR=cpu endif -.PHONY: build publish show +DOCKER_BUILD_ARGS += \ + DEPS_ARGS="$(FETCH_ARGS.$(ARCH))" \ + BUILD_ARGS="$(BUILD_ARGS.$(ARCH))" + +include $(MK)/docker diff --git a/opt/build/docker/version-params b/opt/build/docker/version-params new file mode 100755 index 000000000..a5b6bed07 --- /dev/null +++ b/opt/build/docker/version-params @@ -0,0 +1,12 @@ +#!/bin/bash + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +ROOT=$(cd $HERE/../..; pwd) + +tag=`git describe --abbrev=0 2> /dev/null | sed 's/^v\(.*\)/\1/'` +if [[ $? != 0 || -z $tag ]]; then + branch=`git rev-parse --abbrev-ref HEAD` + echo "VERSION=$branch" +else + echo "VERSION=$tag LATEST=1" +fi diff --git a/opt/pack.sh b/opt/pack.sh index 9f2fe557f..12a88245d 100755 --- a/opt/pack.sh +++ b/opt/pack.sh @@ -6,7 +6,6 @@ [[ -z $DEVICE ]] && { echo DEVICE undefined; exit 1; } [[ -z $BINDIR ]] && { echo BINDIR undefined; exit 1; } [[ -z $INSTALL_DIR ]] && { echo INSTALL_DIR undefined; exit 1; } -[[ ! -z $INTO ]] && INTO=$(realpath $INTO) HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . $HERE/readies/shibumi/functions @@ -18,8 +17,7 @@ REDIS_ENT_LIB_PATH=/opt/redislabs/lib BINDIR=$(realpath $BINDIR) INSTALL_DIR=$(realpath $INSTALL_DIR) -export LC_ALL=C.UTF-8 -export LANG=C.UTF-8 +. $ROOT/opt/readies/bin/enable-utf8 export ARCH=$($ROOT/opt/readies/bin/platform --arch) export OS=$($ROOT/opt/readies/bin/platform --os) @@ -30,8 +28,12 @@ pack_ramp() { cd $ROOT local STEM=$PRODUCT.$OS-$OSNICK-$ARCH - local FQ_VER=$GIT_VER - local FQ_PACKAGE=$STEM.$FQ_VER + local FQ_PACKAGE + if [[ -z $BRANCH ]]; then + FQ_PACKAGE=$STEM.$VERSION + else + FQ_PACKAGE=$STEM.$BRANCH + fi # this is only to extract {semantic_version} into VERSION RAMPOUT=$(mktemp /tmp/ramp.XXXXXX) @@ -39,20 +41,16 @@ pack_ramp() { local rampfile=`realpath $(tail -1 $RAMPOUT)` rm -f $rampfile $RAMPOUT echo `basename $rampfile` | sed -e "s/[^.]*\.[^.]*\.\(.*\)\.zip/\1/" > $BINDIR/VERSION - export VERSION=$(cat $BINDIR/VERSION) + export SEMVER=$(cat $BINDIR/VERSION) $RAMP_PROG pack -m $ROOT/ramp.yml -o $BINDIR/$FQ_PACKAGE.zip \ - -c "BACKENDSPATH $REDIS_ENT_LIB_PATH/$PRODUCT-$DEVICE-$VERSION/backends" $INSTALL_DIR/$PRODUCT.so > /dev/null 2>&1 + -c "BACKENDSPATH $REDIS_ENT_LIB_PATH/$PRODUCT-$DEVICE-$SEMVER/backends" $INSTALL_DIR/$PRODUCT.so > /dev/null 2>&1 cd "$BINDIR" - ln -sf $FQ_PACKAGE.zip $STEM.$VERSION.zip - ln -sf $FQ_PACKAGE.zip $STEM.latest.zip - # [[ ! -z $BRANCH ]] && ln -sf $FQ_PACKAGE.zip $STEM.$BRANCH.zip + if [[ -z $BRANCH ]]; then + ln -sf $FQ_PACKAGE.zip $STEM.latest.zip + fi - export RELEASE_ARTIFACTS="$RELEASE_ARTIFACTS $STEM.$VERSION.zip $RAMP_STEM.latest.zip" - export DEV_ARTIFACTS="$DEV_ARTIFACTS $FQ_PACKAGE.zip $STEM.$BRANCH.zip" - # [[ ! -z $BRANCH ]] && export DEV_ARTIFACTS="$DEV_ARTIFACTS $DEPS.$BRANCH.tgz" - echo "Done." } @@ -61,15 +59,18 @@ pack_deps() { cd $ROOT - VERSION=$(cat $BINDIR/VERSION) - local VARIANT=$OS-$OSNICK-$ARCH.$GIT_VER + SEMVER=$(cat $BINDIR/VERSION) local STEM=$PRODUCT-$DEVICE-dependencies.$OS-$OSNICK-$ARCH - local FQ_VER=$GIT_VER - local FQ_PACKAGE=$STEM.$FQ_VER + local FQ_PACKAGE + if [[ -z $BRANCH ]]; then + FQ_PACKAGE=$STEM.$VERSION + else + FQ_PACKAGE=$STEM.$BRANCH + fi cd $INSTALL_DIR - local BACKENDS_DIR=$PRODUCT-$DEVICE-$VERSION + local BACKENDS_DIR=$PRODUCT-$DEVICE-$SEMVER if [[ ! -h backends ]]; then [[ ! -d backends ]] && { echo "install-$DEVICE/backend directory not found." ; exit 1; } rm -rf $BACKENDS_DIR @@ -81,14 +82,10 @@ pack_deps() { find $BACKENDS_DIR -name "*.so*" | xargs tar pczf $BINDIR/$FQ_PACKAGE.tgz cd "$BINDIR" - ln -sf $FQ_PACKAGE.tgz $STEM.$VERSION.tgz - ln -sf $FQ_PACKAGE.tgz $STEM.latest.tgz - # [[ ! -z $BRANCH ]] && ln -sf $FQ_PACKAGE.tgz $STEM.$BRANCH.tgz + if [[ -z $BRANCH ]]; then + ln -sf $FQ_PACKAGE.tgz $STEM.latest.tgz + fi - export RELEASE_ARTIFACTS="$RELEASE_ARTIFACTS $STEM.$VERSION.tgz $STEM.latest.tgz" - export DEV_ARTIFACTS="$DEV_ARTIFACTS $FQ_PACKAGE.tgz" - # [[ ! -z $BRANCH ]] && export DEV_ARTIFACTS="$DEV_ARTIFACTS $STEM.$BRANCH.tgz" - echo "Done." } @@ -101,7 +98,6 @@ if [[ $1 == --help || $1 == help ]]; then INSTALL_DIR=dir directory in which artifacts are found DEVICE=cpu|gpu BRANCH=branch branch names to serve as an exta package tag - INTO=dir package destination directory (optinal) RAMP=1 build RAMP file DEPS=1 build dependencies file RAMP_PROG path to RAMP program @@ -113,24 +109,37 @@ fi PRODUCT=redisai PRODUCT_LIB=$PRODUCT.so -GIT_VER="" -if [[ -d $ROOT/.git ]]; then - if [[ ! -z $BRANCH ]]; then - GIT_BRANCH="$BRANCH" +if [[ -z $BRANCH ]]; then + tag=`git describe --abbrev=0 2> /dev/null | sed 's/^v\(.*\)/\1/'` + if [[ $? != 0 || -z $tag ]]; then + BRANCH=`git rev-parse --abbrev-ref HEAD` + VERSION= else - GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + VERSION=$tag fi - GIT_COMMIT=$(git describe --always --abbrev=7 --dirty="+") - GIT_VER="${GIT_BRANCH}-${GIT_COMMIT}" else - if [[ ! -z $BRANCH ]]; then - GIT_BRANCH="$BRANCH" - else - GIT_BRANCH=unknown - fi - GIT_VER="$GIT_BRANCH" + VERSION= fi +# GIT_VER="" +# if [[ -d $ROOT/.git ]]; then +# if [[ ! -z $BRANCH ]]; then +# GIT_BRANCH="$BRANCH" +# else +# GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) +# fi +# GIT_COMMIT=$(git describe --always --abbrev=7 --dirty="+") +# # GIT_VER="${GIT_BRANCH}-${GIT_COMMIT}" +# GIT_VER="${GIT_BRANCH}" +# else +# if [[ ! -z $BRANCH ]]; then +# GIT_BRANCH="$BRANCH" +# else +# GIT_BRANCH=unknown +# fi +# GIT_VER="$GIT_BRANCH" +# fi + if ! command -v redis-server > /dev/null; then echo "Cannot find redis-server. Aborting." exit 1 @@ -139,18 +148,4 @@ fi pack_ramp [[ $DEPS == 1 ]] && pack_deps -if [[ ! -z $INTO ]]; then - mkdir -p $INTO - cd $INTO - mkdir -p release branch - - for f in $RELEASE_ARTIFACTS; do - [[ -f $BINDIR/$f ]] && cp $BINDIR/$f release/ - done - - for f in $DEV_ARTIFACTS; do - [[ -f $BINDIR/$f ]] && cp $BINDIR/$f branch/ - done -fi - exit 0 diff --git a/opt/readies b/opt/readies index 73e76f5c4..14abdad48 160000 --- a/opt/readies +++ b/opt/readies @@ -1 +1 @@ -Subproject commit 73e76f5c42fcf70cbe2d181eb8f1ce8d348f7be7 +Subproject commit 14abdad4868c9adb011942830ffe748d1c05f073 diff --git a/opt/system-setup.py b/opt/system-setup.py index 26fe98ca7..c84aacaaf 100755 --- a/opt/system-setup.py +++ b/opt/system-setup.py @@ -2,10 +2,11 @@ import sys import os -from subprocess import Popen, PIPE import argparse -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "readies")) +HERE = os.path.abspath(os.path.dirname(__file__)) +ROOT = os.path.abspath(os.path.join(HERE, "..")) +sys.path.insert(0, os.path.join(HERE, "readies")) import paella #---------------------------------------------------------------------------------------------- @@ -22,53 +23,65 @@ def common_first(self): if self.os == 'linux': self.install("ca-certificates") - self.install("git cmake unzip wget patchelf awscli") + self.install("git unzip wget patchelf awscli") self.install("coreutils") # for realpath def debian_compat(self): - self.install("build-essential") + self.install("build-essential cmake") + self.install("python3-regex") self.install("python3-venv python3-psutil python3-networkx python3-numpy") # python3-skimage self.install_git_lfs_on_linux() def redhat_compat(self): - self.group_install("'Development Tools'") self.install("redhat-lsb-core") + self.run("%s/readies/bin/enable-utf8" % HERE) + + self.group_install("'Development Tools'") + self.install("cmake3") + self.run("ln -s `command -v cmake3` /usr/local/bin/cmake") + + self.install("centos-release-scl") + self.install("devtoolset-8") + self.run("cp /opt/rh/devtoolset-8/enable /etc/profile.d/scl-devtoolset-8.sh") + paella.mkdir_p("%s/profile.d" % ROOT) + self.run("cp /opt/rh/devtoolset-8/enable %s/profile.d/scl-devtoolset-8.sh" % ROOT) if not self.dist == "amzn": self.install("epel-release") - self.install("python36 python36-pip") + self.install("python3-devel libaec-devel") self.install("python36-psutil") else: self.run("amazon-linux-extras install epel", output_on_error=True) - self.install("python3 python3-devel") + self.install("python3-devel") self.pip3_install("psutil") self.install_git_lfs_on_linux() def fedora(self): self.group_install("'Development Tools'") + self.install("cmake") + self.run("ln -s `command -v cmake3` /usr/local/bin/cmake") self.install("python3-venv python3-psutil python3-networkx") self.install_git_lfs_on_linux() def macosx(self): - p = Popen('xcode-select -p', stdout=PIPE, close_fds=True, shell=True) - out, _ = p.communicate() - if out.splitlines() == []: + if sh('xcode-select -p') == '': fatal("Xcode tools are not installed. Please run xcode-select --install.") self.install_gnu_utils() + self.install("cmake") self.install("git-lfs") self.install("redis") def common_last(self): - self.run("python3 -m pip uninstall -y ramp-packer RLTest") + self.run("python3 -m pip uninstall -y ramp-packer RLTest || true") # redis-py-cluster should be installed from git due to redis-py dependency self.pip3_install("--no-cache-dir git+https://github.com/Grokzen/redis-py-cluster.git@master") self.pip3_install("--no-cache-dir git+https://github.com/RedisLabsModules/RLTest.git@master") self.pip3_install("--no-cache-dir git+https://github.com/RedisLabs/RAMP@master") - root = os.path.join(os.path.dirname(__file__), "..") - self.pip3_install("-r {}/test/test_requirements.txt".format(root)) + self.pip3_install("-r %s/readies/paella/requirements.txt" % HERE) + self.pip3_install("-r %s/test/test_requirements.txt" % ROOT) self.pip3_install("mkdocs mkdocs-material mkdocs-extensions")