diff --git a/.circleci/config.yml b/.circleci/config.yml index 774eaf1ed..ef2673482 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,13 +70,18 @@ commands: - store_artifacts: path: tests/logs - platform-build-steps: - parameters: - platform: - type: string + platforms-build-steps: steps: - - setup-executor - checkout + - run: + name: Relocate docker overlay2 dir + command: | + sudo systemctl stop docker + sudo mkdir -p /var2/lib/docker + sudo mv /var/lib/docker/overlay2 /var2/lib/docker + sudo mkdir /var/lib/docker/overlay2 + sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2 + sudo systemctl start docker - run: name: Setup command: | @@ -87,15 +92,16 @@ commands: command: | docker login -u redisfab -p $DOCKER_REDISFAB_PWD pushd opt/build/docker - #@@ make build publish $(./version-params) CPU=1 OSNICK=<> X64=1 ARTIFACTS=1 TEST=1 VERBOSE=1 - make build publish $(./version-params) CPU=1 OSNICK=<> X64=1 ARTIFACTS=1 VERBOSE=1 - make build publish $(./version-params) GPU=1 OSNICK=<> X64=1 ARTIFACTS=1 VERBOSE=1 + for osnick in bionic xenial; do + make CPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 + make GPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 + done popd > /dev/null logstar=bin/artifacts/tests-logs-cpu.tgz logsdir=tests/logs/cpu mkdir -p $logsdir if [[ -e $logstar ]]; then tar -C $logsdir -xzf $logstar; fi - (cd bin/artifacts; tar -cf snapshots-<>.tar snapshots/) + (cd bin/artifacts; tar -cf snapshots.tar snapshots/) no_output_timeout: 40m - persist_to_workspace: root: bin/ @@ -106,27 +112,6 @@ commands: - store_artifacts: path: test/logs - dockerhub-build-steps: - parameters: - build_args: - type: string - steps: - - setup-executor - - checkout - - run: - name: Setup - command: | - git submodule update --init --recursive - ./opt/readies/bin/getpy3 - - run: - name: Build for platform - command: | - docker login -u redisfab -p $DOCKER_REDISFAB_PWD - cd opt/build/docker - make build publish $(./version-params) CPU=1 OSNICK=bionic X64=1 REPO=redislabs VERBOSE=1 <> - make build publish $(./version-params) GPU=1 OSNICK=bionic X64=1 REPO=redislabs VERBOSE=1 <> - no_output_timeout: 40m - deploy-steps: parameters: from: @@ -159,15 +144,14 @@ jobs: - build-steps: platform: debian - platform-build: - parameters: - platform: - type: string - docker: - - image: debian:buster + platforms-build: + machine: + enabled: true + docker_layer_caching: true + resource_class: gpu.nvidia.small + image: ubuntu-1604-cuda-11.1:202012-01 steps: - - platform-build-steps: - platform: <> + - platforms-build-steps coverage: docker: @@ -281,6 +265,15 @@ jobs: - run: name: Submodule checkout command: git submodule update --init --recursive + - run: + name: Relocate docker overlay2 dir + command: | + sudo systemctl stop docker + sudo mkdir -p /var2/lib/docker + sudo mv /var/lib/docker/overlay2 /var2/lib/docker + sudo mkdir /var/lib/docker/overlay2 + sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2 + sudo systemctl start docker - run: name: Build command: | @@ -292,17 +285,7 @@ jobs: docker run --gpus all -v $HOME/tests:/build/tests/logs -it --rm redisai-gpu:latest-x64-bionic-test no_output_timeout: 40m - store_artifacts: - path: test/log - - dockerhub-build: - parameters: - build_args: - type: string - docker: - - image: debian:buster - steps: - - dockerhub-build-steps: - build_args: <> + path: tests/logs deploy-artifacts: parameters: @@ -411,9 +394,7 @@ after-build-and-test: &after-build-and-test after-platform-builds: &after-platform-builds requires: - # - build-centos7 - - build-bionic - - build-xenial + - platforms-build platform-build-defs: &platform-build-defs <<: *after-build-and-test @@ -430,17 +411,7 @@ workflows: - build-and-test: <<: *on-any-branch <<: *after-linter - - platform-build: - name: build-centos7 - platform: centos7 - <<: *never # temporarily disabled - - platform-build: - name: build-bionic - platform: bionic - <<: *platform-build-defs - - platform-build: - name: build-xenial - platform: xenial + - platforms-build: <<: *platform-build-defs - coverage: context: common @@ -474,22 +445,10 @@ workflows: context: common <<: *after-platform-builds <<: *on-integ-branch - - dockerhub-build: - name: dockerhub-edge - context: common - build_args: EDGE=1 - <<: *after-build-and-test - <<: *on-master - deploy-release: context: common <<: *after-platform-builds <<: *on-version-tags - - dockerhub-build: - name: dockerhub-release - context: common - build_args: LATEST=1 - <<: *after-platform-builds - <<: *on-version-tags nightly: triggers: diff --git a/opt/build/docker/Makefile b/opt/build/docker/Makefile index 44ec61812..672becaf8 100755 --- a/opt/build/docker/Makefile +++ b/opt/build/docker/Makefile @@ -6,35 +6,23 @@ include $(ROOT)/opt/readies/mk/main # OSNICK=buster|stretch|bionic|xenial|centos7 OSNICK ?= buster -ifeq ($(REPO),redislabs) -SIMPLE ?= 1 -else -REPO=redisfab -endif - -STEM=$(REPO)/redisai +REPO=redisai -LATEST_BRANCH=1.2 +# LATEST_BRANCH=1.2 +INT_BRANCHES=1.2 REDIS_VER:=6.0.9 ART_DIR=$(ROOT)/bin/artifacts ART_INT_DIR=/var/opt/redislabs/artifacts -FETCH_ARGS.arm64v8=WITH_ORT=0 WITH_PT=0 -BUILD_ARGS.arm64v8=WITH_ORT=0 WITH_PT=0 - -FETCH_ARGS.arm32v7=WITH_ORT=0 WITH_PT=0 -BUILD_ARGS.arm32v7=WITH_ORT=0 WITH_PT=0 +#---------------------------------------------------------------------------------------------- ifeq ($(CPU),1) -VAR=cpu - -FETCH_ARGS=$(FETCH_ARGS.$(ARCH)) -BUILD_ARGS=$(BUILD_ARGS.$(ARCH)) +VARIANT=cpu else ifeq ($(GPU),1) -VAR=gpu +VARIANT=gpu DOCKERFILE_STEM ?= $(ROOT)/Dockerfile.gpu OS.custom:=1 @@ -43,20 +31,12 @@ OS.centos7=centos7 OS.bionic=ubuntu18.04 OS.xenial=ubuntu16.04 -FETCH_ARGS=$(FETCH_ARGS.$(ARCH)) -BUILD_ARGS=$(BUILD_ARGS.$(ARCH)) - else ifeq ($(JETSON),1) -VAR=jetson +VARIANT=jetson DOCKERFILE_STEM ?= $(ROOT)/Dockerfile.jetson -# FETCH_ARGS=WITH_TF=1 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0 -# BUILD_ARGS=WITH_TF=1 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0 - endif -# DOCKER_BUILD_ARGS += \ -# DEPS_ARGS="$(FETCH_ARGS)" \ -# BUILD_ARGS="$(BUILD_ARGS)" +#---------------------------------------------------------------------------------------------- -include $(MK)/docker +include $(MK)/docker.native diff --git a/opt/readies b/opt/readies index 010b2685c..03ba961f1 160000 --- a/opt/readies +++ b/opt/readies @@ -1 +1 @@ -Subproject commit 010b2685ca3f2a8387e9582620374e792078a62d +Subproject commit 03ba961f139ac5e2a9ae395502620e4046c4987b