From 59180302d0e6f72ed8dfe162b90fddbb4f7b63a3 Mon Sep 17 00:00:00 2001 From: Chayim Date: Sun, 30 May 2021 15:28:12 +0300 Subject: [PATCH 01/14] ci order change (#762) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b2c038bb3..9f6d87c72 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -510,7 +510,7 @@ workflows: - lint: <<: *on-any-branch - build-and-test: - <<: *on-any-branch-but-tags + <<: *on-any-branch <<: *after-linter - platforms-build: <<: *after-build-and-test From d7daaee7f32efcc0607bb1e74be1e6e8daa60ad7 Mon Sep 17 00:00:00 2001 From: Guy Korland Date: Sun, 30 May 2021 19:21:55 +0300 Subject: [PATCH 02/14] disable nightly-automation --- .circleci/config.yml | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9f6d87c72..1a49335a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -415,15 +415,15 @@ jobs: name: Run QA Automation command: MODULE_VERSION=$CIRCLE_TAG VERBOSE=1 TEST=release ./tests/qa/run - nightly-automation: - docker: - - image: redisfab/rmbuilder:6.2.1-x64-buster - steps: - - checkout - - setup-automation - - run: - name: Run QA Automation - command: MODULE_VERSION=$CIRCLE_BRANCH VERBOSE=1 TEST=nightly QUICK=1 ./tests/qa/run + #nightly-automation: + # docker: + # - image: redisfab/rmbuilder:6.2.1-x64-buster + # steps: + # - checkout + # - setup-automation + # - run: + # name: Run QA Automation + # command: MODULE_VERSION=$CIRCLE_BRANCH VERBOSE=1 TEST=nightly QUICK=1 ./tests/qa/run on-any-branch: &on-any-branch @@ -557,15 +557,15 @@ workflows: requires: - deploy-release - nightly: - triggers: - - schedule: - cron: "20 17 * * *" - filters: - branches: - only: master - jobs: +# nightly: +# triggers: +# - schedule: +# cron: "20 17 * * *" +# filters: +# branches: +# only: master +# jobs: # - build-macos: # <<: *never # temporarily disabled - - nightly-automation: - context: common +# - nightly-automation: +# context: common From f38a906287ca827d67334d3eb349b4f51a499ac8 Mon Sep 17 00:00:00 2001 From: Chayim Date: Mon, 31 May 2021 15:44:50 +0300 Subject: [PATCH 03/14] changing dockers to apt update latest (#767) --- Dockerfile | 3 +++ Dockerfile.gpu | 3 +++ 2 files changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 95147ad10..158060ede 100755 --- a/Dockerfile +++ b/Dockerfile @@ -37,6 +37,9 @@ ADD ./tests/flow/ tests/flow/ RUN FORCE=1 ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py +RUN apt-get update -qq +RUN apt-get upgrade -yqq +RUN rm -rf /var/cache/apt ARG DEPS_ARGS="" COPY ./get_deps.sh . diff --git a/Dockerfile.gpu b/Dockerfile.gpu index 6d4302805..48af1d07f 100644 --- a/Dockerfile.gpu +++ b/Dockerfile.gpu @@ -42,6 +42,9 @@ ADD ./tests/flow/ tests/flow/ RUN FORCE=1 ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py +RUN apt-get update -qq +RUN apt-get upgrade -yqq +RUN rm -rf /var/cache/apt ARG DEPS_ARGS="" COPY ./get_deps.sh . From 914c3d5b1f6442d28448a3785370ab1d546074eb Mon Sep 17 00:00:00 2001 From: Chayim Date: Mon, 31 May 2021 16:00:52 +0300 Subject: [PATCH 04/14] circleci changes to meet the MOD-1227 goals with how ci runs (#766) --- .circleci/config.yml | 158 ++++++++++++++++++++++++++++--------------- 1 file changed, 104 insertions(+), 54 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9f6d87c72..e8d337d57 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,6 +43,19 @@ commands: circleci step halt fi + only_run_if_forked_pull_request: + description: >- + If this build is from an internal (i.e not a forked) pull request, then end early. + This is required when we want different behaviours for external, versus internal PRs. + steps: + - run: + name: Return early if not a forked pull request + command: | + if [[ ! -n "$CIRCLE_PR_NUMBER" ]]; then + echo "Not a forked PR so marking as successfull and exiting" + circleci step halt + fi + relocate-docker-storage: steps: - run: @@ -127,6 +140,64 @@ commands: - store_artifacts: path: tests/logs + valgrind-general-steps: + parameters: + test_args: + type: string + default: "CLUSTER=0 AOF=0" + steps: + - abort_for_docs + - abort_for_noci + - checkout-all + - restore_cache: + keys: + - build-dependencies-{{ checksum "get_deps.sh" }} + # If no exact match is found will get dependencies from source + - setup-build-system + - run: + name: Install dependencies + command: | + ./opt/readies/bin/getredis -v 6.0 --valgrind --force + ./get_deps.sh cpu + - run: + name: Build for valgrind + command: | + make -C opt all VALGRIND=1 SHOW=1 + - run: + name: Test with valgrind + command: | + make -C opt test VALGRIND=1 <> + no_output_timeout: 120m + + build-and-test-gpu-steps: + steps: + - abort_for_docs + - abort_for_noci + - checkout-all + - run: + name: Relocate docker overlay2 dir + command: | + sudo systemctl stop docker + sudo mkdir -p /var2/lib/docker + sudo mv /var/lib/docker/overlay2 /var2/lib/docker + sudo mkdir /var/lib/docker/overlay2 + sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2 + sudo systemctl start docker + - run: + name: Build + command: | + docker build -f Dockerfile.gpu-test --no-cache -t redisai-gpu:latest-x64-bionic-test . + - run: + name: Test + command: | + mkdir -p $HOME/tests + docker run --gpus all -v $HOME/tests:/build/tests/logs -it --rm redisai-gpu:latest-x64-bionic-test + no_output_timeout: 40m + - store_artifacts: + path: tests/logs + + + jobs: lint: docker: @@ -245,37 +316,18 @@ jobs: make -C opt test VALGRIND=1 <> no_output_timeout: 120m + valgrind-general-for-forked-prs: + docker: + - image: redisfab/rmbuilder:6.2.1-x64-buster + steps: + - only_run_if_forked_pull_request + - valgrind-general-steps + valgrind-general: - parameters: - test_args: - type: string - default: "CLUSTER=0 AOF=0" docker: - image: redisfab/rmbuilder:6.2.1-x64-buster steps: - - abort_for_docs - - abort_for_noci - - early_return_for_forked_pull_requests - - checkout-all - - restore_cache: - keys: - - build-dependencies-{{ checksum "get_deps.sh" }} - # If no exact match is found will get dependencies from source - - setup-build-system - - run: - name: Install dependencies - command: | - ./opt/readies/bin/getredis -v 6.0 --valgrind --force - ./get_deps.sh cpu - - run: - name: Build for valgrind - command: | - make -C opt all VALGRIND=1 SHOW=1 - - run: - name: Test with valgrind - command: | - make -C opt test VALGRIND=1 <> - no_output_timeout: 120m + - valgrind-general-steps # build-macos: # macos: @@ -316,6 +368,8 @@ jobs: # make build # sudo make publish + # internal PRs execute build-and-test either in a workflow or + # via a github action trigger build-and-test-gpu: machine: enabled: true @@ -324,30 +378,20 @@ jobs: image: ubuntu-1604-cuda-11.1:202012-01 steps: - - abort_for_docs - - abort_for_noci - - checkout-all - - run: - name: Relocate docker overlay2 dir - command: | - sudo systemctl stop docker - sudo mkdir -p /var2/lib/docker - sudo mv /var/lib/docker/overlay2 /var2/lib/docker - sudo mkdir /var/lib/docker/overlay2 - sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2 - sudo systemctl start docker - - run: - name: Build - command: | - docker build -f Dockerfile.gpu-test --no-cache -t redisai-gpu:latest-x64-bionic-test . - - run: - name: Test - command: | - mkdir -p $HOME/tests - docker run --gpus all -v $HOME/tests:/build/tests/logs -it --rm redisai-gpu:latest-x64-bionic-test - no_output_timeout: 40m - - store_artifacts: - path: tests/logs + - build-and-test-gpu-steps + + # in the case of a forked PR, we want to run the GPU steps + # hence we check if we're forked, explicitly + build-and-test-gpu-for-forked-prs: + machine: + enabled: true + docker_layer_caching: true + resource_class: gpu.nvidia.small + image: ubuntu-1604-cuda-11.1:202012-01 + + steps: + - only_run_if_forked_pull_request + - build-and-test-gpu-steps deploy-artifacts: parameters: @@ -512,6 +556,12 @@ workflows: - build-and-test: <<: *on-any-branch <<: *after-linter + - build-and-test-gpu: + <<: *on-integ-branch + <<: *after-linter + - build-and-test-gpu-for-forked-prs: + <<: *on-any-branch + <<: *after-linter - platforms-build: <<: *after-build-and-test <<: *on-master-version-tags-and-dockertests @@ -530,6 +580,9 @@ workflows: context: common <<: *on-dev-branches <<: *after-linter + - valgrind-general-for-forked-prs: + <<: *on-any-branch + <<: *after-linter - valgrind: name: valgrind-cluster test_args: GEN=0 AOF=0 @@ -540,9 +593,6 @@ workflows: test_args: GEN=0 CLUSTER=0 <<: *on-integ-branch <<: *after-linter - - build-and-test-gpu: - <<: *on-integ-branch - <<: *after-linter - deploy-snapshot: context: common <<: *after-platform-builds From 6987193cdc124d68fb24ae10b7c074e025b2acd6 Mon Sep 17 00:00:00 2001 From: alonre24 Date: Mon, 31 May 2021 16:45:22 +0300 Subject: [PATCH 05/14] Update min redis enterprise version that RedisAI can run on --- ramp-lite.yml | 2 +- ramp-rce.yml | 2 +- ramp.yml | 2 +- tests/qa/RS_VERSIONS | 4 +--- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ramp-lite.yml b/ramp-lite.yml index dc3de83b8..2bf73d5d3 100644 --- a/ramp-lite.yml +++ b/ramp-lite.yml @@ -7,7 +7,7 @@ license: Redis Source Available License v1.0 command_line_args: "BACKENDSPATH /var/opt/redislabs/modules/ai-lite/{{NUMVER}}/deps" # command_line_args: "BACKENDSPATH /var/opt/redislabs/modules/ai/{{NUMVER}}/deps/backends" min_redis_version: "5.0.7" -min_redis_pack_version: "5.4.14" +min_redis_pack_version: "6.0.12" capabilities: - types - hash_policy diff --git a/ramp-rce.yml b/ramp-rce.yml index 4d7732c1a..8451ba36f 100644 --- a/ramp-rce.yml +++ b/ramp-rce.yml @@ -7,7 +7,7 @@ license: Redis Source Available License v1.0 command_line_args: "BACKENDSPATH /var/opt/redislabs/modules/ai-lite/{{NUMVER}}/deps" # command_line_args: "BACKENDSPATH /var/opt/redislabs/modules/ai/{{NUMVER}}/deps/backends" min_redis_version: "5.0.7" -min_redis_pack_version: "5.4.14" +min_redis_pack_version: "6.0.12" capabilities: - types - hash_policy diff --git a/ramp.yml b/ramp.yml index ee10ec8d9..5e7e5d197 100644 --- a/ramp.yml +++ b/ramp.yml @@ -7,7 +7,7 @@ license: Redis Source Available License v1.0 command_line_args: "BACKENDSPATH /var/opt/redislabs/modules/ai/{{NUMVER}}/deps" # command_line_args: "BACKENDSPATH /var/opt/redislabs/modules/ai/{{NUMVER}}/deps/backends" min_redis_version: "5.0.7" -min_redis_pack_version: "5.4.14" +min_redis_pack_version: "6.2.2" capabilities: - types - hash_policy diff --git a/tests/qa/RS_VERSIONS b/tests/qa/RS_VERSIONS index fb1bf9c5b..50e563e20 100644 --- a/tests/qa/RS_VERSIONS +++ b/tests/qa/RS_VERSIONS @@ -1,4 +1,2 @@ 6.0.12-58 -5.4.14-37 -6.0.8-32 -100.0.0-2199 +100.0.0-2400 From bdffbcd32579170347e2c0a08d6327883d3b386d Mon Sep 17 00:00:00 2001 From: Chayim Date: Tue, 1 Jun 2021 11:29:08 +0300 Subject: [PATCH 06/14] changing REDIS_VER to latest release: 6.2.3 (#772) --- Dockerfile | 2 +- Dockerfile.arm | 2 +- Dockerfile.gpu | 2 +- Dockerfile.gpu-test | 2 +- Dockerfile.jetson | 2 +- opt/build/docker/Makefile | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 158060ede..bd8f17c1d 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK} -ARG REDIS_VER=6.0.9 +ARG REDIS_VER=6.2.3 # OSNICK=bionic|stretch|buster ARG OSNICK=buster diff --git a/Dockerfile.arm b/Dockerfile.arm index 93e5bccc2..26f080e3f 100755 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK} -ARG REDIS_VER=6.0.9 +ARG REDIS_VER=6.2.3 # OSNICK=bionic|stretch|buster ARG OSNICK=buster diff --git a/Dockerfile.gpu b/Dockerfile.gpu index 48af1d07f..163c02e40 100644 --- a/Dockerfile.gpu +++ b/Dockerfile.gpu @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-gpu-${ARCH}-${OSNICK} -ARG REDIS_VER=6.0.9 +ARG REDIS_VER=6.2.3 # OSNICK=bionic|centos7|centos6 ARG OSNICK=bionic diff --git a/Dockerfile.gpu-test b/Dockerfile.gpu-test index b01aa1aee..fc6bd95f3 100644 --- a/Dockerfile.gpu-test +++ b/Dockerfile.gpu-test @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-gpu-${ARCH}-${OSNICK}-test -ARG REDIS_VER=6.0.9 +ARG REDIS_VER=6.2.3 # OSNICK=bionic|centos7|centos6 ARG OSNICK=bionic diff --git a/Dockerfile.jetson b/Dockerfile.jetson index 73dbb0214..f5938eca0 100644 --- a/Dockerfile.jetson +++ b/Dockerfile.jetson @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-jetson-${ARCH}-${OSNICK} -ARG REDIS_VER=6.0.9 +ARG REDIS_VER=6.2.3 # OSNICK=bionic|centos7|centos6 ARG OSNICK=bionic diff --git a/opt/build/docker/Makefile b/opt/build/docker/Makefile index 8f4492967..582f2b3e4 100755 --- a/opt/build/docker/Makefile +++ b/opt/build/docker/Makefile @@ -11,7 +11,7 @@ REPO=redisai # LATEST_BRANCH=1.2 INT_BRANCHES=1.2 -REDIS_VER:=6.0.9 +REDIS_VER:=6.2.3 REDISAI_LITE:=0 export ART_DIR=$(ROOT)/bin/artifacts From 865661d9d0497eb14ecd181a5ae6b7779ee5f655 Mon Sep 17 00:00:00 2001 From: Chayim Date: Tue, 1 Jun 2021 11:29:45 +0300 Subject: [PATCH 07/14] changing REDIS_VER to latest release: 6.2.3 (#771) From d42ad1584a4da695a6469bdd43a24236e7a7041d Mon Sep 17 00:00:00 2001 From: Chayim Date: Tue, 1 Jun 2021 13:02:22 +0300 Subject: [PATCH 08/14] adding lite to the release automation (#773) Co-authored-by: tomerhekredis <72793005+tomerhekredis@users.noreply.github.com> --- .circleci/config.yml | 8 +++++++- tests/qa/RS_VERSIONS | 3 +-- tests/qa/RS_VERSIONS-lite | 2 ++ tests/qa/nightly.json | 4 ++-- tests/qa/release.json | 6 +++--- tests/qa/run | 9 ++++++--- 6 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 tests/qa/RS_VERSIONS-lite diff --git a/.circleci/config.yml b/.circleci/config.yml index e8d337d57..0c3260d10 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -458,6 +458,10 @@ jobs: - run: name: Run QA Automation command: MODULE_VERSION=$CIRCLE_TAG VERBOSE=1 TEST=release ./tests/qa/run + - run: + name: Run QA Automation (AILite) + command: MODULE_VERSION=$CIRCLE_TAG VERBOSE=1 TEST=release VARIANT=lite ./tests/qa/run + nightly-automation: docker: @@ -468,7 +472,9 @@ jobs: - run: name: Run QA Automation command: MODULE_VERSION=$CIRCLE_BRANCH VERBOSE=1 TEST=nightly QUICK=1 ./tests/qa/run - + - run: + name: Run QA Automation (AILite) + command: MODULE_VERSION=$CIRCLE_BRANCH VERBOSE=1 TEST=nightly VARIANT=lite QUICK=1 ./tests/qa/run on-any-branch: &on-any-branch filters: diff --git a/tests/qa/RS_VERSIONS b/tests/qa/RS_VERSIONS index 50e563e20..4808afc03 100644 --- a/tests/qa/RS_VERSIONS +++ b/tests/qa/RS_VERSIONS @@ -1,2 +1 @@ -6.0.12-58 -100.0.0-2400 +100.0.0-2440 diff --git a/tests/qa/RS_VERSIONS-lite b/tests/qa/RS_VERSIONS-lite new file mode 100644 index 000000000..c7eff154e --- /dev/null +++ b/tests/qa/RS_VERSIONS-lite @@ -0,0 +1,2 @@ +6.0.12-58 +100.0.0-2440 diff --git a/tests/qa/nightly.json b/tests/qa/nightly.json index 1884d0477..894c9b154 100644 --- a/tests/qa/nightly.json +++ b/tests/qa/nightly.json @@ -1,6 +1,6 @@ { "service_id": "single_module_test_cycle", - "name": "redisai automation-testing", + "name": "redisai{{VARIANT}} automation-testing", "properties": { "sut_version": "master", "email_recipients": "redisaidev-aaaacob2o7eeecrhkqwua77kku@redislabs.slack.com", @@ -15,7 +15,7 @@ "teardown": true, "name": "bionic-amd64-aws", "concurrency": 1, - "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_FILE_PREFIX}}/snapshots/{{RS_MODULE_FILE_PREFIX}}-cpu.linux-bionic-x64.master.zip" + "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_FILE_PREFIX}}/snapshots/{{RS_MODULE_FILE_PREFIX}}-cpu.linux-bionic-x64.master{{VARIANT}}.zip" } ] } diff --git a/tests/qa/release.json b/tests/qa/release.json index 41393dfdd..136f2a04d 100644 --- a/tests/qa/release.json +++ b/tests/qa/release.json @@ -1,6 +1,6 @@ { "service_id": "single_module_test_cycle", - "name": "redisai automation-testing", + "name": "redisai{{VARIANT}} automation-testing", "properties": { "sut_version": "{{RS_VERSION}}", "email_recipients": "redisaidev-aaaacob2o7eeecrhkqwua77kku@redislabs.slack.com", @@ -15,13 +15,13 @@ "teardown": true, "name": "xenial-amd64-aws", "concurrency": 1, - "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_LC}}/{{RS_MODULE_FILE_PREFIX}}.linux-xenial-x64.{{MODULE_VERSION}}.zip" + "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_LC}}/{{RS_MODULE_FILE_PREFIX}}.linux-xenial-x64.{{MODULE_VERSION}}{{VARIANT}}.zip" }, { "teardown": true, "name": "bionic-amd64-aws", "concurrency": 1, - "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_LC}}/{{RS_MODULE_FILE_PREFIX}}.linux-bionic-x64.{{MODULE_VERSION}}.zip" + "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_LC}}/{{RS_MODULE_FILE_PREFIX}}.linux-bionic-x64.{{MODULE_VERSION}}{{VARIANT}}.zip" } ] } diff --git a/tests/qa/run b/tests/qa/run index 45a9917d8..31b4f0131 100755 --- a/tests/qa/run +++ b/tests/qa/run @@ -30,6 +30,7 @@ fi export RS_MODULE=RedisAI export RS_MODULE_LC=${RS_MODULE,,} export RS_MODULE_FILE_PREFIX=redisai-cpu +export VARIANT=${VARIANT} # for variants such as RedisAILite. We append -lite (see circleci for an example) if [[ -z $QA_AUTOMATION_TOKEN && $NOP != 1 ]]; then echo "Variable QA_AUTOMATION_TOKEN is undefined." >&2 @@ -59,7 +60,7 @@ run_test() { cd $HERE json_in=$(mktemp /tmp/$TEST.json.XXXX) - $READIES/bin/xtx -e RS_MODULE -e RS_MODULE_LC -e RS_MODULE_FILE_PREFIX -e MODULE_VERSION -e RS_VERSION $TEST.json > $json_in + $READIES/bin/xtx -e VARIANT -e RS_MODULE -e RS_MODULE_LC -e RS_MODULE_FILE_PREFIX -e MODULE_VERSION -e RS_VERSION $TEST.json > $json_in (( VERBOSE >= 1 )) && cat $json_in if [[ $NOP == 1 ]]; then @@ -89,10 +90,12 @@ run_test() { } rc=0 +# By default we use RS_VERSIONS, but with variants they may need to specify their own redis verisons, hence this file +VERSIONFILE=${HERE}/RS_VERSIONS${VARIANT} if [[ $QUICK == 1 ]]; then - RS_VERSIONS=$(cat $HERE/RS_VERSIONS | head -1) + RS_VERSIONS=$(cat $VERSIONFILE | head -1) else - RS_VERSIONS=$(cat $HERE/RS_VERSIONS) + RS_VERSIONS=$(cat $VERSIONFILE) fi for RS_VERSION in $RS_VERSIONS; do run_test $RS_VERSION From ee3cc1835f969e0945e9da48fd28ead5dcdabb16 Mon Sep 17 00:00:00 2001 From: Chayim Date: Tue, 1 Jun 2021 11:29:08 +0300 Subject: [PATCH 09/14] changing REDIS_VER to latest release: 6.2.3 (#772) --- Dockerfile | 2 +- Dockerfile.arm | 2 +- Dockerfile.gpu | 2 +- Dockerfile.gpu-test | 2 +- Dockerfile.jetson | 2 +- opt/build/docker/Makefile | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 95147ad10..b2a91f4ff 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK} -ARG REDIS_VER=6.0.9 +ARG REDIS_VER=6.2.3 # OSNICK=bionic|stretch|buster ARG OSNICK=buster diff --git a/Dockerfile.arm b/Dockerfile.arm index 93e5bccc2..26f080e3f 100755 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK} -ARG REDIS_VER=6.0.9 +ARG REDIS_VER=6.2.3 # OSNICK=bionic|stretch|buster ARG OSNICK=buster diff --git a/Dockerfile.gpu b/Dockerfile.gpu index 6d4302805..014454b63 100644 --- a/Dockerfile.gpu +++ b/Dockerfile.gpu @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-gpu-${ARCH}-${OSNICK} -ARG REDIS_VER=6.0.9 +ARG REDIS_VER=6.2.3 # OSNICK=bionic|centos7|centos6 ARG OSNICK=bionic diff --git a/Dockerfile.gpu-test b/Dockerfile.gpu-test index b01aa1aee..fc6bd95f3 100644 --- a/Dockerfile.gpu-test +++ b/Dockerfile.gpu-test @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-gpu-${ARCH}-${OSNICK}-test -ARG REDIS_VER=6.0.9 +ARG REDIS_VER=6.2.3 # OSNICK=bionic|centos7|centos6 ARG OSNICK=bionic diff --git a/Dockerfile.jetson b/Dockerfile.jetson index 73dbb0214..f5938eca0 100644 --- a/Dockerfile.jetson +++ b/Dockerfile.jetson @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-jetson-${ARCH}-${OSNICK} -ARG REDIS_VER=6.0.9 +ARG REDIS_VER=6.2.3 # OSNICK=bionic|centos7|centos6 ARG OSNICK=bionic diff --git a/opt/build/docker/Makefile b/opt/build/docker/Makefile index 8f4492967..582f2b3e4 100755 --- a/opt/build/docker/Makefile +++ b/opt/build/docker/Makefile @@ -11,7 +11,7 @@ REPO=redisai # LATEST_BRANCH=1.2 INT_BRANCHES=1.2 -REDIS_VER:=6.0.9 +REDIS_VER:=6.2.3 REDISAI_LITE:=0 export ART_DIR=$(ROOT)/bin/artifacts From d3852e717b8ca3ac811073be630d18e501cb3cbe Mon Sep 17 00:00:00 2001 From: Chayim Date: Tue, 1 Jun 2021 11:29:45 +0300 Subject: [PATCH 10/14] changing REDIS_VER to latest release: 6.2.3 (#771) From 4c54378662d82cbea3a5b79cadd611c5791242db Mon Sep 17 00:00:00 2001 From: Chayim Date: Tue, 1 Jun 2021 13:02:22 +0300 Subject: [PATCH 11/14] adding lite to the release automation (#773) Co-authored-by: tomerhekredis <72793005+tomerhekredis@users.noreply.github.com> --- .circleci/config.yml | 8 +++++++- tests/qa/RS_VERSIONS | 3 +-- tests/qa/RS_VERSIONS-lite | 2 ++ tests/qa/nightly.json | 4 ++-- tests/qa/release.json | 6 +++--- tests/qa/run | 9 ++++++--- 6 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 tests/qa/RS_VERSIONS-lite diff --git a/.circleci/config.yml b/.circleci/config.yml index 9f6d87c72..3c819b519 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -414,6 +414,10 @@ jobs: - run: name: Run QA Automation command: MODULE_VERSION=$CIRCLE_TAG VERBOSE=1 TEST=release ./tests/qa/run + - run: + name: Run QA Automation (AILite) + command: MODULE_VERSION=$CIRCLE_TAG VERBOSE=1 TEST=release VARIANT=lite ./tests/qa/run + nightly-automation: docker: @@ -424,7 +428,9 @@ jobs: - run: name: Run QA Automation command: MODULE_VERSION=$CIRCLE_BRANCH VERBOSE=1 TEST=nightly QUICK=1 ./tests/qa/run - + - run: + name: Run QA Automation (AILite) + command: MODULE_VERSION=$CIRCLE_BRANCH VERBOSE=1 TEST=nightly VARIANT=lite QUICK=1 ./tests/qa/run on-any-branch: &on-any-branch filters: diff --git a/tests/qa/RS_VERSIONS b/tests/qa/RS_VERSIONS index 50e563e20..4808afc03 100644 --- a/tests/qa/RS_VERSIONS +++ b/tests/qa/RS_VERSIONS @@ -1,2 +1 @@ -6.0.12-58 -100.0.0-2400 +100.0.0-2440 diff --git a/tests/qa/RS_VERSIONS-lite b/tests/qa/RS_VERSIONS-lite new file mode 100644 index 000000000..c7eff154e --- /dev/null +++ b/tests/qa/RS_VERSIONS-lite @@ -0,0 +1,2 @@ +6.0.12-58 +100.0.0-2440 diff --git a/tests/qa/nightly.json b/tests/qa/nightly.json index 1884d0477..894c9b154 100644 --- a/tests/qa/nightly.json +++ b/tests/qa/nightly.json @@ -1,6 +1,6 @@ { "service_id": "single_module_test_cycle", - "name": "redisai automation-testing", + "name": "redisai{{VARIANT}} automation-testing", "properties": { "sut_version": "master", "email_recipients": "redisaidev-aaaacob2o7eeecrhkqwua77kku@redislabs.slack.com", @@ -15,7 +15,7 @@ "teardown": true, "name": "bionic-amd64-aws", "concurrency": 1, - "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_FILE_PREFIX}}/snapshots/{{RS_MODULE_FILE_PREFIX}}-cpu.linux-bionic-x64.master.zip" + "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_FILE_PREFIX}}/snapshots/{{RS_MODULE_FILE_PREFIX}}-cpu.linux-bionic-x64.master{{VARIANT}}.zip" } ] } diff --git a/tests/qa/release.json b/tests/qa/release.json index 41393dfdd..136f2a04d 100644 --- a/tests/qa/release.json +++ b/tests/qa/release.json @@ -1,6 +1,6 @@ { "service_id": "single_module_test_cycle", - "name": "redisai automation-testing", + "name": "redisai{{VARIANT}} automation-testing", "properties": { "sut_version": "{{RS_VERSION}}", "email_recipients": "redisaidev-aaaacob2o7eeecrhkqwua77kku@redislabs.slack.com", @@ -15,13 +15,13 @@ "teardown": true, "name": "xenial-amd64-aws", "concurrency": 1, - "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_LC}}/{{RS_MODULE_FILE_PREFIX}}.linux-xenial-x64.{{MODULE_VERSION}}.zip" + "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_LC}}/{{RS_MODULE_FILE_PREFIX}}.linux-xenial-x64.{{MODULE_VERSION}}{{VARIANT}}.zip" }, { "teardown": true, "name": "bionic-amd64-aws", "concurrency": 1, - "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_LC}}/{{RS_MODULE_FILE_PREFIX}}.linux-bionic-x64.{{MODULE_VERSION}}.zip" + "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_LC}}/{{RS_MODULE_FILE_PREFIX}}.linux-bionic-x64.{{MODULE_VERSION}}{{VARIANT}}.zip" } ] } diff --git a/tests/qa/run b/tests/qa/run index 45a9917d8..31b4f0131 100755 --- a/tests/qa/run +++ b/tests/qa/run @@ -30,6 +30,7 @@ fi export RS_MODULE=RedisAI export RS_MODULE_LC=${RS_MODULE,,} export RS_MODULE_FILE_PREFIX=redisai-cpu +export VARIANT=${VARIANT} # for variants such as RedisAILite. We append -lite (see circleci for an example) if [[ -z $QA_AUTOMATION_TOKEN && $NOP != 1 ]]; then echo "Variable QA_AUTOMATION_TOKEN is undefined." >&2 @@ -59,7 +60,7 @@ run_test() { cd $HERE json_in=$(mktemp /tmp/$TEST.json.XXXX) - $READIES/bin/xtx -e RS_MODULE -e RS_MODULE_LC -e RS_MODULE_FILE_PREFIX -e MODULE_VERSION -e RS_VERSION $TEST.json > $json_in + $READIES/bin/xtx -e VARIANT -e RS_MODULE -e RS_MODULE_LC -e RS_MODULE_FILE_PREFIX -e MODULE_VERSION -e RS_VERSION $TEST.json > $json_in (( VERBOSE >= 1 )) && cat $json_in if [[ $NOP == 1 ]]; then @@ -89,10 +90,12 @@ run_test() { } rc=0 +# By default we use RS_VERSIONS, but with variants they may need to specify their own redis verisons, hence this file +VERSIONFILE=${HERE}/RS_VERSIONS${VARIANT} if [[ $QUICK == 1 ]]; then - RS_VERSIONS=$(cat $HERE/RS_VERSIONS | head -1) + RS_VERSIONS=$(cat $VERSIONFILE | head -1) else - RS_VERSIONS=$(cat $HERE/RS_VERSIONS) + RS_VERSIONS=$(cat $VERSIONFILE) fi for RS_VERSION in $RS_VERSIONS; do run_test $RS_VERSION From 5e6ddf471e634655399531b70daa85ea3c5e8e7a Mon Sep 17 00:00:00 2001 From: DvirDukhan Date: Wed, 2 Jun 2021 13:15:08 +0300 Subject: [PATCH 12/14] removed aof tests (#764) --- .circleci/config.yml | 9 ++------- opt/Makefile | 7 +++---- tests/flow/tests_setup/tests.sh | 3 --- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 03b56091c..1611d165e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -289,7 +289,7 @@ jobs: parameters: test_args: type: string - default: "CLUSTER=0 AOF=0" + default: "CLUSTER=0" docker: - image: redisfab/rmbuilder:6.2.1-x64-buster steps: @@ -588,12 +588,7 @@ workflows: <<: *after-linter - valgrind: name: valgrind-cluster - test_args: GEN=0 AOF=0 - <<: *on-integ-branch - <<: *after-linter - - valgrind: - name: valgrind-aof - test_args: GEN=0 CLUSTER=0 + test_args: GEN=0 <<: *on-integ-branch <<: *after-linter - deploy-snapshot: diff --git a/opt/Makefile b/opt/Makefile index 2de38e0d8..67c2bc8b4 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -240,7 +240,6 @@ endif export GEN ?= 1 export SLAVES ?= 1 -export AOF ?= 1 export CLUSTER ?= 1 define UNIT_TESTS @@ -256,7 +255,7 @@ $(SHOW)\ DEVICE=$(DEVICE) \ MODULE=$(realpath $(INSTALLED_TARGET)) \ CLUSTER=$(CLUSTER) \ -GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \ +GEN=$(GEN) SLAVES=$(SLAVES) \ VALGRIND=$(VALGRIND) \ $(ROOT)/tests/flow/tests_setup/tests.sh endef @@ -272,7 +271,7 @@ flow_tests: build DEVICE=$(DEVICE) \ MODULE=$(realpath $(INSTALLED_TARGET)) \ CLUSTER=$(CLUSTER) \ - GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \ + GEN=$(GEN) SLAVES=$(SLAVES) \ VALGRIND=$(VALGRIND) \ REDIS=$(REDIS) \ $(ROOT)/tests/flow/tests_setup/tests.sh @@ -290,7 +289,7 @@ test: build MODULE=$(realpath $(INSTALLED_TARGET)) \ TESTMOD=$(realpath $(BINDIR)/tests/module/testmod.so) \ CLUSTER=$(CLUSTER) \ - GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \ + GEN=$(GEN) SLAVES=$(SLAVES) \ VALGRIND=$(VALGRIND) \ REDIS=$(REDIS) \ $(ROOT)/tests/flow/tests_setup/tests.sh diff --git a/tests/flow/tests_setup/tests.sh b/tests/flow/tests_setup/tests.sh index 16e557828..f0dd3bb99 100755 --- a/tests/flow/tests_setup/tests.sh +++ b/tests/flow/tests_setup/tests.sh @@ -34,7 +34,6 @@ help() { DEVICE=cpu|gpu Device for testing GEN=0|1 General tests - AOF=0|1 Tests with --test-aof SLAVES=0|1 Tests with --test-slaves TEST=test Run specific test (e.g. test.py:test_name) @@ -109,7 +108,6 @@ DEVICE=${DEVICE:-cpu} GEN=${GEN:-1} SLAVES=${SLAVES:-0} -AOF=${AOF:-0} GDB=${GDB:-0} @@ -146,6 +144,5 @@ check_redis_server [[ $GEN == 1 ]] && run_tests [[ $CLUSTER == 1 ]] && RLTEST_ARGS+=" --env oss-cluster --shards-count 1" run_tests "--env oss-cluster" [[ $VALGRIND != 1 && $SLAVES == 1 ]] && RLTEST_ARGS+=" --use-slaves" run_tests "--use-slaves" -[[ $AOF == 1 ]] && RLTEST_ARGS+=" --use-aof" run_tests "--use-aof" # [[ $VALGRIND == 1 ]] && valgrind_summary exit 0 From 59c57fa661f0b17dfcab973980ad4cf383ecc488 Mon Sep 17 00:00:00 2001 From: DvirDukhan Date: Wed, 2 Jun 2021 13:15:08 +0300 Subject: [PATCH 13/14] removed aof tests (#764) --- .circleci/config.yml | 9 ++------- opt/Makefile | 7 +++---- tests/flow/tests_setup/tests.sh | 3 --- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3c819b519..50d983b8c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -218,7 +218,7 @@ jobs: parameters: test_args: type: string - default: "CLUSTER=0 AOF=0" + default: "CLUSTER=0" docker: - image: redisfab/rmbuilder:6.2.1-x64-buster steps: @@ -538,12 +538,7 @@ workflows: <<: *after-linter - valgrind: name: valgrind-cluster - test_args: GEN=0 AOF=0 - <<: *on-integ-branch - <<: *after-linter - - valgrind: - name: valgrind-aof - test_args: GEN=0 CLUSTER=0 + test_args: GEN=0 <<: *on-integ-branch <<: *after-linter - build-and-test-gpu: diff --git a/opt/Makefile b/opt/Makefile index 2de38e0d8..67c2bc8b4 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -240,7 +240,6 @@ endif export GEN ?= 1 export SLAVES ?= 1 -export AOF ?= 1 export CLUSTER ?= 1 define UNIT_TESTS @@ -256,7 +255,7 @@ $(SHOW)\ DEVICE=$(DEVICE) \ MODULE=$(realpath $(INSTALLED_TARGET)) \ CLUSTER=$(CLUSTER) \ -GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \ +GEN=$(GEN) SLAVES=$(SLAVES) \ VALGRIND=$(VALGRIND) \ $(ROOT)/tests/flow/tests_setup/tests.sh endef @@ -272,7 +271,7 @@ flow_tests: build DEVICE=$(DEVICE) \ MODULE=$(realpath $(INSTALLED_TARGET)) \ CLUSTER=$(CLUSTER) \ - GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \ + GEN=$(GEN) SLAVES=$(SLAVES) \ VALGRIND=$(VALGRIND) \ REDIS=$(REDIS) \ $(ROOT)/tests/flow/tests_setup/tests.sh @@ -290,7 +289,7 @@ test: build MODULE=$(realpath $(INSTALLED_TARGET)) \ TESTMOD=$(realpath $(BINDIR)/tests/module/testmod.so) \ CLUSTER=$(CLUSTER) \ - GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \ + GEN=$(GEN) SLAVES=$(SLAVES) \ VALGRIND=$(VALGRIND) \ REDIS=$(REDIS) \ $(ROOT)/tests/flow/tests_setup/tests.sh diff --git a/tests/flow/tests_setup/tests.sh b/tests/flow/tests_setup/tests.sh index 16e557828..f0dd3bb99 100755 --- a/tests/flow/tests_setup/tests.sh +++ b/tests/flow/tests_setup/tests.sh @@ -34,7 +34,6 @@ help() { DEVICE=cpu|gpu Device for testing GEN=0|1 General tests - AOF=0|1 Tests with --test-aof SLAVES=0|1 Tests with --test-slaves TEST=test Run specific test (e.g. test.py:test_name) @@ -109,7 +108,6 @@ DEVICE=${DEVICE:-cpu} GEN=${GEN:-1} SLAVES=${SLAVES:-0} -AOF=${AOF:-0} GDB=${GDB:-0} @@ -146,6 +144,5 @@ check_redis_server [[ $GEN == 1 ]] && run_tests [[ $CLUSTER == 1 ]] && RLTEST_ARGS+=" --env oss-cluster --shards-count 1" run_tests "--env oss-cluster" [[ $VALGRIND != 1 && $SLAVES == 1 ]] && RLTEST_ARGS+=" --use-slaves" run_tests "--use-slaves" -[[ $AOF == 1 ]] && RLTEST_ARGS+=" --use-aof" run_tests "--use-aof" # [[ $VALGRIND == 1 ]] && valgrind_summary exit 0 From 710511d5a8bd22ff496fe26716450eb81d1745af Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Wed, 2 Jun 2021 15:39:06 +0300 Subject: [PATCH 14/14] merge conflicts --- .circleci/config.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 200394a11..e794b2923 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -559,15 +559,12 @@ workflows: <<: *on-any-branch - build-and-test: <<: *on-any-branch -<<<<<<< HEAD -======= <<: *after-linter - build-and-test-gpu: <<: *on-integ-branch <<: *after-linter - build-and-test-gpu-for-forked-prs: <<: *on-any-branch ->>>>>>> master <<: *after-linter - platforms-build: <<: *after-build-and-test @@ -587,23 +584,14 @@ workflows: context: common <<: *on-dev-branches <<: *after-linter -<<<<<<< HEAD - valgrind: name: valgrind-cluster test_args: GEN=0 <<: *on-integ-branch <<: *after-linter - - build-and-test-gpu: -======= - valgrind-general-for-forked-prs: <<: *on-any-branch <<: *after-linter - - valgrind: - name: valgrind-cluster - test_args: GEN=0 ->>>>>>> master - <<: *on-integ-branch - <<: *after-linter - deploy-snapshot: context: common <<: *after-platform-builds