diff --git a/.circleci/config.yml b/.circleci/config.yml index 9f6d87c72..e794b2923 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,6 +43,19 @@ commands: circleci step halt fi + only_run_if_forked_pull_request: + description: >- + If this build is from an internal (i.e not a forked) pull request, then end early. + This is required when we want different behaviours for external, versus internal PRs. + steps: + - run: + name: Return early if not a forked pull request + command: | + if [[ ! -n "$CIRCLE_PR_NUMBER" ]]; then + echo "Not a forked PR so marking as successfull and exiting" + circleci step halt + fi + relocate-docker-storage: steps: - run: @@ -127,6 +140,64 @@ commands: - store_artifacts: path: tests/logs + valgrind-general-steps: + parameters: + test_args: + type: string + default: "CLUSTER=0 AOF=0" + steps: + - abort_for_docs + - abort_for_noci + - checkout-all + - restore_cache: + keys: + - build-dependencies-{{ checksum "get_deps.sh" }} + # If no exact match is found will get dependencies from source + - setup-build-system + - run: + name: Install dependencies + command: | + ./opt/readies/bin/getredis -v 6.0 --valgrind --force + ./get_deps.sh cpu + - run: + name: Build for valgrind + command: | + make -C opt all VALGRIND=1 SHOW=1 + - run: + name: Test with valgrind + command: | + make -C opt test VALGRIND=1 <> + no_output_timeout: 120m + + build-and-test-gpu-steps: + steps: + - abort_for_docs + - abort_for_noci + - checkout-all + - run: + name: Relocate docker overlay2 dir + command: | + sudo systemctl stop docker + sudo mkdir -p /var2/lib/docker + sudo mv /var/lib/docker/overlay2 /var2/lib/docker + sudo mkdir /var/lib/docker/overlay2 + sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2 + sudo systemctl start docker + - run: + name: Build + command: | + docker build -f Dockerfile.gpu-test --no-cache -t redisai-gpu:latest-x64-bionic-test . + - run: + name: Test + command: | + mkdir -p $HOME/tests + docker run --gpus all -v $HOME/tests:/build/tests/logs -it --rm redisai-gpu:latest-x64-bionic-test + no_output_timeout: 40m + - store_artifacts: + path: tests/logs + + + jobs: lint: docker: @@ -218,7 +289,7 @@ jobs: parameters: test_args: type: string - default: "CLUSTER=0 AOF=0" + default: "CLUSTER=0" docker: - image: redisfab/rmbuilder:6.2.1-x64-buster steps: @@ -245,37 +316,18 @@ jobs: make -C opt test VALGRIND=1 <> no_output_timeout: 120m + valgrind-general-for-forked-prs: + docker: + - image: redisfab/rmbuilder:6.2.1-x64-buster + steps: + - only_run_if_forked_pull_request + - valgrind-general-steps + valgrind-general: - parameters: - test_args: - type: string - default: "CLUSTER=0 AOF=0" docker: - image: redisfab/rmbuilder:6.2.1-x64-buster steps: - - abort_for_docs - - abort_for_noci - - early_return_for_forked_pull_requests - - checkout-all - - restore_cache: - keys: - - build-dependencies-{{ checksum "get_deps.sh" }} - # If no exact match is found will get dependencies from source - - setup-build-system - - run: - name: Install dependencies - command: | - ./opt/readies/bin/getredis -v 6.0 --valgrind --force - ./get_deps.sh cpu - - run: - name: Build for valgrind - command: | - make -C opt all VALGRIND=1 SHOW=1 - - run: - name: Test with valgrind - command: | - make -C opt test VALGRIND=1 <> - no_output_timeout: 120m + - valgrind-general-steps # build-macos: # macos: @@ -316,6 +368,8 @@ jobs: # make build # sudo make publish + # internal PRs execute build-and-test either in a workflow or + # via a github action trigger build-and-test-gpu: machine: enabled: true @@ -324,30 +378,20 @@ jobs: image: ubuntu-1604-cuda-11.1:202012-01 steps: - - abort_for_docs - - abort_for_noci - - checkout-all - - run: - name: Relocate docker overlay2 dir - command: | - sudo systemctl stop docker - sudo mkdir -p /var2/lib/docker - sudo mv /var/lib/docker/overlay2 /var2/lib/docker - sudo mkdir /var/lib/docker/overlay2 - sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2 - sudo systemctl start docker - - run: - name: Build - command: | - docker build -f Dockerfile.gpu-test --no-cache -t redisai-gpu:latest-x64-bionic-test . - - run: - name: Test - command: | - mkdir -p $HOME/tests - docker run --gpus all -v $HOME/tests:/build/tests/logs -it --rm redisai-gpu:latest-x64-bionic-test - no_output_timeout: 40m - - store_artifacts: - path: tests/logs + - build-and-test-gpu-steps + + # in the case of a forked PR, we want to run the GPU steps + # hence we check if we're forked, explicitly + build-and-test-gpu-for-forked-prs: + machine: + enabled: true + docker_layer_caching: true + resource_class: gpu.nvidia.small + image: ubuntu-1604-cuda-11.1:202012-01 + + steps: + - only_run_if_forked_pull_request + - build-and-test-gpu-steps deploy-artifacts: parameters: @@ -414,16 +458,20 @@ jobs: - run: name: Run QA Automation command: MODULE_VERSION=$CIRCLE_TAG VERBOSE=1 TEST=release ./tests/qa/run - - nightly-automation: - docker: - - image: redisfab/rmbuilder:6.2.1-x64-buster - steps: - - checkout - - setup-automation - run: - name: Run QA Automation - command: MODULE_VERSION=$CIRCLE_BRANCH VERBOSE=1 TEST=nightly QUICK=1 ./tests/qa/run + name: Run QA Automation (AILite) + command: MODULE_VERSION=$CIRCLE_TAG VERBOSE=1 TEST=release VARIANT=lite ./tests/qa/run + + +# nightly-automation: +# docker: +# - image: redisfab/rmbuilder:6.2.1-x64-buster +# steps: +# - checkout +# - setup-automation +# - run: +# name: Run QA Automation +# command: MODULE_VERSION=$CIRCLE_BRANCH VERBOSE=1 TEST=nightly QUICK=1 ./tests/qa/run on-any-branch: &on-any-branch @@ -512,6 +560,12 @@ workflows: - build-and-test: <<: *on-any-branch <<: *after-linter + - build-and-test-gpu: + <<: *on-integ-branch + <<: *after-linter + - build-and-test-gpu-for-forked-prs: + <<: *on-any-branch + <<: *after-linter - platforms-build: <<: *after-build-and-test <<: *on-master-version-tags-and-dockertests @@ -532,16 +586,11 @@ workflows: <<: *after-linter - valgrind: name: valgrind-cluster - test_args: GEN=0 AOF=0 + test_args: GEN=0 <<: *on-integ-branch <<: *after-linter - - valgrind: - name: valgrind-aof - test_args: GEN=0 CLUSTER=0 - <<: *on-integ-branch - <<: *after-linter - - build-and-test-gpu: - <<: *on-integ-branch + - valgrind-general-for-forked-prs: + <<: *on-any-branch <<: *after-linter - deploy-snapshot: context: common @@ -557,15 +606,15 @@ workflows: requires: - deploy-release - nightly: - triggers: - - schedule: - cron: "20 17 * * *" - filters: - branches: - only: master - jobs: +# nightly: +# triggers: +# - schedule: +# cron: "20 17 * * *" +# filters: +# branches: +# only: master +# jobs: # - build-macos: # <<: *never # temporarily disabled - - nightly-automation: - context: common +# - nightly-automation: +# context: common diff --git a/Dockerfile b/Dockerfile index 95147ad10..bd8f17c1d 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK} -ARG REDIS_VER=6.0.9 +ARG REDIS_VER=6.2.3 # OSNICK=bionic|stretch|buster ARG OSNICK=buster @@ -37,6 +37,9 @@ ADD ./tests/flow/ tests/flow/ RUN FORCE=1 ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py +RUN apt-get update -qq +RUN apt-get upgrade -yqq +RUN rm -rf /var/cache/apt ARG DEPS_ARGS="" COPY ./get_deps.sh . diff --git a/Dockerfile.arm b/Dockerfile.arm index 93e5bccc2..26f080e3f 100755 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK} -ARG REDIS_VER=6.0.9 +ARG REDIS_VER=6.2.3 # OSNICK=bionic|stretch|buster ARG OSNICK=buster diff --git a/Dockerfile.gpu b/Dockerfile.gpu index 6d4302805..163c02e40 100644 --- a/Dockerfile.gpu +++ b/Dockerfile.gpu @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-gpu-${ARCH}-${OSNICK} -ARG REDIS_VER=6.0.9 +ARG REDIS_VER=6.2.3 # OSNICK=bionic|centos7|centos6 ARG OSNICK=bionic @@ -42,6 +42,9 @@ ADD ./tests/flow/ tests/flow/ RUN FORCE=1 ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py +RUN apt-get update -qq +RUN apt-get upgrade -yqq +RUN rm -rf /var/cache/apt ARG DEPS_ARGS="" COPY ./get_deps.sh . diff --git a/Dockerfile.gpu-test b/Dockerfile.gpu-test index b01aa1aee..fc6bd95f3 100644 --- a/Dockerfile.gpu-test +++ b/Dockerfile.gpu-test @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-gpu-${ARCH}-${OSNICK}-test -ARG REDIS_VER=6.0.9 +ARG REDIS_VER=6.2.3 # OSNICK=bionic|centos7|centos6 ARG OSNICK=bionic diff --git a/Dockerfile.jetson b/Dockerfile.jetson index 73dbb0214..f5938eca0 100644 --- a/Dockerfile.jetson +++ b/Dockerfile.jetson @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-jetson-${ARCH}-${OSNICK} -ARG REDIS_VER=6.0.9 +ARG REDIS_VER=6.2.3 # OSNICK=bionic|centos7|centos6 ARG OSNICK=bionic diff --git a/opt/Makefile b/opt/Makefile index 2de38e0d8..67c2bc8b4 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -240,7 +240,6 @@ endif export GEN ?= 1 export SLAVES ?= 1 -export AOF ?= 1 export CLUSTER ?= 1 define UNIT_TESTS @@ -256,7 +255,7 @@ $(SHOW)\ DEVICE=$(DEVICE) \ MODULE=$(realpath $(INSTALLED_TARGET)) \ CLUSTER=$(CLUSTER) \ -GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \ +GEN=$(GEN) SLAVES=$(SLAVES) \ VALGRIND=$(VALGRIND) \ $(ROOT)/tests/flow/tests_setup/tests.sh endef @@ -272,7 +271,7 @@ flow_tests: build DEVICE=$(DEVICE) \ MODULE=$(realpath $(INSTALLED_TARGET)) \ CLUSTER=$(CLUSTER) \ - GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \ + GEN=$(GEN) SLAVES=$(SLAVES) \ VALGRIND=$(VALGRIND) \ REDIS=$(REDIS) \ $(ROOT)/tests/flow/tests_setup/tests.sh @@ -290,7 +289,7 @@ test: build MODULE=$(realpath $(INSTALLED_TARGET)) \ TESTMOD=$(realpath $(BINDIR)/tests/module/testmod.so) \ CLUSTER=$(CLUSTER) \ - GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \ + GEN=$(GEN) SLAVES=$(SLAVES) \ VALGRIND=$(VALGRIND) \ REDIS=$(REDIS) \ $(ROOT)/tests/flow/tests_setup/tests.sh diff --git a/opt/build/docker/Makefile b/opt/build/docker/Makefile index 8f4492967..582f2b3e4 100755 --- a/opt/build/docker/Makefile +++ b/opt/build/docker/Makefile @@ -11,7 +11,7 @@ REPO=redisai # LATEST_BRANCH=1.2 INT_BRANCHES=1.2 -REDIS_VER:=6.0.9 +REDIS_VER:=6.2.3 REDISAI_LITE:=0 export ART_DIR=$(ROOT)/bin/artifacts diff --git a/tests/flow/tests_setup/tests.sh b/tests/flow/tests_setup/tests.sh index 16e557828..f0dd3bb99 100755 --- a/tests/flow/tests_setup/tests.sh +++ b/tests/flow/tests_setup/tests.sh @@ -34,7 +34,6 @@ help() { DEVICE=cpu|gpu Device for testing GEN=0|1 General tests - AOF=0|1 Tests with --test-aof SLAVES=0|1 Tests with --test-slaves TEST=test Run specific test (e.g. test.py:test_name) @@ -109,7 +108,6 @@ DEVICE=${DEVICE:-cpu} GEN=${GEN:-1} SLAVES=${SLAVES:-0} -AOF=${AOF:-0} GDB=${GDB:-0} @@ -146,6 +144,5 @@ check_redis_server [[ $GEN == 1 ]] && run_tests [[ $CLUSTER == 1 ]] && RLTEST_ARGS+=" --env oss-cluster --shards-count 1" run_tests "--env oss-cluster" [[ $VALGRIND != 1 && $SLAVES == 1 ]] && RLTEST_ARGS+=" --use-slaves" run_tests "--use-slaves" -[[ $AOF == 1 ]] && RLTEST_ARGS+=" --use-aof" run_tests "--use-aof" # [[ $VALGRIND == 1 ]] && valgrind_summary exit 0 diff --git a/tests/qa/RS_VERSIONS b/tests/qa/RS_VERSIONS index 50e563e20..4808afc03 100644 --- a/tests/qa/RS_VERSIONS +++ b/tests/qa/RS_VERSIONS @@ -1,2 +1 @@ -6.0.12-58 -100.0.0-2400 +100.0.0-2440 diff --git a/tests/qa/RS_VERSIONS-lite b/tests/qa/RS_VERSIONS-lite new file mode 100644 index 000000000..c7eff154e --- /dev/null +++ b/tests/qa/RS_VERSIONS-lite @@ -0,0 +1,2 @@ +6.0.12-58 +100.0.0-2440 diff --git a/tests/qa/nightly.json b/tests/qa/nightly.json index 1884d0477..894c9b154 100644 --- a/tests/qa/nightly.json +++ b/tests/qa/nightly.json @@ -1,6 +1,6 @@ { "service_id": "single_module_test_cycle", - "name": "redisai automation-testing", + "name": "redisai{{VARIANT}} automation-testing", "properties": { "sut_version": "master", "email_recipients": "redisaidev-aaaacob2o7eeecrhkqwua77kku@redislabs.slack.com", @@ -15,7 +15,7 @@ "teardown": true, "name": "bionic-amd64-aws", "concurrency": 1, - "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_FILE_PREFIX}}/snapshots/{{RS_MODULE_FILE_PREFIX}}-cpu.linux-bionic-x64.master.zip" + "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_FILE_PREFIX}}/snapshots/{{RS_MODULE_FILE_PREFIX}}-cpu.linux-bionic-x64.master{{VARIANT}}.zip" } ] } diff --git a/tests/qa/release.json b/tests/qa/release.json index 41393dfdd..136f2a04d 100644 --- a/tests/qa/release.json +++ b/tests/qa/release.json @@ -1,6 +1,6 @@ { "service_id": "single_module_test_cycle", - "name": "redisai automation-testing", + "name": "redisai{{VARIANT}} automation-testing", "properties": { "sut_version": "{{RS_VERSION}}", "email_recipients": "redisaidev-aaaacob2o7eeecrhkqwua77kku@redislabs.slack.com", @@ -15,13 +15,13 @@ "teardown": true, "name": "xenial-amd64-aws", "concurrency": 1, - "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_LC}}/{{RS_MODULE_FILE_PREFIX}}.linux-xenial-x64.{{MODULE_VERSION}}.zip" + "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_LC}}/{{RS_MODULE_FILE_PREFIX}}.linux-xenial-x64.{{MODULE_VERSION}}{{VARIANT}}.zip" }, { "teardown": true, "name": "bionic-amd64-aws", "concurrency": 1, - "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_LC}}/{{RS_MODULE_FILE_PREFIX}}.linux-bionic-x64.{{MODULE_VERSION}}.zip" + "module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_LC}}/{{RS_MODULE_FILE_PREFIX}}.linux-bionic-x64.{{MODULE_VERSION}}{{VARIANT}}.zip" } ] } diff --git a/tests/qa/run b/tests/qa/run index 45a9917d8..31b4f0131 100755 --- a/tests/qa/run +++ b/tests/qa/run @@ -30,6 +30,7 @@ fi export RS_MODULE=RedisAI export RS_MODULE_LC=${RS_MODULE,,} export RS_MODULE_FILE_PREFIX=redisai-cpu +export VARIANT=${VARIANT} # for variants such as RedisAILite. We append -lite (see circleci for an example) if [[ -z $QA_AUTOMATION_TOKEN && $NOP != 1 ]]; then echo "Variable QA_AUTOMATION_TOKEN is undefined." >&2 @@ -59,7 +60,7 @@ run_test() { cd $HERE json_in=$(mktemp /tmp/$TEST.json.XXXX) - $READIES/bin/xtx -e RS_MODULE -e RS_MODULE_LC -e RS_MODULE_FILE_PREFIX -e MODULE_VERSION -e RS_VERSION $TEST.json > $json_in + $READIES/bin/xtx -e VARIANT -e RS_MODULE -e RS_MODULE_LC -e RS_MODULE_FILE_PREFIX -e MODULE_VERSION -e RS_VERSION $TEST.json > $json_in (( VERBOSE >= 1 )) && cat $json_in if [[ $NOP == 1 ]]; then @@ -89,10 +90,12 @@ run_test() { } rc=0 +# By default we use RS_VERSIONS, but with variants they may need to specify their own redis verisons, hence this file +VERSIONFILE=${HERE}/RS_VERSIONS${VARIANT} if [[ $QUICK == 1 ]]; then - RS_VERSIONS=$(cat $HERE/RS_VERSIONS | head -1) + RS_VERSIONS=$(cat $VERSIONFILE | head -1) else - RS_VERSIONS=$(cat $HERE/RS_VERSIONS) + RS_VERSIONS=$(cat $VERSIONFILE) fi for RS_VERSION in $RS_VERSIONS; do run_test $RS_VERSION