diff --git a/.circleci/config.yml b/.circleci/config.yml index bada64872..c41aff474 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,12 +46,17 @@ commands: no_output_timeout: 20m - run: name: Package - command: make -C opt pack SHOW=1 + command: | + make -C opt pack SHOW=1 + (cd bin/artifacts; tar -cf snapshots-<>.tar snapshots/) - persist_to_workspace: root: bin/ paths: - artifacts/*.zip - artifacts/*.tgz + - artifacts/*.tar + # - artifacts/shapshots/*.zip + # - artifacts/shapshots/*.tgz - store_artifacts: path: test/logs @@ -71,16 +76,27 @@ commands: name: Build for platform command: | docker login -u redisfab -p $DOCKER_REDISFAB_PWD - cd opt/build/docker + pushd opt/build/docker #@@ make build publish $(./version-params) CPU=1 OSNICK=<> X64=1 ARTIFACTS=1 TEST=1 VERBOSE=1 make build publish $(./version-params) CPU=1 OSNICK=<> X64=1 ARTIFACTS=1 VERBOSE=1 make build publish $(./version-params) GPU=1 OSNICK=<> X64=1 ARTIFACTS=1 VERBOSE=1 + popd > /dev/null + logstar=bin/artifacts/tests-logs-cpu.tgz + logsdir=tests/logs/cpu + mkdir -p $logsdir + if [[ -e $logstar ]]; then tar -C $logsdir -xzf $logstar; fi + (cd bin/artifacts; tar -cf snapshots-<>.tar snapshots/) no_output_timeout: 40m - persist_to_workspace: root: bin/ paths: - artifacts/*.zip - artifacts/*.tgz + - artifacts/*.tar + # - artifacts/shapshots/*.zip + # - artifacts/shapshots/*.tgz + - store_artifacts: + path: test/logs deploy-steps: parameters: @@ -202,11 +218,11 @@ jobs: docker run --gpus all -v $HOME/tests:/build/test/logs -it --rm redisai-gpu:latest-x64-bionic-test no_output_timeout: 40m - store_artifacts: - path: tests + path: test/log deploy-artifacts: parameters: - package: + location: type: string docker: - image: redisfab/rmbuilder:6.0.5-x64-buster @@ -216,8 +232,42 @@ jobs: - run: name: Deploy to S3 command: | - cd workspace - aws s3 cp artifacts/ s3://redismodules/$PACKAGE_NAME/ --acl public-read --recursive --exclude "*" --include "*.zip" --include "*.tgz" + cd "workspace/artifacts/<>" + for f in *.zip snapshot/*.tgz; do + aws s3 cp $f s3://redismodules/$PACKAGE_NAME/<>/ --acl public-read + done + + deploy-snapshot: + docker: + - image: redisfab/rmbuilder:6.0.5-x64-buster + steps: + - attach_workspace: + at: workspace + - run: + name: Deploy Snapshots to S3 + command: | + cd workspace/artifacts + for f in snapshots-*.tar; do + tar xf $f + done + cd snapshots + for f in *.zip *.tgz; do + aws s3 cp --no-progress $f s3://redismodules/$PACKAGE_NAME/snapshots/ --acl public-read + done + + deploy-release: + docker: + - image: redisfab/rmbuilder:6.0.5-x64-buster + steps: + - attach_workspace: + at: workspace + - run: + name: Deploy Releases to S3 + command: | + cd workspace/artifacts + for f in *.zip *.tgz; do + aws s3 cp --no-progress $f s3://redismodules/$PACKAGE_NAME/ --acl public-read + done on-any-branch: &on-any-branch @@ -296,14 +346,10 @@ workflows: - build-multiarch-docker: <<: *never # temporarily disabled # <<: *on-version-tags - - deploy-artifacts: - name: deploy-branch - package: branch + - deploy-snapshot: <<: *after-platform-builds - <<: *on-any-branch - - deploy-artifacts: - name: deploy-release - package: release + <<: *on-master + - deploy-release: <<: *after-platform-builds <<: *on-version-tags diff --git a/CMakeLists.txt b/CMakeLists.txt index ab044343a..d725b5eec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,8 +39,10 @@ set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3") # Add -fno-omit-frame-pointer to avoid seeing incomplete stack traces -set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer -DVALGRIND") -set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer -DVALGRIND") +set(CMAKE_COMMON_FLAGS_DEBUG "-g -ggdb -fno-omit-frame-pointer -D_DEBUG -DVALGRIND -include \ + ${CMAKE_CURRENT_SOURCE_DIR}/src/common.h -I${CMAKE_CURRENT_SOURCE_DIR}/opt") +set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${CMAKE_COMMON_FLAGS_DEBUG}") +set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${CMAKE_COMMON_FLAGS_DEBUG}") #---------------------------------------------------------------------------------------------- diff --git a/Dockerfile b/Dockerfile index 1e2427668..f16b60a34 100755 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ COPY --from=redis /usr/local/ /usr/local/ COPY ./opt/ opt/ COPY ./test/test_requirements.txt test/ -RUN ./opt/readies/bin/getpy3 +RUN PIP=19.3.1 FORCE=1 ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py ARG DEPS_ARGS="" @@ -40,17 +40,24 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_ ARG BUILD_ARGS="" ADD ./ /build -RUN set -e ;\ +RUN bash -c "set -e ;\ . ./opt/readies/bin/sourced ./profile.d ;\ - make -C opt build $BUILD_ARGS SHOW=1 + make -C opt build $BUILD_ARGS SHOW=1" ARG PACK ARG TEST RUN mkdir -p bin/artifacts -RUN if [ "$PACK" = "1" ]; then make -C opt pack; fi +RUN set -e ;\ + if [ "$PACK" = "1" ]; then make -C opt pack; fi -RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test $BUILD_ARGS NO_LFS=1; fi +RUN set -e ;\ + if [ "$TEST" = "1" ]; then \ + TEST= make -C opt test $BUILD_ARGS NO_LFS=1 ;\ + if [[ -d test/logs ]]; then \ + tar -C test/logs -czf bin/artifacts/test-logs-cpu.tgz . ;\ + fi ;\ + fi #---------------------------------------------------------------------------------------------- FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} diff --git a/Dockerfile.gpu b/Dockerfile.gpu index 450b8a268..b0be68cff 100644 --- a/Dockerfile.gpu +++ b/Dockerfile.gpu @@ -5,12 +5,12 @@ ARG REDIS_VER=6.0.5 # OSNICK=bionic|centos7|centos6 ARG OSNICK=bionic -# ARCH=x64|arm64v8|arm32v7 -ARG ARCH=x64 - # OS=ubuntu18.04|ubuntu16.04|centos7 ARG OS=ubuntu18.04 +# ARCH=x64|arm64v8|arm32v7 +ARG ARCH=x64 + ARG CUDA_VER=10.1-cudnn7 ARG PACK=0 @@ -22,10 +22,11 @@ FROM nvidia/cuda:${CUDA_VER}-devel-${OS} AS builder ARG OSNICK ARG OS +ARG ARCH ARG REDIS_VER ARG CUDA_VER -RUN echo "Building for ${OSNICK} (${OS}) [with Redis ${REDIS_VER}]" +RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH} [with Redis ${REDIS_VER}]" ENV NVIDIA_VISIBLE_DEVICES all ENV NVIDIA_DRIVER_CAPABILITIES compute,utility @@ -36,7 +37,7 @@ COPY --from=redis /usr/local/ /usr/local/ COPY ./opt/ opt/ COPY ./test/test_requirements.txt test/ -RUN ./opt/readies/bin/getpy3 +RUN PIP=19.3.1 FORCE=1 ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py ARG DEPS_ARGS="" @@ -45,17 +46,24 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_ ARG BUILD_ARGS="" ADD ./ /build -RUN set -e ;\ +RUN bash -c "set -e ;\ . ./opt/readies/bin/sourced ./profile.d ;\ - make -C opt build GPU=1 $BUILD_ARGS SHOW=1 + make -C opt build GPU=1 $BUILD_ARGS SHOW=1" ARG PACK ARG TEST RUN mkdir -p bin/artifacts -RUN if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi +RUN set -e ;\ + if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi -RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1; fi +RUN set -e ;\ + if [ "$TEST" = "1" ]; then \ + TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1 ;\ + if [[ -d test/logs ]]; then \ + tar -C test/logs -czf bin/artifacts/test-logs-gpu.tgz . ;\ + fi ;\ + fi #---------------------------------------------------------------------------------------------- FROM nvidia/cuda:${CUDA_VER}-runtime-${OS} diff --git a/opt/pack.sh b/opt/pack.sh index cb975b45f..27022c7a3 100755 --- a/opt/pack.sh +++ b/opt/pack.sh @@ -112,8 +112,6 @@ pack_ramp() { >&2 cat /tmp/ramp.err exit 1 fi - - echo "Done." } #---------------------------------------------------------------------------------------------- @@ -130,13 +128,13 @@ pack_deps() { local backends_prefix_dir="" if [[ $depname == all ]]; then - local backens_dir=. + local backends_dir=. else - local backens_dir=${PRODUCT}_$depname + local backends_dir=${PRODUCT}_$depname fi cd $INSTALL_DIR/backends - { find $backens_dir -name "*.so*" | \ + { find $backends_dir -name "*.so*" | \ xargs tar -c --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' --transform "s,^,$backends_prefix_dir," 2>> /tmp/pack.err | \ gzip -n - > $tar_path ; E=$?; } || true sha256sum $tar_path | gawk '{print $1}' > $tar_path.sha256 diff --git a/opt/readies b/opt/readies index 6ff353088..eab95101e 160000 --- a/opt/readies +++ b/opt/readies @@ -1 +1 @@ -Subproject commit 6ff353088a43aa90a3bf268443d7623c474d566d +Subproject commit eab95101ea93261bd35792260711a2b08ad62dae diff --git a/opt/system-setup.py b/opt/system-setup.py index 651f28d84..ce0d38a54 100755 --- a/opt/system-setup.py +++ b/opt/system-setup.py @@ -31,23 +31,29 @@ def debian_compat(self): self.install("gawk") self.install("build-essential cmake") self.install("python3-regex") - self.install("python3-venv python3-psutil python3-networkx python3-numpy") # python3-skimage + self.install("python3-psutil python3-networkx python3-numpy") # python3-skimage self.install_git_lfs_on_linux() def redhat_compat(self): self.install("redhat-lsb-core") self.run("%s/readies/bin/enable-utf8" % HERE) - + self.group_install("'Development Tools'") self.install("cmake3") - self.run("ln -s `command -v cmake3` /usr/local/bin/cmake") - + self.run("ln -sf `command -v cmake3` /usr/local/bin/cmake") + self.install("centos-release-scl") self.install("devtoolset-8") self.run("cp /opt/rh/devtoolset-8/enable /etc/profile.d/scl-devtoolset-8.sh") paella.mkdir_p("%s/profile.d" % ROOT) self.run("cp /opt/rh/devtoolset-8/enable %s/profile.d/scl-devtoolset-8.sh" % ROOT) + self.run(""" + dir=$(mktemp -d /tmp/tar.XXXXXX) + (cd $dir; wget -q -O tar.tgz http://redismodules.s3.amazonaws.com/gnu/gnu-tar-1.32-x64-centos7.tgz; tar -xzf tar.tgz -C /; ) + rm -rf $dir + """) + if not self.dist == "amzn": self.install("epel-release") self.install("python3-devel libaec-devel") @@ -62,8 +68,8 @@ def redhat_compat(self): def fedora(self): self.group_install("'Development Tools'") self.install("cmake") - self.run("ln -s `command -v cmake3` /usr/local/bin/cmake") - self.install("python3-venv python3-psutil python3-networkx") + self.run("ln -sf `command -v cmake3` /usr/local/bin/cmake") + self.install("python3 python3-psutil python3-networkx") self.install_git_lfs_on_linux() def macosx(self): diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fee5dbd15..3c235a2e7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,3 +1,7 @@ +if (CMAKE_BUILD_TYPE STREQUAL Debug) + SET(DEBUG_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../opt/readies/cetara/diag/gdb.c") +endif() + ADD_LIBRARY(redisai_obj OBJECT util/dict.c util/queue.c @@ -19,7 +23,8 @@ ADD_LIBRARY(redisai_obj OBJECT rmutil/args.c rmutil/heap.c rmutil/priority_queue.c - rmutil/vector.c run_info.c) + rmutil/vector.c run_info.c + ${DEBUG_SRC}) IF(BUILD_TF) ADD_LIBRARY(redisai_tensorflow_obj OBJECT diff --git a/src/common.h b/src/common.h new file mode 100644 index 000000000..eb6e92bbd --- /dev/null +++ b/src/common.h @@ -0,0 +1,4 @@ + +#if defined(DEBUG) || defined(_DEBUG) +#include "readies/cetara/diag/gdb.h" +#endif diff --git a/test/includes.py b/test/includes.py index 03c5f3ec2..d779e2e59 100755 --- a/test/includes.py +++ b/test/includes.py @@ -24,8 +24,8 @@ COV = os.environ.get("COV") != "0" and os.environ.get("COV") != "0" DEVICE = os.environ.get('DEVICE', 'CPU').upper().encode('utf-8', 'ignore').decode('utf-8') VALGRIND = os.environ.get("VALGRIND") == "1" -print(f"Running tests on {DEVICE}\n") -print(f"Using a max of {MAX_ITERATIONS} iterations per test\n") +print("Running tests on {}\n".format(DEVICE)) +print("Using a max of {} iterations per test\n".format(MAX_ITERATIONS)) # change this to make inference tests longer MAX_TRANSACTIONS=100 diff --git a/test/tests.sh b/test/tests.sh index 0e3820c13..29396fa2c 100755 --- a/test/tests.sh +++ b/test/tests.sh @@ -80,7 +80,8 @@ valgrind_config() { run_tests() { local title="$1" - [[ ! -z $title ]] && { $ROOT/opt/readies/bin/sep0; printf "Tests with $title:\n\n"; } + [[ ! -z $title ]] && { $ROOT/opt/readies/bin/sep -0; printf "Tests with $title:\n\n"; } + cd $ROOT/test $OP python3 -m RLTest --clear-logs --module $MODULE $RLTEST_ARGS } @@ -102,16 +103,15 @@ OP="" MODULE=${MODULE:-$1} [[ -z $MODULE || ! -f $MODULE ]] && { echo "Module not found. Aborting."; exit 1; } -RLTEST_ARGS="" - [[ $VALGRIND == 1 || $VGD == 1 ]] && valgrind_config if [[ ! -z $TEST ]]; then - RLTEST_ARGS+=" --test $TEST" + RLTEST_ARGS+=" --test $TEST -s" export PYDEBUG=${PYDEBUG:-1} fi [[ $VERBOSE == 1 ]] && RLTEST_ARGS+=" -v" +[[ $GDB == 1 ]] && RLTEST_ARGS+=" -i --verbose" #----------------------------------------------------------------------------------------------