Readies sync (mk/coverage)

.circleci/config.yml

@@ -15,16 +15,15 @@ commands:
       - run:
           name: Install dependencies
           command: |
-            ./opt/readies/bin/getpy
+            ./opt/readies/bin/getpy3
             BREW_NO_UPDATE=1 ./opt/system-setup.py
-            git clone git://github.com/antirez/redis.git --branch 5.0.7
-            (cd redis && make malloc=libc -j $(nproc) && make install)
-            redis-server --version
+            git clone git://github.com/antirez/redis.git --branch 5.0.7; (cd redis; make malloc=libc -j $(nproc); make install); redis-server --version
             ./get_deps.sh cpu
       - save_cache:
           paths:
             - deps
-          key: v1-dependencies-{{ checksum "get_deps.sh" }}
+            - /usr/local
+          key: build-dependencies-{{ checksum "get_deps.sh" }}
       - run:
           name: Set up workspace
           command: |
@@ -38,7 +37,6 @@ commands:
           command: |
             mkdir -p ~/workspace/tests
             make -C opt test SHOW=1 VERBOSE=1
-            cp test/logs/* ~/workspace/tests
       - run:
           name: Package
           command: make -C opt pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=~/workspace/packages SHOW=1
@@ -49,8 +47,9 @@ commands:
             - 'packages/release/*.tgz'
             - 'packages/branch/*.zip'
             - 'packages/branch/*.tgz'
-      - store_test_results:
-          path: ~/workspace/tests
+      - store_artifacts:
+          path: test/logs
+
   deploy:
     parameters:
       from:
@@ -68,16 +67,45 @@ jobs:
     steps:
       - ci_steps:
           platform: debian
+
+  coverage:
+    docker:
+      - image: redisfab/rmbuilder:x64-buster
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            ./opt/readies/bin/getpy3
+            ./opt/system-setup.py
+            # git clone git://github.com/antirez/redis.git --branch 5.0.7; cd redis; make malloc=libc -j $(nproc); make install; redis-server --version
+      - restore_cache:
+          keys:
+          - build-dependencies-{{ checksum "get_deps.sh" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
+      - run:
+          name: Build
+          command: |
+            # make fetch SHOW=1
+            make -C opt build COV=1 SHOW=1
+      - run:
+          name: Test with coverage
+          command: |
+            make -C opt test SHOW=1 COV=1
+            make -C opt cov-upload
+
   build-macos:
     macos:
-      xcode: 10.2.1
+      xcode: 11.1.0
     steps:
       - run:
           name: Fix macOS Python installation
           command: |
             brew reinstall -f python2
       - ci_steps:
           platform: macosx
+
   build-multiarch-docker:
     machine:
       enabled: true
@@ -102,6 +130,7 @@ jobs:
             sudo docker login -u redisfab -p $DOCKER_REDISFAB_PWD
             make -C opt/build/docker build
             sudo make -C opt/build/docker publish
+
   build-gpu:
     machine:
       enabled: true
@@ -110,13 +139,6 @@ jobs:
       image: ubuntu-1604-cuda-10.1:201909-23
     steps:
       - checkout
-#      - run:
-#          name: Checkout LFS
-#          command: |
-#            curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
-#            sudo apt-get install -y git-lfs
-#            git lfs install
-#            git lfs pull
       - run:
           name: Build
           command: |
@@ -127,8 +149,9 @@ jobs:
             mkdir -p ~/workspace/tests
             docker run --gpus all -v $HOME/workspace/tests:/build/test/logs -it --rm redisai-gpu:latest-x64-bionic-test
           no_output_timeout: 30m
-      - store_test_results:
+      - store_artifacts:
           path: ~/workspace/tests
+
   deploy_package:
     parameters:
       package:
@@ -164,6 +187,14 @@ workflows:
           filters:
             tags:
               only: /.*/
+      - coverage:
+          requires:
+            - build
+          filters:
+            branches:
+              only: /.*/
+            tags:
+              only: /.*/ 
       - build-macos:
           filters:
             branches:

.dockerignore

@@ -8,3 +8,4 @@ venv*/
 *.zip
 *.tgz
 *.tar.gz
+/VARAINT

.gitignore

@@ -11,6 +11,7 @@ venv*/
 *.zip
 *.tgz
 *.tar.gz
+/VARIANT
 
 # Misc
 .DS_Store

CMakeLists.txt

@@ -16,16 +16,29 @@ IF(NOT CMAKE_BUILD_TYPE)
             "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
 ENDIF()
 
-SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+#----------------------------------------------------------------------------------------------
+
+SET(CMAKE_CC_COMMON_FLAGS "-fPIC")
+
+IF (USE_COVERAGE)
+    IF (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+        MESSAGE(FATAL_ERROR "Build type must be DEBUG for coverage")
+    ENDIF()
+    SET(CMAKE_CC_COMMON_FLAGS "${CMAKE_CC_COMMON_FLAGS} -coverage")
+ENDIF()
+
+SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_CC_COMMON_FLAGS}")
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CC_COMMON_FLAGS}")
 
 # For adding specific Release flags
 set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3")
 set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
 
 # Add -fno-omit-frame-pointer to avoid seeing incomplete stack traces
-set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer")
-set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer")
+set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer -DVALGRIND")
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer -DVALGRIND")
+
+#----------------------------------------------------------------------------------------------
 
 option(BUILD_TF "Build the TensorFlow backend" ON)
 option(BUILD_TFLITE "Build the TensorFlow Lite backend" ON)

Dockerfile

@@ -21,7 +21,7 @@ COPY --from=redis /usr/local/ /usr/local/
 COPY ./opt/ opt/
 COPY ./test/test_requirements.txt test/
 
-RUN ./opt/readies/bin/getpy
+RUN ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py
 
 ARG DEPS_ARGS=""

Dockerfile.arm

@@ -19,7 +19,7 @@ WORKDIR /build
 COPY ./opt/ opt/
 COPY ./test/test_requirements.txt test/
 
-RUN ./opt/readies/bin/getpy
+RUN ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py
 
 ARG DEPS_ARGS=""

Dockerfile.gpu

@@ -22,7 +22,7 @@ COPY --from=redis /usr/local/ /usr/local/
 COPY ./opt/ opt/
 COPY ./test/test_requirements.txt test/
 
-RUN ./opt/readies/bin/getpy
+RUN ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py
 
 ARG DEPS_ARGS=""

Dockerfile.gpu-test

@@ -12,6 +12,8 @@ ARG ARCH=x64
 FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
 FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 AS builder
 
+SHELL ["/bin/bash", "-c"]
+
 ENV NVIDIA_VISIBLE_DEVICES all
 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
 
@@ -21,21 +23,37 @@ COPY --from=redis /usr/local/ /usr/local/
 COPY ./opt/ opt/
 COPY ./test/test_requirements.txt test/
 
-RUN ./opt/readies/bin/getpy
+RUN ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py
+RUN set -e ;\
+	python3 -m virtualenv venv --system-site-packages;\
+	. venv/bin/activate ;\
+	./opt/system-setup.py
 
 ARG DEPS_ARGS=""
 COPY ./get_deps.sh .
-RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_deps.sh gpu; fi
+RUN set -e ;\
+	. venv/bin/activate ;\
+	if [[ -z $DEPS_ARGS ]]; then \
+		./get_deps.sh gpu ;\
+	else \
+		env $DEPS_ARGS ./get_deps.sh gpu ;\
+	fi
 
 ARG BUILD_ARGS=""
 ADD ./ /build
-RUN make -C opt build GPU=1 $BUILD_ARGS SHOW=1
+RUN set -e ;\
+	. venv/bin/activate ;\
+	make -C opt build GPU=1 $BUILD_ARGS SHOW=1
 
 ARG PACK=1
 
-RUN if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi
+RUN set -e ;\
+	if [[ $PACK == 1 ]]; then \
+		. venv/bin/activate ;\
+		make -C opt pack GPU=1 VERBOSE=1 ;\
+	fi
 
 RUN git remote set-url origin https://github.com/RedisAI/RedisAI
 
-CMD ["bash", "-c", "make -C opt test GPU=1 SHOW=1"]
+CMD ["bash", "-c", ". ./venv/bin/activate; make -C opt test GPU=1 SHOW=1"]

README.md

@@ -3,6 +3,7 @@
 [![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/redisai/redisai.svg)](https://hub.docker.com/r/redisai/redisai/builds/)
 [![Mailing List](https://img.shields.io/badge/Mailing%20List-RedisAI-blue)](https://groups.google.com/forum/#!forum/redisai)
 [![Gitter](https://badges.gitter.im/RedisLabs/RedisAI.svg)](https://gitter.im/RedisLabs/RedisAI?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
+[![codecov](https://codecov.io/gh/RedisAI/RedisAI/branch/master/graph/badge.svg)](https://codecov.io/gh/RedisAI/RedisAI)
 
 
 # RedisAI