Test.refactor.coverage

.circleci/config.yml

@@ -37,8 +37,18 @@ commands:
           name: Test
           command: |
             mkdir -p ~/workspace/tests
-            make -C opt test SHOW=1 VERBOSE=1
+            make -C opt test SHOW=1 VERBOSE=1 WITH_COVERAGE=on
             cp test/logs/* ~/workspace/tests
+      - run:
+          name: Coverage
+          command: |
+            mkdir -p build
+            cd build 
+            cmake -DENABLE_CODECOVERAGE=on ..
+            make
+            make test ARGS="-V"
+            make coverage
+            bash <(curl -s https://codecov.io/bash) -f coverage.info -X gcov -x gcov-7 || echo "Codecov did not collect coverage reports"
       - run:
           name: Package
           command: make -C opt pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=~/workspace/packages SHOW=1
@@ -60,7 +70,6 @@ commands:
           name: Deploy to S3
           command: |
             aws s3 cp <<parameters.from>>/ s3://redismodules/$PACKAGE_NAME/ --acl public-read --recursive --exclude "*" --include "*.zip" --include "*.tgz"
-
 jobs:
   build:
     docker:

CMakeLists.txt

@@ -31,6 +31,7 @@ option(BUILD_TF "Build the TensorFlow backend" ON)
 option(BUILD_TFLITE "Build the TensorFlow Lite backend" ON)
 option(BUILD_ORT "Build the ONNXRuntime backend" ON)
 option(BUILD_TORCH "Build the PyTorch backend" ON)
+OPTION(ENABLE_CODECOVERAGE "Enable code coverage testing support" OFF)
 
 #----------------------------------------------------------------------------------------------
 
@@ -71,6 +72,23 @@ GET_FILENAME_COMPONENT(depsAbs
 GET_FILENAME_COMPONENT(installAbs
         "${INSTALL_PATH}" REALPATH BASE_DIR ${CMAKE_SOURCE_DIR})
 
+if(ENABLE_CODECOVERAGE)
+    message(STATUS "Forcing build type to Debug to run coverage.")
+    set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Choose the type of build." FORCE)
+    include(UseCodeCoverage)
+    find_package(PythonInterp 3 REQUIRED)
+    enable_testing()
+    set( ENV{DEVICE} "CPU" )
+    file(GLOB TEST_FILES
+    ${CMAKE_CURRENT_SOURCE_DIR}/test/tests_*.py
+    )
+    foreach(file ${TEST_FILES})
+        message(STATUS "Creating tests ${file}.")
+        add_test(NAME ${file}_single_master COMMAND ${PYTHON_EXECUTABLE} -m RLTest --check-exitcode --test ${file} -v --module redisai.so --module-args "BACKENDSPATH ${CMAKE_CURRENT_SOURCE_DIR}/build TORCH redisai_torch.so TF redisai_tensorflow.so ONNX redisai_onnxruntime.so TFLITE redisai_tflite.so" WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} )
+        add_test(NAME ${file}_aof COMMAND ${PYTHON_EXECUTABLE} -m RLTest --check-exitcode --test ${file} -v --module redisai.so --module-args "BACKENDSPATH ${CMAKE_CURRENT_SOURCE_DIR}/build TORCH redisai_torch.so TF redisai_tensorflow.so ONNX redisai_onnxruntime.so TFLITE redisai_tflite.so" --use-aof WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} )
+        add_test(NAME ${file}_use_slaves COMMAND ${PYTHON_EXECUTABLE} -m RLTest --check-exitcode --test ${file} -v --module redisai.so --module-args "BACKENDSPATH ${CMAKE_CURRENT_SOURCE_DIR}/build TORCH redisai_torch.so TF redisai_tensorflow.so ONNX redisai_onnxruntime.so TFLITE redisai_tflite.so" --use-slaves WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} )
+  endforeach()
+ENDIF()
 #----------------------------------------------------------------------------------------------
 
 INCLUDE_DIRECTORIES(${depsAbs}/dlpack/include)

README.md

@@ -3,6 +3,7 @@
 [![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/redisai/redisai.svg)](https://hub.docker.com/r/redisai/redisai/builds/)
 [![Mailing List](https://img.shields.io/badge/Mailing%20List-RedisAI-blue)](https://groups.google.com/forum/#!forum/redisai)
 [![Gitter](https://badges.gitter.im/RedisLabs/RedisAI.svg)](https://gitter.im/RedisLabs/RedisAI?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
+[![codecov](https://codecov.io/gh/RedisAI/RedisAI/branch/master/graph/badge.svg)](https://codecov.io/gh/RedisAI/RedisAI)
 
 
 # RedisAI

opt/cmake/modules/UseCodeCoverage.cmake

@@ -0,0 +1,34 @@
+# - Enable Code Coverage
+#
+# Variables you may define are:
+#  CODECOV_HTMLOUTPUTDIR - the name of the directory where HTML results are placed. Defaults to "coverage_html"
+#  CODECOV_XMLOUTPUTFILE - the name of the directory where HTML results are placed. Defaults to "coverage.xml"
+#  CODECOV_GCOVR_OPTIONS - additional options given to gcovr commands.
+#
+
+if(ENABLE_CODECOVERAGE)
+
+    if ( NOT CMAKE_BUILD_TYPE STREQUAL "Debug" )
+        message( WARNING "Code coverage results with an optimised (non-Debug) build may be misleading" )
+    endif ( NOT CMAKE_BUILD_TYPE STREQUAL "Debug" )
+
+    if ( NOT DEFINED CODECOV_OUTPUTFILE )
+        set( CODECOV_OUTPUTFILE coverage.info )
+    endif ( NOT DEFINED CODECOV_OUTPUTFILE )
+
+    if ( NOT DEFINED CODECOV_HTMLOUTPUTDIR )
+        set( CODECOV_HTMLOUTPUTDIR coverage_results )
+    endif ( NOT DEFINED CODECOV_HTMLOUTPUTDIR )
+
+    if ( CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_GNUCXX )
+        find_program( CODECOV_GCOV gcov )
+        find_program( CODECOV_LCOV lcov )
+        find_program( CODECOV_GENHTML genhtml )
+        add_definitions( -fprofile-arcs -ftest-coverage )
+        link_libraries( gcov )
+        set( CMAKE_EXE_LINKER_FLAGS ${CMAKE_EXE_LINKER_FLAGS} --coverage )
+        add_custom_target( coverage_init ALL ${CODECOV_LCOV} --base-directory .  --directory ${CMAKE_BINARY_DIR}/src --output-file ${CODECOV_OUTPUTFILE} --capture --initial )
+        add_custom_target( coverage ${CODECOV_LCOV} --base-directory .  --directory ${CMAKE_BINARY_DIR}/src --output-file ${CODECOV_OUTPUTFILE} --capture COMMAND genhtml -o ${CODECOV_HTMLOUTPUTDIR} ${CODECOV_OUTPUTFILE} )
+    endif ( CMAKE_COMPILER_IS_GNUCXX )
+
+endif(ENABLE_CODECOVERAGE)

opt/system-setup.py

@@ -22,7 +22,7 @@ def common_first(self):
 
         if self.os == 'linux':
             self.install("ca-certificates")
-        self.install("git cmake unzip wget patchelf awscli")
+        self.install("git cmake unzip wget patchelf awscli valgrind")
         self.install("coreutils") # for realpath
 
     def debian_compat(self):

src/backends/tensorflow.c

@@ -354,7 +354,6 @@ void RAI_ModelFreeTF(RAI_Model* model, RAI_Error* error) {
     }
     array_free(model->outputs);
   }
-
   TF_DeleteStatus(status);
 }
 

test/tests_tensorflow.py

@@ -87,7 +87,6 @@ def test_run_mobilenet_multiproc(env):
     con.execute_command('AI.MODELSET', 'mobilenet', 'TF', DEVICE,
                         'INPUTS', input_var, 'OUTPUTS', output_var, model_pb)
     ensureSlaveSynced(con, env)
-
     run_test_multiproc(env, 30, run_mobilenet, (img, input_var, output_var))
 
     ensureSlaveSynced(con, env)