Merge branch 'master' into Refactor_get_model/script_from_keyspace

alonre24 · alonre24 · commit 620a27373dab · 2021-02-01T15:23:39.000+02:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -17,7 +17,7 @@ commands:
       - run:
           name: Setup automation
           command: |
-            PIP=1 ./opt/readies/bin/getpy3
+            ./opt/readies/bin/getpy3
             ./opt/system-setup.py
 
   build-steps:
@@ -125,6 +125,7 @@ jobs:
       - run:
           name: Submodule checkout
           command: git submodule update --init --recursive
+      - setup-automation
       - run:
           name: lint
           command: |
@@ -201,7 +202,63 @@ jobs:
       - run:
           name: Test with valgrind
           command: |
-            make -C opt test VALGRIND=1
+            make -C opt test VALGRIND=1 CLUSTER=0 AOF=0
+          no_output_timeout: 120m
+
+  valgrind-cluster:
+    docker:
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
+    steps:
+      - checkout
+      - run:
+          name: Submodule checkout
+          command: git submodule update --init --recursive
+      - restore_cache:
+          keys:
+            - build-dependencies-{{ checksum "get_deps.sh" }}
+          # If no exact match is found will get dependencies from source
+      - setup-automation
+      - run:
+          name: Install dependencies
+          command: |
+            ./opt/readies/bin/getredis -v 6 --valgrind --force
+            ./get_deps.sh cpu
+      - run:
+          name: Build for valgrind with cluster
+          command: |
+            make -C opt all VALGRIND=1 SHOW=1
+      - run:
+          name: Test with valgrind and cluster
+          command: |
+            make -C opt test VALGRIND=1 GEN=0 AOF=0
+          no_output_timeout: 120m
+
+  valgrind-AOF:
+    docker:
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
+    steps:
+      - checkout
+      - run:
+          name: Submodule checkout
+          command: git submodule update --init --recursive
+      - restore_cache:
+          keys:
+            - build-dependencies-{{ checksum "get_deps.sh" }}
+          # If no exact match is found will get dependencies from source
+      - setup-automation
+      - run:
+          name: Install dependencies
+          command: |
+            ./opt/readies/bin/getredis -v 6 --valgrind --force
+            ./get_deps.sh cpu
+      - run:
+          name: Build for valgrind with AOF
+          command: |
+            make -C opt all VALGRIND=1 SHOW=1
+      - run:
+          name: Test with valgrind and AOF
+          command: |
+            make -C opt test VALGRIND=1 GEN=0 CLUSTER=0
           no_output_timeout: 120m
 
   build-macos:
@@ -385,7 +442,8 @@ workflows:
   version: 2
   build_and_package:
     jobs:
-      - lint
+      - lint:
+          <<: *on-any-branch
       - build-debian:
           <<: *on-any-branch
           <<: *after-linter
@@ -405,7 +463,13 @@ workflows:
           <<: *on-any-branch
           <<: *after-linter
       - valgrind:
-          <<: *on-master
+          <<: *on-any-branch
+          <<: *after-linter
+      - valgrind-cluster:
+          <<: *on-integ-branch
+          <<: *after-linter
+      - valgrind-AOF:
+          <<: *on-integ-branch
           <<: *after-linter
       - build-and-test-gpu:
           <<: *on-any-branch
diff --git a/docs/intro.md b/docs/intro.md
@@ -8,6 +8,10 @@ This introduction is intended to present the core concepts it uses and the funct
 !!! important "Prerequisites"
     Before diving into RedisAI please make sure that you are familiar with the basic concepts of machine learning and Redis.
 
+This video will introduce you to RedisAI at a high level. Give it a watch, then keep reading for more details.
+
+<iframe width="560" height="315" src="https://www.youtube.com/embed/gbiqF-eyTW4" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+
 In broad strokes, RedisAI's looks as follows:
 
 ```
diff --git a/opt/readies b/opt/readies
@@ -1 +1 @@
-Subproject commit b6d26cf551f0f648cc1b04fa60daa4959cfbc796
+Subproject commit b300e14f44c80bf7ca0e74cf60be22720da49e9e
diff --git a/opt/redis_valgrind.sup b/opt/redis_valgrind.sup
@@ -26,6 +26,13 @@
    obj:*/libtorch.so.*
 }
 
+{
+   ignore_unversioned_libs
+   Memcheck:Overlap
+   ...
+   obj:*/libtorch_cpu.so*
+}
+
 {
    ignore_unversioned_libs
    Memcheck:Leak
diff --git a/opt/system-setup.py b/opt/system-setup.py
@@ -18,33 +18,28 @@ def __init__(self, nop=False):
 
     def common_first(self):
         self.install_downloaders()
-        self.setup_pip()
-        self.pip_install("wheel virtualenv")
+        self.pip_install("wheel")
 
-        if self.os == 'linux':
-            self.install("ca-certificates")
-        self.install("git unzip wget patchelf")
+        self.install("git unzip patchelf")
         self.install("coreutils") # for realpath
 
     def debian_compat(self):
         self.install("gawk")
         self.install("build-essential")
         self.install("libssl-dev")
-        self.install("clang-format")
         self.install("python3-regex")
         self.install("python3-psutil python3-networkx python3-numpy")
-        if self.arch == 'arm64v8' or self.arch == 'arm32v7':
+        if self.platform.is_arm():
             self.install("python3-dev") # python3-skimage
         self.install("libmpich-dev libopenblas-dev") # for libtorch
         self.install_git_lfs_on_linux()
 
     def redhat_compat(self):
         self.install("redhat-lsb-core")
-        self.run("%s/readies/bin/enable-utf8" % HERE)
+        self.run("%s/bin/enable-utf8" % READIES)
 
-        self.group_install("'Development Tools'")
         self.run("%s/bin/getgcc --modern" % READIES)
-        self.install("llvm-toolset-7")
+        # self.install("llvm-toolset-7")
 
         if self.arch == 'x64':
             self.install_linux_gnu_tar()
@@ -63,7 +58,6 @@ def redhat_compat(self):
     def fedora(self):
         self.group_install("'Development Tools'")
         self.install("python3 python3-psutil python3-networkx")
-        self.install("clang")
         self.install_git_lfs_on_linux()
 
     def linux_last(self):
@@ -73,18 +67,13 @@ def macos(self):
         self.install_gnu_utils()
         self.install("git-lfs")
         self.install("redis")
-        self.install("clang-format")
 
     def common_last(self):
+        self.run("%s/bin/getclang --format" % READIES)
         self.run("%s/bin/getcmake" % READIES)
 
-        self.run("python3 -m pip uninstall -y ramp-packer RLTest || true")
-        # redis-py-cluster should be installed from git due to redis-py dependency
-        self.pip_install("--no-cache-dir git+https://github.com/Grokzen/redis-py-cluster.git@master")
-        self.pip_install("--no-cache-dir git+https://github.com/RedisLabsModules/RLTest.git@master")
-        self.pip_install("--no-cache-dir git+https://github.com/RedisLabs/RAMP@master")
+        self.run("{PYTHON} {READIES}/bin/getrmpytools".format(PYTHON=self.python, READIES=READIES))
 
-        self.pip_install("-r %s/readies/paella/requirements.txt" % HERE)
         self.pip_install("-r %s/tests/flow/test_requirements.txt" % ROOT)
 
         self.pip_install("awscli")
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -16,6 +16,7 @@ file (GLOB BACKEND_COMMON_SRC
         err.c
         util/dict.c
         tensor.c
+        util/string_utils.c
         serialization/ai_datatypes.c)
 
 ADD_LIBRARY(redisai_obj OBJECT
diff --git a/src/DAG/dag.c b/src/DAG/dag.c
@@ -122,8 +122,7 @@ static int _StoreTensorInKeySpace(RedisModuleCtx *ctx, RAI_Tensor *tensor,
         RedisModule_ReplyWithError(ctx, "ERR could not save tensor");
         goto clean_up;
     }
-    if (RedisModule_ModuleTypeSetValue(key, RedisAI_TensorType, RAI_TensorGetShallowCopy(tensor)) !=
-        REDISMODULE_OK) {
+    if (RedisModule_ModuleTypeSetValue(key, RedisAI_TensorType, tensor) != REDISMODULE_OK) {
         RedisModule_ReplyWithError(ctx, "ERR could not save tensor");
         RedisModule_CloseKey(key);
         goto clean_up;
@@ -152,6 +151,7 @@ static void _DAG_PersistTensors(RedisModuleCtx *ctx, RedisAI_RunInfo *rinfo) {
             persist_entry = AI_dictNext(persist_iter);
             continue;
         }
+        tensor = RAI_TensorGetShallowCopy(tensor);
         if (_StoreTensorInKeySpace(ctx, tensor, persist_key_name, true) == REDISMODULE_ERR) {
             *rinfo->dagError = 1;
             RedisModule_Log(ctx, "warning",
diff --git a/src/backends.c b/src/backends.c
@@ -122,7 +122,7 @@ int RAI_LoadBackend_TensorFlow(RedisModuleCtx *ctx, const char *path) {
     }
 
     backend.model_serialize = (int (*)(RAI_Model *, char **, size_t *, RAI_Error *))(
-        unsigned long)dlsym(handle, "RAI_ModelSerializeTF");
+        (unsigned long)dlsym(handle, "RAI_ModelSerializeTF"));
     if (backend.model_serialize == NULL) {
         dlclose(handle);
         RedisModule_Log(ctx, "warning",
diff --git a/src/backends/tensorflow.c b/src/backends/tensorflow.c
@@ -247,7 +247,7 @@ RAI_Model *RAI_ModelCreateTF(RAI_Backend backend, const char *devicestr, RAI_Mod
 
     for (size_t i = 0; i < ninputs; ++i) {
         TF_Operation *oper = TF_GraphOperationByName(model, inputs[i]);
-        if (oper == NULL) {
+        if (oper == NULL || strcmp(TF_OperationOpType(oper), "Placeholder") != 0) {
             size_t len = strlen(inputs[i]);
             char *msg = RedisModule_Calloc(60 + len, sizeof(*msg));
             sprintf(msg, "ERR Input node named \"%s\" not found in TF graph.", inputs[i]);
diff --git a/src/libtorch_c/torch_extensions/torch_redis.cpp b/src/libtorch_c/torch_extensions/torch_redis.cpp
@@ -56,8 +56,8 @@ torch::IValue redisExecute(std::string fn_name, std::vector<std::string> args )
     RedisModuleCallReply *reply = RedisModule_Call(ctx, fn_name.c_str(), "!v", arguments, len);
     RedisModule_ThreadSafeContextUnlock(ctx);
     torch::IValue value = IValueFromRedisReply(reply);
-    RedisModule_FreeThreadSafeContext(ctx);
     RedisModule_FreeCallReply(reply);
+    RedisModule_FreeThreadSafeContext(ctx);
     for(int i= 0; i < len; i++){
         RedisModule_FreeString(NULL, arguments[i]);
     }
diff --git a/src/tensor.c b/src/tensor.c
@@ -15,6 +15,7 @@
 #include "rmutil/alloc.h"
 #include "tensor_struct.h"
 #include "util/dict.h"
+#include "util/string_utils.h"
 #include <assert.h>
 #include <pthread.h>
 #include <stddef.h>
@@ -96,7 +97,8 @@ int Tensor_DataTypeStr(DLDataType dtype, char *dtypestr) {
 
 RAI_Tensor *RAI_TensorCreateWithDLDataType(DLDataType dtype, long long *dims, int ndims,
                                            int tensorAllocMode) {
-    const size_t dtypeSize = Tensor_DataTypeSize(dtype);
+
+    size_t dtypeSize = Tensor_DataTypeSize(dtype);
     if (dtypeSize == 0) {
         return NULL;
     }
@@ -132,11 +134,6 @@ RAI_Tensor *RAI_TensorCreateWithDLDataType(DLDataType dtype, long long *dims, in
         break;
     }
 
-    if (tensorAllocMode != TENSORALLOC_NONE && data == NULL) {
-        RedisModule_Free(ret);
-        return NULL;
-    }
-
     ret->tensor = (DLManagedTensor){.dl_tensor = (DLTensor){.ctx = ctx,
                                                             .data = data,
                                                             .ndim = ndims,
@@ -162,18 +159,14 @@ void RAI_RStringDataTensorDeleter(DLManagedTensor *arg) {
         RedisModuleString *rstr = (RedisModuleString *)arg->manager_ctx;
         RedisModule_FreeString(NULL, rstr);
     }
-
+    RedisModule_Free(arg->dl_tensor.data);
     RedisModule_Free(arg);
 }
 
-RAI_Tensor *RAI_TensorCreateWithDLDataTypeAndRString(DLDataType dtype, long long *dims, int ndims,
-                                                     RedisModuleString *rstr) {
-    const size_t dtypeSize = Tensor_DataTypeSize(dtype);
-    if (dtypeSize == 0) {
-        return NULL;
-    }
+RAI_Tensor *_TensorCreateWithDLDataTypeAndRString(DLDataType dtype, size_t dtypeSize,
+                                                  long long *dims, int ndims,
+                                                  RedisModuleString *rstr, RAI_Error *err) {
 
-    RAI_Tensor *ret = RedisModule_Alloc(sizeof(*ret));
     int64_t *shape = RedisModule_Alloc(ndims * sizeof(*shape));
     int64_t *strides = RedisModule_Alloc(ndims * sizeof(*strides));
 
@@ -188,9 +181,21 @@ RAI_Tensor *RAI_TensorCreateWithDLDataTypeAndRString(DLDataType dtype, long long
     }
 
     DLContext ctx = (DLContext){.device_type = kDLCPU, .device_id = 0};
+    size_t nbytes = len * dtypeSize;
+
+    size_t blob_len;
+    const char *blob = RedisModule_StringPtrLen(rstr, &blob_len);
+    if (blob_len != nbytes) {
+        RedisModule_Free(shape);
+        RedisModule_Free(strides);
+        RAI_SetError(err, RAI_ETENSORSET, "ERR data length does not match tensor shape and type");
+        return NULL;
+    }
+    char *data = RedisModule_Alloc(nbytes);
+    memcpy(data, blob, nbytes);
+    RAI_HoldString(NULL, rstr);
 
-    char *data = (char *)RedisModule_StringPtrLen(rstr, NULL);
-
+    RAI_Tensor *ret = RedisModule_Alloc(sizeof(*ret));
     ret->tensor = (DLManagedTensor){.dl_tensor = (DLTensor){.ctx = ctx,
                                                             .data = data,
                                                             .ndim = ndims,
@@ -637,13 +642,16 @@ int RAI_parseTensorSetArgs(RedisModuleString **argv, int argc, RAI_Tensor **t, i
         RAI_SetError(error, RAI_ETENSORSET, "wrong number of arguments for 'AI.TENSORSET' command");
         return -1;
     }
+
     // get the tensor datatype
     const char *typestr = RedisModule_StringPtrLen(argv[2], NULL);
-    size_t datasize = RAI_TensorDataSizeFromString(typestr);
-    if (!datasize) {
+    DLDataType datatype = RAI_TensorDataTypeFromString(typestr);
+    size_t datasize = Tensor_DataTypeSize(datatype);
+    if (datasize == 0) {
         RAI_SetError(error, RAI_ETENSORSET, "ERR invalid data type");
         return -1;
     }
+
     const char *fmtstr;
     int datafmt = TENSOR_NONE;
     int tensorAllocMode = TENSORALLOC_CALLOC;
@@ -699,23 +707,17 @@ int RAI_parseTensorSetArgs(RedisModuleString **argv, int argc, RAI_Tensor **t, i
         }
     }
 
-    const long long nbytes = len * datasize;
-    size_t datalen;
-    const char *data;
-    DLDataType datatype = RAI_TensorDataTypeFromString(typestr);
     if (datafmt == TENSOR_BLOB) {
         RedisModuleString *rstr = argv[argpos];
-        RedisModule_RetainString(NULL, rstr);
-        *t = RAI_TensorCreateWithDLDataTypeAndRString(datatype, dims, ndims, rstr);
+        *t = _TensorCreateWithDLDataTypeAndRString(datatype, datasize, dims, ndims, rstr, error);
     } else {
         *t = RAI_TensorCreateWithDLDataType(datatype, dims, ndims, tensorAllocMode);
     }
-
-    if (!t) {
+    if (!(*t)) {
         array_free(dims);
-        RAI_SetError(error, RAI_ETENSORSET, "ERR could not create tensor");
         return -1;
     }
+
     long i = 0;
     if (datafmt == TENSOR_VALUES) {
         for (; (argpos <= argc - 1) && (i < len); argpos++) {
diff --git a/tests/flow/tests.sh b/tests/flow/tests.sh
@@ -72,7 +72,6 @@ valgrind_config() {
 
 	RLTEST_ARGS+="\
 		--use-valgrind \
-		--vg-no-fail-on-errors \
 		--vg-suppressions $VALGRIND_SUPRESSIONS"
 }
 
diff --git a/tests/flow/tests_common.py b/tests/flow/tests_common.py
diff --git a/tests/flow/tests_dag.py b/tests/flow/tests_dag.py
diff --git a/tests/flow/tests_llapi.py b/tests/flow/tests_llapi.py
diff --git a/tests/flow/tests_tensorflow.py b/tests/flow/tests_tensorflow.py

Original file line number	Diff line number	Diff line change
`@@ -26,6 +26,13 @@`
`26`	`26`	`obj:/libtorch.so.`
`27`	`27`	`}`
`28`	`28`
	`29`	`+{`
	`30`	`+ ignore_unversioned_libs`
	`31`	`+ Memcheck:Overlap`
	`32`	`+ ...`
	`33`	`+ obj:/libtorch_cpu.so`
	`34`	`+}`
	`35`	`+`
`29`	`36`	`{`
`30`	`37`	`ignore_unversioned_libs`
`31`	`38`	`Memcheck:Leak`
Original file line number	Diff line number	Diff line change
`@@ -122,7 +122,7 @@ int RAI_LoadBackend_TensorFlow(RedisModuleCtx ctx, const char path) {`
`122`	`122`	`}`
`123`	`123`
`124`	`124`	`backend.model_serialize = (int ()(RAI_Model , char *, size_t , RAI_Error *))(`
`125`		`- unsigned long)dlsym(handle, "RAI_ModelSerializeTF");`
	`125`	`+ (unsigned long)dlsym(handle, "RAI_ModelSerializeTF"));`
`126`	`126`	`if (backend.model_serialize == NULL) {`
`127`	`127`	`dlclose(handle);`
`128`	`128`	`RedisModule_Log(ctx, "warning",`