Merge branch 'master' into Refactor_get_model/script_from_keyspace

Author: alonre24

.circleci/config.yml

@@ -17,7 +17,7 @@ commands:
       - run:
           name: Setup automation
           command: |
-            PIP=1 ./opt/readies/bin/getpy3
+            ./opt/readies/bin/getpy3
             ./opt/system-setup.py
 
   build-steps:
@@ -125,6 +125,7 @@ jobs:
       - run:
           name: Submodule checkout
           command: git submodule update --init --recursive
+      - setup-automation
       - run:
           name: lint
           command: |
@@ -201,7 +202,63 @@ jobs:
       - run:
           name: Test with valgrind
           command: |
-            make -C opt test VALGRIND=1
+            make -C opt test VALGRIND=1 CLUSTER=0 AOF=0
+          no_output_timeout: 120m
+
+  valgrind-cluster:
+    docker:
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
+    steps:
+      - checkout
+      - run:
+          name: Submodule checkout
+          command: git submodule update --init --recursive
+      - restore_cache:
+          keys:
+            - build-dependencies-{{ checksum "get_deps.sh" }}
+          # If no exact match is found will get dependencies from source
+      - setup-automation
+      - run:
+          name: Install dependencies
+          command: |
+            ./opt/readies/bin/getredis -v 6 --valgrind --force
+            ./get_deps.sh cpu
+      - run:
+          name: Build for valgrind with cluster
+          command: |
+            make -C opt all VALGRIND=1 SHOW=1
+      - run:
+          name: Test with valgrind and cluster
+          command: |
+            make -C opt test VALGRIND=1 GEN=0 AOF=0
+          no_output_timeout: 120m
+
+  valgrind-AOF:
+    docker:
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
+    steps:
+      - checkout
+      - run:
+          name: Submodule checkout
+          command: git submodule update --init --recursive
+      - restore_cache:
+          keys:
+            - build-dependencies-{{ checksum "get_deps.sh" }}
+          # If no exact match is found will get dependencies from source
+      - setup-automation
+      - run:
+          name: Install dependencies
+          command: |
+            ./opt/readies/bin/getredis -v 6 --valgrind --force
+            ./get_deps.sh cpu
+      - run:
+          name: Build for valgrind with AOF
+          command: |
+            make -C opt all VALGRIND=1 SHOW=1
+      - run:
+          name: Test with valgrind and AOF
+          command: |
+            make -C opt test VALGRIND=1 GEN=0 CLUSTER=0
           no_output_timeout: 120m
 
   build-macos:
@@ -385,7 +442,8 @@ workflows:
   version: 2
   build_and_package:
     jobs:
-      - lint
+      - lint:
+          <<: *on-any-branch
       - build-debian:
           <<: *on-any-branch
           <<: *after-linter
@@ -405,7 +463,13 @@ workflows:
           <<: *on-any-branch
           <<: *after-linter
       - valgrind:
-          <<: *on-master
+          <<: *on-any-branch
+          <<: *after-linter
+      - valgrind-cluster:
+          <<: *on-integ-branch
+          <<: *after-linter
+      - valgrind-AOF:
+          <<: *on-integ-branch
           <<: *after-linter
       - build-and-test-gpu:
           <<: *on-any-branch

docs/intro.md

@@ -8,6 +8,10 @@ This introduction is intended to present the core concepts it uses and the funct
 !!! important "Prerequisites"
     Before diving into RedisAI please make sure that you are familiar with the basic concepts of machine learning and Redis.
 
+This video will introduce you to RedisAI at a high level. Give it a watch, then keep reading for more details.
+
+<iframe width="560" height="315" src="https://www.youtube.com/embed/gbiqF-eyTW4" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+
 In broad strokes, RedisAI's looks as follows:
 
 ```

opt/readies

@@ -26,6 +26,13 @@
    obj:*/libtorch.so.*
 }
 
+{
+   ignore_unversioned_libs
+   Memcheck:Overlap
+   ...
+   obj:*/libtorch_cpu.so*
+}
+
 {
    ignore_unversioned_libs
    Memcheck:Leak

opt/redis_valgrind.sup

@@ -18,33 +18,28 @@ def __init__(self, nop=False):
 
     def common_first(self):
         self.install_downloaders()
-        self.setup_pip()
-        self.pip_install("wheel virtualenv")
+        self.pip_install("wheel")
 
-        if self.os == 'linux':
-            self.install("ca-certificates")
-        self.install("git unzip wget patchelf")
+        self.install("git unzip patchelf")
         self.install("coreutils") # for realpath
 
     def debian_compat(self):
         self.install("gawk")
         self.install("build-essential")
         self.install("libssl-dev")
-        self.install("clang-format")
         self.install("python3-regex")
         self.install("python3-psutil python3-networkx python3-numpy")
-        if self.arch == 'arm64v8' or self.arch == 'arm32v7':
+        if self.platform.is_arm():
             self.install("python3-dev") # python3-skimage
         self.install("libmpich-dev libopenblas-dev") # for libtorch
         self.install_git_lfs_on_linux()
 
     def redhat_compat(self):
         self.install("redhat-lsb-core")
-        self.run("%s/readies/bin/enable-utf8" % HERE)
+        self.run("%s/bin/enable-utf8" % READIES)
 
-        self.group_install("'Development Tools'")
         self.run("%s/bin/getgcc --modern" % READIES)
-        self.install("llvm-toolset-7")
+        # self.install("llvm-toolset-7")
 
         if self.arch == 'x64':
             self.install_linux_gnu_tar()
@@ -63,7 +58,6 @@ def redhat_compat(self):
     def fedora(self):
         self.group_install("'Development Tools'")
         self.install("python3 python3-psutil python3-networkx")
-        self.install("clang")
         self.install_git_lfs_on_linux()
 
     def linux_last(self):
@@ -73,18 +67,13 @@ def macos(self):
         self.install_gnu_utils()
         self.install("git-lfs")
         self.install("redis")
-        self.install("clang-format")
 
     def common_last(self):
+        self.run("%s/bin/getclang --format" % READIES)
         self.run("%s/bin/getcmake" % READIES)
 
-        self.run("python3 -m pip uninstall -y ramp-packer RLTest || true")
-        # redis-py-cluster should be installed from git due to redis-py dependency
-        self.pip_install("--no-cache-dir git+https://github.com/Grokzen/redis-py-cluster.git@master")
-        self.pip_install("--no-cache-dir git+https://github.com/RedisLabsModules/RLTest.git@master")
-        self.pip_install("--no-cache-dir git+https://github.com/RedisLabs/RAMP@master")
+        self.run("{PYTHON} {READIES}/bin/getrmpytools".format(PYTHON=self.python, READIES=READIES))
 
-        self.pip_install("-r %s/readies/paella/requirements.txt" % HERE)
         self.pip_install("-r %s/tests/flow/test_requirements.txt" % ROOT)
 
         self.pip_install("awscli")

opt/system-setup.py

@@ -16,6 +16,7 @@ file (GLOB BACKEND_COMMON_SRC
         err.c
         util/dict.c
         tensor.c
+        util/string_utils.c
         serialization/ai_datatypes.c)
 
 ADD_LIBRARY(redisai_obj OBJECT

src/CMakeLists.txt

@@ -122,8 +122,7 @@ static int _StoreTensorInKeySpace(RedisModuleCtx *ctx, RAI_Tensor *tensor,
         RedisModule_ReplyWithError(ctx, "ERR could not save tensor");
         goto clean_up;
     }
-    if (RedisModule_ModuleTypeSetValue(key, RedisAI_TensorType, RAI_TensorGetShallowCopy(tensor)) !=
-        REDISMODULE_OK) {
+    if (RedisModule_ModuleTypeSetValue(key, RedisAI_TensorType, tensor) != REDISMODULE_OK) {
         RedisModule_ReplyWithError(ctx, "ERR could not save tensor");
         RedisModule_CloseKey(key);
         goto clean_up;
@@ -152,6 +151,7 @@ static void _DAG_PersistTensors(RedisModuleCtx *ctx, RedisAI_RunInfo *rinfo) {
             persist_entry = AI_dictNext(persist_iter);
             continue;
         }
+        tensor = RAI_TensorGetShallowCopy(tensor);
         if (_StoreTensorInKeySpace(ctx, tensor, persist_key_name, true) == REDISMODULE_ERR) {
             *rinfo->dagError = 1;
             RedisModule_Log(ctx, "warning",

src/DAG/dag.c

@@ -122,7 +122,7 @@ int RAI_LoadBackend_TensorFlow(RedisModuleCtx *ctx, const char *path) {
     }
 
     backend.model_serialize = (int (*)(RAI_Model *, char **, size_t *, RAI_Error *))(
-        unsigned long)dlsym(handle, "RAI_ModelSerializeTF");
+        (unsigned long)dlsym(handle, "RAI_ModelSerializeTF"));
     if (backend.model_serialize == NULL) {
         dlclose(handle);
         RedisModule_Log(ctx, "warning",

src/backends.c

@@ -247,7 +247,7 @@ RAI_Model *RAI_ModelCreateTF(RAI_Backend backend, const char *devicestr, RAI_Mod
 
     for (size_t i = 0; i < ninputs; ++i) {
         TF_Operation *oper = TF_GraphOperationByName(model, inputs[i]);
-        if (oper == NULL) {
+        if (oper == NULL || strcmp(TF_OperationOpType(oper), "Placeholder") != 0) {
             size_t len = strlen(inputs[i]);
             char *msg = RedisModule_Calloc(60 + len, sizeof(*msg));
             sprintf(msg, "ERR Input node named \"%s\" not found in TF graph.", inputs[i]);

src/backends/tensorflow.c

@@ -56,8 +56,8 @@ torch::IValue redisExecute(std::string fn_name, std::vector<std::string> args )
     RedisModuleCallReply *reply = RedisModule_Call(ctx, fn_name.c_str(), "!v", arguments, len);
     RedisModule_ThreadSafeContextUnlock(ctx);
     torch::IValue value = IValueFromRedisReply(reply);
-    RedisModule_FreeThreadSafeContext(ctx);
     RedisModule_FreeCallReply(reply);
+    RedisModule_FreeThreadSafeContext(ctx);
     for(int i= 0; i < len; i++){
         RedisModule_FreeString(NULL, arguments[i]);
     }

src/libtorch_c/torch_extensions/torch_redis.cpp

@@ -15,6 +15,7 @@
 #include "rmutil/alloc.h"
 #include "tensor_struct.h"
 #include "util/dict.h"
+#include "util/string_utils.h"
 #include <assert.h>
 #include <pthread.h>
 #include <stddef.h>
@@ -96,7 +97,8 @@ int Tensor_DataTypeStr(DLDataType dtype, char *dtypestr) {
 
 RAI_Tensor *RAI_TensorCreateWithDLDataType(DLDataType dtype, long long *dims, int ndims,
                                            int tensorAllocMode) {
-    const size_t dtypeSize = Tensor_DataTypeSize(dtype);
+
+    size_t dtypeSize = Tensor_DataTypeSize(dtype);
     if (dtypeSize == 0) {
         return NULL;
     }
@@ -132,11 +134,6 @@ RAI_Tensor *RAI_TensorCreateWithDLDataType(DLDataType dtype, long long *dims, in
         break;
     }
 
-    if (tensorAllocMode != TENSORALLOC_NONE && data == NULL) {
-        RedisModule_Free(ret);
-        return NULL;
-    }
-
     ret->tensor = (DLManagedTensor){.dl_tensor = (DLTensor){.ctx = ctx,
                                                             .data = data,
                                                             .ndim = ndims,
@@ -162,18 +159,14 @@ void RAI_RStringDataTensorDeleter(DLManagedTensor *arg) {
         RedisModuleString *rstr = (RedisModuleString *)arg->manager_ctx;
         RedisModule_FreeString(NULL, rstr);
     }
-
+    RedisModule_Free(arg->dl_tensor.data);
     RedisModule_Free(arg);
 }
 
-RAI_Tensor *RAI_TensorCreateWithDLDataTypeAndRString(DLDataType dtype, long long *dims, int ndims,
-                                                     RedisModuleString *rstr) {
-    const size_t dtypeSize = Tensor_DataTypeSize(dtype);
-    if (dtypeSize == 0) {
-        return NULL;
-    }
+RAI_Tensor *_TensorCreateWithDLDataTypeAndRString(DLDataType dtype, size_t dtypeSize,
+                                                  long long *dims, int ndims,
+                                                  RedisModuleString *rstr, RAI_Error *err) {
 
-    RAI_Tensor *ret = RedisModule_Alloc(sizeof(*ret));
     int64_t *shape = RedisModule_Alloc(ndims * sizeof(*shape));
     int64_t *strides = RedisModule_Alloc(ndims * sizeof(*strides));
 
@@ -188,9 +181,21 @@ RAI_Tensor *RAI_TensorCreateWithDLDataTypeAndRString(DLDataType dtype, long long
     }
 
     DLContext ctx = (DLContext){.device_type = kDLCPU, .device_id = 0};
+    size_t nbytes = len * dtypeSize;
+
+    size_t blob_len;
+    const char *blob = RedisModule_StringPtrLen(rstr, &blob_len);
+    if (blob_len != nbytes) {
+        RedisModule_Free(shape);
+        RedisModule_Free(strides);
+        RAI_SetError(err, RAI_ETENSORSET, "ERR data length does not match tensor shape and type");
+        return NULL;
+    }
+    char *data = RedisModule_Alloc(nbytes);
+    memcpy(data, blob, nbytes);
+    RAI_HoldString(NULL, rstr);
 
-    char *data = (char *)RedisModule_StringPtrLen(rstr, NULL);
-
+    RAI_Tensor *ret = RedisModule_Alloc(sizeof(*ret));
     ret->tensor = (DLManagedTensor){.dl_tensor = (DLTensor){.ctx = ctx,
                                                             .data = data,
                                                             .ndim = ndims,
@@ -637,13 +642,16 @@ int RAI_parseTensorSetArgs(RedisModuleString **argv, int argc, RAI_Tensor **t, i
         RAI_SetError(error, RAI_ETENSORSET, "wrong number of arguments for 'AI.TENSORSET' command");
         return -1;
     }
+
     // get the tensor datatype
     const char *typestr = RedisModule_StringPtrLen(argv[2], NULL);
-    size_t datasize = RAI_TensorDataSizeFromString(typestr);
-    if (!datasize) {
+    DLDataType datatype = RAI_TensorDataTypeFromString(typestr);
+    size_t datasize = Tensor_DataTypeSize(datatype);
+    if (datasize == 0) {
         RAI_SetError(error, RAI_ETENSORSET, "ERR invalid data type");
         return -1;
     }
+
     const char *fmtstr;
     int datafmt = TENSOR_NONE;
     int tensorAllocMode = TENSORALLOC_CALLOC;
@@ -699,23 +707,17 @@ int RAI_parseTensorSetArgs(RedisModuleString **argv, int argc, RAI_Tensor **t, i
         }
     }
 
-    const long long nbytes = len * datasize;
-    size_t datalen;
-    const char *data;
-    DLDataType datatype = RAI_TensorDataTypeFromString(typestr);
     if (datafmt == TENSOR_BLOB) {
         RedisModuleString *rstr = argv[argpos];
-        RedisModule_RetainString(NULL, rstr);
-        *t = RAI_TensorCreateWithDLDataTypeAndRString(datatype, dims, ndims, rstr);
+        *t = _TensorCreateWithDLDataTypeAndRString(datatype, datasize, dims, ndims, rstr, error);
     } else {
         *t = RAI_TensorCreateWithDLDataType(datatype, dims, ndims, tensorAllocMode);
     }
-
-    if (!t) {
+    if (!(*t)) {
         array_free(dims);
-        RAI_SetError(error, RAI_ETENSORSET, "ERR could not create tensor");
         return -1;
     }
+
     long i = 0;
     if (datafmt == TENSOR_VALUES) {
         for (; (argpos <= argc - 1) && (i < len); argpos++) {

src/tensor.c

@@ -72,7 +72,6 @@ valgrind_config() {
 
 	RLTEST_ARGS+="\
 		--use-valgrind \
-		--vg-no-fail-on-errors \
 		--vg-suppressions $VALGRIND_SUPRESSIONS"
 }