[WIP] v1.0 fixes cherry pick

.circleci/config.yml

@@ -30,7 +30,7 @@ commands:
           command: |
             ./opt/readies/bin/getpy3
             BREW_NO_UPDATE=1 ./opt/system-setup.py
-            # ./opt/readies/bin/getredis -v 6 --force
+            ./opt/readies/bin/getredis -v 6 --force
             ./get_deps.sh cpu
       - save_cache:
           paths:
@@ -95,7 +95,7 @@ commands:
 jobs:
   build-debian:
     docker:
-      - image: redisfab/rmbuilder:6.0.1-x64-buster
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
     steps:
       - build-steps:
           platform: debian
@@ -112,7 +112,7 @@ jobs:
 
   coverage:
     docker:
-      - image: redisfab/rmbuilder:6.0.1-x64-buster
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
     steps:
       - checkout
       - run:
@@ -136,7 +136,7 @@ jobs:
       - run:
           name: Test with coverage
           command: |
-            make -C opt test SHOW=1 COV=1
+            make -C opt test SHOW=1 COV=1 CLUSTER=1
             make -C opt cov-upload
           no_output_timeout: 20m
 
@@ -209,7 +209,7 @@ jobs:
       package:
         type: string
     docker:
-      - image: redisfab/rmbuilder:6.0.1-x64-buster
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
     steps:
       - attach_workspace:
           at: workspace

Dockerfile.gpu-test

@@ -1,6 +1,6 @@
 # BUILD redisfab/redisai:${VERSION}-gpu-${ARCH}-${OSNICK}-test
 
-ARG REDIS_VER=6.0.1
+ARG REDIS_VER=6.0.5
 
 # OSNICK=bionic|centos7|centos6
 ARG OSNICK=bionic
@@ -29,11 +29,13 @@ COPY --from=redis /usr/local/ /usr/local/
 COPY ./opt/ opt/
 COPY ./test/test_requirements.txt test/
 
-RUN ./opt/readies/bin/getpy3
-RUN ./opt/system-setup.py
+RUN PIP=19.3.1 ./opt/readies/bin/getpy3
+RUN python3 -m pip install virtualenv
+
 RUN set -e ;\
-	python3 -m virtualenv venv --system-site-packages;\
+	python3 -m virtualenv venv --system-site-packages ;\
 	. venv/bin/activate ;\
+	python -m pip install pip==19.3.1 ;\
 	./opt/system-setup.py
 
 ARG DEPS_ARGS=""

opt/Makefile

@@ -189,12 +189,14 @@ endif
 export GEN ?= 1
 export SLAVES ?= 1
 export AOF ?= 1
+export CLUSTER ?= 1
 
 test:
 	$(COVERAGE_RESET)
 	$(SHOW)\
 		DEVICE=$(DEVICE) \
 		MODULE=$(INSTALLED_TARGET) \
+		CLUSTER=$(CLUSTER) \
 		GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \
 		VALGRIND=$(VALGRIND) \
 		$(ROOT)/test/tests.sh

opt/build/docker/Makefile

@@ -1,5 +1,6 @@
 
 ROOT=../../..
+MK.pyver:=3
 include $(ROOT)/opt/readies/mk/main
 
 # OSNICK=buster|stretch|bionic|xenial|centos7
@@ -8,6 +9,8 @@ OSNICK ?= buster
 REPO=redisfab
 STEM=$(REPO)/redisai
 
+REDIS_VER:=6.0.5
+
 ART_DIR=$(ROOT)/bin/artifacts
 ART_INT_DIR=/var/opt/redislabs/artifacts
 

opt/system-setup.py

@@ -18,15 +18,16 @@ def __init__(self, nop=False):
     def common_first(self):
         self.install_downloaders()
         self.setup_pip()
-        self.pip3_install("wheel virtualenv")
-        self.pip3_install("setuptools --upgrade")
+        self.pip_install("wheel virtualenv")
+        self.pip_install("setuptools --upgrade")
 
         if self.os == 'linux':
             self.install("ca-certificates")
-        self.install("git unzip wget patchelf awscli")
+        self.install("git unzip wget patchelf")
         self.install("coreutils") # for realpath
 
     def debian_compat(self):
+        self.pip_install("-IU --force-reinstall setuptools")
         self.install("build-essential cmake")
         self.install("python3-regex")
         self.install("python3-venv python3-psutil python3-networkx python3-numpy") # python3-skimage
@@ -53,7 +54,7 @@ def redhat_compat(self):
         else:
             self.run("amazon-linux-extras install epel", output_on_error=True)
             self.install("python3-devel")
-            self.pip3_install("psutil")
+            self.pip_install("psutil")
 
         self.install_git_lfs_on_linux()
 
@@ -76,14 +77,15 @@ def macosx(self):
     def common_last(self):
         self.run("python3 -m pip uninstall -y ramp-packer RLTest || true")
         # redis-py-cluster should be installed from git due to redis-py dependency
-        self.pip3_install("--no-cache-dir git+https://github.com/Grokzen/redis-py-cluster.git@master")
-        self.pip3_install("--no-cache-dir git+https://github.com/RedisLabsModules/RLTest.git@master")
-        self.pip3_install("--no-cache-dir git+https://github.com/RedisLabs/RAMP@master")
+        self.pip_install("--no-cache-dir git+https://github.com/Grokzen/redis-py-cluster.git@master")
+        self.pip_install("--no-cache-dir git+https://github.com/RedisLabsModules/RLTest.git@master")
+        self.pip_install("--no-cache-dir git+https://github.com/RedisLabs/RAMP@master")
 
-        self.pip3_install("-r %s/readies/paella/requirements.txt" % HERE)
-        self.pip3_install("-r %s/test/test_requirements.txt" % ROOT)
+        self.pip_install("-r %s/readies/paella/requirements.txt" % HERE)
+        self.pip_install("-r %s/test/test_requirements.txt" % ROOT)
 
-        self.pip3_install("mkdocs mkdocs-material mkdocs-extensions")
+        self.pip_install("awscli")
+        self.pip_install("mkdocs mkdocs-material mkdocs-extensions")
 
 #----------------------------------------------------------------------------------------------
 

ramp.yml

@@ -1,6 +1,6 @@
 display_name: RedisAI
-author: Orobix and RedisLabs
-email: luca.antiga@orobix.com
+author: Tensorwerk and RedisLabs
+email: support@redislabs.com
 description: Serving tensors and executing deep learning graphs
 homepage: https://oss.redislabs.com/redisai/
 license: Redis Source Available License v1.0

src/dag.c

@@ -258,7 +258,6 @@ int RAI_parseDAGLoadArgs(RedisModuleCtx *ctx, RedisModuleString **argv,
         ctx, "ERR invalid or negative value found in number of keys to LOAD");
     return -1;
   }
-
   int number_loaded_keys = 0;
   int separator_flag = 0;
   size_t argpos = 2;
@@ -270,8 +269,7 @@ int RAI_parseDAGLoadArgs(RedisModuleCtx *ctx, RedisModuleString **argv,
     } else {
       RAI_Tensor *t;
       RedisModuleKey *key;
-      const int status = RAI_GetTensorFromKeyspace(ctx, argv[argpos], &key, &t,
-                                                   REDISMODULE_READ);
+      const int status = RAI_GetTensorFromKeyspace(ctx, argv[argpos], &key, &t, REDISMODULE_READ);
       if (status == REDISMODULE_ERR) {
         RedisModule_Log(
             ctx, "warning",
@@ -334,3 +332,26 @@ int RAI_parseDAGPersistArgs(RedisModuleCtx *ctx, RedisModuleString **argv,
   }
   return argpos;
 }
+
+int RedisAI_DagRun_IsKeysPositionRequest_ReportKeys(RedisModuleCtx *ctx,
+                                                    RedisModuleString **argv, int argc){
+    for (size_t argpos = 1; argpos < argc; argpos++){
+        const char *arg_string = RedisModule_StringPtrLen(argv[argpos], NULL);
+        if ( (!strcasecmp(arg_string, "LOAD") || !strcasecmp(arg_string, "PERSIST") ) && (argpos+1 < argc) ) {
+            long long n_keys;
+            argpos++;
+            const int retval = RedisModule_StringToLongLong(argv[argpos], &n_keys);
+            if(retval != REDISMODULE_OK){
+                return REDISMODULE_ERR;
+            }
+            argpos++;
+            if (n_keys > 0){
+                size_t last_persist_argpos = n_keys+argpos;
+                for (; argpos < last_persist_argpos &&  argpos < argc; argpos++){
+                    RedisModule_KeyAtPos(ctx, argpos);
+                }
+            }
+        }
+    }
+    return REDISMODULE_OK;
+}

src/dag.h

@@ -73,4 +73,19 @@ int RAI_parseDAGPersistArgs(RedisModuleCtx *ctx, RedisModuleString **argv,
                             int argc, AI_dict **localContextDict,
                             const char *chaining_operator);
 
+/**
+ * When a module command is called in order to obtain the position of
+ * keys, since it was flagged as "getkeys-api" during the registration,
+ * the command implementation checks for this special call using the
+ * RedisModule_IsKeysPositionRequest() API and uses this function in
+ * order to report keys.
+ * No real execution is done on this special call.
+ * @param ctx Context in which Redis modules operate
+ * @param argv Redis command arguments, as an array of strings
+ * @param argc Redis command number of arguments
+ * @return
+ */
+int RedisAI_DagRun_IsKeysPositionRequest_ReportKeys(RedisModuleCtx *ctx,
+                            RedisModuleString **argv, int argc);
+
 #endif /* SRC_DAG_H_ */

src/model.c

@@ -513,6 +513,18 @@ int RAI_ModelSerialize(RAI_Model *model, char **buffer, size_t *len, RAI_Error *
   return ret;
 }
 
+int RedisAI_ModelRun_IsKeysPositionRequest_ReportKeys(RedisModuleCtx *ctx,
+                                                    RedisModuleString **argv, int argc){
+  RedisModule_KeyAtPos(ctx, 1);
+  for (size_t argpos = 3; argpos < argc; argpos++){
+    const char *str = RedisModule_StringPtrLen(argv[argpos], NULL);
+    if (!strcasecmp(str, "OUTPUTS")) {
+      continue;
+    }
+    RedisModule_KeyAtPos(ctx,argpos);
+  }
+  return REDISMODULE_OK;
+}
 
 int RedisAI_Parse_ModelRun_RedisCommand(RedisModuleCtx *ctx,
                                         RedisModuleString **argv, int argc,
@@ -620,7 +632,6 @@ int RedisAI_Parse_ModelRun_RedisCommand(RedisModuleCtx *ctx,
       }
     }
   }
-
   if ((*mto)->inputs && array_len((*mto)->inputs) != ninputs) {
     if (ctx == NULL) {
       RAI_SetError(

src/model.h

@@ -201,6 +201,21 @@ int RAI_ModelSerialize(RAI_Model* model, char** buffer, size_t* len,
 int RAI_GetModelFromKeyspace(RedisModuleCtx* ctx, RedisModuleString* keyName,
                              RedisModuleKey** key, RAI_Model** model, int mode);
 
+/**
+ * When a module command is called in order to obtain the position of
+ * keys, since it was flagged as "getkeys-api" during the registration,
+ * the command implementation checks for this special call using the
+ * RedisModule_IsKeysPositionRequest() API and uses this function in
+ * order to report keys.
+ * No real execution is done on this special call.
+ * @param ctx Context in which Redis modules operate
+ * @param argv Redis command arguments, as an array of strings
+ * @param argc Redis command number of arguments
+ * @return
+ */
+int RedisAI_ModelRun_IsKeysPositionRequest_ReportKeys(RedisModuleCtx *ctx,
+                            RedisModuleString **argv, int argc);
+
 /**
  * Helper method to parse AI.MODELRUN arguments
  *