Install redis-gears from S3 as part of module deps, and test AI-Gears LLAPI.

Dockerfile.gpu-test

@@ -35,6 +35,7 @@ RUN echo export LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/cuda-11.0/lib64
 
 COPY ./opt/ opt/
 COPY ./tests/flow/test_requirements.txt tests/flow/
+COPY ./tests/flow/Install_RedisGears.sh tests/flow/
 
 RUN VENV=venv FORCE=1 ./opt/readies/bin/getpy3
 

opt/redis_valgrind.sup

@@ -68,3 +68,110 @@
    fun:RAI_LoadBackend
 }
 
+{
+   <ignore_py_error0>
+   Memcheck:Cond
+   fun:lzf_compress
+}
+{
+   <ignore_py_error0.1>
+   Memcheck:Cond
+   ...
+   fun:Py_InitializeEx
+   ...
+}
+{
+   <ignore_py_error0.2>
+   Memcheck:Cond
+   ...
+   fun:_PyEval_EvalFrameDefault
+   ...
+}
+{
+   <ignore_py_error0.3>
+   Memcheck:Leak
+   ...
+   fun:_imp_exec_builtin
+   ...
+}
+{
+   <ignore_py_error0.4>
+   Memcheck:Cond
+   ...
+   fun:PySys_SetArgvEx
+   ...
+}
+{
+   <ignore_py_error0.5>
+   Memcheck:Cond
+   ...
+   fun:_PyMethodDef_RawFastCallKeywords
+   ...
+}
+{
+   <ignore_py_error0>
+   Memcheck:Value8
+   fun:lzf_compress
+}
+{
+   <ignore_py_error1>
+   Memcheck:Addr4
+   obj:*libpython2.7*
+}
+{
+   <ignore_py_error1>
+   Memcheck:Addr8
+   obj:*libpython2.7*
+}
+{
+   <ignore_py_error2>
+   Memcheck:Value8
+   obj:*libpython2.7*
+}
+{
+   <ignore_py_error3>
+   Memcheck:Cond
+   obj:*libpython2.7*
+}
+{
+   <ignore_py_error4>
+   Memcheck:Leak
+   ...
+   obj:*libpython2.7*
+   ...
+}
+{
+   <ignore_libevent_issues>
+   Memcheck:Leak
+   ...
+   fun:redisLibeventAttach
+   ...
+}
+{
+   <ignore_compiler_visit_expr_for_now>
+   Memcheck:Leak
+   ...
+   fun:compiler_visit_expr
+   ...
+}
+{
+   <ignore_compiler_visit_expr_for_now>
+   Memcheck:Leak
+   ...
+   fun:redisConnectNonBlock
+   ...
+}
+{
+   <ignore_connection_leak_for_now>
+   Memcheck:Leak
+   ...
+   fun:_redisContextConnectTcp
+   ...
+}
+{
+   Libuv_epoll_ctl
+   Memcheck:Param
+   epoll_ctl(event)
+   fun:epoll_ctl
+   ...
+}

tests/flow/Install_RedisGears.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+
+set -e
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+ROOT=$(cd $HERE/../.. && pwd)
+READIES=$ROOT/opt/readies
+. $READIES/shibumi/defs
+
+if [[ "$1" == "--help" || "$1" == "help" || "$HELP" == "1" ]]; then
+	cat <<-END
+		Obtain RedisGears module binaries
+
+		Install_RedisGears.sh [--help|help]
+
+		Argument variables:
+		GEARS_OSNICK=nick   Get binaries for give osnick
+		GEARS_PATH=dir      Get binaries from given Gears repo
+		NOP=1               No operation
+		HELP=1              Show help
+
+	END
+	exit 0
+fi
+
+OP=""
+[[ "$NOP" == "1" ]] && OP=echo
+
+os="$($READIES/bin/platform --os)"
+arch="$($READIES/bin/platform --arch)"
+
+if [[ ! -z "$GEARS_PATH" ]]; then
+	platform="$($READIES/bin/platform -t)"
+else
+	if [[ "$os" != "linux" || "$arch" != "x64" ]]; then
+		eprint "Cannot match binary artifacts - build RedisGears and set GEARS_PATH"
+		exit 1
+	fi
+
+	dist="$($READIES/bin/platform --dist)"
+	nick="$($READIES/bin/platform --osnick)"
+
+	if [[ $dist == "ubuntu" ]]; then
+		if [[ $nick != "bionic" && $nick != "xenial" && $nick != "trusty" ]]; then
+			nick="bionic"
+		fi
+	elif [[ $dist == debian ]]; then
+		nick=bionic
+	elif [[ $dist == centos || $dist == redhat || $dist == fedora ]]; then
+		nick=centos7
+	elif [[ ! -z "$GEARS_OSNICK" ]]; then
+		nick=$GEARS_OSNICK
+	else
+		eprint "Cannot match binary artifacts - build RedisGears and set GEARS_PATH"
+		exit 1
+	fi
+	platform="${os}-${nick}-${arch}"
+fi
+
+GEARS_S3_URL="http://redismodules.s3.amazonaws.com/redisgears/snapshots"
+GEARS_MOD="redisgears.${platform}.master.zip"
+GEARS_DEPS="redisgears-python.${platform}.master.tgz"
+
+FINAL_WORK_DIR="$ROOT/bin/$($READIES/bin/platform -t)/RedisGears"
+
+if [[ -d $FINAL_WORK_DIR && -f $FINAL_WORK_DIR/redisgears.so ]]; then
+	echo "RedisGears is in ${FINAL_WORK_DIR}"
+	exit 0
+fi
+
+$OP mkdir -p $(dirname $FINAL_WORK_DIR)
+$OP rm -rf ${FINAL_WORK_DIR}.*
+WORK_DIR=$(mktemp -d ${FINAL_WORK_DIR}.XXXXXX)
+$OP mkdir -p $WORK_DIR
+
+if [[ -z $GEARS_PATH ]]; then
+	F_GEARS_MOD="$WORK_DIR/$GEARS_MOD"
+	if [[ ! -f $F_GEARS_MOD ]]; then
+		echo "Download RedisGears ..."
+		$OP wget -q -P $WORK_DIR $GEARS_S3_URL/$GEARS_MOD
+	fi
+
+	F_GEARS_DEPS="$WORK_DIR/$GEARS_DEPS"
+	if [[ ! -f $F_GEARS_DEPS ]]; then
+		echo "Download RedisGears deps ..."
+		$OP wget -q -P $WORK_DIR $GEARS_S3_URL/$GEARS_DEPS
+	fi
+else
+	F_GEARS_MOD="${GEARS_PATH}/artifacts/snapshot/${GEARS_MOD}"
+	F_GEARS_DEPS="${GEARS_PATH}/artifacts/snapshot/${GEARS_DEPS}"
+	[[ ! -f $F_GEARS_MOD ]] && { eprint "$F_GEARS_MOD is missing"; exit 1; }
+	[[ ! -f $F_GEARS_DEPS ]] && { eprint "$F_GEARS_DEPS is missing"; exit 1; }
+fi
+
+$OP unzip -q $F_GEARS_MOD -d $WORK_DIR
+$OP tar --no-same-owner -C $WORK_DIR -xzf $F_GEARS_DEPS
+$OP mv $WORK_DIR $FINAL_WORK_DIR

tests/flow/includes.py

@@ -10,12 +10,14 @@
 from skimage.io import imread
 from skimage.transform import resize
 
-try:
-    sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../deps/readies"))
-    import paella
-except:
-    pass
 
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../opt/readies"))
+import paella
+
+ROOT = os.environ.get("ROOT", None)
+if ROOT is None:
+    sys.stderr.write("ROOT was not defined in the environment.\n")
+    sys.exit(1)
 MAX_ITERATIONS = 2 if os.environ.get("MAX_ITERATIONS") == None else os.environ.get("MAX_ITERATIONS")
 TEST_TF = os.environ.get("TEST_TF") != "0" and os.environ.get("WITH_TF") != "0"
 TEST_TFLITE = os.environ.get("TEST_TFLITE") != "0" and os.environ.get("WITH_TFLITE") != "0"

tests/flow/tests.sh

@@ -136,6 +136,7 @@ cd $ROOT/tests/flow
 
 install_git_lfs
 check_redis_server
+./Install_RedisGears.sh
 
 [[ ! -z $REDIS ]] && RL_TEST_ARGS+=" --env exiting-env --existing-env-addr $REDIS" run_tests "redis-server: $REDIS"
 [[ $GEN == 1 ]]    && run_tests