From 29548e7c9fdd541e153154b2c2623d705f9757ba Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Thu, 20 Feb 2020 18:38:46 +0000 Subject: [PATCH 1/4] [add] tests refactoring per backend --- opt/Makefile | 22 +-- test/__init__.py | 0 test/includes.py | 88 ++++++++++ test/tests_common.py | 317 +++++++++++++++++++++++++++++++++++ test/tests_onnx.py | 169 +++++++++++++++++++ test/tests_pytorch.py | 353 +++++++++++++++++++++++++++++++++++++++ test/tests_tensorflow.py | 330 ++++++++++++++++++++++++++++++++++++ test/tests_tflite.py | 117 +++++++++++++ 8 files changed, 1386 insertions(+), 10 deletions(-) create mode 100644 test/__init__.py create mode 100755 test/includes.py create mode 100644 test/tests_common.py create mode 100644 test/tests_onnx.py create mode 100644 test/tests_pytorch.py create mode 100644 test/tests_tensorflow.py create mode 100644 test/tests_tflite.py diff --git a/opt/Makefile b/opt/Makefile index aac388dbd..70edf6a1d 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -142,42 +142,44 @@ endif #---------------------------------------------------------------------------------------------- +TEST_CMD=DEVICE=$(DEVICE) PYDEBUG=$(PYDEBUG) python3 -m RLTest $(TEST_ARGS) --module $(INSTALL_DIR)/redisai.so + TEST_REPORT_DIR ?= $(PWD) ifeq ($(VERBOSE),1) TEST_ARGS += -v endif ifeq ($(TEST),) -TEST=basic_tests.py +TEST=tests_common.py tests_pytorch.py tests_onnx.py tests_tflite.py tests_tensorflow.py PYDEBUG= else TEST_ARGS += -s PYDEBUG=1 endif -TEST_PREFIX=set -e; cd $(ROOT)/test -TEST_CMD=\ - DEVICE=$(DEVICE) PYDEBUG=$(PYDEBUG) \ - python3 -m RLTest $(TEST_ARGS) --test $(TEST) --module $(INSTALL_DIR)/redisai.so - GEN ?= 1 SLAVES ?= 1 AOF ?= 1 -test: +TEST_PREFIX=set -e; cd $(ROOT)/test + + +test: $(TEST) + +$(TEST): ifneq ($(NO_LFS),1) $(SHOW)if [ "$(git lfs env > /dev/null 2>&1 ; echo $?)" != "0" ]; then cd $(ROOT); git lfs install; fi $(SHOW)cd $(ROOT); git lfs pull endif ifeq ($(GEN),1) - $(SHOW)$(TEST_PREFIX); $(TEST_CMD) + $(SHOW)$(TEST_PREFIX); $(TEST_CMD) --test $@ endif ifeq ($(AOF),1) $(SHOW)$(TEST_PREFIX); printf "\nTests with --use-aof:\n\n" ;\ - $(TEST_CMD) --use-aof + $(TEST_CMD) --use-aof --test $@ endif ifeq ($(SLAVES),1) $(SHOW)$(TEST_PREFIX); printf "\nTests with --use-slaves:\n\n" ;\ - $(TEST_CMD) --use-slaves + $(TEST_CMD) --use-slaves --test $@ endif #---------------------------------------------------------------------------------------------- diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/includes.py b/test/includes.py new file mode 100755 index 000000000..52611d4bc --- /dev/null +++ b/test/includes.py @@ -0,0 +1,88 @@ +import json +import os +import random +import sys +import time +from multiprocessing import Process + +import numpy as np +from skimage.io import imread +from skimage.transform import resize + +try: + sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../deps/readies")) + import paella +except: + pass + +TEST_TF = os.environ.get("TEST_TF") != "0" and os.environ.get("WITH_TF") != "0" +TEST_TFLITE = os.environ.get("TEST_TFLITE") != "0" and os.environ.get("WITH_TFLITE") != "0" +TEST_PT = os.environ.get("TEST_PT") != "0" and os.environ.get("WITH_PT") != "0" +TEST_ONNX = os.environ.get("TEST_ONNX") != "0" and os.environ.get("WITH_ORT") != "0" +DEVICE = os.environ.get('DEVICE', 'CPU').upper() +print(f"Running tests on {DEVICE}\n") + + +def ensureSlaveSynced(con, env): + if env.useSlaves: + # When WAIT returns, all the previous write commands + # sent in the context of the current connection are + # guaranteed to be received by the number of replicas returned by WAIT. + wait_reply = con.execute_command('WAIT', '1', '1000') + env.assertTrue(wait_reply >= 1) + + +def check_cuda(): + return os.system('which nvcc') + + +def info_to_dict(info): + info = [el.decode('ascii') if type(el) is bytes else el for el in info] + return dict(zip(info[::2], info[1::2])) + + +def load_mobilenet_test_data(): + test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') + labels_filename = os.path.join(test_data_path, 'imagenet_class_index.json') + image_filename = os.path.join(test_data_path, 'panda.jpg') + model_filename = os.path.join(test_data_path, 'mobilenet_v2_1.4_224_frozen.pb') + + with open(model_filename, 'rb') as f: + model_pb = f.read() + + with open(labels_filename, 'r') as f: + labels = json.load(f) + + img_height, img_width = 224, 224 + + img = imread(image_filename) + img = resize(img, (img_height, img_width), mode='constant', anti_aliasing=True) + img = img.astype(np.float32) + + return model_pb, labels, img + + +def run_mobilenet(con, img, input_var, output_var): + time.sleep(0.5 * random.randint(0, 10)) + con.execute_command('AI.TENSORSET', 'input', + 'FLOAT', 1, img.shape[1], img.shape[0], img.shape[2], + 'BLOB', img.tobytes()) + + con.execute_command('AI.MODELRUN', 'mobilenet', + 'INPUTS', 'input', 'OUTPUTS', 'output') + + +def run_test_multiproc(env, n_procs, fn, args=tuple()): + procs = [] + + def tmpfn(): + con = env.getConnection() + fn(con, *args) + return 1 + + for _ in range(n_procs): + p = Process(target=tmpfn) + p.start() + procs.append(p) + + [p.join() for p in procs] diff --git a/test/tests_common.py b/test/tests_common.py new file mode 100644 index 000000000..1768bea45 --- /dev/null +++ b/test/tests_common.py @@ -0,0 +1,317 @@ +import redis + +from includes import * + +''' +python -m RLTest --test tests_common.py --module path/to/redisai.so +''' + + +def example_multiproc_fn(env): + env.execute_command('set', 'x', 1) + + +def test_example_multiproc(env): + run_test_multiproc(env, 10, lambda x: x.execute_command('set', 'x', 1)) + r = env.cmd('get', 'x') + env.assertEqual(r, b'1') + + +def test_set_tensor(env): + con = env.getConnection() + con.execute_command('AI.TENSORSET', 'x', 'FLOAT', 2, 'VALUES', 2, 3) + + ensureSlaveSynced(con, env) + + tensor = con.execute_command('AI.TENSORGET', 'x', 'VALUES') + values = tensor[-1] + env.assertEqual(values, [b'2', b'3']) + + con.execute_command('AI.TENSORSET', 'x', 'INT32', 2, 'VALUES', 2, 3) + + ensureSlaveSynced(con, env) + + tensor = con.execute_command('AI.TENSORGET', 'x', 'VALUES') + values = tensor[-1] + env.assertEqual(values, [2, 3]) + + # ERR unsupported data format + try: + con.execute_command('AI.TENSORSET', 'z', 'INT32', 2, 'unsupported', 2, 3) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual(exception.__str__(), "invalid argument found in tensor shape") + + # ERR invalid value + try: + con.execute_command('AI.TENSORSET', 'z', 'FLOAT', 2, 'VALUES', 2, 'A') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual(exception.__str__(), "invalid value") + + # ERR invalid value + try: + con.execute_command('AI.TENSORSET', 'z', 'INT32', 2, 'VALUES', 2, 'A') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual(exception.__str__(), "invalid value") + + try: + con.execute_command('AI.TENSORSET', 1) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.TENSORSET', 'y', 'FLOAT') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.TENSORSET', 'y', 'FLOAT', '2') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.TENSORSET', 'y', 'FLOAT', 2, 'VALUES') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.TENSORSET', 'y', 'FLOAT', 2, 'VALUES', 1) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.TENSORSET', 'y', 'FLOAT', 2, 'VALUES', '1') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + ensureSlaveSynced(con, env) + + # test barrier + ensureSlaveSynced(con, env) + + +def test_get_tensor(env): + con = env.getConnection() + con.execute_command('AI.TENSORSET', 't_FLOAT', 'FLOAT', 2, 'VALUES', 2, 3) + con.execute_command('AI.TENSORSET', 't_INT8', 'INT8', 2, 'VALUES', 1, 1) + con.execute_command('AI.TENSORSET', 't_INT16', 'INT8', 2, 'VALUES', 1, 1) + con.execute_command('AI.TENSORSET', 't_INT32', 'INT8', 2, 'VALUES', 1, 1) + con.execute_command('AI.TENSORSET', 't_INT64', 'INT8', 2, 'VALUES', 1, 1) + + tensor = con.execute_command('AI.TENSORGET', 't_FLOAT', 'BLOB') + values = tensor[-1] + + tensor = con.execute_command('AI.TENSORGET', 't_INT8', 'VALUES') + values = tensor[-1] + env.assertEqual(values, [1, 1]) + + tensor = con.execute_command('AI.TENSORGET', 't_INT16', 'VALUES') + values = tensor[-1] + env.assertEqual(values, [1, 1]) + + tensor = con.execute_command('AI.TENSORGET', 't_INT32', 'VALUES') + values = tensor[-1] + env.assertEqual(values, [1, 1]) + + tensor = con.execute_command('AI.TENSORGET', 't_INT64', 'VALUES') + values = tensor[-1] + env.assertEqual(values, [1, 1]) + + tensor = con.execute_command('AI.TENSORGET', 't_INT32', 'META') + values = tensor[-1] + env.assertEqual(values, [2]) + + # ERR unsupported data format + try: + con.execute_command('AI.TENSORGET', 't_FLOAT', 'unsupported') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual(exception.__str__(), "unsupported data format") + + +def test_run_onnx_model(env): + if not TEST_ONNX: + return + + con = env.getConnection() + + test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') + model_filename = os.path.join(test_data_path, 'mnist.onnx') + wrong_model_filename = os.path.join(test_data_path, 'graph.pb') + sample_filename = os.path.join(test_data_path, 'one.raw') + + with open(model_filename, 'rb') as f: + model_pb = f.read() + + with open(wrong_model_filename, 'rb') as f: + wrong_model_pb = f.read() + + with open(sample_filename, 'rb') as f: + sample_raw = f.read() + + ret = con.execute_command('AI.MODELSET', 'm', 'ONNX', DEVICE, model_pb) + env.assertEqual(ret, b'OK') + + ensureSlaveSynced(con, env) + + ret = con.execute_command('AI.MODELGET', 'm') + env.assertEqual(len(ret), 3) + # TODO: enable me + # env.assertEqual(ret[0], b'ONNX') + # env.assertEqual(ret[1], b'CPU') + + try: + con.execute_command('AI.MODELSET', 'm', 'ONNX', DEVICE, wrong_model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELSET', 'm_1', 'ONNX', model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELSET', 'm_2', model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + con.execute_command('AI.TENSORSET', 'a', 'FLOAT', 1, 1, 28, 28, 'BLOB', sample_raw) + + try: + con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'a', 'OUTPUTS') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_2', 'INPUTS', 'a', 'b', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_3', 'a', 'b', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_1', 'OUTPUTS', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm', 'OUTPUTS', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'OUTPUTS') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'a', 'OUTPUTS', 'b') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'OUTPUTS', 'b') + + ensureSlaveSynced(con, env) + + tensor = con.execute_command('AI.TENSORGET', 'b', 'VALUES') + values = tensor[-1] + argmax = max(range(len(values)), key=lambda i: values[i]) + + env.assertEqual(argmax, 1) + + ensureSlaveSynced(con, env) + if env.useSlaves: + con2 = env.getSlaveConnection() + tensor2 = con2.execute_command('AI.TENSORGET', 'b', 'VALUES') + env.assertEqual(tensor2, tensor) + + # test barrier + ensureSlaveSynced(con, env) + + +def test_run_onnxml_model(env): + if not TEST_ONNX: + return + + con = env.getConnection() + + test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') + linear_model_filename = os.path.join(test_data_path, 'linear_iris.onnx') + logreg_model_filename = os.path.join(test_data_path, 'logreg_iris.onnx') + + with open(linear_model_filename, 'rb') as f: + linear_model = f.read() + + with open(logreg_model_filename, 'rb') as f: + logreg_model = f.read() + + ret = con.execute_command('AI.MODELSET', 'linear', 'ONNX', DEVICE, linear_model) + env.assertEqual(ret, b'OK') + + ret = con.execute_command('AI.MODELSET', 'logreg', 'ONNX', DEVICE, logreg_model) + env.assertEqual(ret, b'OK') + + con.execute_command('AI.TENSORSET', 'features', 'FLOAT', 1, 4, 'VALUES', 5.1, 3.5, 1.4, 0.2) + + ensureSlaveSynced(con, env) + + con.execute_command('AI.MODELRUN', 'linear', 'INPUTS', 'features', 'OUTPUTS', 'linear_out') + con.execute_command('AI.MODELRUN', 'logreg', 'INPUTS', 'features', 'OUTPUTS', 'logreg_out', 'logreg_probs') + + ensureSlaveSynced(con, env) + + linear_out = con.execute_command('AI.TENSORGET', 'linear_out', 'VALUES') + logreg_out = con.execute_command('AI.TENSORGET', 'logreg_out', 'VALUES') + + env.assertEqual(float(linear_out[2][0]), -0.090524077415466309) + env.assertEqual(logreg_out[2][0], 0) + + if env.useSlaves: + con2 = env.getSlaveConnection() + linear_out2 = con2.execute_command('AI.TENSORGET', 'linear_out', 'VALUES') + logreg_out2 = con2.execute_command('AI.TENSORGET', 'logreg_out', 'VALUES') + env.assertEqual(linear_out, linear_out2) + env.assertEqual(logreg_out, logreg_out2) + + # test barrier + ensureSlaveSynced(con, env) + + +def test_set_tensor_multiproc(env): + run_test_multiproc(env, 10, + lambda env: env.execute_command('AI.TENSORSET', 'x', 'FLOAT', 2, 'VALUES', 2, 3)) + con = env.getConnection() + tensor = con.execute_command('AI.TENSORGET', 'x', 'VALUES') + values = tensor[-1] + env.assertEqual(values, [b'2', b'3']) diff --git a/test/tests_onnx.py b/test/tests_onnx.py new file mode 100644 index 000000000..16cf9d866 --- /dev/null +++ b/test/tests_onnx.py @@ -0,0 +1,169 @@ +import redis + +from includes import * + +''' +python -m RLTest --test tests_common.py --module path/to/redisai.so +''' + + +def test_run_onnx_model(env): + if not TEST_ONNX: + return + + con = env.getConnection() + + test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') + model_filename = os.path.join(test_data_path, 'mnist.onnx') + wrong_model_filename = os.path.join(test_data_path, 'graph.pb') + sample_filename = os.path.join(test_data_path, 'one.raw') + + with open(model_filename, 'rb') as f: + model_pb = f.read() + + with open(wrong_model_filename, 'rb') as f: + wrong_model_pb = f.read() + + with open(sample_filename, 'rb') as f: + sample_raw = f.read() + + ret = con.execute_command('AI.MODELSET', 'm', 'ONNX', DEVICE, model_pb) + env.assertEqual(ret, b'OK') + + ensureSlaveSynced(con, env) + + ret = con.execute_command('AI.MODELGET', 'm') + env.assertEqual(len(ret), 3) + # TODO: enable me + # env.assertEqual(ret[0], b'ONNX') + # env.assertEqual(ret[1], b'CPU') + + try: + con.execute_command('AI.MODELSET', 'm', 'ONNX', DEVICE, wrong_model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELSET', 'm_1', 'ONNX', model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELSET', 'm_2', model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + con.execute_command('AI.TENSORSET', 'a', 'FLOAT', 1, 1, 28, 28, 'BLOB', sample_raw) + + try: + con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'a', 'OUTPUTS') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_2', 'INPUTS', 'a', 'b', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_3', 'a', 'b', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_1', 'OUTPUTS', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm', 'OUTPUTS', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'OUTPUTS') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'a', 'OUTPUTS', 'b') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'OUTPUTS', 'b') + + ensureSlaveSynced(con, env) + + tensor = con.execute_command('AI.TENSORGET', 'b', 'VALUES') + values = tensor[-1] + argmax = max(range(len(values)), key=lambda i: values[i]) + + env.assertEqual(argmax, 1) + + ensureSlaveSynced(con, env) + if env.useSlaves: + con2 = env.getSlaveConnection() + tensor2 = con2.execute_command('AI.TENSORGET', 'b', 'VALUES') + env.assertEqual(tensor2, tensor) + + +def test_run_onnxml_model(env): + if not TEST_ONNX: + return + + con = env.getConnection() + + test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') + linear_model_filename = os.path.join(test_data_path, 'linear_iris.onnx') + logreg_model_filename = os.path.join(test_data_path, 'logreg_iris.onnx') + + with open(linear_model_filename, 'rb') as f: + linear_model = f.read() + + with open(logreg_model_filename, 'rb') as f: + logreg_model = f.read() + + ret = con.execute_command('AI.MODELSET', 'linear', 'ONNX', DEVICE, linear_model) + env.assertEqual(ret, b'OK') + + ret = con.execute_command('AI.MODELSET', 'logreg', 'ONNX', DEVICE, logreg_model) + env.assertEqual(ret, b'OK') + + con.execute_command('AI.TENSORSET', 'features', 'FLOAT', 1, 4, 'VALUES', 5.1, 3.5, 1.4, 0.2) + + ensureSlaveSynced(con, env) + + con.execute_command('AI.MODELRUN', 'linear', 'INPUTS', 'features', 'OUTPUTS', 'linear_out') + con.execute_command('AI.MODELRUN', 'logreg', 'INPUTS', 'features', 'OUTPUTS', 'logreg_out', 'logreg_probs') + + ensureSlaveSynced(con, env) + + linear_out = con.execute_command('AI.TENSORGET', 'linear_out', 'VALUES') + logreg_out = con.execute_command('AI.TENSORGET', 'logreg_out', 'VALUES') + + env.assertEqual(float(linear_out[2][0]), -0.090524077415466309) + env.assertEqual(logreg_out[2][0], 0) + + if env.useSlaves: + con2 = env.getSlaveConnection() + linear_out2 = con2.execute_command('AI.TENSORGET', 'linear_out', 'VALUES') + logreg_out2 = con2.execute_command('AI.TENSORGET', 'logreg_out', 'VALUES') + env.assertEqual(linear_out, linear_out2) + env.assertEqual(logreg_out, logreg_out2) + diff --git a/test/tests_pytorch.py b/test/tests_pytorch.py new file mode 100644 index 000000000..d0be6df4f --- /dev/null +++ b/test/tests_pytorch.py @@ -0,0 +1,353 @@ +import redis + +from includes import * + +''' +python -m RLTest --test tests_common.py --module path/to/redisai.so +''' + +def test_run_torch_model(env): + if not TEST_PT: + return + + con = env.getConnection() + + test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') + model_filename = os.path.join(test_data_path, 'pt-minimal.pt') + wrong_model_filename = os.path.join(test_data_path, 'graph.pb') + + with open(model_filename, 'rb') as f: + model_pb = f.read() + + with open(wrong_model_filename, 'rb') as f: + wrong_model_pb = f.read() + + ret = con.execute_command('AI.MODELSET', 'm', 'TORCH', DEVICE, model_pb) + env.assertEqual(ret, b'OK') + + ensureSlaveSynced(con, env) + + ret = con.execute_command('AI.MODELGET', 'm') + # TODO: enable me + # env.assertEqual(ret[0], b'TORCH') + # env.assertEqual(ret[1], b'CPU') + + try: + con.execute_command('AI.MODELSET', 'm', 'TORCH', DEVICE, wrong_model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELSET', 'm_1', 'TORCH', model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELSET', 'm_2', model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + env.execute_command('AI.TENSORSET', 'a', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) + env.execute_command('AI.TENSORSET', 'b', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) + + try: + con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'a', 'b', 'OUTPUTS') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_2', 'INPUTS', 'a', 'b', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_3', 'a', 'b', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_1', 'OUTPUTS', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm', 'OUTPUTS', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'OUTPUTS') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c', 'd') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') + + ensureSlaveSynced(con, env) + + tensor = con.execute_command('AI.TENSORGET', 'c', 'VALUES') + values = tensor[-1] + env.assertEqual(values, [b'4', b'6', b'4', b'6']) + + ensureSlaveSynced(con, env) + if env.useSlaves: + con2 = env.getSlaveConnection() + tensor2 = con2.execute_command('AI.TENSORGET', 'c', 'VALUES') + env.assertEqual(tensor2, tensor) + + +def test_set_script(env): + if not TEST_PT: + return + + con = env.getConnection() + + try: + con.execute_command('AI.SCRIPTSET', 'ket', DEVICE, 'return 1') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.SCRIPTSET', 'nope') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.SCRIPTSET', 'more', DEVICE) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') + script_filename = os.path.join(test_data_path, 'script.txt') + + with open(script_filename, 'rb') as f: + script = f.read() + + ret = con.execute_command('AI.SCRIPTSET', 'ket', DEVICE, script) + env.assertEqual(ret, b'OK') + + ensureSlaveSynced(con, env) + + +def test_del_script(env): + if not TEST_PT: + return + + con = env.getConnection() + + test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') + script_filename = os.path.join(test_data_path, 'script.txt') + + with open(script_filename, 'rb') as f: + script = f.read() + + ret = con.execute_command('AI.SCRIPTSET', 'ket', DEVICE, script) + env.assertEqual(ret, b'OK') + + ensureSlaveSynced(con, env) + + ret = con.execute_command('AI.SCRIPTDEL', 'ket') + env.assertEqual(ret, b'OK') + + ensureSlaveSynced(con, env) + + env.assertFalse(con.execute_command('EXISTS', 'ket')) + + # ERR no script at key from SCRIPTDEL + try: + con.execute_command('DEL', 'EMPTY') + con.execute_command('AI.SCRIPTDEL', 'EMPTY') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual("no script at key", exception.__str__()) + + # ERR wrong type from SCRIPTDEL + try: + con.execute_command('SET', 'NOT_SCRIPT', 'BAR') + con.execute_command('AI.SCRIPTDEL', 'NOT_SCRIPT') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value", exception.__str__()) + + +def test_run_script(env): + if not TEST_PT: + return + + con = env.getConnection() + + test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') + script_filename = os.path.join(test_data_path, 'script.txt') + + with open(script_filename, 'rb') as f: + script = f.read() + + ret = con.execute_command('AI.SCRIPTSET', 'ket', DEVICE, script) + env.assertEqual(ret, b'OK') + + ret = con.execute_command('AI.TENSORSET', 'a', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) + env.assertEqual(ret, b'OK') + ret = con.execute_command('AI.TENSORSET', 'b', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) + env.assertEqual(ret, b'OK') + + ensureSlaveSynced(con, env) + + # TODO: enable me ( this is hanging CI ) + # ret = con.execute_command('AI.SCRIPTGET', 'ket') + # TODO: enable me + # env.assertEqual([b'CPU',script],ret) + + # ERR no script at key from SCRIPTGET + try: + con.execute_command('DEL', 'EMPTY') + con.execute_command('AI.SCRIPTGET', 'EMPTY') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual("cannot get script from empty key", exception.__str__()) + + # ERR wrong type from SCRIPTGET + try: + con.execute_command('SET', 'NOT_SCRIPT', 'BAR') + con.execute_command('AI.SCRIPTGET', 'NOT_SCRIPT') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value", exception.__str__()) + + # ERR no script at key from SCRIPTRUN + try: + con.execute_command('DEL', 'EMPTY') + con.execute_command('AI.SCRIPTRUN', 'EMPTY', 'bar', 'INPUTS', 'b', 'OUTPUTS', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual("script key is empty", exception.__str__()) + + # ERR wrong type from SCRIPTRUN + try: + con.execute_command('SET', 'NOT_SCRIPT', 'BAR') + con.execute_command('AI.SCRIPTRUN', 'NOT_SCRIPT', 'bar', 'INPUTS', 'b', 'OUTPUTS', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value", exception.__str__()) + + # ERR Input key is empty + try: + con.execute_command('DEL', 'EMPTY') + con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'EMPTY', 'b', 'OUTPUTS', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual("Input key is empty", exception.__str__()) + + # ERR Input key not tensor + try: + con.execute_command('SET', 'NOT_TENSOR', 'BAR') + con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'NOT_TENSOR', 'b', 'OUTPUTS', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value", exception.__str__()) + + try: + con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'b', 'OUTPUTS', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.SCRIPTRUN', 'ket', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'b', 'OUTPUTS') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'OUTPUTS') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') + + ensureSlaveSynced(con, env) + + info = con.execute_command('AI.INFO', 'ket') + info_dict_0 = info_to_dict(info) + + env.assertEqual(info_dict_0['KEY'], 'ket') + env.assertEqual(info_dict_0['TYPE'], 'SCRIPT') + env.assertEqual(info_dict_0['BACKEND'], 'TORCH') + env.assertTrue(info_dict_0['DURATION'] > 0) + env.assertEqual(info_dict_0['SAMPLES'], -1) + env.assertEqual(info_dict_0['CALLS'], 4) + env.assertEqual(info_dict_0['ERRORS'], 3) + + con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') + + ensureSlaveSynced(con, env) + + info = con.execute_command('AI.INFO', 'ket') + info_dict_1 = info_to_dict(info) + + env.assertTrue(info_dict_1['DURATION'] > info_dict_0['DURATION']) + env.assertEqual(info_dict_1['SAMPLES'], -1) + env.assertEqual(info_dict_1['CALLS'], 5) + env.assertEqual(info_dict_1['ERRORS'], 3) + + ret = con.execute_command('AI.INFO', 'ket', 'RESETSTAT') + env.assertEqual(ret, b'OK') + + con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') + + ensureSlaveSynced(con, env) + + info = con.execute_command('AI.INFO', 'ket') + info_dict_2 = info_to_dict(info) + + env.assertTrue(info_dict_2['DURATION'] < info_dict_1['DURATION']) + env.assertEqual(info_dict_2['SAMPLES'], -1) + env.assertEqual(info_dict_2['CALLS'], 1) + env.assertEqual(info_dict_2['ERRORS'], 0) + + tensor = con.execute_command('AI.TENSORGET', 'c', 'VALUES') + values = tensor[-1] + env.assertEqual(values, [b'4', b'6', b'4', b'6']) + + ensureSlaveSynced(con, env) + if env.useSlaves: + con2 = env.getSlaveConnection() + tensor2 = con2.execute_command('AI.TENSORGET', 'c', 'VALUES') + env.assertEqual(tensor2, tensor) diff --git a/test/tests_tensorflow.py b/test/tests_tensorflow.py new file mode 100644 index 000000000..34e2cbf54 --- /dev/null +++ b/test/tests_tensorflow.py @@ -0,0 +1,330 @@ +import redis + +from includes import * + +''' +python -m RLTest --test tests_common.py --module path/to/redisai.so +''' + + +def test_run_mobilenet(env): + if not TEST_TF: + return + + con = env.getConnection() + + input_var = 'input' + output_var = 'MobilenetV2/Predictions/Reshape_1' + + model_pb, labels, img = load_mobilenet_test_data() + + con.execute_command('AI.MODELSET', 'mobilenet', 'TF', DEVICE, + 'INPUTS', input_var, 'OUTPUTS', output_var, model_pb) + + con.execute_command('AI.TENSORSET', 'input', + 'FLOAT', 1, img.shape[1], img.shape[0], img.shape[2], + 'BLOB', img.tobytes()) + + ensureSlaveSynced(con, env) + + con.execute_command('AI.MODELRUN', 'mobilenet', + 'INPUTS', 'input', 'OUTPUTS', 'output') + + ensureSlaveSynced(con, env) + + dtype, shape, data = con.execute_command('AI.TENSORGET', 'output', 'BLOB') + + dtype_map = {b'FLOAT': np.float32} + tensor = np.frombuffer(data, dtype=dtype_map[dtype]).reshape(shape) + label_id = np.argmax(tensor) - 1 + + _, label = labels[str(label_id)] + + env.assertEqual(label, 'giant_panda') + + +def test_run_mobilenet_multiproc(env): + if not TEST_TF: + return + + con = env.getConnection() + + input_var = 'input' + output_var = 'MobilenetV2/Predictions/Reshape_1' + + model_pb, labels, img = load_mobilenet_test_data() + con.execute_command('AI.MODELSET', 'mobilenet', 'TF', DEVICE, + 'INPUTS', input_var, 'OUTPUTS', output_var, model_pb) + ensureSlaveSynced(con, env) + + run_test_multiproc(env, 30, run_mobilenet, (img, input_var, output_var)) + + ensureSlaveSynced(con, env) + + dtype, shape, data = con.execute_command('AI.TENSORGET', 'output', 'BLOB') + + dtype_map = {b'FLOAT': np.float32} + tensor = np.frombuffer(data, dtype=dtype_map[dtype]).reshape(shape) + label_id = np.argmax(tensor) - 1 + + _, label = labels[str(label_id)] + + env.assertEqual( + label, 'giant_panda' + ) + +def test_del_tf_model(env): + if not TEST_PT: + return + + con = env.getConnection() + + test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') + model_filename = os.path.join(test_data_path, 'graph.pb') + + with open(model_filename, 'rb') as f: + model_pb = f.read() + + ret = con.execute_command('AI.MODELSET', 'm', 'TF', DEVICE, + 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) + env.assertEqual(ret, b'OK') + + ensureSlaveSynced(con, env) + + con.execute_command('AI.MODELDEL', 'm') + env.assertFalse(env.execute_command('EXISTS', 'm')) + + # ERR no model at key + try: + con.execute_command('AI.MODELDEL', 'm') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual("no model at key", exception.__str__()) + + # ERR wrong type + try: + con.execute_command('SET', 'NOT_MODEL', 'BAR') + con.execute_command('AI.MODELDEL', 'NOT_MODEL') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value", exception.__str__()) + + +def test_run_tf_model(env): + if not TEST_PT: + return + + con = env.getConnection() + + test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') + model_filename = os.path.join(test_data_path, 'graph.pb') + wrong_model_filename = os.path.join(test_data_path, 'pt-minimal.pt') + + with open(model_filename, 'rb') as f: + model_pb = f.read() + + with open(wrong_model_filename, 'rb') as f: + wrong_model_pb = f.read() + ret = con.execute_command('AI.MODELSET', 'm', 'TF', DEVICE, + 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) + env.assertEqual(ret, b'OK') + + ensureSlaveSynced(con, env) + + ret = con.execute_command('AI.MODELGET', 'm') + env.assertEqual(len(ret), 3) + # TODO: enable me + # env.assertEqual(ret[0], b'TF') + # env.assertEqual(ret[1], b'CPU') + + # ERR WrongArity + try: + con.execute_command('AI.MODELGET') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual("wrong number of arguments for 'AI.MODELGET' command", exception.__str__()) + + # ERR WRONGTYPE + con.execute_command('SET', 'NOT_MODEL', 'BAR') + try: + con.execute_command('AI.MODELGET', 'NOT_MODEL') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value", exception.__str__()) + # cleanup + con.execute_command('DEL', 'NOT_MODEL') + + # ERR cannot get model from empty key + con.execute_command('DEL', 'DONT_EXIST') + try: + con.execute_command('AI.MODELGET', 'DONT_EXIST') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual("cannot get model from empty key", exception.__str__()) + + try: + ret = con.execute_command('AI.MODELSET', 'm', 'TF', DEVICE, + 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', wrong_model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELSET', 'm_1', 'TF', + 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELSET', 'm_2', 'PORCH', DEVICE, + 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELSET', 'm_3', 'TORCH', DEVICE, + 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELSET', 'm_4', 'TF', + 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELSET', 'm_5', 'TF', DEVICE, + 'INPUTS', 'a', 'b', 'c', 'OUTPUTS', 'mul', model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELSET', 'm_6', 'TF', DEVICE, + 'INPUTS', 'a', 'b', 'OUTPUTS', 'mult', model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELSET', 'm_7', 'TF', DEVICE, model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELSET', 'm_8', 'TF', DEVICE, + 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELSET', 'm_8', 'TF', DEVICE, + 'INPUTS', 'a_', 'b', 'OUTPUTS', 'mul') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELSET', 'm_8', 'TF', DEVICE, + 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul_') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + # ERR Invalid GraphDef + try: + con.execute_command('AI.MODELSET', 'm_8', 'TF', DEVICE, + 'INPUTS', 'a', 'b', 'OUTPUTS') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + env.assertEqual(exception.__str__(), "Invalid GraphDef") + + try: + con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm', 'OUTPUTS', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + con.execute_command('AI.TENSORSET', 'a', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) + con.execute_command('AI.TENSORSET', 'b', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) + con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') + + ensureSlaveSynced(con, env) + + info = con.execute_command('AI.INFO', 'm') + info_dict_0 = info_to_dict(info) + + env.assertEqual(info_dict_0['KEY'], 'm') + env.assertEqual(info_dict_0['TYPE'], 'MODEL') + env.assertEqual(info_dict_0['BACKEND'], 'TF') + env.assertTrue(info_dict_0['DURATION'] > 0) + env.assertEqual(info_dict_0['SAMPLES'], 2) + env.assertEqual(info_dict_0['CALLS'], 1) + env.assertEqual(info_dict_0['ERRORS'], 0) + + con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') + + ensureSlaveSynced(con, env) + + info = con.execute_command('AI.INFO', 'm') + info_dict_1 = info_to_dict(info) + + env.assertTrue(info_dict_1['DURATION'] > info_dict_0['DURATION']) + env.assertEqual(info_dict_1['SAMPLES'], 4) + env.assertEqual(info_dict_1['CALLS'], 2) + env.assertEqual(info_dict_1['ERRORS'], 0) + + ret = con.execute_command('AI.INFO', 'm', 'RESETSTAT') + env.assertEqual(ret, b'OK') + + con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') + + ensureSlaveSynced(con, env) + + info = con.execute_command('AI.INFO', 'm') + info_dict_2 = info_to_dict(info) + + env.assertTrue(info_dict_2['DURATION'] < info_dict_1['DURATION']) + env.assertEqual(info_dict_2['SAMPLES'], 2) + env.assertEqual(info_dict_2['CALLS'], 1) + env.assertEqual(info_dict_2['ERRORS'], 0) + + tensor = con.execute_command('AI.TENSORGET', 'c', 'VALUES') + values = tensor[-1] + env.assertEqual(values, [b'4', b'9', b'4', b'9']) + + ensureSlaveSynced(con, env) + if env.useSlaves: + con2 = env.getSlaveConnection() + tensor2 = con2.execute_command('AI.TENSORGET', 'c', 'VALUES') + env.assertEqual(tensor2, tensor) + + for _ in env.reloadingIterator(): + env.assertExists('m') + env.assertExists('a') + env.assertExists('b') + env.assertExists('c') + + con.execute_command('AI.MODELDEL', 'm') + ensureSlaveSynced(con, env) + + env.assertFalse(env.execute_command('EXISTS', 'm')) \ No newline at end of file diff --git a/test/tests_tflite.py b/test/tests_tflite.py new file mode 100644 index 000000000..22efdcced --- /dev/null +++ b/test/tests_tflite.py @@ -0,0 +1,117 @@ +import redis + +from includes import * + +''' +python -m RLTest --test tests_common.py --module path/to/redisai.so +''' + + +def test_run_tflite_model(env): + if not TEST_TFLITE: + return + + con = env.getConnection() + + test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') + model_filename = os.path.join(test_data_path, 'mnist_model_quant.tflite') + wrong_model_filename = os.path.join(test_data_path, 'graph.pb') + sample_filename = os.path.join(test_data_path, 'one.raw') + + with open(model_filename, 'rb') as f: + model_pb = f.read() + + with open(model_filename, 'rb') as f: + model_pb2 = f.read() + + with open(wrong_model_filename, 'rb') as f: + wrong_model_pb = f.read() + + with open(sample_filename, 'rb') as f: + sample_raw = f.read() + + ret = con.execute_command('AI.MODELSET', 'm', 'TFLITE', 'CPU', model_pb) + env.assertEqual(ret, b'OK') + + ensureSlaveSynced(con, env) + + ret = con.execute_command('AI.MODELGET', 'm') + env.assertEqual(len(ret), 3) + # TODO: enable me + # env.assertEqual(ret[0], b'TFLITE') + # env.assertEqual(ret[1], b'CPU') + + try: + con.execute_command('AI.MODELSET', 'm_1', 'TFLITE', model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + ret = con.execute_command('AI.MODELSET', 'm_2', 'TFLITE', 'CPU', model_pb2) + + try: + con.execute_command('AI.MODELSET', 'm_2', model_pb) + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + con.execute_command('AI.TENSORSET', 'a', 'FLOAT', 1, 1, 28, 28, 'BLOB', sample_raw) + + try: + con.execute_command('AI.MODELRUN', 'm_2', 'INPUTS', 'a', 'OUTPUTS') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_2', 'INPUTS', 'a', 'b', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_2', 'a', 'b', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm_2', 'OUTPUTS', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm', 'OUTPUTS', 'c') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'OUTPUTS') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + try: + con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'OUTPUTS', 'b') + except Exception as e: + exception = e + env.assertEqual(type(exception), redis.exceptions.ResponseError) + + con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'OUTPUTS', 'b', 'c') + + ensureSlaveSynced(con, env) + + tensor = con.execute_command('AI.TENSORGET', 'b', 'VALUES') + value = tensor[-1][0] + + env.assertEqual(value, 1) + + From bf4e4de6d8c7159c8f9aaaac217d377d44e0732a Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Thu, 20 Feb 2020 19:09:43 +0000 Subject: [PATCH 2/4] [add] coverage integration --- .circleci/config.yml | 13 +++++++++++-- CMakeLists.txt | 18 ++++++++++++++++++ README.md | 1 + opt/redis_valgrind.sup | 22 ++++++++++++++++++++++ opt/system-setup.py | 2 +- src/backends/tensorflow.c | 1 - 6 files changed, 53 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fa6a97669..7c3f627cf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,8 +37,18 @@ commands: name: Test command: | mkdir -p ~/workspace/tests - make -C opt test SHOW=1 VERBOSE=1 + make -C opt test SHOW=1 VERBOSE=1 WITH_COVERAGE=on cp test/logs/* ~/workspace/tests + - run: + name: Coverage + command: | + mkdir -p build + cd build + cmake -DENABLE_CODECOVERAGE=on .. + make + make test ARGS="-V" + make coverage + bash <(curl -s https://codecov.io/bash) -f coverage.info -X gcov -x gcov-7 || echo "Codecov did not collect coverage reports" - run: name: Package command: make -C opt pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=~/workspace/packages SHOW=1 @@ -60,7 +70,6 @@ commands: name: Deploy to S3 command: | aws s3 cp <>/ s3://redismodules/$PACKAGE_NAME/ --acl public-read --recursive --exclude "*" --include "*.zip" --include "*.tgz" - jobs: build: docker: diff --git a/CMakeLists.txt b/CMakeLists.txt index 69f8e7f7e..d7dacb92e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,7 @@ option(BUILD_TF "Build the TensorFlow backend" ON) option(BUILD_TFLITE "Build the TensorFlow Lite backend" ON) option(BUILD_ORT "Build the ONNXRuntime backend" ON) option(BUILD_TORCH "Build the PyTorch backend" ON) +OPTION(ENABLE_CODECOVERAGE "Enable code coverage testing support" OFF) #---------------------------------------------------------------------------------------------- @@ -71,6 +72,23 @@ GET_FILENAME_COMPONENT(depsAbs GET_FILENAME_COMPONENT(installAbs "${INSTALL_PATH}" REALPATH BASE_DIR ${CMAKE_SOURCE_DIR}) +if(ENABLE_CODECOVERAGE) + message(STATUS "Forcing build type to Debug to run coverage.") + set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Choose the type of build." FORCE) + include(UseCodeCoverage) + find_package(PythonInterp 3 REQUIRED) + enable_testing() + set( ENV{DEVICE} "CPU" ) + file(GLOB TEST_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/test/tests_*.py + ) + foreach(file ${TEST_FILES}) + message(STATUS "Creating tests ${file}.") + add_test(NAME ${file}_single_master COMMAND ${PYTHON_EXECUTABLE} -m RLTest --check-exitcode --test ${file} -v --module redisai.so --module-args "BACKENDSPATH ${CMAKE_CURRENT_SOURCE_DIR}/build TORCH redisai_torch.so TF redisai_tensorflow.so ONNX redisai_onnxruntime.so TFLITE redisai_tflite.so" WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} ) + add_test(NAME ${file}_aof COMMAND ${PYTHON_EXECUTABLE} -m RLTest --check-exitcode --test ${file} -v --module redisai.so --module-args "BACKENDSPATH ${CMAKE_CURRENT_SOURCE_DIR}/build TORCH redisai_torch.so TF redisai_tensorflow.so ONNX redisai_onnxruntime.so TFLITE redisai_tflite.so" --use-aof WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} ) + add_test(NAME ${file}_use_slaves COMMAND ${PYTHON_EXECUTABLE} -m RLTest --check-exitcode --test ${file} -v --module redisai.so --module-args "BACKENDSPATH ${CMAKE_CURRENT_SOURCE_DIR}/build TORCH redisai_torch.so TF redisai_tensorflow.so ONNX redisai_onnxruntime.so TFLITE redisai_tflite.so" --use-slaves WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} ) + endforeach() +ENDIF() #---------------------------------------------------------------------------------------------- INCLUDE_DIRECTORIES(${depsAbs}/dlpack/include) diff --git a/README.md b/README.md index 8c59b051e..d47c9f75d 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/redisai/redisai.svg)](https://hub.docker.com/r/redisai/redisai/builds/) [![Mailing List](https://img.shields.io/badge/Mailing%20List-RedisAI-blue)](https://groups.google.com/forum/#!forum/redisai) [![Gitter](https://badges.gitter.im/RedisLabs/RedisAI.svg)](https://gitter.im/RedisLabs/RedisAI?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![codecov](https://codecov.io/gh/RedisAI/RedisAI/branch/master/graph/badge.svg)](https://codecov.io/gh/RedisAI/RedisAI) # RedisAI diff --git a/opt/redis_valgrind.sup b/opt/redis_valgrind.sup index 3024d63bc..90a043f61 100644 --- a/opt/redis_valgrind.sup +++ b/opt/redis_valgrind.sup @@ -1,3 +1,25 @@ +{ + ignore_unversioned_libs + Memcheck:Leak + ... + obj:*/libtensorflow.so.* +} + +{ + ignore_unversioned_libs + Memcheck:Leak + ... + obj:*/libonnxruntime.so.* +} + +{ + ignore_unversioned_libs + Memcheck:Leak + ... + obj:*/libtorch.so.* +} + + { Memcheck:Cond diff --git a/opt/system-setup.py b/opt/system-setup.py index 910865d66..cd6ca0425 100755 --- a/opt/system-setup.py +++ b/opt/system-setup.py @@ -22,7 +22,7 @@ def common_first(self): if self.os == 'linux': self.install("ca-certificates") - self.install("git cmake unzip wget patchelf awscli") + self.install("git cmake unzip wget patchelf awscli valgrind") self.install("coreutils") # for realpath def debian_compat(self): diff --git a/src/backends/tensorflow.c b/src/backends/tensorflow.c index 5d447845c..27952c4af 100644 --- a/src/backends/tensorflow.c +++ b/src/backends/tensorflow.c @@ -354,7 +354,6 @@ void RAI_ModelFreeTF(RAI_Model* model, RAI_Error* error) { } array_free(model->outputs); } - TF_DeleteStatus(status); } From d4709f026e3d92803bc7e2951ba5f65c96ae9737 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Thu, 20 Feb 2020 19:18:05 +0000 Subject: [PATCH 3/4] [fix] add missing UseCodeCoverage.cmake --- opt/cmake/modules/UseCodeCoverage.cmake | 34 +++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 opt/cmake/modules/UseCodeCoverage.cmake diff --git a/opt/cmake/modules/UseCodeCoverage.cmake b/opt/cmake/modules/UseCodeCoverage.cmake new file mode 100644 index 000000000..cb5292ea9 --- /dev/null +++ b/opt/cmake/modules/UseCodeCoverage.cmake @@ -0,0 +1,34 @@ +# - Enable Code Coverage +# +# Variables you may define are: +# CODECOV_HTMLOUTPUTDIR - the name of the directory where HTML results are placed. Defaults to "coverage_html" +# CODECOV_XMLOUTPUTFILE - the name of the directory where HTML results are placed. Defaults to "coverage.xml" +# CODECOV_GCOVR_OPTIONS - additional options given to gcovr commands. +# + +if(ENABLE_CODECOVERAGE) + + if ( NOT CMAKE_BUILD_TYPE STREQUAL "Debug" ) + message( WARNING "Code coverage results with an optimised (non-Debug) build may be misleading" ) + endif ( NOT CMAKE_BUILD_TYPE STREQUAL "Debug" ) + + if ( NOT DEFINED CODECOV_OUTPUTFILE ) + set( CODECOV_OUTPUTFILE coverage.info ) + endif ( NOT DEFINED CODECOV_OUTPUTFILE ) + + if ( NOT DEFINED CODECOV_HTMLOUTPUTDIR ) + set( CODECOV_HTMLOUTPUTDIR coverage_results ) + endif ( NOT DEFINED CODECOV_HTMLOUTPUTDIR ) + + if ( CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_GNUCXX ) + find_program( CODECOV_GCOV gcov ) + find_program( CODECOV_LCOV lcov ) + find_program( CODECOV_GENHTML genhtml ) + add_definitions( -fprofile-arcs -ftest-coverage ) + link_libraries( gcov ) + set( CMAKE_EXE_LINKER_FLAGS ${CMAKE_EXE_LINKER_FLAGS} --coverage ) + add_custom_target( coverage_init ALL ${CODECOV_LCOV} --base-directory . --directory ${CMAKE_BINARY_DIR}/src --output-file ${CODECOV_OUTPUTFILE} --capture --initial ) + add_custom_target( coverage ${CODECOV_LCOV} --base-directory . --directory ${CMAKE_BINARY_DIR}/src --output-file ${CODECOV_OUTPUTFILE} --capture COMMAND genhtml -o ${CODECOV_HTMLOUTPUTDIR} ${CODECOV_OUTPUTFILE} ) + endif ( CMAKE_COMPILER_IS_GNUCXX ) + +endif(ENABLE_CODECOVERAGE) From 35fd66103ed8e7022e57fb70b6777fbcfc02430e Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Mon, 24 Feb 2020 21:52:34 +0000 Subject: [PATCH 4/4] [merge] merged with master --- opt/Makefile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/opt/Makefile b/opt/Makefile index cf6680357..fb4b4dfca 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -142,8 +142,6 @@ endif #---------------------------------------------------------------------------------------------- -TEST_CMD=DEVICE=$(DEVICE) PYDEBUG=$(PYDEBUG) python3 -m RLTest $(TEST_ARGS) --module $(INSTALL_DIR)/redisai.so - TEST_REPORT_DIR ?= $(PWD) ifeq ($(VERBOSE),1) TEST_ARGS += -v @@ -172,15 +170,15 @@ ifneq ($(NO_LFS),1) $(SHOW)cd $(ROOT); git lfs pull endif ifeq ($(GEN),1) - $(SHOW)$(TEST_PREFIX); $(TEST_CMD) --test $@ + $(SHOW)$(TEST_PREFIX); $(TEST_CMD) endif ifeq ($(AOF),1) $(SHOW)$(TEST_PREFIX); printf "\nTests with --use-aof:\n\n" ;\ - $(TEST_CMD) --use-aof --test $@ + $(TEST_CMD) --use-aof endif ifeq ($(SLAVES),1) $(SHOW)$(TEST_PREFIX); printf "\nTests with --use-slaves:\n\n" ;\ - $(TEST_CMD) --use-slaves --test $@ + $(TEST_CMD) --use-slaves endif ifeq ($(VALGRIND),1) $(SHOW)$(TEST_PREFIX); printf "\nTests with valgrind:\n\n" ;\