From f298f82ed5b7db5b1743ee2153f4220105001448 Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 25 Feb 2020 15:00:25 +0200 Subject: [PATCH 01/18] readies sync (mk/cmake) --- .dockerignore | 1 + .gitignore | 1 + opt/Makefile | 10 ++- opt/readies/bin/filter-colors | 4 + opt/readies/bin/getgosu | 36 +++++++++ opt/readies/bin/{getpy => getpy3} | 0 opt/readies/bin/{sep1 => sep0} | 0 opt/readies/bin/system-setup.py | 48 ++++++++++++ opt/readies/cetara/README.md | 0 opt/readies/cetara/diag/gdb.c | 65 ++++++++-------- opt/readies/mk/README.md | 0 opt/readies/mk/bindirs.defs | 2 + opt/readies/mk/bindirs.rules | 12 +++ opt/readies/mk/build.defs | 3 + opt/readies/mk/build.rules | 25 ++++-- opt/readies/mk/cmake.defs | 4 +- opt/readies/mk/cmake.rules | 6 +- opt/readies/mk/common.defs | 4 + opt/readies/mk/common.rules | 5 +- opt/readies/mk/configure.defs | 13 +++- opt/readies/mk/configure.rules | 25 +++--- opt/readies/mk/coverage.defs | 36 +++++++++ opt/readies/mk/coverage.rules | 7 ++ opt/readies/mk/defs | 11 +++ opt/readies/mk/docker | 123 ++++++++++++++++++++++++++++++ opt/readies/mk/git.defs | 5 ++ opt/readies/mk/macosx.defs | 42 ++++++++++ opt/readies/mk/main | 31 ++++++-- opt/readies/mk/nproc.defs | 6 ++ opt/readies/mk/platform.defs | 7 ++ opt/readies/mk/rules | 16 +++- opt/readies/mk/variant.defs | 23 ++---- opt/readies/paella/README.md | 0 opt/readies/paella/debug.py | 8 +- opt/readies/paella/platform.py | 10 +++ opt/readies/paella/setup.py | 7 +- opt/readies/shibumi/README.md | 0 37 files changed, 506 insertions(+), 90 deletions(-) create mode 100644 opt/readies/bin/filter-colors create mode 100644 opt/readies/bin/getgosu rename opt/readies/bin/{getpy => getpy3} (100%) rename opt/readies/bin/{sep1 => sep0} (100%) create mode 100644 opt/readies/bin/system-setup.py create mode 100644 opt/readies/cetara/README.md create mode 100644 opt/readies/mk/README.md create mode 100644 opt/readies/mk/coverage.defs create mode 100644 opt/readies/mk/coverage.rules create mode 100644 opt/readies/mk/docker create mode 100644 opt/readies/mk/git.defs create mode 100644 opt/readies/mk/macosx.defs create mode 100644 opt/readies/mk/nproc.defs create mode 100644 opt/readies/mk/platform.defs create mode 100644 opt/readies/paella/README.md create mode 100644 opt/readies/shibumi/README.md diff --git a/.dockerignore b/.dockerignore index 3805aedef..9e415bc14 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,3 +8,4 @@ venv*/ *.zip *.tgz *.tar.gz +/VARAINT diff --git a/.gitignore b/.gitignore index be9f4c530..dce0540c4 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ venv*/ *.zip *.tgz *.tar.gz +/VARIANT # Misc .DS_Store diff --git a/opt/Makefile b/opt/Makefile index fb4b4dfca..794711bfa 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -3,7 +3,11 @@ ROOT:=.. include readies/mk/main MK_CMAKE:=1 +MK.cmake:=1 MK_CMAKE_INSTALL:=1 +MK_CUSTOM_CLEAN:=1 + +#---------------------------------------------------------------------------------------------- define HELP make setup # install prerequisited (CAUTION: THIS WILL MODIFY YOUR SYSTEM) @@ -52,10 +56,12 @@ endif SRCDIR=.. BINDIR=$(BINROOT)/src -# INSTALL_DIR=$(ROOT)/install-$(DEVICE) DEPS_DIR=$(ROOT)/deps/$(OS)-$(ARCH)-$(DEVICE) -INSTALL_DIR=$(ROOT)/bin/$(OS)-$(ARCH)-$(DEVICE)/install +# INSTALL_DIR=$(ROOT)/bin/$(OS)-$(ARCH)-$(DEVICE)/install +INSTALL_DIR=$(BINROOT)/install-$(DEVICE) + REDIS_VALGRID_SUPRESS=$(ROOT)/opt/redis_valgrind.sup + TARGET=$(BINDIR)/redisai.so BACKENDS_PATH ?= $(INSTALL_DIR)/backends diff --git a/opt/readies/bin/filter-colors b/opt/readies/bin/filter-colors new file mode 100644 index 000000000..b51496e17 --- /dev/null +++ b/opt/readies/bin/filter-colors @@ -0,0 +1,4 @@ +#!/bin/bash + +sed 's/\x1b\[[0-9;]*m//g' + diff --git a/opt/readies/bin/getgosu b/opt/readies/bin/getgosu new file mode 100644 index 000000000..b1fa736bd --- /dev/null +++ b/opt/readies/bin/getgosu @@ -0,0 +1,36 @@ +#!/usr/bin/env python2 + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +import paella + +platform = paella.Platform() + +gosu_arch={ + "x64": "amd64", + "arm64v8": "arm64", + "arm32v7": "armhf" +} + +version = os.environ.get('GOSU_VERSION', '1.11') + +gosu = "https://github.com/tianon/gosu/releases/download/{}/gosu-{}".format(version, gosu_arch[platform.arch]) +# paella.wget(gosu, dest='/usr/local/bin/gosu') +# the former fails with: +# urllib2.URLError: +# thus falling back to wget +sh("wget -q -O /usr/local/bin/gosu {}".format(gosu)) + +# paella.wget(gosu + ".asc", dest='/usr/local/bin/gosu.asc') + +## note: gpg sometimes fails due to network problems, disabling ipv6 helps +# export GNUPGHOME="$(mktemp -d)" +# echo "disable-ipv6" >> $GNUPGHOME/dirmngr.conf +# gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 +# gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu +# gpgconf --kill all +# rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc + +os.chmod('/usr/local/bin/gosu', 0755) diff --git a/opt/readies/bin/getpy b/opt/readies/bin/getpy3 similarity index 100% rename from opt/readies/bin/getpy rename to opt/readies/bin/getpy3 diff --git a/opt/readies/bin/sep1 b/opt/readies/bin/sep0 similarity index 100% rename from opt/readies/bin/sep1 rename to opt/readies/bin/sep0 diff --git a/opt/readies/bin/system-setup.py b/opt/readies/bin/system-setup.py new file mode 100644 index 000000000..5acb059b1 --- /dev/null +++ b/opt/readies/bin/system-setup.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python2 + +import sys +import os +import argparse + +HERE=os.path.join(os.path.dirname(__file__) +sys.path.insert(0, HERE, "..")) +import paella + +#---------------------------------------------------------------------------------------------- + +class SystemSetup(paella.Setup): + def __init__(self, nop=False): + paella.Setup.__init__(self, nop) + + def common_first(self): + # self.install("") + # self.group_install("") + # self.setup_pip() + # self.pip_install("") + pass + + def debian_compat(self): + pass + + def redhat_compat(self): + pass + + def fedora(self): + pass + + def macosx(self): + pass + + def common_last(self): + pass + +#---------------------------------------------------------------------------------------------- + +parser = argparse.ArgumentParser(description='Set up system for build.') +parser.add_argument('-n', '--nop', action="store_true", help='no operation') +# parser.add_argument('--bool', action="store_true", help="flag") +# parser.add_argument('--int', type=int, default=1, help='number') +# parser.add_argument('--str', type=str, default='str', help='number') +args = parser.parse_args() + +SystemSetup(nop = args.nop).setup() diff --git a/opt/readies/cetara/README.md b/opt/readies/cetara/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/opt/readies/cetara/diag/gdb.c b/opt/readies/cetara/diag/gdb.c index 8afa1697d..d4eb44ca6 100755 --- a/opt/readies/cetara/diag/gdb.c +++ b/opt/readies/cetara/diag/gdb.c @@ -4,21 +4,53 @@ #include "readies/cetara/diag/gdb.h" +#include #include #include +#include #include #include +#include #include #include +#include /////////////////////////////////////////////////////////////////////////////////////////////// #ifdef __linux__ -#if 1 +static inline bool _via_gdb_simple() +{ + const int status_fd = open("/proc/self/status", O_RDONLY); + if (status_fd == -1) + return false; + + char buf[4096]; + const ssize_t num_read = read(status_fd, buf, sizeof(buf) - 1); + if (num_read <= 0) + return false; + + buf[num_read] = '\0'; + char tracer_pid[] = "TracerPid:"; + const char *tracer_pid_p = strstr(buf, tracer_pid); + if (!tracer_pid_p) + return false; + + for (const char *p = tracer_pid_p + sizeof(tracer_pid) - 1; p <= buf + num_read; ++p) + { + if (isspace(*p)) + continue; + return isdigit(*p) && *p != '0'; + } + + return false; +} static inline bool _via_gdb() { + if (_via_gdb_simple()) + return true; + int pid; int from_child[2] = {-1, -1}; @@ -78,37 +110,6 @@ static inline bool _via_gdb() } } -#else - -static inline bool _via_gdb() -{ - const int status_fd = open("/proc/self/status", O_RDONLY); - if (status_fd == -1) - return false; - - char buf[4096]; - const ssize_t num_read = read(status_fd, buf, sizeof(buf) - 1); - if (num_read <= 0) - return false; - - buf[num_read] = '\0'; - constexpr char tracer_pid[] = "TracerPid:"; - const auto tracer_pid_p = strstr(buf, tracer_pid); - if (!tracer_pid_p) - return false; - - for (const char *p = tracer_pid_p + sizeof(tracer_pid) - 1; p <= buf + num_read; ++p) - { - if (isspace(*p)) - continue; - return isdigit(*p) && *p != '0'; - } - - return false; -} - -#endif // 1 - #elif defined(__APPLE__) static inline bool _via_gdb() diff --git a/opt/readies/mk/README.md b/opt/readies/mk/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/opt/readies/mk/bindirs.defs b/opt/readies/mk/bindirs.defs index d84ba40c8..48fadc04e 100755 --- a/opt/readies/mk/bindirs.defs +++ b/opt/readies/mk/bindirs.defs @@ -1,5 +1,7 @@ +ifndef BINROOT BINROOT=$(ROOT)/bin/$(FULL_VARIANT) +endif BINROOT.release=$(ROOT)/bin/$(FULL_VARIANT.release) BIN_DIRS=$(sort $(patsubst %/,%,$(BINDIR) $(dir $(OBJECTS))) $(BINDIRS)) diff --git a/opt/readies/mk/bindirs.rules b/opt/readies/mk/bindirs.rules index 4b57e8d49..bf2d5fe87 100755 --- a/opt/readies/mk/bindirs.rules +++ b/opt/readies/mk/bindirs.rules @@ -1,4 +1,6 @@ +ifneq ($(MK.nobindir),1) + ifeq ($(BINDIR),) $(error BINDIR is undefined) endif @@ -7,4 +9,14 @@ endif bindirs: $(BIN_DIRS) +ifeq ($(DIAG),1) +$(info *** BIN_DIRS=$(BIN_DIRS)) +endif + $(foreach DIR,$(BIN_DIRS),$(eval $(call mkdir_rule,$(DIR)))) + +else + +bindirs: ; + +endif # MK.nobindir diff --git a/opt/readies/mk/build.defs b/opt/readies/mk/build.defs index e69de29bb..78ac8dd8d 100755 --- a/opt/readies/mk/build.defs +++ b/opt/readies/mk/build.defs @@ -0,0 +1,3 @@ + +MK_CLEAN_DIR=$(SRCDIR) +MK_CLEAN_ALL_DIRS += $(BINDIR) diff --git a/opt/readies/mk/build.rules b/opt/readies/mk/build.rules index 1a3c31e2b..02862f8db 100755 --- a/opt/readies/mk/build.rules +++ b/opt/readies/mk/build.rules @@ -1,11 +1,20 @@ .PHONY: build clean -build: $(MK_MAKEFILES) $(TARGET) - -#clean: -#ifeq ($(ALL),1) -# $(SHOW)rm -rf $(BINDIR) -#else -# $(SHOW)$(MAKE) clean -C $(BUILD_DIR) -#endif \ No newline at end of file +ifeq ($(DIAG),1) +$(info *** DEFAULT_TARGETS=$(DEFAULT_TARGETS)) +$(info *** MK_MAKEFILES=$(MK_MAKEFILES)) +endif + +build: $(DEFAULT_TARGETS) $(MK_MAKEFILES) $(TARGET) + +ifneq ($(MK_CUSTOM_CLEAN),1) + +clean: +ifeq ($(ALL),1) + $(SHOW)if [ ! -z "$(MK_CLEAN_ALL_DIRS)" ]; then rm -rf $(sort $(MK_CLEAN_ALL_DIRS)); fi +else + $(SHOW)$(MAKE) clean -C $(MK_CLEAN_DIR) +endif + +endif # MK_CUSTOM_CLEAN diff --git a/opt/readies/mk/cmake.defs b/opt/readies/mk/cmake.defs index 0ed2ecad0..73d6223b6 100755 --- a/opt/readies/mk/cmake.defs +++ b/opt/readies/mk/cmake.defs @@ -1,5 +1,5 @@ -ifneq ($(MK_CMAKE),) +ifneq ($(MK.cmake),) MK_MAKEFILES += $(BINDIR)/Makefile @@ -11,4 +11,4 @@ ifeq ($(WHY),1) CMAKE_WHY=--trace-expand endif -endif # MK_CMAKE +endif # MK.cmake diff --git a/opt/readies/mk/cmake.rules b/opt/readies/mk/cmake.rules index 686fae8f1..841be1f92 100755 --- a/opt/readies/mk/cmake.rules +++ b/opt/readies/mk/cmake.rules @@ -1,7 +1,7 @@ -ifneq ($(MK_CMAKE),) +ifneq ($(MK.cmake),) -CMAKE_FILES ?= $(SRCDIR)/CMakeLists.txt +MK_CMAKE_FILES ?= $(SRCDIR)/CMakeLists.txt $(BINDIR)/Makefile : bindirs $(MK_CMAKE_FILES) $(SHOW)if [ ! -d $(BINDIR) ]; then echo "CMake: $(BINDIR) does not exist."; exit 1; fi @@ -11,4 +11,4 @@ $(BINDIR)/Makefile : bindirs $(MK_CMAKE_FILES) # CMAKE_REL_ROOT=`python -c "import os; print os.path.relpath('$(SRCDIR)', '$$PWD')"` \ # cmake $(CMAKE_FLAGS) $$CMAKE_REL_ROOT -endif # MK_CMAKE +endif # MK.cmake diff --git a/opt/readies/mk/common.defs b/opt/readies/mk/common.defs index 1735cbceb..976aedd61 100755 --- a/opt/readies/mk/common.defs +++ b/opt/readies/mk/common.defs @@ -15,3 +15,7 @@ MAKEFLAGS += --no-builtin-rules --no-print-directory define __SEP import os; rows, cols = os.popen('stty size', 'r').read().split(); print('-' * (int(cols) - 1) + \"\n\") endef + +ifneq ($(filter cov-upload,$(MAKECMDGOALS)),) +COV=1 +endif diff --git a/opt/readies/mk/common.rules b/opt/readies/mk/common.rules index f84f87be2..633cf2f6b 100755 --- a/opt/readies/mk/common.rules +++ b/opt/readies/mk/common.rules @@ -1,5 +1,6 @@ .PHONY: __sep -__sep: - @python -c "$(__SEP)" +__sep: ; +# @python -c "$(__SEP)" + diff --git a/opt/readies/mk/configure.defs b/opt/readies/mk/configure.defs index da576887b..6b4f5fa20 100755 --- a/opt/readies/mk/configure.defs +++ b/opt/readies/mk/configure.defs @@ -1,2 +1,13 @@ -CONFIGURE_FLAGS= +ifneq (,$(MK.configure)) + +CONFIGURE_FLAGS = $(CONFIGURE_FLAGS.$(OS)) + +CONFIGURE_BUILD_DIR ?= $(BINDIR) + +CONFIGURE_TARGET ?= $(TARGET) + +MK_CLEAN_DIR=$(CONFIGURE_BUILD_DIR) +MK_CLEAN_ALL_DIRS += $(CONFIGURE_BUILD_DIR) + +endif # MK.configure diff --git a/opt/readies/mk/configure.rules b/opt/readies/mk/configure.rules index 1ef0cbc51..8378279d9 100755 --- a/opt/readies/mk/configure.rules +++ b/opt/readies/mk/configure.rules @@ -1,15 +1,12 @@ -.PHONY: build clean - -build: $(MK_MAKEFILES) $(TARGET) -ifeq (,$(wildcard $(BUILD_DIR)/Makefile)) - $(SHOW)cd $(BUILD_DIR); $(abspath $(SRCDIR))/configure $(CONFIGURE_FLAGS) -endif - @make -C $(BUILD_DIR) - -clean: -ifeq ($(ALL),1) - $(SHOW)rm -rf $(BINDIR) -else - $(SHOW)$(MAKE) clean -C $(BUILD_DIR) -endif +ifneq (,$(MK.configure)) + +$(CONFIGURE_BUILD_DIR)/Makefile: $(SRCDIR)/configure + $(SHOW)set -e ;\ + cd $(CONFIGURE_BUILD_DIR) ;\ + $(abspath $(SRCDIR))/configure $(CONFIGURE_FLAGS) + +$(CONFIGURE_TARGET): $(CONFIGURE_BUILD_DIR)/Makefile + $(SHOW)make -C $(CONFIGURE_BUILD_DIR) -j $(NPROC) + +endif # MK.configure diff --git a/opt/readies/mk/coverage.defs b/opt/readies/mk/coverage.defs new file mode 100644 index 000000000..2f9983ddc --- /dev/null +++ b/opt/readies/mk/coverage.defs @@ -0,0 +1,36 @@ + +ifeq ($(COV),1) + +CC_FLAGS.coverage += --coverage +LD_FLAGS.coverage += --coverage + +COV_INFO=$(BINROOT)/cov.info +COV_DIR=$(BINROOT)/cov + +COV_EXCLUDE += \ + '/Applications/*' \ + '/usr/*' + +COVERAGE_RESET=\ + $(SHOW)set -e; \ + mkdir -p $(COV_DIR) ;\ + lcov --directory $(BINROOT) --base-directory $(SRCDIR) -z > /dev/null 2>&1 + +COVERAGE_COLLECT=\ + $(SHOW)set -e; \ + echo "Collecting coverage data ..." ;\ + lcov --capture --directory $(BINROOT) --base-directory $(SRCDIR) --output-file $(COV_INFO) > /dev/null 2>&1 ;\ + lcov -o $(COV_INFO) -r $(COV_INFO) $(COV_EXCLUDE) > /dev/null 2>&1 + +COVERAGE_REPORT= \ + $(SHOW)set -e; \ + lcov -l $(COV_INFO) ;\ + genhtml --legend -o $(COV_DIR) $(COV_INFO) > /dev/null 2>&1 ;\ + echo "Coverage info at $(realpath $(COV_DIR))/index.html" + +define COVERAGE_COLLECT_REPORT +$(COVERAGE_COLLECT) +$(COVERAGE_REPORT) +endef + +endif # COV diff --git a/opt/readies/mk/coverage.rules b/opt/readies/mk/coverage.rules new file mode 100644 index 000000000..342109af0 --- /dev/null +++ b/opt/readies/mk/coverage.rules @@ -0,0 +1,7 @@ + +ifeq ($(COV),1) + +cov-upload: + $(SHOW)bash <(curl -s https://codecov.io/bash) -f $(COV_INFO) + +endif # COV diff --git a/opt/readies/mk/defs b/opt/readies/mk/defs index c4f0d4016..7ba0de577 100755 --- a/opt/readies/mk/defs +++ b/opt/readies/mk/defs @@ -1,4 +1,15 @@ include $(MK)/help.defs include $(MK)/build.defs + +ifneq ($(MK.cmake),) include $(MK)/cmake.defs +endif + +ifneq ($(MK.configure),) +include $(MK)/configure.defs +endif + +ifeq ($(COV),1) +include $(MK)/coverage.defs +endif diff --git a/opt/readies/mk/docker b/opt/readies/mk/docker new file mode 100644 index 000000000..1679e95d4 --- /dev/null +++ b/opt/readies/mk/docker @@ -0,0 +1,123 @@ + +.NOTPARALLEL: + +MK.nobindir=1 + +MK_ALL_TARGETS=build publish + +ifeq ($(VERSION),) +VERSION:=$(patsubst v%,%,$(shell git describe --tags `git rev-list --tags --max-count=1`)) +endif +ifeq ($(VERSION),) +$(error Cannot determine version. Aborting.) +endif + +BUILD_OPT=--rm +# --squash + +DOCKERFILE_STEM ?= $(ROOT)/Dockerfile + +#---------------------------------------------------------------------------------------------- + +define HELP +make build # build Docker images + X86=1 # build x64 Docker image + ARM7=1 # build arm32v7 Docker image + ARM8=1 # build arm64v8 Docker image +make publish # build and push multi-arch Docker manifest + X86=1 # push x64 Docker image and multi-arch manifest + ARM7=1 # push arm32v7 Docker image and multi-arch manifest + ARM8=1 # push arm64v8 Docker image and multi-arch manifest + PUSH=0 # only push multi-arch manifest +make show # display registry information from Docker repo + +Other arguments: + VERSION=x.y.z # build and publish version x.y.z + OSNICK=nick # nick=buster|stretch|bionic +endef + +#---------------------------------------------------------------------------------------------- + +define targets # (1=OP, 2=op) +$(1)_TARGETS := +$(1)_TARGETS += $(if $(findstring $(X64),1),$(2)_x64) +$(1)_TARGETS += $(if $(findstring $(ARM7),1),$(2)_arm32v7) +$(1)_TARGETS += $(if $(findstring $(ARM8),1),$(2)_arm64v8) + +$(1)_TARGETS += $$(if $$(strip $$($(1)_TARGETS)),,$(2)_x64 $(2)_arm32v7 $(2)_arm64v8) +endef + +$(eval $(call targets,BUILD,build)) +$(eval $(call targets,PUSH,push)) + +#---------------------------------------------------------------------------------------------- + +define build_x64 # (1=arch) +build_$(1): + @docker build $(BUILD_OPT) -t $(STEM)-$(OSNICK):$(VERSION)-x64 -f $(DOCKERFILE_STEM) \ + $(foreach A,$(DOCKER_BUILD_ARGS),--build-arg $(A)) \ + $(ROOT) + +.PHONY: build_$(1) +endef + +define build_arm # (1=arch) +build_$(1): + @docker build $(BUILD_OPT) -t $(STEM)-$(OSNICK):$(VERSION)-$(1) -f $(DOCKERFILE_STEM).arm \ + $(foreach A,$(DOCKER_BUILD_ARGS),--build-arg $(A)) \ + --build-arg ARCH=$(1) \ + $(ROOT) + +.PHONY: build_$(1) +endef + +define push # (1=arch) +push_$(1): + @docker push $(STEM)-$(OSNICK):$(VERSION)-$(1) + +.PHONY: push_$(1) +endef + +define create_manifest # (1=version) +@docker manifest create -a $(STEM)-$(OSNICK):$(1) \ + -a $(STEM)-$(OSNICK):$(VERSION)-x64 \ + -a $(STEM)-$(OSNICK):$(VERSION)-arm64v8 \ + -a $(STEM)-$(OSNICK):$(VERSION)-arm32v7 +@docker manifest annotate $(STEM)-$(OSNICK):$(1) $(STEM)-$(OSNICK):$(VERSION)-arm32v7 --os linux --arch arm --variant v7 +@docker manifest annotate $(STEM)-$(OSNICK):$(1) $(STEM)-$(OSNICK):$(VERSION)-arm64v8 --os linux --arch arm64 --variant v8 +@docker manifest push -p $(STEM)-$(OSNICK):$(1) +endef + +#---------------------------------------------------------------------------------------------- + +DEFAULT_TARGETS += $(BUILD_TARGETS) + +include $(MK)/defs + +include $(MK)/rules + +$(eval $(call build_x64,x64)) +$(eval $(call build_arm,arm64v8)) +$(eval $(call build_arm,arm32v7)) + +$(eval $(call push,x64)) +$(eval $(call push,arm64v8)) +$(eval $(call push,arm32v7)) + +ifneq ($(PUSH),0) +publish: $(PUSH_TARGETS) +else +publish: +endif + $(call create_manifest,$(VERSION)) + $(call create_manifest,latest) + +show: + @echo "$(STEM)-$(OSNICK):" +ifeq ($(INSPECT),1) + @docker manifest inspect $(STEM)-$(OSNICK):$(VERSION) | jq +else + @curl -s -X "GET" https://cloud.docker.com/v2/repositories/$(STEM)-$(OSNICK)/tags/ | jq +endif + +.PHONY: build publish show diff --git a/opt/readies/mk/git.defs b/opt/readies/mk/git.defs new file mode 100644 index 000000000..6a09915a9 --- /dev/null +++ b/opt/readies/mk/git.defs @@ -0,0 +1,5 @@ + +ifeq ($(shell { [ -d .git ] || git rev-parse --git-dir >/dev/null 2>&1; echo -n $$?; }),0) +GIT_SHA := $(shell git rev-parse HEAD) +GIT_COMMIT := $(shell git describe --always --abbrev=7 --dirty="+" 2>/dev/null || git rev-parse --short HEAD) +endif diff --git a/opt/readies/mk/macosx.defs b/opt/readies/mk/macosx.defs new file mode 100644 index 000000000..65486aa58 --- /dev/null +++ b/opt/readies/mk/macosx.defs @@ -0,0 +1,42 @@ + +ifeq ($(OS),macosx) + +#---------------------------------------------------------------------------------------------- + +# pip install --user puts packages here: +export PATH:=$(PATH):$(HOME)/Library/Python/2.7/bin + +#---------------------------------------------------------------------------------------------- + +ZLIB_PREFIX:=$(shell brew --prefix zlib) +LIBSSL_PREFIX:=$(shell brew --prefix openssl) +READLINE_PREFIX:=$(shell brew --prefix readline) +SDK_PREFIX:=$(shell xcrun --show-sdk-path) + +export CPPFLAGS:=\ + -I$(ZLIB_PREFIX)/include \ + -I$(READLINE_PREFIX)/include \ + -I$(LIBSSL_PREFIX)/include \ + -I$(SDK_PREFIX)/usr/include + +export LDFLAGS:=\ + -L$(ZLIB_PREFIX)/lib \ + -L$(LIBSSL_PREFIX)/lib \ + -L$(READLINE_PREFIX)/lib \ + -L$(SDK_PREFIX)/usr/lib + +PKG_CONFIG_DIRS += \ + $(ZLIB_PREFIX) \ + $(LIBSSL_PREFIX) \ + $(READLINE_PREFIX) + +export PKG_CONFIG_PATH:=$(subst $(__SPACE),:,$(strip $(addsuffix /lib/pkgconfig,$(PKG_CONFIG_DIRS)))):$(PKG_CONFIG_PATH) + +CONFIGURE_FLAGS.macosx += \ + CPPFLAGS='$(CPPFLAGS)' \ + LDFLAGS='$(LDFLAGS)' \ + --with-openssl=$(LIBSSL_PREFIX) + +#---------------------------------------------------------------------------------------------- + +endif # macosx diff --git a/opt/readies/mk/main b/opt/readies/mk/main index 05fa4297b..cd590fce6 100755 --- a/opt/readies/mk/main +++ b/opt/readies/mk/main @@ -7,23 +7,44 @@ endif ifneq ($(wildcard $(ROOT)/deps/readies),) READIES:=$(ROOT)/deps/readies -else -ifneq ($(wildcard $(ROOT)/opt/readies),) +else ifneq ($(wildcard $(ROOT)/opt/readies),) READIES:=$(ROOT)/opt/readies else $(error Cannot find readies root) endif -endif MK:=$(READIES)/mk -ifneq ($(shell { CHECK=1 $(READIES)/bin/getpy; echo $$?; }),0) -$(error It seems prerequisites have not been installed: please run 'make setup'.) +MK.pyver ?= 3 +MK.getpy=getpy$(MK.pyver) + +ifneq ($(shell { CHECK=1 $(READIES)/bin/$(MK.getpy); echo -n $$?; }),0) +ifneq ($(filter setup,$(MAKECMDGOALS)),) +$(error It seems build prerequisites have not been installed: please run 'sudo make setup'.) +endif +__NO_PYTHON=1 endif MK_ALL_TARGETS:=bindirs build +DEFAULT_TARGETS:=bindirs + +SUDO:=$(shell [ $$( command -v sudo >/dev/null 2>&1; echo $$? ) = 0 ] && echo sudo) +USER:=$(shell whoami) include $(MK)/functions include $(MK)/common.defs + +ifneq ($(__NO_PYTHON),1) + +include $(MK)/platform.defs + +ifeq ($(OS),macosx) +include $(MK)/macosx.defs +endif + +include $(MK)/git.defs include $(MK)/variant.defs include $(MK)/bindirs.defs +include $(MK)/nproc.defs + +endif # __NO_PYTHON \ No newline at end of file diff --git a/opt/readies/mk/nproc.defs b/opt/readies/mk/nproc.defs new file mode 100644 index 000000000..95f6d3b1f --- /dev/null +++ b/opt/readies/mk/nproc.defs @@ -0,0 +1,6 @@ + +ifeq ($(OS),linux) +NPROC:=$(shell nproc) +else ifeq ($(OS),macosx) +NPROC:=$(shell sysctl -n hw.physicalcpu) +endif diff --git a/opt/readies/mk/platform.defs b/opt/readies/mk/platform.defs new file mode 100644 index 000000000..50896184e --- /dev/null +++ b/opt/readies/mk/platform.defs @@ -0,0 +1,7 @@ + +OS:=$(shell $(READIES)/bin/platform --os) +# ifeq ($(OS),linux) +# OS:=$(shell $(READIES)/bin/platform --dist) +# endif + +ARCH=$(shell $(READIES)/bin/platform --arch) diff --git a/opt/readies/mk/rules b/opt/readies/mk/rules index e433c92b2..77bd717fa 100755 --- a/opt/readies/mk/rules +++ b/opt/readies/mk/rules @@ -1,11 +1,21 @@ -.PHONY: all - -all: $(MK_ALL_TARGETS) +default: build include $(MK)/common.rules include $(MK)/variant.rules include $(MK)/bindirs.rules include $(MK)/help.rules include $(MK)/build.rules +include $(MK)/coverage.rules + +ifneq ($(MK.cmake),) include $(MK)/cmake.rules +endif + +ifneq ($(MK.configure),) +include $(MK)/configure.rules +endif + +all: $(MK_ALL_TARGETS) + +.PHONY: all default diff --git a/opt/readies/mk/variant.defs b/opt/readies/mk/variant.defs index eccb47d8a..4277263fa 100755 --- a/opt/readies/mk/variant.defs +++ b/opt/readies/mk/variant.defs @@ -1,20 +1,8 @@ -OS:=$(shell $(READIES)/bin/platform --os) -# ifeq ($(OS),linux) -# OS:=$(shell $(READIES)/bin/platform --dist) -# endif - -ARCH=$(shell $(READIES)/bin/platform --arch) - -#---------------------------------------------------------------------------------------------- - -ifeq ($(shell { [ -d .git ] || git rev-parse --git-dir >/dev/null 2>&1; echo $?; }),0) -GIT_SHA := $(shell git rev-parse HEAD) -GIT_COMMIT := $(shell git describe --always --abbrev=7 --dirty="+") +ifeq ($(COV),1) +export DEBUG ?= 1 endif -#---------------------------------------------------------------------------------------------- - ifeq ($(DEBUG),1) FLAVOR=debug else @@ -23,13 +11,18 @@ endif #---------------------------------------------------------------------------------------------- +ifeq ($(COV),1) +VARIANT.list:=$(sort $(VARIANT.list) cov) +endif + VARIANT.file:=$(shell if [ -f $(ROOT)/VARIANT ]; then cat $(ROOT)/VARIANT; fi) # if VARIANT not specified and we're the not in submake, use one from file, if present ifeq ($(origin VARIANT),undefined) - ifneq ($(VARIANT.primary),) + ifeq ($(VARIANT.primary),) ifneq ($(VARIANT.file),) VARIANT:=$(VARIANT.file) + $(info VARIANT is $(VARIANT)) endif endif else # VARIANT specified diff --git a/opt/readies/paella/README.md b/opt/readies/paella/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/opt/readies/paella/debug.py b/opt/readies/paella/debug.py index bf6ee7bbc..c615a09c5 100755 --- a/opt/readies/paella/debug.py +++ b/opt/readies/paella/debug.py @@ -4,13 +4,17 @@ #---------------------------------------------------------------------------------------------- pydebug = os.environ.get('PYDEBUG', '') -if pydebug == '1' or pydebug == 'pudb': +if pydebug == '1': try: - from pudb import set_trace as bb + from ipdb import set_trace as bb except ImportError: from pdb import set_trace as bb +elif pydebug == 'pudb': + from pudb import set_trace as bb elif pydebug == 'pdb': from pdb import set_trace as bb +elif pydebug == 'ipdb': + from ipdb import set_trace as bb else: def bb(): pass diff --git a/opt/readies/paella/platform.py b/opt/readies/paella/platform.py index e91ab6ecb..9105089fb 100755 --- a/opt/readies/paella/platform.py +++ b/opt/readies/paella/platform.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import platform +import os import re #---------------------------------------------------------------------------------------------- @@ -35,6 +36,13 @@ def osnick(self): def __init__(self, strict=False): self.os = self.dist = self.os_ver = self.full_os_ver = self.osnick = self.arch = '?' + # self.os = os.getenv("READIES_PLATFORM_OS", '?') + # self.dist = os.getenv("READIES_PLATFORM_DIST", '?') + # self.os_ver = os.getenv("READIES_PLATFORM_OS_VER", '?') + # self.full_os_ver = os.getenv("READIES_PLATFORM_FULL_OS_VER", '?') + # self.osnick = os.getenv("READIES_PLATFORM_OSNICK", '?') + # self.arch = os.getenv("READIES_PLATFORM_ARCH", '?') + self.os = platform.system().lower() if self.os == 'linux': if False: @@ -189,6 +197,8 @@ def invoke(self): assert(False), "Cannot determine installer" elif os == 'macosx': self.macosx() + elif os == 'freebsd': + self.freebsd() self.common_last() diff --git a/opt/readies/paella/setup.py b/opt/readies/paella/setup.py index be36d0456..b4a3985ab 100755 --- a/opt/readies/paella/setup.py +++ b/opt/readies/paella/setup.py @@ -227,7 +227,12 @@ def install_git_lfs_on_linux(self, _try=False): # self.install("git-lfs", _try=_try) def install_gnu_utils(self, _try=False): - self.install("make findutils gnu-sed gnu-tar") + packs = "" + if self.os == 'macosx': + packs= "make findutils gnu-sed gnu-tar gawk" + elif self.os == 'freebsd': + packs = "gmake findutils gsed gtar gawk" + self.install(packs) for x in ['make', 'find', 'sed', 'tar']: p = "/usr/local/bin/{}".format(x) if not os.path.exists(p): diff --git a/opt/readies/shibumi/README.md b/opt/readies/shibumi/README.md new file mode 100644 index 000000000..e69de29bb From 0407ddf6cea7b579218b8ae2a2964386b278d384 Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 25 Feb 2020 15:21:34 +0200 Subject: [PATCH 02/18] fixes 1 --- opt/Makefile | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/opt/Makefile b/opt/Makefile index 794711bfa..444cd85d3 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -10,22 +10,24 @@ MK_CUSTOM_CLEAN:=1 #---------------------------------------------------------------------------------------------- define HELP -make setup # install prerequisited (CAUTION: THIS WILL MODIFY YOUR SYSTEM) -make fetch # download and prepare dependant modules -make build # compile and link -make clean # remove build artifacts - ALL=1 # remove entire artifacts directory -make test # run tests - TEST=test # run only test `test` with Redis output - TEST_ARGS=args # add extra RLTest `args` - VERBOSE=1 # verbose tests output - TEST_COVERAGE=off # add coverage -make pack # create installation packages - PACK_DEPS=0 # do not pack dependencies - INTO=dir # place artifacts in `dir` - BRANCH=name # use `name` as branch name -make deploy # copy packages to S3 -make release # release a version +make setup # install prerequisited (CAUTION: THIS WILL MODIFY YOUR SYSTEM) +make fetch # download and prepare dependant modules +make build # compile and link +make clean # remove build artifacts + ALL=1 # remove entire artifacts directory + +make test # run tests + TEST=test # run only test `test` with Redis output + TEST_ARGS=args # add extra RLTest `args` + VERBOSE=1 # verbose tests output + COVERAGE=1 # perform coverage analysis + +make pack # create installation packages + PACK_DEPS=0 # do not pack dependencies + INTO=dir # place artifacts in `dir` + BRANCH=name # use `name` as branch name +make deploy # copy packages to S3 +make release # release a version fetch and build options: WITH_TF=0 # SKip TensofFlow @@ -60,7 +62,7 @@ DEPS_DIR=$(ROOT)/deps/$(OS)-$(ARCH)-$(DEVICE) # INSTALL_DIR=$(ROOT)/bin/$(OS)-$(ARCH)-$(DEVICE)/install INSTALL_DIR=$(BINROOT)/install-$(DEVICE) -REDIS_VALGRID_SUPRESS=$(ROOT)/opt/redis_valgrind.sup +REDIS_VALGRIND_SUPRESS=$(ROOT)/opt/redis_valgrind.sup TARGET=$(BINDIR)/redisai.so @@ -71,8 +73,7 @@ CMAKE_FILES += \ $(SRCDIR)/src/CMakeLists.txt \ $(SRCDIR)/libtorch_c/CMakeLists.txt -WITH_COVERAGE=off -ifeq ($(TEST_COVERAGE),on) +ifeq ($(COVERAGE),1) WITH_COVERAGE=on endif @@ -168,7 +169,7 @@ TEST_PREFIX=set -e; cd $(ROOT)/test # TODO: --errors-for-leak-kinds=definite VALGRIND_OPTIONS="--leak-check=full -q --show-reachable=no --show-possibly-lost=no" TEST_CMD= DEVICE=$(DEVICE) PYDEBUG=$(PYDEBUG) python3 -m RLTest $(TEST_ARGS) --module $(INSTALL_DIR)/redisai.so -VALGRIND_TEST_CMD= DEVICE=$(DEVICE) PYDEBUG=$(PYDEBUG) python3 -m RLTest $(TEST_ARGS) --module $(INSTALL_DIR)/redisai.so --no-output-catch --use-valgrind --vg-no-fail-on-errors --vg-verbose --vg-options $(VALGRIND_OPTIONS) --vg-suppressions $(realpath $(REDIS_VALGRID_SUPRESS)) +VALGRIND_TEST_CMD= DEVICE=$(DEVICE) PYDEBUG=$(PYDEBUG) python3 -m RLTest $(TEST_ARGS) --module $(INSTALL_DIR)/redisai.so --no-output-catch --use-valgrind --vg-no-fail-on-errors --vg-verbose --vg-options $(VALGRIND_OPTIONS) --vg-suppressions $(realpath $(REDIS_VALGRIND_SUPRESS)) test: ifneq ($(NO_LFS),1) @@ -198,13 +199,14 @@ docker: #---------------------------------------------------------------------------------------------- # Currently only testing for leaks using TF + MODULE_ARGS=\ - BACKENDSPATH $(realpath $(BINDIR)) \ + BACKENDSPATH $(realpath $(BINDIR)) \ TF redisai_tensorflow.so VALGRIND_ARGS=\ $(VALGRIND_OPTIONS) \ - --suppressions=$(realpath $(REDIS_VALGRID_SUPRESS)) \ + --suppressions=$(realpath $(REDIS_VALGRIND_SUPRESS)) \ -v redis-server --protected-mode no --save "" --appendonly no valgrind: $(TARGET) @@ -220,4 +222,4 @@ CALLGRIND_ARGS=\ -v redis-server --protected-mode no --save "" --appendonly no callgrind: $(TARGET) - $(SHOW)valgrind $(CALLGRIND_ARGS) --loadmodule $(realpath $(TARGET)) $(MODULE_ARGS) \ No newline at end of file + $(SHOW)valgrind $(CALLGRIND_ARGS) --loadmodule $(realpath $(TARGET)) $(MODULE_ARGS) From ca001a5d240f25c05d56c9f9b4a99f71c2fc9dc7 Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 25 Feb 2020 15:32:28 +0200 Subject: [PATCH 03/18] fixes 2 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ba0497c2c..25209f9c5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ commands: - run: name: Install dependencies command: | - ./opt/readies/bin/getpy + ./opt/readies/bin/getpy3 BREW_NO_UPDATE=1 ./opt/system-setup.py git clone git://github.com/antirez/redis.git --branch 5.0.7 (cd redis && make malloc=libc -j $(nproc) && make install) From cce38b095698ad75c5e24315bc1178357e729dae Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 25 Feb 2020 17:14:39 +0200 Subject: [PATCH 04/18] fixes 3 --- CMakeLists.txt | 23 +++++++++++--- Dockerfile | 2 +- Dockerfile.arm | 2 +- Dockerfile.gpu-test | 2 +- opt/Makefile | 64 +++++++++++++++++++++++++++++--------- opt/readies/mk/cmake.defs | 1 + opt/readies/mk/cmake.rules | 2 +- 7 files changed, 74 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 69f8e7f7e..95d3a2bcf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,16 +16,31 @@ IF(NOT CMAKE_BUILD_TYPE) "Debug" "Release" "MinSizeRel" "RelWithDebInfo") ENDIF() -SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") +#---------------------------------------------------------------------------------------------- + +SET(CMAKE_CC_COMMON_FLAGS "-fPIC") + +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_CC_COMMON_FLAGS}") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CC_COMMON_FLAGS}") # For adding specific Release flags set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3") # Add -fno-omit-frame-pointer to avoid seeing incomplete stack traces -set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer") -set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer") +set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer -DVALGRIND") +set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer -DVALGRIND") + +#---------------------------------------------------------------------------------------------- + +IF (USE_COVERAGE) + IF (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + MESSAGE(FATAL_ERROR "Build type must be DEBUG for coverage") + ENDIF() + SET(CMAKE_CC_COMMON_FLAGS "${CMAKE_CC_COMMON_FLAGS} -coverage") +ENDIF() + +#---------------------------------------------------------------------------------------------- option(BUILD_TF "Build the TensorFlow backend" ON) option(BUILD_TFLITE "Build the TensorFlow Lite backend" ON) diff --git a/Dockerfile b/Dockerfile index cfa539476..47893ebf8 100755 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ COPY --from=redis /usr/local/ /usr/local/ COPY ./opt/ opt/ COPY ./test/test_requirements.txt test/ -RUN ./opt/readies/bin/getpy +RUN ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py ARG DEPS_ARGS="" diff --git a/Dockerfile.arm b/Dockerfile.arm index 4c4865066..2cf2f91e1 100755 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -19,7 +19,7 @@ WORKDIR /build COPY ./opt/ opt/ COPY ./test/test_requirements.txt test/ -RUN ./opt/readies/bin/getpy +RUN ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py ARG DEPS_ARGS="" diff --git a/Dockerfile.gpu-test b/Dockerfile.gpu-test index 6dbc1ffa7..2278e6592 100644 --- a/Dockerfile.gpu-test +++ b/Dockerfile.gpu-test @@ -21,7 +21,7 @@ COPY --from=redis /usr/local/ /usr/local/ COPY ./opt/ opt/ COPY ./test/test_requirements.txt test/ -RUN ./opt/readies/bin/getpy +RUN ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py ARG DEPS_ARGS="" diff --git a/opt/Makefile b/opt/Makefile index 444cd85d3..28e21adb4 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -12,7 +12,10 @@ MK_CUSTOM_CLEAN:=1 define HELP make setup # install prerequisited (CAUTION: THIS WILL MODIFY YOUR SYSTEM) make fetch # download and prepare dependant modules + make build # compile and link + DEBUG=1 # build for debugging + WHY=1 # explain CMake decisions (into /tmp/cmake.why) make clean # remove build artifacts ALL=1 # remove entire artifacts directory @@ -20,7 +23,7 @@ make test # run tests TEST=test # run only test `test` with Redis output TEST_ARGS=args # add extra RLTest `args` VERBOSE=1 # verbose tests output - COVERAGE=1 # perform coverage analysis + COV=1 # perform coverage analysis make pack # create installation packages PACK_DEPS=0 # do not pack dependencies @@ -68,19 +71,23 @@ TARGET=$(BINDIR)/redisai.so BACKENDS_PATH ?= $(INSTALL_DIR)/backends +#---------------------------------------------------------------------------------------------- + CMAKE_FILES += \ $(SRCDIR)/CMakeLists.txt \ $(SRCDIR)/src/CMakeLists.txt \ $(SRCDIR)/libtorch_c/CMakeLists.txt -ifeq ($(COVERAGE),1) -WITH_COVERAGE=on +ifeq ($(COV),1) +USE_COVERAGE=on +else +USE_COVERAGE=off endif CMAKE_FLAGS += \ -DDEPS_PATH=$(abspath $(DEPS_DIR)) \ -DINSTALL_PATH=$(abspath $(INSTALL_DIR)) \ - -DENABLE_CODECOVERAGE=$(WITH_COVERAGE) \ + -DUSE_COVERAGE=$(USE_COVERAGE) \ -DDEVICE=$(DEVICE) ifeq ($(WITH_TF),0) @@ -149,6 +156,13 @@ endif #---------------------------------------------------------------------------------------------- +define CHECK_REDIS_SERVER +$(SHOW)if ! command -v redis-server > /dev/null; then \ + echo "Cannot find redis-server. Aborting." ;\ + exit 1 ;\ +fi +endef + TEST_REPORT_DIR ?= $(PWD) ifeq ($(VERBOSE),1) TEST_ARGS += -v @@ -166,30 +180,52 @@ SLAVES ?= 1 AOF ?= 1 TEST_PREFIX=set -e; cd $(ROOT)/test + +define TEST_CMD +DEVICE=$(DEVICE) PYDEBUG=$(PYDEBUG) python3 -m RLTest $(TEST_ARGS) --module $(INSTALL_DIR)/redisai.so +endef + # TODO: --errors-for-leak-kinds=definite -VALGRIND_OPTIONS="--leak-check=full -q --show-reachable=no --show-possibly-lost=no" -TEST_CMD= DEVICE=$(DEVICE) PYDEBUG=$(PYDEBUG) python3 -m RLTest $(TEST_ARGS) --module $(INSTALL_DIR)/redisai.so -VALGRIND_TEST_CMD= DEVICE=$(DEVICE) PYDEBUG=$(PYDEBUG) python3 -m RLTest $(TEST_ARGS) --module $(INSTALL_DIR)/redisai.so --no-output-catch --use-valgrind --vg-no-fail-on-errors --vg-verbose --vg-options $(VALGRIND_OPTIONS) --vg-suppressions $(realpath $(REDIS_VALGRIND_SUPRESS)) +VALGRIND_OPTIONS=\ + -q \ + --leak-check=full \ + --show-reachable=no \ + --show-possibly-lost=no + +define VALGRIND_TEST_CMD +$(TEST_CMD) \ + --no-output-catch \ + --use-valgrind \ + --vg-no-fail-on-errors \ + --vg-verbose + --vg-options "$(VALGRIND_OPTIONS)" \ + --vg-suppressions $(realpath $(REDIS_VALGRIND_SUPRESS)) +endef test: + $(CHECK_REDIS_SERVER) ifneq ($(NO_LFS),1) $(SHOW)if [ "$(git lfs env > /dev/null 2>&1 ; echo $?)" != "0" ]; then cd $(ROOT); git lfs install; fi $(SHOW)cd $(ROOT); git lfs pull endif +ifeq ($(COV),1) + $(COVERAGE_RESET) +endif ifeq ($(GEN),1) $(SHOW)$(TEST_PREFIX); $(TEST_CMD) endif ifeq ($(AOF),1) - $(SHOW)$(TEST_PREFIX); printf "\nTests with --use-aof:\n\n" ;\ - $(TEST_CMD) --use-aof + $(SHOW)$(TEST_PREFIX); printf "\nTests with --use-aof:\n\n" ; $(TEST_CMD) --use-aof endif ifeq ($(SLAVES),1) - $(SHOW)$(TEST_PREFIX); printf "\nTests with --use-slaves:\n\n" ;\ - $(TEST_CMD) --use-slaves + $(SHOW)$(TEST_PREFIX); printf "\nTests with --use-slaves:\n\n" ; $(TEST_CMD) --use-slaves endif -ifeq ($(VALGRIND),1) - $(SHOW)$(TEST_PREFIX); printf "\nTests with valgrind:\n\n" ;\ - $(VALGRIND_TEST_CMD) +# ifeq ($(VALGRIND),1) +# $(SHOW)$(TEST_PREFIX); printf "\nTests with valgrind:\n\n" ;\ +# $(VALGRIND_TEST_CMD) +# endif +ifeq ($(COV),1) + $(COVERAGE_COLLECT_REPORT) endif #---------------------------------------------------------------------------------------------- diff --git a/opt/readies/mk/cmake.defs b/opt/readies/mk/cmake.defs index 73d6223b6..ab8a8741a 100755 --- a/opt/readies/mk/cmake.defs +++ b/opt/readies/mk/cmake.defs @@ -9,6 +9,7 @@ endif ifeq ($(WHY),1) CMAKE_WHY=--trace-expand +CMAKE_SINK=>/tmp/cmake.log endif endif # MK.cmake diff --git a/opt/readies/mk/cmake.rules b/opt/readies/mk/cmake.rules index 841be1f92..f2a4a73b7 100755 --- a/opt/readies/mk/cmake.rules +++ b/opt/readies/mk/cmake.rules @@ -5,7 +5,7 @@ MK_CMAKE_FILES ?= $(SRCDIR)/CMakeLists.txt $(BINDIR)/Makefile : bindirs $(MK_CMAKE_FILES) $(SHOW)if [ ! -d $(BINDIR) ]; then echo "CMake: $(BINDIR) does not exist."; exit 1; fi - $(SHOW)cd $(BINDIR); cmake $(CMAKE_WHY) $(CMAKE_FLAGS) $(abspath $(SRCDIR)) + $(SHOW)cd $(BINDIR); cmake $(CMAKE_WHY) $(CMAKE_FLAGS) $(abspath $(SRCDIR)) $(CMAKE_SINK) # $(SHOW)cd $(BINDIR); \ # CMAKE_REL_ROOT=`python -c "import os; print os.path.relpath('$(SRCDIR)', '$$PWD')"` \ From 936604437170db7f17376455a5bfe645c7f9e094 Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 25 Feb 2020 21:04:21 +0200 Subject: [PATCH 05/18] Tests/Valgrind refactoring --- CMakeLists.txt | 16 +- opt/Makefile | 74 +-- opt/readies/mk/cmake.defs | 2 +- opt/readies/mk/coverage.rules | 4 + test/basic_tests.py | 1158 --------------------------------- test/double-panda.py | 94 --- test/tests.sh | 125 ++++ 7 files changed, 151 insertions(+), 1322 deletions(-) delete mode 100644 test/basic_tests.py delete mode 100755 test/double-panda.py create mode 100755 test/tests.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 95d3a2bcf..d838245f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,13 @@ ENDIF() SET(CMAKE_CC_COMMON_FLAGS "-fPIC") +IF (USE_COVERAGE) + IF (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + MESSAGE(FATAL_ERROR "Build type must be DEBUG for coverage") + ENDIF() + SET(CMAKE_CC_COMMON_FLAGS "${CMAKE_CC_COMMON_FLAGS} -coverage") +ENDIF() + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_CC_COMMON_FLAGS}") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CC_COMMON_FLAGS}") @@ -33,15 +40,6 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-poi #---------------------------------------------------------------------------------------------- -IF (USE_COVERAGE) - IF (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") - MESSAGE(FATAL_ERROR "Build type must be DEBUG for coverage") - ENDIF() - SET(CMAKE_CC_COMMON_FLAGS "${CMAKE_CC_COMMON_FLAGS} -coverage") -ENDIF() - -#---------------------------------------------------------------------------------------------- - option(BUILD_TF "Build the TensorFlow backend" ON) option(BUILD_TFLITE "Build the TensorFlow Lite backend" ON) option(BUILD_ORT "Build the ONNXRuntime backend" ON) diff --git a/opt/Makefile b/opt/Makefile index 28e21adb4..544258343 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -15,6 +15,7 @@ make fetch # download and prepare dependant modules make build # compile and link DEBUG=1 # build for debugging + VARIANT=name # build variant `name` WHY=1 # explain CMake decisions (into /tmp/cmake.why) make clean # remove build artifacts ALL=1 # remove entire artifacts directory @@ -24,6 +25,7 @@ make test # run tests TEST_ARGS=args # add extra RLTest `args` VERBOSE=1 # verbose tests output COV=1 # perform coverage analysis +make cov-upload # upload coverage data to codecov.io (requires CODECOV_TOKEN) make pack # create installation packages PACK_DEPS=0 # do not pack dependencies @@ -68,6 +70,7 @@ INSTALL_DIR=$(BINROOT)/install-$(DEVICE) REDIS_VALGRIND_SUPRESS=$(ROOT)/opt/redis_valgrind.sup TARGET=$(BINDIR)/redisai.so +INSTALLED_TARGET=$(INSTALL_DIR)/redisai.so BACKENDS_PATH ?= $(INSTALL_DIR)/backends @@ -156,41 +159,9 @@ endif #---------------------------------------------------------------------------------------------- -define CHECK_REDIS_SERVER -$(SHOW)if ! command -v redis-server > /dev/null; then \ - echo "Cannot find redis-server. Aborting." ;\ - exit 1 ;\ -fi -endef - -TEST_REPORT_DIR ?= $(PWD) -ifeq ($(VERBOSE),1) -TEST_ARGS += -v -endif -ifeq ($(TEST),) -TEST= -PYDEBUG= -else -TEST_ARGS += -s --test $(TEST) -PYDEBUG=1 -endif - -GEN ?= 1 -SLAVES ?= 1 -AOF ?= 1 - -TEST_PREFIX=set -e; cd $(ROOT)/test - -define TEST_CMD -DEVICE=$(DEVICE) PYDEBUG=$(PYDEBUG) python3 -m RLTest $(TEST_ARGS) --module $(INSTALL_DIR)/redisai.so -endef - -# TODO: --errors-for-leak-kinds=definite -VALGRIND_OPTIONS=\ - -q \ - --leak-check=full \ - --show-reachable=no \ - --show-possibly-lost=no +export GEN ?= 1 +export SLAVES ?= 1 +export AOF ?= 1 define VALGRIND_TEST_CMD $(TEST_CMD) \ @@ -203,27 +174,15 @@ $(TEST_CMD) \ endef test: - $(CHECK_REDIS_SERVER) -ifneq ($(NO_LFS),1) - $(SHOW)if [ "$(git lfs env > /dev/null 2>&1 ; echo $?)" != "0" ]; then cd $(ROOT); git lfs install; fi - $(SHOW)cd $(ROOT); git lfs pull -endif ifeq ($(COV),1) $(COVERAGE_RESET) endif -ifeq ($(GEN),1) - $(SHOW)$(TEST_PREFIX); $(TEST_CMD) -endif -ifeq ($(AOF),1) - $(SHOW)$(TEST_PREFIX); printf "\nTests with --use-aof:\n\n" ; $(TEST_CMD) --use-aof -endif -ifeq ($(SLAVES),1) - $(SHOW)$(TEST_PREFIX); printf "\nTests with --use-slaves:\n\n" ; $(TEST_CMD) --use-slaves -endif -# ifeq ($(VALGRIND),1) -# $(SHOW)$(TEST_PREFIX); printf "\nTests with valgrind:\n\n" ;\ -# $(VALGRIND_TEST_CMD) -# endif + $(SHOW)\ + MODULE=$(INSTALLED_TARGET) \ + GEN=$(GEN) \ + AOF=$(AOF) \ + SLAVES=$(SLAVES) \ + $(ROOT)/test/tests.sh ifeq ($(COV),1) $(COVERAGE_COLLECT_REPORT) endif @@ -234,11 +193,6 @@ docker: $(SHOW)docker build -t redisai --build-arg TEST=1 --build-arg PACK=1 .. #---------------------------------------------------------------------------------------------- -# Currently only testing for leaks using TF - -MODULE_ARGS=\ - BACKENDSPATH $(realpath $(BINDIR)) \ - TF redisai_tensorflow.so VALGRIND_ARGS=\ $(VALGRIND_OPTIONS) \ @@ -246,7 +200,7 @@ VALGRIND_ARGS=\ -v redis-server --protected-mode no --save "" --appendonly no valgrind: $(TARGET) - $(SHOW)valgrind $(VALGRIND_ARGS) --loadmodule $(realpath $(TARGET)) $(MODULE_ARGS) + $(SHOW)valgrind $(VALGRIND_ARGS) --loadmodule $(realpath $(INSTALLED_TARGET)) CALLGRIND_ARGS=\ --tool=callgrind \ @@ -258,4 +212,4 @@ CALLGRIND_ARGS=\ -v redis-server --protected-mode no --save "" --appendonly no callgrind: $(TARGET) - $(SHOW)valgrind $(CALLGRIND_ARGS) --loadmodule $(realpath $(TARGET)) $(MODULE_ARGS) + $(SHOW)valgrind $(CALLGRIND_ARGS) --loadmodule $(realpath $(INSTALLED_TARGET)) diff --git a/opt/readies/mk/cmake.defs b/opt/readies/mk/cmake.defs index ab8a8741a..7ad0213f7 100755 --- a/opt/readies/mk/cmake.defs +++ b/opt/readies/mk/cmake.defs @@ -9,7 +9,7 @@ endif ifeq ($(WHY),1) CMAKE_WHY=--trace-expand -CMAKE_SINK=>/tmp/cmake.log +CMAKE_SINK= >/tmp/cmake.why 2>&1 endif endif # MK.cmake diff --git a/opt/readies/mk/coverage.rules b/opt/readies/mk/coverage.rules index 342109af0..6f3ee0671 100644 --- a/opt/readies/mk/coverage.rules +++ b/opt/readies/mk/coverage.rules @@ -2,6 +2,10 @@ ifeq ($(COV),1) cov-upload: +ifneq ($(CODECOV_TOKEN),) $(SHOW)bash <(curl -s https://codecov.io/bash) -f $(COV_INFO) +else + @echo "To upload to codecov.io, please define CODECOV_TOKEN for this module." +endif endif # COV diff --git a/test/basic_tests.py b/test/basic_tests.py deleted file mode 100644 index b8f4be3e6..000000000 --- a/test/basic_tests.py +++ /dev/null @@ -1,1158 +0,0 @@ -from multiprocessing import Pool, Process -import redis - -import numpy as np -from skimage.io import imread -from skimage.transform import resize -import random -import time -import json -import os -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../opt/readies")) -import paella - -TEST_TF = os.environ.get("TEST_TF") != "0" and os.environ.get("WITH_TF") != "0" -TEST_TFLITE = os.environ.get("TEST_TFLITE") != "0" and os.environ.get("WITH_TFLITE") != "0" -TEST_PT = os.environ.get("TEST_PT") != "0" and os.environ.get("WITH_PT") != "0" -TEST_ONNX = os.environ.get("TEST_ONNX") != "0" and os.environ.get("WITH_ORT") != "0" - - -''' -python -m RLTest --test basic_tests.py --module path/to/redisai.so -''' - -DEVICE = os.environ.get('DEVICE', 'CPU').upper() -print(f"Running tests on {DEVICE}\n") - - -def check_cuda(): - return os.system('which nvcc') - - -def info_to_dict(info): - info = [el.decode('ascii') if type(el) is bytes else el for el in info] - return dict(zip(info[::2], info[1::2])) - - -def run_test_multiproc(env, n_procs, fn, args=tuple()): - procs = [] - - def tmpfn(): - con = env.getConnection() - fn(con, *args) - return 1 - - for _ in range(n_procs): - p = Process(target=tmpfn) - p.start() - procs.append(p) - - [p.join() for p in procs] - - -def example_multiproc_fn(env): - env.execute_command('set', 'x', 1) - - -def test_example_multiproc(env): - run_test_multiproc(env, 10, lambda x: x.execute_command('set', 'x', 1)) - r = env.cmd('get', 'x') - env.assertEqual(r, b'1') - - -def test_set_tensor(env): - con = env.getConnection() - con.execute_command('AI.TENSORSET', 'x', 'FLOAT', 2, 'VALUES', 2, 3) - tensor = con.execute_command('AI.TENSORGET', 'x', 'VALUES') - values = tensor[-1] - env.assertEqual(values, [b'2', b'3']) - con.execute_command('AI.TENSORSET', 'x', 'INT32', 2, 'VALUES', 2, 3) - tensor = con.execute_command('AI.TENSORGET', 'x', 'VALUES') - values = tensor[-1] - env.assertEqual(values, [2, 3]) - - # ERR unsupported data format - try: - con.execute_command('AI.TENSORSET', 'z', 'INT32', 2, 'unsupported', 2, 3) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual(exception.__str__(), "invalid argument found in tensor shape") - - # ERR invalid value - try: - con.execute_command('AI.TENSORSET', 'z', 'FLOAT', 2, 'VALUES', 2, 'A') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual(exception.__str__(), "invalid value") - - # ERR invalid value - try: - con.execute_command('AI.TENSORSET', 'z', 'INT32', 2, 'VALUES', 2, 'A') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual(exception.__str__(), "invalid value") - - try: - con.execute_command('AI.TENSORSET', 1) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.TENSORSET', 'y', 'FLOAT') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.TENSORSET', 'y', 'FLOAT', '2') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.TENSORSET', 'y', 'FLOAT', 2, 'VALUES') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.TENSORSET', 'y', 'FLOAT', 2, 'VALUES', 1) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.TENSORSET', 'y', 'FLOAT', 2, 'VALUES', '1') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - time.sleep(0.1) - - for _ in env.reloadingIterator(): - env.assertExists('x') - - -def test_get_tensor(env): - con = env.getConnection() - con.execute_command('AI.TENSORSET', 't_FLOAT', 'FLOAT', 2, 'VALUES', 2, 3) - con.execute_command('AI.TENSORSET', 't_INT8', 'INT8', 2, 'VALUES', 1, 1) - con.execute_command('AI.TENSORSET', 't_INT16', 'INT8', 2, 'VALUES', 1, 1) - con.execute_command('AI.TENSORSET', 't_INT32', 'INT8', 2, 'VALUES', 1, 1) - con.execute_command('AI.TENSORSET', 't_INT64', 'INT8', 2, 'VALUES', 1, 1) - - tensor = con.execute_command('AI.TENSORGET', 't_FLOAT', 'BLOB') - values = tensor[-1] - - tensor = con.execute_command('AI.TENSORGET', 't_INT8', 'VALUES') - values = tensor[-1] - env.assertEqual(values, [1,1]) - - tensor = con.execute_command('AI.TENSORGET', 't_INT16', 'VALUES') - values = tensor[-1] - env.assertEqual(values,[1,1]) - - tensor = con.execute_command('AI.TENSORGET', 't_INT32', 'VALUES') - values = tensor[-1] - env.assertEqual(values,[1,1]) - - - tensor = con.execute_command('AI.TENSORGET', 't_INT64', 'VALUES') - values = tensor[-1] - env.assertEqual(values, [1,1]) - - - tensor = con.execute_command('AI.TENSORGET', 't_INT32', 'META') - values = tensor[-1] - env.assertEqual(values, [2]) - - # ERR unsupported data format - try: - con.execute_command('AI.TENSORGET', 't_FLOAT', 'unsupported') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual(exception.__str__(), "unsupported data format") - - -def test_del_tf_model(env): - if not TEST_PT: - return - - con = env.getConnection() - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - model_filename = os.path.join(test_data_path, 'graph.pb') - - with open(model_filename, 'rb') as f: - model_pb = f.read() - - ret = con.execute_command('AI.MODELSET', 'm', 'TF', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) - env.assertEqual(ret, b'OK') - - con.execute_command('AI.MODELDEL', 'm') - env.assertFalse(env.execute_command('EXISTS', 'm')) - - # ERR no model at key - try: - con.execute_command('AI.MODELDEL', 'm') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("no model at key",exception.__str__()) - - # ERR wrong type - try: - con.execute_command('SET', 'NOT_MODEL', 'BAR') - con.execute_command('AI.MODELDEL', 'NOT_MODEL') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value",exception.__str__()) - - -def test_run_tf_model(env): - if not TEST_PT: - return - - con = env.getConnection() - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - model_filename = os.path.join(test_data_path, 'graph.pb') - wrong_model_filename = os.path.join(test_data_path, 'pt-minimal.pt') - - with open(model_filename, 'rb') as f: - model_pb = f.read() - - with open(wrong_model_filename, 'rb') as f: - wrong_model_pb = f.read() - ret = con.execute_command('AI.MODELSET', 'm', 'TF', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) - env.assertEqual(ret, b'OK') - - ret = con.execute_command('AI.MODELGET', 'm') - env.assertEqual(len(ret), 3) - # TODO: enable me - # env.assertEqual(ret[0], b'TF') - # env.assertEqual(ret[1], b'CPU') - - # ERR WrongArity - try: - con.execute_command('AI.MODELGET') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("wrong number of arguments for 'AI.MODELGET' command", exception.__str__() ) - - # ERR WRONGTYPE - con.execute_command('SET', 'NOT_MODEL', 'BAR') - try: - con.execute_command('AI.MODELGET', 'NOT_MODEL') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value", exception.__str__()) - # cleanup - con.execute_command('DEL', 'NOT_MODEL') - - # ERR cannot get model from empty key - con.execute_command('DEL', 'DONT_EXIST') - try: - con.execute_command('AI.MODELGET', 'DONT_EXIST') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("cannot get model from empty key", exception.__str__()) - - try: - ret = con.execute_command('AI.MODELSET', 'm', 'TF', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', wrong_model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_1', 'TF', - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_2', 'PORCH', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_3', 'TORCH', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_4', 'TF', - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_5', 'TF', DEVICE, - 'INPUTS', 'a', 'b', 'c', 'OUTPUTS', 'mul', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_6', 'TF', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mult', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_7', 'TF', DEVICE, model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_8', 'TF', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_8', 'TF', DEVICE, - 'INPUTS', 'a_', 'b', 'OUTPUTS', 'mul') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_8', 'TF', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS', 'mul_') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - # ERR Invalid GraphDef - try: - con.execute_command('AI.MODELSET', 'm_8', 'TF', DEVICE, - 'INPUTS', 'a', 'b', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual(exception.__str__(), "Invalid GraphDef") - - try: - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - - - con.execute_command('AI.TENSORSET', 'a', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) - con.execute_command('AI.TENSORSET', 'b', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) - - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') - - info = con.execute_command('AI.INFO', 'm') - info_dict_0 = info_to_dict(info) - - env.assertEqual(info_dict_0['KEY'], 'm') - env.assertEqual(info_dict_0['TYPE'], 'MODEL') - env.assertEqual(info_dict_0['BACKEND'], 'TF') - env.assertTrue(info_dict_0['DURATION'] > 0) - env.assertEqual(info_dict_0['SAMPLES'], 2) - env.assertEqual(info_dict_0['CALLS'], 1) - env.assertEqual(info_dict_0['ERRORS'], 0) - - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') - - info = con.execute_command('AI.INFO', 'm') - info_dict_1 = info_to_dict(info) - - env.assertTrue(info_dict_1['DURATION'] > info_dict_0['DURATION']) - env.assertEqual(info_dict_1['SAMPLES'], 4) - env.assertEqual(info_dict_1['CALLS'], 2) - env.assertEqual(info_dict_1['ERRORS'], 0) - - ret = con.execute_command('AI.INFO', 'm', 'RESETSTAT') - env.assertEqual(ret, b'OK') - - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') - info = con.execute_command('AI.INFO', 'm') - info_dict_2 = info_to_dict(info) - - env.assertTrue(info_dict_2['DURATION'] < info_dict_1['DURATION']) - env.assertEqual(info_dict_2['SAMPLES'], 2) - env.assertEqual(info_dict_2['CALLS'], 1) - env.assertEqual(info_dict_2['ERRORS'], 0) - - tensor = con.execute_command('AI.TENSORGET', 'c', 'VALUES') - values = tensor[-1] - env.assertEqual(values, [b'4', b'9', b'4', b'9']) - - if env.useSlaves: - con2 = env.getSlaveConnection() - time.sleep(0.1) - tensor2 = con2.execute_command('AI.TENSORGET', 'c', 'VALUES') - env.assertEqual(tensor2, tensor) - - for _ in env.reloadingIterator(): - env.assertExists('m') - env.assertExists('a') - env.assertExists('b') - env.assertExists('c') - - con.execute_command('AI.MODELDEL', 'm') - env.assertFalse(env.execute_command('EXISTS', 'm')) - - -def test_run_torch_model(env): - if not TEST_PT: - return - - con = env.getConnection() - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - model_filename = os.path.join(test_data_path, 'pt-minimal.pt') - wrong_model_filename = os.path.join(test_data_path, 'graph.pb') - - with open(model_filename, 'rb') as f: - model_pb = f.read() - - with open(wrong_model_filename, 'rb') as f: - wrong_model_pb = f.read() - - ret = con.execute_command('AI.MODELSET', 'm', 'TORCH', DEVICE, model_pb) - env.assertEqual(ret, b'OK') - - ret = con.execute_command('AI.MODELGET', 'm') - # TODO: enable me - # env.assertEqual(ret[0], b'TORCH') - # env.assertEqual(ret[1], b'CPU') - - try: - con.execute_command('AI.MODELSET', 'm', 'TORCH', DEVICE, wrong_model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_1', 'TORCH', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_2', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - env.execute_command('AI.TENSORSET', 'a', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) - env.execute_command('AI.TENSORSET', 'b', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) - - try: - con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'a', 'b', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_2', 'INPUTS', 'a', 'b', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_3', 'a', 'b', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_1', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c', 'd') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') - - tensor = con.execute_command('AI.TENSORGET', 'c', 'VALUES') - values = tensor[-1] - env.assertEqual(values, [b'4', b'6', b'4', b'6']) - - if env.useSlaves: - con2 = env.getSlaveConnection() - time.sleep(0.1) - tensor2 = con2.execute_command('AI.TENSORGET', 'c', 'VALUES') - env.assertEqual(tensor2, tensor) - - for _ in env.reloadingIterator(): - env.assertExists('m') - env.assertExists('a') - env.assertExists('b') - env.assertExists('c') - - -def test_run_onnx_model(env): - if not TEST_ONNX: - return - - con = env.getConnection() - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - model_filename = os.path.join(test_data_path, 'mnist.onnx') - wrong_model_filename = os.path.join(test_data_path, 'graph.pb') - sample_filename = os.path.join(test_data_path, 'one.raw') - - with open(model_filename, 'rb') as f: - model_pb = f.read() - - with open(wrong_model_filename, 'rb') as f: - wrong_model_pb = f.read() - - with open(sample_filename, 'rb') as f: - sample_raw = f.read() - - ret = con.execute_command('AI.MODELSET', 'm', 'ONNX', DEVICE, model_pb) - env.assertEqual(ret, b'OK') - - ret = con.execute_command('AI.MODELGET', 'm') - env.assertEqual(len(ret), 3) - # TODO: enable me - # env.assertEqual(ret[0], b'ONNX') - # env.assertEqual(ret[1], b'CPU') - - try: - con.execute_command('AI.MODELSET', 'm', 'ONNX', DEVICE, wrong_model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_1', 'ONNX', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_2', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - con.execute_command('AI.TENSORSET', 'a', 'FLOAT', 1, 1, 28, 28, 'BLOB', sample_raw) - - try: - con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'a', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_2', 'INPUTS', 'a', 'b', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_3', 'a', 'b', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_1', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_1', 'INPUTS', 'a', 'OUTPUTS', 'b') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'OUTPUTS', 'b') - - tensor = con.execute_command('AI.TENSORGET', 'b', 'VALUES') - values = tensor[-1] - argmax = max(range(len(values)), key=lambda i: values[i]) - - env.assertEqual(argmax, 1) - - if env.useSlaves: - con2 = env.getSlaveConnection() - time.sleep(0.1) - tensor2 = con2.execute_command('AI.TENSORGET', 'b', 'VALUES') - env.assertEqual(tensor2, tensor) - - for _ in env.reloadingIterator(): - env.assertExists('m') - env.assertExists('a') - env.assertExists('b') - - -def test_run_onnxml_model(env): - if not TEST_ONNX: - return - - con = env.getConnection() - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - linear_model_filename = os.path.join(test_data_path, 'linear_iris.onnx') - logreg_model_filename = os.path.join(test_data_path, 'logreg_iris.onnx') - - with open(linear_model_filename, 'rb') as f: - linear_model = f.read() - - with open(logreg_model_filename, 'rb') as f: - logreg_model = f.read() - - ret = con.execute_command('AI.MODELSET', 'linear', 'ONNX', DEVICE, linear_model) - env.assertEqual(ret, b'OK') - - ret = con.execute_command('AI.MODELSET', 'logreg', 'ONNX', DEVICE, logreg_model) - env.assertEqual(ret, b'OK') - - con.execute_command('AI.TENSORSET', 'features', 'FLOAT', 1, 4, 'VALUES', 5.1, 3.5, 1.4, 0.2) - - con.execute_command('AI.MODELRUN', 'linear', 'INPUTS', 'features', 'OUTPUTS', 'linear_out') - con.execute_command('AI.MODELRUN', 'logreg', 'INPUTS', 'features', 'OUTPUTS', 'logreg_out', 'logreg_probs') - - linear_out = con.execute_command('AI.TENSORGET', 'linear_out', 'VALUES') - logreg_out = con.execute_command('AI.TENSORGET', 'logreg_out', 'VALUES') - - env.assertEqual(float(linear_out[2][0]), -0.090524077415466309) - env.assertEqual(logreg_out[2][0], 0) - - if env.useSlaves: - con2 = env.getSlaveConnection() - time.sleep(0.1) - linear_out2 = con2.execute_command('AI.TENSORGET', 'linear_out', 'VALUES') - logreg_out2 = con2.execute_command('AI.TENSORGET', 'logreg_out', 'VALUES') - env.assertEqual(linear_out, linear_out2) - env.assertEqual(logreg_out, logreg_out2) - - for _ in env.reloadingIterator(): - env.assertExists('linear') - env.assertExists('logreg') - - -def test_run_tflite_model(env): - if not TEST_TFLITE: - return - - con = env.getConnection() - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - model_filename = os.path.join(test_data_path, 'mnist_model_quant.tflite') - wrong_model_filename = os.path.join(test_data_path, 'graph.pb') - sample_filename = os.path.join(test_data_path, 'one.raw') - - with open(model_filename, 'rb') as f: - model_pb = f.read() - - with open(model_filename, 'rb') as f: - model_pb2 = f.read() - - with open(wrong_model_filename, 'rb') as f: - wrong_model_pb = f.read() - - with open(sample_filename, 'rb') as f: - sample_raw = f.read() - - ret = con.execute_command('AI.MODELSET', 'm', 'TFLITE', 'CPU', model_pb) - env.assertEqual(ret, b'OK') - - ret = con.execute_command('AI.MODELGET', 'm') - env.assertEqual(len(ret), 3) - # TODO: enable me - # env.assertEqual(ret[0], b'TFLITE') - # env.assertEqual(ret[1], b'CPU') - - # try: - # con.execute_command('AI.MODELSET', 'm_1', 'TFLITE', 'CPU', wrong_model_pb) - # except Exception as e: - # exception = e - # env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELSET', 'm_1', 'TFLITE', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - ret = con.execute_command('AI.MODELSET', 'm_2', 'TFLITE', 'CPU', model_pb2) - - try: - con.execute_command('AI.MODELSET', 'm_2', model_pb) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - con.execute_command('AI.TENSORSET', 'a', 'FLOAT', 1, 1, 28, 28, 'BLOB', sample_raw) - - try: - con.execute_command('AI.MODELRUN', 'm_2', 'INPUTS', 'a', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_2', 'INPUTS', 'a', 'b', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_2', 'a', 'b', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm_2', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'b') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'OUTPUTS', 'b') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - con.execute_command('AI.MODELRUN', 'm', 'INPUTS', 'a', 'OUTPUTS', 'b', 'c') - - tensor = con.execute_command('AI.TENSORGET', 'b', 'VALUES') - value = tensor[-1][0] - - env.assertEqual(value, 1) - - for _ in env.reloadingIterator(): - env.assertExists('m') - env.assertExists('a') - env.assertExists('b') - env.assertExists('c') - - -def test_set_tensor_multiproc(env): - run_test_multiproc(env, 10, - lambda env: env.execute_command('AI.TENSORSET', 'x', 'FLOAT', 2, 'VALUES', 2, 3)) - - con = env.getConnection() - - tensor = con.execute_command('AI.TENSORGET', 'x', 'VALUES') - values = tensor[-1] - env.assertEqual(values, [b'2', b'3']) - - -def load_mobilenet_test_data(): - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - labels_filename = os.path.join(test_data_path, 'imagenet_class_index.json') - image_filename = os.path.join(test_data_path, 'panda.jpg') - model_filename = os.path.join(test_data_path, 'mobilenet_v2_1.4_224_frozen.pb') - - with open(model_filename, 'rb') as f: - model_pb = f.read() - - with open(labels_filename, 'r') as f: - labels = json.load(f) - - img_height, img_width = 224, 224 - - img = imread(image_filename) - img = resize(img, (img_height, img_width), mode='constant', anti_aliasing=True) - img = img.astype(np.float32) - - return model_pb, labels, img - - -def test_run_mobilenet(env): - if not TEST_TF: - return - - con = env.getConnection() - - input_var = 'input' - output_var = 'MobilenetV2/Predictions/Reshape_1' - - model_pb, labels, img = load_mobilenet_test_data() - - con.execute_command('AI.MODELSET', 'mobilenet', 'TF', DEVICE, - 'INPUTS', input_var, 'OUTPUTS', output_var, model_pb) - - con.execute_command('AI.TENSORSET', 'input', - 'FLOAT', 1, img.shape[1], img.shape[0], img.shape[2], - 'BLOB', img.tobytes()) - - con.execute_command('AI.MODELRUN', 'mobilenet', - 'INPUTS', 'input', 'OUTPUTS', 'output') - - dtype, shape, data = con.execute_command('AI.TENSORGET', 'output', 'BLOB') - - dtype_map = {b'FLOAT': np.float32} - tensor = np.frombuffer(data, dtype=dtype_map[dtype]).reshape(shape) - label_id = np.argmax(tensor) - 1 - - _, label = labels[str(label_id)] - - env.assertEqual(label, 'giant_panda') - - -def run_mobilenet(con, img, input_var, output_var): - time.sleep(0.5 * random.randint(0, 10)) - con.execute_command('AI.TENSORSET', 'input', - 'FLOAT', 1, img.shape[1], img.shape[0], img.shape[2], - 'BLOB', img.tobytes()) - - con.execute_command('AI.MODELRUN', 'mobilenet', - 'INPUTS', 'input', 'OUTPUTS', 'output') - - # env.execute_command('DEL', 'input') - - -def test_run_mobilenet_multiproc(env): - if not TEST_TF: - return - - con = env.getConnection() - - input_var = 'input' - output_var = 'MobilenetV2/Predictions/Reshape_1' - - model_pb, labels, img = load_mobilenet_test_data() - con.execute_command('AI.MODELSET', 'mobilenet', 'TF', DEVICE, - 'INPUTS', input_var, 'OUTPUTS', output_var, model_pb) - - run_test_multiproc(env, 30, run_mobilenet, (img, input_var, output_var)) - - dtype, shape, data = con.execute_command('AI.TENSORGET', 'output', 'BLOB') - - dtype_map = {b'FLOAT': np.float32} - tensor = np.frombuffer(data, dtype=dtype_map[dtype]).reshape(shape) - label_id = np.argmax(tensor) - 1 - - _, label = labels[str(label_id)] - - env.assertEqual( - label, 'giant_panda' - ) - - #@@@ possible workaround for side-effect test failure - # env.restartAndReload() - -def test_set_script(env): - if not TEST_PT: - return - - con = env.getConnection() - - try: - con.execute_command('AI.SCRIPTSET', 'ket', DEVICE, 'return 1') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.SCRIPTSET', 'nope') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.SCRIPTSET', 'more', DEVICE) - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - script_filename = os.path.join(test_data_path, 'script.txt') - - with open(script_filename, 'rb') as f: - script = f.read() - - con.execute_command('AI.SCRIPTSET', 'ket', DEVICE, script) - - for _ in env.reloadingIterator(): - env.assertExists('ket') - - - - -def test_del_script(env): - if not TEST_PT: - return - - con = env.getConnection() - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - script_filename = os.path.join(test_data_path, 'script.txt') - - with open(script_filename, 'rb') as f: - script = f.read() - - ret = con.execute_command('AI.SCRIPTSET', 'ket', DEVICE, script) - env.assertEqual(ret, b'OK') - - ret = con.execute_command('AI.SCRIPTDEL', 'ket') - env.assertFalse(con.execute_command('EXISTS', 'ket')) - - # ERR no script at key from SCRIPTDEL - try: - con.execute_command('DEL', 'EMPTY') - con.execute_command('AI.SCRIPTDEL', 'EMPTY') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("no script at key", exception.__str__()) - - # ERR wrong type from SCRIPTDEL - try: - con.execute_command('SET', 'NOT_SCRIPT', 'BAR') - con.execute_command('AI.SCRIPTDEL', 'NOT_SCRIPT') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value", exception.__str__()) - - -def test_run_script(env): - if not TEST_PT: - return - - con = env.getConnection() - - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - script_filename = os.path.join(test_data_path, 'script.txt') - - with open(script_filename, 'rb') as f: - script = f.read() - - ret = con.execute_command('AI.SCRIPTSET', 'ket', DEVICE, script) - env.assertEqual(ret, b'OK') - - ret = con.execute_command('AI.TENSORSET', 'a', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) - env.assertEqual(ret, b'OK') - ret = con.execute_command('AI.TENSORSET', 'b', 'FLOAT', 2, 2, 'VALUES', 2, 3, 2, 3) - env.assertEqual(ret, b'OK') - - # TODO: enable me ( this is hanging CI ) - # ret = con.execute_command('AI.SCRIPTGET', 'ket') - # TODO: enable me - # env.assertEqual([b'CPU',script],ret) - - # ERR no script at key from SCRIPTGET - try: - con.execute_command('DEL', 'EMPTY') - con.execute_command('AI.SCRIPTGET', 'EMPTY') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("cannot get script from empty key", exception.__str__()) - - # ERR wrong type from SCRIPTGET - try: - con.execute_command('SET', 'NOT_SCRIPT', 'BAR') - con.execute_command('AI.SCRIPTGET', 'NOT_SCRIPT') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value", exception.__str__()) - - # ERR no script at key from SCRIPTRUN - try: - con.execute_command('DEL', 'EMPTY') - con.execute_command('AI.SCRIPTRUN', 'EMPTY', 'bar', 'INPUTS', 'b', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("script key is empty", exception.__str__()) - - # ERR wrong type from SCRIPTRUN - try: - con.execute_command('SET', 'NOT_SCRIPT', 'BAR') - con.execute_command('AI.SCRIPTRUN', 'NOT_SCRIPT', 'bar', 'INPUTS', 'b', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value", exception.__str__()) - - # ERR Input key is empty - try: - con.execute_command('DEL', 'EMPTY') - con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'EMPTY', 'b', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("Input key is empty", exception.__str__()) - - # ERR Input key not tensor - try: - con.execute_command('SET', 'NOT_TENSOR', 'BAR') - con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'NOT_TENSOR', 'b', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - env.assertEqual("WRONGTYPE Operation against a key holding the wrong kind of value", exception.__str__()) - - try: - con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'b', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.SCRIPTRUN', 'ket', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'b', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - try: - con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'OUTPUTS') - except Exception as e: - exception = e - env.assertEqual(type(exception), redis.exceptions.ResponseError) - - con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') - - info = con.execute_command('AI.INFO', 'ket') - info_dict_0 = info_to_dict(info) - - env.assertEqual(info_dict_0['KEY'], 'ket') - env.assertEqual(info_dict_0['TYPE'], 'SCRIPT') - env.assertEqual(info_dict_0['BACKEND'], 'TORCH') - env.assertTrue(info_dict_0['DURATION'] > 0) - env.assertEqual(info_dict_0['SAMPLES'], -1) - env.assertEqual(info_dict_0['CALLS'], 4) - env.assertEqual(info_dict_0['ERRORS'], 3) - - con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') - - info = con.execute_command('AI.INFO', 'ket') - info_dict_1 = info_to_dict(info) - - env.assertTrue(info_dict_1['DURATION'] > info_dict_0['DURATION']) - env.assertEqual(info_dict_1['SAMPLES'], -1) - env.assertEqual(info_dict_1['CALLS'], 5) - env.assertEqual(info_dict_1['ERRORS'], 3) - - ret = con.execute_command('AI.INFO', 'ket', 'RESETSTAT') - env.assertEqual(ret, b'OK') - - con.execute_command('AI.SCRIPTRUN', 'ket', 'bar', 'INPUTS', 'a', 'b', 'OUTPUTS', 'c') - - info = con.execute_command('AI.INFO', 'ket') - info_dict_2 = info_to_dict(info) - - env.assertTrue(info_dict_2['DURATION'] < info_dict_1['DURATION']) - env.assertEqual(info_dict_2['SAMPLES'], -1) - env.assertEqual(info_dict_2['CALLS'], 1) - env.assertEqual(info_dict_2['ERRORS'], 0) - - tensor = con.execute_command('AI.TENSORGET', 'c', 'VALUES') - values = tensor[-1] - env.assertEqual(values, [b'4', b'6', b'4', b'6']) - - time.sleep(0.1) - - if env.useSlaves: - con2 = env.getSlaveConnection() - time.sleep(0.1) - tensor2 = con2.execute_command('AI.TENSORGET', 'c', 'VALUES') - env.assertEqual(tensor2, tensor) - - for _ in env.reloadingIterator(): - env.assertExists('ket') - env.assertExists('a') - env.assertExists('b') - env.assertExists('c') diff --git a/test/double-panda.py b/test/double-panda.py deleted file mode 100755 index ab040da8f..000000000 --- a/test/double-panda.py +++ /dev/null @@ -1,94 +0,0 @@ -from RLTest import Env - -from multiprocessing import Pool, Process -import redis - -import numpy as np -from skimage.io import imread -from skimage.transform import resize -import random -import time -import json -import os -import sys - - -def run_test_multiproc(env, n_procs, fn, args=tuple()): - procs = [] - - def tmpfn(): - e = env.getConnection() - fn(e, *args) - return 1 - - for _ in range(n_procs): - p = Process(target=tmpfn) - p.start() - procs.append(p) - - [p.join() for p in procs] - - -def load_mobilenet_test_data(): - test_data_path = os.path.join(os.path.dirname(__file__), 'test_data') - labels_filename = os.path.join(test_data_path, 'imagenet_class_index.json') - image_filename = os.path.join(test_data_path, 'panda.jpg') - model_filename = os.path.join(test_data_path, 'mobilenet_v2_1.4_224_frozen.pb') - - with open(model_filename, 'rb') as f: - model_pb = f.read() - - with open(labels_filename, 'r') as f: - labels = json.load(f) - - img_height, img_width = 224, 224 - - img = imread(image_filename) - img = resize(img, (img_height, img_width), mode='constant', anti_aliasing=True) - img = img.astype(np.float32) - #@@@ this one instead of the above will not blow up, but the test will obviously fail: - # img = np.zeros([224, 224, 3], dtype=np.float32) - - return model_pb, labels, img - - -def test_1_run_mobilenet_multiproc(env): - input_var = 'input' - output_var = 'MobilenetV2/Predictions/Reshape_1' - - con = env - - model_pb, labels, img = load_mobilenet_test_data() - con.execute_command('AI.MODELSET', 'mobilenet', 'TF', 'CPU', - 'INPUTS', input_var, 'OUTPUTS', output_var, model_pb) - - run_test_multiproc(env, 30, run_mobilenet, (img, input_var, output_var)) - - dtype, shape, data = con.execute_command('AI.TENSORGET', 'output', 'BLOB') - - dtype_map = {b'FLOAT': np.float32} - tensor = np.frombuffer(data, dtype=dtype_map[dtype]).reshape(shape) - label_id = np.argmax(tensor) - 1 - - _, label = labels[str(label_id)] - - env.assertEqual( - label, 'giant_panda' - ) - - #@@@ this one also works as a workaround: - # env.restartAndReload() - - -def run_mobilenet(con, img, input_var, output_var): - time.sleep(0.5 * random.randint(0, 10)) - con.execute_command('AI.TENSORSET', 'input', - 'FLOAT', 1, img.shape[1], img.shape[0], img.shape[2], - 'BLOB', img.tobytes()) - - con.execute_command('AI.MODELRUN', 'mobilenet', 'INPUTS', 'input', 'OUTPUTS', 'output') - # con.execute_command('DEL', 'input') - - -def test_2_run_mobilenet_multiproc(env): - test_1_run_mobilenet_multiproc(env) diff --git a/test/tests.sh b/test/tests.sh new file mode 100755 index 000000000..4a78afa02 --- /dev/null +++ b/test/tests.sh @@ -0,0 +1,125 @@ +#!/bin/bash + +[[ $VERBOSE == 1 ]] && set -x +[[ $IGNERR == 1 ]] || set -e + +error() { + echo "There are errors." + exit 1 +} + +trap error ERR + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +. $HERE/../opt/readies/shibumi/functions + +#---------------------------------------------------------------------------------------------- + +help() { + cat <<-END + Run Python tests. + + [ARGVARS...] tests.sh [--help|help] [] + + Argument variables: + VERBOSE=1 Print commands + IGNERR=1 Do not abort on error + + DEVICE=cpu|gpu Device for testing + GEN=0|1 General tests + AOF=0|1 Tests with --test-aof + SLAVES=0|1 Tests with --test-slaves + + TEST=test Run specific test (e.g. test.py:test_name) + VALGRIND=1 Run with Valgrind + VGD=1 Same as VALGRIND=1 + CALLGRIND=1 Run with Callgrind + + END +} + +#---------------------------------------------------------------------------------------------- + +install_git_lfs() { + [[ $NO_LFS == 1 ]] && return + [[ $(git lfs env > /dev/null 2>&1 ; echo $?) != 0 ]] && git lfs install + git lfs pull +} + +#---------------------------------------------------------------------------------------------- + +check_redis_server() { + if ! command -v redis-server > /dev/null; then + echo "Cannot find redis-server. Aborting." + exit 1 + fi +} + +#---------------------------------------------------------------------------------------------- + +valgrind_config() { + VALGRIND_OPTIONS="\ + -q \ + --leak-check=full \ + --show-reachable=no \ + --show-possibly-lost=no" + + VALGRIND_SUPRESSIONS=$ROOT/opt/redis_valgrind.sup + + RLTEST_ARGS+=" \ + --no-output-catch \ + --use-valgrind \ + --vg-no-fail-on-errors \ + --vg-verbose + --vg-options \"$VALGRIND_OPTIONS\" \ + --vg-suppressions $VALGRIND_SUPRESSIONS" +} + +#---------------------------------------------------------------------------------------------- + +run_tests() { + local title="$1" + [[ ! -z $title ]] && { $ROOT/opt/readies/bin/sep0; printf "Tests with $title:\n\n"; } + $OP python3 -m RLTest --clear-logs --module $MODULE $RLTEST_ARGS +} + +#---------------------------------------------------------------------------------------------- + +DEVICE=${DEVICE:-cpu} + +GEN=${GEN:-1} +SLAVES=${SLAVES:-0} +AOF=${AOF:-0} + +GDB=${GDB:-0} + +OP="" +[[ $NOP == 1 ]] && OP="echo" + +MODULE=${MODULE:-$1} +[[ -z $MODULE || ! -f $MODULE ]] && { echo "Module not found. Aborting."; exit 1; } + +RLTEST_ARGS="" + +[[ $VALGRIND == 1 || $VGD == 1 ]] && valgrind_config + +if [[ ! -z $TEST ]]; then + RLTEST_ARGS+=" -s --test $TEST" + export PYDEBUG=${PYDEBUG:-1} +fi + +[[ $VERBOSE == 1 ]] && RLTEST_ARGS+=" -v" + +#---------------------------------------------------------------------------------------------- + +[[ $1 == --help || $1 == help ]] && { help; exit 0; } + +ROOT=$(realpath $HERE/..) +cd $ROOT/test + +install_git_lfs +check_redis_server + +[[ $GEN == 1 ]] && run_tests +[[ $SLAVES == 1 ]] && RLTEST_ARGS+=" --use-slaves" run_tests "--use-slaves" +[[ $AOF == 1 ]] && RLTEST_ARGS+=" --use-aof" run_tests "--use-aof" From 13bbba7792cffca0780ceace9d6a6bbf4ca3041b Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 25 Feb 2020 22:02:52 +0200 Subject: [PATCH 06/18] fixes 4 --- .circleci/config.yml | 43 +++++++++++++++++++++++++++++++++++++- README.md | 1 + opt/Makefile | 21 ++++++++++--------- opt/readies/mk/cmake.rules | 3 ++- test/tests.sh | 7 +++---- 5 files changed, 59 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 25209f9c5..c4ec23723 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,7 +24,7 @@ commands: - save_cache: paths: - deps - key: v1-dependencies-{{ checksum "get_deps.sh" }} + key: build-dependencies-{{ checksum "get_deps.sh" }} - run: name: Set up workspace command: | @@ -68,6 +68,36 @@ jobs: steps: - ci_steps: platform: debian + + coverage: + docker: + - image: redislabsmodules/rmbuilder:latest + steps: + - checkout + - run: + name: Install dependencies + command: | + ./opt/readies/bin/getpy3 + BREW_NO_UPDATE=1 ./opt/system-setup.py + git clone git://github.com/antirez/redis.git --branch 5.0.7 + (cd redis && make malloc=libc -j $(nproc) && make install) + redis-server --version + - restore_cache: + keys: + - build-dependencies-{{ checksum "get_deps.sh" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- + - run: + name: Build + command: | + # make fetch SHOW=1 + make -C opt build COV=1 SHOW=1 + - run: + name: Test with coverage + command: | + make -C opt test SHOW=1 COV=1 + make cov-upload COV=1 + build-macos: macos: xcode: 10.2.1 @@ -78,6 +108,7 @@ jobs: brew reinstall -f python2 - ci_steps: platform: macosx + build-multiarch-docker: machine: enabled: true @@ -102,6 +133,7 @@ jobs: sudo docker login -u redisfab -p $DOCKER_REDISFAB_PWD make -C opt/build/docker build sudo make -C opt/build/docker publish + build-gpu: machine: enabled: true @@ -129,6 +161,7 @@ jobs: no_output_timeout: 30m - store_test_results: path: ~/workspace/tests + deploy_package: parameters: package: @@ -164,6 +197,14 @@ workflows: filters: tags: only: /.*/ + - coverage: + requires: + - build + filters: + branches: + only: /.*/ + tags: + only: /.*/ - build-macos: filters: branches: diff --git a/README.md b/README.md index 8c59b051e..d47c9f75d 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/redisai/redisai.svg)](https://hub.docker.com/r/redisai/redisai/builds/) [![Mailing List](https://img.shields.io/badge/Mailing%20List-RedisAI-blue)](https://groups.google.com/forum/#!forum/redisai) [![Gitter](https://badges.gitter.im/RedisLabs/RedisAI.svg)](https://gitter.im/RedisLabs/RedisAI?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +[![codecov](https://codecov.io/gh/RedisAI/RedisAI/branch/master/graph/badge.svg)](https://codecov.io/gh/RedisAI/RedisAI) # RedisAI diff --git a/opt/Makefile b/opt/Makefile index 544258343..d95be0f22 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -2,11 +2,11 @@ ROOT:=.. include readies/mk/main -MK_CMAKE:=1 MK.cmake:=1 -MK_CMAKE_INSTALL:=1 MK_CUSTOM_CLEAN:=1 +MK_ALL_TARGETS=bindirs fetch build pack + #---------------------------------------------------------------------------------------------- define HELP @@ -61,19 +61,20 @@ DEPS_FLAGS=cpu DEVICE=cpu endif +#---------------------------------------------------------------------------------------------- + SRCDIR=.. BINDIR=$(BINROOT)/src DEPS_DIR=$(ROOT)/deps/$(OS)-$(ARCH)-$(DEVICE) -# INSTALL_DIR=$(ROOT)/bin/$(OS)-$(ARCH)-$(DEVICE)/install INSTALL_DIR=$(BINROOT)/install-$(DEVICE) -REDIS_VALGRIND_SUPRESS=$(ROOT)/opt/redis_valgrind.sup - TARGET=$(BINDIR)/redisai.so INSTALLED_TARGET=$(INSTALL_DIR)/redisai.so BACKENDS_PATH ?= $(INSTALL_DIR)/backends +REDIS_VALGRIND_SUPRESS=$(ROOT)/opt/redis_valgrind.sup + #---------------------------------------------------------------------------------------------- CMAKE_FILES += \ @@ -113,7 +114,7 @@ include $(MK)/defs #---------------------------------------------------------------------------------------------- -.PHONY: deps fetch pack pack_ramp pack_deps test +.PHONY: deps prebuild fetch pack pack_ramp pack_deps test include $(MK)/rules @@ -122,7 +123,8 @@ include $(MK)/rules prebuild: $(SHOW)if [ ! -d $(DEPS_DIR) ]; then echo $$'Dependencies are not in place.\nPlease run \'make fetch\'.'; exit 1; fi -$(TARGET): prebuild $(MK_MAKEFILES) $(DEPS) +$(info $(TARGET): $(MK_MAKEFILES) $(DEPS)) +$(TARGET): $(DEPS) $(SHOW)mkdir -p $(INSTALL_DIR) $(SHOW)$(MAKE) -C $(BINDIR) $(SHOW)$(MAKE) -C $(BINDIR) install @@ -178,10 +180,9 @@ ifeq ($(COV),1) $(COVERAGE_RESET) endif $(SHOW)\ + DEVICE=$(DEVICE) \ MODULE=$(INSTALLED_TARGET) \ - GEN=$(GEN) \ - AOF=$(AOF) \ - SLAVES=$(SLAVES) \ + GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \ $(ROOT)/test/tests.sh ifeq ($(COV),1) $(COVERAGE_COLLECT_REPORT) diff --git a/opt/readies/mk/cmake.rules b/opt/readies/mk/cmake.rules index f2a4a73b7..673c24c1e 100755 --- a/opt/readies/mk/cmake.rules +++ b/opt/readies/mk/cmake.rules @@ -3,7 +3,8 @@ ifneq ($(MK.cmake),) MK_CMAKE_FILES ?= $(SRCDIR)/CMakeLists.txt -$(BINDIR)/Makefile : bindirs $(MK_CMAKE_FILES) +# bindirs +$(BINDIR)/Makefile : $(MK_CMAKE_FILES) $(SHOW)if [ ! -d $(BINDIR) ]; then echo "CMake: $(BINDIR) does not exist."; exit 1; fi $(SHOW)cd $(BINDIR); cmake $(CMAKE_WHY) $(CMAKE_FLAGS) $(abspath $(SRCDIR)) $(CMAKE_SINK) diff --git a/test/tests.sh b/test/tests.sh index 4a78afa02..6bfa38f9e 100755 --- a/test/tests.sh +++ b/test/tests.sh @@ -30,10 +30,9 @@ help() { AOF=0|1 Tests with --test-aof SLAVES=0|1 Tests with --test-slaves - TEST=test Run specific test (e.g. test.py:test_name) - VALGRIND=1 Run with Valgrind - VGD=1 Same as VALGRIND=1 - CALLGRIND=1 Run with Callgrind + TEST=test Run specific test (e.g. test.py:test_name) + VALGRIND|VGD=1 Run with Valgrind + CALLGRIND|CGD=1 Run with Callgrind END } From 30dfcf7da48a4cbb636d15150d8e9eaf86478f12 Mon Sep 17 00:00:00 2001 From: rafie Date: Thu, 27 Feb 2020 13:14:51 +0200 Subject: [PATCH 07/18] fixes 5 --- opt/Makefile | 87 ++++++++++++++++--------------------- opt/readies/mk/cmake.defs | 3 ++ opt/readies/mk/cmake.rules | 12 ++--- test/tests.sh | 19 ++++---- test/valgrind.sh | 89 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 146 insertions(+), 64 deletions(-) create mode 100755 test/valgrind.sh diff --git a/opt/Makefile b/opt/Makefile index d95be0f22..82ada57e9 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -3,28 +3,37 @@ ROOT:=.. include readies/mk/main MK.cmake:=1 +MK_CUSTOM_CMAKE_BUILD:=1 MK_CUSTOM_CLEAN:=1 MK_ALL_TARGETS=bindirs fetch build pack +ifeq ($(COV),1) +DEBUG ?= 1 +endif + #---------------------------------------------------------------------------------------------- define HELP make setup # install prerequisited (CAUTION: THIS WILL MODIFY YOUR SYSTEM) make fetch # download and prepare dependant modules -make build # compile and link - DEBUG=1 # build for debugging - VARIANT=name # build variant `name` - WHY=1 # explain CMake decisions (into /tmp/cmake.why) -make clean # remove build artifacts - ALL=1 # remove entire artifacts directory +make build # compile and link + DEBUG=1 # build for debugging + COV=1 # build for coverage analysis (implies DEBUG=1) + VARIANT=name # build variant `name` + WHY=1 # explain CMake decisions (into /tmp/cmake.why) +make clean # remove build artifacts + ALL=1 # remove entire artifacts directory +make install # create ready-to-run scheme (module and engines) make test # run tests TEST=test # run only test `test` with Redis output TEST_ARGS=args # add extra RLTest `args` VERBOSE=1 # verbose tests output COV=1 # perform coverage analysis + VALGRIND|VGD=1 # test with Valgrind (implies DEBUG=1) + CALLGRIND|CGD=1 # test with Callgrind (implies DEBUG=1) make cov-upload # upload coverage data to codecov.io (requires CODECOV_TOKEN) make pack # create installation packages @@ -33,6 +42,7 @@ make pack # create installation packages BRANCH=name # use `name` as branch name make deploy # copy packages to S3 make release # release a version +make docker # build docker image fetch and build options: WITH_TF=0 # SKip TensofFlow @@ -73,8 +83,6 @@ INSTALLED_TARGET=$(INSTALL_DIR)/redisai.so BACKENDS_PATH ?= $(INSTALL_DIR)/backends -REDIS_VALGRIND_SUPRESS=$(ROOT)/opt/redis_valgrind.sup - #---------------------------------------------------------------------------------------------- CMAKE_FILES += \ @@ -114,24 +122,21 @@ include $(MK)/defs #---------------------------------------------------------------------------------------------- -.PHONY: deps prebuild fetch pack pack_ramp pack_deps test +.PHONY: fetch deps pack pack_ramp pack_deps test include $(MK)/rules #---------------------------------------------------------------------------------------------- -prebuild: - $(SHOW)if [ ! -d $(DEPS_DIR) ]; then echo $$'Dependencies are not in place.\nPlease run \'make fetch\'.'; exit 1; fi +#prebuild: +# $(SHOW)if [ ! -d $(DEPS_DIR) ]; then echo $$'Dependencies are not in place.\nPlease run \'make fetch\'.'; exit 1; fi -$(info $(TARGET): $(MK_MAKEFILES) $(DEPS)) -$(TARGET): $(DEPS) - $(SHOW)mkdir -p $(INSTALL_DIR) +cmake-build $(TARGET): $(MK_MAKEFILES) $(SHOW)$(MAKE) -C $(BINDIR) + $(SHOW)mkdir -p $(INSTALL_DIR) $(SHOW)$(MAKE) -C $(BINDIR) install -# $(SHOW)cd $(ROOT) ;\ -# if [ ! -e install ]; then ln -sf install-$(DEVICE) install; fi -install: +install $(INSTALLED_TARGET): $(TARGET) $(SHOW)mkdir -p $(INSTALL_DIR) $(SHOW)$(MAKE) -C $(BINDIR) install @@ -146,13 +151,18 @@ endif #---------------------------------------------------------------------------------------------- +setup: + @echo Setting up system... + $(SHOW)$(ROOT)/opt/readies/bin/getpy3 + $(SHOW)$(ROOT)/opt/system-setup.py + fetch deps: @echo Fetching dependencies... $(SHOW)VERBOSE=$(_SHOW) $(ROOT)/get_deps.sh $(DEPS_FLAGS) #---------------------------------------------------------------------------------------------- -pack: +pack: $(INSTALLED_TARGET) ifneq ($(PACK_DEPS),0) $(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) BRANCH=$(BRANCH) INTO=$(INTO) DEPS=1 ./pack.sh else @@ -165,15 +175,9 @@ export GEN ?= 1 export SLAVES ?= 1 export AOF ?= 1 -define VALGRIND_TEST_CMD -$(TEST_CMD) \ - --no-output-catch \ - --use-valgrind \ - --vg-no-fail-on-errors \ - --vg-verbose - --vg-options "$(VALGRIND_OPTIONS)" \ - --vg-suppressions $(realpath $(REDIS_VALGRIND_SUPRESS)) -endef +ifneq ($(VGD),) +VALGRIND=$(VGD) +endif test: ifeq ($(COV),1) @@ -183,34 +187,19 @@ endif DEVICE=$(DEVICE) \ MODULE=$(INSTALLED_TARGET) \ GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \ + VALGRIND=$(VALGRIND) \ $(ROOT)/test/tests.sh ifeq ($(COV),1) $(COVERAGE_COLLECT_REPORT) endif -#---------------------------------------------------------------------------------------------- +valgrind: + $(SHOW)$(ROOT)/test/valgrind.sh $(realpath $(INSTALLED_TARGET)) -docker: - $(SHOW)docker build -t redisai --build-arg TEST=1 --build-arg PACK=1 .. +callgrind: + $(SHOW)CALLGRIND=1 $(ROOT)/test/valgrind.sh $(realpath $(INSTALLED_TARGET)) #---------------------------------------------------------------------------------------------- -VALGRIND_ARGS=\ - $(VALGRIND_OPTIONS) \ - --suppressions=$(realpath $(REDIS_VALGRIND_SUPRESS)) \ - -v redis-server --protected-mode no --save "" --appendonly no - -valgrind: $(TARGET) - $(SHOW)valgrind $(VALGRIND_ARGS) --loadmodule $(realpath $(INSTALLED_TARGET)) - -CALLGRIND_ARGS=\ - --tool=callgrind \ - --dump-instr=yes \ - --simulate-cache=no \ - --collect-jumps=no \ - --collect-atstart=yes \ - --instr-atstart=yes \ - -v redis-server --protected-mode no --save "" --appendonly no - -callgrind: $(TARGET) - $(SHOW)valgrind $(CALLGRIND_ARGS) --loadmodule $(realpath $(INSTALLED_TARGET)) +docker: + $(SHOW)docker build -t redisai --build-arg TEST=1 --build-arg PACK=1 .. diff --git a/opt/readies/mk/cmake.defs b/opt/readies/mk/cmake.defs index 7ad0213f7..7c1d6d693 100755 --- a/opt/readies/mk/cmake.defs +++ b/opt/readies/mk/cmake.defs @@ -1,7 +1,10 @@ ifneq ($(MK.cmake),) +MK_CMAKE_FILES ?= $(SRCDIR)/CMakeLists.txt + MK_MAKEFILES += $(BINDIR)/Makefile +DEFAULT_TARGETS += cmake-build ifeq ($(DEBUG),1) CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=Debug diff --git a/opt/readies/mk/cmake.rules b/opt/readies/mk/cmake.rules index 673c24c1e..e553c7b63 100755 --- a/opt/readies/mk/cmake.rules +++ b/opt/readies/mk/cmake.rules @@ -1,15 +1,15 @@ ifneq ($(MK.cmake),) -MK_CMAKE_FILES ?= $(SRCDIR)/CMakeLists.txt - -# bindirs $(BINDIR)/Makefile : $(MK_CMAKE_FILES) $(SHOW)if [ ! -d $(BINDIR) ]; then echo "CMake: $(BINDIR) does not exist."; exit 1; fi $(SHOW)cd $(BINDIR); cmake $(CMAKE_WHY) $(CMAKE_FLAGS) $(abspath $(SRCDIR)) $(CMAKE_SINK) -# $(SHOW)cd $(BINDIR); \ -# CMAKE_REL_ROOT=`python -c "import os; print os.path.relpath('$(SRCDIR)', '$$PWD')"` \ -# cmake $(CMAKE_FLAGS) $$CMAKE_REL_ROOT +ifneq ($(MK_CUSTOM_CMAKE_BUILD),1) + +cmake-build $(TARGET): $(MK_MAKEFILES) + $(SHOW)$(MAKE) -C $(BINDIR) + +endif endif # MK.cmake diff --git a/test/tests.sh b/test/tests.sh index 6bfa38f9e..4d265b650 100755 --- a/test/tests.sh +++ b/test/tests.sh @@ -4,15 +4,18 @@ [[ $IGNERR == 1 ]] || set -e error() { - echo "There are errors." + echo "There are errors:" + gawk 'NR>L-4 && NR>> ":""),$0 }' L=$1 $0 exit 1 } -trap error ERR +[[ -z $_Dbg_DEBUGGER_LEVEL ]] && trap 'error $LINENO' ERR HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . $HERE/../opt/readies/shibumi/functions +export ROOT=$(realpath $HERE/..) + #---------------------------------------------------------------------------------------------- help() { @@ -57,7 +60,7 @@ check_redis_server() { #---------------------------------------------------------------------------------------------- valgrind_config() { - VALGRIND_OPTIONS="\ + export VG_OPTIONS=" -q \ --leak-check=full \ --show-reachable=no \ @@ -65,12 +68,11 @@ valgrind_config() { VALGRIND_SUPRESSIONS=$ROOT/opt/redis_valgrind.sup - RLTEST_ARGS+=" \ + RLTEST_ARGS+="\ --no-output-catch \ --use-valgrind \ --vg-no-fail-on-errors \ - --vg-verbose - --vg-options \"$VALGRIND_OPTIONS\" \ + --vg-verbose \ --vg-suppressions $VALGRIND_SUPRESSIONS" } @@ -84,6 +86,8 @@ run_tests() { #---------------------------------------------------------------------------------------------- +[[ $1 == --help || $1 == help ]] && { help; exit 0; } + DEVICE=${DEVICE:-cpu} GEN=${GEN:-1} @@ -111,9 +115,6 @@ fi #---------------------------------------------------------------------------------------------- -[[ $1 == --help || $1 == help ]] && { help; exit 0; } - -ROOT=$(realpath $HERE/..) cd $ROOT/test install_git_lfs diff --git a/test/valgrind.sh b/test/valgrind.sh new file mode 100755 index 000000000..7e205b52e --- /dev/null +++ b/test/valgrind.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +[[ $VERBOSE == 1 ]] && set -x +[[ $IGNERR == 1 ]] || set -e + +error() { + echo "There are errors." + exit 1 +} + +# trap error ERR + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +. $HERE/../opt/readies/shibumi/functions + +export ROOT=$(realpath $HERE/..) + +#---------------------------------------------------------------------------------------------- + +help() { + cat <<-END + Run Valgrind/Callgrind on RedisAI. + + [ARGVARS...] valgrind.sh [--help|help] [] + + Argument variables: + VERBOSE=1 Print commands + IGNERR=1 Do not abort on error + + CALLGRIND|CGD=1 Run with Callgrind + + END +} + +#---------------------------------------------------------------------------------------------- + +check_redis_server() { + if ! command -v redis-server > /dev/null; then + echo "Cannot find redis-server. Aborting." + exit 1 + fi +} + +#---------------------------------------------------------------------------------------------- + +valgrind_config() { + VALGRIND_SUPRESSIONS=$ROOT/opt/redis_valgrind.sup + + if [[ $CALLGRIND == 1 ]]; then + + VALGRIND_OPTIONS+="\ + --tool=callgrind \ + --dump-instr=yes \ + --simulate-cache=no \ + --collect-jumps=no \ + --collect-atstart=yes \ + --instr-atstart=yes \ + --callgrind-out-file=$MODULE.call" + else + VALGRIND_OPTIONS+="\ + -q \ + --leak-check=full \ + --show-reachable=no \ + --show-possibly-lost=no \ + --show-leak-kinds=all" + fi + + VALGRIND_OPTIONS+=" -v" +} + +#---------------------------------------------------------------------------------------------- + +[[ $1 == --help || $1 == help ]] && { help; exit 0; } + +OP="" +[[ $NOP == 1 ]] && OP="echo" + +MODULE=${MODULE:-$1} +[[ -z $MODULE || ! -f $MODULE ]] && { echo "Module not found. Aborting."; exit 1; } + +VALGRIND_OPTIONS="" +valgrind_config + +#---------------------------------------------------------------------------------------------- + +cd $ROOT/test + +check_redis_server +$OP valgrind $(echo "$VALGRIND_OPTIONS") redis-server --protected-mode no --save '' --appendonly no --loadmodule $MODULE From 972c8b1d48f64b7491651ec7754427cb1b4331ef Mon Sep 17 00:00:00 2001 From: rafie Date: Thu, 27 Feb 2020 17:04:08 +0200 Subject: [PATCH 08/18] fixes 6 --- .circleci/config.yml | 6 ++--- opt/Makefile | 12 ++++------ opt/readies/mk/coverage.defs | 43 ++++++++++++++++++++++-------------- opt/readies/paella/setup.py | 16 ++++++++++++-- 4 files changed, 48 insertions(+), 29 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c4ec23723..a88321e33 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -90,13 +90,13 @@ jobs: - run: name: Build command: | - # make fetch SHOW=1 - make -C opt build COV=1 SHOW=1 + # make fetch SHOW=1 + make -C opt build COV=1 SHOW=1 - run: name: Test with coverage command: | make -C opt test SHOW=1 COV=1 - make cov-upload COV=1 + make cov-upload COV=1 build-macos: macos: diff --git a/opt/Makefile b/opt/Makefile index 82ada57e9..1815bfd7f 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -8,6 +8,10 @@ MK_CUSTOM_CLEAN:=1 MK_ALL_TARGETS=bindirs fetch build pack +ifneq ($(VGD),) +VALGRIND=$(VGD) +endif + ifeq ($(COV),1) DEBUG ?= 1 endif @@ -175,23 +179,15 @@ export GEN ?= 1 export SLAVES ?= 1 export AOF ?= 1 -ifneq ($(VGD),) -VALGRIND=$(VGD) -endif - test: -ifeq ($(COV),1) $(COVERAGE_RESET) -endif $(SHOW)\ DEVICE=$(DEVICE) \ MODULE=$(INSTALLED_TARGET) \ GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \ VALGRIND=$(VALGRIND) \ $(ROOT)/test/tests.sh -ifeq ($(COV),1) $(COVERAGE_COLLECT_REPORT) -endif valgrind: $(SHOW)$(ROOT)/test/valgrind.sh $(realpath $(INSTALLED_TARGET)) diff --git a/opt/readies/mk/coverage.defs b/opt/readies/mk/coverage.defs index 2f9983ddc..921fbd3ce 100644 --- a/opt/readies/mk/coverage.defs +++ b/opt/readies/mk/coverage.defs @@ -11,26 +11,37 @@ COV_EXCLUDE += \ '/Applications/*' \ '/usr/*' -COVERAGE_RESET=\ - $(SHOW)set -e; \ - mkdir -p $(COV_DIR) ;\ - lcov --directory $(BINROOT) --base-directory $(SRCDIR) -z > /dev/null 2>&1 - -COVERAGE_COLLECT=\ - $(SHOW)set -e; \ - echo "Collecting coverage data ..." ;\ - lcov --capture --directory $(BINROOT) --base-directory $(SRCDIR) --output-file $(COV_INFO) > /dev/null 2>&1 ;\ - lcov -o $(COV_INFO) -r $(COV_INFO) $(COV_EXCLUDE) > /dev/null 2>&1 - -COVERAGE_REPORT= \ - $(SHOW)set -e; \ - lcov -l $(COV_INFO) ;\ - genhtml --legend -o $(COV_DIR) $(COV_INFO) > /dev/null 2>&1 ;\ - echo "Coverage info at $(realpath $(COV_DIR))/index.html" +define COVERAGE_RESET +$(SHOW)set -e ;\ +echo "Starting coverage analysys." ;\ +mkdir -p $(COV_DIR) ;\ +lcov --directory $(BINROOT) --base-directory $(SRCDIR) -z > /dev/null 2>&1 +endef + +define COVERAGE_COLLECT +$(SHOW)set -e ;\ +echo "Collecting coverage data ..." ;\ +lcov --capture --directory $(BINROOT) --base-directory $(SRCDIR) --output-file $(COV_INFO) > /dev/null 2>&1 ;\ +lcov -o $(COV_INFO) -r $(COV_INFO) $(COV_EXCLUDE) > /dev/null 2>&1 +endef + +define COVERAGE_REPORT +$(SHOW)set -e ;\ +lcov -l $(COV_INFO) ;\ +genhtml --legend -o $(COV_DIR) $(COV_INFO) > /dev/null 2>&1 ;\ +echo "Coverage info at $$(realpath $$(COV_DIR))/index.html" +endef define COVERAGE_COLLECT_REPORT $(COVERAGE_COLLECT) $(COVERAGE_REPORT) endef +else + +COVERAGE_RESET= +COVERAGE_COLLECT= +COVERAGE_REPORT= +COVERAGE_COLLECT_REPORT= + endif # COV diff --git a/opt/readies/paella/setup.py b/opt/readies/paella/setup.py index b4a3985ab..b10af35ed 100755 --- a/opt/readies/paella/setup.py +++ b/opt/readies/paella/setup.py @@ -2,6 +2,7 @@ import os import sys import tempfile +import textwrap from .platform import OnPlatform, Platform #---------------------------------------------------------------------------------------------- @@ -11,6 +12,8 @@ def __init__(self, nop=False): self.nop = nop def run(self, cmd, output_on_error=False, _try=False): + if cmd.find('\n') > -1: + cmd = str.lstrip(textwrap.dedent(cmd)).replace('\n', '; ') print(cmd) sys.stdout.flush() if self.nop: @@ -216,8 +219,17 @@ def install_downloaders(self, _try=False): self.install("curl wget", _try=_try) def install_git_lfs_on_linux(self, _try=False): - self.run("set -e; wget -q https://github.com/git-lfs/git-lfs/releases/download/v2.9.2/git-lfs-linux-amd64-v2.9.2.tar.gz -O /tmp/git-lfs.tar.gz") - self.run("cd /tmp; tar xzf git-lfs.tar.gz; ./install.sh") + self.run(""" + set -e + d=$(mktemp -d /tmp/git-lfs.XXXXXX) + mkdir -p $d + wget -q https://github.com/git-lfs/git-lfs/releases/download/v2.9.2/git-lfs-linux-amd64-v2.9.2.tar.gz -O $d/git-lfs.tar.gz + cd $d + tar xf git-lfs.tar.gz + cd - + $d/install.sh + rm -rf $d + """) # cmd = "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.{}.sh | bash" # if self.platform.is_redhat_compat(): From 62d76a67d1e0b0b35798d29eb04e1b39372157d4 Mon Sep 17 00:00:00 2001 From: rafie Date: Thu, 27 Feb 2020 18:19:14 +0200 Subject: [PATCH 09/18] fixes 7 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a88321e33..1e39b1c65 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,7 +71,7 @@ jobs: coverage: docker: - - image: redislabsmodules/rmbuilder:latest + - image: redisfab/rmbuilder:x64-buster steps: - checkout - run: From 7c3ff5cd6be6b8f429abfcf70a99479cab055cbc Mon Sep 17 00:00:00 2001 From: rafie Date: Thu, 27 Feb 2020 18:53:23 +0200 Subject: [PATCH 10/18] fixes 8 --- .circleci/config.yml | 10 +++++----- opt/readies/mk/coverage.defs | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1e39b1c65..86beb5e95 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,7 +38,6 @@ commands: command: | mkdir -p ~/workspace/tests make -C opt test SHOW=1 VERBOSE=1 - cp test/logs/* ~/workspace/tests - run: name: Package command: make -C opt pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=~/workspace/packages SHOW=1 @@ -49,8 +48,9 @@ commands: - 'packages/release/*.tgz' - 'packages/branch/*.zip' - 'packages/branch/*.tgz' - - store_test_results: - path: ~/workspace/tests + - store_artifacts: + path: test/logs + deploy: parameters: from: @@ -100,7 +100,7 @@ jobs: build-macos: macos: - xcode: 10.2.1 + xcode: 11.1.0 steps: - run: name: Fix macOS Python installation @@ -159,7 +159,7 @@ jobs: mkdir -p ~/workspace/tests docker run --gpus all -v $HOME/workspace/tests:/build/test/logs -it --rm redisai-gpu:latest-x64-bionic-test no_output_timeout: 30m - - store_test_results: + - store_artifacts: path: ~/workspace/tests deploy_package: diff --git a/opt/readies/mk/coverage.defs b/opt/readies/mk/coverage.defs index 921fbd3ce..c778bcb6a 100644 --- a/opt/readies/mk/coverage.defs +++ b/opt/readies/mk/coverage.defs @@ -14,21 +14,21 @@ COV_EXCLUDE += \ define COVERAGE_RESET $(SHOW)set -e ;\ echo "Starting coverage analysys." ;\ -mkdir -p $(COV_DIR) ;\ -lcov --directory $(BINROOT) --base-directory $(SRCDIR) -z > /dev/null 2>&1 +mkdir -p $$(COV_DIR) ;\ +lcov --directory $$(BINROOT) --base-directory $$(SRCDIR) -z > /dev/null 2>&1 endef define COVERAGE_COLLECT $(SHOW)set -e ;\ echo "Collecting coverage data ..." ;\ -lcov --capture --directory $(BINROOT) --base-directory $(SRCDIR) --output-file $(COV_INFO) > /dev/null 2>&1 ;\ -lcov -o $(COV_INFO) -r $(COV_INFO) $(COV_EXCLUDE) > /dev/null 2>&1 +lcov --capture --directory $$(BINROOT) --base-directory $$(SRCDIR) --output-file $$(COV_INFO) > /dev/null 2>&1 ;\ +lcov -o $$(COV_INFO) -r $$(COV_INFO) $$(COV_EXCLUDE) > /dev/null 2>&1 endef define COVERAGE_REPORT $(SHOW)set -e ;\ -lcov -l $(COV_INFO) ;\ -genhtml --legend -o $(COV_DIR) $(COV_INFO) > /dev/null 2>&1 ;\ +lcov -l $$(COV_INFO) ;\ +genhtml --legend -o $$(COV_DIR) $$(COV_INFO) > /dev/null 2>&1 ;\ echo "Coverage info at $$(realpath $$(COV_DIR))/index.html" endef From c4775f10c222d851fb4af8c0695c19034faa3474 Mon Sep 17 00:00:00 2001 From: rafie Date: Thu, 27 Feb 2020 19:25:00 +0200 Subject: [PATCH 11/18] fixes 9 --- Dockerfile.gpu-test | 24 +++++++++++++++++++----- opt/Makefile | 8 +++++++- opt/readies/mk/coverage.defs | 14 +++++++------- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/Dockerfile.gpu-test b/Dockerfile.gpu-test index 2278e6592..76adb5b29 100644 --- a/Dockerfile.gpu-test +++ b/Dockerfile.gpu-test @@ -22,20 +22,34 @@ COPY ./opt/ opt/ COPY ./test/test_requirements.txt test/ RUN ./opt/readies/bin/getpy3 -RUN ./opt/system-setup.py +RUN set -e ;\ + python3 -m virtualenv venv ;\ + . ./venv/bin/activate ;\ + ./opt/system-setup.py ARG DEPS_ARGS="" COPY ./get_deps.sh . -RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_deps.sh gpu; fi +RUN set -e ;\ + . ./venv/bin/activate ;\ + if [ "$DEPS_ARGS" = "" ]; then \ + ./get_deps.sh gpu ;\ + else \ + env $DEPS_ARGS ./get_deps.sh gpu ;\ + fi ARG BUILD_ARGS="" ADD ./ /build -RUN make -C opt build GPU=1 $BUILD_ARGS SHOW=1 +RUN set -e ;\ + . ./venv/bin/activate ;\ + make -C opt build GPU=1 $BUILD_ARGS SHOW=1 ARG PACK=1 -RUN if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi +RUN if [ "$PACK" = "1" ]; then \ + . ./venv/bin/activate ;\ + make -C opt pack GPU=1 ;\ + fi RUN git remote set-url origin https://github.com/RedisAI/RedisAI -CMD ["bash", "-c", "make -C opt test GPU=1 SHOW=1"] +CMD ["bash", "-c", ". ./venv/bin/activate; make -C opt test GPU=1 SHOW=1"] diff --git a/opt/Makefile b/opt/Makefile index 1815bfd7f..7641faa2a 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -12,6 +12,10 @@ ifneq ($(VGD),) VALGRIND=$(VGD) endif +ifeq ($(VALGRIND),1) +DEBUG ?= 1 +endif + ifeq ($(COV),1) DEBUG ?= 1 endif @@ -179,6 +183,8 @@ export GEN ?= 1 export SLAVES ?= 1 export AOF ?= 1 +$(info COV_DIR=$(COV_DIR)) + test: $(COVERAGE_RESET) $(SHOW)\ @@ -187,7 +193,7 @@ test: GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \ VALGRIND=$(VALGRIND) \ $(ROOT)/test/tests.sh - $(COVERAGE_COLLECT_REPORT) + # $(COVERAGE_COLLECT_REPORT) valgrind: $(SHOW)$(ROOT)/test/valgrind.sh $(realpath $(INSTALLED_TARGET)) diff --git a/opt/readies/mk/coverage.defs b/opt/readies/mk/coverage.defs index c778bcb6a..836efa758 100644 --- a/opt/readies/mk/coverage.defs +++ b/opt/readies/mk/coverage.defs @@ -14,22 +14,22 @@ COV_EXCLUDE += \ define COVERAGE_RESET $(SHOW)set -e ;\ echo "Starting coverage analysys." ;\ -mkdir -p $$(COV_DIR) ;\ -lcov --directory $$(BINROOT) --base-directory $$(SRCDIR) -z > /dev/null 2>&1 +mkdir -p $(COV_DIR) ;\ +lcov --directory $(BINROOT) --base-directory $(SRCDIR) -z > /dev/null 2>&1 endef define COVERAGE_COLLECT $(SHOW)set -e ;\ echo "Collecting coverage data ..." ;\ -lcov --capture --directory $$(BINROOT) --base-directory $$(SRCDIR) --output-file $$(COV_INFO) > /dev/null 2>&1 ;\ -lcov -o $$(COV_INFO) -r $$(COV_INFO) $$(COV_EXCLUDE) > /dev/null 2>&1 +lcov --capture --directory $(BINROOT) --base-directory $(SRCDIR) --output-file $(COV_INFO) > /dev/null 2>&1 ;\ +lcov -o $(COV_INFO) -r $(COV_INFO) $(COV_EXCLUDE) > /dev/null 2>&1 endef define COVERAGE_REPORT $(SHOW)set -e ;\ -lcov -l $$(COV_INFO) ;\ -genhtml --legend -o $$(COV_DIR) $$(COV_INFO) > /dev/null 2>&1 ;\ -echo "Coverage info at $$(realpath $$(COV_DIR))/index.html" +lcov -l $(COV_INFO) ;\ +genhtml --legend -o $(COV_DIR) $(COV_INFO) > /dev/null 2>&1 ;\ +echo "Coverage info at $$(realpath $(COV_DIR))/index.html" endef define COVERAGE_COLLECT_REPORT From 40d4302211e137bff890babb02ff0d05f319152e Mon Sep 17 00:00:00 2001 From: rafie Date: Thu, 27 Feb 2020 19:42:58 +0200 Subject: [PATCH 12/18] fixes 10 --- .circleci/config.yml | 5 ++--- Dockerfile.gpu-test | 1 + opt/Makefile | 2 +- opt/system-setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 86beb5e95..2800ffd73 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,6 +24,7 @@ commands: - save_cache: paths: - deps + - /usr/local key: build-dependencies-{{ checksum "get_deps.sh" }} - run: name: Set up workspace @@ -79,9 +80,7 @@ jobs: command: | ./opt/readies/bin/getpy3 BREW_NO_UPDATE=1 ./opt/system-setup.py - git clone git://github.com/antirez/redis.git --branch 5.0.7 - (cd redis && make malloc=libc -j $(nproc) && make install) - redis-server --version + # git clone git://github.com/antirez/redis.git --branch 5.0.7; cd redis; make malloc=libc -j $(nproc); make install; redis-server --version - restore_cache: keys: - build-dependencies-{{ checksum "get_deps.sh" }} diff --git a/Dockerfile.gpu-test b/Dockerfile.gpu-test index 76adb5b29..29bbe077a 100644 --- a/Dockerfile.gpu-test +++ b/Dockerfile.gpu-test @@ -22,6 +22,7 @@ COPY ./opt/ opt/ COPY ./test/test_requirements.txt test/ RUN ./opt/readies/bin/getpy3 +RUN ./opt/system-setup.py RUN set -e ;\ python3 -m virtualenv venv ;\ . ./venv/bin/activate ;\ diff --git a/opt/Makefile b/opt/Makefile index 7641faa2a..4e4611476 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -193,7 +193,7 @@ test: GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \ VALGRIND=$(VALGRIND) \ $(ROOT)/test/tests.sh - # $(COVERAGE_COLLECT_REPORT) + $(COVERAGE_COLLECT_REPORT) valgrind: $(SHOW)$(ROOT)/test/valgrind.sh $(realpath $(INSTALLED_TARGET)) diff --git a/opt/system-setup.py b/opt/system-setup.py index 910865d66..179643a90 100755 --- a/opt/system-setup.py +++ b/opt/system-setup.py @@ -17,7 +17,7 @@ def __init__(self, nop=False): def common_first(self): self.install_downloaders() self.setup_pip() - self.pip3_install("wheel") + self.pip3_install("wheel virtualenv") self.pip3_install("setuptools --upgrade") if self.os == 'linux': From bb3955ae6944bdd88facf5a96338cbcdb578649b Mon Sep 17 00:00:00 2001 From: rafie Date: Thu, 27 Feb 2020 20:04:22 +0200 Subject: [PATCH 13/18] fixes 11 --- Dockerfile.gpu-test | 8 +++++--- opt/Makefile | 2 -- opt/readies/mk/coverage.defs | 4 ++++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Dockerfile.gpu-test b/Dockerfile.gpu-test index 29bbe077a..cc7097f87 100644 --- a/Dockerfile.gpu-test +++ b/Dockerfile.gpu-test @@ -12,6 +12,8 @@ ARG ARCH=x64 FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 AS builder +SHELL ["/bin/bash", "-c"] + ENV NVIDIA_VISIBLE_DEVICES all ENV NVIDIA_DRIVER_CAPABILITIES compute,utility @@ -32,7 +34,7 @@ ARG DEPS_ARGS="" COPY ./get_deps.sh . RUN set -e ;\ . ./venv/bin/activate ;\ - if [ "$DEPS_ARGS" = "" ]; then \ + if [[ -z $DEPS_ARGS ]]; then \ ./get_deps.sh gpu ;\ else \ env $DEPS_ARGS ./get_deps.sh gpu ;\ @@ -46,9 +48,9 @@ RUN set -e ;\ ARG PACK=1 -RUN if [ "$PACK" = "1" ]; then \ +RUN if [[ $PACK == 1 ]]; then \ . ./venv/bin/activate ;\ - make -C opt pack GPU=1 ;\ + make -C opt pack GPU=1 VERBOSE=1 ;\ fi RUN git remote set-url origin https://github.com/RedisAI/RedisAI diff --git a/opt/Makefile b/opt/Makefile index 4e4611476..da9f0f9d3 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -183,8 +183,6 @@ export GEN ?= 1 export SLAVES ?= 1 export AOF ?= 1 -$(info COV_DIR=$(COV_DIR)) - test: $(COVERAGE_RESET) $(SHOW)\ diff --git a/opt/readies/mk/coverage.defs b/opt/readies/mk/coverage.defs index 836efa758..f38f8d9ba 100644 --- a/opt/readies/mk/coverage.defs +++ b/opt/readies/mk/coverage.defs @@ -1,4 +1,8 @@ +ifneq ($(filter cov-upload,$(MAKECMDGOALS)),) +COV=1 +endif + ifeq ($(COV),1) CC_FLAGS.coverage += --coverage From 558aea352c1aaa4ab5626f1a560b79ee858a2d73 Mon Sep 17 00:00:00 2001 From: rafie Date: Fri, 28 Feb 2020 18:03:14 +0200 Subject: [PATCH 14/18] fixes 12 --- .circleci/config.yml | 2 +- Dockerfile.gpu-test | 11 ++++++----- opt/system-setup.py | 8 ++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2800ffd73..4935488c4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -95,7 +95,7 @@ jobs: name: Test with coverage command: | make -C opt test SHOW=1 COV=1 - make cov-upload COV=1 + make -C opt cov-upload build-macos: macos: diff --git a/Dockerfile.gpu-test b/Dockerfile.gpu-test index cc7097f87..176d80113 100644 --- a/Dockerfile.gpu-test +++ b/Dockerfile.gpu-test @@ -27,13 +27,13 @@ RUN ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py RUN set -e ;\ python3 -m virtualenv venv ;\ - . ./venv/bin/activate ;\ + . venv/bin/activate ;\ ./opt/system-setup.py ARG DEPS_ARGS="" COPY ./get_deps.sh . RUN set -e ;\ - . ./venv/bin/activate ;\ + . venv/bin/activate ;\ if [[ -z $DEPS_ARGS ]]; then \ ./get_deps.sh gpu ;\ else \ @@ -43,13 +43,14 @@ RUN set -e ;\ ARG BUILD_ARGS="" ADD ./ /build RUN set -e ;\ - . ./venv/bin/activate ;\ + . venv/bin/activate ;\ make -C opt build GPU=1 $BUILD_ARGS SHOW=1 ARG PACK=1 -RUN if [[ $PACK == 1 ]]; then \ - . ./venv/bin/activate ;\ +RUN set -e ;\ + if [[ $PACK == 1 ]]; then \ + . venv/bin/activate ;\ make -C opt pack GPU=1 VERBOSE=1 ;\ fi diff --git a/opt/system-setup.py b/opt/system-setup.py index 179643a90..a030a328d 100755 --- a/opt/system-setup.py +++ b/opt/system-setup.py @@ -66,10 +66,10 @@ def macosx(self): self.install("redis") def common_last(self): - if not self.has_command("RLTest"): - self.pip3_install("git+https://github.com/RedisLabsModules/RLTest.git@master") - if not self.has_command("ramp"): - self.pip3_install("git+https://github.com/RedisLabs/RAMP@master") + # if not self.has_command("RLTest"): + self.pip3_install("git+https://github.com/RedisLabsModules/RLTest.git@master") + # if not self.has_command("ramp"): + self.pip3_install("git+https://github.com/RedisLabs/RAMP@master") root = os.path.join(os.path.dirname(__file__), "..") self.pip3_install("-r {}/test/test_requirements.txt".format(root)) From c80d4584a7375fcaae2145d5839db30fe8b592d5 Mon Sep 17 00:00:00 2001 From: rafie Date: Fri, 28 Feb 2020 18:15:24 +0200 Subject: [PATCH 15/18] fixes 13 --- opt/system-setup.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/opt/system-setup.py b/opt/system-setup.py index a030a328d..94d66cbb4 100755 --- a/opt/system-setup.py +++ b/opt/system-setup.py @@ -41,7 +41,7 @@ def redhat_compat(self): else: self.run("amazon-linux-extras install epel", output_on_error=True) self.install("python3 python3-devel") - self.pip_install("psutil") + self.pip3_install("psutil") self.install_git_lfs_on_linux() @@ -66,12 +66,17 @@ def macosx(self): self.install("redis") def common_last(self): - # if not self.has_command("RLTest"): - self.pip3_install("git+https://github.com/RedisLabsModules/RLTest.git@master") - # if not self.has_command("ramp"): - self.pip3_install("git+https://github.com/RedisLabs/RAMP@master") + # this is due to rmbuilder older versions. should be removed once fixed. + self.run("python3 -m pip uninstall -y -q redis redis-py-cluster ramp-packer RLTest rmtest semantic-version || true") + # redis-py-cluster should be installed from git due to redis-py dependency + self.pip3_install("--no-cache-dir git+https://github.com/Grokzen/redis-py-cluster.git@master") + # the following can be probably installed from pypi + self.pip3_install("--no-cache-dir git+https://github.com/RedisLabsModules/RLTest.git@master") + self.pip3_install("--no-cache-dir git+https://github.com/RedisLabs/RAMP@master") + root = os.path.join(os.path.dirname(__file__), "..") self.pip3_install("-r {}/test/test_requirements.txt".format(root)) + #---------------------------------------------------------------------------------------------- From 91b032045f9add68ae680b3739e35270368cfc81 Mon Sep 17 00:00:00 2001 From: rafie Date: Fri, 28 Feb 2020 18:44:55 +0200 Subject: [PATCH 16/18] fixes 14 --- Dockerfile.gpu-test | 2 +- opt/system-setup.py | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Dockerfile.gpu-test b/Dockerfile.gpu-test index 176d80113..ecc5e93eb 100644 --- a/Dockerfile.gpu-test +++ b/Dockerfile.gpu-test @@ -26,7 +26,7 @@ COPY ./test/test_requirements.txt test/ RUN ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py RUN set -e ;\ - python3 -m virtualenv venv ;\ + python3 -m virtualenv venv --system-site-packages;\ . venv/bin/activate ;\ ./opt/system-setup.py diff --git a/opt/system-setup.py b/opt/system-setup.py index 94d66cbb4..07939b1f6 100755 --- a/opt/system-setup.py +++ b/opt/system-setup.py @@ -56,11 +56,6 @@ def macosx(self): if out.splitlines() == []: fatal("Xcode tools are not installed. Please run xcode-select --install.") - # workaround for ssl issue, needed in CircleCI - #if os.environ.get('MACOS_PYTHON_SSL_FIX') == '1': - # self.run("brew unlink python@2") - # self.run("brew reinstall python3") - self.install_gnu_utils() self.install("git-lfs") self.install("redis") From 412d8afcc5470985759460c4103011855c5dc890 Mon Sep 17 00:00:00 2001 From: rafie Date: Sun, 1 Mar 2020 10:15:15 +0200 Subject: [PATCH 17/18] fixes 15 --- .circleci/config.yml | 13 ++----------- Dockerfile.gpu | 2 +- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4935488c4..8c4eba0ff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,9 +17,7 @@ commands: command: | ./opt/readies/bin/getpy3 BREW_NO_UPDATE=1 ./opt/system-setup.py - git clone git://github.com/antirez/redis.git --branch 5.0.7 - (cd redis && make malloc=libc -j $(nproc) && make install) - redis-server --version + git clone git://github.com/antirez/redis.git --branch 5.0.7; cd redis; make malloc=libc -j $(nproc); make install; redis-server --version ./get_deps.sh cpu - save_cache: paths: @@ -79,7 +77,7 @@ jobs: name: Install dependencies command: | ./opt/readies/bin/getpy3 - BREW_NO_UPDATE=1 ./opt/system-setup.py + ./opt/system-setup.py # git clone git://github.com/antirez/redis.git --branch 5.0.7; cd redis; make malloc=libc -j $(nproc); make install; redis-server --version - restore_cache: keys: @@ -141,13 +139,6 @@ jobs: image: ubuntu-1604-cuda-10.1:201909-23 steps: - checkout -# - run: -# name: Checkout LFS -# command: | -# curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash -# sudo apt-get install -y git-lfs -# git lfs install -# git lfs pull - run: name: Build command: | diff --git a/Dockerfile.gpu b/Dockerfile.gpu index 3e7f94c9c..216b45431 100644 --- a/Dockerfile.gpu +++ b/Dockerfile.gpu @@ -22,7 +22,7 @@ COPY --from=redis /usr/local/ /usr/local/ COPY ./opt/ opt/ COPY ./test/test_requirements.txt test/ -RUN ./opt/readies/bin/getpy +RUN ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py ARG DEPS_ARGS="" From 857a98682b90d836d40363cb8615d3bfd67af42a Mon Sep 17 00:00:00 2001 From: rafie Date: Sun, 1 Mar 2020 10:25:55 +0200 Subject: [PATCH 18/18] fixes 16 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8c4eba0ff..edb39d88e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,7 +17,7 @@ commands: command: | ./opt/readies/bin/getpy3 BREW_NO_UPDATE=1 ./opt/system-setup.py - git clone git://github.com/antirez/redis.git --branch 5.0.7; cd redis; make malloc=libc -j $(nproc); make install; redis-server --version + git clone git://github.com/antirez/redis.git --branch 5.0.7; (cd redis; make malloc=libc -j $(nproc); make install); redis-server --version ./get_deps.sh cpu - save_cache: paths: