From c52ac7c259b5e256da7aef50347473823d73b5b7 Mon Sep 17 00:00:00 2001 From: rafie Date: Wed, 21 Aug 2019 17:00:40 +0300 Subject: [PATCH 01/30] ARM support and bin/os-arch-variant scheme --- .circleci/config.yml | 4 +- .dockerignore | 9 +- .gitignore | 25 +- CMakeLists.txt | 59 +- Dockerfile | 29 +- Dockerfile-gpu | 3 +- Dockerfile.arm | 58 ++ automation/Makefile | 102 ++- automation/pack.sh | 133 ++++ automation/readies/mk/build.defs | 0 automation/readies/mk/build.rules | 11 + automation/readies/mk/cmake.defs | 14 + automation/readies/mk/cmake.rules | 11 + automation/readies/mk/configure.rules | 4 +- automation/readies/mk/defs | 8 +- automation/readies/mk/help.rules | 2 + automation/readies/mk/main | 8 + automation/readies/mk/rules | 7 + automation/system-setup.py | 4 +- build/bazel/Dockerfile | 32 + build/docker/Makefile | 99 +++ build/libtorch/Dockerfile.arm | 52 ++ build/libtorch/Dockerfile.x64 | 42 ++ build/libtorch/Makefile | 95 +++ build/libtorch/collect.py | 82 +++ build/libtorch/repack.sh | 55 ++ build/onnxruntime/Dockerfile.arm | 53 ++ build/onnxruntime/Dockerfile.arm7 | 46 ++ build/onnxruntime/Dockerfile.x64 | 36 ++ build/onnxruntime/Makefile | 72 +++ build/onnxruntime/pack.sh | 21 + build/onnxruntime/werrror.patch | 758 +++++++++++++++++++++++ build/platforms/Makefile | 21 + build/tensorflow/Dockerfile.arm | 60 ++ build/tensorflow/Dockerfile.x64 | 66 ++ build/tensorflow/Makefile | 62 ++ build/tensorflow/collect.py | 77 +++ build/tensorflow/tf-1.14.0-arm64v8.patch | 86 +++ get_deps.sh | 349 +++++++---- 39 files changed, 2417 insertions(+), 238 deletions(-) create mode 100755 Dockerfile.arm create mode 100755 automation/pack.sh create mode 100755 automation/readies/mk/build.defs create mode 100755 automation/readies/mk/build.rules create mode 100755 automation/readies/mk/cmake.defs create mode 100755 automation/readies/mk/cmake.rules create mode 100755 automation/readies/mk/main create mode 100755 build/bazel/Dockerfile create mode 100755 build/docker/Makefile create mode 100755 build/libtorch/Dockerfile.arm create mode 100755 build/libtorch/Dockerfile.x64 create mode 100755 build/libtorch/Makefile create mode 100755 build/libtorch/collect.py create mode 100755 build/libtorch/repack.sh create mode 100755 build/onnxruntime/Dockerfile.arm create mode 100755 build/onnxruntime/Dockerfile.arm7 create mode 100755 build/onnxruntime/Dockerfile.x64 create mode 100755 build/onnxruntime/Makefile create mode 100755 build/onnxruntime/pack.sh create mode 100755 build/onnxruntime/werrror.patch create mode 100755 build/platforms/Makefile create mode 100755 build/tensorflow/Dockerfile.arm create mode 100755 build/tensorflow/Dockerfile.x64 create mode 100755 build/tensorflow/Makefile create mode 100755 build/tensorflow/collect.py create mode 100755 build/tensorflow/tf-1.14.0-arm64v8.patch diff --git a/.circleci/config.yml b/.circleci/config.yml index 56fe7236c..8607f1766 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,13 +20,13 @@ jobs: sudo ./automation/readies/bin/getpy sudo ./automation/system-setup.py ./get_deps.sh cpu - git clone git://github.com/antirez/redis.git --branch 5.0.3 + git clone git://github.com/antirez/redis.git --branch 5.0.5 (cd redis && make malloc=libc -j4 && sudo make install) - save_cache: paths: - deps - key: v2-dependencies-{{ checksum "get_deps.sh" }} + key: v1-dependencies-{{ checksum "get_deps.sh" }} - run: name: Build diff --git a/.dockerignore b/.dockerignore index 9418ba6fc..0a65b637b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,10 @@ +/bin/ /deps/ -/build/ /install*/ -**/*.o -**/*.so .venv/ venv*/ +/[0-9]*/ +/install +*.zip +*.tgz +*.tar.gz diff --git a/.gitignore b/.gitignore index 0bb1f7455..be23442e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,16 @@ +/bin/ +/deps/ +/install*/ +/test/venv/ +/test/logs/ +.venv/ +venv*/ +/install +/[0-9]*/ +*.zip +*.tgz +*.tar.gz + # Misc .DS_Store *.swp @@ -6,18 +19,6 @@ __pycache__ *.pyc -# Dirs -/install*/ -/deps/* -/build/ -examples/js/node_modules/ -examples/js/package-lock.json -/test/venv/ -/test/logs/ -/1/ -/venv/ -/.venv/ - # Eclipse (if that's your jam...) .project .cproject diff --git a/CMakeLists.txt b/CMakeLists.txt index 42458cf7e..d84056e82 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ PROJECT(RedisAI) IF (NOT DEPS_PATH) # dependencies are required! - SET(DEPS_PATH ../deps/install) + SET(DEPS_PATH ../deps) # MESSAGE(FATAL_ERROR "DEPS PATH Missing!") ENDIF() @@ -20,10 +20,10 @@ FUNCTION(ADD_LDFLAGS _TARGET NEW_FLAGS) ENDFUNCTION() GET_FILENAME_COMPONENT(depsAbs - "${DEPS_PATH}" REALPATH - BASE_DIR "${CMAKE_BINARY_DIR}") + "${DEPS_PATH}" REALPATH BASE_DIR ${CMAKE_SOURCE_DIR}) +# BASE_DIR "${CMAKE_BINARY_DIR}") -# SET(CUDA_TOOLKIT_ROOT_DIR /usr/local/cuda-9.0) +# SET(CUDA_TOOLKIT_ROOT_DIR /usr/local/cuda-10.0) INCLUDE_DIRECTORIES(${depsAbs}/dlpack/include) INCLUDE_DIRECTORIES(${depsAbs}/libtensorflow/include) @@ -34,28 +34,49 @@ SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") SET(CMAKE_C_STANDARD 11) -FIND_LIBRARY(TF_LIBRARIES NAMES tensorflow +#---------------------------------------------------------------------------------------------- + +FIND_LIBRARY(TF_LIBRARIES NAMES tensorflow #libtensorflow.so PATHS ${depsAbs}/libtensorflow/lib) IF (NOT TF_LIBRARIES) MESSAGE(FATAL_ERROR "Could not find tensorflow") ENDIF() +#---------------------------------------------------------------------------------------------- + FIND_LIBRARY(ORT_LIBRARIES NAMES onnxruntime - PATHS ${depsAbs}/onnxruntime/lib) + PATHS ${depsAbs}/onnxruntime/lib) IF (NOT ORT_LIBRARIES) MESSAGE(FATAL_ERROR "Could not find ONNXRuntime") ENDIF() +#---------------------------------------------------------------------------------------------- + +IF (APPLE) + FIND_LIBRARY(MKL_LIBRARIES NAMES mklml) + IF (NOT MKL_LIBRARIES) + MESSAGE(FATAL_ERROR "Could not find MKL for Mac") + ENDIF() + SET(platDeps "${MKL_LIBRARIES}") +ENDIF() + +#---------------------------------------------------------------------------------------------- + # Find Torch stuff and build our wrapper SET (Torch_DIR ${depsAbs}/libtorch/share/cmake/Torch) FIND_PACKAGE(Torch REQUIRED) + INCLUDE_DIRECTORIES(util/libtorch_c) ADD_DEFINITIONS(-DREDISMODULE_EXPERIMENTAL_API) ADD_SUBDIRECTORY(util/libtorch_c) + +#---------------------------------------------------------------------------------------------- + ADD_SUBDIRECTORY(src) ADD_LIBRARY(redisai SHARED $) +# TARGET_LINK_LIBRARIES(redisai ${ORT_LIBRARIES} torch_c ${TF_LIBRARIES} ${platDeps} ${CMAKE_DL_LIBS}) TARGET_LINK_LIBRARIES(redisai ${CMAKE_DL_LIBS}) SET_TARGET_PROPERTIES(redisai PROPERTIES PREFIX "") @@ -70,6 +91,15 @@ SET(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install-${DEVICE}) INSTALL(TARGETS redisai LIBRARY DESTINATION .) +IF (APPLE) + SET_TARGET_PROPERTIES(redisai PROPERTIES INSTALL_RPATH "@loader_path/lib") +ELSE () + ADD_LDFLAGS(redisai "-Wl,--enable-new-dtags") + SET_TARGET_PROPERTIES(redisai PROPERTIES INSTALL_RPATH "\$ORIGIN/lib") +ENDIF() + +#---------------------------------------------------------------------------------------------- + ADD_LIBRARY(redisai_tensorflow SHARED $) TARGET_LINK_LIBRARIES(redisai_tensorflow ${TF_LIBRARIES}) SET_TARGET_PROPERTIES(redisai_tensorflow PROPERTIES PREFIX "") @@ -81,7 +111,10 @@ ELSE () SET_TARGET_PROPERTIES(redisai_tensorflow PROPERTIES INSTALL_RPATH "\$ORIGIN/lib") ENDIF() INSTALL(TARGETS redisai_tensorflow LIBRARY DESTINATION backends/redisai_tensorflow) -INSTALL(DIRECTORY ${depsAbs}/libtensorflow/lib DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/redisai_tensorflow) +INSTALL(DIRECTORY ${depsAbs}/libtensorflow/lib DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/redisai_tensorflow + FILES_MATCHING PATTERN "*.so*") + +#---------------------------------------------------------------------------------------------- ADD_LIBRARY(redisai_torch SHARED $) TARGET_LINK_LIBRARIES(redisai_torch torch_c ${TORCH_LIBRARIES}) @@ -94,9 +127,12 @@ ELSE () SET_TARGET_PROPERTIES(redisai_torch PROPERTIES INSTALL_RPATH "\$ORIGIN/lib") ENDIF() INSTALL(TARGETS redisai_torch LIBRARY DESTINATION backends/redisai_torch) -INSTALL(DIRECTORY ${depsAbs}/libtorch/lib DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/redisai_torch) +INSTALL(DIRECTORY ${depsAbs}/libtorch/lib DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/redisai_torch + FILES_MATCHING PATTERN "*.so*") -IF(${DEVICE} STREQUAL "gpu") +#---------------------------------------------------------------------------------------------- + +IF (${DEVICE} STREQUAL "gpu") ADD_DEFINITIONS(-DRAI_ONNXRUNTIME_USE_CUDA) ENDIF() @@ -111,4 +147,7 @@ ELSE () SET_TARGET_PROPERTIES(redisai_onnxruntime PROPERTIES INSTALL_RPATH "\$ORIGIN/lib") ENDIF() INSTALL(TARGETS redisai_onnxruntime LIBRARY DESTINATION backends/redisai_onnxruntime) -INSTALL(DIRECTORY ${depsAbs}/onnxruntime/lib DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/redisai_onnxruntime) +INSTALL(DIRECTORY ${depsAbs}/onnxruntime/lib DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/redisai_onnxruntime + FILES_MATCHING PATTERN "*.so*") + +#---------------------------------------------------------------------------------------------- diff --git a/Dockerfile b/Dockerfile index e01c43072..d937fbb38 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,19 @@ -ARG OS=debian:buster +# BUILD redisfab/redisai-cpu-${OSNICK}:M.m.b-${ARCH} + +# OSNICK=bionic|stretch|buster +ARG OSNICK=buster + +# OS=debian:buster-slim|debian:stretch-slim|ubuntu:bionic +OS=debian:buster-slim + +# ARCH=x64|arm64v8|arm32v7 +ARG ARCH=x64 #---------------------------------------------------------------------------------------------- -FROM redis AS redis +FROM redisfab/redis-${ARCH}-${OSNICK}:5.0.5 AS redis FROM ${OS} AS builder -WORKDIR /redisai +WORKDIR /build COPY --from=redis /usr/local/ /usr/local/ COPY ./automation/ automation/ @@ -17,7 +26,7 @@ COPY ./get_deps.sh . RUN ./get_deps.sh cpu ADD ./ /redisai -RUN make -C automation all +RUN make -C automation all SHOW=1 ARG PACK=0 ARG TEST=0 @@ -26,13 +35,17 @@ RUN if [ "$PACK" = "1" ]; then make -C automation pack; fi RUN if [ "$TEST" = "1" ]; then make -C automation test; fi #---------------------------------------------------------------------------------------------- -FROM redis +# FROM redis:latest +FROM redisfab/redis-${ARCH}-${OSNICK}:5.0.5 + +RUN set -e; apt-get -qq update; apt-get -q install -y libgomp1 -RUN set -e; apt-get -qq update; apt-get install -y libgomp1 +ENV REDIS_MODULES /usr/lib/redis/modules +ENV LD_LIBRARY_PATH $REDIS_MODULES -RUN mkdir -p /usr/lib/redis/modules/ +RUN mkdir -p $REDIS_MODULES/ -COPY --from=builder /redisai/install-cpu/ /usr/lib/redis/modules/ +COPY --from=builder /build/install-cpu/ $REDIS_MODULES/ WORKDIR /data EXPOSE 6379 diff --git a/Dockerfile-gpu b/Dockerfile-gpu index feb7917fa..edc53e6c5 100755 --- a/Dockerfile-gpu +++ b/Dockerfile-gpu @@ -47,8 +47,7 @@ RUN set -ex;\ # Package the runner FROM builder -RUN set -ex;\ - mkdir -p /usr/lib/redis/modules/; +RUN mkdir -p /usr/lib/redis/modules/ COPY --from=builder /redisai/install/ /usr/lib/redis/modules/ diff --git a/Dockerfile.arm b/Dockerfile.arm new file mode 100755 index 000000000..d4e9e7a41 --- /dev/null +++ b/Dockerfile.arm @@ -0,0 +1,58 @@ +# BUILD redisfab/redisai-cpu-${OSNICK}:M.m.b-${ARCH} + +# OSNICK=bionic|stretch|buster +ARG OSNICK=buster + +# OS=debian:buster-slim|debian:stretch-slim|ubuntu:bionic +OS=debian:buster-slim + +# ARCH=arm64v8|arm32v7 +ARG ARCH=arm64v8 + +#---------------------------------------------------------------------------------------------- +FROM redisfab/redis-${ARCH}-${OSNICK}-xbuild:5.0.5 AS builder + +RUN [ "cross-build-start" ] + +ADD ./ /build +WORKDIR /build + +COPY ./automation/ automation/ +COPY ./test/test_requirements.txt test/ + +RUN ./automation/readies/bin/getpy +RUN ./automation/system-setup.py + +COPY ./get_deps.sh . +RUN ./get_deps.sh cpu + +ADD ./ /redisai +RUN make -C automation all SHOW=1 + +ARG PACK=0 +ARG TEST=0 + +RUN if [ "$PACK" = "1" ]; then make -C automation pack; fi +RUN if [ "$TEST" = "1" ]; then make -C automation test; fi + +RUN [ "cross-build-end" ] + +#---------------------------------------------------------------------------------------------- +FROM redisfab/redis-${ARCH}-${OSNICK}-xbuild:5.0.5 + +RUN [ "cross-build-start" ] + +RUN set -e; apt-get -qq update; apt-get -q install -y libgomp1 + +ENV REDIS_MODULES /usr/lib/redis/modules +ENV LD_LIBRARY_PATH $REDIS_MODULES + +RUN mkdir -p $REDIS_MODULES/ + +COPY --from=builder /build/install-cpu/ $REDIS_MODULES/ + +WORKDIR /data +EXPOSE 6379 +CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so"] + +RUN [ "cross-build-end" ] diff --git a/automation/Makefile b/automation/Makefile index f4398ce9c..a60ea609b 100755 --- a/automation/Makefile +++ b/automation/Makefile @@ -1,5 +1,19 @@ ROOT:=.. +include readies/mk/main + +MK_CMAKE:=1 +MK_CMAKE_INSTALL:=1 + +define HELP +make fetch # download and prepare dependant modules +make build # compile and link +make clean # remove build artifacts +make pack # create installation packages +make test # run tests +endef + +#---------------------------------------------------------------------------------------------- override GPU:=$(or $(findstring $(CUDA),1),$(findstring $(GPU),1)) @@ -14,89 +28,65 @@ DEPS_FLAGS=cpu DEVICE=cpu endif -GIT_BRANCH:=$(shell git rev-parse --abbrev-ref HEAD) -GIT_COMMIT:=$(shell git describe --always --abbrev=7 --dirty="+") - -ifeq ($(VERSION),) -PACK_VER:=$(GIT_BRANCH)-$(GIT_COMMIT) -else -PACK_VER:=$(VERSION) -endif +SRCDIR=.. +BINDIR=$(BINROOT)/src +INSTALL_DIR=$(ROOT)/install-$(DEVICE) -BINDIR=$(PWD)/build -INSTALL_DIR=$(PWD)/install-$(DEVICE) +TARGET=$(BINDIR)/redisai.so BACKENDS_PATH ?= $(INSTALL_DIR)/backends -RAMP:=ramp +CMAKE_FLAGS += \ + -DDEPS_PATH=$(realpath $(ROOT)/deps) \ + -DDEVICE=$(DEVICE) -ifeq ($(DEBUG),1) -CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=Debug -endif +include readies/mk/defs #---------------------------------------------------------------------------------------------- -.PHONY: all setup build clean deps pack pack_ramp pack_deps test +.PHONY: deps fetch pack pack_ramp pack_deps test -all: build +include $(MK)/rules -build: -ifeq ($(wildcard build/Makefile),) - mkdir -p $(ROOT)/build - cd $(ROOT)/build; \ - cmake -DDEVICE=$(DEVICE) -DDEPS_PATH=$(ROOT)/deps/install $(CMAKE_FLAGS) $(ROOT) -endif - $(MAKE) -C $(ROOT)/build - $(MAKE) -C $(ROOT)/build install +#---------------------------------------------------------------------------------------------- + +$(TARGET): $(MK_MAKEFILES) $(DEPS) + $(MAKE) -C $(BINDIR) + $(MAKE) -C $(BINDIR) install cd $(ROOT) ;\ - [ ! -e install ] && ln -sf install-$(DEVICE) install + if [ ! -e install ]; then ln -sf install-$(DEVICE) install; fi clean: ifeq ($(ALL),1) cd $(ROOT) ;\ rm -rf build install deps/dlpack deps/install-$(DEVICE) deps/*.tar.gz deps/*.zip deps/*.tgz else - $(MAKE) -C $(ROOT)/build clean + $(SHOW)$(MAKE) -C $(BINDIR) clean endif -deps fetch: +#---------------------------------------------------------------------------------------------- + +fetch: @echo Fetching dependencies... - $(ROOT)/get_deps.sh $(DEPS_FLAGS) + $(SHOW)VERBOSE=$(_SHOW) ./get_deps.sh $(DEPS_FLAGS) -pack: pack_ramp pack_deps +#---------------------------------------------------------------------------------------------- + +pack: $(BINDIR)/BINDIR pack_ramp pack_deps + +$(BINDIR)/BINDIR: $(BIN_DIRS) + $(SHOW)echo $(BINDIR)>$(BINDIR)/BINDIR pack_ramp: - @[ ! -z `command -v redis-server` ] || { echo "Cannot find redis-server - aborting."; exit 1; } - @echo "Building RAMP file ..." - @set -e ;\ - cd $(ROOT) ;\ - RAMPOUT=$$(mktemp /tmp/ramp.XXXXXX) ;\ - $(RAMP) pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" $(INSTALL_DIR)/redisai.so 2> /dev/null | grep '.zip' > $$RAMPOUT ;\ - tail -1 $$RAMPOUT > $(BINDIR)/PACKAGE ;\ - cat $(BINDIR)/PACKAGE | sed -e "s/[^.]*\.[^.]*\.\(.*\)\.zip/\1/" > $(BINDIR)/VERSION ;\ - VERSION=$$(cat $(BINDIR)/VERSION) ;\ - $(RAMP) pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" \ - -c "BACKENDSPATH /opt/redislabs/lib/redisai-cpu-$$VERSION/backends" $(INSTALL_DIR)/redisai.so 2> /dev/null | grep '.zip' > $$RAMPOUT ;\ - rm -f $$RAMPOUT ;\ - echo "Done." + $(SHOW)RAMP=1 DEVICE=$(DEVICE) BINDIR=$(BINROOT) ./pack.sh pack_deps: pack_ramp - @echo "Building dependencies file ..." - @set -e ;\ - cd $(ROOT) ;\ - PACK_FNAME=$$(basename `cat $(BINDIR)/PACKAGE`) ;\ - ARCHOSVER=$$(echo "$$PACK_FNAME" | sed -e "s/^redisai\.\([^.]*\..*\)\.zip/\1/") ;\ - VERSION=$$(cat $(BINDIR)/VERSION) ;\ - cd install-$(DEVICE) ;\ - rm -rf redisai-$(DEVICE)-$$VERSION ;\ - mkdir redisai-$(DEVICE)-$$VERSION ;\ - mv backends redisai-$(DEVICE)-$$VERSION ;\ - ln -sf redisai-$(DEVICE)-$$VERSION/backends backends ;\ - find redisai-$(DEVICE)-$$VERSION -name "*.so*" | xargs tar pczf redisai-$(DEVICE)-dependencies.$$ARCHOSVER.tgz ;\ - echo "Done." + $(SHOW)DEPS=1 DEVICE=$(DEVICE) BINDIR=$(BINROOT) ./pack.sh + +#---------------------------------------------------------------------------------------------- test: - @git lfs install + @[[ $(git lfs env > /dev/null 2>&1 ; echo $?) != 0 ]] && git lfs install @git lfs pull @set -e ;\ cd $(ROOT)/test ;\ diff --git a/automation/pack.sh b/automation/pack.sh new file mode 100755 index 000000000..91b0e01fe --- /dev/null +++ b/automation/pack.sh @@ -0,0 +1,133 @@ +#!/bin/bash + +[[ -z $DEVICE ]] && { echo DEVICE undefined; exit 1; } +[[ -z $BINDIR ]] && { echo BINDIR undefined; exit 1; } + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +. $HERE/../automation/readies/shibumi/functions +ROOT=$(realpath $HERE/..) + +RAMP_PROG=ramp +REDIS_ENT_LIB_PATH=/opt/redislabs/lib + +BINDIR=$(realpath $BINDIR) + +pack_ramp() { + echo "Building RAMP file ..." + cd $ROOT + RAMPOUT=$(mktemp /tmp/ramp.XXXXXX) + echo BINDIR=$BINDIR + $RAMP_PROG pack -m $ROOT/ramp.yml -o $BINDIR/$PRODUCT.{os}-{architecture}.{semantic_version}.zip $INSTALL_DIR/redisai.so 2> /dev/null | grep '.zip' > $RAMPOUT + realpath $(tail -1 $RAMPOUT) > $BINDIR/PACKAGE + cat $BINDIR/PACKAGE | sed -e "s/[^.]*\.[^.]*\.\(.*\)\.zip/\1/" > $BINDIR/VERSION + VERSION=$(cat $BINDIR/VERSION) + $RAMP_PROG pack -m $ROOT/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" \ + -c "BACKENDSPATH $REDIS_ENT_LIB_PATH/redisai-cpu-$VERSION/backends" $INSTALL_DIR/redisai.so 2> /dev/null | grep '.zip' > $RAMPOUT + rm -f $RAMPOUT + echo "Done." +} + +pack_deps() { + echo "Building dependencies file ..." + + cd $ROOT + PACK_FNAME=$(basename `cat $BINDIR/PACKAGE`) + ARCHOSVER=$(echo "$PACK_FNAME" | sed -e "s/^redisai\.\([^.]*\..*\)\.zip/\1/") + VERSION=$(cat $BINDIR/VERSION) + cd install-$DEVICE + if [[ ! -h backends ]]; then + [[ ! -d backends ]] && { echo "install-$DEVICE/backend directory not found." ; exit 1; } + rm -rf redisai-$DEVICE-$VERSION + mkdir redisai-$DEVICE-$VERSION + + mv backends redisai-$DEVICE-$VERSION + ln -s redisai-$DEVICE-$VERSION/backends backends + fi + find redisai-$DEVICE-$VERSION -name "*.so*" | xargs tar pczf redisai-$DEVICE-dependencies.$ARCHOSVER.tgz + echo "Done." + } + +set -e +[[ $VERBOSE == 1 ]] && set -x + +if [[ $1 == --help || $1 == help ]]; then + cat <<-END + pack.sh [cpu|gpu] [--help|help] + + Argument variables: + DEVICE=cpu|gpu + BRANCH=branch branch names to serve as an exta package tag + INTO=dir package destination directory (optinal) + RAMP=1 build RAMP file + DEPS=1 build dependencies file + RAMP_PROG path to RAMP program + + END + exit 0 +fi + +# BINDIR=$(cat $ROOT/BINDIR) +BIN=$ROOT/bin +INSTALL_DIR=$ROOT/install + +PRODUCT=redisai +PRODUCT_LIB=$PRODUCT.so + +GIT_VER="" +if [[ -d $ROOT/.git ]]; then + GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + GIT_COMMIT=$(git describe --always --abbrev=7 --dirty="+") + GIT_VER="${GIT_BRANCH}-${GIT_COMMIT}" +fi + +# OSX="" +# if [[ $(./deps/readies/bin/platform --os) == macosx ]]; then +# OSX=1 +# export PATH=$PATH:$HOME/Library/Python/2.7/bin +# fi + +if ! command -v redis-server > /dev/null; then + echo "Cannot find redis-server. Aborting." + exit 1 +fi + +pack_ramp +pack_deps +echo "Done." +exit 0 + + +cd "$BINDIR" +ln -s $PACK_FNAME $PRODUCT.latest.zip +ln -s $DEPS_FNAME $PRODUCT-dependencies.latest.tgz + +if [[ ! -z $BRANCH ]]; then + ln -s $PACK_FNAME $PRODUCT.${BRANCH}.zip + ln -s $DEPS_FNAME $PRODUCT-dependencies.${BRANCH}.tgz +fi +ln -s $PACK_FNAME $PRODUCT.${GIT_VER}.zip +ln -s $DEPS_FNAME $PRODUCT-dependencies.${GIT_VER}.tgz + +RELEASE_ARTIFACTS=\ + $PACK_FNAME $DEPS_FNAME \ + $PRODUCT.latest.zip $PRODUCT-dependencies.latest.tgz + +DEV_ARTIFACTS=\ + $PRODUCT.${BRANCH}.zip $PRODUCT-dependencies.${BRANCH}.tgz \ + $PRODUCT.${GIT_VER}.zip $PRODUCT-dependencies.${GIT_VER}.tgz + +if [[ ! -z $INTO ]]; then + INTO=$(realpath $INTO) + mkdir -p $INTO/release $INTO/branch + cd $INTO/release + foreach f in ($RELEASE_ARTIFACTS) + ln -s $f + end + + cd $INTO/branch + foreach f in ($DEV_ARTIFACTS) + ln -s $f + end +fi + +echo "Done." diff --git a/automation/readies/mk/build.defs b/automation/readies/mk/build.defs new file mode 100755 index 000000000..e69de29bb diff --git a/automation/readies/mk/build.rules b/automation/readies/mk/build.rules new file mode 100755 index 000000000..1a3c31e2b --- /dev/null +++ b/automation/readies/mk/build.rules @@ -0,0 +1,11 @@ + +.PHONY: build clean + +build: $(MK_MAKEFILES) $(TARGET) + +#clean: +#ifeq ($(ALL),1) +# $(SHOW)rm -rf $(BINDIR) +#else +# $(SHOW)$(MAKE) clean -C $(BUILD_DIR) +#endif \ No newline at end of file diff --git a/automation/readies/mk/cmake.defs b/automation/readies/mk/cmake.defs new file mode 100755 index 000000000..0ed2ecad0 --- /dev/null +++ b/automation/readies/mk/cmake.defs @@ -0,0 +1,14 @@ + +ifneq ($(MK_CMAKE),) + +MK_MAKEFILES += $(BINDIR)/Makefile + +ifeq ($(DEBUG),1) +CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=Debug +endif + +ifeq ($(WHY),1) +CMAKE_WHY=--trace-expand +endif + +endif # MK_CMAKE diff --git a/automation/readies/mk/cmake.rules b/automation/readies/mk/cmake.rules new file mode 100755 index 000000000..f6d8d7e60 --- /dev/null +++ b/automation/readies/mk/cmake.rules @@ -0,0 +1,11 @@ + +ifneq ($(MK_CMAKE),) + +$(BINDIR)/Makefile : $(SRCDIR)/CMakeLists.txt + $(SHOW)cd $(BINDIR); cmake $(CMAKE_WHY) $(CMAKE_FLAGS) $(realpath $(SRCDIR)) + +# $(SHOW)cd $(BINDIR); \ +# CMAKE_REL_ROOT=`python -c "import os; print os.path.relpath('$(SRCDIR)', '$$PWD')"` \ +# cmake $(CMAKE_FLAGS) $$CMAKE_REL_ROOT + +endif # MK_CMAKE diff --git a/automation/readies/mk/configure.rules b/automation/readies/mk/configure.rules index baf4ce4d4..dd5278ea2 100755 --- a/automation/readies/mk/configure.rules +++ b/automation/readies/mk/configure.rules @@ -1,5 +1,7 @@ -build: __sep bindirs +.PHONY: build clean + +build: $(MK_MAKEFILES) $(TARGET) ifeq (,$(wildcard $(BUILD_DIR)/Makefile)) $(SHOW)cd $(BUILD_DIR); $(realpath $(SRCDIR))/configure $(CONFIGURE_FLAGS) endif diff --git a/automation/readies/mk/defs b/automation/readies/mk/defs index 819d0d30c..c4f0d4016 100755 --- a/automation/readies/mk/defs +++ b/automation/readies/mk/defs @@ -1,6 +1,4 @@ -MK=$(ROOT)/automation/readies/mk - -include $(MK)/common.defs -include $(MK)/variant.defs -include $(MK)/bindirs.defs +include $(MK)/help.defs +include $(MK)/build.defs +include $(MK)/cmake.defs diff --git a/automation/readies/mk/help.rules b/automation/readies/mk/help.rules index eae2fed21..f44ae0c85 100755 --- a/automation/readies/mk/help.rules +++ b/automation/readies/mk/help.rules @@ -5,7 +5,9 @@ ifneq ($(HELP),) help: $(file >$(HELPFILE),$(HELP)) + @echo @cat $(HELPFILE) + @echo @-rm -f $(HELPFILE) endif diff --git a/automation/readies/mk/main b/automation/readies/mk/main new file mode 100755 index 000000000..336de23c4 --- /dev/null +++ b/automation/readies/mk/main @@ -0,0 +1,8 @@ + +MK=$(ROOT)/automation/readies/mk + +MK_ALL_TARGETS:=bindirs build + +include $(MK)/common.defs +include $(MK)/variant.defs +include $(MK)/bindirs.defs diff --git a/automation/readies/mk/rules b/automation/readies/mk/rules index 1aa2985e3..e433c92b2 100755 --- a/automation/readies/mk/rules +++ b/automation/readies/mk/rules @@ -1,4 +1,11 @@ +.PHONY: all + +all: $(MK_ALL_TARGETS) + include $(MK)/common.rules include $(MK)/variant.rules include $(MK)/bindirs.rules +include $(MK)/help.rules +include $(MK)/build.rules +include $(MK)/cmake.rules diff --git a/automation/system-setup.py b/automation/system-setup.py index 03fbb8705..588811355 100755 --- a/automation/system-setup.py +++ b/automation/system-setup.py @@ -52,10 +52,10 @@ def install_git_lfs_on_linux(self): self.install("git-lfs") def common_last(self): - if not self.has_command("ramp"): - self.pip3_install("git+https://github.com/RedisLabs/RAMP --upgrade") if not self.has_command("RLTest"): self.pip3_install("git+https://github.com/RedisLabsModules/RLTest.git@master") + if not self.has_command("ramp"): + self.pip3_install("git+https://github.com/RedisLabs/RAMP@master") self.pip3_install("-r test/test_requirements.txt") #---------------------------------------------------------------------------------------------- diff --git a/build/bazel/Dockerfile b/build/bazel/Dockerfile new file mode 100755 index 000000000..7748006b7 --- /dev/null +++ b/build/bazel/Dockerfile @@ -0,0 +1,32 @@ +# BUILD redisfab/bazel-${ARCH}:$(BAZ_VER) + +ARG OSNICK=buster +ARG ARCH=arm64v8 + +FROM redisfab/${ARCH}-xbuild:${OSNICK} + +RUN [ "cross-build-start" ] + +ARG BAZ_VER=0.24.1 + +WORKDIR /build + +RUN set -e ;\ + apt-get -qq update ;\ + apt-get -q install -y pkg-config g++ zlib1g-dev zip unzip python wget ;\ + apt-get -q install -y openjdk-11-jdk + +RUN set -e ;\ + wget -q https://github.com/bazelbuild/bazel/releases/download/0.24.1/bazel-0.24.1-dist.zip ;\ + unzip bazel-0.24.1-dist.zip + +RUN set -e ;\ + env EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" bash ./compile.sh ;\ + copy bazel to /usr/local/bin + +RUN [ "cross-build-end" ] + +# FROM insready/bazel:latest as baz +# FROM ubuntu:latest +# COPY --from=baz /usr/bin/bazel* /usr/local/bin/ + diff --git a/build/docker/Makefile b/build/docker/Makefile new file mode 100755 index 000000000..2af3163f6 --- /dev/null +++ b/build/docker/Makefile @@ -0,0 +1,99 @@ + +.NOTPARALLEL: + +ROOT=../.. + +ifeq ($(VERSION),) +VERSION:=$(patsubst v%,%,$(shell git describe --tags `git rev-list --tags --max-count=1`)) +endif +ifeq ($(VERSION),) +$(error Cannot determine version. Aborting.) +endif + +# OSNICK=buster|stretch|bionic +OSNICK ?= buster + +REPO=redisfab +STEM=$(REPO)/redisai-cpu +BUILD_OPT=--rm +# --squash + +#---------------------------------------------------------------------------------------------- + +define targets # (1=OP, 2=op) +$(1)_TARGETS := +$(1)_TARGETS += $(if $(findstring $(X64),1),$(2)_x64) +$(1)_TARGETS += $(if $(findstring $(ARM7),1),$(2)_arm32v7) +$(1)_TARGETS += $(if $(findstring $(ARM8),1),$(2)_arm64v8) + +$(1)_TARGETS += $$(if $$(strip $$($(1)_TARGETS)),,$(2)_x64 $(2)_arm32v7 $(2)_arm64v8) +endef + +$(eval $(call targets,BUILD,build)) +$(eval $(call targets,PUSH,push)) + +#---------------------------------------------------------------------------------------------- + +define build_x64 # (1=arch) +build_$(1): + @docker build $(BUILD_OPT) -t $(STEM)-$(OSNICK):$(VERSION)-x64 -f $(ROOT)/Dockerfile $(ROOT) + +.PHONY: build_$(1) +endef + +define build_arm # (1=arch) +build_$(1): + @docker build $(BUILD_OPT) -t $(STEM)-$(OSNICK):$(VERSION)-$(1) -f $(ROOT)/Dockerfile.arm \ + --build-arg ARCH=$(1) $(ROOT) + +.PHONY: build_$(1) +endef + +define push # (1=arch) +push_$(1): + @docker push $(STEM)-$(OSNICK):$(VERSION)-$(1) + +.PHONY: push_$(1) +endef + +define create_manifest # (1=version) +docker manifest create -a $(STEM)-$(OSNICK):$(1) \ + -a $(STEM)-$(OSNICK):$(VERSION)-x64 \ + -a $(STEM)-$(OSNICK):$(VERSION)-arm64v8 +# -a $(STEM)-$(OSNICK):$(VERSION)-arm32v7 +# docker manifest annotate $(STEM)-$(OSNICK):$(1) $(STEM)-$(OSNICK):$(VERSION)-arm32v7 --os linux --arch arm --variant v7 +docker manifest annotate $(STEM)-$(OSNICK):$(1) $(STEM)-$(OSNICK):$(VERSION)-arm64v8 --os linux --arch arm64 --variant v8 +docker manifest push -p $(STEM)-$(OSNICK):$(1) +endef + +#---------------------------------------------------------------------------------------------- + +all: build publish + +$(eval $(call build_x64,x64)) +$(eval $(call build_arm,arm64v8)) +$(eval $(call build_arm,arm32v7)) + +$(eval $(call push,x64)) +$(eval $(call push,arm64v8)) +$(eval $(call push,arm32v7)) + +build: $(BUILD_TARGETS) + +ifneq ($(PUSH),0) +publish: $(PUSH_TARGETS) +else +publish: +endif + $(call create_manifest,$(VERSION)) + $(call create_manifest,latest) + +show: + @echo "$(STEM)-$(OSNICK):" +ifeq ($(INSPECT),1) + @docker manifest inspect $(STEM)-$(OSNICK):$(VERSION) | jq +else + @curl -s -X "GET" https://cloud.docker.com/v2/repositories/$(STEM)-$(OSNICK)/tags/ | jq +endif + +.PHONY: build public show diff --git a/build/libtorch/Dockerfile.arm b/build/libtorch/Dockerfile.arm new file mode 100755 index 000000000..f24190c98 --- /dev/null +++ b/build/libtorch/Dockerfile.arm @@ -0,0 +1,52 @@ +# BUILD redisfab/libtorch-cpu-${ARCH}:$(PT_VER) + +# stretch|bionic|buster +ARG OSNICK=buster + +# arm32v7|arm64v8 +ARG ARCH=arm64v8 + +#---------------------------------------------------------------------------------------------- +FROM redisfab/${ARCH}-xbuild:${OSNICK} as builder + +ARG PT_VER=1.1.0 + +RUN [ "cross-build-start" ] + +WORKDIR /build + +RUN set -e ;\ + apt-get -qq update ;\ + apt-get -q install -y git build-essential ninja-build cmake python3-pip python3-cffi +RUN pip3 install setuptools pyyaml typing +RUN pip3 install numpy +# RUN pip3 install mkl mkl-include + +RUN set -e ;\ + git clone https://github.com/pytorch/pytorch.git ;\ + cd pytorch ;\ + git checkout "v${PT_VER}" ;\ + git submodule update --init --recursive + +ENV BUILD_PYTHON=0 +ENV USE_GLOO=1 +ENV USE_OPENCV=0 +ENV BUILD_TORCH=ON +ENV BUILD_BINARY=ON +ENV BUILD_CAFFE2_OPS=ON +ENV NO_CUDA=1 +ENV NO_DISTRIBUTED=1 +ENV NO_MKLDNN=1 +ENV NO_NNPACK=1 +ENV NO_QNNPACK=1 + +RUN set -e ;\ + cd pytorch ;\ + python3 setup.py install + +ADD ./deps/readies/ /build/deps/readies/ +ADD ./build/libtorch-arm/collect.py /build/ + +RUN ./collect.py + +RUN [ "cross-build-end" ] diff --git a/build/libtorch/Dockerfile.x64 b/build/libtorch/Dockerfile.x64 new file mode 100755 index 000000000..eab122c8e --- /dev/null +++ b/build/libtorch/Dockerfile.x64 @@ -0,0 +1,42 @@ +ARG OS=debian:buster + +#---------------------------------------------------------------------------------------------- +FROM ${OS} + +ARG PT_VER=1.1.0 + +WORKDIR /build + +RUN set -e ;\ + apt-get -qq update ;\ + apt-get -qq install -y git build-essential ninja-build cmake python3-pip python3-cffi +RUN pip3 install setuptools pyyaml typing +RUN pip3 install numpy +RUN pip3 install mkl mkl-include + +RUN set -e ;\ + git clone https://github.com/pytorch/pytorch.git ;\ + cd pytorch ;\ + git checkout "v${PT_VER}" ;\ + git submodule update --init --recursive + +ENV BUILD_PYTHON=0 +ENV USE_GLOO=1 +ENV USE_OPENCV=0 +ENV BUILD_TORCH=ON +ENV BUILD_BINARY=ON +ENV BUILD_CAFFE2_OPS=ON +ENV NO_CUDA=1 +ENV NO_DISTRIBUTED=1 +ENV NO_MKLDNN=1 +ENV NO_NNPACK=1 +ENV NO_QNNPACK=1 + +RUN set -e ;\ + cd pytorch ;\ + python3 setup.py install + +ADD ./automation/readies/ /build/deps/readies/ +ADD ./build/libtorch-arm/collect.py /build/ + +RUN ./collect.py diff --git a/build/libtorch/Makefile b/build/libtorch/Makefile new file mode 100755 index 000000000..301aa3bcb --- /dev/null +++ b/build/libtorch/Makefile @@ -0,0 +1,95 @@ + +ROOT=../.. + +VERSION ?= 1.2.0 +OSNICK ?= buster + +#---------------------------------------------------------------------------------------------- + +S3_URL=redismodules/pytorch + +STEM=libtorch-cpu-linux + +OS.bionic=ubuntu:bionic +OS.stretch=debian:stretch-slim +OS.buster=debian:buster-slim +OS=$(OS.$(OSNICK)) + +#---------------------------------------------------------------------------------------------- + +define targets # (1=OP, 2=op) +$(1)_TARGETS := +$(1)_TARGETS += $(if $(findstring $(X64),1),$(2)_x64) +$(1)_TARGETS += $(if $(findstring $(ARM7),1),$(2)_arm32v7) +$(1)_TARGETS += $(if $(findstring $(ARM8),1),$(2)_arm64v8) + +$(1)_TARGETS += $$(if $$(strip $$($(1)_TARGETS)),,$(2)_x64 $(2)_arm32v7 $(2)_arm64v8) +endef + +$(eval $(call targets,BUILD,build)) +$(eval $(call targets,PUBLISH,publish)) + +#---------------------------------------------------------------------------------------------- + +define build_x64 # (1=arch, 2=tar-arch) +build_x64: + @docker build --iidfile $$(IID_$(1)) -t redisfab/$(STEM)-$(1):$(VERSION) -f Dockerfile.x64 \ + --build-arg OS=$(OS) $(ROOT) + @docker create --cidfile $$(CID_$(1)) `cat $$(IID_$(1))` + @docker cp `cat $$(CID_$(1))`:/build/$(STEM)-$(2)-$(VERSION).tar.gz . + +.PHONY: build_x64 +endef + +define build_arm # (1=arch, 2=tar-arch) +IID_$(1)=$(1)_$(VERSION).iid +CID_$(1)=$(1)_$(VERSION).cid + +build_$(1): + @docker build --iidfile $$(IID_$(1)) -t redisfab/$(STEM)-$(1):$(VERSION) -f Dockerfile.arm \ + --build-arg ARCH=$(1) $(ROOT) + @docker create --cidfile $$(CID_$(1)) `cat $$(IID_$(1))` + @docker cp `cat $$(CID_$(1))`:/build/$(STEM)-$(2)-$(VERSION).tar.gz . + +.PHONY: build_$(1) +endef + +#---------------------------------------------------------------------------------------------- + +define publish_x64 # (1=arch, 2=tar-arch) +publish_x64: + @aws s3 cp $(STEM)-$(2)-$(VERSION).tar.gz s3://$(S3_URL)/ --acl public-read + +.PHONY: publish_x64 +endef + +define publish_arm # (1=arch, 2=tar-arch) +publish_$(1): + @aws s3 cp $(STEM)-$(2)-$(VERSION).tar.gz s3://$(S3_URL)/ --acl public-read + +.PHONY: publish_$(1) +endef + +#---------------------------------------------------------------------------------------------- + +all: build publish + +build: $(BUILD_TARGETS) + +$(eval $(call build_x64,x64,x86_64)) +$(eval $(call build_arm,arm64v8,arm64)) +$(eval $(call build_arm,arm32v7,arm)) + +publish: $(PUBLISH_TARGETS) + +$(eval $(call publish_x64,x64,x86_64)) +$(eval $(call publish_arm,arm64v8,arm64)) +$(eval $(call publish_arm,arm32v7,arm)) + +repack: + @PT_VERSION=$(VERSION) ./repack.sh + +help: + @echo "make [build|repack|publish] [X64=1|ARM7=1|ARM8=1]" + +.PHONY: all build repack publish help diff --git a/build/libtorch/collect.py b/build/libtorch/collect.py new file mode 100755 index 000000000..fe6c3c7c3 --- /dev/null +++ b/build/libtorch/collect.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +import os +import sys +import argparse +from pathlib import Path +import shutil +import tarfile + +# this refers to deps directory inside a container +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "deps/readies")) +import paella + +#---------------------------------------------------------------------------------------------- + +PYTORCH_VERSION = '1.2.0' + +parser = argparse.ArgumentParser(description='Prepare RedisAI dependant distribution packages.') +parser.add_argument('--pytorch', default='pytorch', help='root of pytorch repository') +parser.add_argument('--pytorch-ver', default=PYTORCH_VERSION, help='pytorch version') +parser.add_argument('--deps', default='.', help='destination directory') +parser.add_argument('-n', '--nop', action="store_true", help='no operation') +args = parser.parse_args() + +#---------------------------------------------------------------------------------------------- + +pytorch = Path(args.pytorch).resolve() +dest = Path(args.deps).resolve() + +#---------------------------------------------------------------------------------------------- + +pt_build='cpu' + +platform = paella.Platform() + +pt_os = platform.os +if pt_os == 'macosx': + pt_os = 'darwin' + +pt_arch = platform.arch +if pt_arch == 'x64': + pt_arch = 'x86_64' +elif pt_arch == 'arm64v8': + pt_arch = 'arm64' +elif pt_arch == 'arm32v7': + pt_arch = 'arm' + +pt_ver = args.pytorch_ver + +#---------------------------------------------------------------------------------------------- + +def copy_p(src, dest): + f = dest/src + paella.mkdir_p(os.path.dirname(f)) + shutil.copy(src, f, follow_symlinks=False) + +def create_tar(name, basedir, dir='.'): + def reset_uid(tarinfo): + tarinfo.uid = tarinfo.gid = 0 + tarinfo.uname = tarinfo.gname = "root" + return tarinfo + with cwd(basedir): + with tarfile.open(name, 'w:gz') as tar: + tar.add(dir, filter=reset_uid) + +def collect_pytorch(): + d_pytorch = dest/'libtorch' + with cwd(pytorch/'torch/include'): + for f in Path('.').glob('**/*.h'): + copy_p(f, d_pytorch/'include') + with cwd(pytorch/'torch/lib'): + for f in Path('.').glob('*.a'): + copy_p(f, d_pytorch/'lib') + for f in Path('.').glob('*.so*'): + copy_p(f, d_pytorch/'lib') + with cwd(pytorch/'torch'): + shutil.copytree('share', d_pytorch/'share', ignore_dangling_symlinks=True) + create_tar('libtorch-{}-{}-{}-{}.tar.gz'.format(pt_build, pt_os, pt_arch, pt_ver), dest, 'libtorch') + +#---------------------------------------------------------------------------------------------- + +collect_pytorch() diff --git a/build/libtorch/repack.sh b/build/libtorch/repack.sh new file mode 100755 index 000000000..910801191 --- /dev/null +++ b/build/libtorch/repack.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +set -e +[[ $VERBOSE == 1 ]] && set -x + +ROOT=../.. + +if [[ "$1" == "cpu" ]]; then + GPU=no +elif [[ "$1" == "gpu" ]]; then + GPU=yes +else + GPU=${GPU:-no} +fi + +OS=$(python3 $ROOT/automation/readies/bin/platform --os) +ARCH=$(python3 $ROOT/automation/readies/bin/platform --arch) + +if [[ -z $PT_VERSION ]]; then + PT_VERSION=1.2.0 + #PT_VERSION="latest" +fi + +if [[ $OS == linux ]]; then + PT_OS=shared-with-deps + if [[ $GPU == no ]]; then + PT_BUILD=cpu + else + PT_BUILD=cu90 + fi + if [[ $ARCH == x64 ]]; then + PT_ARCH=x86_64 + fi +elif [[ $OS == macosx ]]; then + PT_OS=macos + PT_BUILD=cpu +fi + +[[ "$PT_VERSION" == "latest" ]] && PT_BUILD=nightly/${PT_BUILD} + +LIBTORCH_ARCHIVE=libtorch-${PT_OS}-${PT_VERSION}.zip +[[ -z $LIBTORCH_URL ]] && LIBTORCH_URL=https://download.pytorch.org/libtorch/$PT_BUILD/$LIBTORCH_ARCHIVE + +if [ ! -f $LIBTORCH_ARCHIVE ]; then + echo "Downloading libtorch ${PT_VERSION} ${PT_BUILD}" + wget $LIBTORCH_URL +fi + +if [[ $OS == linux ]]; then + PT_OS=linux +fi + +unzip -q -o ${LIBTORCH_ARCHIVE} +tar czf libtorch-${PT_BUILD}-${PT_OS}-${PT_ARCH}-${PT_VERSION}.tar.gz libtorch/ +rm -rf libtorch/ ${LIBTORCH_ARCHIVE} diff --git a/build/onnxruntime/Dockerfile.arm b/build/onnxruntime/Dockerfile.arm new file mode 100755 index 000000000..8ef38268b --- /dev/null +++ b/build/onnxruntime/Dockerfile.arm @@ -0,0 +1,53 @@ +# BUILD redisfab/onnxruntime-${ARCH}-${OSNICK}:$(ONNXRUNTIME_VER) + +# stretch|bionic|buster +ARG OSNICK=buster + +# arm32v7|arm64v8 +ARG ARCH=arm64v8 + +#---------------------------------------------------------------------------------------------- +FROM redisfab/${ARCH}-xbuild:${OSNICK} as builder + +ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime +ARG ONNXRUNTIME_BRANCH=rel-0.4.0 +ARG ONNXRUNTIME_VER=0.4.0 +ARG ARCH_FLAG="--arm64" + +RUN [ "cross-build-start" ] + +RUN apt-get -qq update +RUN apt-get -qq install -y curl wget tar git patch +RUN apt-get -qq install -y build-essential cmake +RUN apt-get -qq install -y libcurl4-openssl-dev libssl-dev libatlas-base-dev zlib1g-dev + +RUN apt-get -qq install -y python3 python3-pip python3-dev +RUN pip3 install --upgrade pip setuptools wheel +RUN pip3 install numpy + +WORKDIR /build + +ADD ./*.patch /build/ +ADD ./pack.sh /build/ + +ARG BUILDTYPE=MinSizeRel +ARG BUILDARGS="--config ${BUILDTYPE} ${ARM_FLAG} --parallel" + +RUN set -e ;\ + git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXRUNTIME_REPO} onnxruntime ;\ + cd onnxruntime ;\ + git checkout "rel-${ONNXRUNTIME_VER}" + +RUN set -e ;\ + cd onnxruntime ;\ + patch -p1 -i ../werrror.patch + +RUN set -e ;\ + cd onnxruntime ;\ + ./build.sh ${BUILDARGS} --update --build ;\ + ./build.sh ${BUILDARGS} --build_shared_lib +# RUN ./build.sh ${BUILDARGS} --enable_pybind --build_wheel + +RUN ./pack.sh ${ONNXRUNTIME_VER} + +RUN [ "cross-build-end" ] diff --git a/build/onnxruntime/Dockerfile.arm7 b/build/onnxruntime/Dockerfile.arm7 new file mode 100755 index 000000000..0b4a06947 --- /dev/null +++ b/build/onnxruntime/Dockerfile.arm7 @@ -0,0 +1,46 @@ +FROM balenalib/raspberrypi3-python:latest-stretch-build + +ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime +ARG ONNXRUNTIME_BRANCH=rel-0.4.0 +ARG ONNXRUNTIME_VER=0.4.0 +ARG ARCH_FLAG="--arm" + +RUN [ "cross-build-start" ] + +RUN set -e ;\ + apt-get -qq update \; + apt-get -q install -y + git tar wget curl \ + build-essential cmake \ + python3 python3-pip python3-dev \ + libcurl4-openssl-dev libssl-dev libatlas-base-dev + +RUN pip3 install --upgrade pip setuptools wheel +RUN pip3 install numpy + +WORKDIR /build + +ADD ./*.patch /build/ +ADD ./pack.sh /build/ + +ARG BUILDTYPE=MinSizeRel +ARG BUILDARGS="--config ${BUILDTYPE} ${ARM_FLAG} --parallel" + +RUN set -e ;\ + git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXRUNTIME_REPO} onnxruntime ;\ + cd onnxruntime ;\ + git checkout "rel-${ONNXRUNTIME_VER}" + +RUN set -e ;\ + cd onnxruntime ;\ + patch -p1 -i ../werrror.patch + +RUN set -e ;\ + cd onnxruntime ;\ + ./build.sh ${BUILDARGS} --update --build ;\ + ./build.sh ${BUILDARGS} --build_shared_lib +# RUN ./build.sh ${BUILDARGS} --enable_pybind --build_wheel + +RUN ./pack.sh ${ONNXRUNTIME_VER} + +RUN [ "cross-build-end" ] diff --git a/build/onnxruntime/Dockerfile.x64 b/build/onnxruntime/Dockerfile.x64 new file mode 100755 index 000000000..c03080aac --- /dev/null +++ b/build/onnxruntime/Dockerfile.x64 @@ -0,0 +1,36 @@ +# BUILD redisfab/onnxruntime-x64-${OS}:$(ONNXRUNTIME_VER) + +ARG OS=debian:buster + +#---------------------------------------------------------------------------------------------- +FROM ${OS} + +ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime +ARG ONNXRUNTIME_BRANCH=rel-0.4.0 +ARG ONNXRUNTIME_VER=0.4.0 + +RUN apt-get -qq update +RUN apt-get -qq install -y curl wget tar git +RUN apt-get -qq install -y build-essential cmake +RUN apt-get -qq install -y libcurl4-openssl-dev libssl-dev libatlas-base-dev zlib1g-dev + +RUN apt-get -qq install -y python3 python3-pip python3-dev +RUN pip3 install --upgrade pip setuptools wheel +RUN pip3 install numpy + +WORKDIR /build + +ADD ./pack.sh /build/ +ARG BUILDTYPE=MinSizeRel +ARG BUILDARGS="--config ${BUILDTYPE} --parallel" + +RUN set -e ;\ + git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXRUNTIME_REPO} onnxruntime ;\ + cd onnxruntime ;\ + git checkout "rel-${ONNXRUNTIME_VER}" + cd onnxruntime ;\ + ./build.sh ${BUILDARGS} --update --build ;\ + ./build.sh ${BUILDARGS} --build_shared_lib +# RUN ./build.sh ${BUILDARGS} --enable_pybind --build_wheel + +RUN ./pack.sh ${ONNXRUNTIME_VER} diff --git a/build/onnxruntime/Makefile b/build/onnxruntime/Makefile new file mode 100755 index 000000000..e6960b856 --- /dev/null +++ b/build/onnxruntime/Makefile @@ -0,0 +1,72 @@ + +ROOT=. + +VERSION=0.4.0 + +S3_URL=redismodules/onnxruntime + +STEM=onnxruntime-linux + +KEEP ?= 1 + +#---------------------------------------------------------------------------------------------- + +define targets # (1=OP, 2=op) +$(1)_TARGETS := +$(1)_TARGETS += $(if $(findstring $(X64),1),$(2)_x64) +$(1)_TARGETS += $(if $(findstring $(ARM7),1),$(2)_arm32v7) +$(1)_TARGETS += $(if $(findstring $(ARM8),1),$(2)_arm64v8) + +$(1)_TARGETS += $$(if $$(strip $$($(1)_TARGETS)),,$(2)_x64 $(2)_arm32v7 $(2)_arm64v8) +endef + +$(eval $(call targets,BUILD,build)) +$(eval $(call targets,PUBLISH,publish)) + +#---------------------------------------------------------------------------------------------- +# arch=x64|arm32v7|arm64v8 +# arch_1=x86_64|arm|arm64 + +define build_arm # (1=arch, 2=arch_1) +IID_$(1)=$(1)_$(VERSION).iid +CID_$(1)=$(1)_$(VERSION).cid + +build_$(1): + @docker build --iidfile $$(IID_$(1)) -t redisfab/$(STEM)-$(1):$(VERSION) -f Dockerfile.arm \ + --build-arg ARCH=$(1) --build-arg ARCH_FLAG=--$(2) $(ROOT) + @docker create --cidfile $$(CID_$(1)) `cat $$(IID_$(1))` + @docker cp `cat $$(CID_$(1))`:/build/$(STEM)-$(2)-$(VERSION).tgz . +ifneq ($(KEEP),1) + # @rm $$(IID_$(1)) $$(CID_$(1)) +endif + +.PHONY: build_$(1) +endef + +#---------------------------------------------------------------------------------------------- + +define publish_arm # (1=arch, 2=arch_1) +publish_$(1): + @aws s3 cp $(STEM)-$(2)-$(VERSION).tgz s3://$(S3_URL)/ --acl public-read +ifneq ($(KEEP),1) + # @rm $(STEM)-$(2)-$(VERSION).tgz +endif + +.PHONY: publish_$(1) +endef + +#---------------------------------------------------------------------------------------------- + +all: build publish + +build: $(BUILD_TARGETS) + +$(eval $(call build_arm,arm64v8,arm64)) +$(eval $(call build_arm,arm32v7,arm)) + +publish: $(PUBLISH_TARGETS) + +$(eval $(call publish_arm,arm64v8,arm64)) +$(eval $(call publish_arm,arm32v7,arm)) + +.PHONY: all build publish diff --git a/build/onnxruntime/pack.sh b/build/onnxruntime/pack.sh new file mode 100755 index 000000000..52eb55cb7 --- /dev/null +++ b/build/onnxruntime/pack.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e +VER="$1" + +mkdir -p pack/include pack/lib +cp onnxruntime/build/Linux/MinSizeRel/libonnxruntime.so.${VER} pack/lib/ +cp onnxruntime/docs/C_API.md pack/ +cp onnxruntime/LICENSE pack/ +cp onnxruntime/README.md pack/ +cp onnxruntime/ThirdPartyNotices.txt pack/ +cp onnxruntime/VERSION_NUMBER pack/ +cd onnxruntime/ +git rev-parse HEAD > ../pack/GIT_COMMIT_ID +cd .. +cp onnxruntime/include/onnxruntime/core/session/onnxruntime_c_api.h pack/include/ +cd pack/lib/ +ln -s libonnxruntime.so.${VER} libonnxruntime.so +cd ../.. +mv pack onnxruntime-linux-arm64-${VER} +tar czf onnxruntime-linux-arm64-${VER}.tgz onnxruntime-linux-arm64-${VER}/ diff --git a/build/onnxruntime/werrror.patch b/build/onnxruntime/werrror.patch new file mode 100755 index 000000000..be5eb10b7 --- /dev/null +++ b/build/onnxruntime/werrror.patch @@ -0,0 +1,758 @@ +diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt +index 96c89fe..f59edea 100644 +--- a/cmake/CMakeLists.txt ++++ b/cmake/CMakeLists.txt +@@ -411,8 +411,8 @@ else() + + if(onnxruntime_DEV_MODE) + if(NOT onnxruntime_USE_TVM) +- string(APPEND CMAKE_CXX_FLAGS " -Werror") +- string(APPEND CMAKE_C_FLAGS " -Werror") ++# string(APPEND CMAKE_CXX_FLAGS " -Werror") ++# string(APPEND CMAKE_C_FLAGS " -Werror") + endif() + endif() + check_cxx_compiler_flag(-Wunused-but-set-variable HAS_UNUSED_BUT_SET_VARIABLE) +Submodule cmake/external/eigen contains modified content +diff --git a/cmake/external/eigen/CMakeLists.txt b/cmake/external/eigen/CMakeLists.txt +index 76e083314..a515736cd 100644 +--- a/cmake/external/eigen/CMakeLists.txt ++++ b/cmake/external/eigen/CMakeLists.txt +@@ -151,10 +151,10 @@ if(NOT MSVC) + + # clang outputs some warnings for unknown flags that are not caught by check_cxx_compiler_flag + # adding -Werror turns such warnings into errors +- check_cxx_compiler_flag("-Werror" COMPILER_SUPPORT_WERROR) +- if(COMPILER_SUPPORT_WERROR) +- set(CMAKE_REQUIRED_FLAGS "-Werror") +- endif() ++# check_cxx_compiler_flag("-Werror" COMPILER_SUPPORT_WERROR) ++# if(COMPILER_SUPPORT_WERROR) ++# set(CMAKE_REQUIRED_FLAGS "-Werror") ++# endif() + ei_add_cxx_compiler_flag("-pedantic") + ei_add_cxx_compiler_flag("-Wall") + ei_add_cxx_compiler_flag("-Wextra") +Submodule cmake/external/googletest contains modified content +diff --git a/cmake/external/googletest/googletest/cmake/internal_utils.cmake b/cmake/external/googletest/googletest/cmake/internal_utils.cmake +index 6448918f..38e1a864 100644 +--- a/cmake/external/googletest/googletest/cmake/internal_utils.cmake ++++ b/cmake/external/googletest/googletest/cmake/internal_utils.cmake +@@ -94,7 +94,7 @@ macro(config_compiler_and_linker) + set(cxx_no_exception_flags "-EHs-c- -D_HAS_EXCEPTIONS=0") + set(cxx_no_rtti_flags "-GR-") + elseif (CMAKE_COMPILER_IS_GNUCXX) +- set(cxx_base_flags "-Wall -Wshadow -Werror") ++ set(cxx_base_flags "-Wall -Wshadow") + if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0.0) + set(cxx_base_flags "${cxx_base_flags} -Wno-error=dangling-else") + endif() +diff --git a/cmake/external/googletest/googletest/xcode/Config/General.xcconfig b/cmake/external/googletest/googletest/xcode/Config/General.xcconfig +index f23e3222..ac07337f 100644 +--- a/cmake/external/googletest/googletest/xcode/Config/General.xcconfig ++++ b/cmake/external/googletest/googletest/xcode/Config/General.xcconfig +@@ -17,7 +17,7 @@ ZERO_LINK = NO + PREBINDING = NO + + // Strictest warning policy +-WARNING_CFLAGS = -Wall -Werror -Wendif-labels -Wnewline-eof -Wno-sign-compare -Wshadow ++WARNING_CFLAGS = -Wall -Wendif-labels -Wnewline-eof -Wno-sign-compare -Wshadow + + // Work around Xcode bugs by using external strip. See: + // http://lists.apple.com/archives/Xcode-users/2006/Feb/msg00050.html +Submodule cmake/external/gsl contains modified content +diff --git a/cmake/external/gsl/tests/CMakeLists.txt b/cmake/external/gsl/tests/CMakeLists.txt +index 0e08d77..a4d103d 100644 +--- a/cmake/external/gsl/tests/CMakeLists.txt ++++ b/cmake/external/gsl/tests/CMakeLists.txt +@@ -41,7 +41,7 @@ target_compile_options(gsl_tests_config INTERFACE + -Wcast-align + -Wconversion + -Wctor-dtor-privacy +- -Werror ++# -Werror + -Wextra + -Wno-missing-braces + -Wnon-virtual-dtor +@@ -129,7 +129,7 @@ target_compile_options(gsl_tests_config_noexcept INTERFACE + -Wcast-align + -Wconversion + -Wctor-dtor-privacy +- -Werror ++# -Werror + -Wextra + -Wno-missing-braces + -Wnon-virtual-dtor +Submodule cmake/external/nsync contains modified content +diff --git a/cmake/external/nsync/builds/aarch64.linux.c++11/Makefile b/cmake/external/nsync/builds/aarch64.linux.c++11/Makefile +index d7b138b..d76d43f 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.c++11/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.c++11/Makefile +@@ -2,7 +2,7 @@ + # use futex directly. + CC=g++ + PLATFORM_CPPFLAGS=-DNSYNC_USE_CPP11_TIMEPOINT -DNSYNC_ATOMIC_CPP11 -I../../platform/c++11.futex -I../../platform/c++11 -I../../platform/gcc -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-std=c++11 -Werror -Wall -Wextra -pedantic ++PLATFORM_CFLAGS=-std=c++11 -Wall -Wextra -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M -std=c++11 + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/c++11/src/per_thread_waiter.cc ../../platform/c++11/src/yield.cc ../../platform/c++11/src/time_rep_timespec.cc ../../platform/c++11/src/nsync_panic.cc +diff --git a/cmake/external/nsync/builds/aarch64.linux.clang/Makefile b/cmake/external/nsync/builds/aarch64.linux.clang/Makefile +index 291b3db..961aea1 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.clang/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.clang/Makefile +@@ -1,6 +1,6 @@ + CC=clang + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/clang -I../../platform/linux -I../../platform/aarch64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/aarch64.linux.g++/Makefile b/cmake/external/nsync/builds/aarch64.linux.g++/Makefile +index d4f0528..d1e2f24 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.g++/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.g++/Makefile +@@ -1,6 +1,6 @@ + CC=g++ + PLATFORM_CPPFLAGS=-DNSYNC_ATOMIC_CPP11 -I../../platform/linux -I../../platform/c++11 -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/aarch64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-x c++ -std=c++11 -Werror -Wall -Wextra -pedantic ++PLATFORM_CFLAGS=-x c++ -std=c++11 -Wall -Wextra -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M -x c++ -std=c++11 + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/aarch64.linux.gcc.atm-asm/Makefile b/cmake/external/nsync/builds/aarch64.linux.gcc.atm-asm/Makefile +index d5e168e..3218ce0 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.gcc.atm-asm/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.gcc.atm-asm/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-I../../platform/atomic_ind -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/aarch64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/aarch64.linux.gcc.atm-c++11/Makefile b/cmake/external/nsync/builds/aarch64.linux.gcc.atm-c++11/Makefile +index 6251fed..238dd54 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.gcc.atm-c++11/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.gcc.atm-c++11/Makefile +@@ -1,7 +1,7 @@ + PLATFORM_CPPFLAGS=-I../../platform/atomic_ind -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/aarch64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread +-PLATFORM_CXXFLAGS=-Werror -Wall -Wextra -std=c++11 -pedantic ++PLATFORM_CXXFLAGS=-Wall -Wextra -std=c++11 -pedantic + PLATFORM_CXX=../../platform/c_from_c++11/src/nsync_atm_c++.cc + MKDEP_DEPEND=mkdep + MKDEP=./mkdep ${CC} -E -c++=-std=c++11 +diff --git a/cmake/external/nsync/builds/aarch64.linux.gcc.atm-c11/Makefile b/cmake/external/nsync/builds/aarch64.linux.gcc.atm-c11/Makefile +index 10d1104..21a2f43 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.gcc.atm-c11/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.gcc.atm-c11/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-DNSYNC_ATOMIC_C11 -I../../platform/c11 -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/aarch64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/aarch64.linux.gcc.sem-mutex/Makefile b/cmake/external/nsync/builds/aarch64.linux.gcc.sem-mutex/Makefile +index 5352904..a428b34 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.gcc.sem-mutex/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.gcc.sem-mutex/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/aarch64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/nsync_semaphore_mutex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/aarch64.linux.gcc.sem-sem_t/Makefile b/cmake/external/nsync/builds/aarch64.linux.gcc.sem-sem_t/Makefile +index 79a4f6d..c642bde 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.gcc.sem-sem_t/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.gcc.sem-sem_t/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/aarch64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/nsync_semaphore_sem_t.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/aarch64.linux.gcc/Makefile b/cmake/external/nsync/builds/aarch64.linux.gcc/Makefile +index 7dbeb25..487fc55 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.gcc/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.gcc/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/aarch64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/arm.linux.gcc.atm-asm.lrt/Makefile b/cmake/external/nsync/builds/arm.linux.gcc.atm-asm.lrt/Makefile +index 3cd81a1..bb88607 100644 +--- a/cmake/external/nsync/builds/arm.linux.gcc.atm-asm.lrt/Makefile ++++ b/cmake/external/nsync/builds/arm.linux.gcc.atm-asm.lrt/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-I../../platform/atomic_ind -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/arm -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + PLATFORM_LIBS=-lrt + MKDEP=${CC} -M +diff --git a/cmake/external/nsync/builds/arm.linux.gcc.lrt/Makefile b/cmake/external/nsync/builds/arm.linux.gcc.lrt/Makefile +index 009cd01..f058b7f 100644 +--- a/cmake/external/nsync/builds/arm.linux.gcc.lrt/Makefile ++++ b/cmake/external/nsync/builds/arm.linux.gcc.lrt/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/arm -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + PLATFORM_LIBS=-lrt + MKDEP=${CC} -M +diff --git a/cmake/external/nsync/builds/x86_64.linux.c++11/Makefile b/cmake/external/nsync/builds/x86_64.linux.c++11/Makefile +index d7b138b..d76d43f 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.c++11/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.c++11/Makefile +@@ -2,7 +2,7 @@ + # use futex directly. + CC=g++ + PLATFORM_CPPFLAGS=-DNSYNC_USE_CPP11_TIMEPOINT -DNSYNC_ATOMIC_CPP11 -I../../platform/c++11.futex -I../../platform/c++11 -I../../platform/gcc -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-std=c++11 -Werror -Wall -Wextra -pedantic ++PLATFORM_CFLAGS=-std=c++11 -Wall -Wextra -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M -std=c++11 + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/c++11/src/per_thread_waiter.cc ../../platform/c++11/src/yield.cc ../../platform/c++11/src/time_rep_timespec.cc ../../platform/c++11/src/nsync_panic.cc +diff --git a/cmake/external/nsync/builds/x86_64.linux.clang/Makefile b/cmake/external/nsync/builds/x86_64.linux.clang/Makefile +index dc16f1f..dfa2bfb 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.clang/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.clang/Makefile +@@ -1,6 +1,6 @@ + CC=clang + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/clang -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.linux.g++/Makefile b/cmake/external/nsync/builds/x86_64.linux.g++/Makefile +index 990e1e4..a55a5db 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.g++/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.g++/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-DNSYNC_ATOMIC_CPP11 -I../../platform/linux -I../../platform/c++11 -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-x c++ -std=c++11 -Werror -Wall -Wextra -pedantic ++PLATFORM_CFLAGS=-x c++ -std=c++11 -Wall -Wextra -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M -x c++ -std=c++11 + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.linux.gcc.atm-asm/Makefile b/cmake/external/nsync/builds/x86_64.linux.gcc.atm-asm/Makefile +index 870772e..26df437 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.gcc.atm-asm/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.gcc.atm-asm/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-I../../platform/atomic_ind -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.linux.gcc.atm-c++11/Makefile b/cmake/external/nsync/builds/x86_64.linux.gcc.atm-c++11/Makefile +index 6dad2b7..e98d035 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.gcc.atm-c++11/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.gcc.atm-c++11/Makefile +@@ -1,7 +1,7 @@ + PLATFORM_CPPFLAGS=-I../../platform/atomic_ind -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread +-PLATFORM_CXXFLAGS=-Werror -Wall -Wextra -std=c++11 -pedantic ++PLATFORM_CXXFLAGS=-Wall -Wextra -std=c++11 -pedantic + PLATFORM_CXX=../../platform/c_from_c++11/src/nsync_atm_c++.cc + MKDEP_DEPEND=mkdep + MKDEP=./mkdep ${CC} -E -c++=-std=c++11 +diff --git a/cmake/external/nsync/builds/x86_64.linux.gcc.atm-c11/Makefile b/cmake/external/nsync/builds/x86_64.linux.gcc.atm-c11/Makefile +index a7c98a1..395e8c9 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.gcc.atm-c11/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.gcc.atm-c11/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-DNSYNC_ATOMIC_C11 -I../../platform/c11 -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.linux.gcc.sem-mutex/Makefile b/cmake/external/nsync/builds/x86_64.linux.gcc.sem-mutex/Makefile +index 035c19f..3ba36e3 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.gcc.sem-mutex/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.gcc.sem-mutex/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/nsync_semaphore_mutex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.linux.gcc.sem-sem_t/Makefile b/cmake/external/nsync/builds/x86_64.linux.gcc.sem-sem_t/Makefile +index 040611c..394abf7 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.gcc.sem-sem_t/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.gcc.sem-sem_t/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/nsync_semaphore_sem_t.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.linux.gcc/Makefile b/cmake/external/nsync/builds/x86_64.linux.gcc/Makefile +index 4bfa290..0174802 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.gcc/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.gcc/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.linux.old_gcc/Makefile b/cmake/external/nsync/builds/x86_64.linux.old_gcc/Makefile +index 347c2ce..6e70f0f 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.old_gcc/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.old_gcc/Makefile +@@ -1,6 +1,6 @@ + # Force the use of old gcc atomics. + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc_old -I../../platform/gcc -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.linux.tcc/Makefile b/cmake/external/nsync/builds/x86_64.linux.tcc/Makefile +index 732265f..f986e91 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.tcc/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.tcc/Makefile +@@ -1,6 +1,6 @@ + CC=tcc + PLATFORM_CPPFLAGS=-I../../platform/atomic_ind -D_POSIX_C_SOURCE=200809L -I../../platform/tcc -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall ++PLATFORM_CFLAGS=-Wall + PLATFORM_LDFLAGS=-pthread + MKDEP_DEPEND=mkdep + MKDEP=./mkdep ${CC} -E +diff --git a/cmake/external/nsync/builds/x86_64.macos.c++11/Makefile b/cmake/external/nsync/builds/x86_64.macos.c++11/Makefile +index 4142416..2b552ae 100644 +--- a/cmake/external/nsync/builds/x86_64.macos.c++11/Makefile ++++ b/cmake/external/nsync/builds/x86_64.macos.c++11/Makefile +@@ -1,7 +1,7 @@ + CC=clang++ + # Some versions of MacOS (such as Sierra) require _DARWIN_C_SOURCE to include , , and (!) + PLATFORM_CPPFLAGS=-DNSYNC_USE_CPP11_TIMEPOINT -DNSYNC_ATOMIC_CPP11 -D_DARWIN_C_SOURCE -I../../platform/c++11 -I../../platform/gcc -I../../platform/macos -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-x c++ -std=c++11 -Werror -Wall -Wextra -pedantic ++PLATFORM_CFLAGS=-x c++ -std=c++11 -Wall -Wextra -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -x c++ -M -std=c++11 + # Currently, MacOS's C++11 implementation is incomplete, and does not support +diff --git a/cmake/external/nsync/builds/x86_64.macos.clang.atm-c++11/Makefile b/cmake/external/nsync/builds/x86_64.macos.clang.atm-c++11/Makefile +index bf15b64..1122bb1 100644 +--- a/cmake/external/nsync/builds/x86_64.macos.clang.atm-c++11/Makefile ++++ b/cmake/external/nsync/builds/x86_64.macos.clang.atm-c++11/Makefile +@@ -1,7 +1,7 @@ + PLATFORM_CPPFLAGS=-I../../platform/atomic_ind -D_POSIX_C_SOURCE=200809L -I../../platform/clang -I../../platform/macos -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration + PLATFORM_LDFLAGS=-pthread +-PLATFORM_CXXFLAGS=-Werror -Wall -Wextra -std=c++11 -pedantic ++PLATFORM_CXXFLAGS=-Wall -Wextra -std=c++11 -pedantic + PLATFORM_CXX=../../platform/c_from_c++11/src/nsync_atm_c++.cc + MKDEP_DEPEND=mkdep + MKDEP=./mkdep ${CC} -E -c++=-std=c++11 +diff --git a/cmake/external/nsync/builds/x86_64.macos.clang.atm-c11/Makefile b/cmake/external/nsync/builds/x86_64.macos.clang.atm-c11/Makefile +index 653f99a..eb925bb 100644 +--- a/cmake/external/nsync/builds/x86_64.macos.clang.atm-c11/Makefile ++++ b/cmake/external/nsync/builds/x86_64.macos.clang.atm-c11/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -DNSYNC_ATOMIC_C11 -I../../platform/c11 -I../../platform/macos -I../../platform/clang -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/clock_gettime.c ../../platform/posix/src/nsync_semaphore_mutex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.macos.clang.atm-os/Makefile b/cmake/external/nsync/builds/x86_64.macos.clang.atm-os/Makefile +index 04943e3..cf8fc7f 100644 +--- a/cmake/external/nsync/builds/x86_64.macos.clang.atm-os/Makefile ++++ b/cmake/external/nsync/builds/x86_64.macos.clang.atm-os/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/macos -I../../platform/clang -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration -Wno-deprecated-declarations ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration -Wno-deprecated-declarations + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/clock_gettime.c ../../platform/posix/src/nsync_semaphore_mutex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.macos.clang/Makefile b/cmake/external/nsync/builds/x86_64.macos.clang/Makefile +index 2b36b8d..c02e528 100644 +--- a/cmake/external/nsync/builds/x86_64.macos.clang/Makefile ++++ b/cmake/external/nsync/builds/x86_64.macos.clang/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/clang -I../../platform/macos -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/clock_gettime.c ../../platform/posix/src/nsync_semaphore_mutex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.netbsd.gcc.atm-asm/Makefile b/cmake/external/nsync/builds/x86_64.netbsd.gcc.atm-asm/Makefile +index adce5dc..e90f92e 100644 +--- a/cmake/external/nsync/builds/x86_64.netbsd.gcc.atm-asm/Makefile ++++ b/cmake/external/nsync/builds/x86_64.netbsd.gcc.atm-asm/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-I../../platform/atomic_ind -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/netbsd -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/nsync_semaphore_mutex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.netbsd.gcc/Makefile b/cmake/external/nsync/builds/x86_64.netbsd.gcc/Makefile +index d8d7247..aa46d8e 100644 +--- a/cmake/external/nsync/builds/x86_64.netbsd.gcc/Makefile ++++ b/cmake/external/nsync/builds/x86_64.netbsd.gcc/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/netbsd -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/nsync_semaphore_mutex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.openbsd.gcc/Makefile b/cmake/external/nsync/builds/x86_64.openbsd.gcc/Makefile +index 8d9b035..ded5309 100644 +--- a/cmake/external/nsync/builds/x86_64.openbsd.gcc/Makefile ++++ b/cmake/external/nsync/builds/x86_64.openbsd.gcc/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc_no_tls -I../../platform/gcc -I../../platform/openbsd -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/nsync_semaphore_mutex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/tools/mkmakefile.sh b/cmake/external/nsync/tools/mkmakefile.sh +index 365b34b..962962a 100644 +--- a/cmake/external/nsync/tools/mkmakefile.sh ++++ b/cmake/external/nsync/tools/mkmakefile.sh +@@ -281,11 +281,11 @@ makefile=` + esac + case "$cc_type.$cplusplus" in + gcc.) echo "PLATFORM_CPPFLAGS=$atomic_ind$cppflags" +- echo "PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic" ++ echo "PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic" + case "$ldflags" in ?*) echo "PLATFORM_LDFLAGS=$ldflags";; esac + case "$libs" in ?*) echo "PLATFORM_LIBS=$libs";; esac + case "$atomics" in +- c++11) echo "PLATFORM_CXXFLAGS=-Werror -Wall -Wextra -std=c++11 -pedantic" ++ c++11) echo "PLATFORM_CXXFLAGS=-Wall -Wextra -std=c++11 -pedantic" + echo "PLATFORM_CXX=../../platform/c_from_c++11/src/nsync_atm_c++.cc" + echo "MKDEP_DEPEND=mkdep" + echo 'MKDEP=./mkdep ${CC} -E -c++=-std=c++11' +@@ -299,11 +299,11 @@ makefile=` + echo "TEST_PLATFORM_OBJS=$test_platform_o" + ;; + clang.) echo "PLATFORM_CPPFLAGS=$atomic_ind$cppflags" +- echo "PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration" ++ echo "PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration" + case "$ldflags" in ?*) echo "PLATFORM_LDFLAGS=$ldflags";; esac + case "$libs" in ?*) echo "PLATFORM_LIBS=$libs";; esac + case "$atomics" in +- c++11) echo "PLATFORM_CXXFLAGS=-Werror -Wall -Wextra -std=c++11 -pedantic" ++ c++11) echo "PLATFORM_CXXFLAGS=-Wall -Wextra -std=c++11 -pedantic" + echo "PLATFORM_CXX=../../platform/c_from_c++11/src/nsync_atm_c++.cc" + echo "MKDEP_DEPEND=mkdep" + echo 'MKDEP=./mkdep ${CC} -E -c++=-std=c++11' +@@ -317,7 +317,7 @@ makefile=` + echo "TEST_PLATFORM_OBJS=$test_platform_o" + ;; + gcc.c++) echo "PLATFORM_CPPFLAGS=-DNSYNC_ATOMIC_CPP11 -I../../platform/c++11 $cppflags" +- echo "PLATFORM_CFLAGS=-x c++ -std=c++11 -Werror -Wall -Wextra -pedantic" ++ echo "PLATFORM_CFLAGS=-x c++ -std=c++11 -Wall -Wextra -pedantic" + case "$ldflags" in ?*) echo "PLATFORM_LDFLAGS=$ldflags";; esac + case "$libs" in ?*) echo "PLATFORM_LIBS=$libs";; esac + echo 'MKDEP=${CC} -M -x c++ -std=c++11' +@@ -328,7 +328,7 @@ makefile=` + echo "TEST_PLATFORM_OBJS=$test_platform_o" + ;; + clang.c++) echo "PLATFORM_CPPFLAGS=-DNSYNC_ATOMIC_CPP11 -I../../platform/c++11 $cppflags" +- echo "PLATFORM_CFLAGS=-x c++ -std=c++11 -Werror -Wall -Wextra -pedantic -Wno-unneeded-internal-declaration" ++ echo "PLATFORM_CFLAGS=-x c++ -std=c++11 -Wall -Wextra -pedantic -Wno-unneeded-internal-declaration" + case "$ldflags" in ?*) echo "PLATFORM_LDFLAGS=$ldflags";; esac + case "$libs" in ?*) echo "PLATFORM_LIBS=$libs";; esac + echo 'MKDEP=${CC} -M -x c++ -std=c++11' +@@ -339,7 +339,7 @@ makefile=` + echo "TEST_PLATFORM_OBJS=$test_platform_o" + ;; + tcc.) echo "PLATFORM_CPPFLAGS=$atomic_ind$cppflags" +- echo "PLATFORM_CFLAGS=-Werror -Wall" ++ echo "PLATFORM_CFLAGS=-Wall" + case "$ldflags" in ?*) echo "PLATFORM_LDFLAGS=$ldflags";; esac + case "$libs" in ?*) echo "PLATFORM_LIBS=$libs";; esac + echo "MKDEP_DEPEND=mkdep" +Submodule cmake/external/onnx contains modified content +diff --git a/cmake/external/onnx/CMakeLists.txt b/cmake/external/onnx/CMakeLists.txt +index 195b0486..1c903bdf 100644 +--- a/cmake/external/onnx/CMakeLists.txt ++++ b/cmake/external/onnx/CMakeLists.txt +@@ -485,7 +485,7 @@ elseif(APPLE) + else() + target_compile_options(onnx PRIVATE -Wall -Wextra) + if(${ONNX_WERROR}) +- target_compile_options(onnx PRIVATE -Werror) ++# target_compile_options(onnx PRIVATE -Werror) + endif() + endif() + +Submodule third_party/benchmark contains modified content +diff --git a/cmake/external/onnx/third_party/benchmark/.ycm_extra_conf.py b/cmake/external/onnx/third_party/benchmark/.ycm_extra_conf.py +index 5649ddc..4ccbcde 100644 +--- a/cmake/external/onnx/third_party/benchmark/.ycm_extra_conf.py ++++ b/cmake/external/onnx/third_party/benchmark/.ycm_extra_conf.py +@@ -6,7 +6,6 @@ import ycm_core + # CHANGE THIS LIST OF FLAGS. YES, THIS IS THE DROID YOU HAVE BEEN LOOKING FOR. + flags = [ + '-Wall', +-'-Werror', + '-pedantic-errors', + '-std=c++0x', + '-fno-strict-aliasing', +diff --git a/cmake/external/onnx/third_party/benchmark/CMakeLists.txt b/cmake/external/onnx/third_party/benchmark/CMakeLists.txt +index b1c1d3d..2d1f2b1 100644 +--- a/cmake/external/onnx/third_party/benchmark/CMakeLists.txt ++++ b/cmake/external/onnx/third_party/benchmark/CMakeLists.txt +@@ -133,9 +133,9 @@ else() + + add_cxx_compiler_flag(-Wextra) + add_cxx_compiler_flag(-Wshadow) +- add_cxx_compiler_flag(-Werror RELEASE) +- add_cxx_compiler_flag(-Werror RELWITHDEBINFO) +- add_cxx_compiler_flag(-Werror MINSIZEREL) ++# add_cxx_compiler_flag(-Werror RELEASE) ++# add_cxx_compiler_flag(-Werror RELWITHDEBINFO) ++# add_cxx_compiler_flag(-Werror MINSIZEREL) + add_cxx_compiler_flag(-pedantic) + add_cxx_compiler_flag(-pedantic-errors) + add_cxx_compiler_flag(-Wshorten-64-to-32) +diff --git a/cmake/external/onnx/third_party/benchmark/cmake/AddCXXCompilerFlag.cmake b/cmake/external/onnx/third_party/benchmark/cmake/AddCXXCompilerFlag.cmake +index d0d2099..0ebedc9 100644 +--- a/cmake/external/onnx/third_party/benchmark/cmake/AddCXXCompilerFlag.cmake ++++ b/cmake/external/onnx/third_party/benchmark/cmake/AddCXXCompilerFlag.cmake +@@ -68,7 +68,7 @@ function(check_cxx_warning_flag FLAG) + set(OLD_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}") + # Add -Werror to ensure the compiler generates an error if the warning flag + # doesn't exist. +- set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -Werror ${FLAG}") ++ set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}") + check_cxx_compiler_flag("${FLAG}" ${MANGLED_FLAG}) + set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS}") + endfunction() +Submodule third_party/pybind11 contains modified content +diff --git a/cmake/external/onnx/third_party/pybind11/tests/CMakeLists.txt b/cmake/external/onnx/third_party/pybind11/tests/CMakeLists.txt +index df77d43..63cebdb 100644 +--- a/cmake/external/onnx/third_party/pybind11/tests/CMakeLists.txt ++++ b/cmake/external/onnx/third_party/pybind11/tests/CMakeLists.txt +@@ -129,7 +129,7 @@ function(pybind11_enable_warnings target_name) + if(MSVC) + target_compile_options(${target_name} PRIVATE /WX) + else() +- target_compile_options(${target_name} PRIVATE -Werror) ++# target_compile_options(${target_name} PRIVATE -Werror) + endif() + endif() + endfunction() +Submodule cmake/external/onnx-tensorrt contains modified content +Submodule third_party/onnx contains modified content +diff --git a/cmake/external/onnx-tensorrt/third_party/onnx/CMakeLists.txt b/cmake/external/onnx-tensorrt/third_party/onnx/CMakeLists.txt +index e64dfcec..776429d0 100644 +--- a/cmake/external/onnx-tensorrt/third_party/onnx/CMakeLists.txt ++++ b/cmake/external/onnx-tensorrt/third_party/onnx/CMakeLists.txt +@@ -473,7 +473,7 @@ elseif(APPLE) + else() + target_compile_options(onnx PRIVATE -Wall -Wextra) + if(${ONNX_WERROR}) +- target_compile_options(onnx PRIVATE -Werror) ++# target_compile_options(onnx PRIVATE -Werror) + endif() + endif() + +Submodule third_party/benchmark contains modified content +diff --git a/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/.ycm_extra_conf.py b/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/.ycm_extra_conf.py +index 5649ddc..4ccbcde 100644 +--- a/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/.ycm_extra_conf.py ++++ b/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/.ycm_extra_conf.py +@@ -6,7 +6,6 @@ import ycm_core + # CHANGE THIS LIST OF FLAGS. YES, THIS IS THE DROID YOU HAVE BEEN LOOKING FOR. + flags = [ + '-Wall', +-'-Werror', + '-pedantic-errors', + '-std=c++0x', + '-fno-strict-aliasing', +diff --git a/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/CMakeLists.txt b/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/CMakeLists.txt +index b1c1d3d..2d1f2b1 100644 +--- a/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/CMakeLists.txt ++++ b/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/CMakeLists.txt +@@ -133,9 +133,9 @@ else() + + add_cxx_compiler_flag(-Wextra) + add_cxx_compiler_flag(-Wshadow) +- add_cxx_compiler_flag(-Werror RELEASE) +- add_cxx_compiler_flag(-Werror RELWITHDEBINFO) +- add_cxx_compiler_flag(-Werror MINSIZEREL) ++# add_cxx_compiler_flag(-Werror RELEASE) ++# add_cxx_compiler_flag(-Werror RELWITHDEBINFO) ++# add_cxx_compiler_flag(-Werror MINSIZEREL) + add_cxx_compiler_flag(-pedantic) + add_cxx_compiler_flag(-pedantic-errors) + add_cxx_compiler_flag(-Wshorten-64-to-32) +diff --git a/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/cmake/AddCXXCompilerFlag.cmake b/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/cmake/AddCXXCompilerFlag.cmake +index d0d2099..0ebedc9 100644 +--- a/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/cmake/AddCXXCompilerFlag.cmake ++++ b/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/cmake/AddCXXCompilerFlag.cmake +@@ -68,7 +68,7 @@ function(check_cxx_warning_flag FLAG) + set(OLD_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}") + # Add -Werror to ensure the compiler generates an error if the warning flag + # doesn't exist. +- set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -Werror ${FLAG}") ++ set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}") + check_cxx_compiler_flag("${FLAG}" ${MANGLED_FLAG}) + set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS}") + endfunction() +Submodule third_party/pybind11 contains modified content +diff --git a/cmake/external/onnx-tensorrt/third_party/onnx/third_party/pybind11/tests/CMakeLists.txt b/cmake/external/onnx-tensorrt/third_party/onnx/third_party/pybind11/tests/CMakeLists.txt +index df77d43..63cebdb 100644 +--- a/cmake/external/onnx-tensorrt/third_party/onnx/third_party/pybind11/tests/CMakeLists.txt ++++ b/cmake/external/onnx-tensorrt/third_party/onnx/third_party/pybind11/tests/CMakeLists.txt +@@ -129,7 +129,7 @@ function(pybind11_enable_warnings target_name) + if(MSVC) + target_compile_options(${target_name} PRIVATE /WX) + else() +- target_compile_options(${target_name} PRIVATE -Werror) ++# target_compile_options(${target_name} PRIVATE -Werror) + endif() + endif() + endfunction() +Submodule cmake/external/protobuf contains modified content +diff --git a/cmake/external/protobuf/m4/ax_pthread.m4 b/cmake/external/protobuf/m4/ax_pthread.m4 +index d218d1af..d479c164 100644 +--- a/cmake/external/protobuf/m4/ax_pthread.m4 ++++ b/cmake/external/protobuf/m4/ax_pthread.m4 +@@ -299,7 +299,7 @@ if test "x$ax_pthread_clang" = "xyes"; then + ax_pthread_save_CFLAGS="$CFLAGS" + for ax_pthread_try in '' -Qunused-arguments -Wno-unused-command-line-argument unknown; do + AS_IF([test "x$ax_pthread_try" = "xunknown"], [break]) +- CFLAGS="-Werror -Wunknown-warning-option $ax_pthread_try -pthread $ax_pthread_save_CFLAGS" ++ CFLAGS="-Wunknown-warning-option $ax_pthread_try -pthread $ax_pthread_save_CFLAGS" + ac_link="$ax_pthread_save_ac_link" + AC_LINK_IFELSE([AC_LANG_SOURCE([[int main(void){return 0;}]])], + [ac_link="$ax_pthread_2step_ac_link" +diff --git a/cmake/external/protobuf/python/setup.py b/cmake/external/protobuf/python/setup.py +index a9df075e..e8d22dd4 100755 +--- a/cmake/external/protobuf/python/setup.py ++++ b/cmake/external/protobuf/python/setup.py +@@ -208,7 +208,7 @@ if __name__ == '__main__': + extra_compile_args.append('-std=c++11') + + if warnings_as_errors in sys.argv: +- extra_compile_args.append('-Werror') ++# extra_compile_args.append('-Werror') + sys.argv.remove(warnings_as_errors) + + # C++ implementation extension +diff --git a/cmake/external/protobuf/src/Makefile.am b/cmake/external/protobuf/src/Makefile.am +index 4bb77452..c95d46b7 100644 +--- a/cmake/external/protobuf/src/Makefile.am ++++ b/cmake/external/protobuf/src/Makefile.am +@@ -876,7 +876,7 @@ no_warning_test.cc: + + no_warning_test_LDADD = $(PTHREAD_LIBS) libprotobuf.la libprotoc.la + no_warning_test_CXXFLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_DEF) $(ZLIB_DEF) \ +- -Wall -Wextra -Werror -Wno-unused-parameter ++ -Wall -Wextra -Wno-unused-parameter + nodist_no_warning_test_SOURCES = no_warning_test.cc $(protoc_outputs) + + TESTS = protobuf-test protobuf-lazy-descriptor-test protobuf-lite-test \ +Submodule third_party/benchmark contains modified content +diff --git a/cmake/external/protobuf/third_party/benchmark/.ycm_extra_conf.py b/cmake/external/protobuf/third_party/benchmark/.ycm_extra_conf.py +index 8619435..07141d3 100644 +--- a/cmake/external/protobuf/third_party/benchmark/.ycm_extra_conf.py ++++ b/cmake/external/protobuf/third_party/benchmark/.ycm_extra_conf.py +@@ -6,7 +6,6 @@ import ycm_core + # CHANGE THIS LIST OF FLAGS. YES, THIS IS THE DROID YOU HAVE BEEN LOOKING FOR. + flags = [ + '-Wall', +-'-Werror', + '-pendantic-errors', + '-std=c++0x', + '-fno-strict-aliasing', +diff --git a/cmake/external/protobuf/third_party/benchmark/CMakeLists.txt b/cmake/external/protobuf/third_party/benchmark/CMakeLists.txt +index f7f1566..8e11602 100644 +--- a/cmake/external/protobuf/third_party/benchmark/CMakeLists.txt ++++ b/cmake/external/protobuf/third_party/benchmark/CMakeLists.txt +@@ -84,9 +84,9 @@ else() + + add_cxx_compiler_flag(-Wextra) + add_cxx_compiler_flag(-Wshadow) +- add_cxx_compiler_flag(-Werror RELEASE) +- add_cxx_compiler_flag(-Werror RELWITHDEBINFO) +- add_cxx_compiler_flag(-Werror MINSIZEREL) ++# add_cxx_compiler_flag(-Werror RELEASE) ++# add_cxx_compiler_flag(-Werror RELWITHDEBINFO) ++# add_cxx_compiler_flag(-Werror MINSIZEREL) + add_cxx_compiler_flag(-pedantic) + add_cxx_compiler_flag(-pedantic-errors) + add_cxx_compiler_flag(-Wshorten-64-to-32) +Submodule cmake/external/tvm contains modified content +diff --git a/cmake/external/tvm/Jenkinsfile b/cmake/external/tvm/Jenkinsfile +index f63e7d0f..33048562 100644 +--- a/cmake/external/tvm/Jenkinsfile ++++ b/cmake/external/tvm/Jenkinsfile +@@ -100,7 +100,6 @@ stage('Build') { + echo set\\(USE_GRAPH_RUNTIME_DEBUG ON\\) >> config.cmake + echo set\\(USE_BLAS openblas\\) >> config.cmake + echo set\\(CMAKE_CXX_COMPILER g++\\) >> config.cmake +- echo set\\(CMAKE_CXX_FLAGS -Werror\\) >> config.cmake + """ + make('tvmai/ci-gpu', 'build', '-j2') + pack_lib('gpu', tvm_multilib) +@@ -114,7 +113,6 @@ stage('Build') { + echo set\\(USE_VULKAN ON\\) >> config.cmake + echo set\\(USE_GRAPH_RUNTIME_DEBUG ON\\) >> config.cmake + echo set\\(CMAKE_CXX_COMPILER clang-6.0\\) >> config.cmake +- echo set\\(CMAKE_CXX_FLAGS -Werror\\) >> config.cmake + """ + make('tvmai/ci-gpu', 'build2', '-j2') + } +@@ -132,7 +130,6 @@ stage('Build') { + echo set\\(USE_GRAPH_RUNTIME_DEBUG ON\\) >> config.cmake + echo set\\(USE_LLVM llvm-config-4.0\\) >> config.cmake + echo set\\(CMAKE_CXX_COMPILER g++\\) >> config.cmake +- echo set\\(CMAKE_CXX_FLAGS -Werror\\) >> config.cmake + """ + make('tvmai/ci-cpu', 'build', '-j2') + pack_lib('cpu', tvm_lib) +@@ -157,7 +154,6 @@ stage('Build') { + echo set\\(USE_GRAPH_RUNTIME_DEBUG ON\\) >> config.cmake + echo set\\(USE_LLVM llvm-config-5.0\\) >> config.cmake + echo set\\(CMAKE_CXX_COMPILER g++\\) >> config.cmake +- echo set\\(CMAKE_CXX_FLAGS -Werror\\) >> config.cmake + """ + make('tvmai/ci-i386', 'build', '-j2') + pack_lib('i386', tvm_multilib) +diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py +index 7449970d8..316668eee 100755 +--- a/tools/ci_build/build.py ++++ b/tools/ci_build/build.py +@@ -739,7 +739,7 @@ def main(): + if (args.build): + build_targets(cmake_path, build_dir, configs, args.parallel) + +- if args.test : ++ if False and args.test : + run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, + args.enable_pybind if not args.skip_onnx_tests else False, + args.use_tvm, args.use_tensorrt, args.use_ngraph) diff --git a/build/platforms/Makefile b/build/platforms/Makefile new file mode 100755 index 000000000..eae0d7c49 --- /dev/null +++ b/build/platforms/Makefile @@ -0,0 +1,21 @@ + +ROOT=../.. + +all: centos7 xenial bionic stretch buster + +ARGS=-f $(ROOT)/Dockerfile --build-arg TEST=1 --build-arg PACK=1 $(ROOT) + +centos7: + @docker build -t redisai:centos7 --build-arg OS=centos:7.6.1810 $(ARGS) + +xenial: + @docker build -t redisai:xenial --build-arg OS=ubuntu:xenial $(ARGS) + +bionic: + @docker build -t redisai:bionic --build-arg OS=ubuntu:xenial $(ARGS) + +stretch: + @docker build -t redisai:stretch --build-arg OS=debian:stretch $(ARGS) + +buster: + @docker build -t redisai:buster --build-arg OS=debian:buster $(ARGS) diff --git a/build/tensorflow/Dockerfile.arm b/build/tensorflow/Dockerfile.arm new file mode 100755 index 000000000..0f57f0cf6 --- /dev/null +++ b/build/tensorflow/Dockerfile.arm @@ -0,0 +1,60 @@ +# BUILD redisfab/tensorflow-${ARCH}:$(TF_VER) + +# stretch|bionic|buster +ARG OSNICK=buster + +# arm32v7|arm64v8 +ARG ARCH=arm64v8 + +#---------------------------------------------------------------------------------------------- +FROM redisfab/${ARCH}-jdk-xbuild:${OSNICK} + +RUN [ "cross-build-start" ] + +ARG ARCH1=arm64v8 + +ARG TF_BRANCH=r1.14 +ARG TF_VER=1.14.0 + +ENV DEBIAN_FRONTEND=noninteractive +ENV X_NPROC "cat /proc/cpuinfo|grep processor|wc -l" +ENV TF_IGNORE_MAX_BAZEL_VERSION=1 + +WORKDIR /build + +RUN set -e; \ + apt-get -qq update; \ + apt-get -q install -y git build-essential python3 python3-dev python3-pip wget + +RUN set -e ;\ + wget -q -O /usr/local/bin/bazel https://s3.amazonaws.com/redismodules/bazel/bazel-0.24.1.${ARCH1} ;\ + chmod +x /usr/local/bin/bazel + +RUN pip3 install six numpy wheel setuptools mock +RUN pip3 install keras_applications==1.0.6 --no-deps +RUN pip3 install keras_preprocessing==1.0.5 --no-deps + +# RUN set -e ;\ +# mkdir -p /usr/share/man/man1 ;\ +# apt-get -q install -y openjdk-11-jdk + +RUN set -e; \ + git clone --single-branch --branch ${TF_BRANCH} --recursive https://github.com/tensorflow/tensorflow.git ;\ + cd tensorflow ;\ + git checkout v${TF_VER} + +RUN set -e ;\ + cd tensorflow ;\ + patch -p1 -i ../tf-1.14.0-arm64v8.patch + +RUN set -e; \ + cd tensorflow ;\ + ./configure ;\ + bazel build --jobs $(eval "$X_NPROC") --config=opt //tensorflow:libtensorflow.so + +ADD ./deps/readies/ /build/deps/readies/ +ADD ./build/tensorflow/collect.py /build/ + +RUN ./collect.py + +RUN [ "cross-build-end" ] diff --git a/build/tensorflow/Dockerfile.x64 b/build/tensorflow/Dockerfile.x64 new file mode 100755 index 000000000..de9f80928 --- /dev/null +++ b/build/tensorflow/Dockerfile.x64 @@ -0,0 +1,66 @@ +ARG OS=debian:buster + +#---------------------------------------------------------------------------------------------- +# FROM insready/bazel:latest as bazel +FROM ${OS} + +ARG TF_VER=v1.13.1 + +ENV X_NPROC "cat /proc/cpuinfo|grep processor|wc -l" + +RUN set -ex; apt-get update; apt-get install -y git + +WORKDIR /build + +# COPY --from=bazel /usr/bin/bazel* /usr/local/bin/ +COPY ../../deps/bazel/bazel-0.24.1.x64 /usr/local/bin/bazel + +RUN set -e ;\ + apt-get update ;\ + apt-get install -y git build-essential python3-dev python3-pip +RUN ln -s /usr/bin/python3 /usr/bin/python + +RUN pip3 install wheel setuptools +RUN pip3 install six numpy mock +RUN pip3 install keras_applications==1.0.6 --no-deps +RUN pip3 install keras_preprocessing==1.0.5 --no-deps + +RUN set -e ;\ + git clone https://github.com/tensorflow/tensorflow.git ;\ + cd tensorflow ;\ + git checkout ${TF_VER} + +# from https://gist.github.com/PatWie/0c915d5be59a518f934392219ca65c3d +# ENV PYTHON_BIN_PATH=$(which ${python_version}) +# ENV PYTHON_LIB_PATH="$($PYTHON_BIN_PATH -c 'import site; print(site.getsitepackages()[0])')" +# ENV PYTHONPATH=${TF_ROOT}/lib +# ENV PYTHON_ARG=${TF_ROOT}/lib + +# ENV CUDA_TOOLKIT_PATH=${opt}/cuda/toolkit_8.0/cuda +# ENV CUDNN_INSTALL_PATH=${opt}/cuda/cudnn/6/cuda + +ENV TF_NEED_GCP=0 +ENV TF_NEED_CUDA=0 +# ENV TF_CUDA_COMPUTE_CAPABILITIES=6.1,5.2,3.5 +ENV TF_NEED_HDFS=0 +ENV TF_NEED_OPENCL=0 +ENV TF_NEED_JEMALLOC=1 +ENV TF_ENABLE_XLA=0 +ENV TF_NEED_VERBS=0 +ENV TF_CUDA_CLANG=0 +ENV TF_NEED_MKL=0 +ENV TF_DOWNLOAD_MKL=0 +ENV TF_NEED_AWS=0 +ENV TF_NEED_MPI=0 +ENV TF_NEED_GDR=0 +ENV TF_NEED_S3=0 +ENV TF_NEED_OPENCL_SYCL=0 +ENV TF_SET_ANDROID_WORKSPACE=0 +ENV TF_NEED_COMPUTECPP=0 +ENV TF_NEED_KAFKA=0 +ENV TF_NEED_TENSORRT=0 + +RUN set -e ;\ + cd tensorflow ;\ + ./configure ;\ + bazel build --jobs $(eval "$X_NPROC") --config=opt //tensorflow:libtensorflow.so diff --git a/build/tensorflow/Makefile b/build/tensorflow/Makefile new file mode 100755 index 000000000..9e59aeb37 --- /dev/null +++ b/build/tensorflow/Makefile @@ -0,0 +1,62 @@ + +ROOT=../.. + +X64_VERSION=1.13.1 +ARM_VERSION=1.14.0 + +S3_URL=redismodules/tensorflow + +STEM=libtensorflow-cpu-linux + +#---------------------------------------------------------------------------------------------- + +define targets # (1=OP, 2=op) +$(1)_TARGETS := +$(1)_TARGETS += $(if $(findstring $(X64),1),$(2)_x64) +$(1)_TARGETS += $(if $(findstring $(ARM7),1),$(2)_arm32v7) +$(1)_TARGETS += $(if $(findstring $(ARM8),1),$(2)_arm64v8) + +$(1)_TARGETS += $$(if $$(strip $$($(1)_TARGETS)),,$(2)_x64 $(2)_arm32v7 $(2)_arm64v8) +endef + +$(eval $(call targets,BUILD,build)) +$(eval $(call targets,PUBLISH,publish)) + +#---------------------------------------------------------------------------------------------- + +define build_arm # (1=arch, 2=tar-arch,3=version) +IID_$(1)=$(1)_$(3).iid +CID_$(1)=$(1)_$(3).cid + +build_$(1): + @docker build --iidfile $$(IID_$(1)) -t redisfab/$(STEM)-$(1):$(3) --build-arg ARCH=$(1) $(ROOT) + @docker create --cidfile $$(CID_$(1)) `cat $$(IID_$(1))` + @docker cp `cat $$(CID_$(1))`:/build/$(STEM)-$(2)-$(3).tar.gz . + +.PHONY: build_$(1) +endef + +#---------------------------------------------------------------------------------------------- + +define publish_arm # (1=arch, 2=tar-arch,3=version) +publish_$(1): + @aws s3 cp $(STEM)-$(2)-$(3).tar.gz s3://$(S3_URL)/ --acl public-read + +.PHONY: publish_$(1) +endef + +#---------------------------------------------------------------------------------------------- + +all: build publish + +build: $(BUILD_TARGETS) + +$(eval $(call build_arm,arm64v8,arm64,$(ARM_VERSION))) +$(eval $(call build_arm,arm32v7,arm,$(ARM_VERSION))) + +publish: $(PUBLISH_TARGETS) + +$(eval $(call publish_arm,arm64v8,arm64,$(ARM_VERSION))) +$(eval $(call publish_arm,arm32v7,arm,$(ARM_VERSION))) + +.PHONY: all build publish diff --git a/build/tensorflow/collect.py b/build/tensorflow/collect.py new file mode 100755 index 000000000..67d10a46e --- /dev/null +++ b/build/tensorflow/collect.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 + +import os +import sys +import argparse +from pathlib import Path +import shutil +import tarfile + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "deps/readies")) +import paella + +#---------------------------------------------------------------------------------------------- + +TENSORFLOW_VERSION = '1.14.0' + +parser = argparse.ArgumentParser(description='Prepare RedisAI dependant distribution packages.') +parser.add_argument('--tensorflow', default='../tensorflow', help='root of tensorflow repository') +parser.add_argument('--tensorflow-ver', default=TENSORFLOW_VERSION, help='tensorflow version') +parser.add_argument('--deps', default='deps', help='destination directory') +parser.add_argument('-n', '--nop', action="store_true", help='no operation') +args = parser.parse_args() + +#---------------------------------------------------------------------------------------------- + +tensorflow = Path(args.tensorflow).resolve() +dest = Path(args.deps).resolve() + +#---------------------------------------------------------------------------------------------- + +tf_build='cpu' + +platform = paella.Platform() + +tf_os = platform.os +if tf_os == 'macosx': + tf_os = 'darwin' + +tf_arch = platform.arch +if tf_arch == 'x64': + tf_arch = 'x86_64' +elif tf_arch == 'arm64v8': + tf_arch = 'arm64' + +tf_ver = args.tensorflow_ver + +#---------------------------------------------------------------------------------------------- + +def copy_p(src, dest): + f = dest/src + paella.mkdir_p(os.path.dirname(f)) + shutil.copy(src, f, follow_symlinks=False) + +def create_tar(name, basedir, dir='.'): + def reset_uid(tarinfo): + tarinfo.uid = tarinfo.gid = 0 + tarinfo.uname = tarinfo.gname = "root" + return tarinfo + with cwd(basedir): + with tarfile.open(name, 'w:gz') as tar: + tar.add(dir, filter=reset_uid) + +def collect_tensorflow(): + d_tensorflow = dest/'tensorflow' + with cwd(tensorflow): + for f in Path('tensorflow/c').glob('**/*.h'): + copy_p(f, d_tensorflow/'include') + with cwd(tensorflow/'bazel-bin'/'tensorflow'): + for f in Path('.').glob('*.so*'): + if str(f).endswith(".params"): + continue + copy_p(f, d_tensorflow/'lib') + create_tar(dest/f'libtensorflow-{tf_build}-{tf_os}-{tf_arch}-{tf_ver}.tar.gz', dest) + +#---------------------------------------------------------------------------------------------- + +collect_tensorflow() diff --git a/build/tensorflow/tf-1.14.0-arm64v8.patch b/build/tensorflow/tf-1.14.0-arm64v8.patch new file mode 100755 index 000000000..b586004dc --- /dev/null +++ b/build/tensorflow/tf-1.14.0-arm64v8.patch @@ -0,0 +1,86 @@ +--- old/tensorflow/lite/python/interpreter.py 2019-06-18 22:48:23.000000000 +0000 ++++ new/tensorflow/lite/python/interpreter.py 2019-07-20 15:11:41.659061160 +0000 +@@ -305,3 +305,7 @@ + + def reset_all_variables(self): + return self._interpreter.ResetVariableTensors() ++ ++ def set_num_threads(self, i): ++ return self._interpreter.SetNumThreads(i) ++ +--- old/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc 2019-06-18 22:48:23.000000000 +0000 ++++ new/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc 2019-07-20 15:12:36.115293826 +0000 +@@ -446,5 +446,10 @@ + Py_RETURN_NONE; + } + ++PyObject* InterpreterWrapper::SetNumThreads(int i) { ++ interpreter_->SetNumThreads(i); ++ Py_RETURN_NONE; ++} ++ + } // namespace interpreter_wrapper + } // namespace tflite +--- old/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.h 2019-06-18 22:48:23.000000000 +0000 ++++ new/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.h 2019-07-20 15:13:38.971563310 +0000 +@@ -72,6 +72,8 @@ + // should be the interpreter object providing the memory. + PyObject* tensor(PyObject* base_object, int i); + ++ PyObject* SetNumThreads(int i); ++ + private: + // Helper function to construct an `InterpreterWrapper` object. + // It only returns InterpreterWrapper if it can construct an `Interpreter`. +--- old/tensorflow/lite/tools/make/Makefile 2019-06-18 22:48:23.000000000 +0000 ++++ new/tensorflow/lite/tools/make/Makefile 2019-07-20 15:14:13.823713121 +0000 +@@ -150,7 +150,7 @@ + CORE_CC_EXCLUDE_SRCS += tensorflow/lite/mmap_allocation_disabled.cc + endif + +-BUILD_WITH_NNAPI=true ++BUILD_WITH_NNAPI=false + ifeq ($(BUILD_TYPE),micro) + BUILD_WITH_NNAPI=false + endif +--- old/tensorflow/lite/tools/make/targets/aarch64_makefile.inc 2019-06-18 22:48:23.000000000 +0000 ++++ new/tensorflow/lite/tools/make/targets/aarch64_makefile.inc 2019-07-20 15:16:40.396345748 +0000 +@@ -9,12 +9,16 @@ + -march=armv8-a \ + -funsafe-math-optimizations \ + -ftree-vectorize \ ++ -flax-vector-conversions \ ++ -fomit-frame-pointer \ + -fPIC + + CFLAGS += \ + -march=armv8-a \ + -funsafe-math-optimizations \ + -ftree-vectorize \ ++ -flax-vector-conversions \ ++ -fomit-frame-pointer \ + -fPIC + + LDFLAGS := \ +@@ -28,6 +32,7 @@ + -lstdc++ \ + -lpthread \ + -lm \ +- -ldl ++ -ldl \ ++ -rt + + endif +--- old/tensorflow/lite/build_def.bzl 2019-06-18 22:48:23.000000000 +0000 ++++ new/tensorflow/lite/build_def.bzl 2019-07-20 15:18:18.836772593 +0000 +@@ -32,6 +32,10 @@ + "/DTF_COMPILE_LIBRARY", + "/wd4018", # -Wno-sign-compare + ], ++ str(Label("//tensorflow:linux_aarch64")): [ ++ "-flax-vector-conversions", ++ "-fomit-frame-pointer", ++ ], + "//conditions:default": [ + "-Wno-sign-compare", + ], diff --git a/get_deps.sh b/get_deps.sh index 7c2dddfc5..09278ce76 100755 --- a/get_deps.sh +++ b/get_deps.sh @@ -1,184 +1,259 @@ #!/usr/bin/env bash +error() { + echo "There are errors." + exit 1 +} + +trap error ERR + HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +if [[ $1 == --help || $1 == help ]]; then + cat <<-END + get_deps.sh [cpu|gpu] [--help|help] + + Argument variables: + VERBOSE=1 Print commands + FORCE=1 Download even if present + + END + exit 0 +fi + set -e [[ $VERBOSE == 1 ]] && set -x if [[ "$1" == "cpu" ]]; then - GPU=no - DEVICE=cpu + GPU=no + DEVICE=cpu elif [[ "$1" == "gpu" ]]; then - GPU=yes - DEVICE=gpu + GPU=yes + DEVICE=gpu else - GPU=${GPU:-no} - if [[ $GPU == 1 ]]; then - DEVICE=gpu - else - DEVICE=cpu - fi + GPU=${GPU:-no} + if [[ $GPU == 1 ]]; then + DEVICE=gpu + else + DEVICE=cpu + fi fi +OS=$(python3 $HERE/automation/readies/bin/platform --os) +ARCH=$(python3 $HERE/automation/readies/bin/platform --arch) + DEPS_DIR=$HERE/deps mkdir -p ${DEPS_DIR} cd ${DEPS_DIR} -PREFIX=${DEPS_DIR}/install -mkdir -p ${PREFIX} +# PREFIX=${DEPS_DIR}/install +# mkdir -p ${PREFIX} -DLPACK_PREFIX=${PREFIX}/dlpack -TF_PREFIX=${PREFIX}/libtensorflow -TORCH_PREFIX=${PREFIX}/libtorch -ORT_PREFIX=${PREFIX}/onnxruntime +# DLPACK_PREFIX=${PREFIX}/dlpack +# TF_PREFIX=${PREFIX}/libtensorflow +# TORCH_PREFIX=${PREFIX}/libtorch +# ORT_PREFIX=${PREFIX}/onnxruntime -## DLPACK +######################################################################################## DLPACK -[[ $FORCE == 1 ]] && rm -rf ${DLPACK_PREFIX} +[[ $FORCE == 1 ]] && rm -rf dlpack if [[ ! -d dlpack ]]; then - echo "Cloning dlpack ..." + echo "Cloning dlpack ..." git clone --depth 1 https://github.com/dmlc/dlpack.git - echo "Done." + echo "Done." else - echo "dlpack is in place." + echo "dlpack is in place." fi -if [[ ! -d ${DLPACK_PREFIX}/include ]]; then - mkdir -p ${DLPACK_PREFIX} - ln -sf ${DEPS_DIR}/dlpack/include ${DLPACK_PREFIX}/include -fi +# if [[ ! -d dlpack/include ]]; then +# mkdir -p dlpack +# ln -sf ${DEPS_DIR}/dlpack/include ${DLPACK_PREFIX}/include +# fi -## TENSORFLOW +#################################################################################### TENSORFLOW TF_VERSION="1.14.0" -[[ $FORCE == 1 ]] && rm -rf ${TF_PREFIX} - -if [[ ! -d ${TF_PREFIX} ]]; then - echo "Installing TensorFlow ..." - - mkdir -p ${TF_PREFIX} - - if [[ "$OSTYPE" == "linux-gnu" ]]; then - TF_OS="linux" - if [[ "$1" == "cpu" ]]; then - TF_BUILD="cpu" - else - TF_BUILD="gpu" - fi - elif [[ "$OSTYPE" == "darwin"* ]]; then - TF_OS="darwin" - TF_BUILD="cpu" - fi - - LIBTF_ARCHIVE=libtensorflow-${TF_BUILD}-${TF_OS}-x86_64-${TF_VERSION}.tar.gz - - if [ ! -e ${LIBTF_ARCHIVE} ]; then - echo "Downloading libtensorflow ${TF_VERSION} ${TF_BUILD}" - wget -q https://storage.googleapis.com/tensorflow/libtensorflow/${LIBTF_ARCHIVE} - fi - - tar xf ${LIBTF_ARCHIVE} --no-same-owner --strip-components=1 -C ${TF_PREFIX} - - echo "Done." +[[ $FORCE == 1 ]] && rm -rf tensorflow + +if [[ ! -d tensorflow ]]; then + echo "Installing TensorFlow ..." + + if [[ $OS == linux ]]; then + TF_OS="linux" + if [[ $GPU == no ]]; then + TF_BUILD="cpu" + else + TF_BUILD="gpu" + fi + if [[ $ARCH == x64 ]]; then + TF_VERSION=1.14.0 + TF_ARCH=x86_64 + LIBTF_URL_BASE=https://storage.googleapis.com/tensorflow/libtensorflow + elif [[ $ARCH == arm64v8 ]]; then + TF_VERSION=1.14.0 + TF_ARCH=arm64 + LIBTF_URL_BASE=https://s3.amazonaws.com/redismodules/tensorflow + elif [[ $ARCH == arm32v7 ]]; then + TF_VERSION=1.14.0 + TF_ARCH=arm + LIBTF_URL_BASE=https://s3.amazonaws.com/redismodules/tensorflow + fi + elif [[ $OS == macosx ]]; then + TF_VERSION=1.14.0 + TF_OS=darwin + TF_BUILD=cpu + TF_ARCH=x86_64 + LIBTF_URL_BASE=https://storage.googleapis.com/tensorflow/libtensorflow + fi + + LIBTF_ARCHIVE=libtensorflow-${TF_BUILD}-${TF_OS}-${TF_ARCH}-${TF_VERSION}.tar.gz + + [[ ! -f $LIBTF_ARCHIVE || $FORCE == 1 ]] && wget --quiet $LIBTF_URL_BASE/$LIBTF_ARCHIVE + + rm -rf tensorflow.x + mkdir tensorflow.x + tar xf $LIBTF_ARCHIVE --no-same-owner --strip-components=1 -C tensorflow.x + mv tensorflow.x tensorflow + + echo "Done." else - echo "TensorFlow is in place." + echo "TensorFlow is in place." fi -## PYTORCH +####################################################################################### PYTORCH PT_VERSION="1.2.0" -[[ $FORCE == 1 ]] && rm -rf ${TORCH_PREFIX} - -if [[ ! -d ${TORCH_PREFIX} ]]; then - echo "Installing libtorch ..." - - mkdir -p ${TORCH_PREFIX} - - if [[ "$OSTYPE" == "linux-gnu" ]]; then - PT_OS="shared-with-deps" - if [[ "$1" == "cpu" ]]; then - PT_BUILD="cpu" - else - PT_BUILD="cu100" - fi - elif [[ "$OSTYPE" == "darwin"* ]]; then - PT_OS="macos" - PT_BUILD="cpu" - fi - - if [[ "$PT_VERSION" == "latest" ]]; then - PT_BUILD=nightly/${PT_BUILD} - fi - - # Where to get the archive - LIBTORCH_URL=https://download.pytorch.org/libtorch/${PT_BUILD}/libtorch-${PT_OS}-${PT_VERSION}.zip - # Directory where torch is extracted to - LIBTORCH_DIRECTORY=libtorch-${PT_OS}-${PT_VERSION} - - # Archive - specifically named - LIBTORCH_ARCHIVE=libtorch-${PT_OS}-${PT_BUILD}-${PT_VERSION}.zip - - if [ ! -e "${LIBTORCH_ARCHIVE}" ]; then - echo "Downloading libtorch ${PT_VERSION} ${PT_BUILD}" - curl -s -L ${LIBTORCH_URL} > ${LIBTORCH_ARCHIVE} - fi - - unzip -q -o ${LIBTORCH_ARCHIVE} -d ${TORCH_PREFIX}/../ - - echo "Done." - - if [[ "${PT_OS}" == "macos" ]]; then - echo "Installing MKL ..." - # also download mkl - MKL_BUNDLE=mklml_mac_2019.0.3.20190220 - if [ ! -e "${MKL_BUNDLE}.tgz" ]; then - wget -q "https://github.com/intel/mkl-dnn/releases/download/v0.18/${MKL_BUNDLE}.tgz" - fi - tar xzf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${TORCH_PREFIX} - mkdir -p ${ORT_PREFIX} - tar xzf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${ORT_PREFIX} - echo "Done." - fi +[[ $FORCE == 1 ]] && rm -rf libtorch + +if [[ ! -d libtorch ]]; then + echo "Installing libtorch ..." + + if [[ $OS == linux ]]; then + PT_OS=linux + if [[ $GPU == no ]]; then + PT_BUILD=cpu + else + PT_BUILD=cu100 + fi + if [[ $ARCH == x64 ]]; then + PT_ARCH=x86_64 + elif [[ $ARCH == arm64v8 ]]; then + PT_ARCH=arm64 + elif [[ $ARCH == arm32v7 ]]; then + PT_ARCH=arm + fi + elif [[ $OS == macosx ]]; then + PT_OS=macos + PT_BUILD=cpu + fi + + [[ "$PT_VERSION" == "latest" ]] && PT_BUILD=nightly/${PT_BUILD} + + LIBTORCH_ARCHIVE=libtorch-${PT_BUILD}-${PT_OS}-${PT_ARCH}-${PT_VERSION}.tar.gz + LIBTORCH_URL=https://s3.amazonaws.com/redismodules/pytorch/$LIBTORCH_ARCHIVE + + [[ ! -f $LIBTORCH_ARCHIVE || $FORCE == 1 ]] && wget -q $LIBTORCH_URL + + rm -rf libtorch.x + mkdir libtorch.x + + tar xf $LIBTORCH_ARCHIVE --no-same-owner -C libtorch.x + mv libtorch.x/libtorch libtorch + rmdir libtorch.x + + echo "Done." else - echo "librotch is in place." + echo "librotch is in place." fi -## ONNXRUNTIME -ORT_VERSION="0.5.0" -ORT_ARCH="x64" -if [[ "$OSTYPE" == "linux-gnu" ]]; then - if [[ "$1" == "cpu" ]]; then - ORT_OS="linux" - ORT_BUILD="" - else - ORT_OS="linux" - ORT_BUILD="-gpu" - fi -elif [[ "$OSTYPE" == "darwin"* ]]; then - ORT_OS="osx" - ORT_BUILD="" +########################################################################################### MKL + +if [[ ! -d mkl ]]; then + MKL_VERSION=0.18 + MKL_BUNDLE_VER=2019.0.3.20190220 + if [[ $OS == macosx ]]; then + echo "Installing MKL ..." + + MKL_OS=mac + MKL_ARCHIVE=mklml_${MKL_OS}_${MKL_BUNDLE_VER}.tgz + [[ ! -e ${MKL_ARCHIVE} ]] && wget -q https://github.com/intel/mkl-dnn/releases/download/v${MKL_VERSION}/${MKL_ARCHIVE} + + rm -rf mkl.x + mkdir mkl.x + tar xzf ${MKL_ARCHIVE} --no-same-owner --strip-components=1 -C mkl.x + mv mkl.x mkl + + echo "Done." + fi +else + echo "mkl is in place." fi -[[ $FORCE == 1 ]] && rm -rf ${ORT_PREFIX} +################################################################################### ONNXRUNTIME + +ORT_VERSION="0.5.0" + +[[ $FORCE == 1 ]] && rm -rf onnxruntime -ORT_ARCHIVE=onnxruntime-${ORT_OS}-${ORT_ARCH}${ORT_BUILD}-${ORT_VERSION}.tgz +if [[ ! -d onnxruntime ]]; then + echo "Installing ONNXRuntime ..." -if [[ ! -d ${ORT_PREFIX} ]]; then - echo "Installing ONNXRuntime ..." + if [[ $OS == linux ]]; then + ORT_OS=linux + if [[ $GPU == no ]]; then + ORT_BUILD="" + else + ORT_BUILD="-gpu" + fi + if [[ $ARCH == x64 ]]; then + ORT_ARCH=x64 + ORT_URL_BASE=https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION} + elif [[ $ARCH == arm64v8 ]]; then + ORT_ARCH=arm64 + ORT_URL_BASE=https://s3.amazonaws.com/redismodules/onnxruntime + elif [[ $ARCH == arm32v7 ]]; then + ORT_ARCH=arm + ORT_URL_BASE=https://s3.amazonaws.com/redismodules/onnxruntime + fi + elif [[ $OS == macosx ]]; then + ORT_OS=osx + ORT_ARCH=x64 + ORT_BUILD="" + ORT_URL_BASE=https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION} + fi - mkdir -p ${ORT_PREFIX} + ORT_ARCHIVE=onnxruntime-${ORT_OS}-${ORT_ARCH}${ORT_BUILD}-${ORT_VERSION}.tgz - if [ ! -e ${ORT_ARCHIVE} ]; then - echo "Downloading ONNXRuntime ${ORT_VERSION} ${DEVICE}" - wget -q https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION}/${ORT_ARCHIVE} - fi - - tar xf ${ORT_ARCHIVE} --no-same-owner --strip-components=1 -C ${ORT_PREFIX} + if [[ ! -e ${ORT_ARCHIVE} ]]; then + wget -q $ORT_URL_BASE/${ORT_ARCHIVE} + echo "Done." + fi + + rm -rf onnxruntime.x + mkdir onnxruntime.x + tar xzf ${ORT_ARCHIVE} --no-same-owner --strip-components=1 -C onnxruntime.x + mv onnxruntime.x onnxruntime + + echo "Done." else - echo "onnxruntime is in place." + echo "ONNXRuntime is in place." fi -echo "Done." +############################################################################# Collect libraries + +#if [[ ! -d install ]]; then +# echo "Collecting binaries..." +# +# rm -rf install.x +# python3 $HERE/automation/collect-deps.py --into install.x +# mv install.x install +# +# echo "Done." +#else +# echo "Binaries in place." +#fi + +echo "Done." \ No newline at end of file From 3068d2b97e93c3b55a20df63f3ee41a6f5886838 Mon Sep 17 00:00:00 2001 From: rafie Date: Sun, 25 Aug 2019 12:57:39 +0300 Subject: [PATCH 02/30] Build: fixes #1 --- automation/Makefile | 20 +++++++++++++------- automation/readies/bin/getpy | 2 ++ automation/readies/bin/getpy2 | 27 ++++++++++++++++----------- automation/readies/mk/main | 6 ++++++ 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/automation/Makefile b/automation/Makefile index a60ea609b..a1f7239f6 100755 --- a/automation/Makefile +++ b/automation/Makefile @@ -6,11 +6,14 @@ MK_CMAKE:=1 MK_CMAKE_INSTALL:=1 define HELP -make fetch # download and prepare dependant modules -make build # compile and link -make clean # remove build artifacts -make pack # create installation packages -make test # run tests +make setup # install prerequisited (CAUTION: THIS WILL MODIFY YOUR SYSTEM) +make fetch # download and prepare dependant modules +make build # compile and link +make clean # remove build artifacts +make pack # create installation packages +make test # run tests +make deploy # copy packages to S3 +make release # release a version endef #---------------------------------------------------------------------------------------------- @@ -40,7 +43,7 @@ CMAKE_FLAGS += \ -DDEPS_PATH=$(realpath $(ROOT)/deps) \ -DDEVICE=$(DEVICE) -include readies/mk/defs +include $(MK)/defs #---------------------------------------------------------------------------------------------- @@ -50,7 +53,10 @@ include $(MK)/rules #---------------------------------------------------------------------------------------------- -$(TARGET): $(MK_MAKEFILES) $(DEPS) +prebuild: + @if [ -d $(ROOT)/deps ]; then echo $$'Dependencies are not in place.\nPlease run \'make fetch\'.'; exit 1; fi + +$(TARGET): prebuild $(MK_MAKEFILES) $(DEPS) $(MAKE) -C $(BINDIR) $(MAKE) -C $(BINDIR) install cd $(ROOT) ;\ diff --git a/automation/readies/bin/getpy b/automation/readies/bin/getpy index edf875647..3273be40e 100755 --- a/automation/readies/bin/getpy +++ b/automation/readies/bin/getpy @@ -16,6 +16,8 @@ fi [ ! -z $(command -v python3) ] && exit 0 +[[ $CHECK == 1 ]] && exit 1 + if [ ! -z $(command -v apt-get) ]; then runn apt-get -qq update runn apt-get -qq install -y python3 diff --git a/automation/readies/bin/getpy2 b/automation/readies/bin/getpy2 index f957042ca..2ac6417e9 100755 --- a/automation/readies/bin/getpy2 +++ b/automation/readies/bin/getpy2 @@ -2,10 +2,12 @@ [ "$VERBOSE" = "1" ] && set -x -show_if_error() { - { "${@:1}"; } > /tmp/py2.log 2>&1 - [ $? != 0 ] && cat /tmp/py2.log - rm -f /tmp/py2.log +runn() { + [[ $NOP == 1 ]] && { echo "${@:1}"; return; } + __runn_log=$(mktemp /tmp/run.XXXXX) + { "${@:1}"; } > $__runn_log 2>&1 + [ $? != 0 ] && cat $__runn_log + rm -f $__runn_log } if [ ! -z $(command -v python) ]; then @@ -13,19 +15,22 @@ if [ ! -z $(command -v python) ]; then fi [ ! -z $(command -v python2) ] && exit 0 +[ ! -z $(command -v python2.7) ] && exit 0 + +[[ $CHECK == 1 ]] && exit 1 if [ ! -z $(command -v apt-get) ]; then - show_if_error apt-get -qq update - show_if_error apt-get -qq install -y python + runn apt-get -qq update + runn apt-get -qq install -y python elif [ ! -z $(command -v dnf) ]; then - show_if_error dnf install -y python2 + runn dnf install -y python2 elif [ ! -z $(command -v yum) ]; then - show_if_error yum install -y python2 + runn yum install -y python2 elif [ ! -z $(command -v apk) ]; then - show_if_error apk update - show_if_error apk add python2 + runn apk update + runn apk add python2 elif [ ! -z $(command -v brew) ]; then - show_if_error brew install python2 + runn brew install python2 fi if [ -z $(command -v python) ]; then diff --git a/automation/readies/mk/main b/automation/readies/mk/main index 336de23c4..3df0f814a 100755 --- a/automation/readies/mk/main +++ b/automation/readies/mk/main @@ -1,4 +1,10 @@ +SHELL=/bin/bash + +ifneq ($(shell { CHECK=1 $(ROOT)/automation/readies/bin/getpy; echo $$?; }),0) +$(error It seems prerequisites have not been installed: please run 'make setup'.) +endif + MK=$(ROOT)/automation/readies/mk MK_ALL_TARGETS:=bindirs build From cfd388ed38b153a2e946b246ce52f2fac0160626 Mon Sep 17 00:00:00 2001 From: rafie Date: Sun, 25 Aug 2019 13:00:34 +0300 Subject: [PATCH 03/30] Build: fixes #2 --- automation/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/Makefile b/automation/Makefile index a1f7239f6..88851f3dc 100755 --- a/automation/Makefile +++ b/automation/Makefile @@ -74,7 +74,7 @@ endif fetch: @echo Fetching dependencies... - $(SHOW)VERBOSE=$(_SHOW) ./get_deps.sh $(DEPS_FLAGS) + $(SHOW)VERBOSE=$(_SHOW)$(ROOT)/get_deps.sh $(DEPS_FLAGS) #---------------------------------------------------------------------------------------------- From 9543ac9b81b152d207265caf2116547744f25e88 Mon Sep 17 00:00:00 2001 From: rafie Date: Sun, 25 Aug 2019 13:49:10 +0300 Subject: [PATCH 04/30] Build: fixes #3 --- automation/Makefile | 2 +- automation/readies/mk/common.defs | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/automation/Makefile b/automation/Makefile index 88851f3dc..83033595f 100755 --- a/automation/Makefile +++ b/automation/Makefile @@ -74,7 +74,7 @@ endif fetch: @echo Fetching dependencies... - $(SHOW)VERBOSE=$(_SHOW)$(ROOT)/get_deps.sh $(DEPS_FLAGS) + $(SHOW)VERBOSE=$(_SHOW) $(ROOT)/get_deps.sh $(DEPS_FLAGS) #---------------------------------------------------------------------------------------------- diff --git a/automation/readies/mk/common.defs b/automation/readies/mk/common.defs index 206cd969d..1331c523b 100755 --- a/automation/readies/mk/common.defs +++ b/automation/readies/mk/common.defs @@ -1,5 +1,8 @@ _SHOW:=$(SHOW) +ifeq ($(_SHOW),) +_SHOW:=0 +endif ifeq ($(SHOW),1) override SHOW:= else From 5dcdfd2cff32f73af98dff2d3a3a98ac0ab4d08a Mon Sep 17 00:00:00 2001 From: rafie Date: Sun, 25 Aug 2019 14:20:21 +0300 Subject: [PATCH 05/30] Build: fixes #4 --- automation/Makefile | 2 +- get_deps.sh | 73 ++++++++++++++++++++------------------------- 2 files changed, 34 insertions(+), 41 deletions(-) diff --git a/automation/Makefile b/automation/Makefile index 83033595f..fba9637f5 100755 --- a/automation/Makefile +++ b/automation/Makefile @@ -54,7 +54,7 @@ include $(MK)/rules #---------------------------------------------------------------------------------------------- prebuild: - @if [ -d $(ROOT)/deps ]; then echo $$'Dependencies are not in place.\nPlease run \'make fetch\'.'; exit 1; fi + @if [ ! -d $(ROOT)/deps ]; then echo $$'Dependencies are not in place.\nPlease run \'make fetch\'.'; exit 1; fi $(TARGET): prebuild $(MK_MAKEFILES) $(DEPS) $(MAKE) -C $(BINDIR) diff --git a/get_deps.sh b/get_deps.sh index 09278ce76..2f7bbf8a8 100755 --- a/get_deps.sh +++ b/get_deps.sh @@ -49,35 +49,31 @@ cd ${DEPS_DIR} # PREFIX=${DEPS_DIR}/install # mkdir -p ${PREFIX} -# DLPACK_PREFIX=${PREFIX}/dlpack -# TF_PREFIX=${PREFIX}/libtensorflow -# TORCH_PREFIX=${PREFIX}/libtorch -# ORT_PREFIX=${PREFIX}/onnxruntime +DLPACK=dlpack +LIBTENSORFLOW=libtensorflow +LIBTORCH=libtorch +MKL=mkl +ONNXRUNTIME=onnxruntime ######################################################################################## DLPACK -[[ $FORCE == 1 ]] && rm -rf dlpack +[[ $FORCE == 1 ]] && rm -rf $DLPACK -if [[ ! -d dlpack ]]; then +if [[ ! -d $DLPACK ]]; then echo "Cloning dlpack ..." - git clone --depth 1 https://github.com/dmlc/dlpack.git + git clone --depth 1 https://github.com/dmlc/dlpack.git $DLPACK echo "Done." else echo "dlpack is in place." fi -# if [[ ! -d dlpack/include ]]; then -# mkdir -p dlpack -# ln -sf ${DEPS_DIR}/dlpack/include ${DLPACK_PREFIX}/include -# fi - #################################################################################### TENSORFLOW TF_VERSION="1.14.0" -[[ $FORCE == 1 ]] && rm -rf tensorflow +[[ $FORCE == 1 ]] && rm -rf $LIBTENSORFLOW -if [[ ! -d tensorflow ]]; then +if [[ ! -d $LIBTENSORFLOW ]]; then echo "Installing TensorFlow ..." if [[ $OS == linux ]]; then @@ -112,10 +108,10 @@ if [[ ! -d tensorflow ]]; then [[ ! -f $LIBTF_ARCHIVE || $FORCE == 1 ]] && wget --quiet $LIBTF_URL_BASE/$LIBTF_ARCHIVE - rm -rf tensorflow.x - mkdir tensorflow.x - tar xf $LIBTF_ARCHIVE --no-same-owner --strip-components=1 -C tensorflow.x - mv tensorflow.x tensorflow + rm -rf $LIBTENSORFLOW.x + mkdir $LIBTENSORFLOW.x + tar xf $LIBTF_ARCHIVE --no-same-owner --strip-components=1 -C $LIBTENSORFLOW.x + mv $LIBTENSORFLOW.x $LIBTENSORFLOW echo "Done." else @@ -126,9 +122,9 @@ fi PT_VERSION="1.2.0" -[[ $FORCE == 1 ]] && rm -rf libtorch +[[ $FORCE == 1 ]] && rm -rf $LIBTORCH -if [[ ! -d libtorch ]]; then +if [[ ! -d $LIBTORCH ]]; then echo "Installing libtorch ..." if [[ $OS == linux ]]; then @@ -157,12 +153,12 @@ if [[ ! -d libtorch ]]; then [[ ! -f $LIBTORCH_ARCHIVE || $FORCE == 1 ]] && wget -q $LIBTORCH_URL - rm -rf libtorch.x - mkdir libtorch.x + rm -rf $LIBTORCH.x + mkdir $LIBTORCH.x - tar xf $LIBTORCH_ARCHIVE --no-same-owner -C libtorch.x - mv libtorch.x/libtorch libtorch - rmdir libtorch.x + tar xf $LIBTORCH_ARCHIVE --no-same-owner -C $LIBTORCH.x + mv $LIBTORCH.x/libtorch $LIBTORCH + rmdir $LIBTORCH.x echo "Done." else @@ -181,10 +177,10 @@ if [[ ! -d mkl ]]; then MKL_ARCHIVE=mklml_${MKL_OS}_${MKL_BUNDLE_VER}.tgz [[ ! -e ${MKL_ARCHIVE} ]] && wget -q https://github.com/intel/mkl-dnn/releases/download/v${MKL_VERSION}/${MKL_ARCHIVE} - rm -rf mkl.x - mkdir mkl.x - tar xzf ${MKL_ARCHIVE} --no-same-owner --strip-components=1 -C mkl.x - mv mkl.x mkl + rm -rf $MKL.x + mkdir $MKL.x + tar xzf ${MKL_ARCHIVE} --no-same-owner --strip-components=1 -C $MKL.x + mv $MKL.x $MKL echo "Done." fi @@ -196,9 +192,9 @@ fi ORT_VERSION="0.5.0" -[[ $FORCE == 1 ]] && rm -rf onnxruntime +[[ $FORCE == 1 ]] && rm -rf $ONNXRUNTIME -if [[ ! -d onnxruntime ]]; then +if [[ ! -d $ONNXRUNTIME ]]; then echo "Installing ONNXRuntime ..." if [[ $OS == linux ]]; then @@ -227,15 +223,12 @@ if [[ ! -d onnxruntime ]]; then ORT_ARCHIVE=onnxruntime-${ORT_OS}-${ORT_ARCH}${ORT_BUILD}-${ORT_VERSION}.tgz - if [[ ! -e ${ORT_ARCHIVE} ]]; then - wget -q $ORT_URL_BASE/${ORT_ARCHIVE} - echo "Done." - fi + [[ ! -e ${ORT_ARCHIVE} ]] && wget -q $ORT_URL_BASE/${ORT_ARCHIVE} - rm -rf onnxruntime.x - mkdir onnxruntime.x - tar xzf ${ORT_ARCHIVE} --no-same-owner --strip-components=1 -C onnxruntime.x - mv onnxruntime.x onnxruntime + rm -rf $ONNXRUNTIME.x + mkdir $ONNXRUNTIME.x + tar xzf ${ORT_ARCHIVE} --no-same-owner --strip-components=1 -C $ONNXRUNTIME.x + mv $ONNXRUNTIME.x $ONNXRUNTIME echo "Done." else @@ -256,4 +249,4 @@ fi # echo "Binaries in place." #fi -echo "Done." \ No newline at end of file +# echo "Done." \ No newline at end of file From 9b0ba405f703d925689feaa52561e1c93033ac02 Mon Sep 17 00:00:00 2001 From: rafie Date: Mon, 26 Aug 2019 18:15:59 +0300 Subject: [PATCH 06/30] Build fixes #5 --- .dockerignore | 3 +- .gitignore | 2 +- CMakeLists.txt | 74 +++++++++++++++++++++--------- automation/Makefile | 28 ++++++----- automation/readies/mk/bindirs.defs | 1 + automation/readies/mk/cmake.rules | 3 +- automation/readies/mk/common.defs | 2 +- automation/readies/mk/common.rules | 5 +- automation/readies/mk/variant.defs | 2 +- automation/system-setup.py | 6 ++- build/libtorch/Makefile | 24 +++++++--- build/libtorch/repack.sh | 6 ++- get_deps.sh | 7 +-- 13 files changed, 107 insertions(+), 56 deletions(-) diff --git a/.dockerignore b/.dockerignore index 0a65b637b..0f6c14c4a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,10 +1,9 @@ /bin/ /deps/ -/install*/ +/install* .venv/ venv*/ /[0-9]*/ -/install *.zip *.tgz *.tar.gz diff --git a/.gitignore b/.gitignore index be23442e5..8f6f89af3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ /bin/ /deps/ -/install*/ +/install* /test/venv/ /test/logs/ .venv/ diff --git a/CMakeLists.txt b/CMakeLists.txt index d84056e82..209ef8797 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,15 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.0.0) PROJECT(RedisAI) -IF (NOT DEPS_PATH) - # dependencies are required! - SET(DEPS_PATH ../deps) - # MESSAGE(FATAL_ERROR "DEPS PATH Missing!") -ENDIF() - -IF (NOT DEVICE) - SET(DEVICE cpu) -ENDIF() +#---------------------------------------------------------------------------------------------- FUNCTION(ADD_LDFLAGS _TARGET NEW_FLAGS) GET_TARGET_PROPERTY(LD_FLAGS ${_TARGET} LINK_FLAGS) @@ -19,11 +11,32 @@ FUNCTION(ADD_LDFLAGS _TARGET NEW_FLAGS) SET_TARGET_PROPERTIES(${_TARGET} PROPERTIES LINK_FLAGS ${NEW_FLAGS}) ENDFUNCTION() +MACRO(install_symlink filepath sympath) + install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${filepath} ${sympath})") + install(CODE "message(\"-- Created symlink: ${sympath} -> ${filepath}\")") +ENDMACRO(install_symlink) + +#---------------------------------------------------------------------------------------------- + +IF (NOT DEVICE) + SET(DEVICE cpu) +ENDIF() + +IF (NOT DEPS_PATH) + SET(DEPS_PATH ../deps) +ENDIF() + +IF (NOT INSTALL_PATH) + SET(INSTALL_PATH ../install-${DEVICE}) +ENDIF() + GET_FILENAME_COMPONENT(depsAbs "${DEPS_PATH}" REALPATH BASE_DIR ${CMAKE_SOURCE_DIR}) -# BASE_DIR "${CMAKE_BINARY_DIR}") -# SET(CUDA_TOOLKIT_ROOT_DIR /usr/local/cuda-10.0) +GET_FILENAME_COMPONENT(installAbs + "${INSTALL_PATH}" REALPATH BASE_DIR ${CMAKE_SOURCE_DIR}) + +#---------------------------------------------------------------------------------------------- INCLUDE_DIRECTORIES(${depsAbs}/dlpack/include) INCLUDE_DIRECTORIES(${depsAbs}/libtensorflow/include) @@ -34,9 +47,11 @@ SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") SET(CMAKE_C_STANDARD 11) +# SET(CUDA_TOOLKIT_ROOT_DIR /usr/local/cuda-10.0) + #---------------------------------------------------------------------------------------------- -FIND_LIBRARY(TF_LIBRARIES NAMES tensorflow #libtensorflow.so +FIND_LIBRARY(TF_LIBRARIES NAMES tensorflow PATHS ${depsAbs}/libtensorflow/lib) IF (NOT TF_LIBRARIES) MESSAGE(FATAL_ERROR "Could not find tensorflow") @@ -53,7 +68,8 @@ ENDIF() #---------------------------------------------------------------------------------------------- IF (APPLE) - FIND_LIBRARY(MKL_LIBRARIES NAMES mklml) + FIND_LIBRARY(MKL_LIBRARIES NAMES mklml + PATHS ${depsAbs}/mkl/lib) IF (NOT MKL_LIBRARIES) MESSAGE(FATAL_ERROR "Could not find MKL for Mac") ENDIF() @@ -76,7 +92,6 @@ ADD_SUBDIRECTORY(util/libtorch_c) ADD_SUBDIRECTORY(src) ADD_LIBRARY(redisai SHARED $) -# TARGET_LINK_LIBRARIES(redisai ${ORT_LIBRARIES} torch_c ${TF_LIBRARIES} ${platDeps} ${CMAKE_DL_LIBS}) TARGET_LINK_LIBRARIES(redisai ${CMAKE_DL_LIBS}) SET_TARGET_PROPERTIES(redisai PROPERTIES PREFIX "") @@ -87,8 +102,6 @@ IF (APPLE) LINK_FLAGS "-undefined dynamic_lookup") ENDIF() -SET(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install-${DEVICE}) - INSTALL(TARGETS redisai LIBRARY DESTINATION .) IF (APPLE) @@ -98,6 +111,12 @@ ELSE () SET_TARGET_PROPERTIES(redisai PROPERTIES INSTALL_RPATH "\$ORIGIN/lib") ENDIF() +IF (APPLE) + SET(LIB_PATTERN "*.dylib") +ELSE() + SET(LIB_PATTERN "*.so*") +ENDIF() + #---------------------------------------------------------------------------------------------- ADD_LIBRARY(redisai_tensorflow SHARED $) @@ -111,8 +130,8 @@ ELSE () SET_TARGET_PROPERTIES(redisai_tensorflow PROPERTIES INSTALL_RPATH "\$ORIGIN/lib") ENDIF() INSTALL(TARGETS redisai_tensorflow LIBRARY DESTINATION backends/redisai_tensorflow) -INSTALL(DIRECTORY ${depsAbs}/libtensorflow/lib DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/redisai_tensorflow - FILES_MATCHING PATTERN "*.so*") +INSTALL(DIRECTORY ${depsAbs}/libtensorflow/lib DESTINATION ${installAbs}/backends/redisai_tensorflow + FILES_MATCHING PATTERN ${LIB_PATTERN}) #---------------------------------------------------------------------------------------------- @@ -127,8 +146,8 @@ ELSE () SET_TARGET_PROPERTIES(redisai_torch PROPERTIES INSTALL_RPATH "\$ORIGIN/lib") ENDIF() INSTALL(TARGETS redisai_torch LIBRARY DESTINATION backends/redisai_torch) -INSTALL(DIRECTORY ${depsAbs}/libtorch/lib DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/redisai_torch - FILES_MATCHING PATTERN "*.so*") +INSTALL(DIRECTORY ${depsAbs}/libtorch/lib DESTINATION ${installAbs}/backends/redisai_torch + FILES_MATCHING PATTERN ${LIB_PATTERN}) #---------------------------------------------------------------------------------------------- @@ -147,7 +166,18 @@ ELSE () SET_TARGET_PROPERTIES(redisai_onnxruntime PROPERTIES INSTALL_RPATH "\$ORIGIN/lib") ENDIF() INSTALL(TARGETS redisai_onnxruntime LIBRARY DESTINATION backends/redisai_onnxruntime) -INSTALL(DIRECTORY ${depsAbs}/onnxruntime/lib DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/redisai_onnxruntime - FILES_MATCHING PATTERN "*.so*") +INSTALL(DIRECTORY ${depsAbs}/onnxruntime/lib DESTINATION ${installAbs}/backends/redisai_onnxruntime + FILES_MATCHING PATTERN ${LIB_PATTERN}) #---------------------------------------------------------------------------------------------- + +IF (APPLE) + INSTALL(DIRECTORY ${depsAbs}/mkl/lib DESTINATION ${installAbs}/backends/redisai_torch + FILES_MATCHING PATTERN ${LIB_PATTERN}) +ENDIF() + +#---------------------------------------------------------------------------------------------- + +IF (NOT ${installAbs} STREQUAL ${CMAKE_SOURCE_DIR}/install-${DEVICE}) + INSTALL_SYMLINK(${installAbs} ${CMAKE_SOURCE_DIR}/install-${DEVICE}) +ENDIF() diff --git a/automation/Makefile b/automation/Makefile index fba9637f5..5dbe2a146 100755 --- a/automation/Makefile +++ b/automation/Makefile @@ -10,8 +10,8 @@ make setup # install prerequisited (CAUTION: THIS WILL MODIFY YOUR SYSTEM) make fetch # download and prepare dependant modules make build # compile and link make clean # remove build artifacts -make pack # create installation packages make test # run tests +make pack # create installation packages make deploy # copy packages to S3 make release # release a version endef @@ -33,14 +33,17 @@ endif SRCDIR=.. BINDIR=$(BINROOT)/src -INSTALL_DIR=$(ROOT)/install-$(DEVICE) +# INSTALL_DIR=$(ROOT)/install-$(DEVICE) +DEPS_DIR=$(ROOT)/deps/$(OS)-$(ARCH)-$(DEVICE) +INSTALL_DIR=$(ROOT)/bin/$(OS)-$(ARCH)-$(DEVICE)/install TARGET=$(BINDIR)/redisai.so BACKENDS_PATH ?= $(INSTALL_DIR)/backends CMAKE_FLAGS += \ - -DDEPS_PATH=$(realpath $(ROOT)/deps) \ + -DDEPS_PATH=$(realpath $(DEPS_DIR)) \ + -DINSTALL_PATH=$(realpath $(INSTALL_DIR)) \ -DDEVICE=$(DEVICE) include $(MK)/defs @@ -54,17 +57,18 @@ include $(MK)/rules #---------------------------------------------------------------------------------------------- prebuild: - @if [ ! -d $(ROOT)/deps ]; then echo $$'Dependencies are not in place.\nPlease run \'make fetch\'.'; exit 1; fi + $(SHOW)if [ ! -d $(ROOT)/deps ]; then echo $$'Dependencies are not in place.\nPlease run \'make fetch\'.'; exit 1; fi $(TARGET): prebuild $(MK_MAKEFILES) $(DEPS) - $(MAKE) -C $(BINDIR) - $(MAKE) -C $(BINDIR) install - cd $(ROOT) ;\ - if [ ! -e install ]; then ln -sf install-$(DEVICE) install; fi + $(SHOW)mkdir -p $(INSTALL_DIR) + $(SHOW)$(MAKE) -C $(BINDIR) + $(SHOW)$(MAKE) -C $(BINDIR) install +# $(SHOW)cd $(ROOT) ;\ +# if [ ! -e install ]; then ln -sf install-$(DEVICE) install; fi clean: ifeq ($(ALL),1) - cd $(ROOT) ;\ + $(SHOW)cd $(ROOT) ;\ rm -rf build install deps/dlpack deps/install-$(DEVICE) deps/*.tar.gz deps/*.zip deps/*.tgz else $(SHOW)$(MAKE) -C $(BINDIR) clean @@ -92,8 +96,8 @@ pack_deps: pack_ramp #---------------------------------------------------------------------------------------------- test: - @[[ $(git lfs env > /dev/null 2>&1 ; echo $?) != 0 ]] && git lfs install - @git lfs pull - @set -e ;\ + $(SHOW)if [ "$(git lfs env > /dev/null 2>&1 ; echo $?)" != "0" ]; then git lfs install; fi + $(SHOW)git lfs pull + $(SHOW)set -e ;\ cd $(ROOT)/test ;\ python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so diff --git a/automation/readies/mk/bindirs.defs b/automation/readies/mk/bindirs.defs index 1ff57c788..d41701347 100755 --- a/automation/readies/mk/bindirs.defs +++ b/automation/readies/mk/bindirs.defs @@ -1,5 +1,6 @@ BINROOT=$(ROOT)/bin/$(FULL_VARIANT) +BINROOT.release=$(ROOT)/bin/$(FULL_VARIANT.release) BIN_DIRS=$(sort $(patsubst %/,%,$(BINDIR) $(dir $(OBJECTS)))) define mkdir_rule diff --git a/automation/readies/mk/cmake.rules b/automation/readies/mk/cmake.rules index f6d8d7e60..a9e4d6902 100755 --- a/automation/readies/mk/cmake.rules +++ b/automation/readies/mk/cmake.rules @@ -1,7 +1,8 @@ ifneq ($(MK_CMAKE),) -$(BINDIR)/Makefile : $(SRCDIR)/CMakeLists.txt +$(BINDIR)/Makefile : bindirs $(SRCDIR)/CMakeLists.txt + $(SHOW)if [ ! -d $(BINDIR) ]; then echo "CMake: $(BINDIR) does not exist."; exit 1; fi $(SHOW)cd $(BINDIR); cmake $(CMAKE_WHY) $(CMAKE_FLAGS) $(realpath $(SRCDIR)) # $(SHOW)cd $(BINDIR); \ diff --git a/automation/readies/mk/common.defs b/automation/readies/mk/common.defs index 1331c523b..1735cbceb 100755 --- a/automation/readies/mk/common.defs +++ b/automation/readies/mk/common.defs @@ -13,5 +13,5 @@ MAKEFLAGS += --no-builtin-rules --no-print-directory # --no-builtin-variables define __SEP -import os; rows, cols = os.popen('stty size', 'r').read().split(); print(\"\n\" + '-' * (int(cols) - 1) + \"\n\") +import os; rows, cols = os.popen('stty size', 'r').read().split(); print('-' * (int(cols) - 1) + \"\n\") endef diff --git a/automation/readies/mk/common.rules b/automation/readies/mk/common.rules index 633cf2f6b..f84f87be2 100755 --- a/automation/readies/mk/common.rules +++ b/automation/readies/mk/common.rules @@ -1,6 +1,5 @@ .PHONY: __sep -__sep: ; -# @python -c "$(__SEP)" - +__sep: + @python -c "$(__SEP)" diff --git a/automation/readies/mk/variant.defs b/automation/readies/mk/variant.defs index aaccb3849..b0cd5638f 100755 --- a/automation/readies/mk/variant.defs +++ b/automation/readies/mk/variant.defs @@ -35,7 +35,7 @@ else __VARIANT:=-$(VARIANT) endif FULL_VARIANT:=$(OS)-$(ARCH)-$(FLAVOR)$(__VARIANT) -FULL_VARIANT_REL:=$(OS)-$(ARCH)-release$(__VARIANT) +FULL_VARIANT.release:=$(OS)-$(ARCH)-release$(__VARIANT) ifneq ($(origin VARIANT),) $(eval $(shell if [ -z $(VARIANT) ]; then rm -f $(ROOT)/VARIANT; else echo $(VARIANT)>$(ROOT)/VARIANT; fi)) diff --git a/automation/system-setup.py b/automation/system-setup.py index 588811355..8adce802a 100755 --- a/automation/system-setup.py +++ b/automation/system-setup.py @@ -19,7 +19,9 @@ def common_first(self): self.pip3_install("wheel") self.pip3_install("setuptools --upgrade") - self.install("git cmake ca-certificates curl unzip wget patchelf awscli") + if self.os == 'linux': + self.install("ca-certificates") + self.install("git cmake curl unzip wget patchelf awscli") def debian_compat(self): self.install("build-essential") @@ -56,7 +58,7 @@ def common_last(self): self.pip3_install("git+https://github.com/RedisLabsModules/RLTest.git@master") if not self.has_command("ramp"): self.pip3_install("git+https://github.com/RedisLabs/RAMP@master") - self.pip3_install("-r test/test_requirements.txt") + self.pip3_install("-r ../test/test_requirements.txt") #---------------------------------------------------------------------------------------------- diff --git a/build/libtorch/Makefile b/build/libtorch/Makefile index 301aa3bcb..d01b454ff 100755 --- a/build/libtorch/Makefile +++ b/build/libtorch/Makefile @@ -8,12 +8,16 @@ OSNICK ?= buster S3_URL=redismodules/pytorch -STEM=libtorch-cpu-linux - -OS.bionic=ubuntu:bionic -OS.stretch=debian:stretch-slim -OS.buster=debian:buster-slim -OS=$(OS.$(OSNICK)) +OS:=$(shell $(ROOT)/automation/readies/bin/platform --os) +ifeq ($(OS),macosx) +OS:=macos +endif +STEM=libtorch-cpu-$(OS) + +DOCKER_OS.bionic=ubuntu:bionic +DOCKER_OS.stretch=debian:stretch-slim +DOCKER_OS.buster=debian:buster-slim +DOCKER_OS=$(DOCKER_OS.$(OSNICK)) #---------------------------------------------------------------------------------------------- @@ -34,7 +38,7 @@ $(eval $(call targets,PUBLISH,publish)) define build_x64 # (1=arch, 2=tar-arch) build_x64: @docker build --iidfile $$(IID_$(1)) -t redisfab/$(STEM)-$(1):$(VERSION) -f Dockerfile.x64 \ - --build-arg OS=$(OS) $(ROOT) + --build-arg OS=$(DOCKER_OS) $(ROOT) @docker create --cidfile $$(CID_$(1)) `cat $$(IID_$(1))` @docker cp `cat $$(CID_$(1))`:/build/$(STEM)-$(2)-$(VERSION).tar.gz . @@ -80,6 +84,12 @@ $(eval $(call build_x64,x64,x86_64)) $(eval $(call build_arm,arm64v8,arm64)) $(eval $(call build_arm,arm32v7,arm)) +ifneq ($(filter publish,$(MAKECMDGOALS)),) +ifeq ($(wildcard $(HOME)/.aws/credentials),) +$(error Please run 'aws configure' and provide it with access credentials) +endif +endif + publish: $(PUBLISH_TARGETS) $(eval $(call publish_x64,x64,x86_64)) diff --git a/build/libtorch/repack.sh b/build/libtorch/repack.sh index 910801191..34fa4ef0b 100755 --- a/build/libtorch/repack.sh +++ b/build/libtorch/repack.sh @@ -16,6 +16,9 @@ fi OS=$(python3 $ROOT/automation/readies/bin/platform --os) ARCH=$(python3 $ROOT/automation/readies/bin/platform --arch) +# avoid wget warnings on macOS +[[ $OS == macosx ]] && export LC_ALL=en_US.UTF-8 + if [[ -z $PT_VERSION ]]; then PT_VERSION=1.2.0 #PT_VERSION="latest" @@ -33,6 +36,7 @@ if [[ $OS == linux ]]; then fi elif [[ $OS == macosx ]]; then PT_OS=macos + PT_ARCH=x86_64 PT_BUILD=cpu fi @@ -43,7 +47,7 @@ LIBTORCH_ARCHIVE=libtorch-${PT_OS}-${PT_VERSION}.zip if [ ! -f $LIBTORCH_ARCHIVE ]; then echo "Downloading libtorch ${PT_VERSION} ${PT_BUILD}" - wget $LIBTORCH_URL + wget -q $LIBTORCH_URL fi if [[ $OS == linux ]]; then diff --git a/get_deps.sh b/get_deps.sh index 2f7bbf8a8..52e6cdcc6 100755 --- a/get_deps.sh +++ b/get_deps.sh @@ -42,7 +42,7 @@ fi OS=$(python3 $HERE/automation/readies/bin/platform --os) ARCH=$(python3 $HERE/automation/readies/bin/platform --arch) -DEPS_DIR=$HERE/deps +DEPS_DIR=$HERE/deps/$OS-$ARCH-$DEVICE mkdir -p ${DEPS_DIR} cd ${DEPS_DIR} @@ -67,7 +67,7 @@ else echo "dlpack is in place." fi -#################################################################################### TENSORFLOW +################################################################################# LIBTENSORFLOW TF_VERSION="1.14.0" @@ -118,7 +118,7 @@ else echo "TensorFlow is in place." fi -####################################################################################### PYTORCH +###################################################################################### LIBTORCH PT_VERSION="1.2.0" @@ -143,6 +143,7 @@ if [[ ! -d $LIBTORCH ]]; then fi elif [[ $OS == macosx ]]; then PT_OS=macos + PT_ARCH=x86_64 PT_BUILD=cpu fi From 6229369bc7b3b2baf59650f09555999ecfb40938 Mon Sep 17 00:00:00 2001 From: rafie Date: Mon, 26 Aug 2019 19:08:01 +0300 Subject: [PATCH 07/30] CircleCI config.yml refectoring --- .circleci/config.yml | 160 +++++++++++++++++-------------------------- automation/Makefile | 6 +- 2 files changed, 69 insertions(+), 97 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8607f1766..83715ebc2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,127 +1,96 @@ version: 2.1 -jobs: - build: - docker: - - image: circleci/python:3.7.4-buster +commands: + ci_steps: + parameters: + platform: + type: string steps: - checkout - - # Download and cache dependencies - restore_cache: keys: - v1-dependencies-{{ checksum "get_deps.sh" }} # fallback to using the latest cache if no exact match is found # - v1-dependencies- - - run: - name: install dependencies + name: Install dependencies command: | sudo ./automation/readies/bin/getpy sudo ./automation/system-setup.py - ./get_deps.sh cpu git clone git://github.com/antirez/redis.git --branch 5.0.5 - (cd redis && make malloc=libc -j4 && sudo make install) - + (cd redis && make malloc=libc -j $(nproc) && sudo make install) + ./get_deps.sh cpu - save_cache: paths: - deps key: v1-dependencies-{{ checksum "get_deps.sh" }} - + - run: + name: Pull Submodules + command: make -C automation fetch - run: name: Build - command: | - make -C automation all - make -C automation pack - + command: make -C automation all SHOW=1 - run: name: Test - command: make -C automation test - - - run: - name: Persist Artifacts command: | - mkdir -p ~/workspace/build - cp install-cpu/$MODULE_ARTIFACT ~/workspace/ - cp -r install-cpu/backends ~/workspace/ - cp ramp.yml ~/workspace/ - - persist_to_workspace: - root: ~/workspace - paths: - - '*.so*' - - '*.yml' - - build - package_branch: - docker: - - image: 'redislabsmodules/rmbuilder:latest' - steps: - - attach_workspace: - at: /workspace + mkdir -p /workspace/tests + make -C automation test TEST_REPORT_DIR=/workspace/tests SHOW=1 - run: name: Package - command: >- - ramp pack -m /workspace/ramp.yml -o - /workspace/build/$PACKAGE_NAME.{os}-{architecture}.$CIRCLE_BRANCH.zip - /workspace/$MODULE_ARTIFACT - environment: - LD_LIBRARY_PATH: /workspace + command: make -C automation pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=/workspace/packages SHOW=1 - persist_to_workspace: root: /workspace paths: - - build - - store_artifacts: - path: /workspace/build - package_release: - docker: - - image: 'redislabsmodules/rmbuilder:latest' + - 'packages/release/*.zip' + - 'packages/release/*.tgz' + - 'packages/branch/*.zip' + - 'packages/branch/*.tgz' + - store_test_results: + path: /workspace/tests + deploy: + parameters: + from: + type: string steps: - - attach_workspace: - at: /workspace - - run: - name: Package - command: >- - ramp pack -m /workspace/ramp.yml -o - /workspace/build/$PACKAGE_NAME.{os}-{architecture}.{semantic_version}.zip - /workspace/$MODULE_ARTIFACT - environment: - LD_LIBRARY_PATH: /workspace - run: - name: Package + name: Deploy to S3 command: >- - ramp pack -m /workspace/ramp.yml -o - /workspace/build/$PACKAGE_NAME.{os}-{architecture}.latest.zip - /workspace/$MODULE_ARTIFACT - environment: - LD_LIBRARY_PATH: /workspace + aws s3 cp <>/ s3://redismodules/$PACKAGE_NAME/ + --acl public-read --recursive --exclude "*" --include "*.zip" --include "*.tgz" - - persist_to_workspace: - root: /workspace - paths: - - build - - store_artifacts: - path: /workspace/build - deploy_branch: +jobs: + build: docker: - image: 'redislabsmodules/rmbuilder:latest' steps: - - attach_workspace: - at: /workspace + - ci_steps: + platform: debian + build-macos: + macos: + xcode: 10.2.1 + steps: - run: - name: Deploy to S3 - command: >- - aws s3 cp /workspace/build/ s3://redismodules/$PACKAGE_NAME/ --acl - public-read --recursive --exclude "*" --include "*.zip" - deploy_release: + name: Set up workspace + command: | + sudo mkdir -p /workspace + sudo chown $USER /workspace + - ci_steps: + platform: macosx + + deploy_package: + parameters: + package: + type: string docker: - image: 'redislabsmodules/rmbuilder:latest' steps: - attach_workspace: at: /workspace - - run: - name: Deploy to S3 - command: >- - aws s3 cp /workspace/build/ s3://redismodules/$PACKAGE_NAME/ --acl - public-read --recursive --exclude "*" --include "*.zip" + - deploy: + from: /workspace/packages/<> + - store_artifacts: + path: /workspace/packages/<> + deploy_docs: docker: - image: 'redislabsmodules/rmbuilder:latest' @@ -133,8 +102,8 @@ jobs: - run: name: Deploy Docs to S3 command: >- - aws s3 cp site s3://oss.redislabs.com/$WEBSITE_FOLDER/ --acl - public-read --recursive + aws s3 cp site s3://oss.redislabs.com/$WEBSITE_FOLDER/ --acl public-read --recursive + workflows: version: 2 build_and_package: @@ -143,13 +112,21 @@ workflows: filters: tags: only: /.*/ - - package_branch: + - build-macos: + filters: + tags: + only: /.*/ + - deploy_package: + name: deploy_branch + package: branch requires: - build filters: branches: only: master - - package_release: + - deploy_package: + name: deploy_release + package: release requires: - build filters: @@ -157,15 +134,6 @@ workflows: ignore: /.*/ tags: only: '/^v[0-9].*/' - - deploy_branch: - requires: - - package_branch - - deploy_release: - filters: - tags: - only: '/^v[0-9].*/' - requires: - - package_release - deploy_docs: filters: branches: diff --git a/automation/Makefile b/automation/Makefile index 5dbe2a146..738470c0b 100755 --- a/automation/Makefile +++ b/automation/Makefile @@ -95,9 +95,13 @@ pack_deps: pack_ramp #---------------------------------------------------------------------------------------------- +TEST_REPORT_DIR ?= $(PWD) + test: $(SHOW)if [ "$(git lfs env > /dev/null 2>&1 ; echo $?)" != "0" ]; then git lfs install; fi $(SHOW)git lfs pull $(SHOW)set -e ;\ cd $(ROOT)/test ;\ - python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so + python3 -m RLTest $(TEST_ARGS) --test basic_tests.py \ + --junitxml=$(TEST_REPORT_DIR)/tests_report.xml \ + --module $(INSTALL_DIR)/redisai.so From 12945d347523438fff6e2258889a2281642bc9ce Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 10:33:25 +0300 Subject: [PATCH 08/30] Build fixes #6 --- .circleci/config.yml | 3 --- CMakeLists.txt | 4 +++- Dockerfile | 6 +++--- automation/Makefile | 16 +++++++++++----- automation/readies/mk/variant.defs | 4 ++-- automation/system-setup.py | 3 ++- get_deps.sh | 3 +++ 7 files changed, 24 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 83715ebc2..30e47ed34 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,9 +24,6 @@ commands: paths: - deps key: v1-dependencies-{{ checksum "get_deps.sh" }} - - run: - name: Pull Submodules - command: make -C automation fetch - run: name: Build command: make -C automation all SHOW=1 diff --git a/CMakeLists.txt b/CMakeLists.txt index 209ef8797..b228b7381 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ IF (NOT DEPS_PATH) ENDIF() IF (NOT INSTALL_PATH) - SET(INSTALL_PATH ../install-${DEVICE}) + SET(INSTALL_PATH ${CMAKE_SOURCE_DIR}/install-${DEVICE}) ENDIF() GET_FILENAME_COMPONENT(depsAbs @@ -102,6 +102,8 @@ IF (APPLE) LINK_FLAGS "-undefined dynamic_lookup") ENDIF() +SET(CMAKE_INSTALL_PREFIX ${installAbs}) + INSTALL(TARGETS redisai LIBRARY DESTINATION .) IF (APPLE) diff --git a/Dockerfile b/Dockerfile index d937fbb38..ce803a986 100755 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ARG OSNICK=buster # OS=debian:buster-slim|debian:stretch-slim|ubuntu:bionic -OS=debian:buster-slim +ARG OS=debian:buster-slim # ARCH=x64|arm64v8|arm32v7 ARG ARCH=x64 @@ -25,8 +25,8 @@ RUN ./automation/system-setup.py COPY ./get_deps.sh . RUN ./get_deps.sh cpu -ADD ./ /redisai -RUN make -C automation all SHOW=1 +ADD ./ /build +RUN make -C automation build SHOW=1 ARG PACK=0 ARG TEST=0 diff --git a/automation/Makefile b/automation/Makefile index 738470c0b..82a7679c5 100755 --- a/automation/Makefile +++ b/automation/Makefile @@ -57,7 +57,7 @@ include $(MK)/rules #---------------------------------------------------------------------------------------------- prebuild: - $(SHOW)if [ ! -d $(ROOT)/deps ]; then echo $$'Dependencies are not in place.\nPlease run \'make fetch\'.'; exit 1; fi + $(SHOW)if [ ! -d $(DEPS_DIR) ]; then echo $$'Dependencies are not in place.\nPlease run \'make fetch\'.'; exit 1; fi $(TARGET): prebuild $(MK_MAKEFILES) $(DEPS) $(SHOW)mkdir -p $(INSTALL_DIR) @@ -66,17 +66,22 @@ $(TARGET): prebuild $(MK_MAKEFILES) $(DEPS) # $(SHOW)cd $(ROOT) ;\ # if [ ! -e install ]; then ln -sf install-$(DEVICE) install; fi +install: + $(SHOW)mkdir -p $(INSTALL_DIR) + $(SHOW)$(MAKE) -C $(BINDIR) install + clean: ifeq ($(ALL),1) - $(SHOW)cd $(ROOT) ;\ - rm -rf build install deps/dlpack deps/install-$(DEVICE) deps/*.tar.gz deps/*.zip deps/*.tgz + $(SHOW)if [ -d "$(BINROOT)" ]; then rm -rf $(BINROOT); fi + $(SHOW)if [ -d "$(INSTALL_DIR)" ]; then rm -rf $(INSTALL_DIR); fi + $(SHOW)rm -f $(ROOT)/install-$(DEVICE) else $(SHOW)$(MAKE) -C $(BINDIR) clean endif #---------------------------------------------------------------------------------------------- -fetch: +fetch deps: @echo Fetching dependencies... $(SHOW)VERBOSE=$(_SHOW) $(ROOT)/get_deps.sh $(DEPS_FLAGS) @@ -103,5 +108,6 @@ test: $(SHOW)set -e ;\ cd $(ROOT)/test ;\ python3 -m RLTest $(TEST_ARGS) --test basic_tests.py \ - --junitxml=$(TEST_REPORT_DIR)/tests_report.xml \ --module $(INSTALL_DIR)/redisai.so + +# --junitxml=$(TEST_REPORT_DIR)/tests_report.xml diff --git a/automation/readies/mk/variant.defs b/automation/readies/mk/variant.defs index b0cd5638f..b719342bf 100755 --- a/automation/readies/mk/variant.defs +++ b/automation/readies/mk/variant.defs @@ -8,8 +8,8 @@ ARCH=$(shell $(ROOT)/automation/readies/bin/platform --arch) #---------------------------------------------------------------------------------------------- -GIT_SHA := $(shell git rev-parse HEAD) -GIT_COMMIT := $(shell git describe --always --abbrev=7 --dirty="+") +# GIT_SHA := $(shell git rev-parse HEAD) +# GIT_COMMIT := $(shell git describe --always --abbrev=7 --dirty="+") #---------------------------------------------------------------------------------------------- diff --git a/automation/system-setup.py b/automation/system-setup.py index 8adce802a..970d41f3f 100755 --- a/automation/system-setup.py +++ b/automation/system-setup.py @@ -58,7 +58,8 @@ def common_last(self): self.pip3_install("git+https://github.com/RedisLabsModules/RLTest.git@master") if not self.has_command("ramp"): self.pip3_install("git+https://github.com/RedisLabs/RAMP@master") - self.pip3_install("-r ../test/test_requirements.txt") + root = os.path.join(os.path.dirname(__file__), "..") + self.pip3_install("-r {}/test/test_requirements.txt".format(root)) #---------------------------------------------------------------------------------------------- diff --git a/get_deps.sh b/get_deps.sh index 52e6cdcc6..9b03d5e7a 100755 --- a/get_deps.sh +++ b/get_deps.sh @@ -42,6 +42,9 @@ fi OS=$(python3 $HERE/automation/readies/bin/platform --os) ARCH=$(python3 $HERE/automation/readies/bin/platform --arch) +# avoid wget warnings on macOS +[[ $OS == macosx ]] && export LC_ALL=en_US.UTF-8 + DEPS_DIR=$HERE/deps/$OS-$ARCH-$DEVICE mkdir -p ${DEPS_DIR} cd ${DEPS_DIR} From df042d09e34f7fbac61210089c664049e89ac74f Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 11:03:07 +0300 Subject: [PATCH 09/30] Build fixes #7 --- CMakeLists.txt | 6 +++++- README.md | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b228b7381..24d7d1d83 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,11 @@ IF (NOT DEVICE) ENDIF() IF (NOT DEPS_PATH) - SET(DEPS_PATH ../deps) + IF (APPLE) + SET(DEPS_PATH deps/macosx-x64-${DEVICE}) + ELSE() + SET(DEPS_PATH deps/linux-x64-${DEVICE}) + ENDIF() ENDIF() IF (NOT INSTALL_PATH) diff --git a/README.md b/README.md index d2964814f..0470c94b5 100644 --- a/README.md +++ b/README.md @@ -64,8 +64,8 @@ Once the dependencies are downloaded, build the module itself. Note that CMake 3.0 or higher is required. ```sh -mkdir build -cd build +mkdir bin +cd bin cmake .. make && make install cd .. From 068b9dcec45ff693a23edfe9e3449abd1f53359a Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 11:07:47 +0300 Subject: [PATCH 10/30] Build fixes #8 --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 30e47ed34..122bda836 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,8 +15,8 @@ commands: - run: name: Install dependencies command: | - sudo ./automation/readies/bin/getpy - sudo ./automation/system-setup.py + ./automation/readies/bin/getpy + ./automation/system-setup.py git clone git://github.com/antirez/redis.git --branch 5.0.5 (cd redis && make malloc=libc -j $(nproc) && sudo make install) ./get_deps.sh cpu From c4bd986fe39ee9b746e546647c6a63875ecea302 Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 11:19:44 +0300 Subject: [PATCH 11/30] Build fixes #9 --- automation/system-setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/automation/system-setup.py b/automation/system-setup.py index 970d41f3f..ed746515a 100755 --- a/automation/system-setup.py +++ b/automation/system-setup.py @@ -15,13 +15,14 @@ def __init__(self, nop=False): paella.Setup.__init__(self, nop) def common_first(self): + self.install("curl") self.setup_pip() self.pip3_install("wheel") self.pip3_install("setuptools --upgrade") if self.os == 'linux': self.install("ca-certificates") - self.install("git cmake curl unzip wget patchelf awscli") + self.install("git cmake unzip wget patchelf awscli") def debian_compat(self): self.install("build-essential") From 191981e5838b98be2a3ab6369cb82200f4705001 Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 11:32:27 +0300 Subject: [PATCH 12/30] Build fixes #10 --- automation/readies/paella/setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/automation/readies/paella/setup.py b/automation/readies/paella/setup.py index 643042c11..ae4b7a0d7 100755 --- a/automation/readies/paella/setup.py +++ b/automation/readies/paella/setup.py @@ -152,9 +152,11 @@ def pip3_install(self, cmd): def setup_pip(self): get_pip = "set -e; cd /tmp; curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py" - if not self.has_command("pip"): + if not self.has_command("pip3"): self.install("python3-distutils") - self.install("curl ca-certificates") + if self.os == 'linux': + self.install("ca-certificates") + self.install("curl") self.run(get_pip + "; " + self.python + " get-pip.py", output_on_error=True) ## fails on ubuntu 18: # if not has_command("pip3") and has_command("python3"): From b14ec42a40ff5b507793c90d1cd02faa1e5e8a5d Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 11:43:49 +0300 Subject: [PATCH 13/30] Build fixes #11 --- automation/readies/paella/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/readies/paella/setup.py b/automation/readies/paella/setup.py index ae4b7a0d7..4948d2c1b 100755 --- a/automation/readies/paella/setup.py +++ b/automation/readies/paella/setup.py @@ -59,12 +59,12 @@ def __init__(self, nop=False): self.dist = self.platform.dist self.ver = self.platform.os_ver + if self.has_command("python3"): + self.python = "python3" if self.has_command("python"): self.python = "python" elif self.has_command("python2"): self.python = "python2" - elif self.has_command("python3"): - self.python = "python3" if self.os == 'macosx': # this is required because osx pip installed are done with --user From 5f7b53b774d5957f0e47a5103bffc92a1ffa77ba Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 15:12:34 +0300 Subject: [PATCH 14/30] Build fixes #12 --- automation/readies/paella/setup.py | 77 +++++++++++++++--------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/automation/readies/paella/setup.py b/automation/readies/paella/setup.py index 4948d2c1b..8cfbe823c 100755 --- a/automation/readies/paella/setup.py +++ b/automation/readies/paella/setup.py @@ -10,7 +10,7 @@ class Runner: def __init__(self, nop=False): self.nop = nop - def run(self, cmd, output_on_error=False): + def run(self, cmd, output_on_error=False, _try=False): print(cmd) sys.stdout.flush() if self.nop: @@ -26,7 +26,8 @@ def run(self, cmd, output_on_error=False): os.remove(temppath) eprint("command failed: " + cmd) sys.stderr.flush() - sys.exit(1) + if not _try: + sys.exit(1) def has_command(self, cmd): return os.system("command -v " + cmd + " > /dev/null") == 0 @@ -80,56 +81,56 @@ def setup(self): RepoRefresh(self.runner).invoke() self.invoke() - def run(self, cmd, output_on_error=False): - return self.runner.run(cmd, output_on_error=output_on_error) + def run(self, cmd, output_on_error=False, _try=False): + return self.runner.run(cmd, output_on_error=output_on_error, _try=_try) def has_command(self, cmd): return self.runner.has_command(cmd) #------------------------------------------------------------------------------------------ - def apt_install(self, packs, group=False): - self.run("apt-get -qq install -y " + packs, output_on_error=True) + def apt_install(self, packs, group=False, _try=False): + self.run("apt-get -qq install -y " + packs, output_on_error=True, _try=_try) - def yum_install(self, packs, group=False): + def yum_install(self, packs, group=False, _try=False): if not group: - self.run("yum install -q -y " + packs, output_on_error=True) + self.run("yum install -q -y " + packs, output_on_error=True, _try=_try) else: - self.run("yum groupinstall -y " + packs, output_on_error=True) + self.run("yum groupinstall -y " + packs, output_on_error=True, _try=_try) - def dnf_install(self, packs, group=False): + def dnf_install(self, packs, group=False, _try=False): if not group: - self.run("dnf install -y " + packs, output_on_error=True) + self.run("dnf install -y " + packs, output_on_error=True, _try=_try) else: - self.run("dnf groupinstall -y " + packs, output_on_error=True) + self.run("dnf groupinstall -y " + packs, output_on_error=True, _try=_try) - def zypper_install(self, packs, group=False): - self.run("zipper --non-interactive install " + packs, output_on_error=True) + def zypper_install(self, packs, group=False, _try=False): + self.run("zipper --non-interactive install " + packs, output_on_error=True, _try=_try) - def pacman_install(self, packs, group=False): - self.run("pacman --noconfirm -S " + packs, output_on_error=True) + def pacman_install(self, packs, group=False, _try=False): + self.run("pacman --noconfirm -S " + packs, output_on_error=True, _try=_try) - def brew_install(self, packs, group=False): + def brew_install(self, packs, group=False, _try=False): # brew will fail if package is already installed for pack in packs.split(): - self.run("brew list {} &>/dev/null || brew install {}".format(pack, pack), output_on_error=True) + self.run("brew list {} &>/dev/null || brew install {}".format(pack, pack), output_on_error=True, _try=_try) - def install(self, packs, group=False): + def install(self, packs, group=False, _try=False): if self.os == 'linux': if self.dist == 'fedora': - self.dnf_install(packs, group) + self.dnf_install(packs, group=group, _try=_try) elif self.dist == 'ubuntu' or self.dist == 'debian': - self.apt_install(packs, group) + self.apt_install(packs, group=group, _try=_try) elif self.dist == 'centos' or self.dist == 'redhat': - self.yum_install(packs, group) + self.yum_install(packs, group=group, _try=_try) elif self.dist == 'suse': - self.zypper_install(packs, group) + self.zypper_install(packs, group=group, _try=_try) elif self.dist == 'arch': - self.pacman_install(packs, group) + self.pacman_install(packs, group=group, _try=_try) else: Assert(False), "Cannot determine installer" elif self.os == 'macosx': - self.brew_install(packs, group) + self.brew_install(packs, group=group, _try=_try) else: Assert(False), "Cannot determine installer" @@ -138,26 +139,26 @@ def group_install(self, packs): #------------------------------------------------------------------------------------------ - def pip_install(self, cmd): + def pip_install(self, cmd, _try=False): pip_user = '' if self.os == 'macosx': pip_user = '--user ' - self.run("pip install --disable-pip-version-check " + pip_user + cmd, output_on_error=True) + self.run("pip install --disable-pip-version-check " + pip_user + cmd, output_on_error=True, _try=_try) - def pip3_install(self, cmd): + def pip3_install(self, cmd, _try=False): pip_user = '' if self.os == 'macosx': pip_user = '--user ' - self.run("pip3 install --disable-pip-version-check " + pip_user + cmd, output_on_error=True) + self.run("pip3 install --disable-pip-version-check " + pip_user + cmd, output_on_error=True, _try=_try) def setup_pip(self): - get_pip = "set -e; cd /tmp; curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py" + get_pip = "set -e; curl -s https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py" if not self.has_command("pip3"): - self.install("python3-distutils") - if self.os == 'linux': - self.install("ca-certificates") - self.install("curl") - self.run(get_pip + "; " + self.python + " get-pip.py", output_on_error=True) - ## fails on ubuntu 18: - # if not has_command("pip3") and has_command("python3"): - # run(get_pip + "; python3 get-pip.py") + self.install("python3-distutils", _try=True) + self.install_downloaders() + self.run(get_pip + "; " + self.python + " /tmp/get-pip.py", output_on_error=True) + + def install_downloaders(self): + if self.os == 'linux': + self.install("ca-certificates") + self.install("curl wget") From 369a259f3dd60ca5a31db25da0116fdcdcb5a9ed Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 15:29:43 +0300 Subject: [PATCH 15/30] Build fixes #13 --- automation/readies/paella/setup.py | 2 +- automation/system-setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/readies/paella/setup.py b/automation/readies/paella/setup.py index 8cfbe823c..2dc924f3c 100755 --- a/automation/readies/paella/setup.py +++ b/automation/readies/paella/setup.py @@ -62,7 +62,7 @@ def __init__(self, nop=False): if self.has_command("python3"): self.python = "python3" - if self.has_command("python"): + elif self.has_command("python"): self.python = "python" elif self.has_command("python2"): self.python = "python2" diff --git a/automation/system-setup.py b/automation/system-setup.py index ed746515a..53c31d20f 100755 --- a/automation/system-setup.py +++ b/automation/system-setup.py @@ -15,7 +15,7 @@ def __init__(self, nop=False): paella.Setup.__init__(self, nop) def common_first(self): - self.install("curl") + self.setup_downloaders() self.setup_pip() self.pip3_install("wheel") self.pip3_install("setuptools --upgrade") From 11653bf3fda887b574ab6ef7f44d327abcfad9e0 Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 15:31:13 +0300 Subject: [PATCH 16/30] Build fixes #14 --- automation/system-setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/system-setup.py b/automation/system-setup.py index 53c31d20f..7d676f83f 100755 --- a/automation/system-setup.py +++ b/automation/system-setup.py @@ -15,7 +15,7 @@ def __init__(self, nop=False): paella.Setup.__init__(self, nop) def common_first(self): - self.setup_downloaders() + self.install_downloaders() self.setup_pip() self.pip3_install("wheel") self.pip3_install("setuptools --upgrade") From a93efe6f41504bde63be992760bb595dff0cd415 Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 15:34:23 +0300 Subject: [PATCH 17/30] Build fixes #15 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 122bda836..901e3c582 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ commands: ./automation/readies/bin/getpy ./automation/system-setup.py git clone git://github.com/antirez/redis.git --branch 5.0.5 - (cd redis && make malloc=libc -j $(nproc) && sudo make install) + (cd redis && make malloc=libc -j $(nproc) && make install) ./get_deps.sh cpu - save_cache: paths: From 192d4a32c9494cadb70e0b4d5125ab9d5151d633 Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 16:51:09 +0300 Subject: [PATCH 18/30] Build fixes #16 --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 901e3c582..c6b9cff65 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,6 +5,8 @@ commands: parameters: platform: type: string + environment: + DEPS: "" steps: - checkout - restore_cache: From e2483a37634bb808d11822dba94cab553a37214e Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 16:54:34 +0300 Subject: [PATCH 19/30] Build fixes #17 --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c6b9cff65..67f0b8f54 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,8 +5,6 @@ commands: parameters: platform: type: string - environment: - DEPS: "" steps: - checkout - restore_cache: @@ -61,6 +59,8 @@ jobs: build: docker: - image: 'redislabsmodules/rmbuilder:latest' + environment: + DEPS: "" steps: - ci_steps: platform: debian From 05284e73622017babfc9d83a9e62c28ec75e9f1b Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 17:19:45 +0300 Subject: [PATCH 20/30] Build fixes #18 --- .circleci/config.yml | 11 ++++++----- automation/Makefile | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 67f0b8f54..4b540ac65 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,10 +15,10 @@ commands: - run: name: Install dependencies command: | - ./automation/readies/bin/getpy - ./automation/system-setup.py + sudo ./automation/readies/bin/getpy + sudo ./automation/system-setup.py git clone git://github.com/antirez/redis.git --branch 5.0.5 - (cd redis && make malloc=libc -j $(nproc) && make install) + (cd redis && make malloc=libc -j $(nproc) && sudo make install) ./get_deps.sh cpu - save_cache: paths: @@ -31,7 +31,8 @@ commands: name: Test command: | mkdir -p /workspace/tests - make -C automation test TEST_REPORT_DIR=/workspace/tests SHOW=1 + make -C automation test SHOW=1 + cp test/logs/* /workspace/tests - run: name: Package command: make -C automation pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=/workspace/packages SHOW=1 @@ -58,7 +59,7 @@ commands: jobs: build: docker: - - image: 'redislabsmodules/rmbuilder:latest' + - image: circleci/python:3.7.4-buster environment: DEPS: "" steps: diff --git a/automation/Makefile b/automation/Makefile index 82a7679c5..fc010d800 100755 --- a/automation/Makefile +++ b/automation/Makefile @@ -103,8 +103,8 @@ pack_deps: pack_ramp TEST_REPORT_DIR ?= $(PWD) test: - $(SHOW)if [ "$(git lfs env > /dev/null 2>&1 ; echo $?)" != "0" ]; then git lfs install; fi - $(SHOW)git lfs pull + $(SHOW)if [ "$(git lfs env > /dev/null 2>&1 ; echo $?)" != "0" ]; then cd $(ROOT); git lfs install; fi + $(SHOW)cd $(ROOT); git lfs pull $(SHOW)set -e ;\ cd $(ROOT)/test ;\ python3 -m RLTest $(TEST_ARGS) --test basic_tests.py \ From d80e2591fcfd5557eed299fe1e1fec4587c9ed2a Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 17:21:08 +0300 Subject: [PATCH 21/30] Build fixes #19 --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4b540ac65..ceac6f6e2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,11 +7,11 @@ commands: type: string steps: - checkout - - restore_cache: - keys: - - v1-dependencies-{{ checksum "get_deps.sh" }} - # fallback to using the latest cache if no exact match is found - # - v1-dependencies- +# - restore_cache: +# keys: +# - v1-dependencies-{{ checksum "get_deps.sh" }} +# # fallback to using the latest cache if no exact match is found +# # - v1-dependencies- - run: name: Install dependencies command: | From baa1837d45a66fe200fea83b82b4b3b948839b0e Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 17:26:15 +0300 Subject: [PATCH 22/30] Build fixes #20 --- .circleci/config.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ceac6f6e2..2ab323810 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,11 +7,11 @@ commands: type: string steps: - checkout -# - restore_cache: -# keys: -# - v1-dependencies-{{ checksum "get_deps.sh" }} -# # fallback to using the latest cache if no exact match is found -# # - v1-dependencies- + - restore_cache: + keys: + - v1-dependencies-{{ checksum "get_deps.sh" }} + # fallback to using the latest cache if no exact match is found + # - v1-dependencies- - run: name: Install dependencies command: | @@ -30,21 +30,21 @@ commands: - run: name: Test command: | - mkdir -p /workspace/tests + mkdir -p ~/workspace/tests make -C automation test SHOW=1 - cp test/logs/* /workspace/tests + cp test/logs/* ~/workspace/tests - run: name: Package - command: make -C automation pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=/workspace/packages SHOW=1 + command: make -C automation pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=~/workspace/packages SHOW=1 - persist_to_workspace: - root: /workspace + root: ~/workspace paths: - 'packages/release/*.zip' - 'packages/release/*.tgz' - 'packages/branch/*.zip' - 'packages/branch/*.tgz' - store_test_results: - path: /workspace/tests + path: ~/workspace/tests deploy: parameters: from: @@ -69,11 +69,11 @@ jobs: macos: xcode: 10.2.1 steps: - - run: - name: Set up workspace - command: | - sudo mkdir -p /workspace - sudo chown $USER /workspace +# - run: +# name: Set up workspace +# command: | +# sudo mkdir -p /workspace +# sudo chown $USER /workspace - ci_steps: platform: macosx @@ -85,11 +85,11 @@ jobs: - image: 'redislabsmodules/rmbuilder:latest' steps: - attach_workspace: - at: /workspace + at: ~/workspace - deploy: - from: /workspace/packages/<> + from: ~/workspace/packages/<> - store_artifacts: - path: /workspace/packages/<> + path: ~/workspace/packages/<> deploy_docs: docker: From d9c2ab48a6584fa16c2c8e528529552dfd87e188 Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 17:27:25 +0300 Subject: [PATCH 23/30] Build fixes #21 --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2ab323810..4589910ae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,11 +7,11 @@ commands: type: string steps: - checkout - - restore_cache: - keys: - - v1-dependencies-{{ checksum "get_deps.sh" }} - # fallback to using the latest cache if no exact match is found - # - v1-dependencies- +# - restore_cache: +# keys: +# - v1-dependencies-{{ checksum "get_deps.sh" }} +# # fallback to using the latest cache if no exact match is found +# # - v1-dependencies- - run: name: Install dependencies command: | From 66543cae4f1bceba4f11e5c66f2ca080f81159da Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 17:34:27 +0300 Subject: [PATCH 24/30] Build fixes #22 --- automation/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/Makefile b/automation/Makefile index fc010d800..0324dbc87 100755 --- a/automation/Makefile +++ b/automation/Makefile @@ -43,7 +43,7 @@ BACKENDS_PATH ?= $(INSTALL_DIR)/backends CMAKE_FLAGS += \ -DDEPS_PATH=$(realpath $(DEPS_DIR)) \ - -DINSTALL_PATH=$(realpath $(INSTALL_DIR)) \ + -DINSTALL_PATH=$(INSTALL_DIR) \ -DDEVICE=$(DEVICE) include $(MK)/defs From a2db3d25c9e6a3b7b6b3f1f3415e54f422b682dc Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 17:57:27 +0300 Subject: [PATCH 25/30] Build fixes #23 --- automation/Makefile | 4 ++-- automation/readies/mk/cmake.rules | 2 +- automation/readies/mk/configure.rules | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/automation/Makefile b/automation/Makefile index 0324dbc87..ea4d0cb64 100755 --- a/automation/Makefile +++ b/automation/Makefile @@ -42,8 +42,8 @@ TARGET=$(BINDIR)/redisai.so BACKENDS_PATH ?= $(INSTALL_DIR)/backends CMAKE_FLAGS += \ - -DDEPS_PATH=$(realpath $(DEPS_DIR)) \ - -DINSTALL_PATH=$(INSTALL_DIR) \ + -DDEPS_PATH=$(abspath $(DEPS_DIR)) \ + -DINSTALL_PATH=$(abspath $(INSTALL_DIR)) \ -DDEVICE=$(DEVICE) include $(MK)/defs diff --git a/automation/readies/mk/cmake.rules b/automation/readies/mk/cmake.rules index a9e4d6902..51ff497e6 100755 --- a/automation/readies/mk/cmake.rules +++ b/automation/readies/mk/cmake.rules @@ -3,7 +3,7 @@ ifneq ($(MK_CMAKE),) $(BINDIR)/Makefile : bindirs $(SRCDIR)/CMakeLists.txt $(SHOW)if [ ! -d $(BINDIR) ]; then echo "CMake: $(BINDIR) does not exist."; exit 1; fi - $(SHOW)cd $(BINDIR); cmake $(CMAKE_WHY) $(CMAKE_FLAGS) $(realpath $(SRCDIR)) + $(SHOW)cd $(BINDIR); cmake $(CMAKE_WHY) $(CMAKE_FLAGS) $(abspath $(SRCDIR)) # $(SHOW)cd $(BINDIR); \ # CMAKE_REL_ROOT=`python -c "import os; print os.path.relpath('$(SRCDIR)', '$$PWD')"` \ diff --git a/automation/readies/mk/configure.rules b/automation/readies/mk/configure.rules index dd5278ea2..1ef0cbc51 100755 --- a/automation/readies/mk/configure.rules +++ b/automation/readies/mk/configure.rules @@ -3,7 +3,7 @@ build: $(MK_MAKEFILES) $(TARGET) ifeq (,$(wildcard $(BUILD_DIR)/Makefile)) - $(SHOW)cd $(BUILD_DIR); $(realpath $(SRCDIR))/configure $(CONFIGURE_FLAGS) + $(SHOW)cd $(BUILD_DIR); $(abspath $(SRCDIR))/configure $(CONFIGURE_FLAGS) endif @make -C $(BUILD_DIR) From e339bfd71a682064b260d0a14e4a5f4f6a792eaf Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 20:45:55 +0300 Subject: [PATCH 26/30] Build fixes #24 --- automation/Makefile | 23 ++++---- automation/pack.sh | 115 +++++++++++++++++++------------------ automation/system-setup.py | 1 + 3 files changed, 70 insertions(+), 69 deletions(-) diff --git a/automation/Makefile b/automation/Makefile index ea4d0cb64..a173cbdb3 100755 --- a/automation/Makefile +++ b/automation/Makefile @@ -10,8 +10,10 @@ make setup # install prerequisited (CAUTION: THIS WILL MODIFY YOUR SYSTEM) make fetch # download and prepare dependant modules make build # compile and link make clean # remove build artifacts + ALL=1 # remove entire artifacts directory make test # run tests make pack # create installation packages + PACK_DEPS=0 # do not pack dependencies make deploy # copy packages to S3 make release # release a version endef @@ -87,16 +89,14 @@ fetch deps: #---------------------------------------------------------------------------------------------- -pack: $(BINDIR)/BINDIR pack_ramp pack_deps - -$(BINDIR)/BINDIR: $(BIN_DIRS) - $(SHOW)echo $(BINDIR)>$(BINDIR)/BINDIR - -pack_ramp: - $(SHOW)RAMP=1 DEVICE=$(DEVICE) BINDIR=$(BINROOT) ./pack.sh +pack: $(BINDIR)/BINDIR + $(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) DEPS=0 ./pack.sh +ifneq ($(PACK_DEPS),0) + $(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) DEPS=1 ./pack.sh +endif -pack_deps: pack_ramp - $(SHOW)DEPS=1 DEVICE=$(DEVICE) BINDIR=$(BINROOT) ./pack.sh +#$(BINDIR)/BINDIR: $(BIN_DIRS) +# $(SHOW)echo $(BINDIR)>$(BINDIR)/BINDIR #---------------------------------------------------------------------------------------------- @@ -107,7 +107,4 @@ test: $(SHOW)cd $(ROOT); git lfs pull $(SHOW)set -e ;\ cd $(ROOT)/test ;\ - python3 -m RLTest $(TEST_ARGS) --test basic_tests.py \ - --module $(INSTALL_DIR)/redisai.so - -# --junitxml=$(TEST_REPORT_DIR)/tests_report.xml + python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so diff --git a/automation/pack.sh b/automation/pack.sh index 91b0e01fe..9cdee1263 100755 --- a/automation/pack.sh +++ b/automation/pack.sh @@ -2,6 +2,8 @@ [[ -z $DEVICE ]] && { echo DEVICE undefined; exit 1; } [[ -z $BINDIR ]] && { echo BINDIR undefined; exit 1; } +[[ -z $INSTALL_DIR ]] && { echo INSTALL_DIR undefined; exit 1; } +[[ ! -z $INTO ]] && INTO=$(realpath $INTO) HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . $HERE/../automation/readies/shibumi/functions @@ -11,19 +13,33 @@ RAMP_PROG=ramp REDIS_ENT_LIB_PATH=/opt/redislabs/lib BINDIR=$(realpath $BINDIR) +INSTALL_DIR=$(realpath $INSTALL_DIR) pack_ramp() { echo "Building RAMP file ..." cd $ROOT RAMPOUT=$(mktemp /tmp/ramp.XXXXXX) - echo BINDIR=$BINDIR - $RAMP_PROG pack -m $ROOT/ramp.yml -o $BINDIR/$PRODUCT.{os}-{architecture}.{semantic_version}.zip $INSTALL_DIR/redisai.so 2> /dev/null | grep '.zip' > $RAMPOUT + $RAMP_PROG pack -m $ROOT/ramp.yml -o $BINDIR/$PRODUCT.{os}-{architecture}.{semantic_version}.zip $INSTALL_DIR/$PRODUCT.so 2> /dev/null | grep '.zip' > $RAMPOUT realpath $(tail -1 $RAMPOUT) > $BINDIR/PACKAGE cat $BINDIR/PACKAGE | sed -e "s/[^.]*\.[^.]*\.\(.*\)\.zip/\1/" > $BINDIR/VERSION + cat $BINDIR/PACKAGE | sed -e "s/[^.]*\.\([^.]*\)\..*\.zip/\1/" > $BINDIR/OSARCH + PACKAGE=$(cat $BINDIR/PACKAGE) VERSION=$(cat $BINDIR/VERSION) - $RAMP_PROG pack -m $ROOT/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" \ - -c "BACKENDSPATH $REDIS_ENT_LIB_PATH/redisai-cpu-$VERSION/backends" $INSTALL_DIR/redisai.so 2> /dev/null | grep '.zip' > $RAMPOUT + OSARCH=$(cat $BINDIR/OSARCH) + $RAMP_PROG pack -m $ROOT/ramp.yml -o "build/$PRODUCT.{os}-{architecture}.{semantic_version}.zip" \ + -c "BACKENDSPATH $REDIS_ENT_LIB_PATH/$PRODUCT-$DEVICE-$VERSION/backends" $INSTALL_DIR/$PRODUCT.so 2> /dev/null | grep '.zip' > $RAMPOUT rm -f $RAMPOUT + export PACK_FNAME="$(basename $PACKAGE)" + + cd "$BINDIR" + RAMP_STEM=$PRODUCT.$OSARCH + ln -sf $PACK_FNAME $RAMP_STEM.latest.zip + [[ ! -z $BRANCH ]] && ln -sf $PACK_FNAME $RAMP_STEM.${BRANCH}.zip + ln -sf $PACK_FNAME $RAMP_STEM.${GIT_VER}.zip + + export RELEASE_ARTIFACTS="$RELEASE_ARTIFACTS $PACK_FNAME $RAMP_STEM.latest.zip" + export DEV_ARTIFACTS="$DEV_ARTIFACTS $RAMP_STEM.${BRANCH}.zip $RAMP_STEM.${GIT_VER}.zip" + echo "Done." } @@ -32,18 +48,29 @@ pack_deps() { cd $ROOT PACK_FNAME=$(basename `cat $BINDIR/PACKAGE`) - ARCHOSVER=$(echo "$PACK_FNAME" | sed -e "s/^redisai\.\([^.]*\..*\)\.zip/\1/") + ARCHOSVER=$(echo "$PACK_FNAME" | sed -e "s/^[^.]*\.\([^.]*\..*\)\.zip/\1/") VERSION=$(cat $BINDIR/VERSION) - cd install-$DEVICE + cd $INSTALL_DIR if [[ ! -h backends ]]; then [[ ! -d backends ]] && { echo "install-$DEVICE/backend directory not found." ; exit 1; } - rm -rf redisai-$DEVICE-$VERSION - mkdir redisai-$DEVICE-$VERSION + rm -rf $PRODUCT-$DEVICE-$VERSION + mkdir $PRODUCT-$DEVICE-$VERSION - mv backends redisai-$DEVICE-$VERSION - ln -s redisai-$DEVICE-$VERSION/backends backends + mv backends $PRODUCT-$DEVICE-$VERSION + ln -s $PRODUCT-$DEVICE-$VERSION/backends backends fi - find redisai-$DEVICE-$VERSION -name "*.so*" | xargs tar pczf redisai-$DEVICE-dependencies.$ARCHOSVER.tgz + find $PRODUCT-$DEVICE-$VERSION -name "*.so*" | xargs tar pczf $BINDIR/$PRODUCT-$DEVICE-dependencies.$ARCHOSVER.tgz + export DEPS_FNAME="$PRODUCT-$DEVICE-dependencies.$ARCHOSVER.tgz" + + cd "$BINDIR" + DEPS_STEM=$PRODUCT-$DEVICE-dependencies.$OSARCH + ln -sf $DEPS_FNAME $DEPS_STEM.latest.tgz + [[ ! -z $BRANCH ]] && ln -sf $DEPS_FNAME $DEPS_STEM.${BRANCH}.tgz + ln -sf $DEPS_FNAME $DEPS_STEM.${GIT_VER}.tgz + + export RELEASE_ARTIFACTS="$RELEASE_ARTIFACTS $DEPS_FNAME $DEPS_STEM.latest.zip" + export DEV_ARTIFACTS="$DEV_ARTIFACTS $DEPS_STEM.${BRANCH}.zip $DEPS_STEM.${GIT_VER}.zip" + echo "Done." } @@ -55,21 +82,19 @@ if [[ $1 == --help || $1 == help ]]; then pack.sh [cpu|gpu] [--help|help] Argument variables: + BINDIR=dir directory in which packages are created + INSTALL_DIR=dir directory in which artifacts are found DEVICE=cpu|gpu - BRANCH=branch branch names to serve as an exta package tag - INTO=dir package destination directory (optinal) - RAMP=1 build RAMP file - DEPS=1 build dependencies file - RAMP_PROG path to RAMP program + BRANCH=branch branch names to serve as an exta package tag + INTO=dir package destination directory (optinal) + RAMP=1 build RAMP file + DEPS=1 build dependencies file + RAMP_PROG path to RAMP program END exit 0 fi -# BINDIR=$(cat $ROOT/BINDIR) -BIN=$ROOT/bin -INSTALL_DIR=$ROOT/install - PRODUCT=redisai PRODUCT_LIB=$PRODUCT.so @@ -80,11 +105,12 @@ if [[ -d $ROOT/.git ]]; then GIT_VER="${GIT_BRANCH}-${GIT_COMMIT}" fi -# OSX="" -# if [[ $(./deps/readies/bin/platform --os) == macosx ]]; then -# OSX=1 -# export PATH=$PATH:$HOME/Library/Python/2.7/bin -# fi +OSX="" +if [[ $(./readies/bin/platform --os) == macosx ]]; then + # macOS: ramp is installed here + OSX=1 + export PATH=$PATH:$HOME/Library/Python/2.7/bin +fi if ! command -v redis-server > /dev/null; then echo "Cannot find redis-server. Aborting." @@ -92,42 +118,19 @@ if ! command -v redis-server > /dev/null; then fi pack_ramp -pack_deps -echo "Done." -exit 0 - - -cd "$BINDIR" -ln -s $PACK_FNAME $PRODUCT.latest.zip -ln -s $DEPS_FNAME $PRODUCT-dependencies.latest.tgz - -if [[ ! -z $BRANCH ]]; then - ln -s $PACK_FNAME $PRODUCT.${BRANCH}.zip - ln -s $DEPS_FNAME $PRODUCT-dependencies.${BRANCH}.tgz -fi -ln -s $PACK_FNAME $PRODUCT.${GIT_VER}.zip -ln -s $DEPS_FNAME $PRODUCT-dependencies.${GIT_VER}.tgz - -RELEASE_ARTIFACTS=\ - $PACK_FNAME $DEPS_FNAME \ - $PRODUCT.latest.zip $PRODUCT-dependencies.latest.tgz - -DEV_ARTIFACTS=\ - $PRODUCT.${BRANCH}.zip $PRODUCT-dependencies.${BRANCH}.tgz \ - $PRODUCT.${GIT_VER}.zip $PRODUCT-dependencies.${GIT_VER}.tgz +[[ $DEPS == 1 ]] && pack_deps if [[ ! -z $INTO ]]; then - INTO=$(realpath $INTO) mkdir -p $INTO/release $INTO/branch cd $INTO/release - foreach f in ($RELEASE_ARTIFACTS) - ln -s $f - end + for f in $RELEASE_ARTIFACTS; do + [[ -f $BINDIR/$f ]] && ln -sf $(realpath --relative-to . $BINDIR/$f) + done cd $INTO/branch - foreach f in ($DEV_ARTIFACTS) - ln -s $f - end + for f in $DEV_ARTIFACTS; do + [[ -f $BINDIR/$f ]] && ln -sf $(realpath --relative-to . $BINDIR/$f) + done fi -echo "Done." +exit 0 diff --git a/automation/system-setup.py b/automation/system-setup.py index 7d676f83f..b74f1c0c9 100755 --- a/automation/system-setup.py +++ b/automation/system-setup.py @@ -23,6 +23,7 @@ def common_first(self): if self.os == 'linux': self.install("ca-certificates") self.install("git cmake unzip wget patchelf awscli") + self.install("coreutils") # for realpath def debian_compat(self): self.install("build-essential") From 92e4440796470c9a9cff9dc3d439a50cf2fbf6fa Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 20:51:14 +0300 Subject: [PATCH 27/30] Build fixes #25 --- automation/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/Makefile b/automation/Makefile index a173cbdb3..d1e26d72a 100755 --- a/automation/Makefile +++ b/automation/Makefile @@ -89,7 +89,7 @@ fetch deps: #---------------------------------------------------------------------------------------------- -pack: $(BINDIR)/BINDIR +pack: $(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) DEPS=0 ./pack.sh ifneq ($(PACK_DEPS),0) $(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) DEPS=1 ./pack.sh From 90f5503f1eaf01aa672df0fb8a795b0c41443353 Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 27 Aug 2019 20:58:12 +0300 Subject: [PATCH 28/30] Build fixes #26 --- .circleci/config.yml | 23 +++++++++-------------- automation/Makefile | 3 ++- get_deps.sh | 16 ---------------- 3 files changed, 11 insertions(+), 31 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4589910ae..b470d3bfa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,11 +7,11 @@ commands: type: string steps: - checkout -# - restore_cache: -# keys: -# - v1-dependencies-{{ checksum "get_deps.sh" }} -# # fallback to using the latest cache if no exact match is found -# # - v1-dependencies- + - restore_cache: + keys: + - v1-dependencies-{{ checksum "get_deps.sh" }} + # fallback to using the latest cache if no exact match is found + # - v1-dependencies- - run: name: Install dependencies command: | @@ -69,11 +69,6 @@ jobs: macos: xcode: 10.2.1 steps: -# - run: -# name: Set up workspace -# command: | -# sudo mkdir -p /workspace -# sudo chown $USER /workspace - ci_steps: platform: macosx @@ -112,10 +107,10 @@ workflows: filters: tags: only: /.*/ - - build-macos: - filters: - tags: - only: /.*/ +# - build-macos: +# filters: +# tags: +# only: /.*/ - deploy_package: name: deploy_branch package: branch diff --git a/automation/Makefile b/automation/Makefile index d1e26d72a..b2d89ee27 100755 --- a/automation/Makefile +++ b/automation/Makefile @@ -90,8 +90,9 @@ fetch deps: #---------------------------------------------------------------------------------------------- pack: - $(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) DEPS=0 ./pack.sh ifneq ($(PACK_DEPS),0) + $(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) DEPS=0 ./pack.sh +else $(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) DEPS=1 ./pack.sh endif diff --git a/get_deps.sh b/get_deps.sh index 9b03d5e7a..65c238fff 100755 --- a/get_deps.sh +++ b/get_deps.sh @@ -238,19 +238,3 @@ if [[ ! -d $ONNXRUNTIME ]]; then else echo "ONNXRuntime is in place." fi - -############################################################################# Collect libraries - -#if [[ ! -d install ]]; then -# echo "Collecting binaries..." -# -# rm -rf install.x -# python3 $HERE/automation/collect-deps.py --into install.x -# mv install.x install -# -# echo "Done." -#else -# echo "Binaries in place." -#fi - -# echo "Done." \ No newline at end of file From 4fab27fc40cbbf7b7e469b9590a8bf8725dae492 Mon Sep 17 00:00:00 2001 From: rafie Date: Sun, 1 Sep 2019 17:43:27 +0300 Subject: [PATCH 29/30] Filesystem restructuring --- .circleci/config.yml | 10 +++++----- .dockerignore | 1 + .gitignore | 1 + CMakeLists.txt | 2 +- Dockerfile | 12 ++++++------ Dockerfile.arm | 12 ++++++------ get_deps.sh | 4 ++-- {automation => opt}/Makefile | 2 +- {build => opt/build}/bazel/Dockerfile | 0 {build => opt/build}/docker/Makefile | 0 {build => opt/build}/libtorch/Dockerfile.arm | 2 +- {build => opt/build}/libtorch/Dockerfile.x64 | 2 +- {build => opt/build}/libtorch/Makefile | 2 +- {build => opt/build}/libtorch/collect.py | 0 {build => opt/build}/libtorch/repack.sh | 4 ++-- {build => opt/build}/onnxruntime/Dockerfile.arm | 0 {build => opt/build}/onnxruntime/Dockerfile.arm7 | 0 {build => opt/build}/onnxruntime/Dockerfile.x64 | 0 {build => opt/build}/onnxruntime/Makefile | 0 {build => opt/build}/onnxruntime/pack.sh | 0 {build => opt/build}/onnxruntime/werrror.patch | 0 {build => opt/build}/platforms/Makefile | 0 {build => opt/build}/tensorflow/Dockerfile.arm | 2 +- {build => opt/build}/tensorflow/Dockerfile.x64 | 0 {build => opt/build}/tensorflow/Makefile | 0 {build => opt/build}/tensorflow/collect.py | 1 + .../build}/tensorflow/tf-1.14.0-arm64v8.patch | 0 {automation => opt}/pack.sh | 4 ++-- {automation => opt}/readies/.gitignore | 0 {automation => opt}/readies/LICENSE | 0 {automation => opt}/readies/README.md | 0 {automation => opt}/readies/bin/getpy | 0 {automation => opt}/readies/bin/getpy2 | 0 {automation => opt}/readies/bin/getredis5 | 0 {automation => opt}/readies/bin/platform | 0 {automation => opt}/readies/bin/python | 0 {automation => opt}/readies/bin/python2 | 0 opt/readies/bin/sep | 6 ++++++ opt/readies/bin/sep1 | 6 ++++++ {automation => opt}/readies/cetara/diag/gdb.c | 0 {automation => opt}/readies/cetara/diag/gdb.h | 0 {automation => opt}/readies/mk/bindirs.defs | 0 {automation => opt}/readies/mk/bindirs.rules | 0 {automation => opt}/readies/mk/build.defs | 0 {automation => opt}/readies/mk/build.rules | 0 {automation => opt}/readies/mk/cmake.defs | 0 {automation => opt}/readies/mk/cmake.rules | 0 {automation => opt}/readies/mk/common.defs | 0 {automation => opt}/readies/mk/common.rules | 0 {automation => opt}/readies/mk/configure.defs | 0 {automation => opt}/readies/mk/configure.rules | 0 {automation => opt}/readies/mk/defs | 0 {automation => opt}/readies/mk/help.defs | 0 {automation => opt}/readies/mk/help.rules | 0 {automation => opt}/readies/mk/main | 4 ++-- {automation => opt}/readies/mk/rules | 0 {automation => opt}/readies/mk/variant.defs | 6 +++--- {automation => opt}/readies/mk/variant.rules | 0 {automation => opt}/readies/paella/__init__.py | 0 {automation => opt}/readies/paella/debug.py | 0 {automation => opt}/readies/paella/docopt1.py | 0 {automation => opt}/readies/paella/files.py | 0 {automation => opt}/readies/paella/log.py | 0 {automation => opt}/readies/paella/platform.py | 0 {automation => opt}/readies/paella/setup.py | 0 {automation => opt}/readies/paella/utils.py | 0 {automation => opt}/readies/paella/utils2.py | 0 {automation => opt}/readies/paella/utils3.py | 0 {automation => opt}/readies/shibumi/functions | 0 {automation => opt}/readies/shibumi/here | 0 {automation => opt}/system-setup.py | 0 src/CMakeLists.txt | 2 +- src/backends/torch.c | 2 +- {util => src}/libtorch_c/CMakeLists.txt | 0 {util => src}/libtorch_c/torch_c.cpp | 0 {util => src}/libtorch_c/torch_c.h | 0 src/model.c | 2 +- src/redisai.c | 2 +- src/{util => rmutil}/alloc.c | 0 src/{util => rmutil}/alloc.h | 0 src/script.c | 2 +- src/tensor.c | 2 +- 82 files changed, 55 insertions(+), 40 deletions(-) rename {automation => opt}/Makefile (98%) rename {build => opt/build}/bazel/Dockerfile (100%) rename {build => opt/build}/docker/Makefile (100%) rename {build => opt/build}/libtorch/Dockerfile.arm (96%) rename {build => opt/build}/libtorch/Dockerfile.x64 (94%) rename {build => opt/build}/libtorch/Makefile (98%) rename {build => opt/build}/libtorch/collect.py (100%) rename {build => opt/build}/libtorch/repack.sh (90%) rename {build => opt/build}/onnxruntime/Dockerfile.arm (100%) rename {build => opt/build}/onnxruntime/Dockerfile.arm7 (100%) rename {build => opt/build}/onnxruntime/Dockerfile.x64 (100%) rename {build => opt/build}/onnxruntime/Makefile (100%) rename {build => opt/build}/onnxruntime/pack.sh (100%) rename {build => opt/build}/onnxruntime/werrror.patch (100%) rename {build => opt/build}/platforms/Makefile (100%) rename {build => opt/build}/tensorflow/Dockerfile.arm (97%) rename {build => opt/build}/tensorflow/Dockerfile.x64 (100%) rename {build => opt/build}/tensorflow/Makefile (100%) rename {build => opt/build}/tensorflow/collect.py (97%) rename {build => opt/build}/tensorflow/tf-1.14.0-arm64v8.patch (100%) rename {automation => opt}/pack.sh (97%) rename {automation => opt}/readies/.gitignore (100%) rename {automation => opt}/readies/LICENSE (100%) rename {automation => opt}/readies/README.md (100%) rename {automation => opt}/readies/bin/getpy (100%) rename {automation => opt}/readies/bin/getpy2 (100%) rename {automation => opt}/readies/bin/getredis5 (100%) rename {automation => opt}/readies/bin/platform (100%) rename {automation => opt}/readies/bin/python (100%) rename {automation => opt}/readies/bin/python2 (100%) create mode 100755 opt/readies/bin/sep create mode 100755 opt/readies/bin/sep1 rename {automation => opt}/readies/cetara/diag/gdb.c (100%) rename {automation => opt}/readies/cetara/diag/gdb.h (100%) rename {automation => opt}/readies/mk/bindirs.defs (100%) rename {automation => opt}/readies/mk/bindirs.rules (100%) rename {automation => opt}/readies/mk/build.defs (100%) rename {automation => opt}/readies/mk/build.rules (100%) rename {automation => opt}/readies/mk/cmake.defs (100%) rename {automation => opt}/readies/mk/cmake.rules (100%) rename {automation => opt}/readies/mk/common.defs (100%) rename {automation => opt}/readies/mk/common.rules (100%) rename {automation => opt}/readies/mk/configure.defs (100%) rename {automation => opt}/readies/mk/configure.rules (100%) rename {automation => opt}/readies/mk/defs (100%) rename {automation => opt}/readies/mk/help.defs (100%) rename {automation => opt}/readies/mk/help.rules (100%) rename {automation => opt}/readies/mk/main (66%) rename {automation => opt}/readies/mk/rules (100%) rename {automation => opt}/readies/mk/variant.defs (84%) rename {automation => opt}/readies/mk/variant.rules (100%) rename {automation => opt}/readies/paella/__init__.py (100%) rename {automation => opt}/readies/paella/debug.py (100%) rename {automation => opt}/readies/paella/docopt1.py (100%) rename {automation => opt}/readies/paella/files.py (100%) rename {automation => opt}/readies/paella/log.py (100%) rename {automation => opt}/readies/paella/platform.py (100%) rename {automation => opt}/readies/paella/setup.py (100%) rename {automation => opt}/readies/paella/utils.py (100%) rename {automation => opt}/readies/paella/utils2.py (100%) rename {automation => opt}/readies/paella/utils3.py (100%) rename {automation => opt}/readies/shibumi/functions (100%) rename {automation => opt}/readies/shibumi/here (100%) rename {automation => opt}/system-setup.py (100%) rename {util => src}/libtorch_c/CMakeLists.txt (100%) rename {util => src}/libtorch_c/torch_c.cpp (100%) rename {util => src}/libtorch_c/torch_c.h (100%) rename src/{util => rmutil}/alloc.c (100%) rename src/{util => rmutil}/alloc.h (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index b470d3bfa..ce74da557 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,8 +15,8 @@ commands: - run: name: Install dependencies command: | - sudo ./automation/readies/bin/getpy - sudo ./automation/system-setup.py + sudo ./opt/readies/bin/getpy + sudo ./opt/system-setup.py git clone git://github.com/antirez/redis.git --branch 5.0.5 (cd redis && make malloc=libc -j $(nproc) && sudo make install) ./get_deps.sh cpu @@ -26,16 +26,16 @@ commands: key: v1-dependencies-{{ checksum "get_deps.sh" }} - run: name: Build - command: make -C automation all SHOW=1 + command: make -C opt all SHOW=1 - run: name: Test command: | mkdir -p ~/workspace/tests - make -C automation test SHOW=1 + make -C opt test SHOW=1 cp test/logs/* ~/workspace/tests - run: name: Package - command: make -C automation pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=~/workspace/packages SHOW=1 + command: make -C opt pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=~/workspace/packages SHOW=1 - persist_to_workspace: root: ~/workspace paths: diff --git a/.dockerignore b/.dockerignore index 0f6c14c4a..3805aedef 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,7 @@ /bin/ /deps/ /install* +/build/ .venv/ venv*/ /[0-9]*/ diff --git a/.gitignore b/.gitignore index 8f6f89af3..ecd44278d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /bin/ /deps/ +/build/ /install* /test/venv/ /test/logs/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 24d7d1d83..1c9a1b2a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,7 +89,7 @@ FIND_PACKAGE(Torch REQUIRED) INCLUDE_DIRECTORIES(util/libtorch_c) ADD_DEFINITIONS(-DREDISMODULE_EXPERIMENTAL_API) -ADD_SUBDIRECTORY(util/libtorch_c) +ADD_SUBDIRECTORY(src/libtorch_c) #---------------------------------------------------------------------------------------------- diff --git a/Dockerfile b/Dockerfile index ce803a986..b74fb0a3c 100755 --- a/Dockerfile +++ b/Dockerfile @@ -16,23 +16,23 @@ FROM ${OS} AS builder WORKDIR /build COPY --from=redis /usr/local/ /usr/local/ -COPY ./automation/ automation/ +COPY ./opt/ opt/ COPY ./test/test_requirements.txt test/ -RUN ./automation/readies/bin/getpy -RUN ./automation/system-setup.py +RUN ./opt/readies/bin/getpy +RUN ./opt/system-setup.py COPY ./get_deps.sh . RUN ./get_deps.sh cpu ADD ./ /build -RUN make -C automation build SHOW=1 +RUN make -C opt build SHOW=1 ARG PACK=0 ARG TEST=0 -RUN if [ "$PACK" = "1" ]; then make -C automation pack; fi -RUN if [ "$TEST" = "1" ]; then make -C automation test; fi +RUN if [ "$PACK" = "1" ]; then make -C opt pack; fi +RUN if [ "$TEST" = "1" ]; then make -C opt test; fi #---------------------------------------------------------------------------------------------- # FROM redis:latest diff --git a/Dockerfile.arm b/Dockerfile.arm index d4e9e7a41..681f5aaf3 100755 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -17,23 +17,23 @@ RUN [ "cross-build-start" ] ADD ./ /build WORKDIR /build -COPY ./automation/ automation/ +COPY ./opt/ opt/ COPY ./test/test_requirements.txt test/ -RUN ./automation/readies/bin/getpy -RUN ./automation/system-setup.py +RUN ./opt/readies/bin/getpy +RUN ./opt/system-setup.py COPY ./get_deps.sh . RUN ./get_deps.sh cpu ADD ./ /redisai -RUN make -C automation all SHOW=1 +RUN make -C opt all SHOW=1 ARG PACK=0 ARG TEST=0 -RUN if [ "$PACK" = "1" ]; then make -C automation pack; fi -RUN if [ "$TEST" = "1" ]; then make -C automation test; fi +RUN if [ "$PACK" = "1" ]; then make -C opt pack; fi +RUN if [ "$TEST" = "1" ]; then make -C opt test; fi RUN [ "cross-build-end" ] diff --git a/get_deps.sh b/get_deps.sh index 65c238fff..ba5b3c717 100755 --- a/get_deps.sh +++ b/get_deps.sh @@ -39,8 +39,8 @@ else fi fi -OS=$(python3 $HERE/automation/readies/bin/platform --os) -ARCH=$(python3 $HERE/automation/readies/bin/platform --arch) +OS=$(python3 $HERE/opt/readies/bin/platform --os) +ARCH=$(python3 $HERE/opt/readies/bin/platform --arch) # avoid wget warnings on macOS [[ $OS == macosx ]] && export LC_ALL=en_US.UTF-8 diff --git a/automation/Makefile b/opt/Makefile similarity index 98% rename from automation/Makefile rename to opt/Makefile index b2d89ee27..ea76a426f 100755 --- a/automation/Makefile +++ b/opt/Makefile @@ -78,7 +78,7 @@ ifeq ($(ALL),1) $(SHOW)if [ -d "$(INSTALL_DIR)" ]; then rm -rf $(INSTALL_DIR); fi $(SHOW)rm -f $(ROOT)/install-$(DEVICE) else - $(SHOW)$(MAKE) -C $(BINDIR) clean + -$(SHOW)$(MAKE) -C $(BINDIR) clean endif #---------------------------------------------------------------------------------------------- diff --git a/build/bazel/Dockerfile b/opt/build/bazel/Dockerfile similarity index 100% rename from build/bazel/Dockerfile rename to opt/build/bazel/Dockerfile diff --git a/build/docker/Makefile b/opt/build/docker/Makefile similarity index 100% rename from build/docker/Makefile rename to opt/build/docker/Makefile diff --git a/build/libtorch/Dockerfile.arm b/opt/build/libtorch/Dockerfile.arm similarity index 96% rename from build/libtorch/Dockerfile.arm rename to opt/build/libtorch/Dockerfile.arm index f24190c98..d5c00c536 100755 --- a/build/libtorch/Dockerfile.arm +++ b/opt/build/libtorch/Dockerfile.arm @@ -44,7 +44,7 @@ RUN set -e ;\ cd pytorch ;\ python3 setup.py install -ADD ./deps/readies/ /build/deps/readies/ +ADD ./readies/ /build/deps/readies/ ADD ./build/libtorch-arm/collect.py /build/ RUN ./collect.py diff --git a/build/libtorch/Dockerfile.x64 b/opt/build/libtorch/Dockerfile.x64 similarity index 94% rename from build/libtorch/Dockerfile.x64 rename to opt/build/libtorch/Dockerfile.x64 index eab122c8e..7372a8246 100755 --- a/build/libtorch/Dockerfile.x64 +++ b/opt/build/libtorch/Dockerfile.x64 @@ -36,7 +36,7 @@ RUN set -e ;\ cd pytorch ;\ python3 setup.py install -ADD ./automation/readies/ /build/deps/readies/ +ADD ./opt/readies/ /build/deps/readies/ ADD ./build/libtorch-arm/collect.py /build/ RUN ./collect.py diff --git a/build/libtorch/Makefile b/opt/build/libtorch/Makefile similarity index 98% rename from build/libtorch/Makefile rename to opt/build/libtorch/Makefile index d01b454ff..bb80891a8 100755 --- a/build/libtorch/Makefile +++ b/opt/build/libtorch/Makefile @@ -8,7 +8,7 @@ OSNICK ?= buster S3_URL=redismodules/pytorch -OS:=$(shell $(ROOT)/automation/readies/bin/platform --os) +OS:=$(shell $(ROOT)/opt/readies/bin/platform --os) ifeq ($(OS),macosx) OS:=macos endif diff --git a/build/libtorch/collect.py b/opt/build/libtorch/collect.py similarity index 100% rename from build/libtorch/collect.py rename to opt/build/libtorch/collect.py diff --git a/build/libtorch/repack.sh b/opt/build/libtorch/repack.sh similarity index 90% rename from build/libtorch/repack.sh rename to opt/build/libtorch/repack.sh index 34fa4ef0b..058dc8af3 100755 --- a/build/libtorch/repack.sh +++ b/opt/build/libtorch/repack.sh @@ -13,8 +13,8 @@ else GPU=${GPU:-no} fi -OS=$(python3 $ROOT/automation/readies/bin/platform --os) -ARCH=$(python3 $ROOT/automation/readies/bin/platform --arch) +OS=$(python3 $ROOT/opt/readies/bin/platform --os) +ARCH=$(python3 $ROOT/opt/readies/bin/platform --arch) # avoid wget warnings on macOS [[ $OS == macosx ]] && export LC_ALL=en_US.UTF-8 diff --git a/build/onnxruntime/Dockerfile.arm b/opt/build/onnxruntime/Dockerfile.arm similarity index 100% rename from build/onnxruntime/Dockerfile.arm rename to opt/build/onnxruntime/Dockerfile.arm diff --git a/build/onnxruntime/Dockerfile.arm7 b/opt/build/onnxruntime/Dockerfile.arm7 similarity index 100% rename from build/onnxruntime/Dockerfile.arm7 rename to opt/build/onnxruntime/Dockerfile.arm7 diff --git a/build/onnxruntime/Dockerfile.x64 b/opt/build/onnxruntime/Dockerfile.x64 similarity index 100% rename from build/onnxruntime/Dockerfile.x64 rename to opt/build/onnxruntime/Dockerfile.x64 diff --git a/build/onnxruntime/Makefile b/opt/build/onnxruntime/Makefile similarity index 100% rename from build/onnxruntime/Makefile rename to opt/build/onnxruntime/Makefile diff --git a/build/onnxruntime/pack.sh b/opt/build/onnxruntime/pack.sh similarity index 100% rename from build/onnxruntime/pack.sh rename to opt/build/onnxruntime/pack.sh diff --git a/build/onnxruntime/werrror.patch b/opt/build/onnxruntime/werrror.patch similarity index 100% rename from build/onnxruntime/werrror.patch rename to opt/build/onnxruntime/werrror.patch diff --git a/build/platforms/Makefile b/opt/build/platforms/Makefile similarity index 100% rename from build/platforms/Makefile rename to opt/build/platforms/Makefile diff --git a/build/tensorflow/Dockerfile.arm b/opt/build/tensorflow/Dockerfile.arm similarity index 97% rename from build/tensorflow/Dockerfile.arm rename to opt/build/tensorflow/Dockerfile.arm index 0f57f0cf6..6e4b47e86 100755 --- a/build/tensorflow/Dockerfile.arm +++ b/opt/build/tensorflow/Dockerfile.arm @@ -52,7 +52,7 @@ RUN set -e; \ ./configure ;\ bazel build --jobs $(eval "$X_NPROC") --config=opt //tensorflow:libtensorflow.so -ADD ./deps/readies/ /build/deps/readies/ +ADD ./readies/ /build/deps/readies/ ADD ./build/tensorflow/collect.py /build/ RUN ./collect.py diff --git a/build/tensorflow/Dockerfile.x64 b/opt/build/tensorflow/Dockerfile.x64 similarity index 100% rename from build/tensorflow/Dockerfile.x64 rename to opt/build/tensorflow/Dockerfile.x64 diff --git a/build/tensorflow/Makefile b/opt/build/tensorflow/Makefile similarity index 100% rename from build/tensorflow/Makefile rename to opt/build/tensorflow/Makefile diff --git a/build/tensorflow/collect.py b/opt/build/tensorflow/collect.py similarity index 97% rename from build/tensorflow/collect.py rename to opt/build/tensorflow/collect.py index 67d10a46e..662e0d128 100755 --- a/build/tensorflow/collect.py +++ b/opt/build/tensorflow/collect.py @@ -7,6 +7,7 @@ import shutil import tarfile +# this refers to deps directory inside a container sys.path.insert(0, os.path.join(os.path.dirname(__file__), "deps/readies")) import paella diff --git a/build/tensorflow/tf-1.14.0-arm64v8.patch b/opt/build/tensorflow/tf-1.14.0-arm64v8.patch similarity index 100% rename from build/tensorflow/tf-1.14.0-arm64v8.patch rename to opt/build/tensorflow/tf-1.14.0-arm64v8.patch diff --git a/automation/pack.sh b/opt/pack.sh similarity index 97% rename from automation/pack.sh rename to opt/pack.sh index 9cdee1263..c189373e1 100755 --- a/automation/pack.sh +++ b/opt/pack.sh @@ -6,7 +6,7 @@ [[ ! -z $INTO ]] && INTO=$(realpath $INTO) HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -. $HERE/../automation/readies/shibumi/functions +. $HERE/readies/shibumi/functions ROOT=$(realpath $HERE/..) RAMP_PROG=ramp @@ -106,7 +106,7 @@ if [[ -d $ROOT/.git ]]; then fi OSX="" -if [[ $(./readies/bin/platform --os) == macosx ]]; then +if [[ $($HERE/readies/bin/platform --os) == macosx ]]; then # macOS: ramp is installed here OSX=1 export PATH=$PATH:$HOME/Library/Python/2.7/bin diff --git a/automation/readies/.gitignore b/opt/readies/.gitignore similarity index 100% rename from automation/readies/.gitignore rename to opt/readies/.gitignore diff --git a/automation/readies/LICENSE b/opt/readies/LICENSE similarity index 100% rename from automation/readies/LICENSE rename to opt/readies/LICENSE diff --git a/automation/readies/README.md b/opt/readies/README.md similarity index 100% rename from automation/readies/README.md rename to opt/readies/README.md diff --git a/automation/readies/bin/getpy b/opt/readies/bin/getpy similarity index 100% rename from automation/readies/bin/getpy rename to opt/readies/bin/getpy diff --git a/automation/readies/bin/getpy2 b/opt/readies/bin/getpy2 similarity index 100% rename from automation/readies/bin/getpy2 rename to opt/readies/bin/getpy2 diff --git a/automation/readies/bin/getredis5 b/opt/readies/bin/getredis5 similarity index 100% rename from automation/readies/bin/getredis5 rename to opt/readies/bin/getredis5 diff --git a/automation/readies/bin/platform b/opt/readies/bin/platform similarity index 100% rename from automation/readies/bin/platform rename to opt/readies/bin/platform diff --git a/automation/readies/bin/python b/opt/readies/bin/python similarity index 100% rename from automation/readies/bin/python rename to opt/readies/bin/python diff --git a/automation/readies/bin/python2 b/opt/readies/bin/python2 similarity index 100% rename from automation/readies/bin/python2 rename to opt/readies/bin/python2 diff --git a/opt/readies/bin/sep b/opt/readies/bin/sep new file mode 100755 index 000000000..9d41b81fa --- /dev/null +++ b/opt/readies/bin/sep @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +import os +rows, cols = os.popen('stty size', 'r').read().split() +print("\n" + "-" * (int(cols) - 1) + "\n") + diff --git a/opt/readies/bin/sep1 b/opt/readies/bin/sep1 new file mode 100755 index 000000000..d5555910a --- /dev/null +++ b/opt/readies/bin/sep1 @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +import os +rows, cols = os.popen('stty size', 'r').read().split() +print("-" * (int(cols) - 1) + "\n") + diff --git a/automation/readies/cetara/diag/gdb.c b/opt/readies/cetara/diag/gdb.c similarity index 100% rename from automation/readies/cetara/diag/gdb.c rename to opt/readies/cetara/diag/gdb.c diff --git a/automation/readies/cetara/diag/gdb.h b/opt/readies/cetara/diag/gdb.h similarity index 100% rename from automation/readies/cetara/diag/gdb.h rename to opt/readies/cetara/diag/gdb.h diff --git a/automation/readies/mk/bindirs.defs b/opt/readies/mk/bindirs.defs similarity index 100% rename from automation/readies/mk/bindirs.defs rename to opt/readies/mk/bindirs.defs diff --git a/automation/readies/mk/bindirs.rules b/opt/readies/mk/bindirs.rules similarity index 100% rename from automation/readies/mk/bindirs.rules rename to opt/readies/mk/bindirs.rules diff --git a/automation/readies/mk/build.defs b/opt/readies/mk/build.defs similarity index 100% rename from automation/readies/mk/build.defs rename to opt/readies/mk/build.defs diff --git a/automation/readies/mk/build.rules b/opt/readies/mk/build.rules similarity index 100% rename from automation/readies/mk/build.rules rename to opt/readies/mk/build.rules diff --git a/automation/readies/mk/cmake.defs b/opt/readies/mk/cmake.defs similarity index 100% rename from automation/readies/mk/cmake.defs rename to opt/readies/mk/cmake.defs diff --git a/automation/readies/mk/cmake.rules b/opt/readies/mk/cmake.rules similarity index 100% rename from automation/readies/mk/cmake.rules rename to opt/readies/mk/cmake.rules diff --git a/automation/readies/mk/common.defs b/opt/readies/mk/common.defs similarity index 100% rename from automation/readies/mk/common.defs rename to opt/readies/mk/common.defs diff --git a/automation/readies/mk/common.rules b/opt/readies/mk/common.rules similarity index 100% rename from automation/readies/mk/common.rules rename to opt/readies/mk/common.rules diff --git a/automation/readies/mk/configure.defs b/opt/readies/mk/configure.defs similarity index 100% rename from automation/readies/mk/configure.defs rename to opt/readies/mk/configure.defs diff --git a/automation/readies/mk/configure.rules b/opt/readies/mk/configure.rules similarity index 100% rename from automation/readies/mk/configure.rules rename to opt/readies/mk/configure.rules diff --git a/automation/readies/mk/defs b/opt/readies/mk/defs similarity index 100% rename from automation/readies/mk/defs rename to opt/readies/mk/defs diff --git a/automation/readies/mk/help.defs b/opt/readies/mk/help.defs similarity index 100% rename from automation/readies/mk/help.defs rename to opt/readies/mk/help.defs diff --git a/automation/readies/mk/help.rules b/opt/readies/mk/help.rules similarity index 100% rename from automation/readies/mk/help.rules rename to opt/readies/mk/help.rules diff --git a/automation/readies/mk/main b/opt/readies/mk/main similarity index 66% rename from automation/readies/mk/main rename to opt/readies/mk/main index 3df0f814a..1a54494ec 100755 --- a/automation/readies/mk/main +++ b/opt/readies/mk/main @@ -1,11 +1,11 @@ SHELL=/bin/bash -ifneq ($(shell { CHECK=1 $(ROOT)/automation/readies/bin/getpy; echo $$?; }),0) +ifneq ($(shell { CHECK=1 $(ROOT)/opt/readies/bin/getpy; echo $$?; }),0) $(error It seems prerequisites have not been installed: please run 'make setup'.) endif -MK=$(ROOT)/automation/readies/mk +MK=$(ROOT)/opt/readies/mk MK_ALL_TARGETS:=bindirs build diff --git a/automation/readies/mk/rules b/opt/readies/mk/rules similarity index 100% rename from automation/readies/mk/rules rename to opt/readies/mk/rules diff --git a/automation/readies/mk/variant.defs b/opt/readies/mk/variant.defs similarity index 84% rename from automation/readies/mk/variant.defs rename to opt/readies/mk/variant.defs index b719342bf..9dcb8875d 100755 --- a/automation/readies/mk/variant.defs +++ b/opt/readies/mk/variant.defs @@ -1,10 +1,10 @@ -OS:=$(shell $(ROOT)/automation/readies/bin/platform --os) +OS:=$(shell $(ROOT)/opt/readies/bin/platform --os) # ifeq ($(OS),linux) -# OS:=$(shell $(ROOT)/automation/readies/bin/platform --dist) +# OS:=$(shell $(ROOT)/opt/readies/bin/platform --dist) # endif -ARCH=$(shell $(ROOT)/automation/readies/bin/platform --arch) +ARCH=$(shell $(ROOT)/opt/readies/bin/platform --arch) #---------------------------------------------------------------------------------------------- diff --git a/automation/readies/mk/variant.rules b/opt/readies/mk/variant.rules similarity index 100% rename from automation/readies/mk/variant.rules rename to opt/readies/mk/variant.rules diff --git a/automation/readies/paella/__init__.py b/opt/readies/paella/__init__.py similarity index 100% rename from automation/readies/paella/__init__.py rename to opt/readies/paella/__init__.py diff --git a/automation/readies/paella/debug.py b/opt/readies/paella/debug.py similarity index 100% rename from automation/readies/paella/debug.py rename to opt/readies/paella/debug.py diff --git a/automation/readies/paella/docopt1.py b/opt/readies/paella/docopt1.py similarity index 100% rename from automation/readies/paella/docopt1.py rename to opt/readies/paella/docopt1.py diff --git a/automation/readies/paella/files.py b/opt/readies/paella/files.py similarity index 100% rename from automation/readies/paella/files.py rename to opt/readies/paella/files.py diff --git a/automation/readies/paella/log.py b/opt/readies/paella/log.py similarity index 100% rename from automation/readies/paella/log.py rename to opt/readies/paella/log.py diff --git a/automation/readies/paella/platform.py b/opt/readies/paella/platform.py similarity index 100% rename from automation/readies/paella/platform.py rename to opt/readies/paella/platform.py diff --git a/automation/readies/paella/setup.py b/opt/readies/paella/setup.py similarity index 100% rename from automation/readies/paella/setup.py rename to opt/readies/paella/setup.py diff --git a/automation/readies/paella/utils.py b/opt/readies/paella/utils.py similarity index 100% rename from automation/readies/paella/utils.py rename to opt/readies/paella/utils.py diff --git a/automation/readies/paella/utils2.py b/opt/readies/paella/utils2.py similarity index 100% rename from automation/readies/paella/utils2.py rename to opt/readies/paella/utils2.py diff --git a/automation/readies/paella/utils3.py b/opt/readies/paella/utils3.py similarity index 100% rename from automation/readies/paella/utils3.py rename to opt/readies/paella/utils3.py diff --git a/automation/readies/shibumi/functions b/opt/readies/shibumi/functions similarity index 100% rename from automation/readies/shibumi/functions rename to opt/readies/shibumi/functions diff --git a/automation/readies/shibumi/here b/opt/readies/shibumi/here similarity index 100% rename from automation/readies/shibumi/here rename to opt/readies/shibumi/here diff --git a/automation/system-setup.py b/opt/system-setup.py similarity index 100% rename from automation/system-setup.py rename to opt/system-setup.py diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 264d9059e..4d616ef92 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,7 +5,7 @@ ADD_LIBRARY(redisai_obj OBJECT err.c script.c tensor.c - util/alloc.c + rmutil/alloc.c rmutil/sds.c rmutil/args.c rmutil/heap.c diff --git a/src/backends/torch.c b/src/backends/torch.c index b4e78ef20..e817c0467 100644 --- a/src/backends/torch.c +++ b/src/backends/torch.c @@ -1,7 +1,7 @@ #include "backends/torch.h" #include "tensor.h" #include "util/arr_rm_alloc.h" -#include "torch_c.h" +#include "libtorch_c/torch_c.h" int RAI_InitBackendTorch(int (*get_api_fn)(const char *, void *)) { get_api_fn("RedisModule_Alloc", ((void **)&RedisModule_Alloc)); diff --git a/util/libtorch_c/CMakeLists.txt b/src/libtorch_c/CMakeLists.txt similarity index 100% rename from util/libtorch_c/CMakeLists.txt rename to src/libtorch_c/CMakeLists.txt diff --git a/util/libtorch_c/torch_c.cpp b/src/libtorch_c/torch_c.cpp similarity index 100% rename from util/libtorch_c/torch_c.cpp rename to src/libtorch_c/torch_c.cpp diff --git a/util/libtorch_c/torch_c.h b/src/libtorch_c/torch_c.h similarity index 100% rename from util/libtorch_c/torch_c.h rename to src/libtorch_c/torch_c.h diff --git a/src/model.c b/src/model.c index ddf7c891b..407ee8f9a 100644 --- a/src/model.c +++ b/src/model.c @@ -2,7 +2,7 @@ #include "model_struct.h" #include "backends.h" -#include "util/alloc.h" +#include "rmutil/alloc.h" #include "util/arr_rm_alloc.h" RedisModuleType *RedisAI_ModelType = NULL; diff --git a/src/redisai.c b/src/redisai.c index c82836f61..deb32fe54 100644 --- a/src/redisai.c +++ b/src/redisai.c @@ -9,7 +9,7 @@ #include #include -#include "util/alloc.h" +#include "rmutil/alloc.h" #include "util/arr_rm_alloc.h" #include "rmutil/args.h" diff --git a/src/util/alloc.c b/src/rmutil/alloc.c similarity index 100% rename from src/util/alloc.c rename to src/rmutil/alloc.c diff --git a/src/util/alloc.h b/src/rmutil/alloc.h similarity index 100% rename from src/util/alloc.h rename to src/rmutil/alloc.h diff --git a/src/script.c b/src/script.c index 06bcb8dd9..d2cf0bb95 100644 --- a/src/script.c +++ b/src/script.c @@ -2,7 +2,7 @@ #include "script_struct.h" #include "backends.h" -#include "util/alloc.h" +#include "rmutil/alloc.h" #include "util/arr_rm_alloc.h" RedisModuleType *RedisAI_ScriptType = NULL; diff --git a/src/tensor.c b/src/tensor.c index f5b7bed96..c4d3eeaf7 100644 --- a/src/tensor.c +++ b/src/tensor.c @@ -3,7 +3,7 @@ #include #include #include -#include "util/alloc.h" +#include "rmutil/alloc.h" #include RedisModuleType *RedisAI_TensorType = NULL; From 2432c6592b54610994e7fac13ee99e7de026f62c Mon Sep 17 00:00:00 2001 From: rafie Date: Sun, 1 Sep 2019 18:34:11 +0300 Subject: [PATCH 30/30] Pack fixes + docker goal in makefile --- opt/Makefile | 12 +++++++----- opt/pack.sh | 10 +++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/opt/Makefile b/opt/Makefile index ea76a426f..3f2009062 100755 --- a/opt/Makefile +++ b/opt/Makefile @@ -91,14 +91,11 @@ fetch deps: pack: ifneq ($(PACK_DEPS),0) - $(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) DEPS=0 ./pack.sh -else $(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) DEPS=1 ./pack.sh +else + $(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) DEPS=0 ./pack.sh endif -#$(BINDIR)/BINDIR: $(BIN_DIRS) -# $(SHOW)echo $(BINDIR)>$(BINDIR)/BINDIR - #---------------------------------------------------------------------------------------------- TEST_REPORT_DIR ?= $(PWD) @@ -109,3 +106,8 @@ test: $(SHOW)set -e ;\ cd $(ROOT)/test ;\ python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so + +#---------------------------------------------------------------------------------------------- + +docker: + $(SHOW)docker build -t redisai --build-arg TEST=1 --build-arg PACK=1 .. diff --git a/opt/pack.sh b/opt/pack.sh index c189373e1..1e3167e59 100755 --- a/opt/pack.sh +++ b/opt/pack.sh @@ -26,7 +26,7 @@ pack_ramp() { PACKAGE=$(cat $BINDIR/PACKAGE) VERSION=$(cat $BINDIR/VERSION) OSARCH=$(cat $BINDIR/OSARCH) - $RAMP_PROG pack -m $ROOT/ramp.yml -o "build/$PRODUCT.{os}-{architecture}.{semantic_version}.zip" \ + $RAMP_PROG pack -m $ROOT/ramp.yml -o "$BINDIR/$PRODUCT.{os}-{architecture}.{semantic_version}.zip" \ -c "BACKENDSPATH $REDIS_ENT_LIB_PATH/$PRODUCT-$DEVICE-$VERSION/backends" $INSTALL_DIR/$PRODUCT.so 2> /dev/null | grep '.zip' > $RAMPOUT rm -f $RAMPOUT export PACK_FNAME="$(basename $PACKAGE)" @@ -68,8 +68,8 @@ pack_deps() { [[ ! -z $BRANCH ]] && ln -sf $DEPS_FNAME $DEPS_STEM.${BRANCH}.tgz ln -sf $DEPS_FNAME $DEPS_STEM.${GIT_VER}.tgz - export RELEASE_ARTIFACTS="$RELEASE_ARTIFACTS $DEPS_FNAME $DEPS_STEM.latest.zip" - export DEV_ARTIFACTS="$DEV_ARTIFACTS $DEPS_STEM.${BRANCH}.zip $DEPS_STEM.${GIT_VER}.zip" + export RELEASE_ARTIFACTS="$RELEASE_ARTIFACTS $DEPS_FNAME $DEPS_STEM.latest.tgz" + export DEV_ARTIFACTS="$DEV_ARTIFACTS $DEPS_STEM.${BRANCH}.tgz $DEPS_STEM.${GIT_VER}.tgz" echo "Done." } @@ -124,12 +124,12 @@ if [[ ! -z $INTO ]]; then mkdir -p $INTO/release $INTO/branch cd $INTO/release for f in $RELEASE_ARTIFACTS; do - [[ -f $BINDIR/$f ]] && ln -sf $(realpath --relative-to . $BINDIR/$f) + [[ -f $BINDIR/$f ]] && ln -sf $(realpath --relative-to . $BINDIR/$f) $(basename $f) done cd $INTO/branch for f in $DEV_ARTIFACTS; do - [[ -f $BINDIR/$f ]] && ln -sf $(realpath --relative-to . $BINDIR/$f) + [[ -f $BINDIR/$f ]] && ln -sf $(realpath --relative-to . $BINDIR/$f) $(basename $f) done fi