diff --git a/.circleci/config.yml b/.circleci/config.yml index 56fe7236c..ce74da557 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,127 +1,91 @@ version: 2.1 -jobs: - build: - docker: - - image: circleci/python:3.7.4-buster +commands: + ci_steps: + parameters: + platform: + type: string steps: - checkout - - # Download and cache dependencies - restore_cache: keys: - v1-dependencies-{{ checksum "get_deps.sh" }} # fallback to using the latest cache if no exact match is found # - v1-dependencies- - - run: - name: install dependencies + name: Install dependencies command: | - sudo ./automation/readies/bin/getpy - sudo ./automation/system-setup.py + sudo ./opt/readies/bin/getpy + sudo ./opt/system-setup.py + git clone git://github.com/antirez/redis.git --branch 5.0.5 + (cd redis && make malloc=libc -j $(nproc) && sudo make install) ./get_deps.sh cpu - git clone git://github.com/antirez/redis.git --branch 5.0.3 - (cd redis && make malloc=libc -j4 && sudo make install) - - save_cache: paths: - deps - key: v2-dependencies-{{ checksum "get_deps.sh" }} - + key: v1-dependencies-{{ checksum "get_deps.sh" }} - run: name: Build - command: | - make -C automation all - make -C automation pack - + command: make -C opt all SHOW=1 - run: name: Test - command: make -C automation test - - - run: - name: Persist Artifacts command: | - mkdir -p ~/workspace/build - cp install-cpu/$MODULE_ARTIFACT ~/workspace/ - cp -r install-cpu/backends ~/workspace/ - cp ramp.yml ~/workspace/ - - persist_to_workspace: - root: ~/workspace - paths: - - '*.so*' - - '*.yml' - - build - package_branch: - docker: - - image: 'redislabsmodules/rmbuilder:latest' - steps: - - attach_workspace: - at: /workspace + mkdir -p ~/workspace/tests + make -C opt test SHOW=1 + cp test/logs/* ~/workspace/tests - run: name: Package - command: >- - ramp pack -m /workspace/ramp.yml -o - /workspace/build/$PACKAGE_NAME.{os}-{architecture}.$CIRCLE_BRANCH.zip - /workspace/$MODULE_ARTIFACT - environment: - LD_LIBRARY_PATH: /workspace + command: make -C opt pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=~/workspace/packages SHOW=1 - persist_to_workspace: - root: /workspace + root: ~/workspace paths: - - build - - store_artifacts: - path: /workspace/build - package_release: - docker: - - image: 'redislabsmodules/rmbuilder:latest' + - 'packages/release/*.zip' + - 'packages/release/*.tgz' + - 'packages/branch/*.zip' + - 'packages/branch/*.tgz' + - store_test_results: + path: ~/workspace/tests + deploy: + parameters: + from: + type: string steps: - - attach_workspace: - at: /workspace - run: - name: Package - command: >- - ramp pack -m /workspace/ramp.yml -o - /workspace/build/$PACKAGE_NAME.{os}-{architecture}.{semantic_version}.zip - /workspace/$MODULE_ARTIFACT - environment: - LD_LIBRARY_PATH: /workspace - - run: - name: Package + name: Deploy to S3 command: >- - ramp pack -m /workspace/ramp.yml -o - /workspace/build/$PACKAGE_NAME.{os}-{architecture}.latest.zip - /workspace/$MODULE_ARTIFACT - environment: - LD_LIBRARY_PATH: /workspace + aws s3 cp <>/ s3://redismodules/$PACKAGE_NAME/ + --acl public-read --recursive --exclude "*" --include "*.zip" --include "*.tgz" - - persist_to_workspace: - root: /workspace - paths: - - build - - store_artifacts: - path: /workspace/build - deploy_branch: +jobs: + build: docker: - - image: 'redislabsmodules/rmbuilder:latest' + - image: circleci/python:3.7.4-buster + environment: + DEPS: "" steps: - - attach_workspace: - at: /workspace - - run: - name: Deploy to S3 - command: >- - aws s3 cp /workspace/build/ s3://redismodules/$PACKAGE_NAME/ --acl - public-read --recursive --exclude "*" --include "*.zip" - deploy_release: + - ci_steps: + platform: debian + build-macos: + macos: + xcode: 10.2.1 + steps: + - ci_steps: + platform: macosx + + deploy_package: + parameters: + package: + type: string docker: - image: 'redislabsmodules/rmbuilder:latest' steps: - attach_workspace: - at: /workspace - - run: - name: Deploy to S3 - command: >- - aws s3 cp /workspace/build/ s3://redismodules/$PACKAGE_NAME/ --acl - public-read --recursive --exclude "*" --include "*.zip" + at: ~/workspace + - deploy: + from: ~/workspace/packages/<> + - store_artifacts: + path: ~/workspace/packages/<> + deploy_docs: docker: - image: 'redislabsmodules/rmbuilder:latest' @@ -133,8 +97,8 @@ jobs: - run: name: Deploy Docs to S3 command: >- - aws s3 cp site s3://oss.redislabs.com/$WEBSITE_FOLDER/ --acl - public-read --recursive + aws s3 cp site s3://oss.redislabs.com/$WEBSITE_FOLDER/ --acl public-read --recursive + workflows: version: 2 build_and_package: @@ -143,13 +107,21 @@ workflows: filters: tags: only: /.*/ - - package_branch: +# - build-macos: +# filters: +# tags: +# only: /.*/ + - deploy_package: + name: deploy_branch + package: branch requires: - build filters: branches: only: master - - package_release: + - deploy_package: + name: deploy_release + package: release requires: - build filters: @@ -157,15 +129,6 @@ workflows: ignore: /.*/ tags: only: '/^v[0-9].*/' - - deploy_branch: - requires: - - package_branch - - deploy_release: - filters: - tags: - only: '/^v[0-9].*/' - requires: - - package_release - deploy_docs: filters: branches: diff --git a/.dockerignore b/.dockerignore index 9418ba6fc..3805aedef 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,10 @@ +/bin/ /deps/ +/install* /build/ -/install*/ -**/*.o -**/*.so .venv/ venv*/ +/[0-9]*/ +*.zip +*.tgz +*.tar.gz diff --git a/.gitignore b/.gitignore index 0bb1f7455..ecd44278d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,17 @@ +/bin/ +/deps/ +/build/ +/install* +/test/venv/ +/test/logs/ +.venv/ +venv*/ +/install +/[0-9]*/ +*.zip +*.tgz +*.tar.gz + # Misc .DS_Store *.swp @@ -6,18 +20,6 @@ __pycache__ *.pyc -# Dirs -/install*/ -/deps/* -/build/ -examples/js/node_modules/ -examples/js/package-lock.json -/test/venv/ -/test/logs/ -/1/ -/venv/ -/.venv/ - # Eclipse (if that's your jam...) .project .cproject diff --git a/CMakeLists.txt b/CMakeLists.txt index 42458cf7e..1c9a1b2a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,15 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.0.0) PROJECT(RedisAI) -IF (NOT DEPS_PATH) - # dependencies are required! - SET(DEPS_PATH ../deps/install) - # MESSAGE(FATAL_ERROR "DEPS PATH Missing!") -ENDIF() - -IF (NOT DEVICE) - SET(DEVICE cpu) -ENDIF() +#---------------------------------------------------------------------------------------------- FUNCTION(ADD_LDFLAGS _TARGET NEW_FLAGS) GET_TARGET_PROPERTY(LD_FLAGS ${_TARGET} LINK_FLAGS) @@ -19,11 +11,36 @@ FUNCTION(ADD_LDFLAGS _TARGET NEW_FLAGS) SET_TARGET_PROPERTIES(${_TARGET} PROPERTIES LINK_FLAGS ${NEW_FLAGS}) ENDFUNCTION() +MACRO(install_symlink filepath sympath) + install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${filepath} ${sympath})") + install(CODE "message(\"-- Created symlink: ${sympath} -> ${filepath}\")") +ENDMACRO(install_symlink) + +#---------------------------------------------------------------------------------------------- + +IF (NOT DEVICE) + SET(DEVICE cpu) +ENDIF() + +IF (NOT DEPS_PATH) + IF (APPLE) + SET(DEPS_PATH deps/macosx-x64-${DEVICE}) + ELSE() + SET(DEPS_PATH deps/linux-x64-${DEVICE}) + ENDIF() +ENDIF() + +IF (NOT INSTALL_PATH) + SET(INSTALL_PATH ${CMAKE_SOURCE_DIR}/install-${DEVICE}) +ENDIF() + GET_FILENAME_COMPONENT(depsAbs - "${DEPS_PATH}" REALPATH - BASE_DIR "${CMAKE_BINARY_DIR}") + "${DEPS_PATH}" REALPATH BASE_DIR ${CMAKE_SOURCE_DIR}) + +GET_FILENAME_COMPONENT(installAbs + "${INSTALL_PATH}" REALPATH BASE_DIR ${CMAKE_SOURCE_DIR}) -# SET(CUDA_TOOLKIT_ROOT_DIR /usr/local/cuda-9.0) +#---------------------------------------------------------------------------------------------- INCLUDE_DIRECTORIES(${depsAbs}/dlpack/include) INCLUDE_DIRECTORIES(${depsAbs}/libtensorflow/include) @@ -34,25 +51,48 @@ SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") SET(CMAKE_C_STANDARD 11) +# SET(CUDA_TOOLKIT_ROOT_DIR /usr/local/cuda-10.0) + +#---------------------------------------------------------------------------------------------- + FIND_LIBRARY(TF_LIBRARIES NAMES tensorflow PATHS ${depsAbs}/libtensorflow/lib) IF (NOT TF_LIBRARIES) MESSAGE(FATAL_ERROR "Could not find tensorflow") ENDIF() +#---------------------------------------------------------------------------------------------- + FIND_LIBRARY(ORT_LIBRARIES NAMES onnxruntime - PATHS ${depsAbs}/onnxruntime/lib) + PATHS ${depsAbs}/onnxruntime/lib) IF (NOT ORT_LIBRARIES) MESSAGE(FATAL_ERROR "Could not find ONNXRuntime") ENDIF() +#---------------------------------------------------------------------------------------------- + +IF (APPLE) + FIND_LIBRARY(MKL_LIBRARIES NAMES mklml + PATHS ${depsAbs}/mkl/lib) + IF (NOT MKL_LIBRARIES) + MESSAGE(FATAL_ERROR "Could not find MKL for Mac") + ENDIF() + SET(platDeps "${MKL_LIBRARIES}") +ENDIF() + +#---------------------------------------------------------------------------------------------- + # Find Torch stuff and build our wrapper SET (Torch_DIR ${depsAbs}/libtorch/share/cmake/Torch) FIND_PACKAGE(Torch REQUIRED) + INCLUDE_DIRECTORIES(util/libtorch_c) ADD_DEFINITIONS(-DREDISMODULE_EXPERIMENTAL_API) -ADD_SUBDIRECTORY(util/libtorch_c) +ADD_SUBDIRECTORY(src/libtorch_c) + +#---------------------------------------------------------------------------------------------- + ADD_SUBDIRECTORY(src) ADD_LIBRARY(redisai SHARED $) @@ -66,10 +106,25 @@ IF (APPLE) LINK_FLAGS "-undefined dynamic_lookup") ENDIF() -SET(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install-${DEVICE}) +SET(CMAKE_INSTALL_PREFIX ${installAbs}) INSTALL(TARGETS redisai LIBRARY DESTINATION .) +IF (APPLE) + SET_TARGET_PROPERTIES(redisai PROPERTIES INSTALL_RPATH "@loader_path/lib") +ELSE () + ADD_LDFLAGS(redisai "-Wl,--enable-new-dtags") + SET_TARGET_PROPERTIES(redisai PROPERTIES INSTALL_RPATH "\$ORIGIN/lib") +ENDIF() + +IF (APPLE) + SET(LIB_PATTERN "*.dylib") +ELSE() + SET(LIB_PATTERN "*.so*") +ENDIF() + +#---------------------------------------------------------------------------------------------- + ADD_LIBRARY(redisai_tensorflow SHARED $) TARGET_LINK_LIBRARIES(redisai_tensorflow ${TF_LIBRARIES}) SET_TARGET_PROPERTIES(redisai_tensorflow PROPERTIES PREFIX "") @@ -81,7 +136,10 @@ ELSE () SET_TARGET_PROPERTIES(redisai_tensorflow PROPERTIES INSTALL_RPATH "\$ORIGIN/lib") ENDIF() INSTALL(TARGETS redisai_tensorflow LIBRARY DESTINATION backends/redisai_tensorflow) -INSTALL(DIRECTORY ${depsAbs}/libtensorflow/lib DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/redisai_tensorflow) +INSTALL(DIRECTORY ${depsAbs}/libtensorflow/lib DESTINATION ${installAbs}/backends/redisai_tensorflow + FILES_MATCHING PATTERN ${LIB_PATTERN}) + +#---------------------------------------------------------------------------------------------- ADD_LIBRARY(redisai_torch SHARED $) TARGET_LINK_LIBRARIES(redisai_torch torch_c ${TORCH_LIBRARIES}) @@ -94,9 +152,12 @@ ELSE () SET_TARGET_PROPERTIES(redisai_torch PROPERTIES INSTALL_RPATH "\$ORIGIN/lib") ENDIF() INSTALL(TARGETS redisai_torch LIBRARY DESTINATION backends/redisai_torch) -INSTALL(DIRECTORY ${depsAbs}/libtorch/lib DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/redisai_torch) +INSTALL(DIRECTORY ${depsAbs}/libtorch/lib DESTINATION ${installAbs}/backends/redisai_torch + FILES_MATCHING PATTERN ${LIB_PATTERN}) -IF(${DEVICE} STREQUAL "gpu") +#---------------------------------------------------------------------------------------------- + +IF (${DEVICE} STREQUAL "gpu") ADD_DEFINITIONS(-DRAI_ONNXRUNTIME_USE_CUDA) ENDIF() @@ -111,4 +172,18 @@ ELSE () SET_TARGET_PROPERTIES(redisai_onnxruntime PROPERTIES INSTALL_RPATH "\$ORIGIN/lib") ENDIF() INSTALL(TARGETS redisai_onnxruntime LIBRARY DESTINATION backends/redisai_onnxruntime) -INSTALL(DIRECTORY ${depsAbs}/onnxruntime/lib DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/redisai_onnxruntime) +INSTALL(DIRECTORY ${depsAbs}/onnxruntime/lib DESTINATION ${installAbs}/backends/redisai_onnxruntime + FILES_MATCHING PATTERN ${LIB_PATTERN}) + +#---------------------------------------------------------------------------------------------- + +IF (APPLE) + INSTALL(DIRECTORY ${depsAbs}/mkl/lib DESTINATION ${installAbs}/backends/redisai_torch + FILES_MATCHING PATTERN ${LIB_PATTERN}) +ENDIF() + +#---------------------------------------------------------------------------------------------- + +IF (NOT ${installAbs} STREQUAL ${CMAKE_SOURCE_DIR}/install-${DEVICE}) + INSTALL_SYMLINK(${installAbs} ${CMAKE_SOURCE_DIR}/install-${DEVICE}) +ENDIF() diff --git a/Dockerfile b/Dockerfile index e01c43072..b74fb0a3c 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,38 +1,51 @@ -ARG OS=debian:buster +# BUILD redisfab/redisai-cpu-${OSNICK}:M.m.b-${ARCH} + +# OSNICK=bionic|stretch|buster +ARG OSNICK=buster + +# OS=debian:buster-slim|debian:stretch-slim|ubuntu:bionic +ARG OS=debian:buster-slim + +# ARCH=x64|arm64v8|arm32v7 +ARG ARCH=x64 #---------------------------------------------------------------------------------------------- -FROM redis AS redis +FROM redisfab/redis-${ARCH}-${OSNICK}:5.0.5 AS redis FROM ${OS} AS builder -WORKDIR /redisai +WORKDIR /build COPY --from=redis /usr/local/ /usr/local/ -COPY ./automation/ automation/ +COPY ./opt/ opt/ COPY ./test/test_requirements.txt test/ -RUN ./automation/readies/bin/getpy -RUN ./automation/system-setup.py +RUN ./opt/readies/bin/getpy +RUN ./opt/system-setup.py COPY ./get_deps.sh . RUN ./get_deps.sh cpu -ADD ./ /redisai -RUN make -C automation all +ADD ./ /build +RUN make -C opt build SHOW=1 ARG PACK=0 ARG TEST=0 -RUN if [ "$PACK" = "1" ]; then make -C automation pack; fi -RUN if [ "$TEST" = "1" ]; then make -C automation test; fi +RUN if [ "$PACK" = "1" ]; then make -C opt pack; fi +RUN if [ "$TEST" = "1" ]; then make -C opt test; fi #---------------------------------------------------------------------------------------------- -FROM redis +# FROM redis:latest +FROM redisfab/redis-${ARCH}-${OSNICK}:5.0.5 + +RUN set -e; apt-get -qq update; apt-get -q install -y libgomp1 -RUN set -e; apt-get -qq update; apt-get install -y libgomp1 +ENV REDIS_MODULES /usr/lib/redis/modules +ENV LD_LIBRARY_PATH $REDIS_MODULES -RUN mkdir -p /usr/lib/redis/modules/ +RUN mkdir -p $REDIS_MODULES/ -COPY --from=builder /redisai/install-cpu/ /usr/lib/redis/modules/ +COPY --from=builder /build/install-cpu/ $REDIS_MODULES/ WORKDIR /data EXPOSE 6379 diff --git a/Dockerfile-gpu b/Dockerfile-gpu index ff975e904..b9b9a0441 100755 --- a/Dockerfile-gpu +++ b/Dockerfile-gpu @@ -48,8 +48,7 @@ RUN set -ex;\ # Package the runner FROM builder -RUN set -ex;\ - mkdir -p /usr/lib/redis/modules/; +RUN mkdir -p /usr/lib/redis/modules/ COPY --from=builder /redisai/install-gpu/ /usr/lib/redis/modules/ diff --git a/Dockerfile.arm b/Dockerfile.arm new file mode 100755 index 000000000..681f5aaf3 --- /dev/null +++ b/Dockerfile.arm @@ -0,0 +1,58 @@ +# BUILD redisfab/redisai-cpu-${OSNICK}:M.m.b-${ARCH} + +# OSNICK=bionic|stretch|buster +ARG OSNICK=buster + +# OS=debian:buster-slim|debian:stretch-slim|ubuntu:bionic +OS=debian:buster-slim + +# ARCH=arm64v8|arm32v7 +ARG ARCH=arm64v8 + +#---------------------------------------------------------------------------------------------- +FROM redisfab/redis-${ARCH}-${OSNICK}-xbuild:5.0.5 AS builder + +RUN [ "cross-build-start" ] + +ADD ./ /build +WORKDIR /build + +COPY ./opt/ opt/ +COPY ./test/test_requirements.txt test/ + +RUN ./opt/readies/bin/getpy +RUN ./opt/system-setup.py + +COPY ./get_deps.sh . +RUN ./get_deps.sh cpu + +ADD ./ /redisai +RUN make -C opt all SHOW=1 + +ARG PACK=0 +ARG TEST=0 + +RUN if [ "$PACK" = "1" ]; then make -C opt pack; fi +RUN if [ "$TEST" = "1" ]; then make -C opt test; fi + +RUN [ "cross-build-end" ] + +#---------------------------------------------------------------------------------------------- +FROM redisfab/redis-${ARCH}-${OSNICK}-xbuild:5.0.5 + +RUN [ "cross-build-start" ] + +RUN set -e; apt-get -qq update; apt-get -q install -y libgomp1 + +ENV REDIS_MODULES /usr/lib/redis/modules +ENV LD_LIBRARY_PATH $REDIS_MODULES + +RUN mkdir -p $REDIS_MODULES/ + +COPY --from=builder /build/install-cpu/ $REDIS_MODULES/ + +WORKDIR /data +EXPOSE 6379 +CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so"] + +RUN [ "cross-build-end" ] diff --git a/README.md b/README.md index 960116e78..c2ddb65cb 100644 --- a/README.md +++ b/README.md @@ -80,8 +80,8 @@ Once the dependencies are downloaded, build the module itself. Note that CMake 3.0 or higher is required. ```sh -mkdir build -cd build +mkdir bin +cd bin cmake .. make && make install cd .. diff --git a/automation/Makefile b/automation/Makefile deleted file mode 100755 index f4398ce9c..000000000 --- a/automation/Makefile +++ /dev/null @@ -1,103 +0,0 @@ - -ROOT:=.. - -override GPU:=$(or $(findstring $(CUDA),1),$(findstring $(GPU),1)) - -ifeq ($(GPU),1) -ifeq ($(CPU),1) -$(error CPU=1 and GPU=1 (or CUDA=1) are conflicting) -endif -DEPS_FLAGS=gpu -DEVICE=gpu -else -DEPS_FLAGS=cpu -DEVICE=cpu -endif - -GIT_BRANCH:=$(shell git rev-parse --abbrev-ref HEAD) -GIT_COMMIT:=$(shell git describe --always --abbrev=7 --dirty="+") - -ifeq ($(VERSION),) -PACK_VER:=$(GIT_BRANCH)-$(GIT_COMMIT) -else -PACK_VER:=$(VERSION) -endif - -BINDIR=$(PWD)/build -INSTALL_DIR=$(PWD)/install-$(DEVICE) - -BACKENDS_PATH ?= $(INSTALL_DIR)/backends - -RAMP:=ramp - -ifeq ($(DEBUG),1) -CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=Debug -endif - -#---------------------------------------------------------------------------------------------- - -.PHONY: all setup build clean deps pack pack_ramp pack_deps test - -all: build - -build: -ifeq ($(wildcard build/Makefile),) - mkdir -p $(ROOT)/build - cd $(ROOT)/build; \ - cmake -DDEVICE=$(DEVICE) -DDEPS_PATH=$(ROOT)/deps/install $(CMAKE_FLAGS) $(ROOT) -endif - $(MAKE) -C $(ROOT)/build - $(MAKE) -C $(ROOT)/build install - cd $(ROOT) ;\ - [ ! -e install ] && ln -sf install-$(DEVICE) install - -clean: -ifeq ($(ALL),1) - cd $(ROOT) ;\ - rm -rf build install deps/dlpack deps/install-$(DEVICE) deps/*.tar.gz deps/*.zip deps/*.tgz -else - $(MAKE) -C $(ROOT)/build clean -endif - -deps fetch: - @echo Fetching dependencies... - $(ROOT)/get_deps.sh $(DEPS_FLAGS) - -pack: pack_ramp pack_deps - -pack_ramp: - @[ ! -z `command -v redis-server` ] || { echo "Cannot find redis-server - aborting."; exit 1; } - @echo "Building RAMP file ..." - @set -e ;\ - cd $(ROOT) ;\ - RAMPOUT=$$(mktemp /tmp/ramp.XXXXXX) ;\ - $(RAMP) pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" $(INSTALL_DIR)/redisai.so 2> /dev/null | grep '.zip' > $$RAMPOUT ;\ - tail -1 $$RAMPOUT > $(BINDIR)/PACKAGE ;\ - cat $(BINDIR)/PACKAGE | sed -e "s/[^.]*\.[^.]*\.\(.*\)\.zip/\1/" > $(BINDIR)/VERSION ;\ - VERSION=$$(cat $(BINDIR)/VERSION) ;\ - $(RAMP) pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" \ - -c "BACKENDSPATH /opt/redislabs/lib/redisai-cpu-$$VERSION/backends" $(INSTALL_DIR)/redisai.so 2> /dev/null | grep '.zip' > $$RAMPOUT ;\ - rm -f $$RAMPOUT ;\ - echo "Done." - -pack_deps: pack_ramp - @echo "Building dependencies file ..." - @set -e ;\ - cd $(ROOT) ;\ - PACK_FNAME=$$(basename `cat $(BINDIR)/PACKAGE`) ;\ - ARCHOSVER=$$(echo "$$PACK_FNAME" | sed -e "s/^redisai\.\([^.]*\..*\)\.zip/\1/") ;\ - VERSION=$$(cat $(BINDIR)/VERSION) ;\ - cd install-$(DEVICE) ;\ - rm -rf redisai-$(DEVICE)-$$VERSION ;\ - mkdir redisai-$(DEVICE)-$$VERSION ;\ - mv backends redisai-$(DEVICE)-$$VERSION ;\ - ln -sf redisai-$(DEVICE)-$$VERSION/backends backends ;\ - find redisai-$(DEVICE)-$$VERSION -name "*.so*" | xargs tar pczf redisai-$(DEVICE)-dependencies.$$ARCHOSVER.tgz ;\ - echo "Done." - -test: - @git lfs install - @git lfs pull - @set -e ;\ - cd $(ROOT)/test ;\ - python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so diff --git a/automation/readies/mk/common.rules b/automation/readies/mk/common.rules deleted file mode 100755 index 633cf2f6b..000000000 --- a/automation/readies/mk/common.rules +++ /dev/null @@ -1,6 +0,0 @@ - -.PHONY: __sep - -__sep: ; -# @python -c "$(__SEP)" - diff --git a/automation/readies/mk/defs b/automation/readies/mk/defs deleted file mode 100755 index 819d0d30c..000000000 --- a/automation/readies/mk/defs +++ /dev/null @@ -1,6 +0,0 @@ - -MK=$(ROOT)/automation/readies/mk - -include $(MK)/common.defs -include $(MK)/variant.defs -include $(MK)/bindirs.defs diff --git a/automation/readies/mk/rules b/automation/readies/mk/rules deleted file mode 100755 index 1aa2985e3..000000000 --- a/automation/readies/mk/rules +++ /dev/null @@ -1,4 +0,0 @@ - -include $(MK)/common.rules -include $(MK)/variant.rules -include $(MK)/bindirs.rules diff --git a/get_deps.sh b/get_deps.sh index 36c2160f2..ba5b3c717 100755 --- a/get_deps.sh +++ b/get_deps.sh @@ -1,184 +1,240 @@ #!/usr/bin/env bash +error() { + echo "There are errors." + exit 1 +} + +trap error ERR + HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +if [[ $1 == --help || $1 == help ]]; then + cat <<-END + get_deps.sh [cpu|gpu] [--help|help] + + Argument variables: + VERBOSE=1 Print commands + FORCE=1 Download even if present + + END + exit 0 +fi + set -e [[ $VERBOSE == 1 ]] && set -x if [[ "$1" == "cpu" ]]; then - GPU=no - DEVICE=cpu + GPU=no + DEVICE=cpu elif [[ "$1" == "gpu" ]]; then - GPU=yes - DEVICE=gpu + GPU=yes + DEVICE=gpu else - GPU=${GPU:-no} - if [[ $GPU == 1 ]]; then - DEVICE=gpu - else - DEVICE=cpu - fi + GPU=${GPU:-no} + if [[ $GPU == 1 ]]; then + DEVICE=gpu + else + DEVICE=cpu + fi fi -DEPS_DIR=$HERE/deps +OS=$(python3 $HERE/opt/readies/bin/platform --os) +ARCH=$(python3 $HERE/opt/readies/bin/platform --arch) + +# avoid wget warnings on macOS +[[ $OS == macosx ]] && export LC_ALL=en_US.UTF-8 + +DEPS_DIR=$HERE/deps/$OS-$ARCH-$DEVICE mkdir -p ${DEPS_DIR} cd ${DEPS_DIR} -PREFIX=${DEPS_DIR}/install -mkdir -p ${PREFIX} +# PREFIX=${DEPS_DIR}/install +# mkdir -p ${PREFIX} -DLPACK_PREFIX=${PREFIX}/dlpack -TF_PREFIX=${PREFIX}/libtensorflow -TORCH_PREFIX=${PREFIX}/libtorch -ORT_PREFIX=${PREFIX}/onnxruntime +DLPACK=dlpack +LIBTENSORFLOW=libtensorflow +LIBTORCH=libtorch +MKL=mkl +ONNXRUNTIME=onnxruntime -## DLPACK +######################################################################################## DLPACK -[[ $FORCE == 1 ]] && rm -rf ${DLPACK_PREFIX} +[[ $FORCE == 1 ]] && rm -rf $DLPACK -if [[ ! -d dlpack ]]; then - echo "Cloning dlpack ..." - git clone --depth 1 https://github.com/dmlc/dlpack.git - echo "Done." +if [[ ! -d $DLPACK ]]; then + echo "Cloning dlpack ..." + git clone --depth 1 https://github.com/dmlc/dlpack.git $DLPACK + echo "Done." else - echo "dlpack is in place." -fi - -if [[ ! -d ${DLPACK_PREFIX}/include ]]; then - mkdir -p ${DLPACK_PREFIX} - ln -sf ${DEPS_DIR}/dlpack/include ${DLPACK_PREFIX}/include + echo "dlpack is in place." fi -## TENSORFLOW +################################################################################# LIBTENSORFLOW TF_VERSION="1.14.0" -[[ $FORCE == 1 ]] && rm -rf ${TF_PREFIX} - -if [[ ! -d ${TF_PREFIX} ]]; then - echo "Installing TensorFlow ..." - - mkdir -p ${TF_PREFIX} - - if [[ "$OSTYPE" == "linux-gnu" ]]; then - TF_OS="linux" - if [[ "$1" == "cpu" ]]; then - TF_BUILD="cpu" - else - TF_BUILD="gpu" - fi - elif [[ "$OSTYPE" == "darwin"* ]]; then - TF_OS="darwin" - TF_BUILD="cpu" - fi - - LIBTF_ARCHIVE=libtensorflow-${TF_BUILD}-${TF_OS}-x86_64-${TF_VERSION}.tar.gz - - if [ ! -e ${LIBTF_ARCHIVE} ]; then - echo "Downloading libtensorflow ${TF_VERSION} ${TF_BUILD}" - wget -q https://storage.googleapis.com/tensorflow/libtensorflow/${LIBTF_ARCHIVE} - fi - - tar xf ${LIBTF_ARCHIVE} --no-same-owner --strip-components=1 -C ${TF_PREFIX} - - echo "Done." +[[ $FORCE == 1 ]] && rm -rf $LIBTENSORFLOW + +if [[ ! -d $LIBTENSORFLOW ]]; then + echo "Installing TensorFlow ..." + + if [[ $OS == linux ]]; then + TF_OS="linux" + if [[ $GPU == no ]]; then + TF_BUILD="cpu" + else + TF_BUILD="gpu" + fi + if [[ $ARCH == x64 ]]; then + TF_VERSION=1.14.0 + TF_ARCH=x86_64 + LIBTF_URL_BASE=https://storage.googleapis.com/tensorflow/libtensorflow + elif [[ $ARCH == arm64v8 ]]; then + TF_VERSION=1.14.0 + TF_ARCH=arm64 + LIBTF_URL_BASE=https://s3.amazonaws.com/redismodules/tensorflow + elif [[ $ARCH == arm32v7 ]]; then + TF_VERSION=1.14.0 + TF_ARCH=arm + LIBTF_URL_BASE=https://s3.amazonaws.com/redismodules/tensorflow + fi + elif [[ $OS == macosx ]]; then + TF_VERSION=1.14.0 + TF_OS=darwin + TF_BUILD=cpu + TF_ARCH=x86_64 + LIBTF_URL_BASE=https://storage.googleapis.com/tensorflow/libtensorflow + fi + + LIBTF_ARCHIVE=libtensorflow-${TF_BUILD}-${TF_OS}-${TF_ARCH}-${TF_VERSION}.tar.gz + + [[ ! -f $LIBTF_ARCHIVE || $FORCE == 1 ]] && wget --quiet $LIBTF_URL_BASE/$LIBTF_ARCHIVE + + rm -rf $LIBTENSORFLOW.x + mkdir $LIBTENSORFLOW.x + tar xf $LIBTF_ARCHIVE --no-same-owner --strip-components=1 -C $LIBTENSORFLOW.x + mv $LIBTENSORFLOW.x $LIBTENSORFLOW + + echo "Done." else - echo "TensorFlow is in place." + echo "TensorFlow is in place." fi -## PYTORCH +###################################################################################### LIBTORCH PT_VERSION="1.2.0" -[[ $FORCE == 1 ]] && rm -rf ${TORCH_PREFIX} - -if [[ ! -d ${TORCH_PREFIX} ]]; then - echo "Installing libtorch ..." - - mkdir -p ${TORCH_PREFIX} - - if [[ "$OSTYPE" == "linux-gnu" ]]; then - PT_OS="shared-with-deps" - if [[ "$1" == "cpu" ]]; then - PT_BUILD="cpu" - else - PT_BUILD="cu100" - fi - elif [[ "$OSTYPE" == "darwin"* ]]; then - PT_OS="macos" - PT_BUILD="cpu" - fi - - if [[ "$PT_VERSION" == "latest" ]]; then - PT_BUILD=nightly/${PT_BUILD} - fi - - # Where to get the archive - LIBTORCH_URL=https://download.pytorch.org/libtorch/${PT_BUILD}/libtorch-${PT_OS}-${PT_VERSION}.zip - # Directory where torch is extracted to - LIBTORCH_DIRECTORY=libtorch-${PT_OS}-${PT_VERSION} - - # Archive - specifically named - LIBTORCH_ARCHIVE=libtorch-${PT_OS}-${PT_BUILD}-${PT_VERSION}.zip - - if [ ! -e "${LIBTORCH_ARCHIVE}" ]; then - echo "Downloading libtorch ${PT_VERSION} ${PT_BUILD}" - curl -s -L ${LIBTORCH_URL} > ${LIBTORCH_ARCHIVE} - fi - - unzip -q -o ${LIBTORCH_ARCHIVE} -d ${TORCH_PREFIX}/../ - - echo "Done." - - if [[ "${PT_OS}" == "macos" ]]; then - echo "Installing MKL ..." - # also download mkl - MKL_BUNDLE=mklml_mac_2019.0.3.20190220 - if [ ! -e "${MKL_BUNDLE}.tgz" ]; then - wget -q "https://github.com/intel/mkl-dnn/releases/download/v0.18/${MKL_BUNDLE}.tgz" - fi - tar xzf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${TORCH_PREFIX} - mkdir -p ${ORT_PREFIX} - tar xzf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${ORT_PREFIX} - echo "Done." - fi +[[ $FORCE == 1 ]] && rm -rf $LIBTORCH + +if [[ ! -d $LIBTORCH ]]; then + echo "Installing libtorch ..." + + if [[ $OS == linux ]]; then + PT_OS=linux + if [[ $GPU == no ]]; then + PT_BUILD=cpu + else + PT_BUILD=cu100 + fi + if [[ $ARCH == x64 ]]; then + PT_ARCH=x86_64 + elif [[ $ARCH == arm64v8 ]]; then + PT_ARCH=arm64 + elif [[ $ARCH == arm32v7 ]]; then + PT_ARCH=arm + fi + elif [[ $OS == macosx ]]; then + PT_OS=macos + PT_ARCH=x86_64 + PT_BUILD=cpu + fi + + [[ "$PT_VERSION" == "latest" ]] && PT_BUILD=nightly/${PT_BUILD} + + LIBTORCH_ARCHIVE=libtorch-${PT_BUILD}-${PT_OS}-${PT_ARCH}-${PT_VERSION}.tar.gz + LIBTORCH_URL=https://s3.amazonaws.com/redismodules/pytorch/$LIBTORCH_ARCHIVE + + [[ ! -f $LIBTORCH_ARCHIVE || $FORCE == 1 ]] && wget -q $LIBTORCH_URL + + rm -rf $LIBTORCH.x + mkdir $LIBTORCH.x + + tar xf $LIBTORCH_ARCHIVE --no-same-owner -C $LIBTORCH.x + mv $LIBTORCH.x/libtorch $LIBTORCH + rmdir $LIBTORCH.x + + echo "Done." else - echo "librotch is in place." + echo "librotch is in place." fi -## ONNXRUNTIME -ORT_VERSION="0.5.0" -ORT_ARCH="x64" -if [[ "$OSTYPE" == "linux-gnu" ]]; then - if [[ "$1" == "cpu" ]]; then - ORT_OS="linux" - ORT_BUILD="" - else - ORT_OS="linux" - ORT_BUILD="-gpu" - fi -elif [[ "$OSTYPE" == "darwin"* ]]; then - ORT_OS="osx" - ORT_BUILD="" -fi +########################################################################################### MKL -[[ $FORCE == 1 ]] && rm -rf ${ORT_PREFIX} +if [[ ! -d mkl ]]; then + MKL_VERSION=0.18 + MKL_BUNDLE_VER=2019.0.3.20190220 + if [[ $OS == macosx ]]; then + echo "Installing MKL ..." -ORT_ARCHIVE=onnxruntime-${ORT_OS}-${ORT_ARCH}${ORT_BUILD}-${ORT_VERSION}.tgz + MKL_OS=mac + MKL_ARCHIVE=mklml_${MKL_OS}_${MKL_BUNDLE_VER}.tgz + [[ ! -e ${MKL_ARCHIVE} ]] && wget -q https://github.com/intel/mkl-dnn/releases/download/v${MKL_VERSION}/${MKL_ARCHIVE} -if [[ ! -d ${ORT_PREFIX} ]]; then - echo "Installing ONNXRuntime ..." + rm -rf $MKL.x + mkdir $MKL.x + tar xzf ${MKL_ARCHIVE} --no-same-owner --strip-components=1 -C $MKL.x + mv $MKL.x $MKL - mkdir -p ${ORT_PREFIX} - - if [ ! -e ${ORT_ARCHIVE} ]; then - echo "Downloading ONNXRuntime ${ORT_VERSION} ${ORT_BUILD}" - wget -q https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION}/${ORT_ARCHIVE} - fi - - tar xf ${ORT_ARCHIVE} --no-same-owner --strip-components=1 -C ${ORT_PREFIX} + echo "Done." + fi else - echo "onnxruntime is in place." + echo "mkl is in place." fi -echo "Done." +################################################################################### ONNXRUNTIME + +ORT_VERSION="0.5.0" + +[[ $FORCE == 1 ]] && rm -rf $ONNXRUNTIME + +if [[ ! -d $ONNXRUNTIME ]]; then + echo "Installing ONNXRuntime ..." + + if [[ $OS == linux ]]; then + ORT_OS=linux + if [[ $GPU == no ]]; then + ORT_BUILD="" + else + ORT_BUILD="-gpu" + fi + if [[ $ARCH == x64 ]]; then + ORT_ARCH=x64 + ORT_URL_BASE=https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION} + elif [[ $ARCH == arm64v8 ]]; then + ORT_ARCH=arm64 + ORT_URL_BASE=https://s3.amazonaws.com/redismodules/onnxruntime + elif [[ $ARCH == arm32v7 ]]; then + ORT_ARCH=arm + ORT_URL_BASE=https://s3.amazonaws.com/redismodules/onnxruntime + fi + elif [[ $OS == macosx ]]; then + ORT_OS=osx + ORT_ARCH=x64 + ORT_BUILD="" + ORT_URL_BASE=https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION} + fi + + ORT_ARCHIVE=onnxruntime-${ORT_OS}-${ORT_ARCH}${ORT_BUILD}-${ORT_VERSION}.tgz + + [[ ! -e ${ORT_ARCHIVE} ]] && wget -q $ORT_URL_BASE/${ORT_ARCHIVE} + + rm -rf $ONNXRUNTIME.x + mkdir $ONNXRUNTIME.x + tar xzf ${ORT_ARCHIVE} --no-same-owner --strip-components=1 -C $ONNXRUNTIME.x + mv $ONNXRUNTIME.x $ONNXRUNTIME + + echo "Done." +else + echo "ONNXRuntime is in place." +fi diff --git a/opt/Makefile b/opt/Makefile new file mode 100755 index 000000000..3f2009062 --- /dev/null +++ b/opt/Makefile @@ -0,0 +1,113 @@ + +ROOT:=.. +include readies/mk/main + +MK_CMAKE:=1 +MK_CMAKE_INSTALL:=1 + +define HELP +make setup # install prerequisited (CAUTION: THIS WILL MODIFY YOUR SYSTEM) +make fetch # download and prepare dependant modules +make build # compile and link +make clean # remove build artifacts + ALL=1 # remove entire artifacts directory +make test # run tests +make pack # create installation packages + PACK_DEPS=0 # do not pack dependencies +make deploy # copy packages to S3 +make release # release a version +endef + +#---------------------------------------------------------------------------------------------- + +override GPU:=$(or $(findstring $(CUDA),1),$(findstring $(GPU),1)) + +ifeq ($(GPU),1) +ifeq ($(CPU),1) +$(error CPU=1 and GPU=1 (or CUDA=1) are conflicting) +endif +DEPS_FLAGS=gpu +DEVICE=gpu +else +DEPS_FLAGS=cpu +DEVICE=cpu +endif + +SRCDIR=.. +BINDIR=$(BINROOT)/src +# INSTALL_DIR=$(ROOT)/install-$(DEVICE) +DEPS_DIR=$(ROOT)/deps/$(OS)-$(ARCH)-$(DEVICE) +INSTALL_DIR=$(ROOT)/bin/$(OS)-$(ARCH)-$(DEVICE)/install + +TARGET=$(BINDIR)/redisai.so + +BACKENDS_PATH ?= $(INSTALL_DIR)/backends + +CMAKE_FLAGS += \ + -DDEPS_PATH=$(abspath $(DEPS_DIR)) \ + -DINSTALL_PATH=$(abspath $(INSTALL_DIR)) \ + -DDEVICE=$(DEVICE) + +include $(MK)/defs + +#---------------------------------------------------------------------------------------------- + +.PHONY: deps fetch pack pack_ramp pack_deps test + +include $(MK)/rules + +#---------------------------------------------------------------------------------------------- + +prebuild: + $(SHOW)if [ ! -d $(DEPS_DIR) ]; then echo $$'Dependencies are not in place.\nPlease run \'make fetch\'.'; exit 1; fi + +$(TARGET): prebuild $(MK_MAKEFILES) $(DEPS) + $(SHOW)mkdir -p $(INSTALL_DIR) + $(SHOW)$(MAKE) -C $(BINDIR) + $(SHOW)$(MAKE) -C $(BINDIR) install +# $(SHOW)cd $(ROOT) ;\ +# if [ ! -e install ]; then ln -sf install-$(DEVICE) install; fi + +install: + $(SHOW)mkdir -p $(INSTALL_DIR) + $(SHOW)$(MAKE) -C $(BINDIR) install + +clean: +ifeq ($(ALL),1) + $(SHOW)if [ -d "$(BINROOT)" ]; then rm -rf $(BINROOT); fi + $(SHOW)if [ -d "$(INSTALL_DIR)" ]; then rm -rf $(INSTALL_DIR); fi + $(SHOW)rm -f $(ROOT)/install-$(DEVICE) +else + -$(SHOW)$(MAKE) -C $(BINDIR) clean +endif + +#---------------------------------------------------------------------------------------------- + +fetch deps: + @echo Fetching dependencies... + $(SHOW)VERBOSE=$(_SHOW) $(ROOT)/get_deps.sh $(DEPS_FLAGS) + +#---------------------------------------------------------------------------------------------- + +pack: +ifneq ($(PACK_DEPS),0) + $(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) DEPS=1 ./pack.sh +else + $(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) DEPS=0 ./pack.sh +endif + +#---------------------------------------------------------------------------------------------- + +TEST_REPORT_DIR ?= $(PWD) + +test: + $(SHOW)if [ "$(git lfs env > /dev/null 2>&1 ; echo $?)" != "0" ]; then cd $(ROOT); git lfs install; fi + $(SHOW)cd $(ROOT); git lfs pull + $(SHOW)set -e ;\ + cd $(ROOT)/test ;\ + python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so + +#---------------------------------------------------------------------------------------------- + +docker: + $(SHOW)docker build -t redisai --build-arg TEST=1 --build-arg PACK=1 .. diff --git a/opt/build/bazel/Dockerfile b/opt/build/bazel/Dockerfile new file mode 100755 index 000000000..7748006b7 --- /dev/null +++ b/opt/build/bazel/Dockerfile @@ -0,0 +1,32 @@ +# BUILD redisfab/bazel-${ARCH}:$(BAZ_VER) + +ARG OSNICK=buster +ARG ARCH=arm64v8 + +FROM redisfab/${ARCH}-xbuild:${OSNICK} + +RUN [ "cross-build-start" ] + +ARG BAZ_VER=0.24.1 + +WORKDIR /build + +RUN set -e ;\ + apt-get -qq update ;\ + apt-get -q install -y pkg-config g++ zlib1g-dev zip unzip python wget ;\ + apt-get -q install -y openjdk-11-jdk + +RUN set -e ;\ + wget -q https://github.com/bazelbuild/bazel/releases/download/0.24.1/bazel-0.24.1-dist.zip ;\ + unzip bazel-0.24.1-dist.zip + +RUN set -e ;\ + env EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" bash ./compile.sh ;\ + copy bazel to /usr/local/bin + +RUN [ "cross-build-end" ] + +# FROM insready/bazel:latest as baz +# FROM ubuntu:latest +# COPY --from=baz /usr/bin/bazel* /usr/local/bin/ + diff --git a/opt/build/docker/Makefile b/opt/build/docker/Makefile new file mode 100755 index 000000000..2af3163f6 --- /dev/null +++ b/opt/build/docker/Makefile @@ -0,0 +1,99 @@ + +.NOTPARALLEL: + +ROOT=../.. + +ifeq ($(VERSION),) +VERSION:=$(patsubst v%,%,$(shell git describe --tags `git rev-list --tags --max-count=1`)) +endif +ifeq ($(VERSION),) +$(error Cannot determine version. Aborting.) +endif + +# OSNICK=buster|stretch|bionic +OSNICK ?= buster + +REPO=redisfab +STEM=$(REPO)/redisai-cpu +BUILD_OPT=--rm +# --squash + +#---------------------------------------------------------------------------------------------- + +define targets # (1=OP, 2=op) +$(1)_TARGETS := +$(1)_TARGETS += $(if $(findstring $(X64),1),$(2)_x64) +$(1)_TARGETS += $(if $(findstring $(ARM7),1),$(2)_arm32v7) +$(1)_TARGETS += $(if $(findstring $(ARM8),1),$(2)_arm64v8) + +$(1)_TARGETS += $$(if $$(strip $$($(1)_TARGETS)),,$(2)_x64 $(2)_arm32v7 $(2)_arm64v8) +endef + +$(eval $(call targets,BUILD,build)) +$(eval $(call targets,PUSH,push)) + +#---------------------------------------------------------------------------------------------- + +define build_x64 # (1=arch) +build_$(1): + @docker build $(BUILD_OPT) -t $(STEM)-$(OSNICK):$(VERSION)-x64 -f $(ROOT)/Dockerfile $(ROOT) + +.PHONY: build_$(1) +endef + +define build_arm # (1=arch) +build_$(1): + @docker build $(BUILD_OPT) -t $(STEM)-$(OSNICK):$(VERSION)-$(1) -f $(ROOT)/Dockerfile.arm \ + --build-arg ARCH=$(1) $(ROOT) + +.PHONY: build_$(1) +endef + +define push # (1=arch) +push_$(1): + @docker push $(STEM)-$(OSNICK):$(VERSION)-$(1) + +.PHONY: push_$(1) +endef + +define create_manifest # (1=version) +docker manifest create -a $(STEM)-$(OSNICK):$(1) \ + -a $(STEM)-$(OSNICK):$(VERSION)-x64 \ + -a $(STEM)-$(OSNICK):$(VERSION)-arm64v8 +# -a $(STEM)-$(OSNICK):$(VERSION)-arm32v7 +# docker manifest annotate $(STEM)-$(OSNICK):$(1) $(STEM)-$(OSNICK):$(VERSION)-arm32v7 --os linux --arch arm --variant v7 +docker manifest annotate $(STEM)-$(OSNICK):$(1) $(STEM)-$(OSNICK):$(VERSION)-arm64v8 --os linux --arch arm64 --variant v8 +docker manifest push -p $(STEM)-$(OSNICK):$(1) +endef + +#---------------------------------------------------------------------------------------------- + +all: build publish + +$(eval $(call build_x64,x64)) +$(eval $(call build_arm,arm64v8)) +$(eval $(call build_arm,arm32v7)) + +$(eval $(call push,x64)) +$(eval $(call push,arm64v8)) +$(eval $(call push,arm32v7)) + +build: $(BUILD_TARGETS) + +ifneq ($(PUSH),0) +publish: $(PUSH_TARGETS) +else +publish: +endif + $(call create_manifest,$(VERSION)) + $(call create_manifest,latest) + +show: + @echo "$(STEM)-$(OSNICK):" +ifeq ($(INSPECT),1) + @docker manifest inspect $(STEM)-$(OSNICK):$(VERSION) | jq +else + @curl -s -X "GET" https://cloud.docker.com/v2/repositories/$(STEM)-$(OSNICK)/tags/ | jq +endif + +.PHONY: build public show diff --git a/opt/build/libtorch/Dockerfile.arm b/opt/build/libtorch/Dockerfile.arm new file mode 100755 index 000000000..d5c00c536 --- /dev/null +++ b/opt/build/libtorch/Dockerfile.arm @@ -0,0 +1,52 @@ +# BUILD redisfab/libtorch-cpu-${ARCH}:$(PT_VER) + +# stretch|bionic|buster +ARG OSNICK=buster + +# arm32v7|arm64v8 +ARG ARCH=arm64v8 + +#---------------------------------------------------------------------------------------------- +FROM redisfab/${ARCH}-xbuild:${OSNICK} as builder + +ARG PT_VER=1.1.0 + +RUN [ "cross-build-start" ] + +WORKDIR /build + +RUN set -e ;\ + apt-get -qq update ;\ + apt-get -q install -y git build-essential ninja-build cmake python3-pip python3-cffi +RUN pip3 install setuptools pyyaml typing +RUN pip3 install numpy +# RUN pip3 install mkl mkl-include + +RUN set -e ;\ + git clone https://github.com/pytorch/pytorch.git ;\ + cd pytorch ;\ + git checkout "v${PT_VER}" ;\ + git submodule update --init --recursive + +ENV BUILD_PYTHON=0 +ENV USE_GLOO=1 +ENV USE_OPENCV=0 +ENV BUILD_TORCH=ON +ENV BUILD_BINARY=ON +ENV BUILD_CAFFE2_OPS=ON +ENV NO_CUDA=1 +ENV NO_DISTRIBUTED=1 +ENV NO_MKLDNN=1 +ENV NO_NNPACK=1 +ENV NO_QNNPACK=1 + +RUN set -e ;\ + cd pytorch ;\ + python3 setup.py install + +ADD ./readies/ /build/deps/readies/ +ADD ./build/libtorch-arm/collect.py /build/ + +RUN ./collect.py + +RUN [ "cross-build-end" ] diff --git a/opt/build/libtorch/Dockerfile.x64 b/opt/build/libtorch/Dockerfile.x64 new file mode 100755 index 000000000..7372a8246 --- /dev/null +++ b/opt/build/libtorch/Dockerfile.x64 @@ -0,0 +1,42 @@ +ARG OS=debian:buster + +#---------------------------------------------------------------------------------------------- +FROM ${OS} + +ARG PT_VER=1.1.0 + +WORKDIR /build + +RUN set -e ;\ + apt-get -qq update ;\ + apt-get -qq install -y git build-essential ninja-build cmake python3-pip python3-cffi +RUN pip3 install setuptools pyyaml typing +RUN pip3 install numpy +RUN pip3 install mkl mkl-include + +RUN set -e ;\ + git clone https://github.com/pytorch/pytorch.git ;\ + cd pytorch ;\ + git checkout "v${PT_VER}" ;\ + git submodule update --init --recursive + +ENV BUILD_PYTHON=0 +ENV USE_GLOO=1 +ENV USE_OPENCV=0 +ENV BUILD_TORCH=ON +ENV BUILD_BINARY=ON +ENV BUILD_CAFFE2_OPS=ON +ENV NO_CUDA=1 +ENV NO_DISTRIBUTED=1 +ENV NO_MKLDNN=1 +ENV NO_NNPACK=1 +ENV NO_QNNPACK=1 + +RUN set -e ;\ + cd pytorch ;\ + python3 setup.py install + +ADD ./opt/readies/ /build/deps/readies/ +ADD ./build/libtorch-arm/collect.py /build/ + +RUN ./collect.py diff --git a/opt/build/libtorch/Makefile b/opt/build/libtorch/Makefile new file mode 100755 index 000000000..bb80891a8 --- /dev/null +++ b/opt/build/libtorch/Makefile @@ -0,0 +1,105 @@ + +ROOT=../.. + +VERSION ?= 1.2.0 +OSNICK ?= buster + +#---------------------------------------------------------------------------------------------- + +S3_URL=redismodules/pytorch + +OS:=$(shell $(ROOT)/opt/readies/bin/platform --os) +ifeq ($(OS),macosx) +OS:=macos +endif +STEM=libtorch-cpu-$(OS) + +DOCKER_OS.bionic=ubuntu:bionic +DOCKER_OS.stretch=debian:stretch-slim +DOCKER_OS.buster=debian:buster-slim +DOCKER_OS=$(DOCKER_OS.$(OSNICK)) + +#---------------------------------------------------------------------------------------------- + +define targets # (1=OP, 2=op) +$(1)_TARGETS := +$(1)_TARGETS += $(if $(findstring $(X64),1),$(2)_x64) +$(1)_TARGETS += $(if $(findstring $(ARM7),1),$(2)_arm32v7) +$(1)_TARGETS += $(if $(findstring $(ARM8),1),$(2)_arm64v8) + +$(1)_TARGETS += $$(if $$(strip $$($(1)_TARGETS)),,$(2)_x64 $(2)_arm32v7 $(2)_arm64v8) +endef + +$(eval $(call targets,BUILD,build)) +$(eval $(call targets,PUBLISH,publish)) + +#---------------------------------------------------------------------------------------------- + +define build_x64 # (1=arch, 2=tar-arch) +build_x64: + @docker build --iidfile $$(IID_$(1)) -t redisfab/$(STEM)-$(1):$(VERSION) -f Dockerfile.x64 \ + --build-arg OS=$(DOCKER_OS) $(ROOT) + @docker create --cidfile $$(CID_$(1)) `cat $$(IID_$(1))` + @docker cp `cat $$(CID_$(1))`:/build/$(STEM)-$(2)-$(VERSION).tar.gz . + +.PHONY: build_x64 +endef + +define build_arm # (1=arch, 2=tar-arch) +IID_$(1)=$(1)_$(VERSION).iid +CID_$(1)=$(1)_$(VERSION).cid + +build_$(1): + @docker build --iidfile $$(IID_$(1)) -t redisfab/$(STEM)-$(1):$(VERSION) -f Dockerfile.arm \ + --build-arg ARCH=$(1) $(ROOT) + @docker create --cidfile $$(CID_$(1)) `cat $$(IID_$(1))` + @docker cp `cat $$(CID_$(1))`:/build/$(STEM)-$(2)-$(VERSION).tar.gz . + +.PHONY: build_$(1) +endef + +#---------------------------------------------------------------------------------------------- + +define publish_x64 # (1=arch, 2=tar-arch) +publish_x64: + @aws s3 cp $(STEM)-$(2)-$(VERSION).tar.gz s3://$(S3_URL)/ --acl public-read + +.PHONY: publish_x64 +endef + +define publish_arm # (1=arch, 2=tar-arch) +publish_$(1): + @aws s3 cp $(STEM)-$(2)-$(VERSION).tar.gz s3://$(S3_URL)/ --acl public-read + +.PHONY: publish_$(1) +endef + +#---------------------------------------------------------------------------------------------- + +all: build publish + +build: $(BUILD_TARGETS) + +$(eval $(call build_x64,x64,x86_64)) +$(eval $(call build_arm,arm64v8,arm64)) +$(eval $(call build_arm,arm32v7,arm)) + +ifneq ($(filter publish,$(MAKECMDGOALS)),) +ifeq ($(wildcard $(HOME)/.aws/credentials),) +$(error Please run 'aws configure' and provide it with access credentials) +endif +endif + +publish: $(PUBLISH_TARGETS) + +$(eval $(call publish_x64,x64,x86_64)) +$(eval $(call publish_arm,arm64v8,arm64)) +$(eval $(call publish_arm,arm32v7,arm)) + +repack: + @PT_VERSION=$(VERSION) ./repack.sh + +help: + @echo "make [build|repack|publish] [X64=1|ARM7=1|ARM8=1]" + +.PHONY: all build repack publish help diff --git a/opt/build/libtorch/collect.py b/opt/build/libtorch/collect.py new file mode 100755 index 000000000..fe6c3c7c3 --- /dev/null +++ b/opt/build/libtorch/collect.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +import os +import sys +import argparse +from pathlib import Path +import shutil +import tarfile + +# this refers to deps directory inside a container +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "deps/readies")) +import paella + +#---------------------------------------------------------------------------------------------- + +PYTORCH_VERSION = '1.2.0' + +parser = argparse.ArgumentParser(description='Prepare RedisAI dependant distribution packages.') +parser.add_argument('--pytorch', default='pytorch', help='root of pytorch repository') +parser.add_argument('--pytorch-ver', default=PYTORCH_VERSION, help='pytorch version') +parser.add_argument('--deps', default='.', help='destination directory') +parser.add_argument('-n', '--nop', action="store_true", help='no operation') +args = parser.parse_args() + +#---------------------------------------------------------------------------------------------- + +pytorch = Path(args.pytorch).resolve() +dest = Path(args.deps).resolve() + +#---------------------------------------------------------------------------------------------- + +pt_build='cpu' + +platform = paella.Platform() + +pt_os = platform.os +if pt_os == 'macosx': + pt_os = 'darwin' + +pt_arch = platform.arch +if pt_arch == 'x64': + pt_arch = 'x86_64' +elif pt_arch == 'arm64v8': + pt_arch = 'arm64' +elif pt_arch == 'arm32v7': + pt_arch = 'arm' + +pt_ver = args.pytorch_ver + +#---------------------------------------------------------------------------------------------- + +def copy_p(src, dest): + f = dest/src + paella.mkdir_p(os.path.dirname(f)) + shutil.copy(src, f, follow_symlinks=False) + +def create_tar(name, basedir, dir='.'): + def reset_uid(tarinfo): + tarinfo.uid = tarinfo.gid = 0 + tarinfo.uname = tarinfo.gname = "root" + return tarinfo + with cwd(basedir): + with tarfile.open(name, 'w:gz') as tar: + tar.add(dir, filter=reset_uid) + +def collect_pytorch(): + d_pytorch = dest/'libtorch' + with cwd(pytorch/'torch/include'): + for f in Path('.').glob('**/*.h'): + copy_p(f, d_pytorch/'include') + with cwd(pytorch/'torch/lib'): + for f in Path('.').glob('*.a'): + copy_p(f, d_pytorch/'lib') + for f in Path('.').glob('*.so*'): + copy_p(f, d_pytorch/'lib') + with cwd(pytorch/'torch'): + shutil.copytree('share', d_pytorch/'share', ignore_dangling_symlinks=True) + create_tar('libtorch-{}-{}-{}-{}.tar.gz'.format(pt_build, pt_os, pt_arch, pt_ver), dest, 'libtorch') + +#---------------------------------------------------------------------------------------------- + +collect_pytorch() diff --git a/opt/build/libtorch/repack.sh b/opt/build/libtorch/repack.sh new file mode 100755 index 000000000..058dc8af3 --- /dev/null +++ b/opt/build/libtorch/repack.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +set -e +[[ $VERBOSE == 1 ]] && set -x + +ROOT=../.. + +if [[ "$1" == "cpu" ]]; then + GPU=no +elif [[ "$1" == "gpu" ]]; then + GPU=yes +else + GPU=${GPU:-no} +fi + +OS=$(python3 $ROOT/opt/readies/bin/platform --os) +ARCH=$(python3 $ROOT/opt/readies/bin/platform --arch) + +# avoid wget warnings on macOS +[[ $OS == macosx ]] && export LC_ALL=en_US.UTF-8 + +if [[ -z $PT_VERSION ]]; then + PT_VERSION=1.2.0 + #PT_VERSION="latest" +fi + +if [[ $OS == linux ]]; then + PT_OS=shared-with-deps + if [[ $GPU == no ]]; then + PT_BUILD=cpu + else + PT_BUILD=cu90 + fi + if [[ $ARCH == x64 ]]; then + PT_ARCH=x86_64 + fi +elif [[ $OS == macosx ]]; then + PT_OS=macos + PT_ARCH=x86_64 + PT_BUILD=cpu +fi + +[[ "$PT_VERSION" == "latest" ]] && PT_BUILD=nightly/${PT_BUILD} + +LIBTORCH_ARCHIVE=libtorch-${PT_OS}-${PT_VERSION}.zip +[[ -z $LIBTORCH_URL ]] && LIBTORCH_URL=https://download.pytorch.org/libtorch/$PT_BUILD/$LIBTORCH_ARCHIVE + +if [ ! -f $LIBTORCH_ARCHIVE ]; then + echo "Downloading libtorch ${PT_VERSION} ${PT_BUILD}" + wget -q $LIBTORCH_URL +fi + +if [[ $OS == linux ]]; then + PT_OS=linux +fi + +unzip -q -o ${LIBTORCH_ARCHIVE} +tar czf libtorch-${PT_BUILD}-${PT_OS}-${PT_ARCH}-${PT_VERSION}.tar.gz libtorch/ +rm -rf libtorch/ ${LIBTORCH_ARCHIVE} diff --git a/opt/build/onnxruntime/Dockerfile.arm b/opt/build/onnxruntime/Dockerfile.arm new file mode 100755 index 000000000..8ef38268b --- /dev/null +++ b/opt/build/onnxruntime/Dockerfile.arm @@ -0,0 +1,53 @@ +# BUILD redisfab/onnxruntime-${ARCH}-${OSNICK}:$(ONNXRUNTIME_VER) + +# stretch|bionic|buster +ARG OSNICK=buster + +# arm32v7|arm64v8 +ARG ARCH=arm64v8 + +#---------------------------------------------------------------------------------------------- +FROM redisfab/${ARCH}-xbuild:${OSNICK} as builder + +ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime +ARG ONNXRUNTIME_BRANCH=rel-0.4.0 +ARG ONNXRUNTIME_VER=0.4.0 +ARG ARCH_FLAG="--arm64" + +RUN [ "cross-build-start" ] + +RUN apt-get -qq update +RUN apt-get -qq install -y curl wget tar git patch +RUN apt-get -qq install -y build-essential cmake +RUN apt-get -qq install -y libcurl4-openssl-dev libssl-dev libatlas-base-dev zlib1g-dev + +RUN apt-get -qq install -y python3 python3-pip python3-dev +RUN pip3 install --upgrade pip setuptools wheel +RUN pip3 install numpy + +WORKDIR /build + +ADD ./*.patch /build/ +ADD ./pack.sh /build/ + +ARG BUILDTYPE=MinSizeRel +ARG BUILDARGS="--config ${BUILDTYPE} ${ARM_FLAG} --parallel" + +RUN set -e ;\ + git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXRUNTIME_REPO} onnxruntime ;\ + cd onnxruntime ;\ + git checkout "rel-${ONNXRUNTIME_VER}" + +RUN set -e ;\ + cd onnxruntime ;\ + patch -p1 -i ../werrror.patch + +RUN set -e ;\ + cd onnxruntime ;\ + ./build.sh ${BUILDARGS} --update --build ;\ + ./build.sh ${BUILDARGS} --build_shared_lib +# RUN ./build.sh ${BUILDARGS} --enable_pybind --build_wheel + +RUN ./pack.sh ${ONNXRUNTIME_VER} + +RUN [ "cross-build-end" ] diff --git a/opt/build/onnxruntime/Dockerfile.arm7 b/opt/build/onnxruntime/Dockerfile.arm7 new file mode 100755 index 000000000..0b4a06947 --- /dev/null +++ b/opt/build/onnxruntime/Dockerfile.arm7 @@ -0,0 +1,46 @@ +FROM balenalib/raspberrypi3-python:latest-stretch-build + +ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime +ARG ONNXRUNTIME_BRANCH=rel-0.4.0 +ARG ONNXRUNTIME_VER=0.4.0 +ARG ARCH_FLAG="--arm" + +RUN [ "cross-build-start" ] + +RUN set -e ;\ + apt-get -qq update \; + apt-get -q install -y + git tar wget curl \ + build-essential cmake \ + python3 python3-pip python3-dev \ + libcurl4-openssl-dev libssl-dev libatlas-base-dev + +RUN pip3 install --upgrade pip setuptools wheel +RUN pip3 install numpy + +WORKDIR /build + +ADD ./*.patch /build/ +ADD ./pack.sh /build/ + +ARG BUILDTYPE=MinSizeRel +ARG BUILDARGS="--config ${BUILDTYPE} ${ARM_FLAG} --parallel" + +RUN set -e ;\ + git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXRUNTIME_REPO} onnxruntime ;\ + cd onnxruntime ;\ + git checkout "rel-${ONNXRUNTIME_VER}" + +RUN set -e ;\ + cd onnxruntime ;\ + patch -p1 -i ../werrror.patch + +RUN set -e ;\ + cd onnxruntime ;\ + ./build.sh ${BUILDARGS} --update --build ;\ + ./build.sh ${BUILDARGS} --build_shared_lib +# RUN ./build.sh ${BUILDARGS} --enable_pybind --build_wheel + +RUN ./pack.sh ${ONNXRUNTIME_VER} + +RUN [ "cross-build-end" ] diff --git a/opt/build/onnxruntime/Dockerfile.x64 b/opt/build/onnxruntime/Dockerfile.x64 new file mode 100755 index 000000000..c03080aac --- /dev/null +++ b/opt/build/onnxruntime/Dockerfile.x64 @@ -0,0 +1,36 @@ +# BUILD redisfab/onnxruntime-x64-${OS}:$(ONNXRUNTIME_VER) + +ARG OS=debian:buster + +#---------------------------------------------------------------------------------------------- +FROM ${OS} + +ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime +ARG ONNXRUNTIME_BRANCH=rel-0.4.0 +ARG ONNXRUNTIME_VER=0.4.0 + +RUN apt-get -qq update +RUN apt-get -qq install -y curl wget tar git +RUN apt-get -qq install -y build-essential cmake +RUN apt-get -qq install -y libcurl4-openssl-dev libssl-dev libatlas-base-dev zlib1g-dev + +RUN apt-get -qq install -y python3 python3-pip python3-dev +RUN pip3 install --upgrade pip setuptools wheel +RUN pip3 install numpy + +WORKDIR /build + +ADD ./pack.sh /build/ +ARG BUILDTYPE=MinSizeRel +ARG BUILDARGS="--config ${BUILDTYPE} --parallel" + +RUN set -e ;\ + git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXRUNTIME_REPO} onnxruntime ;\ + cd onnxruntime ;\ + git checkout "rel-${ONNXRUNTIME_VER}" + cd onnxruntime ;\ + ./build.sh ${BUILDARGS} --update --build ;\ + ./build.sh ${BUILDARGS} --build_shared_lib +# RUN ./build.sh ${BUILDARGS} --enable_pybind --build_wheel + +RUN ./pack.sh ${ONNXRUNTIME_VER} diff --git a/opt/build/onnxruntime/Makefile b/opt/build/onnxruntime/Makefile new file mode 100755 index 000000000..e6960b856 --- /dev/null +++ b/opt/build/onnxruntime/Makefile @@ -0,0 +1,72 @@ + +ROOT=. + +VERSION=0.4.0 + +S3_URL=redismodules/onnxruntime + +STEM=onnxruntime-linux + +KEEP ?= 1 + +#---------------------------------------------------------------------------------------------- + +define targets # (1=OP, 2=op) +$(1)_TARGETS := +$(1)_TARGETS += $(if $(findstring $(X64),1),$(2)_x64) +$(1)_TARGETS += $(if $(findstring $(ARM7),1),$(2)_arm32v7) +$(1)_TARGETS += $(if $(findstring $(ARM8),1),$(2)_arm64v8) + +$(1)_TARGETS += $$(if $$(strip $$($(1)_TARGETS)),,$(2)_x64 $(2)_arm32v7 $(2)_arm64v8) +endef + +$(eval $(call targets,BUILD,build)) +$(eval $(call targets,PUBLISH,publish)) + +#---------------------------------------------------------------------------------------------- +# arch=x64|arm32v7|arm64v8 +# arch_1=x86_64|arm|arm64 + +define build_arm # (1=arch, 2=arch_1) +IID_$(1)=$(1)_$(VERSION).iid +CID_$(1)=$(1)_$(VERSION).cid + +build_$(1): + @docker build --iidfile $$(IID_$(1)) -t redisfab/$(STEM)-$(1):$(VERSION) -f Dockerfile.arm \ + --build-arg ARCH=$(1) --build-arg ARCH_FLAG=--$(2) $(ROOT) + @docker create --cidfile $$(CID_$(1)) `cat $$(IID_$(1))` + @docker cp `cat $$(CID_$(1))`:/build/$(STEM)-$(2)-$(VERSION).tgz . +ifneq ($(KEEP),1) + # @rm $$(IID_$(1)) $$(CID_$(1)) +endif + +.PHONY: build_$(1) +endef + +#---------------------------------------------------------------------------------------------- + +define publish_arm # (1=arch, 2=arch_1) +publish_$(1): + @aws s3 cp $(STEM)-$(2)-$(VERSION).tgz s3://$(S3_URL)/ --acl public-read +ifneq ($(KEEP),1) + # @rm $(STEM)-$(2)-$(VERSION).tgz +endif + +.PHONY: publish_$(1) +endef + +#---------------------------------------------------------------------------------------------- + +all: build publish + +build: $(BUILD_TARGETS) + +$(eval $(call build_arm,arm64v8,arm64)) +$(eval $(call build_arm,arm32v7,arm)) + +publish: $(PUBLISH_TARGETS) + +$(eval $(call publish_arm,arm64v8,arm64)) +$(eval $(call publish_arm,arm32v7,arm)) + +.PHONY: all build publish diff --git a/opt/build/onnxruntime/pack.sh b/opt/build/onnxruntime/pack.sh new file mode 100755 index 000000000..52eb55cb7 --- /dev/null +++ b/opt/build/onnxruntime/pack.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e +VER="$1" + +mkdir -p pack/include pack/lib +cp onnxruntime/build/Linux/MinSizeRel/libonnxruntime.so.${VER} pack/lib/ +cp onnxruntime/docs/C_API.md pack/ +cp onnxruntime/LICENSE pack/ +cp onnxruntime/README.md pack/ +cp onnxruntime/ThirdPartyNotices.txt pack/ +cp onnxruntime/VERSION_NUMBER pack/ +cd onnxruntime/ +git rev-parse HEAD > ../pack/GIT_COMMIT_ID +cd .. +cp onnxruntime/include/onnxruntime/core/session/onnxruntime_c_api.h pack/include/ +cd pack/lib/ +ln -s libonnxruntime.so.${VER} libonnxruntime.so +cd ../.. +mv pack onnxruntime-linux-arm64-${VER} +tar czf onnxruntime-linux-arm64-${VER}.tgz onnxruntime-linux-arm64-${VER}/ diff --git a/opt/build/onnxruntime/werrror.patch b/opt/build/onnxruntime/werrror.patch new file mode 100755 index 000000000..be5eb10b7 --- /dev/null +++ b/opt/build/onnxruntime/werrror.patch @@ -0,0 +1,758 @@ +diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt +index 96c89fe..f59edea 100644 +--- a/cmake/CMakeLists.txt ++++ b/cmake/CMakeLists.txt +@@ -411,8 +411,8 @@ else() + + if(onnxruntime_DEV_MODE) + if(NOT onnxruntime_USE_TVM) +- string(APPEND CMAKE_CXX_FLAGS " -Werror") +- string(APPEND CMAKE_C_FLAGS " -Werror") ++# string(APPEND CMAKE_CXX_FLAGS " -Werror") ++# string(APPEND CMAKE_C_FLAGS " -Werror") + endif() + endif() + check_cxx_compiler_flag(-Wunused-but-set-variable HAS_UNUSED_BUT_SET_VARIABLE) +Submodule cmake/external/eigen contains modified content +diff --git a/cmake/external/eigen/CMakeLists.txt b/cmake/external/eigen/CMakeLists.txt +index 76e083314..a515736cd 100644 +--- a/cmake/external/eigen/CMakeLists.txt ++++ b/cmake/external/eigen/CMakeLists.txt +@@ -151,10 +151,10 @@ if(NOT MSVC) + + # clang outputs some warnings for unknown flags that are not caught by check_cxx_compiler_flag + # adding -Werror turns such warnings into errors +- check_cxx_compiler_flag("-Werror" COMPILER_SUPPORT_WERROR) +- if(COMPILER_SUPPORT_WERROR) +- set(CMAKE_REQUIRED_FLAGS "-Werror") +- endif() ++# check_cxx_compiler_flag("-Werror" COMPILER_SUPPORT_WERROR) ++# if(COMPILER_SUPPORT_WERROR) ++# set(CMAKE_REQUIRED_FLAGS "-Werror") ++# endif() + ei_add_cxx_compiler_flag("-pedantic") + ei_add_cxx_compiler_flag("-Wall") + ei_add_cxx_compiler_flag("-Wextra") +Submodule cmake/external/googletest contains modified content +diff --git a/cmake/external/googletest/googletest/cmake/internal_utils.cmake b/cmake/external/googletest/googletest/cmake/internal_utils.cmake +index 6448918f..38e1a864 100644 +--- a/cmake/external/googletest/googletest/cmake/internal_utils.cmake ++++ b/cmake/external/googletest/googletest/cmake/internal_utils.cmake +@@ -94,7 +94,7 @@ macro(config_compiler_and_linker) + set(cxx_no_exception_flags "-EHs-c- -D_HAS_EXCEPTIONS=0") + set(cxx_no_rtti_flags "-GR-") + elseif (CMAKE_COMPILER_IS_GNUCXX) +- set(cxx_base_flags "-Wall -Wshadow -Werror") ++ set(cxx_base_flags "-Wall -Wshadow") + if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0.0) + set(cxx_base_flags "${cxx_base_flags} -Wno-error=dangling-else") + endif() +diff --git a/cmake/external/googletest/googletest/xcode/Config/General.xcconfig b/cmake/external/googletest/googletest/xcode/Config/General.xcconfig +index f23e3222..ac07337f 100644 +--- a/cmake/external/googletest/googletest/xcode/Config/General.xcconfig ++++ b/cmake/external/googletest/googletest/xcode/Config/General.xcconfig +@@ -17,7 +17,7 @@ ZERO_LINK = NO + PREBINDING = NO + + // Strictest warning policy +-WARNING_CFLAGS = -Wall -Werror -Wendif-labels -Wnewline-eof -Wno-sign-compare -Wshadow ++WARNING_CFLAGS = -Wall -Wendif-labels -Wnewline-eof -Wno-sign-compare -Wshadow + + // Work around Xcode bugs by using external strip. See: + // http://lists.apple.com/archives/Xcode-users/2006/Feb/msg00050.html +Submodule cmake/external/gsl contains modified content +diff --git a/cmake/external/gsl/tests/CMakeLists.txt b/cmake/external/gsl/tests/CMakeLists.txt +index 0e08d77..a4d103d 100644 +--- a/cmake/external/gsl/tests/CMakeLists.txt ++++ b/cmake/external/gsl/tests/CMakeLists.txt +@@ -41,7 +41,7 @@ target_compile_options(gsl_tests_config INTERFACE + -Wcast-align + -Wconversion + -Wctor-dtor-privacy +- -Werror ++# -Werror + -Wextra + -Wno-missing-braces + -Wnon-virtual-dtor +@@ -129,7 +129,7 @@ target_compile_options(gsl_tests_config_noexcept INTERFACE + -Wcast-align + -Wconversion + -Wctor-dtor-privacy +- -Werror ++# -Werror + -Wextra + -Wno-missing-braces + -Wnon-virtual-dtor +Submodule cmake/external/nsync contains modified content +diff --git a/cmake/external/nsync/builds/aarch64.linux.c++11/Makefile b/cmake/external/nsync/builds/aarch64.linux.c++11/Makefile +index d7b138b..d76d43f 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.c++11/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.c++11/Makefile +@@ -2,7 +2,7 @@ + # use futex directly. + CC=g++ + PLATFORM_CPPFLAGS=-DNSYNC_USE_CPP11_TIMEPOINT -DNSYNC_ATOMIC_CPP11 -I../../platform/c++11.futex -I../../platform/c++11 -I../../platform/gcc -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-std=c++11 -Werror -Wall -Wextra -pedantic ++PLATFORM_CFLAGS=-std=c++11 -Wall -Wextra -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M -std=c++11 + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/c++11/src/per_thread_waiter.cc ../../platform/c++11/src/yield.cc ../../platform/c++11/src/time_rep_timespec.cc ../../platform/c++11/src/nsync_panic.cc +diff --git a/cmake/external/nsync/builds/aarch64.linux.clang/Makefile b/cmake/external/nsync/builds/aarch64.linux.clang/Makefile +index 291b3db..961aea1 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.clang/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.clang/Makefile +@@ -1,6 +1,6 @@ + CC=clang + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/clang -I../../platform/linux -I../../platform/aarch64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/aarch64.linux.g++/Makefile b/cmake/external/nsync/builds/aarch64.linux.g++/Makefile +index d4f0528..d1e2f24 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.g++/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.g++/Makefile +@@ -1,6 +1,6 @@ + CC=g++ + PLATFORM_CPPFLAGS=-DNSYNC_ATOMIC_CPP11 -I../../platform/linux -I../../platform/c++11 -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/aarch64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-x c++ -std=c++11 -Werror -Wall -Wextra -pedantic ++PLATFORM_CFLAGS=-x c++ -std=c++11 -Wall -Wextra -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M -x c++ -std=c++11 + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/aarch64.linux.gcc.atm-asm/Makefile b/cmake/external/nsync/builds/aarch64.linux.gcc.atm-asm/Makefile +index d5e168e..3218ce0 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.gcc.atm-asm/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.gcc.atm-asm/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-I../../platform/atomic_ind -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/aarch64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/aarch64.linux.gcc.atm-c++11/Makefile b/cmake/external/nsync/builds/aarch64.linux.gcc.atm-c++11/Makefile +index 6251fed..238dd54 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.gcc.atm-c++11/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.gcc.atm-c++11/Makefile +@@ -1,7 +1,7 @@ + PLATFORM_CPPFLAGS=-I../../platform/atomic_ind -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/aarch64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread +-PLATFORM_CXXFLAGS=-Werror -Wall -Wextra -std=c++11 -pedantic ++PLATFORM_CXXFLAGS=-Wall -Wextra -std=c++11 -pedantic + PLATFORM_CXX=../../platform/c_from_c++11/src/nsync_atm_c++.cc + MKDEP_DEPEND=mkdep + MKDEP=./mkdep ${CC} -E -c++=-std=c++11 +diff --git a/cmake/external/nsync/builds/aarch64.linux.gcc.atm-c11/Makefile b/cmake/external/nsync/builds/aarch64.linux.gcc.atm-c11/Makefile +index 10d1104..21a2f43 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.gcc.atm-c11/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.gcc.atm-c11/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-DNSYNC_ATOMIC_C11 -I../../platform/c11 -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/aarch64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/aarch64.linux.gcc.sem-mutex/Makefile b/cmake/external/nsync/builds/aarch64.linux.gcc.sem-mutex/Makefile +index 5352904..a428b34 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.gcc.sem-mutex/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.gcc.sem-mutex/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/aarch64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/nsync_semaphore_mutex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/aarch64.linux.gcc.sem-sem_t/Makefile b/cmake/external/nsync/builds/aarch64.linux.gcc.sem-sem_t/Makefile +index 79a4f6d..c642bde 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.gcc.sem-sem_t/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.gcc.sem-sem_t/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/aarch64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/nsync_semaphore_sem_t.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/aarch64.linux.gcc/Makefile b/cmake/external/nsync/builds/aarch64.linux.gcc/Makefile +index 7dbeb25..487fc55 100644 +--- a/cmake/external/nsync/builds/aarch64.linux.gcc/Makefile ++++ b/cmake/external/nsync/builds/aarch64.linux.gcc/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/aarch64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/arm.linux.gcc.atm-asm.lrt/Makefile b/cmake/external/nsync/builds/arm.linux.gcc.atm-asm.lrt/Makefile +index 3cd81a1..bb88607 100644 +--- a/cmake/external/nsync/builds/arm.linux.gcc.atm-asm.lrt/Makefile ++++ b/cmake/external/nsync/builds/arm.linux.gcc.atm-asm.lrt/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-I../../platform/atomic_ind -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/arm -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + PLATFORM_LIBS=-lrt + MKDEP=${CC} -M +diff --git a/cmake/external/nsync/builds/arm.linux.gcc.lrt/Makefile b/cmake/external/nsync/builds/arm.linux.gcc.lrt/Makefile +index 009cd01..f058b7f 100644 +--- a/cmake/external/nsync/builds/arm.linux.gcc.lrt/Makefile ++++ b/cmake/external/nsync/builds/arm.linux.gcc.lrt/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/arm -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + PLATFORM_LIBS=-lrt + MKDEP=${CC} -M +diff --git a/cmake/external/nsync/builds/x86_64.linux.c++11/Makefile b/cmake/external/nsync/builds/x86_64.linux.c++11/Makefile +index d7b138b..d76d43f 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.c++11/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.c++11/Makefile +@@ -2,7 +2,7 @@ + # use futex directly. + CC=g++ + PLATFORM_CPPFLAGS=-DNSYNC_USE_CPP11_TIMEPOINT -DNSYNC_ATOMIC_CPP11 -I../../platform/c++11.futex -I../../platform/c++11 -I../../platform/gcc -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-std=c++11 -Werror -Wall -Wextra -pedantic ++PLATFORM_CFLAGS=-std=c++11 -Wall -Wextra -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M -std=c++11 + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/c++11/src/per_thread_waiter.cc ../../platform/c++11/src/yield.cc ../../platform/c++11/src/time_rep_timespec.cc ../../platform/c++11/src/nsync_panic.cc +diff --git a/cmake/external/nsync/builds/x86_64.linux.clang/Makefile b/cmake/external/nsync/builds/x86_64.linux.clang/Makefile +index dc16f1f..dfa2bfb 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.clang/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.clang/Makefile +@@ -1,6 +1,6 @@ + CC=clang + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/clang -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.linux.g++/Makefile b/cmake/external/nsync/builds/x86_64.linux.g++/Makefile +index 990e1e4..a55a5db 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.g++/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.g++/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-DNSYNC_ATOMIC_CPP11 -I../../platform/linux -I../../platform/c++11 -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-x c++ -std=c++11 -Werror -Wall -Wextra -pedantic ++PLATFORM_CFLAGS=-x c++ -std=c++11 -Wall -Wextra -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M -x c++ -std=c++11 + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.linux.gcc.atm-asm/Makefile b/cmake/external/nsync/builds/x86_64.linux.gcc.atm-asm/Makefile +index 870772e..26df437 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.gcc.atm-asm/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.gcc.atm-asm/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-I../../platform/atomic_ind -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.linux.gcc.atm-c++11/Makefile b/cmake/external/nsync/builds/x86_64.linux.gcc.atm-c++11/Makefile +index 6dad2b7..e98d035 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.gcc.atm-c++11/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.gcc.atm-c++11/Makefile +@@ -1,7 +1,7 @@ + PLATFORM_CPPFLAGS=-I../../platform/atomic_ind -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread +-PLATFORM_CXXFLAGS=-Werror -Wall -Wextra -std=c++11 -pedantic ++PLATFORM_CXXFLAGS=-Wall -Wextra -std=c++11 -pedantic + PLATFORM_CXX=../../platform/c_from_c++11/src/nsync_atm_c++.cc + MKDEP_DEPEND=mkdep + MKDEP=./mkdep ${CC} -E -c++=-std=c++11 +diff --git a/cmake/external/nsync/builds/x86_64.linux.gcc.atm-c11/Makefile b/cmake/external/nsync/builds/x86_64.linux.gcc.atm-c11/Makefile +index a7c98a1..395e8c9 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.gcc.atm-c11/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.gcc.atm-c11/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-DNSYNC_ATOMIC_C11 -I../../platform/c11 -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.linux.gcc.sem-mutex/Makefile b/cmake/external/nsync/builds/x86_64.linux.gcc.sem-mutex/Makefile +index 035c19f..3ba36e3 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.gcc.sem-mutex/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.gcc.sem-mutex/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/nsync_semaphore_mutex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.linux.gcc.sem-sem_t/Makefile b/cmake/external/nsync/builds/x86_64.linux.gcc.sem-sem_t/Makefile +index 040611c..394abf7 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.gcc.sem-sem_t/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.gcc.sem-sem_t/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/nsync_semaphore_sem_t.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.linux.gcc/Makefile b/cmake/external/nsync/builds/x86_64.linux.gcc/Makefile +index 4bfa290..0174802 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.gcc/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.gcc/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.linux.old_gcc/Makefile b/cmake/external/nsync/builds/x86_64.linux.old_gcc/Makefile +index 347c2ce..6e70f0f 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.old_gcc/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.old_gcc/Makefile +@@ -1,6 +1,6 @@ + # Force the use of old gcc atomics. + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc_old -I../../platform/gcc -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.linux.tcc/Makefile b/cmake/external/nsync/builds/x86_64.linux.tcc/Makefile +index 732265f..f986e91 100644 +--- a/cmake/external/nsync/builds/x86_64.linux.tcc/Makefile ++++ b/cmake/external/nsync/builds/x86_64.linux.tcc/Makefile +@@ -1,6 +1,6 @@ + CC=tcc + PLATFORM_CPPFLAGS=-I../../platform/atomic_ind -D_POSIX_C_SOURCE=200809L -I../../platform/tcc -I../../platform/linux -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall ++PLATFORM_CFLAGS=-Wall + PLATFORM_LDFLAGS=-pthread + MKDEP_DEPEND=mkdep + MKDEP=./mkdep ${CC} -E +diff --git a/cmake/external/nsync/builds/x86_64.macos.c++11/Makefile b/cmake/external/nsync/builds/x86_64.macos.c++11/Makefile +index 4142416..2b552ae 100644 +--- a/cmake/external/nsync/builds/x86_64.macos.c++11/Makefile ++++ b/cmake/external/nsync/builds/x86_64.macos.c++11/Makefile +@@ -1,7 +1,7 @@ + CC=clang++ + # Some versions of MacOS (such as Sierra) require _DARWIN_C_SOURCE to include , , and (!) + PLATFORM_CPPFLAGS=-DNSYNC_USE_CPP11_TIMEPOINT -DNSYNC_ATOMIC_CPP11 -D_DARWIN_C_SOURCE -I../../platform/c++11 -I../../platform/gcc -I../../platform/macos -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-x c++ -std=c++11 -Werror -Wall -Wextra -pedantic ++PLATFORM_CFLAGS=-x c++ -std=c++11 -Wall -Wextra -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -x c++ -M -std=c++11 + # Currently, MacOS's C++11 implementation is incomplete, and does not support +diff --git a/cmake/external/nsync/builds/x86_64.macos.clang.atm-c++11/Makefile b/cmake/external/nsync/builds/x86_64.macos.clang.atm-c++11/Makefile +index bf15b64..1122bb1 100644 +--- a/cmake/external/nsync/builds/x86_64.macos.clang.atm-c++11/Makefile ++++ b/cmake/external/nsync/builds/x86_64.macos.clang.atm-c++11/Makefile +@@ -1,7 +1,7 @@ + PLATFORM_CPPFLAGS=-I../../platform/atomic_ind -D_POSIX_C_SOURCE=200809L -I../../platform/clang -I../../platform/macos -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration + PLATFORM_LDFLAGS=-pthread +-PLATFORM_CXXFLAGS=-Werror -Wall -Wextra -std=c++11 -pedantic ++PLATFORM_CXXFLAGS=-Wall -Wextra -std=c++11 -pedantic + PLATFORM_CXX=../../platform/c_from_c++11/src/nsync_atm_c++.cc + MKDEP_DEPEND=mkdep + MKDEP=./mkdep ${CC} -E -c++=-std=c++11 +diff --git a/cmake/external/nsync/builds/x86_64.macos.clang.atm-c11/Makefile b/cmake/external/nsync/builds/x86_64.macos.clang.atm-c11/Makefile +index 653f99a..eb925bb 100644 +--- a/cmake/external/nsync/builds/x86_64.macos.clang.atm-c11/Makefile ++++ b/cmake/external/nsync/builds/x86_64.macos.clang.atm-c11/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -DNSYNC_ATOMIC_C11 -I../../platform/c11 -I../../platform/macos -I../../platform/clang -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/clock_gettime.c ../../platform/posix/src/nsync_semaphore_mutex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.macos.clang.atm-os/Makefile b/cmake/external/nsync/builds/x86_64.macos.clang.atm-os/Makefile +index 04943e3..cf8fc7f 100644 +--- a/cmake/external/nsync/builds/x86_64.macos.clang.atm-os/Makefile ++++ b/cmake/external/nsync/builds/x86_64.macos.clang.atm-os/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/macos -I../../platform/clang -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration -Wno-deprecated-declarations ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration -Wno-deprecated-declarations + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/clock_gettime.c ../../platform/posix/src/nsync_semaphore_mutex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.macos.clang/Makefile b/cmake/external/nsync/builds/x86_64.macos.clang/Makefile +index 2b36b8d..c02e528 100644 +--- a/cmake/external/nsync/builds/x86_64.macos.clang/Makefile ++++ b/cmake/external/nsync/builds/x86_64.macos.clang/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/clang -I../../platform/macos -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/clock_gettime.c ../../platform/posix/src/nsync_semaphore_mutex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.netbsd.gcc.atm-asm/Makefile b/cmake/external/nsync/builds/x86_64.netbsd.gcc.atm-asm/Makefile +index adce5dc..e90f92e 100644 +--- a/cmake/external/nsync/builds/x86_64.netbsd.gcc.atm-asm/Makefile ++++ b/cmake/external/nsync/builds/x86_64.netbsd.gcc.atm-asm/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-I../../platform/atomic_ind -D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/netbsd -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/nsync_semaphore_mutex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.netbsd.gcc/Makefile b/cmake/external/nsync/builds/x86_64.netbsd.gcc/Makefile +index d8d7247..aa46d8e 100644 +--- a/cmake/external/nsync/builds/x86_64.netbsd.gcc/Makefile ++++ b/cmake/external/nsync/builds/x86_64.netbsd.gcc/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc -I../../platform/netbsd -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/nsync_semaphore_mutex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/builds/x86_64.openbsd.gcc/Makefile b/cmake/external/nsync/builds/x86_64.openbsd.gcc/Makefile +index 8d9b035..ded5309 100644 +--- a/cmake/external/nsync/builds/x86_64.openbsd.gcc/Makefile ++++ b/cmake/external/nsync/builds/x86_64.openbsd.gcc/Makefile +@@ -1,5 +1,5 @@ + PLATFORM_CPPFLAGS=-D_POSIX_C_SOURCE=200809L -I../../platform/gcc_no_tls -I../../platform/gcc -I../../platform/openbsd -I../../platform/x86_64 -I../../platform/posix -pthread +-PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic ++PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic + PLATFORM_LDFLAGS=-pthread + MKDEP=${CC} -M + PLATFORM_C=../../platform/posix/src/nsync_semaphore_mutex.c ../../platform/posix/src/per_thread_waiter.c ../../platform/posix/src/yield.c ../../platform/posix/src/time_rep.c ../../platform/posix/src/nsync_panic.c +diff --git a/cmake/external/nsync/tools/mkmakefile.sh b/cmake/external/nsync/tools/mkmakefile.sh +index 365b34b..962962a 100644 +--- a/cmake/external/nsync/tools/mkmakefile.sh ++++ b/cmake/external/nsync/tools/mkmakefile.sh +@@ -281,11 +281,11 @@ makefile=` + esac + case "$cc_type.$cplusplus" in + gcc.) echo "PLATFORM_CPPFLAGS=$atomic_ind$cppflags" +- echo "PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic" ++ echo "PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic" + case "$ldflags" in ?*) echo "PLATFORM_LDFLAGS=$ldflags";; esac + case "$libs" in ?*) echo "PLATFORM_LIBS=$libs";; esac + case "$atomics" in +- c++11) echo "PLATFORM_CXXFLAGS=-Werror -Wall -Wextra -std=c++11 -pedantic" ++ c++11) echo "PLATFORM_CXXFLAGS=-Wall -Wextra -std=c++11 -pedantic" + echo "PLATFORM_CXX=../../platform/c_from_c++11/src/nsync_atm_c++.cc" + echo "MKDEP_DEPEND=mkdep" + echo 'MKDEP=./mkdep ${CC} -E -c++=-std=c++11' +@@ -299,11 +299,11 @@ makefile=` + echo "TEST_PLATFORM_OBJS=$test_platform_o" + ;; + clang.) echo "PLATFORM_CPPFLAGS=$atomic_ind$cppflags" +- echo "PLATFORM_CFLAGS=-Werror -Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration" ++ echo "PLATFORM_CFLAGS=-Wall -Wextra -ansi -pedantic -Wno-unneeded-internal-declaration" + case "$ldflags" in ?*) echo "PLATFORM_LDFLAGS=$ldflags";; esac + case "$libs" in ?*) echo "PLATFORM_LIBS=$libs";; esac + case "$atomics" in +- c++11) echo "PLATFORM_CXXFLAGS=-Werror -Wall -Wextra -std=c++11 -pedantic" ++ c++11) echo "PLATFORM_CXXFLAGS=-Wall -Wextra -std=c++11 -pedantic" + echo "PLATFORM_CXX=../../platform/c_from_c++11/src/nsync_atm_c++.cc" + echo "MKDEP_DEPEND=mkdep" + echo 'MKDEP=./mkdep ${CC} -E -c++=-std=c++11' +@@ -317,7 +317,7 @@ makefile=` + echo "TEST_PLATFORM_OBJS=$test_platform_o" + ;; + gcc.c++) echo "PLATFORM_CPPFLAGS=-DNSYNC_ATOMIC_CPP11 -I../../platform/c++11 $cppflags" +- echo "PLATFORM_CFLAGS=-x c++ -std=c++11 -Werror -Wall -Wextra -pedantic" ++ echo "PLATFORM_CFLAGS=-x c++ -std=c++11 -Wall -Wextra -pedantic" + case "$ldflags" in ?*) echo "PLATFORM_LDFLAGS=$ldflags";; esac + case "$libs" in ?*) echo "PLATFORM_LIBS=$libs";; esac + echo 'MKDEP=${CC} -M -x c++ -std=c++11' +@@ -328,7 +328,7 @@ makefile=` + echo "TEST_PLATFORM_OBJS=$test_platform_o" + ;; + clang.c++) echo "PLATFORM_CPPFLAGS=-DNSYNC_ATOMIC_CPP11 -I../../platform/c++11 $cppflags" +- echo "PLATFORM_CFLAGS=-x c++ -std=c++11 -Werror -Wall -Wextra -pedantic -Wno-unneeded-internal-declaration" ++ echo "PLATFORM_CFLAGS=-x c++ -std=c++11 -Wall -Wextra -pedantic -Wno-unneeded-internal-declaration" + case "$ldflags" in ?*) echo "PLATFORM_LDFLAGS=$ldflags";; esac + case "$libs" in ?*) echo "PLATFORM_LIBS=$libs";; esac + echo 'MKDEP=${CC} -M -x c++ -std=c++11' +@@ -339,7 +339,7 @@ makefile=` + echo "TEST_PLATFORM_OBJS=$test_platform_o" + ;; + tcc.) echo "PLATFORM_CPPFLAGS=$atomic_ind$cppflags" +- echo "PLATFORM_CFLAGS=-Werror -Wall" ++ echo "PLATFORM_CFLAGS=-Wall" + case "$ldflags" in ?*) echo "PLATFORM_LDFLAGS=$ldflags";; esac + case "$libs" in ?*) echo "PLATFORM_LIBS=$libs";; esac + echo "MKDEP_DEPEND=mkdep" +Submodule cmake/external/onnx contains modified content +diff --git a/cmake/external/onnx/CMakeLists.txt b/cmake/external/onnx/CMakeLists.txt +index 195b0486..1c903bdf 100644 +--- a/cmake/external/onnx/CMakeLists.txt ++++ b/cmake/external/onnx/CMakeLists.txt +@@ -485,7 +485,7 @@ elseif(APPLE) + else() + target_compile_options(onnx PRIVATE -Wall -Wextra) + if(${ONNX_WERROR}) +- target_compile_options(onnx PRIVATE -Werror) ++# target_compile_options(onnx PRIVATE -Werror) + endif() + endif() + +Submodule third_party/benchmark contains modified content +diff --git a/cmake/external/onnx/third_party/benchmark/.ycm_extra_conf.py b/cmake/external/onnx/third_party/benchmark/.ycm_extra_conf.py +index 5649ddc..4ccbcde 100644 +--- a/cmake/external/onnx/third_party/benchmark/.ycm_extra_conf.py ++++ b/cmake/external/onnx/third_party/benchmark/.ycm_extra_conf.py +@@ -6,7 +6,6 @@ import ycm_core + # CHANGE THIS LIST OF FLAGS. YES, THIS IS THE DROID YOU HAVE BEEN LOOKING FOR. + flags = [ + '-Wall', +-'-Werror', + '-pedantic-errors', + '-std=c++0x', + '-fno-strict-aliasing', +diff --git a/cmake/external/onnx/third_party/benchmark/CMakeLists.txt b/cmake/external/onnx/third_party/benchmark/CMakeLists.txt +index b1c1d3d..2d1f2b1 100644 +--- a/cmake/external/onnx/third_party/benchmark/CMakeLists.txt ++++ b/cmake/external/onnx/third_party/benchmark/CMakeLists.txt +@@ -133,9 +133,9 @@ else() + + add_cxx_compiler_flag(-Wextra) + add_cxx_compiler_flag(-Wshadow) +- add_cxx_compiler_flag(-Werror RELEASE) +- add_cxx_compiler_flag(-Werror RELWITHDEBINFO) +- add_cxx_compiler_flag(-Werror MINSIZEREL) ++# add_cxx_compiler_flag(-Werror RELEASE) ++# add_cxx_compiler_flag(-Werror RELWITHDEBINFO) ++# add_cxx_compiler_flag(-Werror MINSIZEREL) + add_cxx_compiler_flag(-pedantic) + add_cxx_compiler_flag(-pedantic-errors) + add_cxx_compiler_flag(-Wshorten-64-to-32) +diff --git a/cmake/external/onnx/third_party/benchmark/cmake/AddCXXCompilerFlag.cmake b/cmake/external/onnx/third_party/benchmark/cmake/AddCXXCompilerFlag.cmake +index d0d2099..0ebedc9 100644 +--- a/cmake/external/onnx/third_party/benchmark/cmake/AddCXXCompilerFlag.cmake ++++ b/cmake/external/onnx/third_party/benchmark/cmake/AddCXXCompilerFlag.cmake +@@ -68,7 +68,7 @@ function(check_cxx_warning_flag FLAG) + set(OLD_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}") + # Add -Werror to ensure the compiler generates an error if the warning flag + # doesn't exist. +- set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -Werror ${FLAG}") ++ set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}") + check_cxx_compiler_flag("${FLAG}" ${MANGLED_FLAG}) + set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS}") + endfunction() +Submodule third_party/pybind11 contains modified content +diff --git a/cmake/external/onnx/third_party/pybind11/tests/CMakeLists.txt b/cmake/external/onnx/third_party/pybind11/tests/CMakeLists.txt +index df77d43..63cebdb 100644 +--- a/cmake/external/onnx/third_party/pybind11/tests/CMakeLists.txt ++++ b/cmake/external/onnx/third_party/pybind11/tests/CMakeLists.txt +@@ -129,7 +129,7 @@ function(pybind11_enable_warnings target_name) + if(MSVC) + target_compile_options(${target_name} PRIVATE /WX) + else() +- target_compile_options(${target_name} PRIVATE -Werror) ++# target_compile_options(${target_name} PRIVATE -Werror) + endif() + endif() + endfunction() +Submodule cmake/external/onnx-tensorrt contains modified content +Submodule third_party/onnx contains modified content +diff --git a/cmake/external/onnx-tensorrt/third_party/onnx/CMakeLists.txt b/cmake/external/onnx-tensorrt/third_party/onnx/CMakeLists.txt +index e64dfcec..776429d0 100644 +--- a/cmake/external/onnx-tensorrt/third_party/onnx/CMakeLists.txt ++++ b/cmake/external/onnx-tensorrt/third_party/onnx/CMakeLists.txt +@@ -473,7 +473,7 @@ elseif(APPLE) + else() + target_compile_options(onnx PRIVATE -Wall -Wextra) + if(${ONNX_WERROR}) +- target_compile_options(onnx PRIVATE -Werror) ++# target_compile_options(onnx PRIVATE -Werror) + endif() + endif() + +Submodule third_party/benchmark contains modified content +diff --git a/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/.ycm_extra_conf.py b/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/.ycm_extra_conf.py +index 5649ddc..4ccbcde 100644 +--- a/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/.ycm_extra_conf.py ++++ b/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/.ycm_extra_conf.py +@@ -6,7 +6,6 @@ import ycm_core + # CHANGE THIS LIST OF FLAGS. YES, THIS IS THE DROID YOU HAVE BEEN LOOKING FOR. + flags = [ + '-Wall', +-'-Werror', + '-pedantic-errors', + '-std=c++0x', + '-fno-strict-aliasing', +diff --git a/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/CMakeLists.txt b/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/CMakeLists.txt +index b1c1d3d..2d1f2b1 100644 +--- a/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/CMakeLists.txt ++++ b/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/CMakeLists.txt +@@ -133,9 +133,9 @@ else() + + add_cxx_compiler_flag(-Wextra) + add_cxx_compiler_flag(-Wshadow) +- add_cxx_compiler_flag(-Werror RELEASE) +- add_cxx_compiler_flag(-Werror RELWITHDEBINFO) +- add_cxx_compiler_flag(-Werror MINSIZEREL) ++# add_cxx_compiler_flag(-Werror RELEASE) ++# add_cxx_compiler_flag(-Werror RELWITHDEBINFO) ++# add_cxx_compiler_flag(-Werror MINSIZEREL) + add_cxx_compiler_flag(-pedantic) + add_cxx_compiler_flag(-pedantic-errors) + add_cxx_compiler_flag(-Wshorten-64-to-32) +diff --git a/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/cmake/AddCXXCompilerFlag.cmake b/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/cmake/AddCXXCompilerFlag.cmake +index d0d2099..0ebedc9 100644 +--- a/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/cmake/AddCXXCompilerFlag.cmake ++++ b/cmake/external/onnx-tensorrt/third_party/onnx/third_party/benchmark/cmake/AddCXXCompilerFlag.cmake +@@ -68,7 +68,7 @@ function(check_cxx_warning_flag FLAG) + set(OLD_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}") + # Add -Werror to ensure the compiler generates an error if the warning flag + # doesn't exist. +- set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -Werror ${FLAG}") ++ set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}") + check_cxx_compiler_flag("${FLAG}" ${MANGLED_FLAG}) + set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS}") + endfunction() +Submodule third_party/pybind11 contains modified content +diff --git a/cmake/external/onnx-tensorrt/third_party/onnx/third_party/pybind11/tests/CMakeLists.txt b/cmake/external/onnx-tensorrt/third_party/onnx/third_party/pybind11/tests/CMakeLists.txt +index df77d43..63cebdb 100644 +--- a/cmake/external/onnx-tensorrt/third_party/onnx/third_party/pybind11/tests/CMakeLists.txt ++++ b/cmake/external/onnx-tensorrt/third_party/onnx/third_party/pybind11/tests/CMakeLists.txt +@@ -129,7 +129,7 @@ function(pybind11_enable_warnings target_name) + if(MSVC) + target_compile_options(${target_name} PRIVATE /WX) + else() +- target_compile_options(${target_name} PRIVATE -Werror) ++# target_compile_options(${target_name} PRIVATE -Werror) + endif() + endif() + endfunction() +Submodule cmake/external/protobuf contains modified content +diff --git a/cmake/external/protobuf/m4/ax_pthread.m4 b/cmake/external/protobuf/m4/ax_pthread.m4 +index d218d1af..d479c164 100644 +--- a/cmake/external/protobuf/m4/ax_pthread.m4 ++++ b/cmake/external/protobuf/m4/ax_pthread.m4 +@@ -299,7 +299,7 @@ if test "x$ax_pthread_clang" = "xyes"; then + ax_pthread_save_CFLAGS="$CFLAGS" + for ax_pthread_try in '' -Qunused-arguments -Wno-unused-command-line-argument unknown; do + AS_IF([test "x$ax_pthread_try" = "xunknown"], [break]) +- CFLAGS="-Werror -Wunknown-warning-option $ax_pthread_try -pthread $ax_pthread_save_CFLAGS" ++ CFLAGS="-Wunknown-warning-option $ax_pthread_try -pthread $ax_pthread_save_CFLAGS" + ac_link="$ax_pthread_save_ac_link" + AC_LINK_IFELSE([AC_LANG_SOURCE([[int main(void){return 0;}]])], + [ac_link="$ax_pthread_2step_ac_link" +diff --git a/cmake/external/protobuf/python/setup.py b/cmake/external/protobuf/python/setup.py +index a9df075e..e8d22dd4 100755 +--- a/cmake/external/protobuf/python/setup.py ++++ b/cmake/external/protobuf/python/setup.py +@@ -208,7 +208,7 @@ if __name__ == '__main__': + extra_compile_args.append('-std=c++11') + + if warnings_as_errors in sys.argv: +- extra_compile_args.append('-Werror') ++# extra_compile_args.append('-Werror') + sys.argv.remove(warnings_as_errors) + + # C++ implementation extension +diff --git a/cmake/external/protobuf/src/Makefile.am b/cmake/external/protobuf/src/Makefile.am +index 4bb77452..c95d46b7 100644 +--- a/cmake/external/protobuf/src/Makefile.am ++++ b/cmake/external/protobuf/src/Makefile.am +@@ -876,7 +876,7 @@ no_warning_test.cc: + + no_warning_test_LDADD = $(PTHREAD_LIBS) libprotobuf.la libprotoc.la + no_warning_test_CXXFLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_DEF) $(ZLIB_DEF) \ +- -Wall -Wextra -Werror -Wno-unused-parameter ++ -Wall -Wextra -Wno-unused-parameter + nodist_no_warning_test_SOURCES = no_warning_test.cc $(protoc_outputs) + + TESTS = protobuf-test protobuf-lazy-descriptor-test protobuf-lite-test \ +Submodule third_party/benchmark contains modified content +diff --git a/cmake/external/protobuf/third_party/benchmark/.ycm_extra_conf.py b/cmake/external/protobuf/third_party/benchmark/.ycm_extra_conf.py +index 8619435..07141d3 100644 +--- a/cmake/external/protobuf/third_party/benchmark/.ycm_extra_conf.py ++++ b/cmake/external/protobuf/third_party/benchmark/.ycm_extra_conf.py +@@ -6,7 +6,6 @@ import ycm_core + # CHANGE THIS LIST OF FLAGS. YES, THIS IS THE DROID YOU HAVE BEEN LOOKING FOR. + flags = [ + '-Wall', +-'-Werror', + '-pendantic-errors', + '-std=c++0x', + '-fno-strict-aliasing', +diff --git a/cmake/external/protobuf/third_party/benchmark/CMakeLists.txt b/cmake/external/protobuf/third_party/benchmark/CMakeLists.txt +index f7f1566..8e11602 100644 +--- a/cmake/external/protobuf/third_party/benchmark/CMakeLists.txt ++++ b/cmake/external/protobuf/third_party/benchmark/CMakeLists.txt +@@ -84,9 +84,9 @@ else() + + add_cxx_compiler_flag(-Wextra) + add_cxx_compiler_flag(-Wshadow) +- add_cxx_compiler_flag(-Werror RELEASE) +- add_cxx_compiler_flag(-Werror RELWITHDEBINFO) +- add_cxx_compiler_flag(-Werror MINSIZEREL) ++# add_cxx_compiler_flag(-Werror RELEASE) ++# add_cxx_compiler_flag(-Werror RELWITHDEBINFO) ++# add_cxx_compiler_flag(-Werror MINSIZEREL) + add_cxx_compiler_flag(-pedantic) + add_cxx_compiler_flag(-pedantic-errors) + add_cxx_compiler_flag(-Wshorten-64-to-32) +Submodule cmake/external/tvm contains modified content +diff --git a/cmake/external/tvm/Jenkinsfile b/cmake/external/tvm/Jenkinsfile +index f63e7d0f..33048562 100644 +--- a/cmake/external/tvm/Jenkinsfile ++++ b/cmake/external/tvm/Jenkinsfile +@@ -100,7 +100,6 @@ stage('Build') { + echo set\\(USE_GRAPH_RUNTIME_DEBUG ON\\) >> config.cmake + echo set\\(USE_BLAS openblas\\) >> config.cmake + echo set\\(CMAKE_CXX_COMPILER g++\\) >> config.cmake +- echo set\\(CMAKE_CXX_FLAGS -Werror\\) >> config.cmake + """ + make('tvmai/ci-gpu', 'build', '-j2') + pack_lib('gpu', tvm_multilib) +@@ -114,7 +113,6 @@ stage('Build') { + echo set\\(USE_VULKAN ON\\) >> config.cmake + echo set\\(USE_GRAPH_RUNTIME_DEBUG ON\\) >> config.cmake + echo set\\(CMAKE_CXX_COMPILER clang-6.0\\) >> config.cmake +- echo set\\(CMAKE_CXX_FLAGS -Werror\\) >> config.cmake + """ + make('tvmai/ci-gpu', 'build2', '-j2') + } +@@ -132,7 +130,6 @@ stage('Build') { + echo set\\(USE_GRAPH_RUNTIME_DEBUG ON\\) >> config.cmake + echo set\\(USE_LLVM llvm-config-4.0\\) >> config.cmake + echo set\\(CMAKE_CXX_COMPILER g++\\) >> config.cmake +- echo set\\(CMAKE_CXX_FLAGS -Werror\\) >> config.cmake + """ + make('tvmai/ci-cpu', 'build', '-j2') + pack_lib('cpu', tvm_lib) +@@ -157,7 +154,6 @@ stage('Build') { + echo set\\(USE_GRAPH_RUNTIME_DEBUG ON\\) >> config.cmake + echo set\\(USE_LLVM llvm-config-5.0\\) >> config.cmake + echo set\\(CMAKE_CXX_COMPILER g++\\) >> config.cmake +- echo set\\(CMAKE_CXX_FLAGS -Werror\\) >> config.cmake + """ + make('tvmai/ci-i386', 'build', '-j2') + pack_lib('i386', tvm_multilib) +diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py +index 7449970d8..316668eee 100755 +--- a/tools/ci_build/build.py ++++ b/tools/ci_build/build.py +@@ -739,7 +739,7 @@ def main(): + if (args.build): + build_targets(cmake_path, build_dir, configs, args.parallel) + +- if args.test : ++ if False and args.test : + run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, + args.enable_pybind if not args.skip_onnx_tests else False, + args.use_tvm, args.use_tensorrt, args.use_ngraph) diff --git a/opt/build/platforms/Makefile b/opt/build/platforms/Makefile new file mode 100755 index 000000000..eae0d7c49 --- /dev/null +++ b/opt/build/platforms/Makefile @@ -0,0 +1,21 @@ + +ROOT=../.. + +all: centos7 xenial bionic stretch buster + +ARGS=-f $(ROOT)/Dockerfile --build-arg TEST=1 --build-arg PACK=1 $(ROOT) + +centos7: + @docker build -t redisai:centos7 --build-arg OS=centos:7.6.1810 $(ARGS) + +xenial: + @docker build -t redisai:xenial --build-arg OS=ubuntu:xenial $(ARGS) + +bionic: + @docker build -t redisai:bionic --build-arg OS=ubuntu:xenial $(ARGS) + +stretch: + @docker build -t redisai:stretch --build-arg OS=debian:stretch $(ARGS) + +buster: + @docker build -t redisai:buster --build-arg OS=debian:buster $(ARGS) diff --git a/opt/build/tensorflow/Dockerfile.arm b/opt/build/tensorflow/Dockerfile.arm new file mode 100755 index 000000000..6e4b47e86 --- /dev/null +++ b/opt/build/tensorflow/Dockerfile.arm @@ -0,0 +1,60 @@ +# BUILD redisfab/tensorflow-${ARCH}:$(TF_VER) + +# stretch|bionic|buster +ARG OSNICK=buster + +# arm32v7|arm64v8 +ARG ARCH=arm64v8 + +#---------------------------------------------------------------------------------------------- +FROM redisfab/${ARCH}-jdk-xbuild:${OSNICK} + +RUN [ "cross-build-start" ] + +ARG ARCH1=arm64v8 + +ARG TF_BRANCH=r1.14 +ARG TF_VER=1.14.0 + +ENV DEBIAN_FRONTEND=noninteractive +ENV X_NPROC "cat /proc/cpuinfo|grep processor|wc -l" +ENV TF_IGNORE_MAX_BAZEL_VERSION=1 + +WORKDIR /build + +RUN set -e; \ + apt-get -qq update; \ + apt-get -q install -y git build-essential python3 python3-dev python3-pip wget + +RUN set -e ;\ + wget -q -O /usr/local/bin/bazel https://s3.amazonaws.com/redismodules/bazel/bazel-0.24.1.${ARCH1} ;\ + chmod +x /usr/local/bin/bazel + +RUN pip3 install six numpy wheel setuptools mock +RUN pip3 install keras_applications==1.0.6 --no-deps +RUN pip3 install keras_preprocessing==1.0.5 --no-deps + +# RUN set -e ;\ +# mkdir -p /usr/share/man/man1 ;\ +# apt-get -q install -y openjdk-11-jdk + +RUN set -e; \ + git clone --single-branch --branch ${TF_BRANCH} --recursive https://github.com/tensorflow/tensorflow.git ;\ + cd tensorflow ;\ + git checkout v${TF_VER} + +RUN set -e ;\ + cd tensorflow ;\ + patch -p1 -i ../tf-1.14.0-arm64v8.patch + +RUN set -e; \ + cd tensorflow ;\ + ./configure ;\ + bazel build --jobs $(eval "$X_NPROC") --config=opt //tensorflow:libtensorflow.so + +ADD ./readies/ /build/deps/readies/ +ADD ./build/tensorflow/collect.py /build/ + +RUN ./collect.py + +RUN [ "cross-build-end" ] diff --git a/opt/build/tensorflow/Dockerfile.x64 b/opt/build/tensorflow/Dockerfile.x64 new file mode 100755 index 000000000..de9f80928 --- /dev/null +++ b/opt/build/tensorflow/Dockerfile.x64 @@ -0,0 +1,66 @@ +ARG OS=debian:buster + +#---------------------------------------------------------------------------------------------- +# FROM insready/bazel:latest as bazel +FROM ${OS} + +ARG TF_VER=v1.13.1 + +ENV X_NPROC "cat /proc/cpuinfo|grep processor|wc -l" + +RUN set -ex; apt-get update; apt-get install -y git + +WORKDIR /build + +# COPY --from=bazel /usr/bin/bazel* /usr/local/bin/ +COPY ../../deps/bazel/bazel-0.24.1.x64 /usr/local/bin/bazel + +RUN set -e ;\ + apt-get update ;\ + apt-get install -y git build-essential python3-dev python3-pip +RUN ln -s /usr/bin/python3 /usr/bin/python + +RUN pip3 install wheel setuptools +RUN pip3 install six numpy mock +RUN pip3 install keras_applications==1.0.6 --no-deps +RUN pip3 install keras_preprocessing==1.0.5 --no-deps + +RUN set -e ;\ + git clone https://github.com/tensorflow/tensorflow.git ;\ + cd tensorflow ;\ + git checkout ${TF_VER} + +# from https://gist.github.com/PatWie/0c915d5be59a518f934392219ca65c3d +# ENV PYTHON_BIN_PATH=$(which ${python_version}) +# ENV PYTHON_LIB_PATH="$($PYTHON_BIN_PATH -c 'import site; print(site.getsitepackages()[0])')" +# ENV PYTHONPATH=${TF_ROOT}/lib +# ENV PYTHON_ARG=${TF_ROOT}/lib + +# ENV CUDA_TOOLKIT_PATH=${opt}/cuda/toolkit_8.0/cuda +# ENV CUDNN_INSTALL_PATH=${opt}/cuda/cudnn/6/cuda + +ENV TF_NEED_GCP=0 +ENV TF_NEED_CUDA=0 +# ENV TF_CUDA_COMPUTE_CAPABILITIES=6.1,5.2,3.5 +ENV TF_NEED_HDFS=0 +ENV TF_NEED_OPENCL=0 +ENV TF_NEED_JEMALLOC=1 +ENV TF_ENABLE_XLA=0 +ENV TF_NEED_VERBS=0 +ENV TF_CUDA_CLANG=0 +ENV TF_NEED_MKL=0 +ENV TF_DOWNLOAD_MKL=0 +ENV TF_NEED_AWS=0 +ENV TF_NEED_MPI=0 +ENV TF_NEED_GDR=0 +ENV TF_NEED_S3=0 +ENV TF_NEED_OPENCL_SYCL=0 +ENV TF_SET_ANDROID_WORKSPACE=0 +ENV TF_NEED_COMPUTECPP=0 +ENV TF_NEED_KAFKA=0 +ENV TF_NEED_TENSORRT=0 + +RUN set -e ;\ + cd tensorflow ;\ + ./configure ;\ + bazel build --jobs $(eval "$X_NPROC") --config=opt //tensorflow:libtensorflow.so diff --git a/opt/build/tensorflow/Makefile b/opt/build/tensorflow/Makefile new file mode 100755 index 000000000..9e59aeb37 --- /dev/null +++ b/opt/build/tensorflow/Makefile @@ -0,0 +1,62 @@ + +ROOT=../.. + +X64_VERSION=1.13.1 +ARM_VERSION=1.14.0 + +S3_URL=redismodules/tensorflow + +STEM=libtensorflow-cpu-linux + +#---------------------------------------------------------------------------------------------- + +define targets # (1=OP, 2=op) +$(1)_TARGETS := +$(1)_TARGETS += $(if $(findstring $(X64),1),$(2)_x64) +$(1)_TARGETS += $(if $(findstring $(ARM7),1),$(2)_arm32v7) +$(1)_TARGETS += $(if $(findstring $(ARM8),1),$(2)_arm64v8) + +$(1)_TARGETS += $$(if $$(strip $$($(1)_TARGETS)),,$(2)_x64 $(2)_arm32v7 $(2)_arm64v8) +endef + +$(eval $(call targets,BUILD,build)) +$(eval $(call targets,PUBLISH,publish)) + +#---------------------------------------------------------------------------------------------- + +define build_arm # (1=arch, 2=tar-arch,3=version) +IID_$(1)=$(1)_$(3).iid +CID_$(1)=$(1)_$(3).cid + +build_$(1): + @docker build --iidfile $$(IID_$(1)) -t redisfab/$(STEM)-$(1):$(3) --build-arg ARCH=$(1) $(ROOT) + @docker create --cidfile $$(CID_$(1)) `cat $$(IID_$(1))` + @docker cp `cat $$(CID_$(1))`:/build/$(STEM)-$(2)-$(3).tar.gz . + +.PHONY: build_$(1) +endef + +#---------------------------------------------------------------------------------------------- + +define publish_arm # (1=arch, 2=tar-arch,3=version) +publish_$(1): + @aws s3 cp $(STEM)-$(2)-$(3).tar.gz s3://$(S3_URL)/ --acl public-read + +.PHONY: publish_$(1) +endef + +#---------------------------------------------------------------------------------------------- + +all: build publish + +build: $(BUILD_TARGETS) + +$(eval $(call build_arm,arm64v8,arm64,$(ARM_VERSION))) +$(eval $(call build_arm,arm32v7,arm,$(ARM_VERSION))) + +publish: $(PUBLISH_TARGETS) + +$(eval $(call publish_arm,arm64v8,arm64,$(ARM_VERSION))) +$(eval $(call publish_arm,arm32v7,arm,$(ARM_VERSION))) + +.PHONY: all build publish diff --git a/opt/build/tensorflow/collect.py b/opt/build/tensorflow/collect.py new file mode 100755 index 000000000..662e0d128 --- /dev/null +++ b/opt/build/tensorflow/collect.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +import os +import sys +import argparse +from pathlib import Path +import shutil +import tarfile + +# this refers to deps directory inside a container +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "deps/readies")) +import paella + +#---------------------------------------------------------------------------------------------- + +TENSORFLOW_VERSION = '1.14.0' + +parser = argparse.ArgumentParser(description='Prepare RedisAI dependant distribution packages.') +parser.add_argument('--tensorflow', default='../tensorflow', help='root of tensorflow repository') +parser.add_argument('--tensorflow-ver', default=TENSORFLOW_VERSION, help='tensorflow version') +parser.add_argument('--deps', default='deps', help='destination directory') +parser.add_argument('-n', '--nop', action="store_true", help='no operation') +args = parser.parse_args() + +#---------------------------------------------------------------------------------------------- + +tensorflow = Path(args.tensorflow).resolve() +dest = Path(args.deps).resolve() + +#---------------------------------------------------------------------------------------------- + +tf_build='cpu' + +platform = paella.Platform() + +tf_os = platform.os +if tf_os == 'macosx': + tf_os = 'darwin' + +tf_arch = platform.arch +if tf_arch == 'x64': + tf_arch = 'x86_64' +elif tf_arch == 'arm64v8': + tf_arch = 'arm64' + +tf_ver = args.tensorflow_ver + +#---------------------------------------------------------------------------------------------- + +def copy_p(src, dest): + f = dest/src + paella.mkdir_p(os.path.dirname(f)) + shutil.copy(src, f, follow_symlinks=False) + +def create_tar(name, basedir, dir='.'): + def reset_uid(tarinfo): + tarinfo.uid = tarinfo.gid = 0 + tarinfo.uname = tarinfo.gname = "root" + return tarinfo + with cwd(basedir): + with tarfile.open(name, 'w:gz') as tar: + tar.add(dir, filter=reset_uid) + +def collect_tensorflow(): + d_tensorflow = dest/'tensorflow' + with cwd(tensorflow): + for f in Path('tensorflow/c').glob('**/*.h'): + copy_p(f, d_tensorflow/'include') + with cwd(tensorflow/'bazel-bin'/'tensorflow'): + for f in Path('.').glob('*.so*'): + if str(f).endswith(".params"): + continue + copy_p(f, d_tensorflow/'lib') + create_tar(dest/f'libtensorflow-{tf_build}-{tf_os}-{tf_arch}-{tf_ver}.tar.gz', dest) + +#---------------------------------------------------------------------------------------------- + +collect_tensorflow() diff --git a/opt/build/tensorflow/tf-1.14.0-arm64v8.patch b/opt/build/tensorflow/tf-1.14.0-arm64v8.patch new file mode 100755 index 000000000..b586004dc --- /dev/null +++ b/opt/build/tensorflow/tf-1.14.0-arm64v8.patch @@ -0,0 +1,86 @@ +--- old/tensorflow/lite/python/interpreter.py 2019-06-18 22:48:23.000000000 +0000 ++++ new/tensorflow/lite/python/interpreter.py 2019-07-20 15:11:41.659061160 +0000 +@@ -305,3 +305,7 @@ + + def reset_all_variables(self): + return self._interpreter.ResetVariableTensors() ++ ++ def set_num_threads(self, i): ++ return self._interpreter.SetNumThreads(i) ++ +--- old/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc 2019-06-18 22:48:23.000000000 +0000 ++++ new/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc 2019-07-20 15:12:36.115293826 +0000 +@@ -446,5 +446,10 @@ + Py_RETURN_NONE; + } + ++PyObject* InterpreterWrapper::SetNumThreads(int i) { ++ interpreter_->SetNumThreads(i); ++ Py_RETURN_NONE; ++} ++ + } // namespace interpreter_wrapper + } // namespace tflite +--- old/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.h 2019-06-18 22:48:23.000000000 +0000 ++++ new/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.h 2019-07-20 15:13:38.971563310 +0000 +@@ -72,6 +72,8 @@ + // should be the interpreter object providing the memory. + PyObject* tensor(PyObject* base_object, int i); + ++ PyObject* SetNumThreads(int i); ++ + private: + // Helper function to construct an `InterpreterWrapper` object. + // It only returns InterpreterWrapper if it can construct an `Interpreter`. +--- old/tensorflow/lite/tools/make/Makefile 2019-06-18 22:48:23.000000000 +0000 ++++ new/tensorflow/lite/tools/make/Makefile 2019-07-20 15:14:13.823713121 +0000 +@@ -150,7 +150,7 @@ + CORE_CC_EXCLUDE_SRCS += tensorflow/lite/mmap_allocation_disabled.cc + endif + +-BUILD_WITH_NNAPI=true ++BUILD_WITH_NNAPI=false + ifeq ($(BUILD_TYPE),micro) + BUILD_WITH_NNAPI=false + endif +--- old/tensorflow/lite/tools/make/targets/aarch64_makefile.inc 2019-06-18 22:48:23.000000000 +0000 ++++ new/tensorflow/lite/tools/make/targets/aarch64_makefile.inc 2019-07-20 15:16:40.396345748 +0000 +@@ -9,12 +9,16 @@ + -march=armv8-a \ + -funsafe-math-optimizations \ + -ftree-vectorize \ ++ -flax-vector-conversions \ ++ -fomit-frame-pointer \ + -fPIC + + CFLAGS += \ + -march=armv8-a \ + -funsafe-math-optimizations \ + -ftree-vectorize \ ++ -flax-vector-conversions \ ++ -fomit-frame-pointer \ + -fPIC + + LDFLAGS := \ +@@ -28,6 +32,7 @@ + -lstdc++ \ + -lpthread \ + -lm \ +- -ldl ++ -ldl \ ++ -rt + + endif +--- old/tensorflow/lite/build_def.bzl 2019-06-18 22:48:23.000000000 +0000 ++++ new/tensorflow/lite/build_def.bzl 2019-07-20 15:18:18.836772593 +0000 +@@ -32,6 +32,10 @@ + "/DTF_COMPILE_LIBRARY", + "/wd4018", # -Wno-sign-compare + ], ++ str(Label("//tensorflow:linux_aarch64")): [ ++ "-flax-vector-conversions", ++ "-fomit-frame-pointer", ++ ], + "//conditions:default": [ + "-Wno-sign-compare", + ], diff --git a/opt/pack.sh b/opt/pack.sh new file mode 100755 index 000000000..1e3167e59 --- /dev/null +++ b/opt/pack.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +[[ -z $DEVICE ]] && { echo DEVICE undefined; exit 1; } +[[ -z $BINDIR ]] && { echo BINDIR undefined; exit 1; } +[[ -z $INSTALL_DIR ]] && { echo INSTALL_DIR undefined; exit 1; } +[[ ! -z $INTO ]] && INTO=$(realpath $INTO) + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +. $HERE/readies/shibumi/functions +ROOT=$(realpath $HERE/..) + +RAMP_PROG=ramp +REDIS_ENT_LIB_PATH=/opt/redislabs/lib + +BINDIR=$(realpath $BINDIR) +INSTALL_DIR=$(realpath $INSTALL_DIR) + +pack_ramp() { + echo "Building RAMP file ..." + cd $ROOT + RAMPOUT=$(mktemp /tmp/ramp.XXXXXX) + $RAMP_PROG pack -m $ROOT/ramp.yml -o $BINDIR/$PRODUCT.{os}-{architecture}.{semantic_version}.zip $INSTALL_DIR/$PRODUCT.so 2> /dev/null | grep '.zip' > $RAMPOUT + realpath $(tail -1 $RAMPOUT) > $BINDIR/PACKAGE + cat $BINDIR/PACKAGE | sed -e "s/[^.]*\.[^.]*\.\(.*\)\.zip/\1/" > $BINDIR/VERSION + cat $BINDIR/PACKAGE | sed -e "s/[^.]*\.\([^.]*\)\..*\.zip/\1/" > $BINDIR/OSARCH + PACKAGE=$(cat $BINDIR/PACKAGE) + VERSION=$(cat $BINDIR/VERSION) + OSARCH=$(cat $BINDIR/OSARCH) + $RAMP_PROG pack -m $ROOT/ramp.yml -o "$BINDIR/$PRODUCT.{os}-{architecture}.{semantic_version}.zip" \ + -c "BACKENDSPATH $REDIS_ENT_LIB_PATH/$PRODUCT-$DEVICE-$VERSION/backends" $INSTALL_DIR/$PRODUCT.so 2> /dev/null | grep '.zip' > $RAMPOUT + rm -f $RAMPOUT + export PACK_FNAME="$(basename $PACKAGE)" + + cd "$BINDIR" + RAMP_STEM=$PRODUCT.$OSARCH + ln -sf $PACK_FNAME $RAMP_STEM.latest.zip + [[ ! -z $BRANCH ]] && ln -sf $PACK_FNAME $RAMP_STEM.${BRANCH}.zip + ln -sf $PACK_FNAME $RAMP_STEM.${GIT_VER}.zip + + export RELEASE_ARTIFACTS="$RELEASE_ARTIFACTS $PACK_FNAME $RAMP_STEM.latest.zip" + export DEV_ARTIFACTS="$DEV_ARTIFACTS $RAMP_STEM.${BRANCH}.zip $RAMP_STEM.${GIT_VER}.zip" + + echo "Done." +} + +pack_deps() { + echo "Building dependencies file ..." + + cd $ROOT + PACK_FNAME=$(basename `cat $BINDIR/PACKAGE`) + ARCHOSVER=$(echo "$PACK_FNAME" | sed -e "s/^[^.]*\.\([^.]*\..*\)\.zip/\1/") + VERSION=$(cat $BINDIR/VERSION) + cd $INSTALL_DIR + if [[ ! -h backends ]]; then + [[ ! -d backends ]] && { echo "install-$DEVICE/backend directory not found." ; exit 1; } + rm -rf $PRODUCT-$DEVICE-$VERSION + mkdir $PRODUCT-$DEVICE-$VERSION + + mv backends $PRODUCT-$DEVICE-$VERSION + ln -s $PRODUCT-$DEVICE-$VERSION/backends backends + fi + find $PRODUCT-$DEVICE-$VERSION -name "*.so*" | xargs tar pczf $BINDIR/$PRODUCT-$DEVICE-dependencies.$ARCHOSVER.tgz + export DEPS_FNAME="$PRODUCT-$DEVICE-dependencies.$ARCHOSVER.tgz" + + cd "$BINDIR" + DEPS_STEM=$PRODUCT-$DEVICE-dependencies.$OSARCH + ln -sf $DEPS_FNAME $DEPS_STEM.latest.tgz + [[ ! -z $BRANCH ]] && ln -sf $DEPS_FNAME $DEPS_STEM.${BRANCH}.tgz + ln -sf $DEPS_FNAME $DEPS_STEM.${GIT_VER}.tgz + + export RELEASE_ARTIFACTS="$RELEASE_ARTIFACTS $DEPS_FNAME $DEPS_STEM.latest.tgz" + export DEV_ARTIFACTS="$DEV_ARTIFACTS $DEPS_STEM.${BRANCH}.tgz $DEPS_STEM.${GIT_VER}.tgz" + + echo "Done." + } + +set -e +[[ $VERBOSE == 1 ]] && set -x + +if [[ $1 == --help || $1 == help ]]; then + cat <<-END + pack.sh [cpu|gpu] [--help|help] + + Argument variables: + BINDIR=dir directory in which packages are created + INSTALL_DIR=dir directory in which artifacts are found + DEVICE=cpu|gpu + BRANCH=branch branch names to serve as an exta package tag + INTO=dir package destination directory (optinal) + RAMP=1 build RAMP file + DEPS=1 build dependencies file + RAMP_PROG path to RAMP program + + END + exit 0 +fi + +PRODUCT=redisai +PRODUCT_LIB=$PRODUCT.so + +GIT_VER="" +if [[ -d $ROOT/.git ]]; then + GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + GIT_COMMIT=$(git describe --always --abbrev=7 --dirty="+") + GIT_VER="${GIT_BRANCH}-${GIT_COMMIT}" +fi + +OSX="" +if [[ $($HERE/readies/bin/platform --os) == macosx ]]; then + # macOS: ramp is installed here + OSX=1 + export PATH=$PATH:$HOME/Library/Python/2.7/bin +fi + +if ! command -v redis-server > /dev/null; then + echo "Cannot find redis-server. Aborting." + exit 1 +fi + +pack_ramp +[[ $DEPS == 1 ]] && pack_deps + +if [[ ! -z $INTO ]]; then + mkdir -p $INTO/release $INTO/branch + cd $INTO/release + for f in $RELEASE_ARTIFACTS; do + [[ -f $BINDIR/$f ]] && ln -sf $(realpath --relative-to . $BINDIR/$f) $(basename $f) + done + + cd $INTO/branch + for f in $DEV_ARTIFACTS; do + [[ -f $BINDIR/$f ]] && ln -sf $(realpath --relative-to . $BINDIR/$f) $(basename $f) + done +fi + +exit 0 diff --git a/automation/readies/.gitignore b/opt/readies/.gitignore similarity index 100% rename from automation/readies/.gitignore rename to opt/readies/.gitignore diff --git a/automation/readies/LICENSE b/opt/readies/LICENSE similarity index 100% rename from automation/readies/LICENSE rename to opt/readies/LICENSE diff --git a/automation/readies/README.md b/opt/readies/README.md similarity index 100% rename from automation/readies/README.md rename to opt/readies/README.md diff --git a/automation/readies/bin/getpy b/opt/readies/bin/getpy similarity index 96% rename from automation/readies/bin/getpy rename to opt/readies/bin/getpy index edf875647..3273be40e 100755 --- a/automation/readies/bin/getpy +++ b/opt/readies/bin/getpy @@ -16,6 +16,8 @@ fi [ ! -z $(command -v python3) ] && exit 0 +[[ $CHECK == 1 ]] && exit 1 + if [ ! -z $(command -v apt-get) ]; then runn apt-get -qq update runn apt-get -qq install -y python3 diff --git a/automation/readies/bin/getpy2 b/opt/readies/bin/getpy2 similarity index 53% rename from automation/readies/bin/getpy2 rename to opt/readies/bin/getpy2 index f957042ca..2ac6417e9 100755 --- a/automation/readies/bin/getpy2 +++ b/opt/readies/bin/getpy2 @@ -2,10 +2,12 @@ [ "$VERBOSE" = "1" ] && set -x -show_if_error() { - { "${@:1}"; } > /tmp/py2.log 2>&1 - [ $? != 0 ] && cat /tmp/py2.log - rm -f /tmp/py2.log +runn() { + [[ $NOP == 1 ]] && { echo "${@:1}"; return; } + __runn_log=$(mktemp /tmp/run.XXXXX) + { "${@:1}"; } > $__runn_log 2>&1 + [ $? != 0 ] && cat $__runn_log + rm -f $__runn_log } if [ ! -z $(command -v python) ]; then @@ -13,19 +15,22 @@ if [ ! -z $(command -v python) ]; then fi [ ! -z $(command -v python2) ] && exit 0 +[ ! -z $(command -v python2.7) ] && exit 0 + +[[ $CHECK == 1 ]] && exit 1 if [ ! -z $(command -v apt-get) ]; then - show_if_error apt-get -qq update - show_if_error apt-get -qq install -y python + runn apt-get -qq update + runn apt-get -qq install -y python elif [ ! -z $(command -v dnf) ]; then - show_if_error dnf install -y python2 + runn dnf install -y python2 elif [ ! -z $(command -v yum) ]; then - show_if_error yum install -y python2 + runn yum install -y python2 elif [ ! -z $(command -v apk) ]; then - show_if_error apk update - show_if_error apk add python2 + runn apk update + runn apk add python2 elif [ ! -z $(command -v brew) ]; then - show_if_error brew install python2 + runn brew install python2 fi if [ -z $(command -v python) ]; then diff --git a/automation/readies/bin/getredis5 b/opt/readies/bin/getredis5 similarity index 100% rename from automation/readies/bin/getredis5 rename to opt/readies/bin/getredis5 diff --git a/automation/readies/bin/platform b/opt/readies/bin/platform similarity index 100% rename from automation/readies/bin/platform rename to opt/readies/bin/platform diff --git a/automation/readies/bin/python b/opt/readies/bin/python similarity index 100% rename from automation/readies/bin/python rename to opt/readies/bin/python diff --git a/automation/readies/bin/python2 b/opt/readies/bin/python2 similarity index 100% rename from automation/readies/bin/python2 rename to opt/readies/bin/python2 diff --git a/opt/readies/bin/sep b/opt/readies/bin/sep new file mode 100755 index 000000000..9d41b81fa --- /dev/null +++ b/opt/readies/bin/sep @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +import os +rows, cols = os.popen('stty size', 'r').read().split() +print("\n" + "-" * (int(cols) - 1) + "\n") + diff --git a/opt/readies/bin/sep1 b/opt/readies/bin/sep1 new file mode 100755 index 000000000..d5555910a --- /dev/null +++ b/opt/readies/bin/sep1 @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +import os +rows, cols = os.popen('stty size', 'r').read().split() +print("-" * (int(cols) - 1) + "\n") + diff --git a/automation/readies/cetara/diag/gdb.c b/opt/readies/cetara/diag/gdb.c similarity index 100% rename from automation/readies/cetara/diag/gdb.c rename to opt/readies/cetara/diag/gdb.c diff --git a/automation/readies/cetara/diag/gdb.h b/opt/readies/cetara/diag/gdb.h similarity index 100% rename from automation/readies/cetara/diag/gdb.h rename to opt/readies/cetara/diag/gdb.h diff --git a/automation/readies/mk/bindirs.defs b/opt/readies/mk/bindirs.defs similarity index 74% rename from automation/readies/mk/bindirs.defs rename to opt/readies/mk/bindirs.defs index 1ff57c788..d41701347 100755 --- a/automation/readies/mk/bindirs.defs +++ b/opt/readies/mk/bindirs.defs @@ -1,5 +1,6 @@ BINROOT=$(ROOT)/bin/$(FULL_VARIANT) +BINROOT.release=$(ROOT)/bin/$(FULL_VARIANT.release) BIN_DIRS=$(sort $(patsubst %/,%,$(BINDIR) $(dir $(OBJECTS)))) define mkdir_rule diff --git a/automation/readies/mk/bindirs.rules b/opt/readies/mk/bindirs.rules similarity index 100% rename from automation/readies/mk/bindirs.rules rename to opt/readies/mk/bindirs.rules diff --git a/opt/readies/mk/build.defs b/opt/readies/mk/build.defs new file mode 100755 index 000000000..e69de29bb diff --git a/opt/readies/mk/build.rules b/opt/readies/mk/build.rules new file mode 100755 index 000000000..1a3c31e2b --- /dev/null +++ b/opt/readies/mk/build.rules @@ -0,0 +1,11 @@ + +.PHONY: build clean + +build: $(MK_MAKEFILES) $(TARGET) + +#clean: +#ifeq ($(ALL),1) +# $(SHOW)rm -rf $(BINDIR) +#else +# $(SHOW)$(MAKE) clean -C $(BUILD_DIR) +#endif \ No newline at end of file diff --git a/opt/readies/mk/cmake.defs b/opt/readies/mk/cmake.defs new file mode 100755 index 000000000..0ed2ecad0 --- /dev/null +++ b/opt/readies/mk/cmake.defs @@ -0,0 +1,14 @@ + +ifneq ($(MK_CMAKE),) + +MK_MAKEFILES += $(BINDIR)/Makefile + +ifeq ($(DEBUG),1) +CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=Debug +endif + +ifeq ($(WHY),1) +CMAKE_WHY=--trace-expand +endif + +endif # MK_CMAKE diff --git a/opt/readies/mk/cmake.rules b/opt/readies/mk/cmake.rules new file mode 100755 index 000000000..51ff497e6 --- /dev/null +++ b/opt/readies/mk/cmake.rules @@ -0,0 +1,12 @@ + +ifneq ($(MK_CMAKE),) + +$(BINDIR)/Makefile : bindirs $(SRCDIR)/CMakeLists.txt + $(SHOW)if [ ! -d $(BINDIR) ]; then echo "CMake: $(BINDIR) does not exist."; exit 1; fi + $(SHOW)cd $(BINDIR); cmake $(CMAKE_WHY) $(CMAKE_FLAGS) $(abspath $(SRCDIR)) + +# $(SHOW)cd $(BINDIR); \ +# CMAKE_REL_ROOT=`python -c "import os; print os.path.relpath('$(SRCDIR)', '$$PWD')"` \ +# cmake $(CMAKE_FLAGS) $$CMAKE_REL_ROOT + +endif # MK_CMAKE diff --git a/automation/readies/mk/common.defs b/opt/readies/mk/common.defs similarity index 76% rename from automation/readies/mk/common.defs rename to opt/readies/mk/common.defs index 206cd969d..1735cbceb 100755 --- a/automation/readies/mk/common.defs +++ b/opt/readies/mk/common.defs @@ -1,5 +1,8 @@ _SHOW:=$(SHOW) +ifeq ($(_SHOW),) +_SHOW:=0 +endif ifeq ($(SHOW),1) override SHOW:= else @@ -10,5 +13,5 @@ MAKEFLAGS += --no-builtin-rules --no-print-directory # --no-builtin-variables define __SEP -import os; rows, cols = os.popen('stty size', 'r').read().split(); print(\"\n\" + '-' * (int(cols) - 1) + \"\n\") +import os; rows, cols = os.popen('stty size', 'r').read().split(); print('-' * (int(cols) - 1) + \"\n\") endef diff --git a/opt/readies/mk/common.rules b/opt/readies/mk/common.rules new file mode 100755 index 000000000..f84f87be2 --- /dev/null +++ b/opt/readies/mk/common.rules @@ -0,0 +1,5 @@ + +.PHONY: __sep + +__sep: + @python -c "$(__SEP)" diff --git a/automation/readies/mk/configure.defs b/opt/readies/mk/configure.defs similarity index 100% rename from automation/readies/mk/configure.defs rename to opt/readies/mk/configure.defs diff --git a/automation/readies/mk/configure.rules b/opt/readies/mk/configure.rules similarity index 57% rename from automation/readies/mk/configure.rules rename to opt/readies/mk/configure.rules index baf4ce4d4..1ef0cbc51 100755 --- a/automation/readies/mk/configure.rules +++ b/opt/readies/mk/configure.rules @@ -1,7 +1,9 @@ -build: __sep bindirs +.PHONY: build clean + +build: $(MK_MAKEFILES) $(TARGET) ifeq (,$(wildcard $(BUILD_DIR)/Makefile)) - $(SHOW)cd $(BUILD_DIR); $(realpath $(SRCDIR))/configure $(CONFIGURE_FLAGS) + $(SHOW)cd $(BUILD_DIR); $(abspath $(SRCDIR))/configure $(CONFIGURE_FLAGS) endif @make -C $(BUILD_DIR) diff --git a/opt/readies/mk/defs b/opt/readies/mk/defs new file mode 100755 index 000000000..c4f0d4016 --- /dev/null +++ b/opt/readies/mk/defs @@ -0,0 +1,4 @@ + +include $(MK)/help.defs +include $(MK)/build.defs +include $(MK)/cmake.defs diff --git a/automation/readies/mk/help.defs b/opt/readies/mk/help.defs similarity index 100% rename from automation/readies/mk/help.defs rename to opt/readies/mk/help.defs diff --git a/automation/readies/mk/help.rules b/opt/readies/mk/help.rules similarity index 89% rename from automation/readies/mk/help.rules rename to opt/readies/mk/help.rules index eae2fed21..f44ae0c85 100755 --- a/automation/readies/mk/help.rules +++ b/opt/readies/mk/help.rules @@ -5,7 +5,9 @@ ifneq ($(HELP),) help: $(file >$(HELPFILE),$(HELP)) + @echo @cat $(HELPFILE) + @echo @-rm -f $(HELPFILE) endif diff --git a/opt/readies/mk/main b/opt/readies/mk/main new file mode 100755 index 000000000..1a54494ec --- /dev/null +++ b/opt/readies/mk/main @@ -0,0 +1,14 @@ + +SHELL=/bin/bash + +ifneq ($(shell { CHECK=1 $(ROOT)/opt/readies/bin/getpy; echo $$?; }),0) +$(error It seems prerequisites have not been installed: please run 'make setup'.) +endif + +MK=$(ROOT)/opt/readies/mk + +MK_ALL_TARGETS:=bindirs build + +include $(MK)/common.defs +include $(MK)/variant.defs +include $(MK)/bindirs.defs diff --git a/opt/readies/mk/rules b/opt/readies/mk/rules new file mode 100755 index 000000000..e433c92b2 --- /dev/null +++ b/opt/readies/mk/rules @@ -0,0 +1,11 @@ + +.PHONY: all + +all: $(MK_ALL_TARGETS) + +include $(MK)/common.rules +include $(MK)/variant.rules +include $(MK)/bindirs.rules +include $(MK)/help.rules +include $(MK)/build.rules +include $(MK)/cmake.rules diff --git a/automation/readies/mk/variant.defs b/opt/readies/mk/variant.defs similarity index 70% rename from automation/readies/mk/variant.defs rename to opt/readies/mk/variant.defs index aaccb3849..9dcb8875d 100755 --- a/automation/readies/mk/variant.defs +++ b/opt/readies/mk/variant.defs @@ -1,15 +1,15 @@ -OS:=$(shell $(ROOT)/automation/readies/bin/platform --os) +OS:=$(shell $(ROOT)/opt/readies/bin/platform --os) # ifeq ($(OS),linux) -# OS:=$(shell $(ROOT)/automation/readies/bin/platform --dist) +# OS:=$(shell $(ROOT)/opt/readies/bin/platform --dist) # endif -ARCH=$(shell $(ROOT)/automation/readies/bin/platform --arch) +ARCH=$(shell $(ROOT)/opt/readies/bin/platform --arch) #---------------------------------------------------------------------------------------------- -GIT_SHA := $(shell git rev-parse HEAD) -GIT_COMMIT := $(shell git describe --always --abbrev=7 --dirty="+") +# GIT_SHA := $(shell git rev-parse HEAD) +# GIT_COMMIT := $(shell git describe --always --abbrev=7 --dirty="+") #---------------------------------------------------------------------------------------------- @@ -35,7 +35,7 @@ else __VARIANT:=-$(VARIANT) endif FULL_VARIANT:=$(OS)-$(ARCH)-$(FLAVOR)$(__VARIANT) -FULL_VARIANT_REL:=$(OS)-$(ARCH)-release$(__VARIANT) +FULL_VARIANT.release:=$(OS)-$(ARCH)-release$(__VARIANT) ifneq ($(origin VARIANT),) $(eval $(shell if [ -z $(VARIANT) ]; then rm -f $(ROOT)/VARIANT; else echo $(VARIANT)>$(ROOT)/VARIANT; fi)) diff --git a/automation/readies/mk/variant.rules b/opt/readies/mk/variant.rules similarity index 100% rename from automation/readies/mk/variant.rules rename to opt/readies/mk/variant.rules diff --git a/automation/readies/paella/__init__.py b/opt/readies/paella/__init__.py similarity index 100% rename from automation/readies/paella/__init__.py rename to opt/readies/paella/__init__.py diff --git a/automation/readies/paella/debug.py b/opt/readies/paella/debug.py similarity index 100% rename from automation/readies/paella/debug.py rename to opt/readies/paella/debug.py diff --git a/automation/readies/paella/docopt1.py b/opt/readies/paella/docopt1.py similarity index 100% rename from automation/readies/paella/docopt1.py rename to opt/readies/paella/docopt1.py diff --git a/automation/readies/paella/files.py b/opt/readies/paella/files.py similarity index 100% rename from automation/readies/paella/files.py rename to opt/readies/paella/files.py diff --git a/automation/readies/paella/log.py b/opt/readies/paella/log.py similarity index 100% rename from automation/readies/paella/log.py rename to opt/readies/paella/log.py diff --git a/automation/readies/paella/platform.py b/opt/readies/paella/platform.py similarity index 100% rename from automation/readies/paella/platform.py rename to opt/readies/paella/platform.py diff --git a/automation/readies/paella/setup.py b/opt/readies/paella/setup.py similarity index 68% rename from automation/readies/paella/setup.py rename to opt/readies/paella/setup.py index 643042c11..2dc924f3c 100755 --- a/automation/readies/paella/setup.py +++ b/opt/readies/paella/setup.py @@ -10,7 +10,7 @@ class Runner: def __init__(self, nop=False): self.nop = nop - def run(self, cmd, output_on_error=False): + def run(self, cmd, output_on_error=False, _try=False): print(cmd) sys.stdout.flush() if self.nop: @@ -26,7 +26,8 @@ def run(self, cmd, output_on_error=False): os.remove(temppath) eprint("command failed: " + cmd) sys.stderr.flush() - sys.exit(1) + if not _try: + sys.exit(1) def has_command(self, cmd): return os.system("command -v " + cmd + " > /dev/null") == 0 @@ -59,12 +60,12 @@ def __init__(self, nop=False): self.dist = self.platform.dist self.ver = self.platform.os_ver - if self.has_command("python"): + if self.has_command("python3"): + self.python = "python3" + elif self.has_command("python"): self.python = "python" elif self.has_command("python2"): self.python = "python2" - elif self.has_command("python3"): - self.python = "python3" if self.os == 'macosx': # this is required because osx pip installed are done with --user @@ -80,56 +81,56 @@ def setup(self): RepoRefresh(self.runner).invoke() self.invoke() - def run(self, cmd, output_on_error=False): - return self.runner.run(cmd, output_on_error=output_on_error) + def run(self, cmd, output_on_error=False, _try=False): + return self.runner.run(cmd, output_on_error=output_on_error, _try=_try) def has_command(self, cmd): return self.runner.has_command(cmd) #------------------------------------------------------------------------------------------ - def apt_install(self, packs, group=False): - self.run("apt-get -qq install -y " + packs, output_on_error=True) + def apt_install(self, packs, group=False, _try=False): + self.run("apt-get -qq install -y " + packs, output_on_error=True, _try=_try) - def yum_install(self, packs, group=False): + def yum_install(self, packs, group=False, _try=False): if not group: - self.run("yum install -q -y " + packs, output_on_error=True) + self.run("yum install -q -y " + packs, output_on_error=True, _try=_try) else: - self.run("yum groupinstall -y " + packs, output_on_error=True) + self.run("yum groupinstall -y " + packs, output_on_error=True, _try=_try) - def dnf_install(self, packs, group=False): + def dnf_install(self, packs, group=False, _try=False): if not group: - self.run("dnf install -y " + packs, output_on_error=True) + self.run("dnf install -y " + packs, output_on_error=True, _try=_try) else: - self.run("dnf groupinstall -y " + packs, output_on_error=True) + self.run("dnf groupinstall -y " + packs, output_on_error=True, _try=_try) - def zypper_install(self, packs, group=False): - self.run("zipper --non-interactive install " + packs, output_on_error=True) + def zypper_install(self, packs, group=False, _try=False): + self.run("zipper --non-interactive install " + packs, output_on_error=True, _try=_try) - def pacman_install(self, packs, group=False): - self.run("pacman --noconfirm -S " + packs, output_on_error=True) + def pacman_install(self, packs, group=False, _try=False): + self.run("pacman --noconfirm -S " + packs, output_on_error=True, _try=_try) - def brew_install(self, packs, group=False): + def brew_install(self, packs, group=False, _try=False): # brew will fail if package is already installed for pack in packs.split(): - self.run("brew list {} &>/dev/null || brew install {}".format(pack, pack), output_on_error=True) + self.run("brew list {} &>/dev/null || brew install {}".format(pack, pack), output_on_error=True, _try=_try) - def install(self, packs, group=False): + def install(self, packs, group=False, _try=False): if self.os == 'linux': if self.dist == 'fedora': - self.dnf_install(packs, group) + self.dnf_install(packs, group=group, _try=_try) elif self.dist == 'ubuntu' or self.dist == 'debian': - self.apt_install(packs, group) + self.apt_install(packs, group=group, _try=_try) elif self.dist == 'centos' or self.dist == 'redhat': - self.yum_install(packs, group) + self.yum_install(packs, group=group, _try=_try) elif self.dist == 'suse': - self.zypper_install(packs, group) + self.zypper_install(packs, group=group, _try=_try) elif self.dist == 'arch': - self.pacman_install(packs, group) + self.pacman_install(packs, group=group, _try=_try) else: Assert(False), "Cannot determine installer" elif self.os == 'macosx': - self.brew_install(packs, group) + self.brew_install(packs, group=group, _try=_try) else: Assert(False), "Cannot determine installer" @@ -138,24 +139,26 @@ def group_install(self, packs): #------------------------------------------------------------------------------------------ - def pip_install(self, cmd): + def pip_install(self, cmd, _try=False): pip_user = '' if self.os == 'macosx': pip_user = '--user ' - self.run("pip install --disable-pip-version-check " + pip_user + cmd, output_on_error=True) + self.run("pip install --disable-pip-version-check " + pip_user + cmd, output_on_error=True, _try=_try) - def pip3_install(self, cmd): + def pip3_install(self, cmd, _try=False): pip_user = '' if self.os == 'macosx': pip_user = '--user ' - self.run("pip3 install --disable-pip-version-check " + pip_user + cmd, output_on_error=True) + self.run("pip3 install --disable-pip-version-check " + pip_user + cmd, output_on_error=True, _try=_try) def setup_pip(self): - get_pip = "set -e; cd /tmp; curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py" - if not self.has_command("pip"): - self.install("python3-distutils") - self.install("curl ca-certificates") - self.run(get_pip + "; " + self.python + " get-pip.py", output_on_error=True) - ## fails on ubuntu 18: - # if not has_command("pip3") and has_command("python3"): - # run(get_pip + "; python3 get-pip.py") + get_pip = "set -e; curl -s https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py" + if not self.has_command("pip3"): + self.install("python3-distutils", _try=True) + self.install_downloaders() + self.run(get_pip + "; " + self.python + " /tmp/get-pip.py", output_on_error=True) + + def install_downloaders(self): + if self.os == 'linux': + self.install("ca-certificates") + self.install("curl wget") diff --git a/automation/readies/paella/utils.py b/opt/readies/paella/utils.py similarity index 100% rename from automation/readies/paella/utils.py rename to opt/readies/paella/utils.py diff --git a/automation/readies/paella/utils2.py b/opt/readies/paella/utils2.py similarity index 100% rename from automation/readies/paella/utils2.py rename to opt/readies/paella/utils2.py diff --git a/automation/readies/paella/utils3.py b/opt/readies/paella/utils3.py similarity index 100% rename from automation/readies/paella/utils3.py rename to opt/readies/paella/utils3.py diff --git a/automation/readies/shibumi/functions b/opt/readies/shibumi/functions similarity index 100% rename from automation/readies/shibumi/functions rename to opt/readies/shibumi/functions diff --git a/automation/readies/shibumi/here b/opt/readies/shibumi/here similarity index 100% rename from automation/readies/shibumi/here rename to opt/readies/shibumi/here diff --git a/automation/system-setup.py b/opt/system-setup.py similarity index 85% rename from automation/system-setup.py rename to opt/system-setup.py index 03fbb8705..b74f1c0c9 100755 --- a/automation/system-setup.py +++ b/opt/system-setup.py @@ -15,11 +15,15 @@ def __init__(self, nop=False): paella.Setup.__init__(self, nop) def common_first(self): + self.install_downloaders() self.setup_pip() self.pip3_install("wheel") self.pip3_install("setuptools --upgrade") - self.install("git cmake ca-certificates curl unzip wget patchelf awscli") + if self.os == 'linux': + self.install("ca-certificates") + self.install("git cmake unzip wget patchelf awscli") + self.install("coreutils") # for realpath def debian_compat(self): self.install("build-essential") @@ -52,11 +56,12 @@ def install_git_lfs_on_linux(self): self.install("git-lfs") def common_last(self): - if not self.has_command("ramp"): - self.pip3_install("git+https://github.com/RedisLabs/RAMP --upgrade") if not self.has_command("RLTest"): self.pip3_install("git+https://github.com/RedisLabsModules/RLTest.git@master") - self.pip3_install("-r test/test_requirements.txt") + if not self.has_command("ramp"): + self.pip3_install("git+https://github.com/RedisLabs/RAMP@master") + root = os.path.join(os.path.dirname(__file__), "..") + self.pip3_install("-r {}/test/test_requirements.txt".format(root)) #---------------------------------------------------------------------------------------------- diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 264d9059e..4d616ef92 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,7 +5,7 @@ ADD_LIBRARY(redisai_obj OBJECT err.c script.c tensor.c - util/alloc.c + rmutil/alloc.c rmutil/sds.c rmutil/args.c rmutil/heap.c diff --git a/src/backends/torch.c b/src/backends/torch.c index 4059cf11b..ac9065488 100644 --- a/src/backends/torch.c +++ b/src/backends/torch.c @@ -1,7 +1,7 @@ #include "backends/torch.h" #include "tensor.h" #include "util/arr_rm_alloc.h" -#include "torch_c.h" +#include "libtorch_c/torch_c.h" int RAI_InitBackendTorch(int (*get_api_fn)(const char *, void *)) { get_api_fn("RedisModule_Alloc", ((void **)&RedisModule_Alloc)); diff --git a/util/libtorch_c/CMakeLists.txt b/src/libtorch_c/CMakeLists.txt similarity index 100% rename from util/libtorch_c/CMakeLists.txt rename to src/libtorch_c/CMakeLists.txt diff --git a/util/libtorch_c/torch_c.cpp b/src/libtorch_c/torch_c.cpp similarity index 100% rename from util/libtorch_c/torch_c.cpp rename to src/libtorch_c/torch_c.cpp diff --git a/util/libtorch_c/torch_c.h b/src/libtorch_c/torch_c.h similarity index 100% rename from util/libtorch_c/torch_c.h rename to src/libtorch_c/torch_c.h diff --git a/src/model.c b/src/model.c index 8ca24e4ac..67813f879 100644 --- a/src/model.c +++ b/src/model.c @@ -2,7 +2,7 @@ #include "model_struct.h" #include "backends.h" -#include "util/alloc.h" +#include "rmutil/alloc.h" #include "util/arr_rm_alloc.h" RedisModuleType *RedisAI_ModelType = NULL; diff --git a/src/redisai.c b/src/redisai.c index 8fd524e54..42be7aa27 100644 --- a/src/redisai.c +++ b/src/redisai.c @@ -9,7 +9,7 @@ #include #include -#include "util/alloc.h" +#include "rmutil/alloc.h" #include "util/arr_rm_alloc.h" #include "rmutil/args.h" #include "rmutil/khash.h" diff --git a/src/util/alloc.c b/src/rmutil/alloc.c similarity index 100% rename from src/util/alloc.c rename to src/rmutil/alloc.c diff --git a/src/util/alloc.h b/src/rmutil/alloc.h similarity index 100% rename from src/util/alloc.h rename to src/rmutil/alloc.h diff --git a/src/script.c b/src/script.c index cc537b00a..1edffa733 100644 --- a/src/script.c +++ b/src/script.c @@ -2,7 +2,7 @@ #include "script_struct.h" #include "backends.h" -#include "util/alloc.h" +#include "rmutil/alloc.h" #include "util/arr_rm_alloc.h" RedisModuleType *RedisAI_ScriptType = NULL; diff --git a/src/tensor.c b/src/tensor.c index f5b7bed96..c4d3eeaf7 100644 --- a/src/tensor.c +++ b/src/tensor.c @@ -3,7 +3,7 @@ #include #include #include -#include "util/alloc.h" +#include "rmutil/alloc.h" #include RedisModuleType *RedisAI_TensorType = NULL;