Skip to content

Commit eeada0d

Browse files
rafierafie
committed
fixes 11
1 parent 022cc2c commit eeada0d

File tree

7 files changed

+72
-39
lines changed

7 files changed

+72
-39
lines changed

‎get_deps.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ fi
8787

8888
################################################################################# LIBTENSORFLOW
8989

90-
TF_VERSION="2.3.0"
90+
TF_VERSION="2.3.1"
9191

9292
if [[ $WITH_TF != 0 ]]; then
9393
[[ $FORCE == 1 ]] && rm -rf $LIBTENSORFLOW

‎opt/build/libtorch/Makefile

Lines changed: 53 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,91 +1,120 @@
11

22
ROOT=../../..
33

4-
VERSION ?= 1.5.0
5-
OSNICK ?= buster
4+
VERSION ?= 1.7.0
5+
OSNICK ?= bionic
6+
7+
REDIS_VER=6.0.9
68

79
#----------------------------------------------------------------------------------------------
810

911
S3_URL=redismodules/pytorch
1012

1113
OS:=$(shell $(ROOT)/opt/readies/bin/platform --os)
12-
ifeq ($(OS),macosx)
13-
OS:=macos
14-
endif
15-
STEM=libtorch-cpu-$(OS)
14+
OSNICK:=$(shell $(ROOT)/opt/readies/bin/platform --osnick)
15+
ARCH:=$(shell $(ROOT)/opt/readies/bin/platform --arch)
16+
DEVICE ?= cpu
17+
18+
STEM=libtorch
1619

1720
DOCKER_OS.bionic=ubuntu:bionic
1821
DOCKER_OS.stretch=debian:stretch-slim
1922
DOCKER_OS.buster=debian:buster-slim
2023
DOCKER_OS=$(DOCKER_OS.$(OSNICK))
2124

25+
TAR_ARCH.x64=x86_64
26+
TAR_ARCH.arm64v8=arm64
27+
TAR_ARCH.arm32v7=arm
28+
2229
#----------------------------------------------------------------------------------------------
2330

2431
define targets # (1=OP, 2=op)
2532
$(1)_TARGETS :=
26-
$(1)_TARGETS += $(if $(findstring $(X64),1),$(2)_x64)
33+
$(1)_TARGETS += $(if $(findstring $(X64),1),$(2)_native)
34+
ifeq ($$(CROSS),1)
2735
$(1)_TARGETS += $(if $(findstring $(ARM7),1),$(2)_arm32v7)
2836
$(1)_TARGETS += $(if $(findstring $(ARM8),1),$(2)_arm64v8)
37+
endif
2938

30-
$(1)_TARGETS += $$(if $$(strip $$($(1)_TARGETS)),,$(2)_x64 $(2)_arm32v7 $(2)_arm64v8)
39+
ifeq ($$(CROSS),1)
40+
$(1)_TARGETS += $$(if $$(strip $$($(1)_TARGETS)),,$(2)_native $(2)_arm32v7 $(2)_arm64v8)
41+
else
42+
$(1)_TARGETS += $$(if $$(strip $$($(1)_TARGETS)),,$(2)_native)
43+
endif
3144
endef
3245

3346
$(eval $(call targets,BUILD,build))
3447
$(eval $(call targets,PUBLISH,publish))
3548

3649
#----------------------------------------------------------------------------------------------
3750

38-
define build_x64 # (1=arch, 2=tar-arch)
51+
DOCKER_BUILD_ARGS += \
52+
PT_VER=$(VERSION) \
53+
OSNICK=$(OSNICK) \
54+
OS=$(DOCKER_OS) \
55+
ARCH=$(ARCH)
56+
57+
define build_native # (1=arch, 2=tar-arch)
3958
IID_$(1)=$(1)_$(VERSION).iid
4059
CID_$(1)=$(1)_$(VERSION).cid
4160

42-
build_x64:
43-
@docker build --iidfile $$(IID_$(1)) -t redisfab/$(STEM)-$(1):$(VERSION) -f Dockerfile.x64 \
44-
--build-arg OS=$(DOCKER_OS) $(ROOT)
61+
build_native:
62+
@docker build --iidfile $$(IID_$(1)) -t redisfab/$(STEM):$(VERSION)-$(DEVICE)-$(1) -f Dockerfile.$(VARIANT) \
63+
$(foreach A,$(DOCKER_BUILD_ARGS),--build-arg $(A)) .
4564
@docker create --cidfile $$(CID_$(1)) `cat $$(IID_$(1))`
46-
@docker cp `cat $$(CID_$(1))`:/build/dest/$(STEM)-$(2)-$(VERSION).tar.gz .
65+
@docker cp `cat $$(CID_$(1))`:/build/dest/$(STEM)-$(DEVICE)-$(OS)-$(2)-$(VERSION).tar.gz .
4766

48-
.PHONY: build_x64
67+
.PHONY: build_native
4968
endef
5069

70+
ifeq ($(CROSS),1)
71+
5172
define build_arm # (1=arch, 2=tar-arch)
5273
IID_$(1)=$(1)_$(VERSION).iid
5374
CID_$(1)=$(1)_$(VERSION).cid
5475

5576
build_$(1):
56-
@docker build --iidfile $$(IID_$(1)) -t redisfab/$(STEM)-$(1):$(VERSION) -f Dockerfile.arm \
57-
--build-arg ARCH=$(1) $(ROOT)
77+
@docker build --iidfile $$(IID_$(1)) -t redisfab/$(STEM):$(VERSION)-$(DEVICE)-$(1) -f Dockerfile.$(VARIANT) \
78+
$(foreach A,$(DOCKER_BUILD_ARGS),--build-arg $(A)) .
5879
@docker create --cidfile $$(CID_$(1)) `cat $$(IID_$(1))`
5980
@docker cp `cat $$(CID_$(1))`:/build/$(STEM)-$(2)-$(VERSION).tar.gz .
6081

6182
.PHONY: build_$(1)
6283
endef
6384

85+
endif # CROSS
86+
6487
#----------------------------------------------------------------------------------------------
6588

66-
define publish_x64 # (1=arch, 2=tar-arch)
67-
publish_x64:
89+
define publish_native # (1=arch, 2=tar-arch)
90+
publish_native:
6891
@aws s3 cp $(STEM)-$(2)-$(VERSION).tar.gz s3://$(S3_URL)/ --acl public-read
6992

70-
.PHONY: publish_x64
93+
.PHONY: publish_native
7194
endef
7295

96+
ifeq ($(CROSS),1)
97+
7398
define publish_arm # (1=arch, 2=tar-arch)
7499
publish_$(1):
75100
@aws s3 cp $(STEM)-$(2)-$(VERSION).tar.gz s3://$(S3_URL)/ --acl public-read
76101

77102
.PHONY: publish_$(1)
78103
endef
79104

105+
endif # CROSS
106+
80107
#----------------------------------------------------------------------------------------------
81108

82109
all: build publish
83110

84111
build: $(BUILD_TARGETS)
85112

86-
$(eval $(call build_x64,x64,x86_64))
113+
$(eval $(call build_native,$(ARCH),$(TAR_ARCH.$(ARCH))))
114+
ifeq ($(CROSS),1)
87115
$(eval $(call build_arm,arm64v8,arm64))
88116
$(eval $(call build_arm,arm32v7,arm))
117+
endif
89118

90119
ifneq ($(filter publish,$(MAKECMDGOALS)),)
91120
ifeq ($(wildcard $(HOME)/.aws/credentials),)
@@ -96,13 +125,16 @@ endif
96125
publish: $(PUBLISH_TARGETS)
97126

98127
$(eval $(call publish_x64,x64,x86_64))
128+
ifeq ($(CROSS),1)
99129
$(eval $(call publish_arm,arm64v8,arm64))
100130
$(eval $(call publish_arm,arm32v7,arm))
131+
endif
101132

102133
repack:
103134
@PT_VERSION=$(VERSION) GPU=$(GPU) ./repack.sh
104135

105136
help:
106-
@echo "make [build|repack|publish] [X64=1|ARM7=1|ARM8=1]"
137+
@echo "make [build|repack|publish] [X64=1|ARM7=1|ARM8=1] [CROSS=1]"
107138

108139
.PHONY: all build repack publish help
140+

‎opt/build/libtorch/collect.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313

1414
#----------------------------------------------------------------------------------------------
1515

16-
PYTORCH_VERSION = '1.2.0'
16+
PYTORCH_VERSION = '1.7.0'
1717

1818
parser = argparse.ArgumentParser(description='Prepare RedisAI dependant distribution packages.')
1919
parser.add_argument('--pytorch', default='pytorch', help='root of pytorch repository')

‎opt/build/tensorflow-2.x/Dockerfile.jetson

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ RUN git checkout v${TF_VERSION}
4545
ENV TF_NEED_CUDA=1 \
4646
TF_CUDA_COMPUTE_CAPABILITIES=6.1,5.3,3.5,7.2 \
4747
TF_NEED_TENSORRT=0 \
48-
\
48+
\
4949
TF_NEED_GCP=0 \
5050
TF_NEED_HDFS=0 \
5151
TF_NEED_OPENCL=0 \
@@ -63,19 +63,19 @@ ENV TF_NEED_CUDA=1 \
6363
TF_SET_ANDROID_WORKSPACE=0 \
6464
TF_NEED_COMPUTECPP=0 \
6565
TF_NEED_KAFKA=0 \
66-
\
67-
TF_CUDA_VERSION=${CUDA_VER} \
66+
67+
ENV TF_CUDA_VERSION=${CUDA_VER} \
6868
TF_CUDNN_VERSION=${CUDNN_VERSION} \
6969
TF_CUBLAS_VERSION=${TF_CUDA_VERSION} \
7070
TF_TENSORRT_VERSION=${TENSORRT_VERSION} \
71-
\
71+
\
7272
CUDA_TOOLKIT_PATH="/usr/local/cuda-${TF_CUDA_VERSION}" \
7373
TF_CUDA_PATHS="/usr/include,/usr/include/cuda,/opt/cuda/include,/usr/local/cuda-${TF_CUDA_VERSION}/include,/usr/local/cuda-${TF_CUDA_VERSION}/lib64,/usr/local/cuda-${TF_CUDA_VERSION}/targets/${MACH}-linux/,/usr/local/cuda-${TF_CUDA_VERSION}/targets/${MACH}-linux/lib/,/usr/lib/${MACH}-linux-gnu,/usr/local/lib,/usr/local/cuda-${TF_CUDA_VERSION},"
74-
75-
ENV CUDNN_INSTALL_PATH=/usr/lib/${MACH}-linux-gnu \
74+
\
75+
CUDNN_INSTALL_PATH=/usr/lib/${MACH}-linux-gnu \
7676
TENSORRT_INSTALL_PATH=/usr/lib/${MACH}-linux-gnu
77-
78-
ENV PATH="/usr/local/cuda-${TF_CUDA_VERSION}/bin:$PATH" \
77+
\
78+
PATH="/usr/local/cuda-${TF_CUDA_VERSION}/bin:${PATH}" \
7979
LD_LIBRARY_PATH="/usr/local/cuda-${TF_CUDA_VERSION}/lib64:/usr/lib/${MACH}-linux-gnu:${LD_LIBRARY_PATH}"
8080

8181
RUN yes "" | ./configure

‎opt/build/tensorflow/Dockerfile.arm

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# BUILD redisfab/libtensorflow-cpu-$(OSNICK)-${ARCH}:$(TF_VER)
1+
# BUILD redisfab/libtensorflow:${TF_VER}-cpu-${OSNICK}-${ARCH}
22

33
# stretch|bionic|buster
44
ARG OSNICK=buster

‎opt/build/tensorflow/Dockerfile.x64

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# BUILD redisfab/tensorflow-x64:$(TF_VER)
1+
# BUILD redisfab/tensorflow:${TF_VER}-x64-${OSNICK}
22

33
ARG OS=debian:buster
44

‎opt/build/tensorflow/collect.py

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,24 @@
1414
#----------------------------------------------------------------------------------------------
1515

1616
TENSORFLOW_VERSION = '1.15.0'
17+
VARIANT='cpu'
1718

1819
parser = argparse.ArgumentParser(description='Prepare RedisAI dependant distribution packages.')
19-
parser.add_argument('--tensorflow', default='tensorflow', help='root of tensorflow repository')
20-
parser.add_argument('--tensorflow-ver', default=TENSORFLOW_VERSION, help='tensorflow version')
20+
parser.add_argument('--root', default='tensorflow', help='root of tensorflow repository')
21+
parser.add_argument('--version', default=TENSORFLOW_VERSION, help='tensorflow version')
22+
parser.add_argument('--variant', default=VARIANT, help='build variant')
2123
parser.add_argument('--dest', default='dest', help='destination directory')
2224
parser.add_argument('-n', '--nop', action="store_true", help='no operation')
2325
args = parser.parse_args()
2426

2527
#----------------------------------------------------------------------------------------------
2628

27-
tensorflow = Path(args.tensorflow).resolve()
29+
tensorflow = Path(args.root).resolve()
2830
dest = Path(args.dest).resolve()
2931

30-
3132
#----------------------------------------------------------------------------------------------
3233

33-
tf_build='cpu'
34+
tf_build=args.variant
3435

3536
platform = paella.Platform()
3637

@@ -44,7 +45,7 @@
4445
elif tf_arch == 'arm64v8':
4546
tf_arch = 'arm64'
4647

47-
tf_ver = args.tensorflow_ver
48+
tf_ver = args.version
4849

4950
#----------------------------------------------------------------------------------------------
5051

0 commit comments

Comments
 (0)