Skip to content

Commit 482ad74

Browse files
deliahudeliahu
authored
Replace Argo with in-operator DAG manager (#235)
1 parent 46b2a2e commit 482ad74

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+2100
-1877
lines changed

‎Makefile

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,7 @@ SHELL := /bin/bash
2121
# Cortex
2222

2323
devstart:
24-
@kill $(shell pgrep -f rerun); ./dev/operator_local.sh || true
25-
26-
killdev:
27-
@kill $(shell pgrep -f rerun)
24+
@./dev/operator_local.sh || true
2825

2926
kubectl:
3027
@eksctl utils write-kubeconfig --name="cortex"
@@ -138,8 +135,6 @@ ci-build-images:
138135
@./build/build-image.sh images/fluentd fluentd
139136
@./build/build-image.sh images/nginx-controller nginx-controller
140137
@./build/build-image.sh images/nginx-backend nginx-backend
141-
@./build/build-image.sh images/argo-controller argo-controller
142-
@./build/build-image.sh images/argo-executor argo-executor
143138
@./build/build-image.sh images/python-packager python-packager
144139
@./build/build-image.sh images/cluster-autoscaler cluster-autoscaler
145140
@./build/build-image.sh images/nvidia nvidia
@@ -160,8 +155,6 @@ ci-push-images:
160155
@./build/push-image.sh fluentd
161156
@./build/push-image.sh nginx-controller
162157
@./build/push-image.sh nginx-backend
163-
@./build/push-image.sh argo-controller
164-
@./build/push-image.sh argo-executor
165158
@./build/push-image.sh python-packager
166159
@./build/push-image.sh cluster-autoscaler
167160
@./build/push-image.sh nvidia

‎cortex.sh

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -117,8 +117,6 @@ export CORTEX_NODES_MAX="${CORTEX_NODES_MAX:-5}"
117117
export CORTEX_NAMESPACE="${CORTEX_NAMESPACE:-cortex}"
118118

119119
export CORTEX_IMAGE_MANAGER="${CORTEX_IMAGE_MANAGER:-cortexlabs/manager:$CORTEX_VERSION_STABLE}"
120-
export CORTEX_IMAGE_ARGO_CONTROLLER="${CORTEX_IMAGE_ARGO_CONTROLLER:-cortexlabs/argo-controller:$CORTEX_VERSION_STABLE}"
121-
export CORTEX_IMAGE_ARGO_EXECUTOR="${CORTEX_IMAGE_ARGO_EXECUTOR:-cortexlabs/argo-executor:$CORTEX_VERSION_STABLE}"
122120
export CORTEX_IMAGE_FLUENTD="${CORTEX_IMAGE_FLUENTD:-cortexlabs/fluentd:$CORTEX_VERSION_STABLE}"
123121
export CORTEX_IMAGE_NGINX_BACKEND="${CORTEX_IMAGE_NGINX_BACKEND:-cortexlabs/nginx-backend:$CORTEX_VERSION_STABLE}"
124122
export CORTEX_IMAGE_NGINX_CONTROLLER="${CORTEX_IMAGE_NGINX_CONTROLLER:-cortexlabs/nginx-controller:$CORTEX_VERSION_STABLE}"
@@ -177,8 +175,6 @@ function install_cortex() {
177175
-e CORTEX_NODE_TYPE=$CORTEX_NODE_TYPE \
178176
-e CORTEX_LOG_GROUP=$CORTEX_LOG_GROUP \
179177
-e CORTEX_BUCKET=$CORTEX_BUCKET \
180-
-e CORTEX_IMAGE_ARGO_CONTROLLER=$CORTEX_IMAGE_ARGO_CONTROLLER \
181-
-e CORTEX_IMAGE_ARGO_EXECUTOR=$CORTEX_IMAGE_ARGO_EXECUTOR \
182178
-e CORTEX_IMAGE_FLUENTD=$CORTEX_IMAGE_FLUENTD \
183179
-e CORTEX_IMAGE_NGINX_BACKEND=$CORTEX_IMAGE_NGINX_BACKEND \
184180
-e CORTEX_IMAGE_NGINX_CONTROLLER=$CORTEX_IMAGE_NGINX_CONTROLLER \

‎dev/operator_local.sh

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ export CONST_OPERATOR_TRANSFORMERS_DIR=$ROOT/pkg/transformers
2626
export CONST_OPERATOR_ESTIMATORS_DIR=$ROOT/pkg/estimators
2727
export CONST_OPERATOR_IN_CLUSTER=false
2828

29+
kill $(pgrep -f rerun) >/dev/null 2>&1 || true
30+
2931
rerun -watch $ROOT/pkg $ROOT/cli -ignore $ROOT/vendor $ROOT/bin -run sh -c \
3032
"go build -o $ROOT/bin/operator $ROOT/pkg/operator && go build -installsuffix cgo -o $ROOT/bin/cortex $ROOT/cli && $ROOT/bin/operator"
31-
# go run -race $ROOT/pkg/operator/operator.go
33+
34+
# go run -race $ROOT/pkg/operator/operator.go # Check for race conditions. Doesn't seem to catch them all?

‎dev/registry.sh

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,6 @@ function ecr_login() {
3636

3737
function create_registry() {
3838
aws ecr create-repository --repository-name=cortexlabs/manager --region=$REGISTRY_REGION || true
39-
aws ecr create-repository --repository-name=cortexlabs/argo-controller --region=$REGISTRY_REGION || true
40-
aws ecr create-repository --repository-name=cortexlabs/argo-executor --region=$REGISTRY_REGION || true
4139
aws ecr create-repository --repository-name=cortexlabs/fluentd --region=$REGISTRY_REGION || true
4240
aws ecr create-repository --repository-name=cortexlabs/nginx-backend --region=$REGISTRY_REGION || true
4341
aws ecr create-repository --repository-name=cortexlabs/nginx-controller --region=$REGISTRY_REGION || true
@@ -138,8 +136,6 @@ elif [ "$cmd" = "update" ]; then
138136
build_and_push $ROOT/images/nginx-controller nginx-controller latest
139137
build_and_push $ROOT/images/nginx-backend nginx-backend latest
140138
build_and_push $ROOT/images/fluentd fluentd latest
141-
build_and_push $ROOT/images/argo-controller argo-controller latest
142-
build_and_push $ROOT/images/argo-executor argo-executor latest
143139
build_and_push $ROOT/images/tf-serve tf-serve latest
144140
build_and_push $ROOT/images/tf-serve-gpu tf-serve-gpu latest
145141
build_and_push $ROOT/images/onnx-serve-gpu onnx-serve-gpu latest

‎docs/cluster/config.md

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,6 @@ export CORTEX_NAMESPACE="cortex"
4040

4141
# Image paths
4242
export CORTEX_IMAGE_MANAGER="cortexlabs/manager:master"
43-
export CORTEX_IMAGE_ARGO_CONTROLLER="cortexlabs/argo-controller:master"
44-
export CORTEX_IMAGE_ARGO_EXECUTOR="cortexlabs/argo-executor:master"
4543
export CORTEX_IMAGE_FLUENTD="cortexlabs/fluentd:master"
4644
export CORTEX_IMAGE_NGINX_BACKEND="cortexlabs/nginx-backend:master"
4745
export CORTEX_IMAGE_NGINX_CONTROLLER="cortexlabs/nginx-controller:master"

‎docs/cluster/development.md

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,6 @@ export CORTEX_NODES_MAX="5"
5656
export CORTEX_NAMESPACE="cortex"
5757

5858
export CORTEX_IMAGE_MANAGER="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/manager:latest"
59-
export CORTEX_IMAGE_ARGO_CONTROLLER="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/argo-controller:latest"
60-
export CORTEX_IMAGE_ARGO_EXECUTOR="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/argo-executor:latest"
6159
export CORTEX_IMAGE_FLUENTD="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/fluentd:latest"
6260
export CORTEX_IMAGE_NGINX_BACKEND="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/nginx-backend:latest"
6361
export CORTEX_IMAGE_NGINX_CONTROLLER="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/nginx-controller:latest"

‎go.mod

Lines changed: 9 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,10 @@
44
// go mod tidy
55
// replace these lines in go.mod:
66
// github.com/GoogleCloudPlatform/spark-on-k8s-operator v1alpha1-0.5-2.4.0
7-
// github.com/argoproj/argo v2.3.0
87
// github.com/cortexlabs/yaml v2.2.4
9-
// k8s.io/client-go v10.0.0
10-
// k8s.io/api 89a74a8d264df0e993299876a8cde88379b940ee
11-
// k8s.io/apimachinery 2b1284ed4c93a43499e781493253e2ac5959c4fd
8+
// k8s.io/client-go v12.0.0
9+
// k8s.io/api 7525909cc6da
10+
// k8s.io/apimachinery 1799e75a0719
1211
// (note: go to the commit for the client-go release and browse to Godeps/Godeps.json to find the SHAs for k8s.io/api and k8s.io/apimachinery)
1312
// go mod tidy
1413
// check the diff in this file
@@ -19,39 +18,23 @@ go 1.12
1918

2019
require (
2120
github.com/GoogleCloudPlatform/spark-on-k8s-operator v0.0.0-20181208011959-62db1d66dafa
22-
github.com/argoproj/argo v2.3.0+incompatible
23-
github.com/aws/aws-sdk-go v1.20.12
21+
github.com/aws/aws-sdk-go v1.20.20
2422
github.com/cortexlabs/yaml v0.0.0-20190626164117-202ab3a3d475
2523
github.com/davecgh/go-spew v1.1.1
26-
github.com/emicklei/go-restful v2.9.6+incompatible // indirect
27-
github.com/ghodss/yaml v1.0.0
28-
github.com/go-openapi/spec v0.19.2 // indirect
29-
github.com/gogo/protobuf v1.2.1 // indirect
30-
github.com/google/btree v1.0.0 // indirect
31-
github.com/google/gofuzz v1.0.0 // indirect
32-
github.com/googleapis/gnostic v0.3.0 // indirect
3324
github.com/gorilla/mux v1.7.3
3425
github.com/gorilla/websocket v1.4.0
35-
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
3626
github.com/imdario/mergo v0.3.7 // indirect
37-
github.com/json-iterator/go v1.1.6 // indirect
3827
github.com/mitchellh/go-homedir v1.1.0
39-
github.com/modern-go/reflect2 v1.0.1 // indirect
40-
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
4128
github.com/pkg/errors v0.8.1
4229
github.com/spf13/cobra v0.0.5
4330
github.com/stretchr/testify v1.3.0
4431
github.com/tcnksm/go-input v0.0.0-20180404061846-548a7d7a8ee8
45-
github.com/ugorji/go/codec v1.1.5-pre
32+
github.com/ugorji/go/codec v1.1.7
4633
github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca
4734
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 // indirect
4835
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
49-
gopkg.in/inf.v0 v0.9.1 // indirect
50-
gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5
51-
k8s.io/api v0.0.0-20181204000039-89a74a8d264d
52-
k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93
53-
k8s.io/client-go v10.0.0+incompatible
54-
k8s.io/klog v0.3.0 // indirect
55-
k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208 // indirect
56-
sigs.k8s.io/yaml v1.1.0 // indirect
36+
k8s.io/api v0.0.0-20190620084959-7cf5895f2711
37+
k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719
38+
k8s.io/client-go v0.0.0-20190620085101-78d2af792bab
39+
k8s.io/utils v0.0.0-20190712204705-3dccf664f023 // indirect
5740
)

0 commit comments

Comments
 (0)