From 69184b1c8d92ec98f3a0a75eba1f554edd02f173 Mon Sep 17 00:00:00 2001 From: vishal Date: Fri, 6 Sep 2019 10:33:13 -0400 Subject: [PATCH 1/4] Unzip project to check if file exists and add request_handler file hash to api hash calculation --- pkg/operator/context/apis.go | 14 ++++++++++++-- pkg/operator/context/context.go | 8 +++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/pkg/operator/context/apis.go b/pkg/operator/context/apis.go index 1cb0c1698b..8fdd895773 100644 --- a/pkg/operator/context/apis.go +++ b/pkg/operator/context/apis.go @@ -20,6 +20,7 @@ import ( "bytes" "strings" + "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/hash" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/operator/api/context" @@ -27,7 +28,7 @@ import ( "github.com/cortexlabs/cortex/pkg/operator/api/userconfig" ) -func getAPIs(config *userconfig.Config, deploymentVersion string, projectID string) (context.APIs, error) { +func getAPIs(config *userconfig.Config, deploymentVersion string, projectID string, projectFiles map[string][]byte) (context.APIs, error) { apis := context.APIs{} for _, apiConfig := range config.APIs { @@ -36,9 +37,18 @@ func getAPIs(config *userconfig.Config, deploymentVersion string, projectID stri buf.WriteString(s.Obj(apiConfig.Tracker)) buf.WriteString(apiConfig.ModelFormat.String()) buf.WriteString(deploymentVersion) - buf.WriteString(projectID) buf.WriteString(strings.TrimSuffix(apiConfig.Model, "/")) + if apiConfig.RequestHandler != nil { + buf.WriteString(projectID) + + if val, ok := projectFiles[*apiConfig.RequestHandler]; !ok { + return nil, errors.Wrap(userconfig.ErrorImplDoesNotExist(*apiConfig.RequestHandler), resource.APIType.String(), apiConfig.Name, userconfig.RequestHandlerKey) + } else { + buf.Write(val) + } + } + id := hash.Bytes(buf.Bytes()) apis[apiConfig.Name] = &context.API{ diff --git a/pkg/operator/context/context.go b/pkg/operator/context/context.go index 0ecc1dd0ce..b3068a3f01 100644 --- a/pkg/operator/context/context.go +++ b/pkg/operator/context/context.go @@ -24,6 +24,7 @@ import ( "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/hash" + "github.com/cortexlabs/cortex/pkg/lib/zip" "github.com/cortexlabs/cortex/pkg/operator/api/context" "github.com/cortexlabs/cortex/pkg/operator/api/userconfig" "github.com/cortexlabs/cortex/pkg/operator/config" @@ -60,13 +61,18 @@ func New( ) ctx.ProjectID = hash.Bytes(projectBytes) + unzippedProject, err := zip.UnzipMemToMem(projectBytes) + if err != nil { + return nil, err + } + ctx.ProjectKey = filepath.Join(consts.ProjectsDir, ctx.ProjectID+".zip") if err = config.AWS.UploadBytesToS3(projectBytes, ctx.ProjectKey); err != nil { return nil, err } ctx.StatusPrefix = statusPrefix(ctx.App.Name) - apis, err := getAPIs(userconf, ctx.DeploymentVersion, ctx.ProjectID) + apis, err := getAPIs(userconf, ctx.DeploymentVersion, ctx.ProjectID, unzippedProject) if err != nil { return nil, err From e87696fa657b04ffff4c16e27f2e5e8ad37e6aae Mon Sep 17 00:00:00 2001 From: vishal Date: Fri, 6 Sep 2019 10:39:42 -0400 Subject: [PATCH 2/4] Fix lint --- pkg/operator/context/apis.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/operator/context/apis.go b/pkg/operator/context/apis.go index 8fdd895773..e7ec893d95 100644 --- a/pkg/operator/context/apis.go +++ b/pkg/operator/context/apis.go @@ -42,11 +42,11 @@ func getAPIs(config *userconfig.Config, deploymentVersion string, projectID stri if apiConfig.RequestHandler != nil { buf.WriteString(projectID) - if val, ok := projectFiles[*apiConfig.RequestHandler]; !ok { + fileBytes, ok := projectFiles[*apiConfig.RequestHandler] + if !ok { return nil, errors.Wrap(userconfig.ErrorImplDoesNotExist(*apiConfig.RequestHandler), resource.APIType.String(), apiConfig.Name, userconfig.RequestHandlerKey) - } else { - buf.Write(val) } + buf.Write(fileBytes) } id := hash.Bytes(buf.Bytes()) From 9a8d0169841201105adb4050fcfa9360073c63e8 Mon Sep 17 00:00:00 2001 From: vishal Date: Fri, 6 Sep 2019 16:59:27 -0400 Subject: [PATCH 3/4] Check request handler in user config validation --- pkg/operator/api/userconfig/apis.go | 13 ++++++++++--- pkg/operator/api/userconfig/config.go | 19 ++++++++++--------- pkg/operator/context/apis.go | 10 ++-------- pkg/operator/context/context.go | 7 +------ pkg/operator/endpoints/deploy.go | 12 +++++++++--- 5 files changed, 32 insertions(+), 29 deletions(-) diff --git a/pkg/operator/api/userconfig/apis.go b/pkg/operator/api/userconfig/apis.go index 3421c9f742..56baf40ab4 100644 --- a/pkg/operator/api/userconfig/apis.go +++ b/pkg/operator/api/userconfig/apis.go @@ -206,9 +206,9 @@ func (api *API) UserConfigStr() string { return sb.String() } -func (apis APIs) Validate() error { +func (apis APIs) Validate(projectFileMap map[string][]byte) error { for _, api := range apis { - if err := api.Validate(); err != nil { + if err := api.Validate(projectFileMap); err != nil { return err } } @@ -226,7 +226,7 @@ func (apis APIs) Validate() error { return nil } -func (api *API) Validate() error { +func (api *API) Validate(projectFileMap map[string][]byte) error { awsClient, err := aws.NewFromS3Path(api.Model, false) if err != nil { return err @@ -269,6 +269,13 @@ func (api *API) Validate() error { return errors.Wrap(ErrorTFServingOptionsForTFOnly(api.ModelFormat), Identify(api)) } + if api.RequestHandler != nil { + fmt.Println(projectFileMap[*api.RequestHandler]) + if _, ok := projectFileMap[*api.RequestHandler]; !ok { + return errors.Wrap(ErrorImplDoesNotExist(*api.RequestHandler), Identify(api), RequestHandlerKey) + } + } + if err := api.Compute.Validate(); err != nil { return errors.Wrap(err, Identify(api), ComputeKey) } diff --git a/pkg/operator/api/userconfig/config.go b/pkg/operator/api/userconfig/config.go index 6b5f3d158c..2be02c1a0c 100644 --- a/pkg/operator/api/userconfig/config.go +++ b/pkg/operator/api/userconfig/config.go @@ -22,6 +22,7 @@ import ( cr "github.com/cortexlabs/cortex/pkg/lib/configreader" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/files" + "github.com/cortexlabs/cortex/pkg/lib/zip" "github.com/cortexlabs/cortex/pkg/operator/api/resource" ) @@ -35,13 +36,18 @@ var typeFieldValidation = &cr.StructFieldValidation{ Nil: true, } -func (config *Config) Validate() error { +func (config *Config) Validate(projectBytes []byte) error { if err := config.App.Validate(); err != nil { return err } + projectFileMap, err := zip.UnzipMemToMem(projectBytes) + if err != nil { + return err + } + if config.APIs != nil { - if err := config.APIs.Validate(); err != nil { + if err := config.APIs.Validate(projectFileMap); err != nil { return err } } @@ -49,7 +55,7 @@ func (config *Config) Validate() error { return nil } -func New(filePath string, configBytes []byte, validate bool) (*Config, error) { +func New(filePath string, configBytes []byte) (*Config, error) { var err error configData, err := cr.ReadYAMLBytes(configBytes) @@ -109,11 +115,6 @@ func New(filePath string, configBytes []byte, validate bool) (*Config, error) { return nil, ErrorMissingAppDefinition() } - if validate { - if err := config.Validate(); err != nil { - return nil, err - } - } return config, nil } @@ -123,7 +124,7 @@ func ReadAppName(filePath string, relativePath string) (string, error) { return "", errors.Wrap(err, relativePath, ErrorReadConfig().Error()) } - config, err := New(relativePath, configBytes, false) + config, err := New(relativePath, configBytes) if err != nil { return "", err } diff --git a/pkg/operator/context/apis.go b/pkg/operator/context/apis.go index e7ec893d95..5af9f79dd4 100644 --- a/pkg/operator/context/apis.go +++ b/pkg/operator/context/apis.go @@ -20,7 +20,6 @@ import ( "bytes" "strings" - "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/hash" s "github.com/cortexlabs/cortex/pkg/lib/strings" "github.com/cortexlabs/cortex/pkg/operator/api/context" @@ -28,7 +27,7 @@ import ( "github.com/cortexlabs/cortex/pkg/operator/api/userconfig" ) -func getAPIs(config *userconfig.Config, deploymentVersion string, projectID string, projectFiles map[string][]byte) (context.APIs, error) { +func getAPIs(config *userconfig.Config, deploymentVersion string, projectID string) (context.APIs, error) { apis := context.APIs{} for _, apiConfig := range config.APIs { @@ -41,12 +40,7 @@ func getAPIs(config *userconfig.Config, deploymentVersion string, projectID stri if apiConfig.RequestHandler != nil { buf.WriteString(projectID) - - fileBytes, ok := projectFiles[*apiConfig.RequestHandler] - if !ok { - return nil, errors.Wrap(userconfig.ErrorImplDoesNotExist(*apiConfig.RequestHandler), resource.APIType.String(), apiConfig.Name, userconfig.RequestHandlerKey) - } - buf.Write(fileBytes) + buf.WriteString(*apiConfig.RequestHandler) } id := hash.Bytes(buf.Bytes()) diff --git a/pkg/operator/context/context.go b/pkg/operator/context/context.go index b3068a3f01..d145f5dfd7 100644 --- a/pkg/operator/context/context.go +++ b/pkg/operator/context/context.go @@ -24,7 +24,6 @@ import ( "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/hash" - "github.com/cortexlabs/cortex/pkg/lib/zip" "github.com/cortexlabs/cortex/pkg/operator/api/context" "github.com/cortexlabs/cortex/pkg/operator/api/userconfig" "github.com/cortexlabs/cortex/pkg/operator/config" @@ -61,10 +60,6 @@ func New( ) ctx.ProjectID = hash.Bytes(projectBytes) - unzippedProject, err := zip.UnzipMemToMem(projectBytes) - if err != nil { - return nil, err - } ctx.ProjectKey = filepath.Join(consts.ProjectsDir, ctx.ProjectID+".zip") if err = config.AWS.UploadBytesToS3(projectBytes, ctx.ProjectKey); err != nil { @@ -72,7 +67,7 @@ func New( } ctx.StatusPrefix = statusPrefix(ctx.App.Name) - apis, err := getAPIs(userconf, ctx.DeploymentVersion, ctx.ProjectID, unzippedProject) + apis, err := getAPIs(userconf, ctx.DeploymentVersion, ctx.ProjectID) if err != nil { return nil, err diff --git a/pkg/operator/endpoints/deploy.go b/pkg/operator/endpoints/deploy.go index 00d3279916..9dda4cbe5a 100644 --- a/pkg/operator/endpoints/deploy.go +++ b/pkg/operator/endpoints/deploy.go @@ -47,15 +47,21 @@ func Deploy(w http.ResponseWriter, r *http.Request) { return } - userconf, err := userconfig.New("cortex.yaml", configBytes, true) + projectBytes, err := files.ReadReqFile(r, "project.zip") + if err != nil { + RespondError(w, errors.WithStack(err)) + return + } + + userconf, err := userconfig.New("cortex.yaml", configBytes) if err != nil { RespondError(w, err) return } - projectBytes, err := files.ReadReqFile(r, "project.zip") + err = userconf.Validate(projectBytes) if err != nil { - RespondError(w, errors.WithStack(err)) + RespondError(w, err) return } From 795bdfe7fa30151e07c108625414dd084c312540 Mon Sep 17 00:00:00 2001 From: vishal Date: Mon, 9 Sep 2019 14:46:21 -0400 Subject: [PATCH 4/4] Remove debug line --- pkg/operator/api/userconfig/apis.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/operator/api/userconfig/apis.go b/pkg/operator/api/userconfig/apis.go index 56baf40ab4..a7b5a79490 100644 --- a/pkg/operator/api/userconfig/apis.go +++ b/pkg/operator/api/userconfig/apis.go @@ -270,7 +270,6 @@ func (api *API) Validate(projectFileMap map[string][]byte) error { } if api.RequestHandler != nil { - fmt.Println(projectFileMap[*api.RequestHandler]) if _, ok := projectFileMap[*api.RequestHandler]; !ok { return errors.Wrap(ErrorImplDoesNotExist(*api.RequestHandler), Identify(api), RequestHandlerKey) }