From df5b5155d3887e0429c08a55d89bb228ad6aaaa8 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 5 Jul 2019 16:09:08 -0400 Subject: [PATCH 01/13] iris tf example --- docs/apis/packaging-models.md | 2 +- examples/models/tf.py | 112 ++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 examples/models/tf.py diff --git a/docs/apis/packaging-models.md b/docs/apis/packaging-models.md index 4837829420..1371c23330 100644 --- a/docs/apis/packaging-models.md +++ b/docs/apis/packaging-models.md @@ -5,7 +5,7 @@ Zip the exported estimator output in your checkpoint directory, e.g. ```text -$ ls export/estimator +$ ls export/estimator/1560263597/ saved_model.pb variables/ $ zip -r model.zip export/estimator diff --git a/examples/models/tf.py b/examples/models/tf.py new file mode 100644 index 0000000000..141d29b56b --- /dev/null +++ b/examples/models/tf.py @@ -0,0 +1,112 @@ +# sources copied/modified from https://github.com/tensorflow/models/blob/master/samples/core/get_started/ + +import tensorflow as tf +from sklearn.datasets import load_iris +from sklearn.model_selection import train_test_split +import shutil +import os + +EXPORT_DIR = "iris_tf_export" + +def get_all_file_paths(directory): + file_paths = [] + for root, directories, files in os.walk(directory): + for filename in files: + filepath = os.path.join(root, filename) + file_paths.append(filepath) + + return file_paths + +def input_fn(features, labels, batch_size, mode): + """An input function for training""" + dataset = tf.data.Dataset.from_tensor_slices((features, labels)) + if mode == tf.estimator.ModeKeys.TRAIN: + dataset = dataset.shuffle(1000).repeat() + dataset = dataset.batch(batch_size) + dataset_it = dataset.make_one_shot_iterator() + images, labels = dataset_it.get_next() + return {"inputs": images}, labels + + +def json_serving_input_fn(): + inputs = tf.placeholder(shape=[4], dtype=tf.float64) + features = {"inputs": tf.expand_dims(inputs, 0)} + return tf.estimator.export.ServingInputReceiver(features=features, receiver_tensors=inputs) + + +def my_model(features, labels, mode, params): + """DNN with three hidden layers and learning_rate=0.1.""" + net = features["inputs"] + for units in params['hidden_units']: + net = tf.layers.dense(net, units=units, activation=tf.nn.relu) + + logits = tf.layers.dense(net, params['n_classes'], activation=None) + + predicted_classes = tf.argmax(logits, 1) + if mode == tf.estimator.ModeKeys.PREDICT: + predictions = { + 'class_ids': predicted_classes[:, tf.newaxis], + 'probabilities': tf.nn.softmax(logits), + 'logits': logits, + } + return tf.estimator.EstimatorSpec( + mode=mode, + predictions=predictions, + export_outputs={ + "predict": tf.estimator.export.PredictOutput( + {"class_ids": predicted_classes[:, tf.newaxis], "probabilities": tf.nn.softmax(logits)} + ) + },) + + loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) + + accuracy = tf.metrics.accuracy(labels=labels, + predictions=predicted_classes, + name='acc_op') + metrics = {'accuracy': accuracy} + tf.summary.scalar('accuracy', accuracy[1]) + + if mode == tf.estimator.ModeKeys.EVAL: + return tf.estimator.EstimatorSpec( + mode, loss=loss, eval_metric_ops=metrics) + + optimizer = tf.train.AdagradOptimizer(learning_rate=0.1) + train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step()) + return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op) + + + +iris = load_iris() +X, y = iris.data, iris.target +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.8, random_state=42) + +classifier = tf.estimator.Estimator( + model_fn=my_model, + model_dir=EXPORT_DIR, + params={ + 'hidden_units': [10, 10], + 'n_classes': 3, +}) + + + +train_input_fn = lambda: input_fn(X_train, y_train, 100, tf.estimator.ModeKeys.TRAIN) +eval_input_fn = lambda: input_fn(X_test, y_test, 100, tf.estimator.ModeKeys.EVAL) +serving_input_fn = lambda: json_serving_input_fn() +exporter = tf.estimator.FinalExporter("estimator", serving_input_fn, as_text=False) +train_spec = tf.estimator.TrainSpec(train_input_fn, max_steps=1000) +eval_spec = tf.estimator.EvalSpec( + eval_input_fn, + exporters=[exporter], + name="estimator-eval", +) + +tf.estimator.train_and_evaluate(classifier, train_spec, eval_spec) + +# exported path looks like iris_tf_export/export/estimator/1562353043/variables +# need to zip the versioned dir +estimator_dir = EXPORT_DIR+"/export/estimator" +shutil.make_archive("tensorflow", 'zip', os.path.join(estimator_dir)) + +# clean up +shutil.rmtree(EXPORT_DIR) From c26642ffe6f404ef8c859f461ccd9aeec3924598 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 5 Jul 2019 16:22:41 -0400 Subject: [PATCH 02/13] format and lint --- examples/models/tf.py | 61 +++++++++++++++------------------- pkg/lib/configreader/reader.go | 2 +- 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/examples/models/tf.py b/examples/models/tf.py index 141d29b56b..bacbb45207 100644 --- a/examples/models/tf.py +++ b/examples/models/tf.py @@ -8,6 +8,7 @@ EXPORT_DIR = "iris_tf_export" + def get_all_file_paths(directory): file_paths = [] for root, directories, files in os.walk(directory): @@ -17,6 +18,7 @@ def get_all_file_paths(directory): return file_paths + def input_fn(features, labels, batch_size, mode): """An input function for training""" dataset = tf.data.Dataset.from_tensor_slices((features, labels)) @@ -37,57 +39,52 @@ def json_serving_input_fn(): def my_model(features, labels, mode, params): """DNN with three hidden layers and learning_rate=0.1.""" net = features["inputs"] - for units in params['hidden_units']: + for units in params["hidden_units"]: net = tf.layers.dense(net, units=units, activation=tf.nn.relu) - logits = tf.layers.dense(net, params['n_classes'], activation=None) + logits = tf.layers.dense(net, params["n_classes"], activation=None) predicted_classes = tf.argmax(logits, 1) if mode == tf.estimator.ModeKeys.PREDICT: predictions = { - 'class_ids': predicted_classes[:, tf.newaxis], - 'probabilities': tf.nn.softmax(logits), - 'logits': logits, + "class_ids": predicted_classes[:, tf.newaxis], + "probabilities": tf.nn.softmax(logits), + "logits": logits, } return tf.estimator.EstimatorSpec( - mode=mode, - predictions=predictions, - export_outputs={ - "predict": tf.estimator.export.PredictOutput( - {"class_ids": predicted_classes[:, tf.newaxis], "probabilities": tf.nn.softmax(logits)} - ) - },) + mode=mode, + predictions=predictions, + export_outputs={ + "predict": tf.estimator.export.PredictOutput( + { + "class_ids": predicted_classes[:, tf.newaxis], + "probabilities": tf.nn.softmax(logits), + } + ) + }, + ) loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) - accuracy = tf.metrics.accuracy(labels=labels, - predictions=predicted_classes, - name='acc_op') - metrics = {'accuracy': accuracy} - tf.summary.scalar('accuracy', accuracy[1]) + accuracy = tf.metrics.accuracy(labels=labels, predictions=predicted_classes, name="acc_op") + metrics = {"accuracy": accuracy} + tf.summary.scalar("accuracy", accuracy[1]) if mode == tf.estimator.ModeKeys.EVAL: - return tf.estimator.EstimatorSpec( - mode, loss=loss, eval_metric_ops=metrics) + return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics) optimizer = tf.train.AdagradOptimizer(learning_rate=0.1) train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step()) return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op) - iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.8, random_state=42) classifier = tf.estimator.Estimator( - model_fn=my_model, - model_dir=EXPORT_DIR, - params={ - 'hidden_units': [10, 10], - 'n_classes': 3, -}) - + model_fn=my_model, model_dir=EXPORT_DIR, params={"hidden_units": [10, 10], "n_classes": 3} +) train_input_fn = lambda: input_fn(X_train, y_train, 100, tf.estimator.ModeKeys.TRAIN) @@ -95,18 +92,14 @@ def my_model(features, labels, mode, params): serving_input_fn = lambda: json_serving_input_fn() exporter = tf.estimator.FinalExporter("estimator", serving_input_fn, as_text=False) train_spec = tf.estimator.TrainSpec(train_input_fn, max_steps=1000) -eval_spec = tf.estimator.EvalSpec( - eval_input_fn, - exporters=[exporter], - name="estimator-eval", -) +eval_spec = tf.estimator.EvalSpec(eval_input_fn, exporters=[exporter], name="estimator-eval") tf.estimator.train_and_evaluate(classifier, train_spec, eval_spec) # exported path looks like iris_tf_export/export/estimator/1562353043/variables # need to zip the versioned dir -estimator_dir = EXPORT_DIR+"/export/estimator" -shutil.make_archive("tensorflow", 'zip', os.path.join(estimator_dir)) +estimator_dir = EXPORT_DIR + "/export/estimator" +shutil.make_archive("tensorflow", "zip", os.path.join(estimator_dir)) # clean up shutil.rmtree(EXPORT_DIR) diff --git a/pkg/lib/configreader/reader.go b/pkg/lib/configreader/reader.go index 3b40cfdc34..e4b17411ff 100644 --- a/pkg/lib/configreader/reader.go +++ b/pkg/lib/configreader/reader.go @@ -527,7 +527,7 @@ func ReadInterfaceMapValue(name string, interMap map[string]interface{}) (interf // Prompt // -var ui *input.UI = &input.UI{ +var ui = &input.UI{ Writer: os.Stdout, Reader: os.Stdin, } From 226349e665fb838eeaaeebb6c30669bbf1d71fc5 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 5 Jul 2019 16:23:42 -0400 Subject: [PATCH 03/13] test --- pkg/lib/configreader/reader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/lib/configreader/reader.go b/pkg/lib/configreader/reader.go index e4b17411ff..3b40cfdc34 100644 --- a/pkg/lib/configreader/reader.go +++ b/pkg/lib/configreader/reader.go @@ -527,7 +527,7 @@ func ReadInterfaceMapValue(name string, interMap map[string]interface{}) (interf // Prompt // -var ui = &input.UI{ +var ui *input.UI = &input.UI{ Writer: os.Stdout, Reader: os.Stdin, } From ecb43920862f78f9b01840bd69e569eccc60340a Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 5 Jul 2019 17:20:32 -0400 Subject: [PATCH 04/13] validate unzipped content --- pkg/workloads/tf_api/api.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pkg/workloads/tf_api/api.py b/pkg/workloads/tf_api/api.py index 46fb2f8d40..4bb5475008 100644 --- a/pkg/workloads/tf_api/api.py +++ b/pkg/workloads/tf_api/api.py @@ -379,6 +379,20 @@ def predict(deployment_name, api_name): return jsonify(response) +def valid_model_dir(model_dir): + """ + validates that model_dir has the expected directory tree, e.g: + + {TF serving version number}/ + saved_model.pb + variables/ + variables.data-00000-of-00001 + variables.index + """ + version = os.listdir(model_dir)[0] + if not version.isdigit(): + raise "No versions of servable default found under base path in model_dir" + def start(args): ctx = Context(s3_path=args.context, cache_dir=args.cache_dir, workload_id=args.workload_id) @@ -431,6 +445,13 @@ def start(args): if not os.path.isdir(args.model_dir): ctx.storage.download_and_unzip_external(api["model"], args.model_dir) + + try: + valid_model_dir(args.model_dir) + except Exception as e: + logger.exception(e) + sys.exit(1) + channel = grpc.insecure_channel("localhost:" + str(args.tf_serve_port)) local_cache["stub"] = prediction_service_pb2_grpc.PredictionServiceStub(channel) From cfb85a234d93d0bd5513daffe1cd0f3fad743a8d Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 5 Jul 2019 17:22:19 -0400 Subject: [PATCH 05/13] return user exception --- pkg/workloads/tf_api/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/workloads/tf_api/api.py b/pkg/workloads/tf_api/api.py index 4bb5475008..85f8b8f94f 100644 --- a/pkg/workloads/tf_api/api.py +++ b/pkg/workloads/tf_api/api.py @@ -391,7 +391,7 @@ def valid_model_dir(model_dir): """ version = os.listdir(model_dir)[0] if not version.isdigit(): - raise "No versions of servable default found under base path in model_dir" + raise UserException("No versions of servable default found under base path in model_dir") def start(args): From a845387aa04818524ea52af96f04e569a53e78f6 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 5 Jul 2019 17:34:57 -0400 Subject: [PATCH 06/13] make format --- pkg/workloads/tf_api/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/workloads/tf_api/api.py b/pkg/workloads/tf_api/api.py index 85f8b8f94f..7d0c239f41 100644 --- a/pkg/workloads/tf_api/api.py +++ b/pkg/workloads/tf_api/api.py @@ -379,6 +379,7 @@ def predict(deployment_name, api_name): return jsonify(response) + def valid_model_dir(model_dir): """ validates that model_dir has the expected directory tree, e.g: @@ -445,7 +446,6 @@ def start(args): if not os.path.isdir(args.model_dir): ctx.storage.download_and_unzip_external(api["model"], args.model_dir) - try: valid_model_dir(args.model_dir) except Exception as e: From 2df0ba116cd42d176aeeb446826d45a43a9ff62e Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 5 Jul 2019 17:40:11 -0400 Subject: [PATCH 07/13] address comment --- examples/models/tf.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/examples/models/tf.py b/examples/models/tf.py index bacbb45207..ce7d5c9140 100644 --- a/examples/models/tf.py +++ b/examples/models/tf.py @@ -8,17 +8,6 @@ EXPORT_DIR = "iris_tf_export" - -def get_all_file_paths(directory): - file_paths = [] - for root, directories, files in os.walk(directory): - for filename in files: - filepath = os.path.join(root, filename) - file_paths.append(filepath) - - return file_paths - - def input_fn(features, labels, batch_size, mode): """An input function for training""" dataset = tf.data.Dataset.from_tensor_slices((features, labels)) @@ -26,19 +15,19 @@ def input_fn(features, labels, batch_size, mode): dataset = dataset.shuffle(1000).repeat() dataset = dataset.batch(batch_size) dataset_it = dataset.make_one_shot_iterator() - images, labels = dataset_it.get_next() - return {"inputs": images}, labels + irises, labels = dataset_it.get_next() + return {"irises": irises}, labels def json_serving_input_fn(): inputs = tf.placeholder(shape=[4], dtype=tf.float64) - features = {"inputs": tf.expand_dims(inputs, 0)} + features = {"irises": tf.expand_dims(inputs, 0)} return tf.estimator.export.ServingInputReceiver(features=features, receiver_tensors=inputs) def my_model(features, labels, mode, params): """DNN with three hidden layers and learning_rate=0.1.""" - net = features["inputs"] + net = features["irises"] for units in params["hidden_units"]: net = tf.layers.dense(net, units=units, activation=tf.nn.relu) From 04b8b34a5be096e3a6af916c114ea5e606cd21b6 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 5 Jul 2019 17:45:40 -0400 Subject: [PATCH 08/13] format --- examples/models/tf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/models/tf.py b/examples/models/tf.py index ce7d5c9140..914805d76b 100644 --- a/examples/models/tf.py +++ b/examples/models/tf.py @@ -8,6 +8,7 @@ EXPORT_DIR = "iris_tf_export" + def input_fn(features, labels, batch_size, mode): """An input function for training""" dataset = tf.data.Dataset.from_tensor_slices((features, labels)) From 86ecdbf8b79c31f4e25a7acdeafd22b2d221a76d Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Fri, 5 Jul 2019 18:15:29 -0400 Subject: [PATCH 09/13] address some comments --- examples/models/requirements.txt | 2 ++ pkg/workloads/tf_api/api.py | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 examples/models/requirements.txt diff --git a/examples/models/requirements.txt b/examples/models/requirements.txt new file mode 100644 index 0000000000..8e3c98d33c --- /dev/null +++ b/examples/models/requirements.txt @@ -0,0 +1,2 @@ +tensorflow +sklearn diff --git a/pkg/workloads/tf_api/api.py b/pkg/workloads/tf_api/api.py index 7d0c239f41..fe9c6394f3 100644 --- a/pkg/workloads/tf_api/api.py +++ b/pkg/workloads/tf_api/api.py @@ -392,7 +392,14 @@ def valid_model_dir(model_dir): """ version = os.listdir(model_dir)[0] if not version.isdigit(): - raise UserException("No versions of servable default found under base path in model_dir") + raise UserException( + "No versions of servable default found under base path in model_dir. See docs.cortex.dev for how to properly package your TensorFlow model" + ) + + if "saved_model.pb" not in os.listdir(os.path.join(model_dir, version)): + raise UserException( + "Expected packaged model to have a \"saved_model.pb\" file. See docs.cortex.dev for how to properly package your TensorFlow model" + ) def start(args): From e864cae7fb72118865ff79b979af49d719361117 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 5 Jul 2019 16:38:39 -0700 Subject: [PATCH 10/13] Update model.py --- examples/iris/tensorflow/model.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/iris/tensorflow/model.py b/examples/iris/tensorflow/model.py index 914805d76b..5e197be9bf 100644 --- a/examples/iris/tensorflow/model.py +++ b/examples/iris/tensorflow/model.py @@ -86,8 +86,7 @@ def my_model(features, labels, mode, params): tf.estimator.train_and_evaluate(classifier, train_spec, eval_spec) -# exported path looks like iris_tf_export/export/estimator/1562353043/variables -# need to zip the versioned dir +# zip the estimator export dir (the exported path looks like iris_tf_export/export/estimator/1562353043/) estimator_dir = EXPORT_DIR + "/export/estimator" shutil.make_archive("tensorflow", "zip", os.path.join(estimator_dir)) From 9099b00e691454b26f2a478ac62fbdd8acf7c8c5 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 5 Jul 2019 16:44:27 -0700 Subject: [PATCH 11/13] Update api.py --- pkg/workloads/tf_api/api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/workloads/tf_api/api.py b/pkg/workloads/tf_api/api.py index 268b1df951..99405152fd 100644 --- a/pkg/workloads/tf_api/api.py +++ b/pkg/workloads/tf_api/api.py @@ -393,9 +393,11 @@ def predict(deployment_name, api_name): def valid_model_dir(model_dir): """ - validates that model_dir has the expected directory tree, e.g: + validates that model_dir has the expected directory tree. + + For example (your TF serving version number may be different): - {TF serving version number}/ + 1562353043/ saved_model.pb variables/ variables.data-00000-of-00001 From 6e0d3bb52b86f5abae729a4304f4b48d256a0d1f Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 5 Jul 2019 16:45:57 -0700 Subject: [PATCH 12/13] Update api.py --- pkg/workloads/tf_api/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/workloads/tf_api/api.py b/pkg/workloads/tf_api/api.py index 99405152fd..9af5db5e22 100644 --- a/pkg/workloads/tf_api/api.py +++ b/pkg/workloads/tf_api/api.py @@ -391,7 +391,7 @@ def predict(deployment_name, api_name): return jsonify(response) -def valid_model_dir(model_dir): +def validate_model_dir(model_dir): """ validates that model_dir has the expected directory tree. @@ -470,7 +470,7 @@ def start(args): ) try: - valid_model_dir(args.model_dir) + validate_model_dir(args.model_dir) except Exception as e: logger.exception(e) sys.exit(1) From dc1832f6a9053f2ffd8e1bc84cd8cdf48c15348c Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Fri, 5 Jul 2019 16:53:31 -0700 Subject: [PATCH 13/13] Remove trailing whitespace --- pkg/workloads/tf_api/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/workloads/tf_api/api.py b/pkg/workloads/tf_api/api.py index 9af5db5e22..e2e3e4f8fd 100644 --- a/pkg/workloads/tf_api/api.py +++ b/pkg/workloads/tf_api/api.py @@ -394,10 +394,10 @@ def predict(deployment_name, api_name): def validate_model_dir(model_dir): """ validates that model_dir has the expected directory tree. - + For example (your TF serving version number may be different): - 1562353043/ + 1562353043/ saved_model.pb variables/ variables.data-00000-of-00001