From 696a36615688c3d2ca1b2dafd9fb8332b5ef4847 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Wed, 17 Apr 2019 15:03:05 -0700 Subject: [PATCH 01/24] add interactive cli for python converter --- python/requirements.txt | 1 + python/setup.py | 2 + python/tensorflowjs/BUILD | 12 ++ python/tensorflowjs/__init__.py | 1 + python/tensorflowjs/cli.py | 215 ++++++++++++++++++++ python/tensorflowjs/converters/converter.py | 2 +- 6 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 python/tensorflowjs/cli.py diff --git a/python/requirements.txt b/python/requirements.txt index 706171b2..d40e0569 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -4,3 +4,4 @@ numpy==1.15.1 six==1.11.0 tf-nightly-2.0-preview>=2.0.0.dev20190304 tensorflow-hub==0.3.0 +PyInquirer==1.0.3 diff --git a/python/setup.py b/python/setup.py index 004cb48e..d2979deb 100644 --- a/python/setup.py +++ b/python/setup.py @@ -27,6 +27,7 @@ def _get_requirements(file): CONSOLE_SCRIPTS = [ 'tensorflowjs_converter = tensorflowjs.converters.converter:main', + 'tensorflowjs_cli = tensorflowjs.cli:main', ] setuptools.setup( @@ -54,6 +55,7 @@ def _get_requirements(file): ], py_modules=[ 'tensorflowjs', + 'tensorflowjs.cli', 'tensorflowjs.version', 'tensorflowjs.quantization', 'tensorflowjs.read_weights', diff --git a/python/tensorflowjs/BUILD b/python/tensorflowjs/BUILD index 3fbd2f71..d367b110 100644 --- a/python/tensorflowjs/BUILD +++ b/python/tensorflowjs/BUILD @@ -99,6 +99,18 @@ py_test( ], ) +py_binary( + name = "cli", + srcs = ["cli.py"], + srcs_version = "PY2AND3", + deps = [ + ":converters/converter", + "//tensorflowjs:expect_h5py_installed", + "//tensorflowjs:expect_keras_installed", + "//tensorflowjs:expect_tensorflow_installed", + ], +) + # A filegroup BUILD target that includes all the op list json files in the # the op_list/ folder. The op_list folder itself is a symbolic link to the # actual op_list folder under src/. diff --git a/python/tensorflowjs/__init__.py b/python/tensorflowjs/__init__.py index 946f9495..43c8b3f2 100644 --- a/python/tensorflowjs/__init__.py +++ b/python/tensorflowjs/__init__.py @@ -21,5 +21,6 @@ from tensorflowjs import converters from tensorflowjs import quantization from tensorflowjs import version +from tensorflowjs import cli __version__ = version.version diff --git a/python/tensorflowjs/cli.py b/python/tensorflowjs/cli.py new file mode 100644 index 00000000..382613fe --- /dev/null +++ b/python/tensorflowjs/cli.py @@ -0,0 +1,215 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Interactive command line tool for building tensorflow.js conversion command.""" + +from __future__ import print_function, unicode_literals + +import os +import re + +from PyInquirer import prompt +from examples import custom_style_3 + +'''regex for recognizing valid url for TFHub module.''' +URL_REGEX = re.compile( + r'^(?:http|ftp)s?://' # http:// or https:// + # domain... + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' + r'localhost|' # localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip + r'(?::\d+)?' # optional port + r'(?:/?|[/?]\S+)$', re.IGNORECASE) + + +def quantization_type(value): + """Determine the quantization type based on user selection. + Args: + value: user selected value. + """ + answer = None + try: + if '1/2' in value: + answer = 2 + elif '1/4' in value: + answer = 1 + except ValueError: + answer = None + return answer + + +def of_values(answers, key, values): + """Determine user's answers for the key is within the value list. + Args: + answer: Dict of user's answers to the questions. + key: question key. + values: List of values to check from. + """ + value = answers[key] + return value in values + + +def input_path_message(answers): + """Determine question for model's input path. + Args: + answer: Dict of user's answers to the questions. + key: question key. + """ + answer = answers['input_format'] + if answer == 'keras': + return 'What is the path of input HDF5 file?' + elif answer == 'tfhub': + return 'What is the TFHub module URL?' + else: + return 'What is the directory that contains the model?' + + +def validate_input_path(value, input_format): + """validate the input path for given input format. + Args: + value: input path of the model. + input_format: model format string. + """ + if not os.path.exists(value): + return 'Nonexistent path for the model: %s' % value + if input_format in ['keras_saved_model', 'tf_saved_model']: + if not os.path.isdir(value): + return 'The path provided is not a directory: %s' % value + if not any(fname.endswith('.pb') for fname in os.listdir(value)): + return 'This is an invalid saved model directory: %s' % value + if input_format == 'tfhub': + if re.match(URL_REGEX, value) is None: + return 'This is not an valid url for TFHub module: %s' % value + if input_format == 'tfjs_layers_model': + if not os.path.isfile(value): + return 'The path provided is not a file: %s' % value + if not os.path.exists(value + '/model.json'): + return 'This is not a valid directory for layers model: %s' % value + return True + + +def validate_output_path(value): + if os.path.exists(value): + return 'The output path already exists: %s' % value + return True + + +def default_signature_name(answers): + if of_values(answers, 'input_format', ['tf_saved_model']): + return 'serving_default' + return '' + + +def generate_command(options, paths): + args = 'tensorflowjs_converter' + for key, value in options.items(): + if value is not None: + if key is not 'split_weights_by_layer' or value is not False: + args += ' --%s=%s' % (key, value) + + args += ' %s %s' % (paths['input_path'], paths['output_path']) + return args + + +def main(): + print('Weclome to TensorFlow.js converter.') + + questions = [ + { + 'type': 'list', + 'name': 'input_format', + 'message': 'What is your input format?', + 'choices': ['keras', 'keras_saved_model', + 'tf_saved_model', 'tf_hub', 'tfjs_layers_model', + 'tensorflowjs'] + }, + { + 'type': 'input', + 'name': 'saved_model_tags', + 'default': 'serve', + 'message': 'What is tags for the saved model?', + 'when': lambda answers: of_values(answers, 'input_format', + ['tf_saved_model']) + }, + { + 'type': 'input', + 'name': 'signature_name', + 'message': 'What is signature name of the model?', + 'default': default_signature_name, + 'when': lambda answers: of_values(answers, 'input_format', + ['tf_saved_model', 'tfhub']) + }, + { + 'type': 'list', + 'name': 'quantization_bytes', + 'message': 'Do you want to compress the model? ' + '(this will decrease the model precision.)', + 'choices': ['No compression', + 'compress weights to 1/2 the size', + 'compress weights to 1/4 the size'], + 'filter': quantization_type + }, + { + 'type': 'confirm', + 'name': 'split_weights_by_layer', + 'message': 'Do you want to split weights by layers?', + 'default': False, + 'when': lambda answers: of_values(answers, 'input_format', + ['tf_layers_model']) + }, + { + 'type': 'confirm', + 'name': 'skip_op_check', + 'message': 'Do you want to skip op validation?', + 'default': False, + 'when': lambda answers: of_values(answers, 'input_format', + ['tf_saved_model', 'tfhub']) + }, + { + 'type': 'confirm', + 'name': 'strip_debug_ops', + 'message': 'Do you want to strip debug ops?', + 'default': False, + 'when': lambda answers: of_values(answers, 'input_format', + ['tf_saved_model', 'tfhub']) + } + ] + + options = prompt(questions, style=custom_style_3) + print('Conversion configuration:') + + message = input_path_message(options) + directories = [ + { + 'type': 'input', + 'name': 'input_path', + 'message': message, + 'validate': lambda value: validate_input_path(value, options['input_format']) + }, + { + 'type': 'input', + 'name': 'output_path', + 'message': 'Which directory do you want save the converted model?', + 'validate': lambda value: validate_output_path(value) + }, + ] + + paths = prompt(directories, style=custom_style_3) + command = generate_command(options, paths) + print(command) + os.system(command) + + +if __name__ == '__main__': + main() diff --git a/python/tensorflowjs/converters/converter.py b/python/tensorflowjs/converters/converter.py index 9491bcc1..5eb8b979 100644 --- a/python/tensorflowjs/converters/converter.py +++ b/python/tensorflowjs/converters/converter.py @@ -193,7 +193,7 @@ def _standardize_input_output_formats(input_format, output_format): input_format_is_keras = ( input_format in ['keras', 'keras_saved_model']) input_format_is_tf = ( - input_format in ['tf_frozen_model', 'tf_hub']) + input_format in ['tf_saved_model', 'tf_hub']) if output_format is None: # If no explicit output_format is provided, infer it from input format. if input_format_is_keras: From f0f69895b74ef300eb76ae69cf5c73b073b10f2b Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Tue, 25 Jun 2019 09:18:28 -0700 Subject: [PATCH 02/24] add auto looking up the saved model tags and signatures --- python/tensorflowjs/cli.py | 82 +++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 32 deletions(-) diff --git a/python/tensorflowjs/cli.py b/python/tensorflowjs/cli.py index 382613fe..8762b05a 100644 --- a/python/tensorflowjs/cli.py +++ b/python/tensorflowjs/cli.py @@ -21,6 +21,7 @@ from PyInquirer import prompt from examples import custom_style_3 +from tensorflow.python.saved_model.loader_impl import parse_saved_model '''regex for recognizing valid url for TFHub module.''' URL_REGEX = re.compile( @@ -111,21 +112,39 @@ def default_signature_name(answers): return '' -def generate_command(options, paths): +def generate_command(options, params): + params.update(options) args = 'tensorflowjs_converter' - for key, value in options.items(): - if value is not None: - if key is not 'split_weights_by_layer' or value is not False: + not_param_list = ['input_path', 'output_path'] + no_false_param = ['split_weights_by_layer', 'skip_op_check'] + for key, value in params.items(): + if key not in not_param_list and value is not None: + if key not in no_false_param or value is not False: args += ' --%s=%s' % (key, value) - args += ' %s %s' % (paths['input_path'], paths['output_path']) + args += ' %s %s' % (params['input_path'], params['output_path']) return args +def available_tags(answers): + saved_model = parse_saved_model(answers['input_path']) + tags = [] + for meta_graph in saved_model.meta_graphs: + tags.append(",".join(meta_graph.meta_info_def.tags)) + return tags + +def available_signature_names(answers): + path = answers['input_path'] + tags = answers['saved_model_tags'] + saved_model = parse_saved_model(path) + for meta_graph in saved_model.meta_graphs: + if tags == ",".join(meta_graph.meta_info_def.tags): + return meta_graph.signature_def.keys() + return [] def main(): print('Weclome to TensorFlow.js converter.') - questions = [ + input_format = [ { 'type': 'list', 'name': 'input_format', @@ -133,22 +152,34 @@ def main(): 'choices': ['keras', 'keras_saved_model', 'tf_saved_model', 'tf_hub', 'tfjs_layers_model', 'tensorflowjs'] - }, + } + ] + + options = prompt(input_format, style=custom_style_3) + message = input_path_message(options) + + questions = [ { 'type': 'input', + 'name': 'input_path', + 'message': message, + 'validate': lambda value: validate_input_path(value, options['input_format']) + }, + { + 'type': 'list', 'name': 'saved_model_tags', - 'default': 'serve', + 'choices': available_tags, 'message': 'What is tags for the saved model?', - 'when': lambda answers: of_values(answers, 'input_format', + 'when': lambda answers: of_values(options, 'input_format', ['tf_saved_model']) }, { - 'type': 'input', + 'type': 'list', 'name': 'signature_name', 'message': 'What is signature name of the model?', - 'default': default_signature_name, - 'when': lambda answers: of_values(answers, 'input_format', - ['tf_saved_model', 'tfhub']) + 'choices': available_signature_names, + 'when': lambda answers: of_values(options, 'input_format', + ['tf_saved_model']) }, { 'type': 'list', @@ -165,7 +196,7 @@ def main(): 'name': 'split_weights_by_layer', 'message': 'Do you want to split weights by layers?', 'default': False, - 'when': lambda answers: of_values(answers, 'input_format', + 'when': lambda answers: of_values(options, 'input_format', ['tf_layers_model']) }, { @@ -173,7 +204,7 @@ def main(): 'name': 'skip_op_check', 'message': 'Do you want to skip op validation?', 'default': False, - 'when': lambda answers: of_values(answers, 'input_format', + 'when': lambda answers: of_values(options, 'input_format', ['tf_saved_model', 'tfhub']) }, { @@ -181,32 +212,19 @@ def main(): 'name': 'strip_debug_ops', 'message': 'Do you want to strip debug ops?', 'default': False, - 'when': lambda answers: of_values(answers, 'input_format', + 'when': lambda answers: of_values(options, 'input_format', ['tf_saved_model', 'tfhub']) - } - ] - - options = prompt(questions, style=custom_style_3) - print('Conversion configuration:') - - message = input_path_message(options) - directories = [ - { - 'type': 'input', - 'name': 'input_path', - 'message': message, - 'validate': lambda value: validate_input_path(value, options['input_format']) }, { 'type': 'input', 'name': 'output_path', 'message': 'Which directory do you want save the converted model?', 'validate': lambda value: validate_output_path(value) - }, + } ] + params = prompt(questions, style=custom_style_3) - paths = prompt(directories, style=custom_style_3) - command = generate_command(options, paths) + command = generate_command(options, params) print(command) os.system(command) From e944f24fd8b5be2bbc7ae26eff6d9124a66fca6c Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Fri, 28 Jun 2019 16:23:02 -0700 Subject: [PATCH 03/24] fixed pylint errors and added tests --- python/tensorflowjs/BUILD | 10 + python/tensorflowjs/cli.py | 137 ++++++++------ python/tensorflowjs/cli_test.py | 194 ++++++++++++++++++++ python/tensorflowjs/converters/converter.py | 1 + 4 files changed, 289 insertions(+), 53 deletions(-) create mode 100644 python/tensorflowjs/cli_test.py diff --git a/python/tensorflowjs/BUILD b/python/tensorflowjs/BUILD index d367b110..7a8942c3 100644 --- a/python/tensorflowjs/BUILD +++ b/python/tensorflowjs/BUILD @@ -99,6 +99,16 @@ py_test( ], ) +py_test( + name = "cli_test", + srcs = ["cli_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":expect_numpy_installed", + ":cli", + ], +) + py_binary( name = "cli", srcs = ["cli.py"], diff --git a/python/tensorflowjs/cli.py b/python/tensorflowjs/cli.py index 8762b05a..9862b7d5 100644 --- a/python/tensorflowjs/cli.py +++ b/python/tensorflowjs/cli.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Interactive command line tool for building tensorflow.js conversion command.""" +"""Interactive command line tool for tensorflow.js model conversion.""" from __future__ import print_function, unicode_literals @@ -23,15 +23,10 @@ from examples import custom_style_3 from tensorflow.python.saved_model.loader_impl import parse_saved_model -'''regex for recognizing valid url for TFHub module.''' +# regex for recognizing valid url for TFHub module. URL_REGEX = re.compile( - r'^(?:http|ftp)s?://' # http:// or https:// - # domain... - r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' - r'localhost|' # localhost... - r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip - r'(?::\d+)?' # optional port - r'(?:/?|[/?]\S+)$', re.IGNORECASE) + # http:// or https:// + r'^(?:http)s?://', re.IGNORECASE) def quantization_type(value): @@ -57,8 +52,11 @@ def of_values(answers, key, values): key: question key. values: List of values to check from. """ - value = answers[key] - return value in values + try: + value = answers[key] + return value in values + except KeyError: + return False def input_path_message(answers): @@ -70,7 +68,7 @@ def input_path_message(answers): answer = answers['input_format'] if answer == 'keras': return 'What is the path of input HDF5 file?' - elif answer == 'tfhub': + elif answer == 'tf_hub': return 'What is the TFHub module URL?' else: return 'What is the directory that contains the model?' @@ -82,80 +80,102 @@ def validate_input_path(value, input_format): value: input path of the model. input_format: model format string. """ - if not os.path.exists(value): + value = os.path.expanduser(value) + if input_format == 'tf_hub': + if re.match(URL_REGEX, value) is None: + return 'This is not an valid url for TFHub module: %s' % value + elif not os.path.exists(value): return 'Nonexistent path for the model: %s' % value if input_format in ['keras_saved_model', 'tf_saved_model']: if not os.path.isdir(value): return 'The path provided is not a directory: %s' % value if not any(fname.endswith('.pb') for fname in os.listdir(value)): return 'This is an invalid saved model directory: %s' % value - if input_format == 'tfhub': - if re.match(URL_REGEX, value) is None: - return 'This is not an valid url for TFHub module: %s' % value if input_format == 'tfjs_layers_model': if not os.path.isfile(value): return 'The path provided is not a file: %s' % value - if not os.path.exists(value + '/model.json'): - return 'This is not a valid directory for layers model: %s' % value + if input_format == 'keras': + if not os.path.isfile(value): + return 'The path provided is not a file: %s' % value return True def validate_output_path(value): + value = os.path.expanduser(value) if os.path.exists(value): return 'The output path already exists: %s' % value return True -def default_signature_name(answers): - if of_values(answers, 'input_format', ['tf_saved_model']): - return 'serving_default' - return '' - - def generate_command(options, params): params.update(options) args = 'tensorflowjs_converter' not_param_list = ['input_path', 'output_path'] no_false_param = ['split_weights_by_layer', 'skip_op_check'] - for key, value in params.items(): + for key, value in sorted(params.items()): if key not in not_param_list and value is not None: - if key not in no_false_param or value is not False: + if key in no_false_param: + if value is True: + args += ' --%s' % (key) + else: args += ' --%s=%s' % (key, value) args += ' %s %s' % (params['input_path'], params['output_path']) return args + +def available_output_formats(answers): + input_format = answers['input_format'] + if input_format == 'keras_saved_model': + return ['tfjs_graph_model', 'tfjs_layers_model'] + if input_format == 'tfjs_layers_model': + return ['keras', 'tfjs_graph_model'] + return [] + def available_tags(answers): - saved_model = parse_saved_model(answers['input_path']) - tags = [] - for meta_graph in saved_model.meta_graphs: - tags.append(",".join(meta_graph.meta_info_def.tags)) - return tags + if answers['input_format'] == 'tf_saved_model': + saved_model = parse_saved_model(answers['input_path']) + tags = [] + for meta_graph in saved_model.meta_graphs: + tags.append(",".join(meta_graph.meta_info_def.tags)) + return tags + return [] + def available_signature_names(answers): - path = answers['input_path'] - tags = answers['saved_model_tags'] - saved_model = parse_saved_model(path) - for meta_graph in saved_model.meta_graphs: - if tags == ",".join(meta_graph.meta_info_def.tags): - return meta_graph.signature_def.keys() + if answers['input_format'] == 'tf_saved_model': + path = answers['input_path'] + tags = answers['saved_model_tags'] + saved_model = parse_saved_model(path) + for meta_graph in saved_model.meta_graphs: + if tags == ",".join(meta_graph.meta_info_def.tags): + return meta_graph.signature_def.keys() return [] + def main(): print('Weclome to TensorFlow.js converter.') - input_format = [ + formats = [ { 'type': 'list', 'name': 'input_format', 'message': 'What is your input format?', 'choices': ['keras', 'keras_saved_model', - 'tf_saved_model', 'tf_hub', 'tfjs_layers_model', - 'tensorflowjs'] + 'tf_saved_model', 'tf_hub', 'tfjs_layers_model'] + }, + { + 'type': 'list', + 'name': 'output_format', + 'message': 'What is your output format?', + 'choices': available_output_formats, + 'when': lambda answers: of_values(answers, 'input_format', + ['keras_saved_model', + 'tfjs_layers_model']) } ] - options = prompt(input_format, style=custom_style_3) + options = prompt(formats, style=custom_style_3) message = input_path_message(options) questions = [ @@ -163,14 +183,16 @@ def main(): 'type': 'input', 'name': 'input_path', 'message': message, - 'validate': lambda value: validate_input_path(value, options['input_format']) + 'filter': os.path.expanduser, + 'validate': lambda value: validate_input_path( + value, options['input_format']) }, { 'type': 'list', 'name': 'saved_model_tags', 'choices': available_tags, 'message': 'What is tags for the saved model?', - 'when': lambda answers: of_values(options, 'input_format', + 'when': lambda answers: of_values(answers, 'input_format', ['tf_saved_model']) }, { @@ -178,51 +200,60 @@ def main(): 'name': 'signature_name', 'message': 'What is signature name of the model?', 'choices': available_signature_names, - 'when': lambda answers: of_values(options, 'input_format', + 'when': lambda answers: of_values(answers, 'input_format', ['tf_saved_model']) }, { 'type': 'list', 'name': 'quantization_bytes', 'message': 'Do you want to compress the model? ' - '(this will decrease the model precision.)', + '(this will decrease the model precision.)', 'choices': ['No compression', 'compress weights to 1/2 the size', 'compress weights to 1/4 the size'], 'filter': quantization_type }, + { + 'type': 'input', + 'name': 'weight_shard_size_byte', + 'message': 'Please enter shard size (in bytes) of the weight files?', + 'default': str(4 * 1024 * 1024), + 'when': lambda answers: of_values(answers, 'output_format', + ['tfjs_layers_model']) + }, { 'type': 'confirm', 'name': 'split_weights_by_layer', 'message': 'Do you want to split weights by layers?', 'default': False, - 'when': lambda answers: of_values(options, 'input_format', - ['tf_layers_model']) + 'when': lambda answers: of_values(answers, 'input_format', + ['tfjs_layers_model']) }, { 'type': 'confirm', 'name': 'skip_op_check', 'message': 'Do you want to skip op validation?', 'default': False, - 'when': lambda answers: of_values(options, 'input_format', - ['tf_saved_model', 'tfhub']) + 'when': lambda answers: of_values(answers, 'input_format', + ['tf_saved_model', 'tf_hub']) }, { 'type': 'confirm', 'name': 'strip_debug_ops', 'message': 'Do you want to strip debug ops?', 'default': False, - 'when': lambda answers: of_values(options, 'input_format', - ['tf_saved_model', 'tfhub']) + 'when': lambda answers: of_values(answers, 'input_format', + ['tf_saved_model', 'tf_hub']) }, { 'type': 'input', 'name': 'output_path', 'message': 'Which directory do you want save the converted model?', - 'validate': lambda value: validate_output_path(value) + 'filter': os.path.expanduser, + 'validate': validate_output_path } ] - params = prompt(questions, style=custom_style_3) + params = prompt(questions, options, style=custom_style_3) command = generate_command(options, params) print(command) diff --git a/python/tensorflowjs/cli_test.py b/python/tensorflowjs/cli_test.py new file mode 100644 index 00000000..38bc1ed2 --- /dev/null +++ b/python/tensorflowjs/cli_test.py @@ -0,0 +1,194 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import tempfile +import os +import shutil +import tensorflow as tf +from tensorflow.python.eager import def_function +from tensorflow.python.ops import variables +from tensorflow.python.training.tracking import tracking +from tensorflow.python.saved_model.save import save + +from tensorflowjs import cli + +SAVED_MODEL_DIR = 'saved_model' +HD5_FILE_NAME = 'test.HD5' + + +class CliTest(unittest.TestCase): + def setUp(self): + self._tmp_dir = tempfile.mkdtemp() + super(CliTest, self).setUp() + + def tearDown(self): + if os.path.isdir(self._tmp_dir): + shutil.rmtree(self._tmp_dir) + super(CliTest, self).tearDown() + + def _create_hd5_file(self): + filename = os.path.join(self._tmp_dir, 'test.HD5') + open(filename, 'a').close() + + def _create_saved_model(self): + """Test a basic model with functions to make sure functions are inlined.""" + input_data = tf.constant(1., shape=[1]) + root = tracking.AutoTrackable() + root.v1 = variables.Variable(3.) + root.v2 = variables.Variable(2.) + root.f = def_function.function(lambda x: root.v1 * root.v2 * x) + to_save = root.f.get_concrete_function(input_data) + + save_dir = os.path.join(self._tmp_dir, SAVED_MODEL_DIR) + save(root, save_dir, to_save) + + def testQuantizationType(self): + self.assertEqual(2, cli.quantization_type('1/2')) + self.assertEqual(1, cli.quantization_type('1/4')) + self.assertEqual(None, cli.quantization_type('1')) + + def testOfValues(self): + answers = {'input_path': 'abc', 'input_format': '123'} + self.assertEqual(True, cli.of_values(answers, 'input_path', ['abc'])) + self.assertEqual(False, cli.of_values(answers, 'input_path', ['abd'])) + self.assertEqual(False, cli.of_values(answers, 'input_format2', ['abc'])) + + def testInputPathMessage(self): + answers = {'input_format': 'keras'} + self.assertEqual("What is the path of input HDF5 file?", + cli.input_path_message(answers)) + + answers = {'input_format': 'tf_hub'} + self.assertEqual("What is the TFHub module URL?", + cli.input_path_message(answers)) + + answers = {'input_format': 'tf_saved_model'} + self.assertEqual("What is the directory that contains the model?", + cli.input_path_message(answers)) + + def testValidateInputPathForTFHub(self): + self.assertNotEqual(True, cli.validate_input_path(self._tmp_dir, 'tf_hub')) + self.assertEqual(True, + cli.validate_input_path("https://tfhub.dev/mobilenet", + 'tf_hub')) + + def testValidateInputPathForSavedModel(self): + self.assertNotEqual(True, cli.validate_input_path( + self._tmp_dir, 'tf_saved_model')) + self._create_saved_model() + save_dir = os.path.join(self._tmp_dir, SAVED_MODEL_DIR) + self.assertEqual(True, cli.validate_input_path( + save_dir, 'tf_saved_model')) + + def testValidateInputPathForKerasSavedModel(self): + self.assertNotEqual(True, cli.validate_input_path( + self._tmp_dir, 'keras_saved_model')) + self._create_saved_model() + save_dir = os.path.join(self._tmp_dir, SAVED_MODEL_DIR) + self.assertEqual(True, cli.validate_input_path( + save_dir, 'keras_saved_model')) + + def testValidateInputPathForKerasModel(self): + self.assertNotEqual(True, cli.validate_input_path(self._tmp_dir, 'keras')) + self._create_hd5_file() + save_dir = os.path.join(self._tmp_dir, HD5_FILE_NAME) + self.assertEqual(True, cli.validate_input_path( + save_dir, 'keras')) + + def testValidateOutputPath(self): + self.assertNotEqual(True, cli.validate_output_path(self._tmp_dir)) + output_dir = os.path.join(self._tmp_dir, 'test') + self.assertEqual(True, cli.validate_output_path(output_dir)) + + def testAvailableTags(self): + self._create_saved_model() + save_dir = os.path.join(self._tmp_dir, SAVED_MODEL_DIR) + self.assertEqual(['serve'], cli.available_tags( + {'input_path': save_dir, + 'input_format': 'tf_saved_model'})) + + def testAvailableSignatureNames(self): + self._create_saved_model() + save_dir = os.path.join(self._tmp_dir, SAVED_MODEL_DIR) + self.assertEqual(['__saved_model_init_op', 'serving_default'], + list(cli.available_signature_names( + {'input_path': save_dir, + 'input_format': 'tf_saved_model', + 'saved_model_tags': 'serve'}))) + + def testGenerateCommandForSavedModel(self): + options = {'input_format': 'tf_saved_model'} + params = {'input_path': 'tmp/saved_model', + 'saved_model_tags': 'test', + 'signature_name': 'test_default', + 'quantization_bytes': 2, + 'skip_op_check': False, + 'strip_debug_ops': True, + 'output_path': 'tmp/web_model'} + + self.assertEqual('tensorflowjs_converter --input_format=tf_saved_model ' + + '--quantization_bytes=2 --saved_model_tags=test ' + + '--signature_name=test_default --strip_debug_ops=True ' + + 'tmp/saved_model tmp/web_model', + cli.generate_command(options, params)) + + def testGenerateCommandForKerasSavedModel(self): + options = {'input_format': 'tf_keras_saved_model', + 'output_format': 'tfjs_layers_model'} + params = {'input_path': 'tmp/saved_model', + 'saved_model_tags': 'test', + 'signature_name': 'test_default', + 'quantization_bytes': 1, + 'skip_op_check': True, + 'strip_debug_ops': False, + 'output_path': 'tmp/web_model'} + + self.assertEqual('tensorflowjs_converter ' + + '--input_format=tf_keras_saved_model ' + + '--output_format=tfjs_layers_model ' + + '--quantization_bytes=1 --saved_model_tags=test ' + + '--signature_name=test_default --skip_op_check ' + + '--strip_debug_ops=False tmp/saved_model tmp/web_model', + cli.generate_command(options, params)) + + def testGenerateCommandForKerasModel(self): + options = {'input_format': 'keras'} + params = {'input_path': 'tmp/model.HD5', + 'quantization_bytes': 1, + 'output_path': 'tmp/web_model'} + + self.assertEqual('tensorflowjs_converter --input_format=keras ' + + '--quantization_bytes=1 tmp/model.HD5 tmp/web_model', + cli.generate_command(options, params)) + + def testGenerateCommandForLayerModel(self): + options = {'input_format': 'tfjs_layers_model', + 'output_format': 'keras'} + params = {'input_path': 'tmp/model.json', + 'quantization_bytes': 1, + 'output_path': 'tmp/web_model'} + + self.assertEqual('tensorflowjs_converter ' + + '--input_format=tfjs_layers_model ' + + '--output_format=keras ' + + '--quantization_bytes=1 tmp/model.json tmp/web_model', + cli.generate_command(options, params)) + +if __name__ == '__main__': + unittest.main() diff --git a/python/tensorflowjs/converters/converter.py b/python/tensorflowjs/converters/converter.py index 69a1748f..ebd9bb26 100644 --- a/python/tensorflowjs/converters/converter.py +++ b/python/tensorflowjs/converters/converter.py @@ -556,6 +556,7 @@ def main(): strip_debug_ops=FLAGS.strip_debug_ops) elif (input_format == 'tf_hub' and output_format == 'tfjs_graph_model'): + print(FLAGS) tf_saved_model_conversion_v2.convert_tf_hub_module( FLAGS.input_path, FLAGS.output_path, FLAGS.signature_name, skip_op_check=FLAGS.skip_op_check, From 7e93a4f8f6fbce39c4094ee1bee73204faac7db7 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Fri, 28 Jun 2019 17:25:39 -0700 Subject: [PATCH 04/24] adding more docstrings --- python/tensorflowjs/cli.py | 51 +++++++++++++----- python/tensorflowjs/cli_test.py | 95 +++++++++++++++++---------------- 2 files changed, 87 insertions(+), 59 deletions(-) diff --git a/python/tensorflowjs/cli.py b/python/tensorflowjs/cli.py index 9862b7d5..5b510422 100644 --- a/python/tensorflowjs/cli.py +++ b/python/tensorflowjs/cli.py @@ -46,7 +46,7 @@ def quantization_type(value): def of_values(answers, key, values): - """Determine user's answers for the key is within the value list. + """Determine user's answer for the key is in the value list. Args: answer: Dict of user's answers to the questions. key: question key. @@ -62,8 +62,7 @@ def of_values(answers, key, values): def input_path_message(answers): """Determine question for model's input path. Args: - answer: Dict of user's answers to the questions. - key: question key. + answer: Dict of user's answers to the questions """ answer = answers['input_format'] if answer == 'keras': @@ -101,14 +100,22 @@ def validate_input_path(value, input_format): def validate_output_path(value): + """validate the input path for given input format. + Args: + value: input path of the model. + input_format: model format string. + """ value = os.path.expanduser(value) if os.path.exists(value): return 'The output path already exists: %s' % value return True -def generate_command(options, params): - params.update(options) +def generate_command(params): + """generate the tensorflowjs command string for the selected params. + Args: + params: user selected parameters for the conversion. + """ args = 'tensorflowjs_converter' not_param_list = ['input_path', 'output_path'] no_false_param = ['split_weights_by_layer', 'skip_op_check'] @@ -123,8 +130,20 @@ def generate_command(options, params): args += ' %s %s' % (params['input_path'], params['output_path']) return args +def is_saved_model(answers): + """check if the input path contains saved model. + Args: + params: user selected parameters for the conversion. + """ + return answers['input_format'] == 'tf_saved_model' or \ + answers['input_format'] == 'keras_saved_model' and \ + answers['output_format'] == 'tfjs_graph_model' def available_output_formats(answers): + """generate the output formats for given input format. + Args: + ansowers: user selected parameter dict. + """ input_format = answers['input_format'] if input_format == 'keras_saved_model': return ['tfjs_graph_model', 'tfjs_layers_model'] @@ -132,8 +151,13 @@ def available_output_formats(answers): return ['keras', 'tfjs_graph_model'] return [] + def available_tags(answers): - if answers['input_format'] == 'tf_saved_model': + """generate the available saved model tags from the proto file. + Args: + ansowers: user selected parameter dict. + """ + if is_saved_model(answers): saved_model = parse_saved_model(answers['input_path']) tags = [] for meta_graph in saved_model.meta_graphs: @@ -143,7 +167,12 @@ def available_tags(answers): def available_signature_names(answers): - if answers['input_format'] == 'tf_saved_model': + """generate the available saved model signatures from the proto file + and selected tags. + Args: + ansowers: user selected parameter dict. + """ + if is_saved_model(answers): path = answers['input_path'] tags = answers['saved_model_tags'] saved_model = parse_saved_model(path) @@ -192,16 +221,14 @@ def main(): 'name': 'saved_model_tags', 'choices': available_tags, 'message': 'What is tags for the saved model?', - 'when': lambda answers: of_values(answers, 'input_format', - ['tf_saved_model']) + 'when': is_saved_model }, { 'type': 'list', 'name': 'signature_name', 'message': 'What is signature name of the model?', 'choices': available_signature_names, - 'when': lambda answers: of_values(answers, 'input_format', - ['tf_saved_model']) + 'when': is_saved_model }, { 'type': 'list', @@ -241,7 +268,7 @@ def main(): 'type': 'confirm', 'name': 'strip_debug_ops', 'message': 'Do you want to strip debug ops?', - 'default': False, + 'default': True, 'when': lambda answers: of_values(answers, 'input_format', ['tf_saved_model', 'tf_hub']) }, diff --git a/python/tensorflowjs/cli_test.py b/python/tensorflowjs/cli_test.py index 38bc1ed2..45804912 100644 --- a/python/tensorflowjs/cli_test.py +++ b/python/tensorflowjs/cli_test.py @@ -133,62 +133,63 @@ def testAvailableSignatureNames(self): 'saved_model_tags': 'serve'}))) def testGenerateCommandForSavedModel(self): - options = {'input_format': 'tf_saved_model'} - params = {'input_path': 'tmp/saved_model', - 'saved_model_tags': 'test', - 'signature_name': 'test_default', - 'quantization_bytes': 2, - 'skip_op_check': False, - 'strip_debug_ops': True, - 'output_path': 'tmp/web_model'} - - self.assertEqual('tensorflowjs_converter --input_format=tf_saved_model ' + - '--quantization_bytes=2 --saved_model_tags=test ' + - '--signature_name=test_default --strip_debug_ops=True ' + - 'tmp/saved_model tmp/web_model', - cli.generate_command(options, params)) + options = {'input_format': 'tf_saved_model', + 'input_path': 'tmp/saved_model', + 'saved_model_tags': 'test', + 'signature_name': 'test_default', + 'quantization_bytes': 2, + 'skip_op_check': False, + 'strip_debug_ops': True, + 'output_path': 'tmp/web_model'} + + self.assertEqual(('tensorflowjs_converter --input_format=tf_saved_model ' + '--quantization_bytes=2 --saved_model_tags=test ' + '--signature_name=test_default --strip_debug_ops=True ' + 'tmp/saved_model tmp/web_model'), + cli.generate_command(options)) def testGenerateCommandForKerasSavedModel(self): options = {'input_format': 'tf_keras_saved_model', - 'output_format': 'tfjs_layers_model'} - params = {'input_path': 'tmp/saved_model', - 'saved_model_tags': 'test', - 'signature_name': 'test_default', - 'quantization_bytes': 1, - 'skip_op_check': True, - 'strip_debug_ops': False, - 'output_path': 'tmp/web_model'} - - self.assertEqual('tensorflowjs_converter ' + - '--input_format=tf_keras_saved_model ' + - '--output_format=tfjs_layers_model ' + - '--quantization_bytes=1 --saved_model_tags=test ' + - '--signature_name=test_default --skip_op_check ' + - '--strip_debug_ops=False tmp/saved_model tmp/web_model', - cli.generate_command(options, params)) + 'output_format': 'tfjs_layers_model', + 'input_path': 'tmp/saved_model', + 'saved_model_tags': 'test', + 'signature_name': 'test_default', + 'quantization_bytes': 1, + 'skip_op_check': True, + 'strip_debug_ops': False, + 'output_path': 'tmp/web_model'} + + self.assertEqual(('tensorflowjs_converter ' + '--input_format=tf_keras_saved_model ' + '--output_format=tfjs_layers_model ' + '--quantization_bytes=1 --saved_model_tags=test ' + '--signature_name=test_default --skip_op_check ' + '--strip_debug_ops=False tmp/saved_model tmp/web_model'), + cli.generate_command(options)) def testGenerateCommandForKerasModel(self): - options = {'input_format': 'keras'} - params = {'input_path': 'tmp/model.HD5', - 'quantization_bytes': 1, - 'output_path': 'tmp/web_model'} + options = {'input_format': 'keras', + 'input_path': 'tmp/model.HD5', + 'quantization_bytes': 1, + 'output_path': 'tmp/web_model'} - self.assertEqual('tensorflowjs_converter --input_format=keras ' + - '--quantization_bytes=1 tmp/model.HD5 tmp/web_model', - cli.generate_command(options, params)) + self.assertEqual(('tensorflowjs_converter --input_format=keras ' + '--quantization_bytes=1 tmp/model.HD5 tmp/web_model'), + cli.generate_command(options)) def testGenerateCommandForLayerModel(self): options = {'input_format': 'tfjs_layers_model', - 'output_format': 'keras'} - params = {'input_path': 'tmp/model.json', - 'quantization_bytes': 1, - 'output_path': 'tmp/web_model'} - - self.assertEqual('tensorflowjs_converter ' + - '--input_format=tfjs_layers_model ' + - '--output_format=keras ' + - '--quantization_bytes=1 tmp/model.json tmp/web_model', - cli.generate_command(options, params)) + 'output_format': 'keras', + 'input_path': 'tmp/model.json', + 'quantization_bytes': 1, + 'output_path': 'tmp/web_model'} + + self.assertEqual(('tensorflowjs_converter ' + '--input_format=tfjs_layers_model ' + '--output_format=keras ' + '--quantization_bytes=1 tmp/model.json tmp/web_model'), + cli.generate_command(options)) + if __name__ == '__main__': unittest.main() From c9a63dcc57f5eea6398c468419951549c4f9362a Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Mon, 1 Jul 2019 13:46:25 -0700 Subject: [PATCH 05/24] fix typo --- python/tensorflowjs/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tensorflowjs/cli.py b/python/tensorflowjs/cli.py index 5b510422..f1ac00cd 100644 --- a/python/tensorflowjs/cli.py +++ b/python/tensorflowjs/cli.py @@ -282,7 +282,7 @@ def main(): ] params = prompt(questions, options, style=custom_style_3) - command = generate_command(options, params) + command = generate_command(params) print(command) os.system(command) From 82a064841a4cf8b8c229ad68529b17510e6ddbe7 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Wed, 3 Jul 2019 13:12:49 -0700 Subject: [PATCH 06/24] update the cli workflow according to the design doc comments --- python/tensorflowjs/cli.py | 257 ++++++++++++++++---- python/tensorflowjs/cli_test.py | 54 ++-- python/tensorflowjs/converters/converter.py | 12 +- 3 files changed, 246 insertions(+), 77 deletions(-) diff --git a/python/tensorflowjs/cli.py b/python/tensorflowjs/cli.py index f1ac00cd..a8611ae6 100644 --- a/python/tensorflowjs/cli.py +++ b/python/tensorflowjs/cli.py @@ -18,16 +18,25 @@ import os import re +import json from PyInquirer import prompt from examples import custom_style_3 from tensorflow.python.saved_model.loader_impl import parse_saved_model - +from tensorflowjs.converters.converter import main as convert # regex for recognizing valid url for TFHub module. URL_REGEX = re.compile( # http:// or https:// r'^(?:http)s?://', re.IGNORECASE) +DETECTED_INPUT_FORMAT = 'detected_input_format' +KERAS_SAVED_MODEL = 'keras_saved_model' +KERAS_MODEL = 'keras' +TF_SAVED_MODEL = 'tf_saved_model' +TF_HUB = 'tf_hub' +TFJS_GRAPH_MODEL = 'tfjs_keras_model' +TFJS_LAYERS_MODEL = 'tfjs_layers_model' + def quantization_type(value): """Determine the quantization type based on user selection. @@ -59,18 +68,51 @@ def of_values(answers, key, values): return False +def getTFJSModelType(file): + with open(file) as f: + data = json.load(f) + return data['format'] + + +def detect_input_format(answers): + """Determine the input format from model's input path or file. + Args: + answer: Dict of user's answers to the questions + """ + value = answers['input_path'] + if re.match(URL_REGEX, value): + answers[DETECTED_INPUT_FORMAT] = TF_HUB + elif (os.path.isdir(value) and + any(fname.endswith('.pb') for fname in os.listdir(value))): + answers[DETECTED_INPUT_FORMAT] = TF_SAVED_MODEL + elif os.path.isfile(value) and value.endswith('.HDF5'): + answers[DETECTED_INPUT_FORMAT] = KERAS_MODEL + elif os.path.isdir(value) and value.endswith('model.json'): + if getTFJSModelType(value) == 'layers-model': + answers[DETECTED_INPUT_FORMAT] = TFJS_LAYERS_MODEL + elif os.path.isdir(value): + for fname in os.listdir(value): + if fname.endswith('model.json'): + filename = os.path.join(value, fname) + if getTFJSModelType(filename) == 'layers-model': + answers['input_path'] = os.path.join(value, fname) + answers[DETECTED_INPUT_FORMAT] = TFJS_LAYERS_MODEL + break + + def input_path_message(answers): """Determine question for model's input path. Args: answer: Dict of user's answers to the questions """ answer = answers['input_format'] - if answer == 'keras': - return 'What is the path of input HDF5 file?' - elif answer == 'tf_hub': - return 'What is the TFHub module URL?' + message = 'The original path seems to be wrong, ' + if answer == KERAS_MODEL: + return message + 'what is the path of input HDF5 file?' + elif answer == TF_HUB: + return message + 'what is the TFHub module URL?' else: - return 'What is the directory that contains the model?' + return message + 'what is the directory that contains the model?' def validate_input_path(value, input_format): @@ -80,20 +122,22 @@ def validate_input_path(value, input_format): input_format: model format string. """ value = os.path.expanduser(value) - if input_format == 'tf_hub': + if not value: + return 'Please enter a valid path' + if input_format == TF_HUB: if re.match(URL_REGEX, value) is None: return 'This is not an valid url for TFHub module: %s' % value elif not os.path.exists(value): return 'Nonexistent path for the model: %s' % value - if input_format in ['keras_saved_model', 'tf_saved_model']: + if input_format in [KERAS_SAVED_MODEL, TF_SAVED_MODEL]: if not os.path.isdir(value): return 'The path provided is not a directory: %s' % value if not any(fname.endswith('.pb') for fname in os.listdir(value)): return 'This is an invalid saved model directory: %s' % value - if input_format == 'tfjs_layers_model': + if input_format == TFJS_LAYERS_MODEL: if not os.path.isfile(value): return 'The path provided is not a file: %s' % value - if input_format == 'keras': + if input_format == KERAS_MODEL: if not os.path.isfile(value): return 'The path provided is not a file: %s' % value return True @@ -106,38 +150,43 @@ def validate_output_path(value): input_format: model format string. """ value = os.path.expanduser(value) + if not value: + return 'Please provide a valid output path' if os.path.exists(value): return 'The output path already exists: %s' % value return True -def generate_command(params): +def generate_arguments(params): """generate the tensorflowjs command string for the selected params. Args: params: user selected parameters for the conversion. """ - args = 'tensorflowjs_converter' - not_param_list = ['input_path', 'output_path'] + args = [] + not_param_list = [DETECTED_INPUT_FORMAT, 'input_path', 'output_path'] no_false_param = ['split_weights_by_layer', 'skip_op_check'] for key, value in sorted(params.items()): if key not in not_param_list and value is not None: if key in no_false_param: if value is True: - args += ' --%s' % (key) + args.append('--%s' % (key)) else: - args += ' --%s=%s' % (key, value) + args.append('--%s=%s' % (key, value)) - args += ' %s %s' % (params['input_path'], params['output_path']) + args.append(params['input_path']) + args.append(params['output_path']) return args + def is_saved_model(answers): """check if the input path contains saved model. Args: params: user selected parameters for the conversion. """ - return answers['input_format'] == 'tf_saved_model' or \ - answers['input_format'] == 'keras_saved_model' and \ - answers['output_format'] == 'tfjs_graph_model' + return answers['input_format'] == TF_SAVED_MODEL or \ + answers['input_format'] == KERAS_SAVED_MODEL and \ + answers['output_format'] == TFJS_GRAPH_MODEL + def available_output_formats(answers): """generate the output formats for given input format. @@ -145,10 +194,26 @@ def available_output_formats(answers): ansowers: user selected parameter dict. """ input_format = answers['input_format'] - if input_format == 'keras_saved_model': - return ['tfjs_graph_model', 'tfjs_layers_model'] - if input_format == 'tfjs_layers_model': - return ['keras', 'tfjs_graph_model'] + if input_format == KERAS_SAVED_MODEL: + return [{ + 'key': 'g', + 'name': 'Tensorflow.js Graph Model', + 'value': TFJS_GRAPH_MODEL, + }, { + 'key': 'l', + 'name': 'TensoFlow.js Layers Model', + 'value': TFJS_LAYERS_MODEL, + }] + if input_format == TFJS_LAYERS_MODEL: + return [{ + 'key': 'k', + 'name': 'Keras Model (HDF5)', + 'value': KERAS_MODEL, + }, { + 'key': 'l', + 'name': 'TensoFlow.js Layers Model', + 'value': TFJS_LAYERS_MODEL, + }] return [] @@ -178,33 +243,122 @@ def available_signature_names(answers): saved_model = parse_saved_model(path) for meta_graph in saved_model.meta_graphs: if tags == ",".join(meta_graph.meta_info_def.tags): - return meta_graph.signature_def.keys() + signatures = [] + for key in meta_graph.signature_def: + input_nodes = meta_graph.signature_def[key].inputs + output_nodes = meta_graph.signature_def[key].outputs + signatures.append( + {'value': key, + 'name': format_signature(key, input_nodes, output_nodes)}) + return signatures return [] +def format_signature(name, input_nodes, output_nodes): + string = f"signature name: {name}\n" + string += f" inputs: \n{format_nodes(input_nodes)}" + string += f" outputs: \n{format_nodes(output_nodes)}" + return string + + +def format_nodes(nodes): + string = "" + for key in nodes: + value = nodes[key] + string += f" name: {value.name}, " + string += f"dtype: {value.dtype}, " + string += f"shape: {list(map(lambda x: x.size, value.tensor_shape.dim))}\n" + return string + + +def input_format_string(base, target_format, detected_format): + if target_format == detected_format: + return base + ' *' + else: + return base + + +def input_format_message(answers): + message = 'What is your input model format? ' + if DETECTED_INPUT_FORMAT in answers: + detected_format = answers[DETECTED_INPUT_FORMAT] + message += '(auto-detected format is marked with *)' + else: + message += '(model format cannot be detected.) ' + return message + + +def input_formats(answers): + detected_format = None + if DETECTED_INPUT_FORMAT in answers: + detected_format = answers[DETECTED_INPUT_FORMAT] + formats = [{ + 'key': 'k', + 'name': input_format_string('Keras (HDF5)', KERAS_MODEL, + detected_format), + 'value': KERAS_MODEL + }, { + 'key': 'e', + 'name': input_format_string('Tensorflow Keras Saved Model', + KERAS_SAVED_MODEL, + detected_format), + 'value': KERAS_SAVED_MODEL, + }, { + 'key': 's', + 'name': input_format_string('Tensorflow Saved Model', + TF_SAVED_MODEL, + detected_format), + 'value': TF_SAVED_MODEL, + }, { + 'key': 'h', + 'name': input_format_string('TFHub Module', + TF_HUB, + detected_format), + 'value': TF_HUB, + }, { + 'key': 'l', + 'name': input_format_string('TensoFlow.js Layers Model', + TFJS_LAYERS_MODEL, + detected_format), + 'value': TFJS_LAYERS_MODEL, + }] + formats.sort(key=lambda x: x['value'] != detected_format) + return formats + + def main(): print('Weclome to TensorFlow.js converter.') + input_path = [{ + 'type': 'input', + 'name': 'input_path', + 'message': 'Please provide the path of model file or ' + 'the directory that contains model files. \n' + 'If you are converting TFHub module please provide the URL.', + 'filter': os.path.expanduser, + 'validate': + lambda value: 'Please enter a valid path' if not value else True + }] + + input_params = prompt(input_path, style=custom_style_3) + detect_input_format(input_params) formats = [ { 'type': 'list', 'name': 'input_format', - 'message': 'What is your input format?', - 'choices': ['keras', 'keras_saved_model', - 'tf_saved_model', 'tf_hub', 'tfjs_layers_model'] - }, + 'message': input_format_message(input_params), + 'choices': input_formats(input_params)}, { 'type': 'list', 'name': 'output_format', 'message': 'What is your output format?', 'choices': available_output_formats, 'when': lambda answers: of_values(answers, 'input_format', - ['keras_saved_model', - 'tfjs_layers_model']) + [KERAS_SAVED_MODEL, + TFJS_LAYERS_MODEL]) } ] - - options = prompt(formats, style=custom_style_3) + options = prompt(formats, input_params, style=custom_style_3) message = input_path_message(options) questions = [ @@ -214,7 +368,9 @@ def main(): 'message': message, 'filter': os.path.expanduser, 'validate': lambda value: validate_input_path( - value, options['input_format']) + value, options['input_format']), + 'when': lambda answers: (DETECTED_INPUT_FORMAT not in answers) + }, { 'type': 'list', @@ -235,9 +391,9 @@ def main(): 'name': 'quantization_bytes', 'message': 'Do you want to compress the model? ' '(this will decrease the model precision.)', - 'choices': ['No compression', - 'compress weights to 1/2 the size', - 'compress weights to 1/4 the size'], + 'choices': ['No compression, no accuracy loss.', + '2x compression, medium accuracy loss.', + '4x compression, highest accuracy loss.'], 'filter': quantization_type }, { @@ -246,7 +402,7 @@ def main(): 'message': 'Please enter shard size (in bytes) of the weight files?', 'default': str(4 * 1024 * 1024), 'when': lambda answers: of_values(answers, 'output_format', - ['tfjs_layers_model']) + [TFJS_LAYERS_MODEL]) }, { 'type': 'confirm', @@ -254,38 +410,45 @@ def main(): 'message': 'Do you want to split weights by layers?', 'default': False, 'when': lambda answers: of_values(answers, 'input_format', - ['tfjs_layers_model']) + [TFJS_LAYERS_MODEL]) }, { 'type': 'confirm', 'name': 'skip_op_check', - 'message': 'Do you want to skip op validation?', + 'message': 'Do you want to skip op validation? \n' + 'This will allow conversion of unsupported ops, \n' + 'you can implement them as custom ops in tfjs-converter.', 'default': False, 'when': lambda answers: of_values(answers, 'input_format', - ['tf_saved_model', 'tf_hub']) + [TF_SAVED_MODEL, TF_HUB]) }, { 'type': 'confirm', 'name': 'strip_debug_ops', - 'message': 'Do you want to strip debug ops?', + 'message': 'Do you want to strip debug ops? \n' + 'This will improve model execution performance.', 'default': True, 'when': lambda answers: of_values(answers, 'input_format', - ['tf_saved_model', 'tf_hub']) + [TF_SAVED_MODEL, TF_HUB]) }, { 'type': 'input', 'name': 'output_path', - 'message': 'Which directory do you want save the converted model?', + 'message': 'Which directory do you want to save ' + 'the converted model in?', 'filter': os.path.expanduser, 'validate': validate_output_path } ] params = prompt(questions, options, style=custom_style_3) - command = generate_command(params) - print(command) - os.system(command) - + arguments = generate_arguments(params) + convert(arguments) + print('file generated after conversion:') + with os.scandir(params['output_path']) as it: + for entry in it: + print(entry.name, + os.path.getsize(os.path.join(params['output_path'], entry.name))) if __name__ == '__main__': main() diff --git a/python/tensorflowjs/cli_test.py b/python/tensorflowjs/cli_test.py index 45804912..4c93388f 100644 --- a/python/tensorflowjs/cli_test.py +++ b/python/tensorflowjs/cli_test.py @@ -71,15 +71,18 @@ def testOfValues(self): def testInputPathMessage(self): answers = {'input_format': 'keras'} - self.assertEqual("What is the path of input HDF5 file?", + self.assertEqual("The original path seems to be wrong, " + "what is the path of input HDF5 file?", cli.input_path_message(answers)) answers = {'input_format': 'tf_hub'} - self.assertEqual("What is the TFHub module URL?", + self.assertEqual("The original path seems to be wrong, " + "what is the TFHub module URL?", cli.input_path_message(answers)) answers = {'input_format': 'tf_saved_model'} - self.assertEqual("What is the directory that contains the model?", + self.assertEqual("The original path seems to be wrong, " + "what is the directory that contains the model?", cli.input_path_message(answers)) def testValidateInputPathForTFHub(self): @@ -127,10 +130,11 @@ def testAvailableSignatureNames(self): self._create_saved_model() save_dir = os.path.join(self._tmp_dir, SAVED_MODEL_DIR) self.assertEqual(['__saved_model_init_op', 'serving_default'], - list(cli.available_signature_names( + list(map(lambda x: x['value'], + cli.available_signature_names( {'input_path': save_dir, 'input_format': 'tf_saved_model', - 'saved_model_tags': 'serve'}))) + 'saved_model_tags': 'serve'})))) def testGenerateCommandForSavedModel(self): options = {'input_format': 'tf_saved_model', @@ -142,11 +146,11 @@ def testGenerateCommandForSavedModel(self): 'strip_debug_ops': True, 'output_path': 'tmp/web_model'} - self.assertEqual(('tensorflowjs_converter --input_format=tf_saved_model ' - '--quantization_bytes=2 --saved_model_tags=test ' - '--signature_name=test_default --strip_debug_ops=True ' - 'tmp/saved_model tmp/web_model'), - cli.generate_command(options)) + self.assertEqual(['--input_format=tf_saved_model', + '--quantization_bytes=2', '--saved_model_tags=test', + '--signature_name=test_default', '--strip_debug_ops=True', + 'tmp/saved_model', 'tmp/web_model'], + cli.generate_arguments(options)) def testGenerateCommandForKerasSavedModel(self): options = {'input_format': 'tf_keras_saved_model', @@ -159,13 +163,13 @@ def testGenerateCommandForKerasSavedModel(self): 'strip_debug_ops': False, 'output_path': 'tmp/web_model'} - self.assertEqual(('tensorflowjs_converter ' - '--input_format=tf_keras_saved_model ' - '--output_format=tfjs_layers_model ' - '--quantization_bytes=1 --saved_model_tags=test ' - '--signature_name=test_default --skip_op_check ' - '--strip_debug_ops=False tmp/saved_model tmp/web_model'), - cli.generate_command(options)) + self.assertEqual(['--input_format=tf_keras_saved_model', + '--output_format=tfjs_layers_model', + '--quantization_bytes=1', '--saved_model_tags=test', + '--signature_name=test_default', '--skip_op_check', + '--strip_debug_ops=False', 'tmp/saved_model', + 'tmp/web_model'], + cli.generate_arguments(options)) def testGenerateCommandForKerasModel(self): options = {'input_format': 'keras', @@ -173,9 +177,9 @@ def testGenerateCommandForKerasModel(self): 'quantization_bytes': 1, 'output_path': 'tmp/web_model'} - self.assertEqual(('tensorflowjs_converter --input_format=keras ' - '--quantization_bytes=1 tmp/model.HD5 tmp/web_model'), - cli.generate_command(options)) + self.assertEqual(['--input_format=keras', '--quantization_bytes=1', + 'tmp/model.HD5', 'tmp/web_model'], + cli.generate_arguments(options)) def testGenerateCommandForLayerModel(self): options = {'input_format': 'tfjs_layers_model', @@ -184,11 +188,11 @@ def testGenerateCommandForLayerModel(self): 'quantization_bytes': 1, 'output_path': 'tmp/web_model'} - self.assertEqual(('tensorflowjs_converter ' - '--input_format=tfjs_layers_model ' - '--output_format=keras ' - '--quantization_bytes=1 tmp/model.json tmp/web_model'), - cli.generate_command(options)) + self.assertEqual(['--input_format=tfjs_layers_model', + '--output_format=keras', + '--quantization_bytes=1', 'tmp/model.json', + 'tmp/web_model'], + cli.generate_arguments(options)) if __name__ == '__main__': diff --git a/python/tensorflowjs/converters/converter.py b/python/tensorflowjs/converters/converter.py index d8e44345..50fc7ff6 100644 --- a/python/tensorflowjs/converters/converter.py +++ b/python/tensorflowjs/converters/converter.py @@ -399,7 +399,7 @@ def _parse_quantization_bytes(quantization_bytes): raise ValueError('Unsupported quantization bytes: %s' % quantization_bytes) -def setup_arguments(): +def setup_arguments(arguments=None): parser = argparse.ArgumentParser('TensorFlow.js model converters.') parser.add_argument( 'input_path', @@ -487,11 +487,13 @@ def setup_arguments(): default=None, help='Shard size (in bytes) of the weight files. Curently applicable ' 'only to output_format=tfjs_layers_model.') - return parser.parse_args() - + if arguments: + return parser.parse_args(arguments) + else: + return parser.parse_args() -def main(): - FLAGS = setup_arguments() +def main(arguments=None): + FLAGS = setup_arguments(arguments) if FLAGS.show_version: print('\ntensorflowjs %s\n' % version.version) print('Dependency versions:') From 91f781e3e7100ddcbfe1282cc1cf1bb5a3aba925 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Wed, 3 Jul 2019 13:18:54 -0700 Subject: [PATCH 07/24] fix pylint --- python/tensorflowjs/cli.py | 1 - python/tensorflowjs/cli_test.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/python/tensorflowjs/cli.py b/python/tensorflowjs/cli.py index a8611ae6..d3da289c 100644 --- a/python/tensorflowjs/cli.py +++ b/python/tensorflowjs/cli.py @@ -281,7 +281,6 @@ def input_format_string(base, target_format, detected_format): def input_format_message(answers): message = 'What is your input model format? ' if DETECTED_INPUT_FORMAT in answers: - detected_format = answers[DETECTED_INPUT_FORMAT] message += '(auto-detected format is marked with *)' else: message += '(model format cannot be detected.) ' diff --git a/python/tensorflowjs/cli_test.py b/python/tensorflowjs/cli_test.py index 4c93388f..77406190 100644 --- a/python/tensorflowjs/cli_test.py +++ b/python/tensorflowjs/cli_test.py @@ -131,10 +131,10 @@ def testAvailableSignatureNames(self): save_dir = os.path.join(self._tmp_dir, SAVED_MODEL_DIR) self.assertEqual(['__saved_model_init_op', 'serving_default'], list(map(lambda x: x['value'], - cli.available_signature_names( - {'input_path': save_dir, - 'input_format': 'tf_saved_model', - 'saved_model_tags': 'serve'})))) + cli.available_signature_names( + {'input_path': save_dir, + 'input_format': 'tf_saved_model', + 'saved_model_tags': 'serve'})))) def testGenerateCommandForSavedModel(self): options = {'input_format': 'tf_saved_model', From ea5ed070e4f0c02fb9dd94582344de777abb1ca2 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Wed, 3 Jul 2019 14:17:13 -0700 Subject: [PATCH 08/24] show the dtype string instead of value --- python/tensorflowjs/cli.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/python/tensorflowjs/cli.py b/python/tensorflowjs/cli.py index d3da289c..bb95d818 100644 --- a/python/tensorflowjs/cli.py +++ b/python/tensorflowjs/cli.py @@ -23,6 +23,7 @@ from PyInquirer import prompt from examples import custom_style_3 from tensorflow.python.saved_model.loader_impl import parse_saved_model +from tensorflow.core.framework import types_pb2 from tensorflowjs.converters.converter import main as convert # regex for recognizing valid url for TFHub module. URL_REGEX = re.compile( @@ -255,9 +256,9 @@ def available_signature_names(answers): def format_signature(name, input_nodes, output_nodes): - string = f"signature name: {name}\n" - string += f" inputs: \n{format_nodes(input_nodes)}" - string += f" outputs: \n{format_nodes(output_nodes)}" + string = "signature name: %s\n" % name + string += " inputs: \n%s" % format_nodes(input_nodes) + string += " outputs: \n%s" % format_nodes(output_nodes) return string @@ -265,9 +266,11 @@ def format_nodes(nodes): string = "" for key in nodes: value = nodes[key] - string += f" name: {value.name}, " - string += f"dtype: {value.dtype}, " - string += f"shape: {list(map(lambda x: x.size, value.tensor_shape.dim))}\n" + string += " name: %s, " % value.name + string += "dtype: %s, " % types_pb2.DataType.Name(value.dtype) + shape = 'Unknown' if value.tensor_shape.unknown_rank else list( + map(lambda x: x.size, value.tensor_shape.dim)) + string += "shape: %s\n" % shape return string From 093c8aac2d0ec00fc5ab689c1314f81414a64af6 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Wed, 3 Jul 2019 14:38:49 -0700 Subject: [PATCH 09/24] fix pylint error --- python/tensorflowjs/cli.py | 8 +++++--- python/tensorflowjs/cli_test.py | 9 ++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/python/tensorflowjs/cli.py b/python/tensorflowjs/cli.py index bb95d818..8bf56694 100644 --- a/python/tensorflowjs/cli.py +++ b/python/tensorflowjs/cli.py @@ -268,9 +268,10 @@ def format_nodes(nodes): value = nodes[key] string += " name: %s, " % value.name string += "dtype: %s, " % types_pb2.DataType.Name(value.dtype) - shape = 'Unknown' if value.tensor_shape.unknown_rank else list( - map(lambda x: x.size, value.tensor_shape.dim)) - string += "shape: %s\n" % shape + if value.tensor_shape.unknown_rank: + string += "shape: Unknown\n" + else: + string += "shape: %s\n" % [x.size for x in value.tensor_shape.dim] return string @@ -452,5 +453,6 @@ def main(): print(entry.name, os.path.getsize(os.path.join(params['output_path'], entry.name))) + if __name__ == '__main__': main() diff --git a/python/tensorflowjs/cli_test.py b/python/tensorflowjs/cli_test.py index 77406190..a3fee750 100644 --- a/python/tensorflowjs/cli_test.py +++ b/python/tensorflowjs/cli_test.py @@ -130,11 +130,10 @@ def testAvailableSignatureNames(self): self._create_saved_model() save_dir = os.path.join(self._tmp_dir, SAVED_MODEL_DIR) self.assertEqual(['__saved_model_init_op', 'serving_default'], - list(map(lambda x: x['value'], - cli.available_signature_names( - {'input_path': save_dir, - 'input_format': 'tf_saved_model', - 'saved_model_tags': 'serve'})))) + [x['value'] for x in cli.available_signature_names( + {'input_path': save_dir, + 'input_format': 'tf_saved_model', + 'saved_model_tags': 'serve'})]) def testGenerateCommandForSavedModel(self): options = {'input_format': 'tf_saved_model', From 3431a0724d6fc25f09c75355e450cf6883749d9b Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Wed, 17 Jul 2019 11:02:03 -0700 Subject: [PATCH 10/24] move to questionary pip to support prompt_toolkit 2 --- python/requirements.txt | 2 +- python/tensorflowjs/cli.py | 138 +++++++++++--------- python/tensorflowjs/converters/converter.py | 4 +- 3 files changed, 83 insertions(+), 61 deletions(-) diff --git a/python/requirements.txt b/python/requirements.txt index b3025df3..5f8615d0 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -4,4 +4,4 @@ numpy==1.16.4 six==1.11.0 tensorflow==1.14.0 tensorflow-hub==0.5.0 -PyInquirer==1.0.3 +questionary==1.1.1 diff --git a/python/tensorflowjs/cli.py b/python/tensorflowjs/cli.py index 8bf56694..4d54b0ec 100644 --- a/python/tensorflowjs/cli.py +++ b/python/tensorflowjs/cli.py @@ -20,17 +20,26 @@ import re import json -from PyInquirer import prompt -from examples import custom_style_3 +from questionary import prompt +from prompt_toolkit.styles import Style from tensorflow.python.saved_model.loader_impl import parse_saved_model from tensorflow.core.framework import types_pb2 -from tensorflowjs.converters.converter import main as convert +from tensorflowjs.converters.converter import convert # regex for recognizing valid url for TFHub module. -URL_REGEX = re.compile( +TFHUB_VALID_URL_REGEX = re.compile( # http:// or https:// r'^(?:http)s?://', re.IGNORECASE) -DETECTED_INPUT_FORMAT = 'detected_input_format' +# prompt style +prompt_style = Style([ + ('separator', 'fg:#cc5454'), + ('qmark', 'fg:#673ab7 bold'), + ('question', ''), + ('selected', 'fg:#cc5454'), + ('pointer', 'fg:#673ab7 bold'), + ('answer', 'fg:#f44336 bold'), +]) + KERAS_SAVED_MODEL = 'keras_saved_model' KERAS_MODEL = 'keras' TF_SAVED_MODEL = 'tf_saved_model' @@ -39,23 +48,26 @@ TFJS_LAYERS_MODEL = 'tfjs_layers_model' -def quantization_type(value): +def quantization_type(user_selection_quant): """Determine the quantization type based on user selection. Args: - value: user selected value. + user_selection_quant: user selected quantization value. + + Returns: + int: quantization parameter value for converter. """ answer = None try: - if '1/2' in value: + if '1/2' in user_selection_quant: answer = 2 - elif '1/4' in value: + elif '1/4' in user_selection_quant: answer = 1 except ValueError: answer = None return answer -def of_values(answers, key, values): +def value_in_list(answers, key, values): """Determine user's answer for the key is in the value list. Args: answer: Dict of user's answers to the questions. @@ -69,7 +81,7 @@ def of_values(answers, key, values): return False -def getTFJSModelType(file): +def get_tfjs_model_type(file): with open(file) as f: data = json.load(f) return data['format'] @@ -79,27 +91,31 @@ def detect_input_format(answers): """Determine the input format from model's input path or file. Args: answer: Dict of user's answers to the questions + returns: + string: detected input format + string: normalized input path """ + detected_input_format = None value = answers['input_path'] - if re.match(URL_REGEX, value): - answers[DETECTED_INPUT_FORMAT] = TF_HUB + if re.match(TFHUB_VALID_URL_REGEX, value): + detected_input_format = TF_HUB elif (os.path.isdir(value) and any(fname.endswith('.pb') for fname in os.listdir(value))): - answers[DETECTED_INPUT_FORMAT] = TF_SAVED_MODEL + detected_input_format = TF_SAVED_MODEL elif os.path.isfile(value) and value.endswith('.HDF5'): - answers[DETECTED_INPUT_FORMAT] = KERAS_MODEL + detected_input_format = KERAS_MODEL elif os.path.isdir(value) and value.endswith('model.json'): - if getTFJSModelType(value) == 'layers-model': - answers[DETECTED_INPUT_FORMAT] = TFJS_LAYERS_MODEL + if get_tfjs_model_type(value) == 'layers-model': + detected_input_format = TFJS_LAYERS_MODEL elif os.path.isdir(value): for fname in os.listdir(value): if fname.endswith('model.json'): filename = os.path.join(value, fname) - if getTFJSModelType(filename) == 'layers-model': - answers['input_path'] = os.path.join(value, fname) - answers[DETECTED_INPUT_FORMAT] = TFJS_LAYERS_MODEL + if get_tfjs_model_type(filename) == 'layers-model': + value = os.path.join(value, fname) + detected_input_format = TFJS_LAYERS_MODEL break - + return detect_input_path, value def input_path_message(answers): """Determine question for model's input path. @@ -116,45 +132,48 @@ def input_path_message(answers): return message + 'what is the directory that contains the model?' -def validate_input_path(value, input_format): +def validate_input_path(input_path, input_format): """validate the input path for given input format. Args: - value: input path of the model. + input_path: input path of the model. input_format: model format string. """ - value = os.path.expanduser(value) - if not value: + input_path = os.path.expanduser(input_path) + if not input_path: return 'Please enter a valid path' if input_format == TF_HUB: - if re.match(URL_REGEX, value) is None: - return 'This is not an valid url for TFHub module: %s' % value - elif not os.path.exists(value): + if re.match(TFHUB_VALID_URL_REGEX, input_path) is None: + return """This is not an valid URL for TFHub module: %s, + We expect a URL that starts with http(s)://""" % value + elif not os.path.exists(input_path): return 'Nonexistent path for the model: %s' % value if input_format in [KERAS_SAVED_MODEL, TF_SAVED_MODEL]: - if not os.path.isdir(value): + if not os.path.isdir(input_path): return 'The path provided is not a directory: %s' % value if not any(fname.endswith('.pb') for fname in os.listdir(value)): - return 'This is an invalid saved model directory: %s' % value + return 'Did not find a .pb file inside the model\'s directory: %s' % value if input_format == TFJS_LAYERS_MODEL: - if not os.path.isfile(value): + if not os.path.isfile(input_path): return 'The path provided is not a file: %s' % value if input_format == KERAS_MODEL: - if not os.path.isfile(value): + if not os.path.isfile(input_path): return 'The path provided is not a file: %s' % value return True -def validate_output_path(value): +def validate_output_path(output_path): """validate the input path for given input format. Args: - value: input path of the model. + output_path: input path of the model. input_format: model format string. + Returns: + bool: return true when the output directory does not exist. """ - value = os.path.expanduser(value) - if not value: + output_path = os.path.expanduser(output_path) + if not output_path: return 'Please provide a valid output path' - if os.path.exists(value): - return 'The output path already exists: %s' % value + if os.path.exists(output_path): + return 'The output path already exists: %s' % output_path return True @@ -162,9 +181,11 @@ def generate_arguments(params): """generate the tensorflowjs command string for the selected params. Args: params: user selected parameters for the conversion. + Returns: + list: the argument list for converter. """ args = [] - not_param_list = [DETECTED_INPUT_FORMAT, 'input_path', 'output_path'] + not_param_list = ['input_path', 'output_path'] no_false_param = ['split_weights_by_layer', 'skip_op_check'] for key, value in sorted(params.items()): if key not in not_param_list and value is not None: @@ -183,6 +204,8 @@ def is_saved_model(answers): """check if the input path contains saved model. Args: params: user selected parameters for the conversion. + Returns: + bool: """ return answers['input_format'] == TF_SAVED_MODEL or \ answers['input_format'] == KERAS_SAVED_MODEL and \ @@ -282,19 +305,16 @@ def input_format_string(base, target_format, detected_format): return base -def input_format_message(answers): +def input_format_message(detected_input_format): message = 'What is your input model format? ' - if DETECTED_INPUT_FORMAT in answers: + if detected_input_format: message += '(auto-detected format is marked with *)' else: message += '(model format cannot be detected.) ' return message -def input_formats(answers): - detected_format = None - if DETECTED_INPUT_FORMAT in answers: - detected_format = answers[DETECTED_INPUT_FORMAT] +def input_formats(detected_format): formats = [{ 'key': 'k', 'name': input_format_string('Keras (HDF5)', KERAS_MODEL, @@ -342,26 +362,28 @@ def main(): lambda value: 'Please enter a valid path' if not value else True }] - input_params = prompt(input_path, style=custom_style_3) - detect_input_format(input_params) + input_params = prompt(input_path, style=prompt_style) + detected_input_format, normalized_path = detect_input_format( + input_params['input_path']) + input_params['input_path'] = normalized_path formats = [ { 'type': 'list', 'name': 'input_format', - 'message': input_format_message(input_params), + 'message': input_format_message(detected_input_format), 'choices': input_formats(input_params)}, { 'type': 'list', 'name': 'output_format', 'message': 'What is your output format?', 'choices': available_output_formats, - 'when': lambda answers: of_values(answers, 'input_format', + 'when': lambda answers: value_in_list(answers, 'input_format', [KERAS_SAVED_MODEL, TFJS_LAYERS_MODEL]) } ] - options = prompt(formats, input_params, style=custom_style_3) + options = prompt(formats, input_params, style=prompt_style) message = input_path_message(options) questions = [ @@ -372,8 +394,7 @@ def main(): 'filter': os.path.expanduser, 'validate': lambda value: validate_input_path( value, options['input_format']), - 'when': lambda answers: (DETECTED_INPUT_FORMAT not in answers) - + 'when': lambda answers: detect_input_format }, { 'type': 'list', @@ -404,7 +425,7 @@ def main(): 'name': 'weight_shard_size_byte', 'message': 'Please enter shard size (in bytes) of the weight files?', 'default': str(4 * 1024 * 1024), - 'when': lambda answers: of_values(answers, 'output_format', + 'when': lambda answers: value_in_list(answers, 'output_format', [TFJS_LAYERS_MODEL]) }, { @@ -412,7 +433,7 @@ def main(): 'name': 'split_weights_by_layer', 'message': 'Do you want to split weights by layers?', 'default': False, - 'when': lambda answers: of_values(answers, 'input_format', + 'when': lambda answers: value_in_list(answers, 'input_format', [TFJS_LAYERS_MODEL]) }, { @@ -422,7 +443,7 @@ def main(): 'This will allow conversion of unsupported ops, \n' 'you can implement them as custom ops in tfjs-converter.', 'default': False, - 'when': lambda answers: of_values(answers, 'input_format', + 'when': lambda answers: value_in_list(answers, 'input_format', [TF_SAVED_MODEL, TF_HUB]) }, { @@ -431,7 +452,7 @@ def main(): 'message': 'Do you want to strip debug ops? \n' 'This will improve model execution performance.', 'default': True, - 'when': lambda answers: of_values(answers, 'input_format', + 'when': lambda answers: value_in_list(answers, 'input_format', [TF_SAVED_MODEL, TF_HUB]) }, { @@ -443,15 +464,14 @@ def main(): 'validate': validate_output_path } ] - params = prompt(questions, options, style=custom_style_3) + params = prompt(questions, options, style=prompt_style) arguments = generate_arguments(params) convert(arguments) print('file generated after conversion:') with os.scandir(params['output_path']) as it: for entry in it: - print(entry.name, - os.path.getsize(os.path.join(params['output_path'], entry.name))) + print(entry.name, entry.stat().st_size) if __name__ == '__main__': diff --git a/python/tensorflowjs/converters/converter.py b/python/tensorflowjs/converters/converter.py index 50fc7ff6..100130d8 100644 --- a/python/tensorflowjs/converters/converter.py +++ b/python/tensorflowjs/converters/converter.py @@ -492,7 +492,7 @@ def setup_arguments(arguments=None): else: return parser.parse_args() -def main(arguments=None): +def convert(arguments=None): FLAGS = setup_arguments(arguments) if FLAGS.show_version: print('\ntensorflowjs %s\n' % version.version) @@ -585,6 +585,8 @@ def main(arguments=None): 'Unsupported input_format - output_format pair: %s - %s' % (input_format, output_format)) +def main(): + convert() if __name__ == '__main__': main() From 0c7d70bce4584203fac4ec41c95f96da0ef6fde9 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Mon, 29 Jul 2019 16:44:20 -0700 Subject: [PATCH 11/24] update the README and fixed the tests --- README.md | 21 +++++++- python/requirements.txt | 2 +- python/tensorflowjs/cli.py | 87 +++++++++++++++++---------------- python/tensorflowjs/cli_test.py | 7 +-- 4 files changed, 69 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 1a701e9f..2fd9ee74 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,26 @@ __1. Install the TensorFlow.js pip package:__ pip install tensorflowjs ``` -__2. Run the converter script provided by the pip package:__ +__2. Run the conversion script provided by the pip package:__ + +There are two way to trigger the model conversion: + +- The interactive CLI: tensorflowjs_cli +- Regular conversion script: tensorflowjs_converter + +To start the interactive CLI: +```bash +tensorflowjs_cli +``` + +This tool will walk you through the conversion process and provide you with +details explanations for each choice you need to make. Behind the scene it calls +the converter script (tensorflowjs_converter) in pip package. This is the easier +way to convert a single model. + +To convert a batch of models or integrate the conversion process into your own script, you should look into using the tensorflowjs_converter script. + +Here is detail information of parameters of the converter script. The converter expects a __TensorFlow SavedModel__, __TensorFlow Hub module__, __TensorFlow.js JSON__ format, __Keras HDF5 model__, or __tf.keras SavedModel__ diff --git a/python/requirements.txt b/python/requirements.txt index 5f8615d0..b3025df3 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -4,4 +4,4 @@ numpy==1.16.4 six==1.11.0 tensorflow==1.14.0 tensorflow-hub==0.5.0 -questionary==1.1.1 +PyInquirer==1.0.3 diff --git a/python/tensorflowjs/cli.py b/python/tensorflowjs/cli.py index 4d54b0ec..0aa8b8cc 100644 --- a/python/tensorflowjs/cli.py +++ b/python/tensorflowjs/cli.py @@ -20,8 +20,7 @@ import re import json -from questionary import prompt -from prompt_toolkit.styles import Style +from PyInquirer import prompt, style_from_dict, Token from tensorflow.python.saved_model.loader_impl import parse_saved_model from tensorflow.core.framework import types_pb2 from tensorflowjs.converters.converter import convert @@ -31,14 +30,16 @@ r'^(?:http)s?://', re.IGNORECASE) # prompt style -prompt_style = Style([ - ('separator', 'fg:#cc5454'), - ('qmark', 'fg:#673ab7 bold'), - ('question', ''), - ('selected', 'fg:#cc5454'), - ('pointer', 'fg:#673ab7 bold'), - ('answer', 'fg:#f44336 bold'), -]) +prompt_style = style_from_dict({ + Token.Separator: '#6C6C6C', + Token.QuestionMark: '#FF9D00 bold', + # Token.Selected: '', # default + Token.Selected: '#5F819D', + Token.Pointer: '#FF9D00 bold', + Token.Instruction: '', # default + Token.Answer: '#5F819D bold', + Token.Question: '', +}) KERAS_SAVED_MODEL = 'keras_saved_model' KERAS_MODEL = 'keras' @@ -87,35 +88,35 @@ def get_tfjs_model_type(file): return data['format'] -def detect_input_format(answers): +def detect_input_format(input_path): """Determine the input format from model's input path or file. Args: - answer: Dict of user's answers to the questions + input_path: string of the input model path returns: string: detected input format string: normalized input path """ detected_input_format = None - value = answers['input_path'] - if re.match(TFHUB_VALID_URL_REGEX, value): + if re.match(TFHUB_VALID_URL_REGEX, input_path): detected_input_format = TF_HUB - elif (os.path.isdir(value) and - any(fname.endswith('.pb') for fname in os.listdir(value))): + elif (os.path.isdir(input_path) and + any(fname.endswith('.pb') for fname in os.listdir(input_path))): detected_input_format = TF_SAVED_MODEL - elif os.path.isfile(value) and value.endswith('.HDF5'): + elif os.path.isfile(input_path) and input_path.endswith('.HDF5'): detected_input_format = KERAS_MODEL - elif os.path.isdir(value) and value.endswith('model.json'): - if get_tfjs_model_type(value) == 'layers-model': + elif os.path.isdir(input_path) and input_path.endswith('model.json'): + if get_tfjs_model_type(input_path) == 'layers-model': detected_input_format = TFJS_LAYERS_MODEL - elif os.path.isdir(value): - for fname in os.listdir(value): + elif os.path.isdir(input_path): + for fname in os.listdir(input_path): if fname.endswith('model.json'): - filename = os.path.join(value, fname) + filename = os.path.join(input_path, fname) if get_tfjs_model_type(filename) == 'layers-model': - value = os.path.join(value, fname) + input_path = os.path.join(input_path, fname) detected_input_format = TFJS_LAYERS_MODEL break - return detect_input_path, value + return detected_input_format, input_path + def input_path_message(answers): """Determine question for model's input path. @@ -144,20 +145,20 @@ def validate_input_path(input_path, input_format): if input_format == TF_HUB: if re.match(TFHUB_VALID_URL_REGEX, input_path) is None: return """This is not an valid URL for TFHub module: %s, - We expect a URL that starts with http(s)://""" % value + We expect a URL that starts with http(s)://""" % input_path elif not os.path.exists(input_path): - return 'Nonexistent path for the model: %s' % value + return 'Nonexistent path for the model: %s' % input_path if input_format in [KERAS_SAVED_MODEL, TF_SAVED_MODEL]: if not os.path.isdir(input_path): - return 'The path provided is not a directory: %s' % value - if not any(fname.endswith('.pb') for fname in os.listdir(value)): - return 'Did not find a .pb file inside the model\'s directory: %s' % value + return 'The path provided is not a directory: %s' % input_path + if not any(fname.endswith('.pb') for fname in os.listdir(input_path)): + return 'Did not find a .pb file inside the directory: %s' % input_path if input_format == TFJS_LAYERS_MODEL: if not os.path.isfile(input_path): - return 'The path provided is not a file: %s' % value + return 'The path provided is not a file: %s' % input_path if input_format == KERAS_MODEL: if not os.path.isfile(input_path): - return 'The path provided is not a file: %s' % value + return 'The path provided is not a file: %s' % input_path return True @@ -217,6 +218,7 @@ def available_output_formats(answers): Args: ansowers: user selected parameter dict. """ + print(answers) input_format = answers['input_format'] if input_format == KERAS_SAVED_MODEL: return [{ @@ -372,15 +374,15 @@ def main(): 'type': 'list', 'name': 'input_format', 'message': input_format_message(detected_input_format), - 'choices': input_formats(input_params)}, - { + 'choices': input_formats(detected_input_format) + }, { 'type': 'list', 'name': 'output_format', 'message': 'What is your output format?', 'choices': available_output_formats, 'when': lambda answers: value_in_list(answers, 'input_format', - [KERAS_SAVED_MODEL, - TFJS_LAYERS_MODEL]) + [KERAS_SAVED_MODEL, + TFJS_LAYERS_MODEL]) } ] options = prompt(formats, input_params, style=prompt_style) @@ -394,7 +396,7 @@ def main(): 'filter': os.path.expanduser, 'validate': lambda value: validate_input_path( value, options['input_format']), - 'when': lambda answers: detect_input_format + 'when': lambda answers: (not detect_input_format) }, { 'type': 'list', @@ -426,7 +428,7 @@ def main(): 'message': 'Please enter shard size (in bytes) of the weight files?', 'default': str(4 * 1024 * 1024), 'when': lambda answers: value_in_list(answers, 'output_format', - [TFJS_LAYERS_MODEL]) + [TFJS_LAYERS_MODEL]) }, { 'type': 'confirm', @@ -434,7 +436,7 @@ def main(): 'message': 'Do you want to split weights by layers?', 'default': False, 'when': lambda answers: value_in_list(answers, 'input_format', - [TFJS_LAYERS_MODEL]) + [TFJS_LAYERS_MODEL]) }, { 'type': 'confirm', @@ -444,7 +446,7 @@ def main(): 'you can implement them as custom ops in tfjs-converter.', 'default': False, 'when': lambda answers: value_in_list(answers, 'input_format', - [TF_SAVED_MODEL, TF_HUB]) + [TF_SAVED_MODEL, TF_HUB]) }, { 'type': 'confirm', @@ -453,7 +455,7 @@ def main(): 'This will improve model execution performance.', 'default': True, 'when': lambda answers: value_in_list(answers, 'input_format', - [TF_SAVED_MODEL, TF_HUB]) + [TF_SAVED_MODEL, TF_HUB]) }, { 'type': 'input', @@ -469,9 +471,8 @@ def main(): arguments = generate_arguments(params) convert(arguments) print('file generated after conversion:') - with os.scandir(params['output_path']) as it: - for entry in it: - print(entry.name, entry.stat().st_size) + for entry in os.scandir(params['output_path']): + print(entry.stat().st_size, entry.name) if __name__ == '__main__': diff --git a/python/tensorflowjs/cli_test.py b/python/tensorflowjs/cli_test.py index a3fee750..3708433f 100644 --- a/python/tensorflowjs/cli_test.py +++ b/python/tensorflowjs/cli_test.py @@ -65,9 +65,10 @@ def testQuantizationType(self): def testOfValues(self): answers = {'input_path': 'abc', 'input_format': '123'} - self.assertEqual(True, cli.of_values(answers, 'input_path', ['abc'])) - self.assertEqual(False, cli.of_values(answers, 'input_path', ['abd'])) - self.assertEqual(False, cli.of_values(answers, 'input_format2', ['abc'])) + self.assertEqual(True, cli.value_in_list(answers, 'input_path', ['abc'])) + self.assertEqual(False, cli.value_in_list(answers, 'input_path', ['abd'])) + self.assertEqual(False, cli.value_in_list(answers, + 'input_format2', ['abc'])) def testInputPathMessage(self): answers = {'input_format': 'keras'} From 9acd99a133f77eb07e095dcf25053c3b396e8e4e Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Tue, 30 Jul 2019 11:24:27 -0700 Subject: [PATCH 12/24] addressed the comments and add dryrun arg to generate the raw converter command line --- python/tensorflowjs/cli.py | 133 +++++++++++++------- python/tensorflowjs/cli_test.py | 26 +++- python/tensorflowjs/converters/converter.py | 1 - 3 files changed, 111 insertions(+), 49 deletions(-) diff --git a/python/tensorflowjs/cli.py b/python/tensorflowjs/cli.py index 0aa8b8cc..e2d7a35f 100644 --- a/python/tensorflowjs/cli.py +++ b/python/tensorflowjs/cli.py @@ -19,6 +19,7 @@ import os import re import json +import sys from PyInquirer import prompt, style_from_dict, Token from tensorflow.python.saved_model.loader_impl import parse_saved_model @@ -96,25 +97,31 @@ def detect_input_format(input_path): string: detected input format string: normalized input path """ + input_path = input_path.lower() detected_input_format = None if re.match(TFHUB_VALID_URL_REGEX, input_path): detected_input_format = TF_HUB - elif (os.path.isdir(input_path) and - any(fname.endswith('.pb') for fname in os.listdir(input_path))): - detected_input_format = TF_SAVED_MODEL - elif os.path.isfile(input_path) and input_path.endswith('.HDF5'): - detected_input_format = KERAS_MODEL - elif os.path.isdir(input_path) and input_path.endswith('model.json'): - if get_tfjs_model_type(input_path) == 'layers-model': - detected_input_format = TFJS_LAYERS_MODEL elif os.path.isdir(input_path): - for fname in os.listdir(input_path): - if fname.endswith('model.json'): - filename = os.path.join(input_path, fname) - if get_tfjs_model_type(filename) == 'layers-model': - input_path = os.path.join(input_path, fname) - detected_input_format = TFJS_LAYERS_MODEL - break + if any(fname.lower().endswith('.pb') for fname in os.listdir(input_path)): + detected_input_format = TF_SAVED_MODEL + else: + for fname in os.listdir(input_path): + fname = fname.lower() + if fname.endswith('.json'): + filename = os.path.join(input_path, fname) + if get_tfjs_model_type(filename) == 'layers-model': + input_path = os.path.join(input_path, fname) + detected_input_format = TFJS_LAYERS_MODEL + break + elif os.path.isfile(input_path): + if input_path.endswith('.hdf5'): + detected_input_format = KERAS_MODEL + elif input_path.endswith('.pb'): + detected_input_format = TF_SAVED_MODEL + elif (input_path.endswith('.json') and + get_tfjs_model_type(input_path) == 'layers-model'): + detected_input_format = TFJS_LAYERS_MODEL + return detected_input_format, input_path @@ -139,29 +146,52 @@ def validate_input_path(input_path, input_format): input_path: input path of the model. input_format: model format string. """ - input_path = os.path.expanduser(input_path) - if not input_path: + path = os.path.expanduser(input_path) + if not path: return 'Please enter a valid path' if input_format == TF_HUB: - if re.match(TFHUB_VALID_URL_REGEX, input_path) is None: + if re.match(TFHUB_VALID_URL_REGEX, path) is None: return """This is not an valid URL for TFHub module: %s, - We expect a URL that starts with http(s)://""" % input_path - elif not os.path.exists(input_path): - return 'Nonexistent path for the model: %s' % input_path + We expect a URL that starts with http(s)://""" % path + elif not os.path.exists(path): + return 'Nonexistent path for the model: %s' % path if input_format in [KERAS_SAVED_MODEL, TF_SAVED_MODEL]: - if not os.path.isdir(input_path): - return 'The path provided is not a directory: %s' % input_path - if not any(fname.endswith('.pb') for fname in os.listdir(input_path)): - return 'Did not find a .pb file inside the directory: %s' % input_path + is_dir = os.path.isdir(path) + if not is_dir and not path.endswith('.pb'): + return 'The path provided is not a directory or .pb file: %s' % path + if is_dir and not any(f.endswith('.pb') for f in os.listdir(path)): + return 'Did not find a .pb file inside the directory: %s' % path if input_format == TFJS_LAYERS_MODEL: - if not os.path.isfile(input_path): - return 'The path provided is not a file: %s' % input_path + is_dir = os.path.isdir(path) + if not is_dir and not path.endswith('.json'): + return 'The path provided is not a directory or .json file: %s' % path + if is_dir and not any(f.endswith('.json') for f in os.listdir(path)): + return 'Did not find a .pb file inside the directory: %s' % path if input_format == KERAS_MODEL: - if not os.path.isfile(input_path): - return 'The path provided is not a file: %s' % input_path + if not os.path.isfile(path): + return 'The path provided is not a file: %s' % path return True +def expand_input_path(input_format, input_path): + """expand the relative input path to absolute path, and add layers model file + name to the end if input format is `tfjs_layers_model`. + Args: + output_path: input path of the model. + input_format: model format string. + Returns: + string: return expanded input path. + """ + input_path = os.path.expanduser(input_path) + is_dir = os.path.isdir(input_path) + if is_dir: + for fname in os.listdir(input_path): + if fname.endswith('.json'): + filename = os.path.join(input_path, fname) + return filename + return input_path + + def validate_output_path(output_path): """validate the input path for given input format. Args: @@ -201,16 +231,15 @@ def generate_arguments(params): return args -def is_saved_model(answers): +def is_saved_model(input_format): """check if the input path contains saved model. Args: - params: user selected parameters for the conversion. + input_format: input model format. Returns: - bool: + bool: whether this is for a saved model conversion. """ - return answers['input_format'] == TF_SAVED_MODEL or \ - answers['input_format'] == KERAS_SAVED_MODEL and \ - answers['output_format'] == TFJS_GRAPH_MODEL + return input_format == TF_SAVED_MODEL or \ + input_format == KERAS_SAVED_MODEL def available_output_formats(answers): @@ -248,7 +277,8 @@ def available_tags(answers): Args: ansowers: user selected parameter dict. """ - if is_saved_model(answers): + print(answers) + if is_saved_model(answers['input_format']): saved_model = parse_saved_model(answers['input_path']) tags = [] for meta_graph in saved_model.meta_graphs: @@ -263,7 +293,7 @@ def available_signature_names(answers): Args: ansowers: user selected parameter dict. """ - if is_saved_model(answers): + if is_saved_model(answers['input_format']): path = answers['input_path'] tags = answers['saved_model_tags'] saved_model = parse_saved_model(path) @@ -351,7 +381,7 @@ def input_formats(detected_format): return formats -def main(): +def main(dry_run): print('Weclome to TensorFlow.js converter.') input_path = [{ 'type': 'input', @@ -361,7 +391,7 @@ def main(): 'If you are converting TFHub module please provide the URL.', 'filter': os.path.expanduser, 'validate': - lambda value: 'Please enter a valid path' if not value else True + lambda path: 'Please enter a valid path' if not path else True }] input_params = prompt(input_path, style=prompt_style) @@ -393,7 +423,8 @@ def main(): 'type': 'input', 'name': 'input_path', 'message': message, - 'filter': os.path.expanduser, + 'filter': lambda value: expand_input_path( + value, options['input_format']), 'validate': lambda value: validate_input_path( value, options['input_format']), 'when': lambda answers: (not detect_input_format) @@ -403,14 +434,14 @@ def main(): 'name': 'saved_model_tags', 'choices': available_tags, 'message': 'What is tags for the saved model?', - 'when': is_saved_model + 'when': lambda answers: is_saved_model(answers['input_format']) }, { 'type': 'list', 'name': 'signature_name', 'message': 'What is signature name of the model?', 'choices': available_signature_names, - 'when': is_saved_model + 'when': lambda answers: is_saved_model(answers['input_format']) }, { 'type': 'list', @@ -469,11 +500,19 @@ def main(): params = prompt(questions, options, style=prompt_style) arguments = generate_arguments(params) - convert(arguments) - print('file generated after conversion:') - for entry in os.scandir(params['output_path']): - print(entry.stat().st_size, entry.name) + if dry_run: + print('converter command generated:') + print('tensorflowjs_converter %s' % ' '.join(arguments)) + else: + convert(arguments) + print('file generated after conversion:') + for entry in os.scandir(params['output_path']): + print(entry.stat().st_size, entry.name) if __name__ == '__main__': - main() + if len (sys.argv) > 2 or len(sys.argv) == 1 and not sys.argv[1] == 'dryrun': + print("Usage: tensorflowjs_cli [--dryrun]") + sys.exit (1) + dry_run = sys.argv[1] == '--dryrun' + main(dry_run) diff --git a/python/tensorflowjs/cli_test.py b/python/tensorflowjs/cli_test.py index 3708433f..55f504d7 100644 --- a/python/tensorflowjs/cli_test.py +++ b/python/tensorflowjs/cli_test.py @@ -18,6 +18,7 @@ import unittest import tempfile +import json import os import shutil import tensorflow as tf @@ -29,7 +30,9 @@ from tensorflowjs import cli SAVED_MODEL_DIR = 'saved_model' +SAVED_MODEL_NAME = 'saved_model.pb' HD5_FILE_NAME = 'test.HD5' +LAYERS_MODEL_NAME = 'model.json' class CliTest(unittest.TestCase): @@ -42,6 +45,12 @@ def tearDown(self): shutil.rmtree(self._tmp_dir) super(CliTest, self).tearDown() + def _create_layers_model(self): + data = {'format': 'layers-model'} + filename = os.path.join(self._tmp_dir, 'model.json') + with open(filename, 'a') as file: + json.dump(data, file) + def _create_hd5_file(self): filename = os.path.join(self._tmp_dir, 'test.HD5') open(filename, 'a').close() @@ -100,6 +109,10 @@ def testValidateInputPathForSavedModel(self): self.assertEqual(True, cli.validate_input_path( save_dir, 'tf_saved_model')) + save_dir = os.path.join(self._tmp_dir, SAVED_MODEL_DIR, SAVED_MODEL_NAME) + self.assertEqual(True, cli.validate_input_path( + save_dir, 'tf_saved_model')) + def testValidateInputPathForKerasSavedModel(self): self.assertNotEqual(True, cli.validate_input_path( self._tmp_dir, 'keras_saved_model')) @@ -115,6 +128,17 @@ def testValidateInputPathForKerasModel(self): self.assertEqual(True, cli.validate_input_path( save_dir, 'keras')) + def testValidateInputPathForLayersModel(self): + self.assertNotEqual(True, cli.validate_input_path(self._tmp_dir, 'keras')) + self._create_layers_model() + save_dir = os.path.join(self._tmp_dir) + self.assertEqual(True, cli.validate_input_path( + save_dir, 'tfjs_layers_model')) + + save_dir = os.path.join(self._tmp_dir, 'model.json') + self.assertEqual(True, cli.validate_input_path( + save_dir, 'tfjs_layers_model')) + def testValidateOutputPath(self): self.assertNotEqual(True, cli.validate_output_path(self._tmp_dir)) output_dir = os.path.join(self._tmp_dir, 'test') @@ -133,7 +157,7 @@ def testAvailableSignatureNames(self): self.assertEqual(['__saved_model_init_op', 'serving_default'], [x['value'] for x in cli.available_signature_names( {'input_path': save_dir, - 'input_format': 'tf_saved_model', + 'input_format': 'tf_saved_model' 'saved_model_tags': 'serve'})]) def testGenerateCommandForSavedModel(self): diff --git a/python/tensorflowjs/converters/converter.py b/python/tensorflowjs/converters/converter.py index 1d0754aa..ef1e1a83 100644 --- a/python/tensorflowjs/converters/converter.py +++ b/python/tensorflowjs/converters/converter.py @@ -593,7 +593,6 @@ def convert(arguments=None): strip_debug_ops=FLAGS.strip_debug_ops) elif (input_format == 'tf_hub' and output_format == 'tfjs_graph_model'): - print(FLAGS) tf_saved_model_conversion_v2.convert_tf_hub_module( FLAGS.input_path, FLAGS.output_path, FLAGS.signature_name, FLAGS.saved_model_tags, skip_op_check=FLAGS.skip_op_check, From d24ad9b1747c8f76e412bf17686553f653c9d60e Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Tue, 30 Jul 2019 13:44:16 -0700 Subject: [PATCH 13/24] fixed bugs --- python/tensorflowjs/cli.py | 14 ++++++-------- python/tensorflowjs/cli_test.py | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/python/tensorflowjs/cli.py b/python/tensorflowjs/cli.py index e2d7a35f..90a01d2c 100644 --- a/python/tensorflowjs/cli.py +++ b/python/tensorflowjs/cli.py @@ -166,18 +166,18 @@ def validate_input_path(input_path, input_format): if not is_dir and not path.endswith('.json'): return 'The path provided is not a directory or .json file: %s' % path if is_dir and not any(f.endswith('.json') for f in os.listdir(path)): - return 'Did not find a .pb file inside the directory: %s' % path + return 'Did not find a .json file inside the directory: %s' % path if input_format == KERAS_MODEL: if not os.path.isfile(path): return 'The path provided is not a file: %s' % path return True -def expand_input_path(input_format, input_path): +def expand_input_path(input_path, input_format): """expand the relative input path to absolute path, and add layers model file name to the end if input format is `tfjs_layers_model`. Args: - output_path: input path of the model. + input_path: input path of the model. input_format: model format string. Returns: string: return expanded input path. @@ -247,7 +247,6 @@ def available_output_formats(answers): Args: ansowers: user selected parameter dict. """ - print(answers) input_format = answers['input_format'] if input_format == KERAS_SAVED_MODEL: return [{ @@ -277,7 +276,6 @@ def available_tags(answers): Args: ansowers: user selected parameter dict. """ - print(answers) if is_saved_model(answers['input_format']): saved_model = parse_saved_model(answers['input_path']) tags = [] @@ -427,7 +425,7 @@ def main(dry_run): value, options['input_format']), 'validate': lambda value: validate_input_path( value, options['input_format']), - 'when': lambda answers: (not detect_input_format) + 'when': lambda answers: (not detected_input_format) }, { 'type': 'list', @@ -511,8 +509,8 @@ def main(dry_run): if __name__ == '__main__': - if len (sys.argv) > 2 or len(sys.argv) == 1 and not sys.argv[1] == 'dryrun': + if len(sys.argv) > 2 or len(sys.argv) == 2 and not sys.argv[1] == '--dryrun': print("Usage: tensorflowjs_cli [--dryrun]") sys.exit (1) - dry_run = sys.argv[1] == '--dryrun' + dry_run = len(sys.argv) == 2 and sys.argv[1] == '--dryrun' main(dry_run) diff --git a/python/tensorflowjs/cli_test.py b/python/tensorflowjs/cli_test.py index 55f504d7..f11fe5f3 100644 --- a/python/tensorflowjs/cli_test.py +++ b/python/tensorflowjs/cli_test.py @@ -157,7 +157,7 @@ def testAvailableSignatureNames(self): self.assertEqual(['__saved_model_init_op', 'serving_default'], [x['value'] for x in cli.available_signature_names( {'input_path': save_dir, - 'input_format': 'tf_saved_model' + 'input_format': 'tf_saved_model', 'saved_model_tags': 'serve'})]) def testGenerateCommandForSavedModel(self): From a4ea88fd56c7e9c7157769549c37915737a5aed8 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Tue, 30 Jul 2019 15:57:37 -0700 Subject: [PATCH 14/24] address comments --- README.md | 3 +- python/tensorflowjs/BUILD | 7 + python/tensorflowjs/__init__.py | 2 +- python/tensorflowjs/cli.py | 270 ++++++++++---------- python/tensorflowjs/cli_test.py | 19 +- python/tensorflowjs/converters/common.py | 21 ++ python/tensorflowjs/converters/converter.py | 99 ++++--- 7 files changed, 236 insertions(+), 185 deletions(-) diff --git a/README.md b/README.md index 2fd9ee74..1ee1a767 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,8 @@ details explanations for each choice you need to make. Behind the scene it calls the converter script (tensorflowjs_converter) in pip package. This is the easier way to convert a single model. -To convert a batch of models or integrate the conversion process into your own script, you should look into using the tensorflowjs_converter script. +To convert a batch of models or integrate the conversion process into your own +script, you should look into using the tensorflowjs_converter script. Here is detail information of parameters of the converter script. diff --git a/python/tensorflowjs/BUILD b/python/tensorflowjs/BUILD index ba4fb03d..2ada8596 100644 --- a/python/tensorflowjs/BUILD +++ b/python/tensorflowjs/BUILD @@ -48,6 +48,13 @@ py_library( # `pip install tensorflow` or `pip install tensorflow-gpu`. ) +py_library( + name = "expect_PyInquirer_installed", + # This is a dummy rule used as a PyInquirer dependency in open-source. + # We expect tensorflow to already be installed on the system, e.g. via + # `pip install PyInquirer`. +) + py_library( name = "quantization", srcs = ["quantization.py"], diff --git a/python/tensorflowjs/__init__.py b/python/tensorflowjs/__init__.py index 43c8b3f2..97f34baf 100644 --- a/python/tensorflowjs/__init__.py +++ b/python/tensorflowjs/__init__.py @@ -18,9 +18,9 @@ from __future__ import print_function # pylint: disable=unused-imports +from tensorflowjs import cli from tensorflowjs import converters from tensorflowjs import quantization from tensorflowjs import version -from tensorflowjs import cli __version__ = version.version diff --git a/python/tensorflowjs/cli.py b/python/tensorflowjs/cli.py index 90a01d2c..2070e547 100644 --- a/python/tensorflowjs/cli.py +++ b/python/tensorflowjs/cli.py @@ -16,40 +16,34 @@ from __future__ import print_function, unicode_literals +import json import os import re -import json import sys -from PyInquirer import prompt, style_from_dict, Token -from tensorflow.python.saved_model.loader_impl import parse_saved_model +import PyInquirer +import h5py from tensorflow.core.framework import types_pb2 -from tensorflowjs.converters.converter import convert +from tensorflow.python.saved_model import loader_impl +from tensorflowjs.converters import converter +from tensorflowjs.converters import common + # regex for recognizing valid url for TFHub module. TFHUB_VALID_URL_REGEX = re.compile( # http:// or https:// r'^(?:http)s?://', re.IGNORECASE) # prompt style -prompt_style = style_from_dict({ - Token.Separator: '#6C6C6C', - Token.QuestionMark: '#FF9D00 bold', - # Token.Selected: '', # default - Token.Selected: '#5F819D', - Token.Pointer: '#FF9D00 bold', - Token.Instruction: '', # default - Token.Answer: '#5F819D bold', - Token.Question: '', +prompt_style = PyInquirer.style_from_dict({ + PyInquirer.Token.Separator: '#6C6C6C', + PyInquirer.Token.QuestionMark: '#FF9D00 bold', + PyInquirer.Token.Selected: '#5F819D', + PyInquirer.Token.Pointer: '#FF9D00 bold', + PyInquirer.Token.Instruction: '', # default + PyInquirer.Token.Answer: '#5F819D bold', + PyInquirer.Token.Question: '', }) -KERAS_SAVED_MODEL = 'keras_saved_model' -KERAS_MODEL = 'keras' -TF_SAVED_MODEL = 'tf_saved_model' -TF_HUB = 'tf_hub' -TFJS_GRAPH_MODEL = 'tfjs_keras_model' -TFJS_LAYERS_MODEL = 'tfjs_layers_model' - - def quantization_type(user_selection_quant): """Determine the quantization type based on user selection. Args: @@ -100,27 +94,28 @@ def detect_input_format(input_path): input_path = input_path.lower() detected_input_format = None if re.match(TFHUB_VALID_URL_REGEX, input_path): - detected_input_format = TF_HUB + detected_input_format = common.TF_HUB_MODEL elif os.path.isdir(input_path): - if any(fname.lower().endswith('.pb') for fname in os.listdir(input_path)): - detected_input_format = TF_SAVED_MODEL + if (any(fname.lower().endswith('saved_model.pb') + for fname in os.listdir(input_path))): + detected_input_format = common.TF_SAVED_MODEL else: for fname in os.listdir(input_path): fname = fname.lower() - if fname.endswith('.json'): + if fname.endswith('model.json'): filename = os.path.join(input_path, fname) - if get_tfjs_model_type(filename) == 'layers-model': + if get_tfjs_model_type(filename) == common.TFJS_LAYERS_MODEL_FORMAT: input_path = os.path.join(input_path, fname) - detected_input_format = TFJS_LAYERS_MODEL + detected_input_format = common.TFJS_LAYERS_MODEL break elif os.path.isfile(input_path): - if input_path.endswith('.hdf5'): - detected_input_format = KERAS_MODEL - elif input_path.endswith('.pb'): - detected_input_format = TF_SAVED_MODEL - elif (input_path.endswith('.json') and - get_tfjs_model_type(input_path) == 'layers-model'): - detected_input_format = TFJS_LAYERS_MODEL + if h5py.is_hdf5(input_path): + detected_input_format = common.KERAS_MODEL + elif input_path.endswith('saved_model.pb'): + detected_input_format = common.TF_SAVED_MODEL + elif (input_path.endswith('model.json') and + get_tfjs_model_type(input_path) == common.TFJS_LAYERS_MODEL_FORMAT): + detected_input_format = common.TFJS_LAYERS_MODEL return detected_input_format, input_path @@ -130,18 +125,18 @@ def input_path_message(answers): Args: answer: Dict of user's answers to the questions """ - answer = answers['input_format'] + answer = answers[common.INPUT_FORMAT] message = 'The original path seems to be wrong, ' - if answer == KERAS_MODEL: + if answer == common.KERAS_MODEL: return message + 'what is the path of input HDF5 file?' - elif answer == TF_HUB: + elif answer == common.TF_HUB_MODEL: return message + 'what is the TFHub module URL?' else: return message + 'what is the directory that contains the model?' def validate_input_path(input_path, input_format): - """validate the input path for given input format. + """Validate the input path for given input format. Args: input_path: input path of the model. input_format: model format string. @@ -149,36 +144,36 @@ def validate_input_path(input_path, input_format): path = os.path.expanduser(input_path) if not path: return 'Please enter a valid path' - if input_format == TF_HUB: - if re.match(TFHUB_VALID_URL_REGEX, path) is None: + if input_format == common.TF_HUB_MODEL: + if not re.match(TFHUB_VALID_URL_REGEX, path): return """This is not an valid URL for TFHub module: %s, We expect a URL that starts with http(s)://""" % path elif not os.path.exists(path): return 'Nonexistent path for the model: %s' % path - if input_format in [KERAS_SAVED_MODEL, TF_SAVED_MODEL]: + if input_format in [common.KERAS_SAVED_MODEL, common.TF_SAVED_MODEL]: is_dir = os.path.isdir(path) - if not is_dir and not path.endswith('.pb'): - return 'The path provided is not a directory or .pb file: %s' % path - if is_dir and not any(f.endswith('.pb') for f in os.listdir(path)): + if not is_dir and not path.endswith('saved_model.pb'): + return 'The path provided is not a directory or pb file: %s' % path + if (is_dir and + not any(f.endswith('saved_model.pb') for f in os.listdir(path))): return 'Did not find a .pb file inside the directory: %s' % path - if input_format == TFJS_LAYERS_MODEL: + if input_format == common.TFJS_LAYERS_MODEL: is_dir = os.path.isdir(path) - if not is_dir and not path.endswith('.json'): - return 'The path provided is not a directory or .json file: %s' % path - if is_dir and not any(f.endswith('.json') for f in os.listdir(path)): - return 'Did not find a .json file inside the directory: %s' % path - if input_format == KERAS_MODEL: - if not os.path.isfile(path): - return 'The path provided is not a file: %s' % path + if not is_dir and not path.endswith('model.json'): + return 'The path provided is not a directory or json file: %s' % path + if is_dir and not any(f.endswith('model.json') for f in os.listdir(path)): + return 'Did not find the model.json file inside the directory: %s' % path + if input_format == common.KERAS_MODEL: + if not h5py.is_hdf5(path): + return 'The path provided is not a keras model file: %s' % path return True -def expand_input_path(input_path, input_format): - """expand the relative input path to absolute path, and add layers model file +def expand_input_path(input_path): + """Expand the relative input path to absolute path, and add layers model file name to the end if input format is `tfjs_layers_model`. Args: input_path: input path of the model. - input_format: model format string. Returns: string: return expanded input path. """ @@ -193,7 +188,7 @@ def expand_input_path(input_path, input_format): def validate_output_path(output_path): - """validate the input path for given input format. + """Validate the input path for given input format. Args: output_path: input path of the model. input_format: model format string. @@ -209,15 +204,15 @@ def validate_output_path(output_path): def generate_arguments(params): - """generate the tensorflowjs command string for the selected params. + """Generate the tensorflowjs command string for the selected params. Args: params: user selected parameters for the conversion. Returns: list: the argument list for converter. """ args = [] - not_param_list = ['input_path', 'output_path'] - no_false_param = ['split_weights_by_layer', 'skip_op_check'] + not_param_list = [common.INPUT_PATH, common.OUTPUT_PATH] + no_false_param = [common.SPLIT_WEIGHTS_BY_LAYER, common.SKIP_OP_CHECK] for key, value in sorted(params.items()): if key not in not_param_list and value is not None: if key in no_false_param: @@ -226,58 +221,57 @@ def generate_arguments(params): else: args.append('--%s=%s' % (key, value)) - args.append(params['input_path']) - args.append(params['output_path']) + args.append(params[common.INPUT_PATH]) + args.append(params[common.OUTPUT_PATH]) return args def is_saved_model(input_format): - """check if the input path contains saved model. + """Check if the input path contains saved model. Args: input_format: input model format. Returns: bool: whether this is for a saved model conversion. """ - return input_format == TF_SAVED_MODEL or \ - input_format == KERAS_SAVED_MODEL - + return input_format == common.TF_SAVED_MODEL or \ + input_format == common.KERAS_SAVED_MODEL def available_output_formats(answers): - """generate the output formats for given input format. + """Generate the output formats for given input format. Args: ansowers: user selected parameter dict. """ - input_format = answers['input_format'] - if input_format == KERAS_SAVED_MODEL: + input_format = answers[common.INPUT_FORMAT] + if input_format == common.KERAS_SAVED_MODEL: return [{ - 'key': 'g', + 'key': 'g', # shortcut key for the option 'name': 'Tensorflow.js Graph Model', - 'value': TFJS_GRAPH_MODEL, + 'value': common.TFJS_GRAPH_MODEL, }, { 'key': 'l', 'name': 'TensoFlow.js Layers Model', - 'value': TFJS_LAYERS_MODEL, + 'value': common.TFJS_LAYERS_MODEL, }] - if input_format == TFJS_LAYERS_MODEL: + if input_format == common.TFJS_LAYERS_MODEL: return [{ 'key': 'k', 'name': 'Keras Model (HDF5)', - 'value': KERAS_MODEL, + 'value': common.KERAS_MODEL, }, { 'key': 'l', 'name': 'TensoFlow.js Layers Model', - 'value': TFJS_LAYERS_MODEL, + 'value': common.TFJS_LAYERS_MODEL, }] return [] def available_tags(answers): - """generate the available saved model tags from the proto file. + """Generate the available saved model tags from the proto file. Args: ansowers: user selected parameter dict. """ - if is_saved_model(answers['input_format']): - saved_model = parse_saved_model(answers['input_path']) + if is_saved_model(answers[common.INPUT_FORMAT]): + saved_model = loader_impl.parse_saved_model(answers[common.INPUT_PATH]) tags = [] for meta_graph in saved_model.meta_graphs: tags.append(",".join(meta_graph.meta_info_def.tags)) @@ -286,15 +280,15 @@ def available_tags(answers): def available_signature_names(answers): - """generate the available saved model signatures from the proto file + """Generate the available saved model signatures from the proto file and selected tags. Args: ansowers: user selected parameter dict. """ - if is_saved_model(answers['input_format']): - path = answers['input_path'] - tags = answers['saved_model_tags'] - saved_model = parse_saved_model(path) + if is_saved_model(answers[common.INPUT_FORMAT]): + path = answers[common.INPUT_PATH] + tags = answers[common.SAVED_MODEL_TAGS] + saved_model = loader_impl.parse_saved_model(path) for meta_graph in saved_model.meta_graphs: if tags == ",".join(meta_graph.meta_info_def.tags): signatures = [] @@ -347,43 +341,43 @@ def input_format_message(detected_input_format): def input_formats(detected_format): formats = [{ 'key': 'k', - 'name': input_format_string('Keras (HDF5)', KERAS_MODEL, + 'name': input_format_string('Keras (HDF5)', common.KERAS_MODEL, detected_format), - 'value': KERAS_MODEL + 'value': common.KERAS_MODEL }, { 'key': 'e', 'name': input_format_string('Tensorflow Keras Saved Model', - KERAS_SAVED_MODEL, + common.KERAS_SAVED_MODEL, detected_format), - 'value': KERAS_SAVED_MODEL, + 'value': common.KERAS_SAVED_MODEL, }, { 'key': 's', 'name': input_format_string('Tensorflow Saved Model', - TF_SAVED_MODEL, + common.TF_SAVED_MODEL, detected_format), - 'value': TF_SAVED_MODEL, + 'value': common.TF_SAVED_MODEL, }, { 'key': 'h', 'name': input_format_string('TFHub Module', - TF_HUB, + common.TF_HUB_MODEL, detected_format), - 'value': TF_HUB, + 'value': common.TF_HUB_MODEL, }, { 'key': 'l', 'name': input_format_string('TensoFlow.js Layers Model', - TFJS_LAYERS_MODEL, + common.TFJS_LAYERS_MODEL, detected_format), - 'value': TFJS_LAYERS_MODEL, + 'value': common.TFJS_LAYERS_MODEL, }] formats.sort(key=lambda x: x['value'] != detected_format) return formats -def main(dry_run): - print('Weclome to TensorFlow.js converter.') +def main(dryrun): + print('Welcome to TensorFlow.js Converter.') input_path = [{ 'type': 'input', - 'name': 'input_path', + 'name': common.INPUT_PATH, 'message': 'Please provide the path of model file or ' 'the directory that contains model files. \n' 'If you are converting TFHub module please provide the URL.', @@ -392,58 +386,61 @@ def main(dry_run): lambda path: 'Please enter a valid path' if not path else True }] - input_params = prompt(input_path, style=prompt_style) + input_params = PyInquirer.prompt(input_path, style=prompt_style) detected_input_format, normalized_path = detect_input_format( - input_params['input_path']) - input_params['input_path'] = normalized_path + input_params[common.INPUT_PATH]) + input_params[common.INPUT_PATH] = normalized_path formats = [ { 'type': 'list', - 'name': 'input_format', + 'name': common.INPUT_FORMAT, 'message': input_format_message(detected_input_format), 'choices': input_formats(detected_input_format) }, { 'type': 'list', - 'name': 'output_format', + 'name': common.OUTPUT_FORMAT, 'message': 'What is your output format?', 'choices': available_output_formats, - 'when': lambda answers: value_in_list(answers, 'input_format', - [KERAS_SAVED_MODEL, - TFJS_LAYERS_MODEL]) + 'when': lambda answers: value_in_list(answers, common.INPUT_FORMAT, + [common.KERAS_SAVED_MODEL, + common.TFJS_LAYERS_MODEL]) } ] - options = prompt(formats, input_params, style=prompt_style) - message = input_path_message(options) - + formats = PyInquirer.prompt(formats, input_params, style=prompt_style) + message = input_path_message(formats) + print(formats) questions = [ { 'type': 'input', - 'name': 'input_path', + 'name': common.INPUT_PATH, 'message': message, - 'filter': lambda value: expand_input_path( - value, options['input_format']), + 'filter': expand_input_path, 'validate': lambda value: validate_input_path( - value, options['input_format']), + value, formats[common.INPUT_FORMAT]), 'when': lambda answers: (not detected_input_format) }, { 'type': 'list', - 'name': 'saved_model_tags', + 'name': common.SAVED_MODEL_TAGS, 'choices': available_tags, 'message': 'What is tags for the saved model?', - 'when': lambda answers: is_saved_model(answers['input_format']) + 'when': lambda answers: (is_saved_model(answers[common.INPUT_FORMAT]) + and formats[common.OUTPUT_FORMAT] == + common.TFJS_GRAPH_MODEL) }, { 'type': 'list', - 'name': 'signature_name', + 'name': common.SIGNATURE_NAME, 'message': 'What is signature name of the model?', 'choices': available_signature_names, - 'when': lambda answers: is_saved_model(answers['input_format']) + 'when': lambda answers: (is_saved_model(answers[common.INPUT_FORMAT]) + and formats[common.OUTPUT_FORMAT] == + common.TFJS_GRAPH_MODEL) }, { 'type': 'list', - 'name': 'quantization_bytes', + 'name': common.QUANTIZATION_BYTES, 'message': 'Do you want to compress the model? ' '(this will decrease the model precision.)', 'choices': ['No compression, no accuracy loss.', @@ -453,64 +450,65 @@ def main(dry_run): }, { 'type': 'input', - 'name': 'weight_shard_size_byte', + 'name': common.WEIGHT_SHARD_SIZE_BYTES, 'message': 'Please enter shard size (in bytes) of the weight files?', 'default': str(4 * 1024 * 1024), - 'when': lambda answers: value_in_list(answers, 'output_format', - [TFJS_LAYERS_MODEL]) + 'when': lambda answers: value_in_list(answers, common.OUTPUT_FORMAT, + [common.TFJS_LAYERS_MODEL]) }, { 'type': 'confirm', - 'name': 'split_weights_by_layer', + 'name': common.SPLIT_WEIGHTS_BY_LAYER, 'message': 'Do you want to split weights by layers?', 'default': False, - 'when': lambda answers: value_in_list(answers, 'input_format', - [TFJS_LAYERS_MODEL]) + 'when': lambda answers: value_in_list(answers, common.INPUT_FORMAT, + [common.TFJS_LAYERS_MODEL]) }, { 'type': 'confirm', - 'name': 'skip_op_check', + 'name': common.SKIP_OP_CHECK, 'message': 'Do you want to skip op validation? \n' 'This will allow conversion of unsupported ops, \n' 'you can implement them as custom ops in tfjs-converter.', 'default': False, - 'when': lambda answers: value_in_list(answers, 'input_format', - [TF_SAVED_MODEL, TF_HUB]) + 'when': lambda answers: value_in_list(answers, common.INPUT_FORMAT, + [common.TF_SAVED_MODEL, + common.TF_HUB_MODEL]) }, { 'type': 'confirm', - 'name': 'strip_debug_ops', + 'name': common.STRIP_DEBUG_OPS, 'message': 'Do you want to strip debug ops? \n' 'This will improve model execution performance.', 'default': True, - 'when': lambda answers: value_in_list(answers, 'input_format', - [TF_SAVED_MODEL, TF_HUB]) + 'when': lambda answers: value_in_list(answers, common.INPUT_FORMAT, + [common.TF_SAVED_MODEL, + common.TF_HUB_MODEL]) }, { 'type': 'input', - 'name': 'output_path', + 'name': common.OUTPUT_PATH, 'message': 'Which directory do you want to save ' 'the converted model in?', 'filter': os.path.expanduser, 'validate': validate_output_path } ] - params = prompt(questions, options, style=prompt_style) + params = PyInquirer.prompt(questions, formats, style=prompt_style) arguments = generate_arguments(params) - if dry_run: - print('converter command generated:') - print('tensorflowjs_converter %s' % ' '.join(arguments)) - else: - convert(arguments) + print('converter command generated:') + print('tensorflowjs_converter %s' % ' '.join(arguments)) + if not dryrun: + converter.convert(arguments) print('file generated after conversion:') - for entry in os.scandir(params['output_path']): + for entry in os.scandir(params[common.OUTPUT_PATH]): print(entry.stat().st_size, entry.name) if __name__ == '__main__': if len(sys.argv) > 2 or len(sys.argv) == 2 and not sys.argv[1] == '--dryrun': print("Usage: tensorflowjs_cli [--dryrun]") - sys.exit (1) + sys.exit(1) dry_run = len(sys.argv) == 2 and sys.argv[1] == '--dryrun' main(dry_run) diff --git a/python/tensorflowjs/cli_test.py b/python/tensorflowjs/cli_test.py index f11fe5f3..fc9fe3e0 100644 --- a/python/tensorflowjs/cli_test.py +++ b/python/tensorflowjs/cli_test.py @@ -22,16 +22,17 @@ import os import shutil import tensorflow as tf +from tensorflow import keras from tensorflow.python.eager import def_function from tensorflow.python.ops import variables from tensorflow.python.training.tracking import tracking -from tensorflow.python.saved_model.save import save +from tensorflow.python.saved_model import save from tensorflowjs import cli SAVED_MODEL_DIR = 'saved_model' SAVED_MODEL_NAME = 'saved_model.pb' -HD5_FILE_NAME = 'test.HD5' +HD5_FILE_NAME = 'test.h5' LAYERS_MODEL_NAME = 'model.json' @@ -52,8 +53,16 @@ def _create_layers_model(self): json.dump(data, file) def _create_hd5_file(self): - filename = os.path.join(self._tmp_dir, 'test.HD5') - open(filename, 'a').close() + input_tensor = keras.layers.Input((3,)) + dense1 = keras.layers.Dense( + 4, use_bias=True, kernel_initializer='ones', bias_initializer='zeros', + name='MyDense10')(input_tensor) + output = keras.layers.Dense( + 2, use_bias=False, kernel_initializer='ones', name='MyDense20')(dense1) + model = keras.models.Model(inputs=[input_tensor], outputs=[output]) + h5_path = os.path.join(self._tmp_dir, HD5_FILE_NAME) + print(h5_path) + model.save_weights(h5_path) def _create_saved_model(self): """Test a basic model with functions to make sure functions are inlined.""" @@ -65,7 +74,7 @@ def _create_saved_model(self): to_save = root.f.get_concrete_function(input_data) save_dir = os.path.join(self._tmp_dir, SAVED_MODEL_DIR) - save(root, save_dir, to_save) + save.save(root, save_dir, to_save) def testQuantizationType(self): self.assertEqual(2, cli.quantization_type('1/2')) diff --git a/python/tensorflowjs/converters/common.py b/python/tensorflowjs/converters/common.py index 25da4d64..efbdb000 100644 --- a/python/tensorflowjs/converters/common.py +++ b/python/tensorflowjs/converters/common.py @@ -30,6 +30,27 @@ GENERATED_BY_KEY = 'generatedBy' CONVERTED_BY_KEY = 'convertedBy' +# model formats +KERAS_SAVED_MODEL = 'keras_saved_model' +KERAS_MODEL = 'keras' +TF_SAVED_MODEL = 'tf_saved_model' +TF_HUB_MODEL = 'tf_hub' +TFJS_GRAPH_MODEL = 'tfjs_graph_model' +TFJS_LAYERS_MODEL = 'tfjs_layers_model' + +# cli argument string +INPUT_PATH = 'input_path' +OUTPUT_PATH = 'output_path' +INPUT_FORMAT = 'input_format' +OUTPUT_FORMAT = 'output_format' +SIGNATURE_NAME = 'signature_name' +SAVED_MODEL_TAGS = 'saved_model_tags' +QUANTIZATION_BYTES = 'quantization_bytes' +SPLIT_WEIGHTS_BY_LAYER = 'split_weights_by_layer' +VERSION = 'version' +SKIP_OP_CHECK = 'skip_op_check' +STRIP_DEBUG_OPS = 'strip_debug_ops' +WEIGHT_SHARD_SIZE_BYTES = 'weight_shard_size_bytes' def get_converted_by(): """Get the convertedBy string for storage in model artifacts.""" diff --git a/python/tensorflowjs/converters/converter.py b/python/tensorflowjs/converters/converter.py index ef1e1a83..ebdbed84 100644 --- a/python/tensorflowjs/converters/converter.py +++ b/python/tensorflowjs/converters/converter.py @@ -31,6 +31,7 @@ from tensorflowjs import quantization from tensorflowjs import version +from tensorflowjs.converters import common from tensorflowjs.converters import keras_h5_conversion as conversion from tensorflowjs.converters import keras_tfjs_loader from tensorflowjs.converters import tf_saved_model_conversion_v2 @@ -394,17 +395,17 @@ def _standardize_input_output_formats(input_format, output_format): 'Use --input_format=tfjs_layers_model instead.') input_format_is_keras = ( - input_format in ['keras', 'keras_saved_model']) + input_format in [common.KERAS_MODEL, common.KERAS_SAVED_MODEL]) input_format_is_tf = ( - input_format in ['tf_saved_model', 'tf_hub']) + input_format in [common.TF_SAVED_MODEL, common.TF_HUB_MODEL]) if output_format is None: # If no explicit output_format is provided, infer it from input format. if input_format_is_keras: - output_format = 'tfjs_layers_model' + output_format = common.TFJS_LAYERS_MODEL elif input_format_is_tf: - output_format = 'tfjs_graph_model' - elif input_format == 'tfjs_layers_model': - output_format = 'keras' + output_format = common.TFJS_GRAPH_MODEL + elif input_format == common.TFJS_LAYERS_MODEL: + output_format = common.KERAS_MODEL elif output_format == 'tensorflowjs': # https://github.com/tensorflow/tfjs/issues/1292: Remove the logic for the # explicit error message of the deprecated model type name 'tensorflowjs' @@ -435,9 +436,16 @@ def _parse_quantization_bytes(quantization_bytes): def setup_arguments(arguments=None): + """ + Convert a keras HDF5-format model to tfjs GraphModel artifacts. + + Args: + arguments: list, the argument string list to be parsed from. If arguments is + not given, it will try to parse from the system arguments. + """ parser = argparse.ArgumentParser('TensorFlow.js model converters.') parser.add_argument( - 'input_path', + common.INPUT_PATH, nargs='?', type=str, help='Path to the input file or directory. For input format "keras", ' @@ -445,14 +453,18 @@ def setup_arguments(arguments=None): 'a SavedModel directory, session bundle directory, frozen model file, ' 'or TF-Hub module is expected.') parser.add_argument( - 'output_path', nargs='?', type=str, help='Path for all output artifacts.') + common.OUTPUT_PATH, + nargs='?', + type=str, + help='Path for all output artifacts.') parser.add_argument( - '--input_format', + '--%s' % common.INPUT_FORMAT, type=str, required=False, - default='tf_saved_model', - choices=set(['keras', 'keras_saved_model', - 'tf_saved_model', 'tf_hub', 'tfjs_layers_model', + default=common.TF_SAVED_MODEL, + choices=set([common.KERAS_MODEL, common.KERAS_SAVED_MODEL, + common.TF_SAVED_MODEL, common.TF_HUB_MODEL, + common.TFJS_LAYERS_MODEL, 'tensorflowjs']), help='Input format. ' 'For "keras", the input path can be one of the two following formats:\n' @@ -469,55 +481,56 @@ def setup_arguments(arguments=None): 'For "tf" formats, a SavedModel, frozen model, session bundle model, ' ' or TF-Hub module is expected.') parser.add_argument( - '--output_format', + '--%s' % common.OUTPUT_FORMAT, type=str, required=False, - choices=set(['keras', 'keras_saved_model', 'tfjs_layers_model', - 'tfjs_graph_model', 'tensorflowjs']), + choices=set([common.KERAS_MODEL, common.KERAS_SAVED_MODEL, + common.TFJS_LAYERS_MODEL, common.TFJS_GRAPH_MODEL, + 'tensorflowjs']), help='Output format. Default: tfjs_graph_model.') parser.add_argument( - '--signature_name', + '--%s' % common.SIGNATURE_NAME, type=str, default=None, help='Signature of the SavedModel Graph or TF-Hub module to load. ' 'Applicable only if input format is "tf_hub" or "tf_saved_model".') parser.add_argument( - '--saved_model_tags', + '--%s' % common.SAVED_MODEL_TAGS, type=str, default='serve', help='Tags of the MetaGraphDef to load, in comma separated string ' 'format. Defaults to "serve". Applicable only if input format is ' '"tf_saved_model".') parser.add_argument( - '--quantization_bytes', + '--%s' % common.QUANTIZATION_BYTES, type=int, choices=set(quantization.QUANTIZATION_BYTES_TO_DTYPES.keys()), help='How many bytes to optionally quantize/compress the weights to. 1- ' 'and 2-byte quantizaton is supported. The default (unquantized) size is ' '4 bytes.') parser.add_argument( - '--split_weights_by_layer', + '--%s' % common.SPLIT_WEIGHTS_BY_LAYERS, action='store_true', help='Applicable to keras input_format only: Whether the weights from ' 'different layers are to be stored in separate weight groups, ' 'corresponding to separate binary weight files. Default: False.') parser.add_argument( - '--version', + '--%s' % common.VERSION, '-v', dest='show_version', action='store_true', help='Show versions of tensorflowjs and its dependencies') parser.add_argument( - '--skip_op_check', + '--%s' % common.SKIP_OP_CHECK, action='store_true', help='Skip op validation for TensorFlow model conversion.') parser.add_argument( - '--strip_debug_ops', + '--%s' % common.STRIP_DEBUG_OPS, type=bool, default=True, help='Strip debug ops (Print, Assert, CheckNumerics) from graph.') parser.add_argument( - '--weight_shard_size_bytes', + '--%s' % common.WEIGHT_SHARD_SIZE_BYTES, type=int, default=None, help='Shard size (in bytes) of the weight files. Currently applicable ' @@ -538,9 +551,9 @@ def convert(arguments=None): weight_shard_size_bytes = 1024 * 1024 * 4 if FLAGS.weight_shard_size_bytes: - if FLAGS.output_format != 'tfjs_layers_model': + if FLAGS.output_format != common.TFJS_LAYERS_MODEL: raise ValueError( - 'The --weight_shard_size_byte flag is only supported under ' + 'The --weight_shard_size_bytes flag is only supported under ' 'output_format=tfjs_layers_model.') weight_shard_size_bytes = FLAGS.weight_shard_size_bytes @@ -557,7 +570,7 @@ def convert(arguments=None): if FLAGS.quantization_bytes else None) if (FLAGS.signature_name and input_format not in - ('tf_saved_model', 'tf_hub')): + (common.TF_SAVED_MODEL, common.TF_HUB_MODEL)): raise ValueError( 'The --signature_name flag is applicable only to "tf_saved_model" and ' '"tf_hub" input format, but the current input format is ' @@ -565,25 +578,27 @@ def convert(arguments=None): # TODO(cais, piyu): More conversion logics can be added as additional # branches below. - if input_format == 'keras' and output_format == 'tfjs_layers_model': + if (input_format == common.KERAS_MODEL and + output_format == common.TFJS_LAYERS_MODEL): dispatch_keras_h5_to_tfjs_layers_model_conversion( FLAGS.input_path, output_dir=FLAGS.output_path, quantization_dtype=quantization_dtype, split_weights_by_layer=FLAGS.split_weights_by_layer) - elif input_format == 'keras' and output_format == 'tfjs_graph_model': + elif (input_format == common.KERAS_MODEL and + output_format == common.TFJS_GRAPH_MODEL): dispatch_keras_h5_to_tfjs_graph_model_conversion( FLAGS.input_path, output_dir=FLAGS.output_path, quantization_dtype=quantization_dtype, skip_op_check=FLAGS.skip_op_check, strip_debug_ops=FLAGS.strip_debug_ops) - elif (input_format == 'keras_saved_model' and - output_format == 'tfjs_layers_model'): + elif (input_format == common.KERAS_SAVED_MODEL and + output_format == common.TFJS_LAYERS_MODEL): dispatch_keras_saved_model_to_tensorflowjs_conversion( FLAGS.input_path, FLAGS.output_path, quantization_dtype=quantization_dtype, split_weights_by_layer=FLAGS.split_weights_by_layer) - elif (input_format == 'tf_saved_model' and - output_format == 'tfjs_graph_model'): + elif (input_format == common.TF_SAVED_MODEL and + output_format == common.TFJS_GRAPH_MODEL): tf_saved_model_conversion_v2.convert_tf_saved_model( FLAGS.input_path, FLAGS.output_path, signature_def=FLAGS.signature_name, @@ -591,28 +606,28 @@ def convert(arguments=None): quantization_dtype=quantization_dtype, skip_op_check=FLAGS.skip_op_check, strip_debug_ops=FLAGS.strip_debug_ops) - elif (input_format == 'tf_hub' and - output_format == 'tfjs_graph_model'): + elif (input_format == common.TF_HUB_MODEL and + output_format == common.TFJS_GRAPH_MODEL): tf_saved_model_conversion_v2.convert_tf_hub_module( FLAGS.input_path, FLAGS.output_path, FLAGS.signature_name, FLAGS.saved_model_tags, skip_op_check=FLAGS.skip_op_check, strip_debug_ops=FLAGS.strip_debug_ops) - elif (input_format == 'tfjs_layers_model' and - output_format == 'keras'): + elif (input_format == common.TFJS_LAYERS_MODEL and + output_format == common.KERAS_MODEL): dispatch_tensorflowjs_to_keras_h5_conversion(FLAGS.input_path, FLAGS.output_path) - elif (input_format == 'tfjs_layers_model' and - output_format == 'keras_saved_model'): + elif (input_format == common.TFJS_LAYERS_MODEL and + output_format == common.KERAS_SAVED_MODEL): dispatch_tensorflowjs_to_keras_saved_model_conversion(FLAGS.input_path, FLAGS.output_path) - elif (input_format == 'tfjs_layers_model' and - output_format == 'tfjs_layers_model'): + elif (input_format == common.TFJS_LAYERS_MODEL and + output_format == common.TFJS_LAYERS_MODEL): dispatch_tensorflowjs_to_tensorflowjs_conversion( FLAGS.input_path, FLAGS.output_path, quantization_dtype=_parse_quantization_bytes(FLAGS.quantization_bytes), weight_shard_size_bytes=weight_shard_size_bytes) - elif (input_format == 'tfjs_layers_model' and - output_format == 'tfjs_graph_model'): + elif (input_format == common.TFJS_LAYERS_MODEL and + output_format == common.TFJS_GRAPH_MODEL): dispatch_tfjs_layers_model_to_tfjs_graph_conversion( FLAGS.input_path, FLAGS.output_path, quantization_dtype=_parse_quantization_bytes(FLAGS.quantization_bytes), From 4512586f13855c46d4b91df854ff7c2abd5542b8 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Tue, 30 Jul 2019 16:58:21 -0700 Subject: [PATCH 15/24] use tuple instead of list --- python/tensorflowjs/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tensorflowjs/cli.py b/python/tensorflowjs/cli.py index 2070e547..342841c1 100644 --- a/python/tensorflowjs/cli.py +++ b/python/tensorflowjs/cli.py @@ -150,7 +150,7 @@ def validate_input_path(input_path, input_format): We expect a URL that starts with http(s)://""" % path elif not os.path.exists(path): return 'Nonexistent path for the model: %s' % path - if input_format in [common.KERAS_SAVED_MODEL, common.TF_SAVED_MODEL]: + if input_format in (common.KERAS_SAVED_MODEL, common.TF_SAVED_MODEL): is_dir = os.path.isdir(path) if not is_dir and not path.endswith('saved_model.pb'): return 'The path provided is not a directory or pb file: %s' % path From 066af81f2287706fdd0908847a1420de36f638e2 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Wed, 31 Jul 2019 14:50:16 -0700 Subject: [PATCH 16/24] address the comments --- README.md | 14 +++- python/setup.py | 4 +- python/tensorflowjs/BUILD | 13 ++-- python/tensorflowjs/__init__.py | 2 +- python/tensorflowjs/converters/BUILD | 1 + python/tensorflowjs/converters/common.py | 4 +- python/tensorflowjs/converters/converter.py | 2 +- python/tensorflowjs/{cli.py => wizard.py} | 78 +++++++++++-------- .../{cli_test.py => wizard_test.py} | 63 ++++++++------- 9 files changed, 105 insertions(+), 76 deletions(-) rename python/tensorflowjs/{cli.py => wizard.py} (89%) rename python/tensorflowjs/{cli_test.py => wizard_test.py} (79%) diff --git a/README.md b/README.md index 1ee1a767..df65dab5 100644 --- a/README.md +++ b/README.md @@ -56,12 +56,12 @@ __2. Run the conversion script provided by the pip package:__ There are two way to trigger the model conversion: -- The interactive CLI: tensorflowjs_cli +- The conversion wizard: tensorflowjs_wizard - Regular conversion script: tensorflowjs_converter -To start the interactive CLI: +To start the conversion wizard: ```bash -tensorflowjs_cli +tensorflowjs_wizard ``` This tool will walk you through the conversion process and provide you with @@ -69,6 +69,14 @@ details explanations for each choice you need to make. Behind the scene it calls the converter script (tensorflowjs_converter) in pip package. This is the easier way to convert a single model. +There is also dry run mode for the wizard, which will not perform the actual +conversion but only generate the command for tensorflowjs_converter command. +This command can be used in your own shell script. + +```bash +tensorflowjs_wizard --dryrun +``` + To convert a batch of models or integrate the conversion process into your own script, you should look into using the tensorflowjs_converter script. diff --git a/python/setup.py b/python/setup.py index d2979deb..edfce955 100644 --- a/python/setup.py +++ b/python/setup.py @@ -27,7 +27,7 @@ def _get_requirements(file): CONSOLE_SCRIPTS = [ 'tensorflowjs_converter = tensorflowjs.converters.converter:main', - 'tensorflowjs_cli = tensorflowjs.cli:main', + 'tensorflowjs_wizard = tensorflowjs.wizard:main', ] setuptools.setup( @@ -55,10 +55,10 @@ def _get_requirements(file): ], py_modules=[ 'tensorflowjs', - 'tensorflowjs.cli', 'tensorflowjs.version', 'tensorflowjs.quantization', 'tensorflowjs.read_weights', + 'tensorflowjs.wizard', 'tensorflowjs.write_weights', 'tensorflowjs.converters', 'tensorflowjs.converters.common', diff --git a/python/tensorflowjs/BUILD b/python/tensorflowjs/BUILD index 2ada8596..b1d883ae 100644 --- a/python/tensorflowjs/BUILD +++ b/python/tensorflowjs/BUILD @@ -51,7 +51,7 @@ py_library( py_library( name = "expect_PyInquirer_installed", # This is a dummy rule used as a PyInquirer dependency in open-source. - # We expect tensorflow to already be installed on the system, e.g. via + # We expect PyInquirer to already be installed on the system, e.g. via # `pip install PyInquirer`. ) @@ -124,20 +124,21 @@ py_test( ) py_test( - name = "cli_test", - srcs = ["cli_test.py"], + name = "wizard_test", + srcs = ["wizard_test.py"], srcs_version = "PY2AND3", deps = [ ":expect_numpy_installed", - ":cli", + ":wizard", ], ) py_binary( - name = "cli", - srcs = ["cli.py"], + name = "wizard", + srcs = ["wizard.py"], srcs_version = "PY2AND3", deps = [ + ":converters/common", ":converters/converter", "//tensorflowjs:expect_h5py_installed", "//tensorflowjs:expect_keras_installed", diff --git a/python/tensorflowjs/__init__.py b/python/tensorflowjs/__init__.py index 97f34baf..d120fa33 100644 --- a/python/tensorflowjs/__init__.py +++ b/python/tensorflowjs/__init__.py @@ -18,9 +18,9 @@ from __future__ import print_function # pylint: disable=unused-imports -from tensorflowjs import cli from tensorflowjs import converters from tensorflowjs import quantization from tensorflowjs import version +from tensorflowjs import wizard __version__ = version.version diff --git a/python/tensorflowjs/converters/BUILD b/python/tensorflowjs/converters/BUILD index 3f4ea776..b865528a 100644 --- a/python/tensorflowjs/converters/BUILD +++ b/python/tensorflowjs/converters/BUILD @@ -91,6 +91,7 @@ py_binary( srcs = ["converter.py"], srcs_version = "PY2AND3", deps = [ + ":common", ":keras_h5_conversion", ":keras_tfjs_loader", ":tf_saved_model_conversion_v2", diff --git a/python/tensorflowjs/converters/common.py b/python/tensorflowjs/converters/common.py index efbdb000..f579ce15 100644 --- a/python/tensorflowjs/converters/common.py +++ b/python/tensorflowjs/converters/common.py @@ -30,7 +30,7 @@ GENERATED_BY_KEY = 'generatedBy' CONVERTED_BY_KEY = 'convertedBy' -# model formats +# Model formats. KERAS_SAVED_MODEL = 'keras_saved_model' KERAS_MODEL = 'keras' TF_SAVED_MODEL = 'tf_saved_model' @@ -38,7 +38,7 @@ TFJS_GRAPH_MODEL = 'tfjs_graph_model' TFJS_LAYERS_MODEL = 'tfjs_layers_model' -# cli argument string +# CLI argument strings. INPUT_PATH = 'input_path' OUTPUT_PATH = 'output_path' INPUT_FORMAT = 'input_format' diff --git a/python/tensorflowjs/converters/converter.py b/python/tensorflowjs/converters/converter.py index ebdbed84..ff6642dd 100644 --- a/python/tensorflowjs/converters/converter.py +++ b/python/tensorflowjs/converters/converter.py @@ -509,7 +509,7 @@ def setup_arguments(arguments=None): 'and 2-byte quantizaton is supported. The default (unquantized) size is ' '4 bytes.') parser.add_argument( - '--%s' % common.SPLIT_WEIGHTS_BY_LAYERS, + '--%s' % common.SPLIT_WEIGHTS_BY_LAYER, action='store_true', help='Applicable to keras input_format only: Whether the weights from ' 'different layers are to be stored in separate weight groups, ' diff --git a/python/tensorflowjs/cli.py b/python/tensorflowjs/wizard.py similarity index 89% rename from python/tensorflowjs/cli.py rename to python/tensorflowjs/wizard.py index 342841c1..1cc96476 100644 --- a/python/tensorflowjs/cli.py +++ b/python/tensorflowjs/wizard.py @@ -187,20 +187,16 @@ def expand_input_path(input_path): return input_path -def validate_output_path(output_path): - """Validate the input path for given input format. +def output_path_exists(output_path): + """Check the existence of the output path. Args: output_path: input path of the model. - input_format: model format string. Returns: - bool: return true when the output directory does not exist. + bool: return true when the output directory exists. """ - output_path = os.path.expanduser(output_path) - if not output_path: - return 'Please provide a valid output path' if os.path.exists(output_path): - return 'The output path already exists: %s' % output_path - return True + return True + return False def generate_arguments(params): @@ -211,7 +207,8 @@ def generate_arguments(params): list: the argument list for converter. """ args = [] - not_param_list = [common.INPUT_PATH, common.OUTPUT_PATH] + not_param_list = [common.INPUT_PATH, common.OUTPUT_PATH, + 'overwrite_output_path'] no_false_param = [common.SPLIT_WEIGHTS_BY_LAYER, common.SKIP_OP_CHECK] for key, value in sorted(params.items()): if key not in not_param_list and value is not None: @@ -403,13 +400,13 @@ def main(dryrun): 'message': 'What is your output format?', 'choices': available_output_formats, 'when': lambda answers: value_in_list(answers, common.INPUT_FORMAT, - [common.KERAS_SAVED_MODEL, - common.TFJS_LAYERS_MODEL]) + (common.KERAS_SAVED_MODEL, + common.TFJS_LAYERS_MODEL)) } ] - formats = PyInquirer.prompt(formats, input_params, style=prompt_style) - message = input_path_message(formats) - print(formats) + format_params = PyInquirer.prompt(formats, input_params, style=prompt_style) + message = input_path_message(format_params) + print(format_params) questions = [ { 'type': 'input', @@ -417,7 +414,7 @@ def main(dryrun): 'message': message, 'filter': expand_input_path, 'validate': lambda value: validate_input_path( - value, formats[common.INPUT_FORMAT]), + value, format_params[common.INPUT_FORMAT]), 'when': lambda answers: (not detected_input_format) }, { @@ -426,7 +423,8 @@ def main(dryrun): 'choices': available_tags, 'message': 'What is tags for the saved model?', 'when': lambda answers: (is_saved_model(answers[common.INPUT_FORMAT]) - and formats[common.OUTPUT_FORMAT] == + and not common.OUTPUT_FORMAT in format_params + or format_params[common.OUTPUT_FORMAT] == common.TFJS_GRAPH_MODEL) }, { @@ -435,7 +433,8 @@ def main(dryrun): 'message': 'What is signature name of the model?', 'choices': available_signature_names, 'when': lambda answers: (is_saved_model(answers[common.INPUT_FORMAT]) - and formats[common.OUTPUT_FORMAT] == + and not common.OUTPUT_FORMAT in format_params + or format_params[common.OUTPUT_FORMAT] == common.TFJS_GRAPH_MODEL) }, { @@ -454,7 +453,7 @@ def main(dryrun): 'message': 'Please enter shard size (in bytes) of the weight files?', 'default': str(4 * 1024 * 1024), 'when': lambda answers: value_in_list(answers, common.OUTPUT_FORMAT, - [common.TFJS_LAYERS_MODEL]) + (common.TFJS_LAYERS_MODEL)) }, { 'type': 'confirm', @@ -462,7 +461,7 @@ def main(dryrun): 'message': 'Do you want to split weights by layers?', 'default': False, 'when': lambda answers: value_in_list(answers, common.INPUT_FORMAT, - [common.TFJS_LAYERS_MODEL]) + (common.TFJS_LAYERS_MODEL)) }, { 'type': 'confirm', @@ -472,8 +471,8 @@ def main(dryrun): 'you can implement them as custom ops in tfjs-converter.', 'default': False, 'when': lambda answers: value_in_list(answers, common.INPUT_FORMAT, - [common.TF_SAVED_MODEL, - common.TF_HUB_MODEL]) + (common.TF_SAVED_MODEL, + common.TF_HUB_MODEL)) }, { 'type': 'confirm', @@ -482,33 +481,50 @@ def main(dryrun): 'This will improve model execution performance.', 'default': True, 'when': lambda answers: value_in_list(answers, common.INPUT_FORMAT, - [common.TF_SAVED_MODEL, - common.TF_HUB_MODEL]) - }, - { + (common.TF_SAVED_MODEL, + common.TF_HUB_MODEL)) + } + ] + params = PyInquirer.prompt(questions, format_params, style=prompt_style) + + output_options = [ + { 'type': 'input', 'name': common.OUTPUT_PATH, 'message': 'Which directory do you want to save ' 'the converted model in?', - 'filter': os.path.expanduser, - 'validate': validate_output_path + 'filter': lambda path: os.path.expanduser(path.strip()), + 'validate': lambda path: len(path) > 0 + }, { + 'type': 'confirm', + 'message': 'The output already directory exists, ' + 'do you want to overwrite it?', + 'name': 'overwrite_output_path', + 'default': False, + 'when': lambda ans: output_path_exists(ans[common.OUTPUT_PATH]) } ] - params = PyInquirer.prompt(questions, formats, style=prompt_style) + + while (not common.OUTPUT_PATH in params or + output_path_exists(params[common.OUTPUT_PATH]) and + not params['overwrite_output_path']): + params = PyInquirer.prompt(output_options, params, style=prompt_style) arguments = generate_arguments(params) print('converter command generated:') print('tensorflowjs_converter %s' % ' '.join(arguments)) + print('\n\n') + if not dryrun: converter.convert(arguments) - print('file generated after conversion:') + print('\n\nFile(s) generated after conversion:') for entry in os.scandir(params[common.OUTPUT_PATH]): print(entry.stat().st_size, entry.name) if __name__ == '__main__': if len(sys.argv) > 2 or len(sys.argv) == 2 and not sys.argv[1] == '--dryrun': - print("Usage: tensorflowjs_cli [--dryrun]") + print("Usage: tensorflowjs_wizard [--dryrun]") sys.exit(1) dry_run = len(sys.argv) == 2 and sys.argv[1] == '--dryrun' main(dry_run) diff --git a/python/tensorflowjs/cli_test.py b/python/tensorflowjs/wizard_test.py similarity index 79% rename from python/tensorflowjs/cli_test.py rename to python/tensorflowjs/wizard_test.py index fc9fe3e0..dfa5a4ba 100644 --- a/python/tensorflowjs/cli_test.py +++ b/python/tensorflowjs/wizard_test.py @@ -28,7 +28,7 @@ from tensorflow.python.training.tracking import tracking from tensorflow.python.saved_model import save -from tensorflowjs import cli +from tensorflowjs import wizard SAVED_MODEL_DIR = 'saved_model' SAVED_MODEL_NAME = 'saved_model.pb' @@ -77,86 +77,89 @@ def _create_saved_model(self): save.save(root, save_dir, to_save) def testQuantizationType(self): - self.assertEqual(2, cli.quantization_type('1/2')) - self.assertEqual(1, cli.quantization_type('1/4')) - self.assertEqual(None, cli.quantization_type('1')) + self.assertEqual(2, wizard.quantization_type('1/2')) + self.assertEqual(1, wizard.quantization_type('1/4')) + self.assertEqual(None, wizard.quantization_type('1')) def testOfValues(self): answers = {'input_path': 'abc', 'input_format': '123'} - self.assertEqual(True, cli.value_in_list(answers, 'input_path', ['abc'])) - self.assertEqual(False, cli.value_in_list(answers, 'input_path', ['abd'])) - self.assertEqual(False, cli.value_in_list(answers, + self.assertEqual(True, wizard.value_in_list(answers, 'input_path', ['abc'])) + self.assertEqual(False, wizard.value_in_list(answers, + 'input_path', ['abd'])) + self.assertEqual(False, wizard.value_in_list(answers, 'input_format2', ['abc'])) def testInputPathMessage(self): answers = {'input_format': 'keras'} self.assertEqual("The original path seems to be wrong, " "what is the path of input HDF5 file?", - cli.input_path_message(answers)) + wizard.input_path_message(answers)) answers = {'input_format': 'tf_hub'} self.assertEqual("The original path seems to be wrong, " "what is the TFHub module URL?", - cli.input_path_message(answers)) + wizard.input_path_message(answers)) answers = {'input_format': 'tf_saved_model'} self.assertEqual("The original path seems to be wrong, " "what is the directory that contains the model?", - cli.input_path_message(answers)) + wizard.input_path_message(answers)) def testValidateInputPathForTFHub(self): - self.assertNotEqual(True, cli.validate_input_path(self._tmp_dir, 'tf_hub')) + self.assertNotEqual(True, + wizard.validate_input_path(self._tmp_dir, 'tf_hub')) self.assertEqual(True, - cli.validate_input_path("https://tfhub.dev/mobilenet", + wizard.validate_input_path("https://tfhub.dev/mobilenet", 'tf_hub')) def testValidateInputPathForSavedModel(self): - self.assertNotEqual(True, cli.validate_input_path( + self.assertNotEqual(True, wizard.validate_input_path( self._tmp_dir, 'tf_saved_model')) self._create_saved_model() save_dir = os.path.join(self._tmp_dir, SAVED_MODEL_DIR) - self.assertEqual(True, cli.validate_input_path( + self.assertEqual(True, wizard.validate_input_path( save_dir, 'tf_saved_model')) save_dir = os.path.join(self._tmp_dir, SAVED_MODEL_DIR, SAVED_MODEL_NAME) - self.assertEqual(True, cli.validate_input_path( + self.assertEqual(True, wizard.validate_input_path( save_dir, 'tf_saved_model')) def testValidateInputPathForKerasSavedModel(self): - self.assertNotEqual(True, cli.validate_input_path( + self.assertNotEqual(True, wizard.validate_input_path( self._tmp_dir, 'keras_saved_model')) self._create_saved_model() save_dir = os.path.join(self._tmp_dir, SAVED_MODEL_DIR) - self.assertEqual(True, cli.validate_input_path( + self.assertEqual(True, wizard.validate_input_path( save_dir, 'keras_saved_model')) def testValidateInputPathForKerasModel(self): - self.assertNotEqual(True, cli.validate_input_path(self._tmp_dir, 'keras')) + self.assertNotEqual(True, wizard.validate_input_path(self._tmp_dir, 'keras')) self._create_hd5_file() save_dir = os.path.join(self._tmp_dir, HD5_FILE_NAME) - self.assertEqual(True, cli.validate_input_path( + self.assertEqual(True, wizard.validate_input_path( save_dir, 'keras')) def testValidateInputPathForLayersModel(self): - self.assertNotEqual(True, cli.validate_input_path(self._tmp_dir, 'keras')) + self.assertNotEqual(True, + wizard.validate_input_path(self._tmp_dir, 'keras')) self._create_layers_model() save_dir = os.path.join(self._tmp_dir) - self.assertEqual(True, cli.validate_input_path( + self.assertEqual(True, wizard.validate_input_path( save_dir, 'tfjs_layers_model')) save_dir = os.path.join(self._tmp_dir, 'model.json') - self.assertEqual(True, cli.validate_input_path( + self.assertEqual(True, wizard.validate_input_path( save_dir, 'tfjs_layers_model')) def testValidateOutputPath(self): - self.assertNotEqual(True, cli.validate_output_path(self._tmp_dir)) + self.assertNotEqual(True, wizard.validate_output_path(self._tmp_dir)) output_dir = os.path.join(self._tmp_dir, 'test') - self.assertEqual(True, cli.validate_output_path(output_dir)) + self.assertEqual(True, wizard.validate_output_path(output_dir)) def testAvailableTags(self): self._create_saved_model() save_dir = os.path.join(self._tmp_dir, SAVED_MODEL_DIR) - self.assertEqual(['serve'], cli.available_tags( + self.assertEqual(['serve'], wizard.available_tags( {'input_path': save_dir, 'input_format': 'tf_saved_model'})) @@ -164,7 +167,7 @@ def testAvailableSignatureNames(self): self._create_saved_model() save_dir = os.path.join(self._tmp_dir, SAVED_MODEL_DIR) self.assertEqual(['__saved_model_init_op', 'serving_default'], - [x['value'] for x in cli.available_signature_names( + [x['value'] for x in wizard.available_signature_names( {'input_path': save_dir, 'input_format': 'tf_saved_model', 'saved_model_tags': 'serve'})]) @@ -183,7 +186,7 @@ def testGenerateCommandForSavedModel(self): '--quantization_bytes=2', '--saved_model_tags=test', '--signature_name=test_default', '--strip_debug_ops=True', 'tmp/saved_model', 'tmp/web_model'], - cli.generate_arguments(options)) + wizard.generate_arguments(options)) def testGenerateCommandForKerasSavedModel(self): options = {'input_format': 'tf_keras_saved_model', @@ -202,7 +205,7 @@ def testGenerateCommandForKerasSavedModel(self): '--signature_name=test_default', '--skip_op_check', '--strip_debug_ops=False', 'tmp/saved_model', 'tmp/web_model'], - cli.generate_arguments(options)) + wizard.generate_arguments(options)) def testGenerateCommandForKerasModel(self): options = {'input_format': 'keras', @@ -212,7 +215,7 @@ def testGenerateCommandForKerasModel(self): self.assertEqual(['--input_format=keras', '--quantization_bytes=1', 'tmp/model.HD5', 'tmp/web_model'], - cli.generate_arguments(options)) + wizard.generate_arguments(options)) def testGenerateCommandForLayerModel(self): options = {'input_format': 'tfjs_layers_model', @@ -225,7 +228,7 @@ def testGenerateCommandForLayerModel(self): '--output_format=keras', '--quantization_bytes=1', 'tmp/model.json', 'tmp/web_model'], - cli.generate_arguments(options)) + wizard.generate_arguments(options)) if __name__ == '__main__': From 0cc44e6d6d9017578256dffcc0bb088bd5e0b5f9 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Thu, 1 Aug 2019 11:27:54 -0700 Subject: [PATCH 17/24] update compression choices --- python/tensorflowjs/wizard.py | 55 +++++++++++++----------------- python/tensorflowjs/wizard_test.py | 15 ++++---- 2 files changed, 29 insertions(+), 41 deletions(-) diff --git a/python/tensorflowjs/wizard.py b/python/tensorflowjs/wizard.py index 1cc96476..624e34df 100644 --- a/python/tensorflowjs/wizard.py +++ b/python/tensorflowjs/wizard.py @@ -31,7 +31,7 @@ # regex for recognizing valid url for TFHub module. TFHUB_VALID_URL_REGEX = re.compile( # http:// or https:// - r'^(?:http)s?://', re.IGNORECASE) + r'^(http)s?://', re.IGNORECASE) # prompt style prompt_style = PyInquirer.style_from_dict({ @@ -44,25 +44,6 @@ PyInquirer.Token.Question: '', }) -def quantization_type(user_selection_quant): - """Determine the quantization type based on user selection. - Args: - user_selection_quant: user selected quantization value. - - Returns: - int: quantization parameter value for converter. - """ - answer = None - try: - if '1/2' in user_selection_quant: - answer = 2 - elif '1/4' in user_selection_quant: - answer = 1 - except ValueError: - answer = None - return answer - - def value_in_list(answers, key, values): """Determine user's answer for the key is in the value list. Args: @@ -91,7 +72,7 @@ def detect_input_format(input_path): string: detected input format string: normalized input path """ - input_path = input_path.lower() + input_path = input_path.strip() detected_input_format = None if re.match(TFHUB_VALID_URL_REGEX, input_path): detected_input_format = common.TF_HUB_MODEL @@ -130,7 +111,9 @@ def input_path_message(answers): if answer == common.KERAS_MODEL: return message + 'what is the path of input HDF5 file?' elif answer == common.TF_HUB_MODEL: - return message + 'what is the TFHub module URL?' + return message + ("what is the TFHub module URL? \n" + "(i.e. https://tfhub.dev/google/imagenet/" + "mobilenet_v1_100_224/classification/1)") else: return message + 'what is the directory that contains the model?' @@ -141,7 +124,7 @@ def validate_input_path(input_path, input_format): input_path: input path of the model. input_format: model format string. """ - path = os.path.expanduser(input_path) + path = os.path.expanduser(input_path.strip()) if not path: return 'Please enter a valid path' if input_format == common.TF_HUB_MODEL: @@ -177,7 +160,7 @@ def expand_input_path(input_path): Returns: string: return expanded input path. """ - input_path = os.path.expanduser(input_path) + input_path = os.path.expanduser(input_path.strip()) is_dir = os.path.isdir(input_path) if is_dir: for fname in os.listdir(input_path): @@ -423,9 +406,10 @@ def main(dryrun): 'choices': available_tags, 'message': 'What is tags for the saved model?', 'when': lambda answers: (is_saved_model(answers[common.INPUT_FORMAT]) - and not common.OUTPUT_FORMAT in format_params + and + (not common.OUTPUT_FORMAT in format_params or format_params[common.OUTPUT_FORMAT] == - common.TFJS_GRAPH_MODEL) + common.TFJS_GRAPH_MODEL)) }, { 'type': 'list', @@ -433,19 +417,26 @@ def main(dryrun): 'message': 'What is signature name of the model?', 'choices': available_signature_names, 'when': lambda answers: (is_saved_model(answers[common.INPUT_FORMAT]) - and not common.OUTPUT_FORMAT in format_params + and + (not common.OUTPUT_FORMAT in format_params or format_params[common.OUTPUT_FORMAT] == - common.TFJS_GRAPH_MODEL) + common.TFJS_GRAPH_MODEL)) }, { 'type': 'list', 'name': common.QUANTIZATION_BYTES, 'message': 'Do you want to compress the model? ' '(this will decrease the model precision.)', - 'choices': ['No compression, no accuracy loss.', - '2x compression, medium accuracy loss.', - '4x compression, highest accuracy loss.'], - 'filter': quantization_type + 'choices': [{ + 'name': 'No compression, no accuracy loss.', + 'value': None + }, { + 'name': '2x compression, medium accuracy loss.', + 'value': 2 + }, { + 'name': '4x compression, highest accuracy loss.', + 'value': 1 + }] }, { 'type': 'input', diff --git a/python/tensorflowjs/wizard_test.py b/python/tensorflowjs/wizard_test.py index dfa5a4ba..fbdff1dd 100644 --- a/python/tensorflowjs/wizard_test.py +++ b/python/tensorflowjs/wizard_test.py @@ -76,11 +76,6 @@ def _create_saved_model(self): save_dir = os.path.join(self._tmp_dir, SAVED_MODEL_DIR) save.save(root, save_dir, to_save) - def testQuantizationType(self): - self.assertEqual(2, wizard.quantization_type('1/2')) - self.assertEqual(1, wizard.quantization_type('1/4')) - self.assertEqual(None, wizard.quantization_type('1')) - def testOfValues(self): answers = {'input_path': 'abc', 'input_format': '123'} self.assertEqual(True, wizard.value_in_list(answers, 'input_path', ['abc'])) @@ -97,7 +92,9 @@ def testInputPathMessage(self): answers = {'input_format': 'tf_hub'} self.assertEqual("The original path seems to be wrong, " - "what is the TFHub module URL?", + "what is the TFHub module URL? \n" + "(i.e. https://tfhub.dev/google/imagenet/" + "mobilenet_v1_100_224/classification/1)", wizard.input_path_message(answers)) answers = {'input_format': 'tf_saved_model'} @@ -151,10 +148,10 @@ def testValidateInputPathForLayersModel(self): self.assertEqual(True, wizard.validate_input_path( save_dir, 'tfjs_layers_model')) - def testValidateOutputPath(self): - self.assertNotEqual(True, wizard.validate_output_path(self._tmp_dir)) + def testOutputPathExist(self): + self.assertEqual(True, wizard.output_path_exists(self._tmp_dir)) output_dir = os.path.join(self._tmp_dir, 'test') - self.assertEqual(True, wizard.validate_output_path(output_dir)) + self.assertNotEqual(True, wizard.output_path_exists(output_dir)) def testAvailableTags(self): self._create_saved_model() From e2135a474072b7e16e8488973adcfb05421bc761 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Thu, 1 Aug 2019 14:01:33 -0700 Subject: [PATCH 18/24] fix pylint error --- python/tensorflowjs/wizard.py | 44 +++++++++++++++--------------- python/tensorflowjs/wizard_test.py | 9 +++--- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/python/tensorflowjs/wizard.py b/python/tensorflowjs/wizard.py index 624e34df..7ab7af4e 100644 --- a/python/tensorflowjs/wizard.py +++ b/python/tensorflowjs/wizard.py @@ -112,8 +112,8 @@ def input_path_message(answers): return message + 'what is the path of input HDF5 file?' elif answer == common.TF_HUB_MODEL: return message + ("what is the TFHub module URL? \n" - "(i.e. https://tfhub.dev/google/imagenet/" - "mobilenet_v1_100_224/classification/1)") + "(i.e. https://tfhub.dev/google/imagenet/" + "mobilenet_v1_100_224/classification/1)") else: return message + 'what is the directory that contains the model?' @@ -408,8 +408,8 @@ def main(dryrun): 'when': lambda answers: (is_saved_model(answers[common.INPUT_FORMAT]) and (not common.OUTPUT_FORMAT in format_params - or format_params[common.OUTPUT_FORMAT] == - common.TFJS_GRAPH_MODEL)) + or format_params[common.OUTPUT_FORMAT] == + common.TFJS_GRAPH_MODEL)) }, { 'type': 'list', @@ -419,8 +419,8 @@ def main(dryrun): 'when': lambda answers: (is_saved_model(answers[common.INPUT_FORMAT]) and (not common.OUTPUT_FORMAT in format_params - or format_params[common.OUTPUT_FORMAT] == - common.TFJS_GRAPH_MODEL)) + or format_params[common.OUTPUT_FORMAT] == + common.TFJS_GRAPH_MODEL)) }, { 'type': 'list', @@ -428,15 +428,15 @@ def main(dryrun): 'message': 'Do you want to compress the model? ' '(this will decrease the model precision.)', 'choices': [{ - 'name': 'No compression, no accuracy loss.', - 'value': None - }, { - 'name': '2x compression, medium accuracy loss.', - 'value': 2 - }, { - 'name': '4x compression, highest accuracy loss.', - 'value': 1 - }] + 'name': 'No compression, no accuracy loss.', + 'value': None + }, { + 'name': '2x compression, medium accuracy loss.', + 'value': 2 + }, { + 'name': '4x compression, highest accuracy loss.', + 'value': 1 + }] }, { 'type': 'input', @@ -479,7 +479,7 @@ def main(dryrun): params = PyInquirer.prompt(questions, format_params, style=prompt_style) output_options = [ - { + { 'type': 'input', 'name': common.OUTPUT_PATH, 'message': 'Which directory do you want to save ' @@ -487,12 +487,12 @@ def main(dryrun): 'filter': lambda path: os.path.expanduser(path.strip()), 'validate': lambda path: len(path) > 0 }, { - 'type': 'confirm', - 'message': 'The output already directory exists, ' - 'do you want to overwrite it?', - 'name': 'overwrite_output_path', - 'default': False, - 'when': lambda ans: output_path_exists(ans[common.OUTPUT_PATH]) + 'type': 'confirm', + 'message': 'The output already directory exists, ' + 'do you want to overwrite it?', + 'name': 'overwrite_output_path', + 'default': False, + 'when': lambda ans: output_path_exists(ans[common.OUTPUT_PATH]) } ] diff --git a/python/tensorflowjs/wizard_test.py b/python/tensorflowjs/wizard_test.py index fbdff1dd..a2faa8bf 100644 --- a/python/tensorflowjs/wizard_test.py +++ b/python/tensorflowjs/wizard_test.py @@ -80,9 +80,9 @@ def testOfValues(self): answers = {'input_path': 'abc', 'input_format': '123'} self.assertEqual(True, wizard.value_in_list(answers, 'input_path', ['abc'])) self.assertEqual(False, wizard.value_in_list(answers, - 'input_path', ['abd'])) + 'input_path', ['abd'])) self.assertEqual(False, wizard.value_in_list(answers, - 'input_format2', ['abc'])) + 'input_format2', ['abc'])) def testInputPathMessage(self): answers = {'input_format': 'keras'} @@ -107,7 +107,7 @@ def testValidateInputPathForTFHub(self): wizard.validate_input_path(self._tmp_dir, 'tf_hub')) self.assertEqual(True, wizard.validate_input_path("https://tfhub.dev/mobilenet", - 'tf_hub')) + 'tf_hub')) def testValidateInputPathForSavedModel(self): self.assertNotEqual(True, wizard.validate_input_path( @@ -130,7 +130,8 @@ def testValidateInputPathForKerasSavedModel(self): save_dir, 'keras_saved_model')) def testValidateInputPathForKerasModel(self): - self.assertNotEqual(True, wizard.validate_input_path(self._tmp_dir, 'keras')) + self.assertNotEqual(True, + wizard.validate_input_path(self._tmp_dir, 'keras')) self._create_hd5_file() save_dir = os.path.join(self._tmp_dir, HD5_FILE_NAME) self.assertEqual(True, wizard.validate_input_path( From 4cdcda88e26e78543fa561c62d23c85785c9ee50 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Thu, 1 Aug 2019 14:12:34 -0700 Subject: [PATCH 19/24] more pylint error --- python/tensorflowjs/wizard.py | 4 ++-- python/tensorflowjs/wizard_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/tensorflowjs/wizard.py b/python/tensorflowjs/wizard.py index 7ab7af4e..b61cd165 100644 --- a/python/tensorflowjs/wizard.py +++ b/python/tensorflowjs/wizard.py @@ -58,8 +58,8 @@ def value_in_list(answers, key, values): return False -def get_tfjs_model_type(file): - with open(file) as f: +def get_tfjs_model_type(model_file): + with open(model_file) as f: data = json.load(f) return data['format'] diff --git a/python/tensorflowjs/wizard_test.py b/python/tensorflowjs/wizard_test.py index a2faa8bf..bde7f14b 100644 --- a/python/tensorflowjs/wizard_test.py +++ b/python/tensorflowjs/wizard_test.py @@ -49,8 +49,8 @@ def tearDown(self): def _create_layers_model(self): data = {'format': 'layers-model'} filename = os.path.join(self._tmp_dir, 'model.json') - with open(filename, 'a') as file: - json.dump(data, file) + with open(filename, 'a') as model_file: + json.dump(data, model_file) def _create_hd5_file(self): input_tensor = keras.layers.Input((3,)) From 89d52a1017182978b357779a5ae3e918995126d6 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Fri, 2 Aug 2019 11:10:15 -0700 Subject: [PATCH 20/24] used listdir to support py2 --- python/tensorflowjs/wizard.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/python/tensorflowjs/wizard.py b/python/tensorflowjs/wizard.py index b61cd165..716e5c83 100644 --- a/python/tensorflowjs/wizard.py +++ b/python/tensorflowjs/wizard.py @@ -389,7 +389,7 @@ def main(dryrun): ] format_params = PyInquirer.prompt(formats, input_params, style=prompt_style) message = input_path_message(format_params) - print(format_params) + questions = [ { 'type': 'input', @@ -509,8 +509,15 @@ def main(dryrun): if not dryrun: converter.convert(arguments) print('\n\nFile(s) generated after conversion:') - for entry in os.scandir(params[common.OUTPUT_PATH]): - print(entry.stat().st_size, entry.name) + + print("Filename:{0:20} Size {1}".format('','')) + total_size = 0 + for basename in os.listdir(params[common.OUTPUT_PATH]): + filename = os.path.join(params[common.OUTPUT_PATH], basename) + size = os.path.getsize(filename) + print("{0:30} {1}".format(basename, size)) + total_size += size + print("Total size:{0:20}".format(total_size)) if __name__ == '__main__': From 4995679a3980faa85d46d0c5f4662e6bcdae39dd Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Fri, 2 Aug 2019 11:11:44 -0700 Subject: [PATCH 21/24] update text --- python/tensorflowjs/wizard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tensorflowjs/wizard.py b/python/tensorflowjs/wizard.py index 716e5c83..83a6f1d9 100644 --- a/python/tensorflowjs/wizard.py +++ b/python/tensorflowjs/wizard.py @@ -508,7 +508,7 @@ def main(dryrun): if not dryrun: converter.convert(arguments) - print('\n\nFile(s) generated after conversion:') + print('\n\nFile(s) generated by conversion:') print("Filename:{0:20} Size {1}".format('','')) total_size = 0 From 725ddd272c158fe7b3b194322dd6358e2942edd2 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Mon, 5 Aug 2019 17:11:30 -0700 Subject: [PATCH 22/24] fix pylint error --- python/tensorflowjs/wizard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tensorflowjs/wizard.py b/python/tensorflowjs/wizard.py index 83a6f1d9..6046da35 100644 --- a/python/tensorflowjs/wizard.py +++ b/python/tensorflowjs/wizard.py @@ -510,7 +510,7 @@ def main(dryrun): converter.convert(arguments) print('\n\nFile(s) generated by conversion:') - print("Filename:{0:20} Size {1}".format('','')) + print("Filename:{0:20} Size {1}".format('', '')) total_size = 0 for basename in os.listdir(params[common.OUTPUT_PATH]): filename = os.path.join(params[common.OUTPUT_PATH], basename) From 0193b613624a4dea7a19160c894798443b8dd297 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Tue, 6 Aug 2019 09:24:32 -0700 Subject: [PATCH 23/24] addressed comments --- README.md | 14 ++++++-------- python/tensorflowjs/wizard.py | 20 ++++++++++++-------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index df65dab5..c7421f58 100644 --- a/README.md +++ b/README.md @@ -25,12 +25,10 @@ __0. Please make sure that you run in a Docker container or a virtual environmen The script pulls its own subset of TensorFlow, which might conflict with the existing TensorFlow/Keras installation. -__Note__: *Check that [`tf-nightly-2.0-preview`](https://pypi.org/project/tf-nightly-2.0-preview/#files) is available for your platform.* - -Most of the times, this means that you have to use Python 3.6.8 in your local -environment. To force Python 3.6.8 in your local project, you can install -[`pyenv`](https://github.com/pyenv/pyenv) and proceed as follows in the target -directory: +The converter supports both Python 2 and Python 3. But Python 3.6.8 is recommended +for your local environment. To force Python 3.6.8 in your local project, you can +install [`pyenv`](https://github.com/pyenv/pyenv) and proceed as follows in the +target directory: ```bash pyenv install 3.6.8 @@ -66,11 +64,11 @@ tensorflowjs_wizard This tool will walk you through the conversion process and provide you with details explanations for each choice you need to make. Behind the scene it calls -the converter script (tensorflowjs_converter) in pip package. This is the easier +the converter script (`tensorflowjs_converter`) in pip package. This is the easier way to convert a single model. There is also dry run mode for the wizard, which will not perform the actual -conversion but only generate the command for tensorflowjs_converter command. +conversion but only generate the command for `tensorflowjs_converter` command. This command can be used in your own shell script. ```bash diff --git a/python/tensorflowjs/wizard.py b/python/tensorflowjs/wizard.py index 6046da35..d6e960d5 100644 --- a/python/tensorflowjs/wizard.py +++ b/python/tensorflowjs/wizard.py @@ -1,4 +1,4 @@ -# Copyright 2018 Google LLC +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -284,13 +284,14 @@ def available_signature_names(answers): def format_signature(name, input_nodes, output_nodes): string = "signature name: %s\n" % name - string += " inputs: \n%s" % format_nodes(input_nodes) - string += " outputs: \n%s" % format_nodes(output_nodes) + string += " inputs: %s" % format_nodes(input_nodes) + string += " outputs: %s" % format_nodes(output_nodes) return string def format_nodes(nodes): - string = "" + string = "%s of %s\n" % (3 if len(nodes) > 3 else len(nodes), len(nodes)) + count = 0 for key in nodes: value = nodes[key] string += " name: %s, " % value.name @@ -299,6 +300,9 @@ def format_nodes(nodes): string += "shape: Unknown\n" else: string += "shape: %s\n" % [x.size for x in value.tensor_shape.dim] + count += 1 + if count >= 3: + break return string @@ -510,14 +514,14 @@ def main(dryrun): converter.convert(arguments) print('\n\nFile(s) generated by conversion:') - print("Filename:{0:20} Size {1}".format('', '')) + print("Filename {0:25} Size(bytes)".format('')) total_size = 0 - for basename in os.listdir(params[common.OUTPUT_PATH]): + for basename in sorted(os.listdir(params[common.OUTPUT_PATH])): filename = os.path.join(params[common.OUTPUT_PATH], basename) size = os.path.getsize(filename) - print("{0:30} {1}".format(basename, size)) + print("{0:35} {1}".format(basename, size)) total_size += size - print("Total size:{0:20}".format(total_size)) + print("Total size:{0:24} {1}".format('', total_size)) if __name__ == '__main__': From 9c1d0e4c61feb584b239b2822128bbd50f8c7163 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Wed, 14 Aug 2019 15:17:51 -0700 Subject: [PATCH 24/24] fixed test error --- python/tensorflowjs/converters/converter.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/python/tensorflowjs/converters/converter.py b/python/tensorflowjs/converters/converter.py index 44609150..d7dd88ae 100644 --- a/python/tensorflowjs/converters/converter.py +++ b/python/tensorflowjs/converters/converter.py @@ -435,14 +435,11 @@ def _parse_quantization_bytes(quantization_bytes): else: raise ValueError('Unsupported quantization bytes: %s' % quantization_bytes) -def setup_arguments(arguments=None): +def get_arg_parser(): """ Create the argument parser for the converter binary. - - Args: - arguments: list, the argument string list to be parsed from. If arguments is - not given, it will try to parse from the system arguments. """ + parser = argparse.ArgumentParser('TensorFlow.js model converters.') parser.add_argument( common.INPUT_PATH, @@ -535,13 +532,10 @@ def setup_arguments(arguments=None): default=None, help='Shard size (in bytes) of the weight files. Currently applicable ' 'only to output_format=tfjs_layers_model.') - if arguments: - return parser.parse_args(arguments) - else: - return parser.parse_args() + return parser -def convert(arguments=None): - args = setup_arguments(arguments) +def convert(arguments): + args = get_arg_parser().parse_args(arguments) if args.show_version: print('\ntensorflowjs %s\n' % version.version) print('Dependency versions:') @@ -656,7 +650,7 @@ def pip_main(): def main(argv): - convert(args) + convert(argv) if __name__ == '__main__': tf.app.run(main=main, argv=[' '.join(sys.argv[1:])])