From 8f2b50b6c8ea23e86eed7a00eec5afbc54fb1787 Mon Sep 17 00:00:00 2001 From: Imss27 <1529767689@qq.com> Date: Mon, 30 Sep 2024 01:19:06 -0700 Subject: [PATCH 01/13] initial test --- vllm/scripts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vllm/scripts.py b/vllm/scripts.py index 231a18e99f3d..69d91e4dc1ea 100644 --- a/vllm/scripts.py +++ b/vllm/scripts.py @@ -123,6 +123,7 @@ def main(): help="Start the vLLM OpenAI Compatible API server", usage="vllm serve [options]") serve_parser.add_argument("model_tag", + nargs=argparse.REMAINDER, type=str, help="The model tag to serve") serve_parser.add_argument( @@ -160,6 +161,7 @@ def main(): chat_parser.set_defaults(dispatch_function=interactive_cli, command="chat") args = parser.parse_args() + print(args.model_tag) # One of the sub commands should be executed. if hasattr(args, "dispatch_function"): args.dispatch_function(args) From 1ce9778e043763382406f0036716fab19ef5c559 Mon Sep 17 00:00:00 2001 From: Imss27 <1529767689@qq.com> Date: Mon, 30 Sep 2024 01:42:00 -0700 Subject: [PATCH 02/13] Changed the order of processed_args --- vllm/scripts.py | 2 -- vllm/utils.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/vllm/scripts.py b/vllm/scripts.py index 69d91e4dc1ea..231a18e99f3d 100644 --- a/vllm/scripts.py +++ b/vllm/scripts.py @@ -123,7 +123,6 @@ def main(): help="Start the vLLM OpenAI Compatible API server", usage="vllm serve [options]") serve_parser.add_argument("model_tag", - nargs=argparse.REMAINDER, type=str, help="The model tag to serve") serve_parser.add_argument( @@ -161,7 +160,6 @@ def main(): chat_parser.set_defaults(dispatch_function=interactive_cli, command="chat") args = parser.parse_args() - print(args.model_tag) # One of the sub commands should be executed. if hasattr(args, "dispatch_function"): args.dispatch_function(args) diff --git a/vllm/utils.py b/vllm/utils.py index 43b64263d645..745f7707a4a0 100644 --- a/vllm/utils.py +++ b/vllm/utils.py @@ -1184,7 +1184,7 @@ def _pull_args_from_config(args: List[str]) -> List[str]: # followed by rest of cli args. # maintaining this order will enforce the precedence # of cli > config > defaults - args = [args[0]] + config_args + args[1:index] + args[index + 2:] + args = [args[0]] + args[1:index] + config_args + args[index + 2:] return args From f3b3e955d90314494c43f535039504e10cca9d5a Mon Sep 17 00:00:00 2001 From: Imss27 <1529767689@qq.com> Date: Mon, 30 Sep 2024 10:53:21 -0700 Subject: [PATCH 03/13] Reversed order and added comment --- vllm/commit_id.py | 1 + vllm/utils.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 vllm/commit_id.py diff --git a/vllm/commit_id.py b/vllm/commit_id.py new file mode 100644 index 000000000000..732331d9359b --- /dev/null +++ b/vllm/commit_id.py @@ -0,0 +1 @@ +__commit__ = "cca61642e0484212e6cd78b35b4789afed8d19c6" diff --git a/vllm/utils.py b/vllm/utils.py index b92f938f9f9a..f38a7b075677 100644 --- a/vllm/utils.py +++ b/vllm/utils.py @@ -1205,7 +1205,12 @@ def _pull_args_from_config(args: List[str]) -> List[str]: # followed by rest of cli args. # maintaining this order will enforce the precedence # of cli > config > defaults - args = [args[0]] + args[1:index] + config_args + args[index + 2:] + + # Note if served-model-name is specified at the end of yaml file, + # the positional argument model_tag will not be parsed normally, + # you would see error like the following + # vllm serve: error: the following arguments are required: model_tag + args = [args[0]] + config_args + args[1:index] + args[index + 2:] return args From 1f7a56301cffae510e8bab4622951dcc34d77c70 Mon Sep 17 00:00:00 2001 From: Imss27 <1529767689@qq.com> Date: Mon, 30 Sep 2024 10:54:04 -0700 Subject: [PATCH 04/13] Formatted --- vllm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/utils.py b/vllm/utils.py index f38a7b075677..b5571f6d59d0 100644 --- a/vllm/utils.py +++ b/vllm/utils.py @@ -1205,7 +1205,7 @@ def _pull_args_from_config(args: List[str]) -> List[str]: # followed by rest of cli args. # maintaining this order will enforce the precedence # of cli > config > defaults - + # Note if served-model-name is specified at the end of yaml file, # the positional argument model_tag will not be parsed normally, # you would see error like the following From 6efd7432c3fb504a69bc11500bf18d73a8f3c858 Mon Sep 17 00:00:00 2001 From: Imss27 <1529767689@qq.com> Date: Mon, 30 Sep 2024 10:58:38 -0700 Subject: [PATCH 05/13] Deleted commit_id.py --- vllm/commit_id.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 vllm/commit_id.py diff --git a/vllm/commit_id.py b/vllm/commit_id.py deleted file mode 100644 index 732331d9359b..000000000000 --- a/vllm/commit_id.py +++ /dev/null @@ -1 +0,0 @@ -__commit__ = "cca61642e0484212e6cd78b35b4789afed8d19c6" From 17604bd1503f3fb7170552b460a7526f0b9d4939 Mon Sep 17 00:00:00 2001 From: Imss27 <1529767689@qq.com> Date: Mon, 30 Sep 2024 12:44:37 -0700 Subject: [PATCH 06/13] Updated docs to note the error --- docs/source/serving/openai_compatible_server.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/serving/openai_compatible_server.md b/docs/source/serving/openai_compatible_server.md index e0eba7f09bd6..0591194e3acc 100644 --- a/docs/source/serving/openai_compatible_server.md +++ b/docs/source/serving/openai_compatible_server.md @@ -140,9 +140,12 @@ $ vllm serve SOME_MODEL --config config.yaml ``` --- **NOTE** -In case an argument is supplied using command line and the config file, the value from the commandline will take precedence. +In case an argument is supplied simultaneously using command line and the config file, the value from the commandline will take precedence. The order of priorities is `command line > config file values > defaults`. +Try to avoid specifying `served-model-name` at the end of the config file, the positional argument `model_tag` will not be parsed normally if you do so, and you would likely see an error as below: +`vllm serve: error: the following arguments are required: model_tag` + --- ## Tool calling in the chat completion API From b1d451da2177e48117b24e1d021ae86181824199 Mon Sep 17 00:00:00 2001 From: Imss27 <1529767689@qq.com> Date: Mon, 30 Sep 2024 12:46:49 -0700 Subject: [PATCH 07/13] Deleted comment in utils.py --- vllm/utils.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/vllm/utils.py b/vllm/utils.py index b5571f6d59d0..a025c3c40a43 100644 --- a/vllm/utils.py +++ b/vllm/utils.py @@ -1205,11 +1205,6 @@ def _pull_args_from_config(args: List[str]) -> List[str]: # followed by rest of cli args. # maintaining this order will enforce the precedence # of cli > config > defaults - - # Note if served-model-name is specified at the end of yaml file, - # the positional argument model_tag will not be parsed normally, - # you would see error like the following - # vllm serve: error: the following arguments are required: model_tag args = [args[0]] + config_args + args[1:index] + args[index + 2:] return args From 255df7d5f01615fd530b8bc0956a50ba3a4c5b90 Mon Sep 17 00:00:00 2001 From: Imss27 <1529767689@qq.com> Date: Tue, 1 Oct 2024 00:10:39 -0700 Subject: [PATCH 08/13] add dummy arg for model_tag --- docs/source/serving/openai_compatible_server.md | 3 --- vllm/utils.py | 11 ++++++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/source/serving/openai_compatible_server.md b/docs/source/serving/openai_compatible_server.md index 0591194e3acc..1a388bafd7c7 100644 --- a/docs/source/serving/openai_compatible_server.md +++ b/docs/source/serving/openai_compatible_server.md @@ -143,9 +143,6 @@ $ vllm serve SOME_MODEL --config config.yaml In case an argument is supplied simultaneously using command line and the config file, the value from the commandline will take precedence. The order of priorities is `command line > config file values > defaults`. -Try to avoid specifying `served-model-name` at the end of the config file, the positional argument `model_tag` will not be parsed normally if you do so, and you would likely see an error as below: -`vllm serve: error: the following arguments are required: model_tag` - --- ## Tool calling in the chat completion API diff --git a/vllm/utils.py b/vllm/utils.py index a025c3c40a43..dc014f80ec3a 100644 --- a/vllm/utils.py +++ b/vllm/utils.py @@ -1150,8 +1150,12 @@ def parse_args(self, args=None, namespace=None): arg[len('--'):].replace('_', '-')) else: processed_args.append(arg) - - return super().parse_args(processed_args, namespace) + args, argv = super().parse_known_args(processed_args, namespace) + # Raise error if there are more unknown args than --dummy-tag + if len(argv) > 1: + msg = 'unrecognized arguments: {}'.format(' '.join(argv)) + raise argparse.ArgumentError(None, msg) + return args @staticmethod def _pull_args_from_config(args: List[str]) -> List[str]: @@ -1205,7 +1209,8 @@ def _pull_args_from_config(args: List[str]) -> List[str]: # followed by rest of cli args. # maintaining this order will enforce the precedence # of cli > config > defaults - args = [args[0]] + config_args + args[1:index] + args[index + 2:] + args = [args[0]] + config_args + ['--dummy-tag'] + \ + args[1:index] + args[index + 2:] return args From 7e991596a4d1f4f9cd5b95467eb71439d5b2dcea Mon Sep 17 00:00:00 2001 From: Imss27 <1529767689@qq.com> Date: Tue, 1 Oct 2024 10:16:08 -0700 Subject: [PATCH 09/13] Add model_tag in front of config_args --- vllm/utils.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/vllm/utils.py b/vllm/utils.py index dc014f80ec3a..570222750b17 100644 --- a/vllm/utils.py +++ b/vllm/utils.py @@ -1138,6 +1138,12 @@ def parse_args(self, args=None, namespace=None): args = FlexibleArgumentParser._pull_args_from_config(args) # Convert underscores to dashes and vice versa in argument names + processed_args = self._format_args(args) + + return super().parse_args(processed_args, namespace) + + def _format_args(self, args: List[str]) -> List[str]: + """Convert underscores to dashes and vice versa in argument names""" processed_args = [] for arg in args: if arg.startswith('--'): @@ -1150,13 +1156,9 @@ def parse_args(self, args=None, namespace=None): arg[len('--'):].replace('_', '-')) else: processed_args.append(arg) - args, argv = super().parse_known_args(processed_args, namespace) - # Raise error if there are more unknown args than --dummy-tag - if len(argv) > 1: - msg = 'unrecognized arguments: {}'.format(' '.join(argv)) - raise argparse.ArgumentError(None, msg) - return args + return processed_args + @staticmethod def _pull_args_from_config(args: List[str]) -> List[str]: """Method to pull arguments specified in the config file @@ -1205,12 +1207,12 @@ def _pull_args_from_config(args: List[str]) -> List[str]: config_args = FlexibleArgumentParser._load_config_file(file_path) # 0th index is for {serve,chat,complete} + # followed by model_tag # followed by config args # followed by rest of cli args. # maintaining this order will enforce the precedence - # of cli > config > defaults - args = [args[0]] + config_args + ['--dummy-tag'] + \ - args[1:index] + args[index + 2:] + # of cli > config > defaults + args = [args[0]] + [args[1]] + config_args + args[2:index] + args[index+2:] return args From 4e05e6f4bf398778be5a06d8797025916cdee52c Mon Sep 17 00:00:00 2001 From: Imss27 <1529767689@qq.com> Date: Tue, 1 Oct 2024 10:24:16 -0700 Subject: [PATCH 10/13] Unwrapped encapsulation and fix format --- vllm/utils.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/vllm/utils.py b/vllm/utils.py index 570222750b17..f57c0fdd729a 100644 --- a/vllm/utils.py +++ b/vllm/utils.py @@ -1138,12 +1138,6 @@ def parse_args(self, args=None, namespace=None): args = FlexibleArgumentParser._pull_args_from_config(args) # Convert underscores to dashes and vice versa in argument names - processed_args = self._format_args(args) - - return super().parse_args(processed_args, namespace) - - def _format_args(self, args: List[str]) -> List[str]: - """Convert underscores to dashes and vice versa in argument names""" processed_args = [] for arg in args: if arg.startswith('--'): @@ -1157,8 +1151,8 @@ def _format_args(self, args: List[str]) -> List[str]: else: processed_args.append(arg) - return processed_args - + return super().parse_args(processed_args, namespace) + @staticmethod def _pull_args_from_config(args: List[str]) -> List[str]: """Method to pull arguments specified in the config file @@ -1212,7 +1206,8 @@ def _pull_args_from_config(args: List[str]) -> List[str]: # followed by rest of cli args. # maintaining this order will enforce the precedence # of cli > config > defaults - args = [args[0]] + [args[1]] + config_args + args[2:index] + args[index+2:] + args = [args[0]] + [args[1]] + config_args + \ + args[2:index] + args[index+2:] return args From 65f356aceade3a608a5525ee2b3482d44f5ae96e Mon Sep 17 00:00:00 2001 From: Imss27 <1529767689@qq.com> Date: Tue, 1 Oct 2024 10:28:11 -0700 Subject: [PATCH 11/13] fix format --- vllm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/utils.py b/vllm/utils.py index f57c0fdd729a..f1c9e3459e38 100644 --- a/vllm/utils.py +++ b/vllm/utils.py @@ -1205,7 +1205,7 @@ def _pull_args_from_config(args: List[str]) -> List[str]: # followed by config args # followed by rest of cli args. # maintaining this order will enforce the precedence - # of cli > config > defaults + # of cli > config > defaults args = [args[0]] + [args[1]] + config_args + \ args[2:index] + args[index+2:] From cbb92c7cf4a47f45e47bb07f78c363bfe03efbc4 Mon Sep 17 00:00:00 2001 From: Imss27 <1529767689@qq.com> Date: Tue, 1 Oct 2024 23:25:18 -0700 Subject: [PATCH 12/13] Deal with serve separately --- vllm/utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vllm/utils.py b/vllm/utils.py index f1c9e3459e38..b6e631d6ff0b 100644 --- a/vllm/utils.py +++ b/vllm/utils.py @@ -1201,13 +1201,16 @@ def _pull_args_from_config(args: List[str]) -> List[str]: config_args = FlexibleArgumentParser._load_config_file(file_path) # 0th index is for {serve,chat,complete} - # followed by model_tag + # followed by model_tag (only for serve) # followed by config args # followed by rest of cli args. # maintaining this order will enforce the precedence # of cli > config > defaults - args = [args[0]] + [args[1]] + config_args + \ - args[2:index] + args[index+2:] + if args[0] == "serve": + args = [args[0]] + [args[1] + ] + config_args + args[2:index] + args[index + 2:] + else: + args = [args[0]] + config_args + args[1:index] + args[index + 2:] return args From 937aaea7101976218ca02b089bc6ef9cb948b6f1 Mon Sep 17 00:00:00 2001 From: Imss27 <1529767689@qq.com> Date: Wed, 2 Oct 2024 00:45:54 -0700 Subject: [PATCH 13/13] Improved unit tests and deal with serve separately --- tests/data/test_config.yaml | 1 + tests/test_utils.py | 30 +++++++++++++++++++++++------- vllm/utils.py | 9 +++++++-- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/tests/data/test_config.yaml b/tests/data/test_config.yaml index 20d499624de2..42f4f6f7bb99 100644 --- a/tests/data/test_config.yaml +++ b/tests/data/test_config.yaml @@ -1,2 +1,3 @@ port: 12312 +served_model_name: mymodel tensor_parallel_size: 2 diff --git a/tests/test_utils.py b/tests/test_utils.py index c7cb663068c0..f3017a8582ea 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -136,6 +136,8 @@ def parser(): def parser_with_config(): parser = FlexibleArgumentParser() parser.add_argument('serve') + parser.add_argument('model_tag') + parser.add_argument('--served-model-name', type=str) parser.add_argument('--config', type=str) parser.add_argument('--port', type=int) parser.add_argument('--tensor-parallel-size', type=int) @@ -190,33 +192,47 @@ def test_missing_required_argument(parser): def test_cli_override_to_config(parser_with_config): args = parser_with_config.parse_args([ - 'serve', '--config', './data/test_config.yaml', + 'serve', 'mymodel', '--config', './data/test_config.yaml', '--tensor-parallel-size', '3' ]) assert args.tensor_parallel_size == 3 args = parser_with_config.parse_args([ - 'serve', '--tensor-parallel-size', '3', '--config', + 'serve', 'mymodel', '--tensor-parallel-size', '3', '--config', './data/test_config.yaml' ]) assert args.tensor_parallel_size == 3 + assert args.port == 12312 + args = parser_with_config.parse_args([ + 'serve', 'mymodel', '--tensor-parallel-size', '3', '--config', + './data/test_config.yaml', '--port', '666' + ]) + assert args.tensor_parallel_size == 3 + assert args.port == 666 def test_config_args(parser_with_config): args = parser_with_config.parse_args( - ['serve', '--config', './data/test_config.yaml']) + ['serve', 'mymodel', '--config', './data/test_config.yaml']) assert args.tensor_parallel_size == 2 def test_config_file(parser_with_config): with pytest.raises(FileNotFoundError): - parser_with_config.parse_args(['serve', '--config', 'test_config.yml']) + parser_with_config.parse_args( + ['serve', 'mymodel', '--config', 'test_config.yml']) with pytest.raises(ValueError): parser_with_config.parse_args( - ['serve', '--config', './data/test_config.json']) + ['serve', 'mymodel', '--config', './data/test_config.json']) with pytest.raises(ValueError): parser_with_config.parse_args([ - 'serve', '--tensor-parallel-size', '3', '--config', '--batch-size', - '32' + 'serve', 'mymodel', '--tensor-parallel-size', '3', '--config', + '--batch-size', '32' ]) + + +def test_no_model_tag(parser_with_config): + with pytest.raises(ValueError): + parser_with_config.parse_args( + ['serve', '--config', './data/test_config.yaml']) diff --git a/vllm/utils.py b/vllm/utils.py index b6e631d6ff0b..197584867d8b 100644 --- a/vllm/utils.py +++ b/vllm/utils.py @@ -1207,8 +1207,13 @@ def _pull_args_from_config(args: List[str]) -> List[str]: # maintaining this order will enforce the precedence # of cli > config > defaults if args[0] == "serve": - args = [args[0]] + [args[1] - ] + config_args + args[2:index] + args[index + 2:] + if index == 1: + raise ValueError( + "No model_tag specified! Please check your command-line" + " arguments.") + args = [args[0]] + [ + args[1] + ] + config_args + args[2:index] + args[index + 2:] else: args = [args[0]] + config_args + args[1:index] + args[index + 2:]