From dddba213f73f8feeb36b6c5f7855421c70e20b25 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Fri, 14 Feb 2025 17:51:23 +0000 Subject: [PATCH 1/3] [core] add `extra_args` to `SamplingParams` Signed-off-by: Aviv Keshet --- vllm/sampling_params.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 97f9e2129573..5ef8de1bd441 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -166,6 +166,8 @@ class SamplingParams( allowed_token_ids: If provided, the engine will construct a logits processor which only retains scores for the given token ids. Defaults to None. + extra_args: Arbitrary additional args, that can be used by custom sampling + implementations. Not used by any in-tree sampling implementations. """ n: int = 1 @@ -209,6 +211,7 @@ class SamplingParams( guided_decoding: Optional[GuidedDecodingParams] = None logit_bias: Optional[Dict[int, float]] = None allowed_token_ids: Optional[List[int]] = None + extra_args: Any = None @staticmethod def from_optional( @@ -241,6 +244,7 @@ def from_optional( guided_decoding: Optional[GuidedDecodingParams] = None, logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]] = None, allowed_token_ids: Optional[List[int]] = None, + extra_args: Any = None, ) -> "SamplingParams": if logit_bias is not None: logit_bias = { @@ -280,6 +284,7 @@ def from_optional( guided_decoding=guided_decoding, logit_bias=logit_bias, allowed_token_ids=allowed_token_ids, + extra_args=extra_args, ) def __post_init__(self) -> None: @@ -489,7 +494,8 @@ def __repr__(self) -> str: "spaces_between_special_tokens=" f"{self.spaces_between_special_tokens}, " f"truncate_prompt_tokens={self.truncate_prompt_tokens}, " - f"guided_decoding={self.guided_decoding})") + f"guided_decoding={self.guided_decoding}, " + f"extra_args={self.extra_args})") class BeamSearchParams( From 2a60ce2d35e62c2c2361721621b3bf779c533a58 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Fri, 21 Feb 2025 06:38:38 +0000 Subject: [PATCH 2/3] [core] fix line length Signed-off-by: Aviv Keshet --- vllm/sampling_params.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 5ef8de1bd441..e21dcce976d2 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -166,8 +166,9 @@ class SamplingParams( allowed_token_ids: If provided, the engine will construct a logits processor which only retains scores for the given token ids. Defaults to None. - extra_args: Arbitrary additional args, that can be used by custom sampling - implementations. Not used by any in-tree sampling implementations. + extra_args: Arbitrary additional args, that can be used by custom + sampling implementations. Not used by any in-tree sampling + implementations. """ n: int = 1 From 1254f070b1068b51aa66c22bc76afc3825827118 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Thu, 27 Feb 2025 19:28:46 +0000 Subject: [PATCH 3/3] [core] use `Optional[Dict[str, Any]]` Signed-off-by: Aviv Keshet --- vllm/sampling_params.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 50aff26f5def..b60e14d6fad2 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -231,7 +231,7 @@ class SamplingParams( guided_decoding: Optional[GuidedDecodingParams] = None logit_bias: Optional[Dict[int, float]] = None allowed_token_ids: Optional[List[int]] = None - extra_args: Any = None + extra_args: Optional[Dict[str, Any]] = None @staticmethod def from_optional( @@ -264,7 +264,7 @@ def from_optional( guided_decoding: Optional[GuidedDecodingParams] = None, logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]] = None, allowed_token_ids: Optional[List[int]] = None, - extra_args: Any = None, + extra_args: Optional[Dict[str, Any]] = None, ) -> "SamplingParams": if logit_bias is not None: # Convert token_id to integer