From 8242dbd2e3f4316828eb48c18f3113a05895857f Mon Sep 17 00:00:00 2001 From: Dmitrii Cherkasov Date: Sat, 8 Mar 2025 22:18:41 -0800 Subject: [PATCH] Refactor evaluation service config to remove redundant information --- .../evaluation/evaluation_service_config.py | 186 +----------------- .../evaluation_service_model_config.py | 8 - .../test_data/config/evaluation_config.json | 106 ---------- ...evaluation_config_with_default_params.json | 11 -- .../aqua/test_evaluation_service_config.py | 83 +------- 5 files changed, 6 insertions(+), 388 deletions(-) delete mode 100644 ads/aqua/config/evaluation/evaluation_service_model_config.py diff --git a/ads/aqua/config/evaluation/evaluation_service_config.py b/ads/aqua/config/evaluation/evaluation_service_config.py index 67be9203e..8edaf974b 100644 --- a/ads/aqua/config/evaluation/evaluation_service_config.py +++ b/ads/aqua/config/evaluation/evaluation_service_config.py @@ -1,9 +1,8 @@ #!/usr/bin/env python -# Copyright (c) 2024 Oracle and/or its affiliates. +# Copyright (c) 2024, 2025 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ -from copy import deepcopy from typing import Any, Dict, List, Optional from pydantic import Field @@ -11,139 +10,6 @@ from ads.aqua.config.utils.serializer import Serializable -class ModelParamsOverrides(Serializable): - """Defines overrides for model parameters, including exclusions and additional inclusions.""" - - exclude: Optional[List[str]] = Field(default_factory=list) - include: Optional[Dict[str, Any]] = Field(default_factory=dict) - - class Config: - extra = "ignore" - - -class ModelParamsVersion(Serializable): - """Handles version-specific model parameter overrides.""" - - overrides: Optional[ModelParamsOverrides] = Field( - default_factory=ModelParamsOverrides - ) - - class Config: - extra = "ignore" - - -class ModelParamsContainer(Serializable): - """Represents a container's model configuration, including tasks, defaults, and versions.""" - - name: Optional[str] = None - default: Optional[Dict[str, Any]] = Field(default_factory=dict) - versions: Optional[Dict[str, ModelParamsVersion]] = Field(default_factory=dict) - - class Config: - extra = "ignore" - - -class InferenceParams(Serializable): - """Contains inference-related parameters with defaults.""" - - class Config: - extra = "allow" - - -class InferenceContainer(Serializable): - """Represents the inference parameters specific to a container.""" - - name: Optional[str] = None - params: Optional[Dict[str, Any]] = Field(default_factory=dict) - - class Config: - extra = "ignore" - - -class ReportParams(Serializable): - """Handles the report-related parameters.""" - - default: Optional[Dict[str, Any]] = Field(default_factory=dict) - - class Config: - extra = "ignore" - - -class InferenceParamsConfig(Serializable): - """Combines default inference parameters with container-specific configurations.""" - - default: Optional[InferenceParams] = Field(default_factory=InferenceParams) - containers: Optional[List[InferenceContainer]] = Field(default_factory=list) - - def get_merged_params(self, container_name: str) -> InferenceParams: - """ - Merges default inference params with those specific to the given container. - - Parameters - ---------- - container_name (str): The name of the container. - - Returns - ------- - InferenceParams: The merged inference parameters. - """ - merged_params = self.default.to_dict() - for containers in self.containers: - if containers.name.lower() == container_name.lower(): - merged_params.update(containers.params or {}) - break - return InferenceParams(**merged_params) - - class Config: - extra = "ignore" - - -class InferenceModelParamsConfig(Serializable): - """Encapsulates the model parameters for different containers.""" - - default: Optional[Dict[str, Any]] = Field(default_factory=dict) - containers: Optional[List[ModelParamsContainer]] = Field(default_factory=list) - - def get_merged_model_params( - self, - container_name: str, - version: Optional[str] = None, - ) -> Dict[str, Any]: - """ - Gets the model parameters for a given container, version, - merged with the defaults. - - Parameters - ---------- - container_name (str): The name of the container. - version (Optional[str]): The specific version of the container. - - Returns - ------- - Dict[str, Any]: The merged model parameters. - """ - params = deepcopy(self.default) - - for container in self.containers: - if container.name.lower() == container_name.lower(): - params.update(container.default) - - if version and version in container.versions: - version_overrides = container.versions[version].overrides - if version_overrides: - if version_overrides.include: - params.update(version_overrides.include) - if version_overrides.exclude: - for key in version_overrides.exclude: - params.pop(key, None) - break - - return params - - class Config: - extra = "ignore" - - class ShapeFilterConfig(Serializable): """Represents the filtering options for a specific shape.""" @@ -151,7 +17,7 @@ class ShapeFilterConfig(Serializable): evaluation_target: Optional[List[str]] = Field(default_factory=list) class Config: - extra = "ignore" + extra = "allow" class ShapeConfig(Serializable): @@ -178,7 +44,7 @@ class MetricConfig(Serializable): tags: Optional[List[str]] = Field(default_factory=list) class Config: - extra = "ignore" + extra = "allow" class ModelParamsConfig(Serializable): @@ -223,7 +89,7 @@ def search_shapes( ] class Config: - extra = "ignore" + extra = "allow" protected_namespaces = () @@ -235,49 +101,7 @@ class EvaluationServiceConfig(Serializable): version: Optional[str] = "1.0" kind: Optional[str] = "evaluation_service_config" - report_params: Optional[ReportParams] = Field(default_factory=ReportParams) - inference_params: Optional[InferenceParamsConfig] = Field( - default_factory=InferenceParamsConfig - ) - inference_model_params: Optional[InferenceModelParamsConfig] = Field( - default_factory=InferenceModelParamsConfig - ) ui_config: Optional[UIConfig] = Field(default_factory=UIConfig) - def get_merged_inference_params(self, container_name: str) -> InferenceParams: - """ - Merges default inference params with those specific to the given container. - - Params - ------ - container_name (str): The name of the container. - - Returns - ------- - InferenceParams: The merged inference parameters. - """ - return self.inference_params.get_merged_params(container_name=container_name) - - def get_merged_inference_model_params( - self, - container_name: str, - version: Optional[str] = None, - ) -> Dict[str, Any]: - """ - Gets the model parameters for a given container, version, and task, merged with the defaults. - - Parameters - ---------- - container_name (str): The name of the container. - version (Optional[str]): The specific version of the container. - - Returns - ------- - Dict[str, Any]: The merged model parameters. - """ - return self.inference_model_params.get_merged_model_params( - container_name=container_name, version=version - ) - class Config: - extra = "ignore" + extra = "allow" diff --git a/ads/aqua/config/evaluation/evaluation_service_model_config.py b/ads/aqua/config/evaluation/evaluation_service_model_config.py deleted file mode 100644 index 911fe3176..000000000 --- a/ads/aqua/config/evaluation/evaluation_service_model_config.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2024 Oracle and/or its affiliates. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ - -""" -This serves as a future template for implementing model-specific evaluation configurations. -""" diff --git a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json index f108073f0..3dacdf7df 100644 --- a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json +++ b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json @@ -1,111 +1,5 @@ { - "inference_model_params": { - "containers": [ - { - "default": { - "add_generation_prompt": false - }, - "name": "odsc-vllm-serving", - "versions": { - "0.5.1": { - "overrides": { - "exclude": [ - "max_tokens", - "frequency_penalty" - ], - "include": { - "some_other_param": "some_other_param_value" - } - } - }, - "0.5.3.post1": { - "overrides": { - "exclude": [ - "add_generation_prompt" - ], - "include": {} - } - } - } - }, - { - "default": { - "add_generation_prompt": false - }, - "name": "odsc-tgi-serving", - "versions": { - "2.0.1.4": { - "overrides": { - "exclude": [ - "max_tokens", - "frequency_penalty" - ], - "include": { - "some_other_param": "some_other_param_value" - } - } - } - } - }, - { - "default": { - "add_generation_prompt": false - }, - "name": "odsc-llama-cpp-serving", - "versions": { - "0.2.78.0": { - "overrides": { - "exclude": [], - "include": {} - } - } - } - } - ], - "default": { - "add_generation_prompt": false, - "frequency_penalty": 0.0, - "max_tokens": 500, - "model": "odsc-llm", - "presence_penalty": 0.0, - "some_default_param": "some_default_param", - "stop": [], - "temperature": 0.7, - "top_k": 50, - "top_p": 0.9 - } - }, - "inference_params": { - "containers": [ - { - "name": "odsc-vllm-serving", - "params": {} - }, - { - "name": "odsc-tgi-serving", - "params": {} - }, - { - "name": "odsc-llama-cpp-serving", - "params": { - "inference_delay": 1, - "inference_max_threads": 1 - } - } - ], - "default": { - "inference_backoff_factor": 3, - "inference_delay": 0, - "inference_max_threads": 10, - "inference_retries": 3, - "inference_rps": 25, - "inference_timeout": 120 - } - }, "kind": "evaluation_service_config", - "report_params": { - "default": {} - }, "ui_config": { "metrics": [ { diff --git a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json index 237139476..e52e0ad67 100644 --- a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json +++ b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json @@ -1,16 +1,5 @@ { - "inference_model_params": { - "containers": [], - "default": {} - }, - "inference_params": { - "containers": [], - "default": {} - }, "kind": "evaluation_service_config", - "report_params": { - "default": {} - }, "ui_config": { "metrics": [], "model_params": { diff --git a/tests/unitary/with_extras/aqua/test_evaluation_service_config.py b/tests/unitary/with_extras/aqua/test_evaluation_service_config.py index a090f6ad2..673720ebf 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation_service_config.py +++ b/tests/unitary/with_extras/aqua/test_evaluation_service_config.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2024 Oracle and/or its affiliates. +# Copyright (c) 2024, 2025 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ @@ -46,87 +46,6 @@ def test_read_config(self): assert self.mock_config.to_dict() == expected_config - @pytest.mark.parametrize( - "container_name, extra_params", - [ - ("odsc-vllm-serving", {}), - ("odsc-vllm-serving", {}), - ("odsc-tgi-serving", {}), - ( - "odsc-llama-cpp-serving", - {"inference_max_threads": 1, "inference_delay": 1}, - ), - ("none-exist", {}), - ], - ) - def test_get_merged_inference_params(self, container_name, extra_params): - """Tests merging default inference params with those specific to the given framework.""" - - test_result = self.mock_config.get_merged_inference_params( - container_name=container_name - ) - expected_result = { - "inference_rps": 25, - "inference_timeout": 120, - "inference_max_threads": 10, - "inference_retries": 3, - "inference_backoff_factor": 3.0, - "inference_delay": 0.0, - } - expected_result.update(extra_params) - - assert test_result.to_dict() == expected_result - - @pytest.mark.parametrize( - "container_name, version, exclude, include", - [ - ("odsc-vllm-serving", "0.5.3.post1", ["add_generation_prompt"], {}), - ( - "odsc-vllm-serving", - "0.5.1", - ["max_tokens", "frequency_penalty"], - {"some_other_param": "some_other_param_value"}, - ), - ("odsc-vllm-serving", "none-exist", [], {}), - ("odsc-tgi-serving", None, [], {}), - ("odsc-tgi-serving", "none-exist", [], {}), - ( - "odsc-tgi-serving", - "2.0.1.4", - ["max_tokens", "frequency_penalty"], - {"some_other_param": "some_other_param_value"}, - ), - ("odsc-llama-cpp-serving", "0.2.78.0", [], {}), - ("none-exist", "none-exist", [], {}), - ], - ) - def test_get_merged_inference_model_params( - self, container_name, version, exclude, include - ): - expected_result = {"some_default_param": "some_default_param"} - expected_result.update( - { - "model": "odsc-llm", - "add_generation_prompt": False, - "max_tokens": 500, - "temperature": 0.7, - "top_p": 0.9, - "top_k": 50, - "presence_penalty": 0.0, - "frequency_penalty": 0.0, - "stop": [], - } - ) - expected_result.update(include) - for key in exclude: - expected_result.pop(key, None) - - test_result = self.mock_config.get_merged_inference_model_params( - container_name=container_name, version=version - ) - - assert test_result == expected_result - @pytest.mark.parametrize( "evaluation_container, evaluation_target, shapes_found", [