diff --git a/autosklearn/__version__.py b/autosklearn/__version__.py index 957aad4c9b..ed846e6f88 100644 --- a/autosklearn/__version__.py +++ b/autosklearn/__version__.py @@ -1,4 +1,4 @@ """Version information.""" # The following line *must* be the last in the module, exactly as formatted: -__version__ = "0.14.1" +__version__ = "0.14.2" diff --git a/autosklearn/evaluation/__init__.py b/autosklearn/evaluation/__init__.py index 878fdaf069..589535d085 100644 --- a/autosklearn/evaluation/__init__.py +++ b/autosklearn/evaluation/__init__.py @@ -24,6 +24,7 @@ import autosklearn.evaluation.train_evaluator import autosklearn.evaluation.test_evaluator import autosklearn.evaluation.util +import autosklearn.pipeline.components from autosklearn.evaluation.train_evaluator import TYPE_ADDITIONAL_INFO from autosklearn.util.backend import Backend from autosklearn.util.logging_ import PickableLoggerAdapter, get_named_client_logger @@ -336,6 +337,7 @@ def run( init_params=init_params, budget=budget, budget_type=self.budget_type, + additional_components=autosklearn.pipeline.components.base._addons, ) if self.resampling_strategy != 'test': diff --git a/autosklearn/evaluation/abstract_evaluator.py b/autosklearn/evaluation/abstract_evaluator.py index 3b81c5069d..2e398b00ae 100644 --- a/autosklearn/evaluation/abstract_evaluator.py +++ b/autosklearn/evaluation/abstract_evaluator.py @@ -16,6 +16,7 @@ import autosklearn.pipeline.classification import autosklearn.pipeline.regression +from autosklearn.pipeline.components.base import ThirdPartyComponents, _addons from autosklearn.constants import ( CLASSIFICATION_TASKS, REGRESSION_TASKS, @@ -181,6 +182,7 @@ def __init__( backend: Backend, queue: multiprocessing.Queue, metric: Scorer, + additional_components: Dict[str, ThirdPartyComponents], port: Optional[int], configuration: Optional[Union[int, Configuration]] = None, scoring_functions: Optional[List[Scorer]] = None, @@ -268,6 +270,15 @@ def __init__( self.budget = budget self.budget_type = budget_type + # Add 3rd-party components to the list of 3rd-party components in case this wasn't done + # before (this happens if we run in parallel and the components are only passed to the + # AbstractEvaluator via the TAE and are not there yet because the worker is in its own + # process). + for key in additional_components: + for component_name, component in additional_components[key].components.items(): + if component_name not in _addons[key].components: + _addons[key].add_component(component) + # Please mypy to prevent not defined attr self.model = self._get_model() diff --git a/autosklearn/evaluation/test_evaluator.py b/autosklearn/evaluation/test_evaluator.py index 94199e47f9..e83edb0682 100644 --- a/autosklearn/evaluation/test_evaluator.py +++ b/autosklearn/evaluation/test_evaluator.py @@ -12,6 +12,7 @@ AbstractEvaluator, _fit_and_suppress_warnings, ) +from autosklearn.pipeline.components.base import ThirdPartyComponents from autosklearn.metrics import calculate_loss, Scorer from autosklearn.util.backend import Backend @@ -29,6 +30,7 @@ def __init__( backend: Backend, queue: multiprocessing.Queue, metric: Scorer, + additional_components: Dict[str, ThirdPartyComponents], port: Optional[int], configuration: Optional[Union[int, Configuration]] = None, scoring_functions: Optional[List[Scorer]] = None, @@ -44,6 +46,7 @@ def __init__( port=port, configuration=configuration, metric=metric, + additional_components=additional_components, scoring_functions=scoring_functions, seed=seed, output_y_hat_optimization=False, @@ -120,6 +123,7 @@ def eval_t( exclude: Optional[List[str]], disable_file_output: bool, port: Optional[int], + additional_components: Dict[str, ThirdPartyComponents], init_params: Optional[Dict[str, Any]] = None, budget: Optional[float] = None, budget_type: Optional[str] = None, @@ -131,6 +135,7 @@ def eval_t( scoring_functions=scoring_functions, include=include, exclude=exclude, disable_file_output=disable_file_output, - init_params=init_params) + additional_components=additional_components, + init_params=init_params,) evaluator.fit_predict_and_loss() diff --git a/autosklearn/evaluation/train_evaluator.py b/autosklearn/evaluation/train_evaluator.py index 448183da4b..51b433153d 100644 --- a/autosklearn/evaluation/train_evaluator.py +++ b/autosklearn/evaluation/train_evaluator.py @@ -35,7 +35,7 @@ SUPPORTED_TARGET_TYPES, ) from autosklearn.pipeline.base import PIPELINE_DATA_DTYPE -from autosklearn.pipeline.components.base import IterativeComponent +from autosklearn.pipeline.components.base import IterativeComponent, ThirdPartyComponents from autosklearn.metrics import Scorer from autosklearn.util.backend import Backend from autosklearn.util.logging_ import PicklableClientLogger @@ -160,6 +160,7 @@ def __init__( backend: Backend, queue: multiprocessing.Queue, metric: Scorer, + additional_components: Dict[str, ThirdPartyComponents], port: Optional[int], configuration: Optional[Union[int, Configuration]] = None, scoring_functions: Optional[List[Scorer]] = None, @@ -184,6 +185,7 @@ def __init__( port=port, configuration=configuration, metric=metric, + additional_components=additional_components, scoring_functions=scoring_functions, seed=seed, output_y_hat_optimization=output_y_hat_optimization, @@ -1163,6 +1165,7 @@ def eval_holdout( exclude: Optional[List[str]], disable_file_output: bool, port: Optional[int], + additional_components: Dict[str, ThirdPartyComponents], init_params: Optional[Dict[str, Any]] = None, budget: Optional[float] = 100.0, budget_type: Optional[str] = None, @@ -1183,6 +1186,7 @@ def eval_holdout( include=include, exclude=exclude, disable_file_output=disable_file_output, + additional_components=additional_components, init_params=init_params, budget=budget, budget_type=budget_type, @@ -1206,6 +1210,7 @@ def eval_iterative_holdout( exclude: Optional[List[str]], disable_file_output: bool, port: Optional[int], + additional_components: Dict[str, ThirdPartyComponents], init_params: Optional[Dict[str, Any]] = None, budget: Optional[float] = 100.0, budget_type: Optional[str] = None, @@ -1227,6 +1232,7 @@ def eval_iterative_holdout( instance=instance, disable_file_output=disable_file_output, iterative=True, + additional_components=additional_components, init_params=init_params, budget=budget, budget_type=budget_type @@ -1249,6 +1255,7 @@ def eval_partial_cv( exclude: Optional[List[str]], disable_file_output: bool, port: Optional[int], + additional_components: Dict[str, ThirdPartyComponents], init_params: Optional[Dict[str, Any]] = None, budget: Optional[float] = None, budget_type: Optional[str] = None, @@ -1274,6 +1281,7 @@ def eval_partial_cv( include=include, exclude=exclude, disable_file_output=disable_file_output, + additional_components=additional_components, init_params=init_params, budget=budget, budget_type=budget_type, @@ -1298,6 +1306,7 @@ def eval_partial_cv_iterative( exclude: Optional[List[str]], disable_file_output: bool, port: Optional[int], + additional_components: Dict[str, ThirdPartyComponents], init_params: Optional[Dict[str, Any]] = None, budget: Optional[float] = None, budget_type: Optional[str] = None, @@ -1321,6 +1330,7 @@ def eval_partial_cv_iterative( exclude=exclude, disable_file_output=disable_file_output, iterative=True, + additional_components=additional_components, init_params=init_params, ) @@ -1342,6 +1352,7 @@ def eval_cv( exclude: Optional[List[str]], disable_file_output: bool, port: Optional[int], + additional_components: Dict[str, ThirdPartyComponents], init_params: Optional[Dict[str, Any]] = None, budget: Optional[float] = None, budget_type: Optional[str] = None, @@ -1362,6 +1373,7 @@ def eval_cv( include=include, exclude=exclude, disable_file_output=disable_file_output, + additional_components=additional_components, init_params=init_params, budget=budget, budget_type=budget_type, @@ -1386,6 +1398,7 @@ def eval_iterative_cv( exclude: Optional[List[str]], disable_file_output: bool, port: Optional[int], + additional_components: Dict[str, ThirdPartyComponents], init_params: Optional[Dict[str, Any]] = None, budget: Optional[float] = None, budget_type: Optional[str] = None, @@ -1406,6 +1419,7 @@ def eval_iterative_cv( exclude=exclude, disable_file_output=disable_file_output, port=port, + additional_components=additional_components, init_params=init_params, budget=budget, budget_type=budget_type, diff --git a/autosklearn/pipeline/components/base.py b/autosklearn/pipeline/components/base.py index 25992a5a78..3e02f7d4d8 100644 --- a/autosklearn/pipeline/components/base.py +++ b/autosklearn/pipeline/components/base.py @@ -3,11 +3,14 @@ import inspect import pkgutil import sys +from typing import Dict from sklearn.base import BaseEstimator, TransformerMixin from autosklearn.pipeline.constants import SPARSE +_addons = dict() # type: Dict[str, 'ThirdPartyComponents'] + def find_components(package, directory, base_class): components = OrderedDict() diff --git a/autosklearn/pipeline/components/classification/__init__.py b/autosklearn/pipeline/components/classification/__init__.py index e4acee07a1..2dca6623ae 100644 --- a/autosklearn/pipeline/components/classification/__init__.py +++ b/autosklearn/pipeline/components/classification/__init__.py @@ -5,7 +5,7 @@ import os from ..base import AutoSklearnClassificationAlgorithm, find_components, \ - ThirdPartyComponents, AutoSklearnChoice + ThirdPartyComponents, AutoSklearnChoice, _addons from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import CategoricalHyperparameter @@ -13,11 +13,12 @@ _classifiers = find_components(__package__, classifier_directory, AutoSklearnClassificationAlgorithm) -_addons = ThirdPartyComponents(AutoSklearnClassificationAlgorithm) +additional_components = ThirdPartyComponents(AutoSklearnClassificationAlgorithm) +_addons['classification'] = additional_components def add_classifier(classifier: Type[AutoSklearnClassificationAlgorithm]) -> None: - _addons.add_component(classifier) + additional_components.add_component(classifier) class ClassifierChoice(AutoSklearnChoice): @@ -26,7 +27,7 @@ class ClassifierChoice(AutoSklearnChoice): def get_components(cls): components = OrderedDict() components.update(_classifiers) - components.update(_addons.components) + components.update(additional_components.components) return components def get_available_components(cls, dataset_properties=None, diff --git a/autosklearn/pipeline/components/data_preprocessing/categorical_encoding/__init__.py b/autosklearn/pipeline/components/data_preprocessing/categorical_encoding/__init__.py index 330392964c..c4d34ab306 100644 --- a/autosklearn/pipeline/components/data_preprocessing/categorical_encoding/__init__.py +++ b/autosklearn/pipeline/components/data_preprocessing/categorical_encoding/__init__.py @@ -10,7 +10,7 @@ from sklearn.base import BaseEstimator from ...base import AutoSklearnPreprocessingAlgorithm, find_components, \ - ThirdPartyComponents, AutoSklearnChoice + ThirdPartyComponents, AutoSklearnChoice, _addons from autosklearn.pipeline.base import DATASET_PROPERTIES_TYPE, PIPELINE_DATA_DTYPE @@ -18,11 +18,12 @@ _ohes = find_components(__package__, ohe_directory, AutoSklearnPreprocessingAlgorithm) -_addons = ThirdPartyComponents(AutoSklearnPreprocessingAlgorithm) +additional_components = ThirdPartyComponents(AutoSklearnPreprocessingAlgorithm) +_addons['data_preprocessing.categorical_encoding'] = additional_components def add_ohe(ohe: 'OHEChoice') -> None: - _addons.add_component(ohe) + additional_components.add_component(ohe) class OHEChoice(AutoSklearnChoice): @@ -31,7 +32,7 @@ class OHEChoice(AutoSklearnChoice): def get_components(cls: BaseEstimator) -> Dict[str, BaseEstimator]: components: Dict[str, BaseEstimator] = OrderedDict() components.update(_ohes) - components.update(_addons.components) + components.update(additional_components.components) return components def get_hyperparameter_search_space( diff --git a/autosklearn/pipeline/components/data_preprocessing/minority_coalescense/__init__.py b/autosklearn/pipeline/components/data_preprocessing/minority_coalescense/__init__.py index a3f46b32e0..0db0955cb5 100644 --- a/autosklearn/pipeline/components/data_preprocessing/minority_coalescense/__init__.py +++ b/autosklearn/pipeline/components/data_preprocessing/minority_coalescense/__init__.py @@ -8,7 +8,7 @@ from ConfigSpace.hyperparameters import CategoricalHyperparameter from ...base import AutoSklearnPreprocessingAlgorithm, find_components, \ - ThirdPartyComponents, AutoSklearnChoice + ThirdPartyComponents, AutoSklearnChoice, _addons from sklearn.base import BaseEstimator @@ -17,11 +17,12 @@ mc_directory = os.path.split(__file__)[0] _mcs = find_components( __package__, mc_directory, AutoSklearnPreprocessingAlgorithm) -_addons = ThirdPartyComponents(AutoSklearnPreprocessingAlgorithm) +additional_components = ThirdPartyComponents(AutoSklearnPreprocessingAlgorithm) +_addons['data_preprocessing.minority_coalescense'] = additional_components def add_mc(mc: BaseEstimator) -> None: - _addons.add_component(mc) + additional_components.add_component(mc) class CoalescenseChoice(AutoSklearnChoice): @@ -30,7 +31,7 @@ class CoalescenseChoice(AutoSklearnChoice): def get_components(cls: BaseEstimator) -> Dict[str, BaseEstimator]: components: Dict[str, BaseEstimator] = OrderedDict() components.update(_mcs) - components.update(_addons.components) + components.update(additional_components.components) return components def get_hyperparameter_search_space( diff --git a/autosklearn/pipeline/components/data_preprocessing/rescaling/__init__.py b/autosklearn/pipeline/components/data_preprocessing/rescaling/__init__.py index 298086476a..b37ad3ce24 100644 --- a/autosklearn/pipeline/components/data_preprocessing/rescaling/__init__.py +++ b/autosklearn/pipeline/components/data_preprocessing/rescaling/__init__.py @@ -9,7 +9,7 @@ from sklearn.base import BaseEstimator from ...base import AutoSklearnPreprocessingAlgorithm, find_components, \ - ThirdPartyComponents, AutoSklearnChoice + ThirdPartyComponents, AutoSklearnChoice, _addons from autosklearn.pipeline.base import DATASET_PROPERTIES_TYPE, PIPELINE_DATA_DTYPE from autosklearn.pipeline.components.data_preprocessing.rescaling.abstract_rescaling import ( Rescaling @@ -19,11 +19,12 @@ _rescalers = find_components(__package__, rescaling_directory, AutoSklearnPreprocessingAlgorithm) -_addons = ThirdPartyComponents(AutoSklearnPreprocessingAlgorithm) +additional_components = ThirdPartyComponents(AutoSklearnPreprocessingAlgorithm) +_addons['data_preprocessing.rescaling'] = additional_components def add_rescaler(rescaler: Rescaling) -> None: - _addons.add_component(rescaler) + additional_components.add_component(rescaler) class RescalingChoice(AutoSklearnChoice): @@ -32,7 +33,7 @@ class RescalingChoice(AutoSklearnChoice): def get_components(cls: BaseEstimator) -> Dict[str, BaseEstimator]: components: Dict[str, BaseEstimator] = OrderedDict() components.update(_rescalers) - components.update(_addons.components) + components.update(additional_components.components) return components def get_hyperparameter_search_space( diff --git a/autosklearn/pipeline/components/feature_preprocessing/__init__.py b/autosklearn/pipeline/components/feature_preprocessing/__init__.py index 8c42d30298..e124d135d0 100644 --- a/autosklearn/pipeline/components/feature_preprocessing/__init__.py +++ b/autosklearn/pipeline/components/feature_preprocessing/__init__.py @@ -3,7 +3,7 @@ from typing import Type from ..base import AutoSklearnPreprocessingAlgorithm, find_components, \ - ThirdPartyComponents, AutoSklearnChoice + ThirdPartyComponents, AutoSklearnChoice, _addons from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import CategoricalHyperparameter @@ -11,11 +11,12 @@ _preprocessors = find_components(__package__, classifier_directory, AutoSklearnPreprocessingAlgorithm) -_addons = ThirdPartyComponents(AutoSklearnPreprocessingAlgorithm) +additional_components = ThirdPartyComponents(AutoSklearnPreprocessingAlgorithm) +_addons['feature_preprocessing'] = additional_components def add_preprocessor(preprocessor: Type[AutoSklearnPreprocessingAlgorithm]) -> None: - _addons.add_component(preprocessor) + additional_components.add_component(preprocessor) class FeaturePreprocessorChoice(AutoSklearnChoice): @@ -24,7 +25,7 @@ class FeaturePreprocessorChoice(AutoSklearnChoice): def get_components(cls): components = OrderedDict() components.update(_preprocessors) - components.update(_addons.components) + components.update(additional_components.components) return components def get_available_components(self, dataset_properties=None, diff --git a/autosklearn/pipeline/components/regression/__init__.py b/autosklearn/pipeline/components/regression/__init__.py index d346f532c4..651b49b602 100644 --- a/autosklearn/pipeline/components/regression/__init__.py +++ b/autosklearn/pipeline/components/regression/__init__.py @@ -3,7 +3,7 @@ import os from ..base import AutoSklearnRegressionAlgorithm, find_components, \ - ThirdPartyComponents, AutoSklearnChoice + ThirdPartyComponents, AutoSklearnChoice, _addons from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import CategoricalHyperparameter @@ -11,11 +11,12 @@ _regressors = find_components(__package__, regressor_directory, AutoSklearnRegressionAlgorithm) -_addons = ThirdPartyComponents(AutoSklearnRegressionAlgorithm) +additional_components = ThirdPartyComponents(AutoSklearnRegressionAlgorithm) +_addons['regression'] = additional_components def add_regressor(regressor: Type[AutoSklearnRegressionAlgorithm]) -> None: - _addons.add_component(regressor) + additional_components.add_component(regressor) class RegressorChoice(AutoSklearnChoice): @@ -24,7 +25,7 @@ class RegressorChoice(AutoSklearnChoice): def get_components(cls): components = OrderedDict() components.update(_regressors) - components.update(_addons.components) + components.update(additional_components.components) return components @classmethod diff --git a/doc/releases.rst b/doc/releases.rst index afd3fd5e22..c89418d851 100644 --- a/doc/releases.rst +++ b/doc/releases.rst @@ -9,6 +9,16 @@ Releases ======== +Version 0.14.2 +============== + +* FIX #1290: Fixes a bug where it was not possible to extend Auto-sklearn and run it in parallel. + +Contributors v0.14.2 +******************** + +* Matthias Feurer + Version 0.14.1 ============== diff --git a/test/test_evaluation/test_abstract_evaluator.py b/test/test_evaluation/test_abstract_evaluator.py index 26806efd17..7c3e31b603 100644 --- a/test/test_evaluation/test_abstract_evaluator.py +++ b/test/test_evaluation/test_abstract_evaluator.py @@ -11,6 +11,7 @@ import sklearn.dummy from autosklearn.evaluation.abstract_evaluator import AbstractEvaluator +from autosklearn.pipeline.components.base import _addons from autosklearn.metrics import accuracy from autosklearn.util.backend import Backend, BackendContext from smac.tae import StatusType @@ -61,7 +62,9 @@ def test_finish_up_model_predicts_NaN(self): ae = AbstractEvaluator(backend=self.backend_mock, port=self.port, output_y_hat_optimization=False, - queue=queue_mock, metric=accuracy) + queue=queue_mock, metric=accuracy, + additional_components=dict(), + ) ae.Y_optimization = rs.rand(33, 3) predictions_ensemble = rs.rand(33, 3) predictions_test = rs.rand(25, 3) @@ -136,6 +139,7 @@ def test_disable_file_output(self): disable_file_output=True, metric=accuracy, port=self.port, + additional_components=dict(), ) predictions_ensemble = rs.rand(33, 3) @@ -163,6 +167,7 @@ def test_disable_file_output(self): disable_file_output=[disable], metric=accuracy, port=self.port, + additional_components=dict(), ) ae.Y_optimization = predictions_ensemble ae.model = unittest.mock.Mock() @@ -209,6 +214,7 @@ def test_disable_file_output(self): metric=accuracy, disable_file_output=['y_optimization'], port=self.port, + additional_components=dict(), ) ae.Y_optimization = predictions_ensemble ae.model = 'model' @@ -259,6 +265,7 @@ def test_file_output(self): queue=queue_mock, metric=accuracy, port=self.port, + additional_components=dict(), ) ae.model = sklearn.dummy.DummyClassifier() @@ -278,3 +285,37 @@ def test_file_output(self): '.auto-sklearn', 'runs', '1_0_None'))) shutil.rmtree(self.working_directory, ignore_errors=True) + + def test_add_additional_components(self): + shutil.rmtree(self.working_directory, ignore_errors=True) + os.mkdir(self.working_directory) + + queue_mock = unittest.mock.Mock() + + context = BackendContext( + temporary_directory=os.path.join(self.working_directory, 'tmp'), + delete_tmp_folder_after_terminate=True, + ) + with unittest.mock.patch.object(Backend, 'load_datamanager') as load_datamanager_mock: + load_datamanager_mock.return_value = get_multiclass_classification_datamanager() + backend = Backend(context) + + with unittest.mock.patch.object(_addons['classification'], 'add_component') as _: + + # If the components in the argument `additional_components` are an empty dict + # there is no call to `add_component`, if there's something in it, `add_component + # is called (2nd case) + for fixture, case in ((0, dict()), (1, dict(abc='def'))): + + thirdparty_components_patch = unittest.mock.Mock() + thirdparty_components_patch.components = case + additional_components = dict(classification=thirdparty_components_patch) + AbstractEvaluator( + backend=backend, + output_y_hat_optimization=False, + queue=queue_mock, + metric=accuracy, + port=self.port, + additional_components=additional_components, + ) + self.assertEqual(_addons['classification'].add_component.call_count, fixture) diff --git a/test/test_evaluation/test_test_evaluator.py b/test/test_evaluation/test_test_evaluator.py index 06b522a495..d09ec8504a 100644 --- a/test/test_evaluation/test_test_evaluator.py +++ b/test/test_evaluation/test_test_evaluator.py @@ -67,6 +67,7 @@ def test_datasets(self): queue_, metric=metric_lookup[D.info['task']], port=logging.handlers.DEFAULT_TCP_LOGGING_PORT, + additional_components=dict(), ) evaluator.fit_predict_and_loss() @@ -112,6 +113,7 @@ def test_eval_test(self): disable_file_output=False, instance=self.dataset_name, port=self.port, + additional_components=dict(), ) rval = read_queue(self.queue) self.assertEqual(len(rval), 1) @@ -133,6 +135,7 @@ def test_eval_test_all_loss_functions(self): disable_file_output=False, instance=self.dataset_name, port=self.port, + additional_components=dict(), ) rval = read_queue(self.queue) self.assertEqual(len(rval), 1) diff --git a/test/test_evaluation/test_train_evaluator.py b/test/test_evaluation/test_train_evaluator.py index f43a4dd00e..723abb0d41 100644 --- a/test/test_evaluation/test_train_evaluator.py +++ b/test/test_evaluation/test_train_evaluator.py @@ -104,6 +104,7 @@ def test_holdout(self, pipeline_mock): output_y_hat_optimization=True, metric=accuracy, port=self.port, + additional_components=dict(), ) evaluator.file_output = unittest.mock.Mock(spec=evaluator.file_output) evaluator.file_output.return_value = (None, {}) @@ -170,7 +171,8 @@ def configuration_fully_fitted(self): scoring_functions=None, output_y_hat_optimization=True, metric=accuracy, - budget=0.0) + budget=0.0, + additional_components=dict(),) evaluator.file_output = unittest.mock.Mock(spec=evaluator.file_output) evaluator.file_output.return_value = (None, {}) @@ -268,7 +270,9 @@ def configuration_fully_fitted(self): scoring_functions=None, output_y_hat_optimization=True, metric=accuracy, - budget=0.0) + budget=0.0, + additional_components=dict(), + ) evaluator.file_output = unittest.mock.Mock(spec=evaluator.file_output) evaluator.file_output.return_value = (None, {}) @@ -337,7 +341,9 @@ def test_iterative_holdout_not_iterative(self, pipeline_mock): resampling_strategy='holdout-iterative-fit', scoring_functions=None, output_y_hat_optimization=True, - metric=accuracy) + metric=accuracy, + additional_components=dict(), + ) evaluator.file_output = unittest.mock.Mock(spec=evaluator.file_output) evaluator.file_output.return_value = (None, {}) @@ -380,7 +386,9 @@ def test_cv(self, pipeline_mock): resampling_strategy_args={'folds': 5}, scoring_functions=None, output_y_hat_optimization=True, - metric=accuracy) + metric=accuracy, + additional_components=dict(), + ) evaluator.file_output = unittest.mock.Mock(spec=evaluator.file_output) evaluator.file_output.return_value = (None, {}) @@ -434,7 +442,9 @@ def test_partial_cv(self, pipeline_mock): resampling_strategy_args={'folds': 5}, scoring_functions=None, output_y_hat_optimization=True, - metric=accuracy) + metric=accuracy, + additional_components=dict(), + ) evaluator.file_output = unittest.mock.Mock(spec=evaluator.file_output) evaluator.file_output.return_value = (None, {}) @@ -495,7 +505,9 @@ def configuration_fully_fitted(self): scoring_functions=None, output_y_hat_optimization=True, metric=accuracy, - budget=0.0) + budget=0.0, + additional_components=dict(), + ) evaluator.file_output = unittest.mock.Mock(spec=evaluator.file_output) evaluator.file_output.return_value = (None, {}) @@ -563,7 +575,8 @@ def test_file_output(self, loss_mock, model_mock): resampling_strategy_args={'folds': 5}, scoring_functions=SCORER_LIST, output_y_hat_optimization=True, - metric=accuracy) + metric=accuracy, + additional_components=dict(),) self.backend_mock.get_model_dir.return_value = True evaluator.model = 'model' @@ -647,7 +660,9 @@ def test_subsample_indices_classification(self, mock, backend_mock): configuration=configuration, resampling_strategy='cv', resampling_strategy_args={'folds': 10}, - metric=accuracy) + metric=accuracy, + additional_components=dict(), + ) train_indices = np.arange(69, dtype=int) train_indices1 = subsample_indices( train_indices, 0.1449, evaluator.task_type, evaluator.Y_train) @@ -695,7 +710,9 @@ def test_subsample_indices_regression(self, mock, backend_mock): configuration=configuration, resampling_strategy='cv', resampling_strategy_args={'folds': 10}, - metric=accuracy) + metric=accuracy, + additional_components=dict(), + ) train_indices = np.arange(69, dtype=int) train_indices3 = subsample_indices(train_indices, subsample=0.4347, task_type=evaluator.task_type, @@ -741,7 +758,9 @@ def test_predict_proba_binary_classification(self, mock): resampling_strategy='cv', resampling_strategy_args={'folds': 10}, output_y_hat_optimization=False, - metric=accuracy) + metric=accuracy, + additional_components=dict(), + ) evaluator.fit_predict_and_loss() Y_optimization_pred = self.backend_mock.save_numrun_to_dir.call_args_list[0][1][ @@ -781,6 +800,7 @@ def test_fit_predict_and_loss_standard_additional_run_info( resampling_strategy='holdout', output_y_hat_optimization=False, metric=accuracy, + additional_components=dict(), ) evaluator.file_output = unittest.mock.Mock(spec=evaluator.file_output) evaluator.file_output.return_value = (None, {}) @@ -826,6 +846,7 @@ def __call__(self, *args, **kwargs): resampling_strategy_args={'folds': 2}, output_y_hat_optimization=False, metric=accuracy, + additional_components=dict(), ) evaluator.file_output = unittest.mock.Mock(spec=evaluator.file_output) evaluator.file_output.return_value = (None, {}) @@ -878,7 +899,8 @@ def __call__(self): resampling_strategy='holdout', output_y_hat_optimization=False, metric=accuracy, - budget=0.0 + budget=0.0, + additional_components=dict(), ) evaluator.file_output = unittest.mock.Mock(spec=evaluator.file_output) evaluator.file_output.return_value = (None, {}) @@ -916,6 +938,7 @@ def test_fit_predict_and_loss_iterative_noniterativemodel_additional_run_info( resampling_strategy='holdout', output_y_hat_optimization=False, metric=accuracy, + additional_components=dict(), ) evaluator.file_output = unittest.mock.Mock(spec=evaluator.file_output) evaluator.file_output.return_value = (None, {}) @@ -966,6 +989,7 @@ def __call__(self): metric=accuracy, budget_type='iterations', budget=50, + additional_components=dict(), ) evaluator.file_output = unittest.mock.Mock(spec=evaluator.file_output) evaluator.file_output.return_value = (None, {}) @@ -1006,6 +1030,7 @@ def test_fit_predict_and_loss_budget_2_additional_run_info( metric=accuracy, budget_type='subsample', budget=50, + additional_components=dict(), ) evaluator.file_output = unittest.mock.Mock(spec=evaluator.file_output) evaluator.file_output.return_value = (None, {}) @@ -1049,7 +1074,8 @@ def test_datasets(self): resampling_strategy='cv', resampling_strategy_args={'folds': 2}, output_y_hat_optimization=False, - metric=metric_lookup[D.info['task']]) + metric=metric_lookup[D.info['task']], + additional_components=dict(),) evaluator.fit_predict_and_loss() rval = evaluator.queue.get(timeout=1) @@ -2248,6 +2274,7 @@ def test_eval_holdout(self): disable_file_output=False, instance=self.dataset_name, metric=accuracy, + additional_components=dict(), ) info = read_queue(self.queue) self.assertEqual(len(info), 1) @@ -2272,6 +2299,7 @@ def test_eval_holdout_all_loss_functions(self): disable_file_output=False, instance=self.dataset_name, metric=accuracy, + additional_components=dict(), ) rval = read_queue(self.queue) self.assertEqual(len(rval), 1) @@ -2323,6 +2351,7 @@ def test_eval_holdout_iterative_fit_no_timeout(self): disable_file_output=False, instance=self.dataset_name, metric=accuracy, + additional_components=dict(), ) rval = read_queue(self.queue) self.assertEqual(len(rval), 9) @@ -2348,7 +2377,8 @@ def test_eval_holdout_budget_iterations(self): instance=self.dataset_name, metric=accuracy, budget=1, - budget_type='iterations' + budget_type='iterations', + additional_components=dict(), ) info = read_queue(self.queue) self.assertEqual(len(info), 1) @@ -2378,7 +2408,8 @@ def test_eval_holdout_budget_iterations_converged(self): instance=self.dataset_name, metric=accuracy, budget=80, - budget_type='iterations' + budget_type='iterations', + additional_components=dict(), ) info = read_queue(self.queue) self.assertEqual(len(info), 1) @@ -2404,7 +2435,8 @@ def test_eval_holdout_budget_subsample(self): instance=self.dataset_name, metric=accuracy, budget=30, - budget_type='subsample' + budget_type='subsample', + additional_components=dict(), ) info = read_queue(self.queue) self.assertEqual(len(info), 1) @@ -2431,7 +2463,8 @@ def test_eval_holdout_budget_mixed_iterations(self): instance=self.dataset_name, metric=accuracy, budget=1, - budget_type='mixed' + budget_type='mixed', + additional_components=dict() ) info = read_queue(self.queue) self.assertEqual(len(info), 1) @@ -2460,7 +2493,8 @@ def test_eval_holdout_budget_mixed_subsample(self): instance=self.dataset_name, metric=accuracy, budget=40, - budget_type='mixed' + budget_type='mixed', + additional_components=dict(), ) info = read_queue(self.queue) self.assertEqual(len(info), 1) @@ -2485,6 +2519,7 @@ def test_eval_cv(self): disable_file_output=False, instance=self.dataset_name, metric=accuracy, + additional_components=dict(), ) rval = read_queue(self.queue) self.assertEqual(len(rval), 1) @@ -2509,6 +2544,7 @@ def test_eval_cv_all_loss_functions(self): disable_file_output=False, instance=self.dataset_name, metric=accuracy, + additional_components=dict(), ) rval = read_queue(self.queue) self.assertEqual(len(rval), 1) @@ -2577,6 +2613,7 @@ def test_eval_partial_cv(self): exclude=None, disable_file_output=False, metric=accuracy, + additional_components=dict(), ) rval = read_queue(self.queue) self.assertEqual(len(rval), 1) diff --git a/test/test_pipeline/test_classification.py b/test/test_pipeline/test_classification.py index ef6c4baf0d..d5864f14cd 100644 --- a/test/test_pipeline/test_classification.py +++ b/test/test_pipeline/test_classification.py @@ -24,7 +24,7 @@ from autosklearn.pipeline.classification import SimpleClassificationPipeline from autosklearn.pipeline.components.base import \ AutoSklearnClassificationAlgorithm, AutoSklearnPreprocessingAlgorithm -from autosklearn.pipeline.components.base import AutoSklearnComponent, AutoSklearnChoice +from autosklearn.pipeline.components.base import AutoSklearnComponent, AutoSklearnChoice, _addons import autosklearn.pipeline.components.classification as classification_components import autosklearn.pipeline.components.feature_preprocessing as preprocessing_components from autosklearn.pipeline.util import get_dataset @@ -676,20 +676,24 @@ def test_get_params(self): pass def test_add_classifier(self): - self.assertEqual(len(classification_components._addons.components), 0) + self.assertEqual(len(classification_components.additional_components.components), 0) + self.assertEqual(len(_addons['classification'].components), 0) classification_components.add_classifier(DummyClassifier) - self.assertEqual(len(classification_components._addons.components), 1) + self.assertEqual(len(classification_components.additional_components.components), 1) + self.assertEqual(len(_addons['classification'].components), 1) cs = SimpleClassificationPipeline().get_hyperparameter_search_space() self.assertIn('DummyClassifier', str(cs)) - del classification_components._addons.components['DummyClassifier'] + del classification_components.additional_components.components['DummyClassifier'] def test_add_preprocessor(self): - self.assertEqual(len(preprocessing_components._addons.components), 0) + self.assertEqual(len(preprocessing_components.additional_components.components), 0) + self.assertEqual(len(_addons['feature_preprocessing'].components), 0) preprocessing_components.add_preprocessor(DummyPreprocessor) - self.assertEqual(len(preprocessing_components._addons.components), 1) + self.assertEqual(len(preprocessing_components.additional_components.components), 1) + self.assertEqual(len(_addons['feature_preprocessing'].components), 1) cs = SimpleClassificationPipeline().get_hyperparameter_search_space() self.assertIn('DummyPreprocessor', str(cs)) - del preprocessing_components._addons.components['DummyPreprocessor'] + del preprocessing_components.additional_components.components['DummyPreprocessor'] def _test_set_hyperparameter_choice(self, expected_key, implementation, config_dict): """ @@ -869,7 +873,7 @@ def test_fit_instantiates_component(self): except Exception as e: # In case of failure clean up the components and print enough information # to clean up with check in the future - del preprocessing_components._addons.components['CrashPreprocessor'] + del preprocessing_components.additional_components.components['CrashPreprocessor'] self.fail("cs={} config={} Exception={}".format(cs, config, e)) cls.set_hyperparameters(config) with self.assertRaisesRegex( @@ -880,4 +884,4 @@ def test_fit_instantiates_component(self): X=np.array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]), y=np.array([1, 0, 1, 1]) ) - del preprocessing_components._addons.components['CrashPreprocessor'] + del preprocessing_components.additional_components.components['CrashPreprocessor']