Fixing hps remain active & meta hp configuration

.github/workflows/docker-publish.yml

@@ -26,9 +26,8 @@ jobs:
         run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
         id: extract_branch
 
-      # Updating this to @v2 requires a github auth token
       - name: Push to GitHub Packages
-        uses: docker/build-push-action@v1
+        uses: docker/build-push-action@v3
         with:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
@@ -37,12 +36,11 @@ jobs:
           tag_with_ref: true
           tags: ${{ steps.extract_branch.outputs.branch }}
 
-      # Updating this to @v2 requires a github auth token
       - name: Push to Docker Hub
-        uses: docker/build-push-action@v1
+        uses: docker/build-push-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
+          password: ${{ secrets.DOCKER_TOKEN }}
           repository: mfeurer/auto-sklearn
           tags: ${{ steps.extract_branch.outputs.branch }}
 

autosklearn/askl_typing.py

@@ -0,0 +1,3 @@
+from typing import Dict, Union
+
+FEAT_TYPE_TYPE = Dict[Union[str, int], str]

autosklearn/evaluation/abstract_evaluator.py

@@ -17,6 +17,7 @@
 
 import autosklearn.pipeline.classification
 import autosklearn.pipeline.regression
+from autosklearn.askl_typing import FEAT_TYPE_TYPE
 from autosklearn.automl_common.common.utils.backend import Backend
 from autosklearn.constants import (
     CLASSIFICATION_TASKS,
@@ -45,6 +46,7 @@ def __init__(
         self,
         config: Configuration,
         random_state: Optional[Union[int, np.random.RandomState]],
+        feat_type: Optional[FEAT_TYPE_TYPE] = None,
         init_params: Optional[Dict[str, Any]] = None,
         dataset_properties: Dict[str, Any] = {},
         include: Optional[List[str]] = None,
@@ -61,6 +63,7 @@ def __init__(
         self.dataset_properties = dataset_properties
         self.include = include
         self.exclude = exclude
+        self.feat_type = feat_type
 
     def pre_transform(
         self,
@@ -108,6 +111,7 @@ def __init__(
         self,
         config: Configuration,
         random_state: Optional[Union[int, np.random.RandomState]],
+        feat_type: Optional[FEAT_TYPE_TYPE] = None,
         init_params: Optional[Dict[str, Any]] = None,
         dataset_properties: Dict[str, Any] = {},
         include: Optional[List[str]] = None,
@@ -123,6 +127,7 @@ def __init__(
         self.dataset_properties = dataset_properties
         self.include = include
         self.exclude = exclude
+        self.feat_type = feat_type
 
     def pre_transform(
         self,
@@ -217,6 +222,7 @@ def __init__(
         self.queue = queue
 
         self.datamanager = self.backend.load_datamanager()
+        self.feat_type = self.datamanager.feat_type
         self.include = include
         self.exclude = exclude
 
@@ -294,11 +300,12 @@ def __init__(
                     _addons[key].add_component(component)
 
         # Please mypy to prevent not defined attr
-        self.model = self._get_model()
+        self.model = self._get_model(feat_type=self.feat_type)
 
-    def _get_model(self) -> BaseEstimator:
+    def _get_model(self, feat_type: Optional[FEAT_TYPE_TYPE]) -> BaseEstimator:
         if not isinstance(self.configuration, Configuration):
             model = self.model_class(
+                feat_type=feat_type,
                 config=self.configuration,
                 random_state=self.seed,
                 init_params=self._init_params,
@@ -318,6 +325,7 @@ def _get_model(self) -> BaseEstimator:
                     "multiclass": self.task_type == MULTICLASS_CLASSIFICATION,
                 }
             model = self.model_class(
+                feat_type=feat_type,
                 config=self.configuration,
                 dataset_properties=dataset_properties,
                 random_state=self.seed,

autosklearn/evaluation/test_evaluator.py

@@ -58,7 +58,7 @@ def __init__(
         self.X_test = self.datamanager.data.get("X_test")
         self.Y_test = self.datamanager.data.get("Y_test")
 
-        self.model = self._get_model()
+        self.model = self._get_model(self.feat_type)
 
     def fit_predict_and_loss(self) -> None:
         _fit_and_suppress_warnings(self.logger, self.model, self.X_train, self.Y_train)

autosklearn/evaluation/train_evaluator.py

@@ -247,6 +247,7 @@ def __init__(
             budget_type=budget_type,
         )
 
+        self.feat_type = self.backend.load_datamanager().feat_type
         self.resampling_strategy = resampling_strategy
         if resampling_strategy_args is None:
             self.resampling_strategy_args = {}
@@ -305,7 +306,7 @@ def fit_predict_and_loss(self, iterative: bool = False) -> None:
 
                 # Test if the model allows for an iterative fit, if not,
                 # call this method again without the iterative argument
-                model = self._get_model()
+                model = self._get_model(self.feat_type)
                 if not model.estimator_supports_iterative_fit():
                     self.fit_predict_and_loss(iterative=False)
                     return
@@ -319,7 +320,9 @@ def fit_predict_and_loss(self, iterative: bool = False) -> None:
                 Y_test_pred = [None] * self.num_cv_folds
                 train_splits = [None] * self.num_cv_folds
 
-                self.models = [self._get_model() for i in range(self.num_cv_folds)]
+                self.models = [
+                    self._get_model(self.feat_type) for i in range(self.num_cv_folds)
+                ]
                 iterations = [1] * self.num_cv_folds
                 total_n_iterations = [0] * self.num_cv_folds
                 # model.estimator_supports_iterative_fit -> true
@@ -515,7 +518,7 @@ def fit_predict_and_loss(self, iterative: bool = False) -> None:
                     self.Y_optimization = Y_targets
                     self.Y_actual_train = Y_train_targets
 
-                    self.model = self._get_model()
+                    self.model = self._get_model(self.feat_type)
                     status = StatusType.DONOTADVANCE
                     if any(
                         [
@@ -679,7 +682,7 @@ def fit_predict_and_loss(self, iterative: bool = False) -> None:
             self.Y_actual_train = Y_train_targets
 
             if self.num_cv_folds > 1:
-                self.model = self._get_model()
+                self.model = self._get_model(self.feat_type)
                 # Bad style, but necessary for unit testing that self.model is
                 # actually a new model
                 self._added_empty_model = True
@@ -798,7 +801,7 @@ def _partial_fit_and_predict_iterative(
         test_indices: List[int],
         add_model_to_self: bool,
     ) -> None:
-        model = self._get_model()
+        model = self._get_model(self.feat_type)
 
         self.indices[fold] = (train_indices, test_indices)
 
@@ -939,7 +942,7 @@ def _partial_fit_and_predict_standard(
         PIPELINE_DATA_DTYPE,  # test_pred
         TYPE_ADDITIONAL_INFO,
     ]:
-        model = self._get_model()
+        model = self._get_model(self.feat_type)
 
         self.indices[fold] = (train_indices, test_indices)
 
@@ -1005,7 +1008,7 @@ def _partial_fit_and_predict_budget(
         # Add this statement for mypy
         assert self.budget is not None
 
-        model = self._get_model()
+        model = self._get_model(self.feat_type)
         self.indices[fold] = (train_indices, test_indices)
         self.X_targets[fold] = self.X_train[test_indices]
         self.Y_targets[fold] = self.Y_train[test_indices]

autosklearn/experimental/askl2.py

@@ -51,8 +51,10 @@ def __call__(
         initial_configurations = []
         for member in self.portfolio.values():
             try:
+                hp_names = scenario.cs.get_hyperparameter_names()
+                _member = {key: member[key] for key in member if key in hp_names}
                 initial_configurations.append(
-                    Configuration(configuration_space=scenario.cs, values=member)
+                    Configuration(configuration_space=scenario.cs, values=_member)
                 )
             except ValueError:
                 pass
@@ -103,8 +105,10 @@ def __call__(
         initial_configurations = []
         for member in self.portfolio.values():
             try:
+                hp_names = scenario.cs.get_hyperparameter_names()
+                _member = {key: member[key] for key in member if key in hp_names}
                 initial_configurations.append(
-                    Configuration(configuration_space=scenario.cs, values=member)
+                    Configuration(configuration_space=scenario.cs, values=_member)
                 )
             except ValueError:
                 pass

autosklearn/metalearning/input/aslib_simple.py

@@ -5,13 +5,15 @@
 
 import arff
 import pandas as pd
+from ConfigSpace.configuration_space import ConfigurationSpace
 
 
 class AlgorithmSelectionProblem(object):
-    def __init__(self, directory):
+    def __init__(self, directory: str, cs: ConfigurationSpace):
         self.logger = logging.getLogger(__name__)
 
         # Create data structures
+        self.cs = cs
         self.dir_ = directory
         self.algorithm_runs = None
         self.configurations = None
@@ -143,13 +145,17 @@ def _read_configurations(self, filename):
             csv_reader = csv.DictReader(fh)
 
             configurations = dict()
+            hp_names = self.cs.get_hyperparameter_names()
             for line in csv_reader:
                 configuration = dict()
                 algorithm_id = line["idx"]
                 for hp_name, value in line.items():
                     if not value or hp_name == "idx":
                         continue
-
+                    if hp_name not in hp_names:
+                        # skip hyperparameter
+                        # if it is not existing in the current search space
+                        continue
                     try:
                         value = int(value)
                     except Exception:

autosklearn/metalearning/metalearning/meta_base.py

@@ -42,7 +42,9 @@ def __init__(self, configuration_space, aslib_directory, logger):
         self.configuration_space = configuration_space
         self.aslib_directory = aslib_directory
 
-        aslib_reader = aslib_simple.AlgorithmSelectionProblem(self.aslib_directory)
+        aslib_reader = aslib_simple.AlgorithmSelectionProblem(
+            self.aslib_directory, self.configuration_space
+        )
         self.metafeatures = aslib_reader.metafeatures
         self.algorithm_runs: OrderedDict[
             str, pd.DataFrame