ablation path test update

Author: KrissiHub

deepcave/evaluators/ablation.py

@@ -76,8 +76,6 @@ def calculate(
         objectives: Optional[Union[Objective, List[Objective]]],  # noqa
         budget: Optional[Union[int, float]] = None,  # noqa
         model: Any = None,
-        seed: int = 0,
-        n_trees: int = 50,
     ) -> None:
         """
         Calculate the ablation path performances and improvements.
@@ -95,12 +93,6 @@ def calculate(
         model :
             The surrogate model to use for the prediction of the perfromances.
             By default None.
-        seed : int
-            The seed to use for reproducability.
-            By default 0.
-        n_trees : int
-            The number of trees to use for the Random Forest.
-            By default 50.
         """
         if isinstance(objectives, list) and len(objectives) > 1:
             raise ValueError("Only one objective is supported for ablation paths.")
@@ -131,7 +123,7 @@ def calculate(
         # The default model is a RF Surrogate, but it cant be passed as parameter directly
         # because it needs access to its config space
         if self._model is None:
-            self._model = RandomForestSurrogate(self.cs, seed=seed, n_trees=n_trees)
+            self._model = RandomForestSurrogate(self.cs, seed=0, n_trees=50)
 
         self._model.fit(X, Y)
         # Obtain the predicted cost of the default and incumbent configuration

deepcave/evaluators/mo_ablation.py

@@ -123,8 +123,6 @@ def calculate(
         objectives: Optional[Union[Objective, List[Objective]]],  # noqa
         budget: Optional[Union[int, float]] = None,  # noqa
         model: Any = None,
-        seed: int = 0,  # noqa
-        n_trees: int = 50,  # noqa
     ) -> None:
         """
         Calculate the MO ablation path performances and improvements.
@@ -139,12 +137,6 @@ def calculate(
         model : Any
             For mo ablation this parameter does not do anything, except fit the head.
             By default None.
-        n_trees : int
-            The number of trees for the surrogate model.
-            Default is 50.
-        seed : int
-            The seed for the surrogate model.
-            Default is 0.
         """
         assert isinstance(objectives, list)
         for objective in objectives:
@@ -170,7 +162,8 @@ def calculate(
 
             # train one model per objective
             Y = df[normed].to_numpy()
-            model = RandomForestSurrogate(self.cs, seed=seed, n_trees=n_trees)
+            if model is None:
+                model = RandomForestSurrogate(self.cs, seed=0, n_trees=50)
             model.fit(X, Y)
             self.models.append(model)
 

deepcave/plugins/hyperparameter/ablation_paths.py

@@ -338,7 +338,7 @@ def process(run: AbstractRun, inputs: Dict[str, Any]) -> Dict[str, Any]:
         data: Dict[Any, Any] = {}
         for budget_id, budget in enumerate(budgets):
             assert isinstance(budget, (int, float))
-            evaluator.calculate(objective, budget, n_trees=n_trees, seed=0)
+            evaluator.calculate(objective, budget)
             if isinstance(objective, list):
                 assert isinstance(evaluator, MOAblation)
                 data[budget_id] = evaluator.get_importances()

tests/test_evaluators/test_ablation.py

@@ -23,6 +23,7 @@
 from sympy import lambdify, symbols
 
 from deepcave.evaluators.ablation import Ablation as Evaluator
+from deepcave.evaluators.epm.random_forest_surrogate import RandomForestSurrogate
 from deepcave.runs import AbstractRun
 from deepcave.runs.converters.deepcave import DeepCAVERun
 from deepcave.runs.converters.smac3v2 import SMAC3v2Run
@@ -37,8 +38,9 @@ class PolynomialSurrogateModel:
     """
 
     def __init__(
-        self, n: int, max_degree: int = 2, seed: int = 42, coeffs: Optional[np.ndarray] = None
+        self, n: int, max_degree: int = 1, seed: int = 42, coeffs: Optional[np.ndarray] = None
     ):
+        self.ground_truth: np.ndarray
         self._polynomial(n, max_degree, seed, coeffs)
 
     def fit(self, X: np.ndarray, y: np.ndarray) -> np.ndarray:
@@ -83,7 +85,6 @@ def _polynomial(
         np.ndarray
             The fitting polynomial structure with random variables.
         """
-
         x = symbols(f"x1:{n+1}")
 
         terms = []
@@ -110,12 +111,12 @@ def _polynomial(
 
         if coeffs is None:
             np.random.seed(seed)
-            input_values = np.random.randint(1, 5, size=len(coeff_symb))
+            self.ground_truth = np.random.uniform(1, 5, size=len(coeff_symb))
 
         else:
-            input_values = coeffs
+            self.ground_truth = coeffs
 
-        input_mapping = dict(zip(coeff_symb, input_values))
+        input_mapping = dict(zip(coeff_symb, self.ground_truth))
         partial_expr = expr.subs(input_mapping)
 
         variables = sorted(partial_expr.free_symbols, key=lambda s: str(s))
@@ -134,10 +135,12 @@ def test(self):
         objective = self.run.get_objective(0)
 
         # Calculate
-        self.evaluator.calculate(objective, budget, seed=0)
+        model_1 = RandomForestSurrogate(self.run.configspace, seed=0)
+        self.evaluator.calculate(objective, budget, model=model_1)
         importances = self.evaluator.get_ablation_performances()
 
-        self.evaluator.calculate(objective, budget, seed=42)
+        model_2 = RandomForestSurrogate(self.run.configspace, seed=42)
+        self.evaluator.calculate(objective, budget, model=model_2)
         importances2 = self.evaluator.get_ablation_performances()
 
         # Different seed: Different results
@@ -152,10 +155,12 @@ def test_seed(self):
         objective = self.run.get_objective(0)
 
         # Calculate
-        self.evaluator.calculate(objective, budget, seed=0)
+        model_1 = RandomForestSurrogate(self.run.configspace, seed=0)
+        self.evaluator.calculate(objective, budget, model=model_1)
         importances = self.evaluator.get_ablation_performances()
 
-        self.evaluator.calculate(objective, budget, seed=0)
+        model_2 = RandomForestSurrogate(self.run.configspace, seed=0)
+        self.evaluator.calculate(objective, budget, model=model_2)
         importances2 = self.evaluator.get_ablation_performances()
 
         # Same seed: Same results
@@ -174,9 +179,17 @@ def test_polynomial(self):
         self.evaluator.calculate(objectives=objective, budget=budget, model=model)
 
         # Evaluate the final ablation path
-        # performances = self.evaluator.get_ablation_performances()
-        # importances = self.evaluator.get_ablation_improvements()
-        # TODO: evaluation
+        importances = self.evaluator.get_ablation_improvements()
+        sorted_importances = np.array(
+            [
+                round(float(value[0]), 8)
+                for key, value in sorted(importances.items())
+                if key != "default"
+            ]
+        )
+        ground_truth = model.ground_truth[1:]
+
+        assert np.allclose(sorted_importances, ground_truth, rtol=1e-5, atol=1e-8)
 
 
 if __name__ == "__main__":