From ec13a1c840930121dc32c8aa4fa1f7c1deba8d3c Mon Sep 17 00:00:00 2001 From: Jannis Kastner Date: Tue, 21 Oct 2025 14:43:10 +0200 Subject: [PATCH 1/2] Improve efficiency of acquisition function optimization (#954) - Optional parallelization of Local Search - Scale standard deviation of neighborhood sampling as in Regis and Shoemaker - Optional multiple-coordinate local search - Limit plateau walks for exhausted Hyperparameter --- .../maximizer/local_and_random_search.py | 9 + smac/acquisition/maximizer/local_search.py | 314 ++++++++++-------- smac/utils/configspace.py | 271 +++++++++++++++ 3 files changed, 460 insertions(+), 134 deletions(-) diff --git a/smac/acquisition/maximizer/local_and_random_search.py b/smac/acquisition/maximizer/local_and_random_search.py index a24122869..ca1f5703a 100644 --- a/smac/acquisition/maximizer/local_and_random_search.py +++ b/smac/acquisition/maximizer/local_and_random_search.py @@ -48,6 +48,11 @@ class LocalAndSortedRandomSearch(AbstractAcquisitionMaximizer): The ratio of random samples that are taken from the user-defined ConfigurationSpace, as opposed to the uniform version (needs `uniform_configspace`to be defined). seed : int, defaults to 0 + n_jobs_ls: int, defaults to 1 + Number of parallel jobs used when performing local search. If 1, the search is serial. + If >1, multiple starting points are evaluated in parallel. + exchange_size_ls: int, defaults to 1 + Number of Hyperparameters changed by one step of the Local Search """ def __init__( @@ -61,6 +66,8 @@ def __init__( seed: int = 0, uniform_configspace: ConfigurationSpace | None = None, prior_sampling_fraction: float | None = None, + n_jobs_ls: int = 1, + exchange_size_ls: int = 1, ) -> None: super().__init__( configspace, @@ -98,6 +105,8 @@ def __init__( max_steps=max_steps, n_steps_plateau_walk=n_steps_plateau_walk, seed=seed, + n_jobs=n_jobs_ls, + exchange_size=exchange_size_ls, ) self._local_search_iterations = local_search_iterations diff --git a/smac/acquisition/maximizer/local_search.py b/smac/acquisition/maximizer/local_search.py index 422935f14..e05cb3ad2 100644 --- a/smac/acquisition/maximizer/local_search.py +++ b/smac/acquisition/maximizer/local_search.py @@ -4,10 +4,17 @@ import itertools import time +import warnings import numpy as np from ConfigSpace import Configuration, ConfigurationSpace from ConfigSpace.exceptions import ForbiddenValueError +from ConfigSpace.hyperparameters import ( + CategoricalHyperparameter, + OrdinalHyperparameter, + UniformIntegerHyperparameter, +) +from joblib import Parallel, delayed from smac.acquisition.function import AbstractAcquisitionFunction from smac.acquisition.maximizer.abstract_acquisition_maximizer import ( @@ -15,6 +22,7 @@ ) from smac.utils.configspace import ( convert_configurations_to_array, + get_k_exchange_neighbourhood, get_one_exchange_neighbourhood, ) from smac.utils.logging import get_logger @@ -45,6 +53,14 @@ class LocalSearch(AbstractAcquisitionMaximizer): Maximal number of neighbors to obtain at once for each local search for vectorized calls. Can be tuned to reduce the overhead of SMAC. seed : int, defaults to 0 + n_jobs : int, defaults to 1 + Number of parallel jobs to use when performing local search. If 1, the search is serial. + If >1, multiple starting points are evaluated in parallel. + base_sigma: float, defaults to 2 + Base standard deviation for sampling continous hyperparameters + exchange_size : int, defaults to 1 + Number of hyperparameters to modify in each neighborhood step. + """ def __init__( @@ -57,6 +73,9 @@ def __init__( vectorization_min_obtain: int = 2, vectorization_max_obtain: int = 64, seed: int = 0, + n_jobs: int = 1, + base_sigma: float = 0.2, + exchange_size: int = 1, ) -> None: super().__init__( configspace, @@ -69,6 +88,9 @@ def __init__( self._n_steps_plateau_walk = n_steps_plateau_walk self._vectorization_min_obtain = vectorization_min_obtain self._vectorization_max_obtain = vectorization_max_obtain + self._n_jobs = n_jobs + self._base_sigma = base_sigma + self._exchange_size = exchange_size @property def meta(self) -> dict[str, Any]: # noqa: D102 @@ -278,173 +300,197 @@ def _search( """ assert self._acquisition_function is not None + number_of_hyperparameters = len(start_points[0].config_space.keys()) + if self._exchange_size > number_of_hyperparameters: + warnings.warn( + f"Requested _exchange_size={self._exchange_size} exceeds the number of " + f"available hyperparameters ({number_of_hyperparameters}). " + f"Setting _exchange_size to {number_of_hyperparameters}", + ) + self._exchange_size = number_of_hyperparameters + # Gather data structure for starting points if isinstance(start_points, Configuration): start_points = [start_points] - candidates = start_points - # Compute the acquisition value of the candidates - num_candidates = len(candidates) - acq_val_candidates_ = self._acquisition_function(candidates) + results = Parallel(n_jobs=self._n_jobs)(delayed(self._single_local_search)(sp) for sp in start_points) - if num_candidates == 1: - acq_val_candidates = [acq_val_candidates_[0][0]] - else: - acq_val_candidates = [a[0] for a in acq_val_candidates_] + return results + + def _single_local_search(self, start_point: Configuration) -> tuple[float, Configuration]: + """ + Perform a local search from a single starting configuration. + + The local search iteratively explores the k-exchange neighborhood of the + current candidate configuration. If a neighbor has a better acquisition value, + it becomes the new candidate. Plateau walks are used when neighbors have equal acquisition values. + + + Parameters + ---------- + start_point : Configuration + Starting point for the search. + + Returns + ------- + tuple[float, Configuration] + Candidate with its acquisition function value. (acq value, candidate) + """ + rng = np.random.RandomState(self._rng.randint(low=0, high=10000)) + + candidate = start_point + candidate_list = [candidate] + # Compute the acquisition value of the candidate + if self._acquisition_function is None: + raise ValueError("Acquisition function must be set before running local search.") + + acq_val_candidate = self._acquisition_function(candidate_list)[0][0] # Set up additional variables required to do vectorized local search: - # whether the i-th local search is still running - active = [True] * num_candidates - # number of plateau walks of the i-th local search. Reaching the maximum number is the stopping criterion of + # whether the local search is still running + active = True + # number of plateau walks of the local search. Reaching the maximum number is the stopping criterion of # the local search. - n_no_plateau_walk = [0] * num_candidates + n_no_plateau_walk = 0 # tracking the number of steps for logging purposes - local_search_steps = [0] * num_candidates + local_search_steps = 0 # tracking the number of neighbors looked at for logging purposes - neighbors_looked_at = [0] * num_candidates + neighbors_looked_at = 0 # tracking the number of neighbors generated for logging purposse - neighbors_generated = [0] * num_candidates - # how many neighbors were obtained for the i-th local search. Important to map the individual acquisition + neighbors_generated = 0 + # how many neighbors were obtained for the local search. Important to map the individual acquisition # function values to the correct local search run - obtain_n = [self._vectorization_min_obtain] * num_candidates + obtain_n = self._vectorization_min_obtain # Tracking the time it takes to compute the acquisition function times = [] - # Set up the neighborhood generators - neighborhood_iterators = [] - for i, inc in enumerate(candidates): - neighborhood_iterators.append( - # get_one_exchange_neighbourhood implementational details: - # https://github.com/automl/ConfigSpace/blob/05ab3da2a06c084ba920e8e4e3f62f2e87e81442/ConfigSpace/util.pyx#L95 - # Return all configurations in a one-exchange neighborhood. - # - # The method is implemented as defined by: - # Frank Hutter, Holger H. Hoos and Kevin Leyton-Brown - # Sequential Model-Based Optimization for General Algorithm Configuration - # In Proceedings of the conference on Learning and Intelligent - # Optimization(LION 5) - get_one_exchange_neighbourhood(inc, seed=self._rng.randint(low=0, high=100000)) - ) - local_search_steps[i] += 1 + local_search_steps += 1 + neighbors_w_equal_acq: list[Configuration] = [] - # Keeping track of configurations with equal acquisition value for plateau walking - neighbors_w_equal_acq: list[list[Configuration]] = [[] for _ in range(num_candidates)] + hp_names = list(candidate.config_space.keys()) num_iters = 0 - while np.any(active): + while active: # If the maximum number of steps is reached, stop the local search if num_iters is not None and num_iters == self._max_steps: break num_iters += 1 + + # Compute standard deviation based on Regis and Shoemaker (2013) + # TODO: Maybe _max_steps should not be used and instead a fitting constant + if self._max_steps is not None: + sigma_t = self._base_sigma * (1 - np.log(num_iters + 1) / np.log(self._max_steps + 1)) + else: + sigma_t = self._base_sigma + + hp_names = list(candidate.config_space.keys()) + + # Set up the neighborhood generator + if self._exchange_size == 1: + neighborhood_iterator = get_one_exchange_neighbourhood( + candidate, + seed=rng.randint(low=0, high=100000), + stdev=sigma_t, + ) + elif self._exchange_size > 1: + neighborhood_iterator = get_k_exchange_neighbourhood( + candidate, + seed=rng.randint(low=0, high=100000), + stdev=sigma_t, + exchange_size=self._exchange_size, + ) + # Whether the i-th local search improved. When a new neighborhood is generated, this is used to determine # whether a step was made (improvement) or not (iterator exhausted) - improved = [False] * num_candidates + improved = False # Used to request a new neighborhood for the candidates of the i-th local search - new_neighborhood = [False] * num_candidates + new_neighborhood = False + exhausted_hp = set() + regen_count = {hp: 0 for hp in candidate.config_space} # gather all neighbors neighbors = [] - for i, neighborhood_iterator in enumerate(neighborhood_iterators): - if active[i]: - neighbors_for_i = [] - for j in range(obtain_n[i]): - try: - n = next(neighborhood_iterator) - neighbors_generated[i] += 1 - neighbors_for_i.append(n) - except ValueError as e: - # `neighborhood_iterator` raises `ValueError` with some probability when it reaches - # an invalid configuration. - logger.debug(e) - new_neighborhood[i] = True - except StopIteration: - new_neighborhood[i] = True - break - obtain_n[i] = len(neighbors_for_i) - neighbors.extend(neighbors_for_i) - if len(neighbors) != 0: - start_time = time.time() - acq_val = self._acquisition_function(neighbors) - end_time = time.time() - times.append(end_time - start_time) - if np.ndim(acq_val.shape) == 0: - acq_val = np.asarray([acq_val]) - - # Comparing the acquisition function of the neighbors with the acquisition value of the candidate - acq_index = 0 - # Iterating the all i local searches - for i in range(num_candidates): - if not active[i]: + for _ in range(obtain_n): + try: + n = next(neighborhood_iterator) + + # Lists containing each hyperparameter that was changed by the neighborhood_iterator + changed_hp_idx = (n.get_array() != candidate.get_array()).nonzero()[0] + changed_hp_names = [hp_names[i] for i in changed_hp_idx] + + for hp_name in changed_hp_names: + regen_count[hp_name] = regen_count.get(hp_name, 0) + 1 + node = candidate.config_space[hp_name] + + # number of possible values for this hypeparameter + n_values = ( + len(node.choices) + if isinstance(node, CategoricalHyperparameter) + else node.size + if isinstance(node, UniformIntegerHyperparameter) + else len(node.sequence) + if isinstance(node, OrdinalHyperparameter) + else np.inf + ) + + # Stop adding neighbors that adjust this hyperparameter, + # as all possible configurations were probably tried already + if n_values <= 1.5 * regen_count[hp_name]: + exhausted_hp.add(hp_name) + + if all(hp in exhausted_hp for hp in changed_hp_names): continue - # And for each local search we know how many neighbors we obtained - for j in range(obtain_n[i]): - # The next line is only true if there was an improvement and we basically need to iterate to - # the i+1-th local search - if improved[i]: - acq_index += 1 - else: - neighbors_looked_at[i] += 1 - - # Found a better configuration - if acq_val[acq_index] > acq_val_candidates[i]: - is_valid = False - try: - neighbors[acq_index].check_valid_configuration() - is_valid = True - except (ValueError, ForbiddenValueError) as e: - logger.debug("Local search %d: %s", i, e) - - if is_valid: - # We comment this as it just spams the log - # logger.debug( - # "Local search %d: Switch to one of the neighbors (after %d configurations).", - # i, - # neighbors_looked_at[i], - # ) - candidates[i] = neighbors[acq_index] - acq_val_candidates[i] = acq_val[acq_index] - new_neighborhood[i] = True - improved[i] = True - local_search_steps[i] += 1 - neighbors_w_equal_acq[i] = [] - obtain_n[i] = 1 - # Found an equally well performing configuration, keeping it for plateau walking - elif acq_val[acq_index] == acq_val_candidates[i]: - neighbors_w_equal_acq[i].append(neighbors[acq_index]) - - acq_index += 1 - - # Now we check whether we need to create new neighborhoods and whether we need to increase the number of - # plateau walks for one of the local searches. Also disables local searches if the number of plateau walks - # is reached (and all being switched off is the termination criterion). - for i in range(num_candidates): - if not active[i]: - continue - - if obtain_n[i] == 0 or improved[i]: - obtain_n[i] = 2 - else: - obtain_n[i] = obtain_n[i] * 2 - obtain_n[i] = min(obtain_n[i], self._vectorization_max_obtain) - - if new_neighborhood[i]: - if not improved[i] and n_no_plateau_walk[i] < self._n_steps_plateau_walk: - if len(neighbors_w_equal_acq[i]) != 0: - candidates[i] = neighbors_w_equal_acq[i][0] - neighbors_w_equal_acq[i] = [] - n_no_plateau_walk[i] += 1 - if n_no_plateau_walk[i] >= self._n_steps_plateau_walk: - active[i] = False - continue + neighbors_generated += 1 + neighbors.append(n) + except ValueError as e: + # `neighborhood_iterator` raises `ValueError` with some probability when it reaches + # an invalid configuration. + logger.debug(e) + new_neighborhood = True + except StopIteration: + new_neighborhood = True + break + obtain_n = len(neighbors) + if len(neighbors) > 0: + start_time = time.time() + acq_val = self._acquisition_function(neighbors) + times.append(time.time() - start_time) - neighborhood_iterators[i] = get_one_exchange_neighbourhood( - candidates[i], - seed=self._rng.randint(low=0, high=100000), - ) + for idx, neighbor in enumerate(neighbors): + neighbors_looked_at += 1 + val = acq_val[idx][0] + if val > acq_val_candidate: + try: + neighbor.check_valid_configuration() + candidate = neighbor + acq_val_candidate = val + new_neighborhood = True + improved = True + local_search_steps += 1 + neighbors_w_equal_acq = [] + obtain_n = 1 + break + except (ValueError, ForbiddenValueError) as e: + logger.debug("Local search: %s", e) + elif val == acq_val_candidate: + neighbors_w_equal_acq.append(neighbor) + if obtain_n == 0 or improved: + obtain_n = 2 + else: + obtain_n = min(obtain_n * 2, self._vectorization_max_obtain) + if new_neighborhood: + if not improved and n_no_plateau_walk < self._n_steps_plateau_walk: + if len(neighbors_w_equal_acq) > 0: + candidate = neighbors_w_equal_acq[0] + neighbors_w_equal_acq = [] + n_no_plateau_walk += 1 + if n_no_plateau_walk >= self._n_steps_plateau_walk: + active = False + break logger.debug( "Local searches took %s steps and looked at %s configurations. Computing the acquisition function in " @@ -454,4 +500,4 @@ def _search( np.mean(times), ) - return [(a, i) for a, i in zip(acq_val_candidates, candidates)] + return acq_val_candidate, candidate diff --git a/smac/utils/configspace.py b/smac/utils/configspace.py index 01d419321..f326edb43 100644 --- a/smac/utils/configspace.py +++ b/smac/utils/configspace.py @@ -1,8 +1,11 @@ from __future__ import annotations +from typing import Iterator + import hashlib import logging from functools import partial +from itertools import combinations import numpy as np from ConfigSpace import Configuration, ConfigurationSpace @@ -11,6 +14,7 @@ BetaIntegerHyperparameter, CategoricalHyperparameter, Constant, + Hyperparameter, IntegerHyperparameter, NormalFloatHyperparameter, NormalIntegerHyperparameter, @@ -19,8 +23,10 @@ UniformFloatHyperparameter, UniformIntegerHyperparameter, ) +from ConfigSpace.types import f64 from ConfigSpace.util import ( ForbiddenValueError, + change_hp_value, deactivate_inactive_hyperparameters, get_one_exchange_neighbourhood, ) @@ -243,6 +249,271 @@ def transform_continuous_designs( return configs +def get_k_exchange_neighbourhood( + configuration: Configuration, + seed: int | np.random.RandomState, + num_neighbors: int = 4, + stdev: float = 0.2, + exchange_size: int = 1, +) -> Iterator[Configuration]: + """Generate Configurations in the k-exchange neighborhood of a given configuration. + + Each neighbor is obtained by modifying 'exchange_size' hyperparameters in the original + configuration. Continous/integer hyperparameters are sampled around the current value + using a Gaussian distribution, while categorical/ordinal hyperparameters are sampled from + their discrete neighbors. + + + Parameters + ---------- + configuration: Configuration + Configuration for which neighbors are computed. + seed: int | np.random.RandomState + Sets the random seed to a fixed value. + num_neighbors: int + Number of configurations, which are sampled from the neighbourhood of the input configuration. + stdev: float + Standard deviation used for neighborhood sampling. + exchange_size: int + Number of hyperparameters to modify for each neighbor. + + Returns + ------- + Iterator[Configuration] + Iterator over neighbor configurations + """ + OVER_SAMPLE_CONTINUOUS_MULT = 5 + space = configuration.config_space + config = configuration + arr = configuration._vector + dag = space._dag + + # neighbor_sample_size: How many neighbors we should sample for a given + # hyperparameter at once. + # max_iter_per_selection: How many times we loop trying to generate a valid + # configuration with a given hyperparameter, every time it gets sampled. If + # not a single valid configuration is generated in this many iterations, it's + # marked as failed. + # std: The standard deviation to use for the neighborhood of a hyperparameter when + # sampling neighbors. + # should_shuffle: Whether or not we should shuffle the neighbors of a hyperparameter + # once generated + # generated: Whether or not we have already generated the neighbors for this + # hyperparameter, set to false until sampled. + # should_regen: Whether or not we should regenerate more neighbors for this + # hyperparameter at all. + # -> dict[HP, (neighbor_sample_size, std, should_shuffle, generated, should_regen)] + sample_strategy: dict[str, tuple[int, int, float | None, bool, bool, bool]] = {} + + # n_to_gen: Per hyperparameter, how many configurations we should generate with this + # hyperparameter as the one where the values change. + # neighbors_generated_for_hp: The neighbors that were generated for this hp that can + # be retrieved. + # -> tuple[HP, hp_idx, n_to_gen, neighbors_generated_for_hp] + neighbors_to_generate: list[tuple[Hyperparameter, int, int, list[f64]]] = [] + + nan_hps = np.isnan(arr) + UFH = UniformFloatHyperparameter + UIH = UniformIntegerHyperparameter + n_randints_to_gen = 0 + for hp_name, node in dag.nodes.items(): + hp = node.hp + hp_idx = node.idx + + # inactive hyperparameters skipped + # hps with a size of one can't be modified to a neighbor + # This catches Constants, single value categoricals and ordinals (ints?) + if hp.size == 1 or nan_hps[hp_idx]: + continue + + if isinstance(hp, CategoricalHyperparameter): + neighbor_sample_size = hp.size - 1 + # NOTE: We ignore argument `num_neighbors` for Categoricals, + # don't know why + n_to_gen = neighbor_sample_size + max_iter_per_selection = neighbor_sample_size + _std = None + should_shuffle = True + should_regen = False + elif isinstance(hp, OrdinalHyperparameter): + neighbor_sample_size = int(hp.get_num_neighbors(config[hp_name])) + # NOTE: We can only generate maximum 2 neighbors for Ordinals + # so we just generate all possible ones. + _std = None + n_to_gen = neighbor_sample_size + max_iter_per_selection = neighbor_sample_size + should_shuffle = True + should_regen = False + elif np.isinf(hp.size): # All continuous ones + # We can oversample here as there are an infinite number of unique neighbors + # by oversampling, we can hopefully avoid regeneration of neighbors. + neighbor_sample_size = num_neighbors * OVER_SAMPLE_CONTINUOUS_MULT + n_to_gen = num_neighbors + # NOTE: Not sure it should be this high without increasing the std of + # neighborhood sampling. + max_iter_per_selection = max(neighbor_sample_size, 100) + _std = stdev if isinstance(hp, UFH) else None + should_shuffle = False + should_regen = True + else: # All non-continuous ones + # We don't want to over sample a finite hyperparameter as by specifying + # a large number of neighbors, we end up sampling the entire hyperparameter + # range, not just it's immediate neighbors. + _possible_neighbors = int(hp.size - 1) + neighbor_sample_size = int(min(num_neighbors, _possible_neighbors)) + n_to_gen = num_neighbors + # NOTE: Not sure it should be this high without increasing the std of + # neighborhood sampling. + max_iter_per_selection = neighbor_sample_size + _std = stdev if isinstance(hp, UIH) else None + should_shuffle = True + should_regen = _possible_neighbors >= num_neighbors + + n_forbiddens_on_hp = len(dag.forbidden_lookup.get(hp_name, [])) + hueristic_multiplier = 1 + np.sqrt(n_forbiddens_on_hp) + n_randints_to_gen += int(n_to_gen * hueristic_multiplier) + + generated = False + sample_strategy[hp_name] = ( + neighbor_sample_size, + max_iter_per_selection, + _std, + should_shuffle, + generated, + should_regen, + ) + neighbors_to_generate.append((hp, hp_idx, n_to_gen, [])) + + random = np.random.RandomState(seed) if isinstance(seed, int) else seed + arr = config.get_array() + + if len(neighbors_to_generate) == 0: + return + + assert not any(n_to_gen == 0 for _, _, n_to_gen, _ in neighbors_to_generate) + + # Compose a finite set of hyperparameter index combinations + n_hps = len(neighbors_to_generate) + k = min(exchange_size, n_hps) + + # Store each possible combination of k hyperparameters and shuffle for randomness + combos = list(combinations(range(n_hps), k)) + random.shuffle(combos) + + # cap for guaranteed finite termination + MAX_TOTAL_NEIGHBORS = max(1, num_neighbors) * max(1, len(combos)) + neighbors_generated_total = 0 + + # For each possible combination generate up to num_neighbors neighbors + for combo in combos: + # break after generating fixed amount of neighbors + if neighbors_generated_total >= MAX_TOTAL_NEIGHBORS: + break + + # For each hp in this combination, keep local neighbor pool + # (Tied to original get_one_exchange_neighbourhood logic) + for n_gen_round in range(num_neighbors): + if neighbors_generated_total >= MAX_TOTAL_NEIGHBORS: + break + + # We attempt to build one neighbor that modifies all HPs in this combination + # For each hp in combo, ensure it has a local neighbor pool + local_neighbors_list = {} + failed_hp = False + + # For each hp in the combo, ensure its local neighbor pool is filled + for chosen_hp_idx in combo: + hp, hp_idx, n_left, pool = neighbors_to_generate[chosen_hp_idx] + hp_name = hp.name + + ( + neighbor_sample_size, + max_iter_per_selection, + _std, + _should_shuffle, + _generated, + _should_regen, + ) = sample_strategy[hp_name] + + # If pool is empty try to generate neighbors using original logic from get_one_exchange_neighbourhood + if len(pool) == 0: + # If we've generated before and we should not regen, mark this hp as exhausted + if _generated and not _should_regen: + failed_hp = True # No neighbors available for this hp + break + + vec = arr[hp_idx] + _neighbors = hp._neighborhood(vec, n=neighbor_sample_size, seed=random, std=_std) + + if _should_shuffle: + random.shuffle(_neighbors) + + pool = _neighbors.tolist() + # Update global entry such that future combos can use it + neighbors_to_generate[chosen_hp_idx] = (hp, hp_idx, n_left, pool) + sample_strategy[hp_name] = ( + neighbor_sample_size, + max_iter_per_selection, + _std, + _should_shuffle, + True, # generated flag + _should_regen, + ) + + # We failed generating neighbors for this hp + if len(pool) == 0: + failed_hp = True + break + + local_neighbors_list[chosen_hp_idx] = pool + + if failed_hp: + # Try next hp if this one failed + continue + + # Pick one neighbor value per hp in combo (pop from their pools) + new_arr = arr.copy() + for chosen_hp_idx in combo: + hp, hp_idx, n_left, pool = neighbors_to_generate[chosen_hp_idx] + hp_name = hp.name + + # Pop one neighbor value for this hp + neighbor_vector_val = pool.pop() + + # Update global pool entry + neighbors_to_generate[chosen_hp_idx] = (hp, hp_idx, n_left, pool) + + # use change_hp_value to map new hp value properly into new_arr + new_arr = change_hp_value( + configuration_space=space, + configuration_array=new_arr, + hp_name=hp_name, + hp_value=neighbor_vector_val, + index=hp_idx, + ) + + # Check forbidden constraints + is_valid = True + for forbidden_list in space._dag.forbidden_lookup.values(): + if any(f.is_forbidden_vector(new_arr) for f in forbidden_list): + is_valid = False + break + + if not is_valid: + continue + + neighbors_generated_total += 1 + + # For each hp in combo, mark that we produced one neighbor from its quota + for chosen_hp_idx in combo: + hp, hp_idx, n_left, pool = neighbors_to_generate[chosen_hp_idx] + # reduce n_left, but don't go below 0 + one_less = max(0, n_left - 1) + neighbors_to_generate[chosen_hp_idx] = (hp, hp_idx, one_less, pool) + + yield Configuration(space, vector=new_arr) + + # def check_subspace_points( # X: np.ndarray, # cont_dims: np.ndarray | list = [], From 7cf7d272733f9d0c64e18278846e84cfaea6d2c4 Mon Sep 17 00:00:00 2001 From: Jannis Kastner Date: Wed, 10 Dec 2025 13:39:42 +0100 Subject: [PATCH 2/2] Fix runtime explosion in local search for large parameter spaces Update get_k_exchange_neighbourhood to generate fewer combinations, preventing exponential blow-up in the neighborhood search. --- smac/acquisition/maximizer/local_search.py | 63 ++++--- smac/utils/configspace.py | 196 ++++++--------------- 2 files changed, 97 insertions(+), 162 deletions(-) diff --git a/smac/acquisition/maximizer/local_search.py b/smac/acquisition/maximizer/local_search.py index e05cb3ad2..79b1c1969 100644 --- a/smac/acquisition/maximizer/local_search.py +++ b/smac/acquisition/maximizer/local_search.py @@ -369,38 +369,41 @@ def _single_local_search(self, start_point: Configuration) -> tuple[float, Confi hp_names = list(candidate.config_space.keys()) - num_iters = 0 - while active: - - # If the maximum number of steps is reached, stop the local search - if num_iters is not None and num_iters == self._max_steps: - break + first_iteration = True - num_iters += 1 + num_iters = 1 + while active: # Compute standard deviation based on Regis and Shoemaker (2013) - # TODO: Maybe _max_steps should not be used and instead a fitting constant if self._max_steps is not None: sigma_t = self._base_sigma * (1 - np.log(num_iters + 1) / np.log(self._max_steps + 1)) else: sigma_t = self._base_sigma - hp_names = list(candidate.config_space.keys()) - # Set up the neighborhood generator - if self._exchange_size == 1: - neighborhood_iterator = get_one_exchange_neighbourhood( - candidate, - seed=rng.randint(low=0, high=100000), - stdev=sigma_t, - ) - elif self._exchange_size > 1: - neighborhood_iterator = get_k_exchange_neighbourhood( - candidate, - seed=rng.randint(low=0, high=100000), - stdev=sigma_t, - exchange_size=self._exchange_size, - ) + if first_iteration: + if self._exchange_size == 1: + neighborhood_iterator = get_one_exchange_neighbourhood( + candidate, + seed=rng.randint(low=0, high=100000), + stdev=sigma_t, + ) + elif self._exchange_size > 1: + neighborhood_iterator = get_k_exchange_neighbourhood( + candidate, + seed=rng.randint(low=0, high=100000), + stdev=sigma_t, + exchange_size=self._exchange_size, + ) + first_iteration = False + + # If the maximum number of steps is reached, stop the local search + if num_iters is not None and num_iters == self._max_steps: + break + + num_iters += 1 + + hp_names = list(candidate.config_space.keys()) # Whether the i-th local search improved. When a new neighborhood is generated, this is used to determine # whether a step was made (improvement) or not (iterator exhausted) @@ -492,6 +495,20 @@ def _single_local_search(self, start_point: Configuration) -> tuple[float, Confi active = False break + if self._exchange_size == 1: + neighborhood_iterator = get_one_exchange_neighbourhood( + candidate, + seed=rng.randint(low=0, high=100000), + stdev=sigma_t, + ) + elif self._exchange_size > 1: + neighborhood_iterator = get_k_exchange_neighbourhood( + candidate, + seed=rng.randint(low=0, high=100000), + stdev=sigma_t, + exchange_size=self._exchange_size, + ) + logger.debug( "Local searches took %s steps and looked at %s configurations. Computing the acquisition function in " "vectorized for took %f seconds on average.", diff --git a/smac/utils/configspace.py b/smac/utils/configspace.py index f326edb43..8e71fd4cf 100644 --- a/smac/utils/configspace.py +++ b/smac/utils/configspace.py @@ -5,7 +5,6 @@ import hashlib import logging from functools import partial -from itertools import combinations import numpy as np from ConfigSpace import Configuration, ConfigurationSpace @@ -26,7 +25,6 @@ from ConfigSpace.types import f64 from ConfigSpace.util import ( ForbiddenValueError, - change_hp_value, deactivate_inactive_hyperparameters, get_one_exchange_neighbourhood, ) @@ -258,11 +256,10 @@ def get_k_exchange_neighbourhood( ) -> Iterator[Configuration]: """Generate Configurations in the k-exchange neighborhood of a given configuration. - Each neighbor is obtained by modifying 'exchange_size' hyperparameters in the original - configuration. Continous/integer hyperparameters are sampled around the current value - using a Gaussian distribution, while categorical/ordinal hyperparameters are sampled from - their discrete neighbors. - + Each neighbor is obtained by randomly selectin 'exchange_size' hyperparameters + from the original configuration and modifying them: + - Continous/integer hyperparameters are sampled around the current value using a Gaussian distribution + - Categorical/ordinal hyperparameters are sampled from their discrete neighbors. Parameters ---------- @@ -271,11 +268,11 @@ def get_k_exchange_neighbourhood( seed: int | np.random.RandomState Sets the random seed to a fixed value. num_neighbors: int - Number of configurations, which are sampled from the neighbourhood of the input configuration. + Number of neighbors to attempt generating. stdev: float - Standard deviation used for neighborhood sampling. + Standard deviation used for sampling continous/integer hyperparameters. exchange_size: int - Number of hyperparameters to modify for each neighbor. + Number of hyperparameters to modify in each neighbor. Returns ------- @@ -315,21 +312,17 @@ def get_k_exchange_neighbourhood( nan_hps = np.isnan(arr) UFH = UniformFloatHyperparameter UIH = UniformIntegerHyperparameter - n_randints_to_gen = 0 for hp_name, node in dag.nodes.items(): hp = node.hp hp_idx = node.idx - # inactive hyperparameters skipped - # hps with a size of one can't be modified to a neighbor - # This catches Constants, single value categoricals and ordinals (ints?) + # Skip inactive or fixed hyperparameters if hp.size == 1 or nan_hps[hp_idx]: continue + # Determine neighbor sampling strategy per hyperparameter type if isinstance(hp, CategoricalHyperparameter): neighbor_sample_size = hp.size - 1 - # NOTE: We ignore argument `num_neighbors` for Categoricals, - # don't know why n_to_gen = neighbor_sample_size max_iter_per_selection = neighbor_sample_size _std = None @@ -337,49 +330,33 @@ def get_k_exchange_neighbourhood( should_regen = False elif isinstance(hp, OrdinalHyperparameter): neighbor_sample_size = int(hp.get_num_neighbors(config[hp_name])) - # NOTE: We can only generate maximum 2 neighbors for Ordinals - # so we just generate all possible ones. _std = None n_to_gen = neighbor_sample_size max_iter_per_selection = neighbor_sample_size should_shuffle = True should_regen = False - elif np.isinf(hp.size): # All continuous ones - # We can oversample here as there are an infinite number of unique neighbors - # by oversampling, we can hopefully avoid regeneration of neighbors. + elif np.isinf(hp.size): # Continous hyperparameters neighbor_sample_size = num_neighbors * OVER_SAMPLE_CONTINUOUS_MULT n_to_gen = num_neighbors - # NOTE: Not sure it should be this high without increasing the std of - # neighborhood sampling. max_iter_per_selection = max(neighbor_sample_size, 100) _std = stdev if isinstance(hp, UFH) else None should_shuffle = False should_regen = True - else: # All non-continuous ones - # We don't want to over sample a finite hyperparameter as by specifying - # a large number of neighbors, we end up sampling the entire hyperparameter - # range, not just it's immediate neighbors. + else: # Discrete integer hyperparameters _possible_neighbors = int(hp.size - 1) neighbor_sample_size = int(min(num_neighbors, _possible_neighbors)) n_to_gen = num_neighbors - # NOTE: Not sure it should be this high without increasing the std of - # neighborhood sampling. max_iter_per_selection = neighbor_sample_size _std = stdev if isinstance(hp, UIH) else None should_shuffle = True should_regen = _possible_neighbors >= num_neighbors - n_forbiddens_on_hp = len(dag.forbidden_lookup.get(hp_name, [])) - hueristic_multiplier = 1 + np.sqrt(n_forbiddens_on_hp) - n_randints_to_gen += int(n_to_gen * hueristic_multiplier) - - generated = False sample_strategy[hp_name] = ( neighbor_sample_size, max_iter_per_selection, _std, should_shuffle, - generated, + False, should_regen, ) neighbors_to_generate.append((hp, hp_idx, n_to_gen, [])) @@ -396,122 +373,63 @@ def get_k_exchange_neighbourhood( n_hps = len(neighbors_to_generate) k = min(exchange_size, n_hps) - # Store each possible combination of k hyperparameters and shuffle for randomness - combos = list(combinations(range(n_hps), k)) - random.shuffle(combos) + # Generate neighbors until we reach the target number + while True: - # cap for guaranteed finite termination - MAX_TOTAL_NEIGHBORS = max(1, num_neighbors) * max(1, len(combos)) - neighbors_generated_total = 0 - - # For each possible combination generate up to num_neighbors neighbors - for combo in combos: - # break after generating fixed amount of neighbors - if neighbors_generated_total >= MAX_TOTAL_NEIGHBORS: + # Randomly pick 'exchange_size' hyperparameters to modify + # Only from HP's that were not exhausted before + available_indices = [i for i, (_, _, n_left, _) in enumerate(neighbors_to_generate) if n_left > 0] + if len(available_indices) == 0: break - - # For each hp in this combination, keep local neighbor pool - # (Tied to original get_one_exchange_neighbourhood logic) - for n_gen_round in range(num_neighbors): - if neighbors_generated_total >= MAX_TOTAL_NEIGHBORS: - break - - # We attempt to build one neighbor that modifies all HPs in this combination - # For each hp in combo, ensure it has a local neighbor pool - local_neighbors_list = {} - failed_hp = False - - # For each hp in the combo, ensure its local neighbor pool is filled - for chosen_hp_idx in combo: - hp, hp_idx, n_left, pool = neighbors_to_generate[chosen_hp_idx] - hp_name = hp.name - - ( - neighbor_sample_size, - max_iter_per_selection, - _std, - _should_shuffle, - _generated, - _should_regen, - ) = sample_strategy[hp_name] - - # If pool is empty try to generate neighbors using original logic from get_one_exchange_neighbourhood - if len(pool) == 0: - # If we've generated before and we should not regen, mark this hp as exhausted - if _generated and not _should_regen: - failed_hp = True # No neighbors available for this hp - break - + chosen_indices = random.choice(available_indices, size=min(k, len(available_indices)), replace=False) + new_arr = arr.copy() + valid = True + + # Modify each chosen hyperparameter + for idx in chosen_indices: + hp, hp_idx, n_left, pool = neighbors_to_generate[idx] + hp_name = hp.name + neighbor_sample_size, max_iter, _std, shuffle, generated, regen = sample_strategy[hp_name] + + # Generate new neighbors if pool is empty + if len(pool) == 0: + if generated and not regen: + neighbors_to_generate[idx] = (hp, hp_idx, 0, pool) + continue + elif not generated or regen: vec = arr[hp_idx] _neighbors = hp._neighborhood(vec, n=neighbor_sample_size, seed=random, std=_std) - - if _should_shuffle: + if shuffle: random.shuffle(_neighbors) - pool = _neighbors.tolist() - # Update global entry such that future combos can use it - neighbors_to_generate[chosen_hp_idx] = (hp, hp_idx, n_left, pool) - sample_strategy[hp_name] = ( - neighbor_sample_size, - max_iter_per_selection, - _std, - _should_shuffle, - True, # generated flag - _should_regen, - ) - - # We failed generating neighbors for this hp - if len(pool) == 0: - failed_hp = True - break - local_neighbors_list[chosen_hp_idx] = pool - - if failed_hp: - # Try next hp if this one failed - continue - - # Pick one neighbor value per hp in combo (pop from their pools) - new_arr = arr.copy() - for chosen_hp_idx in combo: - hp, hp_idx, n_left, pool = neighbors_to_generate[chosen_hp_idx] - hp_name = hp.name - - # Pop one neighbor value for this hp - neighbor_vector_val = pool.pop() - - # Update global pool entry - neighbors_to_generate[chosen_hp_idx] = (hp, hp_idx, n_left, pool) - - # use change_hp_value to map new hp value properly into new_arr - new_arr = change_hp_value( - configuration_space=space, - configuration_array=new_arr, - hp_name=hp_name, - hp_value=neighbor_vector_val, - index=hp_idx, - ) - - # Check forbidden constraints - is_valid = True - for forbidden_list in space._dag.forbidden_lookup.values(): - if any(f.is_forbidden_vector(new_arr) for f in forbidden_list): - is_valid = False + if len(pool) == 0: + valid = False + break + + sample_strategy[hp_name] = (neighbor_sample_size, max_iter, _std, shuffle, True, regen) + neighbors_to_generate[idx] = (hp, hp_idx, n_left, pool) + else: + valid = False break - if not is_valid: - continue + # pop one neighbor value for this hyperparameter + val = pool.pop() + neighbors_to_generate[idx] = (hp, hp_idx, n_left - 1, pool) + new_arr[hp_idx] = val - neighbors_generated_total += 1 + if not valid: + continue - # For each hp in combo, mark that we produced one neighbor from its quota - for chosen_hp_idx in combo: - hp, hp_idx, n_left, pool = neighbors_to_generate[chosen_hp_idx] - # reduce n_left, but don't go below 0 - one_less = max(0, n_left - 1) - neighbors_to_generate[chosen_hp_idx] = (hp, hp_idx, one_less, pool) + # Check for forbidden constraints + for forbidden_list in dag.forbidden_lookup.values(): + if any(f.is_forbidden_vector(new_arr) for f in forbidden_list): + valid = False + break + if not valid: + continue - yield Configuration(space, vector=new_arr) + yield Configuration(space, vector=new_arr) # def check_subspace_points(