Optimizers - Bayesian Optimization as a Coverage Tool for Evaluating LLM" > Optimizers - Bayesian Optimization as a Coverage Tool for Evaluating LLM" >
Skip to content

Optimizers

bocoel.core.optim

bocoel.Optimizer 

Optimizer(query_eval: QueryEvaluator, boundary: Boundary, **kwargs: Any)

Bases: Protocol

Source code in bocoel/core/optim/interfaces/optim.py 
21
22
23
24
25
def __init__(
    self, query_eval: QueryEvaluator, boundary: Boundary, **kwargs: Any
) -> None:
    # Included s.t. constructors of Index can be used.
    ...

step abstractmethod 

step() -> Mapping[int, float]

Performs a few steps of optimization. Raises StopIteration if done.

Returns

The state change of the optimizer during the step.

Source code in bocoel/core/optim/interfaces/optim.py 
34
35
36
37
38
39
40
41
42
43
44
45
46
@abc.abstractmethod
def step(self) -> Mapping[int, float]:
    """
    Performs a few steps of optimization.
    Raises StopIteration if done.

    Returns
    -------

    The state change of the optimizer during the step.
    """

    ...

save 

save(path: str | Path) -> None

Saves the optimizer to a file.

Parameters

path: str | Path The path to save the optimizer to.

Source code in bocoel/core/optim/interfaces/optim.py 
48
49
50
51
52
53
54
55
56
57
58
59
60
def save(self, path: str | Path) -> None:
    """
    Saves the optimizer to a file.

    Parameters
    ----------

    `path: str | Path`
    The path to save the optimizer to.
    """

    with open(path, "wb") as f:
        pickle.dump({_VERSION_KEY: common.version(), _OPTIMIZER_KEY: self}, f)

bocoel.AxServiceOptimizer 

AxServiceOptimizer(
    query_eval: QueryEvaluator,
    boundary: Boundary,
    *,
    sobol_steps: int = 0,
    device: Device = "cpu",
    workers: int = 1,
    task: Task = Task.EXPLORE,
    acqf: str | AcquisitionFunc = AcquisitionFunc.AUTO,
    surrogate: str | SurrogateModel = SurrogateModel.AUTO,
    surrogate_kwargs: SurrogateOptions | None = None
)

Bases: Optimizer

The Ax optimizer that uses the service API. See https://ax.dev/tutorials/gpei_hartmann_service.html

Source code in bocoel/core/optim/ax/optim.py 
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
def __init__(
    self,
    query_eval: QueryEvaluator,
    boundary: Boundary,
    *,
    sobol_steps: int = 0,
    device: Device = "cpu",
    workers: int = 1,
    task: Task = Task.EXPLORE,
    acqf: str | AcquisitionFunc = AcquisitionFunc.AUTO,
    surrogate: str | SurrogateModel = SurrogateModel.AUTO,
    surrogate_kwargs: SurrogateOptions | None = None,
) -> None:
    acqf = AcquisitionFunc.lookup(acqf)
    task = Task.lookup(task)

    utils.check_acquisition_task_combo(acqf=acqf, task=task)

    self._device = device
    self._acqf = acqf
    self._surrogate = SurrogateModel.lookup(surrogate).surrogate(surrogate_kwargs)
    self._task = task

    self._ax_client = AxClient(generation_strategy=self._gen_strat(sobol_steps))
    self._create_experiment(boundary)

    self._query_eval = query_eval
    self._workers = workers
    self._terminate = False

save 

save(path: str | Path) -> None

Saves the optimizer to a file.

Parameters

path: str | Path The path to save the optimizer to.

Source code in bocoel/core/optim/interfaces/optim.py 
48
49
50
51
52
53
54
55
56
57
58
59
60
def save(self, path: str | Path) -> None:
    """
    Saves the optimizer to a file.

    Parameters
    ----------

    `path: str | Path`
    The path to save the optimizer to.
    """

    with open(path, "wb") as f:
        pickle.dump({_VERSION_KEY: common.version(), _OPTIMIZER_KEY: self}, f)

bocoel.Task

Bases: StrEnum

EXPLORE class-attribute instance-attribute 

EXPLORE = 'EXPLORE'

MINIMIZE class-attribute instance-attribute 

MINIMIZE = 'MINIMIZE'

MAXIMIZE class-attribute instance-attribute 

MAXIMIZE = 'MAXIMIZE'

lookup classmethod 

lookup(name: str | Self) -> Self
Source code in bocoel/common/enums.py 
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
@classmethod
def lookup(cls, name: str | Self) -> Self:
    if isinstance(name, cls):
        return name

    try:
        return cls[name]
    except KeyError:
        pass

    try:
        return cls(name)
    except ValueError:
        pass

    raise ItemNotFound(f"Item not found in enum. Must be one of {list(cls)}")

bocoel.AcquisitionFunc

Bases: StrEnum

ENTROPY class-attribute instance-attribute 

ENTROPY = 'ENTROPY'

MES class-attribute instance-attribute 

MES = 'MAX_VALUE_ENTROPY'

UCB class-attribute instance-attribute 

UCB = 'UPPER_CONFIDENCE_BOUND'

QUCB class-attribute instance-attribute 

QUCB = 'QUASI_UPPER_CONFIDENCE_BOUND'

EI class-attribute instance-attribute 

EI = 'EXPECTED_IMPROVEMENT'

QEI class-attribute instance-attribute 

QEI = 'QUASI_EXPECTED_IMPROVEMENT'

AUTO class-attribute instance-attribute 

AUTO = 'AUTO'

botorch_acqf_class property 

botorch_acqf_class: type[AcquisitionFunction] | None

lookup classmethod 

lookup(name: str | Self) -> Self
Source code in bocoel/common/enums.py 
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
@classmethod
def lookup(cls, name: str | Self) -> Self:
    if isinstance(name, cls):
        return name

    try:
        return cls[name]
    except KeyError:
        pass

    try:
        return cls(name)
    except ValueError:
        pass

    raise ItemNotFound(f"Item not found in enum. Must be one of {list(cls)}")

bocoel.core.optim.ax.surrogates.SurrogateModel

Bases: StrEnum

SAAS class-attribute instance-attribute 

SAAS = 'SAAS'

AUTO class-attribute instance-attribute 

AUTO = 'AUTO'

lookup classmethod 

lookup(name: str | Self) -> Self
Source code in bocoel/common/enums.py 
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
@classmethod
def lookup(cls, name: str | Self) -> Self:
    if isinstance(name, cls):
        return name

    try:
        return cls[name]
    except KeyError:
        pass

    try:
        return cls(name)
    except ValueError:
        pass

    raise ItemNotFound(f"Item not found in enum. Must be one of {list(cls)}")

surrogate 

surrogate(surrogate_options: SurrogateOptions | None) -> Surrogate | None
Source code in bocoel/core/optim/ax/surrogates/supported.py 
25
26
27
28
29
30
31
32
33
34
35
36
def surrogate(self, surrogate_options: SurrogateOptions | None) -> Surrogate | None:
    if surrogate_options is None:
        surrogate_options = {}

    match self:
        case SurrogateModel.AUTO:
            return None
        case SurrogateModel.SAAS:
            return Surrogate(
                botorch_model_class=SaasFullyBayesianSingleTaskGP,
                **surrogate_options,
            )

bocoel.core.optim.ax.surrogates.SurrogateOptions

Bases: TypedDict

mll_class instance-attribute 

mll_class: NotRequired[type[MarginalLogLikelihood]]

mll_options instance-attribute 

mll_options: NotRequired[MLLOptions]

bocoel.KMeansOptimizer 

KMeansOptimizer(
    query_eval: QueryEvaluator,
    boundary: Boundary,
    *,
    batch_size: int,
    embeddings: NDArray,
    model_kwargs: KMeansOptions
)

Bases: ScikitLearnOptimizer

Source code in bocoel/core/optim/sklearn/kmeans.py 
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
def __init__(
    self,
    query_eval: QueryEvaluator,
    boundary: Boundary,
    *,
    batch_size: int,
    embeddings: NDArray,
    model_kwargs: KMeansOptions,
) -> None:
    model = KMeans(**model_kwargs)
    model.fit(embeddings)
    validation.check_is_fitted(model)

    super().__init__(
        query_eval=query_eval,
        boundary=boundary,
        model=model,
        batch_size=batch_size,
    )

save 

save(path: str | Path) -> None

Saves the optimizer to a file.

Parameters

path: str | Path The path to save the optimizer to.

Source code in bocoel/core/optim/interfaces/optim.py 
48
49
50
51
52
53
54
55
56
57
58
59
60
def save(self, path: str | Path) -> None:
    """
    Saves the optimizer to a file.

    Parameters
    ----------

    `path: str | Path`
    The path to save the optimizer to.
    """

    with open(path, "wb") as f:
        pickle.dump({_VERSION_KEY: common.version(), _OPTIMIZER_KEY: self}, f)

bocoel.KMeansOptions

Bases: TypedDict

n_clusters instance-attribute 

n_clusters: int

init instance-attribute 

init: NotRequired[Literal['k-means++', 'random']]

n_init instance-attribute 

n_init: NotRequired[int | Literal['auto']]

tol instance-attribute 

tol: NotRequired[float]

verbose instance-attribute 

verbose: NotRequired[int]

random_state instance-attribute 

random_state: NotRequired[int]

algorithm instance-attribute 

algorithm: NotRequired[Literal['llyod', 'elkan']]

bocoel.KMedoidsOptimizer 

KMedoidsOptimizer(
    query_eval: QueryEvaluator,
    boundary: Boundary,
    *,
    batch_size: int,
    embeddings: NDArray,
    model_kwargs: KMedoidsOptions
)

Bases: ScikitLearnOptimizer

Source code in bocoel/core/optim/sklearn/kmedoids.py 
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
def __init__(
    self,
    query_eval: QueryEvaluator,
    boundary: Boundary,
    *,
    batch_size: int,
    embeddings: NDArray,
    model_kwargs: KMedoidsOptions,
) -> None:
    # Optional dependency.
    from sklearn_extra.cluster import KMedoids

    model = KMedoids(**model_kwargs)
    model.fit(embeddings)
    validation.check_is_fitted(model)

    super().__init__(
        query_eval=query_eval,
        boundary=boundary,
        model=model,
        batch_size=batch_size,
    )

save 

save(path: str | Path) -> None

Saves the optimizer to a file.

Parameters

path: str | Path The path to save the optimizer to.

Source code in bocoel/core/optim/interfaces/optim.py 
48
49
50
51
52
53
54
55
56
57
58
59
60
def save(self, path: str | Path) -> None:
    """
    Saves the optimizer to a file.

    Parameters
    ----------

    `path: str | Path`
    The path to save the optimizer to.
    """

    with open(path, "wb") as f:
        pickle.dump({_VERSION_KEY: common.version(), _OPTIMIZER_KEY: self}, f)

bocoel.KMedoidsOptions

Bases: TypedDict

n_clusters instance-attribute 

n_clusters: int

metrics instance-attribute 

metrics: NotRequired[str]

method instance-attribute 

method: NotRequired[Literal['alternate', 'pam']]

init instance-attribute 

init: NotRequired[Literal['random', 'heuristic', 'kmedoids++', 'build']]

max_iter instance-attribute 

max_iter: NotRequired[int]

random_state instance-attribute 

random_state: NotRequired[int]

bocoel.RandomOptimizer 

RandomOptimizer(
    query_eval: QueryEvaluator,
    boundary: Boundary,
    *,
    samples: int,
    batch_size: int
)

Bases: Optimizer

Source code in bocoel/core/optim/random.py 
17
18
19
20
21
22
23
24
25
26
27
28
29
def __init__(
    self,
    query_eval: QueryEvaluator,
    boundary: Boundary,
    *,
    samples: int,
    batch_size: int,
) -> None:
    LOGGER.info("Instantiating RandomOptimizer", samples=samples)

    self._query_eval = query_eval
    self._boundary = boundary
    self._generator = iter(BatchedGenerator(self._gen_random(samples), batch_size))

save 

save(path: str | Path) -> None

Saves the optimizer to a file.

Parameters

path: str | Path The path to save the optimizer to.

Source code in bocoel/core/optim/interfaces/optim.py 
48
49
50
51
52
53
54
55
56
57
58
59
60
def save(self, path: str | Path) -> None:
    """
    Saves the optimizer to a file.

    Parameters
    ----------

    `path: str | Path`
    The path to save the optimizer to.
    """

    with open(path, "wb") as f:
        pickle.dump({_VERSION_KEY: common.version(), _OPTIMIZER_KEY: self}, f)

bocoel.UniformOptimizer 

UniformOptimizer(
    query_eval: QueryEvaluator,
    boundary: Boundary,
    *,
    grids: Sequence[int],
    batch_size: int
)

Bases: Optimizer

Source code in bocoel/core/optim/uniform.py 
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
def __init__(
    self,
    query_eval: QueryEvaluator,
    boundary: Boundary,
    *,
    grids: Sequence[int],
    batch_size: int,
) -> None:
    LOGGER.info("Instantiating UnfiromOptimizer", grids=grids)

    self._query_eval = query_eval
    self._boundary = boundary

    self._generator = iter(BatchedGenerator(self._gen_locs(grids), batch_size))

    if len(grids) != self._boundary.dims:
        raise ValueError(f"Expected {self._boundary.dims} strides, got {grids}")

save 

save(path: str | Path) -> None

Saves the optimizer to a file.

Parameters

path: str | Path The path to save the optimizer to.

Source code in bocoel/core/optim/interfaces/optim.py 
48
49
50
51
52
53
54
55
56
57
58
59
60
def save(self, path: str | Path) -> None:
    """
    Saves the optimizer to a file.

    Parameters
    ----------

    `path: str | Path`
    The path to save the optimizer to.
    """

    with open(path, "wb") as f:
        pickle.dump({_VERSION_KEY: common.version(), _OPTIMIZER_KEY: self}, f)

bocoel.QueryEvaluator

Bases: Protocol

__call__ abstractmethod 

__call__(query: ArrayLike) -> OrderedDict[int, float]

Evaluates the given batched query. The order of the results must be kept in the original order.

Source code in bocoel/core/optim/interfaces/evals.py 
17
18
19
20
21
22
23
24
@abc.abstractmethod
def __call__(self, query: ArrayLike, /) -> OrderedDict[int, float]:
    """
    Evaluates the given batched query.
    The order of the results must be kept in the original order.
    """

    ...

bocoel.SearchEvaluator

Bases: Protocol