Skip to content

Commit

Permalink
stuff
Browse files Browse the repository at this point in the history
  • Loading branch information
jacobdenobel committed Oct 21, 2024
1 parent d399157 commit 47d3277
Show file tree
Hide file tree
Showing 14 changed files with 6,353 additions and 397 deletions.
6 changes: 3 additions & 3 deletions modcma/asktellcmaes.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,9 +152,9 @@ def tell(self, xi: np.ndarray, fi: float) -> None:
if self.parameters.population.f[index] == None: # noqa
self.parameters.population.f[index] = fi
break
else:
warnings.warn("Repeated call to tell with same xi", UserWarning)
self.parameters.population.f[index] = fi
else:
warnings.warn("Repeated call to tell with same xi", UserWarning)
self.parameters.population.f[index] = fi

self.parameters.used_budget += 1
if len(self.ask_queue) == 0 and (self.parameters.population.f != None).all(): # noqa
Expand Down
4 changes: 3 additions & 1 deletion modcma/c_maes/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,6 @@
ModularCMAES
)

from .cmaescpp.parameters import Settings
from .cmaescpp.parameters import Settings
from .asktellcmaes import AskTellCMAES

108 changes: 108 additions & 0 deletions modcma/c_maes/asktellcmaes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
import warnings
from collections import deque

import numpy as np
from . import cmaescpp


class AskTellCMAES:
def __init__(
self,
dimension: int = None,
settings: cmaescpp.parameters.Settings = None,
modules: cmaescpp.parameters.Modules = None,
parameters: cmaescpp.Parameters = None,
):
if {settings, dimension, parameters} == {None}:
raise TypeError(
"Either dimension or settings or parametrs should be passed"
)

if parameters is not None:
self.cma = cmaescpp.ModularCMAES(parameters)
elif settings is not None:
self.cma = cmaescpp.ModularCMAES(settings)
else:
settings = cmaescpp.parameters.Settings(dimension, modules)
self.cma = cmaescpp.ModularCMAES(settings)

self.ask_queue = deque()

@property
def is_ask_queue_empty(self):
return len(self.ask_queue) == 0

def register_individual(self, x: np.ndarray) -> float:
self.ask_queue.append(x.reshape(-1, 1))
return float("nan")

def ask(self) -> np.ndarray:
"""Retrieve the next indivual from the ask_queue.
If the ask_queue is empty mutate is called in order to fill it.
Returns
-------
np.ndarray
"""
if self.cma.break_conditions():
raise StopIteration("Break conditions reached, ignoring call to: ask")

if self.is_ask_queue_empty:
self.cma.mutate(self.register_individual)
return self.ask_queue.popleft()

def tell(self, xi: np.ndarray, fi: float):
"""Process a provided fitness value fi for a given individual xi.
Parameters
----------
xi: np.ndarray
An individual previously returned by ask()
fi: float
The fitness value for xi
Raises
------
RuntimeError
When ask() is not called before tell()
ValueError
When an unknown xi is provided to the method
Warns
-----
UserWarning
When the same xi is provided more than once
"""

if self.cma.break_conditions():
raise StopIteration("Break conditions reached, ignoring call to: tell ")

if self.is_ask_queue_empty:
pass

indices, *_ = np.where((self.cma.p.pop.X == xi).all(axis=0))
if len(indices) == 0:
breakpoint()
raise ValueError("Unkown xi provided")

f_copy = self.cma.p.pop.f.copy()
for index in indices:
if np.isnan(self.cma.p.pop.f[index]):
f_copy[index] = fi
break
else:
warnings.warn("Repeated call to tell with same xi", UserWarning)
f_copy[index] = fi

self.cma.p.pop.f = f_copy

if self.is_ask_queue_empty and not np.isnan(f_copy).any():
self.cma.select()
self.cma.recombine()
self.cma.adapt(self.register_individual) # this needs
self.cma.mutate(self.register_individual)

# breakpoint()

2 changes: 0 additions & 2 deletions modcma/modularcmaes.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,6 @@ def mutate(self) -> None:

self.parameters.population = Population(x, y, z, f, s)

# print(self.parameters.population.f)
# breakpoint()

def select(self) -> None:
"""Selection of best individuals in the population.
Expand Down
7 changes: 3 additions & 4 deletions scripts/discrepancy/calc_eaf.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import numpy as np
import pandas as pd

ROOT = os.path.realpath(os.path.dirname(os.path.dirname(__file__)))
ROOT = os.path.realpath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
DATA_PATH = os.path.join(ROOT, "data")
N_CPU = 8
EAF_MIN_LOG_TGT = -8
Expand Down Expand Up @@ -70,8 +70,8 @@ def get_eaf_table(fname, tgt):


def find_files(src):
files_orig = glob.glob(f"{src}/*-1/*/IOHprofiler_f*.dat")
return files_orig
files = glob.glob(f"{src}/*OPT-cache-64*/*/IOHprofiler_f*5.dat")
return files

if __name__ == '__main__':
parser = ArgumentParser()
Expand All @@ -89,7 +89,6 @@ def find_files(src):
# os.remove(f"{tgt}/eaf.db")

files = find_files(src)

auc_func = partial(get_eaf_table, tgt=tgt)
results = run_parallel_function(auc_func, files)

745 changes: 466 additions & 279 deletions scripts/discrepancy/discrepancy.ipynb

Large diffs are not rendered by default.

110 changes: 110 additions & 0 deletions scripts/discrepancy/lambda_is_cache.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
import time
import argparse

import ioh
import numpy as np
from scipy.stats.qmc import discrepancy
from scipy.stats import norm

from modcma.c_maes import ModularCMAES, parameters, options, constants, utils, sampling


class Stats:
cache_discrepancy = float("inf")
population_discrepancy = float("inf")
selected_discrepancy = float("inf")



if __name__ == "__main__":
"""
data/pointsets/OptFib_2_16.txt
data/pointsets/Sub_Sobol_10_16.txt
data/pointsets/Sub_Sobol_20_16.txt
data/pointsets/Sub_Sobol_40_16.txt
data/pointsets/Sub_Sobol_5_16.txt
"""
parser = argparse.ArgumentParser()
parser.add_argument("--path", type=str, default=None)
parser.add_argument("--fid", type=int, default=0)
parser.add_argument("--dim", type=int, default=2)
parser.add_argument("--budget", type=int, default=10_000)
parser.add_argument("--logged", action='store_true')
parser.add_argument("--sampler", type=int, default=0, choices=(0, 1, 2))
parser.add_argument("--lamb", type=int, default=16)

args = parser.parse_args()

if args.fid != 0:
fids = [args.fid]
else:
fids = list(range(1, 25))

modules = parameters.Modules()
if args.path:
points = []
with open(args.path) as f:
next(f)
for line in f:
points.append(list(map(float, line.strip().split())))

points = np.array(points)
cache_size, dim = points.shape

constants.cache_max_doubles = 0
constants.cache_min_samples = cache_size
constants.cache_samples = True
algorithm_name = f"CMA-ES-OPT-cache-{cache_size}-lambda{args.lamb}"
else:
dim = args.dim
sampler = options.BaseSampler(args.sampler)
algorithm_name = f"CMA-ES-{sampler.name}-lambda{args.lamb}"
modules.sampler = sampler


if args.logged:
logger = ioh.logger.Analyzer(
root="data",
folder_name=algorithm_name,
algorithm_name=algorithm_name
)
logger.add_run_attributes(Stats, ["cache_discrepancy"])


for fid in fids:
for iid in range(1, 101):
utils.set_seed(fid * dim * iid)
np.random.seed(fid * dim * iid)
problem = ioh.get_problem(fid, iid, dim)
if args.logged:
problem.attach_logger(logger)

start = time.perf_counter()
settings = parameters.Settings(
problem.meta_data.n_variables,
x0=np.random.uniform(-4, 4, size=dim),
sigma0=(problem.bounds.ub[0] - problem.bounds.lb[0]) *.2,
budget=problem.meta_data.n_variables * args.budget,
target=problem.optimum.y + 1e-8,
lambda0=args.lamb,
modules=modules
)

cma = ModularCMAES(settings)

if args.path:
cma.p.sampler = sampling.CachedSampler(points[np.random.permutation(len(points))], True)
cached_sample = norm.cdf(np.vstack([cma.p.sampler() for _ in range(cache_size)]))
Stats.cache_discrepancy = discrepancy(cached_sample, method="L2-star")
print("cache discrepancy: ", Stats.cache_discrepancy)

cma.run(problem)

print(
problem.meta_data,
problem.state.evaluations,
problem.state.final_target_found,
problem.state.current_best_internal .y,
time.perf_counter() - start
)
problem.reset()
93 changes: 93 additions & 0 deletions scripts/discrepancy/run_opt_p_sets.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
import time
import argparse

import ioh
import numpy as np
from scipy.stats.qmc import discrepancy
from scipy.stats import norm

from modcma.c_maes import ModularCMAES, parameters, options, constants, utils, sampling


class Stats:
cache_discrepancy = float("inf")
population_discrepancy = float("inf")
selected_discrepancy = float("inf")



if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("path")
parser.add_argument("--fid", type=int, default=0)

parser.add_argument("--budget", type=int, default=10_000)
parser.add_argument("--logged", action='store_true')

args = parser.parse_args()

if args.fid != 0:
fids = [args.fid]
else:
fids = list(range(1, 25))


points = []
with open(args.path) as f:
next(f)
for line in f:
points.append(list(map(float, line.strip().split())))

points = np.array(points)
cache_size, dim = points.shape

constants.cache_max_doubles = 0
constants.cache_min_samples = cache_size
constants.cache_samples = True
algorithm_name = f"CMA-ES-OPT-cache-{cache_size}"


if args.logged:
logger = ioh.logger.Analyzer(
root="data",
folder_name=algorithm_name,
algorithm_name=algorithm_name
)
logger.add_run_attributes(Stats, ["cache_discrepancy"])


for fid in fids:
for iid in range(1, 101):
utils.set_seed(fid * dim * iid)
np.random.seed(fid * dim * iid)
problem = ioh.get_problem(fid, iid, dim)
if args.logged:
problem.attach_logger(logger)

start = time.perf_counter()
settings = parameters.Settings(
problem.meta_data.n_variables,
x0=np.random.uniform(-4, 4, size=dim),
sigma0=(problem.bounds.ub[0] - problem.bounds.lb[0]) *.2,
budget=problem.meta_data.n_variables * args.budget,
target=problem.optimum.y + 1e-8,
)

cma = ModularCMAES(settings)

cma.p.sampler = sampling.CachedSampler(points[np.random.permutation(len(points))], True)

cached_sample = norm.cdf(np.vstack([cma.p.sampler() for _ in range(cache_size)]))
Stats.cache_discrepancy = discrepancy(cached_sample, method="L2-star")
print("cache discrepancy: ", Stats.cache_discrepancy)

cma.run(problem)

print(
problem.meta_data,
problem.state.evaluations,
problem.state.final_target_found,
problem.state.current_best_internal .y,
time.perf_counter() - start
)
problem.reset()
Loading

0 comments on commit 47d3277

Please sign in to comment.