stuff

modcma/asktellcmaes.py

@@ -152,9 +152,9 @@ def tell(self, xi: np.ndarray, fi: float) -> None:
             if self.parameters.population.f[index] == None: # noqa
                 self.parameters.population.f[index] = fi
                 break
-        else:
-            warnings.warn("Repeated call to tell with same xi", UserWarning)
-            self.parameters.population.f[index] = fi
+            else:
+                warnings.warn("Repeated call to tell with same xi", UserWarning)
+                self.parameters.population.f[index] = fi
 
         self.parameters.used_budget += 1
         if len(self.ask_queue) == 0 and (self.parameters.population.f != None).all(): # noqa

modcma/c_maes/__init__.py

@@ -14,4 +14,6 @@
     ModularCMAES
 )
 
-from .cmaescpp.parameters import Settings
+from .cmaescpp.parameters import Settings
+from .asktellcmaes import AskTellCMAES    
+

modcma/c_maes/asktellcmaes.py

@@ -0,0 +1,108 @@
+import warnings
+from collections import deque
+
+import numpy as np
+from . import cmaescpp
+
+
+class AskTellCMAES:
+    def __init__(
+        self,
+        dimension: int = None,
+        settings: cmaescpp.parameters.Settings = None,
+        modules: cmaescpp.parameters.Modules = None,
+        parameters: cmaescpp.Parameters = None,
+    ):
+        if {settings, dimension, parameters} == {None}:
+            raise TypeError(
+                "Either dimension or settings or parametrs should be passed"
+            )
+
+        if parameters is not None:
+            self.cma = cmaescpp.ModularCMAES(parameters)
+        elif settings is not None:
+            self.cma = cmaescpp.ModularCMAES(settings)
+        else:
+            settings = cmaescpp.parameters.Settings(dimension, modules)
+            self.cma = cmaescpp.ModularCMAES(settings)
+
+        self.ask_queue = deque()
+
+    @property
+    def is_ask_queue_empty(self):
+        return len(self.ask_queue) == 0
+
+    def register_individual(self, x: np.ndarray) -> float:
+        self.ask_queue.append(x.reshape(-1, 1))
+        return float("nan")
+
+    def ask(self) -> np.ndarray:
+        """Retrieve the next indivual from the ask_queue.
+
+        If the ask_queue is empty mutate is called in order to fill it.
+
+        Returns
+        -------
+        np.ndarray
+
+        """
+        if self.cma.break_conditions():
+            raise StopIteration("Break conditions reached, ignoring call to: ask")
+
+        if self.is_ask_queue_empty:
+            self.cma.mutate(self.register_individual)
+        return self.ask_queue.popleft()
+
+    def tell(self, xi: np.ndarray, fi: float):
+        """Process a provided fitness value fi for a given individual xi.
+
+        Parameters
+        ----------
+        xi: np.ndarray
+            An individual previously returned by ask()
+        fi: float
+            The fitness value for xi
+        Raises
+        ------
+        RuntimeError
+            When ask() is not called before tell()
+        ValueError
+            When an unknown xi is provided to the method
+
+        Warns
+        -----
+        UserWarning
+            When the same xi is provided more than once
+
+        """
+
+        if self.cma.break_conditions():
+            raise StopIteration("Break conditions reached, ignoring call to: tell ")
+
+        if self.is_ask_queue_empty:
+            pass
+
+        indices, *_ = np.where((self.cma.p.pop.X == xi).all(axis=0))
+        if len(indices) == 0:
+            breakpoint()
+            raise ValueError("Unkown xi provided")
+
+        f_copy = self.cma.p.pop.f.copy()
+        for index in indices:
+            if np.isnan(self.cma.p.pop.f[index]):
+                f_copy[index] = fi
+                break
+            else:
+                warnings.warn("Repeated call to tell with same xi", UserWarning)
+                f_copy[index] = fi
+
+        self.cma.p.pop.f = f_copy
+
+        if self.is_ask_queue_empty and not np.isnan(f_copy).any():
+            self.cma.select()
+            self.cma.recombine()
+            self.cma.adapt(self.register_individual) # this needs 
+            self.cma.mutate(self.register_individual)
+
+        # breakpoint()
+

modcma/modularcmaes.py

@@ -117,8 +117,6 @@ def mutate(self) -> None:
 
         self.parameters.population = Population(x, y, z, f, s)
 
-        # print(self.parameters.population.f)
-        # breakpoint()
 
     def select(self) -> None:
         """Selection of best individuals in the population.

scripts/discrepancy/calc_eaf.py

@@ -9,7 +9,7 @@
 import numpy as np
 import pandas as pd
 
-ROOT = os.path.realpath(os.path.dirname(os.path.dirname(__file__)))
+ROOT = os.path.realpath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
 DATA_PATH = os.path.join(ROOT, "data") 
 N_CPU = 8
 EAF_MIN_LOG_TGT = -8
@@ -70,8 +70,8 @@ def get_eaf_table(fname, tgt):
 
 
 def find_files(src):
-    files_orig = glob.glob(f"{src}/*-1/*/IOHprofiler_f*.dat")
-    return files_orig
+    files = glob.glob(f"{src}/*OPT-cache-64*/*/IOHprofiler_f*5.dat")
+    return files
 
 if __name__ == '__main__':
     parser = ArgumentParser()
@@ -89,7 +89,6 @@ def find_files(src):
     #     os.remove(f"{tgt}/eaf.db")
 
     files = find_files(src)
-
     auc_func = partial(get_eaf_table, tgt=tgt)
     results = run_parallel_function(auc_func, files)
 

scripts/discrepancy/lambda_is_cache.py

@@ -0,0 +1,110 @@
+import time
+import argparse
+
+import ioh
+import numpy as np
+from scipy.stats.qmc import discrepancy
+from scipy.stats import norm
+
+from modcma.c_maes import ModularCMAES, parameters, options, constants, utils, sampling
+
+
+class Stats:
+    cache_discrepancy = float("inf")
+    population_discrepancy = float("inf")
+    selected_discrepancy = float("inf")
+
+
+
+if __name__ == "__main__":
+    """
+    data/pointsets/OptFib_2_16.txt
+    data/pointsets/Sub_Sobol_10_16.txt
+    data/pointsets/Sub_Sobol_20_16.txt
+    data/pointsets/Sub_Sobol_40_16.txt
+    data/pointsets/Sub_Sobol_5_16.txt
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--path", type=str, default=None)
+    parser.add_argument("--fid", type=int, default=0)
+    parser.add_argument("--dim", type=int, default=2)
+    parser.add_argument("--budget", type=int, default=10_000)
+    parser.add_argument("--logged", action='store_true')
+    parser.add_argument("--sampler", type=int, default=0, choices=(0, 1, 2))
+    parser.add_argument("--lamb", type=int, default=16)
+
+    args = parser.parse_args()
+
+    if args.fid != 0:
+        fids = [args.fid]
+    else:
+        fids = list(range(1, 25))
+
+    modules = parameters.Modules()
+    if args.path: 
+        points = []
+        with open(args.path) as f:
+            next(f)
+            for line in f:
+                points.append(list(map(float, line.strip().split())))
+
+        points = np.array(points)
+        cache_size, dim = points.shape
+
+        constants.cache_max_doubles = 0
+        constants.cache_min_samples = cache_size
+        constants.cache_samples = True
+        algorithm_name = f"CMA-ES-OPT-cache-{cache_size}-lambda{args.lamb}"
+    else:
+        dim = args.dim
+        sampler = options.BaseSampler(args.sampler)
+        algorithm_name = f"CMA-ES-{sampler.name}-lambda{args.lamb}"
+        modules.sampler = sampler
+
+
+    if args.logged:
+        logger = ioh.logger.Analyzer(
+            root="data",
+            folder_name=algorithm_name,
+            algorithm_name=algorithm_name
+        )
+        logger.add_run_attributes(Stats, ["cache_discrepancy"])   
+
+
+    for fid in fids:
+        for iid in range(1, 101):
+            utils.set_seed(fid * dim * iid)
+            np.random.seed(fid * dim * iid)
+            problem = ioh.get_problem(fid, iid, dim)
+            if args.logged:
+                problem.attach_logger(logger)
+
+            start = time.perf_counter()
+            settings = parameters.Settings(
+                problem.meta_data.n_variables,
+                x0=np.random.uniform(-4, 4, size=dim),
+                sigma0=(problem.bounds.ub[0] - problem.bounds.lb[0]) *.2,
+                budget=problem.meta_data.n_variables * args.budget,
+                target=problem.optimum.y + 1e-8,
+                lambda0=args.lamb,
+                modules=modules
+            )
+
+            cma = ModularCMAES(settings)
+
+            if args.path:
+                cma.p.sampler = sampling.CachedSampler(points[np.random.permutation(len(points))], True)
+                cached_sample = norm.cdf(np.vstack([cma.p.sampler() for _ in range(cache_size)]))
+                Stats.cache_discrepancy = discrepancy(cached_sample, method="L2-star")
+                print("cache discrepancy: ", Stats.cache_discrepancy)
+
+            cma.run(problem)
+
+            print(
+                problem.meta_data, 
+                problem.state.evaluations, 
+                problem.state.final_target_found, 
+                problem.state.current_best_internal .y,
+                time.perf_counter() - start                        
+            )
+            problem.reset()

scripts/discrepancy/run_opt_p_sets.py

@@ -0,0 +1,93 @@
+import time
+import argparse
+
+import ioh
+import numpy as np
+from scipy.stats.qmc import discrepancy
+from scipy.stats import norm
+
+from modcma.c_maes import ModularCMAES, parameters, options, constants, utils, sampling
+
+
+class Stats:
+    cache_discrepancy = float("inf")
+    population_discrepancy = float("inf")
+    selected_discrepancy = float("inf")
+
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("path")
+    parser.add_argument("--fid", type=int, default=0)
+
+    parser.add_argument("--budget", type=int, default=10_000)
+    parser.add_argument("--logged", action='store_true')
+
+    args = parser.parse_args()
+
+    if args.fid != 0:
+        fids = [args.fid]
+    else:
+        fids = list(range(1, 25))
+
+
+    points = []
+    with open(args.path) as f:
+        next(f)
+        for line in f:
+            points.append(list(map(float, line.strip().split())))
+
+    points = np.array(points)
+    cache_size, dim = points.shape
+
+    constants.cache_max_doubles = 0
+    constants.cache_min_samples = cache_size
+    constants.cache_samples = True
+    algorithm_name = f"CMA-ES-OPT-cache-{cache_size}"
+
+
+    if args.logged:
+        logger = ioh.logger.Analyzer(
+            root="data",
+            folder_name=algorithm_name,
+            algorithm_name=algorithm_name
+        )
+        logger.add_run_attributes(Stats, ["cache_discrepancy"])   
+
+
+    for fid in fids:
+        for iid in range(1, 101):
+            utils.set_seed(fid * dim * iid)
+            np.random.seed(fid * dim * iid)
+            problem = ioh.get_problem(fid, iid, dim)
+            if args.logged:
+                problem.attach_logger(logger)
+
+            start = time.perf_counter()
+            settings = parameters.Settings(
+                problem.meta_data.n_variables,
+                x0=np.random.uniform(-4, 4, size=dim),
+                sigma0=(problem.bounds.ub[0] - problem.bounds.lb[0]) *.2,
+                budget=problem.meta_data.n_variables * args.budget,
+                target=problem.optimum.y + 1e-8,
+            )
+
+            cma = ModularCMAES(settings)
+
+            cma.p.sampler = sampling.CachedSampler(points[np.random.permutation(len(points))], True)
+
+            cached_sample = norm.cdf(np.vstack([cma.p.sampler() for _ in range(cache_size)]))
+            Stats.cache_discrepancy = discrepancy(cached_sample, method="L2-star")
+            print("cache discrepancy: ", Stats.cache_discrepancy)
+
+            cma.run(problem)
+
+            print(
+                problem.meta_data, 
+                problem.state.evaluations, 
+                problem.state.final_target_found, 
+                problem.state.current_best_internal .y,
+                time.perf_counter() - start                        
+            )
+            problem.reset()