From d2879b3dc03d6657ce43c98b07f6708ecdd75441 Mon Sep 17 00:00:00 2001 From: Daniel Nowak Date: Thu, 30 Jul 2026 05:00:55 +0200 Subject: [PATCH 1/4] First set of hpo optimizers TO DO: - How to integrate to capymoa to optimize non-moa (java implemented) objects? --- .../AutoClass}/Algorithm.java | 2 +- .../AutoClass}/AutoClass.java | 3 +- .../AutoClass}/BooleanParameter.java | 2 +- .../AutoClass}/CategoricalParameter.java | 3 +- .../HeterogeneousEnsembleAbstract.java | 2 +- .../AutoClass}/IParameter.java | 2 +- .../AutoClass}/IntegerParameter.java | 2 +- .../AutoClass}/NumericalParameter.java | 2 +- .../AutoClass}/OrdinalParameter.java | 2 +- .../AutoClass}/TruncatedNormal.java | 2 +- .../AutoML => AutoML/AutoClass}/settings.json | 0 .../AutoML/BayesianStreamTunerClassifier.java | 745 ++++++++++++++++++ .../AutoML/BayesianStreamTunerRegressor.java | 735 +++++++++++++++++ .../classifiers/AutoML/FSS_SPTClassifier.java | 709 +++++++++++++++++ .../classifiers/AutoML/FSS_SPTRegressor.java | 744 +++++++++++++++++ .../moa/classifiers/AutoML/HPOMethod.java | 128 +++ .../classifiers/AutoML/MESSPTClassifier.java | 675 ++++++++++++++++ .../classifiers/AutoML/MESSPTRegressor.java | 668 ++++++++++++++++ .../moa/classifiers/AutoML/MetricUtils.java | 191 +++++ .../Parameters/CategoricalParameter.java | 55 ++ .../AutoML/Parameters/DoubleParameter.java | 56 ++ .../AutoML/Parameters/IntParameter.java | 56 ++ .../AutoML/Parameters/Parameter.java | 54 ++ .../AutoML/RandomSearchClassifier.java | 631 +++++++++++++++ .../AutoML/RandomSearchRegressor.java | 560 +++++++++++++ .../classifiers/AutoML/SSPTClassifier.java | 728 +++++++++++++++++ .../moa/classifiers/AutoML/SSPTRegressor.java | 711 +++++++++++++++++ .../AutoML/space/ConfigurationSpace.java | 221 ++++++ .../AutoML/space/LearnerAccess.java | 157 ++++ .../AutoML/space/LearnerConfigurator.java | 625 +++++++++++++++ .../AutoML/space/ParameterSpec.java | 126 +++ .../functions/BayesianLinearRegression.java | 324 ++++++++ 32 files changed, 8909 insertions(+), 12 deletions(-) rename moa/src/main/java/moa/classifiers/{meta/AutoML => AutoML/AutoClass}/Algorithm.java (99%) rename moa/src/main/java/moa/classifiers/{meta/AutoML => AutoML/AutoClass}/AutoClass.java (99%) rename moa/src/main/java/moa/classifiers/{meta/AutoML => AutoML/AutoClass}/BooleanParameter.java (98%) rename moa/src/main/java/moa/classifiers/{meta/AutoML => AutoML/AutoClass}/CategoricalParameter.java (97%) rename moa/src/main/java/moa/classifiers/{meta/AutoML => AutoML/AutoClass}/HeterogeneousEnsembleAbstract.java (99%) rename moa/src/main/java/moa/classifiers/{meta/AutoML => AutoML/AutoClass}/IParameter.java (89%) rename moa/src/main/java/moa/classifiers/{meta/AutoML => AutoML/AutoClass}/IntegerParameter.java (98%) rename moa/src/main/java/moa/classifiers/{meta/AutoML => AutoML/AutoClass}/NumericalParameter.java (98%) rename moa/src/main/java/moa/classifiers/{meta/AutoML => AutoML/AutoClass}/OrdinalParameter.java (98%) rename moa/src/main/java/moa/classifiers/{meta/AutoML => AutoML/AutoClass}/TruncatedNormal.java (97%) rename moa/src/main/java/moa/classifiers/{meta/AutoML => AutoML/AutoClass}/settings.json (100%) create mode 100644 moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerClassifier.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerRegressor.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/FSS_SPTClassifier.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/FSS_SPTRegressor.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/HPOMethod.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/MESSPTClassifier.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/MESSPTRegressor.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/MetricUtils.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/Parameters/CategoricalParameter.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/Parameters/DoubleParameter.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/Parameters/IntParameter.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/Parameters/Parameter.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/RandomSearchClassifier.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/RandomSearchRegressor.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/SSPTClassifier.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/SSPTRegressor.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/space/ConfigurationSpace.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/space/LearnerAccess.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/space/LearnerConfigurator.java create mode 100644 moa/src/main/java/moa/classifiers/AutoML/space/ParameterSpec.java create mode 100644 moa/src/main/java/moa/classifiers/functions/BayesianLinearRegression.java diff --git a/moa/src/main/java/moa/classifiers/meta/AutoML/Algorithm.java b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/Algorithm.java similarity index 99% rename from moa/src/main/java/moa/classifiers/meta/AutoML/Algorithm.java rename to moa/src/main/java/moa/classifiers/AutoML/AutoClass/Algorithm.java index 441b68f59..2b655f84a 100755 --- a/moa/src/main/java/moa/classifiers/meta/AutoML/Algorithm.java +++ b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/Algorithm.java @@ -1,4 +1,4 @@ -package moa.classifiers.meta.AutoML; +package moa.classifiers.AutoML.AutoClass; import com.github.javacliparser.Option; import com.github.javacliparser.Options; diff --git a/moa/src/main/java/moa/classifiers/meta/AutoML/AutoClass.java b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/AutoClass.java similarity index 99% rename from moa/src/main/java/moa/classifiers/meta/AutoML/AutoClass.java rename to moa/src/main/java/moa/classifiers/AutoML/AutoClass/AutoClass.java index 94f151fda..330dec1b2 100755 --- a/moa/src/main/java/moa/classifiers/meta/AutoML/AutoClass.java +++ b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/AutoClass.java @@ -1,4 +1,4 @@ -package moa.classifiers.meta.AutoML; +package moa.classifiers.AutoML.AutoClass; import com.github.javacliparser.FileOption; import com.google.gson.Gson; @@ -9,7 +9,6 @@ import moa.classifiers.Classifier; import moa.classifiers.MultiClassClassifier; import moa.classifiers.meta.AdaptiveRandomForestRegressor; -import moa.core.DoubleVector; import moa.core.Measurement; import moa.core.ObjectRepository; import moa.tasks.TaskMonitor; diff --git a/moa/src/main/java/moa/classifiers/meta/AutoML/BooleanParameter.java b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/BooleanParameter.java similarity index 98% rename from moa/src/main/java/moa/classifiers/meta/AutoML/BooleanParameter.java rename to moa/src/main/java/moa/classifiers/AutoML/AutoClass/BooleanParameter.java index 4342668e7..c5ab64edc 100755 --- a/moa/src/main/java/moa/classifiers/meta/AutoML/BooleanParameter.java +++ b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/BooleanParameter.java @@ -1,4 +1,4 @@ -package moa.classifiers.meta.AutoML; +package moa.classifiers.AutoML.AutoClass; import com.yahoo.labs.samoa.instances.Attribute; import java.util.ArrayList; diff --git a/moa/src/main/java/moa/classifiers/meta/AutoML/CategoricalParameter.java b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/CategoricalParameter.java similarity index 97% rename from moa/src/main/java/moa/classifiers/meta/AutoML/CategoricalParameter.java rename to moa/src/main/java/moa/classifiers/AutoML/AutoClass/CategoricalParameter.java index e85e672c8..71928784c 100755 --- a/moa/src/main/java/moa/classifiers/meta/AutoML/CategoricalParameter.java +++ b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/CategoricalParameter.java @@ -1,7 +1,6 @@ -package moa.classifiers.meta.AutoML; +package moa.classifiers.AutoML.AutoClass; import com.yahoo.labs.samoa.instances.Attribute; -import moa.classifiers.meta.AutoML.IParameter; import java.util.ArrayList; import java.util.Arrays; diff --git a/moa/src/main/java/moa/classifiers/meta/AutoML/HeterogeneousEnsembleAbstract.java b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/HeterogeneousEnsembleAbstract.java similarity index 99% rename from moa/src/main/java/moa/classifiers/meta/AutoML/HeterogeneousEnsembleAbstract.java rename to moa/src/main/java/moa/classifiers/AutoML/AutoClass/HeterogeneousEnsembleAbstract.java index 3bcbdbafe..9e73eb992 100755 --- a/moa/src/main/java/moa/classifiers/meta/AutoML/HeterogeneousEnsembleAbstract.java +++ b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/HeterogeneousEnsembleAbstract.java @@ -17,7 +17,7 @@ * along with this program. If not, see . * */ -package moa.classifiers.meta.AutoML; +package moa.classifiers.AutoML.AutoClass; import com.github.javacliparser.FlagOption; import com.github.javacliparser.IntOption; diff --git a/moa/src/main/java/moa/classifiers/meta/AutoML/IParameter.java b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/IParameter.java similarity index 89% rename from moa/src/main/java/moa/classifiers/meta/AutoML/IParameter.java rename to moa/src/main/java/moa/classifiers/AutoML/AutoClass/IParameter.java index 3c3d9af66..d47f454a3 100755 --- a/moa/src/main/java/moa/classifiers/meta/AutoML/IParameter.java +++ b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/IParameter.java @@ -1,4 +1,4 @@ -package moa.classifiers.meta.AutoML; +package moa.classifiers.AutoML.AutoClass; // interface allows us to maintain a single list of parameters public interface IParameter { diff --git a/moa/src/main/java/moa/classifiers/meta/AutoML/IntegerParameter.java b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/IntegerParameter.java similarity index 98% rename from moa/src/main/java/moa/classifiers/meta/AutoML/IntegerParameter.java rename to moa/src/main/java/moa/classifiers/AutoML/AutoClass/IntegerParameter.java index 517e8fe6b..1c9303324 100755 --- a/moa/src/main/java/moa/classifiers/meta/AutoML/IntegerParameter.java +++ b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/IntegerParameter.java @@ -1,4 +1,4 @@ -package moa.classifiers.meta.AutoML; +package moa.classifiers.AutoML.AutoClass; import com.yahoo.labs.samoa.instances.Attribute; diff --git a/moa/src/main/java/moa/classifiers/meta/AutoML/NumericalParameter.java b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/NumericalParameter.java similarity index 98% rename from moa/src/main/java/moa/classifiers/meta/AutoML/NumericalParameter.java rename to moa/src/main/java/moa/classifiers/AutoML/AutoClass/NumericalParameter.java index 8d4fb6fbd..4fe100e19 100755 --- a/moa/src/main/java/moa/classifiers/meta/AutoML/NumericalParameter.java +++ b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/NumericalParameter.java @@ -1,4 +1,4 @@ -package moa.classifiers.meta.AutoML; +package moa.classifiers.AutoML.AutoClass; import com.yahoo.labs.samoa.instances.Attribute; diff --git a/moa/src/main/java/moa/classifiers/meta/AutoML/OrdinalParameter.java b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/OrdinalParameter.java similarity index 98% rename from moa/src/main/java/moa/classifiers/meta/AutoML/OrdinalParameter.java rename to moa/src/main/java/moa/classifiers/AutoML/AutoClass/OrdinalParameter.java index 9a445541e..6d6083aa9 100755 --- a/moa/src/main/java/moa/classifiers/meta/AutoML/OrdinalParameter.java +++ b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/OrdinalParameter.java @@ -1,4 +1,4 @@ -package moa.classifiers.meta.AutoML; +package moa.classifiers.AutoML.AutoClass; import com.yahoo.labs.samoa.instances.Attribute; diff --git a/moa/src/main/java/moa/classifiers/meta/AutoML/TruncatedNormal.java b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/TruncatedNormal.java similarity index 97% rename from moa/src/main/java/moa/classifiers/meta/AutoML/TruncatedNormal.java rename to moa/src/main/java/moa/classifiers/AutoML/AutoClass/TruncatedNormal.java index 192ae1577..935a588d0 100755 --- a/moa/src/main/java/moa/classifiers/meta/AutoML/TruncatedNormal.java +++ b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/TruncatedNormal.java @@ -1,4 +1,4 @@ -package moa.classifiers.meta.AutoML; +package moa.classifiers.AutoML.AutoClass; import org.apache.commons.math3.distribution.NormalDistribution; //import umontreal.iro.lecuyer.probdist.NormalDist; diff --git a/moa/src/main/java/moa/classifiers/meta/AutoML/settings.json b/moa/src/main/java/moa/classifiers/AutoML/AutoClass/settings.json similarity index 100% rename from moa/src/main/java/moa/classifiers/meta/AutoML/settings.json rename to moa/src/main/java/moa/classifiers/AutoML/AutoClass/settings.json diff --git a/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerClassifier.java b/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerClassifier.java new file mode 100644 index 000000000..71875ccd3 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerClassifier.java @@ -0,0 +1,745 @@ +/* + * BayesianStreamTunerClassifier.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML; + +import com.github.javacliparser.FileOption; +import com.github.javacliparser.FlagOption; +import com.github.javacliparser.IntOption; +import com.github.javacliparser.MultiChoiceOption; +import com.yahoo.labs.samoa.instances.Attribute; +import com.yahoo.labs.samoa.instances.DenseInstance; +import com.yahoo.labs.samoa.instances.Instance; +import com.yahoo.labs.samoa.instances.Instances; +import com.yahoo.labs.samoa.instances.InstancesHeader; +import moa.capabilities.CapabilitiesHandler; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.AutoML.Parameters.CategoricalParameter; +import moa.classifiers.AutoML.Parameters.DoubleParameter; +import moa.classifiers.AutoML.Parameters.IntParameter; +import moa.classifiers.AutoML.Parameters.Parameter; +import moa.classifiers.AutoML.space.ConfigurationSpace; +import moa.classifiers.AutoML.space.LearnerConfigurator; +import moa.classifiers.AutoML.space.ParameterSpec; +import moa.classifiers.Classifier; +import moa.classifiers.core.driftdetection.ChangeDetector; +import moa.classifiers.MultiClassClassifier; +import moa.classifiers.functions.BayesianLinearRegression; +import moa.core.InstanceExample; +import moa.core.Measurement; +import moa.core.SizeOf; +import moa.evaluation.BasicClassificationPerformanceEvaluator; +import moa.options.ClassOption; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Random; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Bayesian optimisation over a streaming hyperparameter space: a Bayesian + * linear regression surrogate is fitted on (configuration, stream statistics) + * to observed performance, and each update cycle replaces the worst half of the + * candidate pool by configurations maximising the selected acquisition function. + * + *

Candidates are configured through {@link LearnerConfigurator}, i.e. by + * writing MOA {@link com.github.javacliparser.Option}s, so any MOA classifier + * can be tuned as shipped. + * + *

See details in:
Nilesh Verma, Albert Bifet, Bernhard Pfahringer, + * Maroua Bahri. Bayesian Stream Tuner: Dynamic Hyperparameter Optimization for + * Real-Time Data Streams. In Proceedings of the 31st ACM SIGKDD Conference on + * Knowledge Discovery and Data Mining V.2 (KDD '25), pages 2871-2882, + * DOI: 10.1145/3711896.3736852, ACM, 2025.

+ * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public class BayesianStreamTunerClassifier extends AbstractClassifier + implements MultiClassClassifier, CapabilitiesHandler, Serializable, HPOMethod { + + private static final long serialVersionUID = 1L; + + public FileOption configurationFileOption = new FileOption("configurationFile", 'f', + "Search space in JSON format.", null, ".json", false); + + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + "Number of instances between model update cycles; also the number of recent" + + " instances kept to derive the surrogate's stream statistics.", + 1000, 1, Integer.MAX_VALUE); + + public IntOption numberOfCandidatesOption = new IntOption("numberOfCandidates", 'n', + "Number of candidate models (including default).", 10, 2, Integer.MAX_VALUE); + + public MultiChoiceOption metricOption = new MultiChoiceOption("metric", 'm', + "Metric to optimize the model.", MetricUtils.METRIC_NAMES, MetricUtils.METRIC_DESCRIPTIONS, 4); + + public MultiChoiceOption acquisitionFunctionOption = new MultiChoiceOption( + "acquisitionFunction", 'q', + "Acquisition function for Bayesian optimization.", + new String[]{"PI", "EI", "UCB"}, + new String[]{"Probability of Improvement", "Expected Improvement", "Upper Confidence Bound"}, + 0); + + public FlagOption resetLearningOption = new FlagOption("resetLearning", 'L', + "Reset candidate learning instead of copying internal model state from the best classifier."); + + public FlagOption driftDetectionOption = new FlagOption("driftDetection", 'd', + "Enable drift detection on the incumbent's prediction error."); + + public ClassOption driftDetectorOption = new ClassOption("driftDetector", 'D', + "Change detector to use when drift detection is enabled; on a detected" + + " drift the whole search is reinitialized from the search space.", + ChangeDetector.class, "ADWINChangeDetector"); + + public IntOption numberOfJobsOption = new IntOption("numberOfJobs", 'j', + "Total number of concurrent jobs used for processing (-1 = as much as possible, 0 = do not use multithreading)", + 1, -1, Integer.MAX_VALUE); + + protected static final int SINGLE_THREAD = 0; + + // ---- Model ensemble ---- + protected Classifier[] candidates; + + /** + * Pool training the candidates concurrently; {@code null} when running + * single-threaded. Transient because an executor cannot be serialized, so it + * is (re)created on demand by {@link #initExecutor()}. + */ + protected transient ExecutorService executor; + + /** Boundary through which every learner is configured. */ + protected LearnerConfigurator configurator; + protected BasicClassificationPerformanceEvaluator[] evaluators; + protected ArrayList> candidatesParameters; + protected int bestCandidateIndex; + protected long instanceCount; + protected String algorithmCLIString; + + // ---- BLR surrogate ---- + protected BayesianLinearRegression surrogate; + protected InstancesHeader surrogateHeader; + protected int numParameters; + + // ---- Data window (circular buffer) ---- + protected double[][] dataWindow; + protected int windowHead; + protected int windowCount; + protected int windowFeatures; + + // ---- Subset optimization ---- + /** Detector on the incumbent's error; {@code null} unless drift detection is on. */ + protected ChangeDetector driftDetector; + + /** Cumulative number of drifts signalled over the run; not reset by a restart. */ + protected long driftsDetected; + + protected boolean[] optimizeMask; + + @Override + public void setOptimizableParameters(boolean[] mask) { this.optimizeMask = mask; } + + @Override + public boolean isRandomizable() { return true; } + + @Override + public double[] getVotesForInstance(Instance inst) { + return candidates[bestCandidateIndex].getVotesForInstance(inst); + } + + /** + * Creates the training pool on first use, and again after deserialization. + * A pool larger than the batch trained per instance would leave threads idle, + * so the requested job count is capped at the candidate pool size. + */ + protected void initExecutor() { + if (this.executor != null) return; + int numberOfJobs = this.numberOfJobsOption.getValue() == -1 + ? Runtime.getRuntime().availableProcessors() + : this.numberOfJobsOption.getValue(); + numberOfJobs = Math.min(numberOfJobs, this.numberOfCandidatesOption.getValue()); + // SINGLE_THREAD and requesting a single thread are equivalent: training + // then happens in-place and this.executor stays null. + if (numberOfJobs != SINGLE_THREAD && numberOfJobs != 1) { + // Daemon threads: a live pool must not keep the JVM alive after the + // task that ran this learner has finished. + this.executor = Executors.newFixedThreadPool(numberOfJobs, runnable -> { + Thread thread = new Thread(runnable); + thread.setDaemon(true); + return thread; + }); + } + } + + @Override + public void cleanThreads() { + if (this.executor != null) { + this.executor.shutdownNow(); + this.executor = null; + } + } + + @Override + public void resetLearningImpl() { + driftDetector = driftDetectionOption.isSet() + ? ((ChangeDetector) getPreparedClassOption(driftDetectorOption)).copy() + : null; + cleanThreads(); // shut down any pool from a previous reset before creating a new one + this.candidatesParameters = new ArrayList<>(); + this.instanceCount = 0; + this.bestCandidateIndex = 0; + this.dataWindow = null; + this.windowHead = 0; + this.windowCount = 0; + this.surrogate = null; + this.surrogateHeader = null; + this.setConfigurations(); + + } + + public void setConfigurations() { + try { + ConfigurationSpace space = ConfigurationSpace.fromFile(configurationFileOption.getValue()); + this.configurator = new LearnerConfigurator(space); + this.configurator.validate(); + + this.algorithmCLIString = space.algorithm; + int n = numberOfCandidatesOption.getValue(); + + this.candidates = new Classifier[n]; + this.evaluators = new BasicClassificationPerformanceEvaluator[n]; + for (int i = 0; i < n; i++) { + // Built one by one rather than copied from a single base, so that + // each candidate carries its own random seed. + this.candidates[i] = this.configurator.newLearner(this.classifierRandom.nextInt()); + this.evaluators[i] = new BasicClassificationPerformanceEvaluator(); + MetricUtils.configure(this.evaluators[i]); + this.candidatesParameters.add(new ArrayList<>()); + } + + this.numParameters = 0; + { + for (ParameterSpec spec : space.parameters) { + String pName = spec.name; + switch (spec.type) { + case ParameterSpec.TYPE_CATEGORICAL: { + String[] vals = spec.values; + int defaultActive = spec.active; + + CategoricalParameter p0 = new CategoricalParameter(pName, vals, defaultActive, new Random(this.classifierRandom.nextLong())); + candidatesParameters.get(0).add(p0); + setParameter(candidates[0], p0); + for (int i = 1; i < n; i++) { + int rActive = classifierRandom.nextInt(vals.length); + CategoricalParameter pi = new CategoricalParameter(pName, vals, rActive, new Random(classifierRandom.nextLong())); + candidatesParameters.get(i).add(pi); + setParameter(candidates[i], pi); + } + // Categoricals contribute one dimension to the parameter + // vector (their active index), so they must be counted in + // the BLR feature dimension exactly like int/double params. + this.numParameters++; + break; + } + case ParameterSpec.TYPE_INT: { + int[] range = {(int) spec.range[0], (int) spec.range[1]}; + int defaultVal = (int) spec.value; + + IntParameter p0 = new IntParameter(pName, defaultVal, range, new Random(this.classifierRandom.nextLong())); + candidatesParameters.get(0).add(p0); + setParameter(candidates[0], p0); + for (int i = 1; i < n; i++) { + int rVal = classifierRandom.nextInt(range[1] - range[0] + 1) + range[0]; + IntParameter pi = new IntParameter(pName, rVal, range, new Random(classifierRandom.nextLong())); + candidatesParameters.get(i).add(pi); + setParameter(candidates[i], pi); + } + this.numParameters++; + break; + } + case ParameterSpec.TYPE_DOUBLE: { + double[] range = {spec.range[0], spec.range[1]}; + double defaultVal = spec.value; + + DoubleParameter p0 = new DoubleParameter(pName, defaultVal, range, new Random(this.classifierRandom.nextLong())); + candidatesParameters.get(0).add(p0); + setParameter(candidates[0], p0); + for (int i = 1; i < n; i++) { + double rVal = range[0] + (range[1] - range[0]) * classifierRandom.nextDouble(); + DoubleParameter pi = new DoubleParameter(pName, rVal, range, new Random(classifierRandom.nextLong())); + candidatesParameters.get(i).add(pi); + setParameter(candidates[i], pi); + } + this.numParameters++; + break; + } + } + } + } + + // numParameters + 10 statistical features + this.surrogateHeader = createSurrogateHeader(this.numParameters + 10); + this.surrogate = new BayesianLinearRegression(); + this.surrogate.setModelContext(this.surrogateHeader); + this.surrogate.resetLearning(); + + } catch (Exception e) { + // Fail fast: a swallowed error here leaves candidates/surrogate null and + // surfaces later as a confusing NPE far from the real cause. + throw new RuntimeException( + "Failed to load tuner configuration from " + configurationFileOption.getValue(), e); + } + } + + /** + * Apply one drawn hyperparameter to a candidate through its MOA options. + * The parameter object is the same one recorded in {@code candidatesParameters}, + * so the model and the surrogate's feature vector cannot drift apart. + */ + private void setParameter(Classifier c, Parameter p) { + this.configurator.applyLive(c, p); + } + + // ---- BLR instance helpers ---- + + private InstancesHeader createSurrogateHeader(int numFeatures) { + ArrayList atts = new ArrayList<>(); + for (int i = 0; i < numFeatures; i++) atts.add(new Attribute("f" + i)); + atts.add(new Attribute("performance")); + Instances dataset = new Instances("BLR", atts, 0); + dataset.setClassIndex(numFeatures); + return new InstancesHeader(dataset); + } + + private Instance createSurrogateInstance(double[] features, double performance) { + double[] vals = new double[features.length + 1]; + System.arraycopy(features, 0, vals, 0, features.length); + vals[features.length] = performance; + DenseInstance inst = new DenseInstance(1.0, vals); + inst.setDataset(surrogateHeader); + return inst; + } + + // ---- Data window ---- + + private void initDataWindow(int numFeatures) { + this.windowFeatures = numFeatures; + this.dataWindow = new double[gracePeriodOption.getValue()][numFeatures]; + this.windowHead = 0; + this.windowCount = 0; + } + + private void addToDataWindow(Instance inst) { + int nf = inst.numAttributes() - 1; + if (dataWindow == null) initDataWindow(nf); + double[] row = new double[nf]; + int idx = 0; + for (int i = 0; i < inst.numAttributes(); i++) { + if (i != inst.classIndex()) row[idx++] = inst.value(i); + } + dataWindow[windowHead] = row; + windowHead = (windowHead + 1) % dataWindow.length; + if (windowCount < dataWindow.length) windowCount++; + } + + private double[] extractStatFeatures() { + if (windowCount == 0 || dataWindow == null) return new double[10]; + + double[] flat = new double[windowCount * windowFeatures]; + int idx = 0; + for (int i = 0; i < windowCount; i++) { + int pos = (windowHead - windowCount + i + dataWindow.length) % dataWindow.length; + System.arraycopy(dataWindow[pos], 0, flat, idx, windowFeatures); + idx += windowFeatures; + } + Arrays.sort(flat); + + double sum = 0, sum2 = 0; + for (double v : flat) { sum += v; sum2 += v * v; } + double mean = sum / flat.length; + double std = Math.sqrt(Math.max(0.0, sum2 / flat.length - mean * mean)); + + double sum3 = 0, sum4 = 0; + for (double v : flat) { + double d = v - mean; + double d2 = d * d; + sum3 += d2 * d; + sum4 += d2 * d2; + } + double skewness = (std > 1e-9) ? (sum3 / flat.length) / (std * std * std) : 0.0; + double kurt = (std > 1e-9) ? (sum4 / flat.length) / (std * std * std * std) - 3.0 : 0.0; + + return new double[]{ + mean, std, + percentile(flat, 50), flat[flat.length - 1] - flat[0], + percentile(flat, 25), percentile(flat, 75), + flat[0], flat[flat.length - 1], + skewness, kurt + }; + } + + private double percentile(double[] sorted, double p) { + double index = (p / 100.0) * (sorted.length - 1); + int lo = (int) index, hi = lo + 1; + if (hi >= sorted.length) return sorted[sorted.length - 1]; + return sorted[lo] + (index - lo) * (sorted[hi] - sorted[lo]); + } + + // ---- Parameter vector ---- + + private double[] paramsToVector(ArrayList params) { + double[] v = new double[params.size()]; + for (int i = 0; i < params.size(); i++) { + Parameter p = params.get(i); + switch (p.type) { + case Parameter.TYPE_INT: v[i] = ((IntParameter) p).value; break; + case Parameter.TYPE_DOUBLE: v[i] = ((DoubleParameter) p).value; break; + case Parameter.TYPE_CATEGORICAL: v[i] = ((CategoricalParameter) p).active; break; + } + } + return v; + } + + private double[] combineParamsAndStats(ArrayList params, double[] sv) { + double[] pv = paramsToVector(params); + double[] combined = new double[pv.length + sv.length]; + System.arraycopy(pv, 0, combined, 0, pv.length); + System.arraycopy(sv, 0, combined, pv.length, sv.length); + return combined; + } + + // ---- Acquisition functions ---- + + private double computeAcquisition(double mu, double sigma, double bestF) { + switch (acquisitionFunctionOption.getChosenIndex()) { + case 0: { // PI + double z = (mu - bestF) / (sigma + 1e-9); + return normalCDF(z); + } + case 1: { // EI + double z = (mu - bestF) / (sigma + 1e-9); + return (mu - bestF) * normalCDF(z) + sigma * normalPDF(z); + } + default: { // UCB + double kappa = Math.max(0.1, 2.0 * (1.0 - instanceCount / (10.0 * gracePeriodOption.getValue()))); + return mu + kappa * sigma; + } + } + } + + private double normalCDF(double z) { + return 0.5 * (1.0 + erf(z / Math.sqrt(2.0))); + } + + private double normalPDF(double z) { + return Math.exp(-0.5 * z * z) / Math.sqrt(2.0 * Math.PI); + } + + private double erf(double x) { + double t = 1.0 / (1.0 + 0.3275911 * Math.abs(x)); + double poly = t * (0.254829592 + t * (-0.284496736 + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429)))); + double r = 1.0 - poly * Math.exp(-x * x); + return x >= 0 ? r : -r; + } + + // ---- Training ---- + + @Override + public void trainOnInstanceImpl(Instance inst) { + double[] incumbentVotes = driftDetector != null ? getVotesForInstance(inst) : null; + initExecutor(); + addToDataWindow(inst); + + InstanceExample example = new InstanceExample(inst); + Collection trainers = this.executor == null + ? null : new ArrayList(); + for (int i = 0; i < candidates.length; i++) { + evaluators[i].addResult(example, candidates[i].getVotesForInstance(inst)); + if (trainers == null) { + candidates[i].trainOnInstance(inst); + } else { + // Every candidate owns its model, hence no synchronization here. + trainers.add(new TrainingRunnable(candidates[i], inst)); + } + } + if (trainers != null) { + try { + this.executor.invokeAll(trainers); + } catch (InterruptedException ex) { + throw new RuntimeException("Could not call invokeAll() on training threads."); + } + } + + instanceCount++; + if (instanceCount % gracePeriodOption.getValue() == 0) { + updateModels(); + } + + if (incumbentVotes != null) checkDrift(inst, incumbentVotes); + } + + @Override + public void checkParameterChange() { + updateModels(); + } + + @Override + public void swapClassifiers(int bestPerforming) {} + + @Override + public void deepCopyList(int bestPerforming) {} + + @Override + public void changeStateParameter(Parameter parameter, int index) {} + + private void updateModels() { + int n = numberOfCandidatesOption.getValue(); + double[] performances = new double[n]; + for (int i = 0; i < n; i++) { + performances[i] = MetricUtils.getScore(evaluators[i].getPerformanceMeasurements(), + metricOption.getChosenIndex()); + } + + bestCandidateIndex = 0; + for (int i = 1; i < n; i++) { + if (performances[i] > performances[bestCandidateIndex]) bestCandidateIndex = i; + } + + // Window statistics are identical for every candidate this cycle; compute once. + double[] stats = extractStatFeatures(); + + // Train BLR on all non-default candidates (skip index 0) + for (int i = 1; i < n; i++) { + double[] features = combineParamsAndStats(candidatesParameters.get(i), stats); + surrogate.trainOnInstance(createSurrogateInstance(features, performances[i])); + } + + // Sort by performance ascending (worst first) + Integer[] sortedIdx = new Integer[n]; + for (int i = 0; i < n; i++) sortedIdx[i] = i; + Arrays.sort(sortedIdx, (a, b) -> Double.compare(performances[a], performances[b])); + + // Replace worst half, never touching the best or the default (index 0). + int replaceCount = n / 2; + int replaced = 0; + for (int k = 0; k < n && replaced < replaceCount; k++) { + int idx = sortedIdx[k]; + if (idx == bestCandidateIndex || idx == 0) continue; + replaceCandidate(idx, proposeNextParams(performances, stats)); + replaced++; + } + } + + private ArrayList proposeNextParams(double[] performances, double[] stats) { + double bestPerf = -Double.MAX_VALUE; + for (double p : performances) if (p > bestPerf) bestPerf = p; + + ArrayList best = null; + double bestAcq = Double.NEGATIVE_INFINITY; + + for (int k = 0; k < 100; k++) { + ArrayList candidate = randomConfig(); + double[] pv = paramsToVector(candidate); + double[] features = new double[pv.length + stats.length]; + System.arraycopy(pv, 0, features, 0, pv.length); + System.arraycopy(stats, 0, features, pv.length, stats.length); + + double[] muSigma = surrogate.predictWithVariance(createSurrogateInstance(features, 0.0)); + double acq = computeAcquisition(muSigma[0], muSigma[1], bestPerf); + if (acq > bestAcq) { + bestAcq = acq; + best = candidate; + } + } + return best != null ? best : randomConfig(); + } + + private ArrayList randomConfig() { + ArrayList params = new ArrayList<>(); + for (Parameter p : candidatesParameters.get(0)) { + Parameter clone; + switch (p.type) { + case Parameter.TYPE_INT: { + IntParameter ip = (IntParameter) p; + int rVal = classifierRandom.nextInt(ip.range[1] - ip.range[0] + 1) + ip.range[0]; + clone = new IntParameter(ip.name, rVal, ip.range, new Random(classifierRandom.nextLong())); + break; + } + case Parameter.TYPE_DOUBLE: { + DoubleParameter dp = (DoubleParameter) p; + double rVal = dp.range[0] + (dp.range[1] - dp.range[0]) * classifierRandom.nextDouble(); + clone = new DoubleParameter(dp.name, rVal, dp.range, new Random(classifierRandom.nextLong())); + break; + } + default: { + CategoricalParameter cp = (CategoricalParameter) p; + int rActive = classifierRandom.nextInt(cp.values.length); + clone = new CategoricalParameter(cp.name, cp.values, rActive, new Random(classifierRandom.nextLong())); + break; + } + } + params.add(clone); + } + // Frozen dimensions take the incumbent (best candidate) value, so the + // BLR is only ever queried/trained on real feature vectors - never on + // placeholder values for the parameters we are not optimizing. + HPOMethod.freezeToIncumbent(this.optimizeMask, params, candidatesParameters.get(bestCandidateIndex)); + return params; + } + + private void replaceCandidate(int idx, ArrayList params) { + try { + Classifier c; + if (!this.configurator.canApplyLive()) { + // The space holds a hyperparameter the learner only reads when it + // is built, so the candidate has to be rebuilt around it. + c = this.configurator.instantiate(params, this.classifierRandom.nextInt()); + } else { + c = candidates[bestCandidateIndex].copy(); + LearnerConfigurator.reseedCopy(c, this.classifierRandom.nextInt()); + if (resetLearningOption.isSet()) { + c.resetLearning(); + } + this.configurator.applyLive(c, params); + } + candidates[idx] = c; + candidatesParameters.set(idx, params); + evaluators[idx] = new BasicClassificationPerformanceEvaluator(); + MetricUtils.configure(evaluators[idx]); + } catch (Exception e) { + e.printStackTrace(System.out); + } + } + + // ---- MOA bookkeeping ---- + + @Override + protected Measurement[] getModelMeasurementsImpl() { + ArrayList ms = new ArrayList<>(); + ms.add(new Measurement("driftsDetected", driftsDetected)); + for (Measurement m : this.candidates[bestCandidateIndex].getModelMeasurements()) { + ms.add(m); + } + ms.add(new Measurement("bestCandidateIndex", bestCandidateIndex)); + for (Parameter p : candidatesParameters.get(bestCandidateIndex)) { + switch (p.type) { + case Parameter.TYPE_INT: ms.add(new Measurement(p.name, ((IntParameter) p).value)); break; + case Parameter.TYPE_DOUBLE: ms.add(new Measurement(p.name, ((DoubleParameter) p).value)); break; + case Parameter.TYPE_CATEGORICAL: ms.add(new Measurement(p.name, ((CategoricalParameter) p).active)); break; + } + } + return ms.toArray(new Measurement[0]); + } + + @Override + public void getModelDescription(StringBuilder out, int indent) {} + + @Override + public long measureByteSize() { + long size = SizeOf.sizeOf(this); + for (Classifier c : candidates) size += c.measureByteSize(); + return size; + } + + @Override + public double getCandidateScore(int i) { + return MetricUtils.getScore(evaluators[i].getPerformanceMeasurements(), metricOption.getChosenIndex()); + } + + @Override + public double getClassifierScore() { + return MetricUtils.getScore(evaluators[bestCandidateIndex].getPerformanceMeasurements(), metricOption.getChosenIndex()); + } + + @Override + public int getNumberOfCandidates() { return numberOfCandidatesOption.getValue(); } + + @Override + public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } + + @Override + public int getEvaluationInstancesCount() { return (int) (instanceCount % gracePeriodOption.getValue()); } + + @Override + public int getGracePeriod() { return gracePeriodOption.getValue(); } + + @Override + public Classifier getMainClassifier() { return candidates[bestCandidateIndex]; } + + @Override + public String getConfigurationFile() { return this.configurationFileOption.getValue();} + + + @Override + public ArrayList getReferenceParameters() { + return (this.candidatesParameters != null && !this.candidatesParameters.isEmpty()) + ? this.candidatesParameters.get(0) + : null; + } + + @Override + public ArrayList> getCandidateParameters() { + return this.candidatesParameters; + } + + private int argmax(double[] arr) { + int best = 0; + for (int i = 1; i < arr.length; i++) + if (arr[i] > arr[best]) best = i; + return best; + } + + /** + * Feeds the incumbent's prediction error to the change detector and, when a + * drift is signalled, reinitializes the whole search from the search space. + * The signal is the 0/1 misclassification indicator. + */ + private void checkDrift(Instance inst, double[] incumbentVotes) { + driftDetector.input(argmax(incumbentVotes) != (int) inst.classValue() ? 1.0 : 0.0); + if (driftDetector.getChange()) { + driftsDetected++; + resetLearningImpl(); + } + } + + /** + * Inner class to assist with the multi-thread execution. + */ + protected class TrainingRunnable implements Runnable, Callable { + final private Classifier learner; + final private Instance instance; + + public TrainingRunnable(Classifier learner, Instance instance) { + this.learner = learner; + this.instance = instance; + } + + @Override + public void run() { + this.learner.trainOnInstance(this.instance); + } + + @Override + public Integer call() { + run(); + return 0; + } + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerRegressor.java b/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerRegressor.java new file mode 100644 index 000000000..739ed30d4 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerRegressor.java @@ -0,0 +1,735 @@ +/* + * BayesianStreamTunerRegressor.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML; + +import com.github.javacliparser.FileOption; +import com.github.javacliparser.FlagOption; +import com.github.javacliparser.IntOption; +import com.github.javacliparser.MultiChoiceOption; +import com.yahoo.labs.samoa.instances.Attribute; +import com.yahoo.labs.samoa.instances.DenseInstance; +import com.yahoo.labs.samoa.instances.Instance; +import com.yahoo.labs.samoa.instances.Instances; +import com.yahoo.labs.samoa.instances.InstancesHeader; +import moa.capabilities.CapabilitiesHandler; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.AutoML.Parameters.CategoricalParameter; +import moa.classifiers.AutoML.Parameters.DoubleParameter; +import moa.classifiers.AutoML.Parameters.IntParameter; +import moa.classifiers.AutoML.Parameters.Parameter; +import moa.classifiers.AutoML.space.ConfigurationSpace; +import moa.classifiers.AutoML.space.LearnerConfigurator; +import moa.classifiers.AutoML.space.ParameterSpec; +import moa.classifiers.Classifier; +import moa.classifiers.core.driftdetection.ChangeDetector; +import moa.classifiers.Regressor; +import moa.classifiers.functions.BayesianLinearRegression; +import moa.core.InstanceExample; +import moa.core.Measurement; +import moa.core.SizeOf; +import moa.evaluation.BasicRegressionPerformanceEvaluator; +import moa.options.ClassOption; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Random; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Regression counterpart of {@link BayesianStreamTunerClassifier}: Bayesian + * optimisation over the streaming hyperparameter space, scored with the + * regression metrics of {@link MetricUtils}. + * + *

See details in:
Nilesh Verma, Albert Bifet, Bernhard Pfahringer, + * Maroua Bahri. Bayesian Stream Tuner: Dynamic Hyperparameter Optimization for + * Real-Time Data Streams. In Proceedings of the 31st ACM SIGKDD Conference on + * Knowledge Discovery and Data Mining V.2 (KDD '25), pages 2871-2882, + * DOI: 10.1145/3711896.3736852, ACM, 2025.

+ * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public class BayesianStreamTunerRegressor extends AbstractClassifier + implements Regressor, CapabilitiesHandler, Serializable, HPOMethod { + + private static final long serialVersionUID = 1L; + + public FileOption configurationFileOption = new FileOption("configurationFile", 'f', + "Search space in JSON format.", null, ".json", false); + + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + "Number of instances between model update cycles; also the number of recent" + + " instances kept to derive the surrogate's stream statistics.", + 1000, 1, Integer.MAX_VALUE); + + public IntOption numberOfCandidatesOption = new IntOption("numberOfCandidates", 'n', + "Number of candidate models (including default).", 10, 2, Integer.MAX_VALUE); + + // MetricUtils orients every regression metric so that larger is better, so + // error metrics can be selected here without a separate direction flag. + public MultiChoiceOption metricOption = new MultiChoiceOption("metric", 'm', + "Metric to optimize the model.", MetricUtils.REGRESSION_METRIC_NAMES, + MetricUtils.REGRESSION_METRIC_DESCRIPTIONS, 0); + + public MultiChoiceOption acquisitionFunctionOption = new MultiChoiceOption( + "acquisitionFunction", 'q', + "Acquisition function for Bayesian optimization.", + new String[]{"PI", "EI", "UCB"}, + new String[]{"Probability of Improvement", "Expected Improvement", "Upper Confidence Bound"}, + 0); + + public FlagOption driftDetectionOption = new FlagOption("driftDetection", 'd', + "Enable drift detection on the incumbent's prediction error."); + + public ClassOption driftDetectorOption = new ClassOption("driftDetector", 'D', + "Change detector to use when drift detection is enabled; on a detected" + + " drift the whole search is reinitialized from the search space.", + ChangeDetector.class, "ADWINChangeDetector"); + + public IntOption numberOfJobsOption = new IntOption("numberOfJobs", 'j', + "Total number of concurrent jobs used for processing (-1 = as much as possible, 0 = do not use multithreading)", + 1, -1, Integer.MAX_VALUE); + + protected static final int SINGLE_THREAD = 0; + + // ---- Model ensemble ---- + protected Classifier[] candidates; + + /** + * Pool training the candidates concurrently; {@code null} when running + * single-threaded. Transient because an executor cannot be serialized, so it + * is (re)created on demand by {@link #initExecutor()}. + */ + protected transient ExecutorService executor; + + /** Boundary through which every learner is configured. */ + protected LearnerConfigurator configurator; + protected BasicRegressionPerformanceEvaluator[] evaluators; + protected ArrayList> candidatesParameters; + protected int bestCandidateIndex; + protected long instanceCount; + protected String algorithmCLIString; + + // ---- BLR surrogate ---- + protected BayesianLinearRegression surrogate; + protected InstancesHeader surrogateHeader; + protected int numParameters; + + // ---- Data window (circular buffer) ---- + protected double[][] dataWindow; + protected int windowHead; + protected int windowCount; + protected int windowFeatures; + + // ---- Subset optimization ---- + /** Detector on the incumbent's error; {@code null} unless drift detection is on. */ + protected ChangeDetector driftDetector; + + /** Cumulative number of drifts signalled over the run; not reset by a restart. */ + protected long driftsDetected; + + protected boolean[] optimizeMask; + + @Override + public void setOptimizableParameters(boolean[] mask) { this.optimizeMask = mask; } + + @Override + public boolean isRandomizable() { return true; } + + @Override + public double[] getVotesForInstance(Instance inst) { + return candidates[bestCandidateIndex].getVotesForInstance(inst); + } + + /** + * Creates the training pool on first use, and again after deserialization. + * A pool larger than the batch trained per instance would leave threads idle, + * so the requested job count is capped at the candidate pool size. + */ + protected void initExecutor() { + if (this.executor != null) return; + int numberOfJobs = this.numberOfJobsOption.getValue() == -1 + ? Runtime.getRuntime().availableProcessors() + : this.numberOfJobsOption.getValue(); + numberOfJobs = Math.min(numberOfJobs, this.numberOfCandidatesOption.getValue()); + // SINGLE_THREAD and requesting a single thread are equivalent: training + // then happens in-place and this.executor stays null. + if (numberOfJobs != SINGLE_THREAD && numberOfJobs != 1) { + // Daemon threads: a live pool must not keep the JVM alive after the + // task that ran this learner has finished. + this.executor = Executors.newFixedThreadPool(numberOfJobs, runnable -> { + Thread thread = new Thread(runnable); + thread.setDaemon(true); + return thread; + }); + } + } + + @Override + public void cleanThreads() { + if (this.executor != null) { + this.executor.shutdownNow(); + this.executor = null; + } + } + + @Override + public void resetLearningImpl() { + driftDetector = driftDetectionOption.isSet() + ? ((ChangeDetector) getPreparedClassOption(driftDetectorOption)).copy() + : null; + cleanThreads(); // shut down any pool from a previous reset before creating a new one + this.candidatesParameters = new ArrayList<>(); + this.instanceCount = 0; + this.bestCandidateIndex = 0; + this.dataWindow = null; + this.windowHead = 0; + this.windowCount = 0; + this.surrogate = null; + this.surrogateHeader = null; + this.setConfigurations(); + + } + + public void setConfigurations() { + try { + ConfigurationSpace space = ConfigurationSpace.fromFile(configurationFileOption.getValue()); + this.configurator = new LearnerConfigurator(space); + this.configurator.validate(); + + this.algorithmCLIString = space.algorithm; + int n = numberOfCandidatesOption.getValue(); + + this.candidates = new Classifier[n]; + this.evaluators = new BasicRegressionPerformanceEvaluator[n]; + for (int i = 0; i < n; i++) { + // Built one by one rather than copied from a single base, so that + // each candidate carries its own random seed. + this.candidates[i] = this.configurator.newLearner(this.classifierRandom.nextInt()); + this.evaluators[i] = new BasicRegressionPerformanceEvaluator(); + this.candidatesParameters.add(new ArrayList<>()); + } + + this.numParameters = 0; + { + for (ParameterSpec spec : space.parameters) { + String pName = spec.name; + switch (spec.type) { + case ParameterSpec.TYPE_CATEGORICAL: { + String[] vals = spec.values; + int defaultActive = spec.active; + + CategoricalParameter p0 = new CategoricalParameter(pName, vals, defaultActive, new Random(this.classifierRandom.nextLong())); + candidatesParameters.get(0).add(p0); + setParameter(candidates[0], p0); + for (int i = 1; i < n; i++) { + int rActive = classifierRandom.nextInt(vals.length); + CategoricalParameter pi = new CategoricalParameter(pName, vals, rActive, new Random(classifierRandom.nextLong())); + candidatesParameters.get(i).add(pi); + setParameter(candidates[i], pi); + } + // Categoricals contribute one dimension to the parameter + // vector (their active index), so they must be counted in + // the BLR feature dimension exactly like int/double params. + this.numParameters++; + break; + } + case ParameterSpec.TYPE_INT: { + int[] range = {(int) spec.range[0], (int) spec.range[1]}; + int defaultVal = (int) spec.value; + + IntParameter p0 = new IntParameter(pName, defaultVal, range, new Random(this.classifierRandom.nextLong())); + candidatesParameters.get(0).add(p0); + setParameter(candidates[0], p0); + for (int i = 1; i < n; i++) { + int rVal = classifierRandom.nextInt(range[1] - range[0] + 1) + range[0]; + IntParameter pi = new IntParameter(pName, rVal, range, new Random(classifierRandom.nextLong())); + candidatesParameters.get(i).add(pi); + setParameter(candidates[i], pi); + } + this.numParameters++; + break; + } + case ParameterSpec.TYPE_DOUBLE: { + double[] range = {spec.range[0], spec.range[1]}; + double defaultVal = spec.value; + + DoubleParameter p0 = new DoubleParameter(pName, defaultVal, range, new Random(this.classifierRandom.nextLong())); + candidatesParameters.get(0).add(p0); + setParameter(candidates[0], p0); + for (int i = 1; i < n; i++) { + double rVal = range[0] + (range[1] - range[0]) * classifierRandom.nextDouble(); + DoubleParameter pi = new DoubleParameter(pName, rVal, range, new Random(classifierRandom.nextLong())); + candidatesParameters.get(i).add(pi); + setParameter(candidates[i], pi); + } + this.numParameters++; + break; + } + } + } + } + + this.surrogateHeader = createSurrogateHeader(this.numParameters + 10); + this.surrogate = new BayesianLinearRegression(); + this.surrogate.setModelContext(this.surrogateHeader); + this.surrogate.resetLearning(); + + } catch (Exception e) { + // Fail fast: a swallowed error here leaves candidates/surrogate null and + // surfaces later as a confusing NPE far from the real cause. + throw new RuntimeException( + "Failed to load tuner configuration from " + configurationFileOption.getValue(), e); + } + } + + /** + * Apply one drawn hyperparameter to a candidate through its MOA options. + * The parameter object is the same one recorded in {@code candidatesParameters}, + * so the model and the surrogate's feature vector cannot drift apart. + */ + private void setParameter(Classifier c, Parameter p) { + this.configurator.applyLive(c, p); + } + + // ---- BLR instance helpers ---- + + private InstancesHeader createSurrogateHeader(int numFeatures) { + ArrayList atts = new ArrayList<>(); + for (int i = 0; i < numFeatures; i++) atts.add(new Attribute("f" + i)); + atts.add(new Attribute("performance")); + Instances dataset = new Instances("BLR", atts, 0); + dataset.setClassIndex(numFeatures); + return new InstancesHeader(dataset); + } + + private Instance createSurrogateInstance(double[] features, double performance) { + double[] vals = new double[features.length + 1]; + System.arraycopy(features, 0, vals, 0, features.length); + vals[features.length] = performance; + DenseInstance inst = new DenseInstance(1.0, vals); + inst.setDataset(surrogateHeader); + return inst; + } + + // ---- Data window ---- + + private void initDataWindow(int numFeatures) { + this.windowFeatures = numFeatures; + this.dataWindow = new double[gracePeriodOption.getValue()][numFeatures]; + this.windowHead = 0; + this.windowCount = 0; + } + + private void addToDataWindow(Instance inst) { + int nf = inst.numAttributes() - 1; + if (dataWindow == null) initDataWindow(nf); + double[] row = new double[nf]; + int idx = 0; + for (int i = 0; i < inst.numAttributes(); i++) { + if (i != inst.classIndex()) row[idx++] = inst.value(i); + } + dataWindow[windowHead] = row; + windowHead = (windowHead + 1) % dataWindow.length; + if (windowCount < dataWindow.length) windowCount++; + } + + private double[] extractStatFeatures() { + if (windowCount == 0 || dataWindow == null) return new double[10]; + + double[] flat = new double[windowCount * windowFeatures]; + int idx = 0; + for (int i = 0; i < windowCount; i++) { + int pos = (windowHead - windowCount + i + dataWindow.length) % dataWindow.length; + System.arraycopy(dataWindow[pos], 0, flat, idx, windowFeatures); + idx += windowFeatures; + } + Arrays.sort(flat); + + double sum = 0, sum2 = 0; + for (double v : flat) { sum += v; sum2 += v * v; } + double mean = sum / flat.length; + double std = Math.sqrt(Math.max(0.0, sum2 / flat.length - mean * mean)); + + double sum3 = 0, sum4 = 0; + for (double v : flat) { + double d = v - mean; + double d2 = d * d; + sum3 += d2 * d; + sum4 += d2 * d2; + } + double skewness = (std > 1e-9) ? (sum3 / flat.length) / (std * std * std) : 0.0; + double kurt = (std > 1e-9) ? (sum4 / flat.length) / (std * std * std * std) - 3.0 : 0.0; + + return new double[]{ + mean, std, + percentile(flat, 50), flat[flat.length - 1] - flat[0], + percentile(flat, 25), percentile(flat, 75), + flat[0], flat[flat.length - 1], + skewness, kurt + }; + } + + private double percentile(double[] sorted, double p) { + double index = (p / 100.0) * (sorted.length - 1); + int lo = (int) index, hi = lo + 1; + if (hi >= sorted.length) return sorted[sorted.length - 1]; + return sorted[lo] + (index - lo) * (sorted[hi] - sorted[lo]); + } + + // ---- Parameter vector ---- + + private double[] paramsToVector(ArrayList params) { + double[] v = new double[params.size()]; + for (int i = 0; i < params.size(); i++) { + Parameter p = params.get(i); + switch (p.type) { + case Parameter.TYPE_INT: v[i] = ((IntParameter) p).value; break; + case Parameter.TYPE_DOUBLE: v[i] = ((DoubleParameter) p).value; break; + case Parameter.TYPE_CATEGORICAL: v[i] = ((CategoricalParameter) p).active; break; + } + } + return v; + } + + private double[] combineParamsAndStats(ArrayList params, double[] sv) { + double[] pv = paramsToVector(params); + double[] combined = new double[pv.length + sv.length]; + System.arraycopy(pv, 0, combined, 0, pv.length); + System.arraycopy(sv, 0, combined, pv.length, sv.length); + return combined; + } + + // ---- Acquisition functions ---- + + private double computeAcquisition(double mu, double sigma, double bestF) { + // MetricUtils.getRegressionScore already orients every metric so that + // larger is better, so improvement is a plain difference here. + double improvement = mu - bestF; + switch (acquisitionFunctionOption.getChosenIndex()) { + case 0: { // PI + double z = improvement / (sigma + 1e-9); + return normalCDF(z); + } + case 1: { // EI + double z = improvement / (sigma + 1e-9); + return improvement * normalCDF(z) + sigma * normalPDF(z); + } + default: { // UCB + double kappa = Math.max(0.1, 2.0 * (1.0 - instanceCount / (10.0 * gracePeriodOption.getValue()))); + return mu + kappa * sigma; + } + } + } + + private double normalCDF(double z) { + return 0.5 * (1.0 + erf(z / Math.sqrt(2.0))); + } + + private double normalPDF(double z) { + return Math.exp(-0.5 * z * z) / Math.sqrt(2.0 * Math.PI); + } + + private double erf(double x) { + double t = 1.0 / (1.0 + 0.3275911 * Math.abs(x)); + double poly = t * (0.254829592 + t * (-0.284496736 + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429)))); + double r = 1.0 - poly * Math.exp(-x * x); + return x >= 0 ? r : -r; + } + + // ---- Training ---- + + @Override + public void trainOnInstanceImpl(Instance inst) { + double[] incumbentVotes = driftDetector != null ? getVotesForInstance(inst) : null; + initExecutor(); + addToDataWindow(inst); + + InstanceExample example = new InstanceExample(inst); + Collection trainers = this.executor == null + ? null : new ArrayList(); + for (int i = 0; i < candidates.length; i++) { + evaluators[i].addResult(example, candidates[i].getVotesForInstance(inst)); + if (trainers == null) { + candidates[i].trainOnInstance(inst); + } else { + // Every candidate owns its model, hence no synchronization here. + trainers.add(new TrainingRunnable(candidates[i], inst)); + } + } + if (trainers != null) { + try { + this.executor.invokeAll(trainers); + } catch (InterruptedException ex) { + throw new RuntimeException("Could not call invokeAll() on training threads."); + } + } + + instanceCount++; + if (instanceCount % gracePeriodOption.getValue() == 0) { + updateModels(); + } + + if (incumbentVotes != null) checkDrift(inst, incumbentVotes); + } + + @Override + public void checkParameterChange() { + updateModels(); + } + + @Override + public void swapClassifiers(int bestPerforming) {} + + @Override + public void deepCopyList(int bestPerforming) {} + + @Override + public void changeStateParameter(Parameter parameter, int index) {} + + private void updateModels() { + int n = numberOfCandidatesOption.getValue(); + double[] performances = new double[n]; + for (int i = 0; i < n; i++) { + performances[i] = MetricUtils.getRegressionScore( + evaluators[i].getPerformanceMeasurements(), metricOption.getChosenIndex()); + } + + bestCandidateIndex = 0; + for (int i = 1; i < n; i++) { + if (performances[i] > performances[bestCandidateIndex]) bestCandidateIndex = i; + } + + // Window statistics are identical for every candidate this cycle; compute once. + double[] stats = extractStatFeatures(); + + // Train BLR on all non-default candidates (skip index 0) + for (int i = 1; i < n; i++) { + double[] features = combineParamsAndStats(candidatesParameters.get(i), stats); + surrogate.trainOnInstance(createSurrogateInstance(features, performances[i])); + } + + // Sort worst-first; scores are already oriented so that larger is better. + Integer[] sortedIdx = new Integer[n]; + for (int i = 0; i < n; i++) sortedIdx[i] = i; + Arrays.sort(sortedIdx, (a, b) -> Double.compare(performances[a], performances[b])); + + // Replace worst half, never touching the best or the default (index 0). + int replaceCount = n / 2; + int replaced = 0; + for (int k = 0; k < n && replaced < replaceCount; k++) { + int idx = sortedIdx[k]; + if (idx == bestCandidateIndex || idx == 0) continue; + replaceCandidate(idx, proposeNextParams(performances, stats)); + replaced++; + } + } + + private ArrayList proposeNextParams(double[] performances, double[] stats) { + double bestPerf = -Double.MAX_VALUE; + for (double p : performances) { + if (p > bestPerf) bestPerf = p; + } + + ArrayList best = null; + double bestAcq = Double.NEGATIVE_INFINITY; + + for (int k = 0; k < 100; k++) { + ArrayList candidate = randomConfig(); + double[] pv = paramsToVector(candidate); + double[] features = new double[pv.length + stats.length]; + System.arraycopy(pv, 0, features, 0, pv.length); + System.arraycopy(stats, 0, features, pv.length, stats.length); + + double[] muSigma = surrogate.predictWithVariance(createSurrogateInstance(features, 0.0)); + double acq = computeAcquisition(muSigma[0], muSigma[1], bestPerf); + if (acq > bestAcq) { + bestAcq = acq; + best = candidate; + } + } + return best != null ? best : randomConfig(); + } + + private ArrayList randomConfig() { + ArrayList params = new ArrayList<>(); + for (Parameter p : candidatesParameters.get(0)) { + Parameter clone; + switch (p.type) { + case Parameter.TYPE_INT: { + IntParameter ip = (IntParameter) p; + int rVal = classifierRandom.nextInt(ip.range[1] - ip.range[0] + 1) + ip.range[0]; + clone = new IntParameter(ip.name, rVal, ip.range, new Random(classifierRandom.nextLong())); + break; + } + case Parameter.TYPE_DOUBLE: { + DoubleParameter dp = (DoubleParameter) p; + double rVal = dp.range[0] + (dp.range[1] - dp.range[0]) * classifierRandom.nextDouble(); + clone = new DoubleParameter(dp.name, rVal, dp.range, new Random(classifierRandom.nextLong())); + break; + } + default: { + CategoricalParameter cp = (CategoricalParameter) p; + int rActive = classifierRandom.nextInt(cp.values.length); + clone = new CategoricalParameter(cp.name, cp.values, rActive, new Random(classifierRandom.nextLong())); + break; + } + } + params.add(clone); + } + // Frozen dimensions take the incumbent (best candidate) value, so the + // BLR is only ever queried/trained on real feature vectors - never on + // placeholder values for the parameters we are not optimizing. + HPOMethod.freezeToIncumbent(this.optimizeMask, params, candidatesParameters.get(bestCandidateIndex)); + return params; + } + + private void replaceCandidate(int idx, ArrayList params) { + try { + Classifier c; + if (!this.configurator.canApplyLive()) { + // The space holds a hyperparameter the learner only reads when it + // is built, so the candidate has to be rebuilt around it. + c = this.configurator.instantiate(params, this.classifierRandom.nextInt()); + } else { + c = candidates[bestCandidateIndex].copy(); + LearnerConfigurator.reseedCopy(c, this.classifierRandom.nextInt()); + this.configurator.applyLive(c, params); + } + candidates[idx] = c; + candidatesParameters.set(idx, params); + evaluators[idx] = new BasicRegressionPerformanceEvaluator(); + } catch (Exception e) { + e.printStackTrace(System.out); + } + } + + // ---- MOA bookkeeping ---- + + @Override + protected Measurement[] getModelMeasurementsImpl() { + ArrayList ms = new ArrayList<>(); + ms.add(new Measurement("driftsDetected", driftsDetected)); + for (Measurement m : this.candidates[bestCandidateIndex].getModelMeasurements()) { + ms.add(m); + } + ms.add(new Measurement("bestCandidateIndex", bestCandidateIndex)); + for (Parameter p : candidatesParameters.get(bestCandidateIndex)) { + switch (p.type) { + case Parameter.TYPE_INT: ms.add(new Measurement(p.name, ((IntParameter) p).value)); break; + case Parameter.TYPE_DOUBLE: ms.add(new Measurement(p.name, ((DoubleParameter) p).value)); break; + case Parameter.TYPE_CATEGORICAL: ms.add(new Measurement(p.name, ((CategoricalParameter) p).active)); break; + } + } + return ms.toArray(new Measurement[0]); + } + + @Override + public void getModelDescription(StringBuilder out, int indent) {} + + @Override + public long measureByteSize() { + long size = SizeOf.sizeOf(this); + for (Classifier c : candidates) size += c.measureByteSize(); + return size; + } + + @Override + public double getCandidateScore(int i) { + return MetricUtils.getRegressionScore(evaluators[i].getPerformanceMeasurements(), + metricOption.getChosenIndex()); + } + + @Override + public double getClassifierScore() { + return MetricUtils.getRegressionScore( + evaluators[bestCandidateIndex].getPerformanceMeasurements(), metricOption.getChosenIndex()); + } + + @Override + public int getNumberOfCandidates() { return numberOfCandidatesOption.getValue(); } + + @Override + public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } + + @Override + public int getEvaluationInstancesCount() { return (int) (instanceCount % gracePeriodOption.getValue()); } + + @Override + public int getGracePeriod() { return gracePeriodOption.getValue(); } + + @Override + public Classifier getMainClassifier() { return candidates[bestCandidateIndex]; } + + @Override + public ArrayList getReferenceParameters() { + return (this.candidatesParameters != null && !this.candidatesParameters.isEmpty()) + ? this.candidatesParameters.get(0) + : null; + } + + @Override + public String getConfigurationFile() { return this.configurationFileOption.getValue();} + + + @Override + public ArrayList> getCandidateParameters() { + return this.candidatesParameters; + } + + /** + * Feeds the incumbent's prediction error to the change detector and, when a + * drift is signalled, reinitializes the whole search from the search space. + * The signal is the absolute error |y - y_pred|. + */ + private void checkDrift(Instance inst, double[] incumbentVotes) { + double predicted = incumbentVotes.length > 0 ? incumbentVotes[0] : 0.0; + driftDetector.input(Math.abs(predicted - inst.classValue())); + if (driftDetector.getChange()) { + driftsDetected++; + resetLearningImpl(); + } + } + + /** + * Inner class to assist with the multi-thread execution. + */ + protected class TrainingRunnable implements Runnable, Callable { + final private Classifier learner; + final private Instance instance; + + public TrainingRunnable(Classifier learner, Instance instance) { + this.learner = learner; + this.instance = instance; + } + + @Override + public void run() { + this.learner.trainOnInstance(this.instance); + } + + @Override + public Integer call() { + run(); + return 0; + } + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTClassifier.java b/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTClassifier.java new file mode 100644 index 000000000..a21dd6804 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTClassifier.java @@ -0,0 +1,709 @@ +/* + * FSS_SPTClassifier.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML; + +import com.github.javacliparser.FileOption; +import com.github.javacliparser.FlagOption; +import com.github.javacliparser.FloatOption; +import com.github.javacliparser.IntOption; +import com.github.javacliparser.MultiChoiceOption; +import com.yahoo.labs.samoa.instances.Instance; +import moa.capabilities.CapabilitiesHandler; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.AutoML.Parameters.*; +import moa.classifiers.AutoML.space.ConfigurationSpace; +import moa.classifiers.AutoML.space.LearnerConfigurator; +import moa.classifiers.AutoML.space.ParameterSpec; +import moa.classifiers.Classifier; +import moa.classifiers.core.driftdetection.ChangeDetector; +import moa.classifiers.MultiClassClassifier; +import moa.core.InstanceExample; +import moa.core.Measurement; +import moa.core.SizeOf; +import moa.evaluation.BasicClassificationPerformanceEvaluator; +import moa.options.ClassOption; + +import java.io.Serializable; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Fish School Search over a streaming hyperparameter space: a school of + * candidates moves through the search space by individual, collective + * instinctive and collective volitive steps, with the step size annealed from + * {@code initialStep} down to {@code finalStep}. + * + *

Candidates are configured through {@link LearnerConfigurator}, i.e. by + * writing MOA {@link com.github.javacliparser.Option}s, so any MOA classifier + * can be tuned as shipped. + * + *

See details in:
Bruno Veloso, Hugo Amorim Neto, Fernando Buarque, + * Joao Gama. Fish swarm parameter self-tuning for data streams. In Data Mining + * and Knowledge Discovery, 40(1), DOI: 10.1007/s10618-025-01174-8, Springer, + * 2025.

+ * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public class FSS_SPTClassifier extends AbstractClassifier implements MultiClassClassifier, + CapabilitiesHandler, Serializable, HPOMethod { + + // ========== OPTIONS ========== + + public FileOption configurationFileOption = new FileOption("configurationFile", 'f', + "Search space in JSON format.", null, ".json", false); + + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + "Number of instances between FSS school updates.", 1000, 2, Integer.MAX_VALUE); + + public MultiChoiceOption metricOption = new MultiChoiceOption("metric", 'm', + "Metric to optimize the model.", MetricUtils.METRIC_NAMES, MetricUtils.METRIC_DESCRIPTIONS, 4); + + public IntOption numEstimatorsOption = new IntOption("numEstimators", 'n', + "Number of fish (candidate models) in the school.", 10, 2, Integer.MAX_VALUE); + + public FloatOption initialStepOption = new FloatOption("initialStep", 'a', + "Initial individual movement step size (fraction of parameter range).", 0.1, 0.0, 1.0); + + public FloatOption finalStepOption = new FloatOption("finalStep", 'e', + "Final individual movement step size after numIterations updates.", 0.01, 0.0, 1.0); + + public IntOption numIterationsOption = new IntOption("numIterations", 't', + "Number of step-decay iterations.", 100, 1, Integer.MAX_VALUE); + + public FlagOption resetModelsOption = new FlagOption("resetModels", 'r', + "Create fresh models after each movement instead of warm-starting from best."); + + public FlagOption verboseOption = new FlagOption("verbose", 'v', + "Print FSS events to stdout."); + + public FlagOption driftDetectionOption = new FlagOption("driftDetection", 'd', + "Enable drift detection on the incumbent's prediction error."); + + public ClassOption driftDetectorOption = new ClassOption("driftDetector", 'D', + "Change detector to use when drift detection is enabled; on a detected" + + " drift the whole search is reinitialized from the search space.", + ChangeDetector.class, "ADWINChangeDetector"); + + public IntOption numberOfJobsOption = new IntOption("numberOfJobs", 'j', + "Total number of concurrent jobs used for processing (-1 = as much as possible, 0 = do not use multithreading)", + 1, -1, Integer.MAX_VALUE); + + protected static final int SINGLE_THREAD = 0; + + protected static class FishEntry implements Serializable { + Classifier model; + BasicClassificationPerformanceEvaluator evaluator; + ArrayList params; + long instancesSeen; + double weight; + double difFit; + double oldFit; + boolean initialized; + ArrayList oldPos; // position at first nexteval (never updated after init) + double[] difDist; // displacement relative to oldPos when improvement occurred + + FishEntry(Classifier model, BasicClassificationPerformanceEvaluator evaluator, + ArrayList params) { + this.model = model; + this.evaluator = evaluator; + this.params = params; + this.instancesSeen = 0; + this.weight = 0.1; + this.difFit = 0.0; + this.oldFit = Double.NEGATIVE_INFINITY; + this.initialized = false; + this.oldPos = null; + this.difDist = null; + } + + double getMetric(int metricIndex) { + if (instancesSeen == 0) return Double.NEGATIVE_INFINITY; + return MetricUtils.getScore(evaluator.getPerformanceMeasurements(), metricIndex); + } + + void addResult(InstanceExample example, double[] votes) { + evaluator.addResult(example, votes); + instancesSeen++; + } + } + + // ========== FIELDS ========== + + protected FishEntry[] school; + protected ConfigurationSpace space; + + /** Boundary through which every learner is configured. */ + protected LearnerConfigurator configurator; + protected double currentStep; + protected long instanceCount; + protected int evaluationInstances; + + /** Detector on the incumbent's error; {@code null} unless drift detection is on. */ + protected ChangeDetector driftDetector; + + /** Cumulative number of drifts signalled over the run; not reset by a restart. */ + protected long driftsDetected; + + protected boolean[] optimizeMask; + + /** + * Pool training the school concurrently; {@code null} when running + * single-threaded. Transient because an executor cannot be serialized, so it + * is (re)created on demand by {@link #initExecutor()}. + */ + protected transient ExecutorService executor; + + @Override + public void setOptimizableParameters(boolean[] mask) { this.optimizeMask = mask; } + + // ========== LIFECYCLE ========== + + @Override + public boolean isRandomizable() { return true; } + + /** + * Creates the training pool on first use, and again after deserialization. + * A pool larger than the batch trained per instance would leave threads idle, + * so the requested job count is capped at the school size. + */ + protected void initExecutor() { + if (this.executor != null) return; + int numberOfJobs = this.numberOfJobsOption.getValue() == -1 + ? Runtime.getRuntime().availableProcessors() + : this.numberOfJobsOption.getValue(); + numberOfJobs = Math.min(numberOfJobs, this.numEstimatorsOption.getValue()); + // SINGLE_THREAD and requesting a single thread are equivalent: training + // then happens in-place and this.executor stays null. + if (numberOfJobs != SINGLE_THREAD && numberOfJobs != 1) { + // Daemon threads: a live pool must not keep the JVM alive after the + // task that ran this learner has finished. + this.executor = Executors.newFixedThreadPool(numberOfJobs, runnable -> { + Thread thread = new Thread(runnable); + thread.setDaemon(true); + return thread; + }); + } + } + + @Override + public void cleanThreads() { + if (this.executor != null) { + this.executor.shutdownNow(); + this.executor = null; + } + } + + @Override + public void resetLearningImpl() { + cleanThreads(); // shut down any pool from a previous reset before creating a new one + driftDetector = driftDetectionOption.isSet() + ? ((ChangeDetector) getPreparedClassOption(driftDetectorOption)).copy() + : null; + instanceCount = 0; + evaluationInstances = 0; + currentStep = initialStepOption.getValue(); + + setConfigurations(); + initializeSchool(); + + } + + @Override + public void setConfigurations() { + try { + this.space = ConfigurationSpace.fromFile(configurationFileOption.getValue()); + this.configurator = new LearnerConfigurator(this.space); + this.configurator.validate(); + } catch (Exception e) { + throw new IllegalStateException("Could not set up " + getClass().getSimpleName() + + " from \"" + configurationFileOption.getValue() + "\": " + e.getMessage(), e); + } + } + + private void initializeSchool() { + int n = numEstimatorsOption.getValue(); + school = new FishEntry[n]; + for (int i = 0; i < n; i++) { + ArrayList params = createRandomParams(); + Classifier model = createModelWithParams(params); + school[i] = new FishEntry(model, newEvaluator(), params); + } + if (verboseOption.isSet()) + System.out.println("FSS_SPT: Initialized school with " + n + " fish"); + } + + @Override + public void checkParameterChange() {} + + @Override + public void swapClassifiers(int bestPerforming) {} + + @Override + public void deepCopyList(int bestPerforming) {} + + @Override + public void changeStateParameter(Parameter parameter, int index) {} + + // ========== PARAM CREATION ========== + + private ArrayList createRandomParams() { + ArrayList params = new ArrayList<>(); + for (ParameterSpec spec : this.space.parameters) { + String name = spec.name; + switch (spec.type) { + case ParameterSpec.TYPE_INT: { + int[] range = {(int) spec.range[0], (int) spec.range[1]}; + int value = classifierRandom.nextInt(range[1] - range[0] + 1) + range[0]; + params.add(new IntParameter(name, value, range, new Random(classifierRandom.nextLong()))); + break; + } + case ParameterSpec.TYPE_DOUBLE: { + double[] range = {spec.range[0], spec.range[1]}; + double value = range[0] + classifierRandom.nextDouble() * (range[1] - range[0]); + params.add(new DoubleParameter(name, value, range, new Random(classifierRandom.nextLong()))); + break; + } + case ParameterSpec.TYPE_CATEGORICAL: { + String[] values = spec.values; + int active = classifierRandom.nextInt(values.length); + params.add(new CategoricalParameter(name, values, active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return params; + } + + /** A model built from scratch at {@code params}. */ + private Classifier createModelWithParams(ArrayList params) { + return this.configurator.instantiate(params, this.classifierRandom.nextInt()); + } + + /** + * Reconfigure a warm-started model in place, returning the model to use. + * Falls back to rebuilding when the space contains a hyperparameter the + * learner only reads at construction time, which by definition cannot take + * effect on a model that is already training. + */ + private Classifier applyParamsToModel(Classifier model, ArrayList params) { + if (!this.configurator.canApplyLive()) { + return this.configurator.instantiate(params, this.classifierRandom.nextInt()); + } + this.configurator.applyLive(model, params); + return model; + } + + private BasicClassificationPerformanceEvaluator newEvaluator() { + BasicClassificationPerformanceEvaluator eval = new BasicClassificationPerformanceEvaluator(); + MetricUtils.configure(eval); + return eval; + } + + // ========== TRAINING ========== + + @Override + public double[] getVotesForInstance(Instance inst) { + return getBestFish().model.getVotesForInstance(inst); + } + + private FishEntry getBestFish() { + int metric = metricOption.getChosenIndex(); + FishEntry best = school[0]; + for (FishEntry f : school) + if (f.getMetric(metric) > best.getMetric(metric)) best = f; + return best; + } + + @Override + public void trainOnInstanceImpl(Instance inst) { + double[] incumbentVotes = driftDetector != null ? getVotesForInstance(inst) : null; + initExecutor(); + instanceCount++; + evaluationInstances++; + InstanceExample example = new InstanceExample(inst); + + Collection trainers = this.executor == null + ? null : new ArrayList(); + + for (FishEntry fish : school) { + double[] votes = fish.model.getVotesForInstance(inst); + fish.addResult(example, votes); + // Every fish owns its model, hence no synchronization here. + if (trainers == null) fish.model.trainOnInstance(inst); + else trainers.add(new TrainingRunnable(fish.model, inst)); + } + + if (trainers != null) { + try { + this.executor.invokeAll(trainers); + } catch (InterruptedException ex) { + throw new RuntimeException("Could not call invokeAll() on training threads."); + } + } + int halfPeriod = gracePeriodOption.getValue() / 2; + + // Phase 1 (halfway): assess improvement, apply individual movement + if (evaluationInstances == halfPeriod) { + sortSchool(); + individualMovementNextEval(); + individualMovement(); + rebuildModels(); + } + + // Phase 2 (full period): feeding, instinctive, volitional movements + if (evaluationInstances >= gracePeriodOption.getValue()) { + evaluationInstances = 0; + sortSchool(); + double weightChange = feeding(); + instinctiveMovement(); + double[] barycenter = calculateBarycenter(); + volitionalMovement(barycenter, weightChange); + updateStep(); + rebuildModels(); + if (verboseOption.isSet()) + System.out.printf("FSS_SPT: Updated at instance %d, step=%.4f, weightChange=%.4f%n", + instanceCount, currentStep, weightChange); + } + + if (incumbentVotes != null) checkDrift(inst, incumbentVotes); + } + + // Rebuild all fish models with their current params after position updates. + // Warm-starts from the best fish unless resetModels is set. + private void rebuildModels() { + FishEntry best = getBestFish(); + for (FishEntry fish : school) { + // Freeze the hyperparameters outside the optimized subset to the best fish. + HPOMethod.freezeToIncumbent(this.optimizeMask, fish.params, best.params); + if (resetModelsOption.isSet()) { + fish.model = createModelWithParams(fish.params); + } else { + fish.model = best.model.copy(); + LearnerConfigurator.reseedCopy(fish.model, this.classifierRandom.nextInt()); + applyParamsToModel(fish.model, fish.params); + } + fish.evaluator = newEvaluator(); + fish.instancesSeen = 0; + } + } + + // ========== SCHOOL SORT ========== + + private void sortSchool() { + int metric = metricOption.getChosenIndex(); + Arrays.sort(school, (a, b) -> Double.compare(b.getMetric(metric), a.getMetric(metric))); + } + + // ========== INDIVIDUAL MOVEMENT NEXT EVAL ========== + // Assess whether the fish improved since baseline; record displacement and fitness delta. + // oldPos is fixed at first call (mirrors Python behaviour where old_pos is never updated). + + private void individualMovementNextEval() { + int metric = metricOption.getChosenIndex(); + for (FishEntry fish : school) { + double currentFitness = fish.getMetric(metric); + double[] currentPos = paramsToDoubleArray(fish.params); + + if (!fish.initialized) { + fish.oldFit = currentFitness; + fish.oldPos = cloneParams(fish.params); + fish.difDist = new double[currentPos.length]; + fish.difFit = 0.0; + fish.initialized = true; + } else if (currentFitness > fish.oldFit) { + fish.difFit = currentFitness - fish.oldFit; + fish.oldFit = currentFitness; + double[] oldPosArr = paramsToDoubleArray(fish.oldPos); + fish.difDist = subtractArrays(currentPos, oldPosArr); + } else { + fish.difFit = 0.0; + fish.difDist = new double[currentPos.length]; + } + } + } + + // ========== INDIVIDUAL MOVEMENT ========== + // Perturb each fish randomly within step * rangeWidth. + + private void individualMovement() { + double[] rangeWidths = getRangeWidths(school[0].params); + for (FishEntry fish : school) { + double[] pos = paramsToDoubleArray(fish.params); + for (int j = 0; j < pos.length; j++) { + double direction = classifierRandom.nextDouble() * 2.0 - 1.0; + pos[j] += currentStep * direction * rangeWidths[j]; + } + applyDoubleArrayToParams(fish.params, pos); + for (Parameter p : fish.params) + if (p.type == 2 && classifierRandom.nextDouble() < currentStep) p.changeParameter(); + } + } + + // ========== FEEDING ========== + // Update fish weights proportional to normalised fitness improvement. + // Returns (sum_before - sum_after): negative when school got heavier. + + private double feeding() { + double maxDifFit = 0.0; + for (FishEntry fish : school) + if (fish.difFit > maxDifFit) maxDifFit = fish.difFit; + + if (maxDifFit == 0.0) return 0.0; + + double weightBefore = 0.0; + for (FishEntry fish : school) weightBefore += fish.weight; + + for (FishEntry fish : school) + fish.weight += fish.difFit / maxDifFit; + + double weightAfter = 0.0; + for (FishEntry fish : school) weightAfter += fish.weight; + + return weightBefore - weightAfter; + } + + // ========== INSTINCTIVE MOVEMENT ========== + // Move all fish by the fitness-weighted average of individual displacements. + + private void instinctiveMovement() { + if (school[0].difDist == null) return; + double totalDifFit = 0.0; + for (FishEntry fish : school) totalDifFit += fish.difFit; + if (totalDifFit == 0.0) return; + + int n = school[0].difDist.length; + double[] instinctiveVector = new double[n]; + for (FishEntry fish : school) + for (int j = 0; j < n; j++) + instinctiveVector[j] += fish.difDist[j] * fish.difFit; + for (int j = 0; j < n; j++) + instinctiveVector[j] /= totalDifFit; + + for (FishEntry fish : school) { + double[] pos = paramsToDoubleArray(fish.params); + for (int j = 0; j < n; j++) pos[j] += instinctiveVector[j]; + applyDoubleArrayToParams(fish.params, pos); + } + } + + // ========== BARYCENTER ========== + // Compute the weight-averaged centre of the school in parameter space. + + private double[] calculateBarycenter() { + int n = paramsToDoubleArray(school[0].params).length; + double[] barycenter = new double[n]; + double totalWeight = 0.0; + for (FishEntry fish : school) { + double[] pos = paramsToDoubleArray(fish.params); + totalWeight += fish.weight; + for (int j = 0; j < n; j++) barycenter[j] += pos[j] * fish.weight; + } + if (totalWeight > 0.0) + for (int j = 0; j < n; j++) barycenter[j] /= totalWeight; + return barycenter; + } + + // ========== VOLITIONAL MOVEMENT ========== + // weightChange < 0 → school got heavier (improvement) → move AWAY from barycenter (explore). + // weightChange >= 0 → no improvement → move TOWARD barycenter (converge). + // This matches the sign convention in the original Python implementation. + + private void volitionalMovement(double[] barycenter, double weightChange) { + double[] rangeWidths = getRangeWidths(school[0].params); + int n = barycenter.length; + for (FishEntry fish : school) { + double[] pos = paramsToDoubleArray(fish.params); + double dist = euclideanDistance(pos, barycenter); + if (dist == 0.0) continue; + double direction = classifierRandom.nextDouble(); + double[] newPos = new double[n]; + for (int j = 0; j < n; j++) { + double delta = 2.0 * currentStep * direction * rangeWidths[j] + * (barycenter[j] - pos[j]) / dist; + newPos[j] = (weightChange < 0) ? pos[j] - delta : pos[j] + delta; + } + applyDoubleArrayToParams(fish.params, newPos); + } + } + + // ========== STEP UPDATE ========== + + private void updateStep() { + double decay = (initialStepOption.getValue() - finalStepOption.getValue()) + / numIterationsOption.getValue(); + currentStep = Math.max(finalStepOption.getValue(), currentStep - decay); + } + + // ========== PARAM / ARRAY HELPERS ========== + + private double[] paramsToDoubleArray(ArrayList params) { + int count = 0; + for (Parameter p : params) if (p.type == 0 || p.type == 1) count++; + double[] result = new double[count]; + int idx = 0; + for (Parameter p : params) { + if (p.type == 0) result[idx++] = ((IntParameter) p).value; + else if (p.type == 1) result[idx++] = ((DoubleParameter) p).value; + } + return result; + } + + private void applyDoubleArrayToParams(ArrayList params, double[] vals) { + int idx = 0; + for (Parameter p : params) { + if (p.type == 0) { + IntParameter ip = (IntParameter) p; + ip.value = clampInt((int) Math.round(vals[idx++]), ip.range[0], ip.range[1]); + } else if (p.type == 1) { + DoubleParameter dp = (DoubleParameter) p; + dp.value = clampDouble(vals[idx++], dp.range[0], dp.range[1]); + } + } + } + + private double[] getRangeWidths(ArrayList params) { + int count = 0; + for (Parameter p : params) if (p.type == 0 || p.type == 1) count++; + double[] widths = new double[count]; + int idx = 0; + for (Parameter p : params) { + if (p.type == 0) widths[idx++] = ((IntParameter) p).range[1] - ((IntParameter) p).range[0]; + else if (p.type == 1) widths[idx++] = ((DoubleParameter) p).range[1] - ((DoubleParameter) p).range[0]; + } + return widths; + } + + private double[] subtractArrays(double[] a, double[] b) { + double[] result = new double[a.length]; + for (int i = 0; i < a.length; i++) result[i] = a[i] - b[i]; + return result; + } + + private double euclideanDistance(double[] a, double[] b) { + double sum = 0.0; + for (int i = 0; i < a.length; i++) { double d = a[i] - b[i]; sum += d * d; } + return Math.sqrt(sum); + } + + private ArrayList cloneParams(ArrayList source) { + ArrayList copy = new ArrayList<>(); + for (Parameter p : source) { + switch (p.type) { + case Parameter.TYPE_INT: { + IntParameter ip = (IntParameter) p; + copy.add(new IntParameter(ip.name, ip.value, ip.range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_DOUBLE: { + DoubleParameter dp = (DoubleParameter) p; + copy.add(new DoubleParameter(dp.name, dp.value, dp.range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_CATEGORICAL: { + CategoricalParameter cp = (CategoricalParameter) p; + copy.add(new CategoricalParameter(cp.name, cp.values.clone(), cp.active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return copy; + } + + private int clampInt(int val, int min, int max) { return Math.max(min, Math.min(max, val)); } + private double clampDouble(double val, double min, double max) { return Math.max(min, Math.min(max, val)); } + + // ========== MOA INTERFACE ========== + + @Override + protected Measurement[] getModelMeasurementsImpl() { + ArrayList measurements = new ArrayList<>(); + measurements.add(new Measurement("driftsDetected", driftsDetected)); + FishEntry best = getBestFish(); + for (Measurement m : best.model.getModelMeasurements()) + measurements.add(m); + for (Parameter p : best.params) { + switch (p.type) { + case Parameter.TYPE_INT: measurements.add(new Measurement(p.name, ((IntParameter) p).value)); break; + case Parameter.TYPE_DOUBLE: measurements.add(new Measurement(p.name, ((DoubleParameter) p).value)); break; + case Parameter.TYPE_CATEGORICAL: measurements.add(new Measurement(p.name, ((CategoricalParameter) p).active)); break; + } + } + measurements.add(new Measurement("currentStep", currentStep)); + return measurements.toArray(new Measurement[0]); + } + + @Override + public void getModelDescription(StringBuilder out, int indent) {} + + @Override + public long measureByteSize() { + long size = SizeOf.sizeOf(this); + for (FishEntry fish : school) size += fish.model.measureByteSize(); + return size; + } + + + private int argmax(double[] arr) { + int best = 0; + for (int i = 1; i < arr.length; i++) + if (arr[i] > arr[best]) best = i; + return best; + } + + /** + * Feeds the incumbent's prediction error to the change detector and, when a + * drift is signalled, reinitializes the whole search from the search space. + * The signal is the 0/1 misclassification indicator. + */ + private void checkDrift(Instance inst, double[] incumbentVotes) { + driftDetector.input(argmax(incumbentVotes) != (int) inst.classValue() ? 1.0 : 0.0); + if (driftDetector.getChange()) { + driftsDetected++; + if (verboseOption.isSet()) + System.out.println("FSS: Drift detected at instance " + instanceCount + + ", reinitializing"); + resetLearningImpl(); + } + } + + /** + * Inner class to assist with the multi-thread execution. + */ + protected class TrainingRunnable implements Runnable, Callable { + final private Classifier learner; + final private Instance instance; + + public TrainingRunnable(Classifier learner, Instance instance) { + this.learner = learner; + this.instance = instance; + } + + @Override + public void run() { + this.learner.trainOnInstance(this.instance); + } + + @Override + public Integer call() { + run(); + return 0; + } + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTRegressor.java b/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTRegressor.java new file mode 100644 index 000000000..9af73c11f --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTRegressor.java @@ -0,0 +1,744 @@ +/* + * FSS_SPTRegressor.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML; + +import com.github.javacliparser.FileOption; +import com.github.javacliparser.FlagOption; +import com.github.javacliparser.FloatOption; +import com.github.javacliparser.IntOption; +import com.github.javacliparser.MultiChoiceOption; +import com.yahoo.labs.samoa.instances.Instance; +import moa.capabilities.CapabilitiesHandler; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.AutoML.Parameters.*; +import moa.classifiers.AutoML.space.ConfigurationSpace; +import moa.classifiers.AutoML.space.LearnerConfigurator; +import moa.classifiers.AutoML.space.ParameterSpec; +import moa.classifiers.Classifier; +import moa.classifiers.core.driftdetection.ChangeDetector; +import moa.classifiers.Regressor; +import moa.core.InstanceExample; +import moa.core.Measurement; +import moa.core.SizeOf; +import moa.evaluation.BasicRegressionPerformanceEvaluator; +import moa.options.ClassOption; + +import java.io.Serializable; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Regression counterpart of {@link FSS_SPTClassifier}: Fish School Search over + * the streaming hyperparameter space, scored with the regression metrics of + * {@link MetricUtils}. + * + *

See details in:
Bruno Veloso, Hugo Amorim Neto, Fernando Buarque, + * Joao Gama. Fish swarm parameter self-tuning for data streams. In Data Mining + * and Knowledge Discovery, 40(1), DOI: 10.1007/s10618-025-01174-8, Springer, + * 2025.

+ * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public class FSS_SPTRegressor extends AbstractClassifier implements Regressor, + CapabilitiesHandler, Serializable, HPOMethod { + + // ========== OPTIONS ========== + + public FileOption configurationFileOption = new FileOption("configurationFile", 'f', + "Search space in JSON format.", null, ".json", false); + + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + "Number of instances between FSS school updates.", 1000, 2, Integer.MAX_VALUE); + + public MultiChoiceOption metricOption = new MultiChoiceOption("metric", 'm', + "Metric to optimize the model.", MetricUtils.REGRESSION_METRIC_NAMES, + MetricUtils.REGRESSION_METRIC_DESCRIPTIONS, 0); + + public IntOption numEstimatorsOption = new IntOption("numEstimators", 'n', + "Number of fish (candidate models) in the school.", 10, 2, Integer.MAX_VALUE); + + public FloatOption initialStepOption = new FloatOption("initialStep", 'a', + "Initial individual movement step size (fraction of parameter range).", 0.1, 0.0, 1.0); + + public FloatOption finalStepOption = new FloatOption("finalStep", 'e', + "Final individual movement step size after numIterations updates.", 0.01, 0.0, 1.0); + + public IntOption numIterationsOption = new IntOption("numIterations", 't', + "Number of step-decay iterations.", 100, 1, Integer.MAX_VALUE); + + public FlagOption resetModelsOption = new FlagOption("resetModels", 'r', + "Create fresh models after each movement instead of warm-starting from best."); + + public FlagOption verboseOption = new FlagOption("verbose", 'v', + "Print FSS events to stdout."); + + public FlagOption driftDetectionOption = new FlagOption("driftDetection", 'd', + "Enable drift detection on the incumbent's prediction error."); + + public ClassOption driftDetectorOption = new ClassOption("driftDetector", 'D', + "Change detector to use when drift detection is enabled; on a detected" + + " drift the whole search is reinitialized from the search space.", + ChangeDetector.class, "ADWINChangeDetector"); + + public IntOption numberOfJobsOption = new IntOption("numberOfJobs", 'j', + "Total number of concurrent jobs used for processing (-1 = as much as possible, 0 = do not use multithreading)", + 1, -1, Integer.MAX_VALUE); + + protected static final int SINGLE_THREAD = 0; + + // ========== INNER CLASS ========== + + protected static class FishEntry implements Serializable { + Classifier model; + BasicRegressionPerformanceEvaluator evaluator; + ArrayList params; + long instancesSeen; + double weight; + double difFit; + double oldFit; + boolean initialized; + ArrayList oldPos; // position at first nexteval (never updated after init) + double[] difDist; // displacement relative to oldPos when improvement occurred + + FishEntry(Classifier model, BasicRegressionPerformanceEvaluator evaluator, + ArrayList params) { + this.model = model; + this.evaluator = evaluator; + this.params = params; + this.instancesSeen = 0; + this.weight = 0.1; + this.difFit = 0.0; + this.oldFit = Double.NEGATIVE_INFINITY; + this.initialized = false; + this.oldPos = null; + this.difDist = null; + } + + double getMetric(int metricIndex) { + if (instancesSeen == 0) return Double.NEGATIVE_INFINITY; + return MetricUtils.getRegressionScore(evaluator.getPerformanceMeasurements(), metricIndex); + } + + void addResult(InstanceExample example, double[] votes) { + evaluator.addResult(example, votes); + instancesSeen++; + } + } + + // ========== FIELDS ========== + + protected FishEntry[] school; + protected ConfigurationSpace space; + + /** Boundary through which every learner is configured. */ + protected LearnerConfigurator configurator; + protected double currentStep; + protected long instanceCount; + protected int evaluationInstances; + + /** Detector on the incumbent's error; {@code null} unless drift detection is on. */ + protected ChangeDetector driftDetector; + + /** Cumulative number of drifts signalled over the run; not reset by a restart. */ + protected long driftsDetected; + + protected boolean[] optimizeMask; + + /** + * Pool training the school concurrently; {@code null} when running + * single-threaded. Transient because an executor cannot be serialized, so it + * is (re)created on demand by {@link #initExecutor()}. + */ + protected transient ExecutorService executor; + + @Override + public void setOptimizableParameters(boolean[] mask) { this.optimizeMask = mask; } + + // ========== LIFECYCLE ========== + + @Override + public boolean isRandomizable() { return true; } + + /** + * Creates the training pool on first use, and again after deserialization. + * A pool larger than the batch trained per instance would leave threads idle, + * so the requested job count is capped at the school size. + */ + protected void initExecutor() { + if (this.executor != null) return; + int numberOfJobs = this.numberOfJobsOption.getValue() == -1 + ? Runtime.getRuntime().availableProcessors() + : this.numberOfJobsOption.getValue(); + numberOfJobs = Math.min(numberOfJobs, this.numEstimatorsOption.getValue()); + // SINGLE_THREAD and requesting a single thread are equivalent: training + // then happens in-place and this.executor stays null. + if (numberOfJobs != SINGLE_THREAD && numberOfJobs != 1) { + // Daemon threads: a live pool must not keep the JVM alive after the + // task that ran this learner has finished. + this.executor = Executors.newFixedThreadPool(numberOfJobs, runnable -> { + Thread thread = new Thread(runnable); + thread.setDaemon(true); + return thread; + }); + } + } + + @Override + public void cleanThreads() { + if (this.executor != null) { + this.executor.shutdownNow(); + this.executor = null; + } + } + + @Override + public void resetLearningImpl() { + cleanThreads(); // shut down any pool from a previous reset before creating a new one + driftDetector = driftDetectionOption.isSet() + ? ((ChangeDetector) getPreparedClassOption(driftDetectorOption)).copy() + : null; + instanceCount = 0; + evaluationInstances = 0; + currentStep = initialStepOption.getValue(); + + setConfigurations(); + initializeSchool(); + + } + + @Override + public void setConfigurations() { + try { + this.space = ConfigurationSpace.fromFile(configurationFileOption.getValue()); + this.configurator = new LearnerConfigurator(this.space); + this.configurator.validate(); + } catch (Exception e) { + throw new IllegalStateException("Could not set up " + getClass().getSimpleName() + + " from \"" + configurationFileOption.getValue() + "\": " + e.getMessage(), e); + } + } + + private void initializeSchool() { + int n = numEstimatorsOption.getValue(); + school = new FishEntry[n]; + for (int i = 0; i < n; i++) { + ArrayList params = createRandomParams(); + Classifier model = createModelWithParams(params); + school[i] = new FishEntry(model, newEvaluator(), params); + } + if (verboseOption.isSet()) + System.out.println("FSS_SPT: Initialized school with " + n + " fish"); + } + + @Override + public void checkParameterChange() {} + + @Override + public void swapClassifiers(int bestPerforming) {} + + @Override + public void deepCopyList(int bestPerforming) {} + + @Override + public void changeStateParameter(Parameter parameter, int index) {} + + // ========== PARAM CREATION ========== + + private ArrayList createRandomParams() { + ArrayList params = new ArrayList<>(); + for (ParameterSpec spec : this.space.parameters) { + String name = spec.name; + switch (spec.type) { + case ParameterSpec.TYPE_INT: { + int[] range = {(int) spec.range[0], (int) spec.range[1]}; + int value = classifierRandom.nextInt(range[1] - range[0] + 1) + range[0]; + params.add(new IntParameter(name, value, range, new Random(classifierRandom.nextLong()))); + break; + } + case ParameterSpec.TYPE_DOUBLE: { + double[] range = {spec.range[0], spec.range[1]}; + double value = range[0] + classifierRandom.nextDouble() * (range[1] - range[0]); + params.add(new DoubleParameter(name, value, range, new Random(classifierRandom.nextLong()))); + break; + } + case ParameterSpec.TYPE_CATEGORICAL: { + String[] values = spec.values; + int active = classifierRandom.nextInt(values.length); + params.add(new CategoricalParameter(name, values, active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return params; + } + + /** A model built from scratch at {@code params}. */ + private Classifier createModelWithParams(ArrayList params) { + return this.configurator.instantiate(params, this.classifierRandom.nextInt()); + } + + /** + * Reconfigure a warm-started model in place, returning the model to use. + * Falls back to rebuilding when the space contains a hyperparameter the + * learner only reads at construction time, which by definition cannot take + * effect on a model that is already training. + */ + private Classifier applyParamsToModel(Classifier model, ArrayList params) { + if (!this.configurator.canApplyLive()) { + return this.configurator.instantiate(params, this.classifierRandom.nextInt()); + } + this.configurator.applyLive(model, params); + return model; + } + + private BasicRegressionPerformanceEvaluator newEvaluator() { + return new BasicRegressionPerformanceEvaluator(); + } + + // ========== TRAINING ========== + + @Override + public double[] getVotesForInstance(Instance inst) { + return getBestFish().model.getVotesForInstance(inst); + } + + private FishEntry getBestFish() { + int metric = metricOption.getChosenIndex(); + FishEntry best = school[0]; + for (FishEntry f : school) + if (f.getMetric(metric) > best.getMetric(metric)) best = f; + return best; + } + + @Override + public void trainOnInstanceImpl(Instance inst) { + double[] incumbentVotes = driftDetector != null ? getVotesForInstance(inst) : null; + initExecutor(); + instanceCount++; + evaluationInstances++; + InstanceExample example = new InstanceExample(inst); + + Collection trainers = this.executor == null + ? null : new ArrayList(); + + for (FishEntry fish : school) { + double[] votes = fish.model.getVotesForInstance(inst); + fish.addResult(example, votes); + // Every fish owns its model, hence no synchronization here. + if (trainers == null) fish.model.trainOnInstance(inst); + else trainers.add(new TrainingRunnable(fish.model, inst)); + } + + if (trainers != null) { + try { + this.executor.invokeAll(trainers); + } catch (InterruptedException ex) { + throw new RuntimeException("Could not call invokeAll() on training threads."); + } + } + int halfPeriod = gracePeriodOption.getValue() / 2; + + // Phase 1 (halfway): assess improvement, apply individual movement + if (evaluationInstances == halfPeriod) { + sortSchool(); + individualMovementNextEval(); + individualMovement(); + rebuildModels(); + } + + // Phase 2 (full period): feeding, instinctive, volitional movements + if (evaluationInstances >= gracePeriodOption.getValue()) { + evaluationInstances = 0; + sortSchool(); + double weightChange = feeding(); + instinctiveMovement(); + double[] barycenter = calculateBarycenter(); + volitionalMovement(barycenter, weightChange); + updateStep(); + rebuildModels(); + if (verboseOption.isSet()) + System.out.printf("FSS_SPT: Updated at instance %d, step=%.4f, weightChange=%.4f%n", + instanceCount, currentStep, weightChange); + } + + if (incumbentVotes != null) checkDrift(inst, incumbentVotes); + } + + // Rebuild all fish models with their current params after position updates. + // Warm-starts from the best fish unless resetModels is set. + private void rebuildModels() { + FishEntry best = getBestFish(); + for (FishEntry fish : school) { + // Freeze the hyperparameters outside the optimized subset to the best fish. + HPOMethod.freezeToIncumbent(this.optimizeMask, fish.params, best.params); + if (resetModelsOption.isSet()) { + fish.model = createModelWithParams(fish.params); + } else { + fish.model = best.model.copy(); + LearnerConfigurator.reseedCopy(fish.model, this.classifierRandom.nextInt()); + applyParamsToModel(fish.model, fish.params); + } + fish.evaluator = newEvaluator(); + fish.instancesSeen = 0; + } + } + + // ========== SCHOOL SORT ========== + + private void sortSchool() { + int metric = metricOption.getChosenIndex(); + Arrays.sort(school, (a, b) -> Double.compare(b.getMetric(metric), a.getMetric(metric))); + } + + // ========== INDIVIDUAL MOVEMENT NEXT EVAL ========== + // Assess whether the fish improved since baseline; record displacement and fitness delta. + // oldPos is fixed at first call (mirrors Python behaviour where old_pos is never updated). + + private void individualMovementNextEval() { + int metric = metricOption.getChosenIndex(); + for (FishEntry fish : school) { + double currentFitness = fish.getMetric(metric); + double[] currentPos = paramsToDoubleArray(fish.params); + + if (!fish.initialized) { + fish.oldFit = currentFitness; + fish.oldPos = cloneParams(fish.params); + fish.difDist = new double[currentPos.length]; + fish.difFit = 0.0; + fish.initialized = true; + } else if (currentFitness > fish.oldFit) { + fish.difFit = currentFitness - fish.oldFit; + fish.oldFit = currentFitness; + double[] oldPosArr = paramsToDoubleArray(fish.oldPos); + fish.difDist = subtractArrays(currentPos, oldPosArr); + } else { + fish.difFit = 0.0; + fish.difDist = new double[currentPos.length]; + } + } + } + + // ========== INDIVIDUAL MOVEMENT ========== + // Perturb each fish randomly within step * rangeWidth. + + private void individualMovement() { + double[] rangeWidths = getRangeWidths(school[0].params); + for (FishEntry fish : school) { + double[] pos = paramsToDoubleArray(fish.params); + for (int j = 0; j < pos.length; j++) { + double direction = classifierRandom.nextDouble() * 2.0 - 1.0; + pos[j] += currentStep * direction * rangeWidths[j]; + } + applyDoubleArrayToParams(fish.params, pos); + for (Parameter p : fish.params) + if (p.type == 2 && classifierRandom.nextDouble() < currentStep) p.changeParameter(); + } + } + + // ========== FEEDING ========== + // Update fish weights proportional to normalised fitness improvement. + // Returns (sum_before - sum_after): negative when school got heavier. + + private double feeding() { + double maxDifFit = 0.0; + for (FishEntry fish : school) + if (fish.difFit > maxDifFit) maxDifFit = fish.difFit; + + if (maxDifFit == 0.0) return 0.0; + + double weightBefore = 0.0; + for (FishEntry fish : school) weightBefore += fish.weight; + + for (FishEntry fish : school) + fish.weight += fish.difFit / maxDifFit; + + double weightAfter = 0.0; + for (FishEntry fish : school) weightAfter += fish.weight; + + return weightBefore - weightAfter; + } + + // ========== INSTINCTIVE MOVEMENT ========== + // Move all fish by the fitness-weighted average of individual displacements. + + private void instinctiveMovement() { + if (school[0].difDist == null) return; + double totalDifFit = 0.0; + for (FishEntry fish : school) totalDifFit += fish.difFit; + if (totalDifFit == 0.0) return; + + int n = school[0].difDist.length; + double[] instinctiveVector = new double[n]; + for (FishEntry fish : school) + for (int j = 0; j < n; j++) + instinctiveVector[j] += fish.difDist[j] * fish.difFit; + for (int j = 0; j < n; j++) + instinctiveVector[j] /= totalDifFit; + + for (FishEntry fish : school) { + double[] pos = paramsToDoubleArray(fish.params); + for (int j = 0; j < n; j++) pos[j] += instinctiveVector[j]; + applyDoubleArrayToParams(fish.params, pos); + } + } + + // ========== BARYCENTER ========== + // Compute the weight-averaged centre of the school in parameter space. + + private double[] calculateBarycenter() { + int n = paramsToDoubleArray(school[0].params).length; + double[] barycenter = new double[n]; + double totalWeight = 0.0; + for (FishEntry fish : school) { + double[] pos = paramsToDoubleArray(fish.params); + totalWeight += fish.weight; + for (int j = 0; j < n; j++) barycenter[j] += pos[j] * fish.weight; + } + if (totalWeight > 0.0) + for (int j = 0; j < n; j++) barycenter[j] /= totalWeight; + return barycenter; + } + + // ========== VOLITIONAL MOVEMENT ========== + // weightChange < 0 → school got heavier (improvement) → move AWAY from barycenter (explore). + // weightChange >= 0 → no improvement → move TOWARD barycenter (converge). + // This matches the sign convention in the original Python implementation. + + private void volitionalMovement(double[] barycenter, double weightChange) { + double[] rangeWidths = getRangeWidths(school[0].params); + int n = barycenter.length; + for (FishEntry fish : school) { + double[] pos = paramsToDoubleArray(fish.params); + double dist = euclideanDistance(pos, barycenter); + if (dist == 0.0) continue; + double direction = classifierRandom.nextDouble(); + double[] newPos = new double[n]; + for (int j = 0; j < n; j++) { + double delta = 2.0 * currentStep * direction * rangeWidths[j] + * (barycenter[j] - pos[j]) / dist; + newPos[j] = (weightChange < 0) ? pos[j] - delta : pos[j] + delta; + } + applyDoubleArrayToParams(fish.params, newPos); + } + } + + // ========== STEP UPDATE ========== + + private void updateStep() { + double decay = (initialStepOption.getValue() - finalStepOption.getValue()) + / numIterationsOption.getValue(); + currentStep = Math.max(finalStepOption.getValue(), currentStep - decay); + } + + // ========== PARAM / ARRAY HELPERS ========== + + private double[] paramsToDoubleArray(ArrayList params) { + int count = 0; + for (Parameter p : params) if (p.type == 0 || p.type == 1) count++; + double[] result = new double[count]; + int idx = 0; + for (Parameter p : params) { + if (p.type == 0) result[idx++] = ((IntParameter) p).value; + else if (p.type == 1) result[idx++] = ((DoubleParameter) p).value; + } + return result; + } + + private void applyDoubleArrayToParams(ArrayList params, double[] vals) { + int idx = 0; + for (Parameter p : params) { + if (p.type == 0) { + IntParameter ip = (IntParameter) p; + ip.value = clampInt((int) Math.round(vals[idx++]), ip.range[0], ip.range[1]); + } else if (p.type == 1) { + DoubleParameter dp = (DoubleParameter) p; + dp.value = clampDouble(vals[idx++], dp.range[0], dp.range[1]); + } + } + } + + private double[] getRangeWidths(ArrayList params) { + int count = 0; + for (Parameter p : params) if (p.type == 0 || p.type == 1) count++; + double[] widths = new double[count]; + int idx = 0; + for (Parameter p : params) { + if (p.type == 0) widths[idx++] = ((IntParameter) p).range[1] - ((IntParameter) p).range[0]; + else if (p.type == 1) widths[idx++] = ((DoubleParameter) p).range[1] - ((DoubleParameter) p).range[0]; + } + return widths; + } + + private double[] subtractArrays(double[] a, double[] b) { + double[] result = new double[a.length]; + for (int i = 0; i < a.length; i++) result[i] = a[i] - b[i]; + return result; + } + + private double euclideanDistance(double[] a, double[] b) { + double sum = 0.0; + for (int i = 0; i < a.length; i++) { double d = a[i] - b[i]; sum += d * d; } + return Math.sqrt(sum); + } + + private ArrayList cloneParams(ArrayList source) { + ArrayList copy = new ArrayList<>(); + for (Parameter p : source) { + switch (p.type) { + case Parameter.TYPE_INT: { + IntParameter ip = (IntParameter) p; + copy.add(new IntParameter(ip.name, ip.value, ip.range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_DOUBLE: { + DoubleParameter dp = (DoubleParameter) p; + copy.add(new DoubleParameter(dp.name, dp.value, dp.range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_CATEGORICAL: { + CategoricalParameter cp = (CategoricalParameter) p; + copy.add(new CategoricalParameter(cp.name, cp.values.clone(), cp.active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return copy; + } + + private int clampInt(int val, int min, int max) { return Math.max(min, Math.min(max, val)); } + private double clampDouble(double val, double min, double max) { return Math.max(min, Math.min(max, val)); } + + // ========== MOA INTERFACE ========== + + @Override + protected Measurement[] getModelMeasurementsImpl() { + ArrayList measurements = new ArrayList<>(); + measurements.add(new Measurement("driftsDetected", driftsDetected)); + FishEntry best = getBestFish(); + for (Measurement m : best.model.getModelMeasurements()) + measurements.add(m); + for (Parameter p : best.params) { + switch (p.type) { + case Parameter.TYPE_INT: measurements.add(new Measurement(p.name, ((IntParameter) p).value)); break; + case Parameter.TYPE_DOUBLE: measurements.add(new Measurement(p.name, ((DoubleParameter) p).value)); break; + case Parameter.TYPE_CATEGORICAL: measurements.add(new Measurement(p.name, ((CategoricalParameter) p).active)); break; + } + } + measurements.add(new Measurement("currentStep", currentStep)); + return measurements.toArray(new Measurement[0]); + } + + @Override + public void getModelDescription(StringBuilder out, int indent) {} + + @Override + public long measureByteSize() { + long size = SizeOf.sizeOf(this); + for (FishEntry fish : school) size += fish.model.measureByteSize(); + return size; + } + + @Override + public double getCandidateScore(int i) { + return school[i].getMetric(metricOption.getChosenIndex()); + } + + @Override + public double getClassifierScore() { + return getBestFish().getMetric(metricOption.getChosenIndex()); + } + + @Override + public int getNumberOfCandidates() { return numEstimatorsOption.getValue(); } + + @Override + public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } + + @Override + public int getEvaluationInstancesCount() { return evaluationInstances; } + + @Override + public int getGracePeriod() { return gracePeriodOption.getValue(); } + + @Override + public Classifier getMainClassifier() { return getBestFish().model; } + + @Override + public ArrayList getReferenceParameters() { + return (school != null && school.length > 0) ? getBestFish().params : null; + } + + @Override + public String getConfigurationFile() { return this.configurationFileOption.getValue();} + + + @Override + public ArrayList> getCandidateParameters() { + ArrayList> list = new ArrayList<>(); + if (school != null) { + for (FishEntry f : school) { + list.add(f.params); + } + } + return list; + } + + + /** + * Feeds the incumbent's prediction error to the change detector and, when a + * drift is signalled, reinitializes the whole search from the search space. + * The signal is the absolute error |y - y_pred|. + */ + private void checkDrift(Instance inst, double[] incumbentVotes) { + double predicted = incumbentVotes.length > 0 ? incumbentVotes[0] : 0.0; + driftDetector.input(Math.abs(predicted - inst.classValue())); + if (driftDetector.getChange()) { + driftsDetected++; + if (verboseOption.isSet()) + System.out.println("FSS: Drift detected at instance " + instanceCount + + ", reinitializing"); + resetLearningImpl(); + } + } + + /** + * Inner class to assist with the multi-thread execution. + */ + protected class TrainingRunnable implements Runnable, Callable { + final private Classifier learner; + final private Instance instance; + + public TrainingRunnable(Classifier learner, Instance instance) { + this.learner = learner; + this.instance = instance; + } + + @Override + public void run() { + this.learner.trainOnInstance(this.instance); + } + + @Override + public Integer call() { + run(); + return 0; + } + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/HPOMethod.java b/moa/src/main/java/moa/classifiers/AutoML/HPOMethod.java new file mode 100644 index 000000000..3d78e24a9 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/HPOMethod.java @@ -0,0 +1,128 @@ +/* + * HPOMethod.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML; + +import moa.classifiers.AutoML.Parameters.CategoricalParameter; +import moa.classifiers.AutoML.Parameters.DoubleParameter; +import moa.classifiers.AutoML.Parameters.IntParameter; +import moa.classifiers.AutoML.Parameters.Parameter; +import moa.classifiers.Classifier; + +import java.util.ArrayList; + +/** + * Common surface of the streaming hyperparameter optimisation methods, so that + * wrappers can drive and introspect any of them without knowing which search + * strategy is underneath. + * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public interface HPOMethod { + + void setConfigurations(); + + void checkParameterChange(); + + void swapClassifiers(int bestPerforming); + + void deepCopyList(int bestPerforming); + + void changeStateParameter(Parameter parameter, int index); + + default void cleanThreads() {} + + // Accessors letting an introspecting wrapper read the runtime state of an + // HPO method. Defaults return null/0 so implementers compile unchanged; + // a wrapper should require its wrapped learner to override the ones it needs. + + default ArrayList getReferenceParameters() { return null; } + + default ArrayList> getCandidateParameters() { return null; } + + default double getCandidateScore(int i) { return Double.NaN; } + + default double getClassifierScore() { return Double.NaN; } + + default int getNumberOfCandidates() { return 0; } + + /** + * Number of candidates actually trained and evaluated in the current + * evaluation window. Methods that keep a fixed pool simply report the pool + * size; methods with a "max capacity pool, active prefix" design report the + * prefix length. + */ + default int getActiveCandidates() { return getNumberOfCandidates(); } + + /** + * Request that only {@code n} candidates be trained and evaluated from the + * next evaluation window on. Implementations must clamp {@code n} to + * {@code [1, poolCapacity]} and must only let the change take effect at a + * window boundary, so that all scores compared inside one window come from + * candidates that saw the same instances. Returns the granted value, which + * may differ from {@code n}. + * + *

Pushed by an external controller that sizes the candidate budget. The + * default is a no-op so that existing implementers compile unchanged. + */ + default int setActiveCandidates(int n) { return getActiveCandidates(); } + + default long getStatesEvaluatedCount() { return 0; } + + default int getEvaluationInstancesCount() { return 0; } + + default int getGracePeriod() { return Integer.MAX_VALUE; } + + default Classifier getMainClassifier() { return null; } + + default String getConfigurationFile() { return null; } + + /** + * Restrict optimisation to a subset of hyperparameters. {@code mask[k]} is + * {@code true} when parameter k (in search space order; a categorical + * counts as a single entry) should keep being optimised, and {@code false} + * when it must be frozen to the current incumbent ("best configuration") + * value. A {@code null} mask (the default) means optimise every parameter, + * preserving the unrestricted behaviour. + */ + default void setOptimizableParameters(boolean[] mask) {} + + /** + * Overwrite the frozen dimensions of {@code params} (those with + * {@code mask[k] == false}) with the corresponding value from + * {@code incumbent}. Optimised dimensions are left untouched. Shared by all + * HPO methods so that frozen hyperparameters carry the real best-configuration + * value rather than a placeholder - important for surrogate-based methods + * (e.g. the Bayesian Stream Tuner) whose feature vector must stay consistent. + */ + static void freezeToIncumbent(boolean[] mask, ArrayList params, + ArrayList incumbent) { + if (mask == null || params == null || incumbent == null) return; + int n = Math.min(params.size(), Math.min(incumbent.size(), mask.length)); + for (int k = 0; k < n; k++) { + if (mask[k]) continue; + Parameter dst = params.get(k); + Parameter src = incumbent.get(k); + if (dst.type != src.type) continue; + switch (dst.type) { + case Parameter.TYPE_INT: ((IntParameter) dst).value = ((IntParameter) src).value; break; + case Parameter.TYPE_DOUBLE: ((DoubleParameter) dst).value = ((DoubleParameter) src).value; break; + case Parameter.TYPE_CATEGORICAL: ((CategoricalParameter) dst).active = ((CategoricalParameter) src).active; break; + } + } + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/MESSPTClassifier.java b/moa/src/main/java/moa/classifiers/AutoML/MESSPTClassifier.java new file mode 100644 index 000000000..c0860b773 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/MESSPTClassifier.java @@ -0,0 +1,675 @@ +/* + * MESSPTClassifier.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML; + +import com.github.javacliparser.FileOption; +import com.github.javacliparser.FlagOption; +import com.github.javacliparser.FloatOption; +import com.github.javacliparser.IntOption; +import com.github.javacliparser.MultiChoiceOption; +import com.yahoo.labs.samoa.instances.Instance; +import moa.capabilities.CapabilitiesHandler; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.AutoML.Parameters.*; +import moa.classifiers.AutoML.space.ConfigurationSpace; +import moa.classifiers.AutoML.space.LearnerConfigurator; +import moa.classifiers.AutoML.space.ParameterSpec; +import moa.classifiers.Classifier; +import moa.classifiers.core.driftdetection.ChangeDetector; +import moa.classifiers.MultiClassClassifier; +import moa.core.InstanceExample; +import moa.core.Measurement; +import moa.core.SizeOf; +import moa.evaluation.BasicClassificationPerformanceEvaluator; +import moa.options.ClassOption; + +import java.io.Serializable; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Differential-evolution search over a streaming hyperparameter space: a + * population of candidates is trained in parallel and each generation replaces + * a member by its DE/best/1 trial point when the trial scores higher over the + * evaluation window. + * + *

Candidates are configured through {@link LearnerConfigurator}, i.e. by + * writing MOA {@link com.github.javacliparser.Option}s, so any MOA classifier + * can be tuned as shipped. + * + *

See details in:
Antonio R. Moya, Bruno Veloso, Joao Gama, Sebastian + * Ventura. Improving hyper-parameter self-tuning for data streams by adapting + * an evolutionary approach. In Data Mining and Knowledge Discovery, + * 38(3):1289-1315, Springer, 2023.

+ * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public class MESSPTClassifier extends AbstractClassifier implements MultiClassClassifier, + CapabilitiesHandler, Serializable, HPOMethod { + + public FileOption configurationFileOption = new FileOption("configurationFile", 'f', + "Search space in JSON format.", null, ".json", false); + + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + "Number of instances between DE updates.", 1000, 1, Integer.MAX_VALUE); + + public IntOption populationSizeOption = new IntOption("populationSize", 'p', + "Number of candidate models in the population.", 4, 2, Integer.MAX_VALUE); + + public FloatOption convergenceSphereOption = new FloatOption("convergenceSphere", 'c', + "Convergence threshold: squared distance between best params across generations.", 0.001, 0.0, Double.MAX_VALUE); + + public FloatOption initialMutationFactorOption = new FloatOption("initialMutationFactor", 'F', + "Initial DE mutation factor.", 0.5, 0.0, 1.0); + + public FloatOption initialCrossoverRateOption = new FloatOption("initialCrossoverRate", 'C', + "Initial DE crossover rate.", 0.5, 0.0, 1.0); + + public FloatOption augmentationStepOption = new FloatOption("augmentationStep", 'a', + "Step by which the mutation factor decreases and the crossover rate increases each generation.", 0.025, 0.0, 1.0); + + public MultiChoiceOption metricOption = new MultiChoiceOption("metric", 'm', + "Metric to optimize the model.", MetricUtils.METRIC_NAMES, MetricUtils.METRIC_DESCRIPTIONS, 4); + + public FlagOption resetModelsOption = new FlagOption("resetModels", 'r', + "Create fresh classifiers for trial points instead of warm-starting from best."); + + public FlagOption verboseOption = new FlagOption("verbose", 'v', + "Print DE events to stdout."); + + public FlagOption driftDetectionOption = new FlagOption("driftDetection", 'd', + "Enable drift detection on the incumbent's prediction error."); + + public ClassOption driftDetectorOption = new ClassOption("driftDetector", 'D', + "Change detector to use when drift detection is enabled; on a detected" + + " drift the whole search is reinitialized from the search space.", + ChangeDetector.class, "ADWINChangeDetector"); + + public IntOption numberOfJobsOption = new IntOption("numberOfJobs", 'j', + "Total number of concurrent jobs used for processing (-1 = as much as possible, 0 = do not use multithreading)", + 1, -1, Integer.MAX_VALUE); + + protected static final int SINGLE_THREAD = 0; + + // ========== INNER CLASS ========== + + protected static class PopulationEntry implements Serializable { + Classifier model; + BasicClassificationPerformanceEvaluator evaluator; + ArrayList params; + + PopulationEntry(Classifier model, BasicClassificationPerformanceEvaluator evaluator, + ArrayList params) { + this.model = model; + this.evaluator = evaluator; + this.params = params; + } + + double getMetric(int metricIndex) { + return MetricUtils.getScore(evaluator.getPerformanceMeasurements(), metricIndex); + } + } + + // ========== FIELDS ========== + + protected PopulationEntry[] population; + protected ArrayList oldBestParams; + + protected ConfigurationSpace space; + + /** Boundary through which every learner is configured. */ + protected LearnerConfigurator configurator; + + protected long instanceCount; + protected int evaluationInstances; + protected boolean converged; + + protected double mutationFactor; + protected double crossoverRate; + + protected boolean[] optimizeMask; + + /** Detector on the incumbent's error; {@code null} unless drift detection is on. */ + protected ChangeDetector driftDetector; + + /** Cumulative number of drifts signalled over the run; not reset by a restart. */ + protected long driftsDetected; + + /** + * Pool training the population concurrently; {@code null} when running + * single-threaded. Transient because an executor cannot be serialized, so it + * is (re)created on demand by {@link #initExecutor()}. + */ + protected transient ExecutorService executor; + + @Override + public void setOptimizableParameters(boolean[] mask) { this.optimizeMask = mask; } + + // ========== LIFECYCLE ========== + + @Override + public boolean isRandomizable() { + return true; + } + + /** + * Creates the training pool on first use, and again after deserialization. + * A pool larger than the batch trained per instance would leave threads idle, + * so the requested job count is capped at the population size. + */ + protected void initExecutor() { + if (this.executor != null) return; + int numberOfJobs = this.numberOfJobsOption.getValue() == -1 + ? Runtime.getRuntime().availableProcessors() + : this.numberOfJobsOption.getValue(); + numberOfJobs = Math.min(numberOfJobs, this.populationSizeOption.getValue()); + // SINGLE_THREAD and requesting a single thread are equivalent: training + // then happens in-place and this.executor stays null. + if (numberOfJobs != SINGLE_THREAD && numberOfJobs != 1) { + // Daemon threads: a live pool must not keep the JVM alive after the + // task that ran this learner has finished. + this.executor = Executors.newFixedThreadPool(numberOfJobs, runnable -> { + Thread thread = new Thread(runnable); + thread.setDaemon(true); + return thread; + }); + } + } + + @Override + public void cleanThreads() { + if (this.executor != null) { + this.executor.shutdownNow(); + this.executor = null; + } + } + + @Override + public void resetLearningImpl() { + cleanThreads(); // shut down any pool from a previous reset before creating a new one + converged = false; + instanceCount = 0; + evaluationInstances = 0; + oldBestParams = null; + mutationFactor = initialMutationFactorOption.getValue(); + crossoverRate = initialCrossoverRateOption.getValue(); + + driftDetector = driftDetectionOption.isSet() + ? ((ChangeDetector) getPreparedClassOption(driftDetectorOption)).copy() + : null; + + setConfigurations(); + initializePopulation(); + + } + + @Override + public void setConfigurations() { + try { + this.space = ConfigurationSpace.fromFile(configurationFileOption.getValue()); + this.configurator = new LearnerConfigurator(this.space); + this.configurator.validate(); + } catch (Exception e) { + throw new IllegalStateException("Could not set up " + getClass().getSimpleName() + + " from \"" + configurationFileOption.getValue() + "\": " + e.getMessage(), e); + } + } + + private void initializePopulation() { + int n = populationSizeOption.getValue(); + population = new PopulationEntry[n]; + for (int i = 0; i < n; i++) { + ArrayList params = createRandomParams(); + Classifier model = createModelWithParams(params); + population[i] = new PopulationEntry(model, newEvaluator(), params); + } + if (verboseOption.isSet()) + System.out.println("MESSPTClassifier: Initialized population with " + n + " models"); + } + + private ArrayList createRandomParams() { + ArrayList params = new ArrayList<>(); + for (ParameterSpec spec : this.space.parameters) { + String name = spec.name; + switch (spec.type) { + case ParameterSpec.TYPE_INT: { + int[] range = {(int) spec.range[0], (int) spec.range[1]}; + int value = classifierRandom.nextInt(range[1] - range[0] + 1) + range[0]; + params.add(new IntParameter(name, value, range, new Random(classifierRandom.nextLong()))); + break; + } + case ParameterSpec.TYPE_DOUBLE: { + double[] range = {spec.range[0], spec.range[1]}; + double value = range[0] + classifierRandom.nextDouble() * (range[1] - range[0]); + params.add(new DoubleParameter(name, value, range, new Random(classifierRandom.nextLong()))); + break; + } + case ParameterSpec.TYPE_CATEGORICAL: { + String[] values = spec.values; + int active = classifierRandom.nextInt(values.length); + params.add(new CategoricalParameter(name, values, active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return params; + } + + /** A model built from scratch at {@code params}. */ + private Classifier createModelWithParams(ArrayList params) { + return this.configurator.instantiate(params, this.classifierRandom.nextInt()); + } + + /** + * Reconfigure a warm-started model in place, returning the model to use. + * Falls back to rebuilding when the space contains a hyperparameter the + * learner only reads at construction time, which by definition cannot take + * effect on a model that is already training. + */ + private Classifier applyParamsToModel(Classifier model, ArrayList params) { + if (!this.configurator.canApplyLive()) { + return this.configurator.instantiate(params, this.classifierRandom.nextInt()); + } + this.configurator.applyLive(model, params); + return model; + } + + private BasicClassificationPerformanceEvaluator newEvaluator() { + BasicClassificationPerformanceEvaluator eval = new BasicClassificationPerformanceEvaluator(); + MetricUtils.configure(eval); + return eval; + } + + // ========== TRAINING ========== + + @Override + public double[] getVotesForInstance(Instance inst) { + return population[0].model.getVotesForInstance(inst); + } + + @Override + public void trainOnInstanceImpl(Instance inst) { + double[] incumbentVotes = driftDetector != null ? getVotesForInstance(inst) : null; + initExecutor(); + instanceCount++; + evaluationInstances++; + InstanceExample example = new InstanceExample(inst); + + if (converged) { + trainConverged(inst, example); + } else { + trainNotConverged(inst, example); + } + + if (incumbentVotes != null) checkDrift(inst, incumbentVotes); + } + + private void trainConverged(Instance inst, InstanceExample example) { + PopulationEntry best = population[0]; + double[] votes = best.model.getVotesForInstance(inst); + best.evaluator.addResult(example, votes); + best.model.trainOnInstance(inst); + } + + private void trainNotConverged(Instance inst, InstanceExample example) { + Collection trainers = this.executor == null + ? null : new ArrayList(); + + for (PopulationEntry entry : population) { + double[] votes = entry.model.getVotesForInstance(inst); + entry.evaluator.addResult(example, votes); + + // Every individual owns its model, hence no synchronization here. + if (trainers == null) entry.model.trainOnInstance(inst); + else trainers.add(new TrainingRunnable(entry.model, inst)); + } + + if (trainers != null) { + try { + this.executor.invokeAll(trainers); + } catch (InterruptedException ex) { + throw new RuntimeException("Could not call invokeAll() on training threads."); + } + } + + if (evaluationInstances >= gracePeriodOption.getValue()) { + evaluationInstances = 0; + updatePopulation(); + + if (checkConvergence()) { + if (verboseOption.isSet()) + System.out.println("MESSPTClassifier: Converged at instance " + instanceCount); + converged = true; + } else { + mutationFactor = Math.max(0.0, mutationFactor - augmentationStepOption.getValue()); + crossoverRate = Math.min(1.0, crossoverRate + augmentationStepOption.getValue()); + } + } + } + + // ========== DE UPDATE ========== + + private void updatePopulation() { + sortPopulation(); + oldBestParams = cloneParams(population[0].params); + + int n = population.length; + PopulationEntry[] newPopulation = new PopulationEntry[n]; + newPopulation[0] = population[0]; // elitism: keep best + + for (int i = 1; i < n; i++) { + ArrayList trialParams = deMutationBest1(i); + ArrayList newParams = crossover(population[i].params, trialParams); + + // Freeze the hyperparameters outside the optimized subset to the elite (best) individual. + HPOMethod.freezeToIncumbent(this.optimizeMask, newParams, population[0].params); + + PopulationEntry newEntry; + if (resetModelsOption.isSet()) { + Classifier model = createModelWithParams(newParams); + newEntry = new PopulationEntry(model, newEvaluator(), newParams); + } else { + Classifier model = population[0].model.copy(); + LearnerConfigurator.reseedCopy(model, this.classifierRandom.nextInt()); + model = applyParamsToModel(model, newParams); + newEntry = new PopulationEntry(model, newEvaluator(), newParams); + } + newPopulation[i] = newEntry; + } + + population = newPopulation; + } + + private void sortPopulation() { + int metric = metricOption.getChosenIndex(); + Arrays.sort(population, (a, b) -> Double.compare(b.getMetric(metric), a.getMetric(metric))); + } + + // DE/best/1 mutation: v = best + mutationFactor * (r1 - r2) + private ArrayList deMutationBest1(int targetIndex) { + // Sample two distinct indices from population, neither being 0 (best) nor targetIndex + List pool = new ArrayList<>(); + for (int i = 1; i < population.length; i++) { + if (i != targetIndex) pool.add(i); + } + Collections.shuffle(pool, classifierRandom); + int r1Idx = pool.get(0); + int r2Idx = pool.get(1); + + ArrayList best = population[0].params; + ArrayList r1 = population[r1Idx].params; + ArrayList r2 = population[r2Idx].params; + + ArrayList trial = new ArrayList<>(); + for (int i = 0; i < best.size(); i++) { + Parameter pb = best.get(i); + Parameter p1 = r1.get(i); + Parameter p2 = r2.get(i); + switch (pb.type) { + case Parameter.TYPE_INT: { + int[] range = ((IntParameter) pb).range; + double mutated = ((IntParameter) pb).value + mutationFactor * (((IntParameter) p1).value - ((IntParameter) p2).value); + int val = clampInt((int) Math.round(mutated), range[0], range[1]); + trial.add(new IntParameter(pb.name, val, range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_DOUBLE: { + double[] range = ((DoubleParameter) pb).range; + double mutated = ((DoubleParameter) pb).value + mutationFactor * (((DoubleParameter) p1).value - ((DoubleParameter) p2).value); + double val = clampDouble(mutated, range[0], range[1]); + trial.add(new DoubleParameter(pb.name, val, range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_CATEGORICAL: { + CategoricalParameter cp = (CategoricalParameter) pb; + int active = classifierRandom.nextInt(cp.values.length); + trial.add(new CategoricalParameter(pb.name, cp.values.clone(), active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return trial; + } + + private ArrayList crossover(ArrayList target, ArrayList trial) { + ArrayList result = new ArrayList<>(); + for (int i = 0; i < target.size(); i++) { + Parameter pt = target.get(i); + Parameter pv = trial.get(i); + boolean useTrial = classifierRandom.nextDouble() < crossoverRate; + switch (pt.type) { + case Parameter.TYPE_INT: { + IntParameter ip = (IntParameter) (useTrial ? pv : pt); + result.add(new IntParameter(pt.name, ip.value, ((IntParameter) pt).range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_DOUBLE: { + DoubleParameter dp = (DoubleParameter) (useTrial ? pv : pt); + result.add(new DoubleParameter(pt.name, dp.value, ((DoubleParameter) pt).range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_CATEGORICAL: { + CategoricalParameter ct = (CategoricalParameter) pt; + CategoricalParameter cv = (CategoricalParameter) pv; + int active = useTrial ? cv.active : ct.active; + result.add(new CategoricalParameter(pt.name, ct.values.clone(), active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return result; + } + + private boolean checkConvergence() { + if (oldBestParams == null) return false; + ArrayList current = population[0].params; + double distSq = 0; + for (int i = 0; i < oldBestParams.size(); i++) { + Parameter op = oldBestParams.get(i); + Parameter cp = current.get(i); + if (op.type == 0) { + double diff = ((IntParameter) op).value - ((IntParameter) cp).value; + distSq += diff * diff; + } else if (op.type == 1) { + double diff = ((DoubleParameter) op).value - ((DoubleParameter) cp).value; + distSq += diff * diff; + } + } + double threshold = convergenceSphereOption.getValue(); + return distSq < threshold * threshold; + } + + // ========== HELPERS ========== + + private ArrayList cloneParams(ArrayList source) { + ArrayList copy = new ArrayList<>(); + for (Parameter p : source) { + switch (p.type) { + case Parameter.TYPE_INT: { + IntParameter ip = (IntParameter) p; + copy.add(new IntParameter(ip.name, ip.value, ip.range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_DOUBLE: { + DoubleParameter dp = (DoubleParameter) p; + copy.add(new DoubleParameter(dp.name, dp.value, dp.range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_CATEGORICAL: { + CategoricalParameter cp = (CategoricalParameter) p; + copy.add(new CategoricalParameter(cp.name, cp.values.clone(), cp.active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return copy; + } + + private int clampInt(int val, int min, int max) { + return Math.max(min, Math.min(max, val)); + } + + private double clampDouble(double val, double min, double max) { + return Math.max(min, Math.min(max, val)); + } + + private int argmax(double[] arr) { + int best = 0; + for (int i = 1; i < arr.length; i++) + if (arr[i] > arr[best]) best = i; + return best; + } + + @Override + public void checkParameterChange() {} + + @Override + public void swapClassifiers(int bestPerforming) {} + + @Override + public void deepCopyList(int bestPerforming) {} + + @Override + public void changeStateParameter(Parameter parameter, int index) {} + + // ========== MOA INTERFACE ========== + + @Override + protected Measurement[] getModelMeasurementsImpl() { + ArrayList measurements = new ArrayList<>(); + measurements.add(new Measurement("driftsDetected", driftsDetected)); + for (Measurement m : population[0].model.getModelMeasurements()) + measurements.add(m); + measurements.add(new Measurement("mutationFactor", mutationFactor)); + measurements.add(new Measurement("crossoverRate", crossoverRate)); + for (Parameter p : population[0].params) { + switch (p.type) { + case Parameter.TYPE_INT: + measurements.add(new Measurement(p.name, ((IntParameter) p).value)); + break; + case Parameter.TYPE_DOUBLE: + measurements.add(new Measurement(p.name, ((DoubleParameter) p).value)); + break; + case Parameter.TYPE_CATEGORICAL: + measurements.add(new Measurement(p.name, ((CategoricalParameter) p).active)); + break; + } + } + return measurements.toArray(new Measurement[0]); + } + + @Override + public void getModelDescription(StringBuilder out, int indent) { + } + + @Override + public long measureByteSize() { + long size = SizeOf.sizeOf(this); + for (PopulationEntry entry : population) + size += entry.model.measureByteSize(); + return size; + } + + @Override + public double getCandidateScore(int i) { + return population[i].getMetric(metricOption.getChosenIndex()); + } + + @Override + public double getClassifierScore() { + return population[0].getMetric(metricOption.getChosenIndex()); + } + + @Override + public int getNumberOfCandidates() { return populationSizeOption.getValue(); } + + @Override + public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } + + @Override + public int getEvaluationInstancesCount() { return evaluationInstances; } + + @Override + public int getGracePeriod() { return gracePeriodOption.getValue(); } + + @Override + public Classifier getMainClassifier() { return population[0].model; } + + @Override + public ArrayList getReferenceParameters() { + return (population != null && population.length > 0) ? population[0].params : null; + } + + @Override + public String getConfigurationFile() { return this.configurationFileOption.getValue();} + + + @Override + public ArrayList> getCandidateParameters() { + ArrayList> list = new ArrayList<>(); + if (population != null) { + for (PopulationEntry e : population) { + list.add(e.params); + } + } + return list; + } + + + /** + * Feeds the incumbent's prediction error to the change detector and, when a + * drift is signalled, reinitializes the whole search from the search space. + * The signal is the 0/1 misclassification indicator. + */ + private void checkDrift(Instance inst, double[] incumbentVotes) { + driftDetector.input(argmax(incumbentVotes) != (int) inst.classValue() ? 1.0 : 0.0); + if (driftDetector.getChange()) { + driftsDetected++; + if (verboseOption.isSet()) + System.out.println("MESSPT: Drift detected at instance " + instanceCount + + ", reinitializing"); + resetLearningImpl(); + } + } + + /** + * Inner class to assist with the multi-thread execution. + */ + protected class TrainingRunnable implements Runnable, Callable { + final private Classifier learner; + final private Instance instance; + + public TrainingRunnable(Classifier learner, Instance instance) { + this.learner = learner; + this.instance = instance; + } + + @Override + public void run() { + this.learner.trainOnInstance(this.instance); + } + + @Override + public Integer call() { + run(); + return 0; + } + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/MESSPTRegressor.java b/moa/src/main/java/moa/classifiers/AutoML/MESSPTRegressor.java new file mode 100644 index 000000000..67f8a3169 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/MESSPTRegressor.java @@ -0,0 +1,668 @@ +/* + * MESSPTRegressor.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML; + +import com.github.javacliparser.FileOption; +import com.github.javacliparser.FlagOption; +import com.github.javacliparser.FloatOption; +import com.github.javacliparser.IntOption; +import com.github.javacliparser.MultiChoiceOption; +import com.yahoo.labs.samoa.instances.Instance; +import moa.capabilities.CapabilitiesHandler; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.AutoML.Parameters.*; +import moa.classifiers.AutoML.space.ConfigurationSpace; +import moa.classifiers.AutoML.space.LearnerConfigurator; +import moa.classifiers.AutoML.space.ParameterSpec; +import moa.classifiers.Classifier; +import moa.classifiers.core.driftdetection.ChangeDetector; +import moa.classifiers.Regressor; +import moa.core.InstanceExample; +import moa.core.Measurement; +import moa.core.SizeOf; +import moa.evaluation.BasicRegressionPerformanceEvaluator; +import moa.options.ClassOption; + +import java.io.Serializable; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Regression counterpart of {@link MESSPTClassifier}: differential evolution + * over the streaming hyperparameter space, scored with the regression metrics + * of {@link MetricUtils}. + * + *

See details in:
Antonio R. Moya, Bruno Veloso, Joao Gama, Sebastian + * Ventura. Improving hyper-parameter self-tuning for data streams by adapting + * an evolutionary approach. In Data Mining and Knowledge Discovery, + * 38(3):1289-1315, Springer, 2023.

+ * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public class MESSPTRegressor extends AbstractClassifier implements Regressor, + CapabilitiesHandler, Serializable, HPOMethod { + + public FileOption configurationFileOption = new FileOption("configurationFile", 'f', + "Search space in JSON format.", null, ".json", false); + + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + "Number of instances between DE updates.", 1000, 1, Integer.MAX_VALUE); + + public IntOption populationSizeOption = new IntOption("populationSize", 'p', + "Number of candidate models in the population.", 4, 2, Integer.MAX_VALUE); + + public FloatOption convergenceSphereOption = new FloatOption("convergenceSphere", 'c', + "Convergence threshold: squared distance between best params across generations.", 0.001, 0.0, Double.MAX_VALUE); + + public FloatOption initialMutationFactorOption = new FloatOption("initialMutationFactor", 'F', + "Initial DE mutation factor.", 0.5, 0.0, 1.0); + + public FloatOption initialCrossoverRateOption = new FloatOption("initialCrossoverRate", 'C', + "Initial DE crossover rate.", 0.5, 0.0, 1.0); + + public FloatOption augmentationStepOption = new FloatOption("augmentationStep", 'a', + "Step by which the mutation factor decreases and the crossover rate increases each generation.", 0.025, 0.0, 1.0); + + public MultiChoiceOption metricOption = new MultiChoiceOption("metric", 'm', + "Metric to optimize the model.", MetricUtils.REGRESSION_METRIC_NAMES, + MetricUtils.REGRESSION_METRIC_DESCRIPTIONS, 0); + + public FlagOption resetModelsOption = new FlagOption("resetModels", 'r', + "Create fresh regressors for trial points instead of warm-starting from best."); + + public FlagOption verboseOption = new FlagOption("verbose", 'v', + "Print DE events to stdout."); + + public FlagOption driftDetectionOption = new FlagOption("driftDetection", 'd', + "Enable drift detection on the incumbent's prediction error."); + + public ClassOption driftDetectorOption = new ClassOption("driftDetector", 'D', + "Change detector to use when drift detection is enabled; on a detected" + + " drift the whole search is reinitialized from the search space.", + ChangeDetector.class, "ADWINChangeDetector"); + + public IntOption numberOfJobsOption = new IntOption("numberOfJobs", 'j', + "Total number of concurrent jobs used for processing (-1 = as much as possible, 0 = do not use multithreading)", + 1, -1, Integer.MAX_VALUE); + + protected static final int SINGLE_THREAD = 0; + + // ========== INNER CLASS ========== + + protected static class PopulationEntry implements Serializable { + Classifier model; + BasicRegressionPerformanceEvaluator evaluator; + ArrayList params; + + PopulationEntry(Classifier model, BasicRegressionPerformanceEvaluator evaluator, + ArrayList params) { + this.model = model; + this.evaluator = evaluator; + this.params = params; + } + + double getMetric(int metricIndex) { + return MetricUtils.getRegressionScore(evaluator.getPerformanceMeasurements(), metricIndex); + } + } + + // ========== FIELDS ========== + + protected PopulationEntry[] population; + protected ArrayList oldBestParams; + + protected ConfigurationSpace space; + + /** Boundary through which every learner is configured. */ + protected LearnerConfigurator configurator; + + protected long instanceCount; + protected int evaluationInstances; + protected boolean converged; + + protected double mutationFactor; + protected double crossoverRate; + + protected boolean[] optimizeMask; + + /** Detector on the incumbent's error; {@code null} unless drift detection is on. */ + protected ChangeDetector driftDetector; + + /** Cumulative number of drifts signalled over the run; not reset by a restart. */ + protected long driftsDetected; + + /** + * Pool training the population concurrently; {@code null} when running + * single-threaded. Transient because an executor cannot be serialized, so it + * is (re)created on demand by {@link #initExecutor()}. + */ + protected transient ExecutorService executor; + + @Override + public void setOptimizableParameters(boolean[] mask) { this.optimizeMask = mask; } + + // ========== LIFECYCLE ========== + + @Override + public boolean isRandomizable() { + return true; + } + + /** + * Creates the training pool on first use, and again after deserialization. + * A pool larger than the batch trained per instance would leave threads idle, + * so the requested job count is capped at the population size. + */ + protected void initExecutor() { + if (this.executor != null) return; + int numberOfJobs = this.numberOfJobsOption.getValue() == -1 + ? Runtime.getRuntime().availableProcessors() + : this.numberOfJobsOption.getValue(); + numberOfJobs = Math.min(numberOfJobs, this.populationSizeOption.getValue()); + // SINGLE_THREAD and requesting a single thread are equivalent: training + // then happens in-place and this.executor stays null. + if (numberOfJobs != SINGLE_THREAD && numberOfJobs != 1) { + // Daemon threads: a live pool must not keep the JVM alive after the + // task that ran this learner has finished. + this.executor = Executors.newFixedThreadPool(numberOfJobs, runnable -> { + Thread thread = new Thread(runnable); + thread.setDaemon(true); + return thread; + }); + } + } + + @Override + public void cleanThreads() { + if (this.executor != null) { + this.executor.shutdownNow(); + this.executor = null; + } + } + + @Override + public void resetLearningImpl() { + cleanThreads(); // shut down any pool from a previous reset before creating a new one + converged = false; + instanceCount = 0; + evaluationInstances = 0; + oldBestParams = null; + mutationFactor = initialMutationFactorOption.getValue(); + crossoverRate = initialCrossoverRateOption.getValue(); + + driftDetector = driftDetectionOption.isSet() + ? ((ChangeDetector) getPreparedClassOption(driftDetectorOption)).copy() + : null; + + setConfigurations(); + initializePopulation(); + + } + + + public void setConfigurations() { + try { + this.space = ConfigurationSpace.fromFile(configurationFileOption.getValue()); + this.configurator = new LearnerConfigurator(this.space); + this.configurator.validate(); + } catch (Exception e) { + throw new IllegalStateException("Could not set up " + getClass().getSimpleName() + + " from \"" + configurationFileOption.getValue() + "\": " + e.getMessage(), e); + } + } + + @Override + public void checkParameterChange() { + + } + + @Override + public void swapClassifiers(int bestPerforming) { + + } + + @Override + public void deepCopyList(int bestPerforming) { + + } + + @Override + public void changeStateParameter(Parameter parameter, int index) { + + } + + private void initializePopulation() { + int n = populationSizeOption.getValue(); + population = new PopulationEntry[n]; + for (int i = 0; i < n; i++) { + ArrayList params = createRandomParams(); + Classifier model = createModelWithParams(params); + population[i] = new PopulationEntry(model, newEvaluator(), params); + } + if (verboseOption.isSet()) + System.out.println("MESSPTRegressor: Initialized population with " + n + " models"); + } + + private ArrayList createRandomParams() { + ArrayList params = new ArrayList<>(); + for (ParameterSpec spec : this.space.parameters) { + String name = spec.name; + switch (spec.type) { + case ParameterSpec.TYPE_INT: { + int[] range = {(int) spec.range[0], (int) spec.range[1]}; + int value = classifierRandom.nextInt(range[1] - range[0] + 1) + range[0]; + params.add(new IntParameter(name, value, range, new Random(classifierRandom.nextLong()))); + break; + } + case ParameterSpec.TYPE_DOUBLE: { + double[] range = {spec.range[0], spec.range[1]}; + double value = range[0] + classifierRandom.nextDouble() * (range[1] - range[0]); + params.add(new DoubleParameter(name, value, range, new Random(classifierRandom.nextLong()))); + break; + } + case ParameterSpec.TYPE_CATEGORICAL: { + String[] values = spec.values; + int active = classifierRandom.nextInt(values.length); + params.add(new CategoricalParameter(name, values, active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return params; + } + + /** A model built from scratch at {@code params}. */ + private Classifier createModelWithParams(ArrayList params) { + return this.configurator.instantiate(params, this.classifierRandom.nextInt()); + } + + /** + * Reconfigure a warm-started model in place, returning the model to use. + * Falls back to rebuilding when the space contains a hyperparameter the + * learner only reads at construction time, which by definition cannot take + * effect on a model that is already training. + */ + private Classifier applyParamsToModel(Classifier model, ArrayList params) { + if (!this.configurator.canApplyLive()) { + return this.configurator.instantiate(params, this.classifierRandom.nextInt()); + } + this.configurator.applyLive(model, params); + return model; + } + + private BasicRegressionPerformanceEvaluator newEvaluator() { + return new BasicRegressionPerformanceEvaluator(); + } + + // ========== TRAINING ========== + + @Override + public double[] getVotesForInstance(Instance inst) { + return population[0].model.getVotesForInstance(inst); + } + + @Override + public void trainOnInstanceImpl(Instance inst) { + double[] incumbentVotes = driftDetector != null ? getVotesForInstance(inst) : null; + initExecutor(); + instanceCount++; + evaluationInstances++; + InstanceExample example = new InstanceExample(inst); + + trainNotConverged(inst, example); + + if (incumbentVotes != null) checkDrift(inst, incumbentVotes); + } + + private void trainConverged(Instance inst, InstanceExample example) { + PopulationEntry best = population[0]; + double[] votes = best.model.getVotesForInstance(inst); + best.evaluator.addResult(example, votes); + best.model.trainOnInstance(inst); + } + + private void trainNotConverged(Instance inst, InstanceExample example) { + Collection trainers = this.executor == null + ? null : new ArrayList(); + + for (PopulationEntry entry : population) { + double[] votes = entry.model.getVotesForInstance(inst); + entry.evaluator.addResult(example, votes); + + // Every individual owns its model, hence no synchronization here. + if (trainers == null) entry.model.trainOnInstance(inst); + else trainers.add(new TrainingRunnable(entry.model, inst)); + } + + if (trainers != null) { + try { + this.executor.invokeAll(trainers); + } catch (InterruptedException ex) { + throw new RuntimeException("Could not call invokeAll() on training threads."); + } + } + + if (evaluationInstances >= gracePeriodOption.getValue()) { + evaluationInstances = 0; + updatePopulation(); + + if (checkConvergence()) { + if (verboseOption.isSet()) + System.out.println("MESSPTRegressor: Converged at instance " + instanceCount); + converged = true; + } else { + mutationFactor = Math.max(0.0, mutationFactor - augmentationStepOption.getValue()); + crossoverRate = Math.min(1.0, crossoverRate + augmentationStepOption.getValue()); + } + } + } + + // ========== DE UPDATE ========== + + private void updatePopulation() { + sortPopulation(); + oldBestParams = cloneParams(population[0].params); + + int n = population.length; + PopulationEntry[] newPopulation = new PopulationEntry[n]; + newPopulation[0] = population[0]; // elitism: keep best + + for (int i = 1; i < n; i++) { + ArrayList trialParams = deMutationBest1(i); + ArrayList newParams = crossover(population[i].params, trialParams); + + // Freeze the hyperparameters outside the optimized subset to the elite (best) individual. + HPOMethod.freezeToIncumbent(this.optimizeMask, newParams, population[0].params); + + PopulationEntry newEntry; + if (resetModelsOption.isSet()) { + Classifier model = createModelWithParams(newParams); + newEntry = new PopulationEntry(model, newEvaluator(), newParams); + } else { + Classifier model = population[0].model.copy(); + LearnerConfigurator.reseedCopy(model, this.classifierRandom.nextInt()); + model = applyParamsToModel(model, newParams); + newEntry = new PopulationEntry(model, newEvaluator(), newParams); + } + newPopulation[i] = newEntry; + } + + population = newPopulation; + } + + private void sortPopulation() { + int metric = metricOption.getChosenIndex(); + Arrays.sort(population, (a, b) -> Double.compare(b.getMetric(metric), a.getMetric(metric))); + } + + // DE/best/1 mutation: v = best + mutationFactor * (r1 - r2) + private ArrayList deMutationBest1(int targetIndex) { + List pool = new ArrayList<>(); + for (int i = 1; i < population.length; i++) { + if (i != targetIndex) pool.add(i); + } + Collections.shuffle(pool, classifierRandom); + int r1Idx = pool.get(0); + int r2Idx = pool.get(1); + + ArrayList best = population[0].params; + ArrayList r1 = population[r1Idx].params; + ArrayList r2 = population[r2Idx].params; + + ArrayList trial = new ArrayList<>(); + for (int i = 0; i < best.size(); i++) { + Parameter pb = best.get(i); + Parameter p1 = r1.get(i); + Parameter p2 = r2.get(i); + switch (pb.type) { + case Parameter.TYPE_INT: { + int[] range = ((IntParameter) pb).range; + double mutated = ((IntParameter) pb).value + mutationFactor * (((IntParameter) p1).value - ((IntParameter) p2).value); + int val = clampInt((int) Math.round(mutated), range[0], range[1]); + trial.add(new IntParameter(pb.name, val, range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_DOUBLE: { + double[] range = ((DoubleParameter) pb).range; + double mutated = ((DoubleParameter) pb).value + mutationFactor * (((DoubleParameter) p1).value - ((DoubleParameter) p2).value); + double val = clampDouble(mutated, range[0], range[1]); + trial.add(new DoubleParameter(pb.name, val, range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_CATEGORICAL: { + CategoricalParameter cp = (CategoricalParameter) pb; + int active = classifierRandom.nextInt(cp.values.length); + trial.add(new CategoricalParameter(pb.name, cp.values.clone(), active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return trial; + } + + private ArrayList crossover(ArrayList target, ArrayList trial) { + ArrayList result = new ArrayList<>(); + for (int i = 0; i < target.size(); i++) { + Parameter pt = target.get(i); + Parameter pv = trial.get(i); + boolean useTrial = classifierRandom.nextDouble() < crossoverRate; + switch (pt.type) { + case Parameter.TYPE_INT: { + IntParameter ip = (IntParameter) (useTrial ? pv : pt); + result.add(new IntParameter(pt.name, ip.value, ((IntParameter) pt).range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_DOUBLE: { + DoubleParameter dp = (DoubleParameter) (useTrial ? pv : pt); + result.add(new DoubleParameter(pt.name, dp.value, ((DoubleParameter) pt).range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_CATEGORICAL: { + CategoricalParameter ct = (CategoricalParameter) pt; + CategoricalParameter cv = (CategoricalParameter) pv; + int active = useTrial ? cv.active : ct.active; + result.add(new CategoricalParameter(pt.name, ct.values.clone(), active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return result; + } + + private boolean checkConvergence() { + if (oldBestParams == null) return false; + ArrayList current = population[0].params; + double distSq = 0; + for (int i = 0; i < oldBestParams.size(); i++) { + Parameter op = oldBestParams.get(i); + Parameter cp = current.get(i); + if (op.type == 0) { + double diff = ((IntParameter) op).value - ((IntParameter) cp).value; + distSq += diff * diff; + } else if (op.type == 1) { + double diff = ((DoubleParameter) op).value - ((DoubleParameter) cp).value; + distSq += diff * diff; + } + } + double threshold = convergenceSphereOption.getValue(); + return distSq < threshold * threshold; + } + + // ========== HELPERS ========== + + private ArrayList cloneParams(ArrayList source) { + ArrayList copy = new ArrayList<>(); + for (Parameter p : source) { + switch (p.type) { + case Parameter.TYPE_INT: { + IntParameter ip = (IntParameter) p; + copy.add(new IntParameter(ip.name, ip.value, ip.range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_DOUBLE: { + DoubleParameter dp = (DoubleParameter) p; + copy.add(new DoubleParameter(dp.name, dp.value, dp.range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_CATEGORICAL: { + CategoricalParameter cp = (CategoricalParameter) p; + copy.add(new CategoricalParameter(cp.name, cp.values.clone(), cp.active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return copy; + } + + private int clampInt(int val, int min, int max) { + return Math.max(min, Math.min(max, val)); + } + + private double clampDouble(double val, double min, double max) { + return Math.max(min, Math.min(max, val)); + } + + + + // ========== MOA INTERFACE ========== + + @Override + protected Measurement[] getModelMeasurementsImpl() { + ArrayList measurements = new ArrayList<>(); + measurements.add(new Measurement("driftsDetected", driftsDetected)); + for (Measurement m : population[0].model.getModelMeasurements()) + measurements.add(m); + measurements.add(new Measurement("mutationFactor", mutationFactor)); + measurements.add(new Measurement("crossoverRate", crossoverRate)); + for (Parameter p : population[0].params) { + switch (p.type) { + case Parameter.TYPE_INT: + measurements.add(new Measurement(p.name, ((IntParameter) p).value)); + break; + case Parameter.TYPE_DOUBLE: + measurements.add(new Measurement(p.name, ((DoubleParameter) p).value)); + break; + case Parameter.TYPE_CATEGORICAL: + measurements.add(new Measurement(p.name, ((CategoricalParameter) p).active)); + break; + } + } + return measurements.toArray(new Measurement[0]); + } + + @Override + public void getModelDescription(StringBuilder out, int indent) { + } + + @Override + public long measureByteSize() { + long size = SizeOf.sizeOf(this); + for (PopulationEntry entry : population) + size += entry.model.measureByteSize(); + return size; + } + + @Override + public double getCandidateScore(int i) { + return population[i].getMetric(metricOption.getChosenIndex()); + } + + @Override + public double getClassifierScore() { + return population[0].getMetric(metricOption.getChosenIndex()); + } + + @Override + public int getNumberOfCandidates() { return populationSizeOption.getValue(); } + + @Override + public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } + + @Override + public int getEvaluationInstancesCount() { return evaluationInstances; } + + @Override + public int getGracePeriod() { return gracePeriodOption.getValue(); } + + @Override + public Classifier getMainClassifier() { return population[0].model; } + + @Override + public ArrayList getReferenceParameters() { + return (population != null && population.length > 0) ? population[0].params : null; + } + + @Override + public String getConfigurationFile() { return this.configurationFileOption.getValue();} + + + @Override + public ArrayList> getCandidateParameters() { + ArrayList> list = new ArrayList<>(); + if (population != null) { + for (PopulationEntry e : population) { + list.add(e.params); + } + } + return list; + } + + + /** + * Feeds the incumbent's prediction error to the change detector and, when a + * drift is signalled, reinitializes the whole search from the search space. + * The signal is the absolute error |y - y_pred|. + */ + private void checkDrift(Instance inst, double[] incumbentVotes) { + double predicted = incumbentVotes.length > 0 ? incumbentVotes[0] : 0.0; + driftDetector.input(Math.abs(predicted - inst.classValue())); + if (driftDetector.getChange()) { + driftsDetected++; + if (verboseOption.isSet()) + System.out.println("MESSPT: Drift detected at instance " + instanceCount + + ", reinitializing"); + resetLearningImpl(); + } + } + + /** + * Inner class to assist with the multi-thread execution. + */ + protected class TrainingRunnable implements Runnable, Callable { + final private Classifier learner; + final private Instance instance; + + public TrainingRunnable(Classifier learner, Instance instance) { + this.learner = learner; + this.instance = instance; + } + + @Override + public void run() { + this.learner.trainOnInstance(this.instance); + } + + @Override + public Integer call() { + run(); + return 0; + } + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/MetricUtils.java b/moa/src/main/java/moa/classifiers/AutoML/MetricUtils.java new file mode 100644 index 000000000..b5fbbe150 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/MetricUtils.java @@ -0,0 +1,191 @@ +/* + * MetricUtils.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML; + +import moa.core.Measurement; +import moa.evaluation.BasicClassificationPerformanceEvaluator; + +/** + * Shared metric selection for the AutoML methods. Centralises the list of + * metrics exposed through each method's {@code metricOption}, read out of a + * {@link BasicClassificationPerformanceEvaluator}'s measurements by name so it + * is independent of the number of classes. + * + *

F1, precision and recall are aggregated here rather than taken from the + * evaluator's own aggregate measurements, for two reasons. First, MOA's + * {@code getF1Statistic()} returns the F1 of the averaged precision and + * recall, which is not the macro F1 and is not what a search should rank + * candidates by. Second, a class absent from an evaluation window leaves its + * per-class estimator at {@code 0/0}, so MOA's aggregates collapse to + * {@code NaN} and every candidate ties - fatal for a search that ranks by that + * score. Both are handled here by averaging per-class values under an explicit + * NaN policy, which keeps the correction inside this package instead of + * changing an evaluator the rest of MOA depends on. + * + *

Methods using these metrics must switch on the per-class outputs of their + * evaluators, which {@link #configure} does. + * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public final class MetricUtils { + + public static final String[] METRIC_NAMES = { + "Accuracy", "Kappa", "KappaTemporal", "KappaM", "F1", "Precision", "Recall" + }; + + public static final String[] METRIC_DESCRIPTIONS = { + "Percentage of correctly classified instances", + "Kappa statistic (percent)", + "Kappa Temporal statistic (percent)", + "Kappa M statistic (percent)", + "F1 score, macro averaged over classes (percent)", + "Precision, macro averaged over observed classes (percent)", + "Recall, macro averaged over observed classes (percent)" + }; + + public static final int ACCURACY = 0; + public static final int KAPPA = 1; + public static final int KAPPA_TEMPORAL = 2; + public static final int KAPPA_M = 3; + public static final int F1 = 4; + public static final int PRECISION = 5; + public static final int RECALL = 6; + + private static final String[] AGGREGATE_NAMES = { + "classifications correct (percent)", + "Kappa Statistic (percent)", + "Kappa Temporal Statistic (percent)", + "Kappa M Statistic (percent)", + "F1 Score (percent)", + "Precision (percent)", + "Recall (percent)" + }; + + private static final String F1_PER_CLASS = "F1 Score for class "; + private static final String PRECISION_PER_CLASS = "Precision for class "; + private static final String RECALL_PER_CLASS = "Recall for class "; + + public static final String[] REGRESSION_METRIC_NAMES = { + "R2", "AdjustedR2", "MAE", "RMSE", "RelativeMAE", "RelativeRMSE" + }; + + public static final String[] REGRESSION_METRIC_DESCRIPTIONS = { + "Coefficient of determination", + "Adjusted coefficient of determination", + "Mean absolute error", + "Root mean squared error", + "Relative mean absolute error", + "Relative root mean squared error" + }; + + private static final String[] REGRESSION_MEASUREMENT_NAMES = { + "coefficient of determination", + "adjusted coefficient of determination", + "mean absolute error", + "root mean squared error", + "relative mean absolute error", + "relative root mean squared error" + }; + + /** Whether a larger value of the metric at the same index means a better model. */ + private static final boolean[] REGRESSION_HIGHER_IS_BETTER = { + true, true, false, false, false, false + }; + + private MetricUtils() {} + + /** + * Switches on every output {@link #getScore} needs. Call this on each + * evaluator an AutoML method creates, in place of setting + * {@code precisionRecallOutputOption} alone. + */ + public static void configure(BasicClassificationPerformanceEvaluator evaluator) { + evaluator.precisionRecallOutputOption.setValue(true); + evaluator.f1PerClassOption.setValue(true); + evaluator.precisionPerClassOption.setValue(true); + evaluator.recallPerClassOption.setValue(true); + } + + public static double getScore(Measurement[] measurements, int metricChoice) { + if (measurements == null || metricChoice < 0 || metricChoice >= AGGREGATE_NAMES.length) { + return Double.NEGATIVE_INFINITY; + } + switch (metricChoice) { + case F1: + // An undefined class scores zero, so the average stays over all + // classes and a candidate that ignores a class is penalised. + return macroAverage(measurements, F1_PER_CLASS, AGGREGATE_NAMES[F1], true); + case PRECISION: + return macroAverage(measurements, PRECISION_PER_CLASS, AGGREGATE_NAMES[PRECISION], false); + case RECALL: + return macroAverage(measurements, RECALL_PER_CLASS, AGGREGATE_NAMES[RECALL], false); + default: + return lookup(measurements, AGGREGATE_NAMES[metricChoice]); + } + } + + /** + * Averages the per-class entries whose name starts with {@code prefix}. + * When {@code countUndefined} is set, classes with no observations count as + * zero; otherwise they are left out of the average entirely. Falls back to + * {@code aggregateName} if per-class outputs are switched off. + */ + private static double macroAverage(Measurement[] measurements, String prefix, + String aggregateName, boolean countUndefined) { + double total = 0.0; + int classes = 0; + int defined = 0; + for (Measurement measurement : measurements) { + if (!measurement.getName().startsWith(prefix)) continue; + classes++; + double value = measurement.getValue(); + if (Double.isNaN(value)) continue; + total += value; + defined++; + } + if (classes == 0) return lookup(measurements, aggregateName); + if (countUndefined) return total / classes; + return defined == 0 ? 0.0 : total / defined; + } + + /** + * Regression counterpart of {@link #getScore}, always oriented so that + * larger is better: error metrics are negated, so a search can rank + * candidates with a plain {@code >} whichever metric is selected. + * + *

Measurements are looked up by name rather than by position, and the + * same metric is used for candidates and for the incumbent. Comparing, say, + * a candidate's adjusted R² against the incumbent's plain R² would make + * swaps depend on the number of attributes rather than on model quality. + */ + public static double getRegressionScore(Measurement[] measurements, int metricChoice) { + if (measurements == null || metricChoice < 0 || metricChoice >= REGRESSION_MEASUREMENT_NAMES.length) { + return Double.NEGATIVE_INFINITY; + } + double value = lookup(measurements, REGRESSION_MEASUREMENT_NAMES[metricChoice]); + if (Double.isNaN(value)) return Double.NEGATIVE_INFINITY; + return REGRESSION_HIGHER_IS_BETTER[metricChoice] ? value : -value; + } + + private static double lookup(Measurement[] measurements, String name) { + for (Measurement measurement : measurements) { + if (name.equals(measurement.getName())) return measurement.getValue(); + } + return Double.NEGATIVE_INFINITY; + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/Parameters/CategoricalParameter.java b/moa/src/main/java/moa/classifiers/AutoML/Parameters/CategoricalParameter.java new file mode 100644 index 000000000..bae379689 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/Parameters/CategoricalParameter.java @@ -0,0 +1,55 @@ +/* + * CategoricalParameter.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML.Parameters; + +import java.io.Serializable; +import java.util.Random; + +/** + * A categorical hyperparameter: one of {@link #values} is selected by the + * {@link #active} index, which is what the search actually moves. + * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public class CategoricalParameter extends Parameter implements Serializable { + + private static final long serialVersionUID = 1L; + + public String[] values; + public int active; + public Random random; + + public CategoricalParameter(String name, String[] values, int active, Random random) { + this.name = name; + this.values = values; + this.active = active; + this.type = TYPE_CATEGORICAL; + this.random = random; + } + + @Override + public void changeParameter() { + this.active = this.random.nextInt(this.values.length); + } + + @Override + public String toString() { + return "CategoricalParameter [name=" + this.name + + ", active=" + this.values[this.active] + "]"; + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/Parameters/DoubleParameter.java b/moa/src/main/java/moa/classifiers/AutoML/Parameters/DoubleParameter.java new file mode 100644 index 000000000..14d7756d1 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/Parameters/DoubleParameter.java @@ -0,0 +1,56 @@ +/* + * DoubleParameter.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML.Parameters; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Random; + +/** + * A real-valued hyperparameter, uniformly drawn from an inclusive range. + * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public class DoubleParameter extends Parameter implements Serializable { + + private static final long serialVersionUID = 1L; + + public double value; + public double[] range; + public Random random; + + public DoubleParameter(String name, double value, double[] range, Random random) { + this.name = name; + this.value = value; + this.range = range; + this.type = TYPE_DOUBLE; + this.random = random; + } + + @Override + public void changeParameter() { + this.value = this.range[0] + ((this.range[1] - this.range[0]) * this.random.nextDouble()); + this.value = Math.min(this.range[1], Math.max(this.range[0], this.value)); + } + + @Override + public String toString() { + return "DoubleParameter [name=" + this.name + ", value=" + this.value + + ", range=" + Arrays.toString(this.range) + "]"; + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/Parameters/IntParameter.java b/moa/src/main/java/moa/classifiers/AutoML/Parameters/IntParameter.java new file mode 100644 index 000000000..00299815b --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/Parameters/IntParameter.java @@ -0,0 +1,56 @@ +/* + * IntParameter.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML.Parameters; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Random; + +/** + * An integer hyperparameter, uniformly drawn from an inclusive range. + * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public class IntParameter extends Parameter implements Serializable { + + private static final long serialVersionUID = 1L; + + public int value; + public int[] range; + public Random random; + + public IntParameter(String name, int value, int[] range, Random random) { + this.name = name; + this.value = value; + this.range = range; + this.type = TYPE_INT; + this.random = random; + } + + @Override + public void changeParameter() { + this.value = this.random.nextInt((this.range[1] - this.range[0] + 1)) + this.range[0]; + this.value = Math.min(this.range[1], Math.max(this.range[0], this.value)); + } + + @Override + public String toString() { + return "IntParameter [name=" + this.name + ", value=" + this.value + + ", range=" + Arrays.toString(this.range) + "]"; + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/Parameters/Parameter.java b/moa/src/main/java/moa/classifiers/AutoML/Parameters/Parameter.java new file mode 100644 index 000000000..cd92a3880 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/Parameters/Parameter.java @@ -0,0 +1,54 @@ +/* + * Parameter.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML.Parameters; + +import java.io.Serializable; + +/** + * One hyperparameter carrying a concrete value for one candidate. This is the + * mutable half of a hyperparameter: its immutable description, as read from the + * search space JSON, lives in {@link moa.classifiers.AutoML.space.ParameterSpec}. + * + *

{@link #name} is the option path of the hyperparameter, e.g. + * {@code "gracePeriod"} or {@code "treeLearner/gracePeriod"}, matching the + * {@code "parameter"} entry of the search space JSON. + * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public abstract class Parameter implements Serializable { + + private static final long serialVersionUID = 1L; + + /** Type tag of {@link IntParameter}. */ + public static final int TYPE_INT = 0; + + /** Type tag of {@link DoubleParameter}. */ + public static final int TYPE_DOUBLE = 1; + + /** Type tag of {@link CategoricalParameter}. */ + public static final int TYPE_CATEGORICAL = 2; + + /** One of {@link #TYPE_INT}, {@link #TYPE_DOUBLE}, {@link #TYPE_CATEGORICAL}. */ + public int type; + + /** Option path of this hyperparameter. */ + public String name; + + /** Draws a new value for this hyperparameter from its own range. */ + public abstract void changeParameter(); +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/RandomSearchClassifier.java b/moa/src/main/java/moa/classifiers/AutoML/RandomSearchClassifier.java new file mode 100644 index 000000000..3447cda93 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/RandomSearchClassifier.java @@ -0,0 +1,631 @@ +/* + * RandomSearchClassifier.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML; + +import com.github.javacliparser.FileOption; +import com.github.javacliparser.FlagOption; +import com.github.javacliparser.IntOption; +import com.github.javacliparser.MultiChoiceOption; +import com.yahoo.labs.samoa.instances.Instance; +import moa.capabilities.CapabilitiesHandler; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.AutoML.Parameters.CategoricalParameter; +import moa.classifiers.AutoML.Parameters.DoubleParameter; +import moa.classifiers.AutoML.Parameters.IntParameter; +import moa.classifiers.AutoML.Parameters.Parameter; +import moa.classifiers.AutoML.space.ConfigurationSpace; +import moa.classifiers.AutoML.space.LearnerConfigurator; +import moa.classifiers.AutoML.space.ParameterSpec; +import moa.classifiers.Classifier; +import moa.classifiers.core.driftdetection.ChangeDetector; +import moa.classifiers.MultiClassClassifier; +import moa.core.InstanceExample; +import moa.core.Measurement; +import moa.core.SizeOf; +import moa.options.ClassOption; +import moa.evaluation.BasicClassificationPerformanceEvaluator; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Random; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Random search over a streaming hyperparameter space: a pool of candidates + * carries randomly drawn configurations, and whenever a candidate beats the + * incumbent over an evaluation window it takes its place. + * + *

Candidates are configured through {@link LearnerConfigurator}, i.e. by + * writing MOA {@link com.github.javacliparser.Option}s, so any MOA classifier + * can be tuned as shipped. + * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public class RandomSearchClassifier extends AbstractClassifier implements MultiClassClassifier, + CapabilitiesHandler, Serializable, HPOMethod { + + private static final long serialVersionUID = 1L; + + public FileOption configurationFileOption = new FileOption("configurationFile", 'f', + "Search space in JSON format.", null, ".json", false); + + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + "Number of instances between candidate evaluations.", 1000, 1, Integer.MAX_VALUE); + + public FlagOption randomInitialParametersOption = new FlagOption("randomInitialParameters", 'R', + "Draw the initial configuration at random instead of using the values declared in the search space."); + + public IntOption numberOfCandidatesOption = new IntOption("numberOfCandidates", 'n', + "Number of candidate models in the pool.", 10, 1, Integer.MAX_VALUE); + + public MultiChoiceOption metricOption = new MultiChoiceOption("metric", 'm', + "Metric to optimize the model.", MetricUtils.METRIC_NAMES, MetricUtils.METRIC_DESCRIPTIONS, 4); + + public FlagOption resetLearningOption = new FlagOption("resetLearning", 'L', + "Reset candidate learning instead of copying internal model state from the best classifier."); + + public FlagOption driftDetectionOption = new FlagOption("driftDetection", 'd', + "Enable drift detection on the incumbent's prediction error."); + + public ClassOption driftDetectorOption = new ClassOption("driftDetector", 'D', + "Change detector to use when drift detection is enabled; on a detected" + + " drift the whole search is reinitialized from the search space.", + ChangeDetector.class, "ADWINChangeDetector"); + + public IntOption numberOfJobsOption = new IntOption("numberOfJobs", 'j', + "Total number of concurrent jobs used for processing (-1 = as much as possible, 0 = do not use multithreading)", + 1, -1, Integer.MAX_VALUE); + + protected static final int SINGLE_THREAD = 0; + + public long instanceCount; + + public long statesEvaluated; + + public Classifier classifier; + + protected Classifier[] candidates; + + ArrayList classifierParameters; + + ArrayList> candidatesParameters; + + int numericalParameters; + + protected BasicClassificationPerformanceEvaluator evaluatorClassifier; + + protected BasicClassificationPerformanceEvaluator[] evaluatorCandidates; + + int evaluationInstances = 0; + + /** Detector on the incumbent's error; {@code null} unless drift detection is on. */ + protected ChangeDetector driftDetector; + + /** Cumulative number of drifts signalled over the run; not reset by a restart. */ + protected long driftsDetected; + + protected boolean[] optimizeMask; + + /** Search space and the boundary through which every learner is configured. */ + protected LearnerConfigurator configurator; + + /** + * Parameter list most recently written onto each candidate. Needed when a + * candidate has to be rebuilt rather than reconfigured in place. + */ + protected ArrayList[] appliedParameters; + + /** + * Candidates actually trained and evaluated, i.e. the active prefix of the + * pool. The pool itself is always allocated at {@code numberOfCandidatesOption} + * so the budget can move without reallocating anything. + */ + protected int activeCandidates; + + /** Budget requested by an external controller, applied at the next window boundary. */ + protected int pendingActiveCandidates = -1; + + /** + * Pool training the incumbent and the candidates concurrently; {@code null} + * when running single-threaded. Transient because an executor cannot be + * serialized, so it is (re)created on demand by {@link #initExecutor()}. + */ + protected transient ExecutorService executor; + + @Override + public void setOptimizableParameters(boolean[] mask) { this.optimizeMask = mask; } + + @Override + public int getActiveCandidates() { return this.activeCandidates; } + + @Override + public int setActiveCandidates(int n) { + this.pendingActiveCandidates = Math.max(1, Math.min(n, this.numberOfCandidatesOption.getValue())); + return this.pendingActiveCandidates; + } + + /** + * Pin every frozen hyperparameter (optimizeMask[k] == false) of every + * candidate to the incumbent value, and re-apply it to the candidate model + * so the frozen value is actually used. + */ + private void applyFreezeMask() { + if (this.optimizeMask == null) return; + for (int i = 0; i < this.activeCandidates; i++) { + ArrayList params = this.candidatesParameters.get(i); + HPOMethod.freezeToIncumbent(this.optimizeMask, params, this.classifierParameters); + for (int k = 0; k < params.size() && k < this.optimizeMask.length; k++) { + if (this.optimizeMask[k]) continue; + this.configurator.applyLive(this.candidates[i], params.get(k)); + } + } + } + + @Override + public boolean isRandomizable() { + return true; + } + + @Override + public double[] getVotesForInstance(Instance inst) { + //Main classifiers votes + return this.classifier.getVotesForInstance(inst); + } + + /** + * Creates the training pool on first use, and again after deserialization. + * A pool larger than the batch trained per instance would leave threads idle, + * so the requested job count is capped at the incumbent plus the whole pool. + */ + protected void initExecutor() { + if (this.executor != null) return; + int numberOfJobs = this.numberOfJobsOption.getValue() == -1 + ? Runtime.getRuntime().availableProcessors() + : this.numberOfJobsOption.getValue(); + numberOfJobs = Math.min(numberOfJobs, this.numberOfCandidatesOption.getValue() + 1); + // SINGLE_THREAD and requesting a single thread are equivalent: training + // then happens in-place and this.executor stays null. + if (numberOfJobs != SINGLE_THREAD && numberOfJobs != 1) { + // Daemon threads: a live pool must not keep the JVM alive after the + // task that ran this learner has finished. + this.executor = Executors.newFixedThreadPool(numberOfJobs, runnable -> { + Thread thread = new Thread(runnable); + thread.setDaemon(true); + return thread; + }); + } + } + + @Override + public void cleanThreads() { + if (this.executor != null) { + this.executor.shutdownNow(); + this.executor = null; + } + } + + @Override + @SuppressWarnings("unchecked") + public void resetLearningImpl() { + cleanThreads(); // shut down any pool from a previous reset before creating a new one + driftDetector = driftDetectionOption.isSet() + ? ((ChangeDetector) getPreparedClassOption(driftDetectorOption)).copy() + : null; + this.classifierParameters = new ArrayList(); + this.candidatesParameters = new ArrayList<>(); + for (int i = 0; i < this.numberOfCandidatesOption.getValue(); i++) { + this.candidatesParameters.add(new ArrayList()); + } + + + this.evaluatorClassifier = new BasicClassificationPerformanceEvaluator(); + MetricUtils.configure(this.evaluatorClassifier); + this.evaluatorCandidates = new BasicClassificationPerformanceEvaluator[this.numberOfCandidatesOption.getValue()]; + for (int i = 0; i < this.numberOfCandidatesOption.getValue(); i++) { + this.evaluatorCandidates[i] = new BasicClassificationPerformanceEvaluator(); + MetricUtils.configure(this.evaluatorCandidates[i]); + } + + this.appliedParameters = new ArrayList[this.numberOfCandidatesOption.getValue()]; + + this.setConfigurations(); + this.instanceCount = 0; + this.statesEvaluated = 0; + this.activeCandidates = this.numberOfCandidatesOption.getValue(); + this.pendingActiveCandidates = -1; + } + + /** + * Reads the search space, creates the incumbent and the candidate pool, and + * seeds the per-parameter random generators. + */ + public void setConfigurations() { + try { + this.numericalParameters = 0; + + ConfigurationSpace space = ConfigurationSpace.fromFile(this.configurationFileOption.getValue()); + this.configurator = new LearnerConfigurator(space); + this.configurator.validate(); + + this.classifier = this.configurator.newLearner(this.classifierRandom.nextInt()); + + this.candidates = new Classifier[this.numberOfCandidatesOption.getValue()]; + for (int i = 0; i < this.numberOfCandidatesOption.getValue(); i++) { + // Built one by one rather than copied from a single base, so that + // each candidate carries its own random seed. + this.candidates[i] = this.configurator.newLearner(this.classifierRandom.nextInt()); + } + + for (ParameterSpec spec : space.parameters) { + switch (spec.type) { + case ParameterSpec.TYPE_CATEGORICAL: { + int active; + if (!this.randomInitialParametersOption.isSet()) { + active = spec.active; + } else { + active = this.classifierRandom.nextInt(spec.values.length); + } + this.classifierParameters.add(new CategoricalParameter(spec.name, spec.values, + active, new Random(this.classifierRandom.nextLong()))); + for (int i = 0; i < this.numberOfCandidatesOption.getValue(); i++) { + this.candidatesParameters.get(i).add(new CategoricalParameter(spec.name, + spec.values, active, new Random(this.classifierRandom.nextLong()))); + } + break; + } + case ParameterSpec.TYPE_INT: { + int[] range = new int[]{(int) spec.range[0], (int) spec.range[1]}; + int value; + if (!this.randomInitialParametersOption.isSet()) { + value = (int) spec.value; + } else { + value = this.classifierRandom.nextInt(range[1] + 1 - range[0]) + range[0]; + } + this.classifierParameters.add(new IntParameter(spec.name, value, range, + new Random(this.classifierRandom.nextLong()))); + for (int i = 0; i < this.numberOfCandidatesOption.getValue(); i++) { + this.candidatesParameters.get(i).add(new IntParameter(spec.name, value, range, + new Random(this.classifierRandom.nextLong()))); + } + this.numericalParameters++; + break; + } + case ParameterSpec.TYPE_DOUBLE: { + double[] range = new double[]{spec.range[0], spec.range[1]}; + double value; + if (!this.randomInitialParametersOption.isSet()) { + value = spec.value; + } else { + value = range[0] + (range[1] - range[0]) * this.classifierRandom.nextDouble(); + } + this.classifierParameters.add(new DoubleParameter(spec.name, value, range, + new Random(this.classifierRandom.nextLong()))); + for (int i = 0; i < this.numberOfCandidatesOption.getValue(); i++) { + this.candidatesParameters.get(i).add(new DoubleParameter(spec.name, value, range, + new Random(this.classifierRandom.nextLong()))); + } + this.numericalParameters++; + break; + } + default: + break; + } + } + + // Put the incumbent on its declared starting configuration. + this.classifier = this.configurator.instantiate(this.classifierParameters, this.classifierRandom.nextInt()); + + } catch (Exception e) { + throw new IllegalStateException("Could not set up " + getClass().getSimpleName() + + " from \"" + this.configurationFileOption.getValue() + "\": " + e.getMessage(), e); + } + } + + public void deepCopyList(int bestPerforming) { + //Swap parameters from candidates to main classifier + for (int i = 0; i < this.classifierParameters.size(); i++) { + Parameter parameterS = this.candidatesParameters.get(bestPerforming).get(i); + Parameter parameterC = this.classifierParameters.get(i); + switch (parameterS.type) { + case Parameter.TYPE_INT: + ((IntParameter) parameterC).value = ((IntParameter) parameterS).value; + ((IntParameter) parameterC).range = ((IntParameter) parameterS).range; + break; + case Parameter.TYPE_DOUBLE: + ((DoubleParameter) parameterC).value = ((DoubleParameter) parameterS).value; + ((DoubleParameter) parameterC).range = ((DoubleParameter) parameterS).range; + break; + case Parameter.TYPE_CATEGORICAL: + ((CategoricalParameter) parameterC).values = ((CategoricalParameter) parameterS).values; + ((CategoricalParameter) parameterC).active = ((CategoricalParameter) parameterS).active; + break; + } + } + } + + public void checkParameterChange() { + try { + double maxAccuracy = 0; + double classifierAcc = 0; + int bestPerforming = 0; + + if (this.statesEvaluated != 0) { + maxAccuracy = MetricUtils.getScore(this.evaluatorCandidates[0].getPerformanceMeasurements(), + this.metricOption.getChosenIndex()); + for (int i = 1; i < this.activeCandidates; i++) { + double currentAcc = MetricUtils.getScore(this.evaluatorCandidates[i].getPerformanceMeasurements(), + this.metricOption.getChosenIndex()); + if (currentAcc > maxAccuracy) { + bestPerforming = i; + maxAccuracy = currentAcc; + } + } + + classifierAcc = MetricUtils.getScore(this.evaluatorClassifier.getPerformanceMeasurements(), + this.metricOption.getChosenIndex()); + } + + // Swapping classifiers if candidates acc is greater than main classifier + if (maxAccuracy > classifierAcc) { + this.swapClassifiers(bestPerforming); + } + + // Apply an externally requested budget only here: the window is + // scored with the budget it ran under, and the new one takes effect + // from the candidates regenerated below. + if (this.pendingActiveCandidates > 0) { + this.activeCandidates = this.pendingActiveCandidates; + this.pendingActiveCandidates = -1; + } + + boolean live = this.configurator.canApplyLive(); + + if (live) { + for (int i = 0; i < this.activeCandidates; i++) { + if (resetLearningOption.isSet()) { + this.candidates[i].resetLearning(); + } else { + this.candidates[i] = this.classifier.copy(); + LearnerConfigurator.reseedCopy(this.candidates[i], this.classifierRandom.nextInt()); + } + } + } + + // Parameter change of candidates. Candidate i is drawn from, and + // configured out of, its own parameter list, so its model and its + // bookkeeping agree and deepCopyList records the winner's real + // configuration. + for (int i = 0; i < this.activeCandidates; i++) { + ArrayList listp = this.candidatesParameters.get(i); + for (Parameter parameter : listp) { + this.changeStateParameter(parameter, i); + } + this.appliedParameters[i] = listp; + } + + if (!live) { + // At least one hyperparameter is only read when the learner is + // built, so the candidate has to be rebuilt around it. + for (int i = 0; i < this.activeCandidates; i++) { + this.candidates[i] = this.configurator.instantiate(this.appliedParameters[i], this.classifierRandom.nextInt()); + } + } + + // Freeze the hyperparameters outside the optimized subset to the incumbent. + this.applyFreezeMask(); + + //reset evaluators + this.evaluatorClassifier.reset(); + + for (int i = 0; i < this.activeCandidates; i++) { + this.evaluatorCandidates[i].reset(); + } + + } catch (Exception e) { + throw new IllegalStateException("Failed to update candidates in " + + getClass().getSimpleName() + ": " + e.getMessage(), e); + } + } + + public void swapClassifiers(int bestPerforming) { + this.classifier = this.candidates[bestPerforming].copy(); + this.deepCopyList(bestPerforming); + } + + @Override + public void changeStateParameter(Parameter parameter, int index) { + //Draw a new value and push it into the candidate + parameter.changeParameter(); + if (this.configurator.canApplyLive()) { + this.configurator.applyLive(this.candidates[index], parameter); + } + } + + @Override + public void trainOnInstanceImpl(Instance inst) { + double[] incumbentVotes = driftDetector != null ? getVotesForInstance(inst) : null; + initExecutor(); + //check for change in parameters + this.evaluationInstances++; + + if ((this.statesEvaluated == 0) || this.evaluationInstances >= this.gracePeriodOption.getValue()) { + this.checkParameterChange(); + this.statesEvaluated++; + this.evaluationInstances = 0; + } + + //update evaluators (update metrics) + InstanceExample example = new InstanceExample(inst); + this.evaluatorClassifier.addResult(example, this.classifier.getVotesForInstance(inst)); + for (int i = 0; i < this.activeCandidates; i++) { + this.evaluatorCandidates[i].addResult(example, this.candidates[i].getVotesForInstance(inst)); + } + + //train classifiers + if (this.executor == null) { + this.classifier.trainOnInstance(inst); + for (int i = 0; i < this.activeCandidates; i++) { + this.candidates[i].trainOnInstance(inst); + } + } else { + // The incumbent is an independent model, so it joins the batch as one + // more task. Every model owns its state, hence no synchronization here. + Collection trainers = new ArrayList(); + trainers.add(new TrainingRunnable(this.classifier, inst)); + for (int i = 0; i < this.activeCandidates; i++) { + trainers.add(new TrainingRunnable(this.candidates[i], inst)); + } + try { + this.executor.invokeAll(trainers); + } catch (InterruptedException ex) { + throw new RuntimeException("Could not call invokeAll() on training threads."); + } + } + + this.instanceCount++; + + if (incumbentVotes != null) checkDrift(inst, incumbentVotes); + } + + @Override + public long measureByteSize() { + long candidateSize = 0; + for (int i = 0; i < this.activeCandidates; i++) { + candidateSize += this.candidates[i].measureByteSize(); + } + return SizeOf.sizeOf(this) + this.classifier.measureByteSize() + candidateSize; + } + + @Override + protected Measurement[] getModelMeasurementsImpl() { + + for (Classifier c : candidates) { + c.getModelMeasurements(); + } + + ArrayList parameters = new ArrayList<>(); + parameters.add(new Measurement("driftsDetected", driftsDetected)); + + for (Measurement m : this.classifier.getModelMeasurements()) { + parameters.add(m); + } + for (Parameter p : this.classifierParameters) { + switch (p.type) { + case Parameter.TYPE_INT: + parameters.add(new Measurement(p.name, ((IntParameter) p).value)); + break; + case Parameter.TYPE_DOUBLE: + parameters.add(new Measurement(p.name, ((DoubleParameter) p).value)); + break; + case Parameter.TYPE_CATEGORICAL: + parameters.add(new Measurement(p.name, ((CategoricalParameter) p).active)); + break; + } + } + Measurement[] measurements = new Measurement[parameters.size()]; + measurements = parameters.toArray(measurements); + return measurements; + } + + @Override + public void getModelDescription(StringBuilder out, int indent) { + } + + @Override + public ArrayList getReferenceParameters() { return this.classifierParameters; } + + @Override + public ArrayList> getCandidateParameters() { return this.candidatesParameters; } + + @Override + public double getCandidateScore(int i) { + return MetricUtils.getScore(this.evaluatorCandidates[i].getPerformanceMeasurements(), + this.metricOption.getChosenIndex()); + } + + @Override + public double getClassifierScore() { + return MetricUtils.getScore(this.evaluatorClassifier.getPerformanceMeasurements(), + this.metricOption.getChosenIndex()); + } + + @Override + public int getNumberOfCandidates() { return this.numberOfCandidatesOption.getValue(); } + + @Override + public long getStatesEvaluatedCount() { return this.statesEvaluated; } + + @Override + public int getEvaluationInstancesCount() { return this.evaluationInstances; } + + @Override + public int getGracePeriod() { return this.gracePeriodOption.getValue(); } + + @Override + public Classifier getMainClassifier() { return this.classifier; } + + @Override + public String getConfigurationFile() { return this.configurationFileOption.getValue(); } + + private int argmax(double[] arr) { + int best = 0; + for (int i = 1; i < arr.length; i++) + if (arr[i] > arr[best]) best = i; + return best; + } + + /** + * Feeds the incumbent's prediction error to the change detector and, when a + * drift is signalled, reinitializes the whole search from the search space. + * The signal is the 0/1 misclassification indicator. + */ + private void checkDrift(Instance inst, double[] incumbentVotes) { + driftDetector.input(argmax(incumbentVotes) != (int) inst.classValue() ? 1.0 : 0.0); + if (driftDetector.getChange()) { + driftsDetected++; + resetLearningImpl(); + } + } + + /** + * Inner class to assist with the multi-thread execution. + */ + protected class TrainingRunnable implements Runnable, Callable { + final private Classifier learner; + final private Instance instance; + + public TrainingRunnable(Classifier learner, Instance instance) { + this.learner = learner; + this.instance = instance; + } + + @Override + public void run() { + this.learner.trainOnInstance(this.instance); + } + + @Override + public Integer call() { + run(); + return 0; + } + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/RandomSearchRegressor.java b/moa/src/main/java/moa/classifiers/AutoML/RandomSearchRegressor.java new file mode 100644 index 000000000..d17c14381 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/RandomSearchRegressor.java @@ -0,0 +1,560 @@ +/* + * RandomSearchRegressor.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML; + +import com.github.javacliparser.FileOption; +import com.github.javacliparser.FlagOption; +import com.github.javacliparser.IntOption; +import com.github.javacliparser.MultiChoiceOption; +import com.yahoo.labs.samoa.instances.Instance; +import moa.capabilities.CapabilitiesHandler; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.AutoML.Parameters.CategoricalParameter; +import moa.classifiers.AutoML.Parameters.DoubleParameter; +import moa.classifiers.AutoML.Parameters.IntParameter; +import moa.classifiers.AutoML.Parameters.Parameter; +import moa.classifiers.AutoML.space.ConfigurationSpace; +import moa.classifiers.AutoML.space.LearnerConfigurator; +import moa.classifiers.AutoML.space.ParameterSpec; +import moa.classifiers.Classifier; +import moa.classifiers.core.driftdetection.ChangeDetector; +import moa.classifiers.Regressor; +import moa.core.InstanceExample; +import moa.core.Measurement; +import moa.core.SizeOf; +import moa.options.ClassOption; +import moa.evaluation.BasicRegressionPerformanceEvaluator; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Random; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Regression counterpart of {@link RandomSearchClassifier}: the same + * incumbent-plus-candidate-pool search, scored with a regression metric. + * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public class RandomSearchRegressor extends AbstractClassifier implements Regressor, + CapabilitiesHandler, Serializable, HPOMethod { + + private static final long serialVersionUID = 1L; + + public FileOption configurationFileOption = new FileOption("configurationFile", 'f', + "Search space in JSON format.", null, ".json", false); + + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + "Number of instances between candidate evaluations.", 1000, 1, Integer.MAX_VALUE); + + public FlagOption randomInitialParametersOption = new FlagOption("randomInitialParameters", 'R', + "Draw the initial configuration at random instead of using the values declared in the search space."); + + public IntOption numberOfCandidatesOption = new IntOption("numberOfCandidates", 'n', + "Number of candidate models in the pool.", 10, 1, Integer.MAX_VALUE); + + public MultiChoiceOption metricOption = new MultiChoiceOption("metric", 'm', + "Metric to optimize the model.", MetricUtils.REGRESSION_METRIC_NAMES, + MetricUtils.REGRESSION_METRIC_DESCRIPTIONS, 0); + + public FlagOption resetLearningOption = new FlagOption("resetLearning", 'L', + "Reset candidate learning instead of copying internal model state from the best classifier."); + + public FlagOption driftDetectionOption = new FlagOption("driftDetection", 'd', + "Enable drift detection on the incumbent's prediction error."); + + public ClassOption driftDetectorOption = new ClassOption("driftDetector", 'D', + "Change detector to use when drift detection is enabled; on a detected" + + " drift the whole search is reinitialized from the search space.", + ChangeDetector.class, "ADWINChangeDetector"); + + public IntOption numberOfJobsOption = new IntOption("numberOfJobs", 'j', + "Total number of concurrent jobs used for processing (-1 = as much as possible, 0 = do not use multithreading)", + 1, -1, Integer.MAX_VALUE); + + protected static final int SINGLE_THREAD = 0; + + public long instanceCount; + + public long statesEvaluated; + + public Classifier classifier; + + protected Classifier[] candidates; + + ArrayList classifierParameters; + + ArrayList> candidatesParameters; + + int numericalParameters; + + protected BasicRegressionPerformanceEvaluator evaluatorClassifier; + + protected BasicRegressionPerformanceEvaluator[] evaluatorCandidates; + + int evaluationInstances = 0; + + /** Detector on the incumbent's error; {@code null} unless drift detection is on. */ + protected ChangeDetector driftDetector; + + /** Cumulative number of drifts signalled over the run; not reset by a restart. */ + protected long driftsDetected; + + protected boolean[] optimizeMask; + + protected LearnerConfigurator configurator; + + protected ArrayList[] appliedParameters; + + protected int activeCandidates; + + protected int pendingActiveCandidates = -1; + + /** + * Pool training the incumbent and the candidates concurrently; {@code null} + * when running single-threaded. Transient because an executor cannot be + * serialized, so it is (re)created on demand by {@link #initExecutor()}. + */ + protected transient ExecutorService executor; + + @Override + public void setOptimizableParameters(boolean[] mask) { this.optimizeMask = mask; } + + @Override + public int getActiveCandidates() { return this.activeCandidates; } + + @Override + public int setActiveCandidates(int n) { + this.pendingActiveCandidates = Math.max(1, Math.min(n, this.numberOfCandidatesOption.getValue())); + return this.pendingActiveCandidates; + } + + private void applyFreezeMask() { + if (this.optimizeMask == null) return; + for (int i = 0; i < this.activeCandidates; i++) { + ArrayList params = this.candidatesParameters.get(i); + HPOMethod.freezeToIncumbent(this.optimizeMask, params, this.classifierParameters); + for (int k = 0; k < params.size() && k < this.optimizeMask.length; k++) { + if (this.optimizeMask[k]) continue; + this.configurator.applyLive(this.candidates[i], params.get(k)); + } + } + } + + @Override + public boolean isRandomizable() { + return true; + } + + @Override + public double[] getVotesForInstance(Instance inst) { + return this.classifier.getVotesForInstance(inst); + } + + /** + * Creates the training pool on first use, and again after deserialization. + * A pool larger than the batch trained per instance would leave threads idle, + * so the requested job count is capped at the incumbent plus the whole pool. + */ + protected void initExecutor() { + if (this.executor != null) return; + int numberOfJobs = this.numberOfJobsOption.getValue() == -1 + ? Runtime.getRuntime().availableProcessors() + : this.numberOfJobsOption.getValue(); + numberOfJobs = Math.min(numberOfJobs, this.numberOfCandidatesOption.getValue() + 1); + // SINGLE_THREAD and requesting a single thread are equivalent: training + // then happens in-place and this.executor stays null. + if (numberOfJobs != SINGLE_THREAD && numberOfJobs != 1) { + // Daemon threads: a live pool must not keep the JVM alive after the + // task that ran this learner has finished. + this.executor = Executors.newFixedThreadPool(numberOfJobs, runnable -> { + Thread thread = new Thread(runnable); + thread.setDaemon(true); + return thread; + }); + } + } + + @Override + public void cleanThreads() { + if (this.executor != null) { + this.executor.shutdownNow(); + this.executor = null; + } + } + + @Override + @SuppressWarnings("unchecked") + public void resetLearningImpl() { + cleanThreads(); // shut down any pool from a previous reset before creating a new one + driftDetector = driftDetectionOption.isSet() + ? ((ChangeDetector) getPreparedClassOption(driftDetectorOption)).copy() + : null; + this.classifierParameters = new ArrayList(); + this.candidatesParameters = new ArrayList<>(); + for (int i = 0; i < this.numberOfCandidatesOption.getValue(); i++) { + this.candidatesParameters.add(new ArrayList()); + } + + + this.evaluatorClassifier = new BasicRegressionPerformanceEvaluator(); + this.evaluatorCandidates = new BasicRegressionPerformanceEvaluator[this.numberOfCandidatesOption.getValue()]; + for (int i = 0; i < this.numberOfCandidatesOption.getValue(); i++) { + this.evaluatorCandidates[i] = new BasicRegressionPerformanceEvaluator(); + } + + this.appliedParameters = new ArrayList[this.numberOfCandidatesOption.getValue()]; + + this.setConfigurations(); + this.instanceCount = 0; + this.statesEvaluated = 0; + this.activeCandidates = this.numberOfCandidatesOption.getValue(); + this.pendingActiveCandidates = -1; + } + + public void setConfigurations() { + try { + this.numericalParameters = 0; + + ConfigurationSpace space = ConfigurationSpace.fromFile(this.configurationFileOption.getValue()); + this.configurator = new LearnerConfigurator(space); + this.configurator.validate(); + + this.classifier = this.configurator.newLearner(this.classifierRandom.nextInt()); + + this.candidates = new Classifier[this.numberOfCandidatesOption.getValue()]; + for (int i = 0; i < this.numberOfCandidatesOption.getValue(); i++) { + // Built one by one rather than copied from a single base, so that + // each candidate carries its own random seed. + this.candidates[i] = this.configurator.newLearner(this.classifierRandom.nextInt()); + } + + for (ParameterSpec spec : space.parameters) { + switch (spec.type) { + case ParameterSpec.TYPE_CATEGORICAL: { + int active = this.randomInitialParametersOption.isSet() + ? this.classifierRandom.nextInt(spec.values.length) + : spec.active; + this.classifierParameters.add(new CategoricalParameter(spec.name, spec.values, + active, new Random(this.classifierRandom.nextLong()))); + for (int i = 0; i < this.numberOfCandidatesOption.getValue(); i++) { + this.candidatesParameters.get(i).add(new CategoricalParameter(spec.name, + spec.values, active, new Random(this.classifierRandom.nextLong()))); + } + break; + } + case ParameterSpec.TYPE_INT: { + int[] range = new int[]{(int) spec.range[0], (int) spec.range[1]}; + int value = this.randomInitialParametersOption.isSet() + ? this.classifierRandom.nextInt(range[1] + 1 - range[0]) + range[0] + : (int) spec.value; + this.classifierParameters.add(new IntParameter(spec.name, value, range, + new Random(this.classifierRandom.nextLong()))); + for (int i = 0; i < this.numberOfCandidatesOption.getValue(); i++) { + this.candidatesParameters.get(i).add(new IntParameter(spec.name, value, range, + new Random(this.classifierRandom.nextLong()))); + } + this.numericalParameters++; + break; + } + case ParameterSpec.TYPE_DOUBLE: { + double[] range = new double[]{spec.range[0], spec.range[1]}; + double value = this.randomInitialParametersOption.isSet() + ? range[0] + (range[1] - range[0]) * this.classifierRandom.nextDouble() + : spec.value; + this.classifierParameters.add(new DoubleParameter(spec.name, value, range, + new Random(this.classifierRandom.nextLong()))); + for (int i = 0; i < this.numberOfCandidatesOption.getValue(); i++) { + this.candidatesParameters.get(i).add(new DoubleParameter(spec.name, value, range, + new Random(this.classifierRandom.nextLong()))); + } + this.numericalParameters++; + break; + } + default: + break; + } + } + + this.classifier = this.configurator.instantiate(this.classifierParameters, this.classifierRandom.nextInt()); + + } catch (Exception e) { + throw new IllegalStateException("Could not set up " + getClass().getSimpleName() + + " from \"" + this.configurationFileOption.getValue() + "\": " + e.getMessage(), e); + } + } + + public void deepCopyList(int bestPerforming) { + for (int i = 0; i < this.classifierParameters.size(); i++) { + Parameter parameterS = this.candidatesParameters.get(bestPerforming).get(i); + Parameter parameterC = this.classifierParameters.get(i); + switch (parameterS.type) { + case Parameter.TYPE_INT: + ((IntParameter) parameterC).value = ((IntParameter) parameterS).value; + ((IntParameter) parameterC).range = ((IntParameter) parameterS).range; + break; + case Parameter.TYPE_DOUBLE: + ((DoubleParameter) parameterC).value = ((DoubleParameter) parameterS).value; + ((DoubleParameter) parameterC).range = ((DoubleParameter) parameterS).range; + break; + case Parameter.TYPE_CATEGORICAL: + ((CategoricalParameter) parameterC).values = ((CategoricalParameter) parameterS).values; + ((CategoricalParameter) parameterC).active = ((CategoricalParameter) parameterS).active; + break; + } + } + } + + public void checkParameterChange() { + try { + double best = -Double.MAX_VALUE; + double incumbent = -Double.MAX_VALUE; + int bestPerforming = 0; + + if (this.statesEvaluated != 0) { + best = getCandidateScore(0); + for (int i = 1; i < this.activeCandidates; i++) { + double current = getCandidateScore(i); + if (current > best) { + bestPerforming = i; + best = current; + } + } + incumbent = getClassifierScore(); + } + + if (!Double.isNaN(best) && best > incumbent) { + this.swapClassifiers(bestPerforming); + } + + if (this.pendingActiveCandidates > 0) { + this.activeCandidates = this.pendingActiveCandidates; + this.pendingActiveCandidates = -1; + } + + boolean live = this.configurator.canApplyLive(); + + if (live) { + for (int i = 0; i < this.activeCandidates; i++) { + if (this.resetLearningOption.isSet()) { + this.candidates[i].resetLearning(); + } else { + this.candidates[i] = this.classifier.copy(); + LearnerConfigurator.reseedCopy(this.candidates[i], this.classifierRandom.nextInt()); + } + } + } + + for (int i = 0; i < this.activeCandidates; i++) { + ArrayList listp = this.candidatesParameters.get(i); + for (Parameter parameter : listp) { + this.changeStateParameter(parameter, i); + } + this.appliedParameters[i] = listp; + } + + if (!live) { + for (int i = 0; i < this.activeCandidates; i++) { + this.candidates[i] = this.configurator.instantiate(this.appliedParameters[i], this.classifierRandom.nextInt()); + } + } + + this.applyFreezeMask(); + + this.evaluatorClassifier.reset(); + for (int i = 0; i < this.activeCandidates; i++) { + this.evaluatorCandidates[i].reset(); + } + + } catch (Exception e) { + throw new IllegalStateException("Failed to update candidates in " + + getClass().getSimpleName() + ": " + e.getMessage(), e); + } + } + + public void swapClassifiers(int bestPerforming) { + this.classifier = this.candidates[bestPerforming].copy(); + this.deepCopyList(bestPerforming); + } + + @Override + public void changeStateParameter(Parameter parameter, int index) { + parameter.changeParameter(); + if (this.configurator.canApplyLive()) { + this.configurator.applyLive(this.candidates[index], parameter); + } + } + + @Override + public void trainOnInstanceImpl(Instance inst) { + double[] incumbentVotes = driftDetector != null ? getVotesForInstance(inst) : null; + initExecutor(); + this.evaluationInstances++; + + if ((this.statesEvaluated == 0) || this.evaluationInstances >= this.gracePeriodOption.getValue()) { + this.checkParameterChange(); + this.statesEvaluated++; + this.evaluationInstances = 0; + } + + InstanceExample example = new InstanceExample(inst); + this.evaluatorClassifier.addResult(example, this.classifier.getVotesForInstance(inst)); + for (int i = 0; i < this.activeCandidates; i++) { + this.evaluatorCandidates[i].addResult(example, this.candidates[i].getVotesForInstance(inst)); + } + + if (this.executor == null) { + this.classifier.trainOnInstance(inst); + for (int i = 0; i < this.activeCandidates; i++) { + this.candidates[i].trainOnInstance(inst); + } + } else { + // The incumbent is an independent model, so it joins the batch as one + // more task. Every model owns its state, hence no synchronization here. + Collection trainers = new ArrayList(); + trainers.add(new TrainingRunnable(this.classifier, inst)); + for (int i = 0; i < this.activeCandidates; i++) { + trainers.add(new TrainingRunnable(this.candidates[i], inst)); + } + try { + this.executor.invokeAll(trainers); + } catch (InterruptedException ex) { + throw new RuntimeException("Could not call invokeAll() on training threads."); + } + } + + this.instanceCount++; + + if (incumbentVotes != null) checkDrift(inst, incumbentVotes); + } + + @Override + public long measureByteSize() { + long candidateSize = 0; + for (int i = 0; i < this.activeCandidates; i++) { + candidateSize += this.candidates[i].measureByteSize(); + } + return SizeOf.sizeOf(this) + this.classifier.measureByteSize() + candidateSize; + } + + @Override + protected Measurement[] getModelMeasurementsImpl() { + ArrayList parameters = new ArrayList<>(); + parameters.add(new Measurement("driftsDetected", driftsDetected)); + for (Measurement m : this.classifier.getModelMeasurements()) { + parameters.add(m); + } + for (Parameter p : this.classifierParameters) { + switch (p.type) { + case Parameter.TYPE_INT: + parameters.add(new Measurement(p.name, ((IntParameter) p).value)); + break; + case Parameter.TYPE_DOUBLE: + parameters.add(new Measurement(p.name, ((DoubleParameter) p).value)); + break; + case Parameter.TYPE_CATEGORICAL: + parameters.add(new Measurement(p.name, ((CategoricalParameter) p).active)); + break; + } + } + Measurement[] measurements = new Measurement[parameters.size()]; + measurements = parameters.toArray(measurements); + return measurements; + } + + @Override + public void getModelDescription(StringBuilder out, int indent) { + } + + @Override + public ArrayList getReferenceParameters() { return this.classifierParameters; } + + @Override + public ArrayList> getCandidateParameters() { return this.candidatesParameters; } + + @Override + public double getCandidateScore(int i) { + return MetricUtils.getRegressionScore(this.evaluatorCandidates[i].getPerformanceMeasurements(), + this.metricOption.getChosenIndex()); + } + + @Override + public double getClassifierScore() { + return MetricUtils.getRegressionScore(this.evaluatorClassifier.getPerformanceMeasurements(), + this.metricOption.getChosenIndex()); + } + + @Override + public int getNumberOfCandidates() { return this.numberOfCandidatesOption.getValue(); } + + @Override + public long getStatesEvaluatedCount() { return this.statesEvaluated; } + + @Override + public int getEvaluationInstancesCount() { return this.evaluationInstances; } + + @Override + public int getGracePeriod() { return this.gracePeriodOption.getValue(); } + + @Override + public Classifier getMainClassifier() { return this.classifier; } + + @Override + public String getConfigurationFile() { return this.configurationFileOption.getValue(); } + + /** + * Feeds the incumbent's prediction error to the change detector and, when a + * drift is signalled, reinitializes the whole search from the search space. + * The signal is the absolute error |y - y_pred|. + */ + private void checkDrift(Instance inst, double[] incumbentVotes) { + double predicted = incumbentVotes.length > 0 ? incumbentVotes[0] : 0.0; + driftDetector.input(Math.abs(predicted - inst.classValue())); + if (driftDetector.getChange()) { + driftsDetected++; + resetLearningImpl(); + } + } + + /** + * Inner class to assist with the multi-thread execution. + */ + protected class TrainingRunnable implements Runnable, Callable { + final private Classifier learner; + final private Instance instance; + + public TrainingRunnable(Classifier learner, Instance instance) { + this.learner = learner; + this.instance = instance; + } + + @Override + public void run() { + this.learner.trainOnInstance(this.instance); + } + + @Override + public Integer call() { + run(); + return 0; + } + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/SSPTClassifier.java b/moa/src/main/java/moa/classifiers/AutoML/SSPTClassifier.java new file mode 100644 index 000000000..aa4dc22ae --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/SSPTClassifier.java @@ -0,0 +1,728 @@ +/* + * SSPTClassifier.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML; + +import com.github.javacliparser.FileOption; +import com.github.javacliparser.FlagOption; +import com.github.javacliparser.FloatOption; +import com.github.javacliparser.IntOption; +import com.github.javacliparser.MultiChoiceOption; +import com.yahoo.labs.samoa.instances.Instance; +import moa.capabilities.CapabilitiesHandler; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.AutoML.Parameters.*; +import moa.classifiers.AutoML.space.ConfigurationSpace; +import moa.classifiers.AutoML.space.LearnerConfigurator; +import moa.classifiers.AutoML.space.ParameterSpec; +import moa.classifiers.Classifier; +import moa.classifiers.core.driftdetection.ChangeDetector; +import moa.classifiers.MultiClassClassifier; +import moa.core.InstanceExample; +import moa.core.Measurement; +import moa.core.SizeOf; +import moa.evaluation.BasicClassificationPerformanceEvaluator; +import moa.options.ClassOption; + +import java.io.Serializable; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Self-adjusting Nelder-Mead simplex search over a streaming hyperparameter + * space: three vertices are trained in parallel, and each evaluation window + * replaces the worst vertex by the best of the reflection, expansion, + * contraction and shrink points. + * + *

Candidates are configured through {@link LearnerConfigurator}, i.e. by + * writing MOA {@link com.github.javacliparser.Option}s, so any MOA classifier + * can be tuned as shipped. + * + *

See details in:
Bruno Veloso, Joao Gama, Benedita Malheiro, Joao + * Vinagre. Hyperparameter self-tuning for data streams. In Information Fusion, + * 76:75-86, DOI: 10.1016/j.inffus.2021.04.011, Elsevier, 2021.

+ * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public class SSPTClassifier extends AbstractClassifier implements MultiClassClassifier, + CapabilitiesHandler, Serializable, HPOMethod { + + public FileOption configurationFileOption = new FileOption("configurationFile", 'f', + "Search space in JSON format.", null, ".json", false); + + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + "Number of instances between simplex updates.", 1000, 1, Integer.MAX_VALUE); + + public FloatOption convergenceSphereOption = new FloatOption("convergenceSphere", 'c', + "Convergence threshold: squared distance between centroids.", 0.001, 0.0, Double.MAX_VALUE); + + public MultiChoiceOption metricOption = new MultiChoiceOption("metric", 'm', + "Metric to optimize the model.", MetricUtils.METRIC_NAMES, MetricUtils.METRIC_DESCRIPTIONS, 4); + + public FlagOption resetModelsOption = new FlagOption("resetModels", 'r', + "Create fresh classifiers for expanded points instead of warm-starting from best."); + + public FlagOption verboseOption = new FlagOption("verbose", 'v', + "Print simplex events to stdout."); + + public FlagOption driftDetectionOption = new FlagOption("driftDetection", 'd', + "Enable drift detection on the incumbent's prediction error."); + + public ClassOption driftDetectorOption = new ClassOption("driftDetector", 'D', + "Change detector to use when drift detection is enabled; on a detected" + + " drift the whole search is reinitialized from the search space.", + ChangeDetector.class, "ADWINChangeDetector"); + + public IntOption numberOfJobsOption = new IntOption("numberOfJobs", 'j', + "Total number of concurrent jobs used for processing (-1 = as much as possible, 0 = do not use multithreading)", + 1, -1, Integer.MAX_VALUE); + + protected static final int SINGLE_THREAD = 0; + + /** Largest batch trained per instance: the 3 vertices plus the 6 expansion points. */ + protected static final int MAX_MODELS_PER_INSTANCE = 9; + + // ========== INNER CLASS ========== + + protected static class SimplexEntry implements Serializable { + Classifier model; + BasicClassificationPerformanceEvaluator evaluator; + ArrayList params; + long instancesSeen; + + SimplexEntry(Classifier model, BasicClassificationPerformanceEvaluator evaluator, + ArrayList params) { + this.model = model; + this.evaluator = evaluator; + this.params = params; + this.instancesSeen = 0; + } + + double getMetric(int metricIndex) { + if (instancesSeen == 0) return Double.NEGATIVE_INFINITY; + return MetricUtils.getScore(evaluator.getPerformanceMeasurements(), metricIndex); + } + + void addResult(InstanceExample example, double[] votes) { + evaluator.addResult(example, votes); + instancesSeen++; + } + } + + // ========== FIELDS ========== + + protected SimplexEntry[] simplex; + protected HashMap expanded; + + protected ArrayList lastCentroid; + + protected ConfigurationSpace space; + + /** Boundary through which every learner is configured. */ + protected LearnerConfigurator configurator; + + protected long instanceCount; + protected int evaluationInstances; + protected boolean converged; + + protected boolean[] optimizeMask; + + /** Detector on the incumbent's error; {@code null} unless drift detection is on. */ + protected ChangeDetector driftDetector; + + /** Cumulative number of drifts signalled over the run; not reset by a restart. */ + protected long driftsDetected; + + /** + * Pool training the simplex vertices and expansion points concurrently; + * {@code null} when running single-threaded. Transient because an executor + * cannot be serialized, so it is (re)created by {@link #initExecutor()}. + */ + protected transient ExecutorService executor; + + @Override + public void setOptimizableParameters(boolean[] mask) { this.optimizeMask = mask; } + + // ========== LIFECYCLE ========== + + @Override + public boolean isRandomizable() { + return true; + } + + /** + * Creates the training pool on first use, and again after deserialization. + * A pool larger than the batch trained per instance would leave threads idle, + * so the requested job count is capped at {@link #MAX_MODELS_PER_INSTANCE}. + */ + protected void initExecutor() { + if (this.executor != null) return; + int numberOfJobs = this.numberOfJobsOption.getValue() == -1 + ? Runtime.getRuntime().availableProcessors() + : this.numberOfJobsOption.getValue(); + numberOfJobs = Math.min(numberOfJobs, MAX_MODELS_PER_INSTANCE); + // SINGLE_THREAD and requesting a single thread are equivalent: training + // then happens in-place and this.executor stays null. + if (numberOfJobs != SINGLE_THREAD && numberOfJobs != 1) { + // Daemon threads: a live pool must not keep the JVM alive after the + // task that ran this learner has finished. + this.executor = Executors.newFixedThreadPool(numberOfJobs, runnable -> { + Thread thread = new Thread(runnable); + thread.setDaemon(true); + return thread; + }); + } + } + + @Override + public void cleanThreads() { + if (this.executor != null) { + this.executor.shutdownNow(); + this.executor = null; + } + } + + @Override + public void resetLearningImpl() { + cleanThreads(); // shut down any pool from a previous reset before creating a new one + converged = false; + instanceCount = 0; + evaluationInstances = 0; + lastCentroid = null; + expanded = null; + + driftDetector = driftDetectionOption.isSet() + ? ((ChangeDetector) getPreparedClassOption(driftDetectorOption)).copy() + : null; + + setConfigurations(); + initializeSimplex(); + + } + + @Override + public void setConfigurations() { + try { + this.space = ConfigurationSpace.fromFile(configurationFileOption.getValue()); + this.configurator = new LearnerConfigurator(this.space); + this.configurator.validate(); + } catch (Exception e) { + throw new IllegalStateException("Could not set up " + getClass().getSimpleName() + + " from \"" + configurationFileOption.getValue() + "\": " + e.getMessage(), e); + } + } + + + @Override + public void checkParameterChange() {} + + @Override + public void swapClassifiers(int bestPerforming) {} + + @Override + public void deepCopyList(int bestPerforming) {} + + @Override + public void changeStateParameter(Parameter parameter, int index) {} + + private void initializeSimplex() { + simplex = new SimplexEntry[3]; + for (int i = 0; i < 3; i++) { + ArrayList params = createRandomParams(); + Classifier model = createModelWithParams(params); + simplex[i] = new SimplexEntry(model, newEvaluator(), params); + } + if (verboseOption.isSet()) + System.out.println("SSPT: Initialized simplex with 3 models"); + } + + private ArrayList createRandomParams() { + ArrayList params = new ArrayList<>(); + for (ParameterSpec spec : this.space.parameters) { + String name = spec.name; + switch (spec.type) { + case ParameterSpec.TYPE_INT: { + int[] range = {(int) spec.range[0], (int) spec.range[1]}; + int value = classifierRandom.nextInt(range[1] - range[0] + 1) + range[0]; + params.add(new IntParameter(name, value, range, new Random(classifierRandom.nextLong()))); + break; + } + case ParameterSpec.TYPE_DOUBLE: { + double[] range = {spec.range[0], spec.range[1]}; + double value = range[0] + classifierRandom.nextDouble() * (range[1] - range[0]); + params.add(new DoubleParameter(name, value, range, new Random(classifierRandom.nextLong()))); + break; + } + case ParameterSpec.TYPE_CATEGORICAL: { + String[] values = spec.values; + int active = classifierRandom.nextInt(values.length); + params.add(new CategoricalParameter(name, values, active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return params; + } + + /** A model built from scratch at {@code params}. */ + private Classifier createModelWithParams(ArrayList params) { + return this.configurator.instantiate(params, this.classifierRandom.nextInt()); + } + + /** + * Reconfigure a warm-started model in place, returning the model to use. + * Falls back to rebuilding when the space contains a hyperparameter the + * learner only reads at construction time, which by definition cannot take + * effect on a model that is already training. + */ + private Classifier applyParamsToModel(Classifier model, ArrayList params) { + if (!this.configurator.canApplyLive()) { + return this.configurator.instantiate(params, this.classifierRandom.nextInt()); + } + this.configurator.applyLive(model, params); + return model; + } + + private BasicClassificationPerformanceEvaluator newEvaluator() { + BasicClassificationPerformanceEvaluator eval = new BasicClassificationPerformanceEvaluator(); + MetricUtils.configure(eval); + return eval; + } + + // ========== TRAINING ========== + + @Override + public double[] getVotesForInstance(Instance inst) { + return simplex[0].model.getVotesForInstance(inst); + } + + @Override + public void trainOnInstanceImpl(Instance inst) { + double[] incumbentVotes = driftDetector != null ? getVotesForInstance(inst) : null; + initExecutor(); + instanceCount++; + evaluationInstances++; + InstanceExample example = new InstanceExample(inst); + + if (converged) { + trainConverged(inst, example); + } + else { + trainNotConverged(inst, example); + } + + if (incumbentVotes != null) checkDrift(inst, incumbentVotes); + } + + private void trainConverged(Instance inst, InstanceExample example) { + SimplexEntry best = simplex[0]; + double[] votes = best.model.getVotesForInstance(inst); + best.addResult(example, votes); + best.model.trainOnInstance(inst); + } + + private void trainNotConverged(Instance inst, InstanceExample example) { + + // Vertices and expansion points form a single batch: two barriers per + // instance would halve the gain on such a small number of models. + Collection trainers = this.executor == null + ? null : new ArrayList(); + + for (SimplexEntry entry : simplex) { + double[] votes = entry.model.getVotesForInstance(inst); + entry.addResult(example, votes); + + if (trainers == null) entry.model.trainOnInstance(inst); + else trainers.add(new TrainingRunnable(entry.model, inst)); + } + + if (expanded != null) { + for (SimplexEntry entry : expanded.values()) { + double[] votes = entry.model.getVotesForInstance(inst); + entry.addResult(example, votes); + + if (trainers == null) entry.model.trainOnInstance(inst); + else trainers.add(new TrainingRunnable(entry.model, inst)); + } + } + + if (trainers != null) { + try { + this.executor.invokeAll(trainers); + } catch (InterruptedException ex) { + throw new RuntimeException("Could not call invokeAll() on training threads."); + } + } + + if (evaluationInstances >= gracePeriodOption.getValue()) { + evaluationInstances = 0; + updateSimplex(); + if (checkConvergence()) { + if (verboseOption.isSet()) + System.out.println("SSPT: Converged at instance " + instanceCount); + converged = true; + } + } + } + + // ========== SIMPLEX UPDATE ========== + + private void updateSimplex() { + sortSimplex(); + lastCentroid = computeCentroid(); + + if (expanded == null || expanded.isEmpty()) { + expanded = createExpanded(); + } + + applyNelderMeadOperators(); + expanded = null; + } + + private void sortSimplex() { + int metric = metricOption.getChosenIndex(); + // Insertion sort (3 elements, descending by metric) + for (int i = 1; i < 3; i++) { + SimplexEntry key = simplex[i]; + int j = i - 1; + while (j >= 0 && simplex[j].getMetric(metric) < key.getMetric(metric)) { + simplex[j + 1] = simplex[j]; + j--; + } + simplex[j + 1] = key; + } + } + + private ArrayList computeCentroid() { + ArrayList centroid = cloneParams(simplex[0].params); + for (int i = 0; i < centroid.size(); i++) { + Parameter p = centroid.get(i); + if (p.type == 0) { + double sum = 0; + for (SimplexEntry e : simplex) sum += ((IntParameter) e.params.get(i)).value; + int val = clampInt((int) Math.round(sum / 3.0), + ((IntParameter) p).range[0], ((IntParameter) p).range[1]); + ((IntParameter) p).value = val; + } else if (p.type == 1) { + double sum = 0; + for (SimplexEntry e : simplex) sum += ((DoubleParameter) e.params.get(i)).value; + double val = clampDouble(sum / 3.0, + ((DoubleParameter) p).range[0], ((DoubleParameter) p).range[1]); + ((DoubleParameter) p).value = val; + } else { + // Mode for categorical + Map counts = new HashMap<>(); + for (SimplexEntry e : simplex) + counts.merge(((CategoricalParameter) e.params.get(i)).active, 1, Integer::sum); + int mode = Collections.max(counts.entrySet(), Map.Entry.comparingByValue()).getKey(); + ((CategoricalParameter) p).active = mode; + } + } + return centroid; + } + + private boolean checkConvergence() { + if (lastCentroid == null) return false; + ArrayList current = computeCentroid(); + double distSq = 0; + for (int i = 0; i < lastCentroid.size(); i++) { + Parameter lp = lastCentroid.get(i); + Parameter cp = current.get(i); + if (lp.type == 0) { + double diff = ((IntParameter) lp).value - ((IntParameter) cp).value; + distSq += diff * diff; + } else if (lp.type == 1) { + double diff = ((DoubleParameter) lp).value - ((DoubleParameter) cp).value; + distSq += diff * diff; + } + } + double threshold = convergenceSphereOption.getValue(); + return distSq < threshold * threshold; + } + + // ========== NELDER-MEAD ========== + + private interface DoubleOp { + double apply(double a, double b); + } + + private HashMap createExpanded() { + HashMap exp = new HashMap<>(); + + ArrayList bestP = simplex[0].params; + ArrayList goodP = simplex[1].params; + ArrayList worstP = simplex[2].params; + + ArrayList midP = combine(bestP, goodP, (a, b) -> (a + b) / 2.0); + ArrayList reflP = combine(midP, worstP, (a, b) -> 2 * a - b); + ArrayList expP = combine(reflP, midP, (a, b) -> 2 * a - b); + ArrayList shrP = combine(bestP, worstP, (a, b) -> (a + b) / 2.0); + ArrayList cont1P = combine(midP, worstP, (a, b) -> (a + b) / 2.0); + ArrayList cont2P = combine(midP, reflP, (a, b) -> (a + b) / 2.0); + + String[] names = {"midpoint", "reflection", "expansion", "shrink", "contraction1", "contraction2"}; + @SuppressWarnings("unchecked") + ArrayList[] paramSets = new ArrayList[]{midP, reflP, expP, shrP, cont1P, cont2P}; + + // Freeze the hyperparameters outside the optimized subset to the best + // simplex vertex before any model is built or evaluated. + for (ArrayList ps : paramSets) + HPOMethod.freezeToIncumbent(this.optimizeMask, ps, simplex[0].params); + + for (int i = 0; i < names.length; i++) { + SimplexEntry entry; + if (resetModelsOption.isSet()) { + Classifier model = createModelWithParams(paramSets[i]); + entry = new SimplexEntry(model, newEvaluator(), paramSets[i]); + } else { + // Warm-start: copy best model's learned state, change parameters + Classifier model = simplex[0].model.copy(); + LearnerConfigurator.reseedCopy(model, this.classifierRandom.nextInt()); + model = applyParamsToModel(model, paramSets[i]); + entry = new SimplexEntry(model, newEvaluator(), paramSets[i]); + } + exp.put(names[i], entry); + } + return exp; + } + + private void applyNelderMeadOperators() { + int metric = metricOption.getChosenIndex(); + SimplexEntry b = simplex[0]; + SimplexEntry g = simplex[1]; + SimplexEntry w = simplex[2]; + SimplexEntry r = expanded.get("reflection"); + SimplexEntry c1 = expanded.get("contraction1"); + SimplexEntry c2 = expanded.get("contraction2"); + SimplexEntry e = expanded.get("expansion"); + SimplexEntry s = expanded.get("shrink"); + SimplexEntry mid = expanded.get("midpoint"); + + SimplexEntry contraction = c1.getMetric(metric) > c2.getMetric(metric) ? c1 : c2; + + if (r.getMetric(metric) > g.getMetric(metric)) { + if (b.getMetric(metric) > r.getMetric(metric)) { + simplex[2] = r; + } else { + simplex[2] = e.getMetric(metric) > b.getMetric(metric) ? e : r; + } + } else { + if (r.getMetric(metric) > w.getMetric(metric)) { + simplex[2] = r; + } else { + if (contraction.getMetric(metric) > w.getMetric(metric)) { + simplex[2] = contraction; + } else { + simplex[2] = s; + simplex[1] = mid; + } + } + } + sortSimplex(); + } + + // ========== HELPERS ========== + + private ArrayList combine(ArrayList p1, ArrayList p2, DoubleOp op) { + ArrayList result = new ArrayList<>(); + for (int i = 0; i < p1.size(); i++) { + Parameter a = p1.get(i); + Parameter b = p2.get(i); + switch (a.type) { + case Parameter.TYPE_INT: { + int[] range = ((IntParameter) a).range; + double combined = op.apply(((IntParameter) a).value, ((IntParameter) b).value); + int val = clampInt((int) Math.round(combined), range[0], range[1]); + result.add(new IntParameter(a.name, val, range, new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_DOUBLE: { + double[] range = ((DoubleParameter) a).range; + double val = clampDouble( + op.apply(((DoubleParameter) a).value, ((DoubleParameter) b).value), + range[0], range[1]); + result.add(new DoubleParameter(a.name, val, range, new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_CATEGORICAL: { + CategoricalParameter ca = (CategoricalParameter) a; + CategoricalParameter cb = (CategoricalParameter) b; + int active = classifierRandom.nextBoolean() ? ca.active : cb.active; + result.add(new CategoricalParameter(a.name, ca.values, active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return result; + } + + private ArrayList cloneParams(ArrayList source) { + ArrayList copy = new ArrayList<>(); + for (Parameter p : source) { + switch (p.type) { + case Parameter.TYPE_INT: { + IntParameter ip = (IntParameter) p; + copy.add(new IntParameter(ip.name, ip.value, ip.range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_DOUBLE: { + DoubleParameter dp = (DoubleParameter) p; + copy.add(new DoubleParameter(dp.name, dp.value, dp.range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_CATEGORICAL: { + CategoricalParameter cp = (CategoricalParameter) p; + copy.add(new CategoricalParameter(cp.name, cp.values.clone(), cp.active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return copy; + } + + private int clampInt(int val, int min, int max) { + return Math.max(min, Math.min(max, val)); + } + + private double clampDouble(double val, double min, double max) { + return Math.max(min, Math.min(max, val)); + } + + private int argmax(double[] arr) { + int best = 0; + for (int i = 1; i < arr.length; i++) + if (arr[i] > arr[best]) best = i; + return best; + } + + // ========== MOA INTERFACE ========== + + @Override + protected Measurement[] getModelMeasurementsImpl() { + ArrayList measurements = new ArrayList<>(); + measurements.add(new Measurement("driftsDetected", driftsDetected)); + for (Measurement m : simplex[0].model.getModelMeasurements()) + measurements.add(m); + for (Parameter p : simplex[0].params) { + switch (p.type) { + case Parameter.TYPE_INT: + measurements.add(new Measurement(p.name, ((IntParameter) p).value)); + break; + case Parameter.TYPE_DOUBLE: + measurements.add(new Measurement(p.name, ((DoubleParameter) p).value)); + break; + case Parameter.TYPE_CATEGORICAL: + measurements.add(new Measurement(p.name, ((CategoricalParameter) p).active)); + break; + } + } + return measurements.toArray(new Measurement[0]); + } + + @Override + public void getModelDescription(StringBuilder out, int indent) { + } + + @Override + public long measureByteSize() { + long size = SizeOf.sizeOf(this); + for (SimplexEntry entry : simplex) + size += entry.model.measureByteSize(); + return size; + } + + @Override + public double getCandidateScore(int i) { + return simplex[i].getMetric(metricOption.getChosenIndex()); + } + + @Override + public double getClassifierScore() { + return simplex[0].getMetric(metricOption.getChosenIndex()); + } + + @Override + public int getNumberOfCandidates() { return 9; } + + @Override + public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } + + @Override + public int getEvaluationInstancesCount() { return evaluationInstances; } + + @Override + public int getGracePeriod() { return gracePeriodOption.getValue(); } + + @Override + public Classifier getMainClassifier() { return simplex[0].model; } + + @Override + public ArrayList getReferenceParameters() { return simplex[0].params; } + + @Override + public String getConfigurationFile() { return this.configurationFileOption.getValue();} + + + @Override + public ArrayList> getCandidateParameters() { + ArrayList> list = new ArrayList<>(); + for (SimplexEntry e : simplex) list.add(e.params); + return list; + } + + /** + * Feeds the incumbent's prediction error to the change detector and, when a + * drift is signalled, reinitializes the whole search from the search space. + * The signal is the 0/1 misclassification indicator. + */ + private void checkDrift(Instance inst, double[] incumbentVotes) { + driftDetector.input(argmax(incumbentVotes) != (int) inst.classValue() ? 1.0 : 0.0); + if (driftDetector.getChange()) { + driftsDetected++; + if (verboseOption.isSet()) + System.out.println("SSPT: Drift detected at instance " + instanceCount + + ", reinitializing"); + resetLearningImpl(); + } + } + + /** + * Inner class to assist with the multi-thread execution. + */ + protected class TrainingRunnable implements Runnable, Callable { + final private Classifier learner; + final private Instance instance; + + public TrainingRunnable(Classifier learner, Instance instance) { + this.learner = learner; + this.instance = instance; + } + + @Override + public void run() { + this.learner.trainOnInstance(this.instance); + } + + @Override + public Integer call() { + run(); + return 0; + } + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/SSPTRegressor.java b/moa/src/main/java/moa/classifiers/AutoML/SSPTRegressor.java new file mode 100644 index 000000000..12587a0f3 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/SSPTRegressor.java @@ -0,0 +1,711 @@ +/* + * SSPTRegressor.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML; + +import com.github.javacliparser.FileOption; +import com.github.javacliparser.FlagOption; +import com.github.javacliparser.FloatOption; +import com.github.javacliparser.IntOption; +import com.github.javacliparser.MultiChoiceOption; +import com.yahoo.labs.samoa.instances.Instance; +import moa.capabilities.CapabilitiesHandler; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.AutoML.Parameters.*; +import moa.classifiers.AutoML.space.ConfigurationSpace; +import moa.classifiers.AutoML.space.LearnerConfigurator; +import moa.classifiers.AutoML.space.ParameterSpec; +import moa.classifiers.Classifier; +import moa.classifiers.core.driftdetection.ChangeDetector; +import moa.classifiers.Regressor; +import moa.core.InstanceExample; +import moa.core.Measurement; +import moa.core.SizeOf; +import moa.evaluation.BasicRegressionPerformanceEvaluator; +import moa.options.ClassOption; + +import java.io.Serializable; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Regression counterpart of {@link SSPTClassifier}: a Nelder-Mead simplex over + * the streaming hyperparameter space, scored with the regression metrics of + * {@link MetricUtils}. + * + *

See details in:
Bruno Veloso, Joao Gama, Benedita Malheiro, Joao + * Vinagre. Hyperparameter self-tuning for data streams. In Information Fusion, + * 76:75-86, DOI: 10.1016/j.inffus.2021.04.011, Elsevier, 2021.

+ * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public class SSPTRegressor extends AbstractClassifier implements Regressor, + CapabilitiesHandler, Serializable, HPOMethod { + + public FileOption configurationFileOption = new FileOption("configurationFile", 'f', + "Search space in JSON format.", null, ".json", false); + + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + "Number of instances between simplex updates.", 1000, 1, Integer.MAX_VALUE); + + public FloatOption convergenceSphereOption = new FloatOption("convergenceSphere", 'c', + "Convergence threshold: squared distance between centroids.", 0.001, 0.0, Double.MAX_VALUE); + + public MultiChoiceOption metricOption = new MultiChoiceOption("metric", 'm', + "Metric to optimize the model.", MetricUtils.REGRESSION_METRIC_NAMES, + MetricUtils.REGRESSION_METRIC_DESCRIPTIONS, 0); + + public FlagOption resetModelsOption = new FlagOption("resetModels", 'r', + "Create fresh classifiers for expanded points instead of warm-starting from best."); + + public FlagOption verboseOption = new FlagOption("verbose", 'v', + "Print simplex events to stdout."); + + public FlagOption driftDetectionOption = new FlagOption("driftDetection", 'd', + "Enable drift detection on the incumbent's prediction error."); + + public ClassOption driftDetectorOption = new ClassOption("driftDetector", 'D', + "Change detector to use when drift detection is enabled; on a detected" + + " drift the whole search is reinitialized from the search space.", + ChangeDetector.class, "ADWINChangeDetector"); + + public IntOption numberOfJobsOption = new IntOption("numberOfJobs", 'j', + "Total number of concurrent jobs used for processing (-1 = as much as possible, 0 = do not use multithreading)", + 1, -1, Integer.MAX_VALUE); + + protected static final int SINGLE_THREAD = 0; + + /** Largest batch trained per instance: the 3 vertices plus the 6 expansion points. */ + protected static final int MAX_MODELS_PER_INSTANCE = 9; + + // ========== INNER CLASS ========== + + protected static class SimplexEntry implements Serializable { + Classifier model; + BasicRegressionPerformanceEvaluator evaluator; + ArrayList params; + long instancesSeen; + + SimplexEntry(Classifier model, BasicRegressionPerformanceEvaluator evaluator, + ArrayList params) { + this.model = model; + this.evaluator = evaluator; + this.params = params; + this.instancesSeen = 0; + } + + double getMetric(int metricIndex) { + if (instancesSeen == 0) return Double.NEGATIVE_INFINITY; + return MetricUtils.getRegressionScore(evaluator.getPerformanceMeasurements(), metricIndex); + } + + void addResult(InstanceExample example, double[] votes) { + evaluator.addResult(example, votes); + instancesSeen++; + } + } + + // ========== FIELDS ========== + + protected SimplexEntry[] simplex; + protected HashMap expanded; + + protected ArrayList lastCentroid; + + protected ConfigurationSpace space; + + /** Boundary through which every learner is configured. */ + protected LearnerConfigurator configurator; + + protected long instanceCount; + protected int evaluationInstances; + protected boolean converged; + + protected boolean[] optimizeMask; + + /** Detector on the incumbent's error; {@code null} unless drift detection is on. */ + protected ChangeDetector driftDetector; + + /** Cumulative number of drifts signalled over the run; not reset by a restart. */ + protected long driftsDetected; + + /** + * Pool training the simplex vertices and expansion points concurrently; + * {@code null} when running single-threaded. Transient because an executor + * cannot be serialized, so it is (re)created by {@link #initExecutor()}. + */ + protected transient ExecutorService executor; + + @Override + public void setOptimizableParameters(boolean[] mask) { this.optimizeMask = mask; } + + // ========== LIFECYCLE ========== + + @Override + public boolean isRandomizable() { + return true; + } + + /** + * Creates the training pool on first use, and again after deserialization. + * A pool larger than the batch trained per instance would leave threads idle, + * so the requested job count is capped at {@link #MAX_MODELS_PER_INSTANCE}. + */ + protected void initExecutor() { + if (this.executor != null) return; + int numberOfJobs = this.numberOfJobsOption.getValue() == -1 + ? Runtime.getRuntime().availableProcessors() + : this.numberOfJobsOption.getValue(); + numberOfJobs = Math.min(numberOfJobs, MAX_MODELS_PER_INSTANCE); + // SINGLE_THREAD and requesting a single thread are equivalent: training + // then happens in-place and this.executor stays null. + if (numberOfJobs != SINGLE_THREAD && numberOfJobs != 1) { + // Daemon threads: a live pool must not keep the JVM alive after the + // task that ran this learner has finished. + this.executor = Executors.newFixedThreadPool(numberOfJobs, runnable -> { + Thread thread = new Thread(runnable); + thread.setDaemon(true); + return thread; + }); + } + } + + @Override + public void cleanThreads() { + if (this.executor != null) { + this.executor.shutdownNow(); + this.executor = null; + } + } + + @Override + public void resetLearningImpl() { + cleanThreads(); // shut down any pool from a previous reset before creating a new one + converged = false; + instanceCount = 0; + evaluationInstances = 0; + lastCentroid = null; + expanded = null; + + driftDetector = driftDetectionOption.isSet() + ? ((ChangeDetector) getPreparedClassOption(driftDetectorOption)).copy() + : null; + + setConfigurations(); + initializeSimplex(); + + } + + @Override + public void setConfigurations() { + try { + this.space = ConfigurationSpace.fromFile(configurationFileOption.getValue()); + this.configurator = new LearnerConfigurator(this.space); + this.configurator.validate(); + } catch (Exception e) { + throw new IllegalStateException("Could not set up " + getClass().getSimpleName() + + " from \"" + configurationFileOption.getValue() + "\": " + e.getMessage(), e); + } + } + + + @Override + public void checkParameterChange() {} + + @Override + public void swapClassifiers(int bestPerforming) {} + + @Override + public void deepCopyList(int bestPerforming) {} + + @Override + public void changeStateParameter(Parameter parameter, int index) {} + + private void initializeSimplex() { + simplex = new SimplexEntry[3]; + for (int i = 0; i < 3; i++) { + ArrayList params = createRandomParams(); + Classifier model = createModelWithParams(params); + simplex[i] = new SimplexEntry(model, newEvaluator(), params); + } + if (verboseOption.isSet()) + System.out.println("SSPT: Initialized simplex with 3 models"); + } + + private ArrayList createRandomParams() { + ArrayList params = new ArrayList<>(); + for (ParameterSpec spec : this.space.parameters) { + String name = spec.name; + switch (spec.type) { + case ParameterSpec.TYPE_INT: { + int[] range = {(int) spec.range[0], (int) spec.range[1]}; + int value = classifierRandom.nextInt(range[1] - range[0] + 1) + range[0]; + params.add(new IntParameter(name, value, range, new Random(classifierRandom.nextLong()))); + break; + } + case ParameterSpec.TYPE_DOUBLE: { + double[] range = {spec.range[0], spec.range[1]}; + double value = range[0] + classifierRandom.nextDouble() * (range[1] - range[0]); + params.add(new DoubleParameter(name, value, range, new Random(classifierRandom.nextLong()))); + break; + } + case ParameterSpec.TYPE_CATEGORICAL: { + String[] values = spec.values; + int active = classifierRandom.nextInt(values.length); + params.add(new CategoricalParameter(name, values, active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return params; + } + + /** A model built from scratch at {@code params}. */ + private Classifier createModelWithParams(ArrayList params) { + return this.configurator.instantiate(params, this.classifierRandom.nextInt()); + } + + /** + * Reconfigure a warm-started model in place, returning the model to use. + * Falls back to rebuilding when the space contains a hyperparameter the + * learner only reads at construction time, which by definition cannot take + * effect on a model that is already training. + */ + private Classifier applyParamsToModel(Classifier model, ArrayList params) { + if (!this.configurator.canApplyLive()) { + return this.configurator.instantiate(params, this.classifierRandom.nextInt()); + } + this.configurator.applyLive(model, params); + return model; + } + + private BasicRegressionPerformanceEvaluator newEvaluator() { + return new BasicRegressionPerformanceEvaluator(); + } + + // ========== TRAINING ========== + + @Override + public double[] getVotesForInstance(Instance inst) { + return simplex[0].model.getVotesForInstance(inst); + } + + @Override + public void trainOnInstanceImpl(Instance inst) { + double[] incumbentVotes = driftDetector != null ? getVotesForInstance(inst) : null; + initExecutor(); + instanceCount++; + evaluationInstances++; + InstanceExample example = new InstanceExample(inst); + + trainNotConverged(inst, example); + + if (incumbentVotes != null) checkDrift(inst, incumbentVotes); + } + + private void trainConverged(Instance inst, InstanceExample example) { + SimplexEntry best = simplex[0]; + double[] votes = best.model.getVotesForInstance(inst); + best.addResult(example, votes); + best.model.trainOnInstance(inst); + } + + private void trainNotConverged(Instance inst, InstanceExample example) { + + // Vertices and expansion points form a single batch: two barriers per + // instance would halve the gain on such a small number of models. + Collection trainers = this.executor == null + ? null : new ArrayList(); + + for (SimplexEntry entry : simplex) { + double[] votes = entry.model.getVotesForInstance(inst); + entry.addResult(example, votes); + + if (trainers == null) entry.model.trainOnInstance(inst); + else trainers.add(new TrainingRunnable(entry.model, inst)); + } + + if (expanded != null) { + for (SimplexEntry entry : expanded.values()) { + double[] votes = entry.model.getVotesForInstance(inst); + entry.addResult(example, votes); + + if (trainers == null) entry.model.trainOnInstance(inst); + else trainers.add(new TrainingRunnable(entry.model, inst)); + } + } + + if (trainers != null) { + try { + this.executor.invokeAll(trainers); + } catch (InterruptedException ex) { + throw new RuntimeException("Could not call invokeAll() on training threads."); + } + } + + if (evaluationInstances >= gracePeriodOption.getValue()) { + evaluationInstances = 0; + updateSimplex(); + if (checkConvergence()) { + if (verboseOption.isSet()) + System.out.println("SSPT: Converged at instance " + instanceCount); + converged = true; + } + } + } + + // ========== SIMPLEX UPDATE ========== + + private void updateSimplex() { + sortSimplex(); + lastCentroid = computeCentroid(); + + if (expanded == null || expanded.isEmpty()) { + expanded = createExpanded(); + } + + applyNelderMeadOperators(); + expanded = null; + } + + private void sortSimplex() { + int metric = metricOption.getChosenIndex(); + // Insertion sort (3 elements, descending by metric) + for (int i = 1; i < 3; i++) { + SimplexEntry key = simplex[i]; + int j = i - 1; + while (j >= 0 && simplex[j].getMetric(metric) < key.getMetric(metric)) { + simplex[j + 1] = simplex[j]; + j--; + } + simplex[j + 1] = key; + } + } + + private ArrayList computeCentroid() { + ArrayList centroid = cloneParams(simplex[0].params); + for (int i = 0; i < centroid.size(); i++) { + Parameter p = centroid.get(i); + if (p.type == 0) { + double sum = 0; + for (SimplexEntry e : simplex) sum += ((IntParameter) e.params.get(i)).value; + int val = clampInt((int) Math.round(sum / 3.0), + ((IntParameter) p).range[0], ((IntParameter) p).range[1]); + ((IntParameter) p).value = val; + } else if (p.type == 1) { + double sum = 0; + for (SimplexEntry e : simplex) sum += ((DoubleParameter) e.params.get(i)).value; + double val = clampDouble(sum / 3.0, + ((DoubleParameter) p).range[0], ((DoubleParameter) p).range[1]); + ((DoubleParameter) p).value = val; + } else { + // Mode for categorical + Map counts = new HashMap<>(); + for (SimplexEntry e : simplex) + counts.merge(((CategoricalParameter) e.params.get(i)).active, 1, Integer::sum); + int mode = Collections.max(counts.entrySet(), Map.Entry.comparingByValue()).getKey(); + ((CategoricalParameter) p).active = mode; + } + } + return centroid; + } + + private boolean checkConvergence() { + if (lastCentroid == null) return false; + ArrayList current = computeCentroid(); + double distSq = 0; + for (int i = 0; i < lastCentroid.size(); i++) { + Parameter lp = lastCentroid.get(i); + Parameter cp = current.get(i); + if (lp.type == 0) { + double diff = ((IntParameter) lp).value - ((IntParameter) cp).value; + distSq += diff * diff; + } else if (lp.type == 1) { + double diff = ((DoubleParameter) lp).value - ((DoubleParameter) cp).value; + distSq += diff * diff; + } + } + double threshold = convergenceSphereOption.getValue(); + return distSq < threshold * threshold; + } + + // ========== NELDER-MEAD ========== + + private interface DoubleOp { + double apply(double a, double b); + } + + private HashMap createExpanded() { + HashMap exp = new HashMap<>(); + + ArrayList bestP = simplex[0].params; + ArrayList goodP = simplex[1].params; + ArrayList worstP = simplex[2].params; + + ArrayList midP = combine(bestP, goodP, (a, b) -> (a + b) / 2.0); + ArrayList reflP = combine(midP, worstP, (a, b) -> 2 * a - b); + ArrayList expP = combine(reflP, midP, (a, b) -> 2 * a - b); + ArrayList shrP = combine(bestP, worstP, (a, b) -> (a + b) / 2.0); + ArrayList cont1P = combine(midP, worstP, (a, b) -> (a + b) / 2.0); + ArrayList cont2P = combine(midP, reflP, (a, b) -> (a + b) / 2.0); + + String[] names = {"midpoint", "reflection", "expansion", "shrink", "contraction1", "contraction2"}; + @SuppressWarnings("unchecked") + ArrayList[] paramSets = new ArrayList[]{midP, reflP, expP, shrP, cont1P, cont2P}; + + // Freeze the hyperparameters outside the optimized subset to the best + // simplex vertex before any model is built or evaluated. + for (ArrayList ps : paramSets) + HPOMethod.freezeToIncumbent(this.optimizeMask, ps, simplex[0].params); + + for (int i = 0; i < names.length; i++) { + SimplexEntry entry; + if (resetModelsOption.isSet()) { + Classifier model = createModelWithParams(paramSets[i]); + entry = new SimplexEntry(model, newEvaluator(), paramSets[i]); + } else { + // Warm-start: copy best model's learned state, change parameters + Classifier model = simplex[0].model.copy(); + LearnerConfigurator.reseedCopy(model, this.classifierRandom.nextInt()); + model = applyParamsToModel(model, paramSets[i]); + entry = new SimplexEntry(model, newEvaluator(), paramSets[i]); + } + exp.put(names[i], entry); + } + return exp; + } + + private void applyNelderMeadOperators() { + int metric = metricOption.getChosenIndex(); + SimplexEntry b = simplex[0]; + SimplexEntry g = simplex[1]; + SimplexEntry w = simplex[2]; + SimplexEntry r = expanded.get("reflection"); + SimplexEntry c1 = expanded.get("contraction1"); + SimplexEntry c2 = expanded.get("contraction2"); + SimplexEntry e = expanded.get("expansion"); + SimplexEntry s = expanded.get("shrink"); + SimplexEntry mid = expanded.get("midpoint"); + + SimplexEntry contraction = c1.getMetric(metric) > c2.getMetric(metric) ? c1 : c2; + + if (r.getMetric(metric) > g.getMetric(metric)) { + if (b.getMetric(metric) > r.getMetric(metric)) { + simplex[2] = r; + } else { + simplex[2] = e.getMetric(metric) > b.getMetric(metric) ? e : r; + } + } else { + if (r.getMetric(metric) > w.getMetric(metric)) { + simplex[2] = r; + } else { + if (contraction.getMetric(metric) > w.getMetric(metric)) { + simplex[2] = contraction; + } else { + simplex[2] = s; + simplex[1] = mid; + } + } + } + sortSimplex(); + } + + // ========== HELPERS ========== + + private ArrayList combine(ArrayList p1, ArrayList p2, DoubleOp op) { + ArrayList result = new ArrayList<>(); + for (int i = 0; i < p1.size(); i++) { + Parameter a = p1.get(i); + Parameter b = p2.get(i); + switch (a.type) { + case Parameter.TYPE_INT: { + int[] range = ((IntParameter) a).range; + double combined = op.apply(((IntParameter) a).value, ((IntParameter) b).value); + int val = clampInt((int) Math.round(combined), range[0], range[1]); + result.add(new IntParameter(a.name, val, range, new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_DOUBLE: { + double[] range = ((DoubleParameter) a).range; + double val = clampDouble( + op.apply(((DoubleParameter) a).value, ((DoubleParameter) b).value), + range[0], range[1]); + result.add(new DoubleParameter(a.name, val, range, new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_CATEGORICAL: { + CategoricalParameter ca = (CategoricalParameter) a; + CategoricalParameter cb = (CategoricalParameter) b; + int active = classifierRandom.nextBoolean() ? ca.active : cb.active; + result.add(new CategoricalParameter(a.name, ca.values, active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return result; + } + + private ArrayList cloneParams(ArrayList source) { + ArrayList copy = new ArrayList<>(); + for (Parameter p : source) { + switch (p.type) { + case Parameter.TYPE_INT: { + IntParameter ip = (IntParameter) p; + copy.add(new IntParameter(ip.name, ip.value, ip.range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_DOUBLE: { + DoubleParameter dp = (DoubleParameter) p; + copy.add(new DoubleParameter(dp.name, dp.value, dp.range.clone(), new Random(classifierRandom.nextLong()))); + break; + } + case Parameter.TYPE_CATEGORICAL: { + CategoricalParameter cp = (CategoricalParameter) p; + copy.add(new CategoricalParameter(cp.name, cp.values.clone(), cp.active, new Random(classifierRandom.nextLong()))); + break; + } + } + } + return copy; + } + + private int clampInt(int val, int min, int max) { + return Math.max(min, Math.min(max, val)); + } + + private double clampDouble(double val, double min, double max) { + return Math.max(min, Math.min(max, val)); + } + + // ========== MOA INTERFACE ========== + + @Override + protected Measurement[] getModelMeasurementsImpl() { + ArrayList measurements = new ArrayList<>(); + measurements.add(new Measurement("driftsDetected", driftsDetected)); + for (Measurement m : simplex[0].model.getModelMeasurements()) + measurements.add(m); + for (Parameter p : simplex[0].params) { + switch (p.type) { + case Parameter.TYPE_INT: + measurements.add(new Measurement(p.name, ((IntParameter) p).value)); + break; + case Parameter.TYPE_DOUBLE: + measurements.add(new Measurement(p.name, ((DoubleParameter) p).value)); + break; + case Parameter.TYPE_CATEGORICAL: + measurements.add(new Measurement(p.name, ((CategoricalParameter) p).active)); + break; + } + } + return measurements.toArray(new Measurement[0]); + } + + @Override + public void getModelDescription(StringBuilder out, int indent) { + } + + @Override + public long measureByteSize() { + long size = SizeOf.sizeOf(this); + for (SimplexEntry entry : simplex) + size += entry.model.measureByteSize(); + return size; + } + + @Override + public double getCandidateScore(int i) { + return simplex[i].getMetric(metricOption.getChosenIndex()); + } + + @Override + public double getClassifierScore() { + return simplex[0].getMetric(metricOption.getChosenIndex()); + } + + @Override + public int getNumberOfCandidates() { return 3; } + + @Override + public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } + + @Override + public int getEvaluationInstancesCount() { return evaluationInstances; } + + @Override + public int getGracePeriod() { return gracePeriodOption.getValue(); } + + @Override + public Classifier getMainClassifier() { return simplex[0].model; } + + @Override + public ArrayList getReferenceParameters() { return simplex[0].params; } + + @Override + public String getConfigurationFile() { return this.configurationFileOption.getValue();} + + + @Override + public ArrayList> getCandidateParameters() { + ArrayList> list = new ArrayList<>(); + for (SimplexEntry e : simplex) list.add(e.params); + return list; + } + + /** + * Feeds the incumbent's prediction error to the change detector and, when a + * drift is signalled, reinitializes the whole search from the search space. + * The signal is the absolute error |y - y_pred|. + */ + private void checkDrift(Instance inst, double[] incumbentVotes) { + double predicted = incumbentVotes.length > 0 ? incumbentVotes[0] : 0.0; + driftDetector.input(Math.abs(predicted - inst.classValue())); + if (driftDetector.getChange()) { + driftsDetected++; + if (verboseOption.isSet()) + System.out.println("SSPT: Drift detected at instance " + instanceCount + + ", reinitializing"); + resetLearningImpl(); + } + } + + /** + * Inner class to assist with the multi-thread execution. + */ + protected class TrainingRunnable implements Runnable, Callable { + final private Classifier learner; + final private Instance instance; + + public TrainingRunnable(Classifier learner, Instance instance) { + this.learner = learner; + this.instance = instance; + } + + @Override + public void run() { + this.learner.trainOnInstance(this.instance); + } + + @Override + public Integer call() { + run(); + return 0; + } + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/space/ConfigurationSpace.java b/moa/src/main/java/moa/classifiers/AutoML/space/ConfigurationSpace.java new file mode 100644 index 000000000..1ee1c85d0 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/space/ConfigurationSpace.java @@ -0,0 +1,221 @@ +/* + * ConfigurationSpace.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML.space; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.Reader; +import java.io.Serializable; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + +/** + * The search space shared by every AutoML method: the CLI string of the learner + * being tuned, plus the list of tunable hyperparameters. + * + *

Expected JSON, as produced by hand or by the CapyMOA wrappers: + * + * 

+ * {
+ *   "algorithm": "moa.classifiers.trees.HoeffdingTree",
+ *   "parameters": [
+ *     {"parameter": "gracePeriod",    "type": "integer",     "value": 200,  "range": [50, 500], "step": 10},
+ *     {"parameter": "splitConfidence","type": "double",      "value": 1E-7, "range": [1E-7, 0.05]},
+ *     {"parameter": "splitCriterion", "type": "categorical", "active": 0,
+ *      "values": ["moa.classifiers.core.splitcriteria.InfoGainSplitCriterion",
+ *                 "moa.classifiers.core.splitcriteria.GiniSplitCriterion"]}
+ *   ]
+ * }
+ *
+ * + *

{@code "parameter"} is an option path resolved against the learner's MOA + * options - see {@link ParameterSpec}. {@code "type"} accepts {@code integer} / + * {@code int}, {@code double} / {@code float} / {@code real}, and + * {@code categorical} / {@code nominal}. The optional {@code "onChange"} key + * takes {@code "live"} (default) or {@code "reset"}. + * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public class ConfigurationSpace implements Serializable { + + private static final long serialVersionUID = 1L; + + /** CLI string of the learner to tune, e.g. {@code "moa.classifiers.trees.HoeffdingTree -g 50"}. */ + public final String algorithm; + + /** Tunable hyperparameters, in declaration order. */ + public final List parameters; + + public ConfigurationSpace(String algorithm, List parameters) { + this.algorithm = algorithm; + this.parameters = parameters; + } + + public static ConfigurationSpace fromFile(String path) throws IOException { + if (path == null || path.trim().isEmpty()) { + throw new IOException("No configuration file given. Set the -f option to a search space JSON file."); + } + File file = new File(path); + if (!file.isFile()) { + throw new IOException("Configuration file not found: " + file.getAbsolutePath()); + } + try (Reader reader = new FileReader(file)) { + return read(reader, file.getAbsolutePath()); + } + } + + public static ConfigurationSpace fromString(String json) throws IOException { + return read(new StringReader(json), ""); + } + + private static ConfigurationSpace read(Reader reader, String origin) throws IOException { + JsonElement rootElement = JsonParser.parseReader(reader); + if (rootElement == null || !rootElement.isJsonObject()) { + throw new IOException("Configuration in " + origin + " is not a JSON object."); + } + JsonObject root = rootElement.getAsJsonObject(); + + if (!root.has("algorithm")) { + throw new IOException("Configuration in " + origin + " has no \"algorithm\" entry."); + } + String algorithm = root.get("algorithm").getAsString(); + + List specs = new ArrayList<>(); + if (root.has("parameters")) { + JsonArray array = root.getAsJsonArray("parameters"); + for (int i = 0; i < array.size(); i++) { + specs.add(readParameter(array.get(i).getAsJsonObject(), origin, i)); + } + } + return new ConfigurationSpace(algorithm, specs); + } + + private static ParameterSpec readParameter(JsonObject node, String origin, int index) throws IOException { + ParameterSpec spec = new ParameterSpec(); + + if (!node.has("parameter")) { + throw new IOException("Parameter " + index + " in " + origin + " has no \"parameter\" name."); + } + spec.name = node.get("parameter").getAsString(); + + String type = node.has("type") ? node.get("type").getAsString().toLowerCase() : ""; + switch (type) { + case "integer": + case "int": + spec.type = ParameterSpec.TYPE_INT; + break; + case "double": + case "float": + case "real": + spec.type = ParameterSpec.TYPE_DOUBLE; + break; + case "categorical": + case "nominal": + spec.type = ParameterSpec.TYPE_CATEGORICAL; + break; + default: + throw new IOException("Parameter \"" + spec.name + "\" in " + origin + + " has unknown type \"" + type + "\"."); + } + + if (spec.type == ParameterSpec.TYPE_CATEGORICAL) { + if (!node.has("values")) { + throw new IOException("Categorical parameter \"" + spec.name + "\" in " + origin + + " has no \"values\" list."); + } + JsonArray values = node.getAsJsonArray("values"); + spec.values = new String[values.size()]; + for (int i = 0; i < values.size(); i++) { + spec.values[i] = values.get(i).getAsString(); + } + spec.active = node.has("active") ? node.get("active").getAsInt() : 0; + if (spec.active < 0 || spec.active >= spec.values.length) { + throw new IOException("Categorical parameter \"" + spec.name + "\" in " + origin + + " has \"active\" outside its \"values\" list."); + } + } else { + if (!node.has("range")) { + throw new IOException("Numeric parameter \"" + spec.name + "\" in " + origin + + " has no \"range\"."); + } + JsonArray range = node.getAsJsonArray("range"); + if (range.size() != 2) { + throw new IOException("Numeric parameter \"" + spec.name + "\" in " + origin + + " needs a \"range\" of exactly two entries."); + } + spec.range = new double[]{range.get(0).getAsDouble(), range.get(1).getAsDouble()}; + if (spec.range[0] > spec.range[1]) { + throw new IOException("Numeric parameter \"" + spec.name + "\" in " + origin + + " has an inverted \"range\"."); + } + spec.value = node.has("value") ? node.get("value").getAsDouble() : spec.range[0]; + if (node.has("step")) { + spec.step = node.get("step").getAsDouble(); + } + } + + if (node.has("onChange")) { + String onChange = node.get("onChange").getAsString().toLowerCase(); + switch (onChange) { + case "live": + spec.live = true; + break; + case "reset": + spec.live = false; + break; + default: + throw new IOException("Parameter \"" + spec.name + "\" in " + origin + + " has unknown \"onChange\" value \"" + onChange + "\"; expected live or reset."); + } + } + + return spec; + } + + /** Number of {@code integer} and {@code double} parameters, i.e. the surrogate feature count. */ + public int numericalCount() { + int count = 0; + for (ParameterSpec spec : this.parameters) { + if (spec.type != ParameterSpec.TYPE_CATEGORICAL) count++; + } + return count; + } + + /** Whether every hyperparameter can be pushed into an already-trained learner. */ + public boolean allLive() { + for (ParameterSpec spec : this.parameters) { + if (!spec.live) return false; + } + return true; + } + + public int size() { + return this.parameters.size(); + } + + public ParameterSpec get(int index) { + return this.parameters.get(index); + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/space/LearnerAccess.java b/moa/src/main/java/moa/classifiers/AutoML/space/LearnerAccess.java new file mode 100644 index 000000000..655da1698 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/space/LearnerAccess.java @@ -0,0 +1,157 @@ +/* + * LearnerAccess.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML.space; + +import com.yahoo.labs.samoa.instances.Instance; +import moa.classifiers.Classifier; +import moa.options.AbstractOptionHandler; +import moa.options.ClassOption; +import moa.options.OptionHandler; +import moa.options.OptionsHandler; +import moa.tasks.NullMonitor; + +import java.lang.reflect.Array; +import java.lang.reflect.Field; + +/** + * Read-only access to learner internals that MOA does not expose publicly, for + * the surrogate machinery of the adaptive and multi-fidelity methods. + * + *

The reference implementation obtained these by widening the visibility of + * fields inside {@code AdaptiveRandomForest} and + * {@code AdaptivePredictionInterval}. Doing it here instead keeps every learner + * exactly as MOA ships it, at the cost of a little reflection confined to this + * class - and, unlike a field widening, it degrades gracefully: a learner that + * does not look like an ensemble simply reports no member predictions, and the + * caller falls back to its interval-based uncertainty estimate. + * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public final class LearnerAccess { + + private static final Field CONFIG_FIELD = resolveConfigField(); + + private LearnerAccess() {} + + /** + * Per-member predictions of an ensemble learner, or {@code null} when + * {@code learner} does not expose an {@code ensemble} of sub-learners. + * Members that abstain are skipped, so the result may be shorter than the + * ensemble; a result of fewer than two entries carries no spread and is + * reported as {@code null}. + */ + public static double[] memberPredictions(Object learner, Instance inst) { + if (learner == null || inst == null) return null; + Object ensemble = readField(learner, "ensemble"); + if (ensemble == null || !ensemble.getClass().isArray()) return null; + + int length = Array.getLength(ensemble); + double[] predictions = new double[length]; + int count = 0; + for (int i = 0; i < length; i++) { + Classifier member = asClassifier(Array.get(ensemble, i)); + if (member == null) continue; + double[] votes = member.getVotesForInstance(inst); + if (votes == null || votes.length == 0 || Double.isNaN(votes[0])) continue; + predictions[count++] = votes[0]; + } + if (count < 2) return null; + double[] trimmed = new double[count]; + System.arraycopy(predictions, 0, trimmed, 0, count); + return trimmed; + } + + /** Standard deviation of the member predictions, or {@code NaN} when there are none. */ + public static double ensembleStd(Object learner, Instance inst) { + double[] predictions = memberPredictions(learner, inst); + if (predictions == null) return Double.NaN; + double mean = 0.0; + for (double p : predictions) mean += p / predictions.length; + double variance = 0.0; + for (double p : predictions) { + double d = p - mean; + variance += d * d / predictions.length; + } + return Math.sqrt(Math.max(0.0, variance)); + } + + /** + * Point an existing {@link ClassOption} at an object the caller already + * owns, so a wrapper can hand its own model to a learner that would + * otherwise build one from the option's CLI string. + * + * @return whether the option was found and set + */ + public static boolean setNestedLearner(OptionHandler owner, String optionName, Object learner) { + if (owner == null) return false; + com.github.javacliparser.Option option = owner.getOptions().getOption(optionName); + if (!(option instanceof ClassOption)) return false; + ((ClassOption) option).setCurrentObject(learner); + refreshClassOptions(owner); + return true; + } + + /** + * Rebuild the memo of materialised class options, so a freshly assigned + * {@link ClassOption} value is the one the learner goes on to use. + */ + public static void refreshClassOptions(OptionHandler owner) { + if (CONFIG_FIELD == null || !(owner instanceof AbstractOptionHandler)) return; + try { + Object config = CONFIG_FIELD.get(owner); + if (config instanceof OptionsHandler) { + ((OptionsHandler) config).prepareClassOptions(new NullMonitor(), null); + } + } catch (IllegalAccessException ignored) { + // Nothing to refresh; the caller's option write simply will not be seen. + } + } + + /** The member itself when it is a classifier, otherwise the classifier it wraps. */ + private static Classifier asClassifier(Object member) { + if (member instanceof Classifier) return (Classifier) member; + Object inner = readField(member, "classifier"); + return inner instanceof Classifier ? (Classifier) inner : null; + } + + private static Object readField(Object target, String name) { + if (target == null) return null; + for (Class c = target.getClass(); c != null && c != Object.class; c = c.getSuperclass()) { + try { + Field field = c.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } catch (NoSuchFieldException ignored) { + // keep walking up + } catch (RuntimeException | IllegalAccessException e) { + return null; + } + } + return null; + } + + private static Field resolveConfigField() { + try { + Field field = AbstractOptionHandler.class.getDeclaredField("config"); + field.setAccessible(true); + return field; + } catch (Throwable ignored) { + return null; + } + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/space/LearnerConfigurator.java b/moa/src/main/java/moa/classifiers/AutoML/space/LearnerConfigurator.java new file mode 100644 index 000000000..b95693f77 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/space/LearnerConfigurator.java @@ -0,0 +1,625 @@ +/* + * LearnerConfigurator.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML.space; + +import com.github.javacliparser.FlagOption; +import com.github.javacliparser.FloatOption; +import com.github.javacliparser.IntOption; +import com.github.javacliparser.MultiChoiceOption; +import com.github.javacliparser.Option; +import com.github.javacliparser.StringOption; +import moa.classifiers.AutoML.Parameters.CategoricalParameter; +import moa.classifiers.AutoML.Parameters.DoubleParameter; +import moa.classifiers.AutoML.Parameters.IntParameter; +import moa.classifiers.AutoML.Parameters.Parameter; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.Classifier; +import moa.options.AbstractOptionHandler; +import moa.options.ClassOption; +import moa.options.OptionHandler; +import moa.options.OptionsHandler; +import moa.tasks.NullMonitor; + +import java.io.Serializable; +import java.lang.reflect.Array; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +/** + * The single point of contact between an AutoML method and the learner it + * tunes. Every hyperparameter write in this package goes through here, and no + * learner in MOA needs to be modified to support it. + * + *

Hyperparameters are addressed as MOA {@link Option}s rather than as plain + * Java fields, so a value is set with e.g. {@code IntOption.setValue(250)} on + * the very {@code IntOption} instance the learner already dereferences. Two + * ways of getting a configuration into a learner are offered: + * + *

    + *
  • {@link #instantiate} - the cold path. Builds a fresh learner + * from the search space's CLI string, writes every option, and resets it. + * Correct for any learner and any hyperparameter, at the cost of the + * model state.
    • + *
  • {@link #applyLive} - the warm path. Pushes options into a + * learner that is already training, so a candidate can inherit the + * incumbent's model. Reaches nested learners by walking the object graph: + * on an {@code AdaptiveRandomForest} it writes {@code gracePeriod} on the + * {@code treeLearner} prototype and on all + * {@code ensemble[i].classifier} trees, which is what makes a warm-started + * forest actually honour a hyperparameter change.
    • + *
+ * + *

The warm path only works for hyperparameters the learner re-reads as it + * trains. Ones that are consumed once to derive structural state - see + * {@link ParameterSpec#live} - must be declared {@code "onChange": "reset"} in + * the search space and are routed to the cold path instead. + * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public class LearnerConfigurator implements Serializable { + + private static final long serialVersionUID = 1L; + + /** + * Depth cap for the object-graph walk. Sub-learners sit very close to the + * root - an {@code AdaptiveRandomForest} tree is at depth three, via + * {@code ensemble} then {@code ARFBaseLearner} then {@code classifier} - so + * this is generous while still bounding pathological graphs. + */ + private static final int MAX_DEPTH = 8; + + /** Hard cap on visited objects, a backstop against unforeseen graph shapes. */ + private static final int MAX_VISITS = 50000; + + /** + * Types the walk must not descend into: learner model internals, which can + * hold hundreds of thousands of objects and never contain a nested learner. + * Resolved by name so that a MOA build missing one of them still works. + */ + private static final String[] OPAQUE_TYPE_NAMES = { + "moa.classifiers.trees.HoeffdingTree$Node", + "moa.classifiers.trees.FIMTDD$Node", + "moa.classifiers.trees.EFDT$EFDTNode", + "moa.classifiers.core.attributeclassobservers.AttributeClassObserver", + "moa.core.DoubleVector", + "moa.core.GaussianEstimator", + "moa.core.GreenwaldKhannaQuantileSummary", + }; + + private static final List> OPAQUE_TYPES = resolveOpaqueTypes(); + + /** + * {@code AbstractOptionHandler.config} holds the {@link OptionsHandler} that + * memoises materialised {@link ClassOption} objects. Replacing a + * ClassOption's value leaves that memo stale, and MOA offers no public way + * to invalidate it, so this one protected field of MOA core is read + * reflectively. It is the only reflective access in this class, and it does + * not touch any learner-specific member. + */ + private static final Field CONFIG_FIELD = resolveConfigField(); + + private final ConfigurationSpace space; + + /** Cached per-class field lists for the object-graph walk. */ + private transient Map, Field[]> fieldCache; + + public LearnerConfigurator(ConfigurationSpace space) { + this.space = space; + } + + public ConfigurationSpace getSpace() { + return this.space; + } + + /** Whether every hyperparameter in the space may be pushed into a live learner. */ + public boolean canApplyLive() { + return this.space.allLive(); + } + + // ------------------------------------------------------------------ + // Construction + // ------------------------------------------------------------------ + + /** A fresh learner at the search space's base configuration, reset and ready to train. */ + public Classifier newLearner() { + Classifier learner = rawLearner(); + learner.resetLearning(); + return learner; + } + + /** As {@link #newLearner()}, with {@code seed} pushed into a randomizable learner. */ + public Classifier newLearner(int seed) { + Classifier learner = rawLearner(); + seed(learner, seed); + learner.resetLearning(); + return learner; + } + + private Classifier rawLearner() { + try { + return (Classifier) ClassOption.cliStringToObject(this.space.algorithm, Classifier.class, null); + } catch (Exception e) { + throw new IllegalStateException("Could not create the learner to tune from \"" + + this.space.algorithm + "\": " + e.getMessage(), e); + } + } + + /** + * Seeds a candidate, so that two candidates carrying identical + * hyperparameters still behave differently and the pool does not collapse + * into copies of one model. + * + *

Only learners that declare {@link Classifier#isRandomizable()} are + * seeded: {@code AbstractClassifier} creates its seed option, and reseeds + * {@code classifierRandom} from it in {@code resetLearning}, only under that + * flag, so seeding a non-randomizable learner would be silently discarded. + * This must run before {@code resetLearning}, which is what turns the + * seed into the learner's random stream. + */ + private static void seed(Classifier learner, int seed) { + if (learner.isRandomizable()) { + learner.setRandomSeed(seed); + } + } + + /** + * Reseeds a warm-started candidate in place, i.e. one obtained by copying + * an incumbent rather than by building it from the search space. + * + *

{@link #seed} is not enough on such a candidate: {@code setRandomSeed} + * only records the seed, and {@code AbstractClassifier} turns it into a + * random stream in {@code resetLearning} - which is precisely what warm + * starting avoids, since that would discard the learned state being reused. + * The generator is therefore replaced directly. + * + *

Without this, every candidate copied from one incumbent in the same + * evaluation window shares that incumbent's generator and its position + * in the stream, because MOA's {@code copy()} is serialization based. Two + * candidates carrying identical hyperparameters would then be exact + * duplicates and make identical random decisions, wasting a pool slot. + * + *

Only randomizable {@link AbstractClassifier}s expose a generator, so + * anything else is left untouched. The reseed applies to the learner itself, + * not to randomizable learners nested inside its {@link ClassOption}s. + */ + public static void reseedCopy(Classifier learner, int seed) { + if (learner instanceof AbstractClassifier && learner.isRandomizable()) { + AbstractClassifier randomizable = (AbstractClassifier) learner; + randomizable.setRandomSeed(seed); + randomizable.classifierRandom = new Random(seed); + } + } + + /** + * Builds a new learner carrying {@code params}. Used whenever a candidate + * must start from scratch, and as the fallback for hyperparameters that + * cannot be changed on a running learner. + */ + public Classifier instantiate(List params) { + Classifier learner = rawLearner(); + writeByPath(learner, params); + learner.resetLearning(); + return learner; + } + + /** As {@link #instantiate(List)}, with {@code seed} pushed into a randomizable learner. */ + public Classifier instantiate(List params, int seed) { + Classifier learner = rawLearner(); + writeByPath(learner, params); + seed(learner, seed); + learner.resetLearning(); + return learner; + } + + /** + * Writes {@code params} into {@code learner} by resolving each option path + * from the root. Nested learners are reached through the live objects held + * by their {@link ClassOption}s, so no CLI round trip is involved. + */ + private void writeByPath(Classifier learner, List params) { + for (int i = 0; i < params.size(); i++) { + Parameter param = params.get(i); + ParameterSpec spec = specFor(param, i); + OptionHandler owner = resolveOwner(learner, spec); + Option option = owner.getOptions().getOption(spec.optionName()); + if (option == null) { + throw new IllegalStateException("Learner " + owner.getClass().getName() + + " has no option named \"" + spec.optionName() + "\" (from parameter \"" + + spec.name + "\")."); + } + if (write(option, param) && option instanceof ClassOption) { + invalidateClassOptionCache(owner); + } + } + } + + /** Walks a path prefix such as {@code treeLearner/...} down to the learner that owns the option. */ + private OptionHandler resolveOwner(OptionHandler root, ParameterSpec spec) { + OptionHandler current = root; + for (String segment : spec.pathPrefix()) { + Option option = current.getOptions().getOption(segment); + if (!(option instanceof ClassOption)) { + throw new IllegalStateException("Path segment \"" + segment + "\" of parameter \"" + + spec.name + "\" is not a class option of " + + current.getClass().getName() + "."); + } + Object nested = ((ClassOption) option).getPreMaterializedObject(); + if (!(nested instanceof OptionHandler)) { + throw new IllegalStateException("Path segment \"" + segment + "\" of parameter \"" + + spec.name + "\" does not resolve to a configurable object."); + } + current = (OptionHandler) nested; + } + return current; + } + + // ------------------------------------------------------------------ + // Live reconfiguration + // ------------------------------------------------------------------ + + /** + * Pushes {@code params} into an already-training learner, returning the + * number of options actually written. + * + *

Options are matched by leaf name across the whole reachable object + * graph, not by path. That is deliberate: an ensemble's sub-learners are + * private copies of the prototype held by its {@link ClassOption}, so a + * path-scoped write would update the prototype and leave every tree already + * in the ensemble at its old setting. Matching by name reaches both. + * + *

A return value of zero means the learner ignored the configuration + * entirely, which is worth surfacing rather than silently tolerating. + */ + public int applyLive(Object learner, List params) { + if (learner == null || params == null || params.isEmpty()) return 0; + + Map byName = new HashMap<>(); + for (int i = 0; i < params.size(); i++) { + Parameter param = params.get(i); + ParameterSpec spec = specFor(param, i); + if (!spec.live) continue; + byName.put(spec.optionName(), param); + } + if (byName.isEmpty()) return 0; + + int written = 0; + IdentityHashMap seen = new IdentityHashMap<>(); + ArrayDeque queue = new ArrayDeque<>(); + ArrayDeque depths = new ArrayDeque<>(); + queue.add(learner); + depths.add(0); + int visits = 0; + + while (!queue.isEmpty() && visits < MAX_VISITS) { + Object node = queue.poll(); + int depth = depths.poll(); + if (node == null || seen.put(node, Boolean.TRUE) != null) continue; + visits++; + + if (node instanceof OptionHandler) { + OptionHandler handler = (OptionHandler) node; + boolean classOptionWritten = false; + for (Option option : handler.getOptions().getOptionArray()) { + Parameter param = byName.get(option.getName()); + if (param != null && write(option, param)) { + written++; + classOptionWritten |= option instanceof ClassOption; + } + // Descend through nested learners held by class options: this + // is how an ensemble's prototype gets reconfigured. + if (depth < MAX_DEPTH && option instanceof ClassOption) { + Object nested = ((ClassOption) option).getPreMaterializedObject(); + if (shouldVisit(nested)) { + queue.add(nested); + depths.add(depth + 1); + } + } + } + if (classOptionWritten) invalidateClassOptionCache(handler); + } + + if (depth >= MAX_DEPTH) continue; + for (Object child : childrenOf(node)) { + if (shouldVisit(child)) { + queue.add(child); + depths.add(depth + 1); + } + } + } + return written; + } + + /** Single-parameter convenience used by the per-candidate mutation step. */ + public int applyLive(Object learner, Parameter param) { + List one = new ArrayList<>(1); + one.add(param); + return applyLive(learner, one); + } + + // ------------------------------------------------------------------ + // Option writing + // ------------------------------------------------------------------ + + /** Writes one parameter value onto one option, returning whether it applied. */ + private static boolean write(Option option, Parameter param) { + switch (param.type) { + case ParameterSpec.TYPE_INT: { + int value = ((IntParameter) param).value; + if (option instanceof IntOption) { + ((IntOption) option).setValue(value); + return true; + } + if (option instanceof FloatOption) { + ((FloatOption) option).setValue(value); + return true; + } + return false; + } + case ParameterSpec.TYPE_DOUBLE: { + double value = ((DoubleParameter) param).value; + if (option instanceof FloatOption) { + ((FloatOption) option).setValue(value); + return true; + } + if (option instanceof IntOption) { + ((IntOption) option).setValue((int) Math.round(value)); + return true; + } + return false; + } + case ParameterSpec.TYPE_CATEGORICAL: { + CategoricalParameter categorical = (CategoricalParameter) param; + String value = categorical.values[categorical.active]; + if (option instanceof ClassOption) { + ((ClassOption) option).setValueViaCLIString(value); + return true; + } + if (option instanceof MultiChoiceOption) { + ((MultiChoiceOption) option).setChosenLabel(value); + return true; + } + if (option instanceof FlagOption) { + ((FlagOption) option).setValue(Boolean.parseBoolean(value)); + return true; + } + if (option instanceof StringOption) { + ((StringOption) option).setValue(value); + return true; + } + return false; + } + default: + return false; + } + } + + /** + * Drops the memo of materialised class options so the learner picks up a + * newly written {@link ClassOption} value. Only called after such a write. + */ + private static void invalidateClassOptionCache(OptionHandler handler) { + if (CONFIG_FIELD == null || !(handler instanceof AbstractOptionHandler)) return; + try { + Object config = CONFIG_FIELD.get(handler); + if (config instanceof OptionsHandler) { + ((OptionsHandler) config).prepareClassOptions(new NullMonitor(), null); + } + } catch (IllegalAccessException ignored) { + // Without the memo refresh a categorical change would not be seen; + // validate() reports this up front rather than failing silently here. + } + } + + // ------------------------------------------------------------------ + // Validation + // ------------------------------------------------------------------ + + /** + * Checks the search space against the learner before any training starts: + * every option path must resolve, every option must accept the declared + * type, and both ends of every numeric range must be within the bounds MOA + * declares for that option. Fails loudly here rather than thousands of + * instances into a run. + */ + public void validate() { + Classifier probe = rawLearner(); + + if (CONFIG_FIELD == null) { + for (ParameterSpec spec : this.space.parameters) { + if (spec.type == ParameterSpec.TYPE_CATEGORICAL) { + throw new IllegalStateException("Cannot tune categorical parameter \"" + + spec.name + "\": this MOA build does not expose " + + "AbstractOptionHandler.config, so class option changes could not take effect."); + } + } + } + + for (ParameterSpec spec : this.space.parameters) { + OptionHandler owner = resolveOwner(probe, spec); + Option option = owner.getOptions().getOption(spec.optionName()); + if (option == null) { + throw new IllegalStateException("Parameter \"" + spec.name + "\": " + + owner.getClass().getName() + " has no option named \"" + spec.optionName() + + "\". Use the MOA option name, not the Java field name -" + + " for instance \"gracePeriod\", not \"gracePeriodOption\"."); + } + switch (spec.type) { + case ParameterSpec.TYPE_INT: + case ParameterSpec.TYPE_DOUBLE: + if (!(option instanceof IntOption) && !(option instanceof FloatOption)) { + throw new IllegalStateException("Parameter \"" + spec.name + + "\" is declared numeric but option \"" + spec.optionName() + "\" of " + + owner.getClass().getName() + " is a " + option.getClass().getSimpleName() + "."); + } + checkBound(option, spec, spec.range[0]); + checkBound(option, spec, spec.range[1]); + break; + case ParameterSpec.TYPE_CATEGORICAL: + if (!(option instanceof ClassOption) && !(option instanceof MultiChoiceOption) + && !(option instanceof FlagOption) && !(option instanceof StringOption)) { + throw new IllegalStateException("Parameter \"" + spec.name + + "\" is declared categorical but option \"" + spec.optionName() + "\" of " + + owner.getClass().getName() + " is a " + option.getClass().getSimpleName() + "."); + } + for (String value : spec.values) { + try { + CategoricalParameter probeParam = + new CategoricalParameter(spec.optionName(), new String[]{value}, 0, null); + write(option, probeParam); + } catch (RuntimeException e) { + throw new IllegalStateException("Parameter \"" + spec.name + + "\" cannot take value \"" + value + "\": " + e.getMessage(), e); + } + } + break; + default: + break; + } + } + } + + private static void checkBound(Option option, ParameterSpec spec, double bound) { + try { + if (option instanceof IntOption) { + ((IntOption) option).setValue((int) Math.round(bound)); + } else { + ((FloatOption) option).setValue(bound); + } + } catch (RuntimeException e) { + throw new IllegalStateException("Parameter \"" + spec.name + "\" has a range reaching " + + bound + ", which the learner rejects: " + e.getMessage(), e); + } + } + + private ParameterSpec specFor(Parameter param, int index) { + if (index < this.space.size()) { + ParameterSpec spec = this.space.get(index); + if (spec.name.equals(param.name) || spec.optionName().equals(param.name)) { + return spec; + } + } + for (ParameterSpec spec : this.space.parameters) { + if (spec.name.equals(param.name) || spec.optionName().equals(param.name)) { + return spec; + } + } + throw new IllegalStateException("No search space entry for parameter \"" + param.name + "\"."); + } + + // ------------------------------------------------------------------ + // Object-graph walk + // ------------------------------------------------------------------ + + private boolean shouldVisit(Object object) { + if (object == null) return false; + Class type = object.getClass(); + if (type.isArray()) return !type.getComponentType().isPrimitive(); + if (object instanceof Option) return false; + if (object instanceof Collection || object instanceof Map) return true; + if (object instanceof OptionHandler) return true; + + String name = type.getName(); + if (!name.startsWith("moa.")) return false; + for (Class opaque : OPAQUE_TYPES) { + if (opaque.isInstance(object)) return false; + } + return true; + } + + /** Object-valued members of {@code node} that the walk should consider. */ + private Iterable childrenOf(Object node) { + List children = new ArrayList<>(); + Class type = node.getClass(); + + if (type.isArray()) { + int length = Array.getLength(node); + for (int i = 0; i < length; i++) children.add(Array.get(node, i)); + return children; + } + if (node instanceof Collection) { + children.addAll((Collection) node); + return children; + } + if (node instanceof Map) { + children.addAll(((Map) node).values()); + return children; + } + + for (Field field : fieldsOf(type)) { + try { + children.add(field.get(node)); + } catch (IllegalAccessException | RuntimeException ignored) { + // Inaccessible members simply do not take part in the walk. + } + } + return children; + } + + private Field[] fieldsOf(Class type) { + if (this.fieldCache == null) this.fieldCache = new HashMap<>(); + Field[] cached = this.fieldCache.get(type); + if (cached != null) return cached; + + List fields = new ArrayList<>(); + for (Class current = type; current != null && current != Object.class; current = current.getSuperclass()) { + for (Field field : current.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers())) continue; + if (field.getType().isPrimitive()) continue; + try { + field.setAccessible(true); + } catch (RuntimeException e) { + continue; + } + fields.add(field); + } + } + cached = fields.toArray(new Field[0]); + this.fieldCache.put(type, cached); + return cached; + } + + private static List> resolveOpaqueTypes() { + List> types = new ArrayList<>(); + for (String name : OPAQUE_TYPE_NAMES) { + try { + types.add(Class.forName(name)); + } catch (Throwable ignored) { + // Absent in this build; nothing to exclude. + } + } + return types; + } + + private static Field resolveConfigField() { + try { + Field field = AbstractOptionHandler.class.getDeclaredField("config"); + field.setAccessible(true); + return field; + } catch (Throwable ignored) { + return null; + } + } +} diff --git a/moa/src/main/java/moa/classifiers/AutoML/space/ParameterSpec.java b/moa/src/main/java/moa/classifiers/AutoML/space/ParameterSpec.java new file mode 100644 index 000000000..644b2275c --- /dev/null +++ b/moa/src/main/java/moa/classifiers/AutoML/space/ParameterSpec.java @@ -0,0 +1,126 @@ +/* + * ParameterSpec.java + * Copyright (C) 2026 University of Waikato, Hamilton, New Zealand + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.AutoML.space; + +import moa.classifiers.AutoML.Parameters.Parameter; + +import java.io.Serializable; +import java.util.Arrays; + +/** + * Static description of one tunable hyperparameter, as read from the search + * space JSON. This is the immutable half of a hyperparameter: the mutable + * per-candidate value lives in {@link moa.classifiers.AutoML.Parameters.Parameter}. + * + *

The {@code parameter} field is an option path. A bare name such as + * {@code "gracePeriod"} refers to an option of the learner itself; a slash + * separated path such as {@code "treeLearner/gracePeriod"} refers to an option + * of a nested learner reached through a {@link moa.options.ClassOption}. The + * leaf segment is always the MOA option name, i.e. the first argument given to + * the {@code IntOption} / {@code FloatOption} / {@code ClassOption} constructor + * in the learner - not the Java field name, which conventionally carries an + * {@code Option} suffix. + * + * @author Daniel Nowak Assis (daniel dot nowak-assis at lip6 dot fr) + */ +public class ParameterSpec implements Serializable { + + private static final long serialVersionUID = 1L; + + /** Matches {@link moa.classifiers.AutoML.Parameters.IntParameter}. */ + public static final int TYPE_INT = Parameter.TYPE_INT; + + /** Matches {@link moa.classifiers.AutoML.Parameters.DoubleParameter}. */ + public static final int TYPE_DOUBLE = Parameter.TYPE_DOUBLE; + + /** Matches {@link moa.classifiers.AutoML.Parameters.CategoricalParameter}. */ + public static final int TYPE_CATEGORICAL = Parameter.TYPE_CATEGORICAL; + + /** + * Option path, e.g. {@code "gracePeriod"} or {@code "treeLearner/gracePeriod"}. + * Read from the {@code "parameter"} entry of the search space JSON. + */ + public String name; + + /** One of {@link #TYPE_INT}, {@link #TYPE_DOUBLE}, {@link #TYPE_CATEGORICAL}. */ + public int type; + + /** Initial value for numeric parameters. */ + public double value; + + /** Inclusive {@code [low, high]} bounds for numeric parameters. */ + public double[] range; + + /** Optional step hint used by the local-search methods; {@code NaN} when absent. */ + public double step = Double.NaN; + + /** Candidate CLI strings for categorical parameters. */ + public String[] values; + + /** Index into {@link #values} used as the initial setting. */ + public int active; + + /** + * Whether this hyperparameter can be pushed into an already-trained learner + * ({@code "onChange": "live"}, the default) or requires the candidate to be + * rebuilt from scratch ({@code "onChange": "reset"}). + * + *

Declare {@code reset} whenever the learner only consumes the option + * once, to derive some structural quantity it will not recompute: MOA + * examples are {@code AdaptiveRandomForest.mFeaturesPerTreeSize}, which is + * turned into {@code subspaceSize} in {@code initEnsemble}, and + * {@code kNN.limit}, which sizes the sliding window on first use. Writing + * the option after that point is silently ignored by the learner, so a + * {@code live} declaration there would freeze the search. + */ + public boolean live = true; + + /** Leaf segment of {@link #name}, i.e. the MOA option name. */ + public String optionName() { + int slash = this.name.lastIndexOf('/'); + return slash < 0 ? this.name : this.name.substring(slash + 1); + } + + /** Path segments leading to the owning learner, empty when the option is on the root. */ + public String[] pathPrefix() { + int slash = this.name.lastIndexOf('/'); + if (slash < 0) return new String[0]; + return this.name.substring(0, slash).split("/"); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("ParameterSpec [").append(this.name).append(", "); + switch (this.type) { + case TYPE_INT: + sb.append("integer, value=").append((int) this.value) + .append(", range=").append(Arrays.toString(new int[]{(int) this.range[0], (int) this.range[1]})); + break; + case TYPE_DOUBLE: + sb.append("double, value=").append(this.value) + .append(", range=").append(Arrays.toString(this.range)); + break; + default: + sb.append("categorical, active=").append(this.active) + .append(", values=").append(Arrays.toString(this.values)); + } + sb.append(this.live ? ", live]" : ", reset]"); + return sb.toString(); + } +} diff --git a/moa/src/main/java/moa/classifiers/functions/BayesianLinearRegression.java b/moa/src/main/java/moa/classifiers/functions/BayesianLinearRegression.java new file mode 100644 index 000000000..c67d58279 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/functions/BayesianLinearRegression.java @@ -0,0 +1,324 @@ +/* + * BayesianLinearRegression.java + * Port of River's BayesianLinearRegression to MOA. + * Based on Bishop's Pattern Recognition and Machine Learning (2006), equations 3.50-3.59. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package moa.classifiers.functions; + +import com.github.javacliparser.FloatOption; +import com.yahoo.labs.samoa.instances.Instance; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.Regressor; +import moa.core.Measurement; +import moa.core.StringUtils; + + +public class BayesianLinearRegression extends AbstractClassifier implements Regressor { + + private static final long serialVersionUID = 1L; + + @Override + public String getPurposeString() { + return "Bayesian linear regression. Does not require feature scaling. " + + "Supports concept drift via the smoothing parameter. " + + "Port of River's BayesianLinearRegression."; + } + + public FloatOption alphaOption = new FloatOption("alpha", 'a', + "Prior precision. Controls the strength of the Gaussian prior over the weights.", + 1.0, 1e-10, Double.MAX_VALUE); + + public FloatOption betaOption = new FloatOption("beta", 'b', + "Noise precision (inverse of the assumed noise variance).", + 1.0, 1e-10, Double.MAX_VALUE); + + public FloatOption smoothingOption = new FloatOption("smoothing", 's', + "Smoothing factor in (0, 1) for concept drift adaptation. " + + "Set to 0 (default) to disable smoothing and use the fast Sherman-Morrison update. " + + "A value such as 0.8 makes the model gradually forget older observations.", + 0.0, 0.0, 1.0); + + // Posterior precision matrix S_N (numFeatures x numFeatures) + private double[][] S; + + // Posterior covariance matrix S_N^{-1} + private double[][] Sinv; + + // Posterior mean vector m_N + private double[] m; + + // Number of input features (excluding class attribute) + private int numFeatures; + + @Override + public void resetLearningImpl() { + S = null; + Sinv = null; + m = null; + numFeatures = 0; + } + + /** Initialises S, S_inv, and m for n features using the configured alpha. */ + private void initModel(int n) { + double alpha = alphaOption.getValue(); + numFeatures = n; + S = new double[n][n]; + Sinv = new double[n][n]; + m = new double[n]; + // Prior: S_0 = alpha * I, S_0^{-1} = (1/alpha) * I + for (int i = 0; i < n; i++) { + S[i][i] = alpha; + Sinv[i][i] = 1.0 / alpha; + } + } + + /** Extracts the input feature values from an instance, skipping the class attribute. */ + private double[] featureVector(Instance inst) { + int n = inst.numAttributes() - 1; + double[] x = new double[n]; + int idx = 0; + for (int i = 0; i < inst.numAttributes(); i++) { + if (i != inst.classIndex()) { + double v = inst.value(i); + x[idx++] = Double.isNaN(v) ? 0.0 : v; + } + } + return x; + } + + @Override + public void trainOnInstanceImpl(Instance inst) { + if (inst.classIsMissing()) return; + + double[] x = featureVector(inst); + int n = x.length; + if (n == 0) return; + + if (S == null) initModel(n); + + double y = inst.classValue(); + double beta = betaOption.getValue(); + double smoothing = smoothingOption.getValue(); + + // beta * x + double[] bx = scale(x, beta); + + // S * m (needed in both branches) + double[] Sm = matVec(S, m); + + if (smoothing <= 0.0) { + // ---- No smoothing: Sherman-Morrison rank-1 update ---- + // + // S_N = S_{N-1} + beta * x * x^T (Bishop eq. 3.51) + // S_N^-1 updated via Sherman-Morrison: + // S_N^-1 = S_{N-1}^-1 + // - (S_{N-1}^-1 * bx)(x^T * S_{N-1}^-1) + // / (1 + x^T * S_{N-1}^-1 * bx) + // m_N = S_N^-1 * (S_{N-1} * m_{N-1} + beta * y * x) (eq. 3.50) + + double[] Sinv_bx = matVec(Sinv, bx); // S^-1 * (beta*x) + double[] xT_Sinv = vecMat(x, Sinv); // x^T * S^-1 + double denom = 1.0 + dot(x, Sinv_bx); + + for (int i = 0; i < n; i++) + for (int j = 0; j < n; j++) + Sinv[i][j] -= Sinv_bx[i] * xT_Sinv[j] / denom; + + double[] rhs = new double[n]; + for (int i = 0; i < n; i++) + rhs[i] = Sm[i] + bx[i] * y; + m = matVec(Sinv, rhs); + + for (int i = 0; i < n; i++) + for (int j = 0; j < n; j++) + S[i][j] += bx[i] * x[j]; + + } else { + // ---- With smoothing: full matrix inversion ---- + // + // S_N = smoothing * S_{N-1} + (1-smoothing) * beta * x * x^T + // S_N^-1 = inv(S_N) + // m_N = S_N^-1 * (smoothing * S_{N-1} * m_{N-1} + // + (1-smoothing) * beta * y * x) + + double[][] newS = new double[n][n]; + for (int i = 0; i < n; i++) + for (int j = 0; j < n; j++) + newS[i][j] = smoothing * S[i][j] + (1.0 - smoothing) * bx[i] * x[j]; + + double[][] newSinv = invertMatrix(newS); + + double[] rhs = new double[n]; + for (int i = 0; i < n; i++) + rhs[i] = smoothing * Sm[i] + (1.0 - smoothing) * bx[i] * y; + m = matVec(newSinv, rhs); + + S = newS; + Sinv = newSinv; + } + } + + /** + * Returns the predicted value as the posterior mean estimate (Bishop eq. 3.58): + * y_hat = m_N^T * x + */ + @Override + public double[] getVotesForInstance(Instance inst) { + if (m == null) return new double[]{0.0}; + double[] x = featureVector(inst); + return new double[]{dot(m, x)}; + } + + /** + * Returns {mean, std} of the predictive distribution (Bishop eq. 3.58-3.59): + * mean = m_N^T * x + * var = 1/beta + x^T * S_N^{-1} * x + */ + public double[] predictWithVariance(Instance inst) { + if (m == null) return new double[]{0.0, 1.0}; + double[] x = featureVector(inst); + double mean = dot(m, x); + double[] Sinv_x = matVec(Sinv, x); + double variance = 1.0 / betaOption.getValue() + dot(x, Sinv_x); + return new double[]{mean, Math.sqrt(Math.max(0.0, variance))}; + } + + // ---- Matrix / vector helpers ---- + + /** Matrix-vector product: A * v */ + private double[] matVec(double[][] A, double[] v) { + int n = v.length; + double[] r = new double[n]; + for (int i = 0; i < n; i++) + for (int j = 0; j < n; j++) + r[i] += A[i][j] * v[j]; + return r; + } + + /** Row-vector-matrix product: v^T * A (returns row vector) */ + private double[] vecMat(double[] v, double[][] A) { + int n = v.length; + double[] r = new double[n]; + for (int j = 0; j < n; j++) + for (int i = 0; i < n; i++) + r[j] += v[i] * A[i][j]; + return r; + } + + /** Dot product of two vectors. */ + private double dot(double[] a, double[] b) { + double s = 0.0; + for (int i = 0; i < a.length; i++) s += a[i] * b[i]; + return s; + } + + /** Returns a new vector: scalar * v. */ + private double[] scale(double[] v, double scalar) { + double[] r = new double[v.length]; + for (int i = 0; i < v.length; i++) r[i] = scalar * v[i]; + return r; + } + + /** + * Inverts a square matrix using Gauss-Jordan elimination with partial pivoting. + * Returns the identity matrix if the input is (near-)singular. + */ + private double[][] invertMatrix(double[][] A) { + int n = A.length; + // Build augmented matrix [A | I] + double[][] aug = new double[n][2 * n]; + for (int i = 0; i < n; i++) { + System.arraycopy(A[i], 0, aug[i], 0, n); + aug[i][n + i] = 1.0; + } + + for (int col = 0; col < n; col++) { + // Partial pivot + int pivotRow = col; + for (int row = col + 1; row < n; row++) + if (Math.abs(aug[row][col]) > Math.abs(aug[pivotRow][col])) + pivotRow = row; + double[] tmp = aug[col]; aug[col] = aug[pivotRow]; aug[pivotRow] = tmp; + + double pivotVal = aug[col][col]; + if (Math.abs(pivotVal) < 1e-15) { + // Singular column — leave as-is (graceful degradation) + continue; + } + + // Scale pivot row + double inv = 1.0 / pivotVal; + for (int j = col; j < 2 * n; j++) aug[col][j] *= inv; + + // Eliminate column in all other rows + for (int row = 0; row < n; row++) { + if (row == col) continue; + double factor = aug[row][col]; + if (factor == 0.0) continue; + for (int j = col; j < 2 * n; j++) + aug[row][j] -= factor * aug[col][j]; + } + } + + // Extract right half as the inverse + double[][] inv = new double[n][n]; + for (int i = 0; i < n; i++) + System.arraycopy(aug[i], n, inv[i], 0, n); + return inv; + } + + // ---- MOA bookkeeping ---- + + @Override + protected Measurement[] getModelMeasurementsImpl() { + return null; + } + + @Override + public void getModelDescription(StringBuilder out, int indent) { + StringUtils.appendIndented(out, indent, "Bayesian Linear Regression"); + StringUtils.appendNewline(out); + if (m == null) { + StringUtils.appendIndented(out, indent, " Model not yet trained."); + StringUtils.appendNewline(out); + return; + } + StringUtils.appendIndented(out, indent, String.format( + " alpha=%.4f beta=%.4f smoothing=%.4f%n", + alphaOption.getValue(), betaOption.getValue(), smoothingOption.getValue())); + + StringUtils.appendIndented(out, indent, " Posterior mean weights (m_N):"); + StringUtils.appendNewline(out); + // Attempt to print feature names from model context when available + com.yahoo.labs.samoa.instances.InstancesHeader header = getModelContext(); + for (int i = 0, fi = 0; header != null && i < header.numAttributes(); i++) { + if (i == header.classIndex()) continue; + StringUtils.appendIndented(out, indent, + String.format(" %s: %.6f%n", header.attribute(i).name(), m[fi++])); + } + if (header == null) { + for (int i = 0; i < m.length; i++) { + StringUtils.appendIndented(out, indent, + String.format(" w[%d]: %.6f%n", i, m[i])); + } + } + } + + @Override + public boolean isRandomizable() { + return false; + } +} \ No newline at end of file From efdbeebe9069c597b75fe94512eb7c74b8194e01 Mon Sep 17 00:00:00 2001 From: Daniel Nowak Date: Thu, 30 Jul 2026 07:05:52 +0200 Subject: [PATCH 2/4] search space for capymoa --- .../AutoML/BayesianStreamTunerClassifier.java | 9 +++- .../AutoML/BayesianStreamTunerRegressor.java | 9 +++- .../classifiers/AutoML/FSS_SPTClassifier.java | 53 ++++++++++++++++++- .../classifiers/AutoML/FSS_SPTRegressor.java | 9 +++- .../classifiers/AutoML/MESSPTClassifier.java | 9 +++- .../classifiers/AutoML/MESSPTRegressor.java | 9 +++- .../moa/classifiers/AutoML/MetricUtils.java | 6 +++ .../AutoML/RandomSearchClassifier.java | 9 +++- .../AutoML/RandomSearchRegressor.java | 9 +++- .../classifiers/AutoML/SSPTClassifier.java | 11 ++-- .../moa/classifiers/AutoML/SSPTRegressor.java | 11 ++-- .../AutoML/space/ConfigurationSpace.java | 29 +++++++++- 12 files changed, 150 insertions(+), 23 deletions(-) diff --git a/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerClassifier.java b/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerClassifier.java index 71875ccd3..3375a8da6 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerClassifier.java +++ b/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerClassifier.java @@ -21,6 +21,7 @@ import com.github.javacliparser.FlagOption; import com.github.javacliparser.IntOption; import com.github.javacliparser.MultiChoiceOption; +import com.github.javacliparser.StringOption; import com.yahoo.labs.samoa.instances.Attribute; import com.yahoo.labs.samoa.instances.DenseInstance; import com.yahoo.labs.samoa.instances.Instance; @@ -80,6 +81,10 @@ public class BayesianStreamTunerClassifier extends AbstractClassifier public FileOption configurationFileOption = new FileOption("configurationFile", 'f', "Search space in JSON format.", null, ".json", false); + public StringOption searchSpaceOption = new StringOption("searchSpace", 's', + "Search space as inline JSON. Takes precedence over configurationFile when set," + + " so that a caller holding the space in memory need not write a file.", ""); + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', "Number of instances between model update cycles; also the number of recent" + " instances kept to derive the surrogate's stream statistics.", @@ -216,7 +221,7 @@ public void resetLearningImpl() { public void setConfigurations() { try { - ConfigurationSpace space = ConfigurationSpace.fromFile(configurationFileOption.getValue()); + ConfigurationSpace space = ConfigurationSpace.resolve(configurationFileOption.getValue(), searchSpaceOption.getValue()); this.configurator = new LearnerConfigurator(space); this.configurator.validate(); @@ -304,7 +309,7 @@ public void setConfigurations() { // Fail fast: a swallowed error here leaves candidates/surrogate null and // surfaces later as a confusing NPE far from the real cause. throw new RuntimeException( - "Failed to load tuner configuration from " + configurationFileOption.getValue(), e); + "Failed to load tuner configuration from " + ConfigurationSpace.describeSource(configurationFileOption.getValue(), searchSpaceOption.getValue()), e); } } diff --git a/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerRegressor.java b/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerRegressor.java index 739ed30d4..a36de275f 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerRegressor.java +++ b/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerRegressor.java @@ -21,6 +21,7 @@ import com.github.javacliparser.FlagOption; import com.github.javacliparser.IntOption; import com.github.javacliparser.MultiChoiceOption; +import com.github.javacliparser.StringOption; import com.yahoo.labs.samoa.instances.Attribute; import com.yahoo.labs.samoa.instances.DenseInstance; import com.yahoo.labs.samoa.instances.Instance; @@ -75,6 +76,10 @@ public class BayesianStreamTunerRegressor extends AbstractClassifier public FileOption configurationFileOption = new FileOption("configurationFile", 'f', "Search space in JSON format.", null, ".json", false); + public StringOption searchSpaceOption = new StringOption("searchSpace", 's', + "Search space as inline JSON. Takes precedence over configurationFile when set," + + " so that a caller holding the space in memory need not write a file.", ""); + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', "Number of instances between model update cycles; also the number of recent" + " instances kept to derive the surrogate's stream statistics.", @@ -211,7 +216,7 @@ public void resetLearningImpl() { public void setConfigurations() { try { - ConfigurationSpace space = ConfigurationSpace.fromFile(configurationFileOption.getValue()); + ConfigurationSpace space = ConfigurationSpace.resolve(configurationFileOption.getValue(), searchSpaceOption.getValue()); this.configurator = new LearnerConfigurator(space); this.configurator.validate(); @@ -297,7 +302,7 @@ public void setConfigurations() { // Fail fast: a swallowed error here leaves candidates/surrogate null and // surfaces later as a confusing NPE far from the real cause. throw new RuntimeException( - "Failed to load tuner configuration from " + configurationFileOption.getValue(), e); + "Failed to load tuner configuration from " + ConfigurationSpace.describeSource(configurationFileOption.getValue(), searchSpaceOption.getValue()), e); } } diff --git a/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTClassifier.java b/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTClassifier.java index a21dd6804..e472220ee 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTClassifier.java +++ b/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTClassifier.java @@ -22,6 +22,7 @@ import com.github.javacliparser.FloatOption; import com.github.javacliparser.IntOption; import com.github.javacliparser.MultiChoiceOption; +import com.github.javacliparser.StringOption; import com.yahoo.labs.samoa.instances.Instance; import moa.capabilities.CapabilitiesHandler; import moa.classifiers.AbstractClassifier; @@ -69,6 +70,10 @@ public class FSS_SPTClassifier extends AbstractClassifier implements MultiClassC public FileOption configurationFileOption = new FileOption("configurationFile", 'f', "Search space in JSON format.", null, ".json", false); + public StringOption searchSpaceOption = new StringOption("searchSpace", 's', + "Search space as inline JSON. Takes precedence over configurationFile when set," + + " so that a caller holding the space in memory need not write a file.", ""); + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', "Number of instances between FSS school updates.", 1000, 2, Integer.MAX_VALUE); @@ -228,12 +233,12 @@ public void resetLearningImpl() { @Override public void setConfigurations() { try { - this.space = ConfigurationSpace.fromFile(configurationFileOption.getValue()); + this.space = ConfigurationSpace.resolve(configurationFileOption.getValue(), searchSpaceOption.getValue()); this.configurator = new LearnerConfigurator(this.space); this.configurator.validate(); } catch (Exception e) { throw new IllegalStateException("Could not set up " + getClass().getSimpleName() - + " from \"" + configurationFileOption.getValue() + "\": " + e.getMessage(), e); + + " from " + ConfigurationSpace.describeSource(configurationFileOption.getValue(), searchSpaceOption.getValue()) + ": " + e.getMessage(), e); } } @@ -659,6 +664,50 @@ public long measureByteSize() { return size; } + @Override + public double getCandidateScore(int i) { + return school[i].getMetric(metricOption.getChosenIndex()); + } + + @Override + public double getClassifierScore() { + return getBestFish().getMetric(metricOption.getChosenIndex()); + } + + @Override + public int getNumberOfCandidates() { return numEstimatorsOption.getValue(); } + + @Override + public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } + + @Override + public int getEvaluationInstancesCount() { return evaluationInstances; } + + @Override + public int getGracePeriod() { return gracePeriodOption.getValue(); } + + @Override + public Classifier getMainClassifier() { return getBestFish().model; } + + @Override + public ArrayList getReferenceParameters() { + return (school != null && school.length > 0) ? getBestFish().params : null; + } + + @Override + public String getConfigurationFile() { return this.configurationFileOption.getValue(); } + + @Override + public ArrayList> getCandidateParameters() { + ArrayList> list = new ArrayList<>(); + if (school != null) { + for (FishEntry f : school) { + list.add(f.params); + } + } + return list; + } + private int argmax(double[] arr) { int best = 0; diff --git a/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTRegressor.java b/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTRegressor.java index 9af73c11f..e8523ce7c 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTRegressor.java +++ b/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTRegressor.java @@ -22,6 +22,7 @@ import com.github.javacliparser.FloatOption; import com.github.javacliparser.IntOption; import com.github.javacliparser.MultiChoiceOption; +import com.github.javacliparser.StringOption; import com.yahoo.labs.samoa.instances.Instance; import moa.capabilities.CapabilitiesHandler; import moa.classifiers.AbstractClassifier; @@ -64,6 +65,10 @@ public class FSS_SPTRegressor extends AbstractClassifier implements Regressor, public FileOption configurationFileOption = new FileOption("configurationFile", 'f', "Search space in JSON format.", null, ".json", false); + public StringOption searchSpaceOption = new StringOption("searchSpace", 's', + "Search space as inline JSON. Takes precedence over configurationFile when set," + + " so that a caller holding the space in memory need not write a file.", ""); + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', "Number of instances between FSS school updates.", 1000, 2, Integer.MAX_VALUE); @@ -226,12 +231,12 @@ public void resetLearningImpl() { @Override public void setConfigurations() { try { - this.space = ConfigurationSpace.fromFile(configurationFileOption.getValue()); + this.space = ConfigurationSpace.resolve(configurationFileOption.getValue(), searchSpaceOption.getValue()); this.configurator = new LearnerConfigurator(this.space); this.configurator.validate(); } catch (Exception e) { throw new IllegalStateException("Could not set up " + getClass().getSimpleName() - + " from \"" + configurationFileOption.getValue() + "\": " + e.getMessage(), e); + + " from " + ConfigurationSpace.describeSource(configurationFileOption.getValue(), searchSpaceOption.getValue()) + ": " + e.getMessage(), e); } } diff --git a/moa/src/main/java/moa/classifiers/AutoML/MESSPTClassifier.java b/moa/src/main/java/moa/classifiers/AutoML/MESSPTClassifier.java index c0860b773..4db0c64e4 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/MESSPTClassifier.java +++ b/moa/src/main/java/moa/classifiers/AutoML/MESSPTClassifier.java @@ -22,6 +22,7 @@ import com.github.javacliparser.FloatOption; import com.github.javacliparser.IntOption; import com.github.javacliparser.MultiChoiceOption; +import com.github.javacliparser.StringOption; import com.yahoo.labs.samoa.instances.Instance; import moa.capabilities.CapabilitiesHandler; import moa.classifiers.AbstractClassifier; @@ -67,6 +68,10 @@ public class MESSPTClassifier extends AbstractClassifier implements MultiClassCl public FileOption configurationFileOption = new FileOption("configurationFile", 'f', "Search space in JSON format.", null, ".json", false); + public StringOption searchSpaceOption = new StringOption("searchSpace", 's', + "Search space as inline JSON. Takes precedence over configurationFile when set," + + " so that a caller holding the space in memory need not write a file.", ""); + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', "Number of instances between DE updates.", 1000, 1, Integer.MAX_VALUE); @@ -223,12 +228,12 @@ public void resetLearningImpl() { @Override public void setConfigurations() { try { - this.space = ConfigurationSpace.fromFile(configurationFileOption.getValue()); + this.space = ConfigurationSpace.resolve(configurationFileOption.getValue(), searchSpaceOption.getValue()); this.configurator = new LearnerConfigurator(this.space); this.configurator.validate(); } catch (Exception e) { throw new IllegalStateException("Could not set up " + getClass().getSimpleName() - + " from \"" + configurationFileOption.getValue() + "\": " + e.getMessage(), e); + + " from " + ConfigurationSpace.describeSource(configurationFileOption.getValue(), searchSpaceOption.getValue()) + ": " + e.getMessage(), e); } } diff --git a/moa/src/main/java/moa/classifiers/AutoML/MESSPTRegressor.java b/moa/src/main/java/moa/classifiers/AutoML/MESSPTRegressor.java index 67f8a3169..fc5140c1e 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/MESSPTRegressor.java +++ b/moa/src/main/java/moa/classifiers/AutoML/MESSPTRegressor.java @@ -22,6 +22,7 @@ import com.github.javacliparser.FloatOption; import com.github.javacliparser.IntOption; import com.github.javacliparser.MultiChoiceOption; +import com.github.javacliparser.StringOption; import com.yahoo.labs.samoa.instances.Instance; import moa.capabilities.CapabilitiesHandler; import moa.classifiers.AbstractClassifier; @@ -62,6 +63,10 @@ public class MESSPTRegressor extends AbstractClassifier implements Regressor, public FileOption configurationFileOption = new FileOption("configurationFile", 'f', "Search space in JSON format.", null, ".json", false); + public StringOption searchSpaceOption = new StringOption("searchSpace", 's', + "Search space as inline JSON. Takes precedence over configurationFile when set," + + " so that a caller holding the space in memory need not write a file.", ""); + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', "Number of instances between DE updates.", 1000, 1, Integer.MAX_VALUE); @@ -219,12 +224,12 @@ public void resetLearningImpl() { public void setConfigurations() { try { - this.space = ConfigurationSpace.fromFile(configurationFileOption.getValue()); + this.space = ConfigurationSpace.resolve(configurationFileOption.getValue(), searchSpaceOption.getValue()); this.configurator = new LearnerConfigurator(this.space); this.configurator.validate(); } catch (Exception e) { throw new IllegalStateException("Could not set up " + getClass().getSimpleName() - + " from \"" + configurationFileOption.getValue() + "\": " + e.getMessage(), e); + + " from " + ConfigurationSpace.describeSource(configurationFileOption.getValue(), searchSpaceOption.getValue()) + ": " + e.getMessage(), e); } } diff --git a/moa/src/main/java/moa/classifiers/AutoML/MetricUtils.java b/moa/src/main/java/moa/classifiers/AutoML/MetricUtils.java index b5fbbe150..c7276f022 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/MetricUtils.java +++ b/moa/src/main/java/moa/classifiers/AutoML/MetricUtils.java @@ -119,6 +119,12 @@ public static void configure(BasicClassificationPerformanceEvaluator evaluator) evaluator.f1PerClassOption.setValue(true); evaluator.precisionPerClassOption.setValue(true); evaluator.recallPerClassOption.setValue(true); + // A freshly constructed evaluator allocates its estimators only in + // reset(), so getPerformanceMeasurements() would throw on one that has + // not scored an instance yet. That happens whenever the search state is + // read at an evaluation window boundary, just after candidates were + // respawned with new evaluators. + evaluator.reset(); } public static double getScore(Measurement[] measurements, int metricChoice) { diff --git a/moa/src/main/java/moa/classifiers/AutoML/RandomSearchClassifier.java b/moa/src/main/java/moa/classifiers/AutoML/RandomSearchClassifier.java index 3447cda93..3c30b90ca 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/RandomSearchClassifier.java +++ b/moa/src/main/java/moa/classifiers/AutoML/RandomSearchClassifier.java @@ -21,6 +21,7 @@ import com.github.javacliparser.FlagOption; import com.github.javacliparser.IntOption; import com.github.javacliparser.MultiChoiceOption; +import com.github.javacliparser.StringOption; import com.yahoo.labs.samoa.instances.Instance; import moa.capabilities.CapabilitiesHandler; import moa.classifiers.AbstractClassifier; @@ -67,6 +68,10 @@ public class RandomSearchClassifier extends AbstractClassifier implements MultiC public FileOption configurationFileOption = new FileOption("configurationFile", 'f', "Search space in JSON format.", null, ".json", false); + public StringOption searchSpaceOption = new StringOption("searchSpace", 's', + "Search space as inline JSON. Takes precedence over configurationFile when set," + + " so that a caller holding the space in memory need not write a file.", ""); + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', "Number of instances between candidate evaluations.", 1000, 1, Integer.MAX_VALUE); @@ -261,7 +266,7 @@ public void setConfigurations() { try { this.numericalParameters = 0; - ConfigurationSpace space = ConfigurationSpace.fromFile(this.configurationFileOption.getValue()); + ConfigurationSpace space = ConfigurationSpace.resolve(this.configurationFileOption.getValue(), this.searchSpaceOption.getValue()); this.configurator = new LearnerConfigurator(space); this.configurator.validate(); @@ -335,7 +340,7 @@ public void setConfigurations() { } catch (Exception e) { throw new IllegalStateException("Could not set up " + getClass().getSimpleName() - + " from \"" + this.configurationFileOption.getValue() + "\": " + e.getMessage(), e); + + " from " + ConfigurationSpace.describeSource(this.configurationFileOption.getValue(), this.searchSpaceOption.getValue()) + ": " + e.getMessage(), e); } } diff --git a/moa/src/main/java/moa/classifiers/AutoML/RandomSearchRegressor.java b/moa/src/main/java/moa/classifiers/AutoML/RandomSearchRegressor.java index d17c14381..b64f4193e 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/RandomSearchRegressor.java +++ b/moa/src/main/java/moa/classifiers/AutoML/RandomSearchRegressor.java @@ -21,6 +21,7 @@ import com.github.javacliparser.FlagOption; import com.github.javacliparser.IntOption; import com.github.javacliparser.MultiChoiceOption; +import com.github.javacliparser.StringOption; import com.yahoo.labs.samoa.instances.Instance; import moa.capabilities.CapabilitiesHandler; import moa.classifiers.AbstractClassifier; @@ -62,6 +63,10 @@ public class RandomSearchRegressor extends AbstractClassifier implements Regress public FileOption configurationFileOption = new FileOption("configurationFile", 'f', "Search space in JSON format.", null, ".json", false); + public StringOption searchSpaceOption = new StringOption("searchSpace", 's', + "Search space as inline JSON. Takes precedence over configurationFile when set," + + " so that a caller holding the space in memory need not write a file.", ""); + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', "Number of instances between candidate evaluations.", 1000, 1, Integer.MAX_VALUE); @@ -234,7 +239,7 @@ public void setConfigurations() { try { this.numericalParameters = 0; - ConfigurationSpace space = ConfigurationSpace.fromFile(this.configurationFileOption.getValue()); + ConfigurationSpace space = ConfigurationSpace.resolve(this.configurationFileOption.getValue(), this.searchSpaceOption.getValue()); this.configurator = new LearnerConfigurator(space); this.configurator.validate(); @@ -298,7 +303,7 @@ public void setConfigurations() { } catch (Exception e) { throw new IllegalStateException("Could not set up " + getClass().getSimpleName() - + " from \"" + this.configurationFileOption.getValue() + "\": " + e.getMessage(), e); + + " from " + ConfigurationSpace.describeSource(this.configurationFileOption.getValue(), this.searchSpaceOption.getValue()) + ": " + e.getMessage(), e); } } diff --git a/moa/src/main/java/moa/classifiers/AutoML/SSPTClassifier.java b/moa/src/main/java/moa/classifiers/AutoML/SSPTClassifier.java index aa4dc22ae..f914cf5c1 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/SSPTClassifier.java +++ b/moa/src/main/java/moa/classifiers/AutoML/SSPTClassifier.java @@ -22,6 +22,7 @@ import com.github.javacliparser.FloatOption; import com.github.javacliparser.IntOption; import com.github.javacliparser.MultiChoiceOption; +import com.github.javacliparser.StringOption; import com.yahoo.labs.samoa.instances.Instance; import moa.capabilities.CapabilitiesHandler; import moa.classifiers.AbstractClassifier; @@ -66,6 +67,10 @@ public class SSPTClassifier extends AbstractClassifier implements MultiClassClas public FileOption configurationFileOption = new FileOption("configurationFile", 'f', "Search space in JSON format.", null, ".json", false); + public StringOption searchSpaceOption = new StringOption("searchSpace", 's', + "Search space as inline JSON. Takes precedence over configurationFile when set," + + " so that a caller holding the space in memory need not write a file.", ""); + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', "Number of instances between simplex updates.", 1000, 1, Integer.MAX_VALUE); @@ -219,12 +224,12 @@ public void resetLearningImpl() { @Override public void setConfigurations() { try { - this.space = ConfigurationSpace.fromFile(configurationFileOption.getValue()); + this.space = ConfigurationSpace.resolve(configurationFileOption.getValue(), searchSpaceOption.getValue()); this.configurator = new LearnerConfigurator(this.space); this.configurator.validate(); } catch (Exception e) { throw new IllegalStateException("Could not set up " + getClass().getSimpleName() - + " from \"" + configurationFileOption.getValue() + "\": " + e.getMessage(), e); + + " from " + ConfigurationSpace.describeSource(configurationFileOption.getValue(), searchSpaceOption.getValue()) + ": " + e.getMessage(), e); } } @@ -658,7 +663,7 @@ public double getClassifierScore() { } @Override - public int getNumberOfCandidates() { return 9; } + public int getNumberOfCandidates() { return simplex == null ? 0 : simplex.length; } @Override public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } diff --git a/moa/src/main/java/moa/classifiers/AutoML/SSPTRegressor.java b/moa/src/main/java/moa/classifiers/AutoML/SSPTRegressor.java index 12587a0f3..ee0ed69c0 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/SSPTRegressor.java +++ b/moa/src/main/java/moa/classifiers/AutoML/SSPTRegressor.java @@ -22,6 +22,7 @@ import com.github.javacliparser.FloatOption; import com.github.javacliparser.IntOption; import com.github.javacliparser.MultiChoiceOption; +import com.github.javacliparser.StringOption; import com.yahoo.labs.samoa.instances.Instance; import moa.capabilities.CapabilitiesHandler; import moa.classifiers.AbstractClassifier; @@ -61,6 +62,10 @@ public class SSPTRegressor extends AbstractClassifier implements Regressor, public FileOption configurationFileOption = new FileOption("configurationFile", 'f', "Search space in JSON format.", null, ".json", false); + public StringOption searchSpaceOption = new StringOption("searchSpace", 's', + "Search space as inline JSON. Takes precedence over configurationFile when set," + + " so that a caller holding the space in memory need not write a file.", ""); + public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', "Number of instances between simplex updates.", 1000, 1, Integer.MAX_VALUE); @@ -215,12 +220,12 @@ public void resetLearningImpl() { @Override public void setConfigurations() { try { - this.space = ConfigurationSpace.fromFile(configurationFileOption.getValue()); + this.space = ConfigurationSpace.resolve(configurationFileOption.getValue(), searchSpaceOption.getValue()); this.configurator = new LearnerConfigurator(this.space); this.configurator.validate(); } catch (Exception e) { throw new IllegalStateException("Could not set up " + getClass().getSimpleName() - + " from \"" + configurationFileOption.getValue() + "\": " + e.getMessage(), e); + + " from " + ConfigurationSpace.describeSource(configurationFileOption.getValue(), searchSpaceOption.getValue()) + ": " + e.getMessage(), e); } } @@ -640,7 +645,7 @@ public double getClassifierScore() { } @Override - public int getNumberOfCandidates() { return 3; } + public int getNumberOfCandidates() { return simplex == null ? 0 : simplex.length; } @Override public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } diff --git a/moa/src/main/java/moa/classifiers/AutoML/space/ConfigurationSpace.java b/moa/src/main/java/moa/classifiers/AutoML/space/ConfigurationSpace.java index 1ee1c85d0..69fa2ac0c 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/space/ConfigurationSpace.java +++ b/moa/src/main/java/moa/classifiers/AutoML/space/ConfigurationSpace.java @@ -73,9 +73,36 @@ public ConfigurationSpace(String algorithm, List parameters) { this.parameters = parameters; } + /** + * The search space a tuner should use, given both of the ways one can be + * supplied: inline JSON wins over a file when set, so that a caller holding + * the space in memory - the CapyMOA wrappers, or a script - never has to + * write a temporary file just to satisfy the {@code -f} option. + * + * @param path path from the {@code configurationFile} option, may be null + * @param json JSON from the {@code searchSpace} option, may be null or empty + */ + public static ConfigurationSpace resolve(String path, String json) throws IOException { + if (json != null && !json.trim().isEmpty()) { + return fromString(json); + } + return fromFile(path); + } + + /** + * How to refer to the search space in an error message, matching whichever + * of the two options {@link #resolve} would have read. + */ + public static String describeSource(String path, String json) { + if (json != null && !json.trim().isEmpty()) return "the inline search space"; + if (path == null || path.trim().isEmpty()) return "(no search space given)"; + return "\"" + path + "\""; + } + public static ConfigurationSpace fromFile(String path) throws IOException { if (path == null || path.trim().isEmpty()) { - throw new IOException("No configuration file given. Set the -f option to a search space JSON file."); + throw new IOException("No search space given. Set -f to a search space JSON file," + + " or -s to the search space JSON itself."); } File file = new File(path); if (!file.isFile()) { From ce82115c5aee2f7dd02470d24c23de0204d652c2 Mon Sep 17 00:00:00 2001 From: Daniel Nowak Date: Thu, 6 Aug 2026 04:26:39 +0200 Subject: [PATCH 3/4] moa final commit --- .../AutoML/BayesianStreamTunerClassifier.java | 14 ++--- .../AutoML/BayesianStreamTunerRegressor.java | 14 ++--- .../classifiers/AutoML/FSS_SPTClassifier.java | 10 ++-- .../classifiers/AutoML/FSS_SPTRegressor.java | 10 ++-- .../moa/classifiers/AutoML/HPOMethod.java | 2 +- .../classifiers/AutoML/MESSPTClassifier.java | 8 +-- .../classifiers/AutoML/MESSPTRegressor.java | 8 +-- .../AutoML/RandomSearchClassifier.java | 6 +-- .../AutoML/RandomSearchRegressor.java | 6 +-- .../classifiers/AutoML/SSPTClassifier.java | 52 +++++++++++++++---- .../moa/classifiers/AutoML/SSPTRegressor.java | 52 +++++++++++++++---- .../AutoML/space/ConfigurationSpace.java | 6 +-- 12 files changed, 126 insertions(+), 62 deletions(-) diff --git a/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerClassifier.java b/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerClassifier.java index 3375a8da6..4c43a6809 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerClassifier.java +++ b/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerClassifier.java @@ -85,7 +85,7 @@ public class BayesianStreamTunerClassifier extends AbstractClassifier "Search space as inline JSON. Takes precedence over configurationFile when set," + " so that a caller holding the space in memory need not write a file.", ""); - public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + public IntOption periodicityOption = new IntOption("periodicity", 'g', "Number of instances between model update cycles; also the number of recent" + " instances kept to derive the surrogate's stream statistics.", 1000, 1, Integer.MAX_VALUE); @@ -346,7 +346,7 @@ private Instance createSurrogateInstance(double[] features, double performance) private void initDataWindow(int numFeatures) { this.windowFeatures = numFeatures; - this.dataWindow = new double[gracePeriodOption.getValue()][numFeatures]; + this.dataWindow = new double[periodicityOption.getValue()][numFeatures]; this.windowHead = 0; this.windowCount = 0; } @@ -443,7 +443,7 @@ private double computeAcquisition(double mu, double sigma, double bestF) { return (mu - bestF) * normalCDF(z) + sigma * normalPDF(z); } default: { // UCB - double kappa = Math.max(0.1, 2.0 * (1.0 - instanceCount / (10.0 * gracePeriodOption.getValue()))); + double kappa = Math.max(0.1, 2.0 * (1.0 - instanceCount / (10.0 * periodicityOption.getValue()))); return mu + kappa * sigma; } } @@ -493,7 +493,7 @@ public void trainOnInstanceImpl(Instance inst) { } instanceCount++; - if (instanceCount % gracePeriodOption.getValue() == 0) { + if (instanceCount % periodicityOption.getValue() == 0) { updateModels(); } @@ -677,13 +677,13 @@ public double getClassifierScore() { public int getNumberOfCandidates() { return numberOfCandidatesOption.getValue(); } @Override - public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } + public long getStatesEvaluatedCount() { return instanceCount / periodicityOption.getValue(); } @Override - public int getEvaluationInstancesCount() { return (int) (instanceCount % gracePeriodOption.getValue()); } + public int getEvaluationInstancesCount() { return (int) (instanceCount % periodicityOption.getValue()); } @Override - public int getGracePeriod() { return gracePeriodOption.getValue(); } + public int getPeriodicity() { return periodicityOption.getValue(); } @Override public Classifier getMainClassifier() { return candidates[bestCandidateIndex]; } diff --git a/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerRegressor.java b/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerRegressor.java index a36de275f..627cd5319 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerRegressor.java +++ b/moa/src/main/java/moa/classifiers/AutoML/BayesianStreamTunerRegressor.java @@ -80,7 +80,7 @@ public class BayesianStreamTunerRegressor extends AbstractClassifier "Search space as inline JSON. Takes precedence over configurationFile when set," + " so that a caller holding the space in memory need not write a file.", ""); - public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + public IntOption periodicityOption = new IntOption("periodicity", 'g', "Number of instances between model update cycles; also the number of recent" + " instances kept to derive the surrogate's stream statistics.", 1000, 1, Integer.MAX_VALUE); @@ -339,7 +339,7 @@ private Instance createSurrogateInstance(double[] features, double performance) private void initDataWindow(int numFeatures) { this.windowFeatures = numFeatures; - this.dataWindow = new double[gracePeriodOption.getValue()][numFeatures]; + this.dataWindow = new double[periodicityOption.getValue()][numFeatures]; this.windowHead = 0; this.windowCount = 0; } @@ -439,7 +439,7 @@ private double computeAcquisition(double mu, double sigma, double bestF) { return improvement * normalCDF(z) + sigma * normalPDF(z); } default: { // UCB - double kappa = Math.max(0.1, 2.0 * (1.0 - instanceCount / (10.0 * gracePeriodOption.getValue()))); + double kappa = Math.max(0.1, 2.0 * (1.0 - instanceCount / (10.0 * periodicityOption.getValue()))); return mu + kappa * sigma; } } @@ -489,7 +489,7 @@ public void trainOnInstanceImpl(Instance inst) { } instanceCount++; - if (instanceCount % gracePeriodOption.getValue() == 0) { + if (instanceCount % periodicityOption.getValue() == 0) { updateModels(); } @@ -673,13 +673,13 @@ public double getClassifierScore() { public int getNumberOfCandidates() { return numberOfCandidatesOption.getValue(); } @Override - public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } + public long getStatesEvaluatedCount() { return instanceCount / periodicityOption.getValue(); } @Override - public int getEvaluationInstancesCount() { return (int) (instanceCount % gracePeriodOption.getValue()); } + public int getEvaluationInstancesCount() { return (int) (instanceCount % periodicityOption.getValue()); } @Override - public int getGracePeriod() { return gracePeriodOption.getValue(); } + public int getPeriodicity() { return periodicityOption.getValue(); } @Override public Classifier getMainClassifier() { return candidates[bestCandidateIndex]; } diff --git a/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTClassifier.java b/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTClassifier.java index e472220ee..3841b6982 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTClassifier.java +++ b/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTClassifier.java @@ -74,7 +74,7 @@ public class FSS_SPTClassifier extends AbstractClassifier implements MultiClassC "Search space as inline JSON. Takes precedence over configurationFile when set," + " so that a caller holding the space in memory need not write a file.", ""); - public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + public IntOption periodicityOption = new IntOption("periodicity", 'g', "Number of instances between FSS school updates.", 1000, 2, Integer.MAX_VALUE); public MultiChoiceOption metricOption = new MultiChoiceOption("metric", 'm', @@ -362,7 +362,7 @@ public void trainOnInstanceImpl(Instance inst) { throw new RuntimeException("Could not call invokeAll() on training threads."); } } - int halfPeriod = gracePeriodOption.getValue() / 2; + int halfPeriod = periodicityOption.getValue() / 2; // Phase 1 (halfway): assess improvement, apply individual movement if (evaluationInstances == halfPeriod) { @@ -373,7 +373,7 @@ public void trainOnInstanceImpl(Instance inst) { } // Phase 2 (full period): feeding, instinctive, volitional movements - if (evaluationInstances >= gracePeriodOption.getValue()) { + if (evaluationInstances >= periodicityOption.getValue()) { evaluationInstances = 0; sortSchool(); double weightChange = feeding(); @@ -678,13 +678,13 @@ public double getClassifierScore() { public int getNumberOfCandidates() { return numEstimatorsOption.getValue(); } @Override - public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } + public long getStatesEvaluatedCount() { return instanceCount / periodicityOption.getValue(); } @Override public int getEvaluationInstancesCount() { return evaluationInstances; } @Override - public int getGracePeriod() { return gracePeriodOption.getValue(); } + public int getPeriodicity() { return periodicityOption.getValue(); } @Override public Classifier getMainClassifier() { return getBestFish().model; } diff --git a/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTRegressor.java b/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTRegressor.java index e8523ce7c..c5982d095 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTRegressor.java +++ b/moa/src/main/java/moa/classifiers/AutoML/FSS_SPTRegressor.java @@ -69,7 +69,7 @@ public class FSS_SPTRegressor extends AbstractClassifier implements Regressor, "Search space as inline JSON. Takes precedence over configurationFile when set," + " so that a caller holding the space in memory need not write a file.", ""); - public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + public IntOption periodicityOption = new IntOption("periodicity", 'g', "Number of instances between FSS school updates.", 1000, 2, Integer.MAX_VALUE); public MultiChoiceOption metricOption = new MultiChoiceOption("metric", 'm', @@ -358,7 +358,7 @@ public void trainOnInstanceImpl(Instance inst) { throw new RuntimeException("Could not call invokeAll() on training threads."); } } - int halfPeriod = gracePeriodOption.getValue() / 2; + int halfPeriod = periodicityOption.getValue() / 2; // Phase 1 (halfway): assess improvement, apply individual movement if (evaluationInstances == halfPeriod) { @@ -369,7 +369,7 @@ public void trainOnInstanceImpl(Instance inst) { } // Phase 2 (full period): feeding, instinctive, volitional movements - if (evaluationInstances >= gracePeriodOption.getValue()) { + if (evaluationInstances >= periodicityOption.getValue()) { evaluationInstances = 0; sortSchool(); double weightChange = feeding(); @@ -674,13 +674,13 @@ public double getClassifierScore() { public int getNumberOfCandidates() { return numEstimatorsOption.getValue(); } @Override - public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } + public long getStatesEvaluatedCount() { return instanceCount / periodicityOption.getValue(); } @Override public int getEvaluationInstancesCount() { return evaluationInstances; } @Override - public int getGracePeriod() { return gracePeriodOption.getValue(); } + public int getPeriodicity() { return periodicityOption.getValue(); } @Override public Classifier getMainClassifier() { return getBestFish().model; } diff --git a/moa/src/main/java/moa/classifiers/AutoML/HPOMethod.java b/moa/src/main/java/moa/classifiers/AutoML/HPOMethod.java index 3d78e24a9..baec50153 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/HPOMethod.java +++ b/moa/src/main/java/moa/classifiers/AutoML/HPOMethod.java @@ -85,7 +85,7 @@ default void cleanThreads() {} default int getEvaluationInstancesCount() { return 0; } - default int getGracePeriod() { return Integer.MAX_VALUE; } + default int getPeriodicity() { return Integer.MAX_VALUE; } default Classifier getMainClassifier() { return null; } diff --git a/moa/src/main/java/moa/classifiers/AutoML/MESSPTClassifier.java b/moa/src/main/java/moa/classifiers/AutoML/MESSPTClassifier.java index 4db0c64e4..c60bf65d1 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/MESSPTClassifier.java +++ b/moa/src/main/java/moa/classifiers/AutoML/MESSPTClassifier.java @@ -72,7 +72,7 @@ public class MESSPTClassifier extends AbstractClassifier implements MultiClassCl "Search space as inline JSON. Takes precedence over configurationFile when set," + " so that a caller holding the space in memory need not write a file.", ""); - public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + public IntOption periodicityOption = new IntOption("periodicity", 'g', "Number of instances between DE updates.", 1000, 1, Integer.MAX_VALUE); public IntOption populationSizeOption = new IntOption("populationSize", 'p', @@ -354,7 +354,7 @@ private void trainNotConverged(Instance inst, InstanceExample example) { } } - if (evaluationInstances >= gracePeriodOption.getValue()) { + if (evaluationInstances >= periodicityOption.getValue()) { evaluationInstances = 0; updatePopulation(); @@ -606,13 +606,13 @@ public double getClassifierScore() { public int getNumberOfCandidates() { return populationSizeOption.getValue(); } @Override - public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } + public long getStatesEvaluatedCount() { return instanceCount / periodicityOption.getValue(); } @Override public int getEvaluationInstancesCount() { return evaluationInstances; } @Override - public int getGracePeriod() { return gracePeriodOption.getValue(); } + public int getPeriodicity() { return periodicityOption.getValue(); } @Override public Classifier getMainClassifier() { return population[0].model; } diff --git a/moa/src/main/java/moa/classifiers/AutoML/MESSPTRegressor.java b/moa/src/main/java/moa/classifiers/AutoML/MESSPTRegressor.java index fc5140c1e..7a8314ea4 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/MESSPTRegressor.java +++ b/moa/src/main/java/moa/classifiers/AutoML/MESSPTRegressor.java @@ -67,7 +67,7 @@ public class MESSPTRegressor extends AbstractClassifier implements Regressor, "Search space as inline JSON. Takes precedence over configurationFile when set," + " so that a caller holding the space in memory need not write a file.", ""); - public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + public IntOption periodicityOption = new IntOption("periodicity", 'g', "Number of instances between DE updates.", 1000, 1, Integer.MAX_VALUE); public IntOption populationSizeOption = new IntOption("populationSize", 'p', @@ -364,7 +364,7 @@ private void trainNotConverged(Instance inst, InstanceExample example) { } } - if (evaluationInstances >= gracePeriodOption.getValue()) { + if (evaluationInstances >= periodicityOption.getValue()) { evaluationInstances = 0; updatePopulation(); @@ -598,13 +598,13 @@ public double getClassifierScore() { public int getNumberOfCandidates() { return populationSizeOption.getValue(); } @Override - public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } + public long getStatesEvaluatedCount() { return instanceCount / periodicityOption.getValue(); } @Override public int getEvaluationInstancesCount() { return evaluationInstances; } @Override - public int getGracePeriod() { return gracePeriodOption.getValue(); } + public int getPeriodicity() { return periodicityOption.getValue(); } @Override public Classifier getMainClassifier() { return population[0].model; } diff --git a/moa/src/main/java/moa/classifiers/AutoML/RandomSearchClassifier.java b/moa/src/main/java/moa/classifiers/AutoML/RandomSearchClassifier.java index 3c30b90ca..0ab799f75 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/RandomSearchClassifier.java +++ b/moa/src/main/java/moa/classifiers/AutoML/RandomSearchClassifier.java @@ -72,7 +72,7 @@ public class RandomSearchClassifier extends AbstractClassifier implements MultiC "Search space as inline JSON. Takes precedence over configurationFile when set," + " so that a caller holding the space in memory need not write a file.", ""); - public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + public IntOption periodicityOption = new IntOption("periodicity", 'g', "Number of instances between candidate evaluations.", 1000, 1, Integer.MAX_VALUE); public FlagOption randomInitialParametersOption = new FlagOption("randomInitialParameters", 'R', @@ -471,7 +471,7 @@ public void trainOnInstanceImpl(Instance inst) { //check for change in parameters this.evaluationInstances++; - if ((this.statesEvaluated == 0) || this.evaluationInstances >= this.gracePeriodOption.getValue()) { + if ((this.statesEvaluated == 0) || this.evaluationInstances >= this.periodicityOption.getValue()) { this.checkParameterChange(); this.statesEvaluated++; this.evaluationInstances = 0; @@ -582,7 +582,7 @@ public double getClassifierScore() { public int getEvaluationInstancesCount() { return this.evaluationInstances; } @Override - public int getGracePeriod() { return this.gracePeriodOption.getValue(); } + public int getPeriodicity() { return this.periodicityOption.getValue(); } @Override public Classifier getMainClassifier() { return this.classifier; } diff --git a/moa/src/main/java/moa/classifiers/AutoML/RandomSearchRegressor.java b/moa/src/main/java/moa/classifiers/AutoML/RandomSearchRegressor.java index b64f4193e..96416573f 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/RandomSearchRegressor.java +++ b/moa/src/main/java/moa/classifiers/AutoML/RandomSearchRegressor.java @@ -67,7 +67,7 @@ public class RandomSearchRegressor extends AbstractClassifier implements Regress "Search space as inline JSON. Takes precedence over configurationFile when set," + " so that a caller holding the space in memory need not write a file.", ""); - public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + public IntOption periodicityOption = new IntOption("periodicity", 'g', "Number of instances between candidate evaluations.", 1000, 1, Integer.MAX_VALUE); public FlagOption randomInitialParametersOption = new FlagOption("randomInitialParameters", 'R', @@ -414,7 +414,7 @@ public void trainOnInstanceImpl(Instance inst) { initExecutor(); this.evaluationInstances++; - if ((this.statesEvaluated == 0) || this.evaluationInstances >= this.gracePeriodOption.getValue()) { + if ((this.statesEvaluated == 0) || this.evaluationInstances >= this.periodicityOption.getValue()) { this.checkParameterChange(); this.statesEvaluated++; this.evaluationInstances = 0; @@ -517,7 +517,7 @@ public double getClassifierScore() { public int getEvaluationInstancesCount() { return this.evaluationInstances; } @Override - public int getGracePeriod() { return this.gracePeriodOption.getValue(); } + public int getPeriodicity() { return this.periodicityOption.getValue(); } @Override public Classifier getMainClassifier() { return this.classifier; } diff --git a/moa/src/main/java/moa/classifiers/AutoML/SSPTClassifier.java b/moa/src/main/java/moa/classifiers/AutoML/SSPTClassifier.java index f914cf5c1..29a07bc52 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/SSPTClassifier.java +++ b/moa/src/main/java/moa/classifiers/AutoML/SSPTClassifier.java @@ -71,7 +71,7 @@ public class SSPTClassifier extends AbstractClassifier implements MultiClassClas "Search space as inline JSON. Takes precedence over configurationFile when set," + " so that a caller holding the space in memory need not write a file.", ""); - public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + public IntOption periodicityOption = new IntOption("periodicity", 'g', "Number of instances between simplex updates.", 1000, 1, Integer.MAX_VALUE); public FloatOption convergenceSphereOption = new FloatOption("convergenceSphere", 'c', @@ -128,6 +128,12 @@ void addResult(InstanceExample example, double[] votes) { evaluator.addResult(example, votes); instancesSeen++; } + + /** Forget what this entry scored, so a fresh window starts even. */ + void resetEvaluation() { + evaluator.reset(); + instancesSeen = 0; + } } // ========== FIELDS ========== @@ -375,10 +381,10 @@ private void trainNotConverged(Instance inst, InstanceExample example) { } } - if (evaluationInstances >= gracePeriodOption.getValue()) { + if (evaluationInstances >= periodicityOption.getValue()) { evaluationInstances = 0; - updateSimplex(); - if (checkConvergence()) { + boolean moved = updateSimplex(); + if (moved && checkConvergence()) { if (verboseOption.isSet()) System.out.println("SSPT: Converged at instance " + instanceCount); converged = true; @@ -388,16 +394,42 @@ private void trainNotConverged(Instance inst, InstanceExample example) { // ========== SIMPLEX UPDATE ========== - private void updateSimplex() { - sortSimplex(); - lastCentroid = computeCentroid(); - + /** + * Advances the simplex, alternating between laying down the Nelder-Mead + * candidate points and choosing among them. + * + *

The two phases must be separate windows. {@link #createExpanded()} + * builds the reflection, expansion, contraction, shrink and midpoint + * models but trains none of them, and + * {@link #applyNelderMeadOperators()} ranks them by measured performance - + * so applying the operators in the same call that creates the points + * compares six models that have seen no instances at all. Every + * comparison then fails, the operators fall through to their last branch, + * and the simplex is overwritten with untrained shrink and midpoint models + * on every update, whatever the data says. + * + *

Splitting the phases lets the candidate points train alongside the + * vertices for a full window first, which is what makes the choice between + * them meaningful. + * + * @return whether the simplex moved, i.e. whether this call applied + * the operators rather than laying the candidate points down + */ + private boolean updateSimplex() { if (expanded == null || expanded.isEmpty()) { + sortSimplex(); + lastCentroid = computeCentroid(); expanded = createExpanded(); + // The vertices have been accumulating since they were created and + // the candidate points have seen nothing, so the next window is + // scored from a common start for all nine models. + for (SimplexEntry vertex : simplex) vertex.resetEvaluation(); + return false; } applyNelderMeadOperators(); expanded = null; + return true; } private void sortSimplex() { @@ -666,13 +698,13 @@ public double getClassifierScore() { public int getNumberOfCandidates() { return simplex == null ? 0 : simplex.length; } @Override - public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } + public long getStatesEvaluatedCount() { return instanceCount / periodicityOption.getValue(); } @Override public int getEvaluationInstancesCount() { return evaluationInstances; } @Override - public int getGracePeriod() { return gracePeriodOption.getValue(); } + public int getPeriodicity() { return periodicityOption.getValue(); } @Override public Classifier getMainClassifier() { return simplex[0].model; } diff --git a/moa/src/main/java/moa/classifiers/AutoML/SSPTRegressor.java b/moa/src/main/java/moa/classifiers/AutoML/SSPTRegressor.java index ee0ed69c0..743617197 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/SSPTRegressor.java +++ b/moa/src/main/java/moa/classifiers/AutoML/SSPTRegressor.java @@ -66,7 +66,7 @@ public class SSPTRegressor extends AbstractClassifier implements Regressor, "Search space as inline JSON. Takes precedence over configurationFile when set," + " so that a caller holding the space in memory need not write a file.", ""); - public IntOption gracePeriodOption = new IntOption("gracePeriod", 'g', + public IntOption periodicityOption = new IntOption("periodicity", 'g', "Number of instances between simplex updates.", 1000, 1, Integer.MAX_VALUE); public FloatOption convergenceSphereOption = new FloatOption("convergenceSphere", 'c', @@ -124,6 +124,12 @@ void addResult(InstanceExample example, double[] votes) { evaluator.addResult(example, votes); instancesSeen++; } + + /** Forget what this entry scored, so a fresh window starts even. */ + void resetEvaluation() { + evaluator.reset(); + instancesSeen = 0; + } } // ========== FIELDS ========== @@ -364,10 +370,10 @@ private void trainNotConverged(Instance inst, InstanceExample example) { } } - if (evaluationInstances >= gracePeriodOption.getValue()) { + if (evaluationInstances >= periodicityOption.getValue()) { evaluationInstances = 0; - updateSimplex(); - if (checkConvergence()) { + boolean moved = updateSimplex(); + if (moved && checkConvergence()) { if (verboseOption.isSet()) System.out.println("SSPT: Converged at instance " + instanceCount); converged = true; @@ -377,16 +383,42 @@ private void trainNotConverged(Instance inst, InstanceExample example) { // ========== SIMPLEX UPDATE ========== - private void updateSimplex() { - sortSimplex(); - lastCentroid = computeCentroid(); - + /** + * Advances the simplex, alternating between laying down the Nelder-Mead + * candidate points and choosing among them. + * + *

The two phases must be separate windows. {@link #createExpanded()} + * builds the reflection, expansion, contraction, shrink and midpoint + * models but trains none of them, and + * {@link #applyNelderMeadOperators()} ranks them by measured performance - + * so applying the operators in the same call that creates the points + * compares six models that have seen no instances at all. Every + * comparison then fails, the operators fall through to their last branch, + * and the simplex is overwritten with untrained shrink and midpoint models + * on every update, whatever the data says. + * + *

Splitting the phases lets the candidate points train alongside the + * vertices for a full window first, which is what makes the choice between + * them meaningful. + * + * @return whether the simplex moved, i.e. whether this call applied + * the operators rather than laying the candidate points down + */ + private boolean updateSimplex() { if (expanded == null || expanded.isEmpty()) { + sortSimplex(); + lastCentroid = computeCentroid(); expanded = createExpanded(); + // The vertices have been accumulating since they were created and + // the candidate points have seen nothing, so the next window is + // scored from a common start for all nine models. + for (SimplexEntry vertex : simplex) vertex.resetEvaluation(); + return false; } applyNelderMeadOperators(); expanded = null; + return true; } private void sortSimplex() { @@ -648,13 +680,13 @@ public double getClassifierScore() { public int getNumberOfCandidates() { return simplex == null ? 0 : simplex.length; } @Override - public long getStatesEvaluatedCount() { return instanceCount / gracePeriodOption.getValue(); } + public long getStatesEvaluatedCount() { return instanceCount / periodicityOption.getValue(); } @Override public int getEvaluationInstancesCount() { return evaluationInstances; } @Override - public int getGracePeriod() { return gracePeriodOption.getValue(); } + public int getPeriodicity() { return periodicityOption.getValue(); } @Override public Classifier getMainClassifier() { return simplex[0].model; } diff --git a/moa/src/main/java/moa/classifiers/AutoML/space/ConfigurationSpace.java b/moa/src/main/java/moa/classifiers/AutoML/space/ConfigurationSpace.java index 69fa2ac0c..7998118c8 100644 --- a/moa/src/main/java/moa/classifiers/AutoML/space/ConfigurationSpace.java +++ b/moa/src/main/java/moa/classifiers/AutoML/space/ConfigurationSpace.java @@ -35,7 +35,7 @@ * The search space shared by every AutoML method: the CLI string of the learner * being tuned, plus the list of tunable hyperparameters. * - *

Expected JSON, as produced by hand or by the CapyMOA wrappers: + *

Expected JSON: * * 

  * {
@@ -76,8 +76,8 @@ public ConfigurationSpace(String algorithm, List parameters) {
     /**
      * The search space a tuner should use, given both of the ways one can be
      * supplied: inline JSON wins over a file when set, so that a caller holding
-     * the space in memory - the CapyMOA wrappers, or a script - never has to
-     * write a temporary file just to satisfy the {@code -f} option.
+     * the space in memory - a script, say - never has to write a temporary file
+     * just to satisfy the {@code -f} option.
      *
      * @param path path from the {@code configurationFile} option, may be null
      * @param json JSON from the {@code searchSpace} option, may be null or empty

From 283d2dd06a469baa679951713e8e9eed3e67e9ed Mon Sep 17 00:00:00 2001
From: Daniel Nowak 
Date: Thu, 6 Aug 2026 11:25:29 +0200
Subject: [PATCH 4/4] update blr

---
 .../moa/classifiers/functions/BayesianLinearRegression.java   | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/moa/src/main/java/moa/classifiers/functions/BayesianLinearRegression.java b/moa/src/main/java/moa/classifiers/functions/BayesianLinearRegression.java
index c67d58279..19b67eab7 100644
--- a/moa/src/main/java/moa/classifiers/functions/BayesianLinearRegression.java
+++ b/moa/src/main/java/moa/classifiers/functions/BayesianLinearRegression.java
@@ -1,6 +1,5 @@
 /*
  *    BayesianLinearRegression.java
- *    Port of River's BayesianLinearRegression to MOA.
  *    Based on Bishop's Pattern Recognition and Machine Learning (2006), equations 3.50-3.59.
  *
  *    This program is free software; you can redistribute it and/or modify
@@ -33,8 +32,7 @@ public class BayesianLinearRegression extends AbstractClassifier implements Regr
     @Override
     public String getPurposeString() {
         return "Bayesian linear regression. Does not require feature scaling. "
-                + "Supports concept drift via the smoothing parameter. "
-                + "Port of River's BayesianLinearRegression.";
+                + "Supports concept drift via the smoothing parameter. ";
     }
 
     public FloatOption alphaOption = new FloatOption("alpha", 'a',