From 17b7b390ba0665c75cab9a6bc152375695394874 Mon Sep 17 00:00:00 2001 From: Nuwan Gunasekara Date: Mon, 4 Jul 2022 10:36:07 +1200 Subject: [PATCH 1/7] Initial version of MLP support using DJL --- moa/pom.xml | 43 ++ .../moa/classifiers/neuralNetworks/MLP.java | 551 ++++++++++++++++++ 2 files changed, 594 insertions(+) create mode 100644 moa/src/main/java/moa/classifiers/neuralNetworks/MLP.java diff --git a/moa/pom.xml b/moa/pom.xml index 004d4d744..126569b37 100644 --- a/moa/pom.xml +++ b/moa/pom.xml @@ -161,6 +161,49 @@ JMathIO 1.0 + + + + + ai.djl + api + 0.9.0 + + + org.slf4j + slf4j-api + + + + + + org.slf4j + slf4j-log4j12 + 1.7.26 + + + + + ai.djl + model-zoo + 0.9.0 + + + + + ai.djl.pytorch + pytorch-engine + 0.9.0 + + + + + ai.djl.pytorch + pytorch-native-auto + 1.7.0 + runtime + + diff --git a/moa/src/main/java/moa/classifiers/neuralNetworks/MLP.java b/moa/src/main/java/moa/classifiers/neuralNetworks/MLP.java new file mode 100644 index 000000000..ed8a9e3af --- /dev/null +++ b/moa/src/main/java/moa/classifiers/neuralNetworks/MLP.java @@ -0,0 +1,551 @@ +/* + * MLP.java + * Copyright (C) 2022 University of Waikato, Hamilton, New Zealand + * @author Nuwan Gunasekara (ng98@students.waikato.ac.nz) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ +package moa.classifiers.neuralNetworks; + +import com.github.javacliparser.FlagOption; +import com.github.javacliparser.IntOption; +import moa.capabilities.Capability; +import moa.capabilities.ImmutableCapabilities; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.MultiClassClassifier; +import moa.classifiers.core.driftdetection.ADWIN; +import moa.core.Measurement; +import com.yahoo.labs.samoa.instances.Instance; +import com.yahoo.labs.samoa.instances.InstancesHeader; +import com.github.javacliparser.FloatOption; +import com.github.javacliparser.MultiChoiceOption; + +import ai.djl.Device; +import ai.djl.Model; +import ai.djl.basicmodelzoo.basic.Mlp; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.NDList; +import ai.djl.ndarray.types.Shape; +import ai.djl.nn.Block; +import ai.djl.training.DefaultTrainingConfig; +import ai.djl.training.GradientCollector; +import ai.djl.training.Trainer; +import ai.djl.training.loss.Loss; +import ai.djl.training.tracker.Tracker; +import ai.djl.training.optimizer.Optimizer; + +import java.text.DecimalFormat; +import java.lang.Math; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static ai.djl.ndarray.types.DataType.FLOAT32; + +class MiniBatch { + private transient NDManager trainingNDManager; + public transient NDArray trainMiniBatchData = null; + public transient NDArray trainMiniBatchLabels = null; + public int itemsInMiniBatch = 0; + public int miniBatchSize = 1; + public NDList d = null; + public NDList l = null; + + public MiniBatch(Device device, int miniBatchSize) { + this.trainingNDManager = NDManager.newBaseManager(device); + this.miniBatchSize = miniBatchSize; + } + + public void addToMiniBatch(double[] featureValues, double [] classValue) { + if (itemsInMiniBatch == 0) { + // initialize and create NDArrays + trainMiniBatchData = trainingNDManager.create(featureValues).toType(FLOAT32, false); + trainMiniBatchLabels = trainingNDManager.create(classValue); + } else { + // add item to the mini batch + if (itemsInMiniBatch == 1) { + trainMiniBatchData = trainMiniBatchData.stack(trainingNDManager.create(featureValues).toType(FLOAT32, false), 0); + trainMiniBatchLabels = trainMiniBatchLabels.stack(trainingNDManager.create(classValue), 0); + } else { + trainMiniBatchData = trainMiniBatchData.concat(trainingNDManager.create(featureValues, new Shape(1, featureValues.length)).toType(FLOAT32, false), 0); + trainMiniBatchLabels = trainMiniBatchLabels.concat(trainingNDManager.create(classValue, new Shape(1, classValue.length)), 0); + } + } + itemsInMiniBatch++; + if (itemsInMiniBatch == miniBatchSize){ + d = new NDList(trainMiniBatchData); + l = new NDList(trainMiniBatchLabels); + } + } + + public void addToMiniBatch(Instance inst) { + double [] instDoubleA = inst.toDoubleArray(); + NDArray dataAndLabel = trainingNDManager.create(instDoubleA).toType(FLOAT32, false); + int featureLength = instDoubleA.length -1; + int classLength = 1; + NDArray data = dataAndLabel.get("0:" + Integer.toString(featureLength)); + NDArray label = dataAndLabel.get(Integer.toString(featureLength) +":" + Integer.toString(featureLength+1)); + + if (itemsInMiniBatch == 0) { + // initialize and create NDArrays + trainMiniBatchData = data; + trainMiniBatchLabels = label; + } else { + // add item to the mini batch + if (itemsInMiniBatch == 1) { + trainMiniBatchData = trainMiniBatchData.stack(data, 0); + trainMiniBatchLabels = trainMiniBatchLabels.stack(label, 0); + } else { + trainMiniBatchData = trainMiniBatchData.concat(data.reshape(new Shape(1, featureLength)), 0); + trainMiniBatchLabels = trainMiniBatchLabels.concat(label.reshape(new Shape(1, classLength)), 0); + } + } + itemsInMiniBatch++; + if (itemsInMiniBatch == miniBatchSize){ + d = new NDList(trainMiniBatchData); + l = new NDList(trainMiniBatchLabels); + } + } + + public boolean miniBatchFull(){ + return (itemsInMiniBatch == miniBatchSize); + } + + public void discardMiniBatch(){ + if (d != null){ + d.close(); + d = null; + } + if (trainMiniBatchData != null){ + trainMiniBatchData.close(); + trainMiniBatchData = null; + } + if (l != null){ + l.close(); + l = null; + } + if (trainMiniBatchLabels != null){ + trainMiniBatchLabels.close(); + trainMiniBatchLabels = null; + } + trainingNDManager.close(); + trainingNDManager = null; + itemsInMiniBatch = 0; + } +} + +public class MLP extends AbstractClassifier implements MultiClassClassifier { + + private static final long serialVersionUID = 1L; + + public static class NormalizeInfo{ + double sumOfValues = 0.0f; + double sumOfSquares = 0.0f; + } + + public static final int OPTIMIZER_SGD = 0; + public static final int OPTIMIZER_RMSPROP = 1; + public static final int OPTIMIZER_RMSPROP_RESET = 2; + public static final int OPTIMIZER_ADAGRAD = 3; + public static final int OPTIMIZER_ADAGRAD_RESET = 4; + public static final int OPTIMIZER_ADAM = 5; + public static final int OPTIMIZER_ADAM_RESET = 6; + + protected long samplesSeen = 0; + protected long trainedCount = 0; + protected NormalizeInfo[] normalizeInfo = null; + + private double[] pFeatureValues = null; + private double [] pClassValue = null; + + public FloatOption learningRateOption = new FloatOption( + "learningRate", + 'r', + "Learning Rate", + 0.03, 0.0000001, 1.0); + + public FloatOption backPropLossThreshold = new FloatOption( + "backPropLossThreshold", + 'b', + "Back propagation loss threshold", + 0.0, 0.0, Math.pow(10,10)); + + public MultiChoiceOption optimizerTypeOption = new MultiChoiceOption("optimizer", 'o', + "Choose optimizer", + new String[]{"SGD", "RMSPROP", "RMSPROP_RESET", "ADAGRAD", "ADAGRAD_RESET", "ADAM", "ADAM_RESET"}, + new String[]{"oSGD", "oRMSPROP", "oRMSPROP_RESET", "oADAGRAD", "oADAGRAD_RESET", "oADAM", "oADAM_RESET"}, + 0); + + public FlagOption useOneHotEncode = new FlagOption("useOneHotEncode", 'h', + "use one hot encoding"); + + public FlagOption useNormalization = new FlagOption("useNormalization", 'n', + "Normalize data"); + + public IntOption numberOfNeuronsInEachLayerInLog2 = new IntOption( + "numberOfNeuronsInEachLayerInLog2", + 'N', + "Number of neurons in the each layer in log2", + 10, 0, 20); + + public IntOption numberOfLayers = new IntOption( + "numberOfLayers", + 'L', + "Number of layers", + 1, 1, 4); + + public IntOption miniBatchSize = new IntOption( + "miniBatchSize", + 'B', + "Mini Batch Size", + 1, 1, 2048); + + public static final int deviceTypeOptionGPU = 0; + public static final int deviceTypeOptionCPU = 1; + public MultiChoiceOption deviceTypeOption = new MultiChoiceOption("deviceType", 'd', + "Choose device to run the model(use CPU if GPUs are not available)", + new String[]{"GPU","CPU"}, + new String[]{"GPU (use CPU if not available)", "CPU"}, + deviceTypeOptionCPU); + + + public double deltaForADWIN = 1.0E-5; + + @Override + public String getPurposeString() { + return "NN: special."; + } + + public ADWIN lossEstimator; + public String modelName; + protected Model nnmodel = null; + protected Trainer trainer = null; + protected int featureValuesArraySize = 0; + private MiniBatch miniBatch = null; + private int numberOfClasses; + private double [] votes; + private int gpuCount; + private final static DecimalFormat decimalFormat = new DecimalFormat("0.00000"); + + + @Override + public void resetLearningImpl() { + } + + public void trainOnMiniBatch(MiniBatch batch, boolean trainNet){ + NDList d = batch.d; + NDList l = batch.l; + try{ + samplesSeen ++; + // train mini batch + GradientCollector collector = trainer.newGradientCollector(); + NDList preds = trainer.forward(d, l); + NDArray lossValue = trainer.getLoss().evaluate(l, preds); + double loss = lossValue.getFloat(); + + if ((trainNet) && (loss > backPropLossThreshold.getValue())){ + trainedCount++; + try { + collector.backward(lossValue); + trainer.step(); // enforce the calculated weights + }catch (IllegalStateException e) + { + // trainer.step() throws above exception if all gradients are zero. + } + } + this.lossEstimator.setInput(loss); + collector.close(); + preds.close(); + lossValue.close(); + }catch (Exception e) { + System.err.println(e); + e.printStackTrace(); + System.exit(1); + } + } + + @Override + public void trainOnInstanceImpl(Instance inst) { + initializeNetwork(inst); + + + if (miniBatch == null){ + miniBatch = new MiniBatch(nnmodel.getNDManager().getDevice(), miniBatchSize.getValue()); + } + + if (useNormalization.isSet() || useOneHotEncode.isSet()){ + pClassValue[0] = inst.classValue(); + miniBatch.addToMiniBatch(pFeatureValues, pClassValue); + }else { + miniBatch.addToMiniBatch(inst); + } + + if (miniBatch.miniBatchFull() ){ + trainOnMiniBatch(miniBatch, true); + miniBatch.discardMiniBatch(); + miniBatch = null; + } + } + + public double[] getVotesForFeatureValues(Instance inst, double[] featureValues) { + initializeNetwork(inst); + + try { + NDManager testingNDManager = NDManager.newBaseManager(nnmodel.getNDManager().getDevice()); + NDList d = new NDList(testingNDManager.create(featureValues).toType(FLOAT32, false)); + NDList preds = trainer.evaluate(d); + + for (int i = 0; i < inst.numClasses(); i++) { + votes[i] = (double) preds.get(0).toFloatArray()[i]; + } + preds.close(); + d.close(); + testingNDManager.close(); + }catch (Exception e) { + System.err.println(e); + e.printStackTrace(); + System.exit(1); + } + + return votes; + } + + + public double[] getVotesForFeatureValues(Instance inst) { + initializeNetwork(inst); + + try { + NDManager testingNDManager = NDManager.newBaseManager(nnmodel.getNDManager().getDevice()); + double [] instDoubleA = inst.toDoubleArray(); + int featureLength = instDoubleA.length -1; + NDArray dataAndLabel = testingNDManager.create(instDoubleA).toType(FLOAT32, false); + NDArray data = dataAndLabel.get("0:" + Integer.toString(featureLength)); + + NDList d = new NDList(data); + NDList preds = trainer.evaluate(d); + + for (int i = 0; i < inst.numClasses(); i++) { + votes[i] = (double) preds.get(0).toFloatArray()[i]; + } + preds.close(); + d.close(); + testingNDManager.close(); + }catch (Exception e) { + System.err.println(e); + e.printStackTrace(); + System.exit(1); + } + + return votes; + } + + @Override + public double[] getVotesForInstance(Instance inst) { + + initializeNetwork(inst); + + if (useNormalization.isSet() || useOneHotEncode.isSet()){ + setFeatureValuesArray(inst, pFeatureValues, useOneHotEncode.isSet(), true, normalizeInfo, samplesSeen); + return getVotesForFeatureValues(inst, pFeatureValues); + }else{ + return getVotesForFeatureValues(inst); + } + } + + + @Override + public ImmutableCapabilities defineImmutableCapabilities() { + return new ImmutableCapabilities(Capability.VIEW_STANDARD, Capability.VIEW_LITE); + } + + @Override + protected Measurement[] getModelMeasurementsImpl() { + return null; + } + + @Override + public void getModelDescription(StringBuilder out, int indent) { + } + + public boolean isRandomizable() { + return false; + } + + public static int getFeatureValuesArraySize(Instance inst, boolean useOneHotEncoding){ + int totalOneHotEncodedSize = 0; + int totalOneHotEncodedInstances = 0; + for(int i=0; i < inst.numInputAttributes(); i++){ + if (useOneHotEncoding && inst.attribute(i).isNominal() && (inst.attribute(i).numValues() > 2) ){ + totalOneHotEncodedSize += inst.attribute(i).numValues(); + totalOneHotEncodedInstances ++; + } + } + return inst.numInputAttributes() + totalOneHotEncodedSize - totalOneHotEncodedInstances; + } + + public static double getNormalizedValue(double value, double sumOfValues, double sumOfSquares, long samplesSeen){ + // Normalize data + double variance = 0.0f; + double sd; + double mean = 0.0f; + if (samplesSeen > 1){ + mean = sumOfValues / samplesSeen; + variance = (sumOfSquares - ((sumOfValues * sumOfValues) / samplesSeen)) / samplesSeen; + } + sd = Math.sqrt(variance); + if (sd > 0.0f){ + return (value - mean) / (3 * sd); + } else{ + return 0.0f; + } + } + + public static void setFeatureValuesArray(Instance inst, double[] featureValuesArrayToSet, boolean useOneHotEncoding, boolean testing, NormalizeInfo[] normalizeInfo, long samplesSeen){ + int totalOneHotEncodedSize = 0; + int totalOneHotEncodedInstances = 0; + for(int i=0; i < inst.numInputAttributes(); i++){ + int index = i + totalOneHotEncodedSize - totalOneHotEncodedInstances; + if (useOneHotEncoding && inst.attribute(i).isNominal() && (inst.attribute(i).numValues() > 2) ){ + // Do one hot-encoding + featureValuesArrayToSet[index + (int)inst.value(i)] = 1.0f; + totalOneHotEncodedSize += inst.attribute(i).numValues(); + totalOneHotEncodedInstances ++; + }else + { + if( inst.attribute(i).isNumeric() && (normalizeInfo != null) && (normalizeInfo[index] != null) ){ + // Normalize data + if (testing) { + normalizeInfo[index].sumOfSquares += inst.value(i) * inst.value(i); + normalizeInfo[index].sumOfValues += inst.value(i); + } + featureValuesArrayToSet[index] = getNormalizedValue(inst.value(i), normalizeInfo[index].sumOfValues, normalizeInfo[index].sumOfSquares, samplesSeen); + }else{ + featureValuesArrayToSet[index] = inst.value(i); + } + } + } + } + + public void initializeNetwork(Instance inst) { + if (nnmodel != null){ + return; + } + + votes = new double [inst.numClasses()]; + + if (useNormalization.isSet() || useOneHotEncode.isSet()) { + pClassValue = new double[1]; + featureValuesArraySize = getFeatureValuesArraySize(inst, useOneHotEncode.isSet()); + pFeatureValues = new double[featureValuesArraySize]; + if (useNormalization.isSet()) { + normalizeInfo = new NormalizeInfo[featureValuesArraySize]; + for (int i = 0; i < normalizeInfo.length; i++) { + normalizeInfo[i] = new NormalizeInfo(); + } + } + }else{ + featureValuesArraySize = inst.numInputAttributes(); + } + + try { + gpuCount = Device.getGpuCount(); + + numberOfClasses = inst.numClasses(); + setModel(); + + lossEstimator = new ADWIN(deltaForADWIN); + + switch (this.optimizerTypeOption.getChosenIndex()){ + case MLP.OPTIMIZER_RMSPROP_RESET: + case MLP.OPTIMIZER_ADAGRAD_RESET: + case MLP.OPTIMIZER_ADAM_RESET: + break; + default: + break; + } + setTrainer(); + }catch (Exception e) { + System.err.println(e); + e.printStackTrace(); + } + modelName = "L" + numberOfLayers.getValue() + "_N" + numberOfNeuronsInEachLayerInLog2.getValue() +"_" + optimizerTypeOption.getChosenLabel() +"_" + decimalFormat.format(learningRateOption.getValue()); + } + + public double getLossEstimation(){ + return lossEstimator.getEstimation(); + } + + protected void setModel(){ + try{ + if ((deviceTypeOption.getChosenIndex() == deviceTypeOptionGPU) && (gpuCount == 0)){ + throw new RuntimeException("GPU selected as device. But NO GPUs detected."); + } + nnmodel = Model.newInstance("mlp", (deviceTypeOption.getChosenIndex() == deviceTypeOptionGPU) ? Device.gpu() : Device.cpu()); + // Construct a neural network and set it in the block + Integer [] neuronsInEachHiddenLayer = {}; + List neuronsInEachHiddenLayerArrayList = new ArrayList(Arrays.asList(neuronsInEachHiddenLayer)); + for(int i=0; i < numberOfLayers.getValue(); i++){ + neuronsInEachHiddenLayerArrayList.add((int) Math.pow(2, numberOfNeuronsInEachLayerInLog2.getValue())); + } + neuronsInEachHiddenLayer = neuronsInEachHiddenLayerArrayList.toArray(neuronsInEachHiddenLayer); + Block block = new Mlp(featureValuesArraySize, numberOfClasses, Arrays.stream(neuronsInEachHiddenLayer).mapToInt(Integer::intValue).toArray()); + nnmodel.setBlock(block); + System.out.println("System GPU count: "+ gpuCount + " Model using Device: " + nnmodel.getNDManager().getDevice()); + + }catch (Exception e) { + System.err.println(e); + e.printStackTrace(); + } + } + + protected void setTrainer(){ + if (trainer != null){ + trainer.close(); + trainer = null; + } + try { + Tracker learningRateTracker = Tracker.fixed((float) this.learningRateOption.getValue()); + Optimizer optimizer; + + switch(this.optimizerTypeOption.getChosenIndex()) { + case MLP.OPTIMIZER_RMSPROP_RESET: + case MLP.OPTIMIZER_RMSPROP: + optimizer = Optimizer.rmsprop().optLearningRateTracker(learningRateTracker).build(); + break; + case MLP.OPTIMIZER_ADAGRAD_RESET: + case MLP.OPTIMIZER_ADAGRAD: + optimizer = Optimizer.adagrad().optLearningRateTracker(learningRateTracker).build(); + break; + case MLP.OPTIMIZER_ADAM_RESET: + case MLP.OPTIMIZER_ADAM: + optimizer = Optimizer.adam().optLearningRateTracker(learningRateTracker).build(); + break; + case MLP.OPTIMIZER_SGD: + default: + optimizer = Optimizer.sgd().setLearningRateTracker(learningRateTracker).build(); + break; + } + Loss loss = Loss.softmaxCrossEntropyLoss(); + DefaultTrainingConfig config = new DefaultTrainingConfig(loss); + config.optOptimizer(optimizer); + trainer = nnmodel.newTrainer(config); + trainer.initialize(new Shape(miniBatchSize.getValue(), featureValuesArraySize)); + }catch (Exception e) { + System.err.println(e); + e.printStackTrace(); + } + } +} \ No newline at end of file From a120daf10673e5787027ca78aee64002652ee3fa Mon Sep 17 00:00:00 2001 From: Nuwan Gunasekara Date: Mon, 4 Jul 2022 10:36:58 +1200 Subject: [PATCH 2/7] Initial version of CAND --- .../moa/classifiers/neuralNetworks/CAND.java | 450 ++++++++++++++++++ 1 file changed, 450 insertions(+) create mode 100644 moa/src/main/java/moa/classifiers/neuralNetworks/CAND.java diff --git a/moa/src/main/java/moa/classifiers/neuralNetworks/CAND.java b/moa/src/main/java/moa/classifiers/neuralNetworks/CAND.java new file mode 100644 index 000000000..79c8212b6 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/neuralNetworks/CAND.java @@ -0,0 +1,450 @@ +/* + * CAND.java + * Copyright (C) 2022 University of Waikato, Hamilton, New Zealand + * @author Nuwan Gunasekara (ng98@students.waikato.ac.nz) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package moa.classifiers.neuralNetworks; + +import com.github.javacliparser.*; +import com.yahoo.labs.samoa.instances.Instance; +import moa.capabilities.CapabilitiesHandler; +import moa.capabilities.Capability; +import moa.capabilities.ImmutableCapabilities; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.MultiClassClassifier; +import moa.classifiers.core.driftdetection.ADWIN; +import moa.core.InstanceExample; +import moa.core.Measurement; +import moa.evaluation.BasicClassificationPerformanceEvaluator; + +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.*; + + +public class CAND extends AbstractClassifier implements MultiClassClassifier, CapabilitiesHandler { + + private static final long serialVersionUID = 1L; + protected MLP[] nn = null; + protected int featureValuesArraySize = 0; + protected long samplesSeen = 0; + protected MLP.NormalizeInfo[] normalizeInfo = null; + private double[] featureValues = null; + private double [] class_value = null; + private ExecutorService exService = null; + private FileWriter statsDumpFile = null; + private FileWriter votesDumpFile = null; + private BasicClassificationPerformanceEvaluator performanceEvaluator = new BasicClassificationPerformanceEvaluator(); + private long driftsDetectedPerSampleFrequency = 0; + private long totalDriftsDetected = 0; + private long avgMLPsPerSampleFrequency = 0; + private long lastGetModelMeasurementsImplCalledAt=0; + + private MiniBatch miniBatch = null; + + private ADWIN accEstimator = new ADWIN(1.0E-3); + + public static final int LARGER_P_POOL_10 = 0; + public static final int LARGER_P_POOL_30 = 1; + + public MultiChoiceOption largerPool = new MultiChoiceOption("largerPool", 'P', + "The larger pool type", + new String[]{"P10", "P30"}, + new String[]{ + "P10 = { learning rates: 5.0E-(1 to 5), optimizes: SGD,Adam, neurons in 1st layer: 2^(8 to 10) }", + "P30 = { learning rates: 5.0E-(1 to 5), optimizes: Adam, neurons in 1st layer: 2^9 }"}, + LARGER_P_POOL_30); + + public IntOption numberOfMLPsToTrainOption = new IntOption( + "numberOfMLPsToTrain", + 'o', + "Number of MLPs to train at a given time (after numberOfInstancesToTrainAllMLPsAtStart instances)", + 2, 2, Integer.MAX_VALUE); + + public IntOption numberOfLayersInEachMLP = new IntOption( + "numberOfLayersInEachMLP", + 'L', + "Number of layers in each MLP", + 1, 1, 4); + + public IntOption numberOfInstancesToTrainAllMLPsAtStartOption = new IntOption( + "numberOfInstancesToTrainAllMLPsAtStart", + 's', + "Number of instances to train all MLPs at start", + 100, 0, Integer.MAX_VALUE); + + public IntOption miniBatchSize = new IntOption( + "miniBatchSize", + 'B', + "Mini Batch Size", + 1, 1, 2048); + + + public FlagOption useOneHotEncode = new FlagOption("useOneHotEncode", 'h', + "use one hot encoding"); + + public FlagOption useNormalization = new FlagOption("useNormalization", 'n', + "Normalize data"); + + public FloatOption backPropLossThreshold = new FloatOption( + "backPropLossThreshold", + 'b', + "Back propagation loss threshold", + 0.0, 0.0, Math.pow(10,10)); + + public MultiChoiceOption deviceTypeOption = new MultiChoiceOption("deviceType", 'd', + "Choose device to run the model(For GPU, needs CUDA installed on the system. Use CPU if GPUs are not available)", + new String[]{"GPU","CPU"}, + new String[]{"GPU (Needs CUDA installed on the system. Use CPU if not available)", "CPU"}, + MLP.deviceTypeOptionCPU); + + public FlagOption doNotTrainEachMLPUsingASeparateThread = new FlagOption("doNotTrainEachMLPUsingASeparateThread", 't', + "Do NOT train each MLP using a separate thread"); + public StringOption votesDumpFileName = new StringOption("votesDumpFileName", 'f', + "Votes dump file name", + "" ); + public StringOption statsDumpFileName = new StringOption("statsDumpFileName", 'F', + "Stats dump file name", + "" ); + + @Override + public void resetLearningImpl() { + if (nn != null) { + exService.shutdownNow(); + exService = null; + for (int i = 0; i < this.nn.length; i++) { + nn[i] = null; + } + nn = null; + featureValuesArraySize = 0; + samplesSeen = 0; + normalizeInfo = null; + featureValues = null; + class_value = null; + } + } + + @Override + public void trainOnInstanceImpl(Instance instance) { + if(this.nn == null){ + initNNs(instance); + } + + class_value[0] = instance.classValue(); + + if (miniBatch == null){ + miniBatch = new MiniBatch(this.nn[0].nnmodel.getNDManager().getDevice(), miniBatchSize.getValue()); +// System.out.println("For training mini batch using device: " + trainingNDManager.getDevice()); + } + + miniBatch.addToMiniBatch(featureValues, class_value); + if (miniBatch.miniBatchFull() ){ + int numberOfMLPsToTrain = numberOfMLPsToTrainOption.getValue(); + int numberOfTopMLPsToTrain = numberOfMLPsToTrain /2; + if (samplesSeen < numberOfInstancesToTrainAllMLPsAtStartOption.getValue()){ + numberOfMLPsToTrain = nn.length; + numberOfTopMLPsToTrain = nn.length; + } + avgMLPsPerSampleFrequency += numberOfMLPsToTrain; + + Arrays.sort(this.nn, new Comparator() { + @Override + public int compare(MLP o1, MLP o2) { + return Double.compare(o1.getLossEstimation(), o2.getLossEstimation()); + } + }); + + boolean [] trainNetwork = new boolean[this.nn.length]; + for (int i =0; i < numberOfMLPsToTrain; i++) { + int nnIndex; + if (i < numberOfTopMLPsToTrain){ + // top most train + nnIndex = i; + }else { + // Random train + int offSet = (int) ((samplesSeen + i) % (this.nn.length - numberOfTopMLPsToTrain)); + nnIndex = numberOfTopMLPsToTrain + offSet; + } + trainNetwork[nnIndex] = true; + } + + class TrainThread implements Callable { + private final MLP mlp; + private final MiniBatch miniBatch; + private final boolean trainNet; + + public TrainThread(MLP mlp, MiniBatch miniBatch, boolean trainNet) { + this.mlp = mlp; + this.miniBatch = miniBatch; + this.trainNet = trainNet; + } + + @Override + public Boolean call() { + try { + this.mlp.trainOnMiniBatch(this.miniBatch, this.trainNet); + } catch (NullPointerException e){ + e.printStackTrace(); + System.exit(1); + } + return Boolean.TRUE; + } + } + + final Future [] runFuture = new Future[this.nn.length]; + + for (int i =0; i < this.nn.length; i++) { + if (! this.doNotTrainEachMLPUsingASeparateThread.isSet()){ + //start thread + runFuture[i] = exService.submit(new TrainThread(this.nn[i], this.miniBatch, trainNetwork[i])); + }else{ + this.nn[i].initializeNetwork(instance); + this.nn[i].trainOnMiniBatch(miniBatch, trainNetwork[i]); + } + } + + if (! this.doNotTrainEachMLPUsingASeparateThread.isSet()){ + // wait for threads to complete + int runningCount = this.nn.length; + while (runningCount != 0){ + runningCount = 0; + for (int i =0; i < this.nn.length; i++) { + try { + final Boolean returnedValue = runFuture[i].get(); + if (!returnedValue.equals(Boolean.TRUE)){ + runningCount++; + } + } catch (InterruptedException | ExecutionException e) { + e.printStackTrace(); + } + } + } + } + miniBatch.discardMiniBatch(); + miniBatch = null; + } + } + + private void printStats(){ + long sampleFrequency = samplesSeen - lastGetModelMeasurementsImplCalledAt; + if (statsDumpFile != null) { + for (int i = 0; i < this.nn.length; i++) { + try { + statsDumpFile.write(samplesSeen + "," + + this.nn[i].samplesSeen + "," + + this.nn[i].trainedCount + "," + + this.nn[i].modelName + "," + + performanceEvaluator.getPerformanceMeasurements()[1].getValue() + "," + + this.nn[i].lossEstimator.getEstimation() + "," + + totalDriftsDetected + "," + + sampleFrequency + "," + + driftsDetectedPerSampleFrequency + "," + + avgMLPsPerSampleFrequency / sampleFrequency + "\n"); + statsDumpFile.flush(); + } catch (IOException e) { + System.out.println("An error occurred."); + e.printStackTrace(); + } + } + } + } + + private void printVotes(Instance instance){ + if (votesDumpFile != null) { + for (int i = 0; i < this.nn.length; i++) { + try { + votesDumpFile.write(samplesSeen + "," + + this.nn[i].modelName + "," + + this.nn[i].lossEstimator.getEstimation() + "," + + instance.classValue() + "," + + instance.classIndex() + "," + + Arrays.toString(this.nn[i].getVotesForFeatureValues(instance, featureValues)) + + "\n"); + } catch (IOException e) { + System.out.println("An error occurred."); + e.printStackTrace(); + } + } + } + } + + @Override + public double[] getVotesForInstance(Instance instance) { + int chosenIndex = 0; + double [] votes; + samplesSeen ++; + + if(this.nn == null) { + initNNs(instance); + }else { + double minEstimation = Double.MAX_VALUE; + for (int i = 0 ; i < this.nn.length ; i++) { + if (this.nn[i].getLossEstimation() < minEstimation){ + minEstimation = this.nn[i].getLossEstimation(); + chosenIndex = i; + } + } + } + MLP.setFeatureValuesArray(instance, featureValues, useOneHotEncode.isSet(), true, normalizeInfo, samplesSeen); + + votes = this.nn[chosenIndex].getVotesForFeatureValues(instance, featureValues); + performanceEvaluator.addResult(new InstanceExample(instance), votes); + double lastAcc = accEstimator.getEstimation(); + accEstimator.setInput(performanceEvaluator.getPerformanceMeasurements()[1].getValue()); + if (accEstimator.getChange() && (accEstimator.getEstimation() < lastAcc)){ + totalDriftsDetected++; + driftsDetectedPerSampleFrequency++; + } + printVotes(instance); + return votes; + } + + @Override + public boolean isRandomizable() { + return false; + } + + @Override + public void getModelDescription(StringBuilder arg0, int arg1) { + } + + @Override + protected Measurement[] getModelMeasurementsImpl() { + printStats(); + driftsDetectedPerSampleFrequency = 0; + avgMLPsPerSampleFrequency = 0; + lastGetModelMeasurementsImplCalledAt = samplesSeen; + try{ + if (votesDumpFile != null) { + votesDumpFile.flush(); + } + }catch (IOException e) { + System.out.println("An error occurred."); + e.printStackTrace(); + } + return null; + } + + protected void initNNs(Instance instance) { + class MLPConfigs{ + private final int numberOfNeuronsInL1InLog2; + private final int optimizerType; + private final float learningRate; + private final double deltaForADWIN; + + MLPConfigs(int numberOfNeuronsInL1InLog2, int optimizerType, float learningRate, double deltaForADWIN){ + this.numberOfNeuronsInL1InLog2 = numberOfNeuronsInL1InLog2; + this.optimizerType = optimizerType; + this.learningRate = learningRate; + this.deltaForADWIN = deltaForADWIN; + } + } + + MLPConfigs [] nnConfigs = {}; + List nnConfigsArrayList = new ArrayList(Arrays.asList(nnConfigs)); + float [] denominator = {10.0f, 100.0f, 1000.0f, 10000.0f, 100000.0f}; + float [] numerator = new float [] {5.0f}; + for (int n=0; n < numerator.length; n++){ + for (int d=0; d < denominator.length; d++){ + float lr = numerator[n]/denominator[d]; + for (int numberOfNeuronsInL1InLog2 = 8; numberOfNeuronsInL1InLog2 < 11; numberOfNeuronsInL1InLog2++){ + + if (largerPool.getChosenIndex() == LARGER_P_POOL_30) { + nnConfigsArrayList.add(new MLPConfigs(numberOfNeuronsInL1InLog2, MLP.OPTIMIZER_SGD, lr, 1.0E-3)); + }else{ // LARGER_P_POOL_10 + if ( numberOfNeuronsInL1InLog2 == 9){ + continue; + }else{ + // numberOfNeuronsInL1InLog2 = 8 or 10 + } + } + nnConfigsArrayList.add(new MLPConfigs(numberOfNeuronsInL1InLog2, MLP.OPTIMIZER_ADAM, lr, 1.0E-3)); + } + } + } + nnConfigs = nnConfigsArrayList.toArray(nnConfigs); + + this.nn = new MLP[nnConfigs.length]; + for(int i=0; i < nnConfigs.length; i++){ + this.nn[i] = new MLP(); + this.nn[i].optimizerTypeOption.setChosenIndex(nnConfigs[i].optimizerType); + this.nn[i].learningRateOption.setValue(nnConfigs[i].learningRate); + this.nn[i].useOneHotEncode.setValue(useOneHotEncode.isSet()); + this.nn[i].deviceTypeOption.setChosenIndex(deviceTypeOption.getChosenIndex()); + this.nn[i].numberOfNeuronsInEachLayerInLog2.setValue(nnConfigs[i].numberOfNeuronsInL1InLog2); + this.nn[i].numberOfLayers.setValue(numberOfLayersInEachMLP.getValue()); + this.nn[i].deltaForADWIN = nnConfigs[i].deltaForADWIN; + this.nn[i].backPropLossThreshold.setValue(backPropLossThreshold.getValue()); + this.nn[i].initializeNetwork(instance); + } + + try { + if (statsDumpFileName.getValue().length() > 0) { + statsDumpFile = new FileWriter(statsDumpFileName.getValue()); + statsDumpFile.write("id," + + "samplesSeenAtTrain," + + "trainedCount," + + "optimizer_type_learning_rate_delta," + + "acc," + + "estimated_loss," + + "totalDriftsDetected," + + "sampleFrequency," + + "driftsDetectedPerSampleFrequency," + + "avgMLPsPerSampleFrequency" + + "\n"); + statsDumpFile.flush(); + } + if (votesDumpFileName.getValue().length() > 0 ) { + votesDumpFile = new FileWriter(votesDumpFileName.getValue()); + votesDumpFile.write("id," + + "modelName," + + "estimated_loss," + + "classValue," + + "classIndex," + + "votes," + + "\n"); + votesDumpFile.flush(); + } + } catch (IOException e) { + System.out.println("An error occurred."); + e.printStackTrace(); + } + + exService = Executors.newFixedThreadPool(nnConfigs.length); + + class_value = new double[1]; + featureValuesArraySize = MLP.getFeatureValuesArraySize(instance, useOneHotEncode.isSet()); + System.out.println("Number of features before one-hot encode: " + instance.numInputAttributes() + " : Number of features after one-hot encode: " + featureValuesArraySize); + featureValues = new double [featureValuesArraySize]; + if (useNormalization.isSet()) { + normalizeInfo = new MLP.NormalizeInfo[featureValuesArraySize]; + for(int i=0; i < normalizeInfo.length; i++){ + normalizeInfo[i] = new MLP.NormalizeInfo(); + } + } + } + + @Override + public ImmutableCapabilities defineImmutableCapabilities() { + return new ImmutableCapabilities(Capability.VIEW_STANDARD, Capability.VIEW_LITE); + } +} From 283a6ffb8119cba8954f5e9c8ac72880feb38174 Mon Sep 17 00:00:00 2001 From: Nuwan Gunasekara Date: Sun, 31 Jul 2022 13:55:01 +1200 Subject: [PATCH 3/7] Unit tests for CAND and MLP --- .../classifiers/neuralNetworks/CANDTest.java | 88 +++++++++++++++++++ .../classifiers/neuralNetworks/MLPTest.java | 87 ++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 moa/src/test/java/moa/classifiers/neuralNetworks/CANDTest.java create mode 100644 moa/src/test/java/moa/classifiers/neuralNetworks/MLPTest.java diff --git a/moa/src/test/java/moa/classifiers/neuralNetworks/CANDTest.java b/moa/src/test/java/moa/classifiers/neuralNetworks/CANDTest.java new file mode 100644 index 000000000..9c1667c61 --- /dev/null +++ b/moa/src/test/java/moa/classifiers/neuralNetworks/CANDTest.java @@ -0,0 +1,88 @@ +/* + * StreamingRandomPatchesTest.java + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/** + * + */ +package moa.classifiers.neuralNetworks; + +import junit.framework.Test; +import junit.framework.TestSuite; +import moa.classifiers.AbstractMultipleClassifierTestCase; +import moa.classifiers.Classifier; + +/** + * Tests the CAND classifier. + * + * @author Nuwan Gunasekara (ng98 at students dot waikato dot ac dot nz) + * @version $Revision$ + */ +public class CANDTest + extends AbstractMultipleClassifierTestCase { + + /** + * Constructs the test case. Called by subclasses. + * + * @param name the name of the test + */ + public CANDTest(String name) { + super(name); + this.setNumberTests(1); + } + + /** + * Returns the classifier setups to use in the regression test. + * + * @return the setups + */ + @Override + protected Classifier[] getRegressionClassifierSetups() { + CAND CANDtest = new CAND(); + CANDtest.largerPool.setChosenIndex(1); + CANDtest.numberOfMLPsToTrainOption.setValue(2); + CANDtest.numberOfLayersInEachMLP.setValue(1); + CANDtest.numberOfInstancesToTrainAllMLPsAtStartOption.setValue(100); + CANDtest.miniBatchSize.setValue(4); + CANDtest.useOneHotEncode.setValue(true); + CANDtest.useNormalization.setValue(true); + CANDtest.backPropLossThreshold.setValue(0.3); + CANDtest.deviceTypeOption.setChosenIndex(1); + + + return new Classifier[]{ + CANDtest, + }; + } + + /** + * Returns a test suite. + * + * @return the test suite + */ + public static Test suite() { + return new TestSuite(CANDTest.class); + } + + /** + * Runs the test from commandline. + * + * @param args ignored + */ + public static void main(String[] args) { + runTest(suite()); + } +} diff --git a/moa/src/test/java/moa/classifiers/neuralNetworks/MLPTest.java b/moa/src/test/java/moa/classifiers/neuralNetworks/MLPTest.java new file mode 100644 index 000000000..9c98bd12c --- /dev/null +++ b/moa/src/test/java/moa/classifiers/neuralNetworks/MLPTest.java @@ -0,0 +1,87 @@ +/* + * StreamingRandomPatchesTest.java + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/** + * + */ +package moa.classifiers.neuralNetworks; + +import junit.framework.Test; +import junit.framework.TestSuite; +import moa.classifiers.AbstractMultipleClassifierTestCase; +import moa.classifiers.Classifier; + +/** + * Tests the MLP classifier. + * + * @author Nuwan Gunasekara (ng98 at students dot waikato dot ac dot nz) + * @version $Revision$ + */ +public class MLPTest + extends AbstractMultipleClassifierTestCase { + + /** + * Constructs the test case. Called by subclasses. + * + * @param name the name of the test + */ + public MLPTest(String name) { + super(name); + this.setNumberTests(1); + } + + /** + * Returns the classifier setups to use in the regression test. + * + * @return the setups + */ + @Override + protected Classifier[] getRegressionClassifierSetups() { + MLP MLPtest = new MLP(); + MLPtest.learningRateOption.setValue(0.03); + MLPtest.backPropLossThreshold.setValue(0.3); + MLPtest.optimizerTypeOption.setChosenIndex(5); + MLPtest.useOneHotEncode.setValue(true); + MLPtest.useNormalization.setValue(true); + MLPtest.numberOfNeuronsInEachLayerInLog2.setValue(9); + MLPtest.numberOfLayers.setValue(1); + MLPtest.miniBatchSize.setValue(4); + MLPtest.deviceTypeOption.setChosenIndex(1); + + return new Classifier[]{ + MLPtest, + }; + } + + /** + * Returns a test suite. + * + * @return the test suite + */ + public static Test suite() { + return new TestSuite(MLPTest.class); + } + + /** + * Runs the test from commandline. + * + * @param args ignored + */ + public static void main(String[] args) { + runTest(suite()); + } +} \ No newline at end of file From 38fac44188fd6a87d8fd0ecfd29fbd7e62247285 Mon Sep 17 00:00:00 2001 From: Nuwan Gunasekara Date: Sun, 31 Jul 2022 14:46:22 +1200 Subject: [PATCH 4/7] Add reference and description. --- .../moa/classifiers/neuralNetworks/CAND.java | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/moa/src/main/java/moa/classifiers/neuralNetworks/CAND.java b/moa/src/main/java/moa/classifiers/neuralNetworks/CAND.java index 79c8212b6..b035ec9de 100644 --- a/moa/src/main/java/moa/classifiers/neuralNetworks/CAND.java +++ b/moa/src/main/java/moa/classifiers/neuralNetworks/CAND.java @@ -40,6 +40,40 @@ import java.util.List; import java.util.concurrent.*; +/** + * Continuously Adaptive Neural networks for Data streams + * + *

Continuously Adaptive Neural networks for Data streams (CAND). For every prediction, CAND chooses the current best + * network from a pool of candidates by continuously monitoring the performance of all candidate networks. The candidates + * are trained using different optimizers and hyperparameters. There are two orthogonal heuristics (skip back propagation, + * smaller training pool) for accelerating CAND, which trade-off small amounts of accuracy for significant runtime gains. + * When training, small mini-batches yields similar accuracy to single-instance fully incremental training, even on + * evolving data streams.

+ * + *

See details in:
Nuwan Gunasekara, Heitor Murilo Gomes, Bernhard Pfahringer, Albert Bifet. + * Online Hyperparameter Optimization for Streaming Neural Networks. + * International Joint Conference on Neural Networks (IJCNN), 2022.

+ * + *

Parameters:

    + *
  • -P : The larger pool type. + * P10 = { learning rates: 5.0E-(1 to 5), optimizes: SGD,Adam, neurons in 1st layer: 2^(8 to 10) }, + * P30 = { learning rates: 5.0E-(1 to 5), optimizes: Adam, neurons in 1st layer: 2^9 }
    • + *
  • -o : Number of MLPs to train at a given time (after -s numberOfInstancesToTrainAllMLPsAtStart instances).
    • + *
  • -L : Number of layers in each MLP.
    • + *
  • -s : Number of instances to train all MLPs at start.
    • + *
  • -B : Mini Batch Size.
    • + *
  • -h : Use one hot encoding.
    • + *
  • -n : Normalize data.
    • + *
  • -b : Skip back propagation loss threshold.
    • + *
  • -d : Choose device to run the model(For GPU, needs CUDA installed on the system. Use CPU if GPUs are not available)
    • + *
  • -t : Do NOT train each MLP using a separate thread.
    • + *
  • -f : Votes dump file name.
    • + *
  • -F : Stats dump file name.
    • + *
+ * + * @author Nuwan Gunasekara (ng98 at students dot waikato dot ac dot nz) + * @version $Revision: 1 $ + */ public class CAND extends AbstractClassifier implements MultiClassClassifier, CapabilitiesHandler { @@ -108,7 +142,7 @@ public class CAND extends AbstractClassifier implements MultiClassClassifier, Ca public FloatOption backPropLossThreshold = new FloatOption( "backPropLossThreshold", 'b', - "Back propagation loss threshold", + "Skip back propagation loss threshold", 0.0, 0.0, Math.pow(10,10)); public MultiChoiceOption deviceTypeOption = new MultiChoiceOption("deviceType", 'd', From 4457873e15c8d4a338ca9a48d5b9eecfe60c405e Mon Sep 17 00:00:00 2001 From: Nuwan Gunasekara Date: Thu, 18 Aug 2022 22:18:49 +1200 Subject: [PATCH 5/7] Rename the package directory to 'deeplearning' --- .../moa/classifiers/{neuralNetworks => deeplearning}/CAND.java | 2 +- .../moa/classifiers/{neuralNetworks => deeplearning}/MLP.java | 3 +-- .../classifiers/{neuralNetworks => deeplearning}/CANDTest.java | 2 +- .../classifiers/{neuralNetworks => deeplearning}/MLPTest.java | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) rename moa/src/main/java/moa/classifiers/{neuralNetworks => deeplearning}/CAND.java (99%) rename moa/src/main/java/moa/classifiers/{neuralNetworks => deeplearning}/MLP.java (99%) rename moa/src/test/java/moa/classifiers/{neuralNetworks => deeplearning}/CANDTest.java (98%) rename moa/src/test/java/moa/classifiers/{neuralNetworks => deeplearning}/MLPTest.java (98%) diff --git a/moa/src/main/java/moa/classifiers/neuralNetworks/CAND.java b/moa/src/main/java/moa/classifiers/deeplearning/CAND.java similarity index 99% rename from moa/src/main/java/moa/classifiers/neuralNetworks/CAND.java rename to moa/src/main/java/moa/classifiers/deeplearning/CAND.java index b035ec9de..855778feb 100644 --- a/moa/src/main/java/moa/classifiers/neuralNetworks/CAND.java +++ b/moa/src/main/java/moa/classifiers/deeplearning/CAND.java @@ -18,7 +18,7 @@ * */ -package moa.classifiers.neuralNetworks; +package moa.classifiers.deeplearning; import com.github.javacliparser.*; import com.yahoo.labs.samoa.instances.Instance; diff --git a/moa/src/main/java/moa/classifiers/neuralNetworks/MLP.java b/moa/src/main/java/moa/classifiers/deeplearning/MLP.java similarity index 99% rename from moa/src/main/java/moa/classifiers/neuralNetworks/MLP.java rename to moa/src/main/java/moa/classifiers/deeplearning/MLP.java index ed8a9e3af..d1a54794d 100644 --- a/moa/src/main/java/moa/classifiers/neuralNetworks/MLP.java +++ b/moa/src/main/java/moa/classifiers/deeplearning/MLP.java @@ -17,7 +17,7 @@ * along with this program. If not, see . * */ -package moa.classifiers.neuralNetworks; +package moa.classifiers.deeplearning; import com.github.javacliparser.FlagOption; import com.github.javacliparser.IntOption; @@ -28,7 +28,6 @@ import moa.classifiers.core.driftdetection.ADWIN; import moa.core.Measurement; import com.yahoo.labs.samoa.instances.Instance; -import com.yahoo.labs.samoa.instances.InstancesHeader; import com.github.javacliparser.FloatOption; import com.github.javacliparser.MultiChoiceOption; diff --git a/moa/src/test/java/moa/classifiers/neuralNetworks/CANDTest.java b/moa/src/test/java/moa/classifiers/deeplearning/CANDTest.java similarity index 98% rename from moa/src/test/java/moa/classifiers/neuralNetworks/CANDTest.java rename to moa/src/test/java/moa/classifiers/deeplearning/CANDTest.java index 9c1667c61..cc954704a 100644 --- a/moa/src/test/java/moa/classifiers/neuralNetworks/CANDTest.java +++ b/moa/src/test/java/moa/classifiers/deeplearning/CANDTest.java @@ -18,7 +18,7 @@ /** * */ -package moa.classifiers.neuralNetworks; +package moa.classifiers.deeplearning; import junit.framework.Test; import junit.framework.TestSuite; diff --git a/moa/src/test/java/moa/classifiers/neuralNetworks/MLPTest.java b/moa/src/test/java/moa/classifiers/deeplearning/MLPTest.java similarity index 98% rename from moa/src/test/java/moa/classifiers/neuralNetworks/MLPTest.java rename to moa/src/test/java/moa/classifiers/deeplearning/MLPTest.java index 9c98bd12c..2f7a3fb6d 100644 --- a/moa/src/test/java/moa/classifiers/neuralNetworks/MLPTest.java +++ b/moa/src/test/java/moa/classifiers/deeplearning/MLPTest.java @@ -18,7 +18,7 @@ /** * */ -package moa.classifiers.neuralNetworks; +package moa.classifiers.deeplearning; import junit.framework.Test; import junit.framework.TestSuite; From 1e15e3a1fa5db9f1616190f2cdd95760bae72b5c Mon Sep 17 00:00:00 2001 From: Nuwan Gunasekara Date: Thu, 18 Aug 2022 23:36:42 +1200 Subject: [PATCH 6/7] As per the paper, use the best performing defaults. --- moa/src/main/java/moa/classifiers/deeplearning/CAND.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moa/src/main/java/moa/classifiers/deeplearning/CAND.java b/moa/src/main/java/moa/classifiers/deeplearning/CAND.java index 855778feb..0316affa2 100644 --- a/moa/src/main/java/moa/classifiers/deeplearning/CAND.java +++ b/moa/src/main/java/moa/classifiers/deeplearning/CAND.java @@ -112,7 +112,7 @@ public class CAND extends AbstractClassifier implements MultiClassClassifier, Ca "numberOfMLPsToTrain", 'o', "Number of MLPs to train at a given time (after numberOfInstancesToTrainAllMLPsAtStart instances)", - 2, 2, Integer.MAX_VALUE); + 10, 2, Integer.MAX_VALUE); public IntOption numberOfLayersInEachMLP = new IntOption( "numberOfLayersInEachMLP", @@ -143,7 +143,7 @@ public class CAND extends AbstractClassifier implements MultiClassClassifier, Ca "backPropLossThreshold", 'b', "Skip back propagation loss threshold", - 0.0, 0.0, Math.pow(10,10)); + 0.3, 0.0, Math.pow(10,10)); public MultiChoiceOption deviceTypeOption = new MultiChoiceOption("deviceType", 'd', "Choose device to run the model(For GPU, needs CUDA installed on the system. Use CPU if GPUs are not available)", From 21edd30576b12b90f95a978534ef6a7d4fdd238d Mon Sep 17 00:00:00 2001 From: Nuwan Gunasekara Date: Tue, 6 Sep 2022 21:06:55 +1200 Subject: [PATCH 7/7] Fixing issue #260 - Add items to ADWIN ensemble only if drift detection is enabled. Drift detection is off by default. --- .../java/moa/classifiers/meta/imbalanced/OnlineAdaBoost.java | 4 +++- .../java/moa/classifiers/meta/imbalanced/OnlineAdaC2.java | 4 +++- .../main/java/moa/classifiers/meta/imbalanced/OnlineCSB2.java | 4 +++- .../java/moa/classifiers/meta/imbalanced/OnlineRUSBoost.java | 4 +++- .../moa/classifiers/meta/imbalanced/OnlineSMOTEBagging.java | 4 +++- .../classifiers/meta/imbalanced/OnlineUnderOverBagging.java | 4 +++- 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineAdaBoost.java b/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineAdaBoost.java index 98276ff75..0bde95042 100644 --- a/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineAdaBoost.java +++ b/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineAdaBoost.java @@ -227,7 +227,9 @@ protected void adjustEnsembleSize(int nClasses) { for (int i = this.nEstimators; i < nClasses; i++) { this.ensemble.add(this.baseLearner.copy()); this.nEstimators ++; - this.adwinEnsemble.add(new ADWIN()); + if (this.driftDetection) { + this.adwinEnsemble.add(new ADWIN()); + } this.lambdaSc.add(0.0); this.lambdaSw.add(0.0); this.epsilon.add(0.0); diff --git a/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineAdaC2.java b/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineAdaC2.java index 110370401..7048033a1 100644 --- a/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineAdaC2.java +++ b/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineAdaC2.java @@ -256,7 +256,9 @@ protected void adjustEnsembleSize(int nClasses) { for (int i = this.nEstimators; i < nClasses; i++) { this.ensemble.add(this.baseLearner.copy()); this.nEstimators ++; - this.adwinEnsemble.add(new ADWIN()); + if (this.driftDetection) { + this.adwinEnsemble.add(new ADWIN()); + } this.lambdaTP.add(0.0); this.lambdaTN.add(0.0); this.lambdaFP.add(0.0); diff --git a/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineCSB2.java b/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineCSB2.java index f7fb8575a..f84805014 100644 --- a/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineCSB2.java +++ b/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineCSB2.java @@ -249,7 +249,9 @@ protected void adjustEnsembleSize(int nClasses) { for (int i = this.nEstimators; i < nClasses; i++) { this.ensemble.add(this.baseLearner.copy()); this.nEstimators ++; - this.adwinEnsemble.add(new ADWIN()); + if (this.driftDetection) { + this.adwinEnsemble.add(new ADWIN()); + } this.lambdaFP.add(0.0); this.lambdaFN.add(0.0); this.lambdaSum.add(0.0); diff --git a/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineRUSBoost.java b/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineRUSBoost.java index 80bd2e828..f9b93d5ea 100644 --- a/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineRUSBoost.java +++ b/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineRUSBoost.java @@ -291,7 +291,9 @@ protected void adjustEnsembleSize(int nClasses) { for (int i = this.nEstimators; i < nClasses; i++) { this.ensemble.add(this.baseLearner.copy()); this.nEstimators ++; - this.adwinEnsemble.add(new ADWIN()); + if (this.driftDetection) { + this.adwinEnsemble.add(new ADWIN()); + } this.lambdaSc.add(0.0); this.lambdaPos.add(0.0); this.lambdaNeg.add(0.0); diff --git a/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineSMOTEBagging.java b/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineSMOTEBagging.java index f4191a558..eaaf8b1ef 100644 --- a/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineSMOTEBagging.java +++ b/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineSMOTEBagging.java @@ -231,7 +231,9 @@ protected void adjustEnsembleSize(int nClasses) { for (int i = this.nEstimators; i < nClasses; i++) { this.ensemble.add(this.baseLearner.copy()); this.nEstimators ++; - this.adwinEnsemble.add(new ADWIN()); + if (this.driftDetection) { + this.adwinEnsemble.add(new ADWIN()); + } } } } diff --git a/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineUnderOverBagging.java b/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineUnderOverBagging.java index aed93282f..9235dfe5c 100644 --- a/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineUnderOverBagging.java +++ b/moa/src/main/java/moa/classifiers/meta/imbalanced/OnlineUnderOverBagging.java @@ -209,7 +209,9 @@ protected void adjustEnsembleSize(int nClasses) { for (int i = this.nEstimators; i < nClasses; i++) { this.ensemble.add(this.baseLearner.copy()); this.nEstimators ++; - this.adwinEnsemble.add(new ADWIN()); + if (this.driftDetection) { + this.adwinEnsemble.add(new ADWIN()); + } } } }