Converted neural networks to use new flat volume structure, part of #31

vol3/vol3-java-examples/src/main/java/com/heatonresearch/aifh/ann/BasicLayer.java

@@ -31,6 +31,7 @@
 import com.heatonresearch.aifh.AIFHError;
 import com.heatonresearch.aifh.ann.activation.ActivationFunction;
 import com.heatonresearch.aifh.ann.train.GradientCalc;
+import com.heatonresearch.aifh.flat.FlatVolume;
 import com.heatonresearch.aifh.randomize.GenerateRandom;
 
 /**
@@ -48,6 +49,8 @@ public class BasicLayer extends WeightedLayer {
      */
     private boolean hasBias;
 
+    private FlatVolume layerOutput;
+    private FlatVolume layerSums;
 
     /**
      * Do not use this constructor.  This was added to support serialization.
@@ -70,6 +73,9 @@ public BasicLayer(final ActivationFunction theActivation, boolean theHasBias, in
         setActivation(theActivation);
         this.hasBias = theHasBias;
         this.count = theCount;
+
+        this.layerOutput = new FlatVolume(theCount, hasBias);
+        this.layerSums = new FlatVolume(theCount, hasBias);
     }
 
     /**
@@ -79,7 +85,7 @@ public BasicLayer(final ActivationFunction theActivation, boolean theHasBias, in
      * @param theCount The neuron count.
      */
     public BasicLayer(final ActivationFunction theActivation, boolean theHasBias, int theCount) {
-        this(theActivation,theHasBias,new int[] {theCount});
+        this(theActivation,theHasBias,new int[] {theCount,1,1});
     }
 
 
@@ -192,4 +198,12 @@ public int getNeuronDepthUnit() {
             return this.count[0];
         }
     }
+
+    public FlatVolume getLayerOutput() {
+        return layerOutput;
+    }
+
+    public FlatVolume getLayerSums() {
+        return layerSums;
+    }
 }

vol3/vol3-java-examples/src/main/java/com/heatonresearch/aifh/ann/BasicNetwork.java

@@ -30,6 +30,7 @@
 
 import com.heatonresearch.aifh.AIFHError;
 import com.heatonresearch.aifh.ann.randomize.XaiverRandomizeNetwork;
+import com.heatonresearch.aifh.flat.FlatData;
 import com.heatonresearch.aifh.learning.ClassificationAlgorithm;
 import com.heatonresearch.aifh.learning.RegressionAlgorithm;
 import com.heatonresearch.aifh.util.ArrayUtil;
@@ -51,16 +52,6 @@ public class BasicNetwork implements RegressionAlgorithm, ClassificationAlgorith
      */
     private int inputCount;
 
-    /**
-     * The outputs from each of the neurons.
-     */
-    private double[] layerOutput;
-
-    /**
-     * The sum of the layer, before the activation function is applied, producing the layerOutput.
-     */
-    private double[] layerSums;
-
     /**
      * The number of output neurons in this network.
      */
@@ -81,6 +72,8 @@ public class BasicNetwork implements RegressionAlgorithm, ClassificationAlgorith
      */
     private boolean networkTraining;
 
+    private FlatData layerOutput = new FlatData();
+
     /**
      * Default constructor.
      */
@@ -99,18 +92,15 @@ public BasicNetwork() {
     public void compute(final double[] input, final double[] output) {
         clearOutput();
 
-        final int sourceIndex = getNeuronCount()
-                - this.layers.get(0).getTotalCount();
-
-        System.arraycopy(input, 0, this.layerOutput, sourceIndex,
+        System.arraycopy(input, 0, getLayerOutput().getData() ,getInputLayer().getLayerOutput().getOffset(),
                 this.inputCount);
 
         for (int i = 1; i<this.layers.size(); i++) {
             this.layers.get(i).computeLayer();
         }
 
-
-        System.arraycopy(this.layerOutput, 0, output, 0, this.outputCount);
+        System.arraycopy( getLayerOutput().getData() ,getOutputLayer().getLayerOutput().getOffset(),
+                output, 0, this.outputCount);
     }
 
     /**
@@ -139,13 +129,6 @@ public int getInputCount() {
         return this.inputCount;
     }
 
-    /**
-     * @return The output for each layer.
-     */
-    public double[] getLayerOutput() {
-        return this.layerOutput;
-    }
-
     /**
      * @return The number of output neurons.
      */
@@ -179,14 +162,6 @@ public void setOutputCount(final int outputCount) {
         this.outputCount = outputCount;
     }
 
-
-    /**
-     * @return the layerSums
-     */
-    public double[] getLayerSums() {
-        return this.layerSums;
-    }
-
     /**
      * Get the weight between the two layers.
      * @param fromLayer The from layer.
@@ -263,16 +238,14 @@ public void finalizeStructure() {
         TempStructureCounts counts = new TempStructureCounts();
 
         for (int i = this.layers.size() - 1; i >= 0; i--) {
-
             final Layer layer = this.layers.get(i);
-
             layer.finalizeStructure(this, i, counts);
-
+            this.layerOutput.addFlatObject(layer.getLayerSums());
+            this.layerOutput.addFlatObject(layer.getLayerOutput());
         }
 
+        this.layerOutput.finalizeStructure();
         this.weights = new double[counts.getWeightCount()];
-        this.layerOutput = new double[counts.getNeuronCount()];
-        this.layerSums = new double[counts.getNeuronCount()];
 
         clearOutput();
     }
@@ -281,18 +254,14 @@ public void finalizeStructure() {
      * Clear the outputs of each layer.
      */
     public void clearOutput() {
-        // Clear all outputs to 0
-        for(int i=0;i<this.layerOutput.length;i++) {
-            this.layerOutput[i] = 0.0;
-            this.layerSums[i] = 0.0;
-        }
-        // Init the output arrays by filling in bias values
-        int index = 0;
-        for (int i = 0; i < this.layers.size(); i++) {
-            Layer layer = this.layers.get(this.layers.size()-1-i);
-            index += layer.getCount();
-            if (layer.hasBias()) {
-                this.layerOutput[index++] = 1.0;
+        for(Layer layer: this.layers) {
+            for(int i=0;i<layer.getLayerOutput().getLength();i++) {
+                layer.getLayerOutput().set(i, 0);
+                layer.getLayerSums().set(i, 0);
+            }
+
+            if( layer.hasBias()) {
+                layer.getLayerOutput().set(layer.getCount(), 1);
             }
         }
     }
@@ -424,4 +393,25 @@ public boolean isNetworkTraining() {
     public void setNetworkTraining(boolean networkTraining) {
         this.networkTraining = networkTraining;
     }
+
+    public Layer getInputLayer() {
+        return this.layers.get(0);
+    }
+
+    public Layer getOutputLayer() {
+        return this.layers.get(this.layers.size()-1);
+    }
+
+    public FlatData getLayerOutput() {
+        return layerOutput;
+    }
+
+    public void dumpOutputs() {
+        int i = 0;
+        for(Layer layer: this.layers) {
+            i++;
+            System.out.println("Layer #" + i + ":Sums=" + layer.getLayerSums()
+                    + ",Output=" + layer.getLayerOutput()  );
+        }
+    }
 }

vol3/vol3-java-examples/src/main/java/com/heatonresearch/aifh/ann/Conv2DLayer.java

@@ -31,6 +31,7 @@
 import com.heatonresearch.aifh.AIFHError;
 import com.heatonresearch.aifh.ann.activation.ActivationFunction;
 import com.heatonresearch.aifh.ann.train.GradientCalc;
+import com.heatonresearch.aifh.flat.FlatVolume;
 import com.heatonresearch.aifh.randomize.GenerateRandom;
 
 /**
@@ -81,6 +82,10 @@ public class Conv2DLayer extends WeightedLayer {
      */
     private int inDepth;
 
+    private FlatVolume layerOutput;
+    private FlatVolume layerSums;
+
+
     /**
      * Construct a 2D convolution layer.
      * @param theActivation The activation function.
@@ -93,6 +98,11 @@ public Conv2DLayer(final ActivationFunction theActivation, int theNumFilters, in
         this.filterRows = theFilterRows;
         this.filterColumns = theFilterColumns;
         this.numFilters = theNumFilters;
+        int[] shape = {theFilterRows, theFilterColumns, theNumFilters};
+
+        this.layerOutput = new FlatVolume(shape, true);
+        this.layerSums = new FlatVolume(shape, true);
+
     }
 
     /**
@@ -134,6 +144,16 @@ public int getNeuronDepthUnit() {
         return this.filterColumns * this.filterRows;
     }
 
+    @Override
+    public FlatVolume getLayerOutput() {
+        return this.layerOutput;
+    }
+
+    @Override
+    public FlatVolume getLayerSums() {
+        return this.layerSums;
+    }
+
     /**
      * {@inheritDoc}
      */

vol3/vol3-java-examples/src/main/java/com/heatonresearch/aifh/ann/Layer.java

@@ -30,6 +30,7 @@
 
 import com.heatonresearch.aifh.ann.activation.ActivationFunction;
 import com.heatonresearch.aifh.ann.train.GradientCalc;
+import com.heatonresearch.aifh.flat.FlatVolume;
 import com.heatonresearch.aifh.randomize.GenerateRandom;
 
 /**
@@ -130,4 +131,9 @@ void finalizeStructure(BasicNetwork theOwner, int theLayerIndex,
      */
     int getNeuronDepthUnit();
 
+    FlatVolume getLayerOutput();
+
+    FlatVolume getLayerSums();
+
+
 }

vol3/vol3-java-examples/src/main/java/com/heatonresearch/aifh/ann/MaxPoolLayer.java

@@ -30,6 +30,7 @@
 
 import com.heatonresearch.aifh.ann.activation.ActivationFunction;
 import com.heatonresearch.aifh.ann.train.GradientCalc;
+import com.heatonresearch.aifh.flat.FlatVolume;
 import com.heatonresearch.aifh.randomize.GenerateRandom;
 
 /**
@@ -136,4 +137,14 @@ public int getWeightDepthUnit() {
     public int getNeuronDepthUnit() {
         return 0;
     }
+
+    @Override
+    public FlatVolume getLayerOutput() {
+        return null;
+    }
+
+    @Override
+    public FlatVolume getLayerSums() {
+        return null;
+    }
 }

vol3/vol3-java-examples/src/main/java/com/heatonresearch/aifh/ann/WeightedLayer.java

@@ -118,16 +118,24 @@ public void computeLayer(int inputOffset, int outputOffset, int fromCount, int t
 
             for (int y = 0; y < fromCount; y++) {
                 if(prev.isActive(ix) && isActive(y)) {
-                    sum += weights[index] * getOwner().getLayerOutput()[prev.getNeuronIndex()+y];
+                    double temp = weights[index] * prev.getLayerOutput().get(y);
+                    sum += weights[index] * prev.getLayerOutput().get(y);
+                    //System.out.println(index+":" + weights[index] + "*" + prev.getLayerOutput().get(y) + "=" + temp );
+                    //System.out.println("Sum:" + sum);
                 }
                 index++;
             }
-            getOwner().getLayerSums()[x] += sum;
-            getOwner().getLayerOutput()[x] += sum;
+            //System.out.println("sums["+ix+"]="+sum);
+            getLayerSums().add(ix, sum);
+            getLayerOutput().add(ix, sum);
         }
 
         getActivation().activationFunction(
-                getOwner().getLayerOutput(), getNeuronIndex(), toCount);
+                this.owner.getLayerOutput().getData(),getLayerOutput().getOffset(), toCount);
+
+        //this.owner.dumpOutputs();
+        //System.out.println("AF");
+
     }
 
     /**
@@ -153,11 +161,9 @@ public void computeGradient(GradientCalc calc, int inputOffset, int outputOffset
         // array references are made method local to avoid one indirection
         final double[] layerDelta = calc.getLayerDelta();
         final double[] weights = this.getOwner().getWeights();
-        final double[] layerOutput = getOwner().getLayerOutput();
-        final double[] layerSums = getOwner().getLayerSums();
         int y = fromLayerIndex;
         for (int yi = 0; yi < fromLayerSize; yi++) {
-            final double output = layerOutput[y];
+            final double output = prev.getLayerOutput().get(yi);
             double sum = 0;
 
             int wi = index + yi;
@@ -170,7 +176,7 @@ public void computeGradient(GradientCalc calc, int inputOffset, int outputOffset
                 sum += weights[wi] * layerDelta[x];
             }
             layerDelta[y] = sum
-                    * (activation.derivativeFunction(layerSums[y], layerOutput[y]));
+                    * (activation.derivativeFunction(prev.getLayerSums().get(yi), prev.getLayerOutput().get(yi)));
 
             y++;
         }
@@ -238,4 +244,6 @@ public String toString() {
         result.append("]");
         return result.toString();
     }
+
+
 }

vol3/vol3-java-examples/src/main/java/com/heatonresearch/aifh/ann/train/GradientCalc.java

@@ -54,15 +54,6 @@ public class GradientCalc {
      */
     private final double[] layerDelta;
 
-    /**
-     * The output from each layer.
-     */
-    private final double[] layerOutput;
-
-    /**
-     * The sums.
-     */
-    private final double[] layerSums;
 
     /**
      * The gradients.
@@ -95,13 +86,11 @@ public GradientCalc(final BasicNetwork theNetwork,
         this.network = theNetwork;
         this.errorFunction = ef;
 
-        this.layerDelta = new double[this.network.getLayerOutput().length];
+        this.layerDelta = new double[this.network.getNeuronCount()];
         this.gradients = new double[this.network.getWeights().length];
         this.actual = new double[this.network.getOutputCount()];
 
         this.weights = this.network.getWeights();
-        this.layerOutput = this.network.getLayerOutput();
-        this.layerSums = this.network.getLayerSums();
         this.owner = theOwner;
     }
 
@@ -134,7 +123,8 @@ public void process(ErrorCalculation errorCalc, double[] input, double[] ideal)
         int outputLayerIndex = this.network.getLayers().size() - 1;
         ActivationFunction outputActivation = this.network.getLayers().get(outputLayerIndex).getActivation();
         this.errorFunction.calculateError(
-                outputActivation, this.layerSums, this.layerOutput,
+                outputActivation, this.network.getOutputLayer().getLayerSums(),
+                this.network.getOutputLayer().getLayerOutput(),
                 ideal, this.actual, this.layerDelta, 0, 1.0);
 
         // Apply regularization, if requested.