Merge branch 'v1.x' into mubaidr-patch-1

Author: robertleeplummerjr

bower.json

@@ -31,5 +31,5 @@
     "node_modules",
     "test"
   ],
-  "version": "1.3.1"
+  "version": "1.4.0"
 }

browser.js

@@ -6,7 +6,7 @@
  *   license: MIT (http://opensource.org/licenses/MIT)
  *   author: Heather Arthur <[email protected]>
  *   homepage: https://github.com/brainjs/brain.js#readme
- *   version: 1.3.1
+ *   version: 1.4.0
  *
  * acorn:
  *   license: MIT (http://opensource.org/licenses/MIT)
@@ -1092,7 +1092,11 @@ var NeuralNetwork = function () {
         momentum: 0.1, // multiply's against the specified "change" then adds to learning rate for change
         callback: null, // a periodic call back that can be triggered while training
         callbackPeriod: 10, // the number of iterations through the training data between callback calls
-        timeout: Infinity // the max number of milliseconds to train for
+        timeout: Infinity, // the max number of milliseconds to train for
+        praxis: null,
+        beta1: 0.9,
+        beta2: 0.999,
+        epsilon: 1e-8
       };
     }
   }, {
@@ -1531,6 +1535,10 @@ var NeuralNetwork = function () {
       endTime = _prepTraining2.endTime;
 
 
+      if (options.praxis === 'adam') {
+        this._setupAdam();
+      }
+
       while (this._trainingTick(data, status, endTime)) {}
       return status;
     }
@@ -1737,6 +1745,71 @@ var NeuralNetwork = function () {
         }
       }
     }
+  }, {
+    key: '_setupAdam',
+    value: function _setupAdam() {
+      this.biasChangesLow = [];
+      this.biasChangesHigh = [];
+      this.changesLow = [];
+      this.changesHigh = [];
+      this.iterations = 0;
+
+      for (var layer = 0; layer <= this.outputLayer; layer++) {
+        var size = this.sizes[layer];
+        if (layer > 0) {
+          this.biasChangesLow[layer] = (0, _zeros2.default)(size);
+          this.biasChangesHigh[layer] = (0, _zeros2.default)(size);
+          this.changesLow[layer] = new Array(size);
+          this.changesHigh[layer] = new Array(size);
+
+          for (var node = 0; node < size; node++) {
+            var prevSize = this.sizes[layer - 1];
+            this.changesLow[layer][node] = (0, _zeros2.default)(prevSize);
+            this.changesHigh[layer][node] = (0, _zeros2.default)(prevSize);
+          }
+        }
+      }
+
+      this._adjustWeights = this._adjustWeightsAdam;
+    }
+  }, {
+    key: '_adjustWeightsAdam',
+    value: function _adjustWeightsAdam() {
+      var trainOpts = this.trainOpts;
+      this.iterations++;
+
+      for (var layer = 1; layer <= this.outputLayer; layer++) {
+        var incoming = this.outputs[layer - 1];
+
+        for (var node = 0; node < this.sizes[layer]; node++) {
+          var delta = this.deltas[layer][node];
+
+          for (var k = 0; k < incoming.length; k++) {
+            var gradient = delta * incoming[k];
+            var changeLow = this.changesLow[layer][node][k] * trainOpts.beta1 + (1 - trainOpts.beta1) * gradient;
+            var changeHigh = this.changesHigh[layer][node][k] * trainOpts.beta2 + (1 - trainOpts.beta2) * gradient * gradient;
+
+            var momentumCorrection = changeLow / (1 - Math.pow(trainOpts.beta1, this.iterations));
+            var gradientCorrection = changeHigh / (1 - Math.pow(trainOpts.beta2, this.iterations));
+
+            this.changesLow[layer][node][k] = changeLow;
+            this.changesHigh[layer][node][k] = changeHigh;
+            this.weights[layer][node][k] += this.trainOpts.learningRate * momentumCorrection / (Math.sqrt(gradientCorrection) + trainOpts.epsilon);
+          }
+
+          var biasGradient = this.deltas[layer][node];
+          var biasChangeLow = this.biasChangesLow[layer][node] * trainOpts.beta1 + (1 - trainOpts.beta1) * biasGradient;
+          var biasChangeHigh = this.biasChangesHigh[layer][node] * trainOpts.beta2 + (1 - trainOpts.beta2) * biasGradient * biasGradient;
+
+          var biasMomentumCorrection = this.biasChangesLow[layer][node] / (1 - Math.pow(trainOpts.beta1, this.iterations));
+          var biasGradientCorrection = this.biasChangesHigh[layer][node] / (1 - Math.pow(trainOpts.beta2, this.iterations));
+
+          this.biasChangesLow[layer][node] = biasChangeLow;
+          this.biasChangesHigh[layer][node] = biasChangeHigh;
+          this.biases[layer][node] += trainOpts.learningRate * biasMomentumCorrection / (Math.sqrt(biasGradientCorrection) + trainOpts.epsilon);
+        }
+      }
+    }
 
     /**
      *

browser.min.js

@@ -1,7 +1,7 @@
 {
   "name": "brain.js",
   "description": "Neural network library",
-  "version": "1.3.1",
+  "version": "1.4.0",
   "author": "Heather Arthur <[email protected]>",
   "repository": {
     "type": "git",

dist/neural-network.js

@@ -22,7 +22,11 @@ export default class NeuralNetwork {
       momentum: 0.1,        // multiply's against the specified "change" then adds to learning rate for change
       callback: null,       // a periodic call back that can be triggered while training
       callbackPeriod: 10,   // the number of iterations through the training data between callback calls
-      timeout: Infinity     // the max number of milliseconds to train for
+      timeout: Infinity,    // the max number of milliseconds to train for
+      praxis: null,
+      beta1: 0.9,
+      beta2: 0.999,
+      epsilon: 1e-8,
     };
   }
 
@@ -443,6 +447,10 @@ export default class NeuralNetwork {
     let status, endTime;
     ({ data, status, endTime } = this._prepTraining(data, options));
 
+    if (options.praxis === 'adam') {
+      this._setupAdam();
+    }
+
     while (this._trainingTick(data, status, endTime));
     return status;
   }
@@ -619,6 +627,69 @@ export default class NeuralNetwork {
     }
   }
 
+  _setupAdam() {
+    this.biasChangesLow = [];
+    this.biasChangesHigh = [];
+    this.changesLow = [];
+    this.changesHigh = [];
+    this.iterations = 0;
+
+    for (let layer = 0; layer <= this.outputLayer; layer++) {
+      let size = this.sizes[layer];
+      if (layer > 0) {
+        this.biasChangesLow[layer] = zeros(size);
+        this.biasChangesHigh[layer] = zeros(size);
+        this.changesLow[layer] = new Array(size);
+        this.changesHigh[layer] = new Array(size);
+
+        for (let node = 0; node < size; node++) {
+          let prevSize = this.sizes[layer - 1];
+          this.changesLow[layer][node] = zeros(prevSize);
+          this.changesHigh[layer][node] = zeros(prevSize);
+        }
+      }
+    }
+
+    this._adjustWeights = this._adjustWeightsAdam;
+  }
+
+  _adjustWeightsAdam() {
+    const trainOpts = this.trainOpts;
+    this.iterations++;
+
+    for (let layer = 1; layer <= this.outputLayer; layer++) {
+      const incoming = this.outputs[layer - 1];
+
+      for (let node = 0; node < this.sizes[layer]; node++) {
+        const delta = this.deltas[layer][node];
+
+        for (let k = 0; k < incoming.length; k++) {
+          const gradient = delta * incoming[k];
+          const changeLow = this.changesLow[layer][node][k] * trainOpts.beta1 + (1 - trainOpts.beta1) * gradient;
+          const changeHigh = this.changesHigh[layer][node][k] * trainOpts.beta2 + (1 - trainOpts.beta2) * gradient * gradient;
+          
+          const momentumCorrection = changeLow / (1 - Math.pow(trainOpts.beta1, this.iterations));
+          const gradientCorrection = changeHigh / (1 - Math.pow(trainOpts.beta2, this.iterations));
+
+          this.changesLow[layer][node][k] = changeLow;
+          this.changesHigh[layer][node][k] = changeHigh;
+          this.weights[layer][node][k] += this.trainOpts.learningRate * momentumCorrection / (Math.sqrt(gradientCorrection) + trainOpts.epsilon);
+        }
+
+        const biasGradient = this.deltas[layer][node];
+        const biasChangeLow = this.biasChangesLow[layer][node] * trainOpts.beta1 + (1 - trainOpts.beta1) * biasGradient;
+        const biasChangeHigh = this.biasChangesHigh[layer][node] * trainOpts.beta2 + (1 - trainOpts.beta2) * biasGradient * biasGradient;
+
+        const biasMomentumCorrection = this.biasChangesLow[layer][node] / (1 - Math.pow(trainOpts.beta1, this.iterations));
+        const biasGradientCorrection = this.biasChangesHigh[layer][node] / (1 - Math.pow(trainOpts.beta2, this.iterations));
+
+        this.biasChangesLow[layer][node] = biasChangeLow;
+        this.biasChangesHigh[layer][node] = biasChangeHigh;
+        this.biases[layer][node] += trainOpts.learningRate * biasMomentumCorrection / (Math.sqrt(biasGradientCorrection) + trainOpts.epsilon);
+      }
+    }
+  }
+
   /**
    *
    * @param data

dist/neural-network.js.map

@@ -14,9 +14,20 @@ function testBitwise(data, op) {
   let res = net.train(data, { errorThresh: 0.003 });
 
   data.forEach(d => {
-    var actual = net.run(d.input)
+    var actual = net.run(d.input);
     var expected = d.output;
-    assert.ok(isAround(actual, expected), `failed to train "${op}" - expected: ${expected}, actual: ${actual}`);
+    assert.ok(isAround(actual[0], expected[0]), `failed to train "${op}" - expected: ${expected}, actual: ${actual}`);
+  });
+}
+
+function testBitwiseAdam(data, op) {
+  let net = new brain.NeuralNetwork();
+  let res = net.train(data, { errorThresh: 0.003, learningRate: 0.05, praxis: 'adam' });
+
+  data.forEach(d => {
+    var actual = net.run(d.input);
+    var expected = d.output;
+    assert.ok(isAround(actual[0], expected[0]), `failed to train "${op}" - expected: ${expected}, actual: ${actual}`);
   });
 }
 
@@ -45,10 +56,10 @@ describe('bitwise functions sync training', () => {
   });
 
   it('XOR function', () => {
-    let xor = [{input: [0, 0], output: [0]},
-               {input: [0, 1], output: [1]},
-               {input: [1, 0], output: [1]},
-               {input: [1, 1], output: [0]}];
+    let xor = [{input: [0.001, 0.001], output: [0.001]},
+               {input: [0.001, 1], output: [1]},
+               {input: [1, 0.001], output: [1]},
+               {input: [1, 1], output: [0.001]}];
     testBitwise(xor, 'xor');
   });
 
@@ -67,4 +78,36 @@ describe('bitwise functions sync training', () => {
                {input: [1, 1], output: [1]}];
     testBitwise(and, 'and');
   });
+});
+
+describe('bitwise using adam praxis functions sync training', () => {
+  it('NOT function', () => {
+    let not = [{input: [0], output: [1]},
+      {input: [1], output: [0]}];
+    testBitwiseAdam(not, 'not');
+  });
+
+  it('XOR function', () => {
+    let xor = [{input: [0.001, 0.001], output: [0.001]},
+      {input: [0.001, 1], output: [1]},
+      {input: [1, 0.001], output: [1]},
+      {input: [1, 1], output: [0.001]}];
+    testBitwiseAdam(xor, 'xor');
+  });
+
+  it('OR function', () => {
+    let or = [{input: [0, 0], output: [0]},
+      {input: [0, 1], output: [1]},
+      {input: [1, 0], output: [1]},
+      {input: [1, 1], output: [1]}];
+    testBitwiseAdam(or, 'or');
+  });
+
+  it('AND function', () => {
+    let and = [{input: [0, 0], output: [0]},
+      {input: [0, 1], output: [0]},
+      {input: [1, 0], output: [0]},
+      {input: [1, 1], output: [1]}];
+    testBitwiseAdam(and, 'and');
+  });
 });