Hyper-Parameter Optimization Framework - WIP

bindings/py/cpp_src/bindings/algorithms/py_SpatialPooler.cpp

@@ -58,7 +58,7 @@ using namespace sdr;
             , Real
             , bool
             , Real
-            , UInt
+            , Int
             , UInt
             , Real
             , Real

src/examples/mnist_nni/config.yml

@@ -0,0 +1,16 @@
+authorName: default
+experimentName: example_mnist
+trialConcurrency: 6
+trainingServicePlatform: local
+searchSpacePath: search_space.json
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
+  #SMAC (SMAC should be installed through nnictl)
+  builtinTunerName: TPE
+  classArgs:
+    optimize_mode: maximize
+trial:
+  command: python3 mnist.py
+  codeDir: .
+  gpuNum: 0

src/examples/mnist_nni/config_assessor.yml

@@ -0,0 +1,29 @@
+authorName: default
+experimentName: example_mnist
+trialConcurrency: 1
+maxExecDuration: 1h
+maxTrialNum: 50
+#choice: local, remote
+trainingServicePlatform: local
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
+  #SMAC (SMAC should be installed through nnictl)
+  builtinTunerName: TPE
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+assessor:
+  #choice: Medianstop, Curvefitting
+  builtinAssessorName: Curvefitting
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+    epoch_num: 20
+    threshold: 0.9
+trial:
+  command: python3 mnist.py
+  codeDir: .
+  gpuNum: 0

src/examples/mnist_nni/config_frameworkcontroller.yml

@@ -0,0 +1,41 @@
+authorName: default
+experimentName: example_mnist
+trialConcurrency: 1
+maxExecDuration: 1h
+maxTrialNum: 10
+#choice: local, remote, pai, kubeflow
+trainingServicePlatform: frameworkcontroller
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
+  builtinTunerName: TPE
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+assessor:
+  builtinAssessorName: Medianstop
+  classArgs:
+    optimize_mode: maximize
+  gpuNum: 0
+trial:
+  codeDir: .
+  taskRoles:
+    - name: worker
+      taskNum: 1
+      command: python3 mnist.py
+      gpuNum: 1
+      cpuNum: 1
+      memoryMB: 8192
+      image: msranni/nni:latest
+      frameworkAttemptCompletionPolicy:
+        minFailedTaskCount: 1
+        minSucceededTaskCount: 1
+frameworkcontrollerConfig:
+  storage: nfs
+  nfs:
+    # Your NFS server IP, like 10.10.10.10
+    server: {your_nfs_server_ip}
+    # Your NFS server export path, like /var/nfs/nni
+    path: {your_nfs_server_export_path}

src/examples/mnist_nni/config_kubeflow.yml

@@ -0,0 +1,32 @@
+authorName: default
+experimentName: example_dist
+trialConcurrency: 1
+maxExecDuration: 1h
+maxTrialNum: 1
+#choice: local, remote, pai, kubeflow
+trainingServicePlatform: kubeflow
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
+  builtinTunerName: TPE
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+trial:
+  codeDir: .
+  worker:
+    replicas: 1
+    command: python3 mnist.py
+    gpuNum: 0
+    cpuNum: 1
+    memoryMB: 8192
+    image: msranni/nni:latest
+kubeflowConfig:
+  operator: tf-operator
+  apiVersion: v1alpha2
+  storage: nfs
+  nfs:
+    server: 10.10.10.10
+    path: /var/nfs/general

src/examples/mnist_nni/config_pai.yml

@@ -0,0 +1,36 @@
+authorName: default
+experimentName: example_mnist
+trialConcurrency: 1
+maxExecDuration: 1h
+maxTrialNum: 10
+#choice: local, remote, pai
+trainingServicePlatform: pai
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
+  #SMAC (SMAC should be installed through nnictl)
+  builtinTunerName: TPE
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+trial:
+  command: python3 mnist.py
+  codeDir: .
+  gpuNum: 0
+  cpuNum: 1
+  memoryMB: 8196
+  #The docker image to run nni job on pai
+  image: msranni/nni:latest
+  #The hdfs directory to store data on pai, format 'hdfs://host:port/directory'
+  dataDir: hdfs://10.10.10.10:9000/username/nni
+  #The hdfs directory to store output data generated by nni, format 'hdfs://host:port/directory'
+  outputDir: hdfs://10.10.10.10:9000/username/nni
+paiConfig:
+  #The username to login pai
+  userName: username
+  #The password to login pai
+  passWord: password
+  #The host of restful server of pai
+  host: 10.10.10.10

src/examples/mnist_nni/config_windows.yml

@@ -0,0 +1,21 @@
+authorName: default
+experimentName: example_mnist
+trialConcurrency: 1
+maxExecDuration: 1h
+maxTrialNum: 10
+#choice: local, remote, pai
+trainingServicePlatform: local
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
+  #SMAC (SMAC should be installed through nnictl)
+  builtinTunerName: TPE
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+trial:
+  command: python mnist.py
+  codeDir: .
+  gpuNum: 0

src/examples/mnist_nni/mnist.py

@@ -0,0 +1,163 @@
+""" An MNIST classifier using Spatial Pooler."""
+
+import argparse
+import logging
+import math
+import random
+import gzip
+import numpy as np
+import os
+import threading
+from pprint import pprint
+
+from nupic.bindings.algorithms import SpatialPooler, Classifier
+from nupic.bindings.sdr import SDR, Metrics
+
+import nni
+
+
+def load_mnist(path):
+    """See: http://yann.lecun.com/exdb/mnist/ for MNIST download and binary file format spec."""
+    def int32(b):
+        i = 0
+        for char in b:
+            i *= 256
+            # i += ord(char)    # python2
+            i += char
+        return i
+
+    def load_labels(file_name):
+        with gzip.open(file_name, 'rb') as f:
+            raw = f.read()
+            assert(int32(raw[0:4]) == 2049)  # Magic number
+            labels = []
+            for char in raw[8:]:
+                # labels.append(ord(char))      # python2
+                labels.append(char)
+        return labels
+
+    def load_images(file_name):
+        with gzip.open(file_name, 'rb') as f:
+            raw = f.read()
+            assert(int32(raw[0:4]) == 2051)    # Magic number
+            num_imgs   = int32(raw[4:8])
+            rows       = int32(raw[8:12])
+            cols       = int32(raw[12:16])
+            assert(rows == 28)
+            assert(cols == 28)
+            img_size   = rows*cols
+            data_start = 4*4
+            imgs = []
+            for img_index in range(num_imgs):
+                vec = raw[data_start + img_index*img_size : data_start + (img_index+1)*img_size]
+                # vec = [ord(c) for c in vec]   # python2
+                vec = list(vec)
+                vec = np.array(vec, dtype=np.uint8)
+                buf = np.reshape(vec, (rows, cols, 1))
+                imgs.append(buf)
+            assert(len(raw) == data_start + img_size * num_imgs)   # All data should be used.
+        return imgs
+
+    train_labels = load_labels(os.path.join(path, 'train-labels-idx1-ubyte.gz'))
+    train_images = load_images(os.path.join(path, 'train-images-idx3-ubyte.gz'))
+    test_labels  = load_labels(os.path.join(path, 't10k-labels-idx1-ubyte.gz'))
+    test_images  = load_images(os.path.join(path, 't10k-images-idx3-ubyte.gz'))
+
+    return train_labels, train_images, test_labels, test_images
+
+
+class BWImageEncoder:
+    """Simple grey scale image encoder for MNIST."""
+    def __init__(self, input_space):
+        self.output = SDR(tuple(input_space))
+
+    def encode(self, image):
+        self.output.dense = image >= np.mean(image)
+        return self.output
+
+
+def main(args):
+    # Load data.
+    train_labels, train_images, test_labels, test_images = load_mnist(args['data_dir'])
+    training_data = list(zip(train_images, train_labels))
+    test_data     = list(zip(test_images, test_labels))
+    random.shuffle(training_data)
+    random.shuffle(test_data)
+
+    # Setup the AI.
+    enc = BWImageEncoder(train_images[0].shape[:2])
+    sp = SpatialPooler(
+        inputDimensions            = enc.output.dimensions,
+        columnDimensions           = [int(args['columnDimensions']), 1],
+        potentialRadius            = 99999999,
+        potentialPct               = args['potentialPct'],
+        globalInhibition           = True,
+        localAreaDensity           = args['localAreaDensity'],
+        numActiveColumnsPerInhArea = -1,
+        stimulusThreshold          = int(round(args['stimulusThreshold'])),
+        synPermInactiveDec         = args['synPermInactiveDec'],
+        synPermActiveInc           = args['synPermActiveInc'],
+        synPermConnected           = args['synPermConnected'],
+        minPctOverlapDutyCycle     = args['minPctOverlapDutyCycle'],
+        dutyCyclePeriod            = int(round(args['dutyCyclePeriod'])),
+        boostStrength              = args['boostStrength'],
+        seed                       = 42,
+        spVerbosity                = 99,
+        wrapAround                 = False)
+    columns = SDR( sp.getColumnDimensions() )
+    columns_stats = Metrics( columns, 99999999 )
+    sdrc = Classifier()
+
+    # Training Loop
+    for i in range(len(train_images)):
+        img, lbl = random.choice(training_data)
+        enc.encode(np.squeeze(img))
+        sp.compute( enc.output, True, columns )
+        sdrc.learn( columns, lbl )
+
+    print(str(sp))
+    print(str(columns_stats))
+
+    # Testing Loop
+    score = 0
+    for img, lbl in test_data:
+        enc.encode(np.squeeze(img))
+        sp.compute( enc.output, False, columns )
+        if lbl == np.argmax( sdrc.infer( columns ) ):
+            score += 1
+
+    print('Score:', 100 * score / len(test_data), '%')
+    nni.report_final_result( score / len(test_data) )
+
+
+def get_params():
+    ''' Get parameters from command line '''
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data_dir", type=str, default="./MNIST_data")
+
+    parser.add_argument("--columnDimensions",       type=int,   default = 10000)
+    parser.add_argument("--potentialPct",           type=float, default = 0.5)
+    parser.add_argument("--localAreaDensity",       type=float, default = .015)
+    parser.add_argument("--stimulusThreshold",      type=int,   default = 6)
+    parser.add_argument("--synPermActiveInc",       type=float, default = 0.01)
+    parser.add_argument("--synPermInactiveDec",     type=float, default = 0.005)
+    parser.add_argument("--synPermConnected",       type=float, default = 0.4)
+    parser.add_argument("--minPctOverlapDutyCycle", type=float, default = 0.001)
+    parser.add_argument("--dutyCyclePeriod",        type=int,   default = 1000)
+    parser.add_argument("--boostStrength",          type=float, default = 2.5)
+
+    args, _ = parser.parse_known_args()
+    return args
+
+if __name__ == '__main__':
+    params = vars(get_params())
+    tuner_params = nni.get_next_parameter()
+    if tuner_params is not None:
+        params.update(tuner_params)
+    pprint(params)
+
+    timer = threading.Timer( 30 * 60, lambda: os._exit(1))
+    timer.daemon = True
+    timer.start()
+    main(params)
+    timer.cancel()