NAS Envelopenet Suggestion and Job Example (#425)

* NAS Envelopenet Suggestion and Job Example

* NAS Envelopenet Suggestion and Job Example

* Typo correction  in nasjob

* Readme

* Readme added

* Link to paper

cmd/suggestion/nasenvelopenet/Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.6
+
+ADD . /usr/src/app/github.com/kubeflow/katib
+WORKDIR /usr/src/app/github.com/kubeflow/katib/cmd/suggestion/nasenvelopenet
+RUN pip install --no-cache-dir -r requirements.txt
+ENV PYTHONPATH /usr/src/app/github.com/kubeflow/katib:/usr/src/app/github.com/kubeflow/katib/pkg/api/python
+
+ENTRYPOINT ["python", "-u", "main.py"]

cmd/suggestion/nasenvelopenet/__init__.py

@@ -0,0 +1 @@
+

cmd/suggestion/nasenvelopenet/main.py

@@ -0,0 +1,29 @@
+import grpc
+from concurrent import futures
+
+import time
+
+from pkg.api.python import api_pb2_grpc
+from pkg.suggestion.nasenvelopenet_service import EnvelopenetService
+from pkg.suggestion.types import DEFAULT_PORT
+from logging import getLogger, StreamHandler, INFO, DEBUG
+
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+
+
+def serve():
+    print("NAS Envelopenet Suggestion Service")
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+    api_pb2_grpc.add_SuggestionServicer_to_server(EnvelopenetService(), server)
+    server.add_insecure_port(DEFAULT_PORT)
+    print("Listening...")
+    server.start()
+    try:
+        while True:
+            time.sleep(_ONE_DAY_IN_SECONDS)
+    except KeyboardInterrupt:
+        server.stop(0)
+
+if __name__ == "__main__":
+    serve()

cmd/suggestion/nasenvelopenet/requirements.txt

@@ -0,0 +1,3 @@
+grpcio
+protobuf
+googleapis-common-protos

examples/nasjob-example-envelopenet.yaml

@@ -0,0 +1,126 @@
+apiVersion: "kubeflow.org/v1alpha1"
+kind: StudyJob
+metadata:
+  namespace: kubeflow
+  labels:
+    controller-tools.k8s.io: "1.0"
+  name: nasenvelopenet
+spec:
+  studyName: nasenvelopenet
+  owner: crd
+  optimizationtype: maximize
+  objectivevaluename: MeanSSS
+  optimizationgoal: 0.99
+  requestcount: 6
+  metricsnames:
+    - accuracy
+  nasConfig:
+    graphConfig:
+      numLayers: 3
+      inputSize:
+        - 32
+        - 32
+        - 3
+      outputSize:
+        - 10
+    operations:
+      - operationType: envelopenet
+        parameterconfigs:
+          - name: envelopecell
+            parametertype: categorical
+            feasible:
+              list:
+              - "1"
+              - "3"
+              - "5"
+              - "3sep"
+              - "5sep"
+              - "7sep"
+          - name: layers_per_stage
+            parametertype: categorical
+            feasible: 
+              list:
+              - "3"
+              - "3"
+              - "3"
+          - name: construction
+            parametertype: categorical
+            feasible: 
+              list:
+              - "1"
+              - "1"
+              - "0"
+          - name: parameter_limits
+            parametertype: categorical
+            feasible: 
+              list:
+              - "0"
+              - "0"
+              - "0"
+          - name: max_filter_prune
+            parametertype: categorical
+            feasible: 
+              list: 
+              - "6"
+          - name: skip
+            parametertype: categorical
+            feasible: 
+              list:
+              - "1"
+          - name: outputs
+            parametertype: categorical
+            feasible:
+              list:
+              - "64"
+              - "128"
+              - "256"
+              - "512"
+
+  workerSpec:
+    goTemplate:
+        rawTemplate: |-
+          apiVersion: batch/v1
+          kind: Job
+          metadata:
+            name: {{.WorkerID}}
+            namespace: {{.NameSpace}}
+          spec:
+            template:
+              spec:
+                containers:
+                - name: {{.WorkerID}}
+                  image: docker.io/anubhavgarg/envelopenet_training_container
+                  command:
+                  - "python3.5"
+                  - "-u"
+                  - "run_trial.py"
+                  {{- with .HyperParameters}}
+                  {{- range .}}
+                  - "--{{.Name}}={{.Value}}"
+                  {{- end}}
+                  {{- end}}
+                  resources:
+                    limits:
+                      nvidia.com/gpu: 1
+                restartPolicy: OnFailure
+  suggestionSpec:
+    suggestionAlgorithm: "nasenvelopenet"
+    suggestionParameters:
+      - name: "max_layers_per_stage"
+        value: "763"
+      - name: "gpus"
+        value: "01"
+      - name: "gpu_usage"
+        value: "0.47"
+      - name: "steps"
+        value: "1000"
+      - name: "batch_size"
+        value: "50"
+      - name: "dataset"
+        value: "cifar10"
+      - name: "data_dir"
+        value: "data/"
+      - name: "iterations"
+        value: "5"
+      - name: "log_stats"
+        value: "True"

manifests/vizier/suggestion/NAS-Envelopenet/deployment.yaml

@@ -0,0 +1,23 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: vizier-suggestion-nasenvelopenet
+  namespace: kubeflow
+  labels:
+    app: vizier
+    component: suggestion-nasenvelopenet
+spec:
+  replicas: 1
+  template:
+    metadata:
+      name: vizier-suggestion-nasenvelopenet
+      labels:
+        app: vizier
+        component: suggestion-nasenvelopenet
+    spec:
+      containers:
+      - name: vizier-suggestion-nasenvelopenet
+        image: docker.io/anubhavgarg/nas_envelopenet_service
+        ports:
+        - name: api
+          containerPort: 6789

manifests/vizier/suggestion/NAS-Envelopenet/service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: vizier-suggestion-nasenvelopenet
+  namespace: kubeflow
+  labels:
+    app: vizier
+    component: suggestion-nasenvelopenet
+spec:
+  type: ClusterIP
+  ports:
+    - port: 6789
+      protocol: TCP
+      name: api
+  selector:
+    app: vizier
+    component: suggestion-nasenvelopenet

pkg/suggestion/NAS_Envelopenet/README.md

@@ -0,0 +1,28 @@
+# About the Neural Architecture Search with Envelopenet Suggestion
+
+The algorithm follows the idea proposed in *Fast Neural Architecture Construction using EnvelopeNets* by Kamath et al.(https://arxiv.org/pdf/1803.06744.pdf). It is not a Reinforcement Learning or evolution based NAS, rather a method to construct deep network
+architectures by pruning and expansion of a base network. This approach directly compares the utility of different filters using statistics derived from filter featuremaps reach a state where the utility of different filters
+within a network can be compared and hence can be used to construct networks. 
+
+# Envelopenets
+
+The EnvelopeCell is a set of M convolution blocks connected in parallel. E.g. one of the EnvelopeCells used in this work has 6 convolution blocks connected in parallel: 1x1 convolution,
+3x3 convolution, 3x3 separable convolution, 5x5 convolution, 5x5 separable convolution and 7x7 separable convolution. The EnvelopeNet consists of a number of the EnvelopeCells stacked in series organized into stages of
+n layers, separated by wideners.
+
+# Output of `GetSuggestion()`
+
+The output of `GetSuggestion()` is the `architecture`
+
+`architecture` is a json string of the definition of a neural architecture. The format is as stated above. One example is:
+```
+{'type': 'macro', 'network': [{'filters': ['1', '3', '5', '3sep', '5sep', '7sep'], 'outputs': 26, 'inputs': []}, {'filters': ['1', '3', '5', '3sep', '5sep', '7sep'], 'outputs': 26, 'inputs': []},
+ {'filters': ['1', '3', '5', '3sep', '5sep', '7sep'], 'outputs': 64, 'inputs': [1]}, {'widener': {}}, {'filters': ['1', '3', '5', '3sep', '5sep', '7sep'], 'outputs': 53, 'inputs': [3, 2, 1]},
+ {'filters': ['1', '3', '5', '3sep', '5sep', '7sep'], 'outputs': 53, 'inputs': [4, 3, 2, 1]}, {'filters': ['1', '3', '5', '3sep', '5sep', '7sep'], 'outputs': 128, 'inputs': [5, 4, 3, 2, 1]}, {'widener': {}},
+ {'filters': ['1', '3', '5', '3sep', '5sep', '7sep'], 'outputs': 106, 'inputs': [7, 6, 5, 4, 3, 2, 1]}, {'filters': ['1', '3', '5', '3sep', '5sep', '7sep'], 'outputs': 106, 'inputs': [8, 7, 6, 5, 4, 3, 2, 1]},
+ {'filters': ['1', '3', '5', '3sep', '5sep', '7sep'], 'outputs': 256, 'inputs': [9, 8, 7, 6, 5, 4, 3, 2, 1]}]}
+```
+# Flow of example
+
+The parameters and initial architecture from yaml file is passed to ModelConstructor, which constructs the model and trains. Then it sends the featuremap statistics from this truncated training and the process to refine the architecture
+continues till the max_iterations parameter.