Custom Layer Support (#315)

* Added basic custom layer editor. Fixes #1

* Added custom layer editing. Fixes #156

WIP #156 Updated rootviz component config

WIP #156 Added template creation from parent if no code

WIP #156 Added root viz panel. Changed ordering

WIP #156 Added some support for 'create layer' button

WIP #156 Fixed create custom layer type button

WIP #156 Fixed bug with checking node for attribute

WIP #156. Added support for updating meta from constructor args

WIP #156

WIP #156 Added seeds

WIP #156 Added attr inheritance in layer def

WIP #156 Removed extra TODO

* Added custom layer support to executions. Fixes #296

WIP #296 Fixed execution support

* Updated GenArch for custom layers. Fixes #157

WIP #157 Updated 'standalone' default to true

WIP #157 Fixed code-climate issues

WIP #157 Fixed bad default

WIP Removed outdated tests

config/components.json

@@ -61,23 +61,29 @@
                 "description": "Pipelines compose operations together to effectively train, test and/or ensemble models."
             },
             {
-                "nodeName": "MyArtifacts",
-                "icon": "view_quilt",
+                "nodeName": "MyLayers",
+                "icon": "clear_all",
                 "rank": 4,
                 "color": "blue-grey",
+                "description": "Custom torch layers can be created and stored here for use in neural network architectures."
+            },
+            {
+                "nodeName": "MyArtifacts",
+                "icon": "view_quilt",
+                "rank": 5,
                 "description": "Artifacts from pipeline executions are stored here."
             },
             {
                 "nodeName": "MyExecutions",
                 "icon": "list",
-                "rank": 5,
+                "rank": 6,
+                "color": "blue-grey",
                 "description": "Executions are read-only snapshots of pipelines that have been executed. Past and current executing pipelines are stored here."
             },
             {
                 "nodeName": "MyDataTypes",
                 "icon": "settings",
-                "rank": 6,
-                "color": "blue-grey",
+                "rank": 7,
                 "description": "Custom defined data types are stored here."
             }
         ]

src/plugins/ExecutePipeline/ExecutePipeline.js

@@ -591,6 +591,7 @@ define([
 
         // add the given files
         return this.createEntryFile(node, files)
+            .then(() => this.createCustomLayers(node, files))
             .then(() => this.createInputs(node, files))
             .then(() => this.createOutputs(node, files))
             .then(() => this.createMainFile(node, files))
@@ -600,6 +601,31 @@ define([
             });
     };
 
+    ExecutePipeline.prototype.createCustomLayers = function (node, files) {
+        var isCustomLayer = {},
+            metaDict = this.core.getAllMetaNodes(this.rootNode),
+            metanodes,
+            customLayers,
+            code;
+
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        // Get all the custom layers
+        metanodes.forEach(node => {
+            if (this.core.getAttribute(node, 'name') === 'CustomLayer') {
+                isCustomLayer[this.core.getPath(node)] = true;
+            }
+        });
+        customLayers = metanodes.filter(node =>
+            this.core.getMixinPaths(node).some(id => isCustomLayer[id]));
+
+        // Get the code definitions for each
+        code = 'require \'nn\'\n\n' + customLayers
+            .map(node => this.core.getAttribute(node, 'code')).join('\n');
+
+        // Create the custom layers file
+        files['custom-layers.lua'] = code;
+    };
+
     ExecutePipeline.prototype.createInputs = function (node, files) {
         var tplContents;
         return this.getInputs(node)

src/plugins/ExecutePipeline/templates/main.ejs

@@ -1,3 +1,6 @@
+-- load custom layers
+require './custom-layers'
+
 -- input data<% inputs.forEach(function(pair) { var input = pair[0], isNil = pair[1];%>
 <%= isNil ? 'local ' : ''%><%= input %> = <% if (isNil) { %>nil<% } else { %>require './inputs/<%= input %>'<%}}); %>
 

src/plugins/GenerateArchitecture/GenerateArchitecture.js

@@ -43,33 +43,47 @@ define([
     GenerateArchitecture.prototype.constructor = GenerateArchitecture;
 
     GenerateArchitecture.prototype.main = function () {
+        this.addCustomLayersToMeta();
         this.LayerDict = createLayerDict(this.core, this.META);
         this.uniqueId = 2;
         this._oldTemplateSettings = _.templateSettings;
         return PluginBase.prototype.main.apply(this, arguments);
     };
 
+    GenerateArchitecture.prototype.addCustomLayersToMeta = function () {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode);
+
+        Object.keys(metaDict).map(id => metaDict[id])
+            // Get all custom layers
+            .filter(node => this.core.isTypeOf(node, this.META.Layer))
+            // Add them to the meta
+            .forEach(node => this.META[this.core.getAttribute(node, 'name')] = node);
+    };
+
     GenerateArchitecture.prototype.createOutputFiles = function (tree) {
         var layers = tree[Constants.CHILDREN],
             //initialLayers,
             result = {},
-            code;
+            code = 'require \'nn\'\n';
 
         //initialLayers = layers.filter(layer => layer[Constants.PREV].length === 0);
         // Add an index to each layer
         layers.forEach((l, index) => l[INDEX] = index);
-        code = this.genArchCode(layers);
+
+        // Define custom layers
+        if (this.getCurrentConfig().standalone) {
+            code += this.genLayerDefinitions(layers);
+        }
+
+        code += this.genArchCode(layers);
 
         result[tree.name + '.lua'] = code;
         _.templateSettings = this._oldTemplateSettings;  // FIXME: Fix this in SimpleNodes
         return result;
     };
 
     GenerateArchitecture.prototype.genArchCode = function (layers) {
-        // Create a 'null' start layer
-
         return [
-            'require \'nn\'',
             this.createSequential(layers[0], 'net').code,
             '\nreturn net'
         ].join('\n');
@@ -174,5 +188,21 @@ define([
         return !(value === undefined || value === null || value === '');
     };
 
+    GenerateArchitecture.prototype.genLayerDefinitions = function(layers) {
+        var code = '',
+            customLayerId = this.core.getPath(this.META.CustomLayer),
+            customLayers = layers.filter(layer => {  // Get the custom layers
+                var node = this.META[layer.name];
+                return this.core.getMixinPaths(node).indexOf(customLayerId) !== -1;
+            });
+
+        if (customLayers.length) {
+            code += '\n-------------- Custom Layer Definitions --------------\n\n';
+            code += customLayers.map(layer => layer.code).join('\n');
+            code += '\n\n-------------- Network --------------\n';
+        }
+
+        return code;
+    };
     return GenerateArchitecture;
 });

src/plugins/GenerateArchitecture/metadata.json

@@ -9,5 +9,14 @@
   },
   "disableServerSideExecution": false,
   "disableBrowserSideExecution": false,
-  "configStructure": []
+  "configStructure": [
+      {
+            "name": "standalone",
+            "displayName": "Standalone",
+            "description": "Prepend custom layer definitions",
+            "value": false,
+            "valueType": "boolean",
+            "readOnly": false
+      }
+  ]
 }

src/visualizers/Visualizers.json

@@ -5,24 +5,12 @@
     "panel": "panels/AutoViz/AutoVizPanel",
     "DEBUG_ONLY": false
   },
-  {
-    "id": "EasyDAG",
-    "title": "EasyDAG",
-    "panel": "panels/EasyDAG/EasyDAGPanel",
-    "DEBUG_ONLY": false
-  },
   {
     "id": "ArchEditor",
     "title": "ArchEditor",
     "panel": "panels/ArchEditor/ArchEditorPanel",
     "DEBUG_ONLY": false
   },
-  {
-    "id": "TextEditor",
-    "title": "TextEditor",
-    "panel": "panels/TextEditor/TextEditorPanel",
-    "DEBUG_ONLY": false
-  },
   {
     "id": "OperationEditor",
     "title": "OperationEditor",
@@ -94,5 +82,11 @@
     "title": "LogViewer",
     "panel": "panels/LogViewer/LogViewerPanel",
     "DEBUG_ONLY": false
+  },
+  {
+    "id": "LayerEditor",
+    "title": "LayerEditor",
+    "panel": "panels/LayerEditor/LayerEditorPanel",
+    "DEBUG_ONLY": false
   }
-]
+]

src/visualizers/panels/ForgeActionButton/Actions.js

@@ -2,13 +2,9 @@
 // These are actions defined for specific meta types. They are evaluated from
 // the context of the ForgeActionButton
 define([
-    'js/RegistryKeys',
-    'js/Panels/MetaEditor/MetaEditorConstants',
-    'js/Constants'
+    'js/RegistryKeys'
 ], function(
-    REGISTRY_KEYS,
-    META_CONSTANTS,
-    CONSTANTS
+    REGISTRY_KEYS
 ) {
     var instances = [
             'Architecture',
@@ -20,88 +16,29 @@ define([
         ],
         create = {};
 
-    var getUniqueName = function(parentId, basename) {
-        var pNode = this.client.getNode(parentId),
-            children = pNode.getChildrenIds().map(id => this.client.getNode(id)),
-            name = basename,
-            exists = {},
-            i = 2;
-
-        children.forEach(child => exists[child.getAttribute('name')] = true);
-
-        while (exists[name]) {
-            name = basename + '_' + i;
-            i++;
-        }
-
-        return name;
-    };
-
     var createNew = function(type, metasheetName) {
-        // Create CNN node in the current dir
-        // Get CNN node type
-        var parentId = this._currentNodeId,
-            newId,
-            baseId;
+        var newId,
+            baseId,
+            msg = `Created new ${type + (metasheetName ? ' prototype' : '')}`;
 
         baseId = this.client.getAllMetaNodes()
                 .find(node => node.getAttribute('name') === type)
                 .getId();
 
-        this.client.startTransaction('Created new operation prototype');
-        newId = this.client.createChild({parentId, baseId});
+        this.client.startTransaction(msg);
+        newId = this.createNamedNode(baseId, !!metasheetName);
 
-        // Name the new node
-        var basename = 'New' + this.client.getNode(baseId).getAttribute('name'),
-            newName = getUniqueName.call(this, parentId, basename);
-
-        // If instance, make the first char lowercase
-        if (!metasheetName) {
-            newName = newName.substring(0, 1).toLowerCase() + newName.substring(1);
+        if (metasheetName) {
+            this.addToMetaSheet(newId, metasheetName);
         }
 
-        this.client.setAttributes(newId, 'name', newName);
-        if (metasheetName) {  // Add to metasheet
-            var root = this.client.getNode(CONSTANTS.PROJECT_ROOT_ID),
-                metatabs = root.getRegistry(REGISTRY_KEYS.META_SHEETS),
-                metatab = metatabs.find(tab => tab.title === metasheetName) || metatabs[0],
-                metatabId = metatab.SetID;
-
-            // Add to the general meta
-            this.client.addMember(
-                CONSTANTS.PROJECT_ROOT_ID,
-                newId,
-                META_CONSTANTS.META_ASPECT_SET_NAME
-            );
-            this.client.setMemberRegistry(
-                CONSTANTS.PROJECT_ROOT_ID,
-                newId,
-                META_CONSTANTS.META_ASPECT_SET_NAME,
-                REGISTRY_KEYS.POSITION,
-                {
-                    x: 100,
-                    y: 100
-                }
-            );
-
-            // Add to the specific sheet
-            this.client.addMember(CONSTANTS.PROJECT_ROOT_ID, newId, metatabId);
-            this.client.setMemberRegistry(
-                CONSTANTS.PROJECT_ROOT_ID,
-                newId,
-                metatabId,
-                REGISTRY_KEYS.POSITION,
-                {
-                    x: 100,
-                    y: 100
-                }
-            );
-        }
         this.client.completeTransaction();
 
         WebGMEGlobal.State.registerActiveObject(newId);
+        return newId;
     };
 
+
     instances.forEach(type => {
         create[type] = function() {
             return createNew.call(this, type);
@@ -114,6 +51,37 @@ define([
         };
     });
 
+    var createLayer = function() {
+        // Prompt the base type
+        this.promptLayerType().then(selected => {
+            var baseId = selected.node.id,
+                typeName = this.client.getNode(baseId).getAttribute('name'),
+                metanodes = this.client.getAllMetaNodes(),
+                msg = `Created new custom ${typeName} layer`,
+                newId,
+                customLayerId,
+                name;
+
+            for (var i = metanodes.length; i--;) {
+                name = metanodes[i].getAttribute('name');
+                if (name === 'CustomLayer') {
+                    customLayerId = metanodes[i].getId();
+                    break;
+                }
+            }
+
+            this.client.startTransaction(msg);
+
+            newId = this.createNamedNode(baseId, true);
+            this.addToMetaSheet(newId, 'CustomLayers');
+            this.client.addMixin(newId, customLayerId);
+            this.client.setRegistry(newId, REGISTRY_KEYS.IS_ABSTRACT, false);
+
+            this.client.completeTransaction();
+
+            WebGMEGlobal.State.registerActiveObject(newId);
+        });
+    };
     ////////////// Downloading files //////////////
     var downloadAttrs = [
             'data',
@@ -216,6 +184,13 @@ define([
                 action: create.Data
             }
         ],
+        MyLayers_META: [
+            {
+                name: 'Create new layer',
+                icon: 'queue',
+                action: createLayer
+            }
+        ],
         MyOperations_META: [
             {
                 name: 'Create new operation',