Refine storage utilization page

.gitignore

@@ -43,11 +43,13 @@ smart-zeppelin/zeppelin-web/.flattened-pom.xml
 smart-zeppelin/zeppelin-web/.tmp
 smart-zeppelin/zeppelin-web/bower_components
 smart-zeppelin/zeppelin-web/dist
+smart-zeppelin/zeppelin-web/src/assets/resp.json
 smart-zeppelin/zeppelin-web/node
 smart-zeppelin/zeppelin-web/node_modules
 smart-zeppelin/zeppelin-web/reports
 smart-zeppelin/zeppelin-web/src/fonts
 smart-zeppelin/zeppelin-web/yarn-error.log
 smart-zeppelin/zeppelin-web/yarn.lock
+smart-zeppelin/zeppelin-web/package-lock.json
 smart-zeppelin/zeppelin-zengine/.flattened-pom.xml
 smart-zeppelin/zeppelin-distribution/.flattened-pom.xml

smart-common/src/main/java/org/smartdata/NodeHosts.java

@@ -0,0 +1,74 @@
+package org.smartdata;
+
+import org.smartdata.conf.SmartConf;
+import org.smartdata.conf.SmartConfKeys;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.HashSet;
+import java.util.Scanner;
+import java.util.Set;
+
+public class NodeHosts {
+
+    private SmartConf conf;
+    public Set<String> agentHosts;
+    public Set<String> serverHosts;
+    public NodeHosts(SmartConf conf) {
+        this.conf = conf;
+        this.serverHosts = init("server");
+        this.agentHosts = init("agent");
+    }
+
+    public Set<String> init(String role) {
+        String fileName = "/agents";
+        switch (role) {
+            case "agent":
+                fileName = "/agents";
+                break;
+            case "server":
+                fileName = "/servers";
+                break;
+        }
+        String hostName = "";
+        try {
+            InetAddress address = InetAddress.getLocalHost();
+            hostName = address.getHostName();
+        } catch (UnknownHostException e) {
+            e.printStackTrace();
+        }
+
+        String agentConfFile = conf.get(SmartConfKeys.SMART_CONF_DIR_KEY,
+                SmartConfKeys.SMART_CONF_DIR_DEFAULT) + fileName;
+        Scanner sc = null;
+        HashSet<String> hosts = new HashSet<>();
+        try {
+            sc = new Scanner(new File(agentConfFile));
+        } catch (FileNotFoundException ex) {
+            ex.printStackTrace();
+        }
+
+        while (sc != null && sc.hasNextLine()) {
+            String host = sc.nextLine().trim();
+            if (!host.startsWith("#") && !host.isEmpty()) {
+                if (host.equals("localhost")) {
+                    hosts.add(hostName);
+                } else {
+                    hosts.add(host);
+                }
+            }
+        }
+
+        return hosts;
+    }
+
+    public Set<String> getServerHosts() {
+        return serverHosts;
+    }
+
+    public Set<String> getAgentHosts() {
+        return agentHosts;
+    }
+}

smart-engine/src/main/java/org/smartdata/server/SmartEngine.java

@@ -20,6 +20,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.smartdata.AbstractService;
+import org.smartdata.NodeHosts;
 import org.smartdata.conf.SmartConf;
 import org.smartdata.model.StorageCapacity;
 import org.smartdata.model.Utilization;
@@ -36,11 +37,13 @@
 import org.smartdata.server.engine.cmdlet.agent.AgentInfo;
 
 import java.io.IOException;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Random;
+import java.util.Set;
 
 public class SmartEngine extends AbstractService {
   private ConfManager confMgr;
@@ -50,6 +53,7 @@ public class SmartEngine extends AbstractService {
   private RuleManager ruleMgr;
   private CmdletManager cmdletManager;
   private AgentExecutorService agentService;
+  private NodeHosts nodeHosts;
   private HazelcastExecutorService hazelcastService;
   private List<AbstractService> services = new ArrayList<>();
   public static final Logger LOG = LoggerFactory.getLogger(SmartEngine.class);
@@ -67,6 +71,7 @@ public void init() throws IOException {
     cmdletManager = new CmdletManager(serverContext);
     services.add(cmdletManager);
     agentService = new AgentExecutorService(conf, cmdletManager);
+    nodeHosts = new NodeHosts(conf);
     hazelcastService = new HazelcastExecutorService(cmdletManager);
     cmdletManager.registerExecutorService(agentService);
     cmdletManager.registerExecutorService(hazelcastService);
@@ -120,6 +125,14 @@ public List<StandbyServerInfo> getStandbyServers() {
     return hazelcastService.getStandbyServers();
   }
 
+  public Set<String> getAgentHosts() {
+    return nodeHosts.getAgentHosts();
+  }
+
+  public Set<String> getServerHosts() {
+    return nodeHosts.getServerHosts();
+  }
+
   public List<AgentInfo> getAgents() {
     return agentService.getAgentInfos();
   }

smart-zeppelin/zeppelin-server/src/main/java/org/smartdata/server/rest/SystemRestApi.java

@@ -60,4 +60,16 @@ public Response agents() {
     // return list of agents and their states
     return new JsonResponse<>(Response.Status.OK, smartEngine.getAgents()).build();
   }
+
+  @GET
+  @Path("/all_agent_hosts")
+  public Response allAgentHosts() {
+    return new JsonResponse<>(Response.Status.OK, smartEngine.getAgentHosts()).build();
+  }
+
+  @GET
+  @Path("/all_server_hosts")
+  public Response allMasterHosts() {
+    return new JsonResponse<>(Response.Status.OK, smartEngine.getServerHosts()).build();
+  }
 }

smart-zeppelin/zeppelin-web/src/app/app.js

@@ -182,7 +182,13 @@ var zeppelinWebApp = angular.module('zeppelinWebApp', [
         resolve: {
           nodes0: ['models', function (models) {
             return models.$get.nodes();
-          }]
+          }],
+          serverHosts: ['models', function (models) {
+            return models.$get.serverHosts();
+          }],
+          agentHosts: ['models', function (models) {
+            return models.$get.agentHosts();
+          }],
         }
       })
       .when('/jobmanager', {

smart-zeppelin/zeppelin-web/src/app/dashboard/services/models/models.js

@@ -20,7 +20,6 @@ angular.module('org.apache.hadoop.ssm.models', [])
 
   .factory('models', ['$timeout', 'conf', 'restapi', 'locator',
     function ($timeout, conf, restapi, locator) {
-      'use strict';
 
       var util = {
         usage: function (current, total) {
@@ -263,6 +262,12 @@ angular.module('org.apache.hadoop.ssm.models', [])
         nodes: function () {
           return get('cluster/primary/ssmnodescmdletmetrics', decoder.nodes);
         },
+        agentHosts: function () {
+          return get('system/all_agent_hosts', decoder.nodes);
+        },
+        serverHosts: function () {
+          return get('system/all_server_hosts', decoder.nodes);
+        },
         notebookInfo: function () {
           return get('note/info', decoder.default)
         }

smart-zeppelin/zeppelin-web/src/app/dashboard/views/cluster/nodeinfo/nodeStatus.css

@@ -0,0 +1,11 @@
+.statusPoint {
+  width: 15px;
+  height: 15px;
+  border-radius: 15px;
+}
+
+.statusPoint-container {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+}

smart-zeppelin/zeppelin-web/src/app/dashboard/views/cluster/nodeinfo/nodes.html

@@ -16,17 +16,37 @@
 specific language governing permissions and limitations
 under the License.
 -->
-<div class="col-md-12">
+<div class="col-md-2"></div>
+<div class="col-md-8">
   <br/>
-  <table class="table table-bordered">
-    <caption class="text-center"><h4>Node Info</h4></caption>
+  <table class="table table-bordered table-hover table-condensed" style="margin-bottom: 40px;margin-top: 40px">
+    <!--<caption class="text-center" style="margin-bottom: 20px; float: left"><h3><strong>Node Info</strong></h3></caption>-->
+    <tr>
+      <td class="col-md-5">Live Nodes</td>
+      <td>{{liveNumber}}</td>
+    </tr>
+    <tr class="{{deadNumber !== 0 ? 'danger' : ''}}">
+      <td>Dead Nodes</td>
+      <td>{{deadNumber}}</td>
+    </tr>
+    <tr>
+      <td>Pending Cmdlets for Scheduling</td>
+      <td>{{(nodes[0].nodeInfo.executorType == "LOCAL" ? nodes[0].numPendingSchedule : "")}}</td>
+    </tr>
+    <tr>
+      <td>Pending Cmdlets for Dispatching</td>
+      <td>{{(nodes[0].nodeInfo.executorType == "LOCAL" ? nodes[0].numPendingDispatch : "")}}</td>
+    </tr>
+  </table>
+  <table class="table table-bordered table-hover table-condensed">
     <thead ng-show="nodes.length > 0">
     <tr>
-      <th>Id</th>
-      <th>Regist Time</th>
-      <th>Number of Executors</th>
+      <th>ID</th>
+      <th>Register Time</th>
+      <th>Executors</th>
+      <th>Cmdlets in Execution</th>
       <th>Cmdlets Executed</th>
-      <th>Cmdlets In Execution</th>
+      <th>Status</th>
     </tr>
     </thead>
     <thead ng-if="nodes.length <= 0">
@@ -35,21 +55,25 @@
         <div class="table-no-data" style="width: 100%;">
           <h2 class="glyphicon glyphicon-bullhorn"></h2>
           <h4>No node is running</h4>
-
           <p>Please start a node first.</p>
         </div>
       </th>
     </tr>
     </thead>
-    <tbody>
-    <tr ng-repeat="node in nodes">
-      <td>{{node.nodeInfo.id + (node.nodeInfo.executorType == "LOCAL" ? " (PendingSchedule:" + node.numPendingSchedule + ", PendingDispatch:" + node.numPendingDispatch + ")" : "")}}</td>
-      <td>{{node.registTime}}</td>
-      <td>{{node.numExecutors}}</td>
-      <td>{{node.cmdletsExecuted}}</td>
-      <td>{{node.cmdletsInExecution}}</td>
+    <tbody ng-init="index=-1">
+    <tr ng-repeat="node in nodes" ng-init="isLive=node.isLive">
+      <td>{{
+        isLive
+        ? (node.nodeInfo.id)
+        : node.nodeInfo.host + (node.type === "agent" ? "@agent" : "@server")
+        }}</td>
+      <td>{{isLive ? node.registTime : '-'}}</td>
+      <td>{{isLive ? node.numExecutors : '-'}}</td>
+      <td>{{isLive ? node.cmdletsInExecution : '-'}}</td>
+      <td>{{isLive ? node.cmdletsExecuted : '-'}}</td>
+      <td class="statusPoint-container"><div class="statusPoint {{isLive ? 'btn-success' : 'btn-danger'}}"></div></td>
     </tr>
     </tbody>
   </table>
 </div>
-
+<div class="col-md-2"></div>

smart-zeppelin/zeppelin-web/src/app/dashboard/views/cluster/nodeinfo/nodes.js

@@ -19,11 +19,59 @@
 angular.module('zeppelinWebApp')
 
   .controller('NodesCtrl', NodesCtrl);
-  NodesCtrl.$inject = ['$scope', '$filter', 'nodes0'];
-  function NodesCtrl($scope, $filter, nodes0) {
+  NodesCtrl.$inject = ['$scope', '$filter', 'nodes0', 'serverHosts', 'agentHosts'];
+  function NodesCtrl($scope, $filter, nodes0, serverHosts, agentHosts) {
     $scope.nodes = nodes0.body;
+    $scope.serverHosts = serverHosts.body;
+    $scope.agentHosts = agentHosts.body;
+
+    angular.forEach($scope.nodes, function (data, index) {
+      if ('maxInExecution' in data) {
+        let index = $scope.serverHosts.indexOf(data.nodeInfo.host);
+        if (index >= 0) {
+          data.isLive = true;
+          data.type = 'server';
+          $scope.serverHosts.splice(index, 1);
+        }
+      } else {
+        let index = $scope.agentHosts.indexOf(data.nodeInfo.host);
+        if (index >= 0) {
+          data.isLive = true;
+          data.type = 'agent';
+          $scope.agentHosts.splice(index, 1);
+        }
+      }
+    });
+
+    $scope.liveNumber = $scope.nodes.length;
+    $scope.deadNumber = $scope.serverHosts.length + $scope.agentHosts.length;
+
+    angular.forEach($scope.serverHosts, function (host, index) {
+      $scope.nodes.push(
+        {
+          nodeInfo: {
+            host: host
+          },
+          isLive: false,
+          type: 'server'
+        }
+      );
+    });
+
+    angular.forEach($scope.agentHosts, function (host, index) {
+      $scope.nodes.push(
+        {
+          nodeInfo: {
+            host: host
+          },
+          isLive: false,
+          type: 'agent'
+        }
+      );
+    });
+
     angular.forEach($scope.nodes, function (data,index) {
-      data.registTime = data.registTime === 0 ? "-" :
+      data.registTime = data.registTime === 0 ? '-' :
         $filter('date')(data.registTime,'yyyy-MM-dd HH:mm:ss');
     });
   }

smart-zeppelin/zeppelin-web/src/app/dashboard/views/cluster/storage/bigChart.css

@@ -0,0 +1,18 @@
+.thumbnail-thumbnail-margin-right {
+  margin-right: 10px;
+}
+
+.storage-thumbnail {
+  position: relative;
+  transition: all 0.4s;
+  cursor: pointer;
+}
+
+.storage-thumbnail:hover {
+  top: -5px;
+  box-shadow: 0 5px 25px -6px #8f8f8f ;
+}
+
+.nvtooltip {
+  z-index: 10003;
+}