From 14232ec0fbaee0bdbb3f59bbe29e0addeb29784f Mon Sep 17 00:00:00 2001 From: Yuwei Date: Wed, 26 Sep 2018 13:31:42 +0800 Subject: [PATCH] Solve #1910, Enhance list file action (#1942) --- .../smartdata/hdfs/action/ListFileAction.java | 65 +++++++++++++++-- .../app/dashboard/views/actions/actions.html | 2 +- .../app/dashboard/views/actions/actions.js | 70 ++++++++++++++++++- 3 files changed, 130 insertions(+), 7 deletions(-) diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ListFileAction.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ListFileAction.java index 013489f4f1b..8ef0c1897db 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ListFileAction.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ListFileAction.java @@ -31,7 +31,11 @@ import java.io.IOException; import java.net.URI; +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; +import java.util.LinkedList; import java.util.Map; +import java.util.Queue; /** * An action to list files in a directory. @@ -39,16 +43,41 @@ @ActionSignature( actionId = "list", displayName = "list", - usage = HdfsAction.FILE_PATH + " $src" + usage = HdfsAction.FILE_PATH + " $src1" + ListFileAction.RECURSIVELY + " $src2" + ListFileAction.DUMP + " $src3" + + ListFileAction.HUMAN + " $src4" ) public class ListFileAction extends HdfsAction { private static final Logger LOG = LoggerFactory.getLogger(ListFileAction.class); private String srcPath; + private boolean recursively = false; + private boolean dump = false; + private boolean human = false; + + // Options + public static final String RECURSIVELY = "-R"; + public static final String DUMP = "-d"; + public static final String HUMAN = "-h"; @Override public void init(Map args) { super.init(args); - this.srcPath = args.get(FILE_PATH); + if (args.containsKey(RECURSIVELY)) { + this.recursively = true; + if (this.srcPath == null || this.srcPath == "") + this.srcPath = args.get(RECURSIVELY); + } + if (args.containsKey(DUMP)) { + this.dump = true; + if (this.srcPath == null || this.srcPath == "") + this.srcPath = args.get(DUMP); + } + if (args.containsKey(HUMAN)) { + this.human = true; + if (this.srcPath == null || this.srcPath == "") + this.srcPath = args.get(HUMAN); + } + if (this.srcPath == null || this.srcPath == "") + this.srcPath = args.get(FILE_PATH); } @Override @@ -62,6 +91,17 @@ protected void execute() throws Exception { listDirectory(srcPath); } + private static String readableFileSize(long size) { + if(size <= 0) + return "0"; + final String[] units = new String[] { "", "K", "M", "G", "T" }; + int digitGroups = (int) (Math.log10(size)/Math.log10(1024)); + if (digitGroups == 0) { + return Long.toString(size); + } + return new DecimalFormat("#,##0.#").format(size/Math.pow(1024, digitGroups)) + " " + units[digitGroups]; + } + private void listDirectory(String src) throws IOException { if (!src.startsWith("hdfs")) { //list file in local Dir @@ -70,15 +110,30 @@ private void listDirectory(String src) throws IOException { appendLog("File not found!"); return; } - if (hdfsFileStatus.isDir()) { + if (hdfsFileStatus.isDir() && !dump) { DirectoryListing listing = dfsClient.listPaths(src, HdfsFileStatus.EMPTY_NAME); HdfsFileStatus[] fileList = listing.getPartialListing(); + SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm"); for (int i = 0; i < fileList.length; i++) { appendLog( - String.format("%s", fileList[i].getFullPath(new Path(src)))); + String.format("%s%s %5d %s\t%s\t%13s %s %s", fileList[i].isDir() ? 'd' : '-', + fileList[i].getPermission(), fileList[i].getReplication(), + fileList[i].getOwner(), fileList[i].getGroup(), + human ? readableFileSize(fileList[i].getLen()) : fileList[i].getLen(), + formatter.format(fileList[i].getModificationTime()), fileList[i].getFullPath(new Path(src)))); + if (recursively && fileList[i].isDir()) { + listDirectory(fileList[i].getFullPath(new Path(src)).toString()); + } } } else { - appendLog(String.format("%s", src)); + SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm"); + appendLog( + String.format("%s%s %5d %s\t%s\t%13s %s %s", hdfsFileStatus.isDir() ? 'd' : '-', + hdfsFileStatus.getPermission(), + hdfsFileStatus.getReplication(), + hdfsFileStatus.getOwner(), hdfsFileStatus.getGroup(), + human ? readableFileSize(hdfsFileStatus.getLen()) : hdfsFileStatus.getLen(), + formatter.format(hdfsFileStatus.getModificationTime()), hdfsFileStatus.getFullPath(new Path(src)))); } } else { //list file in remote Directory diff --git a/smart-zeppelin/zeppelin-web/src/app/dashboard/views/actions/actions.html b/smart-zeppelin/zeppelin-web/src/app/dashboard/views/actions/actions.html index 717731bf4d0..d8754265632 100644 --- a/smart-zeppelin/zeppelin-web/src/app/dashboard/views/actions/actions.html +++ b/smart-zeppelin/zeppelin-web/src/app/dashboard/views/actions/actions.html @@ -92,7 +92,7 @@

No action is running

{{action.execHost}} {{action.createTime}} {{action.finishTime}} - {{action.finished ? action.runTime : "-"}}ms + {{action.finished ? action.runTime : "-"}}{{action.finished ? "ms" : ""}} {{action.finished ? action.successful ? "Successful" : "Failed" : "-"}}
diff --git a/smart-zeppelin/zeppelin-web/src/app/dashboard/views/actions/actions.js b/smart-zeppelin/zeppelin-web/src/app/dashboard/views/actions/actions.js index aaf4b37f0f1..78f610479ea 100644 --- a/smart-zeppelin/zeppelin-web/src/app/dashboard/views/actions/actions.js +++ b/smart-zeppelin/zeppelin-web/src/app/dashboard/views/actions/actions.js @@ -124,6 +124,72 @@ angular.module('zeppelinWebApp') search($scope.path); }; + $scope.jumpToPage = function () { + var index = document.getElementById('page').value; + $scope.gotoPage(Number(index)); + }; + + function __search__ (text) { + if (!$scope.searching) { + $scope.currentSearchPage = 1; + } + $http.get(baseUrlSrv.getSmartApiRoot() + conf.restapiProtocol + '/actions/search/' + + text + '/' + $scope.currentSearchPage + '/' + $scope.pageNumber + '/' + $scope.orderby + '/' + $scope.isDesc) + .then(function(response) { + var actionData = angular.fromJson(response.data); + $scope.totalNumber = actionData.body.totalNumOfActions; + $scope.actions = actionData.body.actions; + angular.forEach($scope.actions, function (data,index) { + data.runTime = data.finishTime - data.createTime; + data.createTime = data.createTime === 0 ? "-" : + $filter('date')(data.createTime,'yyyy-MM-dd HH:mm:ss'); + data.finishTime = data.finished ? data.finishTime === 0 ? "-" : + $filter('date')(data.finishTime,'yyyy-MM-dd HH:mm:ss') : '-'; + data.progress = Math.round(data.progress * 100); + data.progressColor = data.finished ? data.successful ? 'success' : 'danger' : 'warning'; + }); + $scope.totalPage = Math.ceil($scope.totalNumber / $scope.pageNumber); + }, function(errorResponse) { + $scope.totalNumber = 0; + }); + }; + + function search(text) { + if (text == "") { + $scope.searching = false; + getActions(); + } + else { + __search__(text); + $scope.searching = true; + } + }; + + $scope.getContent = function () { + var tmp = document.getElementById('search').value; + var res = ""; + for (var i = 0; i < tmp.length; i++) { + if (tmp.charAt(i) == '%') { + res += "%25"; + } + else if (tmp.charAt(i) == '/'){ + res += "%2F"; + } + else if (tmp.charAt(i) == '?'){ + res += "%3F"; + } + else if (tmp.charAt(i) == '@'){ + res += "%40"; + } + else { + res += tmp.charAt(i); + } + } + $scope.path = res; + search($scope.path); + + }; + $scope.gotoPage = function (index) { if (!$scope.searching) { $scope.currentPage = index; @@ -168,7 +234,9 @@ angular.module('zeppelinWebApp') $(document).keyup(function(event) { if (event.keyCode == 27) { + $scope.searching = false; getActions(); + document.getElementById("search").value = ""; } }); @@ -179,7 +247,7 @@ angular.module('zeppelinWebApp') else { __search__($scope.path); } - },60000); + },2000); $scope.$on('$destroy',function(){ $interval.cancel(timer);