Skip to content
This repository has been archived by the owner on Jan 3, 2023. It is now read-only.

Commit

Permalink
Solve #1910, Enhance list file action (#1942)
Browse files Browse the repository at this point in the history
  • Loading branch information
@rum2mojito @littlezhou
rum2mojito authored and littlezhou committed Sep 26, 2018
1 parent 67a8e9f commit 14232ec
Show file tree
Hide file tree
Showing 3 changed files with 130 additions and 7 deletions.
65 changes: 60 additions & 5 deletions ...t-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/action/ListFileAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,24 +31,53 @@

import java.io.IOException;
import java.net.URI;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.LinkedList;
import java.util.Map;
import java.util.Queue;

/**
* An action to list files in a directory.
*/
@ActionSignature(
actionId = "list",
displayName = "list",
usage = HdfsAction.FILE_PATH + " $src"
usage = HdfsAction.FILE_PATH + " $src1" + ListFileAction.RECURSIVELY + " $src2" + ListFileAction.DUMP + " $src3"
+ ListFileAction.HUMAN + " $src4"
)
public class ListFileAction extends HdfsAction {
private static final Logger LOG = LoggerFactory.getLogger(ListFileAction.class);
private String srcPath;
private boolean recursively = false;
private boolean dump = false;
private boolean human = false;

// Options
public static final String RECURSIVELY = "-R";
public static final String DUMP = "-d";
public static final String HUMAN = "-h";

@Override
public void init(Map<String, String> args) {
super.init(args);
this.srcPath = args.get(FILE_PATH);
if (args.containsKey(RECURSIVELY)) {
this.recursively = true;
if (this.srcPath == null || this.srcPath == "")
this.srcPath = args.get(RECURSIVELY);
}
if (args.containsKey(DUMP)) {
this.dump = true;
if (this.srcPath == null || this.srcPath == "")
this.srcPath = args.get(DUMP);
}
if (args.containsKey(HUMAN)) {
this.human = true;
if (this.srcPath == null || this.srcPath == "")
this.srcPath = args.get(HUMAN);
}
if (this.srcPath == null || this.srcPath == "")
this.srcPath = args.get(FILE_PATH);
}

@Override
Expand All @@ -62,6 +91,17 @@ protected void execute() throws Exception {
listDirectory(srcPath);
}

private static String readableFileSize(long size) {
if(size <= 0)
return "0";
final String[] units = new String[] { "", "K", "M", "G", "T" };
int digitGroups = (int) (Math.log10(size)/Math.log10(1024));
if (digitGroups == 0) {
return Long.toString(size);
}
return new DecimalFormat("#,##0.#").format(size/Math.pow(1024, digitGroups)) + " " + units[digitGroups];
}

private void listDirectory(String src) throws IOException {
if (!src.startsWith("hdfs")) {
//list file in local Dir
Expand All @@ -70,15 +110,30 @@ private void listDirectory(String src) throws IOException {
appendLog("File not found!");
return;
}
if (hdfsFileStatus.isDir()) {
if (hdfsFileStatus.isDir() && !dump) {
DirectoryListing listing = dfsClient.listPaths(src, HdfsFileStatus.EMPTY_NAME);
HdfsFileStatus[] fileList = listing.getPartialListing();
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm");
for (int i = 0; i < fileList.length; i++) {
appendLog(
String.format("%s", fileList[i].getFullPath(new Path(src))));
String.format("%s%s %5d %s\t%s\t%13s %s %s", fileList[i].isDir() ? 'd' : '-',
fileList[i].getPermission(), fileList[i].getReplication(),
fileList[i].getOwner(), fileList[i].getGroup(),
human ? readableFileSize(fileList[i].getLen()) : fileList[i].getLen(),
formatter.format(fileList[i].getModificationTime()), fileList[i].getFullPath(new Path(src))));
if (recursively && fileList[i].isDir()) {
listDirectory(fileList[i].getFullPath(new Path(src)).toString());
}
}
} else {
appendLog(String.format("%s", src));
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm");
appendLog(
String.format("%s%s %5d %s\t%s\t%13s %s %s", hdfsFileStatus.isDir() ? 'd' : '-',
hdfsFileStatus.getPermission(),
hdfsFileStatus.getReplication(),
hdfsFileStatus.getOwner(), hdfsFileStatus.getGroup(),
human ? readableFileSize(hdfsFileStatus.getLen()) : hdfsFileStatus.getLen(),
formatter.format(hdfsFileStatus.getModificationTime()), hdfsFileStatus.getFullPath(new Path(src))));
}
} else {
//list file in remote Directory
Expand Down
2 changes: 1 addition & 1 deletion smart-zeppelin/zeppelin-web/src/app/dashboard/views/actions/actions.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ <h4>No action is running</h4>
<td class="bk">{{action.execHost}}</td>
<td>{{action.createTime}}</td>
<td>{{action.finishTime}}</td>
<td>{{action.finished ? action.runTime : "-"}}ms</td>
<td>{{action.finished ? action.runTime : "-"}}{{action.finished ? "ms" : ""}}</td>
<td>{{action.finished ? action.successful ? "Successful" : "Failed" : "-"}}</td>
<td>
<div style="width: 100%"><span class="small pull-left"></span>
Expand Down
70 changes: 69 additions & 1 deletion smart-zeppelin/zeppelin-web/src/app/dashboard/views/actions/actions.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,72 @@ angular.module('zeppelinWebApp')
search($scope.path);
};

$scope.jumpToPage = function () {
var index = document.getElementById('page').value;
$scope.gotoPage(Number(index));
};

function __search__ (text) {
if (!$scope.searching) {
$scope.currentSearchPage = 1;
}
$http.get(baseUrlSrv.getSmartApiRoot() + conf.restapiProtocol + '/actions/search/'
+ text + '/' + $scope.currentSearchPage + '/' + $scope.pageNumber + '/' + $scope.orderby + '/' + $scope.isDesc)
.then(function(response) {
var actionData = angular.fromJson(response.data);
$scope.totalNumber = actionData.body.totalNumOfActions;
$scope.actions = actionData.body.actions;
angular.forEach($scope.actions, function (data,index) {
data.runTime = data.finishTime - data.createTime;
data.createTime = data.createTime === 0 ? "-" :
$filter('date')(data.createTime,'yyyy-MM-dd HH:mm:ss');
data.finishTime = data.finished ? data.finishTime === 0 ? "-" :
$filter('date')(data.finishTime,'yyyy-MM-dd HH:mm:ss') : '-';
data.progress = Math.round(data.progress * 100);
data.progressColor = data.finished ? data.successful ? 'success' : 'danger' : 'warning';
});
$scope.totalPage = Math.ceil($scope.totalNumber / $scope.pageNumber);
}, function(errorResponse) {
$scope.totalNumber = 0;
});
};

function search(text) {
if (text == "") {
$scope.searching = false;
getActions();
}
else {
__search__(text);
$scope.searching = true;
}
};

$scope.getContent = function () {
var tmp = document.getElementById('search').value;
var res = "";
for (var i = 0; i < tmp.length; i++) {
if (tmp.charAt(i) == '%') {
res += "%25";
}
else if (tmp.charAt(i) == '/'){
res += "%2F";
}
else if (tmp.charAt(i) == '?'){
res += "%3F";
}
else if (tmp.charAt(i) == '@'){
res += "%40";
}
else {
res += tmp.charAt(i);
}
}
$scope.path = res;
search($scope.path);

};

$scope.gotoPage = function (index) {
if (!$scope.searching) {
$scope.currentPage = index;
Expand Down Expand Up @@ -168,7 +234,9 @@ angular.module('zeppelinWebApp')

$(document).keyup(function(event) {
if (event.keyCode == 27) {
$scope.searching = false;
getActions();
document.getElementById("search").value = "";
}
});

Expand All @@ -179,7 +247,7 @@ angular.module('zeppelinWebApp')
else {
__search__($scope.path);
}
},60000);
},2000);

$scope.$on('$destroy',function(){
$interval.cancel(timer);
Expand Down

0 comments on commit 14232ec

Please sign in to comment.