Skip to content

Commit

Permalink
Make capacity command show worker state
Browse files Browse the repository at this point in the history 
### What changes are proposed in this pull request?

```
bin/alluxio fsadmin report capacity
Capacity information for all workers:
    Total Capacity: 10.67GB
        Tier: MEM  Size: 10.67GB
    Used Capacity: 0B
        Tier: MEM  Size: 0B
    Used Percentage: 0%
    Free Percentage: 100%
Format is short: true
Format is %-16s %-15s %-16s %-13s %s %-16s %-40s

Worker Name      State           Last Heartbeat   Storage       MEM              Version          Revision
192.168.3.8      ACTIVE           166              capacity      10.67GB          2.10.0-SNAPSHOT  ffcb706497bf47e43d5b2efc90f664c7a3e7014e
                                                  used          0B (0%)
```


### What changes are proposed in this pull request?

Please outline the changes and how this PR fixes the issue.

### Why are the changes needed?

Please clarify why the changes are needed. For instance,
  1. If you propose a new API, clarify the use case for a new API.
  2. If you fix a bug, describe the bug.

### Does this PR introduce any user facing changes?

Please list the user-facing changes introduced by your change, including
  1. change in user-facing APIs
  2. addition or removal of property keys
  3. webui

			pr-link: #17325
			change-id: cid-598ce3cd88ee00e5981b0de273b27a3ea18d710b
  • Loading branch information
@jiacheliu3
jiacheliu3 authored Apr 28, 2023
1 parent 141ee0e commit 26257e6
Show file tree
Hide file tree
Showing 8 changed files with 68 additions and 43 deletions.
9 changes: 5 additions & 4 deletions ...lluxio/master/block/meta/WorkerState.java → ...main/java/alluxio/master/WorkerState.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,16 @@
* See the NOTICE file distributed with this work for information regarding copyright ownership.
*/

package alluxio.master.block.meta;
package alluxio.master;

/***
* The worker state maintained by master.
*/
public enum WorkerState {
LIVE("In Service"),
LOST("Out of Service"),
DECOMMISSIONED("Decommissioned");
LIVE("ACTIVE"),
LOST("LOST"),
DECOMMISSIONED("Decommissioned"),
DISABLED("Disabled");
private final String mState;

WorkerState(String s) {
Expand Down
27 changes: 22 additions & 5 deletions core/server/master/src/main/java/alluxio/master/block/DefaultBlockMaster.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@
import alluxio.heartbeat.HeartbeatThread;
import alluxio.master.CoreMaster;
import alluxio.master.CoreMasterContext;
import alluxio.master.WorkerState;
import alluxio.master.block.meta.MasterWorkerInfo;
import alluxio.master.block.meta.WorkerMetaLockSection;
import alluxio.master.block.meta.WorkerState;
import alluxio.master.journal.JournalContext;
import alluxio.master.journal.SingleEntryJournaled;
import alluxio.master.journal.checkpoint.CheckpointName;
Expand Down Expand Up @@ -677,6 +677,8 @@ public List<WorkerInfo> getWorkerInfoList() throws UnavailableException {
}

private List<WorkerInfo> constructWorkerInfoList() {
// TODO(jiacheng): investigate why this cache is refreshed so many times by the
// alluxio.master.scheduler.Scheduler L239
List<WorkerInfo> workerInfoList = new ArrayList<>(mWorkers.size());
for (MasterWorkerInfo worker : mWorkers) {
// extractWorkerInfo handles the locking internally
Expand Down Expand Up @@ -791,16 +793,31 @@ public List<WorkerInfo> getWorkerReport(GetWorkerReportOptions options)
+ selectedDecommissionedWorkers.size());
for (MasterWorkerInfo worker : selectedLiveWorkers) {
// extractWorkerInfo handles the locking internally
workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), WorkerState.LIVE));
if (mRejectWorkers.contains(worker.getWorkerAddress())) {
workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(),
WorkerState.DISABLED));
} else {
workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), WorkerState.LIVE));
}
}
for (MasterWorkerInfo worker : selectedLostWorkers) {
// extractWorkerInfo handles the locking internally
workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), WorkerState.LOST));
if (mRejectWorkers.contains(worker.getWorkerAddress())) {
workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(),
WorkerState.DISABLED));
} else {
workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), WorkerState.LOST));
}
}
for (MasterWorkerInfo worker : selectedDecommissionedWorkers) {
// extractWorkerInfo handles the locking internally
workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(),
WorkerState.DECOMMISSIONED));
if (mRejectWorkers.contains(worker.getWorkerAddress())) {
workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(),
WorkerState.DISABLED));
} else {
workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(),
WorkerState.DECOMMISSIONED));
}
}
return workerInfoList;
}
Expand Down
1 change: 1 addition & 0 deletions core/server/master/src/main/java/alluxio/master/block/meta/MasterWorkerInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import alluxio.client.block.options.GetWorkerReportOptions.WorkerInfoField;
import alluxio.grpc.BuildVersion;
import alluxio.grpc.StorageList;
import alluxio.master.WorkerState;
import alluxio.master.block.DefaultBlockMaster;
import alluxio.resource.LockResource;
import alluxio.util.CommonUtils;
Expand Down
3 changes: 1 addition & 2 deletions core/server/master/src/test/java/alluxio/master/block/BlockMasterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@
import alluxio.master.CoreMasterContext;
import alluxio.master.MasterRegistry;
import alluxio.master.MasterTestUtils;
import alluxio.master.WorkerState;
import alluxio.master.block.meta.MasterWorkerInfo;
import alluxio.master.block.meta.WorkerState;
import alluxio.master.journal.JournalSystem;
import alluxio.master.journal.noop.NoopJournalSystem;
import alluxio.master.metrics.MetricsMaster;
Expand Down Expand Up @@ -113,7 +113,6 @@ public class BlockMasterTest {
private MasterRegistry mRegistry;
private ManualClock mClock;
private ExecutorService mExecutorService;
private ExecutorService mClientExecutorService;
private MetricsMaster mMetricsMaster;
private List<Metric> mMetrics;

Expand Down
3 changes: 2 additions & 1 deletion core/server/master/src/test/java/alluxio/master/block/meta/MasterWorkerInfoTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import alluxio.DefaultStorageTierAssoc;
import alluxio.StorageTierAssoc;
import alluxio.client.block.options.GetWorkerReportOptions;
import alluxio.master.WorkerState;
import alluxio.wire.WorkerInfo;
import alluxio.wire.WorkerNetAddress;

Expand Down Expand Up @@ -142,7 +143,7 @@ public void workerInfoGeneration() {
WorkerState.LIVE);
assertEquals(mInfo.getId(), workerInfo.getId());
assertEquals(mInfo.getWorkerAddress(), workerInfo.getAddress());
assertEquals("In Service", workerInfo.getState());
assertEquals(WorkerState.LIVE.toString(), workerInfo.getState());
assertEquals(mInfo.getCapacityBytes(), workerInfo.getCapacityBytes());
assertEquals(mInfo.getUsedBytes(), workerInfo.getUsedBytes());
assertEquals(mInfo.getStartTime(), workerInfo.getStartTimeMs());
Expand Down
19 changes: 11 additions & 8 deletions shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,7 @@ private void printWorkerInfo(List<WorkerInfo> workerInfoList) {
if (mCapacityTierInfoMap.size() == 0) {
return;
} else if (mCapacityTierInfoMap.size() == 1) {
// TODO(jiacheng): test BOTH long and short output
// Do not print Total value when only one tier exists
printShortWorkerInfo(workerInfoList);
return;
Expand All @@ -309,7 +310,8 @@ private void printWorkerInfo(List<WorkerInfo> workerInfoList) {
String tiersInfo = String.format(Strings.repeat("%-14s", tiers.size()), tiers.toArray());
String longInfoFormat = getInfoFormat(workerInfoList, false);
print(String.format("%n" + longInfoFormat,
"Worker Name", "Last Heartbeat", "Storage", "Total", tiersInfo, "Version", "Revision"));
"Worker Name", "State", "Last Heartbeat", "Storage", "Total", tiersInfo,
"Version", "Revision"));

for (WorkerInfo info : workerInfoList) {
String workerName = info.getAddress().getHost();
Expand All @@ -326,10 +328,11 @@ private void printWorkerInfo(List<WorkerInfo> workerInfoList) {
String capacityTierInfo = getWorkerFormattedTierValues(mCapacityTierInfoMap, workerName);
String usedTierInfo = getWorkerFormattedTierValues(mUsedTierInfoMap, workerName);

print(String.format(longInfoFormat, workerName, info.getLastContactSec(), "capacity",
print(String.format(longInfoFormat, workerName, info.getState(),
info.getLastContactSec(), "capacity",
FormatUtils.getSizeFromBytes(capacityBytes), capacityTierInfo,
info.getVersion(), info.getRevision()));
print(String.format(longInfoFormat, "", "", "used",
print(String.format(longInfoFormat, "", "", "", "used",
FormatUtils.getSizeFromBytes(usedBytes) + usedPercentageInfo, usedTierInfo,
"", ""));
}
Expand All @@ -344,7 +347,7 @@ private void printShortWorkerInfo(List<WorkerInfo> workerInfoList) {
String tier = String.format("%-16s", mCapacityTierInfoMap.firstKey());
String shortInfoFormat = getInfoFormat(workerInfoList, true);
print(String.format("%n" + shortInfoFormat,
"Worker Name", "Last Heartbeat", "Storage", tier, "Version", "Revision"));
"Worker Name", "State", "Last Heartbeat", "Storage", tier, "Version", "Revision"));

for (WorkerInfo info : workerInfoList) {
long capacityBytes = info.getCapacityBytes();
Expand All @@ -355,11 +358,11 @@ private void printShortWorkerInfo(List<WorkerInfo> workerInfoList) {
int usedPercentage = (int) (100L * usedBytes / capacityBytes);
usedPercentageInfo = String.format(" (%s%%)", usedPercentage);
}
print(String.format(shortInfoFormat, info.getAddress().getHost(),
print(String.format(shortInfoFormat, info.getAddress().getHost(), info.getState(),
info.getLastContactSec(), "capacity",
String.format("%-16s", FormatUtils.getSizeFromBytes(capacityBytes)),
info.getVersion(), info.getRevision()));
print(String.format(shortInfoFormat, "", "", "used",
print(String.format(shortInfoFormat, "", "", "", "used",
String.format("%-16s", FormatUtils.getSizeFromBytes(usedBytes) + usedPercentageInfo),
"", ""));
}
Expand All @@ -380,9 +383,9 @@ private String getInfoFormat(List<WorkerInfo> workerInfoList, boolean isShort) {
firstIndent = maxWorkerNameLength + 5;
}
if (isShort) {
return "%-" + firstIndent + "s %-16s %-13s %s %-16s %-40s";
return "%-" + firstIndent + "s %-15s %-16s %-13s %s %-16s %-40s";
}
return "%-" + firstIndent + "s %-16s %-13s %-16s %s %-16s %-40s";
return "%-" + firstIndent + "s %-15s %-16s %-13s %-16s %s %-16s %-40s";
}

/**
Expand Down
43 changes: 22 additions & 21 deletions shell/src/test/java/alluxio/cli/fsadmin/report/CapacityCommandTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import alluxio.client.block.BlockMasterClient;
import alluxio.client.block.options.GetWorkerReportOptions;
import alluxio.conf.Configuration;
import alluxio.master.WorkerState;
import alluxio.wire.WorkerInfo;
import alluxio.wire.WorkerNetAddress;

Expand Down Expand Up @@ -73,11 +74,11 @@ public void longCapacity() throws IOException {
" Used Percentage: 34%",
" Free Percentage: 66%",
"",
"Worker Name Last Heartbeat Storage Total MEM SSD HDD DOM RAM Version Revision ",
"216.239.33.96 542 capacity 18.63GB 4768.37MB 4768.37MB - 9.31GB - 2.10.0-SNAPSHOT 0123456789abcdef0123456789abcdef01234567",
" used 953.67MB (5%) 190.73MB 286.10MB - 476.84MB - ",
"64.68.90.1 3123 capacity 11.18GB 3814.70MB - 1907.35MB - 5.59GB 2.9.3 0123456789012345678901234567890123456789",
" used 9.31GB (83%) 2861.02MB - 1907.35MB - 4768.37MB ");
"Worker Name State Last Heartbeat Storage Total MEM SSD HDD DOM RAM Version Revision ",
"216.239.33.96 ACTIVE 542 capacity 18.63GB 4768.37MB 4768.37MB - 9.31GB - 2.10.0-SNAPSHOT 0123456789abcdef0123456789abcdef01234567",
" used 953.67MB (5%) 190.73MB 286.10MB - 476.84MB - ",
"64.68.90.1 ACTIVE 3123 capacity 11.18GB 3814.70MB - 1907.35MB - 5.59GB 2.9.3 0123456789012345678901234567890123456789",
" used 9.31GB (83%) 2861.02MB - 1907.35MB - 4768.37MB ");
// CHECKSTYLE.ON: LineLengthExceed
List<String> testOutput = Arrays.asList(output.split("\n"));
Assert.assertThat(testOutput,
Expand Down Expand Up @@ -107,11 +108,11 @@ public void shortCapacity() throws IOException {
" Used Percentage: 34%",
" Free Percentage: 66%",
"",
"Worker Name Last Heartbeat Storage RAM Version Revision ",
"215.42.95.24 953 capacity 9.31GB 2.2.4 000111222333444555666777888999aaabbbcccd",
" used 476.84MB (5%) ",
"29.53.5.124 6424122 capacity 5.59GB 2.2.3 00112233445566778899aabbccddeeff00112233",
" used 4768.37MB (83%) ");
"Worker Name State Last Heartbeat Storage RAM Version Revision ",
"215.42.95.24 ACTIVE 953 capacity 9.31GB 2.2.4 000111222333444555666777888999aaabbbcccd",
" used 476.84MB (5%) ",
"29.53.5.124 LOST 6424122 capacity 5.59GB 2.2.3 00112233445566778899aabbccddeeff00112233",
" used 4768.37MB (83%) ");
List<String> testOutput = Arrays.asList(output.split("\n"));
Assert.assertThat(testOutput,
IsIterableContainingInOrder.contains(expectedOutput.toArray()));
Expand Down Expand Up @@ -145,11 +146,11 @@ public void longWorkerNameCapacity() throws IOException {
" Used Percentage: 34%",
" Free Percentage: 66%",
"",
"Worker Name Last Heartbeat Storage Total MEM SSD HDD Version Revision ",
"org.apache.hdp1 681 capacity 1907.35MB 572.20MB 572.20MB - 2.10.0-rc1 abababababababababababababababababababab",
" used 95.37MB (5%) 19.07MB 28.61MB - ",
"org.alluxio.long.host1 6211 capacity 1144.41MB 572.20MB - 190.73MB 2.10.0-rc2 0101010101010101010101010101010101010101",
" used 953.67MB (83%) 286.10MB - 190.73MB ");
"Worker Name State Last Heartbeat Storage Total MEM SSD HDD Version Revision ",
"org.apache.hdp1 ACTIVE 681 capacity 1907.35MB 572.20MB 572.20MB - 2.10.0-rc1 abababababababababababababababababababab",
" used 95.37MB (5%) 19.07MB 28.61MB - ",
"org.alluxio.long.host1 ACTIVE 6211 capacity 1144.41MB 572.20MB - 190.73MB 2.10.0-rc2 0101010101010101010101010101010101010101",
" used 953.67MB (83%) 286.10MB - 190.73MB ");
// CHECKSTYLE.ON: LineLengthExceed
List<String> testOutput = Arrays.asList(output.split("\n"));

Expand Down Expand Up @@ -178,7 +179,7 @@ private List<WorkerInfo> prepareLongInfoList() {
.setId(1)
.setLastContactSec(3123)
.setStartTimeMs(1331231121212L)
.setState("In Service")
.setState(WorkerState.LIVE.toString())
.setUsedBytes(10000000000L)
.setUsedBytesOnTiers(usedBytesOnTiersOne)
.setVersion("2.9.3")
Expand All @@ -199,7 +200,7 @@ private List<WorkerInfo> prepareLongInfoList() {
.setId(2)
.setLastContactSec(542)
.setStartTimeMs(1131231121212L)
.setState("In Service")
.setState(WorkerState.LIVE.toString())
.setUsedBytes(1000000000L)
.setUsedBytesOnTiers(usedBytesOnTiersSec)
.setVersion("2.10.0-SNAPSHOT")
Expand All @@ -226,7 +227,7 @@ private List<WorkerInfo> prepareShortInfoList() {
.setId(1)
.setLastContactSec(6424122)
.setStartTimeMs(19365332L)
.setState("Out of Service")
.setState(WorkerState.LOST.toString())
.setUsedBytes(5000000000L)
.setUsedBytesOnTiers(usedBytesOnTiersOne)
.setVersion("2.2.3")
Expand All @@ -243,7 +244,7 @@ private List<WorkerInfo> prepareShortInfoList() {
.setId(2)
.setLastContactSec(953)
.setStartTimeMs(112495222L)
.setState("In Service")
.setState(WorkerState.LIVE.toString())
.setUsedBytes(500000000L)
.setUsedBytesOnTiers(usedBytesOnTiersSec)
.setVersion("2.2.4")
Expand Down Expand Up @@ -272,7 +273,7 @@ private List<WorkerInfo> prepareLongWorkerNameInfoList() {
.setId(1)
.setLastContactSec(6211)
.setStartTimeMs(1529222699127L)
.setState("In Service")
.setState(WorkerState.LIVE.toString())
.setUsedBytes(1000000000L)
.setUsedBytesOnTiers(usedBytesOnTiersOne)
.setVersion("2.10.0-rc2")
Expand All @@ -291,7 +292,7 @@ private List<WorkerInfo> prepareLongWorkerNameInfoList() {
.setId(2)
.setLastContactSec(681)
.setStartTimeMs(1529222699127L)
.setState("In Service")
.setState(WorkerState.LIVE.toString())
.setUsedBytes(100000000L)
.setUsedBytesOnTiers(usedBytesOnTiersSec)
.setVersion("2.10.0-rc1")
Expand Down
6 changes: 4 additions & 2 deletions tests/src/test/java/alluxio/client/cli/fsadmin/command/CapacityCommandIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ public void allCapacity() {
Assert.assertEquals(" Free Percentage: 100%", lines[6]);
Assert.assertEquals("", lines[7]);
Assert.assertTrue(lines[8].matches(
"Worker Name {6,}Last Heartbeat {3}Storage {7}MEM {14}Version {10}Revision *"));
"Worker Name {6,}State {11,}Last Heartbeat {3}Storage {7}MEM {14}Version {10}Revision *"));
Assert.assertTrue(lines[9].contains("ACTIVE"));
Assert.assertTrue(lines[9].contains("capacity " + size));
Assert.assertTrue(lines[10].contains("used 0B (0%)"));
}
Expand Down Expand Up @@ -68,7 +69,8 @@ public void liveCapacity() {
Assert.assertEquals(" Free Percentage: 100%", lines[6]);
Assert.assertEquals("", lines[7]);
Assert.assertTrue(lines[8].matches(
"Worker Name {6,}Last Heartbeat {3}Storage {7}MEM {14}Version {10}Revision *"));
"Worker Name {6,}State {11,}Last Heartbeat {3}Storage {7}MEM {14}Version {10}Revision *"));
Assert.assertTrue(lines[9].contains("ACTIVE"));
Assert.assertTrue(lines[9].contains("capacity " + size));
Assert.assertTrue(lines[10].contains("used 0B (0%)"));
}
Expand Down

0 comments on commit 26257e6

Please sign in to comment.