diff --git a/core/server/master/src/main/java/alluxio/master/block/meta/WorkerState.java b/core/common/src/main/java/alluxio/master/WorkerState.java similarity index 86% rename from core/server/master/src/main/java/alluxio/master/block/meta/WorkerState.java rename to core/common/src/main/java/alluxio/master/WorkerState.java index 0f5d2327ba28..8b4572fa7cde 100644 --- a/core/server/master/src/main/java/alluxio/master/block/meta/WorkerState.java +++ b/core/common/src/main/java/alluxio/master/WorkerState.java @@ -9,15 +9,16 @@ * See the NOTICE file distributed with this work for information regarding copyright ownership. */ -package alluxio.master.block.meta; +package alluxio.master; /*** * The worker state maintained by master. */ public enum WorkerState { - LIVE("In Service"), - LOST("Out of Service"), - DECOMMISSIONED("Decommissioned"); + LIVE("ACTIVE"), + LOST("LOST"), + DECOMMISSIONED("Decommissioned"), + DISABLED("Disabled"); private final String mState; WorkerState(String s) { diff --git a/core/server/master/src/main/java/alluxio/master/block/DefaultBlockMaster.java b/core/server/master/src/main/java/alluxio/master/block/DefaultBlockMaster.java index 73d0f6c83180..0a87d0086797 100644 --- a/core/server/master/src/main/java/alluxio/master/block/DefaultBlockMaster.java +++ b/core/server/master/src/main/java/alluxio/master/block/DefaultBlockMaster.java @@ -49,9 +49,9 @@ import alluxio.heartbeat.HeartbeatThread; import alluxio.master.CoreMaster; import alluxio.master.CoreMasterContext; +import alluxio.master.WorkerState; import alluxio.master.block.meta.MasterWorkerInfo; import alluxio.master.block.meta.WorkerMetaLockSection; -import alluxio.master.block.meta.WorkerState; import alluxio.master.journal.JournalContext; import alluxio.master.journal.SingleEntryJournaled; import alluxio.master.journal.checkpoint.CheckpointName; @@ -677,6 +677,8 @@ public List getWorkerInfoList() throws UnavailableException { } private List constructWorkerInfoList() { + // TODO(jiacheng): investigate why this cache is refreshed so many times by the + // alluxio.master.scheduler.Scheduler L239 List workerInfoList = new ArrayList<>(mWorkers.size()); for (MasterWorkerInfo worker : mWorkers) { // extractWorkerInfo handles the locking internally @@ -791,16 +793,31 @@ public List getWorkerReport(GetWorkerReportOptions options) + selectedDecommissionedWorkers.size()); for (MasterWorkerInfo worker : selectedLiveWorkers) { // extractWorkerInfo handles the locking internally - workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), WorkerState.LIVE)); + if (mRejectWorkers.contains(worker.getWorkerAddress())) { + workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), + WorkerState.DISABLED)); + } else { + workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), WorkerState.LIVE)); + } } for (MasterWorkerInfo worker : selectedLostWorkers) { // extractWorkerInfo handles the locking internally - workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), WorkerState.LOST)); + if (mRejectWorkers.contains(worker.getWorkerAddress())) { + workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), + WorkerState.DISABLED)); + } else { + workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), WorkerState.LOST)); + } } for (MasterWorkerInfo worker : selectedDecommissionedWorkers) { // extractWorkerInfo handles the locking internally - workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), - WorkerState.DECOMMISSIONED)); + if (mRejectWorkers.contains(worker.getWorkerAddress())) { + workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), + WorkerState.DISABLED)); + } else { + workerInfoList.add(extractWorkerInfo(worker, options.getFieldRange(), + WorkerState.DECOMMISSIONED)); + } } return workerInfoList; } diff --git a/core/server/master/src/main/java/alluxio/master/block/meta/MasterWorkerInfo.java b/core/server/master/src/main/java/alluxio/master/block/meta/MasterWorkerInfo.java index b2d08a66fcb3..d4dae4783b1d 100644 --- a/core/server/master/src/main/java/alluxio/master/block/meta/MasterWorkerInfo.java +++ b/core/server/master/src/main/java/alluxio/master/block/meta/MasterWorkerInfo.java @@ -17,6 +17,7 @@ import alluxio.client.block.options.GetWorkerReportOptions.WorkerInfoField; import alluxio.grpc.BuildVersion; import alluxio.grpc.StorageList; +import alluxio.master.WorkerState; import alluxio.master.block.DefaultBlockMaster; import alluxio.resource.LockResource; import alluxio.util.CommonUtils; diff --git a/core/server/master/src/test/java/alluxio/master/block/BlockMasterTest.java b/core/server/master/src/test/java/alluxio/master/block/BlockMasterTest.java index 71a200c1010f..d1cbb7a2a686 100644 --- a/core/server/master/src/test/java/alluxio/master/block/BlockMasterTest.java +++ b/core/server/master/src/test/java/alluxio/master/block/BlockMasterTest.java @@ -42,8 +42,8 @@ import alluxio.master.CoreMasterContext; import alluxio.master.MasterRegistry; import alluxio.master.MasterTestUtils; +import alluxio.master.WorkerState; import alluxio.master.block.meta.MasterWorkerInfo; -import alluxio.master.block.meta.WorkerState; import alluxio.master.journal.JournalSystem; import alluxio.master.journal.noop.NoopJournalSystem; import alluxio.master.metrics.MetricsMaster; @@ -113,7 +113,6 @@ public class BlockMasterTest { private MasterRegistry mRegistry; private ManualClock mClock; private ExecutorService mExecutorService; - private ExecutorService mClientExecutorService; private MetricsMaster mMetricsMaster; private List mMetrics; diff --git a/core/server/master/src/test/java/alluxio/master/block/meta/MasterWorkerInfoTest.java b/core/server/master/src/test/java/alluxio/master/block/meta/MasterWorkerInfoTest.java index c855f79fb6c7..e5bc773cce51 100644 --- a/core/server/master/src/test/java/alluxio/master/block/meta/MasterWorkerInfoTest.java +++ b/core/server/master/src/test/java/alluxio/master/block/meta/MasterWorkerInfoTest.java @@ -19,6 +19,7 @@ import alluxio.DefaultStorageTierAssoc; import alluxio.StorageTierAssoc; import alluxio.client.block.options.GetWorkerReportOptions; +import alluxio.master.WorkerState; import alluxio.wire.WorkerInfo; import alluxio.wire.WorkerNetAddress; @@ -142,7 +143,7 @@ public void workerInfoGeneration() { WorkerState.LIVE); assertEquals(mInfo.getId(), workerInfo.getId()); assertEquals(mInfo.getWorkerAddress(), workerInfo.getAddress()); - assertEquals("In Service", workerInfo.getState()); + assertEquals(WorkerState.LIVE.toString(), workerInfo.getState()); assertEquals(mInfo.getCapacityBytes(), workerInfo.getCapacityBytes()); assertEquals(mInfo.getUsedBytes(), workerInfo.getUsedBytes()); assertEquals(mInfo.getStartTime(), workerInfo.getStartTimeMs()); diff --git a/shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java b/shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java index 1d5699acbf41..908e705406e9 100644 --- a/shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java +++ b/shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java @@ -301,6 +301,7 @@ private void printWorkerInfo(List workerInfoList) { if (mCapacityTierInfoMap.size() == 0) { return; } else if (mCapacityTierInfoMap.size() == 1) { + // TODO(jiacheng): test BOTH long and short output // Do not print Total value when only one tier exists printShortWorkerInfo(workerInfoList); return; @@ -309,7 +310,8 @@ private void printWorkerInfo(List workerInfoList) { String tiersInfo = String.format(Strings.repeat("%-14s", tiers.size()), tiers.toArray()); String longInfoFormat = getInfoFormat(workerInfoList, false); print(String.format("%n" + longInfoFormat, - "Worker Name", "Last Heartbeat", "Storage", "Total", tiersInfo, "Version", "Revision")); + "Worker Name", "State", "Last Heartbeat", "Storage", "Total", tiersInfo, + "Version", "Revision")); for (WorkerInfo info : workerInfoList) { String workerName = info.getAddress().getHost(); @@ -326,10 +328,11 @@ private void printWorkerInfo(List workerInfoList) { String capacityTierInfo = getWorkerFormattedTierValues(mCapacityTierInfoMap, workerName); String usedTierInfo = getWorkerFormattedTierValues(mUsedTierInfoMap, workerName); - print(String.format(longInfoFormat, workerName, info.getLastContactSec(), "capacity", + print(String.format(longInfoFormat, workerName, info.getState(), + info.getLastContactSec(), "capacity", FormatUtils.getSizeFromBytes(capacityBytes), capacityTierInfo, info.getVersion(), info.getRevision())); - print(String.format(longInfoFormat, "", "", "used", + print(String.format(longInfoFormat, "", "", "", "used", FormatUtils.getSizeFromBytes(usedBytes) + usedPercentageInfo, usedTierInfo, "", "")); } @@ -344,7 +347,7 @@ private void printShortWorkerInfo(List workerInfoList) { String tier = String.format("%-16s", mCapacityTierInfoMap.firstKey()); String shortInfoFormat = getInfoFormat(workerInfoList, true); print(String.format("%n" + shortInfoFormat, - "Worker Name", "Last Heartbeat", "Storage", tier, "Version", "Revision")); + "Worker Name", "State", "Last Heartbeat", "Storage", tier, "Version", "Revision")); for (WorkerInfo info : workerInfoList) { long capacityBytes = info.getCapacityBytes(); @@ -355,11 +358,11 @@ private void printShortWorkerInfo(List workerInfoList) { int usedPercentage = (int) (100L * usedBytes / capacityBytes); usedPercentageInfo = String.format(" (%s%%)", usedPercentage); } - print(String.format(shortInfoFormat, info.getAddress().getHost(), + print(String.format(shortInfoFormat, info.getAddress().getHost(), info.getState(), info.getLastContactSec(), "capacity", String.format("%-16s", FormatUtils.getSizeFromBytes(capacityBytes)), info.getVersion(), info.getRevision())); - print(String.format(shortInfoFormat, "", "", "used", + print(String.format(shortInfoFormat, "", "", "", "used", String.format("%-16s", FormatUtils.getSizeFromBytes(usedBytes) + usedPercentageInfo), "", "")); } @@ -380,9 +383,9 @@ private String getInfoFormat(List workerInfoList, boolean isShort) { firstIndent = maxWorkerNameLength + 5; } if (isShort) { - return "%-" + firstIndent + "s %-16s %-13s %s %-16s %-40s"; + return "%-" + firstIndent + "s %-15s %-16s %-13s %s %-16s %-40s"; } - return "%-" + firstIndent + "s %-16s %-13s %-16s %s %-16s %-40s"; + return "%-" + firstIndent + "s %-15s %-16s %-13s %-16s %s %-16s %-40s"; } /** diff --git a/shell/src/test/java/alluxio/cli/fsadmin/report/CapacityCommandTest.java b/shell/src/test/java/alluxio/cli/fsadmin/report/CapacityCommandTest.java index 129fc67a0095..d4aa63800ffe 100644 --- a/shell/src/test/java/alluxio/cli/fsadmin/report/CapacityCommandTest.java +++ b/shell/src/test/java/alluxio/cli/fsadmin/report/CapacityCommandTest.java @@ -15,6 +15,7 @@ import alluxio.client.block.BlockMasterClient; import alluxio.client.block.options.GetWorkerReportOptions; import alluxio.conf.Configuration; +import alluxio.master.WorkerState; import alluxio.wire.WorkerInfo; import alluxio.wire.WorkerNetAddress; @@ -73,11 +74,11 @@ public void longCapacity() throws IOException { " Used Percentage: 34%", " Free Percentage: 66%", "", - "Worker Name Last Heartbeat Storage Total MEM SSD HDD DOM RAM Version Revision ", - "216.239.33.96 542 capacity 18.63GB 4768.37MB 4768.37MB - 9.31GB - 2.10.0-SNAPSHOT 0123456789abcdef0123456789abcdef01234567", - " used 953.67MB (5%) 190.73MB 286.10MB - 476.84MB - ", - "64.68.90.1 3123 capacity 11.18GB 3814.70MB - 1907.35MB - 5.59GB 2.9.3 0123456789012345678901234567890123456789", - " used 9.31GB (83%) 2861.02MB - 1907.35MB - 4768.37MB "); + "Worker Name State Last Heartbeat Storage Total MEM SSD HDD DOM RAM Version Revision ", + "216.239.33.96 ACTIVE 542 capacity 18.63GB 4768.37MB 4768.37MB - 9.31GB - 2.10.0-SNAPSHOT 0123456789abcdef0123456789abcdef01234567", + " used 953.67MB (5%) 190.73MB 286.10MB - 476.84MB - ", + "64.68.90.1 ACTIVE 3123 capacity 11.18GB 3814.70MB - 1907.35MB - 5.59GB 2.9.3 0123456789012345678901234567890123456789", + " used 9.31GB (83%) 2861.02MB - 1907.35MB - 4768.37MB "); // CHECKSTYLE.ON: LineLengthExceed List testOutput = Arrays.asList(output.split("\n")); Assert.assertThat(testOutput, @@ -107,11 +108,11 @@ public void shortCapacity() throws IOException { " Used Percentage: 34%", " Free Percentage: 66%", "", - "Worker Name Last Heartbeat Storage RAM Version Revision ", - "215.42.95.24 953 capacity 9.31GB 2.2.4 000111222333444555666777888999aaabbbcccd", - " used 476.84MB (5%) ", - "29.53.5.124 6424122 capacity 5.59GB 2.2.3 00112233445566778899aabbccddeeff00112233", - " used 4768.37MB (83%) "); + "Worker Name State Last Heartbeat Storage RAM Version Revision ", + "215.42.95.24 ACTIVE 953 capacity 9.31GB 2.2.4 000111222333444555666777888999aaabbbcccd", + " used 476.84MB (5%) ", + "29.53.5.124 LOST 6424122 capacity 5.59GB 2.2.3 00112233445566778899aabbccddeeff00112233", + " used 4768.37MB (83%) "); List testOutput = Arrays.asList(output.split("\n")); Assert.assertThat(testOutput, IsIterableContainingInOrder.contains(expectedOutput.toArray())); @@ -145,11 +146,11 @@ public void longWorkerNameCapacity() throws IOException { " Used Percentage: 34%", " Free Percentage: 66%", "", - "Worker Name Last Heartbeat Storage Total MEM SSD HDD Version Revision ", - "org.apache.hdp1 681 capacity 1907.35MB 572.20MB 572.20MB - 2.10.0-rc1 abababababababababababababababababababab", - " used 95.37MB (5%) 19.07MB 28.61MB - ", - "org.alluxio.long.host1 6211 capacity 1144.41MB 572.20MB - 190.73MB 2.10.0-rc2 0101010101010101010101010101010101010101", - " used 953.67MB (83%) 286.10MB - 190.73MB "); + "Worker Name State Last Heartbeat Storage Total MEM SSD HDD Version Revision ", + "org.apache.hdp1 ACTIVE 681 capacity 1907.35MB 572.20MB 572.20MB - 2.10.0-rc1 abababababababababababababababababababab", + " used 95.37MB (5%) 19.07MB 28.61MB - ", + "org.alluxio.long.host1 ACTIVE 6211 capacity 1144.41MB 572.20MB - 190.73MB 2.10.0-rc2 0101010101010101010101010101010101010101", + " used 953.67MB (83%) 286.10MB - 190.73MB "); // CHECKSTYLE.ON: LineLengthExceed List testOutput = Arrays.asList(output.split("\n")); @@ -178,7 +179,7 @@ private List prepareLongInfoList() { .setId(1) .setLastContactSec(3123) .setStartTimeMs(1331231121212L) - .setState("In Service") + .setState(WorkerState.LIVE.toString()) .setUsedBytes(10000000000L) .setUsedBytesOnTiers(usedBytesOnTiersOne) .setVersion("2.9.3") @@ -199,7 +200,7 @@ private List prepareLongInfoList() { .setId(2) .setLastContactSec(542) .setStartTimeMs(1131231121212L) - .setState("In Service") + .setState(WorkerState.LIVE.toString()) .setUsedBytes(1000000000L) .setUsedBytesOnTiers(usedBytesOnTiersSec) .setVersion("2.10.0-SNAPSHOT") @@ -226,7 +227,7 @@ private List prepareShortInfoList() { .setId(1) .setLastContactSec(6424122) .setStartTimeMs(19365332L) - .setState("Out of Service") + .setState(WorkerState.LOST.toString()) .setUsedBytes(5000000000L) .setUsedBytesOnTiers(usedBytesOnTiersOne) .setVersion("2.2.3") @@ -243,7 +244,7 @@ private List prepareShortInfoList() { .setId(2) .setLastContactSec(953) .setStartTimeMs(112495222L) - .setState("In Service") + .setState(WorkerState.LIVE.toString()) .setUsedBytes(500000000L) .setUsedBytesOnTiers(usedBytesOnTiersSec) .setVersion("2.2.4") @@ -272,7 +273,7 @@ private List prepareLongWorkerNameInfoList() { .setId(1) .setLastContactSec(6211) .setStartTimeMs(1529222699127L) - .setState("In Service") + .setState(WorkerState.LIVE.toString()) .setUsedBytes(1000000000L) .setUsedBytesOnTiers(usedBytesOnTiersOne) .setVersion("2.10.0-rc2") @@ -291,7 +292,7 @@ private List prepareLongWorkerNameInfoList() { .setId(2) .setLastContactSec(681) .setStartTimeMs(1529222699127L) - .setState("In Service") + .setState(WorkerState.LIVE.toString()) .setUsedBytes(100000000L) .setUsedBytesOnTiers(usedBytesOnTiersSec) .setVersion("2.10.0-rc1") diff --git a/tests/src/test/java/alluxio/client/cli/fsadmin/command/CapacityCommandIntegrationTest.java b/tests/src/test/java/alluxio/client/cli/fsadmin/command/CapacityCommandIntegrationTest.java index 2096f672d062..5014d3814d12 100644 --- a/tests/src/test/java/alluxio/client/cli/fsadmin/command/CapacityCommandIntegrationTest.java +++ b/tests/src/test/java/alluxio/client/cli/fsadmin/command/CapacityCommandIntegrationTest.java @@ -39,7 +39,8 @@ public void allCapacity() { Assert.assertEquals(" Free Percentage: 100%", lines[6]); Assert.assertEquals("", lines[7]); Assert.assertTrue(lines[8].matches( - "Worker Name {6,}Last Heartbeat {3}Storage {7}MEM {14}Version {10}Revision *")); + "Worker Name {6,}State {11,}Last Heartbeat {3}Storage {7}MEM {14}Version {10}Revision *")); + Assert.assertTrue(lines[9].contains("ACTIVE")); Assert.assertTrue(lines[9].contains("capacity " + size)); Assert.assertTrue(lines[10].contains("used 0B (0%)")); } @@ -68,7 +69,8 @@ public void liveCapacity() { Assert.assertEquals(" Free Percentage: 100%", lines[6]); Assert.assertEquals("", lines[7]); Assert.assertTrue(lines[8].matches( - "Worker Name {6,}Last Heartbeat {3}Storage {7}MEM {14}Version {10}Revision *")); + "Worker Name {6,}State {11,}Last Heartbeat {3}Storage {7}MEM {14}Version {10}Revision *")); + Assert.assertTrue(lines[9].contains("ACTIVE")); Assert.assertTrue(lines[9].contains("capacity " + size)); Assert.assertTrue(lines[10].contains("used 0B (0%)")); }