Solve #1211, Refine cmdlet scheduler

littlezhou · Nov 6, 2017 · 528cde5 · 528cde5
1 parent 59b7c6b
commit 528cde5
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 1 deletion.
diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/CmdletManager.java b/smart-engine/src/main/java/org/smartdata/server/engine/CmdletManager.java
@@ -397,6 +397,15 @@ private synchronized Set<String> lockMovefileActionFiles(List<ActionInfo> action
     return filesToLock.keySet();
   }
 
+  private boolean shouldStopSchedule() {
+    int left = dispatcher.getTotalSlotsLeft();
+    int total = dispatcher.getTotalSlots();
+    if (scheduledCmdlet.size() >= left + total * 0.2) {
+      return true;
+    }
+    return false;
+  }
+
   public int scheduleCmdlet() throws IOException {
     int nScheduled = 0;
 
@@ -408,7 +417,7 @@ public int scheduleCmdlet() throws IOException {
     }
 
     Iterator<Long> it = schedulingCmdlet.iterator();
-    while (it.hasNext()) {
+    while (it.hasNext() && !shouldStopSchedule()) {
       long id = it.next();
       CmdletInfo cmdlet = idToCmdlets.get(id);
       synchronized (cmdlet) {

diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/CmdletDispatcher.java b/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/CmdletDispatcher.java
@@ -307,6 +307,18 @@ private int updateSlotsLeft(int index, int delta) {
     }
   }
 
+  public int getTotalSlotsLeft() {
+    int total = 0;
+    for (int i : cmdExecSrvInstsSlotsLeft) {
+      total += i;
+    }
+    return total;
+  }
+
+  public int getTotalSlots() {
+    return cmdExecSrvTotalInsts * defaultSlots;
+  }
+
   public void start() {
     schExecService.scheduleAtFixedRate(
         new DispatchTask(this), 200, 100, TimeUnit.MILLISECONDS);