From 2dab9442943979c94fa7814a923075766f06cbe2 Mon Sep 17 00:00:00 2001
From: Dipin Hora <dipinhora+github@gmail.com>
Date: Wed, 28 Mar 2018 15:35:27 -0400
Subject: [PATCH] Ensure ASIO thread cannot be stopped prematurely (#2612)

Prior to this commit, it was possible for the ASIO thread to
be stopped in anticipation of runtime termination when runtime
termination didn't actually occur. This would leave the runtime
in an invalid state where a new ASIO thread would need to be
restarted.

This commit changes the behavior to ensure that the ASIO thread
is only stopped when the runtime is about to be terminated to
ensure the runtime cannot be in an invalid state where the ASIO
thread isn't running.
---
 src/libponyrt/asio/asio.c       |  8 +++++++-
 src/libponyrt/asio/asio.h       |  7 +++++++
 src/libponyrt/sched/scheduler.c | 23 ++++++++---------------
 src/libponyrt/sched/scheduler.h |  2 +-
 4 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/src/libponyrt/asio/asio.c b/src/libponyrt/asio/asio.c
index 04581cbb54..48f0097ee5 100644
--- a/src/libponyrt/asio/asio.c
+++ b/src/libponyrt/asio/asio.c
@@ -59,7 +59,7 @@ bool ponyint_asio_start()
 
 bool ponyint_asio_stop()
 {
-  if(atomic_load_explicit(&running_base.noisy_count, memory_order_relaxed) > 0)
+  if(!ponyint_asio_stoppable())
     return false;
 
   if(running_base.backend != NULL)
@@ -74,6 +74,12 @@ bool ponyint_asio_stop()
   return true;
 }
 
+bool ponyint_asio_stoppable()
+{
+  // can only stop if we have no noisy actors
+  return atomic_load_explicit(&running_base.noisy_count, memory_order_relaxed) == 0;
+}
+
 uint64_t ponyint_asio_noisy_add()
 {
   return atomic_fetch_add_explicit(&running_base.noisy_count, 1, memory_order_relaxed);
diff --git a/src/libponyrt/asio/asio.h b/src/libponyrt/asio/asio.h
index 80ed6c7c9a..0c9f59d90b 100644
--- a/src/libponyrt/asio/asio.h
+++ b/src/libponyrt/asio/asio.h
@@ -83,6 +83,13 @@ uint32_t ponyint_asio_get_cpu();
  */
 bool ponyint_asio_stop();
 
+/** Checks if it is safe to stop the asynchronous event mechanism.
+ *
+ * Stopping an event mechanism is only possible if there are no pending "noisy"
+ * subscriptions.
+ */
+bool ponyint_asio_stoppable();
+
 /** Add a noisy event subscription.
  */
 uint64_t ponyint_asio_noisy_add();
diff --git a/src/libponyrt/sched/scheduler.c b/src/libponyrt/sched/scheduler.c
index 9ed9acf086..0fb3d3902a 100644
--- a/src/libponyrt/sched/scheduler.c
+++ b/src/libponyrt/sched/scheduler.c
@@ -212,18 +212,6 @@ static void handle_sched_block(scheduler_t* sched)
 // handle SCHED_UNBLOCK message
 static void handle_sched_unblock(scheduler_t* sched)
 {
-  // if the ASIO thread has already been stopped
-  if (sched->asio_stopped)
-  {
-    // restart the ASIO thread
-    ponyint_asio_init(asio_cpu);
-    sched->asio_stopped = !ponyint_asio_start();
-  }
-
-  // make sure asio hasn't already been stopped or else runtime is in
-  // an invalid state without the ASIO thread running
-  pony_assert(!sched->asio_stopped);
-
   // Cancel all acks and increment the ack token, so that any pending
   // acks in the queue will be dropped when they are received.
   sched->block_count--;
@@ -328,21 +316,26 @@ static bool quiescent(scheduler_t* sched, uint64_t tsc, uint64_t tsc2)
 
   if(sched->ack_count >= current_active_scheduler_count)
   {
-    if(sched->asio_stopped)
+    if(sched->asio_stoppable && ponyint_asio_stop())
     {
+      // successfully stopped ASIO thread
+      // tell all scheduler threads to terminate
       send_msg_all(sched->index, SCHED_TERMINATE, 0);
 
       wake_suspended_threads(sched->index);
 
       sched->ack_token++;
       sched->ack_count = 0;
-    } else if(ponyint_asio_stop()) {
-      sched->asio_stopped = true;
+    } else if(ponyint_asio_stoppable()) {
+      sched->asio_stoppable = true;
       sched->ack_token++;
       sched->ack_count = 0;
 
       // Run another CNF/ACK cycle.
       send_msg_all_active(sched->index, SCHED_CNF, sched->ack_token);
+    } else {
+      // ASIO is not stoppable
+      sched->asio_stoppable = false;
     }
   }
 
diff --git a/src/libponyrt/sched/scheduler.h b/src/libponyrt/sched/scheduler.h
index 789a7397ad..6c401d4052 100644
--- a/src/libponyrt/sched/scheduler.h
+++ b/src/libponyrt/sched/scheduler.h
@@ -56,7 +56,7 @@ struct scheduler_t
   uint32_t cpu;
   uint32_t node;
   bool terminate;
-  bool asio_stopped;
+  bool asio_stoppable;
   bool asio_noisy;
   pony_signal_event_t sleep_object;