diff --git a/src/diagnostics.rs b/src/diagnostics.rs
index 504863b134..cf27a9c376 100644
--- a/src/diagnostics.rs
+++ b/src/diagnostics.rs
@@ -1,7 +1,12 @@
 use rustc_mir::interpret::InterpErrorInfo;
+use std::cell::RefCell;
 
 use crate::*;
 
+pub enum NonHaltingDiagnostic {
+    PoppedTrackedPointerTag(Item),
+}
+
 pub fn report_err<'tcx, 'mir>(
     ecx: &InterpCx<'mir, 'tcx, Evaluator<'tcx>>,
     mut e: InterpErrorInfo<'tcx>,
@@ -12,8 +17,6 @@ pub fn report_err<'tcx, 'mir>(
             let info = info.downcast_ref::<TerminationInfo>().expect("invalid MachineStop payload");
             match info {
                 TerminationInfo::Exit(code) => return Some(*code),
-                TerminationInfo::PoppedTrackedPointerTag(item) =>
-                    format!("popped tracked tag for item {:?}", item),
                 TerminationInfo::Abort => format!("the evaluated program aborted execution"),
             }
         }
@@ -25,11 +28,23 @@ pub fn report_err<'tcx, 'mir>(
         _ => e.to_string(),
     };
     e.print_backtrace();
+    report_msg(ecx, msg, true)
+}
+
+pub fn report_msg<'tcx, 'mir>(
+    ecx: &InterpCx<'mir, 'tcx, Evaluator<'tcx>>,
+    msg: String,
+    error: bool,
+) -> Option<i64> {
     if let Some(frame) = ecx.stack().last() {
         let span = frame.current_source_info().unwrap().span;
 
-        let msg = format!("Miri evaluation error: {}", msg);
-        let mut err = ecx.tcx.sess.struct_span_err(span, msg.as_str());
+        let mut err = if error {
+            let msg = format!("Miri evaluation error: {}", msg);
+            ecx.tcx.sess.struct_span_err(span, msg.as_str())
+        } else {
+            ecx.tcx.sess.diagnostic().span_note_diag(span, msg.as_str())
+        };
         let frames = ecx.generate_stacktrace(None);
         err.span_label(span, msg);
         // We iterate with indices because we need to look at the next frame (the caller).
@@ -61,12 +76,11 @@ pub fn report_err<'tcx, 'mir>(
     return None;
 }
 
-use std::cell::RefCell;
 thread_local! {
-    static ECX: RefCell<Vec<InterpErrorInfo<'static>>> = RefCell::new(Vec::new());
+    static ECX: RefCell<Vec<NonHaltingDiagnostic>> = RefCell::new(Vec::new());
 }
 
-pub fn register_err(e: InterpErrorInfo<'static>) {
+pub fn register_err(e: NonHaltingDiagnostic) {
     ECX.with(|ecx| ecx.borrow_mut().push(e));
 }
 
@@ -76,7 +90,11 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
         let this = self.eval_context_ref();
         ECX.with(|ecx| {
             for e in ecx.borrow_mut().drain(..) {
-                report_err(this, e);
+                let msg = match e {
+                    NonHaltingDiagnostic::PoppedTrackedPointerTag(item) =>
+                        format!("popped tracked tag for item {:?}", item),
+                };
+                report_msg(this, msg, false);
             }
         });
     }
diff --git a/src/eval.rs b/src/eval.rs
index 1e9332c9a3..171f2627ad 100644
--- a/src/eval.rs
+++ b/src/eval.rs
@@ -33,7 +33,6 @@ pub struct MiriConfig {
 /// Details of premature program termination.
 pub enum TerminationInfo {
     Exit(i64),
-    PoppedTrackedPointerTag(Item),
     Abort,
 }
 
diff --git a/src/lib.rs b/src/lib.rs
index ae92a777d8..60e5217721 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -43,7 +43,7 @@ pub use crate::shims::tls::{EvalContextExt as TlsEvalContextExt, TlsData};
 pub use crate::shims::EvalContextExt as ShimsEvalContextExt;
 
 pub use crate::diagnostics::{
-    register_err, report_err, EvalContextExt as DiagnosticsEvalContextExt,
+    register_err, report_err, EvalContextExt as DiagnosticsEvalContextExt, NonHaltingDiagnostic,
 };
 pub use crate::eval::{create_ecx, eval_main, MiriConfig, TerminationInfo};
 pub use crate::helpers::EvalContextExt as HelpersEvalContextExt;
diff --git a/src/stacked_borrows.rs b/src/stacked_borrows.rs
index 7bb7dd6cec..a98c8e26c6 100644
--- a/src/stacked_borrows.rs
+++ b/src/stacked_borrows.rs
@@ -10,7 +10,6 @@ use std::rc::Rc;
 use rustc_hir::Mutability;
 use rustc::mir::RetagKind;
 use rustc::ty::{self, layout::Size};
-use rustc_mir::interpret::InterpError;
 
 use crate::*;
 
@@ -267,12 +266,7 @@ impl<'tcx> Stack {
     fn check_protector(item: &Item, tag: Option<Tag>, global: &GlobalState) -> InterpResult<'tcx> {
         if let Tag::Tagged(id) = item.tag {
             if Some(id) == global.tracked_pointer_tag {
-                register_err(
-                    InterpError::MachineStop(Box::new(TerminationInfo::PoppedTrackedPointerTag(
-                        item.clone(),
-                    )))
-                    .into(),
-                );
+                register_err(NonHaltingDiagnostic::PoppedTrackedPointerTag(item.clone()));
             }
         }
         if let Some(call) = item.protector {