From fe0641dcca1ddcd3dc47760aeaf3db1411c77d8c Mon Sep 17 00:00:00 2001
From: Nate Bosch <nbosch@google.com>
Date: Tue, 9 Jul 2024 01:15:34 +0000
Subject: [PATCH] Expand error message for async test declaration

Add suggestions for fixing the declared tests. Differentiate the message
based on whether the test `main` is running within the test runner, or
as a standalone executable.
---
 pkgs/test_api/lib/src/backend/declarer.dart | 67 +++++++++++++++------
 pkgs/test_core/lib/src/scaffolding.dart     |  2 +-
 2 files changed, 49 insertions(+), 20 deletions(-)

diff --git a/pkgs/test_api/lib/src/backend/declarer.dart b/pkgs/test_api/lib/src/backend/declarer.dart
index 648e3232d..76f7563a7 100644
--- a/pkgs/test_api/lib/src/backend/declarer.dart
+++ b/pkgs/test_api/lib/src/backend/declarer.dart
@@ -112,6 +112,16 @@ class Declarer {
   /// `null`.
   final Set<String>? _seenNames;
 
+  /// Whether this declarer is running in a standalone test executation.
+  ///
+  /// The full test runner awaits asynchronous `main` declarations, and so
+  /// asynchronous work can be performed in between calls to `group`, and `test`
+  /// etc. When running as a standalone file tests are run synchronously
+  /// following the first call to declare a test, so all tests must be declared
+  /// synchronously starting at that point. Track whether we are running in this
+  /// more limited mode to customize the error message for tests declared late.
+  final bool _isStandalone;
+
   /// Creates a new declarer for the root group.
   ///
   /// This is the implicit group that exists outside of any calls to `group()`.
@@ -139,16 +149,19 @@ class Declarer {
     String? fullTestName,
     // TODO: Change the default https://github.com/dart-lang/test/issues/1571
     bool allowDuplicateTestNames = true,
+    bool isStandalone = false,
   }) : this._(
-            null,
-            null,
-            metadata ?? Metadata(),
-            platformVariables ?? const UnmodifiableSetView.empty(),
-            collectTraces,
-            null,
-            noRetry,
-            fullTestName,
-            allowDuplicateTestNames ? null : <String>{});
+          null,
+          null,
+          metadata ?? Metadata(),
+          platformVariables ?? const UnmodifiableSetView.empty(),
+          collectTraces,
+          null,
+          noRetry,
+          fullTestName,
+          allowDuplicateTestNames ? null : <String>{},
+          isStandalone,
+        );
 
   Declarer._(
     this._parent,
@@ -160,6 +173,7 @@ class Declarer {
     this._noRetry,
     this._fullTestName,
     this._seenNames,
+    this._isStandalone,
   );
 
   /// Runs [body] with this declarer as [Declarer.current].
@@ -252,15 +266,17 @@ class Declarer {
     var trace = _collectTraces ? Trace.current(2) : null;
 
     var declarer = Declarer._(
-        this,
-        fullTestPrefix,
-        metadata,
-        _platformVariables,
-        _collectTraces,
-        trace,
-        _noRetry,
-        _fullTestName,
-        _seenNames);
+      this,
+      fullTestPrefix,
+      metadata,
+      _platformVariables,
+      _collectTraces,
+      trace,
+      _noRetry,
+      _fullTestName,
+      _seenNames,
+      _isStandalone,
+    );
     declarer.declare(() {
       // Cast to dynamic to avoid the analyzer complaining about us using the
       // result of a void method.
@@ -340,7 +356,20 @@ class Declarer {
   /// [name] should be the name of the method being called.
   void _checkNotBuilt(String name) {
     if (!_built) return;
-    throw StateError("Can't call $name() once tests have begun running.");
+    final restrictionMessage = _isStandalone
+        ? 'When running a test as an executable directly '
+            '(not as a suite by the test runner), '
+            'tests must be declared in a synchronous block.\n'
+            'If async work is required before any tests are run '
+            'use a `setUpAll` callback.\n'
+            'If async work cannot be avoided before declaring tests, '
+            'all async events must be complete before declaring the first test.'
+        : 'If async work is required before any tests are run '
+            'use a `setUpAll` callback.\n'
+            'If async work cannot be avoided before declaring tests it must '
+            'all be awaited within the Future returned from `main`.';
+    throw StateError("Can't call $name() once tests have begun running.\n"
+        '$restrictionMessage');
   }
 
   /// Run the set-up functions for this and any parent groups.
diff --git a/pkgs/test_core/lib/src/scaffolding.dart b/pkgs/test_core/lib/src/scaffolding.dart
index 93343fef1..31b3d899a 100644
--- a/pkgs/test_core/lib/src/scaffolding.dart
+++ b/pkgs/test_core/lib/src/scaffolding.dart
@@ -44,7 +44,7 @@ Declarer get _declarer {
   // In order to run the tests, we set up our own Declarer via
   // [_globalDeclarer], and pump the event queue as a best effort to wait for
   // all tests to be defined before starting them.
-  _globalDeclarer = Declarer();
+  _globalDeclarer = Declarer(isStandalone: true);
 
   () async {
     await pumpEventQueue();