Improvements to macro_tool for benchmarking.

Split functionality into explicit "commands", add patching for analyzer for benchmarking as an alternative to patching for CFE for running, add benchmarking.

pkgs/_macro_tool/README.md

@@ -1,15 +1,74 @@
-A tool for running macro examples end to end.
+# macro_tool
 
-## Example usage
+A tool for running and benchmarking with macros.
 
-The following will run the json_codable example, assuming you start at the root
-of this repo:
+Specify a list of commands in addition to options:
+
+  apply: runs macros, outputs to disk
+  patch_for_analyzer: fixes macro output for the analyzer
+  patch_for_cfe: fixes macro output for the CFE
+  run: runs the specified script
+  revert: reverts patches to files
+  benchmark_apply: benchmarks applying macros
+  benchmark_analyze: benchmarks analyzing macros
+  bust_caches: modifies files like the benchmarks do
+  watch: loops watching for changes to the specified script and applying
+
+## Examples
+
+All examples are run from the root of this repo.
+
+Benchmarks required that you first create sources to benchmark with:
+
+```bash
+dart tool/benchmark_generator/bin/main.dart large macro 16
+```
+
+Benchmark running macros:
+
+```bash
+dart pkgs/_macro_tool/bin/main.dart \
+    --workspace=goldens/foo/lib/generated/large \
+    --packageConfig=.dart_tool/package_config.json \
+    benchmark_apply
+```
+
+Benchmark analysis on macro output without running macros:
+
+```bash
+dart pkgs/_macro_tool/bin/main.dart \
+    --workspace=goldens/foo/lib/generated/large \
+    --packageConfig=.dart_tool/package_config.json \
+    apply patch_for_analyzer benchmark_analyze revert
+```
+
+Run a script with macros:
 
 ```bash
 dart pkgs/_macro_tool/bin/main.dart \
   --workspace=goldens/foo \
   --packageConfig=.dart_tool/package_config.json \
-  --script=goldens/foo/lib/json_codable_test.dart
+  --script=goldens/foo/lib/json_codable_test.dart \
+  apply patch_for_cfe run revert
 ```
 
-You can also enable watch mode using `--watch`.
+Watch for changes to a script, applying macros when it changes, writing the
+output to disk and reporting how long it took:
+
+```bash
+dart pkgs/_macro_tool/bin/main.dart \
+  --workspace=goldens/foo \
+  --packageConfig=.dart_tool/package_config.json \
+  --script=goldens/foo/lib/json_codable_test.dart \
+  watch
+```
+
+Clean up output after using `watch`:
+
+```bash
+dart pkgs/_macro_tool/bin/main.dart \
+  --workspace=goldens/foo \
+  --packageConfig=.dart_tool/package_config.json \
+  --script=goldens/foo/lib/json_codable_test.dart \
+  revert
+```

pkgs/_macro_tool/bin/main.dart

@@ -4,45 +4,8 @@
 
 import 'dart:io';
 
-import 'package:_macro_tool/macro_tool.dart';
-import 'package:args/args.dart';
-import 'package:path/path.dart' as p;
-
-final argParser = ArgParser()
-  ..addOption('host',
-      defaultsTo: 'analyzer', help: 'The macro host: "analyzer" or "cfe".')
-  ..addOption('workspace', help: 'Path to workspace.')
-  ..addOption('packageConfig', help: 'Path to package config.')
-  ..addOption('script', help: 'Path to script.')
-  ..addFlag('skip-cleanup',
-      help: 'Whether to skip delete of augmentations and revert of script.')
-  ..addFlag('watch', help: 'Whether to watch for changes.');
+import 'package:_macro_tool/main.dart' as macro_tool;
 
 Future<void> main(List<String> arguments) async {
-  final args = argParser.parse(arguments);
-
-  final host = HostOption.forString(args['host'] as String?);
-  final workspace = args['workspace'] as String?;
-  final packageConfig = args['packageConfig'] as String?;
-  final script = args['script'] as String?;
-
-  if (host == null ||
-      workspace == null ||
-      packageConfig == null ||
-      script == null) {
-    print('''
-Runs a Dart script with `dart_model` macros.
-
-${argParser.usage}''');
-    exit(1);
-  }
-
-  final tool = MacroTool(
-      host: host,
-      workspacePath: p.canonicalize(workspace),
-      packageConfigPath: p.canonicalize(packageConfig),
-      scriptPath: p.canonicalize(script),
-      skipCleanup: args['skip-cleanup'] as bool,
-      watch: args['watch'] as bool);
-  await tool.run();
+  exit(await macro_tool.main(arguments));
 }

pkgs/_macro_tool/lib/analyzer_macro_runner.dart

@@ -0,0 +1,89 @@
+// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:_analyzer_macros/macro_implementation.dart';
+import 'package:analyzer/dart/analysis/analysis_context.dart';
+import 'package:analyzer/dart/analysis/analysis_context_collection.dart';
+import 'package:analyzer/dart/analysis/results.dart' hide FileResult;
+import 'package:analyzer/diagnostic/diagnostic.dart';
+import 'package:analyzer/src/summary2/macro_injected_impl.dart'
+    as injected_analyzer;
+import 'package:macro_service/macro_service.dart';
+
+import 'macro_runner.dart';
+import 'source_file.dart';
+
+class AnalyzerMacroRunner implements MacroRunner {
+  final String workspacePath;
+  final String packageConfigPath;
+
+  @override
+  final List<SourceFile> sourceFiles;
+
+  late final AnalysisContext analysisContext;
+  AnalyzerMacroImplementation? analyzerMacroImplementation;
+
+  AnalyzerMacroRunner(
+      {required this.workspacePath, required this.packageConfigPath})
+      : sourceFiles = Directory(workspacePath)
+            .listSync(recursive: true)
+            .whereType<File>()
+            .where((f) => f.path.endsWith('.dart'))
+            .map((f) => SourceFile(f.path))
+            .toList() {
+    final contextCollection =
+        AnalysisContextCollection(includedPaths: [workspacePath]);
+    analysisContext = contextCollection.contexts.first;
+  }
+
+  void notifyChange(SourceFile sourceFile) {
+    analysisContext.changeFile(sourceFile.path);
+  }
+
+  @override
+  Future<WorkspaceResult> run({bool injectImplementation = true}) async {
+    if (injectImplementation) {
+      analyzerMacroImplementation ??= await AnalyzerMacroImplementation.start(
+          protocol: Protocol(
+              encoding: ProtocolEncoding.binary,
+              version: ProtocolVersion.macros1),
+          packageConfig: Uri.file(packageConfigPath));
+      injected_analyzer.macroImplementation = analyzerMacroImplementation;
+    } else {
+      injected_analyzer.macroImplementation = null;
+    }
+
+    final fileResults = <FileResult>[];
+    final stopwatch = Stopwatch()..start();
+    Duration? firstDuration;
+    for (final sourceFile in sourceFiles) {
+      await analysisContext.applyPendingFileChanges();
+      ResolvedLibraryResult resolvedLibrary =
+          (await analysisContext.currentSession.getResolvedLibrary(sourceFile))
+              as ResolvedLibraryResult;
+
+      final errors = ((await analysisContext.currentSession
+              .getErrors(sourceFile)) as ErrorsResult)
+          .errors
+          .where((e) => e.severity == Severity.error)
+          .map((e) => e.toString())
+          .toList();
+
+      final augmentationUnits =
+          resolvedLibrary.units.where((u) => u.isMacroPart).toList();
+      final output = augmentationUnits.singleOrNull?.content;
+
+      fileResults.add(
+          FileResult(sourceFile: sourceFile, output: output, errors: errors));
+      if (firstDuration == null) firstDuration = stopwatch.elapsed;
+    }
+
+    return WorkspaceResult(
+        fileResults: fileResults,
+        firstResultAfter: firstDuration!,
+        lastResultAfter: stopwatch.elapsed);
+  }
+}