From b81f43999b9d9b5f75117776ebbb1b9442e66edc Mon Sep 17 00:00:00 2001 From: Nate Bosch Date: Mon, 3 Aug 2020 15:59:13 -0700 Subject: [PATCH] Try out using msgpack for asset graph Towards #41 See if this makes a meaningful difference for performance. --- build_runner/bin/graph_inspector.dart | 2 +- build_runner/test/generate/watch_test.dart | 2 +- build_runner_core/lib/src/asset_graph/graph.dart | 5 ++++- .../lib/src/asset_graph/serialization.dart | 10 +++++----- build_runner_core/lib/src/util/constants.dart | 2 +- build_runner_core/pubspec.yaml | 1 + build_runner_core/test/generate/build_test.dart | 6 +++--- 7 files changed, 16 insertions(+), 12 deletions(-) diff --git a/build_runner/bin/graph_inspector.dart b/build_runner/bin/graph_inspector.dart index db03cb17f..8b90b697b 100644 --- a/build_runner/bin/graph_inspector.dart +++ b/build_runner/bin/graph_inspector.dart @@ -30,7 +30,7 @@ Future main(List args) async { final argParser = ArgParser() ..addOption('graph-file', - abbr: 'g', help: 'Specify the asset_graph.json file to inspect.') + abbr: 'g', help: 'Specify the asset_graph.msgpack file to inspect.') ..addOption('build-script', abbr: 'b', help: 'Specify the build script to find the asset graph for.', diff --git a/build_runner/test/generate/watch_test.dart b/build_runner/test/generate/watch_test.dart index 38001fb2d..4b5f879a7 100644 --- a/build_runner/test/generate/watch_test.dart +++ b/build_runner/test/generate/watch_test.dart @@ -231,7 +231,7 @@ a:file://fake/pkg/path decodedMatches('b')); }); - test('rebuilds properly update asset_graph.json', () async { + test('rebuilds properly update asset_graph.msgpack', () async { var buildState = await startWatch([copyABuildApplication], {'a|web/a.txt': 'a', 'a|web/b.txt': 'b'}, writer); var results = StreamQueue(buildState.buildResults); diff --git a/build_runner_core/lib/src/asset_graph/graph.dart b/build_runner_core/lib/src/asset_graph/graph.dart index 62d6093a1..135f50b98 100644 --- a/build_runner_core/lib/src/asset_graph/graph.dart +++ b/build_runner_core/lib/src/asset_graph/graph.dart @@ -6,6 +6,7 @@ import 'dart:async'; import 'dart:collection'; import 'dart:convert'; import 'dart:io'; +import 'dart:typed_data'; import 'package:build/build.dart'; import 'package:build/experiments.dart' as experiments_zone; @@ -13,6 +14,7 @@ import 'package:convert/convert.dart'; import 'package:crypto/crypto.dart'; import 'package:glob/glob.dart'; import 'package:meta/meta.dart'; +import 'package:msgpack_dart/msgpack_dart.dart' as msgpack; import 'package:package_config/package_config.dart'; import 'package:watcher/watcher.dart'; @@ -48,7 +50,8 @@ class AssetGraph { /// Deserializes this graph. factory AssetGraph.deserialize(List serializedGraph) => - _AssetGraphDeserializer(serializedGraph).deserialize(); + _AssetGraphDeserializer(Uint8List.fromList(serializedGraph)) + .deserialize(); static Future build( List buildPhases, diff --git a/build_runner_core/lib/src/asset_graph/serialization.dart b/build_runner_core/lib/src/asset_graph/serialization.dart index e6754a692..7e80d275d 100644 --- a/build_runner_core/lib/src/asset_graph/serialization.dart +++ b/build_runner_core/lib/src/asset_graph/serialization.dart @@ -8,7 +8,7 @@ part of 'graph.dart'; /// /// This should be incremented any time the serialize/deserialize formats /// change. -const _version = 24; +const _version = 1; /// Deserializes an [AssetGraph] from a [Map]. class _AssetGraphDeserializer { @@ -18,10 +18,10 @@ class _AssetGraphDeserializer { _AssetGraphDeserializer._(this._serializedGraph); - factory _AssetGraphDeserializer(List bytes) { + factory _AssetGraphDeserializer(Uint8List bytes) { dynamic decoded; try { - decoded = jsonDecode(utf8.decode(bytes)); + decoded = msgpack.deserialize(bytes); } on FormatException { throw AssetGraphCorruptedException(); } @@ -211,7 +211,7 @@ class _AssetGraphSerializer { _AssetGraphSerializer(this._graph); /// Perform the serialization, should only be called once. - List serialize() { + Uint8List serialize() { var pathId = 0; // [path0, packageId0, path1, packageId1, ...] var assetPaths = []; @@ -233,7 +233,7 @@ class _AssetGraphSerializer { .map((pkg, version) => MapEntry(pkg, version?.toString())), 'enabledExperiments': _graph.enabledExperiments, }; - return utf8.encode(json.encode(result)); + return msgpack.serialize(result); } List _serializeNode(AssetNode node) { diff --git a/build_runner_core/lib/src/util/constants.dart b/build_runner_core/lib/src/util/constants.dart index 8b55f55f3..e421355ff 100644 --- a/build_runner_core/lib/src/util/constants.dart +++ b/build_runner_core/lib/src/util/constants.dart @@ -13,7 +13,7 @@ final String assetGraphPath = assetGraphPathFor(_scriptPath); /// Relative path to the asset graph for a build script at [path] String assetGraphPathFor(String path) => - '$cacheDir/${_scriptHashFor(path)}/asset_graph.json'; + '$cacheDir/${_scriptHashFor(path)}/asset_graph.msgpack'; /// Relative path to the directory which holds serialized versions of errors /// reported during previous builds. diff --git a/build_runner_core/pubspec.yaml b/build_runner_core/pubspec.yaml index 8c4fc3315..e74643aa2 100644 --- a/build_runner_core/pubspec.yaml +++ b/build_runner_core/pubspec.yaml @@ -26,6 +26,7 @@ dependencies: timing: ^0.1.1 watcher: ^0.9.7 yaml: ^2.1.0 + msgpack_dart: ^0.0.7 dev_dependencies: build_test: ^1.0.0 diff --git a/build_runner_core/test/generate/build_test.dart b/build_runner_core/test/generate/build_test.dart index 98d519bfe..8119a5446 100644 --- a/build_runner_core/test/generate/build_test.dart +++ b/build_runner_core/test/generate/build_test.dart @@ -842,7 +842,7 @@ void main() { }); }); - test('tracks dependency graph in a asset_graph.json file', () async { + test('tracks dependency graph in a asset_graph.msgpack file', () async { final writer = InMemoryRunnerAssetWriter(); await testBuilders([ requiresPostProcessBuilderApplication, @@ -1019,7 +1019,7 @@ void main() { outputs: {}, writer: writer); }); - test('can recover from a deleted asset_graph.json cache', () async { + test('can recover from a deleted asset_graph.msgpack cache', () async { final writer = InMemoryRunnerAssetWriter(); var inputs = {'a|web/a.txt': 'a', 'a|lib/b.txt': 'b'}; var outputs = { @@ -1030,7 +1030,7 @@ void main() { await testBuilders([copyABuilderApplication], inputs, outputs: outputs, writer: writer); - // Delete the `asset_graph.json` file! + // Delete the `asset_graph.msgpack` file! var outputId = makeAssetId('a|$assetGraphPath'); await writer.delete(outputId);