Skip to content

Commit

Permalink
Try out using msgpack for asset graph
Browse files Browse the repository at this point in the history
Towards #41

See if this makes a meaningful difference for performance.
  • Loading branch information
natebosch committed Aug 3, 2020
1 parent 4c76bbf commit b81f439
Show file tree
Hide file tree
Showing 7 changed files with 16 additions and 12 deletions.
2 changes: 1 addition & 1 deletion build_runner/bin/graph_inspector.dart
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ Future<void> main(List<String> args) async {

final argParser = ArgParser()
..addOption('graph-file',
abbr: 'g', help: 'Specify the asset_graph.json file to inspect.')
abbr: 'g', help: 'Specify the asset_graph.msgpack file to inspect.')
..addOption('build-script',
abbr: 'b',
help: 'Specify the build script to find the asset graph for.',
Expand Down
2 changes: 1 addition & 1 deletion build_runner/test/generate/watch_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ a:file://fake/pkg/path
decodedMatches('b'));
});

test('rebuilds properly update asset_graph.json', () async {
test('rebuilds properly update asset_graph.msgpack', () async {
var buildState = await startWatch([copyABuildApplication],
{'a|web/a.txt': 'a', 'a|web/b.txt': 'b'}, writer);
var results = StreamQueue(buildState.buildResults);
Expand Down
5 changes: 4 additions & 1 deletion build_runner_core/lib/src/asset_graph/graph.dart
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@ import 'dart:async';
import 'dart:collection';
import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';

import 'package:build/build.dart';
import 'package:build/experiments.dart' as experiments_zone;
import 'package:convert/convert.dart';
import 'package:crypto/crypto.dart';
import 'package:glob/glob.dart';
import 'package:meta/meta.dart';
import 'package:msgpack_dart/msgpack_dart.dart' as msgpack;
import 'package:package_config/package_config.dart';
import 'package:watcher/watcher.dart';

Expand Down Expand Up @@ -48,7 +50,8 @@ class AssetGraph {

/// Deserializes this graph.
factory AssetGraph.deserialize(List<int> serializedGraph) =>
_AssetGraphDeserializer(serializedGraph).deserialize();
_AssetGraphDeserializer(Uint8List.fromList(serializedGraph))
.deserialize();

static Future<AssetGraph> build(
List<BuildPhase> buildPhases,
Expand Down
10 changes: 5 additions & 5 deletions build_runner_core/lib/src/asset_graph/serialization.dart
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ part of 'graph.dart';
///
/// This should be incremented any time the serialize/deserialize formats
/// change.
const _version = 24;
const _version = 1;

/// Deserializes an [AssetGraph] from a [Map].
class _AssetGraphDeserializer {
Expand All @@ -18,10 +18,10 @@ class _AssetGraphDeserializer {

_AssetGraphDeserializer._(this._serializedGraph);

factory _AssetGraphDeserializer(List<int> bytes) {
factory _AssetGraphDeserializer(Uint8List bytes) {
dynamic decoded;
try {
decoded = jsonDecode(utf8.decode(bytes));
decoded = msgpack.deserialize(bytes);
} on FormatException {
throw AssetGraphCorruptedException();
}
Expand Down Expand Up @@ -211,7 +211,7 @@ class _AssetGraphSerializer {
_AssetGraphSerializer(this._graph);

/// Perform the serialization, should only be called once.
List<int> serialize() {
Uint8List serialize() {
var pathId = 0;
// [path0, packageId0, path1, packageId1, ...]
var assetPaths = <dynamic>[];
Expand All @@ -233,7 +233,7 @@ class _AssetGraphSerializer {
.map((pkg, version) => MapEntry(pkg, version?.toString())),
'enabledExperiments': _graph.enabledExperiments,
};
return utf8.encode(json.encode(result));
return msgpack.serialize(result);
}

List _serializeNode(AssetNode node) {
Expand Down
2 changes: 1 addition & 1 deletion build_runner_core/lib/src/util/constants.dart
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ final String assetGraphPath = assetGraphPathFor(_scriptPath);

/// Relative path to the asset graph for a build script at [path]
String assetGraphPathFor(String path) =>
'$cacheDir/${_scriptHashFor(path)}/asset_graph.json';
'$cacheDir/${_scriptHashFor(path)}/asset_graph.msgpack';

/// Relative path to the directory which holds serialized versions of errors
/// reported during previous builds.
Expand Down
1 change: 1 addition & 0 deletions build_runner_core/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ dependencies:
timing: ^0.1.1
watcher: ^0.9.7
yaml: ^2.1.0
msgpack_dart: ^0.0.7

dev_dependencies:
build_test: ^1.0.0
Expand Down
6 changes: 3 additions & 3 deletions build_runner_core/test/generate/build_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -842,7 +842,7 @@ void main() {
});
});

test('tracks dependency graph in a asset_graph.json file', () async {
test('tracks dependency graph in a asset_graph.msgpack file', () async {
final writer = InMemoryRunnerAssetWriter();
await testBuilders([
requiresPostProcessBuilderApplication,
Expand Down Expand Up @@ -1019,7 +1019,7 @@ void main() {
outputs: {}, writer: writer);
});

test('can recover from a deleted asset_graph.json cache', () async {
test('can recover from a deleted asset_graph.msgpack cache', () async {
final writer = InMemoryRunnerAssetWriter();
var inputs = <String, String>{'a|web/a.txt': 'a', 'a|lib/b.txt': 'b'};
var outputs = <String, String>{
Expand All @@ -1030,7 +1030,7 @@ void main() {
await testBuilders([copyABuilderApplication], inputs,
outputs: outputs, writer: writer);

// Delete the `asset_graph.json` file!
// Delete the `asset_graph.msgpack` file!
var outputId = makeAssetId('a|$assetGraphPath');
await writer.delete(outputId);

Expand Down

0 comments on commit b81f439

Please sign in to comment.