diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 00000000..475b2c9e --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,48 @@ +name: Unicode Conformance Testing +on: + pull_request: + branches: [ main ] + paths: + - '.github/workflows/conformance.yml' + - 'pkgs/intl4x/**' + push: + branches: [ main ] + paths: + - '.github/workflows/conformance.yml' + - 'pkgs/intl4x/**' + +jobs: + run_all: + runs-on: ubuntu-latest + steps: + - uses: dart-lang/setup-dart@d6a63dab3335f427404425de0fbfed4686d93c4f + with: + sdk: stable + + - uses: actions/checkout@7739b9ba2efcda9dde65ad1e3c2dbe65b41dfba7 + + - uses: actions/checkout@7739b9ba2efcda9dde65ad1e3c2dbe65b41dfba7 + with: + repository: unicode-org/conformance + path: 'conformance' + + - run: mv pkgs/intl4x/test/tools/conformance_config.json conformance/conformance_config.json + + - run: (cd conformance; bash generateDataAndRun.sh conformance_config.json) + + - name: Download Reference Exec Summary + continue-on-error: true + uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b + with: + name: referenceExecSummary + path: reference + + - run: (cd pkgs/intl4x; dart pub get) + - run: dart run pkgs/intl4x/test/tools/conformance_parser.dart --current-path conformance/TEMP_DATA/testReports/exec_summary.json --reference-path reference/TEMP_DATA/testReports/exec_summary.json >> $GITHUB_STEP_SUMMARY + + - name: Upload Reference Summary iff on main branch + if: github.ref == 'refs/heads/main' + uses: actions/upload-artifact@65d862660abb392b8c4a3d1195a2108db131dd05 + with: + name: referenceExecSummary + path: TEMP_DATA/testReports/ diff --git a/pkgs/intl4x/CHANGELOG.md b/pkgs/intl4x/CHANGELOG.md index 1d20cdb9..bd452f67 100644 --- a/pkgs/intl4x/CHANGELOG.md +++ b/pkgs/intl4x/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.6.1-wip + +- Add conformance testing workflow. + ## 0.6.0 - Add full ECMA locale. diff --git a/pkgs/intl4x/pubspec.yaml b/pkgs/intl4x/pubspec.yaml index 36fa270b..32105d18 100644 --- a/pkgs/intl4x/pubspec.yaml +++ b/pkgs/intl4x/pubspec.yaml @@ -1,7 +1,7 @@ name: intl4x description: >- A lightweight modular library for internationalization (i18n) functionality. -version: 0.6.0 +version: 0.6.1-wip repository: https://github.com/dart-lang/i18n/tree/main/pkgs/intl4x platforms: ## TODO: Add native platforms once ICU4X is integrated. web: @@ -14,6 +14,7 @@ dependencies: js: ^0.6.5 dev_dependencies: + args: ^2.4.2 build_runner: ^2.1.4 build_web_compilers: ^3.2.1 dart_flutter_team_lints: ^1.0.0 diff --git a/pkgs/intl4x/test/tools/conformance_config.json b/pkgs/intl4x/test/tools/conformance_config.json new file mode 100644 index 00000000..10e9841f --- /dev/null +++ b/pkgs/intl4x/test/tools/conformance_config.json @@ -0,0 +1,18 @@ +[ + { + "prereq": { + "name": "nvm", + "version": "20.1.0", + "command": "nvm install 20.1.0;nvm use 20.1.0" + }, + "run": { + "icu_version": "icu73", + "exec": "dart_web", + "test_type": [ + "coll_shift_short", + "number_fmt" + ], + "per_execution": 10000 + } + } +] \ No newline at end of file diff --git a/pkgs/intl4x/test/tools/conformance_parser.dart b/pkgs/intl4x/test/tools/conformance_parser.dart new file mode 100644 index 00000000..f9942996 --- /dev/null +++ b/pkgs/intl4x/test/tools/conformance_parser.dart @@ -0,0 +1,157 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; +import 'dart:math'; + +import 'package:args/args.dart'; + +void main(List args) { + final argParser = ArgParser(); + final referencePathOption = 'reference-path'; + final currentPathOption = 'current-path'; + argParser.addOption( + referencePathOption, + mandatory: true, + ); + argParser.addOption( + currentPathOption, + mandatory: true, + ); + final parse = argParser.parse(args); + final pathToReference = parse[referencePathOption] as String; + final pathToCurrent = parse[currentPathOption] as String; + + final infos = getInfos(getJson(pathToCurrent, 'dart_web')); + final referenceInfos = getInfos(getJson(pathToReference, 'dart_web')); + + final markdown = StringBuffer(''' +| Case | Total | Passing | Failing | Error | Unsupported | +| ---- | ----- | ------- | ------- | ----- | ----------- | +'''); + for (final entry in infos.entries) { + final referenceInfo = referenceInfos[entry.key]; + markdown.writeln( + '| ${entry.key} ${entry.value.getRow(referenceInfo ?? Info())}'); + } + print(markdown); + + final errorMessage = compareToReference(infos, referenceInfos); + if (errorMessage == null) { + exit(0); + } else { + exit(1); + } +} + +String? compareToReference( + Map infos, + Map referenceInfos, +) { + for (final entry in infos.entries) { + final info = entry.value; + final referenceInfo = referenceInfos[entry.key]; + if (referenceInfo != null) { + final failureMessage = shouldFail(info, referenceInfo); + if (failureMessage != null) { + return failureMessage; + } + } + } + return null; +} + +String? shouldFail(Info info, Info referenceInfo) { + final moreErrors = + info.error > referenceInfo.error ? 'Too many new errors' : null; + final moreFailing = + info.failing > referenceInfo.failing ? 'Too many new failing' : null; + final moreUnsupported = info.unsupported > referenceInfo.unsupported + ? 'Too many new unsupported' + : null; + return moreErrors ?? moreFailing ?? moreUnsupported; +} + +Map getInfos(Map current) { + final infos = {}; + for (final entry in current.entries) { + final caseName = entry.key; + final caseInfos = entry.value as List; + final caseInfo = caseInfos.firstOrNull as Map?; + if (caseInfo != null) { + infos[caseName] = Info( + total: caseInfo['test_count'] as int, + error: caseInfo['error_count'] as int, + failing: caseInfo['fail_count'] as int, + passing: caseInfo['pass_count'] as int, + unsupported: caseInfo['unsupported_count'] as int, + ); + } + } + return infos; +} + +Map getJson(String pathToCurrent, String exec) { + final file = File(pathToCurrent); + if (!file.existsSync()) return {}; + final currentStr = file.readAsStringSync(); + final decoded = jsonDecode(currentStr) as Map; + + return decoded.map((key, value) { + final list = (value as List) + .where((element) => (element as Map)['exec'] == exec) + .toList(); + return MapEntry(key, list); + }); +} + +class Info { + final int total; + final int passing; + final int failing; + final int error; + final int unsupported; + + Info({ + this.total = 0, + this.passing = 0, + this.failing = 0, + this.error = 0, + this.unsupported = 0, + }); + + String getRow(Info reference) { + final columnItems = [ + _getString(total, reference.total), + _getString(passing, reference.passing), + _getString(failing, reference.failing), + _getString(error, reference.error), + _getString(unsupported, reference.unsupported), + ]; + return '| ${columnItems.join(' | ')} |'; + } + + String _getString(int current, int reference) { + final change = (current - reference) / current; + String changeStr; + if (!change.isNaN) { + final changePercent = change * 100; + final changeClamped = max(min(changePercent, 100), -100); + String prefix; + if (changeClamped > 0) { + prefix = ':arrow_upper_right:'; + } else if (changeClamped < 0) { + prefix = ':arrow_lower_right:'; + } else { + prefix = ':arrow_right:'; + } + changeStr = '$prefix ${changeClamped.toStringAsFixed(2)} %'; + } else { + changeStr = ''; + } + final s = '$current $changeStr'; + return s; + } +}