Add conformance testing (#706)

* Add `copyWith` to all classes

* Add changelog

* Add mixin

* Add mixin

* Revert "Add mixin"

This reverts commit 1e9a32e.

* Revert "Add mixin"

This reverts commit e2c1326.

* Return default ECMA

* Fix bug

* Clean up digits

* Fix options

* Remove unused parameters

* Switch useGrouping to bool

* Add conformance testing workflow for intl4x

* Fix

* Run in subdir

* Handle missing reference file

* Continue on error

* Add debug msg

* Do not fail on non existing reference

* Run dart only

* Add config file

* Make it look nicer

* Correct order

* Changes as per review

* Fixes

* Add pub get call

* cd to directory

* Fix typo

* Add number_fmt to tests

* Switch to unicode/conformance repo

* Add ref to main

* Fix typo

.github/workflows/conformance.yml

@@ -0,0 +1,48 @@
+name: Unicode Conformance Testing
+on:
+  pull_request:
+    branches: [ main ]
+    paths:
+      - '.github/workflows/conformance.yml'
+      - 'pkgs/intl4x/**'
+  push:
+    branches: [ main ]
+    paths:
+      - '.github/workflows/conformance.yml'
+      - 'pkgs/intl4x/**'
+
+jobs:
+    run_all:
+      runs-on: ubuntu-latest
+      steps:
+        - uses: dart-lang/setup-dart@d6a63dab3335f427404425de0fbfed4686d93c4f
+          with:
+            sdk: stable
+
+        - uses: actions/checkout@7739b9ba2efcda9dde65ad1e3c2dbe65b41dfba7
+
+        - uses: actions/checkout@7739b9ba2efcda9dde65ad1e3c2dbe65b41dfba7
+          with:
+            repository: unicode-org/conformance
+            path: 'conformance'
+
+        - run: mv pkgs/intl4x/test/tools/conformance_config.json conformance/conformance_config.json
+
+        - run: (cd conformance; bash generateDataAndRun.sh conformance_config.json)
+
+        - name: Download Reference Exec Summary
+          continue-on-error: true
+          uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b
+          with:
+            name: referenceExecSummary
+            path: reference
+
+        - run: (cd pkgs/intl4x; dart pub get)
+        - run: dart run pkgs/intl4x/test/tools/conformance_parser.dart --current-path conformance/TEMP_DATA/testReports/exec_summary.json --reference-path reference/TEMP_DATA/testReports/exec_summary.json >> $GITHUB_STEP_SUMMARY
+
+        - name: Upload Reference Summary iff on main branch
+          if: github.ref == 'refs/heads/main'
+          uses: actions/upload-artifact@65d862660abb392b8c4a3d1195a2108db131dd05
+          with:
+            name: referenceExecSummary
+            path: TEMP_DATA/testReports/

pkgs/intl4x/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.6.1-wip
+
+- Add conformance testing workflow.
+
 ## 0.6.0
 
 - Add full ECMA locale.

pkgs/intl4x/pubspec.yaml

@@ -1,7 +1,7 @@
 name: intl4x
 description: >-
   A lightweight modular library for internationalization (i18n) functionality.
-version: 0.6.0
+version: 0.6.1-wip
 repository: https://github.com/dart-lang/i18n/tree/main/pkgs/intl4x
 platforms: ## TODO: Add native platforms once ICU4X is integrated.
   web:
@@ -14,6 +14,7 @@ dependencies:
   js: ^0.6.5
 
 dev_dependencies:
+  args: ^2.4.2
   build_runner: ^2.1.4
   build_web_compilers: ^3.2.1
   dart_flutter_team_lints: ^1.0.0

pkgs/intl4x/test/tools/conformance_config.json

@@ -0,0 +1,18 @@
+[
+    {
+        "prereq": {
+            "name": "nvm",
+            "version": "20.1.0",
+            "command": "nvm install 20.1.0;nvm use 20.1.0"
+        },
+        "run": {
+            "icu_version": "icu73",
+            "exec": "dart_web",
+            "test_type": [
+                "coll_shift_short",
+                "number_fmt"
+            ],
+            "per_execution": 10000
+        }
+    }
+]

pkgs/intl4x/test/tools/conformance_parser.dart

@@ -0,0 +1,157 @@
+// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+import 'dart:math';
+
+import 'package:args/args.dart';
+
+void main(List<String> args) {
+  final argParser = ArgParser();
+  final referencePathOption = 'reference-path';
+  final currentPathOption = 'current-path';
+  argParser.addOption(
+    referencePathOption,
+    mandatory: true,
+  );
+  argParser.addOption(
+    currentPathOption,
+    mandatory: true,
+  );
+  final parse = argParser.parse(args);
+  final pathToReference = parse[referencePathOption] as String;
+  final pathToCurrent = parse[currentPathOption] as String;
+
+  final infos = getInfos(getJson(pathToCurrent, 'dart_web'));
+  final referenceInfos = getInfos(getJson(pathToReference, 'dart_web'));
+
+  final markdown = StringBuffer('''
+| Case | Total | Passing | Failing | Error | Unsupported |
+| ---- | ----- | ------- | ------- | ----- | ----------- |
+''');
+  for (final entry in infos.entries) {
+    final referenceInfo = referenceInfos[entry.key];
+    markdown.writeln(
+        '| ${entry.key} ${entry.value.getRow(referenceInfo ?? Info())}');
+  }
+  print(markdown);
+
+  final errorMessage = compareToReference(infos, referenceInfos);
+  if (errorMessage == null) {
+    exit(0);
+  } else {
+    exit(1);
+  }
+}
+
+String? compareToReference(
+  Map<String, Info> infos,
+  Map<String, Info> referenceInfos,
+) {
+  for (final entry in infos.entries) {
+    final info = entry.value;
+    final referenceInfo = referenceInfos[entry.key];
+    if (referenceInfo != null) {
+      final failureMessage = shouldFail(info, referenceInfo);
+      if (failureMessage != null) {
+        return failureMessage;
+      }
+    }
+  }
+  return null;
+}
+
+String? shouldFail(Info info, Info referenceInfo) {
+  final moreErrors =
+      info.error > referenceInfo.error ? 'Too many new errors' : null;
+  final moreFailing =
+      info.failing > referenceInfo.failing ? 'Too many new failing' : null;
+  final moreUnsupported = info.unsupported > referenceInfo.unsupported
+      ? 'Too many new unsupported'
+      : null;
+  return moreErrors ?? moreFailing ?? moreUnsupported;
+}
+
+Map<String, Info> getInfos(Map<String, dynamic> current) {
+  final infos = <String, Info>{};
+  for (final entry in current.entries) {
+    final caseName = entry.key;
+    final caseInfos = entry.value as List;
+    final caseInfo = caseInfos.firstOrNull as Map<String, dynamic>?;
+    if (caseInfo != null) {
+      infos[caseName] = Info(
+        total: caseInfo['test_count'] as int,
+        error: caseInfo['error_count'] as int,
+        failing: caseInfo['fail_count'] as int,
+        passing: caseInfo['pass_count'] as int,
+        unsupported: caseInfo['unsupported_count'] as int,
+      );
+    }
+  }
+  return infos;
+}
+
+Map<String, dynamic> getJson(String pathToCurrent, String exec) {
+  final file = File(pathToCurrent);
+  if (!file.existsSync()) return <String, dynamic>{};
+  final currentStr = file.readAsStringSync();
+  final decoded = jsonDecode(currentStr) as Map<String, dynamic>;
+
+  return decoded.map((key, value) {
+    final list = (value as List)
+        .where((element) => (element as Map)['exec'] == exec)
+        .toList();
+    return MapEntry(key, list);
+  });
+}
+
+class Info {
+  final int total;
+  final int passing;
+  final int failing;
+  final int error;
+  final int unsupported;
+
+  Info({
+    this.total = 0,
+    this.passing = 0,
+    this.failing = 0,
+    this.error = 0,
+    this.unsupported = 0,
+  });
+
+  String getRow(Info reference) {
+    final columnItems = [
+      _getString(total, reference.total),
+      _getString(passing, reference.passing),
+      _getString(failing, reference.failing),
+      _getString(error, reference.error),
+      _getString(unsupported, reference.unsupported),
+    ];
+    return '| ${columnItems.join(' | ')} |';
+  }
+
+  String _getString(int current, int reference) {
+    final change = (current - reference) / current;
+    String changeStr;
+    if (!change.isNaN) {
+      final changePercent = change * 100;
+      final changeClamped = max(min(changePercent, 100), -100);
+      String prefix;
+      if (changeClamped > 0) {
+        prefix = ':arrow_upper_right:';
+      } else if (changeClamped < 0) {
+        prefix = ':arrow_lower_right:';
+      } else {
+        prefix = ':arrow_right:';
+      }
+      changeStr = '$prefix ${changeClamped.toStringAsFixed(2)} %';
+    } else {
+      changeStr = '';
+    }
+    final s = '$current $changeStr';
+    return s;
+  }
+}