From ceb217893553df3c9b69f65fdfef6219f348dd07 Mon Sep 17 00:00:00 2001 From: Seth Falco Date: Sat, 25 May 2024 23:44:26 +0100 Subject: [PATCH] test: expand and optimize regression testing (#1918) --- .github/workflows/main.yml | 35 ++++--- test/regression-extract.js | 185 +++++++++++++++++++++++++++++-------- test/regression.js | 1 - 3 files changed, 169 insertions(+), 52 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 684a8d1cd..c61170ce9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,11 +1,10 @@ name: CI + env: FORCE_COLOR: 2 NODE: 20 + on: - push: - branches: - - main pull_request: branches: - main @@ -26,18 +25,6 @@ jobs: - run: yarn install - run: yarn lint - run: yarn typecheck - regression: - name: Test regressions - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE }} - cache: yarn - - run: yarn install - - run: yarn playwright install --with-deps chromium - - run: yarn test-regression test: name: ${{ matrix.os }} Node.js ${{ matrix.node-version }} strategy: @@ -61,3 +48,21 @@ jobs: - run: yarn playwright install --with-deps chromium - run: yarn test - run: yarn test-bundles + regression: + name: Test regressions + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + needs: + - lint + - test + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE }} + cache: yarn + - run: yarn install + - run: yarn playwright install --with-deps chromium + - run: yarn test-regression diff --git a/test/regression-extract.js b/test/regression-extract.js index 8ac9898db..76063bd77 100644 --- a/test/regression-extract.js +++ b/test/regression-extract.js @@ -11,55 +11,169 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); const pipeline = util.promisify(stream.pipeline); +/** Files to skip regression testing for due to parsing issues. */ const exclude = [ // animated - 'svg/filters-light-04-f.svg', - 'svg/filters-composite-05-f.svg', + 'svgs/W3C_SVG_11_TestSuite/svg/filters-light-04-f.svg', + 'svgs/W3C_SVG_11_TestSuite/svg/filters-composite-05-f.svg', // messed gradients - 'svg/pservers-grad-18-b.svg', + 'svgs/W3C_SVG_11_TestSuite/svg/pservers-grad-18-b.svg', // removing wrapping breaks :first-child pseudo-class - 'svg/styling-pres-04-f.svg', + 'svgs/W3C_SVG_11_TestSuite/svg/styling-pres-04-f.svg', // rect is converted to path which matches wrong styles - 'svg/styling-css-08-f.svg', + 'svgs/W3C_SVG_11_TestSuite/svg/styling-css-08-f.svg', // complex selectors are messed because of converting shapes to paths - 'svg/struct-use-10-f.svg', - 'svg/struct-use-11-f.svg', - 'svg/styling-css-01-b.svg', - 'svg/styling-css-04-f.svg', - // strange artifact breaks inconsistently breaks regression tests - 'svg/filters-conv-05-f.svg', + 'svgs/W3C_SVG_11_TestSuite/svg/struct-use-10-f.svg', + 'svgs/W3C_SVG_11_TestSuite/svg/struct-use-11-f.svg', + 'svgs/W3C_SVG_11_TestSuite/svg/styling-css-01-b.svg', + 'svgs/W3C_SVG_11_TestSuite/svg/styling-css-04-f.svg', + // strange artifact breaks inconsistently breaks regression tests + 'svgs/W3C_SVG_11_TestSuite/svg/filters-conv-05-f.svg', + // broken upon adding dataset and pending fix + 'svgs/oxygen-icons-5.113.0/scalable/actions/document-print-preview.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/hidef/tools-rip-audio-cd.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/im-ban-kick-user.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/im-ban-user.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/irc-close-channel.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/irc-remove-operator.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/irc-unvoice.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/list-add.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/list-remove-user.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/list-remove.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/mail-send.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/player-volume.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/16x16/document-edit-decrypt.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/16x16/document-edit-encrypt.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/16x16/document-edit-sign.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/16x16/document-edit-verify.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/16x16/im-ban-user.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/16x16/im-yahoo.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/16x16/irc-close-channel.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/16x16/irc-remove-operator.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/16x16/irc-unvoice.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/16x16/list-add-user.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/16x16/tools-media-optical-burn-image.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/22x22/edit-select-none.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/22x22/im-google.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/22x22/irc-close-channel.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/22x22/list-add-user.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/22x22/mail-receive.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/22x22/mixer-cd.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/22x22/news-unsubscribe.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/22x22/tools-media-optical-burn-image.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/22x22/tools-media-optical-copy.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/22x22/view-time-schedule-baselined-remove.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/32x32/documentation.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/32x32/edit-table-delete-column.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/32x32/edit-table-delete-row.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/32x32/im-ban-kick-user.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/32x32/im-ban-user.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/32x32/im-google.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/32x32/mail-mark-junk.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/32x32/tools-media-optical-burn-image.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/32x32/tools-rip-video-dvd.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/32x32/view-calendar-birthday.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/32x32/view-calendar-holiday.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/32x32/view-calendar-special-occasion.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/32x32/view-calendar-wedding-anniversary.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/48x48/im-google.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/small/48x48/tools-media-optical-burn-image.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/tools-media-optical-burn-image.svg', + 'svgs/oxygen-icons-5.113.0/scalable/actions/tools-rip-audio-cd.svg', + 'svgs/oxygen-icons-5.113.0/scalable/applets/org.kde.plasma.clipboard.svg', + 'svgs/oxygen-icons-5.113.0/scalable/applets/org.kde.plasma.devicenotifier.svg', + 'svgs/oxygen-icons-5.113.0/scalable/applets/org.kde.plasma.icontasks.svg', + 'svgs/oxygen-icons-5.113.0/scalable/applets/org.kde.plasma.kickerdash.svg', + 'svgs/oxygen-icons-5.113.0/scalable/applets/org.kde.plasma.quicklaunch.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/basket.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/calligraauthor.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/hardware.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/hidef/kmail2.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/hidef/preferences-desktop-locale.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/hwinfo.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/kde-gtk-config.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/kjournal.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/kmail2.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/kmymoney.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/kplato.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/krfb.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/ksudoku.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/okteta.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/picmi.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/preferences-desktop-user-password.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/preferences-system-time.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/small/16x16/kchart.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/small/16x16/system-file-manager.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/small/22x22/basket.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/small/32x32/preferences-system-windows-move.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/small/32x32/system-file-manager.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/small/48x48/kig.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/small/64x64/kplato.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/strigi.svg', + 'svgs/oxygen-icons-5.113.0/scalable/apps/timevault.svg', + 'svgs/oxygen-icons-5.113.0/scalable/categories/applications-toys.svg', + 'svgs/oxygen-icons-5.113.0/scalable/categories/hidef/preferences-system.svg', + 'svgs/oxygen-icons-5.113.0/scalable/categories/preferences-system.svg', + 'svgs/oxygen-icons-5.113.0/scalable/devices/audio-card.svg', + 'svgs/oxygen-icons-5.113.0/scalable/devices/camera-web.svg', + 'svgs/oxygen-icons-5.113.0/scalable/devices/cpu.svg', + 'svgs/oxygen-icons-5.113.0/scalable/devices/hidef/input-keyboard.svg', + 'svgs/oxygen-icons-5.113.0/scalable/devices/hidef/media-optical-audio.svg', + 'svgs/oxygen-icons-5.113.0/scalable/devices/input-keyboard.svg', + 'svgs/oxygen-icons-5.113.0/scalable/devices/media-optical-audio.svg', + 'svgs/oxygen-icons-5.113.0/scalable/devices/scanner.svg', + 'svgs/oxygen-icons-5.113.0/scalable/devices/small/16x16/media-optical-audio.svg', + 'svgs/oxygen-icons-5.113.0/scalable/devices/small/16x16/media-optical-data.svg', + 'svgs/oxygen-icons-5.113.0/scalable/devices/small/22x22/media-optical-audio.svg', + 'svgs/oxygen-icons-5.113.0/scalable/devices/small/22x22/media-optical-data.svg', + 'svgs/oxygen-icons-5.113.0/scalable/devices/small/32x32/media-optical-audio.svg', + 'svgs/oxygen-icons-5.113.0/scalable/devices/small/48x48/media-optical-audio.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/application-x-cd-image.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/application-x-cue.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/application-x-kvtml.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/hidef/application-rtf.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/hidef/application-sxw.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/small/22x22/application-javascript.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/small/22x22/application-pdf.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/small/22x22/application-x-javascript.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/small/22x22/application-x-srt.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/small/22x22/text-css.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/small/22x22/text-plain.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/small/22x22/text-x-changelog.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/small/22x22/text-x-csharp.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/small/48x48/application-x-cd-image.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/small/48x48/application-x-cda.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/small/48x48/application-x-cue.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/small/48x48/help.svg', + 'svgs/oxygen-icons-5.113.0/scalable/mimetypes/small/64x64/application-x-cue.svg', + 'svgs/oxygen-icons-5.113.0/scalable/places/small/64x64/folder-tar.svg', + 'svgs/oxygen-icons-5.113.0/scalable/places/small/64x64/network-server-database.svg', + 'svgs/oxygen-icons-5.113.0/scalable/places/small/64x64/server-database.svg', + 'svgs/oxygen-icons-5.113.0/scalable/status/small/22x22/weather-showers-day.svg', + 'svgs/oxygen-icons-5.113.0/scalable/status/small/32x32/weather-showers-day.svg', + 'svgs/oxygen-icons-5.113.0/scalable/status/user-busy.svg', + 'svgs/oxygen-icons-5.113.0/scalable/status/user-online.svg', + 'svgs/oxygen-icons-5.113.0/scalable/text-formatting.svg', ]; /** * @param {string} url * @param {string} baseDir - * @param {RegExp} include */ -const extractTarGz = async (url, baseDir, include) => { +const extractTarGz = async (url, baseDir) => { const extract = tarStream.extract(); extract.on('entry', async (header, stream, next) => { const name = header.name; try { - if (include == null || include.test(name)) { - if ( - name.endsWith('.svg') && - !exclude.includes(name) && - !name.startsWith('svg/animate-') - ) { - const file = path.join(baseDir, name); - await fs.promises.mkdir(path.dirname(file), { recursive: true }); - await pipeline(stream, fs.createWriteStream(file)); - } else if (name.endsWith('.svgz')) { - // .svgz -> .svg - const file = path.join(baseDir, name.slice(0, -1)); - await fs.promises.mkdir(path.dirname(file), { recursive: true }); - await pipeline( - stream, - zlib.createGunzip(), - fs.createWriteStream(file), - ); - } + if ( + name.endsWith('.svg') && + !exclude.includes(name) && + !name.startsWith('svgs/W3C_SVG_11_TestSuite/svg/animate-') + ) { + const file = path.join(baseDir, header.name); + await fs.promises.mkdir(path.dirname(file), { recursive: true }); + await pipeline(stream, fs.createWriteStream(file)); } } catch (error) { console.error(error); @@ -74,11 +188,10 @@ const extractTarGz = async (url, baseDir, include) => { (async () => { try { - console.info('Downloading W3C SVG 1.1 Test Suite and extracting files'); + console.info('Downloading SVGO Test Suite and extracting files'); await extractTarGz( - 'https://www.w3.org/Graphics/SVG/Test/20110816/archives/W3C_SVG_11_TestSuite.tar.gz', - path.join(__dirname, 'regression-fixtures', 'w3c-svg-11-test-suite'), - /^svg\//, + 'https://svg.github.io/svgo-test-suite/svgo-test-suite.tar.gz', + path.join(__dirname, 'regression-fixtures'), ); } catch (error) { console.error(error); diff --git a/test/regression.js b/test/regression.js index 20592ee10..88f7f58a5 100644 --- a/test/regression.js +++ b/test/regression.js @@ -56,7 +56,6 @@ const runTests = async (list) => { ); // ignore small aliasing issues if (matched <= 4) { - console.info(`${name} is passed`); passed++; } else { mismatched++;