diff --git a/.eslintrc.json b/.eslintrc.json index 56bbcf0952..dce9b3443d 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -5,8 +5,8 @@ "plugin:jsonc/recommended-with-json" ], "parserOptions": { - "ecmaVersion": 9, - "sourceType": "script", + "ecmaVersion": 2022, + "sourceType": "module", "ecmaFeatures": { "globalReturn": false, "impliedStrict": true @@ -14,7 +14,7 @@ }, "env": { "browser": true, - "es2018": true, + "es2022": true, "webextensions": true }, "plugins": [ @@ -24,7 +24,8 @@ "jsonc" ], "ignorePatterns": [ - "/ext/lib/" + "/ext/lib/", + "/dev/lib/handlebars/" ], "rules": { "arrow-parens": [ @@ -50,6 +51,7 @@ "no-case-declarations": "error", "no-const-assign": "error", "no-constant-condition": "off", + "no-console": "warn", "no-global-assign": "error", "no-param-reassign": "off", "no-prototype-builtins": "error", @@ -180,6 +182,7 @@ "after": true } ], + "no-implicit-globals": "error", "no-trailing-spaces": "error", "no-whitespace-before-property": "error", "object-curly-spacing": [ @@ -250,6 +253,7 @@ "jsdoc/multiline-blocks": "error", "jsdoc/no-bad-blocks": "error", "jsdoc/no-multi-asterisks": "error", + "jsdoc/no-undefined-types": 1, "jsdoc/require-asterisk-prefix": "error", "jsdoc/require-hyphen-before-param-description": [ "error", @@ -375,49 +379,6 @@ "webextensions": false } }, - { - "files": [ - "ext/**/*.js" - ], - "excludedFiles": [ - "ext/js/core.js", - "ext/js/accessibility/google-docs.js", - "ext/js/**/sandbox/**/*.js" - ], - "globals": { - "serializeError": "readonly", - "deserializeError": "readonly", - "isObject": "readonly", - "stringReverse": "readonly", - "promiseTimeout": "readonly", - "escapeRegExp": "readonly", - "deferPromise": "readonly", - "clone": "readonly", - "deepEqual": "readonly", - "generateId": "readonly", - "promiseAnimationFrame": "readonly", - "invokeMessageHandler": "readonly", - "log": "readonly", - "DynamicProperty": "readonly", - "EventDispatcher": "readonly", - "EventListenerCollection": "readonly", - "Logger": "readonly" - } - }, - { - "files": [ - "ext/**/*.js" - ], - "excludedFiles": [ - "ext/js/core.js", - "ext/js/accessibility/google-docs.js", - "ext/js/yomichan.js", - "ext/js/**/sandbox/**/*.js" - ], - "globals": { - "yomichan": "readonly" - } - }, { "files": [ "ext/js/yomichan.js" @@ -431,49 +392,26 @@ "test/**/*.js", "dev/**/*.js" ], - "excludedFiles": [ - "test/data/html/*.js" - ], - "parserOptions": { - "ecmaVersion": 8, - "sourceType": "module" - }, "env": { "browser": false, - "es2017": true, "node": true, "webextensions": false } }, { "files": [ - "playwright.config.js" + "test/data/html/*.js" ], - "env": { - "browser": false, - "es2017": true, - "node": true, - "webextensions": false + "parserOptions": { + "sourceType": "script" }, - "rules": { - "no-undefined": "off" - } - }, - { - "files": [ - "integration.spec.js", - "playwright-util.js", - "visual.spec.js" - ], "env": { - "browser": false, - "es2017": true, - "node": true, + "browser": true, + "node": false, "webextensions": false }, "rules": { - "no-undefined": "off", - "no-empty-pattern": "off" + "no-implicit-globals": "off" } }, { @@ -514,13 +452,13 @@ "env": { "browser": false, "serviceworker": true, - "es2017": true, "webextensions": true }, "globals": { "FileReader": "readonly", "Intl": "readonly", - "crypto": "readonly" + "crypto": "readonly", + "AbortController": "readonly" } }, { @@ -538,29 +476,52 @@ "env": { "browser": false, "worker": true, - "es2017": true, "webextensions": true } }, { "files": [ - "ext/js/**/*.js" - ], - "excludedFiles": [ - "ext/js/core.js", - "ext/js/**/*main.js" + "playwright.config.js" ], + "env": { + "browser": false, + "node": true, + "webextensions": false + }, "rules": { - "no-implicit-globals": "error" + "no-undefined": "off" } }, { "files": [ - "ext/js/**/*.js" + "integration.spec.js", + "playwright-util.js", + "visual.spec.js" ], - "globals": { - "AbortController": "readonly" + "env": { + "browser": false, + "node": true, + "webextensions": false + }, + "rules": { + "no-undefined": "off", + "no-empty-pattern": "off" } + }, + { + "files": [ + "test/**" + ], + "plugins": [ + "vitest" + ], + "extends": [ + "plugin:vitest/recommended" + ], + "rules": { + "vitest/prefer-to-be": "off" + }, + "env": {} } ] } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc70925fe7..358ac1def2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,8 +21,8 @@ jobs: - name: Install dependencies run: npm ci - - name: Lint - run: npm run test-lint + - name: Lint JS + run: npm run test-lint-js env: CI: true @@ -36,6 +36,9 @@ jobs: env: CI: true + - name: Build Libs + run: npm run build-libs + - name: Tests run: npm run test-code env: diff --git a/.gitignore b/.gitignore index 405fead0aa..40e825c68e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,17 @@ +.DS_Store + node_modules/ + builds/ -.DS_Store + dictionaries/ + /test-results/ /playwright-report/ /playwright/.cache/ /test/playwright/__screenshots__/ + ext/manifest.json + +ext/lib/* +!ext/lib/__mocks__/ diff --git a/.vscode/settings.json b/.vscode/settings.json index 9318d9f68e..2480961b96 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,10 +1,13 @@ { "markdown.extension.toc.levels": "1..3", "editor.codeActionsOnSave": { - "source.fixAll.eslint": true + "source.addMissingImports": true, + "source.organizeImports": true, + "source.fixAll.eslint": true, }, "eslint.format.enable": true, "playwright.env": { "PW_EXPERIMENTAL_SERVICE_WORKER_NETWORK_EVENTS": 1 - } + }, + "javascript.preferences.importModuleSpecifierEnding": "js", } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2b19e68ee5..3b48236ffd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,7 +42,7 @@ There are two scripts to build the extension to a packaged file for various buil - [build.bat](build.bat) on Windows - [build.sh](build.sh) on Linux -Both of these files are convenience scripts which invoke node [dev/build.js](dev/build.js). +Both of these files are convenience scripts which invoke node [dev/bin/build.js](dev/bin/build.js). The build script can produce several different build files based on manifest configurations defined in [manifest-variants.json](dev/data/manifest-variants.json). Several command line arguments are available for these scripts: diff --git a/dev/bin/build-libs.js b/dev/bin/build-libs.js new file mode 100644 index 0000000000..07d27188a5 --- /dev/null +++ b/dev/bin/build-libs.js @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * Copyright (C) 2020-2022 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import {buildLibs} from '../build-libs.js'; + +buildLibs(); diff --git a/dev/build.js b/dev/bin/build.js similarity index 92% rename from dev/build.js rename to dev/bin/build.js index 24b1e2d0f5..282f0414e1 100644 --- a/dev/build.js +++ b/dev/bin/build.js @@ -16,15 +16,17 @@ * along with this program. If not, see . */ -const fs = require('fs'); -const path = require('path'); -const assert = require('assert'); -const readline = require('readline'); -const childProcess = require('child_process'); -const util = require('./util'); -const {getAllFiles, getArgs, testMain} = util; -const {ManifestUtil} = require('./manifest-util'); - +import assert from 'assert'; +import childProcess from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import readline from 'readline'; +import {fileURLToPath} from 'url'; +import {buildLibs} from '../build-libs.js'; +import {ManifestUtil} from '../manifest-util.js'; +import {getAllFiles, getArgs, testMain} from '../util.js'; + +const dirname = path.dirname(fileURLToPath(import.meta.url)); async function createZip(directory, excludeFiles, outputFileName, sevenZipExes, onUpdate, dryRun) { try { @@ -59,7 +61,7 @@ async function createZip(directory, excludeFiles, outputFileName, sevenZipExes, } async function createJSZip(directory, excludeFiles, outputFileName, onUpdate, dryRun) { - const JSZip = util.JSZip; + const JSZip = null; const files = getAllFiles(directory); removeItemsFromArray(files, excludeFiles); const zip = new JSZip(); @@ -178,7 +180,7 @@ function ensureFilesExist(directory, files) { } -async function main(argv) { +export async function main(argv) { const args = getArgs(argv, new Map([ ['all', false], ['default', false], @@ -195,12 +197,13 @@ async function main(argv) { const manifestUtil = new ManifestUtil(); - const rootDir = path.join(__dirname, '..'); + const rootDir = path.join(dirname, '..', '..'); const extDir = path.join(rootDir, 'ext'); const buildDir = path.join(rootDir, 'builds'); const manifestPath = path.join(extDir, 'manifest.json'); try { + await buildLibs(); const variantNames = ( argv.length === 0 || args.get('all') ? manifestUtil.getVariants().filter(({buildable}) => buildable !== false).map(({name}) => name) : @@ -218,12 +221,4 @@ async function main(argv) { } } - -if (require.main === module) { - testMain(main, process.argv.slice(2)); -} - - -module.exports = { - main -}; +testMain(main, process.argv.slice(2)); diff --git a/test/test-build-libs.js b/dev/bin/dictionary-validate.js similarity index 50% rename from test/test-build-libs.js rename to dev/bin/dictionary-validate.js index 496f43f8b8..78ad5198a2 100644 --- a/test/test-build-libs.js +++ b/dev/bin/dictionary-validate.js @@ -1,6 +1,6 @@ /* * Copyright (C) 2023 Yomitan Authors - * Copyright (C) 2022 Yomichan Authors + * Copyright (C) 2020-2022 Yomichan Authors * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,27 +16,25 @@ * along with this program. If not, see . */ -const fs = require('fs'); -const assert = require('assert'); -const {getBuildTargets} = require('../dev/build-libs'); +import {testDictionaryFiles} from '../dictionary-validate.js'; async function main() { - try { - for (const {path: path2, build} of getBuildTargets()) { - let expectedContent = await build(); - if (typeof expectedContent !== 'string') { - // Buffer - expectedContent = expectedContent.toString('utf8'); - } - const actualContent = fs.readFileSync(path2, {encoding: 'utf8'}); - assert.strictEqual(actualContent, expectedContent); - } - } catch (e) { - console.error(e); - process.exit(-1); + const dictionaryFileNames = process.argv.slice(2); + if (dictionaryFileNames.length === 0) { + console.log([ + 'Usage:', + ' node dictionary-validate [--ajv] ...' + ].join('\n')); return; } - process.exit(0); + + let mode = null; + if (dictionaryFileNames[0] === '--ajv') { + mode = 'ajv'; + dictionaryFileNames.splice(0, 1); + } + + await testDictionaryFiles(mode, dictionaryFileNames); } -if (require.main === module) { main(); } +main(); diff --git a/test/test-css-json.js b/dev/bin/generate-css-json.js similarity index 58% rename from test/test-css-json.js rename to dev/bin/generate-css-json.js index ddeee6bded..48b42c65c1 100644 --- a/test/test-css-json.js +++ b/dev/bin/generate-css-json.js @@ -1,6 +1,6 @@ /* * Copyright (C) 2023 Yomitan Authors - * Copyright (C) 2021-2022 Yomichan Authors + * Copyright (C) 2020-2022 Yomichan Authors * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,22 +16,14 @@ * along with this program. If not, see . */ -const fs = require('fs'); -const assert = require('assert'); -const {testMain} = require('../dev/util'); -const {formatRulesJson, generateRules} = require('../dev/css-to-json-util'); -const {getTargets} = require('../dev/generate-css-json'); - +import fs from 'fs'; +import {formatRulesJson, generateRules, getTargets} from '../generate-css-json.js'; function main() { for (const {cssFile, overridesCssFile, outputPath} of getTargets()) { - const actual = fs.readFileSync(outputPath, {encoding: 'utf8'}); - const expected = formatRulesJson(generateRules(cssFile, overridesCssFile)); - assert.deepStrictEqual(actual, expected); + const json = formatRulesJson(generateRules(cssFile, overridesCssFile)); + fs.writeFileSync(outputPath, json, {encoding: 'utf8'}); } } - -if (require.main === module) { - testMain(main, process.argv.slice(2)); -} +main(); diff --git a/dev/bin/schema-validate.js b/dev/bin/schema-validate.js new file mode 100644 index 0000000000..86cfebae07 --- /dev/null +++ b/dev/bin/schema-validate.js @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * Copyright (C) 2020-2022 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import fs from 'fs'; +import performance from 'perf_hooks'; +import {createJsonSchema} from '../util.js'; + +function main() { + const args = process.argv.slice(2); + if (args.length < 2) { + console.log([ + 'Usage:', + ' node schema-validate [--ajv] ...' + ].join('\n')); + return; + } + + let mode = null; + if (args[0] === '--ajv') { + mode = 'ajv'; + args.splice(0, 1); + } + + const schemaSource = fs.readFileSync(args[0], {encoding: 'utf8'}); + const schema = JSON.parse(schemaSource); + + for (const dataFileName of args.slice(1)) { + const start = performance.now(); + try { + console.log(`Validating ${dataFileName}...`); + const dataSource = fs.readFileSync(dataFileName, {encoding: 'utf8'}); + const data = JSON.parse(dataSource); + createJsonSchema(mode, schema).validate(data); + const end = performance.now(); + console.log(`No issues detected (${((end - start) / 1000).toFixed(2)}s)`); + } catch (e) { + const end = performance.now(); + console.log(`Encountered an error (${((end - start) / 1000).toFixed(2)}s)`); + console.warn(e); + } + } +} + + +main(); diff --git a/dev/build-libs.js b/dev/build-libs.js index 36c07edd0c..8320a94785 100644 --- a/dev/build-libs.js +++ b/dev/build-libs.js @@ -16,51 +16,48 @@ * along with this program. If not, see . */ -const fs = require('fs'); -const path = require('path'); -const browserify = require('browserify'); +import Ajv from 'ajv'; +import standaloneCode from 'ajv/dist/standalone/index.js'; +import esbuild from 'esbuild'; +import fs from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; -async function buildParse5() { - const parse5Path = require.resolve('parse5'); - const cwd = process.cwd(); - try { - const baseDir = path.dirname(parse5Path); - process.chdir(baseDir); // This is necessary to ensure relative source map file names are consistent - return await new Promise((resolve, reject) => { - browserify({ - entries: [parse5Path], - standalone: 'parse5', - debug: true, - baseDir - }).bundle((error, result) => { - if (error) { - reject(error); - } else { - resolve(result); - } - }); - }); - } finally { - process.chdir(cwd); - } -} +const dirname = path.dirname(fileURLToPath(import.meta.url)); +const extDir = path.join(dirname, '..', 'ext'); -function getBuildTargets() { - const extLibPath = path.join(__dirname, '..', 'ext', 'lib'); - return [ - {path: path.join(extLibPath, 'parse5.js'), build: buildParse5} - ]; +async function buildLib(p) { + await esbuild.build({ + entryPoints: [p], + bundle: true, + minify: false, + sourcemap: true, + target: 'es2020', + format: 'esm', + outfile: path.join(extDir, 'lib', path.basename(p)), + external: ['fs'] + }); } -async function main() { - for (const {path: path2, build} of getBuildTargets()) { - const content = await build(); - fs.writeFileSync(path2, content); +export async function buildLibs() { + const devLibPath = path.join(dirname, 'lib'); + const files = await fs.promises.readdir(devLibPath, { + withFileTypes: true + }); + for (const f of files) { + if (f.isFile()) { + await buildLib(path.join(devLibPath, f.name)); + } } -} -if (require.main === module) { main(); } + const schemaDir = path.join(extDir, 'data/schemas/'); + const schemaFileNames = fs.readdirSync(schemaDir); + const schemas = schemaFileNames.map((schemaFileName) => JSON.parse(fs.readFileSync(path.join(schemaDir, schemaFileName)))); + const ajv = new Ajv({schemas: schemas, code: {source: true, esm: true}}); + const moduleCode = standaloneCode(ajv); -module.exports = { - getBuildTargets -}; + // https://github.com/ajv-validator/ajv/issues/2209 + const patchedModuleCode = "import {ucs2length} from './ucs2length.js';" + moduleCode.replaceAll('require("ajv/dist/runtime/ucs2length").default', 'ucs2length'); + + fs.writeFileSync(path.join(extDir, 'lib/validate-schemas.js'), patchedModuleCode); +} diff --git a/dev/css-to-json-util.js b/dev/css-to-json-util.js deleted file mode 100644 index 79aae3c9e1..0000000000 --- a/dev/css-to-json-util.js +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright (C) 2023 Yomitan Authors - * Copyright (C) 2021-2022 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -const fs = require('fs'); -const css = require('css'); - -function indexOfRule(rules, selectors) { - const jj = selectors.length; - for (let i = 0, ii = rules.length; i < ii; ++i) { - const ruleSelectors = rules[i].selectors; - if (ruleSelectors.length !== jj) { continue; } - let okay = true; - for (let j = 0; j < jj; ++j) { - if (selectors[j] !== ruleSelectors[j]) { - okay = false; - break; - } - } - if (okay) { return i; } - } - return -1; -} - -function removeProperty(styles, property, removedProperties) { - let removeCount = removedProperties.get(property); - if (typeof removeCount !== 'undefined') { return removeCount; } - removeCount = 0; - for (let i = 0, ii = styles.length; i < ii; ++i) { - const key = styles[i][0]; - if (key !== property) { continue; } - styles.splice(i, 1); - --i; - --ii; - ++removeCount; - } - removedProperties.set(property, removeCount); - return removeCount; -} - -function formatRulesJson(rules) { - // Manually format JSON, for improved compactness - // return JSON.stringify(rules, null, 4); - const indent1 = ' '; - const indent2 = indent1.repeat(2); - const indent3 = indent1.repeat(3); - let result = ''; - result += '['; - let index1 = 0; - for (const {selectors, styles} of rules) { - if (index1 > 0) { result += ','; } - result += `\n${indent1}{\n${indent2}"selectors": `; - if (selectors.length === 1) { - result += `[${JSON.stringify(selectors[0], null, 4)}]`; - } else { - result += JSON.stringify(selectors, null, 4).replace(/\n/g, '\n' + indent2); - } - result += `,\n${indent2}"styles": [`; - let index2 = 0; - for (const [key, value] of styles) { - if (index2 > 0) { result += ','; } - result += `\n${indent3}[${JSON.stringify(key)}, ${JSON.stringify(value)}]`; - ++index2; - } - if (index2 > 0) { result += `\n${indent2}`; } - result += `]\n${indent1}}`; - ++index1; - } - if (index1 > 0) { result += '\n'; } - result += ']'; - return result; -} - -function generateRules(cssFile, overridesCssFile) { - const content1 = fs.readFileSync(cssFile, {encoding: 'utf8'}); - const content2 = fs.readFileSync(overridesCssFile, {encoding: 'utf8'}); - const stylesheet1 = css.parse(content1, {}).stylesheet; - const stylesheet2 = css.parse(content2, {}).stylesheet; - - const removePropertyPattern = /^remove-property\s+([\w\W]+)$/; - const removeRulePattern = /^remove-rule$/; - const propertySeparator = /\s+/; - - const rules = []; - - // Default stylesheet - for (const rule of stylesheet1.rules) { - if (rule.type !== 'rule') { continue; } - const {selectors, declarations} = rule; - const styles = []; - for (const declaration of declarations) { - if (declaration.type !== 'declaration') { console.log(declaration); continue; } - const {property, value} = declaration; - styles.push([property, value]); - } - if (styles.length > 0) { - rules.push({selectors, styles}); - } - } - - // Overrides - for (const rule of stylesheet2.rules) { - if (rule.type !== 'rule') { continue; } - const {selectors, declarations} = rule; - const removedProperties = new Map(); - for (const declaration of declarations) { - switch (declaration.type) { - case 'declaration': - { - const index = indexOfRule(rules, selectors); - let entry; - if (index >= 0) { - entry = rules[index]; - } else { - entry = {selectors, styles: []}; - rules.push(entry); - } - const {property, value} = declaration; - removeProperty(entry.styles, property, removedProperties); - entry.styles.push([property, value]); - } - break; - case 'comment': - { - const index = indexOfRule(rules, selectors); - if (index < 0) { throw new Error('Could not find rule with matching selectors'); } - const comment = declaration.comment.trim(); - let m; - if ((m = removePropertyPattern.exec(comment)) !== null) { - for (const property of m[1].split(propertySeparator)) { - const removeCount = removeProperty(rules[index].styles, property, removedProperties); - if (removeCount === 0) { throw new Error(`Property removal is unnecessary; ${property} does not exist`); } - } - } else if (removeRulePattern.test(comment)) { - rules.splice(index, 1); - } - } - break; - } - } - } - - // Remove empty - for (let i = 0, ii = rules.length; i < ii; ++i) { - if (rules[i].styles.length > 0) { continue; } - rules.splice(i, 1); - --i; - --ii; - } - - return rules; -} - - -module.exports = { - formatRulesJson, - generateRules -}; diff --git a/dev/data/manifest-variants.json b/dev/data/manifest-variants.json index 1eae211252..e6113b7506 100644 --- a/dev/data/manifest-variants.json +++ b/dev/data/manifest-variants.json @@ -28,7 +28,8 @@ "default_popup": "action-popup.html" }, "background": { - "service_worker": "sw.js" + "service_worker": "sw.js", + "type": "module" }, "content_scripts": [ { @@ -41,28 +42,7 @@ "match_about_blank": true, "all_frames": true, "js": [ - "js/core.js", - "js/yomichan.js", - "js/app/frontend.js", - "js/app/popup.js", - "js/app/popup-factory.js", - "js/app/popup-proxy.js", - "js/app/popup-window.js", - "js/app/theme-controller.js", - "js/comm/api.js", - "js/comm/cross-frame-api.js", - "js/comm/frame-ancestry-handler.js", - "js/comm/frame-client.js", - "js/comm/frame-offset-forwarder.js", - "js/data/sandbox/string-util.js", - "js/dom/dom-text-scanner.js", - "js/dom/document-util.js", - "js/dom/text-source-element.js", - "js/dom/text-source-range.js", - "js/input/hotkey-handler.js", - "js/language/text-scanner.js", - "js/script/dynamic-loader.js", - "js/app/content-script-main.js" + "js/app/content-script-wrapper.js" ] } ], @@ -118,7 +98,8 @@ { "resources": [ "popup.html", - "template-renderer.html" + "template-renderer.html", + "js/*" ], "matches": [ "" @@ -141,8 +122,7 @@ "inherit": "base", "fileName": "yomitan-chrome.zip", "excludeFiles": [ - "background.html", - "js/dom/native-simple-dom-parser.js" + "background.html" ] }, { @@ -187,7 +167,9 @@ "path": [ "permissions" ], - "items": ["clipboardRead"] + "items": [ + "clipboardRead" + ] } ] }, @@ -203,6 +185,13 @@ "service_worker" ] }, + { + "action": "delete", + "path": [ + "background", + "type" + ] + }, { "action": "set", "path": [ @@ -268,9 +257,7 @@ "sw.js", "offscreen.html", "js/background/offscreen.js", - "js/background/offscreen-main.js", - "js/dom/simple-dom-parser.js", - "lib/parse5.js" + "js/background/offscreen-main.js" ] }, { @@ -319,9 +306,7 @@ "sw.js", "offscreen.html", "js/background/offscreen.js", - "js/background/offscreen-main.js", - "js/dom/simple-dom-parser.js", - "lib/parse5.js" + "js/background/offscreen-main.js" ] }, { @@ -368,9 +353,7 @@ "sw.js", "offscreen.html", "js/background/offscreen.js", - "js/background/offscreen-main.js", - "js/dom/simple-dom-parser.js", - "lib/parse5.js" + "js/background/offscreen-main.js" ] } ] diff --git a/dev/database-vm.js b/dev/database-vm.js deleted file mode 100644 index d557069114..0000000000 --- a/dev/database-vm.js +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2023 Yomitan Authors - * Copyright (C) 2020-2022 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -const fs = require('fs'); -const url = require('url'); -const path = require('path'); -const {JSZip} = require('./util'); -const {VM} = require('./vm'); -require('fake-indexeddb/auto'); - -const chrome = { - runtime: { - getURL: (path2) => { - return url.pathToFileURL(path.join(__dirname, '..', 'ext', path2.replace(/^\//, ''))).href; - } - } -}; - -async function fetch(url2) { - const extDir = path.join(__dirname, '..', 'ext'); - let filePath; - try { - filePath = url.fileURLToPath(url2); - } catch (e) { - filePath = path.resolve(extDir, url2.replace(/^[/\\]/, '')); - } - await Promise.resolve(); - const content = fs.readFileSync(filePath, {encoding: null}); - return { - ok: true, - status: 200, - statusText: 'OK', - text: async () => Promise.resolve(content.toString('utf8')), - json: async () => Promise.resolve(JSON.parse(content.toString('utf8'))) - }; -} - -function atob(data) { - return Buffer.from(data, 'base64').toString('ascii'); -} - -class DatabaseVM extends VM { - constructor(globals={}) { - super(Object.assign({ - chrome, - fetch, - indexedDB: global.indexedDB, - IDBKeyRange: global.IDBKeyRange, - JSZip, - atob - }, globals)); - this.context.window = this.context; - this.indexedDB = global.indexedDB; - } -} - -class DatabaseVMDictionaryImporterMediaLoader { - async getImageDetails(content) { - // Placeholder values - return {content, width: 100, height: 100}; - } -} - -module.exports = { - DatabaseVM, - DatabaseVMDictionaryImporterMediaLoader -}; diff --git a/dev/dictionary-validate.js b/dev/dictionary-validate.js index 0c926acc93..eb40beda04 100644 --- a/dev/dictionary-validate.js +++ b/dev/dictionary-validate.js @@ -16,12 +16,11 @@ * along with this program. If not, see . */ -const fs = require('fs'); -const path = require('path'); -const {performance} = require('perf_hooks'); -const {JSZip} = require('./util'); -const {createJsonSchema} = require('./schema-validate'); - +import fs from 'fs'; +import JSZip from 'jszip'; +import path from 'path'; +import {performance} from 'perf_hooks'; +import {createJsonSchema} from './schema-validate.js'; function readSchema(relativeFileName) { const fileName = path.join(__dirname, relativeFileName); @@ -29,7 +28,6 @@ function readSchema(relativeFileName) { return JSON.parse(source); } - async function validateDictionaryBanks(mode, zip, fileNameFormat, schema) { let jsonSchema; try { @@ -57,7 +55,7 @@ async function validateDictionaryBanks(mode, zip, fileNameFormat, schema) { } } -async function validateDictionary(mode, archive, schemas) { +export async function validateDictionary(mode, archive, schemas) { const fileName = 'index.json'; const indexFile = archive.files[fileName]; if (!indexFile) { @@ -82,7 +80,7 @@ async function validateDictionary(mode, archive, schemas) { await validateDictionaryBanks(mode, archive, 'tag_bank_?.json', schemas.tagBankV3); } -function getSchemas() { +export function getSchemas() { return { index: readSchema('../ext/data/schemas/dictionary-index-schema.json'), kanjiBankV1: readSchema('../ext/data/schemas/dictionary-kanji-bank-v1-schema.json'), @@ -95,8 +93,7 @@ function getSchemas() { }; } - -async function testDictionaryFiles(mode, dictionaryFileNames) { +export async function testDictionaryFiles(mode, dictionaryFileNames) { const schemas = getSchemas(); for (const dictionaryFileName of dictionaryFileNames) { @@ -115,33 +112,3 @@ async function testDictionaryFiles(mode, dictionaryFileNames) { } } } - - -async function main() { - const dictionaryFileNames = process.argv.slice(2); - if (dictionaryFileNames.length === 0) { - console.log([ - 'Usage:', - ' node dictionary-validate [--ajv] ...' - ].join('\n')); - return; - } - - let mode = null; - if (dictionaryFileNames[0] === '--ajv') { - mode = 'ajv'; - dictionaryFileNames.splice(0, 1); - } - - await testDictionaryFiles(mode, dictionaryFileNames); -} - - -if (require.main === module) { main(); } - - -module.exports = { - getSchemas, - validateDictionary, - testDictionaryFiles -}; diff --git a/dev/generate-css-json.js b/dev/generate-css-json.js index 787173abba..914c14522c 100644 --- a/dev/generate-css-json.js +++ b/dev/generate-css-json.js @@ -16,12 +16,10 @@ * along with this program. If not, see . */ -const fs = require('fs'); -const path = require('path'); -const {testMain} = require('./util'); -const {formatRulesJson, generateRules} = require('./css-to-json-util'); +import fs from 'fs'; +import path from 'path'; -function getTargets() { +export function getTargets() { return [ { cssFile: path.join(__dirname, '..', 'ext/css/structured-content.css'), @@ -36,19 +34,150 @@ function getTargets() { ]; } -function main() { - for (const {cssFile, overridesCssFile, outputPath} of getTargets()) { - const json = formatRulesJson(generateRules(cssFile, overridesCssFile)); - fs.writeFileSync(outputPath, json, {encoding: 'utf8'}); +import css from 'css'; + +function indexOfRule(rules, selectors) { + const jj = selectors.length; + for (let i = 0, ii = rules.length; i < ii; ++i) { + const ruleSelectors = rules[i].selectors; + if (ruleSelectors.length !== jj) { continue; } + let okay = true; + for (let j = 0; j < jj; ++j) { + if (selectors[j] !== ruleSelectors[j]) { + okay = false; + break; + } + } + if (okay) { return i; } } + return -1; } +function removeProperty(styles, property, removedProperties) { + let removeCount = removedProperties.get(property); + if (typeof removeCount !== 'undefined') { return removeCount; } + removeCount = 0; + for (let i = 0, ii = styles.length; i < ii; ++i) { + const key = styles[i][0]; + if (key !== property) { continue; } + styles.splice(i, 1); + --i; + --ii; + ++removeCount; + } + removedProperties.set(property, removeCount); + return removeCount; +} -if (require.main === module) { - testMain(main, process.argv.slice(2)); +export function formatRulesJson(rules) { + // Manually format JSON, for improved compactness + // return JSON.stringify(rules, null, 4); + const indent1 = ' '; + const indent2 = indent1.repeat(2); + const indent3 = indent1.repeat(3); + let result = ''; + result += '['; + let index1 = 0; + for (const {selectors, styles} of rules) { + if (index1 > 0) { result += ','; } + result += `\n${indent1}{\n${indent2}"selectors": `; + if (selectors.length === 1) { + result += `[${JSON.stringify(selectors[0], null, 4)}]`; + } else { + result += JSON.stringify(selectors, null, 4).replace(/\n/g, '\n' + indent2); + } + result += `,\n${indent2}"styles": [`; + let index2 = 0; + for (const [key, value] of styles) { + if (index2 > 0) { result += ','; } + result += `\n${indent3}[${JSON.stringify(key)}, ${JSON.stringify(value)}]`; + ++index2; + } + if (index2 > 0) { result += `\n${indent2}`; } + result += `]\n${indent1}}`; + ++index1; + } + if (index1 > 0) { result += '\n'; } + result += ']'; + return result; } +export function generateRules(cssFile, overridesCssFile) { + const content1 = fs.readFileSync(cssFile, {encoding: 'utf8'}); + const content2 = fs.readFileSync(overridesCssFile, {encoding: 'utf8'}); + const stylesheet1 = css.parse(content1, {}).stylesheet; + const stylesheet2 = css.parse(content2, {}).stylesheet; + + const removePropertyPattern = /^remove-property\s+([\w\W]+)$/; + const removeRulePattern = /^remove-rule$/; + const propertySeparator = /\s+/; -module.exports = { - getTargets -}; + const rules = []; + + // Default stylesheet + for (const rule of stylesheet1.rules) { + if (rule.type !== 'rule') { continue; } + const {selectors, declarations} = rule; + const styles = []; + for (const declaration of declarations) { + if (declaration.type !== 'declaration') { console.log(declaration); continue; } + const {property, value} = declaration; + styles.push([property, value]); + } + if (styles.length > 0) { + rules.push({selectors, styles}); + } + } + + // Overrides + for (const rule of stylesheet2.rules) { + if (rule.type !== 'rule') { continue; } + const {selectors, declarations} = rule; + const removedProperties = new Map(); + for (const declaration of declarations) { + switch (declaration.type) { + case 'declaration': + { + const index = indexOfRule(rules, selectors); + let entry; + if (index >= 0) { + entry = rules[index]; + } else { + entry = {selectors, styles: []}; + rules.push(entry); + } + const {property, value} = declaration; + removeProperty(entry.styles, property, removedProperties); + entry.styles.push([property, value]); + } + break; + case 'comment': + { + const index = indexOfRule(rules, selectors); + if (index < 0) { throw new Error('Could not find rule with matching selectors'); } + const comment = declaration.comment.trim(); + let m; + if ((m = removePropertyPattern.exec(comment)) !== null) { + for (const property of m[1].split(propertySeparator)) { + const removeCount = removeProperty(rules[index].styles, property, removedProperties); + if (removeCount === 0) { throw new Error(`Property removal is unnecessary; ${property} does not exist`); } + } + } else if (removeRulePattern.test(comment)) { + rules.splice(index, 1); + } + } + break; + } + } + } + + // Remove empty + for (let i = 0, ii = rules.length; i < ii; ++i) { + if (rules[i].styles.length > 0) { continue; } + rules.splice(i, 1); + --i; + --ii; + } + + return rules; +} diff --git a/dev/lib/dexie-export-import.js b/dev/lib/dexie-export-import.js new file mode 100644 index 0000000000..8d2ec2068f --- /dev/null +++ b/dev/lib/dexie-export-import.js @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +export * from 'dexie-export-import'; diff --git a/dev/lib/dexie.js b/dev/lib/dexie.js new file mode 100644 index 0000000000..aa3f2b7d95 --- /dev/null +++ b/dev/lib/dexie.js @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +export * from 'dexie'; diff --git a/dev/lib/handlebars.js b/dev/lib/handlebars.js new file mode 100644 index 0000000000..5b57efdd63 --- /dev/null +++ b/dev/lib/handlebars.js @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +export {Handlebars} from './handlebars/src/handlebars.js'; + diff --git a/dev/lib/handlebars/LICENSE b/dev/lib/handlebars/LICENSE new file mode 100644 index 0000000000..5d971a1754 --- /dev/null +++ b/dev/lib/handlebars/LICENSE @@ -0,0 +1,29 @@ +The MIT License (MIT) + +Copyright (c) Elasticsearch BV +Copyright (c) Copyright (C) 2011-2019 by Yehuda Katz + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +This software consists of voluntary contributions made by many +individuals. For exact contribution history, see the revision history +available at the following locations: + - https://github.com/handlebars-lang/handlebars.js + - https://github.com/elastic/kibana/tree/main/packages/kbn-handlebars diff --git a/dev/lib/handlebars/README.md b/dev/lib/handlebars/README.md new file mode 100644 index 0000000000..cc151645ef --- /dev/null +++ b/dev/lib/handlebars/README.md @@ -0,0 +1,164 @@ +# @kbn/handlebars + +A custom version of the handlebars package which, to improve security, does not use `eval` or `new Function`. This means that templates can't be compiled into JavaScript functions in advance and hence, rendering the templates is a lot slower. + +## Limitations + +- Only the following compile options are supported: + - `data` + - `knownHelpers` + - `knownHelpersOnly` + - `noEscape` + - `strict` + - `assumeObjects` + - `preventIndent` + - `explicitPartialContext` + +- Only the following runtime options are supported: + - `data` + - `helpers` + - `partials` + - `decorators` (not documented in the official Handlebars [runtime options documentation](https://handlebarsjs.com/api-reference/runtime-options.html)) + - `blockParams` (not documented in the official Handlebars [runtime options documentation](https://handlebarsjs.com/api-reference/runtime-options.html)) + +## Implementation differences + +The standard `handlebars` implementation: + +1. When given a template string, e.g. `Hello {{x}}`, return a "render" function which takes an "input" object, e.g. `{ x: 'World' }`. +1. The first time the "render" function is called the following happens: + 1. Turn the template string into an Abstract Syntax Tree (AST). + 1. Convert the AST into a hyper optimized JavaScript function which takes the input object as an argument. + 1. Call the generate JavaScript function with the given "input" object to produce and return the final output string (`Hello World`). +1. Subsequent calls to the "render" function will re-use the already generated JavaScript function. + +The custom `@kbn/handlebars` implementation: + +1. When given a template string, e.g. `Hello {{x}}`, return a "render" function which takes an "input" object, e.g. `{ x: 'World' }`. +1. The first time the "render" function is called the following happens: + 1. Turn the template string into an Abstract Syntax Tree (AST). + 1. Process the AST with the given "input" object to produce and return the final output string (`Hello World`). +1. Subsequent calls to the "render" function will re-use the already generated AST. + +_Note: Not parsing of the template string until the first call to the "render" function is deliberate as it mimics the original `handlebars` implementation. This means that any errors that occur due to an invalid template string will not be thrown until the first call to the "render" function._ + +## Technical details + +The `handlebars` library exposes the API for both [generating the AST](https://github.com/handlebars-lang/handlebars.js/blob/master/docs/compiler-api.md#ast) and walking it by implementing the [Visitor API](https://github.com/handlebars-lang/handlebars.js/blob/master/docs/compiler-api.md#ast-visitor). We can leverage that to our advantage and create our own "render" function, which internally calls this API to generate the AST and then the API to walk the AST. + +The `@kbn/handlebars` implementation of the `Visitor` class implements all the necessary methods called by the parent `Visitor` code when instructed to walk the AST. They all start with an upppercase letter, e.g. `MustacheStatement` or `SubExpression`. We call this class `ElasticHandlebarsVisitor`. + +To parse the template string to an AST representation, we call `Handlebars.parse(templateString)`, which returns an AST object. + +The AST object contains a bunch of nodes, one for each element of the template string, all arranged in a tree-like structure. The root of the AST object is a node of type `Program`. This is a special node, which we do not need to worry about, but each of its direct children has a type named like the method which will be called when the walking algorithm reaches that node, e.g. `ContentStatement` or `BlockStatement`. These are the methods that our `Visitor` implementation implements. + +To instruct our `ElasticHandlebarsVisitor` class to start walking the AST object, we call the `accept()` method inherited from the parent `Visitor` class with the main AST object. The `Visitor` will walk each node in turn that is directly attached to the root `Program` node. For each node it traverses, it will call the matching method in our `ElasticHandlebarsVisitor` class. + +To instruct the `Visitor` code to traverse any child nodes of a given node, our implementation needs to manually call `accept(childNode)`, `acceptArray(arrayOfChildNodes)`, `acceptKey(node, childKeyName)`, or `acceptRequired(node, childKeyName)` from within any of the "node" methods, otherwise the child nodes are ignored. + +### State + +We keep state internally in the `ElasticHandlebarsVisitor` object using the following private properties: + +- `contexts`: An array (stack) of `context` objects. In a simple template this array will always only contain a single element: The main `context` object. In more complicated scenarios, new `context` objects will be pushed and popped to and from the `contexts` stack as needed. +- `output`: An array containing the "rendered" output of each node (normally just one element per node). In the most simple template, this is simply joined together into a the final output string after the AST has been traversed. In more complicated templates, we use this array temporarily to collect parameters to give to helper functions (see the `getParams` function). + +## Testing + +The tests for `@kbn/handlebars` are integrated into the regular test suite of Kibana and are all jest tests. To run them all, simply do: + +```sh +node scripts/jest packages/kbn-handlebars +``` + +By default, each test will run both the original `handlebars` code and the modified `@kbn/handlebars` code to compare if the output of the two are identical. To isolate a test run to just one or the other, you can use the following environment variables: + +- `EVAL=1` - Set to only run the original `handlebars` implementation that uses `eval`. +- `AST=1` - Set to only run the modified `@kbn/handlebars` implementation that doesn't use `eval`. + +## Development + +Some of the tests have been copied from the upstream `handlebars` project and modified to fit our use-case, test-suite, and coding conventions. They are all located under the `packages/kbn-handlebars/src/spec` directory. To check if any of the copied files have received updates upstream that we might want to include in our copies, you can run the following script: + +```sh +./packages/kbn-handlebars/scripts/check_for_upstream_updates.sh +``` + +_Note: This will look for changes in the `4.x` branch of the `handlebars.js` repo only. Changes in the `master` branch are ignored._ + +Once all updates have been manually merged with our versions of the files, run the following script to "lock" us into the new updates: + +```sh +./packages/kbn-handlebars/scripts/update_upstream_git_hash.sh +``` + +This will update file `packages/kbn-handlebars/src/spec/.upstream_git_hash`. Make sure to commit changes to this file as well. + +## Debugging + +### Print AST + +To output the generated AST object structure in a somewhat readable form, use the following script: + +```sh +./packages/kbn-handlebars/scripts/print_ast.js +``` + +Example: + +```sh +./packages/kbn-handlebars/scripts/print_ast.js '{{value}}' +``` + +Output: + +```js +{ + type: 'Program', + body: [ + { + type: 'MustacheStatement', + path: { + type: 'PathExpression', + data: false, + depth: 0, + parts: [ 'value' ], + original: 'value' + }, + params: [], + hash: undefined, + escaped: true + } + ] +} +``` + +By default certain properties will be hidden in the output. +For more control over the output, check out the options by running the script without any arguments. + +### Print generated code + +It's possible to see the generated JavaScript code that `handlebars` create for a given template using the following command line tool: + +```sh +./node_modules/handlebars/print-script