From f45dc851351e2736ab4a775e6d79136350a94957 Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Mon, 1 Jun 2015 18:53:38 +0200 Subject: [PATCH 01/23] Install mocha --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 06152b4fa8..74b926099b 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "gulp-concat": "^2.3.4", "gulp-header": "^1.0.5", "gulp-rename": "^1.2.0", - "gulp-uglify": "^0.3.1" + "gulp-uglify": "^0.3.1", + "mocha": "^2.2.5" } } From d1f905209ae44960a956481f7133be504471d39f Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Mon, 1 Jun 2015 20:39:11 +0200 Subject: [PATCH 02/23] Implemented test runner --- package.json | 3 +- tests/helper/components.js | 11 ++++ tests/helper/prism-loader.js | 101 +++++++++++++++++++++++++++++ tests/helper/test-case.js | 71 ++++++++++++++++++++ tests/helper/test-discovery.js | 63 ++++++++++++++++++ tests/languages/javascript/test.js | 0 tests/run.js | 32 +++++++++ 7 files changed, 280 insertions(+), 1 deletion(-) create mode 100644 tests/helper/components.js create mode 100644 tests/helper/prism-loader.js create mode 100644 tests/helper/test-case.js create mode 100644 tests/helper/test-discovery.js create mode 100644 tests/languages/javascript/test.js create mode 100644 tests/run.js diff --git a/package.json b/package.json index 74b926099b..e254ad770a 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "Lightweight, robust, elegant syntax highlighting. A spin-off project from Dabblet.", "main": "prism.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "mocha tests/run.js" }, "repository": { "type": "git", @@ -18,6 +18,7 @@ "license": "MIT", "readmeFilename": "README.md", "devDependencies": { + "chai": "^2.3.0", "gulp": "^3.8.6", "gulp-concat": "^2.3.4", "gulp-header": "^1.0.5", diff --git a/tests/helper/components.js b/tests/helper/components.js new file mode 100644 index 0000000000..274ad146f7 --- /dev/null +++ b/tests/helper/components.js @@ -0,0 +1,11 @@ +"use strict"; + +var fs = require("fs"); +var vm = require('vm'); + +var fileContent = fs.readFileSync(__dirname + "/../../components.js", "utf8"); +var context = {}; +vm.createContext(context); +vm.runInContext(fileContent, context); + +module.exports = context.components; diff --git a/tests/helper/prism-loader.js b/tests/helper/prism-loader.js new file mode 100644 index 0000000000..1f1426ac92 --- /dev/null +++ b/tests/helper/prism-loader.js @@ -0,0 +1,101 @@ +"use strict"; + +var fs = require("fs"); +var vm = require('vm'); +var components = require("./components"); +var languagesCatalog = components.languages; + + +module.exports = { + + /** + * Creates a new Prism instance with the given language loaded + * + * @param {string} language + * @returns {Prism} + */ + createInstance: function (language) { + var Prism = this.createEmptyPrism(); + return this.loadLanguage(language, Prism); + }, + + + /** + * Loads the given language (including recursively loading the dependencies) and + * appends the config to the given Prism object + * + * @private + * @param {string} language + * @param {Prism} Prism + * @returns {Prism} + */ + loadLanguage: function (language, Prism) { + if (!languagesCatalog[language]) + { + throw new Error("Language '" + language + "' not found."); + } + + // if the language has a dependency -> load it first + if (languagesCatalog[language].require) + { + Prism = this.loadLanguage(languagesCatalog[language].require, Prism); + } + + // load the language itself + var languageSource = this.loadFileSource(language); + var context = this.runFileWithContext(languageSource, {Prism: Prism}); + + return context.Prism; + }, + + + /** + * Creates a new empty prism instance + * + * @private + * @returns {Prism} + */ + createEmptyPrism: function () { + var coreSource = this.loadFileSource("core"); + var context = this.runFileWithContext(coreSource); + return context.Prism; + }, + + + /** + * Cached file sources, to prevent massive HDD work + * + * @private + * @type {Object.} + */ + fileSourceCache: {}, + + + /** + * Loads the given file source as string + * + * @private + * @param {string} name + * @returns {string} + */ + loadFileSource: function (name) { + return this.fileSourceCache[name] = this.fileSourceCache[name] || fs.readFileSync(__dirname + "/../../components/prism-" + name + ".js", "utf8"); + }, + + + /** + * Runs a VM for a given file source with the given context + * + * @private + * @param {string} fileSource + * @param {*}context + * + * @returns {*} + */ + runFileWithContext: function (fileSource, context) { + context = context || {}; + vm.createContext(context); + vm.runInContext(fileSource, context); + return context; + } +}; diff --git a/tests/helper/test-case.js b/tests/helper/test-case.js new file mode 100644 index 0000000000..6afce39ae3 --- /dev/null +++ b/tests/helper/test-case.js @@ -0,0 +1,71 @@ +"use strict"; + +var fs = require("fs"); +var expect = require("chai").expect; +var PrismLoader = require("./prism-loader"); + +module.exports = { + + /** + * Runs the given test case file and asserts the result + * + * @param {string} language + * @param {string} filePath + */ + runTestCase: function (language, filePath) { + var testCase = this.parseTestCaseFile(filePath); + + if (null === testCase) { + throw new Error("Test case file has invalid format, please read the docs."); + } + + var Prism = PrismLoader.createInstance(language); + var compiledTokenStream = Prism.tokenize(testCase.testSource, Prism.languages[language]); + var simplifiedTokenStream = this.transformCompiledTokenStream(compiledTokenStream); + + expect(simplifiedTokenStream).to.eql(testCase.expectedTokenStream); + }, + + + /** + * Simplifies the token stream to ease the matching with the expected token stream + * + * @param {string} tokenStream + * @returns {Array.} + */ + transformCompiledTokenStream: function (tokenStream) { + return tokenStream.filter( + function (token) { + // only support objects + return (typeof token === "object"); + } + ).map( + function (entry) + { + return [entry.type, entry.content]; + } + ); + }, + + + /** + * Parses the test case from the given test case file + * + * @private + * @param {string} filePath + * @returns {{testSource: string, expectedTokenStream: *}|null} + */ + parseTestCaseFile: function (filePath) { + var testCaseSource = fs.readFileSync(filePath, "utf8"); + var testCase = testCaseSource.split(/^----*\w*$/m); + + if (2 === testCase.length) { + return { + testSource: testCase[0].trim(), + expectedTokenStream: JSON.parse(testCase[1]) + }; + } + + return null; + } +}; diff --git a/tests/helper/test-discovery.js b/tests/helper/test-discovery.js new file mode 100644 index 0000000000..9f9ea2dca4 --- /dev/null +++ b/tests/helper/test-discovery.js @@ -0,0 +1,63 @@ +"use strict"; + +var fs = require("fs"); +var path = require('path'); + + +module.exports = { + + /** + * Loads the list of all available tests + * + * @param {string} rootDir + * @returns {Object.} + */ + loadAllTests: function (rootDir) { + var testSuite = {}; + var self = this; + + this.getAllDirectories(rootDir).forEach( + function (language) { + testSuite[language] = self.getAllFiles(path.join(rootDir, language)); + } + ); + + return testSuite; + }, + + + /** + * Returns a list of all (sub)directories (just the directory names, not full paths) + * in the given src directory + * + * @param {string} src + * @returns {Array.} + */ + getAllDirectories: function (src) { + return fs.readdirSync(src).filter( + function (file) { + return fs.statSync(path.join(src, file)).isDirectory(); + } + ); + }, + + + /** + * Returns a list of all full file paths to all files in the given src directory + * + * @private + * @param {string} src + * @returns {Array.} + */ + getAllFiles: function (src) { + return fs.readdirSync(src).filter( + function (fileName) { + return fs.statSync(path.join(src, fileName)).isFile(); + } + ).map( + function (fileName) { + return path.join(src, fileName); + } + ); + } +}; diff --git a/tests/languages/javascript/test.js b/tests/languages/javascript/test.js new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/run.js b/tests/run.js new file mode 100644 index 0000000000..50bd20f137 --- /dev/null +++ b/tests/run.js @@ -0,0 +1,32 @@ +"use strict"; + +var TestDiscovery = require("./helper/test-discovery"); +var TestCase = require("./helper/test-case"); +var path = require("path"); + +// load complete test suite +var testSuite = TestDiscovery.loadAllTests(__dirname + "/languages"); + +// define tests for all tests in all languages in the test suite +for (var language in testSuite) +{ + if (!testSuite.hasOwnProperty(language)) { + continue; + } + + (function (language, testFiles) { + describe("Testing language '" + language + "'", function() { + testFiles.forEach( + function (filePath) { + var fileName = path.basename(filePath); + + it("Should pass test case '" + fileName + "'", + function () { + TestCase.runTestCase(language, filePath); + } + ); + } + ); + }); + })(language, testSuite[language]); +} From a806d1cc55f1d22d5a964c241bca40a74923763c Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Mon, 1 Jun 2015 20:39:28 +0200 Subject: [PATCH 03/23] Added first simple test case --- tests/languages/javascript/test.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/languages/javascript/test.js b/tests/languages/javascript/test.js index e69de29bb2..df16f2e405 100644 --- a/tests/languages/javascript/test.js +++ b/tests/languages/javascript/test.js @@ -0,0 +1,10 @@ +var a = 5; + +---------------------------------------------------- + +[ + ["keyword", "var"], + ["operator", "="], + ["number", "5"], + ["punctuation", ";"] +] From 8c64e4779203ef3f9cce8adb8d3afd896ca364e1 Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Mon, 1 Jun 2015 21:12:37 +0200 Subject: [PATCH 04/23] Fixed internal test names --- tests/run.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run.js b/tests/run.js index 50bd20f137..32dab49fb5 100644 --- a/tests/run.js +++ b/tests/run.js @@ -20,7 +20,7 @@ for (var language in testSuite) function (filePath) { var fileName = path.basename(filePath); - it("Should pass test case '" + fileName + "'", + it("– should pass test case '" + fileName + "'", function () { TestCase.runTestCase(language, filePath); } From be5e037b171be70be362574ebfffe4f0499bb4c5 Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Sat, 13 Jun 2015 15:12:21 +0200 Subject: [PATCH 05/23] Added support for test case comments --- tests/helper/test-case.js | 59 ++++++++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/tests/helper/test-case.js b/tests/helper/test-case.js index 6afce39ae3..9bec510323 100644 --- a/tests/helper/test-case.js +++ b/tests/helper/test-case.js @@ -1,9 +1,38 @@ "use strict"; var fs = require("fs"); -var expect = require("chai").expect; +var assert = require("chai").assert; var PrismLoader = require("./prism-loader"); +/** + * Handles parsing of a test case file. + * + * + * A test case file consists of at least two parts, separated by a line of dashes. + * This separation line must start at the beginning of the line and consist of at least three dashes. + * + * The test case file can either consist of two parts: + * + * {source code} + * ---- + * {expected token stream} + * + * + * or of three parts: + * + * {source code} + * ---- + * {expected token stream} + * ---- + * {text comment explaining the test case} + * + * If the file contains more than three parts, the remaining parts are just ignored. + * If the file however does not contain at least two parts (so no expected token stream), + * the test case will later be marked as failed. + * + * + * @type {{runTestCase: Function, transformCompiledTokenStream: Function, parseTestCaseFile: Function}} + */ module.exports = { /** @@ -23,7 +52,7 @@ module.exports = { var compiledTokenStream = Prism.tokenize(testCase.testSource, Prism.languages[language]); var simplifiedTokenStream = this.transformCompiledTokenStream(compiledTokenStream); - expect(simplifiedTokenStream).to.eql(testCase.expectedTokenStream); + assert.deepEqual(simplifiedTokenStream, testCase.expectedTokenStream, testCase.comment); }, @@ -53,19 +82,29 @@ module.exports = { * * @private * @param {string} filePath - * @returns {{testSource: string, expectedTokenStream: *}|null} + * @returns {{testSource: string, expectedTokenStream: Array.>, comment:string?}|null} */ parseTestCaseFile: function (filePath) { var testCaseSource = fs.readFileSync(filePath, "utf8"); - var testCase = testCaseSource.split(/^----*\w*$/m); + var testCaseParts = testCaseSource.split(/^----*\w*$/m); - if (2 === testCase.length) { - return { - testSource: testCase[0].trim(), - expectedTokenStream: JSON.parse(testCase[1]) - }; + // No expected token stream found + if (2 > testCaseParts.length) { + return null; } - return null; + var testCase = { + testSource: testCaseParts[0].trim(), + expectedTokenStream: JSON.parse(testCaseParts[1]), + comment: null + }; + + // if there are three parts, the third one is the comment + // explaining the test case + if (testCaseParts[2]) { + testCase.comment = testCaseParts[2].trim(); + } + + return testCase; } }; From 829f8a860dbd6426213c3e0c3cd2afb22d3591d1 Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Sat, 13 Jun 2015 15:24:53 +0200 Subject: [PATCH 06/23] New file extension for test case files: .test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## All test case files are now required to have the ".test" extension. This prevents issues with syntax highlighting in common IDEs and also prevents the test system to choke on non-testcase files in these directories (like `.DS_Store` or `Thumbs.db`). ## Hide the ".test" extension in the description of the test cases The message for a testcase `blabla.test` will now just read: > 1) Testing language 'css' – should pass test case 'blabla': --- tests/helper/test-discovery.js | 4 +++- tests/run.js | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/helper/test-discovery.js b/tests/helper/test-discovery.js index 9f9ea2dca4..7cbaef8193 100644 --- a/tests/helper/test-discovery.js +++ b/tests/helper/test-discovery.js @@ -52,7 +52,9 @@ module.exports = { getAllFiles: function (src) { return fs.readdirSync(src).filter( function (fileName) { - return fs.statSync(path.join(src, fileName)).isFile(); + // only find files that have the ".test" extension + return ".test" === path.extname(fileName) && + fs.statSync(path.join(src, fileName)).isFile(); } ).map( function (fileName) { diff --git a/tests/run.js b/tests/run.js index 32dab49fb5..0f0475fa6e 100644 --- a/tests/run.js +++ b/tests/run.js @@ -18,7 +18,7 @@ for (var language in testSuite) describe("Testing language '" + language + "'", function() { testFiles.forEach( function (filePath) { - var fileName = path.basename(filePath); + var fileName = path.basename(filePath, path.extname(filePath)); it("– should pass test case '" + fileName + "'", function () { From dec517f38299413bd66f82cb8f64636e46737433 Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Sat, 13 Jun 2015 15:25:53 +0200 Subject: [PATCH 07/23] Refreshed example test case for new file extension and comment support. --- tests/languages/javascript/{test.js => testcase1.test} | 4 ++++ 1 file changed, 4 insertions(+) rename tests/languages/javascript/{test.js => testcase1.test} (60%) diff --git a/tests/languages/javascript/test.js b/tests/languages/javascript/testcase1.test similarity index 60% rename from tests/languages/javascript/test.js rename to tests/languages/javascript/testcase1.test index df16f2e405..e6cd4a59aa 100644 --- a/tests/languages/javascript/test.js +++ b/tests/languages/javascript/testcase1.test @@ -8,3 +8,7 @@ var a = 5; ["number", "5"], ["punctuation", ";"] ] + +---------------------------------------------------- + +This is a comment explaining this test case. \ No newline at end of file From 09898e8e77a38f812e080c3f1ffbdd28182bd14f Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Sat, 13 Jun 2015 16:37:07 +0200 Subject: [PATCH 08/23] Support language inclusion tests By using composed language names "language+language2+language3" you can test language inclusion or do integration tests. --- tests/helper/prism-loader.js | 36 +++++++++++++++++++++++++----------- tests/helper/test-case.js | 20 ++++++++++++++++---- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/tests/helper/prism-loader.js b/tests/helper/prism-loader.js index 1f1426ac92..df34951be0 100644 --- a/tests/helper/prism-loader.js +++ b/tests/helper/prism-loader.js @@ -11,12 +11,21 @@ module.exports = { /** * Creates a new Prism instance with the given language loaded * - * @param {string} language + * @param {string|string[]} languages * @returns {Prism} */ - createInstance: function (language) { - var Prism = this.createEmptyPrism(); - return this.loadLanguage(language, Prism); + createInstance: function (languages) { + var context = { + loadedLanguages: [], + Prism: this.createEmptyPrism() + }; + languages = Array.isArray(languages) ? languages : [languages]; + + for (var i = 0, l = languages.length; i < l; i++) { + context = this.loadLanguage(languages[i], context); + } + + return context.Prism; }, @@ -26,26 +35,31 @@ module.exports = { * * @private * @param {string} language - * @param {Prism} Prism - * @returns {Prism} + * @param {{loadedLanguages: string[], Prism: Prism}} context */ - loadLanguage: function (language, Prism) { + loadLanguage: function (language, context) { if (!languagesCatalog[language]) { throw new Error("Language '" + language + "' not found."); } + // the given language was already loaded + if (-1 < context.loadedLanguages.indexOf(language)) { + return context; + } + // if the language has a dependency -> load it first if (languagesCatalog[language].require) { - Prism = this.loadLanguage(languagesCatalog[language].require, Prism); + context = this.loadLanguage(languagesCatalog[language].require, context); } // load the language itself var languageSource = this.loadFileSource(language); - var context = this.runFileWithContext(languageSource, {Prism: Prism}); + context.Prism = this.runFileWithContext(languageSource, {Prism: context.Prism}).Prism; + context.loadedLanguages.push(language); - return context.Prism; + return context; }, @@ -88,7 +102,7 @@ module.exports = { * * @private * @param {string} fileSource - * @param {*}context + * @param {*} [context] * * @returns {*} */ diff --git a/tests/helper/test-case.js b/tests/helper/test-case.js index 9bec510323..095cba0eda 100644 --- a/tests/helper/test-case.js +++ b/tests/helper/test-case.js @@ -38,18 +38,30 @@ module.exports = { /** * Runs the given test case file and asserts the result * - * @param {string} language + * The passed language identifier can either be a language like "css" or a composed language + * identifier like "css+markup". Composed identifiers can be used for testing language inclusion. + * + * When testing language inclusion, the first given language is the main language which will be passed + * to Prism for highlighting ("css+markup" will result in a call to Prism to highlight with the "css" grammar). + * But it will be ensured, that the additional passed languages will be loaded too. + * + * The languages will be loaded in the order they were provided. + * + * @param {string} languageIdentifier * @param {string} filePath */ - runTestCase: function (language, filePath) { + runTestCase: function (languageIdentifier, filePath) { var testCase = this.parseTestCaseFile(filePath); + var languages = languageIdentifier.split("+"); if (null === testCase) { throw new Error("Test case file has invalid format, please read the docs."); } - var Prism = PrismLoader.createInstance(language); - var compiledTokenStream = Prism.tokenize(testCase.testSource, Prism.languages[language]); + var Prism = PrismLoader.createInstance(languages); + // the first language is the main language to highlight + var mainLanguageGrammar = Prism.languages[languages[0]]; + var compiledTokenStream = Prism.tokenize(testCase.testSource, mainLanguageGrammar); var simplifiedTokenStream = this.transformCompiledTokenStream(compiledTokenStream); assert.deepEqual(simplifiedTokenStream, testCase.expectedTokenStream, testCase.comment); From aec5fcc4a2f8d9e60fe9c9172c5b26a495c9893b Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Sat, 13 Jun 2015 16:41:31 +0200 Subject: [PATCH 09/23] Added return type definition --- tests/helper/prism-loader.js | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/helper/prism-loader.js b/tests/helper/prism-loader.js index df34951be0..87739fd2e9 100644 --- a/tests/helper/prism-loader.js +++ b/tests/helper/prism-loader.js @@ -36,6 +36,7 @@ module.exports = { * @private * @param {string} language * @param {{loadedLanguages: string[], Prism: Prism}} context + * @returns {{loadedLanguages: string[], Prism: Prism}} */ loadLanguage: function (language, context) { if (!languagesCatalog[language]) From 9619d4cb37dda40f1553c156a7b2658f1c1117e1 Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Tue, 14 Jul 2015 10:46:57 +0200 Subject: [PATCH 10/23] Implement support for simplifying nested token streams --- tests/helper/test-case.js | 39 ++++++++++++++++++++------- tests/languages/apacheconf/test2.test | 6 +++++ 2 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 tests/languages/apacheconf/test2.test diff --git a/tests/helper/test-case.js b/tests/helper/test-case.js index 095cba0eda..92a9951508 100644 --- a/tests/helper/test-case.js +++ b/tests/helper/test-case.js @@ -55,7 +55,7 @@ module.exports = { var languages = languageIdentifier.split("+"); if (null === testCase) { - throw new Error("Test case file has invalid format, please read the docs."); + throw new Error("Test case file has invalid format (or the provided token stream is invalid JSON), please read the docs."); } var Prism = PrismLoader.createInstance(languages); @@ -72,20 +72,41 @@ module.exports = { * Simplifies the token stream to ease the matching with the expected token stream * * @param {string} tokenStream - * @returns {Array.} + * @returns {Array.} */ transformCompiledTokenStream: function (tokenStream) { + // First filter all top-level non-objects as non-objects are not-identified tokens + // + // we don't want to filter them in the lower levels as we want to support nested content-structures return tokenStream.filter( function (token) { - // only support objects return (typeof token === "object"); } - ).map( - function (entry) - { - return [entry.type, entry.content]; - } - ); + ).map(this.transformCompiledRecursivelyTokenStream.bind(this)); + }, + + + /** + * Walks the token stream and recursively simplifies it + * + * @private + * @param {Array|{type: string, content: *}|string} token + * @returns {Array|string} + */ + transformCompiledRecursivelyTokenStream: function (token) + { + if (Array.isArray(token)) + { + return token.map(this.transformCompiledRecursivelyTokenStream.bind(this)); + } + else if (typeof token === "object") + { + return [token.type, this.transformCompiledRecursivelyTokenStream(token.content)]; + } + else + { + return token; + } }, diff --git a/tests/languages/apacheconf/test2.test b/tests/languages/apacheconf/test2.test new file mode 100644 index 0000000000..fbc0728b0b --- /dev/null +++ b/tests/languages/apacheconf/test2.test @@ -0,0 +1,6 @@ +"foo bar" +'foo bar' +"%{REMOTE_HOST}" + +----- +[] From 9ce5838a4f8812462d2f664ee7b68d32c01ebe1b Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Tue, 14 Jul 2015 10:51:35 +0200 Subject: [PATCH 11/23] Catch JSON parse error to provide a unified error reporting in the test runner --- tests/helper/test-case.js | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/tests/helper/test-case.js b/tests/helper/test-case.js index 92a9951508..240599e06b 100644 --- a/tests/helper/test-case.js +++ b/tests/helper/test-case.js @@ -121,23 +121,25 @@ module.exports = { var testCaseSource = fs.readFileSync(filePath, "utf8"); var testCaseParts = testCaseSource.split(/^----*\w*$/m); - // No expected token stream found - if (2 > testCaseParts.length) { - return null; - } - - var testCase = { - testSource: testCaseParts[0].trim(), - expectedTokenStream: JSON.parse(testCaseParts[1]), - comment: null - }; + try { + var testCase = { + testSource: testCaseParts[0].trim(), + expectedTokenStream: JSON.parse(testCaseParts[1]), + comment: null + }; + + // if there are three parts, the third one is the comment + // explaining the test case + if (testCaseParts[2]) { + testCase.comment = testCaseParts[2].trim(); + } - // if there are three parts, the third one is the comment - // explaining the test case - if (testCaseParts[2]) { - testCase.comment = testCaseParts[2].trim(); + return testCase; + } + catch (e) + { + // the JSON can't be parsed (e.g. it could be empty) + return null; } - - return testCase; } }; From ea2349349b677b5f07822bbf1161533348aeb486 Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Tue, 14 Jul 2015 10:51:56 +0200 Subject: [PATCH 12/23] Fully defined Apacheconf test case --- tests/languages/apacheconf/test2.test | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/languages/apacheconf/test2.test b/tests/languages/apacheconf/test2.test index fbc0728b0b..a136917c03 100644 --- a/tests/languages/apacheconf/test2.test +++ b/tests/languages/apacheconf/test2.test @@ -3,4 +3,14 @@ "%{REMOTE_HOST}" ----- -[] +[ + ["string", ["\"foo bar\""]], + ["string", ["'foo bar'"]], + [ + "string", [ + "\"", + ["variable", "%{REMOTE_HOST}"], + "\"" + ] + ] +] From 5c2f9e0a500661473fc9ca068b90e3afa2eb5c77 Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Wed, 15 Jul 2015 19:29:27 +0200 Subject: [PATCH 13/23] Use consistent quote style --- tests/helper/components.js | 2 +- tests/helper/prism-loader.js | 2 +- tests/helper/test-discovery.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/helper/components.js b/tests/helper/components.js index 274ad146f7..09dd6aa957 100644 --- a/tests/helper/components.js +++ b/tests/helper/components.js @@ -1,7 +1,7 @@ "use strict"; var fs = require("fs"); -var vm = require('vm'); +var vm = require("vm"); var fileContent = fs.readFileSync(__dirname + "/../../components.js", "utf8"); var context = {}; diff --git a/tests/helper/prism-loader.js b/tests/helper/prism-loader.js index 87739fd2e9..ea82c0bc47 100644 --- a/tests/helper/prism-loader.js +++ b/tests/helper/prism-loader.js @@ -1,7 +1,7 @@ "use strict"; var fs = require("fs"); -var vm = require('vm'); +var vm = require("vm"); var components = require("./components"); var languagesCatalog = components.languages; diff --git a/tests/helper/test-discovery.js b/tests/helper/test-discovery.js index 7cbaef8193..741dc7591b 100644 --- a/tests/helper/test-discovery.js +++ b/tests/helper/test-discovery.js @@ -1,7 +1,7 @@ "use strict"; var fs = require("fs"); -var path = require('path'); +var path = require("path"); module.exports = { From 3a3cd26838a2c8181c050dfc437d24d8cd4e153c Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Wed, 15 Jul 2015 19:35:07 +0200 Subject: [PATCH 14/23] Extract simplification of token stream Pull the token stream transformation out of the test case into its own component. --- tests/helper/test-case.js | 44 ++--------------------- tests/helper/token-stream-transformer.js | 45 ++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 42 deletions(-) create mode 100644 tests/helper/token-stream-transformer.js diff --git a/tests/helper/test-case.js b/tests/helper/test-case.js index 240599e06b..b62d230da9 100644 --- a/tests/helper/test-case.js +++ b/tests/helper/test-case.js @@ -3,6 +3,7 @@ var fs = require("fs"); var assert = require("chai").assert; var PrismLoader = require("./prism-loader"); +var TokenStreamTransformer = require("./token-stream-transformer"); /** * Handles parsing of a test case file. @@ -62,53 +63,12 @@ module.exports = { // the first language is the main language to highlight var mainLanguageGrammar = Prism.languages[languages[0]]; var compiledTokenStream = Prism.tokenize(testCase.testSource, mainLanguageGrammar); - var simplifiedTokenStream = this.transformCompiledTokenStream(compiledTokenStream); + var simplifiedTokenStream = TokenStreamTransformer.simplify(compiledTokenStream); assert.deepEqual(simplifiedTokenStream, testCase.expectedTokenStream, testCase.comment); }, - /** - * Simplifies the token stream to ease the matching with the expected token stream - * - * @param {string} tokenStream - * @returns {Array.} - */ - transformCompiledTokenStream: function (tokenStream) { - // First filter all top-level non-objects as non-objects are not-identified tokens - // - // we don't want to filter them in the lower levels as we want to support nested content-structures - return tokenStream.filter( - function (token) { - return (typeof token === "object"); - } - ).map(this.transformCompiledRecursivelyTokenStream.bind(this)); - }, - - - /** - * Walks the token stream and recursively simplifies it - * - * @private - * @param {Array|{type: string, content: *}|string} token - * @returns {Array|string} - */ - transformCompiledRecursivelyTokenStream: function (token) - { - if (Array.isArray(token)) - { - return token.map(this.transformCompiledRecursivelyTokenStream.bind(this)); - } - else if (typeof token === "object") - { - return [token.type, this.transformCompiledRecursivelyTokenStream(token.content)]; - } - else - { - return token; - } - }, - /** * Parses the test case from the given test case file diff --git a/tests/helper/token-stream-transformer.js b/tests/helper/token-stream-transformer.js new file mode 100644 index 0000000000..fdc635bf5d --- /dev/null +++ b/tests/helper/token-stream-transformer.js @@ -0,0 +1,45 @@ +"use strict"; + + +module.exports = { + /** + * Simplifies the token stream to ease the matching with the expected token stream + * + * @param {string} tokenStream + * @returns {Array.} + */ + simplify: function (tokenStream) { + // First filter all top-level non-objects as non-objects are not-identified tokens + // + // we don't want to filter them in the lower levels as we want to support nested content-structures + return tokenStream.filter( + function (token) { + return (typeof token === "object"); + } + ).map(this.simplifyRecursively.bind(this)); + }, + + + /** + * Walks the token stream and recursively simplifies it + * + * @private + * @param {Array|{type: string, content: *}|string} token + * @returns {Array|string} + */ + simplifyRecursively: function (token) + { + if (Array.isArray(token)) + { + return token.map(this.simplifyRecursively.bind(this)); + } + else if (typeof token === "object") + { + return [token.type, this.simplifyRecursively(token.content)]; + } + else + { + return token; + } + } +}; From f05e0797648e816d38c86edaead9095c71cfdbf7 Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Wed, 15 Jul 2015 20:01:39 +0200 Subject: [PATCH 15/23] Updated simplification of token stream It now strips empty values --- tests/helper/token-stream-transformer.js | 45 ++++++++++-------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/tests/helper/token-stream-transformer.js b/tests/helper/token-stream-transformer.js index fdc635bf5d..38aa77ca24 100644 --- a/tests/helper/token-stream-transformer.js +++ b/tests/helper/token-stream-transformer.js @@ -3,43 +3,34 @@ module.exports = { /** - * Simplifies the token stream to ease the matching with the expected token stream + * Simplifies the token stream to ease the matching with the expected token stream. * - * @param {string} tokenStream + * * Strings are kept as-is + * * In arrays each value is transformed individually + * * Values that are empty (empty arrays or strings only containing whitespace) + * + * + * @param {Array} tokenStream * @returns {Array.} */ simplify: function (tokenStream) { - // First filter all top-level non-objects as non-objects are not-identified tokens - // - // we don't want to filter them in the lower levels as we want to support nested content-structures - return tokenStream.filter( - function (token) { - return (typeof token === "object"); - } - ).map(this.simplifyRecursively.bind(this)); - }, - - - /** - * Walks the token stream and recursively simplifies it - * - * @private - * @param {Array|{type: string, content: *}|string} token - * @returns {Array|string} - */ - simplifyRecursively: function (token) - { - if (Array.isArray(token)) + if (Array.isArray(tokenStream)) { - return token.map(this.simplifyRecursively.bind(this)); + return tokenStream + .map(this.simplify.bind(this)) + .filter(function(value) + { + return !(Array.isArray(value) && !value.length) && !(typeof value === "string" && !value.trim().length); + } + ); } - else if (typeof token === "object") + else if (typeof tokenStream === "object") { - return [token.type, this.simplifyRecursively(token.content)]; + return [tokenStream.type, this.simplify(tokenStream.content)]; } else { - return token; + return tokenStream; } } }; From 8bdf4c87d899692e6ce4422309531ecae25cc9a4 Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Wed, 15 Jul 2015 20:02:49 +0200 Subject: [PATCH 16/23] Test the test runner itself We are at a point where we probably should test the test runner (especially the token stream transformer) itself. --- package.json | 2 +- tests/testrunner-tests.js | 91 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 tests/testrunner-tests.js diff --git a/package.json b/package.json index e254ad770a..2f8af95a5f 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "Lightweight, robust, elegant syntax highlighting. A spin-off project from Dabblet.", "main": "prism.js", "scripts": { - "test": "mocha tests/run.js" + "test": "mocha tests/testrunner-tests.js && mocha tests/run.js" }, "repository": { "type": "git", diff --git a/tests/testrunner-tests.js b/tests/testrunner-tests.js new file mode 100644 index 0000000000..f414e8b808 --- /dev/null +++ b/tests/testrunner-tests.js @@ -0,0 +1,91 @@ +"use strict"; + +var assert = require("chai").assert; +var TokenStreamTransformer = require("./helper/token-stream-transformer"); + +describe("The token stream transformer", + function () + { + it("should handle all kinds of simple transformations", + function () + { + var tokens = [ + {type: "type", content: "content"}, + "string" + ]; + + var expected = [ + ["type", "content"], + "string" + ]; + + assert.deepEqual(TokenStreamTransformer.simplify(tokens), expected); + } + ); + + + it("should handle nested structures", + function () + { + var tokens = [ + {type: "type", content: [ + {type: "insideType", content: [ + {type: "insideInsideType", content: "content"} + ]} + ]} + ]; + + var expected = [ + ["type", [ + ["insideType", [ + ["insideInsideType", "content"] + ]] + ]] + ]; + + assert.deepEqual(TokenStreamTransformer.simplify(tokens), expected); + } + ); + + + it("should strip empty tokens", + function () + { + var tokenStream = [ + "", + "\r\n", + "\t", + " " + ]; + + var expectedSimplified = []; + + assert.deepEqual(TokenStreamTransformer.simplify(tokenStream), expectedSimplified); + } + ); + + + it("should strip empty token tree branches", + function () + { + var tokenStream = [ + {type: "type", content: [ + ["", ""], + "", + {type: "nested", content: [""]} + ]}, + [[[[[[[""]]]]]]] + ]; + + var expectedSimplified = [ + ["type", [ + ["nested", []] + ]] + ]; + + assert.deepEqual(TokenStreamTransformer.simplify(tokenStream), expectedSimplified); + } + ); + } +); + From b8d92aa75ca1560aba65e0d08bf672c4baf2cf4b Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Wed, 15 Jul 2015 20:06:21 +0200 Subject: [PATCH 17/23] Verify that the token stream transfomer ignores all token properties except type and content --- tests/testrunner-tests.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/testrunner-tests.js b/tests/testrunner-tests.js index f414e8b808..b9f79dc88e 100644 --- a/tests/testrunner-tests.js +++ b/tests/testrunner-tests.js @@ -86,6 +86,23 @@ describe("The token stream transformer", assert.deepEqual(TokenStreamTransformer.simplify(tokenStream), expectedSimplified); } ); + + + it("should ignore all properties in tokens except value and content", + function () + { + + var tokenStream = [ + {type: "type", content: "content", alias: "alias"} + ]; + + var expectedSimplified = [ + ["type", "content"] + ]; + + assert.deepEqual(TokenStreamTransformer.simplify(tokenStream), expectedSimplified); + } + ); } ); From 1e0b8d9da944949e52bd7f2c5d38406e2d534f34 Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Sun, 26 Jul 2015 14:08:47 +0200 Subject: [PATCH 18/23] Fixed code style issues --- tests/helper/prism-loader.js | 6 +-- tests/helper/test-case.js | 4 +- tests/helper/token-stream-transformer.js | 12 ++---- tests/run.js | 5 +-- tests/testrunner-tests.js | 47 +++++++++++++----------- 5 files changed, 34 insertions(+), 40 deletions(-) diff --git a/tests/helper/prism-loader.js b/tests/helper/prism-loader.js index ea82c0bc47..9bc515b425 100644 --- a/tests/helper/prism-loader.js +++ b/tests/helper/prism-loader.js @@ -39,8 +39,7 @@ module.exports = { * @returns {{loadedLanguages: string[], Prism: Prism}} */ loadLanguage: function (language, context) { - if (!languagesCatalog[language]) - { + if (!languagesCatalog[language]) { throw new Error("Language '" + language + "' not found."); } @@ -50,8 +49,7 @@ module.exports = { } // if the language has a dependency -> load it first - if (languagesCatalog[language].require) - { + if (languagesCatalog[language].require) { context = this.loadLanguage(languagesCatalog[language].require, context); } diff --git a/tests/helper/test-case.js b/tests/helper/test-case.js index b62d230da9..ec9ae06d25 100644 --- a/tests/helper/test-case.js +++ b/tests/helper/test-case.js @@ -69,7 +69,6 @@ module.exports = { }, - /** * Parses the test case from the given test case file * @@ -96,8 +95,7 @@ module.exports = { return testCase; } - catch (e) - { + catch (e) { // the JSON can't be parsed (e.g. it could be empty) return null; } diff --git a/tests/helper/token-stream-transformer.js b/tests/helper/token-stream-transformer.js index 38aa77ca24..deb831c45b 100644 --- a/tests/helper/token-stream-transformer.js +++ b/tests/helper/token-stream-transformer.js @@ -14,22 +14,18 @@ module.exports = { * @returns {Array.} */ simplify: function (tokenStream) { - if (Array.isArray(tokenStream)) - { + if (Array.isArray(tokenStream)) { return tokenStream .map(this.simplify.bind(this)) - .filter(function(value) - { + .filter(function (value) { return !(Array.isArray(value) && !value.length) && !(typeof value === "string" && !value.trim().length); } ); } - else if (typeof tokenStream === "object") - { + else if (typeof tokenStream === "object") { return [tokenStream.type, this.simplify(tokenStream.content)]; } - else - { + else { return tokenStream; } } diff --git a/tests/run.js b/tests/run.js index 0f0475fa6e..5e0050487f 100644 --- a/tests/run.js +++ b/tests/run.js @@ -8,14 +8,13 @@ var path = require("path"); var testSuite = TestDiscovery.loadAllTests(__dirname + "/languages"); // define tests for all tests in all languages in the test suite -for (var language in testSuite) -{ +for (var language in testSuite) { if (!testSuite.hasOwnProperty(language)) { continue; } (function (language, testFiles) { - describe("Testing language '" + language + "'", function() { + describe("Testing language '" + language + "'", function () { testFiles.forEach( function (filePath) { var fileName = path.basename(filePath, path.extname(filePath)); diff --git a/tests/testrunner-tests.js b/tests/testrunner-tests.js index b9f79dc88e..23731078e4 100644 --- a/tests/testrunner-tests.js +++ b/tests/testrunner-tests.js @@ -4,11 +4,9 @@ var assert = require("chai").assert; var TokenStreamTransformer = require("./helper/token-stream-transformer"); describe("The token stream transformer", - function () - { + function () { it("should handle all kinds of simple transformations", - function () - { + function () { var tokens = [ {type: "type", content: "content"}, "string" @@ -25,14 +23,19 @@ describe("The token stream transformer", it("should handle nested structures", - function () - { + function () { var tokens = [ - {type: "type", content: [ - {type: "insideType", content: [ - {type: "insideInsideType", content: "content"} - ]} - ]} + { + type: "type", + content: [ + { + type: "insideType", content: + [ + {type: "insideInsideType", content: "content"} + ] + } + ] + } ]; var expected = [ @@ -49,8 +52,7 @@ describe("The token stream transformer", it("should strip empty tokens", - function () - { + function () { var tokenStream = [ "", "\r\n", @@ -66,14 +68,16 @@ describe("The token stream transformer", it("should strip empty token tree branches", - function () - { + function () { var tokenStream = [ - {type: "type", content: [ - ["", ""], - "", - {type: "nested", content: [""]} - ]}, + { + type: "type", + content: [ + ["", ""], + "", + {type: "nested", content: [""]} + ] + }, [[[[[[[""]]]]]]] ]; @@ -89,8 +93,7 @@ describe("The token stream transformer", it("should ignore all properties in tokens except value and content", - function () - { + function () { var tokenStream = [ {type: "type", content: "content", alias: "alias"} From 799570f9cbff3b68ef8a8873977d9a78d6a584ed Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Sun, 26 Jul 2015 14:25:30 +0200 Subject: [PATCH 19/23] Fixed javascript test case --- tests/languages/javascript/testcase1.test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/languages/javascript/testcase1.test b/tests/languages/javascript/testcase1.test index e6cd4a59aa..78a197175a 100644 --- a/tests/languages/javascript/testcase1.test +++ b/tests/languages/javascript/testcase1.test @@ -4,6 +4,7 @@ var a = 5; [ ["keyword", "var"], + " a ", ["operator", "="], ["number", "5"], ["punctuation", ";"] @@ -11,4 +12,4 @@ var a = 5; ---------------------------------------------------- -This is a comment explaining this test case. \ No newline at end of file +This is a comment explaining this test case. From a13c87b9e3712233472814d536ffd45bec381c77 Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Sun, 26 Jul 2015 14:27:38 +0200 Subject: [PATCH 20/23] Added support for specifying the main language to load in test cases You can add an exclamation mark anywhere in the name to load it as main language. If you do not specify anything, the first entry is used as main language css+markup! --> markup is main language css+markup --> css is main language --- tests/helper/test-case.js | 48 +++++++++++++++++++++++++++++++--- tests/testrunner-tests.js | 55 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/tests/helper/test-case.js b/tests/helper/test-case.js index ec9ae06d25..34835a3326 100644 --- a/tests/helper/test-case.js +++ b/tests/helper/test-case.js @@ -53,15 +53,15 @@ module.exports = { */ runTestCase: function (languageIdentifier, filePath) { var testCase = this.parseTestCaseFile(filePath); - var languages = languageIdentifier.split("+"); + var usedLanguages = this.parseLanguageNames(languageIdentifier); if (null === testCase) { throw new Error("Test case file has invalid format (or the provided token stream is invalid JSON), please read the docs."); } - var Prism = PrismLoader.createInstance(languages); + var Prism = PrismLoader.createInstance(usedLanguages.languages); // the first language is the main language to highlight - var mainLanguageGrammar = Prism.languages[languages[0]]; + var mainLanguageGrammar = Prism.languages[usedLanguages.mainLanguage]; var compiledTokenStream = Prism.tokenize(testCase.testSource, mainLanguageGrammar); var simplifiedTokenStream = TokenStreamTransformer.simplify(compiledTokenStream); @@ -69,6 +69,48 @@ module.exports = { }, + /** + * Parses the language names and finds the main language. + * + * It is either the first language or the language followed by a exclamation mark “!”. + * There should only be one language with an exclamation mark. + * + * @param {string} languageIdentifier + * + * @returns {{languages: string[], mainLanguage: string}} + */ + parseLanguageNames: function (languageIdentifier) { + var languages = languageIdentifier.split("+"); + var mainLanguage = null; + + languages = languages.map( + function (language) { + var pos = language.indexOf("!"); + + if (-1 < pos) { + if (mainLanguage) { + throw "There are multiple main languages defined."; + } + + mainLanguage = language.replace("!", ""); + return mainLanguage; + } + + return language; + } + ); + + if (!mainLanguage) { + mainLanguage = languages[0]; + } + + return { + languages: languages, + mainLanguage: mainLanguage + }; + }, + + /** * Parses the test case from the given test case file * diff --git a/tests/testrunner-tests.js b/tests/testrunner-tests.js index 23731078e4..c4e0fd51f0 100644 --- a/tests/testrunner-tests.js +++ b/tests/testrunner-tests.js @@ -2,7 +2,10 @@ var assert = require("chai").assert; var TokenStreamTransformer = require("./helper/token-stream-transformer"); +var TestCase = require("./helper/test-case"); + +//region Token Stream Transformer describe("The token stream transformer", function () { it("should handle all kinds of simple transformations", @@ -108,4 +111,56 @@ describe("The token stream transformer", ); } ); +//endregion + + +//region Language name parsing +describe("The language name parsing", + function () { + it("should use the first language as the main language if no language is specified", + function () { + assert.deepEqual( + TestCase.parseLanguageNames("a"), + { + languages: ["a"], + mainLanguage: "a" + } + ); + + assert.deepEqual( + TestCase.parseLanguageNames("a+b+c"), + { + languages: ["a", "b", "c"], + mainLanguage: "a" + } + ); + } + ); + + it("should use the specified language as main language", + function () { + assert.deepEqual( + TestCase.parseLanguageNames("a+b!+c"), + { + languages: ["a", "b", "c"], + mainLanguage: "b" + } + ); + } + ); + + + it("should throw an error if there are multiple main languages", + function () { + assert.throw( + function () { + TestCase.parseLanguageNames("a+b!+c!"); + }, + "There are multiple main languages defined." + ); + } + ); + } +); +//endregion From 9e6703ada54ec98b6c90c1127c6c4104066e5ba0 Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Sun, 26 Jul 2015 14:32:00 +0200 Subject: [PATCH 21/23] Added travis.yml to run tests in travis --- .travis.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..68fbf6d87c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,11 @@ +language: node_js + +node_js: + - "0.10" + - "0.12" + +before_script: + - npm install -g gulp + - gulp + +script: npm test From 2bc551851c753b14a5cc5cd9293d4474c6656b3f Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Sun, 26 Jul 2015 14:34:41 +0200 Subject: [PATCH 22/23] Improve support of node 10.x --- tests/helper/components.js | 3 +-- tests/helper/prism-loader.js | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/helper/components.js b/tests/helper/components.js index 09dd6aa957..68cb7abe3a 100644 --- a/tests/helper/components.js +++ b/tests/helper/components.js @@ -5,7 +5,6 @@ var vm = require("vm"); var fileContent = fs.readFileSync(__dirname + "/../../components.js", "utf8"); var context = {}; -vm.createContext(context); -vm.runInContext(fileContent, context); +vm.runInNewContext(fileContent, context); module.exports = context.components; diff --git a/tests/helper/prism-loader.js b/tests/helper/prism-loader.js index 9bc515b425..31f3689d63 100644 --- a/tests/helper/prism-loader.js +++ b/tests/helper/prism-loader.js @@ -107,8 +107,7 @@ module.exports = { */ runFileWithContext: function (fileSource, context) { context = context || {}; - vm.createContext(context); - vm.runInContext(fileSource, context); + vm.runInNewContext(fileSource, context); return context; } }; From e8884a96ed6aa5fb53710078afe3aea68176ec44 Mon Sep 17 00:00:00 2001 From: Jannik Zschiesche Date: Sun, 26 Jul 2015 15:28:39 +0200 Subject: [PATCH 23/23] Added documentation about the test runner and test suite --- test-suite.html | 147 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 test-suite.html diff --git a/test-suite.html b/test-suite.html new file mode 100644 index 0000000000..accd149d92 --- /dev/null +++ b/test-suite.html @@ -0,0 +1,147 @@ + + + + + + +Running the test suite ▲ Prism + + + + + + + + + +
+
+ +

Running the test suite

+

Prism has a test suite, that ensures that the correct tokens are matched.

+
+ +
+

Running the test suite

+ +

Running the test suite is simple: just call npm test.

+

All test files are run in isolation. A new prism instance is created for each test case. This will slow the test runner a bit down, but we can be sure that nothing leaks into the next test case.

+
+ +
+

Writing tests

+ +

Thank you for writing tests! Tests are awesome! They ensure, that we can improve the codebase without breaking anything. Also, this way, we can ensure that upgrading Prism is as painless as possible for you.

+

You can add new tests by creating a new test case file (with the .test file extension) in the tests directory which is located at /tests/languages/${language}.

+ +
+

Language directories

+

All tests are sorted into directories in the tests/languages directory. Each directory name encodes, which language you are currently testing.

+

All language names must match the names from the definition in components.js.

+ +

Example 1: testing a language in isolation (default use case)

+

Just put your test file into the directory of the language you want to test.

+

So, if you want to test CSS, put your test file in /tests/languages/css to test CSS only. If you create a test case in this directory, the test runner will ensure that the css language definition including all required language definitions are correctly loaded.

+ +

Example 2: testing language injection

+

If you want to test language injection, you typically need to load two or more languages where one language is the “main” language that is being tested, with all other languages being injected into it.

+

You need to define multiple languages by separating them using a + sign: markup+php.

+

The languages are loaded in order, so first markup (+ dependencies) is loaded, then php (+ dependencies). The test loader ensures that no language is loaded more than once (for example if two languages have the same dependencies).

+

By default the first language is the main language: markup+php will have markup as main language. This is equal to putting your code in the following code block:

+ 
...
+<pre><code class="language-markup">
+	<!-- your code here -->
+</code><pre>
+...
+ +

If you need to load the languages in a given order, but you don't want to use the first language as main language, you can mark the main language with an exclamation mark: markup+php!. This will use php as main language. (You can only define one main language. The test runner will fail all tests in directories with more than one main language.)

+ +

Note: by loading multiple languages you can do integration tests (ensure that loading two or more languages together won't break anything).

+
+ +
+

Creating your test case file

+

At first you need to create a new file in the language directory, you want to test.

+

Use a proper name for your test case. Please use one case of the following conventions:

+
    +
  • issue{issueid}: reference a github issue id (example: issue588.test).
    • +
  • {featurename}_feature: group all tests to one feature in one file (example: string_interpolation_feature.test).
    • +
  • {language}_inclusion: test inclusion of one language into the other (example: markup/php_inclusion.test will test php inclusion into markup).
    • +
+

You can use all conventions as a prefix, so string_interpolation_feature_inline.test is possible. But please take a minute or two to think of a proper name of your test case file. You are writing code not only for the computers, but also for your fellow developers.

+
+ +
+

Writing your test

+

The structure of a test case file is as follows:

+ 

+... language snippet...
+----
+... the simplified token stream you expect ...
+ +

Your file is built up of two or three sections, separated by three or more dashes -, starting at the begin of the line:

+
    +
  1. Your language snippet. The code you want to compile using Prism. (required)
    2. +
  2. The simplified token stream you expect. Needs to be valid JSON. (required)
    3. +
  3. A comment explaining the test case. (optional)
    4. +
+

The easiest way would be to look at an existing test file:

+ 
var a = 5;
+
+----------------------------------------------------
+
+[
+	["keyword", "var"],
+	" a ",
+	["operator", "="],
+	["number", "5"],
+	["punctuation", ";"]
+]
+
+----------------------------------------------------
+
+This is a comment explaining this test case.
+
+ +
+

Explaining the simplified token stream

+

While compiling, Prism transforms your source code into a token stream. This is basically a tree of nested tokens (or arrays, or strings).

+

As these trees are hard to write by hand, the test runner uses a simplified version of it.

+

It uses the following rules:

+
    +
  • Token objects are transformed into an array: [token.type, token.content] (whereas token.content can be a nested structure).
    • +
  • All strings that are either empty or only contain whitespace, are removed from the token stream.
    • +
  • All empty structures are removed.
    • +
+

For further information: reading the tests of the test runner (tests/testrunner-tests.js) will help you understand the transformation.

+
+
+ + +
+

Test runner tests

+

The test runner itself is tested in a separate test case. You can find all “test core” related tests in tests/testrunner-tests.js.

+

You shouldn't need to touch this file ever, except you modify the test runner code.

+
+ +
+

Internal structure

+

The global test flow is at follows:

+
    +
  1. Run all internal tests (test the test runner).
    2. +
  2. Find all language tests.
    3. +
  3. Run all language tests individually.
    4. +
  4. Report the results.
    5. +
+
+ + +
+ + + + + + + +