diff --git a/package.json b/package.json index f6b7fbd7e..457dd2ddc 100755 --- a/package.json +++ b/package.json @@ -81,7 +81,8 @@ "toml": "^3.0.0", "viperhtml": "^2.17.0", "vue": "^2.6.10", - "vue-server-renderer": "^2.6.10" + "vue-server-renderer": "^2.6.10", + "js-yaml": "^3.13.1" }, "dependencies": { "@11ty/dependency-tree": "^1.0.0", @@ -119,4 +120,4 @@ }, "pre-commit": "lint-staged", "pre-push": "test" -} \ No newline at end of file +} diff --git a/src/TemplateData.js b/src/TemplateData.js index 8aa5802de..83862e14f 100644 --- a/src/TemplateData.js +++ b/src/TemplateData.js @@ -111,10 +111,19 @@ class TemplateData { async getTemplateDataFileGlob() { let dir = await this.getInputDir(); - return TemplatePath.addLeadingDotSlashArray([ + let paths = [ `${dir}/**/*.json`, // covers .11tydata.json too `${dir}/**/*${this.config.jsDataFileSuffix}.js` - ]); + ]; + + if (this.hasUserDataExtensions()) { + let userPaths = this.getUserDataExtensions().map( + extension => `${dir}/**/*.${extension}` // covers .11tydata.{extension} too + ); + paths = userPaths.concat(paths); + } + + return TemplatePath.addLeadingDotSlashArray(paths); } async getTemplateJavaScriptDataFileGlob() { @@ -126,7 +135,15 @@ class TemplateData { async getGlobalDataGlob() { let dir = await this.getInputDir(); - return [this._getGlobalDataGlobByExtension(dir, "(json|js)")]; + let userExtensions = ""; + // creating glob string for user extensions + if (this.hasUserDataExtensions()) { + userExtensions = this.getUserDataExtensions().join("|") + "|"; + } + + return [ + this._getGlobalDataGlobByExtension(dir, "(" + userExtensions + "json|js)") + ]; } getWatchPathCache() { @@ -157,6 +174,7 @@ class TemplateData { let files = TemplatePath.addLeadingDotSlashArray( await this.getGlobalDataFiles() ); + let dataFileConflicts = {}; for (let j = 0, k = files.length; j < k; j++) { @@ -218,7 +236,31 @@ class TemplateData { return localData; } - async _getLocalJsonString(path) { + getUserDataExtensions() { + if (!this.config.dataExtensions) { + return []; + } + + // returning extensions in reverse order to create proper extension order + // later added formats will override first ones + return Array.from(this.config.dataExtensions.keys()).reverse(); + } + + getUserDataParser(extension) { + return this.config.dataExtensions.get(extension); + } + + isUserDataExtension(extension) { + return ( + this.config.dataExtensions && this.config.dataExtensions.has(extension) + ); + } + + hasUserDataExtensions() { + return this.config.dataExtensions && this.config.dataExtensions.size > 0; + } + + async _loadFileContents(path) { let rawInput; try { rawInput = await fs.readFile(path, "utf-8"); @@ -228,56 +270,90 @@ class TemplateData { return rawInput; } + async _parseDataFile(path, rawImports, ignoreProcessing, parser) { + let rawInput = await this._loadFileContents(path); + let engineName = this.dataTemplateEngine; + + if (!rawInput) { + return {}; + } + + if (ignoreProcessing || engineName === false) { + try { + return parser(rawInput); + } catch (e) { + throw new TemplateDataParseError( + `Having trouble parsing data file ${path}`, + e + ); + } + } else { + let fn = await new TemplateRender(engineName).getCompiledTemplate( + rawInput + ); + + try { + // pass in rawImports, don’t pass in global data, that’s what we’re parsing + let raw = await fn(rawImports); + return parser(raw); + } catch (e) { + throw new TemplateDataParseError( + `Having trouble parsing data file ${path}`, + e + ); + } + } + } + async getDataValue(path, rawImports, ignoreProcessing) { - if (ignoreProcessing || TemplatePath.getExtension(path) === "js") { + let extension = TemplatePath.getExtension(path); + + // ignoreProcessing = false for global data files + // ignoreProcessing = true for local data files + if ( + extension === "js" || + (extension === "json" && (ignoreProcessing || !this.dataTemplateEngine)) + ) { + // JS data file or require’d JSON (no preprocessing needed) let localPath = TemplatePath.absolutePath(path); - if (await fs.pathExists(localPath)) { - let dataBench = bench.get(`\`${path}\``); - dataBench.before(); - deleteRequireCache(localPath); - let returnValue = require(localPath); - if (typeof returnValue === "function") { - returnValue = await returnValue(); - } - - dataBench.after(); - return returnValue; - } else { + if (!(await fs.pathExists(localPath))) { return {}; } - } else { - let rawInput = await this._getLocalJsonString(path); - let engineName = this.dataTemplateEngine; - - if (rawInput) { - if (ignoreProcessing || engineName === false) { - try { - return JSON.parse(rawInput); - } catch (e) { - throw new TemplateDataParseError( - `Having trouble parsing data file ${path}`, - e - ); - } - } else { - let fn = await new TemplateRender(engineName).getCompiledTemplate( - rawInput - ); - try { - // pass in rawImports, don’t pass in global data, that’s what we’re parsing - return JSON.parse(await fn(rawImports)); - } catch (e) { - throw new TemplateDataParseError( - `Having trouble parsing data file ${path}`, - e - ); - } - } + let dataBench = bench.get(`\`${path}\``); + dataBench.before(); + deleteRequireCache(localPath); + + let returnValue = require(localPath); + if (typeof returnValue === "function") { + returnValue = await returnValue(); } + + dataBench.after(); + return returnValue; + } else if (this.isUserDataExtension(extension)) { + // Other extensions + var parser = this.getUserDataParser(extension); + return this._parseDataFile(path, rawImports, ignoreProcessing, parser); + } else if (extension === "json") { + // File to string, parse with JSON (preprocess) + return this._parseDataFile( + path, + rawImports, + ignoreProcessing, + JSON.parse + ); + } else { + throw new TemplateDataParseError( + `Could not find an appropriate data parser for ${path}. Do you need to add a plugin to your config file?` + ); } + } - return {}; + _pushExtensionsToPaths(paths, curpath, extensions) { + for (let extension of extensions) { + paths.push(curpath + "." + extension); + } } async getLocalDataPaths(templatePath) { @@ -286,19 +362,34 @@ class TemplateData { let inputDir = TemplatePath.addLeadingDotSlash( TemplatePath.normalize(this.inputDir) ); + debugDev("getLocalDataPaths(%o)", templatePath); debugDev("parsed.dir: %o", parsed.dir); + let userExtensions = this.getUserDataExtensions(); + if (parsed.dir) { let fileNameNoExt = EleventyExtensionMap.removeTemplateExtension( parsed.base ); + let filePathNoExt = parsed.dir + "/" + fileNameNoExt; let dataSuffix = this.config.jsDataFileSuffix; debug("Using %o to find data files.", dataSuffix); + + // data suffix paths.push(filePathNoExt + dataSuffix + ".js"); paths.push(filePathNoExt + dataSuffix + ".json"); + // inject user extensions + this._pushExtensionsToPaths( + paths, + filePathNoExt + dataSuffix, + userExtensions + ); + + // top level paths.push(filePathNoExt + ".json"); + this._pushExtensionsToPaths(paths, filePathNoExt, userExtensions); let allDirs = TemplatePath.getAllDirs(parsed.dir); debugDev("allDirs: %o", allDirs); @@ -307,15 +398,33 @@ class TemplateData { let dirPathNoExt = dir + "/" + lastDir; if (!inputDir) { + // data suffix paths.push(dirPathNoExt + dataSuffix + ".js"); paths.push(dirPathNoExt + dataSuffix + ".json"); + this._pushExtensionsToPaths( + paths, + dirPathNoExt + dataSuffix, + userExtensions + ); + + // top level paths.push(dirPathNoExt + ".json"); + this._pushExtensionsToPaths(paths, dirPathNoExt, userExtensions); } else { debugDev("dirStr: %o; inputDir: %o", dir, inputDir); if (dir.indexOf(inputDir) === 0 && dir !== inputDir) { + // data suffix paths.push(dirPathNoExt + dataSuffix + ".js"); paths.push(dirPathNoExt + dataSuffix + ".json"); + this._pushExtensionsToPaths( + paths, + dirPathNoExt + dataSuffix, + userExtensions + ); + + // top level paths.push(dirPathNoExt + ".json"); + this._pushExtensionsToPaths(paths, dirPathNoExt, userExtensions); } } } diff --git a/src/UserConfig.js b/src/UserConfig.js index 76df00d94..fc82a9277 100644 --- a/src/UserConfig.js +++ b/src/UserConfig.js @@ -56,6 +56,9 @@ class UserConfig { // this.templateExtensionAliases = {}; this.watchJavaScriptDependencies = true; this.browserSyncConfig = {}; + + // using Map to preserve insertion order + this.dataExtensions = new Map(); } versionCheck(expected) { @@ -603,13 +606,18 @@ class UserConfig { // templateExtensionAliases: this.templateExtensionAliases, watchJavaScriptDependencies: this.watchJavaScriptDependencies, browserSyncConfig: this.browserSyncConfig, - frontMatterParsingOptions: this.frontMatterParsingOptions + frontMatterParsingOptions: this.frontMatterParsingOptions, + dataExtensions: this.dataExtensions }; } // addExtension(fileExtension, userClass) { // this.userExtensionMap[ fileExtension ] = userClass; // } + + addDataExtension(formatExtension, formatParser) { + this.dataExtensions.set(formatExtension, formatParser); + } } module.exports = UserConfig; diff --git a/test/TemplateDataTest.js b/test/TemplateDataTest.js index 5892367f1..9350c062f 100644 --- a/test/TemplateDataTest.js +++ b/test/TemplateDataTest.js @@ -52,6 +52,7 @@ test("Add local data", async t => { t.is(withLocalData.localdatakey1, "localdatavalue1"); // from the js file + // this checks priority/overrides t.is(withLocalData.localdatakeyfromjs, "howdydoody"); t.is(withLocalData.localdatakeyfromjs2, "howdy2"); }); diff --git a/test/UserDataExtensionsTest.js b/test/UserDataExtensionsTest.js new file mode 100644 index 000000000..fa5cc1e62 --- /dev/null +++ b/test/UserDataExtensionsTest.js @@ -0,0 +1,92 @@ +import test from "ava"; +import TemplateData from "../src/TemplateData"; +let yaml = require("js-yaml"); + +function injectDataExtensions(dataObj) { + dataObj.config.dataExtensions = new Map([ + ["yaml", s => yaml.safeLoad(s)], + ["nosj", JSON.parse] + ]); +} + +test("Local data", async t => { + let dataObj = new TemplateData("./test/stubs-630/"); + injectDataExtensions(dataObj); + + let data = await dataObj.getData(); + + // YAML GLOBAL DATA + t.is(data.globalData2.datakey1, "datavalue2"); + t.is(data.globalData2.datakey2, "@11ty/eleventy--yaml"); + + // NOSJ (JSON) GLOBAL DATA + t.is(data.globalData3.datakey1, "datavalue3"); + t.is(data.globalData3.datakey2, "@11ty/eleventy--nosj"); + + let withLocalData = await dataObj.getLocalData( + "./test/stubs-630/component-yaml/component.njk" + ); + // console.log("localdata", withLocalData); + + t.is(withLocalData.yamlKey1, "yaml1"); + t.is(withLocalData.yamlKey2, "yaml2"); + t.is(withLocalData.yamlKey3, "yaml3"); + t.is(withLocalData.nosjKey1, "nosj1"); + t.is(withLocalData.jsonKey1, "json1"); + t.is(withLocalData.jsonKey2, "json2"); + t.is(withLocalData.jsKey1, "js1"); +}); + +test("Local files", async t => { + let dataObj = new TemplateData("./test/stubs-630/"); + injectDataExtensions(dataObj); + let files = await dataObj.getLocalDataPaths( + "./test/stubs-630/component-yaml/component.njk" + ); + t.deepEqual(files, [ + "./test/stubs-630/component-yaml/component-yaml.yaml", + "./test/stubs-630/component-yaml/component-yaml.nosj", + "./test/stubs-630/component-yaml/component-yaml.json", + "./test/stubs-630/component-yaml/component-yaml.11tydata.yaml", + "./test/stubs-630/component-yaml/component-yaml.11tydata.nosj", + "./test/stubs-630/component-yaml/component-yaml.11tydata.json", + "./test/stubs-630/component-yaml/component-yaml.11tydata.js", + "./test/stubs-630/component-yaml/component.yaml", + "./test/stubs-630/component-yaml/component.nosj", + "./test/stubs-630/component-yaml/component.json", + "./test/stubs-630/component-yaml/component.11tydata.yaml", + "./test/stubs-630/component-yaml/component.11tydata.nosj", + "./test/stubs-630/component-yaml/component.11tydata.json", + "./test/stubs-630/component-yaml/component.11tydata.js" + ]); +}); + +test("Global data", async t => { + let dataObj = new TemplateData("./test/stubs-630/"); + + injectDataExtensions(dataObj); + + t.deepEqual(await dataObj.getGlobalDataGlob(), [ + "./test/stubs-630/_data/**/*.(nosj|yaml|json|js)" + ]); + + let dataFilePaths = await dataObj.getGlobalDataFiles(); + let data = await dataObj.getData(); + + // JS GLOBAL DATA + t.is(data.globalData0.datakey1, "datavalue0"); + + // JSON GLOBAL DATA + t.is(data.globalData1.datakey1, "datavalue1"); + t.is(data.globalData1.datakey2, "@11ty/eleventy--json"); + + // YAML GLOBAL DATA + t.is(data.globalData2.datakey1, "datavalue2"); + t.is(data.globalData2.datakey2, "@11ty/eleventy--yaml"); + + // NOSJ (JSON) GLOBAL DATA + t.is(data.globalData3.datakey1, "datavalue3"); + t.is(data.globalData3.datakey2, "@11ty/eleventy--nosj"); + + t.is(data.subdir.globalDataSubdir.keyyaml, "yaml"); +}); diff --git a/test/stubs-630/_data/globalData0.js b/test/stubs-630/_data/globalData0.js new file mode 100644 index 000000000..4f79aca51 --- /dev/null +++ b/test/stubs-630/_data/globalData0.js @@ -0,0 +1,3 @@ +module.exports = { + datakey1: "datavalue0" +}; diff --git a/test/stubs-630/_data/globalData1.json b/test/stubs-630/_data/globalData1.json new file mode 100644 index 000000000..5f02feff3 --- /dev/null +++ b/test/stubs-630/_data/globalData1.json @@ -0,0 +1,4 @@ +{ + "datakey1": "datavalue1", + "datakey2": "{{pkg.name}}--json" +} diff --git a/test/stubs-630/_data/globalData2.yaml b/test/stubs-630/_data/globalData2.yaml new file mode 100644 index 000000000..9a852bfa8 --- /dev/null +++ b/test/stubs-630/_data/globalData2.yaml @@ -0,0 +1,2 @@ +datakey1: datavalue2 +datakey2: "{{pkg.name}}--yaml" diff --git a/test/stubs-630/_data/globalData3.nosj b/test/stubs-630/_data/globalData3.nosj new file mode 100644 index 000000000..3f01a07ea --- /dev/null +++ b/test/stubs-630/_data/globalData3.nosj @@ -0,0 +1,4 @@ +{ + "datakey1": "datavalue3", + "datakey2": "{{pkg.name}}--nosj" +} diff --git a/test/stubs-630/_data/subdir/globalDataSubdir.yaml b/test/stubs-630/_data/subdir/globalDataSubdir.yaml new file mode 100644 index 000000000..7e610b578 --- /dev/null +++ b/test/stubs-630/_data/subdir/globalDataSubdir.yaml @@ -0,0 +1 @@ +keyyaml: "yaml" diff --git a/test/stubs-630/component-yaml/component.11tydata.js b/test/stubs-630/component-yaml/component.11tydata.js new file mode 100644 index 000000000..f8e5b9878 --- /dev/null +++ b/test/stubs-630/component-yaml/component.11tydata.js @@ -0,0 +1,3 @@ +module.exports = { + jsKey1: "js1" +}; diff --git a/test/stubs-630/component-yaml/component.11tydata.json b/test/stubs-630/component-yaml/component.11tydata.json new file mode 100644 index 000000000..0af98b490 --- /dev/null +++ b/test/stubs-630/component-yaml/component.11tydata.json @@ -0,0 +1,3 @@ +{ + "jsonKey1": "json1" +} diff --git a/test/stubs-630/component-yaml/component.11tydata.nosj b/test/stubs-630/component-yaml/component.11tydata.nosj new file mode 100644 index 000000000..76bc8b93e --- /dev/null +++ b/test/stubs-630/component-yaml/component.11tydata.nosj @@ -0,0 +1,3 @@ +{ + "nosjKey1": "nosj1" +} diff --git a/test/stubs-630/component-yaml/component.11tydata.yaml b/test/stubs-630/component-yaml/component.11tydata.yaml new file mode 100644 index 000000000..b61c3866f --- /dev/null +++ b/test/stubs-630/component-yaml/component.11tydata.yaml @@ -0,0 +1,5 @@ +yamlKey2: "yaml2" +yamlKey3: "yaml3" +jsonKey1: "overriden" +jsKey1: "overriden" +nosjKey1: "overriden" diff --git a/test/stubs-630/component-yaml/component.json b/test/stubs-630/component-yaml/component.json new file mode 100644 index 000000000..21e49a8ce --- /dev/null +++ b/test/stubs-630/component-yaml/component.json @@ -0,0 +1,5 @@ +{ + "jsonKey2": "json2", + "jsKey1": "overriden", + "yamlKey3": "overriden" +} diff --git a/test/stubs-630/component-yaml/component.njk b/test/stubs-630/component-yaml/component.njk new file mode 100644 index 000000000..4772f7452 --- /dev/null +++ b/test/stubs-630/component-yaml/component.njk @@ -0,0 +1 @@ +{{localkeyOverride}} \ No newline at end of file diff --git a/test/stubs-630/component-yaml/component.yaml b/test/stubs-630/component-yaml/component.yaml new file mode 100644 index 000000000..f65aadd9d --- /dev/null +++ b/test/stubs-630/component-yaml/component.yaml @@ -0,0 +1,5 @@ +yamlKey1: "yaml1" +yamlKey2: "overriden" +jsonKey1: "overriden" +jsonKey2: "overriden" +jsKey1: "overriden"