Gloryofrobots data hooks via #635

package.json

@@ -81,7 +81,8 @@
     "toml": "^3.0.0",
     "viperhtml": "^2.17.0",
     "vue": "^2.6.10",
-    "vue-server-renderer": "^2.6.10"
+    "vue-server-renderer": "^2.6.10",
+    "js-yaml": "^3.13.1"
   },
   "dependencies": {
     "@11ty/dependency-tree": "^1.0.0",
@@ -119,4 +120,4 @@
   },
   "pre-commit": "lint-staged",
   "pre-push": "test"
-}
+}

src/TemplateData.js

@@ -111,10 +111,19 @@ class TemplateData {
 
   async getTemplateDataFileGlob() {
     let dir = await this.getInputDir();
-    return TemplatePath.addLeadingDotSlashArray([
+    let paths = [
       `${dir}/**/*.json`, // covers .11tydata.json too
       `${dir}/**/*${this.config.jsDataFileSuffix}.js`
-    ]);
+    ];
+
+    if (this.hasUserDataExtensions()) {
+      let userPaths = this.getUserDataExtensions().map(
+        extension => `${dir}/**/*.${extension}` // covers .11tydata.{extension} too
+      );
+      paths = userPaths.concat(paths);
+    }
+
+    return TemplatePath.addLeadingDotSlashArray(paths);
   }
 
   async getTemplateJavaScriptDataFileGlob() {
@@ -126,7 +135,15 @@ class TemplateData {
 
   async getGlobalDataGlob() {
     let dir = await this.getInputDir();
-    return [this._getGlobalDataGlobByExtension(dir, "(json|js)")];
+    let userExtensions = "";
+    // creating glob string for user extensions
+    if (this.hasUserDataExtensions()) {
+      userExtensions = this.getUserDataExtensions().join("|") + "|";
+    }
+
+    return [
+      this._getGlobalDataGlobByExtension(dir, "(" + userExtensions + "json|js)")
+    ];
   }
 
   getWatchPathCache() {
@@ -157,6 +174,7 @@ class TemplateData {
     let files = TemplatePath.addLeadingDotSlashArray(
       await this.getGlobalDataFiles()
     );
+
     let dataFileConflicts = {};
 
     for (let j = 0, k = files.length; j < k; j++) {
@@ -218,7 +236,31 @@ class TemplateData {
     return localData;
   }
 
-  async _getLocalJsonString(path) {
+  getUserDataExtensions() {
+    if (!this.config.dataExtensions) {
+      return [];
+    }
+
+    // returning extensions in reverse order to create proper extension order
+    // later added formats will override first ones
+    return Array.from(this.config.dataExtensions.keys()).reverse();
+  }
+
+  getUserDataParser(extension) {
+    return this.config.dataExtensions.get(extension);
+  }
+
+  isUserDataExtension(extension) {
+    return (
+      this.config.dataExtensions && this.config.dataExtensions.has(extension)
+    );
+  }
+
+  hasUserDataExtensions() {
+    return this.config.dataExtensions && this.config.dataExtensions.size > 0;
+  }
+
+  async _loadFileContents(path) {
     let rawInput;
     try {
       rawInput = await fs.readFile(path, "utf-8");
@@ -228,56 +270,90 @@ class TemplateData {
     return rawInput;
   }
 
+  async _parseDataFile(path, rawImports, ignoreProcessing, parser) {
+    let rawInput = await this._loadFileContents(path);
+    let engineName = this.dataTemplateEngine;
+
+    if (!rawInput) {
+      return {};
+    }
+
+    if (ignoreProcessing || engineName === false) {
+      try {
+        return parser(rawInput);
+      } catch (e) {
+        throw new TemplateDataParseError(
+          `Having trouble parsing data file ${path}`,
+          e
+        );
+      }
+    } else {
+      let fn = await new TemplateRender(engineName).getCompiledTemplate(
+        rawInput
+      );
+
+      try {
+        // pass in rawImports, don’t pass in global data, that’s what we’re parsing
+        let raw = await fn(rawImports);
+        return parser(raw);
+      } catch (e) {
+        throw new TemplateDataParseError(
+          `Having trouble parsing data file ${path}`,
+          e
+        );
+      }
+    }
+  }
+
   async getDataValue(path, rawImports, ignoreProcessing) {
-    if (ignoreProcessing || TemplatePath.getExtension(path) === "js") {
+    let extension = TemplatePath.getExtension(path);
+
+    // ignoreProcessing = false for global data files
+    // ignoreProcessing = true for local data files
+    if (
+      extension === "js" ||
+      (extension === "json" && (ignoreProcessing || !this.dataTemplateEngine))
+    ) {
+      // JS data file or require’d JSON (no preprocessing needed)
       let localPath = TemplatePath.absolutePath(path);
-      if (await fs.pathExists(localPath)) {
-        let dataBench = bench.get(`\`${path}\``);
-        dataBench.before();
-        deleteRequireCache(localPath);
-        let returnValue = require(localPath);
-        if (typeof returnValue === "function") {
-          returnValue = await returnValue();
-        }
-
-        dataBench.after();
-        return returnValue;
-      } else {
+      if (!(await fs.pathExists(localPath))) {
         return {};
       }
-    } else {
-      let rawInput = await this._getLocalJsonString(path);
-      let engineName = this.dataTemplateEngine;
-
-      if (rawInput) {
-        if (ignoreProcessing || engineName === false) {
-          try {
-            return JSON.parse(rawInput);
-          } catch (e) {
-            throw new TemplateDataParseError(
-              `Having trouble parsing data file ${path}`,
-              e
-            );
-          }
-        } else {
-          let fn = await new TemplateRender(engineName).getCompiledTemplate(
-            rawInput
-          );
 
-          try {
-            // pass in rawImports, don’t pass in global data, that’s what we’re parsing
-            return JSON.parse(await fn(rawImports));
-          } catch (e) {
-            throw new TemplateDataParseError(
-              `Having trouble parsing data file ${path}`,
-              e
-            );
-          }
-        }
+      let dataBench = bench.get(`\`${path}\``);
+      dataBench.before();
+      deleteRequireCache(localPath);
+
+      let returnValue = require(localPath);
+      if (typeof returnValue === "function") {
+        returnValue = await returnValue();
       }
+
+      dataBench.after();
+      return returnValue;
+    } else if (this.isUserDataExtension(extension)) {
+      // Other extensions
+      var parser = this.getUserDataParser(extension);
+      return this._parseDataFile(path, rawImports, ignoreProcessing, parser);
+    } else if (extension === "json") {
+      // File to string, parse with JSON (preprocess)
+      return this._parseDataFile(
+        path,
+        rawImports,
+        ignoreProcessing,
+        JSON.parse
+      );
+    } else {
+      throw new TemplateDataParseError(
+        `Could not find an appropriate data parser for ${path}. Do you need to add a plugin to your config file?`
+      );
     }
+  }
 
-    return {};
+  _pushExtensionsToPaths(paths, curpath, extensions) {
+    for (let extension of extensions) {
+      paths.push(curpath + "." + extension);
+    }
   }
 
   async getLocalDataPaths(templatePath) {
@@ -286,19 +362,34 @@ class TemplateData {
     let inputDir = TemplatePath.addLeadingDotSlash(
       TemplatePath.normalize(this.inputDir)
     );
+
     debugDev("getLocalDataPaths(%o)", templatePath);
     debugDev("parsed.dir: %o", parsed.dir);
 
+    let userExtensions = this.getUserDataExtensions();
+
     if (parsed.dir) {
       let fileNameNoExt = EleventyExtensionMap.removeTemplateExtension(
         parsed.base
       );
+
       let filePathNoExt = parsed.dir + "/" + fileNameNoExt;
       let dataSuffix = this.config.jsDataFileSuffix;
       debug("Using %o to find data files.", dataSuffix);
+
+      // data suffix
       paths.push(filePathNoExt + dataSuffix + ".js");
       paths.push(filePathNoExt + dataSuffix + ".json");
+      // inject user extensions
+      this._pushExtensionsToPaths(
+        paths,
+        filePathNoExt + dataSuffix,
+        userExtensions
+      );
+
+      // top level
       paths.push(filePathNoExt + ".json");
+      this._pushExtensionsToPaths(paths, filePathNoExt, userExtensions);
 
       let allDirs = TemplatePath.getAllDirs(parsed.dir);
       debugDev("allDirs: %o", allDirs);
@@ -307,15 +398,33 @@ class TemplateData {
         let dirPathNoExt = dir + "/" + lastDir;
 
         if (!inputDir) {
+          // data suffix
           paths.push(dirPathNoExt + dataSuffix + ".js");
           paths.push(dirPathNoExt + dataSuffix + ".json");
+          this._pushExtensionsToPaths(
+            paths,
+            dirPathNoExt + dataSuffix,
+            userExtensions
+          );
+
+          // top level
           paths.push(dirPathNoExt + ".json");
+          this._pushExtensionsToPaths(paths, dirPathNoExt, userExtensions);
         } else {
           debugDev("dirStr: %o; inputDir: %o", dir, inputDir);
           if (dir.indexOf(inputDir) === 0 && dir !== inputDir) {
+            // data suffix
             paths.push(dirPathNoExt + dataSuffix + ".js");
             paths.push(dirPathNoExt + dataSuffix + ".json");
+            this._pushExtensionsToPaths(
+              paths,
+              dirPathNoExt + dataSuffix,
+              userExtensions
+            );
+
+            // top level
             paths.push(dirPathNoExt + ".json");
+            this._pushExtensionsToPaths(paths, dirPathNoExt, userExtensions);
           }
         }
       }

src/UserConfig.js

@@ -56,6 +56,9 @@ class UserConfig {
     // this.templateExtensionAliases = {};
     this.watchJavaScriptDependencies = true;
     this.browserSyncConfig = {};
+
+    // using Map to preserve insertion order
+    this.dataExtensions = new Map();
   }
 
   versionCheck(expected) {
@@ -603,13 +606,18 @@ class UserConfig {
       // templateExtensionAliases: this.templateExtensionAliases,
       watchJavaScriptDependencies: this.watchJavaScriptDependencies,
       browserSyncConfig: this.browserSyncConfig,
-      frontMatterParsingOptions: this.frontMatterParsingOptions
+      frontMatterParsingOptions: this.frontMatterParsingOptions,
+      dataExtensions: this.dataExtensions
     };
   }
 
   // addExtension(fileExtension, userClass) {
   //   this.userExtensionMap[ fileExtension ] = userClass;
   // }
+
+  addDataExtension(formatExtension, formatParser) {
+    this.dataExtensions.set(formatExtension, formatParser);
+  }
 }
 
 module.exports = UserConfig;

test/TemplateDataTest.js

@@ -52,6 +52,7 @@ test("Add local data", async t => {
   t.is(withLocalData.localdatakey1, "localdatavalue1");
 
   // from the js file
+  // this checks priority/overrides
   t.is(withLocalData.localdatakeyfromjs, "howdydoody");
   t.is(withLocalData.localdatakeyfromjs2, "howdy2");
 });