Skip to content

Commit

Permalink
Adds requestId option (to add your own unique id for cache/reuse/dedupe)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zachleat
zachleat committed Nov 12, 2024
1 parent 4d24451 commit aea5b9e
Show file tree
Hide file tree
Showing 7 changed files with 224 additions and 160 deletions.
55 changes: 14 additions & 41 deletions eleventy-fetch.js
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
const { default: PQueue } = require("p-queue");
const debug = require("debug")("Eleventy:Fetch");

const RemoteAssetCache = require("./src/RemoteAssetCache");
const AssetCache = require("./src/AssetCache");
const Sources = require("./src/Sources.js");
const RemoteAssetCache = require("./src/RemoteAssetCache.js");
const AssetCache = require("./src/AssetCache.js");

const globalOptions = {
type: "buffer",
Expand All @@ -24,38 +25,6 @@ const globalOptions = {
hashLength: 30,
};

function isFullUrl(url) {
try {
new URL(url);
return true;
} catch (e) {
// invalid url OR already a local path
return false;
}
}

function isAwaitable(maybeAwaitable) {
return (
(typeof maybeAwaitable === "object" && typeof maybeAwaitable.then === "function") ||
maybeAwaitable.constructor.name === "AsyncFunction"
);
}

async function save(source, options) {
if (!(isFullUrl(source) || isAwaitable(source))) {
return Promise.reject(new Error("Caching an already local asset is not yet supported."));
}

if (isAwaitable(source) && !options.formatUrlForDisplay) {
return Promise.reject(
new Error("formatUrlForDisplay must be implemented, as a Promise has been provided."),
);
}

let asset = new RemoteAssetCache(source, options.directory, options);
return asset.fetch(options);
}

/* Queue */
let queue = new PQueue({
concurrency: globalOptions.concurrency,
Expand All @@ -68,11 +37,9 @@ queue.on("active", () => {
let inProgress = {};

function queueSave(source, queueCallback, options) {
let sourceKey;
if(typeof source === "string") {
sourceKey = source;
} else {
sourceKey = RemoteAssetCache.getUid(source, options);
let sourceKey = RemoteAssetCache.getRequestId(source, options);
if(!sourceKey) {
return Promise.reject(Sources.getInvalidSourceError(source));
}

if (!inProgress[sourceKey]) {
Expand All @@ -85,9 +52,14 @@ function queueSave(source, queueCallback, options) {
}

module.exports = function (source, options) {
if (!Sources.isFullUrl(source) && !Sources.isValidSource(source)) {
throw new Error("Caching an already local asset is not yet supported.");
}

let mergedOptions = Object.assign({}, globalOptions, options);
return queueSave(source, () => {
return save(source, mergedOptions);
let asset = new RemoteAssetCache(source, mergedOptions.directory, mergedOptions);
return asset.fetch(mergedOptions);
}, mergedOptions);
};

Expand All @@ -102,7 +74,8 @@ Object.defineProperty(module.exports, "concurrency", {

module.exports.queue = queueSave;
module.exports.Util = {
isFullUrl,
isFullUrl: Sources.isFullUrl,
};
module.exports.RemoteAssetCache = RemoteAssetCache;
module.exports.AssetCache = AssetCache;
module.exports.Sources = Sources;
83 changes: 44 additions & 39 deletions src/AssetCache.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,61 +4,32 @@ const path = require("path");
const { create: FlatCacheCreate } = require("flat-cache");
const { createHash } = require("crypto");

const Sources = require("./Sources.js");

const debug = require("debug")("Eleventy:Fetch");

class AssetCache {
#customFilename;

static getCacheKey(source, options) {
// RemoteAssetCache sends this an Array, which skips this altogether
if (
(typeof source === "object" && typeof source.then === "function") ||
(typeof source === "function" && source.constructor.name === "AsyncFunction")
) {
if(typeof options.formatUrlForDisplay !== "function") {
throw new Error("When caching an arbitrary promise source, an options.formatUrlForDisplay() callback is required.");
}

return options.formatUrlForDisplay();
constructor(source, cacheDirectory, options = {}) {
if(!Sources.isValidSource(source)) {
throw Sources.getInvalidSourceError(source);
}

return source;
}

constructor(url, cacheDirectory, options = {}) {
let uniqueKey;
// RemoteAssetCache passes in an array
if(Array.isArray(uniqueKey)) {
uniqueKey = uniqueKey.join(",");
} else {
uniqueKey = AssetCache.getCacheKey(url, options);
}
let uniqueKey = AssetCache.getCacheKey(source, options);
this.uniqueKey = uniqueKey;

this.hash = AssetCache.getHash(uniqueKey, options.hashLength);

this.cacheDirectory = cacheDirectory || ".cache";
this.defaultDuration = "1d";
this.options = options;

// Compute the filename only once
if (typeof this.options.filenameFormat === "function") {
this.#customFilename = this.options.filenameFormat(uniqueKey, this.hash);

if (typeof this.#customFilename !== "string") {
throw new Error(`The provided cacheFilename callback function did not return a string.`);
}
this.#customFilename = AssetCache.cleanFilename(this.options.filenameFormat(uniqueKey, this.hash));

if (typeof this.#customFilename.length === 0) {
throw new Error(`The provided cacheFilename callback function returned an empty string.`);
}

// Ensure no illegal characters are present (Windows or Linux: forward/backslash, chevrons, colon, double-quote, pipe, question mark, asterisk)
if (this.#customFilename.match(/([\/\\<>:"|?*]+?)/)) {
const sanitizedFilename = this.#customFilename.replace(/[\/\\<>:"|?*]+/g, "");
console.warn(
`[AssetCache] Some illegal characters were removed from the cache filename: ${this.#customFilename} will be cached as ${sanitizedFilename}.`,
);
this.#customFilename = sanitizedFilename;
if (typeof this.#customFilename !== "string" || this.#customFilename.length === 0) {
throw new Error(`The provided filenameFormat callback function needs to return valid filename characters.`);
}
}
}
Expand All @@ -71,6 +42,40 @@ class AssetCache {
}
}

static cleanFilename(filename) {
// Ensure no illegal characters are present (Windows or Linux: forward/backslash, chevrons, colon, double-quote, pipe, question mark, asterisk)
if (filename.match(/([\/\\<>:"|?*]+?)/)) {
let sanitizedFilename = filename.replace(/[\/\\<>:"|?*]+/g, "");
debug(
`[@11ty/eleventy-fetch] Some illegal characters were removed from the cache filename: ${filename} will be cached as ${sanitizedFilename}.`,
);
return sanitizedFilename;
}

return filename;
}

static getCacheKey(source, options) {
// RemoteAssetCache passes in a string here, which skips this check (requestId is already used upstream)
if (Sources.isValidComplexSource(source)) {
if(options.requestId) {
return options.requestId;
}

if(typeof source.toString === "function") {
// return source.toString();
let toStr = source.toString();
if(toStr !== "function() {}" && toStr !== "[object Object]") {
return toStr;
}
}

throw Sources.getInvalidSourceError(source);
}

return source;
}

// Defult hashLength also set in global options, duplicated here for tests
// v5.0+ key can be Array or literal
static getHash(key, hashLength = 30) {
Expand Down
87 changes: 46 additions & 41 deletions src/RemoteAssetCache.js
View file
Original file line number Diff line number Diff line change
@@ -1,57 +1,63 @@
const AssetCache = require("./AssetCache");
const Sources = require("./Sources.js");
const AssetCache = require("./AssetCache.js");
// const debug = require("debug")("Eleventy:Fetch");

class RemoteAssetCache extends AssetCache {
constructor(url, cacheDirectory, options = {}) {
let cleanUrl = url;
if (options.removeUrlQueryParams) {
cleanUrl = RemoteAssetCache.cleanUrl(cleanUrl);
}

// Must run after removeUrlQueryParams
let displayUrl = RemoteAssetCache.convertUrlToString(cleanUrl, options);
let cacheKeyArray = RemoteAssetCache.getCacheKey(displayUrl, options);
constructor(source, cacheDirectory, options = {}) {
let requestId = RemoteAssetCache.getRequestId(source, options);
super(requestId, cacheDirectory, options);

super(cacheKeyArray, cacheDirectory, options);

this.url = url;
this.source = source;
this.options = options;

this.displayUrl = displayUrl;
this.displayUrl = RemoteAssetCache.convertUrlToString(source, options);
}

static getUid(source, options) {
let displayUrl = RemoteAssetCache.convertUrlToString(source, options);
let cacheKeyArray = RemoteAssetCache.getCacheKey(displayUrl, options);
return cacheKeyArray.join(",");
static getRequestId(source, options) {
if (Sources.isValidComplexSource(source)) {
return this.getCacheKey(source, options);
}

if (options.removeUrlQueryParams) {
let cleaned = this.cleanUrl(source);
return this.getCacheKey(cleaned, options);
}

return this.getCacheKey(source, options);
}

static getCacheKey(source, options) {
// Promise sources are handled upstream
let cacheKey = [source];
let cacheKey = {
source: AssetCache.getCacheKey(source, options),
};

if (options.fetchOptions) {
if (options.fetchOptions.method && options.fetchOptions.method !== "GET") {
cacheKey.push(options.fetchOptions.method);
cacheKey.method = options.fetchOptions.method;
}
if (options.fetchOptions.body) {
cacheKey.push(options.fetchOptions.body);
cacheKey.body = options.fetchOptions.body;
}
}

return cacheKey;
if(Object.keys(cacheKey).length > 1) {
return JSON.stringify(cacheKey);
}

return cacheKey.source;
}

static cleanUrl(url) {
if(typeof url !== "string" && !(url instanceof URL)) {
if(!Sources.isFullUrl(url)) {
return url;
}

let cleanUrl;
if(typeof url === "string") {
if(typeof url === "string" || typeof url.toString === "function") {
cleanUrl = new URL(url);
} else if(url instanceof URL) {
cleanUrl = url;
} else {
throw new Error("Invalid source for cleanUrl: " + url)
}

cleanUrl.search = new URLSearchParams([]);
Expand All @@ -60,20 +66,15 @@ class RemoteAssetCache extends AssetCache {
}

static convertUrlToString(source, options = {}) {
// removes query params
source = RemoteAssetCache.cleanUrl(source);

let { formatUrlForDisplay } = options;
if (formatUrlForDisplay && typeof formatUrlForDisplay === "function") {
return formatUrlForDisplay(source);
return "" + formatUrlForDisplay(source);
}

return source;
}

get url() {
return this._url;
}

set url(url) {
this._url = url;
return "" + source;
}

async getResponseValue(response, type) {
Expand All @@ -100,15 +101,19 @@ class RemoteAssetCache extends AssetCache {

let body;
let type = optionsOverride.type || this.options.type;
if (typeof this.url === "object" && typeof this.url.then === "function") {
body = await this.url;
} else if (typeof this.url === "function" && this.url.constructor.name === "AsyncFunction") {
body = await this.url();
if (typeof this.source === "object" && typeof this.source.then === "function") {
body = await this.source;
} else if (typeof this.source === "function") {
// sync or async function
body = await this.source();
} else {
let fetchOptions = optionsOverride.fetchOptions || this.options.fetchOptions || {};
if(!Sources.isFullUrl(this.source)) {
throw Sources.getInvalidSourceError(this.source);
}

// v5: now using global (Node-native or otherwise) fetch instead of node-fetch
let response = await fetch(this.url, fetchOptions);
let response = await fetch(this.source, fetchOptions);
if (!response.ok) {
throw new Error(
`Bad response for ${this.displayUrl} (${response.status}): ${response.statusText}`,
Expand Down
50 changes: 50 additions & 0 deletions src/Sources.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
class Sources {
static isFullUrl(url) {
try {
if(url instanceof URL) {
return true;
}

new URL(url);
return true;
} catch (e) {
// invalid url OR already a local path
return false;
}
}

static isValidSource(source) {
// String (url?)
if(typeof source === "string") {
return true;
}
if(this.isValidComplexSource(source)) {
return true;
}
return false;
}

static isValidComplexSource(source) {
// Async/sync Function
if(typeof source === "function") {
return true;
}
if(typeof source === "object") {
// Raw promise
if(typeof source.then === "function") {
return true;
}
// anything string-able
if(typeof source.toString === "function") {
return true;
}
}
return false;
}

static getInvalidSourceError(source, errorCause) {
return new Error("Invalid source: must be a string, function, or Promise. If a function or Promise, you must provide a `toString()` method or an `options.requestId` unique key. Received: " + source, { cause: errorCause });
}
}

module.exports = Sources;
Loading

0 comments on commit aea5b9e

Please sign in to comment.