From 6ca2dab0dfa11e2a9edde907a906ba1cdb19b690 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Thu, 5 Sep 2024 12:44:37 +0200 Subject: [PATCH 1/2] feat(http, server): support native `ttl` --- docs/2.drivers/http.md | 4 ++- src/drivers/http.ts | 50 +++++++++++++++++++---------- src/server.ts | 73 ++++++++++++++++++++++-------------------- src/types.ts | 12 ++++++- test/server.test.ts | 65 +++++++++++++++++++++++++++++++++++-- 5 files changed, 147 insertions(+), 57 deletions(-) diff --git a/docs/2.drivers/http.md b/docs/2.drivers/http.md index 7976a33c..37367f3d 100644 --- a/docs/2.drivers/http.md +++ b/docs/2.drivers/http.md @@ -32,10 +32,12 @@ const storage = createStorage({ - `getItem`: Maps to http `GET`. Returns deserialized value if response is ok - `hasItem`: Maps to http `HEAD`. Returns `true` if response is ok (200) -- `setItem`: Maps to http `PUT`. Sends serialized value using body +- `getMeta`: Maps to http `HEAD` (headers: `last-modified` => `mtime`, `x-ttl` => `ttl`) +- `setItem`: Maps to http `PUT`. Sends serialized value using body (`ttl` option will be sent as `x-ttl` header). - `removeItem`: Maps to `DELETE` - `clear`: Not supported **Transaction Options:** - `headers`: Custom headers to be sent on each operation (`getItem`, `setItem`, etc) +- `ttl`: Custom `ttl` (in seconds) for supported drivers. Will be mapped to `x-ttl` http header. diff --git a/src/drivers/http.ts b/src/drivers/http.ts index 8ede6986..60d9a261 100644 --- a/src/drivers/http.ts +++ b/src/drivers/http.ts @@ -1,3 +1,4 @@ +import type { TransactionOptions } from "../types"; import { defineDriver } from "./utils"; import { type FetchError, $fetch as _fetch } from "ofetch"; import { joinURL } from "ufo"; @@ -22,82 +23,97 @@ export default defineDriver((opts: HTTPOptions) => { throw error; }; + const getHeaders = ( + topts: TransactionOptions | undefined, + defaultHeaders?: Record + ) => { + const headers = { + ...defaultHeaders, + ...opts.headers, + ...topts?.headers, + }; + if (topts?.ttl && !headers["x-ttl"]) { + headers["x-ttl"] = topts.ttl + ""; + } + return headers; + }; + return { name: DRIVER_NAME, options: opts, hasItem(key, topts) { return _fetch(r(key), { method: "HEAD", - headers: { ...opts.headers, ...topts.headers }, + headers: getHeaders(topts), }) .then(() => true) .catch((err) => catchFetchError(err, false)); }, - async getItem(key, tops = {}) { + async getItem(key, tops) { const value = await _fetch(r(key), { - headers: { ...opts.headers, ...tops.headers }, + headers: getHeaders(tops), }).catch(catchFetchError); return value; }, async getItemRaw(key, topts) { const value = await _fetch(r(key), { - headers: { - accept: "application/octet-stream", - ...opts.headers, - ...topts.headers, - }, + headers: getHeaders(topts, { accept: "application/octet-stream" }), }).catch(catchFetchError); return value; }, async getMeta(key, topts) { const res = await _fetch.raw(r(key), { method: "HEAD", - headers: { ...opts.headers, ...topts.headers }, + headers: getHeaders(topts), }); let mtime = undefined; + let ttl = undefined; const _lastModified = res.headers.get("last-modified"); if (_lastModified) { mtime = new Date(_lastModified); } + const _ttl = res.headers.get("x-ttl"); + if (_ttl) { + ttl = Number.parseInt(_ttl, 10); + } return { status: res.status, mtime, + ttl, }; }, async setItem(key, value, topts) { await _fetch(r(key), { method: "PUT", body: value, - headers: { ...opts.headers, ...topts?.headers }, + headers: getHeaders(topts), }); }, async setItemRaw(key, value, topts) { await _fetch(r(key), { method: "PUT", body: value, - headers: { + headers: getHeaders(topts, { "content-type": "application/octet-stream", - ...opts.headers, - ...topts.headers, - }, + }), }); }, async removeItem(key, topts) { await _fetch(r(key), { method: "DELETE", - headers: { ...opts.headers, ...topts.headers }, + headers: getHeaders(topts), }); }, async getKeys(base, topts) { const value = await _fetch(rBase(base), { - headers: { ...opts.headers, ...topts.headers }, + headers: getHeaders(topts), }); return Array.isArray(value) ? value : []; }, async clear(base, topts) { await _fetch(rBase(base), { method: "DELETE", - headers: { ...opts.headers, ...topts.headers }, + headers: getHeaders(topts), }); }, }; diff --git a/src/server.ts b/src/server.ts index dadc6035..08875f5e 100644 --- a/src/server.ts +++ b/src/server.ts @@ -11,7 +11,7 @@ import { EventHandler, H3Event, } from "h3"; -import { Storage } from "./types"; +import type { Storage, TransactionOptions, StorageMeta } from "./types"; import { stringify } from "./_utils"; import { normalizeKey, normalizeBaseKey } from "./utils"; @@ -74,50 +74,36 @@ export function createH3StorageHandler( throw _httpError; } - // GET => getItem + // GET => getItem / getKeys if (event.method === "GET") { if (isBaseKey) { const keys = await storage.getKeys(key); return keys.map((key) => key.replace(/:/g, "/")); } - const isRaw = getRequestHeader(event, "accept") === "application/octet-stream"; - - const checkNotFound = (value: any) => { - if (value === null) { - throw createError({ - statusMessage: "KV value not found", - statusCode: 404, - }); - } - }; - - if (isRaw) { - const value = await storage.getItemRaw(key); - checkNotFound(value); - return value; - } else { - const value = await storage.getItem(key); - checkNotFound(value); - return stringify(value); + const driverValue = await (isRaw + ? storage.getItemRaw(key) + : storage.getItem(key)); + if (driverValue === null) { + throw createError({ + statusCode: 404, + statusMessage: "KV value not found", + }); } + setMetaHeaders(event, await storage.getMeta(key)); + return isRaw ? driverValue : stringify(driverValue); } - // HEAD => hasItem + meta (mtime) + // HEAD => hasItem + meta (mtime, ttl) if (event.method === "HEAD") { - const _hasItem = await storage.hasItem(key); - event.node.res.statusCode = _hasItem ? 200 : 404; - if (_hasItem) { - const meta = await storage.getMeta(key); - if (meta.mtime) { - setResponseHeader( - event, - "last-modified", - new Date(meta.mtime).toUTCString() - ); - } + if (!(await storage.hasItem(key))) { + throw createError({ + statusCode: 404, + statusMessage: "KV value not found", + }); } + setMetaHeaders(event, await storage.getMeta(key)); return ""; } @@ -125,13 +111,16 @@ export function createH3StorageHandler( if (event.method === "PUT") { const isRaw = getRequestHeader(event, "content-type") === "application/octet-stream"; + const topts: TransactionOptions = { + ttl: Number(getRequestHeader(event, "x-ttl")) || undefined, + }; if (isRaw) { const value = await readRawBody(event, false); - await storage.setItemRaw(key, value); + await storage.setItemRaw(key, value, topts); } else { const value = await readRawBody(event, "utf8"); if (value !== undefined) { - await storage.setItem(key, value); + await storage.setItem(key, value, topts); } } return "OK"; @@ -150,6 +139,20 @@ export function createH3StorageHandler( }); } +function setMetaHeaders(event: H3Event, meta: StorageMeta) { + if (meta.mtime) { + setResponseHeader( + event, + "last-modified", + new Date(meta.mtime).toUTCString() + ); + } + if (meta.ttl) { + setResponseHeader(event, "x-ttl", `${meta.ttl}`); + setResponseHeader(event, "cache-control", `max-age=${meta.ttl}`); + } +} + /** * This function creates a node-compatible handler for your custom storage server. * diff --git a/src/types.ts b/src/types.ts index b408bbd7..e72a14f5 100644 --- a/src/types.ts +++ b/src/types.ts @@ -11,10 +11,20 @@ export type Unwatch = () => MaybePromise; export interface StorageMeta { atime?: Date; mtime?: Date; + ttl?: number; [key: string]: StorageValue | Date | undefined; } -export type TransactionOptions = Record; +export interface TransactionOptions { + [key: string]: any; + + /** + * Time to live in seconds + * + * **Note:** Native TTL support is not supported by all drivers + */ + ttl?: number; +} export interface Driver { name?: string; diff --git a/test/server.test.ts b/test/server.test.ts index bc123e74..4991d638 100644 --- a/test/server.test.ts +++ b/test/server.test.ts @@ -4,11 +4,12 @@ import { listen } from "listhen"; import { $fetch } from "ofetch"; import { createStorage } from "../src"; import { createStorageServer } from "../src/server"; -import fs from "../src/drivers/fs.ts"; +import fsDriver from "../src/drivers/fs.ts"; +import httpDriver from "../src/drivers/http.ts"; describe("server", () => { it("basic", async () => { - const storage = createStorage(); + const storage = createTestStorage(); const storageServer = createStorageServer(storage, { authorize(req) { if (req.type === "read" && req.key.startsWith("private:")) { @@ -23,6 +24,10 @@ describe("server", () => { const fetchStorage = (url: string, options?: any) => $fetch(url, { baseURL: serverURL, ...options }); + const remoteStorage = createStorage({ + driver: httpDriver({ base: serverURL }), + }); + expect(await fetchStorage("foo/", {})).toMatchObject([]); await storage.setItem("foo/bar", "bar"); @@ -56,12 +61,17 @@ describe("server", () => { statusMessage: "Unauthorized Read", }); + // TTL + await storage.setItem("ttl", "ttl", { ttl: 1000 }); + expect(await storage.getMeta("ttl")).toMatchObject({ ttl: 1000 }); + expect(await remoteStorage.getMeta("ttl")).toMatchObject({ ttl: 1000 }); + await close(); }); it("properly encodes raw items", async () => { const storage = createStorage({ - driver: fs({ base: "./test/fs-storage" }), + driver: fsDriver({ base: "./test/fs-storage" }), }); const storageServer = createStorageServer(storage); const { close, url: serverURL } = await listen(storageServer.handle, { @@ -91,3 +101,52 @@ describe("server", () => { await close(); }); }); + +function createTestStorage() { + const data = new Map(); + const ttl = new Map(); + const storage = createStorage({ + driver: { + hasItem(key) { + return data.has(key); + }, + getItem(key) { + return data.get(key) ?? null; + }, + getItemRaw(key) { + return data.get(key) ?? null; + }, + setItem(key, value, opts) { + data.set(key, value); + if (opts?.ttl) { + ttl.set(key, opts.ttl); + } + }, + setItemRaw(key, value, opts) { + data.set(key, value); + if (opts?.ttl) { + ttl.set(key, opts.ttl); + } + }, + getMeta(key) { + return { + ttl: ttl.get(key), + }; + }, + removeItem(key) { + data.delete(key); + }, + getKeys() { + return [...data.keys()]; + }, + clear() { + data.clear(); + }, + dispose() { + data.clear(); + }, + }, + }); + + return storage; +} From 8dbd6051f254e2a3c68ba99cd5248cf25266b791 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Thu, 5 Sep 2024 12:52:17 +0200 Subject: [PATCH 2/2] revert type change for now --- src/types.ts | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/types.ts b/src/types.ts index e72a14f5..c92f528e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -15,16 +15,8 @@ export interface StorageMeta { [key: string]: StorageValue | Date | undefined; } -export interface TransactionOptions { - [key: string]: any; - - /** - * Time to live in seconds - * - * **Note:** Native TTL support is not supported by all drivers - */ - ttl?: number; -} +// TODO: type ttl +export type TransactionOptions = Record; export interface Driver { name?: string;