From 3a5d865b54c1e5118915fc4c37e5a6e631b0d38d Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Wed, 8 Sep 2021 13:52:07 +0200 Subject: [PATCH] feat: meta support resolves #3 --- README.md | 46 ++++++++++++++++++++++++++++++++++++--- src/drivers/fs.ts | 7 +++++- src/drivers/http.ts | 12 +++++++++- src/server.ts | 8 ++++++- src/storage.ts | 42 ++++++++++++++++++++++++++++++----- src/types.ts | 22 +++++++++++++++---- test/drivers/fs.test.ts | 6 +++++ test/drivers/http.test.ts | 5 +++++ test/server.test.ts | 3 ++- 9 files changed, 135 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 995404b1..a137f10b 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ - Unix-style mountable paths (multi driver) - Default in-memory storage - Tree-shakable and lightweight core +- Driver native and custom user metadata support - Native aware value serialization and deserialization - Restore initial state (hydration) - State snapshot @@ -30,7 +31,10 @@ - [`storage.hasItem(key)`](#storagehasitemkey) - [`storage.getItem(key)`](#storagegetitemkey) - [`storage.setItem(key, value)`](#storagesetitemkey-value) - - [`storage.removeItem(key)`](#storageremoveitemkey) + - [`storage.removeItem(key, removeMeta = true)`](#storageremoveitemkey-removemeta--true) + - [`storage.getMeta(key, nativeOnly?)`](#storagegetmetakey-nativeonly) + - [`storage.setMeta(key)`](#storagesetmetakey) + - [`storage.removeMeta(key)`](#storageremovemetakey) - [`storage.getKeys(base?)`](#storagegetkeysbase) - [`storage.clear(base?)`](#storageclearbase) - [`storage.dispose()`](#storagedispose) @@ -105,18 +109,50 @@ If value is `undefined`, it is same as calling `removeItem(key)`. await storage.setItem('foo:bar', 'baz') ``` -### `storage.removeItem(key)` +### `storage.removeItem(key, removeMeta = true)` -Remove a value from storage. +Remove a value (and it's meta) from storage. ```js await storage.removeItem('foo:bar') ``` +### `storage.getMeta(key, nativeOnly?)` + +Get metadata object for a specific key. + +This data is fetched from two sources: +- Driver native meta (like file creation time) +- Custom meta set by `storage.setMeta` (overrides driver native meta) + +```js +await storage.getMeta('foo:bar') // For fs driver returns an object like { mtime, atime, size } +``` + +### `storage.setMeta(key)` + +Set custom meta for a specific key by adding a `$` suffix. + +```js +await storage.setMeta('foo:bar', { flag: 1 }) +// Same as storage.setItem('foo:bar$', { flag: 1 }) +``` + +### `storage.removeMeta(key)` + +Remove meta for a specific key by adding a `$` suffix. + +```js +await storage.removeMeta('foo:bar',) +// Same as storage.removeMeta('foo:bar$') +``` + ### `storage.getKeys(base?)` Get all keys. Returns an array of `string`. +Meta keys (ending with `$`) will be filtered. + If a base is provided, only keys starting with base will be returned also only mounts starting with base will be queried. Keys still have full path. ```js @@ -240,6 +276,8 @@ npx unstorage . Maps data to real filesystem using directory structure for nested keys. Supports watching using [chokidar](https://github.com/paulmillr/chokidar). +This driver implements meta for each key including `mtime` (last modified time), `atime` (last access time), and `size` (file size) using `fs.stat`. + ```js import { createStorage } from 'unstorage' import fsDriver from 'unstorage/drivers/memory' @@ -293,6 +331,8 @@ const storage = createStorage({ Use a remote HTTP/HTTPS endpoint as data storage. Supports built-in [http server](#storage-server) methods. +This driver implements meta for each key including `mtime` (last modified time) and `status` from HTTP headers by making a `HEAD` request. + ```js import { createStorage } from 'unstorage' import httpDriver from 'unstorage/drivers/http' diff --git a/src/drivers/fs.ts b/src/drivers/fs.ts index cb489de2..68f0ef94 100644 --- a/src/drivers/fs.ts +++ b/src/drivers/fs.ts @@ -1,4 +1,4 @@ -import { existsSync } from 'fs' +import { existsSync, promises as fsp } from 'fs' import { resolve, relative, join } from 'path' import { FSWatcher, WatchOptions, watch } from 'chokidar' import { defineDriver } from './utils' @@ -35,6 +35,11 @@ export default defineDriver((opts: FSStorageOptions = {}) => { getItem (key) { return readFile(r(key)) }, + async getMeta (key) { + const { atime, mtime, size } = await fsp.stat(r(key)) + .catch(() => ({ atime: undefined, mtime: undefined, size: undefined })) + return { atime, mtime, size } + }, setItem (key, value) { return writeFile(r(key), value) }, diff --git a/src/drivers/http.ts b/src/drivers/http.ts index 4a6f23aa..3f0e014a 100644 --- a/src/drivers/http.ts +++ b/src/drivers/http.ts @@ -17,9 +17,19 @@ export default defineDriver((opts: HTTPOptions = {}) => { .catch(() => false) }, async getItem (key) { - const value = $fetch(r(key)) + const value = await $fetch(r(key)) return value }, + async getMeta (key) { + const res = await $fetch.raw(r(key), { method: 'HEAD' }) + let mtime = undefined + const _lastModified = res.headers.get('last-modified') + if (_lastModified) { mtime = new Date(_lastModified) } + return { + status: res.status, + mtime + } + }, async setItem(key, value) { await $fetch(r(key), { method: 'PUT', body: stringify(value) }) }, diff --git a/src/server.ts b/src/server.ts index 0f0630c5..fed7f6f5 100644 --- a/src/server.ts +++ b/src/server.ts @@ -22,10 +22,16 @@ export function createStorageServer (storage: Storage, _opts: StorageServerOptio } return stringify(val) } - // HEAD => hasItem + // HEAD => hasItem + meta (mtime) if (req.method === 'HEAD') { const _hasItem = await storage.hasItem(req.url!) res.statusCode = _hasItem ? 200 : 404 + if (_hasItem) { + const meta = await storage.getMeta(req.url!) + if (meta.mtime) { + res.setHeader('Last-Modified', new Date(meta.mtime).toUTCString()) + } + } return '' } // PUT => setItem diff --git a/src/storage.ts b/src/storage.ts index f79a6ca3..4ef792b6 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -63,6 +63,7 @@ export function createStorage (opts: CreateStorageOptions = {}): Storage { } const storage: Storage = { + // Item hasItem (key) { key = normalizeKey(key) const { relativeKey, driver } = getMount(key) @@ -84,14 +85,43 @@ export function createStorage (opts: CreateStorageOptions = {}): Storage { onChange('update', key) } }, - async removeItem (key) { + async removeItem (key, removeMeta = true) { key = normalizeKey(key) const { relativeKey, driver } = getMount(key) await asyncCall(driver.removeItem, relativeKey) + if (removeMeta) { + await asyncCall(driver.removeItem, relativeKey + '$') + } if (!driver.watch) { onChange('remove', key) } }, + // Meta + async getMeta (key, nativeMetaOnly) { + key = normalizeKey(key) + const { relativeKey, driver } = getMount(key) + const meta = Object.create(null) + if (driver.getMeta) { + Object.assign(meta, await asyncCall(driver.getMeta, relativeKey)) + } + if (!nativeMetaOnly) { + const val = await asyncCall(driver.getItem, relativeKey + '$').then(val => destr(val)) + if (val && typeof val === 'object') { + // TODO: Support date by destr? + if (typeof val.atime === 'string') { val.atime = new Date(val.atime) } + if (typeof val.mtime === 'string') { val.mtime = new Date(val.mtime) } + Object.assign(meta, val) + } + } + return meta + }, + setMeta (key: string, value: any) { + return this.setItem(key + '$', value) + }, + removeMeta (key: string) { + return this.removeItem(key + '$') + }, + // Keys async getKeys (base) { base = normalizeBase(base) const keyGroups = await Promise.all(getMounts(base).map(async (mount) => { @@ -101,6 +131,7 @@ export function createStorage (opts: CreateStorageOptions = {}): Storage { const keys = keyGroups.flat() return base ? keys.filter(key => key.startsWith(base!)) : keys }, + // Utils async clear (base) { base = normalizeBase(base) await Promise.all(getMounts(base).map(m => asyncCall(m.driver.clear))) @@ -108,6 +139,11 @@ export function createStorage (opts: CreateStorageOptions = {}): Storage { async dispose () { await Promise.all(Object.values(ctx.mounts).map(driver => dispose(driver))) }, + async watch (callback) { + await startWatch() + ctx.watchListeners.push(callback) + }, + // Mount mount (base, driver) { base = normalizeBase(base) if (base && ctx.mounts[base]) { @@ -134,10 +170,6 @@ export function createStorage (opts: CreateStorageOptions = {}): Storage { } ctx.mountpoints = ctx.mountpoints.filter(key => key !== base) delete ctx.mounts[base] - }, - async watch (callback) { - await startWatch() - ctx.watchListeners.push(callback) } } diff --git a/src/types.ts b/src/types.ts index c2f9a44c..83875758 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,13 +1,19 @@ export type StorageValue = null | string | String | number | Number | boolean | Boolean | object - export type WatchEvent = 'update' | 'remove' export type WatchCallback = (event: WatchEvent, key: string) => any +export interface StorageMeta { + atime?: Date, + mtime?: Date + [key: string]: StorageValue | Date | undefined +} + export interface Driver { hasItem: (key: string) => boolean | Promise getItem: (key: string) => StorageValue setItem: (key: string, value: string) => void | Promise removeItem: (key: string) => void | Promise + getMeta?: (key: string) => StorageMeta | Promise getKeys: () => string[] | Promise clear: () => void | Promise dispose?: () => void | Promise @@ -15,14 +21,22 @@ export interface Driver { } export interface Storage { + // Item hasItem: (key: string) => Promise getItem: (key: string) => Promise setItem: (key: string, value: StorageValue) => Promise - removeItem: (key: string) => Promise + removeItem: (key: string, removeMeta?: boolean) => Promise + // Meta + getMeta: (key: string, nativeMetaOnly?: true) => StorageMeta | Promise + setMeta: (key: string, value: StorageMeta) => Promise + removeMeta: (key: string) => Promise + // Keys getKeys: (base?: string) => Promise + // Utils clear: (base?: string) => Promise - mount: (base: string, driver: Driver) => Storage - unmount: (base: string, dispose?: boolean) => Promise dispose: () => Promise watch: (callback: WatchCallback) => Promise + // Mount + mount: (base: string, driver: Driver) => Storage + unmount: (base: string, dispose?: boolean) => Promise } diff --git a/test/drivers/fs.test.ts b/test/drivers/fs.test.ts index 0ffd62dc..4feae088 100644 --- a/test/drivers/fs.test.ts +++ b/test/drivers/fs.test.ts @@ -12,6 +12,12 @@ describe('drivers: fs', () => { it('check filesystem', async () => { expect(await readFile(resolve(dir, 's1/a'))).toBe('test_data') }) + it('native meta', async () => { + const meta = await ctx.storage.getMeta('/s1/a') + expect(meta.atime?.constructor.name).toBe('Date') + expect(meta.mtime?.constructor.name).toBe('Date') + expect(meta.size).toBeGreaterThan(0) + }) it('watch filesystem', async () => { const watcher = jest.fn() await ctx.storage.watch(watcher) diff --git a/test/drivers/http.test.ts b/test/drivers/http.test.ts index 3e7ad4d2..d0d24226 100644 --- a/test/drivers/http.test.ts +++ b/test/drivers/http.test.ts @@ -17,6 +17,11 @@ describe('drivers: http', () => { expect(await storage.getItem('http:foo')).toBe('bar') expect(await storage.hasItem('/http/foo')).toBe(true) + const date = new Date() + await storage.setMeta('/http/foo', { mtime: date }) + + expect(await storage.getMeta('/http/foo')).toMatchObject({ mtime: date, status: 200 }) + await close() }) }) diff --git a/test/server.test.ts b/test/server.test.ts index b69e7fcd..2b9fe607 100644 --- a/test/server.test.ts +++ b/test/server.test.ts @@ -14,11 +14,12 @@ describe('server', () => { expect(await fetchStorage('foo', {})).toMatchObject([]) await storage.setItem('foo/bar', 'bar') + await storage.setMeta('foo/bar', { mtime: new Date() }) expect(await fetchStorage('foo/bar')).toBe('bar') expect(await fetchStorage('foo/bar', { method: 'PUT', body: 'updated' })).toBe('OK') expect(await fetchStorage('foo/bar')).toBe('updated') - expect(await fetchStorage('/')).toMatchObject(['foo/bar']) + expect(await fetchStorage('/')).toMatchObject(['foo/bar', 'foo/bar$']) expect(await fetchStorage('foo/bar', { method: 'DELETE' })).toBe('OK') expect(await fetchStorage('foo/bar', {})).toMatchObject([])