diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 425ccc10a..ce17dbd87 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,7 +10,7 @@ You can contribute changes to this repo by opening a pull request: 1) After forking this repository to your Git account, make the proposed changes on your forked branch. 2) Run tests and linting locally. - - [Install and run Docker](https://docs.docker.com/get-docker/) if you aren't already. + - [Install and run Docker](https://docs.docker.com/get-docker/) if you aren't already. NOTE: on docker set `enable host networking` to true as it is required for the tests in redis clustering. - Run `pnpm test:services:start`, allow for the services to come up. - Run `pnpm test`. 3) Commit your changes and push them to your forked repository. diff --git a/README.md b/README.md index c9d840380..a37489662 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ Keyv and its storage adapters are in this mono repo and there are details below on how to use this repository. In addtion we have a couple of other documents for review: -* [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) - Our code of conduct -* [CONTRIBUTING.md](CONTRIBUTING.md) - How to contribute to this project -* [SECURITY.md](SECURITY.md) - Security guidelines and supported versions +* [CODE_OF_CONDUCT](CODE_OF_CONDUCT.md) - Our code of conduct +* [CONTRIBUTING](CONTRIBUTING.md) - How to contribute to this project +* [SECURITY](SECURITY.md) - Security guidelines and supported versions ## Getting Started @@ -27,7 +27,7 @@ You can contribute changes to this repo by opening a pull request: 1) After forking this repository to your Git account, make the proposed changes on your forked branch. 2) Run tests and linting locally. - - [Install and run Docker](https://docs.docker.com/get-docker/) if you aren't already. + - [Install and run Docker](https://docs.docker.com/get-docker/) if you aren't already. NOTE: on docker set `enable host networking` to true as it is required for the tests in redis clustering. - Run `pnpm test:services:start`, allow for the services to come up. - Run `pnpm test`. 3) Commit your changes and push them to your forked repository. diff --git a/docker-compose-arm64.yaml b/docker-compose-arm64.yaml index bc24b9cb0..1f8295923 100644 --- a/docker-compose-arm64.yaml +++ b/docker-compose-arm64.yaml @@ -69,7 +69,7 @@ services: REDIS_HOST: redis ports: - 6379:6379 - keyv_redis_1: + keyv_redis_tls_1: image: redis:latest command: redis-server --port 0 --tls-port 6380 --tls-cert-file /tls/redis.crt --tls-key-file /tls/redis.key --tls-ca-cert-file /tls/ca.crt --tls-auth-clients no environment: @@ -94,4 +94,5 @@ services: - ALLOW_NONE_AUTHENTICATION=yes ports: - 2379:2379 - - 2380:2380 \ No newline at end of file + - 2380:2380 + \ No newline at end of file diff --git a/docker-compose-redis-cluster.yaml b/docker-compose-redis-cluster.yaml new file mode 100644 index 000000000..177a0c09b --- /dev/null +++ b/docker-compose-redis-cluster.yaml @@ -0,0 +1,39 @@ +x-redis-cluster-base: &redis-cluster-base + image: docker.io/bitnami/redis-cluster:latest + network_mode: host + +services: + redis-cluster-1: + container_name: redis-cluster-1 + <<: *redis-cluster-base + environment: + - 'ALLOW_EMPTY_PASSWORD=yes' + - 'REDIS_NODES=127.0.0.1:7001 127.0.0.1:7002 127.0.0.1:7003' + - 'REDIS_CLUSTER_DYNAMIC_IPS=no' + - 'REDIS_CLUSTER_ANNOUNCE_IP=127.0.0.1' + - 'REDIS_PORT_NUMBER=7001' + + redis-cluster-2: + container_name: redis-cluster-2 + <<: *redis-cluster-base + environment: + - 'ALLOW_EMPTY_PASSWORD=yes' + - 'REDIS_NODES=127.0.0.1:7001 127.0.0.1:7002 127.0.0.1:7003' + - 'REDIS_CLUSTER_DYNAMIC_IPS=no' + - 'REDIS_CLUSTER_ANNOUNCE_IP=127.0.0.1' + - 'REDIS_PORT_NUMBER=7002' + + redis-cluster-3: + container_name: redis-cluster-3 + <<: *redis-cluster-base + depends_on: + - redis-cluster-1 + - redis-cluster-2 + environment: + - 'ALLOW_EMPTY_PASSWORD=yes' + - 'REDIS_NODES=127.0.0.1:7001 127.0.0.1:7002 127.0.0.1:7003' + - 'REDIS_CLUSTER_DYNAMIC_IPS=no' + - 'REDIS_CLUSTER_ANNOUNCE_IP=127.0.0.1' + - 'REDIS_PORT_NUMBER=7003' + - 'REDIS_CLUSTER_REPLICAS=0' + - 'REDIS_CLUSTER_CREATOR=yes' \ No newline at end of file diff --git a/packages/redis/README.md b/packages/redis/README.md index 808b30f29..49d1b93af 100644 --- a/packages/redis/README.md +++ b/packages/redis/README.md @@ -183,7 +183,9 @@ const cluster = createCluster({ const keyv = new Keyv({ store: new KeyvRedis(cluster) }); ``` -You can learn more about the `createCluster` function in the [documentation](https://github.com/redis/node-redis/blob/master/docs/clustering.md) at https://github.com/redis/node-redis/tree/master/docs. +There are some features that are not supported in clustering such as `clear()` and `iterator()`. This is because the `SCAN` command is not supported in clustering. If you need to clear or delete keys you can use the `deleteMany()` method. + +You can learn more about the `createCluster` function in the [documentation](https://github.com/redis/node-redis/blob/master/docs/clustering.md) at https://github.com/redis/node-redis/tree/master/docs. Here is an example of how to use TLS: diff --git a/packages/redis/src/index.ts b/packages/redis/src/index.ts index 8a86dcd48..6151bd957 100644 --- a/packages/redis/src/index.ts +++ b/packages/redis/src/index.ts @@ -1,5 +1,11 @@ import EventEmitter from 'node:events'; -import {createClient, type RedisClientType, type RedisClientOptions} from 'redis'; +import { + createClient, createCluster, type RedisClientType, type RedisClientOptions, type RedisClusterType, + type RedisClusterOptions, + type RedisModules, + type RedisFunctions, + type RedisScripts, +} from 'redis'; import {Keyv, type KeyvStoreAdapter} from 'keyv'; export type KeyvRedisOptions = { @@ -47,9 +53,12 @@ export type KeyvRedisEntry = { */ ttl?: number; }; + +export type RedisClientConnectionType = RedisClientType | RedisClusterType; + // eslint-disable-next-line unicorn/prefer-event-target export default class KeyvRedis extends EventEmitter implements KeyvStoreAdapter { - private _client: RedisClientType = createClient() as RedisClientType; + private _client: RedisClientConnectionType = createClient() as RedisClientType; private _namespace: string | undefined; private _keyPrefixSeparator = '::'; private _clearBatchSize = 1000; @@ -60,35 +69,34 @@ export default class KeyvRedis extends EventEmitter implements KeyvStoreAdapter * @param {string | RedisClientOptions | RedisClientType} [connect] How to connect to the Redis server. If string pass in the url, if object pass in the options, if RedisClient pass in the client. * @param {KeyvRedisOptions} [options] Options for the adapter such as namespace, keyPrefixSeparator, and clearBatchSize. */ - constructor(connect?: string | RedisClientOptions | RedisClientType, options?: KeyvRedisOptions) { + constructor(connect?: string | RedisClientOptions | RedisClusterOptions | RedisClientConnectionType, options?: KeyvRedisOptions) { super(); if (connect) { if (typeof connect === 'string') { this._client = createClient({url: connect}) as RedisClientType; - } else if ((connect as RedisClientType).connect !== undefined) { - this._client = connect as RedisClientType; + } else if ((connect as any).connect !== undefined) { + this._client = this.isClientCluster(connect as RedisClientConnectionType) ? connect as RedisClusterType : connect as RedisClientType; } else if (connect instanceof Object) { - this._client = createClient(connect as RedisClientOptions) as RedisClientType; + this._client = (connect as any).rootNodes === undefined ? createClient(connect as RedisClientOptions) as RedisClientType : createCluster(connect as RedisClusterOptions) as RedisClusterType; } } this.setOptions(options); - this.initClient(); } /** * Get the Redis client. */ - public get client(): RedisClientType { + public get client(): RedisClientConnectionType { return this._client; } /** * Set the Redis client. */ - public set client(value: RedisClientType) { + public set client(value: RedisClientConnectionType) { this._client = value; this.initClient(); } @@ -97,13 +105,20 @@ export default class KeyvRedis extends EventEmitter implements KeyvStoreAdapter * Get the options for the adapter. */ public get opts(): KeyvRedisPropertyOptions { - return { + let url = ''; + if ((this._client as RedisClientType).options) { + url = (this._client as RedisClientType).options?.url ?? 'redis://localhost:6379'; + } + + const results: KeyvRedisPropertyOptions = { namespace: this._namespace, keyPrefixSeparator: this._keyPrefixSeparator, clearBatchSize: this._clearBatchSize, dialect: 'redis', - url: this._client?.options?.url ?? 'redis://localhost:6379', + url, }; + + return results; } /** @@ -176,7 +191,7 @@ export default class KeyvRedis extends EventEmitter implements KeyvStoreAdapter /** * Get the Redis URL used to connect to the server. This is used to get a connected client. */ - public async getClient(): Promise { + public async getClient(): Promise { if (!this._client.isOpen) { await this._client.connect(); } @@ -368,18 +383,39 @@ export default class KeyvRedis extends EventEmitter implements KeyvStoreAdapter return key; } + /** + * Is the client a cluster. + * @returns {boolean} - true if the client is a cluster, false if not + */ + public isCluster(): boolean { + return this.isClientCluster(this._client); + } + /** * Get an async iterator for the keys and values in the store. If a namespace is provided, it will only iterate over keys with that namespace. * @param {string} [namespace] - the namespace to iterate over * @returns {AsyncGenerator<[string, T | undefined], void, unknown>} - async iterator with key value pairs */ public async * iterator(namespace?: string): AsyncGenerator<[string, Value | undefined], void, unknown> { + if (this.isCluster()) { + throw new Error('Iterating over keys in a cluster is not supported.'); + } else { + yield * this.iteratorClient(namespace); + } + } + + /** + * Get an async iterator for the keys and values in the store. If a namespace is provided, it will only iterate over keys with that namespace. + * @param {string} [namespace] - the namespace to iterate over + * @returns {AsyncGenerator<[string, T | undefined], void, unknown>} - async iterator with key value pairs + */ + public async * iteratorClient(namespace?: string): AsyncGenerator<[string, Value | undefined], void, unknown> { const client = await this.getClient(); const match = namespace ? `${namespace}${this._keyPrefixSeparator}*` : '*'; let cursor = '0'; do { // eslint-disable-next-line no-await-in-loop, @typescript-eslint/naming-convention - const result = await client.scan(Number.parseInt(cursor, 10), {MATCH: match, TYPE: 'string'}); + const result = await (client as RedisClientType).scan(Number.parseInt(cursor, 10), {MATCH: match, TYPE: 'string'}); cursor = result.cursor.toString(); let {keys} = result; @@ -401,13 +437,13 @@ export default class KeyvRedis extends EventEmitter implements KeyvStoreAdapter /** * Clear all keys in the store. - * IMPORTANT: this can cause performance issues if there are a large number of keys in the store. Use with caution as not recommended for production. + * IMPORTANT: this can cause performance issues if there are a large number of keys in the store and worse with clusters. Use with caution as not recommended for production. * If a namespace is not set it will clear all keys with no prefix. * If a namespace is set it will clear all keys with that namespace. * @returns {Promise} */ public async clear(): Promise { - await this.clearNamespace(this._namespace); + await (this.isCluster() ? this.clearNamespaceCluster(this._namespace) : this.clearNamespace(this._namespace)); } private async clearNamespace(namespace?: string): Promise { @@ -418,9 +454,8 @@ export default class KeyvRedis extends EventEmitter implements KeyvStoreAdapter const client = await this.getClient(); do { - // Use SCAN to find keys incrementally in batches // eslint-disable-next-line no-await-in-loop, @typescript-eslint/naming-convention - const result = await client.scan(Number.parseInt(cursor, 10), {MATCH: match, COUNT: batchSize, TYPE: 'string'}); + const result = await (client as RedisClientType).scan(Number.parseInt(cursor, 10), {MATCH: match, COUNT: batchSize, TYPE: 'string'}); cursor = result.cursor.toString(); let {keys} = result; @@ -450,6 +485,18 @@ export default class KeyvRedis extends EventEmitter implements KeyvStoreAdapter } } + private async clearNamespaceCluster(namespace?: string): Promise { + throw new Error('Clearing all keys in a cluster is not supported.'); + } + + private isClientCluster(client: RedisClientConnectionType): boolean { + if ((client as any).options === undefined && (client as any).scan === undefined) { + return true; + } + + return false; + } + private setOptions(options?: KeyvRedisOptions): void { if (!options) { return; @@ -493,7 +540,7 @@ export function createKeyv(connect?: string | RedisClientOptions | RedisClientTy } export { - createClient, createCluster, type RedisClientOptions, type RedisClientType, + createClient, createCluster, type RedisClientOptions, type RedisClientType, type RedisClusterType, type RedisClusterOptions, } from 'redis'; export { diff --git a/packages/redis/test/cluster.ts b/packages/redis/test/cluster.ts new file mode 100644 index 000000000..e3b05b54b --- /dev/null +++ b/packages/redis/test/cluster.ts @@ -0,0 +1,100 @@ +import {describe, test, expect} from 'vitest'; +import KeyvRedis, {createCluster} from '../src/index.js'; + +const defaultClusterOptions = { + rootNodes: [ + { + url: 'redis://localhost:7001', + }, + { + url: 'redis://localhost:7002', + }, + { + url: 'redis://localhost:7003', + }, + ], + useReplicas: true, +}; + +describe('KeyvRedis Cluster', () => { + test('should be able to connect to a cluster', async () => { + const cluster = createCluster(defaultClusterOptions); + + const keyvRedis = new KeyvRedis(cluster); + + expect(keyvRedis).toBeDefined(); + expect(keyvRedis.client).toEqual(cluster); + }); + + test('should be able to send in cluster options', async () => { + const keyvRedis = new KeyvRedis(defaultClusterOptions); + expect(keyvRedis.isCluster()).toBe(true); + }); + + test('shoudl be able to set the redis cluster client', async () => { + const cluster = createCluster(defaultClusterOptions); + + const keyvRedis = new KeyvRedis(); + expect(keyvRedis.isCluster()).toBe(false); + + keyvRedis.client = cluster; + expect(keyvRedis.client).toEqual(cluster); + expect(keyvRedis.isCluster()).toBe(true); + }); + + test('should be able to set a value', async () => { + const cluster = createCluster(defaultClusterOptions); + + const keyvRedis = new KeyvRedis(cluster); + + await keyvRedis.delete('test-cl1'); + + const undefinedResult = await keyvRedis.get('test-cl1'); + expect(undefinedResult).toBeUndefined(); + + await keyvRedis.set('test-cl1', 'test'); + + const result = await keyvRedis.get('test-cl1'); + + expect(result).toBe('test'); + + await keyvRedis.delete('test-cl1'); + }); + + test('should thrown an error on clear', async () => { + const cluster = createCluster(defaultClusterOptions); + + const keyvRedis = new KeyvRedis(cluster); + + let errorThrown = false; + try { + await keyvRedis.clear(); + } catch (error) { + expect(error).toBeDefined(); + errorThrown = true; + } + + expect(errorThrown).toBe(true); + }); + + test('should throw an error on iterator', async () => { + const cluster = createCluster(defaultClusterOptions); + + const keyvRedis = new KeyvRedis(cluster); + + let errorThrown = false; + try { + const keys = []; + const values = []; + for await (const [key, value] of keyvRedis.iterator('foo')) { + keys.push(key); + values.push(value); + } + } catch (error) { + expect(error).toBeDefined(); + errorThrown = true; + } + + expect(errorThrown).toBe(true); + }); +}); diff --git a/packages/redis/test/suite.ts b/packages/redis/test/suite.ts index b1e0e9894..cbf51b663 100644 --- a/packages/redis/test/suite.ts +++ b/packages/redis/test/suite.ts @@ -1,13 +1,13 @@ import * as test from 'vitest'; import keyvTestSuite, {keyvIteratorTests} from '@keyv/test-suite'; import {Keyv} from 'keyv'; -import KeyvRedis from '../src/index.js'; +import KeyvRedis, {type RedisClientType} from '../src/index.js'; const redisUrl = 'redis://localhost:6379/5'; const store = () => new KeyvRedis(redisUrl); test.afterAll(async () => { - const client = await store().getClient(); + const client = await store().getClient() as RedisClientType; await client.flushDb(); await store().disconnect(); }); diff --git a/packages/redis/test/test.ts b/packages/redis/test/test.ts index edc7b69e2..4365fadc5 100644 --- a/packages/redis/test/test.ts +++ b/packages/redis/test/test.ts @@ -3,7 +3,7 @@ import { } from 'vitest'; import {createClient, type RedisClientType} from 'redis'; import {delay} from '@keyv/test-suite'; -import KeyvRedis, {createKeyv} from '../src/index.js'; +import KeyvRedis, {createKeyv, createCluster} from '../src/index.js'; describe('KeyvRedis', () => { test('should be a class', () => { @@ -43,13 +43,13 @@ describe('KeyvRedis', () => { test('should be able to pass in client options to constructor', () => { const uri = 'redis://foo:6379'; const keyvRedis = new KeyvRedis({url: uri}); - expect(keyvRedis.client.options?.url).toBe(uri); + expect((keyvRedis.client as RedisClientType).options?.url).toBe(uri); }); test('should be able to pass in the url and options to constructor', () => { const uri = 'redis://localhost:6379'; const keyvRedis = new KeyvRedis(uri, {namespace: 'test'}); - expect(keyvRedis.client.options?.url).toBe(uri); + expect((keyvRedis.client as RedisClientType).options?.url).toBe(uri); expect(keyvRedis.namespace).toBe('test'); }); @@ -62,7 +62,6 @@ describe('KeyvRedis', () => { useUnlink: true, }; const keyvRedis = new KeyvRedis(uri, options); - expect(keyvRedis.client.options?.url).toBe(uri); expect(keyvRedis.namespace).toBe('test'); expect(keyvRedis.keyPrefixSeparator).toBe('->'); expect(keyvRedis.clearBatchSize).toBe(100); @@ -81,9 +80,10 @@ describe('KeyvRedis', () => { expect(keyvRedis.useUnlink).toBe(false); }); - test('should be able to get and set opts', () => { + test('should be able to get and set opts', async () => { const keyvRedis = new KeyvRedis(); keyvRedis.opts = {namespace: 'test', keyPrefixSeparator: ':1', clearBatchSize: 2000}; + expect(keyvRedis.opts).toEqual({ namespace: 'test', keyPrefixSeparator: ':1', clearBatchSize: 2000, dialect: 'redis', url: 'redis://localhost:6379', }); @@ -93,7 +93,7 @@ describe('KeyvRedis', () => { describe('KeyvRedis Methods', () => { beforeEach(async () => { const keyvRedis = new KeyvRedis(); - const client = await keyvRedis.getClient(); + const client = await keyvRedis.getClient() as RedisClientType; await client.flushDb(); await keyvRedis.disconnect(); }); @@ -150,7 +150,7 @@ describe('KeyvRedis Methods', () => { test('should do nothing if no keys on clear', async () => { const keyvRedis = new KeyvRedis(); - const client = await keyvRedis.getClient(); + const client = await keyvRedis.getClient() as RedisClientType; await client.flushDb(); await keyvRedis.clear(); keyvRedis.namespace = 'ns1'; @@ -235,7 +235,7 @@ describe('KeyvRedis Methods', () => { describe('KeyvRedis Namespace', () => { beforeEach(async () => { const keyvRedis = new KeyvRedis(); - const client = await keyvRedis.getClient(); + const client = await keyvRedis.getClient() as RedisClientType; await client.flushDb(); await keyvRedis.disconnect(); }); @@ -264,7 +264,7 @@ describe('KeyvRedis Namespace', () => { test('should clear with no namespace but not the namespace ones', async () => { const keyvRedis = new KeyvRedis(); - const client = await keyvRedis.getClient(); + const client = await keyvRedis.getClient() as RedisClientType; await client.flushDb(); keyvRedis.namespace = 'ns1'; await keyvRedis.set('foo91', 'bar'); @@ -280,7 +280,7 @@ describe('KeyvRedis Namespace', () => { test('should clear namespace but not other ones', async () => { const keyvRedis = new KeyvRedis(); - const client = await keyvRedis.getClient(); + const client = await keyvRedis.getClient() as RedisClientType; await client.flushDb(); keyvRedis.namespace = 'ns1'; await keyvRedis.set('foo921', 'bar'); @@ -333,7 +333,7 @@ describe('KeyvRedis Namespace', () => { describe('KeyvRedis Iterators', () => { beforeEach(async () => { const keyvRedis = new KeyvRedis(); - const client = await keyvRedis.getClient(); + const client = await keyvRedis.getClient() as RedisClientType; await client.flushDb(); await keyvRedis.disconnect(); }); diff --git a/packages/website/site/docs/index.md b/packages/website/site/docs/index.md index 23721dc60..e9a68ae45 100644 --- a/packages/website/site/docs/index.md +++ b/packages/website/site/docs/index.md @@ -25,6 +25,7 @@ By default, everything is stored in memory; you can optionally also install a st ```sh npm install --save @keyv/redis +npm install --save @keyv/valkey npm install --save @keyv/memcache npm install --save @keyv/mongo npm install --save @keyv/sqlite diff --git a/test-services-start.sh b/test-services-start.sh index 2fbc84e94..afe0de4a0 100755 --- a/test-services-start.sh +++ b/test-services-start.sh @@ -5,9 +5,9 @@ architecture=$(uname -m) if [[ "$architecture" == "arm"* || "$architecture" == "aarch64" ]]; then echo "Running on an ARM platform" # Call ARM-specific command - docker compose -f ./docker-compose-arm64.yaml up -d + docker compose -f ./docker-compose-arm64.yaml -f ./docker-compose-redis-cluster.yaml up -d else echo "Running on a non-ARM platform" # Call non-ARM command - docker compose -f ./docker-compose.yaml up -d + docker compose -f ./docker-compose.yaml -f ./docker-compose-redis-cluster.yaml up -d fi diff --git a/test-services-stop.sh b/test-services-stop.sh index 5dc6d4ee9..1d56984d2 100755 --- a/test-services-stop.sh +++ b/test-services-stop.sh @@ -5,9 +5,9 @@ architecture=$(uname -m) if [[ "$architecture" == "arm"* || "$architecture" == "aarch64" ]]; then echo "Running on an ARM platform" # Call ARM-specific command - docker compose -f ./docker-compose-arm64.yaml down -v + docker compose -f ./docker-compose-arm64.yaml -f ./docker-compose-redis-cluster.yaml down -v else echo "Running on a non-ARM platform" # Call non-ARM command - docker compose -f ./docker-compose.yaml down -v + docker compose -f ./docker-compose.yaml -f ./docker-compose-redis-cluster.yaml down -v fi