From 7f68f3272f479b368fa20ee3c5cded49bdbf66e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Arturo=20Cabral=20Mej=C3=ADa?= Date: Sun, 20 Oct 2024 19:26:30 -0400 Subject: [PATCH 1/5] fix: save event from mirrors --- .gitignore | 5 +- Dockerfile | 2 +- docker-compose.yml | 4 +- src/@types/settings.ts | 3 + src/app/static-mirroring-worker.ts | 198 +++++++++++++++++- .../static-mirroring.worker-factory.ts | 15 +- src/utils/settings.ts | 2 +- 7 files changed, 217 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index e18b6ad3..ae5f179c 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,7 @@ dist *.env # Nostr data folder -.nostr \ No newline at end of file +.nostr + +# Docker Compose overrides +docker-compose.overrides.yml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2ab01f64..6e1c9709 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18-alpine3.16 as build +FROM node:18-alpine3.16 AS build WORKDIR /build diff --git a/docker-compose.yml b/docker-compose.yml index 80bd6412..b7cacded 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -75,6 +75,7 @@ services: restart: on-failure networks: default: + nostream-db: image: postgres container_name: nostream-db @@ -96,6 +97,7 @@ services: timeout: 5s retries: 5 start_period: 360s + nostream-cache: image: redis:7.0.5-alpine3.16 container_name: nostream-cache @@ -110,6 +112,7 @@ services: interval: 1s timeout: 5s retries: 5 + nostream-migrate: image: node:18-alpine3.16 container_name: nostream-migrate @@ -131,7 +134,6 @@ services: condition: service_healthy networks: default: - ipv4_address: 10.10.10.254 networks: default: diff --git a/src/@types/settings.ts b/src/@types/settings.ts index 3717fbe5..4af81bb4 100644 --- a/src/@types/settings.ts +++ b/src/@types/settings.ts @@ -203,6 +203,9 @@ export interface Mirror { address: string filters?: SubscriptionFilter[] secret?: Secret + limits?: { + event?: EventLimits + } } export interface Mirroring { diff --git a/src/app/static-mirroring-worker.ts b/src/app/static-mirroring-worker.ts index 9c2500ec..3ce95278 100644 --- a/src/app/static-mirroring-worker.ts +++ b/src/app/static-mirroring-worker.ts @@ -1,12 +1,15 @@ -import { anyPass, map, path } from 'ramda' +import { anyPass, map, mergeDeepRight, path } from 'ramda' import { RawData, WebSocket } from 'ws' import cluster from 'cluster' import { randomUUID } from 'crypto' import { createRelayedEventMessage, createSubscriptionMessage } from '../utils/messages' -import { isEventIdValid, isEventMatchingFilter, isEventSignatureValid } from '../utils/event' -import { Mirror, Settings } from '../@types/settings' +import { EventLimits, FeeSchedule, Mirror, Settings } from '../@types/settings' +import { getEventExpiration, getEventProofOfWork, getPubkeyProofOfWork, getPublicKey, getRelayPrivateKey, isEventIdValid, isEventKindOrRangeMatch, isEventMatchingFilter, isEventSignatureValid, isExpiredEvent } from '../utils/event' +import { IEventRepository, IUserRepository } from '../@types/repositories' import { createLogger } from '../factories/logger-factory' +import { Event } from '../@types/event' +import { EventExpirationTimeMetadataKey } from '../constants/base' import { IRunnable } from '../@types/base' import { OutgoingEventMessage } from '../@types/messages' import { RelayedEvent } from '../@types/event' @@ -19,6 +22,8 @@ export class StaticMirroringWorker implements IRunnable { private config: Mirror public constructor( + private readonly eventRepository: IEventRepository, + private readonly userRepository: IUserRepository, private readonly process: NodeJS.Process, private readonly settings: () => Settings, ) { @@ -57,7 +62,7 @@ export class StaticMirroringWorker implements IRunnable { this.send(JSON.stringify(createSubscriptionMessage(subscriptionId, filters))) } }) - .on('message', async function (raw: RawData) { + .on('message', async (raw: RawData) => { try { const message = JSON.parse(raw.toString('utf8')) as OutgoingEventMessage @@ -70,7 +75,7 @@ export class StaticMirroringWorker implements IRunnable { return } - const event = message[2] + let event = message[2] if (!anyPass(map(isEventMatchingFilter, config.filters))(event)) { return @@ -80,10 +85,34 @@ export class StaticMirroringWorker implements IRunnable { return } + if (isExpiredEvent(event)) { + return + } + + const eventExpiration = getEventExpiration(event) + if (eventExpiration) { + event = { + ...event, + [EventExpirationTimeMetadataKey]: eventExpiration, + } as any + } + + if (!this.canAcceptEvent(event)) { + return + } + + if (!await this.isUserAdmitted(event)) { + return + } + since = Math.floor(Date.now() / 1000) - 30 - if (cluster.isWorker && typeof process.send === 'function') { - debug('%s >> local: %s', config.address, event.id) + debug('%s >> local: %s', config.address, event.id) + + const inserted = await this.eventRepository.create(event) + + if (inserted && cluster.isWorker && typeof process.send === 'function') { + process.send({ eventName: WebSocketServerAdapterEvent.Broadcast, event, @@ -110,6 +139,161 @@ export class StaticMirroringWorker implements IRunnable { this.client = createMirror(this.config) } + private getRelayPublicKey(): string { + const relayPrivkey = getRelayPrivateKey(this.settings().info.relay_url) + return getPublicKey(relayPrivkey) + } + + private canAcceptEvent(event: Event): boolean { + if (this.getRelayPublicKey() === event.pubkey) { + debug(`event ${event.id} not accepted: pubkey is relay pubkey`) + return false + } + + const now = Math.floor(Date.now() / 1000) + + const eventLimits = this.settings().limits?.event ?? {} + + const eventLimitOverrides = this.config.limits.event ?? {} + + const limits = mergeDeepRight(eventLimits, eventLimitOverrides) as EventLimits + + if (Array.isArray(limits.content)) { + for (const limit of limits.content) { + if ( + typeof limit.maxLength !== 'undefined' + && limit.maxLength > 0 + && event.content.length > limit.maxLength + && ( + !Array.isArray(limit.kinds) + || limit.kinds.some(isEventKindOrRangeMatch(event)) + ) + ) { + debug(`event ${event.id} not accepted: content is longer than ${limit.maxLength} bytes`) + return false + } + } + } else if ( + typeof limits.content?.maxLength !== 'undefined' + && limits.content?.maxLength > 0 + && event.content.length > limits.content.maxLength + && ( + !Array.isArray(limits.content.kinds) + || limits.content.kinds.some(isEventKindOrRangeMatch(event)) + ) + ) { + debug(`event ${event.id} not accepted: content is longer than ${limits.content.maxLength} bytes`) + return false + } + + if ( + typeof limits.createdAt?.maxPositiveDelta !== 'undefined' + && limits.createdAt.maxPositiveDelta > 0 + && event.created_at > now + limits.createdAt.maxPositiveDelta) { + debug(`event ${event.id} not accepted: created_at is more than ${limits.createdAt.maxPositiveDelta} seconds in the future`) + return false + } + + if ( + typeof limits.createdAt?.maxNegativeDelta !== 'undefined' + && limits.createdAt.maxNegativeDelta > 0 + && event.created_at < now - limits.createdAt.maxNegativeDelta) { + debug(`event ${event.id} not accepted: created_at is more than ${limits.createdAt.maxNegativeDelta} seconds in the past`) + return false + } + + if ( + typeof limits.eventId?.minLeadingZeroBits !== 'undefined' + && limits.eventId.minLeadingZeroBits > 0 + ) { + const pow = getEventProofOfWork(event.id) + if (pow < limits.eventId.minLeadingZeroBits) { + debug(`event ${event.id} not accepted: pow difficulty ${pow}<${limits.eventId.minLeadingZeroBits}`) + return false + } + } + + if ( + typeof limits.pubkey?.minLeadingZeroBits !== 'undefined' + && limits.pubkey.minLeadingZeroBits > 0 + ) { + const pow = getPubkeyProofOfWork(event.pubkey) + if (pow < limits.pubkey.minLeadingZeroBits) { + debug(`event ${event.id} not accepted: pow pubkey difficulty ${pow}<${limits.pubkey.minLeadingZeroBits}`) + return false + } + } + + if ( + typeof limits.pubkey?.whitelist !== 'undefined' + && limits.pubkey.whitelist.length > 0 + && !limits.pubkey.whitelist.some((prefix) => event.pubkey.startsWith(prefix)) + ) { + debug(`event ${event.id} not accepted: pubkey not allowed: ${event.pubkey}`) + return false + } + + if ( + typeof limits.pubkey?.blacklist !== 'undefined' + && limits.pubkey.blacklist.length > 0 + && limits.pubkey.blacklist.some((prefix) => event.pubkey.startsWith(prefix)) + ) { + debug(`event ${event.id} not accepted: pubkey not allowed: ${event.pubkey}`) + return false + } + + if ( + typeof limits.kind?.whitelist !== 'undefined' + && limits.kind.whitelist.length > 0 + && !limits.kind.whitelist.some(isEventKindOrRangeMatch(event))) { + debug(`blocked: event kind ${event.kind} not allowed`) + return false + } + + if ( + typeof limits.kind?.blacklist !== 'undefined' + && limits.kind.blacklist.length > 0 + && limits.kind.blacklist.some(isEventKindOrRangeMatch(event))) { + debug(`blocked: event kind ${event.kind} not allowed`) + return false + } + + return true + } + + protected async isUserAdmitted(event: Event): Promise { + const currentSettings = this.settings() + + if (currentSettings.payments?.enabled !== true) { + return true + } + + const isApplicableFee = (feeSchedule: FeeSchedule) => + feeSchedule.enabled + && !feeSchedule.whitelists?.pubkeys?.some((prefix) => event.pubkey.startsWith(prefix)) + && !feeSchedule.whitelists?.event_kinds?.some(isEventKindOrRangeMatch(event)) + + const feeSchedules = currentSettings.payments?.feeSchedules?.admission?.filter(isApplicableFee) + + if (!Array.isArray(feeSchedules) || !feeSchedules.length) { + return true + } + + const user = await this.userRepository.findByPubkey(event.pubkey) + if (user?.isAdmitted !== true) { + debug(`user not admitted: ${event.pubkey}`) + return false + } + + const minBalance = currentSettings.limits?.event?.pubkey?.minBalance + if (minBalance && user.balance < minBalance) { + debug(`user not admitted: user balance ${user.balance} < ${minBalance}`) + return false + } + + return true + } + private onMessage(message: { eventName: string, event: unknown, source: string }): void { if ( message.eventName !== WebSocketServerAdapterEvent.Broadcast diff --git a/src/factories/static-mirroring.worker-factory.ts b/src/factories/static-mirroring.worker-factory.ts index 99908145..234430e4 100644 --- a/src/factories/static-mirroring.worker-factory.ts +++ b/src/factories/static-mirroring.worker-factory.ts @@ -1,6 +1,19 @@ +import { getMasterDbClient, getReadReplicaDbClient } from '../database/client' import { createSettings } from './settings-factory' +import { EventRepository } from '../repositories/event-repository' import { StaticMirroringWorker } from '../app/static-mirroring-worker' +import { UserRepository } from '../repositories/user-repository' export const staticMirroringWorkerFactory = () => { - return new StaticMirroringWorker(process, createSettings) + const dbClient = getMasterDbClient() + const readReplicaDbClient = getReadReplicaDbClient() + const eventRepository = new EventRepository(dbClient, readReplicaDbClient) + const userRepository = new UserRepository(dbClient) + + return new StaticMirroringWorker( + eventRepository, + userRepository, + process, + createSettings, + ) } diff --git a/src/utils/settings.ts b/src/utils/settings.ts index 55092c33..e1708706 100644 --- a/src/utils/settings.ts +++ b/src/utils/settings.ts @@ -125,7 +125,7 @@ export class SettingsStatic { const settingsFilePath = join(basePath, `settings.${fileType}`) const reload = () => { - console.log('reloading settings') + debug('reloading settings') SettingsStatic._settings = undefined SettingsStatic.createSettings() } From f57d93e986d80298226ab4c1ad839c1ae3c7f181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Arturo=20Cabral=20Mej=C3=ADa?= Date: Sun, 20 Oct 2024 19:32:50 -0400 Subject: [PATCH 2/5] docs: add config for mirroring event limits --- CONFIGURATION.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index adcc4de6..4118be6e 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -87,6 +87,7 @@ Running `nostream` for the first time creates the settings file in ` Date: Tue, 22 Oct 2024 08:38:19 -0400 Subject: [PATCH 3/5] chore: add skip admission check (thanks YEGHRO) --- CONFIGURATION.md | 1 + src/@types/settings.ts | 1 + src/app/static-mirroring-worker.ts | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 4118be6e..9b1d86a7 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -88,6 +88,7 @@ Running `nostream` for the first time creates the settings file in ` { const currentSettings = this.settings() + if (this.config.skipAdmissionCheck === true) { + return true + } + if (currentSettings.payments?.enabled !== true) { return true } From 5bcbd52b7cd11277244abfc29440437795af406c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Arturo=20Cabral=20Mej=C3=ADa?= Date: Tue, 22 Oct 2024 08:57:59 -0400 Subject: [PATCH 4/5] 2.0.1 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index cbd38ec3..c6de95be 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "nostream", - "version": "2.0.0", + "version": "2.0.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "nostream", - "version": "2.0.0", + "version": "2.0.1", "license": "MIT", "dependencies": { "@noble/secp256k1": "1.7.1", diff --git a/package.json b/package.json index eebe5622..b12304b0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "nostream", - "version": "2.0.0", + "version": "2.0.1", "description": "A Nostr relay written in Typescript.", "supportedNips": [ 1, From 509008b3937b4a1cfeae0a7a9ff5cf1a67608d31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Arturo=20Cabral=20Mej=C3=ADa?= Date: Tue, 22 Oct 2024 09:01:37 -0400 Subject: [PATCH 5/5] chore: bump some deps --- package-lock.json | 58 +++++++++++++++++++++++------------------------ package.json | 6 ++--- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/package-lock.json b/package-lock.json index c6de95be..fe61d518 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "MIT", "dependencies": { "@noble/secp256k1": "1.7.1", - "axios": "1.6.5", + "axios": "^1.7.7", "bech32": "2.0.0", "body-parser": "1.20.1", "debug": "4.3.4", @@ -27,7 +27,7 @@ "redis": "4.5.1", "rxjs": "7.8.0", "tor-control-ts": "^1.0.0", - "ws": "8.12.0" + "ws": "^8.18.0" }, "devDependencies": { "@commitlint/cli": "17.2.0", @@ -50,7 +50,7 @@ "@types/ramda": "^0.28.13", "@types/sinon": "^10.0.11", "@types/sinon-chai": "^3.2.8", - "@types/ws": "^8.5.3", + "@types/ws": "^8.5.12", "@typescript-eslint/eslint-plugin": "^5.19.0", "@typescript-eslint/parser": "^5.19.0", "chai": "^4.3.6", @@ -2385,9 +2385,9 @@ "dev": true }, "node_modules/@types/ws": { - "version": "8.5.3", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.3.tgz", - "integrity": "sha512-6YOoWjruKj1uLf3INHH7D3qTXwFfEsg1kf3c0uDdSBJwfa/llkwIjrAGV7j7mVgGNbzTQ3HiHKKDXl6bJPD97w==", + "version": "8.5.12", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.12.tgz", + "integrity": "sha512-3tPRkv1EtkDpzlgyKyI8pGsGZAGPEaXeu0DOj5DI25Ja91bdAYddYHbADRYVrZMRbfW+1l5YwXVDKohDJNQxkQ==", "dev": true, "dependencies": { "@types/node": "*" @@ -2909,11 +2909,11 @@ } }, "node_modules/axios": { - "version": "1.6.5", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.5.tgz", - "integrity": "sha512-Ii012v05KEVuUoFWmMW/UQv9aRIc3ZwkWDcM+h5Il8izZCtRVpDUfwpoFf7eOtajT3QiGR4yDUx7lPqHJULgbg==", + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", "dependencies": { - "follow-redirects": "^1.15.4", + "follow-redirects": "^1.15.6", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" } @@ -5095,9 +5095,9 @@ "dev": true }, "node_modules/follow-redirects": { - "version": "1.15.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", - "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==", + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", "funding": [ { "type": "individual", @@ -13444,9 +13444,9 @@ } }, "node_modules/ws": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.12.0.tgz", - "integrity": "sha512-kU62emKIdKVeEIOIKVegvqpXMSTAMLJozpHZaJNDYqBjzlSYXQGviYwN1osDLJ9av68qHd4a2oSjd7yD4pacig==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "engines": { "node": ">=10.0.0" }, @@ -15477,9 +15477,9 @@ "dev": true }, "@types/ws": { - "version": "8.5.3", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.3.tgz", - "integrity": "sha512-6YOoWjruKj1uLf3INHH7D3qTXwFfEsg1kf3c0uDdSBJwfa/llkwIjrAGV7j7mVgGNbzTQ3HiHKKDXl6bJPD97w==", + "version": "8.5.12", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.12.tgz", + "integrity": "sha512-3tPRkv1EtkDpzlgyKyI8pGsGZAGPEaXeu0DOj5DI25Ja91bdAYddYHbADRYVrZMRbfW+1l5YwXVDKohDJNQxkQ==", "dev": true, "requires": { "@types/node": "*" @@ -15841,11 +15841,11 @@ "dev": true }, "axios": { - "version": "1.6.5", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.5.tgz", - "integrity": "sha512-Ii012v05KEVuUoFWmMW/UQv9aRIc3ZwkWDcM+h5Il8izZCtRVpDUfwpoFf7eOtajT3QiGR4yDUx7lPqHJULgbg==", + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", "requires": { - "follow-redirects": "^1.15.4", + "follow-redirects": "^1.15.6", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" } @@ -17529,9 +17529,9 @@ "dev": true }, "follow-redirects": { - "version": "1.15.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", - "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==" + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==" }, "foreground-child": { "version": "2.0.0", @@ -23673,9 +23673,9 @@ } }, "ws": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.12.0.tgz", - "integrity": "sha512-kU62emKIdKVeEIOIKVegvqpXMSTAMLJozpHZaJNDYqBjzlSYXQGviYwN1osDLJ9av68qHd4a2oSjd7yD4pacig==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "requires": {} }, "xmlbuilder": { diff --git a/package.json b/package.json index b12304b0..bf69445f 100644 --- a/package.json +++ b/package.json @@ -90,7 +90,7 @@ "@types/ramda": "^0.28.13", "@types/sinon": "^10.0.11", "@types/sinon-chai": "^3.2.8", - "@types/ws": "^8.5.3", + "@types/ws": "^8.5.12", "@typescript-eslint/eslint-plugin": "^5.19.0", "@typescript-eslint/parser": "^5.19.0", "chai": "^4.3.6", @@ -115,7 +115,7 @@ }, "dependencies": { "@noble/secp256k1": "1.7.1", - "axios": "1.6.5", + "axios": "^1.7.7", "bech32": "2.0.0", "body-parser": "1.20.1", "debug": "4.3.4", @@ -132,7 +132,7 @@ "redis": "4.5.1", "rxjs": "7.8.0", "tor-control-ts": "^1.0.0", - "ws": "8.12.0" + "ws": "^8.18.0" }, "config": { "commitizen": {