From 02dfe0a3d56d2a21de6a96f7b6a3228a77619a8e Mon Sep 17 00:00:00 2001 From: tamaina Date: Mon, 26 Feb 2024 20:49:40 +0000 Subject: [PATCH 001/134] =?UTF-8?q?1.=20ed25519=E3=82=AD=E3=83=BC=E3=83=9A?= =?UTF-8?q?=E3=82=A2=E3=82=92=E7=99=BA=E8=A1=8C=E3=83=BBPerson=E3=81=A8?= =?UTF-8?q?=E3=81=97=E3=81=A6=E5=85=AC=E9=96=8B=E9=8D=B5=E3=82=92=E9=80=81?= =?UTF-8?q?=E5=8F=97=E4=BF=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../migration/1708980134301-APMultipleKeys.js | 43 +++++++++++++++++ .../backend/src/core/GlobalEventService.ts | 1 + .../backend/src/core/UserKeypairService.ts | 45 +++++++++++++++++- .../core/activitypub/ApDbResolverService.ts | 6 +-- .../activitypub/ApDeliverManagerService.ts | 21 ++++++--- .../src/core/activitypub/ApRendererService.ts | 12 +++-- .../activitypub/models/ApPersonService.ts | 46 +++++++++++++++++-- packages/backend/src/core/activitypub/type.ts | 17 +++++-- packages/backend/src/misc/gen-key-pair.ts | 23 ++++++++-- packages/backend/src/models/UserKeypair.ts | 40 ++++++++++++++++ packages/backend/src/models/UserPublickey.ts | 14 +++--- .../src/server/ActivityPubServerService.ts | 2 +- .../backend/test/unit/misc/gen-key-pair.ts | 40 ++++++++++++++++ 13 files changed, 274 insertions(+), 36 deletions(-) create mode 100644 packages/backend/migration/1708980134301-APMultipleKeys.js create mode 100644 packages/backend/test/unit/misc/gen-key-pair.ts diff --git a/packages/backend/migration/1708980134301-APMultipleKeys.js b/packages/backend/migration/1708980134301-APMultipleKeys.js new file mode 100644 index 000000000000..d0687fa7faae --- /dev/null +++ b/packages/backend/migration/1708980134301-APMultipleKeys.js @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class APMultipleKeys1708980134301 { + name = 'APMultipleKeys1708980134301' + + async up(queryRunner) { + await queryRunner.query(`DROP INDEX "public"."IDX_171e64971c780ebd23fae140bb"`); + await queryRunner.query(`ALTER TABLE "user_keypair" ADD "ed25519PublicKey" character varying(128)`); + await queryRunner.query(`ALTER TABLE "user_keypair" ADD "ed25519PrivateKey" character varying(128)`); + await queryRunner.query(`ALTER TABLE "user_keypair" ADD "ed25519PublicKeySignature" character varying(720)`); + await queryRunner.query(`ALTER TABLE "user_keypair" ADD "ed25519SignatureAlgorithm" character varying(32)`); + await queryRunner.query(`ALTER TABLE "user_publickey" DROP CONSTRAINT "FK_10c146e4b39b443ede016f6736d"`); + await queryRunner.query(`ALTER TABLE "user_publickey" DROP CONSTRAINT "PK_10c146e4b39b443ede016f6736d"`); + await queryRunner.query(`ALTER TABLE "user_publickey" ADD CONSTRAINT "PK_0db6a5fdb992323449edc8ee421" PRIMARY KEY ("userId", "keyId")`); + await queryRunner.query(`ALTER TABLE "user_publickey" DROP CONSTRAINT "PK_0db6a5fdb992323449edc8ee421"`); + await queryRunner.query(`ALTER TABLE "user_publickey" ADD CONSTRAINT "PK_171e64971c780ebd23fae140bba" PRIMARY KEY ("keyId")`); + await queryRunner.query(`ALTER TABLE "user_publickey" ADD CONSTRAINT "UQ_10c146e4b39b443ede016f6736d" UNIQUE ("userId")`); + await queryRunner.query(`CREATE INDEX "IDX_10c146e4b39b443ede016f6736" ON "user_publickey" ("userId") `); + await queryRunner.query(`ALTER TABLE "user_publickey" ADD CONSTRAINT "FK_10c146e4b39b443ede016f6736d" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "user_publickey" DROP CONSTRAINT "FK_10c146e4b39b443ede016f6736d"`); + await queryRunner.query(`DROP INDEX "public"."IDX_10c146e4b39b443ede016f6736"`); + await queryRunner.query(`ALTER TABLE "user_publickey" DROP CONSTRAINT "UQ_10c146e4b39b443ede016f6736d"`); + await queryRunner.query(`ALTER TABLE "user_publickey" DROP CONSTRAINT "PK_171e64971c780ebd23fae140bba"`); + await queryRunner.query(`ALTER TABLE "user_publickey" ADD CONSTRAINT "PK_0db6a5fdb992323449edc8ee421" PRIMARY KEY ("userId", "keyId")`); + await queryRunner.query(`ALTER TABLE "user_publickey" DROP CONSTRAINT "PK_0db6a5fdb992323449edc8ee421"`); + await queryRunner.query(`ALTER TABLE "user_publickey" ADD CONSTRAINT "PK_10c146e4b39b443ede016f6736d" PRIMARY KEY ("userId")`); + await queryRunner.query(`ALTER TABLE "user_publickey" ADD CONSTRAINT "FK_10c146e4b39b443ede016f6736d" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "user_profile" ALTER COLUMN "followersVisibility" DROP DEFAULT`); + await queryRunner.query(`ALTER TABLE "user_profile" ALTER COLUMN "followersVisibility" TYPE "public"."user_profile_followersVisibility_enum_old" USING "followersVisibility"::"text"::"public"."user_profile_followersVisibility_enum_old"`); + await queryRunner.query(`ALTER TABLE "user_profile" ALTER COLUMN "followersVisibility" SET DEFAULT 'public'`); + await queryRunner.query(`ALTER TABLE "user_keypair" DROP COLUMN "ed25519SignatureAlgorithm"`); + await queryRunner.query(`ALTER TABLE "user_keypair" DROP COLUMN "ed25519PublicKeySignature"`); + await queryRunner.query(`ALTER TABLE "user_keypair" DROP COLUMN "ed25519PrivateKey"`); + await queryRunner.query(`ALTER TABLE "user_keypair" DROP COLUMN "ed25519PublicKey"`); + await queryRunner.query(`CREATE UNIQUE INDEX "IDX_171e64971c780ebd23fae140bb" ON "user_publickey" ("keyId") `); + } +} diff --git a/packages/backend/src/core/GlobalEventService.ts b/packages/backend/src/core/GlobalEventService.ts index 7c1b34da05ea..331df4315da7 100644 --- a/packages/backend/src/core/GlobalEventService.ts +++ b/packages/backend/src/core/GlobalEventService.ts @@ -240,6 +240,7 @@ export interface InternalEventTypes { unmute: { muterId: MiUser['id']; muteeId: MiUser['id']; }; userListMemberAdded: { userListId: MiUserList['id']; memberId: MiUser['id']; }; userListMemberRemoved: { userListId: MiUserList['id']; memberId: MiUser['id']; }; + userKeypairUpdated: { userId: MiUser['id']; }; } // name/messages(spec) pairs dictionary diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 51ac99179a6a..7946f410cf3f 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -3,6 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import { sign } from 'node:crypto'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; import type { MiUser } from '@/models/User.js'; @@ -11,6 +12,8 @@ import { RedisKVCache } from '@/misc/cache.js'; import type { MiUserKeypair } from '@/models/UserKeypair.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; +import { ED25519_SIGN_ALGORITHM, genEd25519KeyPair } from '@/misc/gen-key-pair.js'; +import { GlobalEventService, GlobalEvents } from '@/core/GlobalEventService.js'; @Injectable() export class UserKeypairService implements OnApplicationShutdown { @@ -19,9 +22,12 @@ export class UserKeypairService implements OnApplicationShutdown { constructor( @Inject(DI.redis) private redisClient: Redis.Redis, - + @Inject(DI.redisForSub) + private redisForSub: Redis.Redis, @Inject(DI.userKeypairsRepository) private userKeypairsRepository: UserKeypairsRepository, + + private globalEventService: GlobalEventService, ) { this.cache = new RedisKVCache(this.redisClient, 'userKeypair', { lifetime: 1000 * 60 * 60 * 24, // 24h @@ -30,6 +36,8 @@ export class UserKeypairService implements OnApplicationShutdown { toRedisConverter: (value) => JSON.stringify(value), fromRedisConverter: (value) => JSON.parse(value), }); + + this.redisForSub.on('message', this.onMessage); } @bindThis @@ -37,6 +45,41 @@ export class UserKeypairService implements OnApplicationShutdown { return await this.cache.fetch(userId); } + @bindThis + public async refresh(userId: MiUser['id']): Promise { + return await this.cache.refresh(userId); + } + + @bindThis + public async prepareEd25519KeyPair(userId: MiUser['id']): Promise { + await this.refresh(userId); + const keypair = await this.cache.fetch(userId); + if (keypair.ed25519PublicKey != null) return; + const ed25519 = await genEd25519KeyPair(); + const ed25519PublicKeySignature = sign(ED25519_SIGN_ALGORITHM, Buffer.from(ed25519.publicKey), keypair.privateKey).toString('base64'); + await this.userKeypairsRepository.update({ userId }, { + ed25519PublicKey: ed25519.publicKey, + ed25519PrivateKey: ed25519.privateKey, + ed25519PublicKeySignature, + ed25519SignatureAlgorithm: `rsa-${ED25519_SIGN_ALGORITHM}`, + }); + this.globalEventService.publishInternalEvent('userKeypairUpdated', { userId }); + } + + @bindThis + private async onMessage(_: string, data: string): Promise { + const obj = JSON.parse(data); + + if (obj.channel === 'internal') { + const { type, body } = obj.message as GlobalEvents['internal']['payload']; + switch (type) { + case 'userKeypairUpdated': { + this.refresh(body.userId); + break; + } + } + } + } @bindThis public dispose(): void { this.cache.dispose(); diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index f6b70ead4455..913070fdd461 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -36,7 +36,7 @@ export type UriParseResult = { @Injectable() export class ApDbResolverService implements OnApplicationShutdown { private publicKeyCache: MemoryKVCache; - private publicKeyByUserIdCache: MemoryKVCache; + private publicKeyByUserIdCache: MemoryKVCache; constructor( @Inject(DI.config) @@ -55,7 +55,7 @@ export class ApDbResolverService implements OnApplicationShutdown { private apPersonService: ApPersonService, ) { this.publicKeyCache = new MemoryKVCache(Infinity); - this.publicKeyByUserIdCache = new MemoryKVCache(Infinity); + this.publicKeyByUserIdCache = new MemoryKVCache(Infinity); } @bindThis @@ -159,7 +159,7 @@ export class ApDbResolverService implements OnApplicationShutdown { const key = await this.publicKeyByUserIdCache.fetch( user.id, - () => this.userPublickeysRepository.findOneBy({ userId: user.id }), + () => this.userPublickeysRepository.find({ where: { userId: user.id } }), v => v != null, ); diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index 5d07cd8e8f63..27a9c11daba6 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -9,10 +9,10 @@ import { DI } from '@/di-symbols.js'; import type { FollowingsRepository } from '@/models/_.js'; import type { MiLocalUser, MiRemoteUser, MiUser } from '@/models/User.js'; import { QueueService } from '@/core/QueueService.js'; -import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; import type { IActivity } from '@/core/activitypub/type.js'; import { ThinUser } from '@/queue/types.js'; +import { UserKeypairService } from '../UserKeypairService.js'; interface IRecipe { type: string; @@ -40,14 +40,14 @@ class DeliverManager { /** * Constructor - * @param userEntityService + * @param userKeypairService * @param followingsRepository * @param queueService * @param actor Actor * @param activity Activity to deliver */ constructor( - private userEntityService: UserEntityService, + private userKeypairService: UserKeypairService, private followingsRepository: FollowingsRepository, private queueService: QueueService, @@ -105,6 +105,13 @@ class DeliverManager { */ @bindThis public async execute(): Promise { + //#region MIGRATION + /** + * ed25519の署名がなければ追加する + */ + await this.userKeypairService.prepareEd25519KeyPair(this.actor.id); + //#endregion + // The value flags whether it is shared or not. // key: inbox URL, value: whether it is sharedInbox const inboxes = new Map(); @@ -154,7 +161,7 @@ export class ApDeliverManagerService { @Inject(DI.followingsRepository) private followingsRepository: FollowingsRepository, - private userEntityService: UserEntityService, + private userKeypairService: UserKeypairService, private queueService: QueueService, ) { } @@ -167,7 +174,7 @@ export class ApDeliverManagerService { @bindThis public async deliverToFollowers(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity): Promise { const manager = new DeliverManager( - this.userEntityService, + this.userKeypairService, this.followingsRepository, this.queueService, actor, @@ -186,7 +193,7 @@ export class ApDeliverManagerService { @bindThis public async deliverToUser(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity, to: MiRemoteUser): Promise { const manager = new DeliverManager( - this.userEntityService, + this.userKeypairService, this.followingsRepository, this.queueService, actor, @@ -199,7 +206,7 @@ export class ApDeliverManagerService { @bindThis public createDeliverManager(actor: { id: MiUser['id']; host: null; }, activity: IActivity | null): DeliverManager { return new DeliverManager( - this.userEntityService, + this.userKeypairService, this.followingsRepository, this.queueService, diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index d7fb977a99d3..66322f68703b 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -250,15 +250,16 @@ export class ApRendererService { } @bindThis - public renderKey(user: MiLocalUser, key: MiUserKeypair, postfix?: string): IKey { + public renderKey(user: MiLocalUser, publicKey: string, postfix?: string, signature?: IKey['signature']): IKey { return { id: `${this.config.url}/users/${user.id}${postfix ?? '/publickey'}`, type: 'Key', owner: this.userEntityService.genLocalUserUri(user.id), - publicKeyPem: createPublicKey(key.publicKey).export({ + publicKeyPem: createPublicKey(publicKey).export({ type: 'spki', format: 'pem', - }), + }) as string, + signature, }; } @@ -498,7 +499,10 @@ export class ApRendererService { tag, manuallyApprovesFollowers: user.isLocked, discoverable: user.isExplorable, - publicKey: this.renderKey(user, keypair, '#main-key'), + publicKey: this.renderKey(user, keypair.publicKey, '#main-key'), + additionalPublicKeys: [ + ...(keypair.ed25519PublicKey ? [this.renderKey(user, keypair.ed25519PublicKey, '#ed25519-key', { type: keypair.ed25519SignatureAlgorithm!, signatureValue: keypair.ed25519PublicKeySignature! })] : []), + ], isCat: user.isCat, attachment: attachment.length ? attachment : undefined, }; diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 744b1ea68376..c31aa0326178 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -3,9 +3,10 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import { verify } from 'crypto'; import { Inject, Injectable } from '@nestjs/common'; import promiseLimit from 'promise-limit'; -import { DataSource } from 'typeorm'; +import { DataSource, In, Not } from 'typeorm'; import { ModuleRef } from '@nestjs/core'; import { DI } from '@/di-symbols.js'; import type { FollowingsRepository, InstancesRepository, UserProfilesRepository, UserPublickeysRepository, UsersRepository } from '@/models/_.js'; @@ -357,10 +358,25 @@ export class ApPersonService implements OnModuleInit { if (person.publicKey) { await transactionalEntityManager.save(new MiUserPublickey({ - userId: user.id, keyId: person.publicKey.id, + userId: user.id, keyPem: person.publicKey.publicKeyPem, })); + + if (person.additionalPublicKeys) { + for (const key of person.additionalPublicKeys) { + if ( + key.signature && key.signature.type && key.signature.signatureValue && + verify(key.signature.type, Buffer.from(key.publicKeyPem), person.publicKey.publicKeyPem, Buffer.from(key.signature.signatureValue, 'base64')) + ) { + await transactionalEntityManager.save(new MiUserPublickey({ + keyId: key.id, + userId: user.id, + keyPem: key.publicKeyPem, + })); + } + } + } } }); } catch (e) { @@ -506,13 +522,35 @@ export class ApPersonService implements OnModuleInit { // Update user await this.usersRepository.update(exist.id, updates); + const availablePublicKeys = new Set(); if (person.publicKey) { - await this.userPublickeysRepository.update({ userId: exist.id }, { - keyId: person.publicKey.id, + await this.userPublickeysRepository.update({ keyId: person.publicKey.id }, { + userId: exist.id, keyPem: person.publicKey.publicKeyPem, }); + availablePublicKeys.add(person.publicKey.id); + + if (person.additionalPublicKeys) { + for (const key of person.additionalPublicKeys) { + if ( + key.signature && key.signature.type && key.signature.signatureValue && + verify(key.signature.type, Buffer.from(key.publicKeyPem), person.publicKey.publicKeyPem, Buffer.from(key.signature.signatureValue, 'base64')) + ) { + await this.userPublickeysRepository.update({ keyId: key.id }, { + userId: exist.id, + keyPem: key.publicKeyPem, + }); + availablePublicKeys.add(key.id); + } + } + } } + this.userPublickeysRepository.delete({ + keyId: Not(In(Array.from(availablePublicKeys))), + userId: exist.id, + }); + let _description: string | null = null; if (person._misskey_summary) { diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index b43dddad61d2..72d383442eca 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -168,10 +168,8 @@ export interface IActor extends IObject { discoverable?: boolean; inbox: string; sharedInbox?: string; // 後方互換性のため - publicKey?: { - id: string; - publicKeyPem: string; - }; + publicKey?: IKey; + additionalPublicKeys?: IKey[]; followers?: string | ICollection | IOrderedCollection; following?: string | ICollection | IOrderedCollection; featured?: string | IOrderedCollection; @@ -235,8 +233,17 @@ export const isEmoji = (object: IObject): object is IApEmoji => export interface IKey extends IObject { type: 'Key'; + id: string; owner: string; - publicKeyPem: string | Buffer; + publicKeyPem: string; + + /** + * Signature of publicKeyPem, signed by root privateKey (for additionalPublicKey) + */ + signature?: { + type: string; + signatureValue: string + }; } export interface IApDocument extends IObject { diff --git a/packages/backend/src/misc/gen-key-pair.ts b/packages/backend/src/misc/gen-key-pair.ts index 02a303dc0a17..7b2e84d99151 100644 --- a/packages/backend/src/misc/gen-key-pair.ts +++ b/packages/backend/src/misc/gen-key-pair.ts @@ -8,7 +8,9 @@ import * as util from 'node:util'; const generateKeyPair = util.promisify(crypto.generateKeyPair); -export async function genRsaKeyPair(modulusLength = 2048) { +export const ED25519_SIGN_ALGORITHM = 'sha256'; + +export async function genRsaKeyPair(modulusLength = 4096) { return await generateKeyPair('rsa', { modulusLength, publicKeyEncoding: { @@ -24,9 +26,8 @@ export async function genRsaKeyPair(modulusLength = 2048) { }); } -export async function genEcKeyPair(namedCurve: 'prime256v1' | 'secp384r1' | 'secp521r1' | 'curve25519' = 'prime256v1') { - return await generateKeyPair('ec', { - namedCurve, +export async function genEd25519KeyPair() { + return await generateKeyPair('ed25519', { publicKeyEncoding: { type: 'spki', format: 'pem', @@ -39,3 +40,17 @@ export async function genEcKeyPair(namedCurve: 'prime256v1' | 'secp384r1' | 'sec }, }); } + +export async function genRSAAndEd25519KeyPair(rsaModulusLength = 4096) { + const rsa = await genRsaKeyPair(rsaModulusLength); + const ed25519 = await genEd25519KeyPair(); + const ed25519PublicKeySignature = crypto.sign(ED25519_SIGN_ALGORITHM, Buffer.from(ed25519.publicKey), rsa.privateKey).toString('base64'); + return { + publicKey: rsa.publicKey, + privateKey: rsa.privateKey, + ed25519PublicKey: ed25519.publicKey, + ed25519PrivateKey: ed25519.privateKey, + ed25519PublicKeySignature, + ed25519SignatureAlgorithm: `rsa-${ED25519_SIGN_ALGORITHM}`, + }; +} diff --git a/packages/backend/src/models/UserKeypair.ts b/packages/backend/src/models/UserKeypair.ts index f5252d126c5e..47d44af849ff 100644 --- a/packages/backend/src/models/UserKeypair.ts +++ b/packages/backend/src/models/UserKeypair.ts @@ -18,16 +18,56 @@ export class MiUserKeypair { @JoinColumn() public user: MiUser | null; + /** + * RSA public key + */ @Column('varchar', { length: 4096, }) public publicKey: string; + /** + * RSA private key + */ @Column('varchar', { length: 4096, }) public privateKey: string; + @Column('varchar', { + length: 128, + nullable: true, + default: null, + }) + public ed25519PublicKey: string | null; + + @Column('varchar', { + length: 128, + nullable: true, + default: null, + }) + public ed25519PrivateKey: string | null; + + /** + * Signature of ed25519PublicKey, signed by privateKey. (base64) + */ + @Column('varchar', { + length: 720, + nullable: true, + default: null, + }) + public ed25519PublicKeySignature: string | null; + + /** + * Signature algorithm of ed25519PublicKeySignature. + */ + @Column('varchar', { + length: 32, + nullable: true, + default: null, + }) + public ed25519SignatureAlgorithm: string | null; + constructor(data: Partial) { if (data == null) return; diff --git a/packages/backend/src/models/UserPublickey.ts b/packages/backend/src/models/UserPublickey.ts index 6bcd78530477..0ecff2bcbed1 100644 --- a/packages/backend/src/models/UserPublickey.ts +++ b/packages/backend/src/models/UserPublickey.ts @@ -9,7 +9,13 @@ import { MiUser } from './User.js'; @Entity('user_publickey') export class MiUserPublickey { - @PrimaryColumn(id()) + @PrimaryColumn('varchar', { + length: 256, + }) + public keyId: string; + + @Index() + @Column(id()) public userId: MiUser['id']; @OneToOne(type => MiUser, { @@ -18,12 +24,6 @@ export class MiUserPublickey { @JoinColumn() public user: MiUser | null; - @Index({ unique: true }) - @Column('varchar', { - length: 256, - }) - public keyId: string; - @Column('varchar', { length: 4096, }) diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index 60366dd5c231..5e6cb0a05a16 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -640,7 +640,7 @@ export class ActivityPubServerService { if (this.userEntityService.isLocalUser(user)) { reply.header('Cache-Control', 'public, max-age=180'); this.setResponseType(request, reply); - return (this.apRendererService.addContext(this.apRendererService.renderKey(user, keypair))); + return (this.apRendererService.addContext(this.apRendererService.renderKey(user, keypair.publicKey))); } else { reply.code(400); return; diff --git a/packages/backend/test/unit/misc/gen-key-pair.ts b/packages/backend/test/unit/misc/gen-key-pair.ts new file mode 100644 index 000000000000..91c62f621d07 --- /dev/null +++ b/packages/backend/test/unit/misc/gen-key-pair.ts @@ -0,0 +1,40 @@ +import * as crypto from 'node:crypto'; +import { genRSAAndEd25519KeyPair } from '@/misc/gen-key-pair.js'; + +describe(genRSAAndEd25519KeyPair, () => { + test('generates key pair', async () => { + const keyPair = await genRSAAndEd25519KeyPair(); + // 毎回違うキーペアが生成されることを確認するために2回生成して比較してみる + const keyPair2 = await genRSAAndEd25519KeyPair(); + console.log(Object.entries(keyPair).map(([k, v]) => `${k}: ${v.length}`).join('\n')); + console.log(Object.entries(keyPair).map(([k, v]) => `${k}\n${v}`).join('\n')); + + expect(keyPair.publicKey).toMatch(/^-----BEGIN PUBLIC KEY-----/); + expect(keyPair.publicKey).toMatch(/-----END PUBLIC KEY-----\n$/); + expect(keyPair.publicKey).not.toBe(keyPair2.publicKey); + + const publicKeyObj = crypto.createPublicKey(keyPair.publicKey); + expect(publicKeyObj.asymmetricKeyType).toBe('rsa'); + + expect(keyPair.privateKey).toMatch(/^-----BEGIN PRIVATE KEY-----/); + expect(keyPair.privateKey).toMatch(/-----END PRIVATE KEY-----\n$/); + expect(keyPair.privateKey).not.toBe(keyPair2.privateKey); + expect(keyPair.ed25519PublicKey).toMatch(/^-----BEGIN PUBLIC KEY-----/); + expect(keyPair.ed25519PublicKey).toMatch(/-----END PUBLIC KEY-----\n$/); + expect(keyPair.ed25519PublicKey).not.toBe(keyPair2.ed25519PublicKey); + + const ed25519PublicKeyObj = crypto.createPublicKey(keyPair.ed25519PublicKey); + expect(ed25519PublicKeyObj.asymmetricKeyType).toBe('ed25519'); + + expect(keyPair.ed25519PrivateKey).toMatch(/^-----BEGIN PRIVATE KEY-----/); + expect(keyPair.ed25519PrivateKey).toMatch(/-----END PRIVATE KEY-----\n$/); + expect(keyPair.ed25519PrivateKey).not.toBe(keyPair2.ed25519PrivateKey); + expect(keyPair.ed25519PublicKeySignature).toBe( + crypto.sign(keyPair.ed25519SignatureAlgorithm.split('-').pop(), Buffer.from(keyPair.ed25519PublicKey), keyPair.privateKey).toString('base64'), + ); + expect(crypto.verify(keyPair.ed25519SignatureAlgorithm, Buffer.from(keyPair.ed25519PublicKey), keyPair.publicKey, Buffer.from(keyPair.ed25519PublicKeySignature, 'base64'))).toBe(true); + expect(keyPair.ed25519PublicKeySignature).not.toBe(keyPair2.ed25519PublicKeySignature); + + //const imported = await webCrypto.subtle.importKey('spki', Buffer.from(keyPair.publicKey).buffer, { name: 'rsa-pss', hash: 'sha-256' }, false, ['verify']); + }); +}); From 18353973859d20c4be96d7b9ea91da35cb92cf52 Mon Sep 17 00:00:00 2001 From: tamaina Date: Mon, 26 Feb 2024 21:06:05 +0000 Subject: [PATCH 002/134] validate additionalPublicKeys --- .../core/activitypub/ApDbResolverService.ts | 2 +- .../activitypub/models/ApPersonService.ts | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 913070fdd461..6cce3951bdad 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -152,7 +152,7 @@ export class ApDbResolverService implements OnApplicationShutdown { @bindThis public async getAuthUserFromApId(uri: string): Promise<{ user: MiRemoteUser; - key: MiUserPublickey | null; + key: MiUserPublickey[] | null; } | null> { const user = await this.apPersonService.resolvePerson(uri) as MiRemoteUser; if (user.isDeleted) return null; diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index c31aa0326178..4c1f79e1b1ea 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -194,6 +194,37 @@ export class ApPersonService implements OnModuleInit { } } + if (x.additionalPublicKeys) { + if (!x.publicKey) { + throw new Error('invalid Actor: additionalPublicKeys is set but publicKey is not'); + } + + if (!Array.isArray(x.additionalPublicKeys)) { + throw new Error('invalid Actor: additionalPublicKeys is not an array'); + } + + for (const key of x.additionalPublicKeys) { + if (typeof key.id !== 'string') { + throw new Error('invalid Actor: additionalPublicKeys.id is not a string'); + } + + const keyIdHost = this.punyHost(key.id); + if (keyIdHost !== expectHost) { + throw new Error('invalid Actor: additionalPublicKeys.id has different host'); + } + + if (!key.signature) { + throw new Error('invalid Actor: additionalPublicKeys.signature is not set'); + } + if (typeof key.signature.type !== 'string') { + throw new Error('invalid Actor: additionalPublicKeys.signature.type is not a string'); + } + if (typeof key.signature.signatureValue !== 'string') { + throw new Error('invalid Actor: additionalPublicKeys.signature.signatureValue is not a string'); + } + } + } + return x; } From 5b7b8503cd4f1b76b65d18b8f291e10db12143b3 Mon Sep 17 00:00:00 2001 From: tamaina Date: Mon, 26 Feb 2024 21:27:50 +0000 Subject: [PATCH 003/134] =?UTF-8?q?getAuthUserFromApId=E3=81=AFmain?= =?UTF-8?q?=E3=82=92=E9=81=B8=E3=81=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/activitypub/ApDbResolverService.ts | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 6cce3951bdad..fe211157b9ee 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -152,17 +152,36 @@ export class ApDbResolverService implements OnApplicationShutdown { @bindThis public async getAuthUserFromApId(uri: string): Promise<{ user: MiRemoteUser; - key: MiUserPublickey[] | null; + key: MiUserPublickey | null; } | null> { const user = await this.apPersonService.resolvePerson(uri) as MiRemoteUser; if (user.isDeleted) return null; - const key = await this.publicKeyByUserIdCache.fetch( + const keys = await this.publicKeyByUserIdCache.fetch( user.id, () => this.userPublickeysRepository.find({ where: { userId: user.id } }), v => v != null, ); + if (keys == null || keys.length === 8) return null; + + // 公開鍵は複数あるが、mainっぽいのを選ぶ + const key = keys.length === 1 ? + keys[0] : + keys.find(x => { + try { + const url = new URL(x.keyId); + if ( + url.hash.toLowerCase().includes('main') || + url.pathname.split('/').pop()?.toLowerCase().includes('main') + ) { + return true; + } + } catch { /* noop */ } + + return false; + }) ?? keys[0]; + return { user, key, From 00738b90c2e02d648422ac495859e169562af25c Mon Sep 17 00:00:00 2001 From: tamaina Date: Mon, 26 Feb 2024 21:31:43 +0000 Subject: [PATCH 004/134] :v: --- .../src/core/activitypub/ApDbResolverService.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index fe211157b9ee..0cb3b78941e0 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -171,10 +171,12 @@ export class ApDbResolverService implements OnApplicationShutdown { keys.find(x => { try { const url = new URL(x.keyId); - if ( - url.hash.toLowerCase().includes('main') || - url.pathname.split('/').pop()?.toLowerCase().includes('main') - ) { + const path = url.pathname.split('/').pop()?.toLowerCase(); + if (url.hash) { + if (url.hash.toLowerCase().includes('main')) { + return true; + } + } else if (path?.includes('main') || path === 'publickey') { return true; } } catch { /* noop */ } From 172546f3efb0ef96812d988d18ac7f3eadff0be8 Mon Sep 17 00:00:00 2001 From: tamaina Date: Mon, 26 Feb 2024 21:44:33 +0000 Subject: [PATCH 005/134] fix --- .../src/core/CreateSystemUserService.ts | 7 +++---- packages/backend/src/core/SignupService.ts | 21 +++---------------- 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/packages/backend/src/core/CreateSystemUserService.ts b/packages/backend/src/core/CreateSystemUserService.ts index 6c5b0f6a36ae..60ddc9cde290 100644 --- a/packages/backend/src/core/CreateSystemUserService.ts +++ b/packages/backend/src/core/CreateSystemUserService.ts @@ -7,7 +7,7 @@ import { randomUUID } from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; import bcrypt from 'bcryptjs'; import { IsNull, DataSource } from 'typeorm'; -import { genRsaKeyPair } from '@/misc/gen-key-pair.js'; +import { genRSAAndEd25519KeyPair } from '@/misc/gen-key-pair.js'; import { MiUser } from '@/models/User.js'; import { MiUserProfile } from '@/models/UserProfile.js'; import { IdService } from '@/core/IdService.js'; @@ -38,7 +38,7 @@ export class CreateSystemUserService { // Generate secret const secret = generateNativeUserToken(); - const keyPair = await genRsaKeyPair(); + const keyPair = await genRSAAndEd25519KeyPair(); let account!: MiUser; @@ -64,9 +64,8 @@ export class CreateSystemUserService { }).then(x => transactionalEntityManager.findOneByOrFail(MiUser, x.identifiers[0])); await transactionalEntityManager.insert(MiUserKeypair, { - publicKey: keyPair.publicKey, - privateKey: keyPair.privateKey, userId: account.id, + ...keyPair, }); await transactionalEntityManager.insert(MiUserProfile, { diff --git a/packages/backend/src/core/SignupService.ts b/packages/backend/src/core/SignupService.ts index 5522ecd6cca5..78a77d2adf67 100644 --- a/packages/backend/src/core/SignupService.ts +++ b/packages/backend/src/core/SignupService.ts @@ -21,6 +21,7 @@ import { bindThis } from '@/decorators.js'; import UsersChart from '@/core/chart/charts/users.js'; import { UtilityService } from '@/core/UtilityService.js'; import { MetaService } from '@/core/MetaService.js'; +import { genRSAAndEd25519KeyPair } from '@/misc/gen-key-pair.js'; @Injectable() export class SignupService { @@ -93,22 +94,7 @@ export class SignupService { } } - const keyPair = await new Promise((res, rej) => - generateKeyPair('rsa', { - modulusLength: 2048, - publicKeyEncoding: { - type: 'spki', - format: 'pem', - }, - privateKeyEncoding: { - type: 'pkcs8', - format: 'pem', - cipher: undefined, - passphrase: undefined, - }, - }, (err, publicKey, privateKey) => - err ? rej(err) : res([publicKey, privateKey]), - )); + const keyPair = await genRSAAndEd25519KeyPair(); let account!: MiUser; @@ -131,9 +117,8 @@ export class SignupService { })); await transactionalEntityManager.save(new MiUserKeypair({ - publicKey: keyPair[0], - privateKey: keyPair[1], userId: account.id, + ...keyPair, })); await transactionalEntityManager.save(new MiUserProfile({ From 1d780ac010875314928fcba257d4ca34ade3ea15 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 27 Feb 2024 01:41:34 +0000 Subject: [PATCH 006/134] signatureAlgorithm --- .../src/core/activitypub/ApRendererService.ts | 3 ++- .../src/core/activitypub/models/ApPersonService.ts | 12 ++++++------ packages/backend/src/core/activitypub/type.ts | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 66322f68703b..66a15b2d1889 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -501,7 +501,7 @@ export class ApRendererService { discoverable: user.isExplorable, publicKey: this.renderKey(user, keypair.publicKey, '#main-key'), additionalPublicKeys: [ - ...(keypair.ed25519PublicKey ? [this.renderKey(user, keypair.ed25519PublicKey, '#ed25519-key', { type: keypair.ed25519SignatureAlgorithm!, signatureValue: keypair.ed25519PublicKeySignature! })] : []), + ...(keypair.ed25519PublicKey ? [this.renderKey(user, keypair.ed25519PublicKey, '#ed25519-key', { signatureAlgorithm: keypair.ed25519SignatureAlgorithm!, signatureValue: keypair.ed25519PublicKeySignature! })] : []), ], isCat: user.isCat, attachment: attachment.length ? attachment : undefined, @@ -649,6 +649,7 @@ export class ApRendererService { '_misskey_votes': 'misskey:_misskey_votes', '_misskey_summary': 'misskey:_misskey_summary', 'isCat': 'misskey:isCat', + additionalPublicKeys: 'misskey:additionalPublicKeys', // vcard vcard: 'http://www.w3.org/2006/vcard/ns#', }, diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 4c1f79e1b1ea..36fad4c7594e 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -216,8 +216,8 @@ export class ApPersonService implements OnModuleInit { if (!key.signature) { throw new Error('invalid Actor: additionalPublicKeys.signature is not set'); } - if (typeof key.signature.type !== 'string') { - throw new Error('invalid Actor: additionalPublicKeys.signature.type is not a string'); + if (typeof key.signature.signatureAlgorithm !== 'string') { + throw new Error('invalid Actor: additionalPublicKeys.signature.signatureAlgorithm is not a string'); } if (typeof key.signature.signatureValue !== 'string') { throw new Error('invalid Actor: additionalPublicKeys.signature.signatureValue is not a string'); @@ -397,8 +397,8 @@ export class ApPersonService implements OnModuleInit { if (person.additionalPublicKeys) { for (const key of person.additionalPublicKeys) { if ( - key.signature && key.signature.type && key.signature.signatureValue && - verify(key.signature.type, Buffer.from(key.publicKeyPem), person.publicKey.publicKeyPem, Buffer.from(key.signature.signatureValue, 'base64')) + key.signature && key.signature.signatureAlgorithm && key.signature.signatureValue && + verify(key.signature.signatureAlgorithm, Buffer.from(key.publicKeyPem), person.publicKey.publicKeyPem, Buffer.from(key.signature.signatureValue, 'base64')) ) { await transactionalEntityManager.save(new MiUserPublickey({ keyId: key.id, @@ -564,8 +564,8 @@ export class ApPersonService implements OnModuleInit { if (person.additionalPublicKeys) { for (const key of person.additionalPublicKeys) { if ( - key.signature && key.signature.type && key.signature.signatureValue && - verify(key.signature.type, Buffer.from(key.publicKeyPem), person.publicKey.publicKeyPem, Buffer.from(key.signature.signatureValue, 'base64')) + key.signature && key.signature.signatureAlgorithm && key.signature.signatureValue && + verify(key.signature.signatureAlgorithm, Buffer.from(key.publicKeyPem), person.publicKey.publicKeyPem, Buffer.from(key.signature.signatureValue, 'base64')) ) { await this.userPublickeysRepository.update({ keyId: key.id }, { userId: exist.id, diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 72d383442eca..ff0d340b6bd8 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -241,7 +241,7 @@ export interface IKey extends IObject { * Signature of publicKeyPem, signed by root privateKey (for additionalPublicKey) */ signature?: { - type: string; + signatureAlgorithm: string; signatureValue: string }; } From 13e0a64a7744d369983e6ec0ee8a56075945e95f Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 27 Feb 2024 01:57:35 +0000 Subject: [PATCH 007/134] set publicKeyCache lifetime --- packages/backend/src/core/activitypub/ApDbResolverService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 0cb3b78941e0..273572e0da8c 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -54,8 +54,8 @@ export class ApDbResolverService implements OnApplicationShutdown { private cacheService: CacheService, private apPersonService: ApPersonService, ) { - this.publicKeyCache = new MemoryKVCache(Infinity); - this.publicKeyByUserIdCache = new MemoryKVCache(Infinity); + this.publicKeyCache = new MemoryKVCache(1e3 * 60 * 20); // 20分 + this.publicKeyByUserIdCache = new MemoryKVCache(1e3 * 60 * 20); // 20分 } @bindThis From e2a8f4f88091554bc0fca8ec67f38673ffa1c822 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 27 Feb 2024 02:15:49 +0000 Subject: [PATCH 008/134] refresh --- .../src/core/activitypub/ApDbResolverService.ts | 12 +++++++++++- .../src/queue/processors/InboxProcessorService.ts | 3 ++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 273572e0da8c..904c89b990a5 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -5,7 +5,7 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; -import type { NotesRepository, UserPublickeysRepository, UsersRepository } from '@/models/_.js'; +import type { MiUser, NotesRepository, UserPublickeysRepository, UsersRepository } from '@/models/_.js'; import type { Config } from '@/config.js'; import { MemoryKVCache } from '@/misc/cache.js'; import type { MiUserPublickey } from '@/models/UserPublickey.js'; @@ -190,6 +190,16 @@ export class ApDbResolverService implements OnApplicationShutdown { }; } + @bindThis + public refreshCacheByUserId(userId: MiUser['id']): void { + this.publicKeyByUserIdCache.delete(userId); + for (const [k, v] of this.publicKeyCache.cache.entries()) { + if (v.value?.userId === userId) { + this.publicKeyCache.delete(k); + } + } + } + @bindThis public dispose(): void { this.publicKeyCache.dispose(); diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 0a713149e5f1..bc593147bbae 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -119,7 +119,8 @@ export class InboxProcessorService { // みたいになっててUserを引っ張れば公開キーも入ることを期待する if (activity.signature.creator) { const candicate = activity.signature.creator.replace(/#.*/, ''); - await this.apPersonService.resolvePerson(candicate).catch(() => null); + const user = await this.apPersonService.resolvePerson(candicate).catch(() => null); + if (user) this.apDbResolverService.refreshCacheByUserId(user.id); } // keyIdからLD-Signatureのユーザーを取得 From 5876a28f1e815069ddf312f32d097562b1ad8dcf Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 27 Feb 2024 03:01:07 +0000 Subject: [PATCH 009/134] httpMessageSignatureAcceptable --- .../src/server/NodeinfoServerService.ts | 51 ++++++++++++++++--- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/server/NodeinfoServerService.ts b/packages/backend/src/server/NodeinfoServerService.ts index c1e5af08c903..f9a9ce04491a 100644 --- a/packages/backend/src/server/NodeinfoServerService.ts +++ b/packages/backend/src/server/NodeinfoServerService.ts @@ -37,12 +37,12 @@ export class NodeinfoServerService { @bindThis public getLinks() { return [{ - rel: 'http://nodeinfo.diaspora.software/ns/schema/2.1', - href: this.config.url + nodeinfo2_1path - }, { - rel: 'http://nodeinfo.diaspora.software/ns/schema/2.0', - href: this.config.url + nodeinfo2_0path, - }]; + rel: 'http://nodeinfo.diaspora.software/ns/schema/2.1', + href: this.config.url + nodeinfo2_1path, + }, { + rel: 'http://nodeinfo.diaspora.software/ns/schema/2.0', + href: this.config.url + nodeinfo2_0path, + }]; } @bindThis @@ -94,6 +94,45 @@ export class NodeinfoServerService { localComments: 0, }, metadata: { + httpMessageSignatureAcceptable: [ + { + version: 'draft-ietf-httpbis-message-signatures-02', + /** + * https://datatracker.ietf.org/doc/html/draft-ietf-httpbis-message-signatures-02#name-initial-contents + */ + algorithms: [ + 'hs2019', + // 'rsa-sha1', [Deprecated] + 'rsa-sha256', + 'hmac-sha256', + 'ecdsa-sha256', + ], + /** + * https://datatracker.ietf.org/doc/html/rfc9421#section-6.2 + */ + hs2019: [ + 'ecdsa-p384-sha384', + 'ed25519', + ], + }, + /** + * rfc9421 algorithms: + * https://datatracker.ietf.org/doc/html/rfc9421#section-6.2 + * Misskeyはnode:crypto.verifyに食わせるだけなので、だいたいなんでもいけるはず + */ + /*{ + version: 'rfc9421', + algorithms: [ + 'rsa-pss-sha512', + 'rsa-v1_5-sha256', + 'hmac-sha256', + 'ecdsa-p256-sha256', + 'ecdsa-p384-sha384', + 'ed25519', + ], + },*/ + ], + nodeName: meta.name, nodeDescription: meta.description, nodeAdmins: [{ From eb8bef486d82618c3a6b2ee6d7094eea685fb1f1 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 27 Feb 2024 03:06:19 +0000 Subject: [PATCH 010/134] ED25519_SIGNED_ALGORITHM --- packages/backend/src/core/UserKeypairService.ts | 6 +++--- packages/backend/src/misc/gen-key-pair.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 7946f410cf3f..78a18b438d7a 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -12,7 +12,7 @@ import { RedisKVCache } from '@/misc/cache.js'; import type { MiUserKeypair } from '@/models/UserKeypair.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; -import { ED25519_SIGN_ALGORITHM, genEd25519KeyPair } from '@/misc/gen-key-pair.js'; +import { ED25519_SIGNED_ALGORITHM, genEd25519KeyPair } from '@/misc/gen-key-pair.js'; import { GlobalEventService, GlobalEvents } from '@/core/GlobalEventService.js'; @Injectable() @@ -56,12 +56,12 @@ export class UserKeypairService implements OnApplicationShutdown { const keypair = await this.cache.fetch(userId); if (keypair.ed25519PublicKey != null) return; const ed25519 = await genEd25519KeyPair(); - const ed25519PublicKeySignature = sign(ED25519_SIGN_ALGORITHM, Buffer.from(ed25519.publicKey), keypair.privateKey).toString('base64'); + const ed25519PublicKeySignature = sign(ED25519_SIGNED_ALGORITHM, Buffer.from(ed25519.publicKey), keypair.privateKey).toString('base64'); await this.userKeypairsRepository.update({ userId }, { ed25519PublicKey: ed25519.publicKey, ed25519PrivateKey: ed25519.privateKey, ed25519PublicKeySignature, - ed25519SignatureAlgorithm: `rsa-${ED25519_SIGN_ALGORITHM}`, + ed25519SignatureAlgorithm: `rsa-${ED25519_SIGNED_ALGORITHM}`, }); this.globalEventService.publishInternalEvent('userKeypairUpdated', { userId }); } diff --git a/packages/backend/src/misc/gen-key-pair.ts b/packages/backend/src/misc/gen-key-pair.ts index 7b2e84d99151..51ea093b2985 100644 --- a/packages/backend/src/misc/gen-key-pair.ts +++ b/packages/backend/src/misc/gen-key-pair.ts @@ -8,7 +8,7 @@ import * as util from 'node:util'; const generateKeyPair = util.promisify(crypto.generateKeyPair); -export const ED25519_SIGN_ALGORITHM = 'sha256'; +export const ED25519_SIGNED_ALGORITHM = 'sha256'; export async function genRsaKeyPair(modulusLength = 4096) { return await generateKeyPair('rsa', { @@ -44,13 +44,13 @@ export async function genEd25519KeyPair() { export async function genRSAAndEd25519KeyPair(rsaModulusLength = 4096) { const rsa = await genRsaKeyPair(rsaModulusLength); const ed25519 = await genEd25519KeyPair(); - const ed25519PublicKeySignature = crypto.sign(ED25519_SIGN_ALGORITHM, Buffer.from(ed25519.publicKey), rsa.privateKey).toString('base64'); + const ed25519PublicKeySignature = crypto.sign(ED25519_SIGNED_ALGORITHM, Buffer.from(ed25519.publicKey), rsa.privateKey).toString('base64'); return { publicKey: rsa.publicKey, privateKey: rsa.privateKey, ed25519PublicKey: ed25519.publicKey, ed25519PrivateKey: ed25519.privateKey, ed25519PublicKeySignature, - ed25519SignatureAlgorithm: `rsa-${ED25519_SIGN_ALGORITHM}`, + ed25519SignatureAlgorithm: `rsa-${ED25519_SIGNED_ALGORITHM}`, }; } From 437e69cfc48f51b5be168cddc9a61f7b5dbfb4d7 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 27 Feb 2024 03:08:55 +0000 Subject: [PATCH 011/134] ED25519_PUBLIC_KEY_SIGNATURE_ALGORITHM --- packages/backend/src/core/UserKeypairService.ts | 6 +++--- packages/backend/src/misc/gen-key-pair.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 78a18b438d7a..246b74757619 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -12,7 +12,7 @@ import { RedisKVCache } from '@/misc/cache.js'; import type { MiUserKeypair } from '@/models/UserKeypair.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; -import { ED25519_SIGNED_ALGORITHM, genEd25519KeyPair } from '@/misc/gen-key-pair.js'; +import { ED25519_PUBLIC_KEY_SIGNATURE_ALGORITHM, genEd25519KeyPair } from '@/misc/gen-key-pair.js'; import { GlobalEventService, GlobalEvents } from '@/core/GlobalEventService.js'; @Injectable() @@ -56,12 +56,12 @@ export class UserKeypairService implements OnApplicationShutdown { const keypair = await this.cache.fetch(userId); if (keypair.ed25519PublicKey != null) return; const ed25519 = await genEd25519KeyPair(); - const ed25519PublicKeySignature = sign(ED25519_SIGNED_ALGORITHM, Buffer.from(ed25519.publicKey), keypair.privateKey).toString('base64'); + const ed25519PublicKeySignature = sign(ED25519_PUBLIC_KEY_SIGNATURE_ALGORITHM, Buffer.from(ed25519.publicKey), keypair.privateKey).toString('base64'); await this.userKeypairsRepository.update({ userId }, { ed25519PublicKey: ed25519.publicKey, ed25519PrivateKey: ed25519.privateKey, ed25519PublicKeySignature, - ed25519SignatureAlgorithm: `rsa-${ED25519_SIGNED_ALGORITHM}`, + ed25519SignatureAlgorithm: `rsa-${ED25519_PUBLIC_KEY_SIGNATURE_ALGORITHM}`, }); this.globalEventService.publishInternalEvent('userKeypairUpdated', { userId }); } diff --git a/packages/backend/src/misc/gen-key-pair.ts b/packages/backend/src/misc/gen-key-pair.ts index 51ea093b2985..c4b44597c5a4 100644 --- a/packages/backend/src/misc/gen-key-pair.ts +++ b/packages/backend/src/misc/gen-key-pair.ts @@ -8,7 +8,7 @@ import * as util from 'node:util'; const generateKeyPair = util.promisify(crypto.generateKeyPair); -export const ED25519_SIGNED_ALGORITHM = 'sha256'; +export const ED25519_PUBLIC_KEY_SIGNATURE_ALGORITHM = 'sha256'; export async function genRsaKeyPair(modulusLength = 4096) { return await generateKeyPair('rsa', { @@ -44,13 +44,13 @@ export async function genEd25519KeyPair() { export async function genRSAAndEd25519KeyPair(rsaModulusLength = 4096) { const rsa = await genRsaKeyPair(rsaModulusLength); const ed25519 = await genEd25519KeyPair(); - const ed25519PublicKeySignature = crypto.sign(ED25519_SIGNED_ALGORITHM, Buffer.from(ed25519.publicKey), rsa.privateKey).toString('base64'); + const ed25519PublicKeySignature = crypto.sign(ED25519_PUBLIC_KEY_SIGNATURE_ALGORITHM, Buffer.from(ed25519.publicKey), rsa.privateKey).toString('base64'); return { publicKey: rsa.publicKey, privateKey: rsa.privateKey, ed25519PublicKey: ed25519.publicKey, ed25519PrivateKey: ed25519.privateKey, ed25519PublicKeySignature, - ed25519SignatureAlgorithm: `rsa-${ED25519_SIGNED_ALGORITHM}`, + ed25519SignatureAlgorithm: `rsa-${ED25519_PUBLIC_KEY_SIGNATURE_ALGORITHM}`, }; } From 9705ec4a47544f6c8cedb90479f6183faaf87007 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 27 Feb 2024 03:33:50 +0000 Subject: [PATCH 012/134] remove sign additionalPublicKeys signature requirements --- .../migration/1708980134301-APMultipleKeys.js | 4 -- .../backend/src/core/UserKeypairService.ts | 6 +-- .../src/core/activitypub/ApRendererService.ts | 5 +-- .../activitypub/models/ApPersonService.ts | 40 +++++-------------- packages/backend/src/core/activitypub/type.ts | 8 ---- packages/backend/src/misc/gen-key-pair.ts | 5 --- packages/backend/src/models/UserKeypair.ts | 20 ---------- .../backend/test/unit/misc/gen-key-pair.ts | 6 --- 8 files changed, 13 insertions(+), 81 deletions(-) diff --git a/packages/backend/migration/1708980134301-APMultipleKeys.js b/packages/backend/migration/1708980134301-APMultipleKeys.js index d0687fa7faae..ca55526c6e03 100644 --- a/packages/backend/migration/1708980134301-APMultipleKeys.js +++ b/packages/backend/migration/1708980134301-APMultipleKeys.js @@ -10,8 +10,6 @@ export class APMultipleKeys1708980134301 { await queryRunner.query(`DROP INDEX "public"."IDX_171e64971c780ebd23fae140bb"`); await queryRunner.query(`ALTER TABLE "user_keypair" ADD "ed25519PublicKey" character varying(128)`); await queryRunner.query(`ALTER TABLE "user_keypair" ADD "ed25519PrivateKey" character varying(128)`); - await queryRunner.query(`ALTER TABLE "user_keypair" ADD "ed25519PublicKeySignature" character varying(720)`); - await queryRunner.query(`ALTER TABLE "user_keypair" ADD "ed25519SignatureAlgorithm" character varying(32)`); await queryRunner.query(`ALTER TABLE "user_publickey" DROP CONSTRAINT "FK_10c146e4b39b443ede016f6736d"`); await queryRunner.query(`ALTER TABLE "user_publickey" DROP CONSTRAINT "PK_10c146e4b39b443ede016f6736d"`); await queryRunner.query(`ALTER TABLE "user_publickey" ADD CONSTRAINT "PK_0db6a5fdb992323449edc8ee421" PRIMARY KEY ("userId", "keyId")`); @@ -34,8 +32,6 @@ export class APMultipleKeys1708980134301 { await queryRunner.query(`ALTER TABLE "user_profile" ALTER COLUMN "followersVisibility" DROP DEFAULT`); await queryRunner.query(`ALTER TABLE "user_profile" ALTER COLUMN "followersVisibility" TYPE "public"."user_profile_followersVisibility_enum_old" USING "followersVisibility"::"text"::"public"."user_profile_followersVisibility_enum_old"`); await queryRunner.query(`ALTER TABLE "user_profile" ALTER COLUMN "followersVisibility" SET DEFAULT 'public'`); - await queryRunner.query(`ALTER TABLE "user_keypair" DROP COLUMN "ed25519SignatureAlgorithm"`); - await queryRunner.query(`ALTER TABLE "user_keypair" DROP COLUMN "ed25519PublicKeySignature"`); await queryRunner.query(`ALTER TABLE "user_keypair" DROP COLUMN "ed25519PrivateKey"`); await queryRunner.query(`ALTER TABLE "user_keypair" DROP COLUMN "ed25519PublicKey"`); await queryRunner.query(`CREATE UNIQUE INDEX "IDX_171e64971c780ebd23fae140bb" ON "user_publickey" ("keyId") `); diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 246b74757619..10818cfc7f8a 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -3,7 +3,6 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { sign } from 'node:crypto'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; import type { MiUser } from '@/models/User.js'; @@ -12,7 +11,7 @@ import { RedisKVCache } from '@/misc/cache.js'; import type { MiUserKeypair } from '@/models/UserKeypair.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; -import { ED25519_PUBLIC_KEY_SIGNATURE_ALGORITHM, genEd25519KeyPair } from '@/misc/gen-key-pair.js'; +import { genEd25519KeyPair } from '@/misc/gen-key-pair.js'; import { GlobalEventService, GlobalEvents } from '@/core/GlobalEventService.js'; @Injectable() @@ -56,12 +55,9 @@ export class UserKeypairService implements OnApplicationShutdown { const keypair = await this.cache.fetch(userId); if (keypair.ed25519PublicKey != null) return; const ed25519 = await genEd25519KeyPair(); - const ed25519PublicKeySignature = sign(ED25519_PUBLIC_KEY_SIGNATURE_ALGORITHM, Buffer.from(ed25519.publicKey), keypair.privateKey).toString('base64'); await this.userKeypairsRepository.update({ userId }, { ed25519PublicKey: ed25519.publicKey, ed25519PrivateKey: ed25519.privateKey, - ed25519PublicKeySignature, - ed25519SignatureAlgorithm: `rsa-${ED25519_PUBLIC_KEY_SIGNATURE_ALGORITHM}`, }); this.globalEventService.publishInternalEvent('userKeypairUpdated', { userId }); } diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 66a15b2d1889..d929fa4ee1cd 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -250,7 +250,7 @@ export class ApRendererService { } @bindThis - public renderKey(user: MiLocalUser, publicKey: string, postfix?: string, signature?: IKey['signature']): IKey { + public renderKey(user: MiLocalUser, publicKey: string, postfix?: string): IKey { return { id: `${this.config.url}/users/${user.id}${postfix ?? '/publickey'}`, type: 'Key', @@ -259,7 +259,6 @@ export class ApRendererService { type: 'spki', format: 'pem', }) as string, - signature, }; } @@ -501,7 +500,7 @@ export class ApRendererService { discoverable: user.isExplorable, publicKey: this.renderKey(user, keypair.publicKey, '#main-key'), additionalPublicKeys: [ - ...(keypair.ed25519PublicKey ? [this.renderKey(user, keypair.ed25519PublicKey, '#ed25519-key', { signatureAlgorithm: keypair.ed25519SignatureAlgorithm!, signatureValue: keypair.ed25519PublicKeySignature! })] : []), + ...(keypair.ed25519PublicKey ? [this.renderKey(user, keypair.ed25519PublicKey, '#ed25519-key')] : []), ], isCat: user.isCat, attachment: attachment.length ? attachment : undefined, diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 36fad4c7594e..5100a2460095 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -212,16 +212,6 @@ export class ApPersonService implements OnModuleInit { if (keyIdHost !== expectHost) { throw new Error('invalid Actor: additionalPublicKeys.id has different host'); } - - if (!key.signature) { - throw new Error('invalid Actor: additionalPublicKeys.signature is not set'); - } - if (typeof key.signature.signatureAlgorithm !== 'string') { - throw new Error('invalid Actor: additionalPublicKeys.signature.signatureAlgorithm is not a string'); - } - if (typeof key.signature.signatureValue !== 'string') { - throw new Error('invalid Actor: additionalPublicKeys.signature.signatureValue is not a string'); - } } } @@ -396,16 +386,11 @@ export class ApPersonService implements OnModuleInit { if (person.additionalPublicKeys) { for (const key of person.additionalPublicKeys) { - if ( - key.signature && key.signature.signatureAlgorithm && key.signature.signatureValue && - verify(key.signature.signatureAlgorithm, Buffer.from(key.publicKeyPem), person.publicKey.publicKeyPem, Buffer.from(key.signature.signatureValue, 'base64')) - ) { - await transactionalEntityManager.save(new MiUserPublickey({ - keyId: key.id, - userId: user.id, - keyPem: key.publicKeyPem, - })); - } + await transactionalEntityManager.save(new MiUserPublickey({ + keyId: key.id, + userId: user.id, + keyPem: key.publicKeyPem, + })); } } } @@ -563,16 +548,11 @@ export class ApPersonService implements OnModuleInit { if (person.additionalPublicKeys) { for (const key of person.additionalPublicKeys) { - if ( - key.signature && key.signature.signatureAlgorithm && key.signature.signatureValue && - verify(key.signature.signatureAlgorithm, Buffer.from(key.publicKeyPem), person.publicKey.publicKeyPem, Buffer.from(key.signature.signatureValue, 'base64')) - ) { - await this.userPublickeysRepository.update({ keyId: key.id }, { - userId: exist.id, - keyPem: key.publicKeyPem, - }); - availablePublicKeys.add(key.id); - } + await this.userPublickeysRepository.update({ keyId: key.id }, { + userId: exist.id, + keyPem: key.publicKeyPem, + }); + availablePublicKeys.add(key.id); } } } diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index ff0d340b6bd8..0b34fd766447 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -236,14 +236,6 @@ export interface IKey extends IObject { id: string; owner: string; publicKeyPem: string; - - /** - * Signature of publicKeyPem, signed by root privateKey (for additionalPublicKey) - */ - signature?: { - signatureAlgorithm: string; - signatureValue: string - }; } export interface IApDocument extends IObject { diff --git a/packages/backend/src/misc/gen-key-pair.ts b/packages/backend/src/misc/gen-key-pair.ts index c4b44597c5a4..a214591a81b3 100644 --- a/packages/backend/src/misc/gen-key-pair.ts +++ b/packages/backend/src/misc/gen-key-pair.ts @@ -8,8 +8,6 @@ import * as util from 'node:util'; const generateKeyPair = util.promisify(crypto.generateKeyPair); -export const ED25519_PUBLIC_KEY_SIGNATURE_ALGORITHM = 'sha256'; - export async function genRsaKeyPair(modulusLength = 4096) { return await generateKeyPair('rsa', { modulusLength, @@ -44,13 +42,10 @@ export async function genEd25519KeyPair() { export async function genRSAAndEd25519KeyPair(rsaModulusLength = 4096) { const rsa = await genRsaKeyPair(rsaModulusLength); const ed25519 = await genEd25519KeyPair(); - const ed25519PublicKeySignature = crypto.sign(ED25519_PUBLIC_KEY_SIGNATURE_ALGORITHM, Buffer.from(ed25519.publicKey), rsa.privateKey).toString('base64'); return { publicKey: rsa.publicKey, privateKey: rsa.privateKey, ed25519PublicKey: ed25519.publicKey, ed25519PrivateKey: ed25519.privateKey, - ed25519PublicKeySignature, - ed25519SignatureAlgorithm: `rsa-${ED25519_PUBLIC_KEY_SIGNATURE_ALGORITHM}`, }; } diff --git a/packages/backend/src/models/UserKeypair.ts b/packages/backend/src/models/UserKeypair.ts index 47d44af849ff..59b37bb9e9e4 100644 --- a/packages/backend/src/models/UserKeypair.ts +++ b/packages/backend/src/models/UserKeypair.ts @@ -48,26 +48,6 @@ export class MiUserKeypair { }) public ed25519PrivateKey: string | null; - /** - * Signature of ed25519PublicKey, signed by privateKey. (base64) - */ - @Column('varchar', { - length: 720, - nullable: true, - default: null, - }) - public ed25519PublicKeySignature: string | null; - - /** - * Signature algorithm of ed25519PublicKeySignature. - */ - @Column('varchar', { - length: 32, - nullable: true, - default: null, - }) - public ed25519SignatureAlgorithm: string | null; - constructor(data: Partial) { if (data == null) return; diff --git a/packages/backend/test/unit/misc/gen-key-pair.ts b/packages/backend/test/unit/misc/gen-key-pair.ts index 91c62f621d07..771147d299dc 100644 --- a/packages/backend/test/unit/misc/gen-key-pair.ts +++ b/packages/backend/test/unit/misc/gen-key-pair.ts @@ -29,12 +29,6 @@ describe(genRSAAndEd25519KeyPair, () => { expect(keyPair.ed25519PrivateKey).toMatch(/^-----BEGIN PRIVATE KEY-----/); expect(keyPair.ed25519PrivateKey).toMatch(/-----END PRIVATE KEY-----\n$/); expect(keyPair.ed25519PrivateKey).not.toBe(keyPair2.ed25519PrivateKey); - expect(keyPair.ed25519PublicKeySignature).toBe( - crypto.sign(keyPair.ed25519SignatureAlgorithm.split('-').pop(), Buffer.from(keyPair.ed25519PublicKey), keyPair.privateKey).toString('base64'), - ); - expect(crypto.verify(keyPair.ed25519SignatureAlgorithm, Buffer.from(keyPair.ed25519PublicKey), keyPair.publicKey, Buffer.from(keyPair.ed25519PublicKeySignature, 'base64'))).toBe(true); - expect(keyPair.ed25519PublicKeySignature).not.toBe(keyPair2.ed25519PublicKeySignature); - //const imported = await webCrypto.subtle.importKey('spki', Buffer.from(keyPair.publicKey).buffer, { name: 'rsa-pss', hash: 'sha-256' }, false, ['verify']); }); }); From f6b7872a0285f0578d302c82ff516e759c7bfc71 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 27 Feb 2024 03:37:02 +0000 Subject: [PATCH 013/134] httpMessageSignaturesSupported --- packages/backend/src/server/NodeinfoServerService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/server/NodeinfoServerService.ts b/packages/backend/src/server/NodeinfoServerService.ts index f9a9ce04491a..1b50d7183986 100644 --- a/packages/backend/src/server/NodeinfoServerService.ts +++ b/packages/backend/src/server/NodeinfoServerService.ts @@ -94,7 +94,7 @@ export class NodeinfoServerService { localComments: 0, }, metadata: { - httpMessageSignatureAcceptable: [ + httpMessageSignaturesSupported: [ { version: 'draft-ietf-httpbis-message-signatures-02', /** From 8579cb222f269ce490512112f7f2157a7dbabf86 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 27 Feb 2024 04:25:51 +0000 Subject: [PATCH 014/134] httpMessageSignaturesImplementationLevel --- .../src/server/NodeinfoServerService.ts | 44 +++---------------- 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/packages/backend/src/server/NodeinfoServerService.ts b/packages/backend/src/server/NodeinfoServerService.ts index 1b50d7183986..3b7ee79ca380 100644 --- a/packages/backend/src/server/NodeinfoServerService.ts +++ b/packages/backend/src/server/NodeinfoServerService.ts @@ -94,44 +94,12 @@ export class NodeinfoServerService { localComments: 0, }, metadata: { - httpMessageSignaturesSupported: [ - { - version: 'draft-ietf-httpbis-message-signatures-02', - /** - * https://datatracker.ietf.org/doc/html/draft-ietf-httpbis-message-signatures-02#name-initial-contents - */ - algorithms: [ - 'hs2019', - // 'rsa-sha1', [Deprecated] - 'rsa-sha256', - 'hmac-sha256', - 'ecdsa-sha256', - ], - /** - * https://datatracker.ietf.org/doc/html/rfc9421#section-6.2 - */ - hs2019: [ - 'ecdsa-p384-sha384', - 'ed25519', - ], - }, - /** - * rfc9421 algorithms: - * https://datatracker.ietf.org/doc/html/rfc9421#section-6.2 - * Misskeyはnode:crypto.verifyに食わせるだけなので、だいたいなんでもいけるはず - */ - /*{ - version: 'rfc9421', - algorithms: [ - 'rsa-pss-sha512', - 'rsa-v1_5-sha256', - 'hmac-sha256', - 'ecdsa-p256-sha256', - 'ecdsa-p384-sha384', - 'ed25519', - ], - },*/ - ], + /** + * 00: Draft, RSA only + * 01: Draft, Ed25519 suported + * 11: RFC 9421, Ed25519 supported + */ + httpMessageSignaturesImplementationLevel: 1, nodeName: meta.name, nodeDescription: meta.description, From 59ae73516976da0f1d34dd320a2bb76f6df14d69 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 27 Feb 2024 04:52:40 +0000 Subject: [PATCH 015/134] httpMessageSignaturesImplementationLevel: '01' --- packages/backend/src/server/NodeinfoServerService.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/server/NodeinfoServerService.ts b/packages/backend/src/server/NodeinfoServerService.ts index 3b7ee79ca380..a5d4fdbddc5f 100644 --- a/packages/backend/src/server/NodeinfoServerService.ts +++ b/packages/backend/src/server/NodeinfoServerService.ts @@ -95,11 +95,11 @@ export class NodeinfoServerService { }, metadata: { /** - * 00: Draft, RSA only - * 01: Draft, Ed25519 suported - * 11: RFC 9421, Ed25519 supported + * '00': Draft, RSA only + * '01': Draft, Ed25519 suported + * '11': RFC 9421, Ed25519 supported */ - httpMessageSignaturesImplementationLevel: 1, + httpMessageSignaturesImplementationLevel: '01', nodeName: meta.name, nodeDescription: meta.description, From aaacfabc1b33c8ea95c0cd6d232abb3767dc77a0 Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Wed, 28 Feb 2024 16:44:01 +0900 Subject: [PATCH 016/134] perf(federation): Use hint for getAuthUserFromApId (#13470) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Hint for getAuthUserFromApId * とどのつまりこれでいいのか？ --- .../core/activitypub/ApDbResolverService.ts | 41 +------------------ .../queue/processors/InboxProcessorService.ts | 32 +++++---------- 2 files changed, 13 insertions(+), 60 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 904c89b990a5..4013eaddc0fd 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -35,7 +35,6 @@ export type UriParseResult = { @Injectable() export class ApDbResolverService implements OnApplicationShutdown { - private publicKeyCache: MemoryKVCache; private publicKeyByUserIdCache: MemoryKVCache; constructor( @@ -54,7 +53,6 @@ export class ApDbResolverService implements OnApplicationShutdown { private cacheService: CacheService, private apPersonService: ApPersonService, ) { - this.publicKeyCache = new MemoryKVCache(1e3 * 60 * 20); // 20分 this.publicKeyByUserIdCache = new MemoryKVCache(1e3 * 60 * 20); // 20分 } @@ -116,41 +114,11 @@ export class ApDbResolverService implements OnApplicationShutdown { } } - /** - * AP KeyId => Misskey User and Key - */ - @bindThis - public async getAuthUserFromKeyId(keyId: string): Promise<{ - user: MiRemoteUser; - key: MiUserPublickey; - } | null> { - const key = await this.publicKeyCache.fetch(keyId, async () => { - const key = await this.userPublickeysRepository.findOneBy({ - keyId, - }); - - if (key == null) return null; - - return key; - }, key => key != null); - - if (key == null) return null; - - const user = await this.cacheService.findUserById(key.userId).catch(() => null) as MiRemoteUser | null; - if (user == null) return null; - if (user.isDeleted) return null; - - return { - user, - key, - }; - } - /** * AP Actor id => Misskey User and Key */ @bindThis - public async getAuthUserFromApId(uri: string): Promise<{ + public async getAuthUserFromApId(uri: string, keyId?: string): Promise<{ user: MiRemoteUser; key: MiUserPublickey | null; } | null> { @@ -170,6 +138,7 @@ export class ApDbResolverService implements OnApplicationShutdown { keys[0] : keys.find(x => { try { + if (x.keyId === keyId) return true; const url = new URL(x.keyId); const path = url.pathname.split('/').pop()?.toLowerCase(); if (url.hash) { @@ -193,16 +162,10 @@ export class ApDbResolverService implements OnApplicationShutdown { @bindThis public refreshCacheByUserId(userId: MiUser['id']): void { this.publicKeyByUserIdCache.delete(userId); - for (const [k, v] of this.publicKeyCache.cache.entries()) { - if (v.value?.userId === userId) { - this.publicKeyCache.delete(k); - } - } } @bindThis public dispose(): void { - this.publicKeyCache.dispose(); this.publicKeyByUserIdCache.dispose(); } diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index bc593147bbae..ac7f8e6cd920 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -77,20 +77,18 @@ export class InboxProcessorService { let authUser: { user: MiRemoteUser; key: MiUserPublickey | null; - } | null = await this.apDbResolverService.getAuthUserFromKeyId(signature.keyId); + } | null = null; // keyIdでわからなければ、activity.actorを元にDBから取得 || activity.actorを元にリモートから取得 - if (authUser == null) { - try { - authUser = await this.apDbResolverService.getAuthUserFromApId(getApId(activity.actor)); - } catch (err) { - // 対象が4xxならスキップ - if (err instanceof StatusError) { - if (!err.isRetryable) { - throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`); - } - throw new Error(`Error in actor ${activity.actor} - ${err.statusCode}`); + try { + authUser = await this.apDbResolverService.getAuthUserFromApId(getApId(activity.actor), signature.keyId); + } catch (err) { + // 対象が4xxならスキップ + if (err instanceof StatusError) { + if (!err.isRetryable) { + throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`); } + throw new Error(`Error in actor ${activity.actor} - ${err.statusCode}`); } } @@ -110,21 +108,13 @@ export class InboxProcessorService { // また、signatureのsignerは、activity.actorと一致する必要がある if (!httpSignatureValidated || authUser.user.uri !== activity.actor) { // 一致しなくても、でもLD-Signatureがありそうならそっちも見る - if (activity.signature) { + if (activity.signature?.creator) { if (activity.signature.type !== 'RsaSignature2017') { throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${activity.signature.type}`); } - // activity.signature.creator: https://example.oom/users/user#main-key - // みたいになっててUserを引っ張れば公開キーも入ることを期待する - if (activity.signature.creator) { - const candicate = activity.signature.creator.replace(/#.*/, ''); - const user = await this.apPersonService.resolvePerson(candicate).catch(() => null); - if (user) this.apDbResolverService.refreshCacheByUserId(user.id); - } + authUser = await this.apDbResolverService.getAuthUserFromApId(activity.signature.creator.replace(/#.*/, '')); - // keyIdからLD-Signatureのユーザーを取得 - authUser = await this.apDbResolverService.getAuthUserFromKeyId(activity.signature.creator); if (authUser == null) { throw new Bull.UnrecoverableError('skip: LD-Signatureのユーザーが取得できませんでした'); } From a1e6cb02b85e68eab23aa30e09751257186d033b Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 29 Feb 2024 21:05:31 +0000 Subject: [PATCH 017/134] use @misskey-dev/node-http-message-signatures --- packages/backend/package.json | 2 +- .../backend/src/@types/http-signature.d.ts | 82 ------------------- packages/backend/src/core/QueueService.ts | 6 +- .../queue/processors/InboxProcessorService.ts | 6 +- packages/backend/src/queue/types.ts | 20 ++++- .../src/server/ActivityPubServerService.ts | 60 +++----------- packages/backend/test/unit/ap-request.ts | 61 -------------- pnpm-lock.yaml | 19 ++--- 8 files changed, 45 insertions(+), 211 deletions(-) delete mode 100644 packages/backend/src/@types/http-signature.d.ts delete mode 100644 packages/backend/test/unit/ap-request.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index 9b38fd6228b6..072b9ece14fc 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -79,12 +79,12 @@ "@fastify/multipart": "8.1.0", "@fastify/static": "6.12.0", "@fastify/view": "8.2.0", + "@misskey-dev/node-http-message-signatures": "0.0.0-alpha.7", "@misskey-dev/sharp-read-bmp": "1.2.0", "@misskey-dev/summaly": "5.0.3", "@nestjs/common": "10.2.10", "@nestjs/core": "10.2.10", "@nestjs/testing": "10.2.10", - "@peertube/http-signature": "1.7.0", "@simplewebauthn/server": "9.0.3", "@sinonjs/fake-timers": "11.2.2", "@smithy/node-http-handler": "2.1.10", diff --git a/packages/backend/src/@types/http-signature.d.ts b/packages/backend/src/@types/http-signature.d.ts deleted file mode 100644 index 75b62e55f0fd..000000000000 --- a/packages/backend/src/@types/http-signature.d.ts +++ /dev/null @@ -1,82 +0,0 @@ -/* - * SPDX-FileCopyrightText: syuilo and misskey-project - * SPDX-License-Identifier: AGPL-3.0-only - */ - -declare module '@peertube/http-signature' { - import type { IncomingMessage, ClientRequest } from 'node:http'; - - interface ISignature { - keyId: string; - algorithm: string; - headers: string[]; - signature: string; - } - - interface IOptions { - headers?: string[]; - algorithm?: string; - strict?: boolean; - authorizationHeaderName?: string; - } - - interface IParseRequestOptions extends IOptions { - clockSkew?: number; - } - - interface IParsedSignature { - scheme: string; - params: ISignature; - signingString: string; - algorithm: string; - keyId: string; - } - - type RequestSignerConstructorOptions = - IRequestSignerConstructorOptionsFromProperties | - IRequestSignerConstructorOptionsFromFunction; - - interface IRequestSignerConstructorOptionsFromProperties { - keyId: string; - key: string | Buffer; - algorithm?: string; - } - - interface IRequestSignerConstructorOptionsFromFunction { - sign?: (data: string, cb: (err: any, sig: ISignature) => void) => void; - } - - class RequestSigner { - constructor(options: RequestSignerConstructorOptions); - - public writeHeader(header: string, value: string): string; - - public writeDateHeader(): string; - - public writeTarget(method: string, path: string): void; - - public sign(cb: (err: any, authz: string) => void): void; - } - - interface ISignRequestOptions extends IOptions { - keyId: string; - key: string; - httpVersion?: string; - } - - export function parse(request: IncomingMessage, options?: IParseRequestOptions): IParsedSignature; - export function parseRequest(request: IncomingMessage, options?: IParseRequestOptions): IParsedSignature; - - export function sign(request: ClientRequest, options: ISignRequestOptions): boolean; - export function signRequest(request: ClientRequest, options: ISignRequestOptions): boolean; - export function createSigner(): RequestSigner; - export function isSigner(obj: any): obj is RequestSigner; - - export function sshKeyToPEM(key: string): string; - export function sshKeyFingerprint(key: string): string; - export function pemToRsaSSHKey(pem: string, comment: string): string; - - export function verify(parsedSignature: IParsedSignature, pubkey: string | Buffer): boolean; - export function verifySignature(parsedSignature: IParsedSignature, pubkey: string | Buffer): boolean; - export function verifyHMAC(parsedSignature: IParsedSignature, secret: string): boolean; -} diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index c258a22927d4..ef2ec0d2ffee 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -12,11 +12,11 @@ import type { Config } from '@/config.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import type { Antenna } from '@/server/api/endpoints/i/import-antennas.js'; +import { ApRequestCreator } from '@/core/activitypub/ApRequestService.js'; import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, RelationshipQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js'; import type { DbJobData, DeliverJobData, RelationshipJobData, ThinUser } from '../queue/types.js'; -import type httpSignature from '@peertube/http-signature'; import type * as Bull from 'bullmq'; -import { ApRequestCreator } from '@/core/activitypub/ApRequestService.js'; +import type { ParsedSignature } from '@misskey-dev/node-http-message-signatures'; @Injectable() export class QueueService { @@ -136,7 +136,7 @@ export class QueueService { } @bindThis - public inbox(activity: IActivity, signature: httpSignature.IParsedSignature) { + public inbox(activity: IActivity, signature: ParsedSignature) { const data = { activity: activity, signature, diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index ac7f8e6cd920..85ce6417b822 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -5,8 +5,8 @@ import { URL } from 'node:url'; import { Injectable } from '@nestjs/common'; -import httpSignature from '@peertube/http-signature'; import * as Bull from 'bullmq'; +import { verifyDraftSignature } from '@misskey-dev/node-http-message-signatures'; import type Logger from '@/logger.js'; import { MetaService } from '@/core/MetaService.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; @@ -51,7 +51,7 @@ export class InboxProcessorService { @bindThis public async process(job: Bull.Job): Promise { - const signature = job.data.signature; // HTTP-signature + const signature = 'version' in job.data.signature ? job.data.signature.value : job.data.signature; const activity = job.data.activity; //#region Log @@ -103,7 +103,7 @@ export class InboxProcessorService { } // HTTP-Signatureの検証 - const httpSignatureValidated = httpSignature.verifySignature(signature, authUser.key.keyPem); + const httpSignatureValidated = verifyDraftSignature(signature, authUser.key.keyPem); // また、signatureのsignerは、activity.actorと一致する必要がある if (!httpSignatureValidated || authUser.user.uri !== activity.actor) { diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index ce57ba745eb4..135bccb60c0f 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -9,7 +9,23 @@ import type { MiNote } from '@/models/Note.js'; import type { MiUser } from '@/models/User.js'; import type { MiWebhook } from '@/models/Webhook.js'; import type { IActivity } from '@/core/activitypub/type.js'; -import type httpSignature from '@peertube/http-signature'; +import type { ParsedSignature } from '@misskey-dev/node-http-message-signatures'; + +/** + * @peertube/http-signature 時代の古いデータにも対応しておく + */ +export interface OldParsedSignature { + scheme: 'Signature'; + params: { + keyId: string; + algorithm: string; + headers: string[]; + signature: string; + }; + signingString: string; + algorithm: string; + keyId: string; +} export type DeliverJobData = { /** Actor */ @@ -26,7 +42,7 @@ export type DeliverJobData = { export type InboxJobData = { activity: IActivity; - signature: httpSignature.IParsedSignature; + signature: ParsedSignature | OldParsedSignature; }; export type RelationshipJobData = { diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index 5e6cb0a05a16..269bc3fb1162 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -7,7 +7,7 @@ import * as crypto from 'node:crypto'; import { IncomingMessage } from 'node:http'; import { Inject, Injectable } from '@nestjs/common'; import fastifyAccepts from '@fastify/accepts'; -import httpSignature from '@peertube/http-signature'; +import { verifyDigestHeader, parseRequestSignature } from '@misskey-dev/node-http-message-signatures'; import { Brackets, In, IsNull, LessThan, Not } from 'typeorm'; import accepts from 'accepts'; import vary from 'vary'; @@ -103,63 +103,29 @@ export class ActivityPubServerService { private inbox(request: FastifyRequest, reply: FastifyReply) { let signature; + const verifyDigest = verifyDigestHeader(request.raw, request.rawBody || '', true); + if (!verifyDigest) { + reply.code(401); + return; + } + try { - signature = httpSignature.parseRequest(request.raw, { 'headers': [] }); + signature = parseRequestSignature(request.raw); } catch (e) { reply.code(401); return; } - if (signature.params.headers.indexOf('host') === -1 - || request.headers.host !== this.config.host) { - // Host not specified or not match. + if (!signature) { reply.code(401); return; } - if (signature.params.headers.indexOf('digest') === -1) { - // Digest not found. + if (signature.value.params.headers.indexOf('host') === -1 + || request.headers.host !== this.config.host) { + // Host not specified or not match. reply.code(401); - } else { - const digest = request.headers.digest; - - if (typeof digest !== 'string') { - // Huh? - reply.code(401); - return; - } - - const re = /^([a-zA-Z0-9\-]+)=(.+)$/; - const match = digest.match(re); - - if (match == null) { - // Invalid digest - reply.code(401); - return; - } - - const algo = match[1].toUpperCase(); - const digestValue = match[2]; - - if (algo !== 'SHA-256') { - // Unsupported digest algorithm - reply.code(401); - return; - } - - if (request.rawBody == null) { - // Bad request - reply.code(400); - return; - } - - const hash = crypto.createHash('sha256').update(request.rawBody).digest('base64'); - - if (hash !== digestValue) { - // Invalid digest - reply.code(401); - return; - } + return; } this.queueService.inbox(request.body as IActivity, signature); diff --git a/packages/backend/test/unit/ap-request.ts b/packages/backend/test/unit/ap-request.ts deleted file mode 100644 index d3d39240dc9a..000000000000 --- a/packages/backend/test/unit/ap-request.ts +++ /dev/null @@ -1,61 +0,0 @@ -/* - * SPDX-FileCopyrightText: syuilo and misskey-project - * SPDX-License-Identifier: AGPL-3.0-only - */ - -import * as assert from 'assert'; -import httpSignature from '@peertube/http-signature'; - -import { genRsaKeyPair } from '@/misc/gen-key-pair.js'; -import { ApRequestCreator } from '@/core/activitypub/ApRequestService.js'; - -export const buildParsedSignature = (signingString: string, signature: string, algorithm: string) => { - return { - scheme: 'Signature', - params: { - keyId: 'KeyID', // dummy, not used for verify - algorithm: algorithm, - headers: ['(request-target)', 'date', 'host', 'digest'], // dummy, not used for verify - signature: signature, - }, - signingString: signingString, - algorithm: algorithm.toUpperCase(), - keyId: 'KeyID', // dummy, not used for verify - }; -}; - -describe('ap-request', () => { - test('createSignedPost with verify', async () => { - const keypair = await genRsaKeyPair(); - const key = { keyId: 'x', 'privateKeyPem': keypair.privateKey }; - const url = 'https://example.com/inbox'; - const activity = { a: 1 }; - const body = JSON.stringify(activity); - const headers = { - 'User-Agent': 'UA', - }; - - const req = ApRequestCreator.createSignedPost({ key, url, body, additionalHeaders: headers }); - - const parsed = buildParsedSignature(req.signingString, req.signature, 'rsa-sha256'); - - const result = httpSignature.verifySignature(parsed, keypair.publicKey); - assert.deepStrictEqual(result, true); - }); - - test('createSignedGet with verify', async () => { - const keypair = await genRsaKeyPair(); - const key = { keyId: 'x', 'privateKeyPem': keypair.privateKey }; - const url = 'https://example.com/outbox'; - const headers = { - 'User-Agent': 'UA', - }; - - const req = ApRequestCreator.createSignedGet({ key, url, additionalHeaders: headers }); - - const parsed = buildParsedSignature(req.signingString, req.signature, 'rsa-sha256'); - - const result = httpSignature.verifySignature(parsed, keypair.publicKey); - assert.deepStrictEqual(result, true); - }); -}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 26add9a11e24..922234f34d98 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -110,6 +110,9 @@ importers: '@fastify/view': specifier: 8.2.0 version: 8.2.0 + '@misskey-dev/node-http-message-signatures': + specifier: 0.0.0-alpha.7 + version: 0.0.0-alpha.7 '@misskey-dev/sharp-read-bmp': specifier: 1.2.0 version: 1.2.0 @@ -125,9 +128,6 @@ importers: '@nestjs/testing': specifier: 10.2.10 version: 10.2.10(@nestjs/common@10.2.10)(@nestjs/core@10.2.10)(@nestjs/platform-express@10.3.3) - '@peertube/http-signature': - specifier: 1.7.0 - version: 1.7.0 '@simplewebauthn/server': specifier: 9.0.3 version: 9.0.3 @@ -4735,6 +4735,10 @@ packages: eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.18.1)(eslint@8.56.0) dev: true + /@misskey-dev/node-http-message-signatures@0.0.0-alpha.7: + resolution: {integrity: sha512-iM1nZ3YT+G4AEhbUnsK7PqnMY9MjBP5JomQAgi2OyxDtZ/wBpgLP6MCVz3ElCqZ8NQS1f+c4E1m6/dSN8MtU9Q==} + dev: false + /@misskey-dev/sharp-read-bmp@1.2.0: resolution: {integrity: sha512-er4pRakXzHYfEgOFAFfQagqDouG+wLm+kwNq1I30oSdIHDa0wM3KjFpfIGQ25Fks4GcmOl1s7Zh6xoQu5dNjTw==} dependencies: @@ -5057,15 +5061,6 @@ packages: tslib: 2.6.2 dev: false - /@peertube/http-signature@1.7.0: - resolution: {integrity: sha512-aGQIwo6/sWtyyqhVK4e1MtxYz4N1X8CNt6SOtCc+Wnczs5S5ONaLHDDR8LYaGn0MgOwvGgXyuZ5sJIfd7iyoUw==} - engines: {node: '>=0.10'} - dependencies: - assert-plus: 1.0.0 - jsprim: 1.4.2 - sshpk: 1.17.0 - dev: false - /@pkgjs/parseargs@0.11.0: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} From fc20ef018198ac0b5781062940af3349d555a40f Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 29 Feb 2024 21:18:46 +0000 Subject: [PATCH 018/134] fix --- .../src/server/api/endpoints/admin/queue/inbox-delayed.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/server/api/endpoints/admin/queue/inbox-delayed.ts b/packages/backend/src/server/api/endpoints/admin/queue/inbox-delayed.ts index 305ae1af1da5..7839d8b6f5a1 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/inbox-delayed.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/inbox-delayed.ts @@ -56,7 +56,8 @@ export default class extends Endpoint { // eslint- const res = [] as [string, number][]; for (const job of jobs) { - const host = new URL(job.data.signature.keyId).host; + const signature = 'version' in job.data.signature ? job.data.signature.value : job.data.signature; + const host = new URL(signature.keyId).host; if (res.find(x => x[0] === host)) { res.find(x => x[0] === host)![1]++; } else { From 735714d61ccd943d07b0831de44384cdcfadb7d4 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 29 Feb 2024 22:20:48 +0000 Subject: [PATCH 019/134] signedPost, signedGet --- .../migration/1709242519122-HttpSignImplLv.js | 16 ++ .../src/core/FetchInstanceMetadataService.ts | 6 +- .../src/core/activitypub/ApRequestService.ts | 185 +++++++----------- .../src/core/activitypub/ApResolverService.ts | 8 +- packages/backend/src/models/Instance.ts | 5 + .../processors/DeliverProcessorService.ts | 29 +-- 6 files changed, 114 insertions(+), 135 deletions(-) create mode 100644 packages/backend/migration/1709242519122-HttpSignImplLv.js diff --git a/packages/backend/migration/1709242519122-HttpSignImplLv.js b/packages/backend/migration/1709242519122-HttpSignImplLv.js new file mode 100644 index 000000000000..7748bae00612 --- /dev/null +++ b/packages/backend/migration/1709242519122-HttpSignImplLv.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class HttpSignImplLv1709242519122 { + name = 'HttpSignImplLv1709242519122' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" ADD "httpMessageSignaturesImplementationLevel" character varying(16) NOT NULL DEFAULT '00'`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "httpMessageSignaturesImplementationLevel"`); + } +} diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index bc270bd28fd3..63168afd7d05 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -24,6 +24,7 @@ type NodeInfo = { version?: unknown; }; metadata?: { + httpMessageSignaturesImplementationLevel?: unknown, name?: unknown; nodeName?: unknown; nodeDescription?: unknown; @@ -70,7 +71,7 @@ export class FetchInstanceMetadataService { if (!force) { const _instance = await this.federatedInstanceService.fetch(host); const now = Date.now(); - if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) { + if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 3)) { // unlock at the finally caluse return; } @@ -104,6 +105,9 @@ export class FetchInstanceMetadataService { updates.openRegistrations = info.openRegistrations; updates.maintainerName = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name ?? null) : null : null; updates.maintainerEmail = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email ?? null) : null : null; + if (info.metadata && info.metadata.httpMessageSignaturesImplementationLevel) { + updates.httpMessageSignaturesImplementationLevel = info.metadata.httpMessageSignaturesImplementationLevel.toString() ?? '00'; + } } if (name) updates.name = name; diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 93ac8ce9a74b..717e47d4b41f 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -3,9 +3,9 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import * as crypto from 'node:crypto'; import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; +import { genRFC3230DigestHeader, RequestLike, signAsDraftToRequest } from '@misskey-dev/node-http-message-signatures'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import type { MiUser } from '@/models/User.js'; @@ -16,12 +16,6 @@ import { bindThis } from '@/decorators.js'; import type Logger from '@/logger.js'; import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js'; -type Request = { - url: string; - method: string; - headers: Record; -}; - type Signed = { request: Request; signingString: string; @@ -34,103 +28,51 @@ type PrivateKey = { keyId: string; }; -export class ApRequestCreator { - static createSignedPost(args: { key: PrivateKey, url: string, body: string, digest?: string, additionalHeaders: Record }): Signed { - const u = new URL(args.url); - const digestHeader = args.digest ?? this.createDigest(args.body); - - const request: Request = { - url: u.href, - method: 'POST', - headers: this.#objectAssignWithLcKey({ - 'Date': new Date().toUTCString(), - 'Host': u.host, - 'Content-Type': 'application/activity+json', - 'Digest': digestHeader, - }, args.additionalHeaders), - }; - - const result = this.#signToRequest(request, args.key, ['(request-target)', 'date', 'host', 'digest']); - - return { - request, - signingString: result.signingString, - signature: result.signature, - signatureHeader: result.signatureHeader, - }; - } - - static createDigest(body: string) { - return `SHA-256=${crypto.createHash('sha256').update(body).digest('base64')}`; - } - - static createSignedGet(args: { key: PrivateKey, url: string, additionalHeaders: Record }): Signed { - const u = new URL(args.url); - - const request: Request = { - url: u.href, - method: 'GET', - headers: this.#objectAssignWithLcKey({ - 'Accept': 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"', - 'Date': new Date().toUTCString(), - 'Host': new URL(args.url).host, - }, args.additionalHeaders), - }; - - const result = this.#signToRequest(request, args.key, ['(request-target)', 'date', 'host', 'accept']); - - return { - request, - signingString: result.signingString, - signature: result.signature, - signatureHeader: result.signatureHeader, - }; - } - - static #signToRequest(request: Request, key: PrivateKey, includeHeaders: string[]): Signed { - const signingString = this.#genSigningString(request, includeHeaders); - const signature = crypto.sign('sha256', Buffer.from(signingString), key.privateKeyPem).toString('base64'); - const signatureHeader = `keyId="${key.keyId}",algorithm="rsa-sha256",headers="${includeHeaders.join(' ')}",signature="${signature}"`; - - request.headers = this.#objectAssignWithLcKey(request.headers, { - Signature: signatureHeader, - }); - // node-fetch will generate this for us. if we keep 'Host', it won't change with redirects! - delete request.headers['host']; - - return { - request, - signingString, - signature, - signatureHeader, - }; - } - - static #genSigningString(request: Request, includeHeaders: string[]): string { - request.headers = this.#lcObjectKey(request.headers); - - const results: string[] = []; - - for (const key of includeHeaders.map(x => x.toLowerCase())) { - if (key === '(request-target)') { - results.push(`(request-target): ${request.method.toLowerCase()} ${new URL(request.url).pathname}`); - } else { - results.push(`${key}: ${request.headers[key]}`); - } - } - - return results.join('\n'); - } - - static #lcObjectKey(src: Record): Record { - const dst: Record = {}; - for (const key of Object.keys(src).filter(x => x !== '__proto__' && typeof src[x] === 'string')) dst[key.toLowerCase()] = src[key]; - return dst; - } +export function createSignedPost(args: { level: string; key: PrivateKey; url: string; body: string; digest?: string; additionalHeaders: Record }) { + const u = new URL(args.url); + const request: RequestLike = { + url: u.href, + method: 'POST', + headers: { + 'Date': new Date().toUTCString(), + 'Host': u.host, + 'Content-Type': 'application/activity+json', + ...args.additionalHeaders, + }, + }; + + // TODO: levelによって処理を分ける + const digestHeader = args.digest ?? genRFC3230DigestHeader(args.body); + request.headers['Digest'] = digestHeader; + + const result = signAsDraftToRequest(request, args.key, ['(request-target)', 'date', 'host', 'digest']); + + return { + request, + ...result, + }; +} - static #objectAssignWithLcKey(a: Record, b: Record): Record { - return Object.assign(this.#lcObjectKey(a), this.#lcObjectKey(b)); - } +export function createSignedGet(args: { level: string; key: PrivateKey; url: string; additionalHeaders: Record }) { + const u = new URL(args.url); + const request: RequestLike = { + url: u.href, + method: 'GET', + headers: { + 'Accept': 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"', + 'Date': new Date().toUTCString(), + 'Host': new URL(args.url).host, + ...args.additionalHeaders, + }, + }; + + // TODO: levelによって処理を分ける + const result = signAsDraftToRequest(request, args.key, ['(request-target)', 'date', 'host', 'accept']); + + return { + request, + ...result, + }; } @Injectable() @@ -150,16 +92,25 @@ export class ApRequestService { } @bindThis - public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, digest?: string): Promise { - const body = typeof object === 'string' ? object : JSON.stringify(object); + private async getPrivateKey(userId: MiUser['id'], level: string): Promise { + const keypair = await this.userKeypairService.getUserKeypair(userId); + + return (level !== '00' && keypair.ed25519PrivateKey) ? { + privateKeyPem: keypair.ed25519PrivateKey, + keyId: `${this.config.url}/users/${userId}#ed25519-key`, + } : { + privateKeyPem: keypair.privateKey, + keyId: `${this.config.url}/users/${userId}#main-key`, + }; + } - const keypair = await this.userKeypairService.getUserKeypair(user.id); + @bindThis + public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, level: string, digest?: string): Promise { + const body = typeof object === 'string' ? object : JSON.stringify(object); - const req = ApRequestCreator.createSignedPost({ - key: { - privateKeyPem: keypair.privateKey, - keyId: `${this.config.url}/users/${user.id}#main-key`, - }, + const req = createSignedPost({ + level, + key: await this.getPrivateKey(user.id, level), url, body, digest, @@ -180,14 +131,10 @@ export class ApRequestService { * @param url URL to fetch */ @bindThis - public async signedGet(url: string, user: { id: MiUser['id'] }): Promise { - const keypair = await this.userKeypairService.getUserKeypair(user.id); - - const req = ApRequestCreator.createSignedGet({ - key: { - privateKeyPem: keypair.privateKey, - keyId: `${this.config.url}/users/${user.id}#main-key`, - }, + public async signedGet(url: string, user: { id: MiUser['id'] }, level: string): Promise { + const req = createSignedGet({ + level, + key: await this.getPrivateKey(user.id, level), url, additionalHeaders: { }, diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index bb3c40f0939b..727ff6f95626 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -16,6 +16,7 @@ import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { LoggerService } from '@/core/LoggerService.js'; import type Logger from '@/logger.js'; +import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { isCollectionOrOrderedCollection } from './type.js'; import { ApDbResolverService } from './ApDbResolverService.js'; import { ApRendererService } from './ApRendererService.js'; @@ -41,6 +42,7 @@ export class Resolver { private httpRequestService: HttpRequestService, private apRendererService: ApRendererService, private apDbResolverService: ApDbResolverService, + private federatedInstanceService: FederatedInstanceService, private loggerService: LoggerService, private recursionLimit = 100, ) { @@ -103,8 +105,10 @@ export class Resolver { this.user = await this.instanceActorService.getInstanceActor(); } + const server = await this.federatedInstanceService.fetch(host); + const object = (this.user - ? await this.apRequestService.signedGet(value, this.user) as IObject + ? await this.apRequestService.signedGet(value, this.user, server.httpMessageSignaturesImplementationLevel) as IObject : await this.httpRequestService.getActivityJson(value)) as IObject; if ( @@ -200,6 +204,7 @@ export class ApResolverService { private httpRequestService: HttpRequestService, private apRendererService: ApRendererService, private apDbResolverService: ApDbResolverService, + private federatedInstanceService: FederatedInstanceService, private loggerService: LoggerService, ) { } @@ -220,6 +225,7 @@ export class ApResolverService { this.httpRequestService, this.apRendererService, this.apDbResolverService, + this.federatedInstanceService, this.loggerService, ); } diff --git a/packages/backend/src/models/Instance.ts b/packages/backend/src/models/Instance.ts index 9863c9d75da7..0c94f9b27751 100644 --- a/packages/backend/src/models/Instance.ts +++ b/packages/backend/src/models/Instance.ts @@ -149,4 +149,9 @@ export class MiInstance { length: 16384, default: '', }) public moderationNote: string; + + @Column('varchar', { + length: 16, default: '00', nullable: false, + }) + public httpMessageSignaturesImplementationLevel: string; } diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 5fed070929e8..8f3782a04842 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -72,24 +72,25 @@ export class DeliverProcessorService { } try { - await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content, job.data.digest); + const _server = await this.federatedInstanceService.fetch(host); + await this.fetchInstanceMetadataService.fetchInstanceMetadata(_server).then(() => {}); + const server = await this.federatedInstanceService.fetch(host); + + await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content, server.httpMessageSignaturesImplementationLevel, job.data.digest); // Update stats - this.federatedInstanceService.fetch(host).then(i => { - if (i.isNotResponding) { - this.federatedInstanceService.update(i.id, { - isNotResponding: false, - }); - } + if (server.isNotResponding) { + this.federatedInstanceService.update(server.id, { + isNotResponding: false, + }); + } - this.fetchInstanceMetadataService.fetchInstanceMetadata(i); - this.apRequestChart.deliverSucc(); - this.federationChart.deliverd(i.host, true); + this.apRequestChart.deliverSucc(); + this.federationChart.deliverd(server.host, true); - if (meta.enableChartsForFederatedInstances) { - this.instanceChart.requestSent(i.host, true); - } - }); + if (meta.enableChartsForFederatedInstances) { + this.instanceChart.requestSent(server.host, true); + } return 'Success'; } catch (res) { From 434520a14ec1b85c3e05dc38a912667e34b6b392 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 29 Feb 2024 22:36:19 +0000 Subject: [PATCH 020/134] =?UTF-8?q?ap-request.ts=E3=82=92=E5=BE=A9?= =?UTF-8?q?=E6=B4=BB=E3=81=95=E3=81=9B=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/test/unit/ap-request.ts | 71 ++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 packages/backend/test/unit/ap-request.ts diff --git a/packages/backend/test/unit/ap-request.ts b/packages/backend/test/unit/ap-request.ts new file mode 100644 index 000000000000..2aeeed1ebc4e --- /dev/null +++ b/packages/backend/test/unit/ap-request.ts @@ -0,0 +1,71 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import * as assert from 'assert'; +import { verifyDraftSignature, parseRequestSignature } from '@misskey-dev/node-http-message-signatures'; +import { genEd25519KeyPair, genRsaKeyPair } from '@/misc/gen-key-pair.js'; +import { createSignedGet, createSignedPost } from '@/core/activitypub/ApRequestService.js'; + +export const buildParsedSignature = (signingString: string, signature: string, algorithm: string) => { + return { + scheme: 'Signature', + params: { + keyId: 'KeyID', // dummy, not used for verify + algorithm: algorithm, + headers: ['(request-target)', 'date', 'host', 'digest'], // dummy, not used for verify + signature: signature, + }, + signingString: signingString, + algorithm: algorithm.toUpperCase(), + keyId: 'KeyID', // dummy, not used for verify + }; +}; + +async function getKeyPair(level: string) { + if (level === '00') { + return await genRsaKeyPair(); + } else if (level === '01') { + return await genEd25519KeyPair(); + } + throw new Error('Invalid level'); +} + +describe('ap-request', () => { + describe.each(['00', '01'])('createSignedPost with verify', async (level) => { + const keypair = await getKeyPair(level); + const key = { keyId: 'x', 'privateKeyPem': keypair.privateKey }; + const url = 'https://example.com/inbox'; + const activity = { a: 1 }; + const body = JSON.stringify(activity); + const headers = { + 'User-Agent': 'UA', + }; + + const req = createSignedPost({ level, key, url, body, additionalHeaders: headers }); + + const parsed = parseRequestSignature(req.request); + expect(parsed?.version).toBe('draft'); + if (!parsed) return; + const verify = verifyDraftSignature(parsed?.value, keypair.publicKey); + assert.deepStrictEqual(verify, true); + }); + + describe.each(['00', '01'])('createSignedGet with verify', async (level) => { + const keypair = await getKeyPair(level); + const key = { keyId: 'x', 'privateKeyPem': keypair.privateKey }; + const url = 'https://example.com/outbox'; + const headers = { + 'User-Agent': 'UA', + }; + + const req = createSignedGet({ level, key, url, additionalHeaders: headers }); + + const parsed = parseRequestSignature(req.request); + expect(parsed?.version).toBe('draft'); + if (!parsed) return; + const verify = verifyDraftSignature(parsed?.value, keypair.publicKey); + assert.deepStrictEqual(verify, true); + }); +}); From 5f89b0a2a38f96629cd2e06659a1128db59c9734 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 29 Feb 2024 22:41:45 +0000 Subject: [PATCH 021/134] remove digest prerender --- packages/backend/src/core/QueueService.ts | 5 ----- packages/backend/src/core/activitypub/ApRequestService.ts | 7 +++---- .../src/queue/processors/DeliverProcessorService.ts | 2 +- packages/backend/src/queue/types.ts | 2 -- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index ef2ec0d2ffee..7ba1853f84a9 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -12,7 +12,6 @@ import type { Config } from '@/config.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import type { Antenna } from '@/server/api/endpoints/i/import-antennas.js'; -import { ApRequestCreator } from '@/core/activitypub/ApRequestService.js'; import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, RelationshipQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js'; import type { DbJobData, DeliverJobData, RelationshipJobData, ThinUser } from '../queue/types.js'; import type * as Bull from 'bullmq'; @@ -76,14 +75,12 @@ export class QueueService { if (to == null) return null; const contentBody = JSON.stringify(content); - const digest = ApRequestCreator.createDigest(contentBody); const data: DeliverJobData = { user: { id: user.id, }, content: contentBody, - digest, to, isSharedInbox, }; @@ -109,7 +106,6 @@ export class QueueService { public async deliverMany(user: ThinUser, content: IActivity | null, inboxes: Map) { if (content == null) return null; const contentBody = JSON.stringify(content); - const digest = ApRequestCreator.createDigest(contentBody); const opts = { attempts: this.config.deliverJobMaxAttempts ?? 12, @@ -125,7 +121,6 @@ export class QueueService { data: { user, content: contentBody, - digest, to: d[0], isSharedInbox: d[1], } as DeliverJobData, diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 717e47d4b41f..cc2f05aba2b7 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -28,7 +28,7 @@ type PrivateKey = { keyId: string; }; -export function createSignedPost(args: { level: string; key: PrivateKey; url: string; body: string; digest?: string; additionalHeaders: Record }) { +export function createSignedPost(args: { level: string; key: PrivateKey; url: string; body: string; additionalHeaders: Record }) { const u = new URL(args.url); const request: RequestLike = { url: u.href, @@ -42,7 +42,7 @@ export function createSignedPost(args: { level: string; key: PrivateKey; url: st }; // TODO: levelによって処理を分ける - const digestHeader = args.digest ?? genRFC3230DigestHeader(args.body); + const digestHeader = genRFC3230DigestHeader(args.body); request.headers['Digest'] = digestHeader; const result = signAsDraftToRequest(request, args.key, ['(request-target)', 'date', 'host', 'digest']); @@ -105,7 +105,7 @@ export class ApRequestService { } @bindThis - public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, level: string, digest?: string): Promise { + public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, level: string): Promise { const body = typeof object === 'string' ? object : JSON.stringify(object); const req = createSignedPost({ @@ -113,7 +113,6 @@ export class ApRequestService { key: await this.getPrivateKey(user.id, level), url, body, - digest, additionalHeaders: { }, }); diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 8f3782a04842..6bbe7a4973aa 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -76,7 +76,7 @@ export class DeliverProcessorService { await this.fetchInstanceMetadataService.fetchInstanceMetadata(_server).then(() => {}); const server = await this.federatedInstanceService.fetch(host); - await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content, server.httpMessageSignaturesImplementationLevel, job.data.digest); + await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content, server.httpMessageSignaturesImplementationLevel); // Update stats if (server.isNotResponding) { diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 135bccb60c0f..fcae4c259680 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -32,8 +32,6 @@ export type DeliverJobData = { user: ThinUser; /** Activity */ content: string; - /** Digest header */ - digest: string; /** inbox URL to deliver */ to: string; /** whether it is sharedInbox */ From 66c0942d7e67bea66988f2b437214f1ed31e10d3 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 29 Feb 2024 22:50:46 +0000 Subject: [PATCH 022/134] fix test? --- packages/backend/test/misc/mock-resolver.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend/test/misc/mock-resolver.ts b/packages/backend/test/misc/mock-resolver.ts index 3c7e796700da..485506ee64dd 100644 --- a/packages/backend/test/misc/mock-resolver.ts +++ b/packages/backend/test/misc/mock-resolver.ts @@ -14,6 +14,7 @@ import type { InstanceActorService } from '@/core/InstanceActorService.js'; import type { LoggerService } from '@/core/LoggerService.js'; import type { MetaService } from '@/core/MetaService.js'; import type { UtilityService } from '@/core/UtilityService.js'; +import type { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { bindThis } from '@/decorators.js'; import type { FollowRequestsRepository, @@ -47,6 +48,7 @@ export class MockResolver extends Resolver { {} as HttpRequestService, {} as ApRendererService, {} as ApDbResolverService, + {} as FederatedInstanceService, loggerService, ); } From 7751d8005649993a80abe7d9df96488921ec1d44 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 29 Feb 2024 22:57:10 +0000 Subject: [PATCH 023/134] fix test --- packages/backend/test/unit/ap-request.ts | 60 +++++++++++++----------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/packages/backend/test/unit/ap-request.ts b/packages/backend/test/unit/ap-request.ts index 2aeeed1ebc4e..d04de6800197 100644 --- a/packages/backend/test/unit/ap-request.ts +++ b/packages/backend/test/unit/ap-request.ts @@ -33,39 +33,43 @@ async function getKeyPair(level: string) { } describe('ap-request', () => { - describe.each(['00', '01'])('createSignedPost with verify', async (level) => { - const keypair = await getKeyPair(level); - const key = { keyId: 'x', 'privateKeyPem': keypair.privateKey }; - const url = 'https://example.com/inbox'; - const activity = { a: 1 }; - const body = JSON.stringify(activity); - const headers = { - 'User-Agent': 'UA', - }; + describe.each(['00', '01'])('createSignedPost with verify', (level) => { + test('pass', async () => { + const keypair = await getKeyPair(level); + const key = { keyId: 'x', 'privateKeyPem': keypair.privateKey }; + const url = 'https://example.com/inbox'; + const activity = { a: 1 }; + const body = JSON.stringify(activity); + const headers = { + 'User-Agent': 'UA', + }; - const req = createSignedPost({ level, key, url, body, additionalHeaders: headers }); + const req = createSignedPost({ level, key, url, body, additionalHeaders: headers }); - const parsed = parseRequestSignature(req.request); - expect(parsed?.version).toBe('draft'); - if (!parsed) return; - const verify = verifyDraftSignature(parsed?.value, keypair.publicKey); - assert.deepStrictEqual(verify, true); + const parsed = parseRequestSignature(req.request); + expect(parsed?.version).toBe('draft'); + if (!parsed) return; + const verify = verifyDraftSignature(parsed?.value, keypair.publicKey); + assert.deepStrictEqual(verify, true); + }); }); - describe.each(['00', '01'])('createSignedGet with verify', async (level) => { - const keypair = await getKeyPair(level); - const key = { keyId: 'x', 'privateKeyPem': keypair.privateKey }; - const url = 'https://example.com/outbox'; - const headers = { - 'User-Agent': 'UA', - }; + describe.each(['00', '01'])('createSignedGet with verify', (level) => { + test('pass', async () => { + const keypair = await getKeyPair(level); + const key = { keyId: 'x', 'privateKeyPem': keypair.privateKey }; + const url = 'https://example.com/outbox'; + const headers = { + 'User-Agent': 'UA', + }; - const req = createSignedGet({ level, key, url, additionalHeaders: headers }); + const req = createSignedGet({ level, key, url, additionalHeaders: headers }); - const parsed = parseRequestSignature(req.request); - expect(parsed?.version).toBe('draft'); - if (!parsed) return; - const verify = verifyDraftSignature(parsed?.value, keypair.publicKey); - assert.deepStrictEqual(verify, true); + const parsed = parseRequestSignature(req.request); + expect(parsed?.version).toBe('draft'); + if (!parsed) return; + const verify = verifyDraftSignature(parsed?.value, keypair.publicKey); + assert.deepStrictEqual(verify, true); + }); }); }); From 16cea7d3b66a8cf3b2e1827fae6f64483b1df862 Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 1 Mar 2024 04:38:42 +0000 Subject: [PATCH 024/134] add httpMessageSignaturesImplementationLevel to FederationInstance --- packages/backend/src/core/entities/InstanceEntityService.ts | 1 + .../backend/src/models/json-schema/federation-instance.ts | 4 ++++ packages/misskey-js/src/autogen/types.ts | 1 + 3 files changed, 6 insertions(+) diff --git a/packages/backend/src/core/entities/InstanceEntityService.ts b/packages/backend/src/core/entities/InstanceEntityService.ts index e46bd8b9637e..1482a22a6c2f 100644 --- a/packages/backend/src/core/entities/InstanceEntityService.ts +++ b/packages/backend/src/core/entities/InstanceEntityService.ts @@ -55,6 +55,7 @@ export class InstanceEntityService { infoUpdatedAt: instance.infoUpdatedAt ? instance.infoUpdatedAt.toISOString() : null, latestRequestReceivedAt: instance.latestRequestReceivedAt ? instance.latestRequestReceivedAt.toISOString() : null, moderationNote: iAmModerator ? instance.moderationNote : null, + httpMessageSignaturesImplementationLevel: instance.httpMessageSignaturesImplementationLevel, }; } diff --git a/packages/backend/src/models/json-schema/federation-instance.ts b/packages/backend/src/models/json-schema/federation-instance.ts index 42d98fe5238e..5da3f87d429e 100644 --- a/packages/backend/src/models/json-schema/federation-instance.ts +++ b/packages/backend/src/models/json-schema/federation-instance.ts @@ -111,5 +111,9 @@ export const packedFederationInstanceSchema = { type: 'string', optional: true, nullable: true, }, + httpMessageSignaturesImplementationLevel: { + type: 'string', + optional: false, nullable: false, + }, }, } as const; diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 227295fbb80d..0e32c91484a5 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -4492,6 +4492,7 @@ export type components = { /** Format: date-time */ latestRequestReceivedAt: string | null; moderationNote?: string | null; + httpMessageSignaturesImplementationLevel: string; }; GalleryPost: { /** From 87ded2bd1c2c96bcd5719f244998100a65f65feb Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 1 Mar 2024 05:04:02 +0000 Subject: [PATCH 025/134] ManyToOne --- .../1709269211718-APMultipleKeysFix1.js | 16 ++++++++++++++++ packages/backend/src/models/UserKeypair.ts | 4 ++-- 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 packages/backend/migration/1709269211718-APMultipleKeysFix1.js diff --git a/packages/backend/migration/1709269211718-APMultipleKeysFix1.js b/packages/backend/migration/1709269211718-APMultipleKeysFix1.js new file mode 100644 index 000000000000..d2011802f264 --- /dev/null +++ b/packages/backend/migration/1709269211718-APMultipleKeysFix1.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class APMultipleKeys1709269211718 { + name = 'APMultipleKeys1709269211718' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "user_publickey" DROP CONSTRAINT "UQ_10c146e4b39b443ede016f6736d"`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "user_publickey" ADD CONSTRAINT "UQ_10c146e4b39b443ede016f6736d" UNIQUE ("userId")`); + } +} diff --git a/packages/backend/src/models/UserKeypair.ts b/packages/backend/src/models/UserKeypair.ts index 59b37bb9e9e4..afa74ef11a0c 100644 --- a/packages/backend/src/models/UserKeypair.ts +++ b/packages/backend/src/models/UserKeypair.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { PrimaryColumn, Entity, JoinColumn, Column, OneToOne } from 'typeorm'; +import { PrimaryColumn, Entity, JoinColumn, Column, ManyToOne } from 'typeorm'; import { id } from './util/id.js'; import { MiUser } from './User.js'; @@ -12,7 +12,7 @@ export class MiUserKeypair { @PrimaryColumn(id()) public userId: MiUser['id']; - @OneToOne(type => MiUser, { + @ManyToOne(type => MiUser, { onDelete: 'CASCADE', }) @JoinColumn() From 54fe8ca600bbecb15236a5ccddef5f5f7a9e2fb8 Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 1 Mar 2024 06:49:38 +0000 Subject: [PATCH 026/134] fetchPersonWithRenewal --- .../src/core/FetchInstanceMetadataService.ts | 23 ++++++++++++------- .../core/activitypub/ApDbResolverService.ts | 2 +- .../src/core/activitypub/ApInboxService.ts | 10 -------- .../src/core/activitypub/ApRequestService.ts | 23 ++++++++++++++++--- .../activitypub/models/ApPersonService.ts | 20 ++++++++++++++-- .../queue/processors/InboxProcessorService.ts | 12 ++++++++-- 6 files changed, 64 insertions(+), 26 deletions(-) diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 63168afd7d05..21e84a7454f3 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -40,6 +40,7 @@ type NodeInfo = { @Injectable() export class FetchInstanceMetadataService { private logger: Logger; + private httpColon = 'https://'; constructor( private httpRequestService: HttpRequestService, @@ -49,6 +50,7 @@ export class FetchInstanceMetadataService { private redisClient: Redis.Redis, ) { this.logger = this.loggerService.getLogger('metadata', 'cyan'); + this.httpColon = 'http://'; } @bindThis @@ -72,8 +74,7 @@ export class FetchInstanceMetadataService { const _instance = await this.federatedInstanceService.fetch(host); const now = Date.now(); if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 3)) { - // unlock at the finally caluse - return; + throw new Error('Skip because updated recently'); } } @@ -119,6 +120,12 @@ export class FetchInstanceMetadataService { await this.federatedInstanceService.update(instance.id, updates); this.logger.succ(`Successfuly updated metadata of ${instance.host}`); + this.logger.debug('Updated metadata:', { + info: !!info, + dom: !!dom, + manifest: !!manifest, + updates, + }); } catch (e) { this.logger.error(`Failed to update metadata of ${instance.host}: ${e}`); } finally { @@ -131,7 +138,7 @@ export class FetchInstanceMetadataService { this.logger.info(`Fetching nodeinfo of ${instance.host} ...`); try { - const wellknown = await this.httpRequestService.getJson('https://' + instance.host + '/.well-known/nodeinfo') + const wellknown = await this.httpRequestService.getJson(this.httpColon + instance.host + '/.well-known/nodeinfo') .catch(err => { if (err.statusCode === 404) { throw new Error('No nodeinfo provided'); @@ -174,7 +181,7 @@ export class FetchInstanceMetadataService { private async fetchDom(instance: MiInstance): Promise { this.logger.info(`Fetching HTML of ${instance.host} ...`); - const url = 'https://' + instance.host; + const url = this.httpColon + instance.host; const html = await this.httpRequestService.getHtml(url); @@ -186,7 +193,7 @@ export class FetchInstanceMetadataService { @bindThis private async fetchManifest(instance: MiInstance): Promise | null> { - const url = 'https://' + instance.host; + const url = this.httpColon + instance.host; const manifestUrl = url + '/manifest.json'; @@ -197,7 +204,7 @@ export class FetchInstanceMetadataService { @bindThis private async fetchFaviconUrl(instance: MiInstance, doc: DOMWindow['document'] | null): Promise { - const url = 'https://' + instance.host; + const url = this.httpColon + instance.host; if (doc) { // https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043 @@ -224,12 +231,12 @@ export class FetchInstanceMetadataService { @bindThis private async fetchIconUrl(instance: MiInstance, doc: DOMWindow['document'] | null, manifest: Record | null): Promise { if (manifest && manifest.icons && manifest.icons.length > 0 && manifest.icons[0].src) { - const url = 'https://' + instance.host; + const url = this.httpColon + instance.host; return (new URL(manifest.icons[0].src, url)).href; } if (doc) { - const url = 'https://' + instance.host; + const url = this.httpColon + instance.host; // https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043 const links = Array.from(doc.getElementsByTagName('link')).reverse(); diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 4013eaddc0fd..d95bd3bbda01 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -122,7 +122,7 @@ export class ApDbResolverService implements OnApplicationShutdown { user: MiRemoteUser; key: MiUserPublickey | null; } | null> { - const user = await this.apPersonService.resolvePerson(uri) as MiRemoteUser; + const user = await this.apPersonService.resolvePerson(uri, undefined, true) as MiRemoteUser; if (user.isDeleted) return null; const keys = await this.publicKeyByUserIdCache.fetch( diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index b0f56a5d8284..13b86c292010 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -36,7 +36,6 @@ import { ApResolverService } from './ApResolverService.js'; import { ApAudienceService } from './ApAudienceService.js'; import { ApPersonService } from './models/ApPersonService.js'; import { ApQuestionService } from './models/ApQuestionService.js'; -import { CacheService } from '@/core/CacheService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import type { Resolver } from './ApResolverService.js'; import type { IAccept, IAdd, IAnnounce, IBlock, ICreate, IDelete, IFlag, IFollow, ILike, IObject, IReject, IRemove, IUndo, IUpdate, IMove } from './type.js'; @@ -109,15 +108,6 @@ export class ApInboxService { } else { await this.performOneActivity(actor, activity); } - - // ついでにリモートユーザーの情報が古かったら更新しておく - if (actor.uri) { - if (actor.lastFetchedAt == null || Date.now() - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) { - setImmediate(() => { - this.apPersonService.updatePerson(actor.uri); - }); - } - } } @bindThis diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index cc2f05aba2b7..36a4b34de1d7 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -107,16 +107,24 @@ export class ApRequestService { @bindThis public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, level: string): Promise { const body = typeof object === 'string' ? object : JSON.stringify(object); - + const key = await this.getPrivateKey(user.id, level); const req = createSignedPost({ level, - key: await this.getPrivateKey(user.id, level), + key, url, body, additionalHeaders: { + 'User-Agent': this.config.userAgent, }, }); + this.logger.debug('create signed post', { + version: 'draft', + level, + url, + keyId: key.keyId, + }); + await this.httpRequestService.send(url, { method: req.request.method, headers: req.request.headers, @@ -131,14 +139,23 @@ export class ApRequestService { */ @bindThis public async signedGet(url: string, user: { id: MiUser['id'] }, level: string): Promise { + const key = await this.getPrivateKey(user.id, level); const req = createSignedGet({ level, - key: await this.getPrivateKey(user.id, level), + key, url, additionalHeaders: { + 'User-Agent': this.config.userAgent, }, }); + this.logger.debug('create signed get', { + version: 'draft', + level, + url, + keyId: key.keyId, + }); + const res = await this.httpRequestService.send(url, { method: req.request.method, headers: req.request.headers, diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 5100a2460095..da4a42f73e3d 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -248,6 +248,22 @@ export class ApPersonService implements OnModuleInit { return null; } + @bindThis + async fetchPersonWithRenewal(uri: string): Promise { + const exist = await this.fetchPerson(uri); + if (exist == null) return null; + + // ついでにリモートユーザーの情報が古かったら更新しておく + if (this.userEntityService.isRemoteUser(exist)) { + if (exist.lastFetchedAt == null || Date.now() - exist.lastFetchedAt.getTime() > 1000 * 60 * 60 * 3) { + await this.updatePerson(exist.uri); + return await this.fetchPerson(uri); + } + } + + return exist; + } + private async resolveAvatarAndBanner(user: MiRemoteUser, icon: any, image: any): Promise>> { if (user == null) throw new Error('failed to create user: user is null'); @@ -624,9 +640,9 @@ export class ApPersonService implements OnModuleInit { * リモートサーバーからフェッチしてMisskeyに登録しそれを返します。 */ @bindThis - public async resolvePerson(uri: string, resolver?: Resolver): Promise { + public async resolvePerson(uri: string, resolver?: Resolver, withRenewal = false): Promise { //#region このサーバーに既に登録されていたらそれを返す - const exist = await this.fetchPerson(uri); + const exist = withRenewal ? await this.fetchPersonWithRenewal(uri) : await this.fetchPerson(uri); if (exist) return exist; //#endregion diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 85ce6417b822..4d7f76c0d960 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -19,6 +19,7 @@ import type { MiRemoteUser } from '@/models/User.js'; import type { MiUserPublickey } from '@/models/UserPublickey.js'; import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; import { StatusError } from '@/misc/status-error.js'; +import * as Acct from '@/misc/acct.js'; import { UtilityService } from '@/core/UtilityService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { LdSignatureService } from '@/core/activitypub/LdSignatureService.js'; @@ -79,7 +80,6 @@ export class InboxProcessorService { key: MiUserPublickey | null; } | null = null; - // keyIdでわからなければ、activity.actorを元にDBから取得 || activity.actorを元にリモートから取得 try { authUser = await this.apDbResolverService.getAuthUserFromApId(getApId(activity.actor), signature.keyId); } catch (err) { @@ -103,7 +103,15 @@ export class InboxProcessorService { } // HTTP-Signatureの検証 - const httpSignatureValidated = verifyDraftSignature(signature, authUser.key.keyPem); + const errorLogger = (ms: any) => this.logger.error(ms); + const httpSignatureValidated = verifyDraftSignature(signature, authUser.key.keyPem, errorLogger); + this.logger.debug('Inbox message validation: ', { + userId: authUser.user.id, + userAcct: Acct.toString(authUser.user), + parsedKeyId: signature.keyId, + foundKeyId: authUser.key.keyId, + httpSignatureValidated, + }); // また、signatureのsignerは、activity.actorと一致する必要がある if (!httpSignatureValidated || authUser.user.uri !== activity.actor) { From bec6159b4a19905efa79acce105e63a80f0d1100 Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 1 Mar 2024 06:58:43 +0000 Subject: [PATCH 027/134] exactKey --- .../core/activitypub/ApDbResolverService.ts | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index d95bd3bbda01..25a37ca09611 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -131,31 +131,43 @@ export class ApDbResolverService implements OnApplicationShutdown { v => v != null, ); - if (keys == null || keys.length === 8) return null; + if (keys == null || !Array.isArray(keys)) return null; + + if (keys.length === 0) { + return { + user, + key: keys[0], + }; + } + + const exactKey = keys.find(x => x.keyId === keyId); + if (exactKey) { + return { + user, + key: exactKey, + }; + } // 公開鍵は複数あるが、mainっぽいのを選ぶ - const key = keys.length === 1 ? - keys[0] : - keys.find(x => { - try { - if (x.keyId === keyId) return true; - const url = new URL(x.keyId); - const path = url.pathname.split('/').pop()?.toLowerCase(); - if (url.hash) { - if (url.hash.toLowerCase().includes('main')) { - return true; - } - } else if (path?.includes('main') || path === 'publickey') { + const mainKey = keys.find(x => { + try { + if (x.keyId === keyId) return true; + const url = new URL(x.keyId); + const path = url.pathname.split('/').pop()?.toLowerCase(); + if (url.hash) { + if (url.hash.toLowerCase().includes('main')) { return true; } - } catch { /* noop */ } - - return false; - }) ?? keys[0]; + } else if (path?.includes('main') || path === 'publickey') { + return true; + } + } catch { /* noop */ } + return false; + }); return { user, - key, + key: mainKey ?? keys[0], }; } From 743b74077583b7a14881618de567563fb7992c60 Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 1 Mar 2024 07:18:16 +0000 Subject: [PATCH 028/134] :v: --- CONTRIBUTING.md | 2 +- packages/backend/src/core/FetchInstanceMetadataService.ts | 2 +- packages/backend/src/core/WebfingerService.ts | 2 +- packages/backend/src/core/activitypub/models/ApPersonService.ts | 2 ++ 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a3263bf6aa13..b155e4403b58 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -185,7 +185,7 @@ TODO ## Environment Variable - `MISSKEY_CONFIG_YML`: Specify the file path of config.yml instead of default.yml (e.g. `2nd.yml`). -- `MISSKEY_WEBFINGER_USE_HTTP`: If it's set true, WebFinger requests will be http instead of https, useful for testing federation between servers in localhost. NEVER USE IN PRODUCTION. +- `MISSKEY_USE_HTTP`: If it's set true, federation requests (like nodeinfo and webfinger) will be http instead of https, useful for testing federation between servers in localhost. NEVER USE IN PRODUCTION. (was `MISSKEY_WEBFINGER_USE_HTTP`) ## Continuous integration Misskey uses GitHub Actions for executing automated tests. diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 21e84a7454f3..fc4f335d54df 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -50,7 +50,7 @@ export class FetchInstanceMetadataService { private redisClient: Redis.Redis, ) { this.logger = this.loggerService.getLogger('metadata', 'cyan'); - this.httpColon = 'http://'; + this.httpColon = process.env.MISSKEY_USE_HTTP?.toLowerCase() === 'true' ? 'http://' : 'https://'; } @bindThis diff --git a/packages/backend/src/core/WebfingerService.ts b/packages/backend/src/core/WebfingerService.ts index 374536a74101..aa1144778cba 100644 --- a/packages/backend/src/core/WebfingerService.ts +++ b/packages/backend/src/core/WebfingerService.ts @@ -46,7 +46,7 @@ export class WebfingerService { const m = query.match(mRegex); if (m) { const hostname = m[2]; - const useHttp = process.env.MISSKEY_WEBFINGER_USE_HTTP && process.env.MISSKEY_WEBFINGER_USE_HTTP.toLowerCase() === 'true'; + const useHttp = process.env.MISSKEY_USE_HTTP && process.env.MISSKEY_USE_HTTP.toLowerCase() === 'true'; return `http${useHttp ? '' : 's'}://${hostname}/.well-known/webfinger?${urlQuery({ resource: `acct:${query}` })}`; } diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index da4a42f73e3d..000d1d414479 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -256,9 +256,11 @@ export class ApPersonService implements OnModuleInit { // ついでにリモートユーザーの情報が古かったら更新しておく if (this.userEntityService.isRemoteUser(exist)) { if (exist.lastFetchedAt == null || Date.now() - exist.lastFetchedAt.getTime() > 1000 * 60 * 60 * 3) { + this.logger.debug('fetchPersonWithRenewal: renew', { uri, lastFetchedAt: exist.lastFetchedAt }); await this.updatePerson(exist.uri); return await this.fetchPerson(uri); } + this.logger.debug('fetchPersonWithRenewal: use cache', { uri, lastFetchedAt: exist.lastFetchedAt }); } return exist; From fd71ad7a5f641abfb7316965e7ecedce01264a19 Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 1 Mar 2024 07:27:43 +0000 Subject: [PATCH 029/134] use const --- packages/backend/src/const.ts | 5 +++++ packages/backend/src/core/FetchInstanceMetadataService.ts | 3 ++- .../backend/src/core/activitypub/models/ApPersonService.ts | 5 +++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/const.ts b/packages/backend/src/const.ts index a238f4973a95..8579f5d1875b 100644 --- a/packages/backend/src/const.ts +++ b/packages/backend/src/const.ts @@ -8,6 +8,11 @@ export const MAX_NOTE_TEXT_LENGTH = 3000; export const USER_ONLINE_THRESHOLD = 1000 * 60 * 10; // 10min export const USER_ACTIVE_THRESHOLD = 1000 * 60 * 60 * 24 * 3; // 3days +export const REMOTE_USER_CACHE_TTL = 1000 * 60 * 60 * 3; // 3hours +export const REMOTE_USER_MOVE_COOLDOWN = 1000 * 60 * 60 * 24 * 14; // 14days + +export const REMOTE_SERVER_CACHE_TTL = 1000 * 60 * 60 * 3; // 3hours + //#region hard limits // If you change DB_* values, you must also change the DB schema. diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index fc4f335d54df..d038646d0d3f 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -15,6 +15,7 @@ import { LoggerService } from '@/core/LoggerService.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; +import { REMOTE_SERVER_CACHE_TTL } from '@/const'; import type { DOMWindow } from 'jsdom'; type NodeInfo = { @@ -73,7 +74,7 @@ export class FetchInstanceMetadataService { if (!force) { const _instance = await this.federatedInstanceService.fetch(host); const now = Date.now(); - if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 3)) { + if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < REMOTE_SERVER_CACHE_TTL)) { throw new Error('Skip because updated recently'); } } diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 000d1d414479..97abfcbde8a6 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -40,6 +40,7 @@ import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.j import type { AccountMoveService } from '@/core/AccountMoveService.js'; import { checkHttps } from '@/misc/check-https.js'; import { isNotNull } from '@/misc/is-not-null.js'; +import { REMOTE_USER_CACHE_TTL, REMOTE_USER_MOVE_COOLDOWN } from '@/const.js'; import { getApId, getApType, getOneApHrefNullable, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js'; import { extractApHashtags } from './tag.js'; import type { OnModuleInit } from '@nestjs/common'; @@ -255,7 +256,7 @@ export class ApPersonService implements OnModuleInit { // ついでにリモートユーザーの情報が古かったら更新しておく if (this.userEntityService.isRemoteUser(exist)) { - if (exist.lastFetchedAt == null || Date.now() - exist.lastFetchedAt.getTime() > 1000 * 60 * 60 * 3) { + if (exist.lastFetchedAt == null || Date.now() - exist.lastFetchedAt.getTime() > REMOTE_USER_CACHE_TTL) { this.logger.debug('fetchPersonWithRenewal: renew', { uri, lastFetchedAt: exist.lastFetchedAt }); await this.updatePerson(exist.uri); return await this.fetchPerson(uri); @@ -619,7 +620,7 @@ export class ApPersonService implements OnModuleInit { exist.movedAt == null || // 以前のmovingから14日以上経過した場合のみ移行処理を許可 // （Mastodonのクールダウン期間は30日だが若干緩めに設定しておく） - exist.movedAt.getTime() + 1000 * 60 * 60 * 24 * 14 < updated.movedAt.getTime() + exist.movedAt.getTime() + REMOTE_USER_MOVE_COOLDOWN < updated.movedAt.getTime() )) { this.logger.info(`Start to process Move of @${updated.username}@${updated.host} (${uri})`); return this.processRemoteMove(updated, movePreventUris) From 67758d2d1e8c5686db242af0c059c8d6ca3bef44 Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 1 Mar 2024 12:52:46 +0000 Subject: [PATCH 030/134] use gen-key-pair fn. from '@misskey-dev/node-http-message-signatures' --- .../backend/src/core/UserKeypairService.ts | 2 +- packages/backend/src/misc/gen-key-pair.ts | 36 +------------------ packages/backend/test/unit/ap-request.ts | 7 ++-- 3 files changed, 5 insertions(+), 40 deletions(-) diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 10818cfc7f8a..4cc1fa78d643 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -11,7 +11,7 @@ import { RedisKVCache } from '@/misc/cache.js'; import type { MiUserKeypair } from '@/models/UserKeypair.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; -import { genEd25519KeyPair } from '@/misc/gen-key-pair.js'; +import { genEd25519KeyPair } from '@misskey-dev/node-http-message-signatures'; import { GlobalEventService, GlobalEvents } from '@/core/GlobalEventService.js'; @Injectable() diff --git a/packages/backend/src/misc/gen-key-pair.ts b/packages/backend/src/misc/gen-key-pair.ts index a214591a81b3..58e3091369c0 100644 --- a/packages/backend/src/misc/gen-key-pair.ts +++ b/packages/backend/src/misc/gen-key-pair.ts @@ -3,41 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import * as crypto from 'node:crypto'; -import * as util from 'node:util'; - -const generateKeyPair = util.promisify(crypto.generateKeyPair); - -export async function genRsaKeyPair(modulusLength = 4096) { - return await generateKeyPair('rsa', { - modulusLength, - publicKeyEncoding: { - type: 'spki', - format: 'pem', - }, - privateKeyEncoding: { - type: 'pkcs8', - format: 'pem', - cipher: undefined, - passphrase: undefined, - }, - }); -} - -export async function genEd25519KeyPair() { - return await generateKeyPair('ed25519', { - publicKeyEncoding: { - type: 'spki', - format: 'pem', - }, - privateKeyEncoding: { - type: 'pkcs8', - format: 'pem', - cipher: undefined, - passphrase: undefined, - }, - }); -} +import { genEd25519KeyPair, genRsaKeyPair } from '@misskey-dev/node-http-message-signatures'; export async function genRSAAndEd25519KeyPair(rsaModulusLength = 4096) { const rsa = await genRsaKeyPair(rsaModulusLength); diff --git a/packages/backend/test/unit/ap-request.ts b/packages/backend/test/unit/ap-request.ts index d04de6800197..fc3855534de3 100644 --- a/packages/backend/test/unit/ap-request.ts +++ b/packages/backend/test/unit/ap-request.ts @@ -4,8 +4,7 @@ */ import * as assert from 'assert'; -import { verifyDraftSignature, parseRequestSignature } from '@misskey-dev/node-http-message-signatures'; -import { genEd25519KeyPair, genRsaKeyPair } from '@/misc/gen-key-pair.js'; +import { verifyDraftSignature, parseRequestSignature, genEd25519KeyPair, genRsaKeyPair } from '@misskey-dev/node-http-message-signatures'; import { createSignedGet, createSignedPost } from '@/core/activitypub/ApRequestService.js'; export const buildParsedSignature = (signingString: string, signature: string, algorithm: string) => { @@ -49,7 +48,7 @@ describe('ap-request', () => { const parsed = parseRequestSignature(req.request); expect(parsed?.version).toBe('draft'); if (!parsed) return; - const verify = verifyDraftSignature(parsed?.value, keypair.publicKey); + const verify = verifyDraftSignature(parsed.value, keypair.publicKey); assert.deepStrictEqual(verify, true); }); }); @@ -68,7 +67,7 @@ describe('ap-request', () => { const parsed = parseRequestSignature(req.request); expect(parsed?.version).toBe('draft'); if (!parsed) return; - const verify = verifyDraftSignature(parsed?.value, keypair.publicKey); + const verify = verifyDraftSignature(parsed.value, keypair.publicKey); assert.deepStrictEqual(verify, true); }); }); From 86c9f0b0fba5c5bcc2b5684a5feecb3ae64a8ea3 Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 1 Mar 2024 15:08:12 +0000 Subject: [PATCH 031/134] update node-http-message-signatures --- packages/backend/package.json | 2 +- packages/backend/src/server/ActivityPubServerService.ts | 7 +------ pnpm-lock.yaml | 8 ++++---- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index c07f31c77797..b80956e5bc27 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -79,7 +79,7 @@ "@fastify/multipart": "8.1.0", "@fastify/static": "6.12.0", "@fastify/view": "8.2.0", - "@misskey-dev/node-http-message-signatures": "0.0.0-alpha.7", + "@misskey-dev/node-http-message-signatures": "0.0.0-alpha.10", "@misskey-dev/sharp-read-bmp": "1.2.0", "@misskey-dev/summaly": "5.0.3", "@nestjs/common": "10.3.3", diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index 269bc3fb1162..204b9e7aa738 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -101,7 +101,7 @@ export class ActivityPubServerService { @bindThis private inbox(request: FastifyRequest, reply: FastifyReply) { - let signature; + let signature: ReturnType; const verifyDigest = verifyDigestHeader(request.raw, request.rawBody || '', true); if (!verifyDigest) { @@ -116,11 +116,6 @@ export class ActivityPubServerService { return; } - if (!signature) { - reply.code(401); - return; - } - if (signature.value.params.headers.indexOf('host') === -1 || request.headers.host !== this.config.host) { // Host not specified or not match. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b989c052541e..2d2d6a23d107 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,8 +111,8 @@ importers: specifier: 8.2.0 version: 8.2.0 '@misskey-dev/node-http-message-signatures': - specifier: 0.0.0-alpha.7 - version: 0.0.0-alpha.7 + specifier: 0.0.0-alpha.10 + version: 0.0.0-alpha.10 '@misskey-dev/sharp-read-bmp': specifier: 1.2.0 version: 1.2.0 @@ -4750,8 +4750,8 @@ packages: eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.1.0)(eslint@8.57.0) dev: true - /@misskey-dev/node-http-message-signatures@0.0.0-alpha.7: - resolution: {integrity: sha512-iM1nZ3YT+G4AEhbUnsK7PqnMY9MjBP5JomQAgi2OyxDtZ/wBpgLP6MCVz3ElCqZ8NQS1f+c4E1m6/dSN8MtU9Q==} + /@misskey-dev/node-http-message-signatures@0.0.0-alpha.10: + resolution: {integrity: sha512-4weLJVBm06bx4fLJHL9YtKXQ8ofKSNDI4UAaoI0JkzbbU8pyJHK4LFGfERrpUX9C+WsvSwVFZAgAzOxFwJ7z2w==} dev: false /@misskey-dev/sharp-read-bmp@1.2.0: From 65bd187d85b64ce95ccee88cd7136d2b88d97490 Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 1 Mar 2024 15:51:09 +0000 Subject: [PATCH 032/134] fix --- packages/backend/src/core/FetchInstanceMetadataService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index d038646d0d3f..a1313fc2578e 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -15,7 +15,7 @@ import { LoggerService } from '@/core/LoggerService.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; -import { REMOTE_SERVER_CACHE_TTL } from '@/const'; +import { REMOTE_SERVER_CACHE_TTL } from '@/const.js'; import type { DOMWindow } from 'jsdom'; type NodeInfo = { From 9111b5c4826021bbede6ba4fde1f4e68b90990a0 Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 1 Mar 2024 16:33:08 +0000 Subject: [PATCH 033/134] @misskey-dev/node-http-message-signatures@0.0.0-alpha.11 --- packages/backend/package.json | 2 +- pnpm-lock.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index b80956e5bc27..1d3f2fcd2a6b 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -79,7 +79,7 @@ "@fastify/multipart": "8.1.0", "@fastify/static": "6.12.0", "@fastify/view": "8.2.0", - "@misskey-dev/node-http-message-signatures": "0.0.0-alpha.10", + "@misskey-dev/node-http-message-signatures": "0.0.0-alpha.11", "@misskey-dev/sharp-read-bmp": "1.2.0", "@misskey-dev/summaly": "5.0.3", "@nestjs/common": "10.3.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2d2d6a23d107..52efc4c2170b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,8 +111,8 @@ importers: specifier: 8.2.0 version: 8.2.0 '@misskey-dev/node-http-message-signatures': - specifier: 0.0.0-alpha.10 - version: 0.0.0-alpha.10 + specifier: 0.0.0-alpha.11 + version: 0.0.0-alpha.11 '@misskey-dev/sharp-read-bmp': specifier: 1.2.0 version: 1.2.0 @@ -4750,8 +4750,8 @@ packages: eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.1.0)(eslint@8.57.0) dev: true - /@misskey-dev/node-http-message-signatures@0.0.0-alpha.10: - resolution: {integrity: sha512-4weLJVBm06bx4fLJHL9YtKXQ8ofKSNDI4UAaoI0JkzbbU8pyJHK4LFGfERrpUX9C+WsvSwVFZAgAzOxFwJ7z2w==} + /@misskey-dev/node-http-message-signatures@0.0.0-alpha.11: + resolution: {integrity: sha512-DeLzSIquddDQp+hqD9/zS/IkpJg8Zs1h/4viavCFe37CW2W4ABFumXhYJPB07n775QEihZNuvdipoMNXl83jzw==} dev: false /@misskey-dev/sharp-read-bmp@1.2.0: From d86b8c8752bb9eb5f6b4431403ed4bb0bc615c59 Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 1 Mar 2024 18:29:30 +0000 Subject: [PATCH 034/134] =?UTF-8?q?getAuthUserFromApId=E3=81=A7updatePerso?= =?UTF-8?q?n=E3=81=AE=E9=A0=BB=E5=BA=A6=E3=82=92=E5=A2=97=E3=82=84?= =?UTF-8?q?=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/activitypub/ApDbResolverService.ts | 82 ++++++++++--------- .../activitypub/models/ApPersonService.ts | 9 +- 2 files changed, 49 insertions(+), 42 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 25a37ca09611..9111906538d8 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -116,6 +116,8 @@ export class ApDbResolverService implements OnApplicationShutdown { /** * AP Actor id => Misskey User and Key + * @param uri AP Actor id + * @param keyId Key id to find. If not specified, main key will be selected. */ @bindThis public async getAuthUserFromApId(uri: string, keyId?: string): Promise<{ @@ -125,50 +127,56 @@ export class ApDbResolverService implements OnApplicationShutdown { const user = await this.apPersonService.resolvePerson(uri, undefined, true) as MiRemoteUser; if (user.isDeleted) return null; - const keys = await this.publicKeyByUserIdCache.fetch( - user.id, - () => this.userPublickeysRepository.find({ where: { userId: user.id } }), - v => v != null, - ); - - if (keys == null || !Array.isArray(keys)) return null; + const keys = await this.getPublicKeyByUserId(user.id); + + if (keys == null || !Array.isArray(keys)) return { user, key: null }; + + if (!keyId) { + // mainっぽいのを選ぶ + const mainKey = keys.find(x => { + try { + const url = new URL(x.keyId); + const path = url.pathname.split('/').pop()?.toLowerCase(); + if (url.hash) { + if (url.hash.toLowerCase().includes('main')) { + return true; + } + } else if (path?.includes('main') || path === 'publickey') { + return true; + } + } catch { /* noop */ } - if (keys.length === 0) { - return { - user, - key: keys[0], - }; + return false; + }); + return { user, key: mainKey ?? keys[0] }; } const exactKey = keys.find(x => x.keyId === keyId); - if (exactKey) { - return { - user, - key: exactKey, - }; + if (exactKey) return { user, key: exactKey }; + + // keyIdで見つからない場合、lastFetchedAtでの更新制限を弱めて再取得 + if (user.lastFetchedAt == null || user.lastFetchedAt < new Date(Date.now() - 1000 * 60 * 12)) { + const renewed = await this.apPersonService.fetchPersonWithRenewal(uri, 0); + if (renewed == null || renewed.isDeleted) return null; + + this.refreshCacheByUserId(user.id); + const keys = await this.getPublicKeyByUserId(user.id); + if (keys == null || !Array.isArray(keys)) return null; + + const exactKey = keys.find(x => x.keyId === keyId); + if (exactKey) return { user, key: exactKey }; } - // 公開鍵は複数あるが、mainっぽいのを選ぶ - const mainKey = keys.find(x => { - try { - if (x.keyId === keyId) return true; - const url = new URL(x.keyId); - const path = url.pathname.split('/').pop()?.toLowerCase(); - if (url.hash) { - if (url.hash.toLowerCase().includes('main')) { - return true; - } - } else if (path?.includes('main') || path === 'publickey') { - return true; - } - } catch { /* noop */ } + return { user, key: null }; + } - return false; - }); - return { - user, - key: mainKey ?? keys[0], - }; + @bindThis + public async getPublicKeyByUserId(userId: MiUser['id']): Promise { + return await this.publicKeyByUserIdCache.fetch( + userId, + () => this.userPublickeysRepository.find({ where: { userId } }), + v => v != null, + ); } @bindThis diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 97abfcbde8a6..e331ea5395b5 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -250,18 +250,17 @@ export class ApPersonService implements OnModuleInit { } @bindThis - async fetchPersonWithRenewal(uri: string): Promise { + async fetchPersonWithRenewal(uri: string, TTL = REMOTE_USER_CACHE_TTL): Promise { const exist = await this.fetchPerson(uri); if (exist == null) return null; - // ついでにリモートユーザーの情報が古かったら更新しておく if (this.userEntityService.isRemoteUser(exist)) { - if (exist.lastFetchedAt == null || Date.now() - exist.lastFetchedAt.getTime() > REMOTE_USER_CACHE_TTL) { - this.logger.debug('fetchPersonWithRenewal: renew', { uri, lastFetchedAt: exist.lastFetchedAt }); + if (TTL === 0 || exist.lastFetchedAt == null || Date.now() - exist.lastFetchedAt.getTime() > TTL) { + this.logger.debug('fetchPersonWithRenewal: renew', { uri, TTL, lastFetchedAt: exist.lastFetchedAt }); await this.updatePerson(exist.uri); return await this.fetchPerson(uri); } - this.logger.debug('fetchPersonWithRenewal: use cache', { uri, lastFetchedAt: exist.lastFetchedAt }); + this.logger.debug('fetchPersonWithRenewal: use cache', { uri, TTL, lastFetchedAt: exist.lastFetchedAt }); } return exist; From ea6c38cc6b095e268e2825c8765b80756a6a3bf4 Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 1 Mar 2024 18:38:31 +0000 Subject: [PATCH 035/134] cacheRaw.date --- .../src/core/activitypub/ApDbResolverService.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 9111906538d8..1495ba7a95fe 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -53,7 +53,7 @@ export class ApDbResolverService implements OnApplicationShutdown { private cacheService: CacheService, private apPersonService: ApPersonService, ) { - this.publicKeyByUserIdCache = new MemoryKVCache(1e3 * 60 * 20); // 20分 + this.publicKeyByUserIdCache = new MemoryKVCache(Infinity); } @bindThis @@ -154,7 +154,19 @@ export class ApDbResolverService implements OnApplicationShutdown { const exactKey = keys.find(x => x.keyId === keyId); if (exactKey) return { user, key: exactKey }; - // keyIdで見つからない場合、lastFetchedAtでの更新制限を弱めて再取得 + // keyIdで見つからない場合 + // まずはキャッシュを更新して再取得 + const cacheRaw = this.publicKeyByUserIdCache.cache.get(user.id); + if (cacheRaw && cacheRaw.date > Date.now() - 1000 * 60 * 12) { + this.refreshCacheByUserId(user.id); + const keys = await this.getPublicKeyByUserId(user.id); + if (keys == null || !Array.isArray(keys)) return null; + + const exactKey = keys.find(x => x.keyId === keyId); + if (exactKey) return { user, key: exactKey }; + } + + // lastFetchedAtでの更新制限を弱めて再取得 if (user.lastFetchedAt == null || user.lastFetchedAt < new Date(Date.now() - 1000 * 60 * 12)) { const renewed = await this.apPersonService.fetchPersonWithRenewal(uri, 0); if (renewed == null || renewed.isDeleted) return null; From 1357b076d02a54dbbbde3c5f19ec1a829a81ebb7 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 2 Mar 2024 13:53:52 +0000 Subject: [PATCH 036/134] use requiredInputs https://github.com/misskey-dev/misskey/pull/13464#discussion_r1509964359 --- packages/backend/src/server/ActivityPubServerService.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index 204b9e7aa738..ba0adac03b05 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -110,7 +110,11 @@ export class ActivityPubServerService { } try { - signature = parseRequestSignature(request.raw); + signature = parseRequestSignature(request.raw, { + requiredInputs: { + draft: ['(request-target)', 'digest', 'host', 'date'], + }, + }); } catch (e) { reply.code(401); return; From a405b6282729a3aac18f80777f134a95d4fca945 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 3 Mar 2024 21:02:23 +0000 Subject: [PATCH 037/134] update @misskey-dev/node-http-message-signatures --- packages/backend/package.json | 2 +- .../src/core/activitypub/ApRequestService.ts | 14 ++++++------- .../queue/processors/InboxProcessorService.ts | 2 +- .../src/server/ActivityPubServerService.ts | 12 ++--------- packages/backend/test/unit/ap-request.ts | 8 ++++---- pnpm-lock.yaml | 20 ++++++++++++++----- 6 files changed, 30 insertions(+), 28 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 1d3f2fcd2a6b..82675fafc2c4 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -79,7 +79,7 @@ "@fastify/multipart": "8.1.0", "@fastify/static": "6.12.0", "@fastify/view": "8.2.0", - "@misskey-dev/node-http-message-signatures": "0.0.0-alpha.11", + "@misskey-dev/node-http-message-signatures": "0.0.1", "@misskey-dev/sharp-read-bmp": "1.2.0", "@misskey-dev/summaly": "5.0.3", "@nestjs/common": "10.3.3", diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 36a4b34de1d7..9dda59543e56 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -28,7 +28,7 @@ type PrivateKey = { keyId: string; }; -export function createSignedPost(args: { level: string; key: PrivateKey; url: string; body: string; additionalHeaders: Record }) { +export async function createSignedPost(args: { level: string; key: PrivateKey; url: string; body: string; additionalHeaders: Record }) { const u = new URL(args.url); const request: RequestLike = { url: u.href, @@ -42,10 +42,10 @@ export function createSignedPost(args: { level: string; key: PrivateKey; url: st }; // TODO: levelによって処理を分ける - const digestHeader = genRFC3230DigestHeader(args.body); + const digestHeader = await genRFC3230DigestHeader(args.body, 'SHA-256'); request.headers['Digest'] = digestHeader; - const result = signAsDraftToRequest(request, args.key, ['(request-target)', 'date', 'host', 'digest']); + const result = await signAsDraftToRequest(request, args.key, ['(request-target)', 'date', 'host', 'digest']); return { request, @@ -53,7 +53,7 @@ export function createSignedPost(args: { level: string; key: PrivateKey; url: st }; } -export function createSignedGet(args: { level: string; key: PrivateKey; url: string; additionalHeaders: Record }) { +export async function createSignedGet(args: { level: string; key: PrivateKey; url: string; additionalHeaders: Record }) { const u = new URL(args.url); const request: RequestLike = { url: u.href, @@ -67,7 +67,7 @@ export function createSignedGet(args: { level: string; key: PrivateKey; url: str }; // TODO: levelによって処理を分ける - const result = signAsDraftToRequest(request, args.key, ['(request-target)', 'date', 'host', 'accept']); + const result = await signAsDraftToRequest(request, args.key, ['(request-target)', 'date', 'host', 'accept']); return { request, @@ -108,7 +108,7 @@ export class ApRequestService { public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, level: string): Promise { const body = typeof object === 'string' ? object : JSON.stringify(object); const key = await this.getPrivateKey(user.id, level); - const req = createSignedPost({ + const req = await createSignedPost({ level, key, url, @@ -140,7 +140,7 @@ export class ApRequestService { @bindThis public async signedGet(url: string, user: { id: MiUser['id'] }, level: string): Promise { const key = await this.getPrivateKey(user.id, level); - const req = createSignedGet({ + const req = await createSignedGet({ level, key, url, diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 3f10a128ed1a..94163a24958e 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -104,7 +104,7 @@ export class InboxProcessorService { // HTTP-Signatureの検証 const errorLogger = (ms: any) => this.logger.error(ms); - const httpSignatureValidated = verifyDraftSignature(signature, authUser.key.keyPem, errorLogger); + const httpSignatureValidated = await verifyDraftSignature(signature, authUser.key.keyPem, errorLogger); this.logger.debug('Inbox message validation: ', { userId: authUser.user.id, userAcct: Acct.toString(authUser.user), diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index ba0adac03b05..1dbb94a2c2b3 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -3,7 +3,6 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import * as crypto from 'node:crypto'; import { IncomingMessage } from 'node:http'; import { Inject, Injectable } from '@nestjs/common'; import fastifyAccepts from '@fastify/accepts'; @@ -100,10 +99,10 @@ export class ActivityPubServerService { } @bindThis - private inbox(request: FastifyRequest, reply: FastifyReply) { + private async inbox(request: FastifyRequest, reply: FastifyReply) { let signature: ReturnType; - const verifyDigest = verifyDigestHeader(request.raw, request.rawBody || '', true); + const verifyDigest = await verifyDigestHeader(request.raw, request.rawBody || '', true); if (!verifyDigest) { reply.code(401); return; @@ -120,13 +119,6 @@ export class ActivityPubServerService { return; } - if (signature.value.params.headers.indexOf('host') === -1 - || request.headers.host !== this.config.host) { - // Host not specified or not match. - reply.code(401); - return; - } - this.queueService.inbox(request.body as IActivity, signature); reply.code(202); diff --git a/packages/backend/test/unit/ap-request.ts b/packages/backend/test/unit/ap-request.ts index fc3855534de3..797ef73e62a5 100644 --- a/packages/backend/test/unit/ap-request.ts +++ b/packages/backend/test/unit/ap-request.ts @@ -43,12 +43,12 @@ describe('ap-request', () => { 'User-Agent': 'UA', }; - const req = createSignedPost({ level, key, url, body, additionalHeaders: headers }); + const req = await createSignedPost({ level, key, url, body, additionalHeaders: headers }); const parsed = parseRequestSignature(req.request); expect(parsed?.version).toBe('draft'); if (!parsed) return; - const verify = verifyDraftSignature(parsed.value, keypair.publicKey); + const verify = await verifyDraftSignature(parsed.value, keypair.publicKey); assert.deepStrictEqual(verify, true); }); }); @@ -62,12 +62,12 @@ describe('ap-request', () => { 'User-Agent': 'UA', }; - const req = createSignedGet({ level, key, url, additionalHeaders: headers }); + const req = await createSignedGet({ level, key, url, additionalHeaders: headers }); const parsed = parseRequestSignature(req.request); expect(parsed?.version).toBe('draft'); if (!parsed) return; - const verify = verifyDraftSignature(parsed.value, keypair.publicKey); + const verify = await verifyDraftSignature(parsed.value, keypair.publicKey); assert.deepStrictEqual(verify, true); }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 52efc4c2170b..5853c00bb7ab 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,8 +111,8 @@ importers: specifier: 8.2.0 version: 8.2.0 '@misskey-dev/node-http-message-signatures': - specifier: 0.0.0-alpha.11 - version: 0.0.0-alpha.11 + specifier: 0.0.1 + version: 0.0.1 '@misskey-dev/sharp-read-bmp': specifier: 1.2.0 version: 1.2.0 @@ -4621,6 +4621,10 @@ packages: resolution: {integrity: sha512-fuscdXJ9G1qb7W8VdHi+IwRqij3lBkosAm4ydQtEmbY58OzHXqQhvlxqEkoz0yssNVn38bcpRWgA9PP+OGoisw==} dev: false + /@lapo/asn1js@1.2.4: + resolution: {integrity: sha512-mdInpQZaYUWu5QbKIB2+Vd+j6Y7cc6xQYNwYBPC9jri2rwy3tbxom0IhhT4G5WOKWO7Iht10SxYpKq+AfuH6dw==} + dev: false + /@levischuck/tiny-cbor@0.2.2: resolution: {integrity: sha512-f5CnPw997Y2GQ8FAvtuVVC19FX8mwNNC+1XJcIi16n/LTJifKO6QBgGLgN3YEmqtGMk17SKSuoWES3imJVxAVw==} dev: false @@ -4750,8 +4754,10 @@ packages: eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.1.0)(eslint@8.57.0) dev: true - /@misskey-dev/node-http-message-signatures@0.0.0-alpha.11: - resolution: {integrity: sha512-DeLzSIquddDQp+hqD9/zS/IkpJg8Zs1h/4viavCFe37CW2W4ABFumXhYJPB07n775QEihZNuvdipoMNXl83jzw==} + /@misskey-dev/node-http-message-signatures@0.0.1: + resolution: {integrity: sha512-BaMYEOSBwBtDW0NJcljE4S39gFpgNigVzbVFlVsKzu4+k7snL72mXEaAoMCHWCw7XmdY3+fKADbQoqc4SFrpLw==} + dependencies: + '@lapo/asn1js': 1.2.4 dev: false /@misskey-dev/sharp-read-bmp@1.2.0: @@ -6641,7 +6647,7 @@ packages: ts-dedent: 2.2.0 type-fest: 2.19.0 vue: 3.4.21(typescript@5.3.3) - vue-component-type-helpers: 1.8.27 + vue-component-type-helpers: 2.0.3 transitivePeerDependencies: - encoding - supports-color @@ -19440,6 +19446,10 @@ packages: resolution: {integrity: sha512-6bnLkn8O0JJyiFSIF0EfCogzeqNXpnjJ0vW/SZzNHfe6sPx30lTtTXlE5TFs2qhJlAtDFybStVNpL73cPe3OMQ==} dev: true + /vue-component-type-helpers@2.0.3: + resolution: {integrity: sha512-dVPXmrwul+lLt2ErNqwfIGTWkZoPQjlarFx5pwXnCwdJ80ZaJ2nmqtt3KcUgVaKBwBQBpJqOlJPaMxICBay+Cg==} + dev: true + /vue-demi@0.14.7(vue@3.4.21): resolution: {integrity: sha512-EOG8KXDQNwkJILkx/gPcoL/7vH+hORoBaKgGe+6W7VFMvCYJfmF2dGbvgDroVnI8LU7/kTu8mbjRZGBU1z9NTA==} engines: {node: '>=12'} From 13af6f2313ff5cda466c50ead25bb91edb9fbbd2 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 3 Mar 2024 21:22:47 +0000 Subject: [PATCH 038/134] clean up --- packages/backend/src/core/SignupService.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/backend/src/core/SignupService.ts b/packages/backend/src/core/SignupService.ts index 78a77d2adf67..54c61700622d 100644 --- a/packages/backend/src/core/SignupService.ts +++ b/packages/backend/src/core/SignupService.ts @@ -3,7 +3,6 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { generateKeyPair } from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; import bcrypt from 'bcryptjs'; import { DataSource, IsNull } from 'typeorm'; From aabdb666b7b84a259d48a39012710d9b95178c3b Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 3 Mar 2024 22:03:25 +0000 Subject: [PATCH 039/134] err msg --- packages/backend/src/core/FetchInstanceMetadataService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index a1313fc2578e..4dc81cd6be93 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -75,7 +75,7 @@ export class FetchInstanceMetadataService { const _instance = await this.federatedInstanceService.fetch(host); const now = Date.now(); if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < REMOTE_SERVER_CACHE_TTL)) { - throw new Error('Skip because updated recently'); + throw new Error(`Skip because updated recently ${_instance.infoUpdatedAt.toJSON()}`); } } From 25d5a8cb7e905221b464316893d9aa79926e6c39 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 3 Mar 2024 23:04:33 +0000 Subject: [PATCH 040/134] =?UTF-8?q?fix(backend):=20fetchInstanceMetadata?= =?UTF-8?q?=E3=81=AELock=E3=81=8C=E6=B0=B8=E9=81=A0=E3=81=AB=E8=A7=A3?= =?UTF-8?q?=E9=99=A4=E3=81=95=E3=82=8C=E3=81=AA=E3=81=84=E5=95=8F=E9=A1=8C?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: まっちゃとーにゅ <17376330+u1-liquid@users.noreply.github.com> --- .../src/core/FetchInstanceMetadataService.ts | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index bc270bd28fd3..7e5185e960ef 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -51,23 +51,33 @@ export class FetchInstanceMetadataService { } @bindThis - public async tryLock(host: string): Promise { - const mutex = await this.redisClient.set(`fetchInstanceMetadata:mutex:${host}`, '1', 'GET'); - return mutex !== '1'; + private async tryLock(host: string): Promise { + // TODO: マイグレーションなのであとで消す (2024.3.1) + this.redisClient.del(`fetchInstanceMetadata:mutex:${host}`); + + return await this.redisClient.set( + `fetchInstanceMetadata:mutex:v2:${host}`, '1', + 'EX', 30, // 30秒したら自動でロック解除 https://github.com/misskey-dev/misskey/issues/13506#issuecomment-1975375395 + 'GET' // 古い値を返す（なかったらnull） + ); } @bindThis - public unlock(host: string): Promise<'OK'> { - return this.redisClient.set(`fetchInstanceMetadata:mutex:${host}`, '0'); + private unlock(host: string): Promise { + return this.redisClient.del(`fetchInstanceMetadata:mutex:v2:${host}`); } @bindThis public async fetchInstanceMetadata(instance: MiInstance, force = false): Promise { const host = instance.host; - // Acquire mutex to ensure no parallel runs - if (!await this.tryLock(host)) return; + try { if (!force) { + if (await this.tryLock(host) === '1') { + // 1が返ってきていたらロックされている = 何もしない + return; + } + const _instance = await this.federatedInstanceService.fetch(host); const now = Date.now(); if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) { From 89e1ff699adc8bacce521d8d73e22d7439390e01 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 3 Mar 2024 23:19:13 +0000 Subject: [PATCH 041/134] fix httpMessageSignaturesImplementationLevel validation --- .../backend/src/core/FetchInstanceMetadataService.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 4dc81cd6be93..2398d948190f 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -17,6 +17,7 @@ import { bindThis } from '@/decorators.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { REMOTE_SERVER_CACHE_TTL } from '@/const.js'; import type { DOMWindow } from 'jsdom'; +import { el } from 'date-fns/locale'; type NodeInfo = { openRegistrations?: unknown; @@ -107,8 +108,13 @@ export class FetchInstanceMetadataService { updates.openRegistrations = info.openRegistrations; updates.maintainerName = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name ?? null) : null : null; updates.maintainerEmail = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email ?? null) : null : null; - if (info.metadata && info.metadata.httpMessageSignaturesImplementationLevel) { - updates.httpMessageSignaturesImplementationLevel = info.metadata.httpMessageSignaturesImplementationLevel.toString() ?? '00'; + if (info.metadata && info.metadata.httpMessageSignaturesImplementationLevel && ( + info.metadata.httpMessageSignaturesImplementationLevel === '01' || + info.metadata.httpMessageSignaturesImplementationLevel === '11' + )) { + updates.httpMessageSignaturesImplementationLevel = info.metadata.httpMessageSignaturesImplementationLevel; + } else { + updates.httpMessageSignaturesImplementationLevel = '00'; } } From 2dde845738846e021fee6e6c3c0f625cd0ce1e47 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 3 Mar 2024 23:26:35 +0000 Subject: [PATCH 042/134] fix test --- packages/backend/src/core/FetchInstanceMetadataService.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 7e5185e960ef..d8be1153ce64 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -51,7 +51,8 @@ export class FetchInstanceMetadataService { } @bindThis - private async tryLock(host: string): Promise { + // public for test + public async tryLock(host: string): Promise { // TODO: マイグレーションなのであとで消す (2024.3.1) this.redisClient.del(`fetchInstanceMetadata:mutex:${host}`); @@ -63,7 +64,8 @@ export class FetchInstanceMetadataService { } @bindThis - private unlock(host: string): Promise { + // public for test + public unlock(host: string): Promise { return this.redisClient.del(`fetchInstanceMetadata:mutex:v2:${host}`); } From 41a461edbe1dfe347e9a8c7394c2d6ab9e869973 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 3 Mar 2024 23:33:08 +0000 Subject: [PATCH 043/134] fix --- .../backend/src/core/FetchInstanceMetadataService.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index d8be1153ce64..d937605b3c63 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -73,13 +73,14 @@ export class FetchInstanceMetadataService { public async fetchInstanceMetadata(instance: MiInstance, force = false): Promise { const host = instance.host; + // unlockされてしまうのでtry内でロックチェックをしない + if (!force && await this.tryLock(host) === '1') { + // 1が返ってきていたらロックされているという意味なので、何もしない + return; + } + try { if (!force) { - if (await this.tryLock(host) === '1') { - // 1が返ってきていたらロックされている = 何もしない - return; - } - const _instance = await this.federatedInstanceService.fetch(host); const now = Date.now(); if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) { From 2926f68d8e3cd15865f4969778ac4fed00cdb4bf Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 3 Mar 2024 23:33:25 +0000 Subject: [PATCH 044/134] comment --- packages/backend/src/core/FetchInstanceMetadataService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index d937605b3c63..dc3780e270c8 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -73,7 +73,7 @@ export class FetchInstanceMetadataService { public async fetchInstanceMetadata(instance: MiInstance, force = false): Promise { const host = instance.host; - // unlockされてしまうのでtry内でロックチェックをしない + // finallyでunlockされてしまうのでtry内でロックチェックをしない if (!force && await this.tryLock(host) === '1') { // 1が返ってきていたらロックされているという意味なので、何もしない return; From 64fcf736cccafc6d548a5bbd4bda29a4154ed686 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 3 Mar 2024 23:36:03 +0000 Subject: [PATCH 045/134] comment --- packages/backend/src/core/FetchInstanceMetadataService.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index dc3780e270c8..8d173855f33d 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -74,6 +74,7 @@ export class FetchInstanceMetadataService { const host = instance.host; // finallyでunlockされてしまうのでtry内でロックチェックをしない + // （returnであってもfinallyは実行される） if (!force && await this.tryLock(host) === '1') { // 1が返ってきていたらロックされているという意味なので、何もしない return; From 7eb19d5a8e413ceb8f134fc53c2917e1a1a91fa2 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 3 Mar 2024 23:45:47 +0000 Subject: [PATCH 046/134] improve test --- .../test/unit/FetchInstanceMetadataService.ts | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/packages/backend/test/unit/FetchInstanceMetadataService.ts b/packages/backend/test/unit/FetchInstanceMetadataService.ts index 510b84b68036..bf8f3ab0e306 100644 --- a/packages/backend/test/unit/FetchInstanceMetadataService.ts +++ b/packages/backend/test/unit/FetchInstanceMetadataService.ts @@ -56,6 +56,7 @@ describe('FetchInstanceMetadataService', () => { } else if (token === DI.redis) { return mockRedis; } + return null; }) .compile(); @@ -78,6 +79,7 @@ describe('FetchInstanceMetadataService', () => { httpRequestService.getJson.mockImplementation(() => { throw Error(); }); const tryLockSpy = jest.spyOn(fetchInstanceMetadataService, 'tryLock'); const unlockSpy = jest.spyOn(fetchInstanceMetadataService, 'unlock'); + await fetchInstanceMetadataService.fetchInstanceMetadata({ host: 'example.com' } as any); expect(tryLockSpy).toHaveBeenCalledTimes(1); expect(unlockSpy).toHaveBeenCalledTimes(1); @@ -92,6 +94,7 @@ describe('FetchInstanceMetadataService', () => { httpRequestService.getJson.mockImplementation(() => { throw Error(); }); const tryLockSpy = jest.spyOn(fetchInstanceMetadataService, 'tryLock'); const unlockSpy = jest.spyOn(fetchInstanceMetadataService, 'unlock'); + await fetchInstanceMetadataService.fetchInstanceMetadata({ host: 'example.com' } as any); expect(tryLockSpy).toHaveBeenCalledTimes(1); expect(unlockSpy).toHaveBeenCalledTimes(1); @@ -104,13 +107,30 @@ describe('FetchInstanceMetadataService', () => { const now = Date.now(); federatedInstanceService.fetch.mockResolvedValue({ infoUpdatedAt: { getTime: () => now - 10 * 1000 * 60 * 60 * 24 } } as any); httpRequestService.getJson.mockImplementation(() => { throw Error(); }); + await fetchInstanceMetadataService.tryLock('example.com'); const tryLockSpy = jest.spyOn(fetchInstanceMetadataService, 'tryLock'); const unlockSpy = jest.spyOn(fetchInstanceMetadataService, 'unlock'); - await fetchInstanceMetadataService.tryLock('example.com'); + await fetchInstanceMetadataService.fetchInstanceMetadata({ host: 'example.com' } as any); - expect(tryLockSpy).toHaveBeenCalledTimes(2); + expect(tryLockSpy).toHaveBeenCalledTimes(1); expect(unlockSpy).toHaveBeenCalledTimes(0); expect(federatedInstanceService.fetch).toHaveBeenCalledTimes(0); expect(httpRequestService.getJson).toHaveBeenCalledTimes(0); }); + + test('Do when lock not acquired but forced', async () => { + redisClient.set = mockRedis(); + const now = Date.now(); + federatedInstanceService.fetch.mockResolvedValue({ infoUpdatedAt: { getTime: () => now - 10 * 1000 * 60 * 60 * 24 } } as any); + httpRequestService.getJson.mockImplementation(() => { throw Error(); }); + await fetchInstanceMetadataService.tryLock('example.com'); + const tryLockSpy = jest.spyOn(fetchInstanceMetadataService, 'tryLock'); + const unlockSpy = jest.spyOn(fetchInstanceMetadataService, 'unlock'); + + await fetchInstanceMetadataService.fetchInstanceMetadata({ host: 'example.com' } as any, true); + expect(tryLockSpy).toHaveBeenCalledTimes(0); + expect(unlockSpy).toHaveBeenCalledTimes(1); + expect(federatedInstanceService.fetch).toHaveBeenCalledTimes(0); + expect(httpRequestService.getJson).toHaveBeenCalled(); + }); }); From c7eed1c360cfec165757aa714fbd075346bbbb7a Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 3 Mar 2024 23:49:04 +0000 Subject: [PATCH 047/134] fix --- packages/backend/src/core/FetchInstanceMetadataService.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 54dc925540e6..e8d9cec80455 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -17,7 +17,6 @@ import { bindThis } from '@/decorators.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { REMOTE_SERVER_CACHE_TTL } from '@/const.js'; import type { DOMWindow } from 'jsdom'; -import { el } from 'date-fns/locale'; type NodeInfo = { openRegistrations?: unknown; From 6a56aea4227d3459d37c17ae67ad6eb686f391d1 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 3 Mar 2024 23:53:30 +0000 Subject: [PATCH 048/134] use Promise.all in genRSAAndEd25519KeyPair --- packages/backend/src/misc/gen-key-pair.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/backend/src/misc/gen-key-pair.ts b/packages/backend/src/misc/gen-key-pair.ts index 58e3091369c0..0b033ec33e2a 100644 --- a/packages/backend/src/misc/gen-key-pair.ts +++ b/packages/backend/src/misc/gen-key-pair.ts @@ -6,8 +6,7 @@ import { genEd25519KeyPair, genRsaKeyPair } from '@misskey-dev/node-http-message-signatures'; export async function genRSAAndEd25519KeyPair(rsaModulusLength = 4096) { - const rsa = await genRsaKeyPair(rsaModulusLength); - const ed25519 = await genEd25519KeyPair(); + const [rsa, ed25519] = await Promise.all([genRsaKeyPair(rsaModulusLength), genEd25519KeyPair()]); return { publicKey: rsa.publicKey, privateKey: rsa.privateKey, From d772eacfa144ce4e2ac943c42248dcc78735b666 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 3 Mar 2024 23:57:30 +0000 Subject: [PATCH 049/134] refreshAndprepareEd25519KeyPair --- packages/backend/src/core/UserKeypairService.ts | 4 ++-- .../backend/src/core/activitypub/ApDeliverManagerService.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 4cc1fa78d643..c27f8fc71a41 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -5,13 +5,13 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; +import { genEd25519KeyPair } from '@misskey-dev/node-http-message-signatures'; import type { MiUser } from '@/models/User.js'; import type { UserKeypairsRepository } from '@/models/_.js'; import { RedisKVCache } from '@/misc/cache.js'; import type { MiUserKeypair } from '@/models/UserKeypair.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; -import { genEd25519KeyPair } from '@misskey-dev/node-http-message-signatures'; import { GlobalEventService, GlobalEvents } from '@/core/GlobalEventService.js'; @Injectable() @@ -50,7 +50,7 @@ export class UserKeypairService implements OnApplicationShutdown { } @bindThis - public async prepareEd25519KeyPair(userId: MiUser['id']): Promise { + public async refreshAndprepareEd25519KeyPair(userId: MiUser['id']): Promise { await this.refresh(userId); const keypair = await this.cache.fetch(userId); if (keypair.ed25519PublicKey != null) return; diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index 27a9c11daba6..cd3b6a8795c7 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -109,7 +109,7 @@ class DeliverManager { /** * ed25519の署名がなければ追加する */ - await this.userKeypairService.prepareEd25519KeyPair(this.actor.id); + await this.userKeypairService.refreshAndprepareEd25519KeyPair(this.actor.id); //#endregion // The value flags whether it is shared or not. From 941aed6a141d12f3abc33ce07d5f7930c3d50881 Mon Sep 17 00:00:00 2001 From: tamaina Date: Mon, 4 Mar 2024 00:05:48 +0000 Subject: [PATCH 050/134] refreshAndfindKey --- .../core/activitypub/ApDbResolverService.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 1495ba7a95fe..61fcc6731dc9 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -114,6 +114,15 @@ export class ApDbResolverService implements OnApplicationShutdown { } } + @bindThis + private async refreshAndfindKey(userId: MiUser['id'], keyId: string): Promise { + this.refreshCacheByUserId(userId); + const keys = await this.getPublicKeyByUserId(userId); + if (keys == null || !Array.isArray(keys)) return null; + + return keys.find(x => x.keyId === keyId) ?? null; + } + /** * AP Actor id => Misskey User and Key * @param uri AP Actor id @@ -158,11 +167,7 @@ export class ApDbResolverService implements OnApplicationShutdown { // まずはキャッシュを更新して再取得 const cacheRaw = this.publicKeyByUserIdCache.cache.get(user.id); if (cacheRaw && cacheRaw.date > Date.now() - 1000 * 60 * 12) { - this.refreshCacheByUserId(user.id); - const keys = await this.getPublicKeyByUserId(user.id); - if (keys == null || !Array.isArray(keys)) return null; - - const exactKey = keys.find(x => x.keyId === keyId); + const exactKey = await this.refreshAndfindKey(user.id, keyId); if (exactKey) return { user, key: exactKey }; } @@ -171,12 +176,7 @@ export class ApDbResolverService implements OnApplicationShutdown { const renewed = await this.apPersonService.fetchPersonWithRenewal(uri, 0); if (renewed == null || renewed.isDeleted) return null; - this.refreshCacheByUserId(user.id); - const keys = await this.getPublicKeyByUserId(user.id); - if (keys == null || !Array.isArray(keys)) return null; - - const exactKey = keys.find(x => x.keyId === keyId); - if (exactKey) return { user, key: exactKey }; + return { user, key: await this.refreshAndfindKey(user.id, keyId) }; } return { user, key: null }; From 83f635835eb4ed4abed1d7afdc480dbe82c7cad7 Mon Sep 17 00:00:00 2001 From: tamaina Date: Mon, 4 Mar 2024 00:12:11 +0000 Subject: [PATCH 051/134] commetn --- packages/backend/src/core/activitypub/ApRequestService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 9dda59543e56..0e96a80165e1 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -41,7 +41,7 @@ export async function createSignedPost(args: { level: string; key: PrivateKey; u }, }; - // TODO: levelによって処理を分ける + // TODO: httpMessageSignaturesImplementationLevelによって新規格で通信をするようにする const digestHeader = await genRFC3230DigestHeader(args.body, 'SHA-256'); request.headers['Digest'] = digestHeader; @@ -66,7 +66,7 @@ export async function createSignedGet(args: { level: string; key: PrivateKey; ur }, }; - // TODO: levelによって処理を分ける + // TODO: httpMessageSignaturesImplementationLevelによって新規格で通信をするようにする const result = await signAsDraftToRequest(request, args.key, ['(request-target)', 'date', 'host', 'accept']); return { From 25cc9e0bf10b6f9204ad9d3c8cee2d63b0e9117a Mon Sep 17 00:00:00 2001 From: tamaina Date: Mon, 4 Mar 2024 00:34:37 +0000 Subject: [PATCH 052/134] refactor public keys add --- .../activitypub/models/ApPersonService.ts | 53 ++++++++----------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index e331ea5395b5..983bf0ebc549 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -50,7 +50,7 @@ import type { ApResolverService, Resolver } from '../ApResolverService.js'; import type { ApLoggerService } from '../ApLoggerService.js'; // eslint-disable-next-line @typescript-eslint/consistent-type-imports import type { ApImageService } from './ApImageService.js'; -import type { IActor, IObject } from '../type.js'; +import type { IActor, IKey, IObject } from '../type.js'; const nameLength = 128; const summaryLength = 2048; @@ -396,20 +396,17 @@ export class ApPersonService implements OnModuleInit { })); if (person.publicKey) { - await transactionalEntityManager.save(new MiUserPublickey({ - keyId: person.publicKey.id, - userId: user.id, - keyPem: person.publicKey.publicKeyPem, - })); - - if (person.additionalPublicKeys) { - for (const key of person.additionalPublicKeys) { - await transactionalEntityManager.save(new MiUserPublickey({ - keyId: key.id, - userId: user.id, - keyPem: key.publicKeyPem, - })); - } + const keys = new Map([ + ...(person.additionalPublicKeys ? person.additionalPublicKeys.map(key => [key.id, key] as const) : []), + [person.publicKey.id, person.publicKey], + ]); + + for (const key of keys.values()) { + await transactionalEntityManager.save(new MiUserPublickey({ + keyId: key.id, + userId: user.id, + keyPem: key.publicKeyPem, + })); } } }); @@ -556,27 +553,21 @@ export class ApPersonService implements OnModuleInit { // Update user await this.usersRepository.update(exist.id, updates); - const availablePublicKeys = new Set(); + const publicKeys = new Map(); if (person.publicKey) { - await this.userPublickeysRepository.update({ keyId: person.publicKey.id }, { - userId: exist.id, - keyPem: person.publicKey.publicKeyPem, - }); - availablePublicKeys.add(person.publicKey.id); - - if (person.additionalPublicKeys) { - for (const key of person.additionalPublicKeys) { - await this.userPublickeysRepository.update({ keyId: key.id }, { - userId: exist.id, - keyPem: key.publicKeyPem, - }); - availablePublicKeys.add(key.id); - } + (person.additionalPublicKeys ?? []).forEach(key => publicKeys.set(key.id, key)); + publicKeys.set(person.publicKey.id, person.publicKey); + + for (const key of publicKeys.values()) { + await this.userPublickeysRepository.update({ keyId: key.id }, { + userId: exist.id, + keyPem: key.publicKeyPem, + }); } } this.userPublickeysRepository.delete({ - keyId: Not(In(Array.from(availablePublicKeys))), + keyId: Not(In(Array.from(publicKeys.keys()))), userId: exist.id, }); From eefca034fc622db86c2527bed034bb9b64d7ba77 Mon Sep 17 00:00:00 2001 From: tamaina Date: Mon, 4 Mar 2024 02:20:09 +0000 Subject: [PATCH 053/134] =?UTF-8?q?digest=E3=83=97=E3=83=AA=E3=83=AC?= =?UTF-8?q?=E3=83=B3=E3=83=80=E3=82=92=E5=BE=A9=E6=B4=BB=E3=81=95=E3=81=9B?= =?UTF-8?q?=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RFC実装時にどうするか考える --- packages/backend/src/core/QueueService.ts | 3 ++- .../backend/src/core/activitypub/ApRequestService.ts | 9 +++++---- .../src/queue/processors/DeliverProcessorService.ts | 2 +- packages/backend/src/queue/types.ts | 2 ++ 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 7ba1853f84a9..1e96adedfd42 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -15,7 +15,7 @@ import type { Antenna } from '@/server/api/endpoints/i/import-antennas.js'; import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, RelationshipQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js'; import type { DbJobData, DeliverJobData, RelationshipJobData, ThinUser } from '../queue/types.js'; import type * as Bull from 'bullmq'; -import type { ParsedSignature } from '@misskey-dev/node-http-message-signatures'; +import { genRFC3230DigestHeader, type ParsedSignature } from '@misskey-dev/node-http-message-signatures'; @Injectable() export class QueueService { @@ -81,6 +81,7 @@ export class QueueService { id: user.id, }, content: contentBody, + digest: await genRFC3230DigestHeader(contentBody, 'SHA-256'), to, isSharedInbox, }; diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 0e96a80165e1..8469dfbce46c 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -28,7 +28,7 @@ type PrivateKey = { keyId: string; }; -export async function createSignedPost(args: { level: string; key: PrivateKey; url: string; body: string; additionalHeaders: Record }) { +export async function createSignedPost(args: { level: string; key: PrivateKey; url: string; body: string; digest?: string, additionalHeaders: Record }) { const u = new URL(args.url); const request: RequestLike = { url: u.href, @@ -40,9 +40,9 @@ export async function createSignedPost(args: { level: string; key: PrivateKey; u ...args.additionalHeaders, }, }; - +c // TODO: httpMessageSignaturesImplementationLevelによって新規格で通信をするようにする - const digestHeader = await genRFC3230DigestHeader(args.body, 'SHA-256'); + const digestHeader = args.digest ?? await genRFC3230DigestHeader(args.body, 'SHA-256'); request.headers['Digest'] = digestHeader; const result = await signAsDraftToRequest(request, args.key, ['(request-target)', 'date', 'host', 'digest']); @@ -105,7 +105,7 @@ export class ApRequestService { } @bindThis - public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, level: string): Promise { + public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, level: string, digest?: string): Promise { const body = typeof object === 'string' ? object : JSON.stringify(object); const key = await this.getPrivateKey(user.id, level); const req = await createSignedPost({ @@ -116,6 +116,7 @@ export class ApRequestService { additionalHeaders: { 'User-Agent': this.config.userAgent, }, + digest, }); this.logger.debug('create signed post', { diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 6bbe7a4973aa..8f3782a04842 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -76,7 +76,7 @@ export class DeliverProcessorService { await this.fetchInstanceMetadataService.fetchInstanceMetadata(_server).then(() => {}); const server = await this.federatedInstanceService.fetch(host); - await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content, server.httpMessageSignaturesImplementationLevel); + await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content, server.httpMessageSignaturesImplementationLevel, job.data.digest); // Update stats if (server.isNotResponding) { diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index fcae4c259680..135bccb60c0f 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -32,6 +32,8 @@ export type DeliverJobData = { user: ThinUser; /** Activity */ content: string; + /** Digest header */ + digest: string; /** inbox URL to deliver */ to: string; /** whether it is sharedInbox */ From 79249a0514e955d4f6cffebdd03620873e8bcbe7 Mon Sep 17 00:00:00 2001 From: tamaina Date: Mon, 4 Mar 2024 02:26:16 +0000 Subject: [PATCH 054/134] fix, async --- packages/backend/src/core/QueueService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 1e96adedfd42..6177aba32e75 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -70,7 +70,7 @@ export class QueueService { } @bindThis - public deliver(user: ThinUser, content: IActivity | null, to: string | null, isSharedInbox: boolean) { + public async deliver(user: ThinUser, content: IActivity | null, to: string | null, isSharedInbox: boolean) { if (content == null) return null; if (to == null) return null; From 7a334a5e281a696416b707945a42d7a57cb8e512 Mon Sep 17 00:00:00 2001 From: tamaina Date: Mon, 4 Mar 2024 03:12:17 +0000 Subject: [PATCH 055/134] fix --- packages/backend/src/core/activitypub/ApRequestService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 8469dfbce46c..6c58bee211d9 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -40,7 +40,7 @@ export async function createSignedPost(args: { level: string; key: PrivateKey; u ...args.additionalHeaders, }, }; -c + // TODO: httpMessageSignaturesImplementationLevelによって新規格で通信をするようにする const digestHeader = args.digest ?? await genRFC3230DigestHeader(args.body, 'SHA-256'); request.headers['Digest'] = digestHeader; From 821a79ff28492ae1142a064a33e1a314464a1e3b Mon Sep 17 00:00:00 2001 From: tamaina Date: Mon, 4 Mar 2024 09:33:31 +0000 Subject: [PATCH 056/134] !== true --- packages/backend/src/queue/processors/InboxProcessorService.ts | 2 +- packages/backend/src/server/ActivityPubServerService.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 94163a24958e..23e377d7907e 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -114,7 +114,7 @@ export class InboxProcessorService { }); // また、signatureのsignerは、activity.actorと一致する必要がある - if (!httpSignatureValidated || authUser.user.uri !== activity.actor) { + if (httpSignatureValidated !== true || authUser.user.uri !== activity.actor) { // 一致しなくても、でもLD-Signatureがありそうならそっちも見る if (activity.signature?.creator) { if (activity.signature.type !== 'RsaSignature2017') { diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index 1dbb94a2c2b3..78b89e868464 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -103,7 +103,7 @@ export class ActivityPubServerService { let signature: ReturnType; const verifyDigest = await verifyDigestHeader(request.raw, request.rawBody || '', true); - if (!verifyDigest) { + if (verifyDigest !== true) { reply.code(401); return; } From 1af1bc87bd58e7655e72473fd82c21394f52a0a0 Mon Sep 17 00:00:00 2001 From: tamaina Date: Mon, 4 Mar 2024 10:05:45 +0000 Subject: [PATCH 057/134] use save --- .../src/core/activitypub/models/ApPersonService.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 983bf0ebc549..a977c9251d50 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -558,12 +558,11 @@ export class ApPersonService implements OnModuleInit { (person.additionalPublicKeys ?? []).forEach(key => publicKeys.set(key.id, key)); publicKeys.set(person.publicKey.id, person.publicKey); - for (const key of publicKeys.values()) { - await this.userPublickeysRepository.update({ keyId: key.id }, { - userId: exist.id, - keyPem: key.publicKeyPem, - }); - } + await this.userPublickeysRepository.save(Array.from(publicKeys.values(), key => ({ + keyId: key.id, + userId: exist.id, + keyPem: key.publicKeyPem, + }))); } this.userPublickeysRepository.delete({ From 7d77c7044e49e753d2bbecaabd0ff4b832d48231 Mon Sep 17 00:00:00 2001 From: tamaina Date: Mon, 4 Mar 2024 18:47:07 +0000 Subject: [PATCH 058/134] Deliver update person when new key generated (not tested) https://github.com/misskey-dev/misskey/pull/13464#issuecomment-1977049061 --- .../backend/src/core/AccountUpdateService.ts | 13 +++++++--- packages/backend/src/core/QueueService.ts | 7 +++-- packages/backend/src/core/RelayService.ts | 4 +-- .../activitypub/ApDeliverManagerService.ts | 26 +++++++++++++------ .../src/core/activitypub/ApRequestService.ts | 7 ++++- .../processors/DeliverProcessorService.ts | 11 +++++++- packages/backend/src/queue/types.ts | 2 ++ 7 files changed, 53 insertions(+), 17 deletions(-) diff --git a/packages/backend/src/core/AccountUpdateService.ts b/packages/backend/src/core/AccountUpdateService.ts index 69a57b485468..4c89f3a04892 100644 --- a/packages/backend/src/core/AccountUpdateService.ts +++ b/packages/backend/src/core/AccountUpdateService.ts @@ -27,15 +27,22 @@ export class AccountUpdateService { } @bindThis - public async publishToFollowers(userId: MiUser['id']) { + /** + * ユーザーのアップデートをフォロワーに配信する + * @param userId ユーザーID + * @param isKeyUpdation Ed25519キーの作成など公開鍵のアップデートによる呼び出しか？ trueにするとメインキーを使うようになる + */ + public async publishToFollowers(userId: MiUser['id'], isKeyUpdation: boolean = false) { const user = await this.usersRepository.findOneBy({ id: userId }); if (user == null) throw new Error('user not found'); // フォロワーがリモートユーザーかつ投稿者がローカルユーザーならUpdateを配信 if (this.userEntityService.isLocalUser(user)) { const content = this.apRendererService.addContext(this.apRendererService.renderUpdate(await this.apRendererService.renderPerson(user), user)); - this.apDeliverManagerService.deliverToFollowers(user, content); - this.relayService.deliverToRelays(user, content); + await Promise.allSettled([ + this.apDeliverManagerService.deliverToFollowers(user, content, isKeyUpdation), + this.relayService.deliverToRelays(user, content, isKeyUpdation), + ]); } } } diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 6177aba32e75..cb5dca2d39ef 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -70,7 +70,7 @@ export class QueueService { } @bindThis - public async deliver(user: ThinUser, content: IActivity | null, to: string | null, isSharedInbox: boolean) { + public async deliver(user: ThinUser, content: IActivity | null, to: string | null, isSharedInbox: boolean, forceMainKey?: boolean) { if (content == null) return null; if (to == null) return null; @@ -84,6 +84,7 @@ export class QueueService { digest: await genRFC3230DigestHeader(contentBody, 'SHA-256'), to, isSharedInbox, + forceMainKey, }; return this.deliverQueue.add(to, data, { @@ -101,10 +102,11 @@ export class QueueService { * @param user `{ id: string; }` この関数ではThinUserに変換しないので前もって変換してください * @param content IActivity | null * @param inboxes `Map` / key: to (inbox url), value: isSharedInbox (whether it is sharedInbox) + * @param forceMainKey boolean | undefined, force to use main (rsa) key * @returns void */ @bindThis - public async deliverMany(user: ThinUser, content: IActivity | null, inboxes: Map) { + public async deliverMany(user: ThinUser, content: IActivity | null, inboxes: Map, forceMainKey?: boolean) { if (content == null) return null; const contentBody = JSON.stringify(content); @@ -124,6 +126,7 @@ export class QueueService { content: contentBody, to: d[0], isSharedInbox: d[1], + forceMainKey, } as DeliverJobData, opts, }))); diff --git a/packages/backend/src/core/RelayService.ts b/packages/backend/src/core/RelayService.ts index e9dc9b57afef..52bf5d8e6aba 100644 --- a/packages/backend/src/core/RelayService.ts +++ b/packages/backend/src/core/RelayService.ts @@ -111,7 +111,7 @@ export class RelayService { } @bindThis - public async deliverToRelays(user: { id: MiUser['id']; host: null; }, activity: any): Promise { + public async deliverToRelays(user: { id: MiUser['id']; host: null; }, activity: any, forceMainKey?: boolean): Promise { if (activity == null) return; const relays = await this.relaysCache.fetch(() => this.relaysRepository.findBy({ @@ -125,7 +125,7 @@ export class RelayService { const signed = await this.apRendererService.attachLdSignature(copy, user); for (const relay of relays) { - this.queueService.deliver(user, signed, relay.inbox, false); + this.queueService.deliver(user, signed, relay.inbox, false, forceMainKey); } } } diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index cd3b6a8795c7..cf3dce465e73 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -13,6 +13,7 @@ import { bindThis } from '@/decorators.js'; import type { IActivity } from '@/core/activitypub/type.js'; import { ThinUser } from '@/queue/types.js'; import { UserKeypairService } from '../UserKeypairService.js'; +import { AccountUpdateService } from '@/core/AccountUpdateService.js'; interface IRecipe { type: string; @@ -50,6 +51,7 @@ class DeliverManager { private userKeypairService: UserKeypairService, private followingsRepository: FollowingsRepository, private queueService: QueueService, + private accountUpdateService: AccountUpdateService, actor: { id: MiUser['id']; host: null; }, activity: IActivity | null, @@ -104,12 +106,16 @@ class DeliverManager { * Execute delivers */ @bindThis - public async execute(): Promise { + public async execute(opts?: { forceMainKey?: boolean }): Promise { //#region MIGRATION - /** - * ed25519の署名がなければ追加する - */ - await this.userKeypairService.refreshAndprepareEd25519KeyPair(this.actor.id); + if (opts?.forceMainKey !== true) { + /** + * ed25519の署名がなければ追加する + */ + await this.userKeypairService.refreshAndprepareEd25519KeyPair(this.actor.id); + // リモートに配信 + await this.accountUpdateService.publishToFollowers(this.actor.id, true); + } //#endregion // The value flags whether it is shared or not. @@ -163,6 +169,7 @@ export class ApDeliverManagerService { private userKeypairService: UserKeypairService, private queueService: QueueService, + private accountUpdateService: AccountUpdateService, ) { } @@ -170,18 +177,20 @@ export class ApDeliverManagerService { * Deliver activity to followers * @param actor * @param activity Activity + * @param forceMainKey Force to use main (rsa) key */ @bindThis - public async deliverToFollowers(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity): Promise { + public async deliverToFollowers(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity, forceMainKey?: boolean): Promise { const manager = new DeliverManager( this.userKeypairService, this.followingsRepository, this.queueService, + this.accountUpdateService, actor, activity, ); manager.addFollowersRecipe(); - await manager.execute(); + await manager.execute({ forceMainKey }); } /** @@ -196,6 +205,7 @@ export class ApDeliverManagerService { this.userKeypairService, this.followingsRepository, this.queueService, + this.accountUpdateService, actor, activity, ); @@ -209,7 +219,7 @@ export class ApDeliverManagerService { this.userKeypairService, this.followingsRepository, this.queueService, - + this.accountUpdateService, actor, activity, ); diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 6c58bee211d9..132593679ed2 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -91,11 +91,16 @@ export class ApRequestService { this.logger = this.loggerService?.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる } + /** + * Get private key by user id and implementation level + * @param userId User id + * @param level Implementation level + */ @bindThis private async getPrivateKey(userId: MiUser['id'], level: string): Promise { const keypair = await this.userKeypairService.getUserKeypair(userId); - return (level !== '00' && keypair.ed25519PrivateKey) ? { + return (level !== '00' && level !== '10' && keypair.ed25519PrivateKey) ? { privateKeyPem: keypair.ed25519PrivateKey, keyId: `${this.config.url}/users/${userId}#ed25519-key`, } : { diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 8f3782a04842..2a4dd5f9976a 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -76,7 +76,16 @@ export class DeliverProcessorService { await this.fetchInstanceMetadataService.fetchInstanceMetadata(_server).then(() => {}); const server = await this.federatedInstanceService.fetch(host); - await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content, server.httpMessageSignaturesImplementationLevel, job.data.digest); + /** + * RSAキーを強制するかでレベルを変える + */ + const level = job.data.forceMainKey ? + server.httpMessageSignaturesImplementationLevel === '11' ? + '10' : + '00' + : server.httpMessageSignaturesImplementationLevel; + + await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content, level, job.data.digest); // Update stats if (server.isNotResponding) { diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 135bccb60c0f..74e0c0a66eed 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -38,6 +38,8 @@ export type DeliverJobData = { to: string; /** whether it is sharedInbox */ isSharedInbox: boolean; + /** force to use main (rsa) key */ + forceMainKey?: boolean; }; export type InboxJobData = { From 15782f7f47320dd7e64b011f92073a1fa52ae770 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 03:31:13 +0000 Subject: [PATCH 059/134] =?UTF-8?q?=E5=BE=AA=E7=92=B0=E5=8F=82=E7=85=A7?= =?UTF-8?q?=E3=81=A7=E8=90=BD=E3=81=A1=E3=82=8B=E3=81=AE=E3=82=92=E8=A7=A3?= =?UTF-8?q?=E6=B6=88=EF=BC=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/core/AccountUpdateService.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/core/AccountUpdateService.ts b/packages/backend/src/core/AccountUpdateService.ts index 4c89f3a04892..fd790913b456 100644 --- a/packages/backend/src/core/AccountUpdateService.ts +++ b/packages/backend/src/core/AccountUpdateService.ts @@ -3,7 +3,8 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Inject, Injectable } from '@nestjs/common'; +import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; +import { ModuleRef } from '@nestjs/core'; import { DI } from '@/di-symbols.js'; import type { UsersRepository } from '@/models/_.js'; import type { MiUser } from '@/models/User.js'; @@ -14,18 +15,24 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; @Injectable() -export class AccountUpdateService { +export class AccountUpdateService implements OnModuleInit { + private apDeliverManagerService: ApDeliverManagerService; constructor( + private moduleRef: ModuleRef, + @Inject(DI.usersRepository) private usersRepository: UsersRepository, private userEntityService: UserEntityService, private apRendererService: ApRendererService, - private apDeliverManagerService: ApDeliverManagerService, private relayService: RelayService, ) { } + async onModuleInit() { + this.apDeliverManagerService = this.moduleRef.get(ApDeliverManagerService.name); + } + @bindThis /** * ユーザーのアップデートをフォロワーに配信する From 0082f6f8e8c5d5febd14933ba9a1ac643f70ca92 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 05:15:56 +0000 Subject: [PATCH 060/134] fix? --- packages/backend/src/core/AccountUpdateService.ts | 2 ++ packages/backend/src/core/UserKeypairService.ts | 5 +++++ .../src/core/activitypub/ApDeliverManagerService.ts | 8 -------- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/core/AccountUpdateService.ts b/packages/backend/src/core/AccountUpdateService.ts index fd790913b456..5102d0508d1c 100644 --- a/packages/backend/src/core/AccountUpdateService.ts +++ b/packages/backend/src/core/AccountUpdateService.ts @@ -30,6 +30,8 @@ export class AccountUpdateService implements OnModuleInit { } async onModuleInit() { + // Circular dependency + // AccountUpdateService - ApDeliverManagerSevice( - DeliverManager) - UserKeypairService - AccountUpdateService this.apDeliverManagerService = this.moduleRef.get(ApDeliverManagerService.name); } diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index c27f8fc71a41..34199a68f463 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -13,6 +13,7 @@ import type { MiUserKeypair } from '@/models/UserKeypair.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { GlobalEventService, GlobalEvents } from '@/core/GlobalEventService.js'; +import { AccountUpdateService } from '@/core/AccountUpdateService.js'; @Injectable() export class UserKeypairService implements OnApplicationShutdown { @@ -27,6 +28,7 @@ export class UserKeypairService implements OnApplicationShutdown { private userKeypairsRepository: UserKeypairsRepository, private globalEventService: GlobalEventService, + private accountUpdateService: AccountUpdateService, ) { this.cache = new RedisKVCache(this.redisClient, 'userKeypair', { lifetime: 1000 * 60 * 60 * 24, // 24h @@ -54,12 +56,15 @@ export class UserKeypairService implements OnApplicationShutdown { await this.refresh(userId); const keypair = await this.cache.fetch(userId); if (keypair.ed25519PublicKey != null) return; + const ed25519 = await genEd25519KeyPair(); await this.userKeypairsRepository.update({ userId }, { ed25519PublicKey: ed25519.publicKey, ed25519PrivateKey: ed25519.privateKey, }); this.globalEventService.publishInternalEvent('userKeypairUpdated', { userId }); + // リモートに配信 + await this.accountUpdateService.publishToFollowers(userId, true); } @bindThis diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index cf3dce465e73..1ba6416ad493 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -13,7 +13,6 @@ import { bindThis } from '@/decorators.js'; import type { IActivity } from '@/core/activitypub/type.js'; import { ThinUser } from '@/queue/types.js'; import { UserKeypairService } from '../UserKeypairService.js'; -import { AccountUpdateService } from '@/core/AccountUpdateService.js'; interface IRecipe { type: string; @@ -51,7 +50,6 @@ class DeliverManager { private userKeypairService: UserKeypairService, private followingsRepository: FollowingsRepository, private queueService: QueueService, - private accountUpdateService: AccountUpdateService, actor: { id: MiUser['id']; host: null; }, activity: IActivity | null, @@ -113,8 +111,6 @@ class DeliverManager { * ed25519の署名がなければ追加する */ await this.userKeypairService.refreshAndprepareEd25519KeyPair(this.actor.id); - // リモートに配信 - await this.accountUpdateService.publishToFollowers(this.actor.id, true); } //#endregion @@ -169,7 +165,6 @@ export class ApDeliverManagerService { private userKeypairService: UserKeypairService, private queueService: QueueService, - private accountUpdateService: AccountUpdateService, ) { } @@ -185,7 +180,6 @@ export class ApDeliverManagerService { this.userKeypairService, this.followingsRepository, this.queueService, - this.accountUpdateService, actor, activity, ); @@ -205,7 +199,6 @@ export class ApDeliverManagerService { this.userKeypairService, this.followingsRepository, this.queueService, - this.accountUpdateService, actor, activity, ); @@ -219,7 +212,6 @@ export class ApDeliverManagerService { this.userKeypairService, this.followingsRepository, this.queueService, - this.accountUpdateService, actor, activity, ); From 2a622b02dc86add6f84319115b45e4e64df71282 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 05:20:37 +0000 Subject: [PATCH 061/134] Revert "fix?" This reverts commit 0082f6f8e8c5d5febd14933ba9a1ac643f70ca92. --- packages/backend/src/core/AccountUpdateService.ts | 2 -- packages/backend/src/core/UserKeypairService.ts | 5 ----- .../src/core/activitypub/ApDeliverManagerService.ts | 8 ++++++++ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/core/AccountUpdateService.ts b/packages/backend/src/core/AccountUpdateService.ts index 5102d0508d1c..fd790913b456 100644 --- a/packages/backend/src/core/AccountUpdateService.ts +++ b/packages/backend/src/core/AccountUpdateService.ts @@ -30,8 +30,6 @@ export class AccountUpdateService implements OnModuleInit { } async onModuleInit() { - // Circular dependency - // AccountUpdateService - ApDeliverManagerSevice( - DeliverManager) - UserKeypairService - AccountUpdateService this.apDeliverManagerService = this.moduleRef.get(ApDeliverManagerService.name); } diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 34199a68f463..c27f8fc71a41 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -13,7 +13,6 @@ import type { MiUserKeypair } from '@/models/UserKeypair.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { GlobalEventService, GlobalEvents } from '@/core/GlobalEventService.js'; -import { AccountUpdateService } from '@/core/AccountUpdateService.js'; @Injectable() export class UserKeypairService implements OnApplicationShutdown { @@ -28,7 +27,6 @@ export class UserKeypairService implements OnApplicationShutdown { private userKeypairsRepository: UserKeypairsRepository, private globalEventService: GlobalEventService, - private accountUpdateService: AccountUpdateService, ) { this.cache = new RedisKVCache(this.redisClient, 'userKeypair', { lifetime: 1000 * 60 * 60 * 24, // 24h @@ -56,15 +54,12 @@ export class UserKeypairService implements OnApplicationShutdown { await this.refresh(userId); const keypair = await this.cache.fetch(userId); if (keypair.ed25519PublicKey != null) return; - const ed25519 = await genEd25519KeyPair(); await this.userKeypairsRepository.update({ userId }, { ed25519PublicKey: ed25519.publicKey, ed25519PrivateKey: ed25519.privateKey, }); this.globalEventService.publishInternalEvent('userKeypairUpdated', { userId }); - // リモートに配信 - await this.accountUpdateService.publishToFollowers(userId, true); } @bindThis diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index 1ba6416ad493..cf3dce465e73 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -13,6 +13,7 @@ import { bindThis } from '@/decorators.js'; import type { IActivity } from '@/core/activitypub/type.js'; import { ThinUser } from '@/queue/types.js'; import { UserKeypairService } from '../UserKeypairService.js'; +import { AccountUpdateService } from '@/core/AccountUpdateService.js'; interface IRecipe { type: string; @@ -50,6 +51,7 @@ class DeliverManager { private userKeypairService: UserKeypairService, private followingsRepository: FollowingsRepository, private queueService: QueueService, + private accountUpdateService: AccountUpdateService, actor: { id: MiUser['id']; host: null; }, activity: IActivity | null, @@ -111,6 +113,8 @@ class DeliverManager { * ed25519の署名がなければ追加する */ await this.userKeypairService.refreshAndprepareEd25519KeyPair(this.actor.id); + // リモートに配信 + await this.accountUpdateService.publishToFollowers(this.actor.id, true); } //#endregion @@ -165,6 +169,7 @@ export class ApDeliverManagerService { private userKeypairService: UserKeypairService, private queueService: QueueService, + private accountUpdateService: AccountUpdateService, ) { } @@ -180,6 +185,7 @@ export class ApDeliverManagerService { this.userKeypairService, this.followingsRepository, this.queueService, + this.accountUpdateService, actor, activity, ); @@ -199,6 +205,7 @@ export class ApDeliverManagerService { this.userKeypairService, this.followingsRepository, this.queueService, + this.accountUpdateService, actor, activity, ); @@ -212,6 +219,7 @@ export class ApDeliverManagerService { this.userKeypairService, this.followingsRepository, this.queueService, + this.accountUpdateService, actor, activity, ); From 31bf1dbc95953fe481975fcf1edbd4b092be1f18 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 05:22:09 +0000 Subject: [PATCH 062/134] a --- packages/backend/src/core/UserKeypairService.ts | 13 +++++++++++-- .../src/core/activitypub/ApDeliverManagerService.ts | 10 ++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index c27f8fc71a41..2341976068ec 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -49,17 +49,26 @@ export class UserKeypairService implements OnApplicationShutdown { return await this.cache.refresh(userId); } + /** + * + * @param userId user id + * @returns Promise true if keypair is created, false if keypair is already exists + */ @bindThis - public async refreshAndprepareEd25519KeyPair(userId: MiUser['id']): Promise { + public async refreshAndprepareEd25519KeyPair(userId: MiUser['id']): Promise { await this.refresh(userId); const keypair = await this.cache.fetch(userId); - if (keypair.ed25519PublicKey != null) return; + if (keypair.ed25519PublicKey != null) { + return false; + } + const ed25519 = await genEd25519KeyPair(); await this.userKeypairsRepository.update({ userId }, { ed25519PublicKey: ed25519.publicKey, ed25519PrivateKey: ed25519.privateKey, }); this.globalEventService.publishInternalEvent('userKeypairUpdated', { userId }); + return true; } @bindThis diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index cf3dce465e73..e161ac997330 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -12,8 +12,8 @@ import { QueueService } from '@/core/QueueService.js'; import { bindThis } from '@/decorators.js'; import type { IActivity } from '@/core/activitypub/type.js'; import { ThinUser } from '@/queue/types.js'; -import { UserKeypairService } from '../UserKeypairService.js'; import { AccountUpdateService } from '@/core/AccountUpdateService.js'; +import { UserKeypairService } from '../UserKeypairService.js'; interface IRecipe { type: string; @@ -112,9 +112,11 @@ class DeliverManager { /** * ed25519の署名がなければ追加する */ - await this.userKeypairService.refreshAndprepareEd25519KeyPair(this.actor.id); - // リモートに配信 - await this.accountUpdateService.publishToFollowers(this.actor.id, true); + const created = await this.userKeypairService.refreshAndprepareEd25519KeyPair(this.actor.id); + if (created) { + // リモートに配信 + await this.accountUpdateService.publishToFollowers(this.actor.id, true); + } } //#endregion From 4b9ffb8dc0a6c05a2a7c9e7ddb0c5a44df6b83ba Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 05:38:27 +0000 Subject: [PATCH 063/134] logger --- .../backend/src/core/activitypub/ApDeliverManagerService.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index e161ac997330..2046d980d4b7 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -14,6 +14,7 @@ import type { IActivity } from '@/core/activitypub/type.js'; import { ThinUser } from '@/queue/types.js'; import { AccountUpdateService } from '@/core/AccountUpdateService.js'; import { UserKeypairService } from '../UserKeypairService.js'; +import Logger from '@/logger.js'; interface IRecipe { type: string; @@ -23,6 +24,8 @@ interface IFollowersRecipe extends IRecipe { type: 'Followers'; } +const logger = new Logger('deliver-manager', 'azure'); + interface IDirectRecipe extends IRecipe { type: 'Direct'; to: MiRemoteUser; @@ -114,6 +117,7 @@ class DeliverManager { */ const created = await this.userKeypairService.refreshAndprepareEd25519KeyPair(this.actor.id); if (created) { + logger.info(`ed25519 key pair created for user ${this.actor.id} and publishing to followers`); // リモートに配信 await this.accountUpdateService.publishToFollowers(this.actor.id, true); } @@ -160,6 +164,7 @@ class DeliverManager { // deliver await this.queueService.deliverMany(this.actor, this.activity, inboxes); + logger.info(`Deliver queues dispatched to ${inboxes.size} inboxes`); } } From ac4336db43ff6cdc86b9b5a99b6a7dd454ebc1a3 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 05:41:11 +0000 Subject: [PATCH 064/134] log --- .../backend/src/core/activitypub/ApDeliverManagerService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index 2046d980d4b7..cbe17dca4a66 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -164,7 +164,7 @@ class DeliverManager { // deliver await this.queueService.deliverMany(this.actor, this.activity, inboxes); - logger.info(`Deliver queues dispatched to ${inboxes.size} inboxes`); + logger.info(`Deliver queues dispatched: inboxes=${inboxes.size} actorId=${this.actor?.id} activityId=${this.activity?.id}`); } } From 6e4357c3784a9b24b407886d6c855c7c0baefe52 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 05:49:12 +0000 Subject: [PATCH 065/134] change logger --- .../core/activitypub/ApDeliverManagerService.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index cbe17dca4a66..449524c5ffca 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -13,8 +13,9 @@ import { bindThis } from '@/decorators.js'; import type { IActivity } from '@/core/activitypub/type.js'; import { ThinUser } from '@/queue/types.js'; import { AccountUpdateService } from '@/core/AccountUpdateService.js'; +import type Logger from '@/logger.js'; import { UserKeypairService } from '../UserKeypairService.js'; -import Logger from '@/logger.js'; +import { ApLoggerService } from './ApLoggerService.js'; interface IRecipe { type: string; @@ -24,8 +25,6 @@ interface IFollowersRecipe extends IRecipe { type: 'Followers'; } -const logger = new Logger('deliver-manager', 'azure'); - interface IDirectRecipe extends IRecipe { type: 'Direct'; to: MiRemoteUser; @@ -55,6 +54,7 @@ class DeliverManager { private followingsRepository: FollowingsRepository, private queueService: QueueService, private accountUpdateService: AccountUpdateService, + private logger: Logger, actor: { id: MiUser['id']; host: null; }, activity: IActivity | null, @@ -117,7 +117,7 @@ class DeliverManager { */ const created = await this.userKeypairService.refreshAndprepareEd25519KeyPair(this.actor.id); if (created) { - logger.info(`ed25519 key pair created for user ${this.actor.id} and publishing to followers`); + this.logger.info(`ed25519 key pair created for user ${this.actor.id} and publishing to followers`); // リモートに配信 await this.accountUpdateService.publishToFollowers(this.actor.id, true); } @@ -164,12 +164,14 @@ class DeliverManager { // deliver await this.queueService.deliverMany(this.actor, this.activity, inboxes); - logger.info(`Deliver queues dispatched: inboxes=${inboxes.size} actorId=${this.actor?.id} activityId=${this.activity?.id}`); + this.logger.info(`Deliver queues dispatched: inboxes=${inboxes.size} actorId=${this.actor.id} activityId=${this.activity?.id}`); } } @Injectable() export class ApDeliverManagerService { + private logger: Logger; + constructor( @Inject(DI.followingsRepository) private followingsRepository: FollowingsRepository, @@ -177,7 +179,9 @@ export class ApDeliverManagerService { private userKeypairService: UserKeypairService, private queueService: QueueService, private accountUpdateService: AccountUpdateService, + private apLoggerService: ApLoggerService, ) { + this.logger = this.apLoggerService.logger.createSubLogger('deliver-manager'); } /** @@ -193,6 +197,7 @@ export class ApDeliverManagerService { this.followingsRepository, this.queueService, this.accountUpdateService, + this.logger, actor, activity, ); @@ -213,6 +218,7 @@ export class ApDeliverManagerService { this.followingsRepository, this.queueService, this.accountUpdateService, + this.logger, actor, activity, ); @@ -227,6 +233,7 @@ export class ApDeliverManagerService { this.followingsRepository, this.queueService, this.accountUpdateService, + this.logger, actor, activity, ); From 430f0b7911066d9f74ae70d975ba8cb9db98422e Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 11:57:43 +0000 Subject: [PATCH 066/134] =?UTF-8?q?=E7=A7=98=E5=AF=86=E9=8D=B5=E3=81=AE?= =?UTF-8?q?=E5=A4=89=E6=9B=B4=E3=81=AF=E3=80=81=E3=83=95=E3=83=A9=E3=82=B0?= =?UTF-8?q?=E3=81=A7=E3=81=AF=E3=81=AA=E3=81=8F=E9=8D=B5=E3=82=92=E5=BC=95?= =?UTF-8?q?=E3=81=8D=E5=9B=9E=E3=81=99=E3=82=88=E3=81=86=E3=81=AB=E3=81=99?= =?UTF-8?q?=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/src/core/AccountUpdateService.ts | 7 +-- packages/backend/src/core/QueueService.ts | 10 ++--- packages/backend/src/core/RelayService.ts | 5 ++- .../backend/src/core/UserKeypairService.ts | 43 +++++++++++++++++-- .../activitypub/ApDeliverManagerService.ts | 16 ++++--- .../src/core/activitypub/ApRequestService.ts | 27 +++--------- packages/backend/src/core/activitypub/type.ts | 5 +++ .../processors/DeliverProcessorService.ts | 18 ++++---- packages/backend/src/queue/types.ts | 4 +- 9 files changed, 82 insertions(+), 53 deletions(-) diff --git a/packages/backend/src/core/AccountUpdateService.ts b/packages/backend/src/core/AccountUpdateService.ts index fd790913b456..208131ed6e4b 100644 --- a/packages/backend/src/core/AccountUpdateService.ts +++ b/packages/backend/src/core/AccountUpdateService.ts @@ -13,6 +13,7 @@ import { RelayService } from '@/core/RelayService.js'; import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; +import type { PrivateKey } from './activitypub/type.js'; @Injectable() export class AccountUpdateService implements OnModuleInit { @@ -39,7 +40,7 @@ export class AccountUpdateService implements OnModuleInit { * @param userId ユーザーID * @param isKeyUpdation Ed25519キーの作成など公開鍵のアップデートによる呼び出しか？ trueにするとメインキーを使うようになる */ - public async publishToFollowers(userId: MiUser['id'], isKeyUpdation: boolean = false) { + public async publishToFollowers(userId: MiUser['id'], deliverKey?: PrivateKey) { const user = await this.usersRepository.findOneBy({ id: userId }); if (user == null) throw new Error('user not found'); @@ -47,8 +48,8 @@ export class AccountUpdateService implements OnModuleInit { if (this.userEntityService.isLocalUser(user)) { const content = this.apRendererService.addContext(this.apRendererService.renderUpdate(await this.apRendererService.renderPerson(user), user)); await Promise.allSettled([ - this.apDeliverManagerService.deliverToFollowers(user, content, isKeyUpdation), - this.relayService.deliverToRelays(user, content, isKeyUpdation), + this.apDeliverManagerService.deliverToFollowers(user, content, deliverKey), + this.relayService.deliverToRelays(user, content, deliverKey), ]); } } diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index cb5dca2d39ef..b73d0de4d013 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -5,7 +5,7 @@ import { randomUUID } from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; -import type { IActivity } from '@/core/activitypub/type.js'; +import type { IActivity, PrivateKey } from '@/core/activitypub/type.js'; import type { MiDriveFile } from '@/models/DriveFile.js'; import type { MiWebhook, webhookEventTypes } from '@/models/Webhook.js'; import type { Config } from '@/config.js'; @@ -70,7 +70,7 @@ export class QueueService { } @bindThis - public async deliver(user: ThinUser, content: IActivity | null, to: string | null, isSharedInbox: boolean, forceMainKey?: boolean) { + public async deliver(user: ThinUser, content: IActivity | null, to: string | null, isSharedInbox: boolean, privateKey?: PrivateKey) { if (content == null) return null; if (to == null) return null; @@ -84,7 +84,7 @@ export class QueueService { digest: await genRFC3230DigestHeader(contentBody, 'SHA-256'), to, isSharedInbox, - forceMainKey, + privateKey, }; return this.deliverQueue.add(to, data, { @@ -106,7 +106,7 @@ export class QueueService { * @returns void */ @bindThis - public async deliverMany(user: ThinUser, content: IActivity | null, inboxes: Map, forceMainKey?: boolean) { + public async deliverMany(user: ThinUser, content: IActivity | null, inboxes: Map, privateKey?: PrivateKey) { if (content == null) return null; const contentBody = JSON.stringify(content); @@ -126,7 +126,7 @@ export class QueueService { content: contentBody, to: d[0], isSharedInbox: d[1], - forceMainKey, + privateKey, } as DeliverJobData, opts, }))); diff --git a/packages/backend/src/core/RelayService.ts b/packages/backend/src/core/RelayService.ts index 52bf5d8e6aba..c5099a542556 100644 --- a/packages/backend/src/core/RelayService.ts +++ b/packages/backend/src/core/RelayService.ts @@ -16,6 +16,7 @@ import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { DI } from '@/di-symbols.js'; import { deepClone } from '@/misc/clone.js'; import { bindThis } from '@/decorators.js'; +import { PrivateKey } from './activitypub/type.js'; const ACTOR_USERNAME = 'relay.actor' as const; @@ -111,7 +112,7 @@ export class RelayService { } @bindThis - public async deliverToRelays(user: { id: MiUser['id']; host: null; }, activity: any, forceMainKey?: boolean): Promise { + public async deliverToRelays(user: { id: MiUser['id']; host: null; }, activity: any, privateKey?: PrivateKey): Promise { if (activity == null) return; const relays = await this.relaysCache.fetch(() => this.relaysRepository.findBy({ @@ -125,7 +126,7 @@ export class RelayService { const signed = await this.apRendererService.attachLdSignature(copy, user); for (const relay of relays) { - this.queueService.deliver(user, signed, relay.inbox, false, forceMainKey); + this.queueService.deliver(user, signed, relay.inbox, false, privateKey); } } } diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 2341976068ec..26190024cdf5 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -13,6 +13,7 @@ import type { MiUserKeypair } from '@/models/UserKeypair.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { GlobalEventService, GlobalEvents } from '@/core/GlobalEventService.js'; +import { UserEntityService } from '@/core/entities/UserEntityService.js'; @Injectable() export class UserKeypairService implements OnApplicationShutdown { @@ -27,6 +28,7 @@ export class UserKeypairService implements OnApplicationShutdown { private userKeypairsRepository: UserKeypairsRepository, private globalEventService: GlobalEventService, + private userEntityService: UserEntityService, ) { this.cache = new RedisKVCache(this.redisClient, 'userKeypair', { lifetime: 1000 * 60 * 60 * 24, // 24h @@ -44,6 +46,35 @@ export class UserKeypairService implements OnApplicationShutdown { return await this.cache.fetch(userId); } + /** + * + * @param userIdOrHint user id or MiUserKeypair + * @param preferType 'main' or 'ed25519'; If 'ed25519' is specified and ed25519 keypair is not exists, it will return main keypair + * @returns + */ + @bindThis + public async getLocalUserKeypairWithKeyId( + userIdOrHint: MiUser['id'] | MiUserKeypair, preferType: 'main' | 'ed25519' + ): Promise<{ keyId: string; publicKey: string; privateKey: string; }> { + const keypair = typeof userIdOrHint === 'string' ? await this.getUserKeypair(userIdOrHint) : userIdOrHint; + if (preferType === 'ed25519' && keypair.ed25519PublicKey != null && keypair.ed25519PrivateKey != null) { + return { + keyId: `${this.userEntityService.genLocalUserUri(keypair.userId)}#ed25519-key`, + publicKey: keypair.ed25519PublicKey, + privateKey: keypair.ed25519PrivateKey, + }; + } + if (preferType === 'main') { + return { + keyId: `${this.userEntityService.genLocalUserUri(keypair.userId)}#main-key`, + publicKey: keypair.publicKey, + privateKey: keypair.privateKey, + }; + } + + throw new Error('invalid type'); + } + @bindThis public async refresh(userId: MiUser['id']): Promise { return await this.cache.refresh(userId); @@ -52,14 +83,14 @@ export class UserKeypairService implements OnApplicationShutdown { /** * * @param userId user id - * @returns Promise true if keypair is created, false if keypair is already exists + * @returns MiUserKeypair if keypair is created, void if keypair is already exists */ @bindThis - public async refreshAndprepareEd25519KeyPair(userId: MiUser['id']): Promise { + public async refreshAndprepareEd25519KeyPair(userId: MiUser['id']): Promise { await this.refresh(userId); const keypair = await this.cache.fetch(userId); if (keypair.ed25519PublicKey != null) { - return false; + return; } const ed25519 = await genEd25519KeyPair(); @@ -68,7 +99,11 @@ export class UserKeypairService implements OnApplicationShutdown { ed25519PrivateKey: ed25519.privateKey, }); this.globalEventService.publishInternalEvent('userKeypairUpdated', { userId }); - return true; + return { + ...keypair, + ed25519PublicKey: ed25519.publicKey, + ed25519PrivateKey: ed25519.privateKey, + }; } @bindThis diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index 449524c5ffca..7f22f167af5d 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -10,7 +10,7 @@ import type { FollowingsRepository } from '@/models/_.js'; import type { MiLocalUser, MiRemoteUser, MiUser } from '@/models/User.js'; import { QueueService } from '@/core/QueueService.js'; import { bindThis } from '@/decorators.js'; -import type { IActivity } from '@/core/activitypub/type.js'; +import type { IActivity, PrivateKey } from '@/core/activitypub/type.js'; import { ThinUser } from '@/queue/types.js'; import { AccountUpdateService } from '@/core/AccountUpdateService.js'; import type Logger from '@/logger.js'; @@ -109,17 +109,19 @@ class DeliverManager { * Execute delivers */ @bindThis - public async execute(opts?: { forceMainKey?: boolean }): Promise { + public async execute(opts?: { privateKey?: PrivateKey }): Promise { //#region MIGRATION - if (opts?.forceMainKey !== true) { + if (!opts?.privateKey) { /** * ed25519の署名がなければ追加する */ const created = await this.userKeypairService.refreshAndprepareEd25519KeyPair(this.actor.id); if (created) { + // createdが存在するということは新規作成されたということなので、フォロワーに配信する this.logger.info(`ed25519 key pair created for user ${this.actor.id} and publishing to followers`); // リモートに配信 - await this.accountUpdateService.publishToFollowers(this.actor.id, true); + const keyPair = await this.userKeypairService.getLocalUserKeypairWithKeyId(created, 'ed25519'); + await this.accountUpdateService.publishToFollowers(this.actor.id, { keyId: keyPair.keyId, privateKeyPem: keyPair.privateKey }); } } //#endregion @@ -163,7 +165,7 @@ class DeliverManager { } // deliver - await this.queueService.deliverMany(this.actor, this.activity, inboxes); + await this.queueService.deliverMany(this.actor, this.activity, inboxes, opts?.privateKey); this.logger.info(`Deliver queues dispatched: inboxes=${inboxes.size} actorId=${this.actor.id} activityId=${this.activity?.id}`); } } @@ -191,7 +193,7 @@ export class ApDeliverManagerService { * @param forceMainKey Force to use main (rsa) key */ @bindThis - public async deliverToFollowers(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity, forceMainKey?: boolean): Promise { + public async deliverToFollowers(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity, privateKey?: PrivateKey): Promise { const manager = new DeliverManager( this.userKeypairService, this.followingsRepository, @@ -202,7 +204,7 @@ export class ApDeliverManagerService { activity, ); manager.addFollowersRecipe(); - await manager.execute({ forceMainKey }); + await manager.execute({ privateKey }); } /** diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 132593679ed2..a6b905370624 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -15,18 +15,7 @@ import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import type Logger from '@/logger.js'; import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js'; - -type Signed = { - request: Request; - signingString: string; - signature: string; - signatureHeader: string; -}; - -type PrivateKey = { - privateKeyPem: string; - keyId: string; -}; +import type { PrivateKey } from './type.js'; export async function createSignedPost(args: { level: string; key: PrivateKey; url: string; body: string; digest?: string, additionalHeaders: Record }) { const u = new URL(args.url); @@ -98,21 +87,19 @@ export class ApRequestService { */ @bindThis private async getPrivateKey(userId: MiUser['id'], level: string): Promise { - const keypair = await this.userKeypairService.getUserKeypair(userId); + const type = level === '00' || level === '10' ? 'ed25519' : 'main'; + const keypair = await this.userKeypairService.getLocalUserKeypairWithKeyId(userId, type); - return (level !== '00' && level !== '10' && keypair.ed25519PrivateKey) ? { - privateKeyPem: keypair.ed25519PrivateKey, - keyId: `${this.config.url}/users/${userId}#ed25519-key`, - } : { + return { + keyId: keypair.keyId, privateKeyPem: keypair.privateKey, - keyId: `${this.config.url}/users/${userId}#main-key`, }; } @bindThis - public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, level: string, digest?: string): Promise { + public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, level: string, digest?: string, key?: PrivateKey): Promise { const body = typeof object === 'string' ? object : JSON.stringify(object); - const key = await this.getPrivateKey(user.id, level); + key = key ?? await this.getPrivateKey(user.id, level); const req = await createSignedPost({ level, key, diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 0b34fd766447..54d1c52dc090 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -326,3 +326,8 @@ export const isAnnounce = (object: IObject): object is IAnnounce => getApType(ob export const isBlock = (object: IObject): object is IBlock => getApType(object) === 'Block'; export const isFlag = (object: IObject): object is IFlag => getApType(object) === 'Flag'; export const isMove = (object: IObject): object is IMove => getApType(object) === 'Move'; + +export type PrivateKey = { + privateKeyPem: string; + keyId: string; +}; diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 2a4dd5f9976a..47a69e762213 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -76,16 +76,14 @@ export class DeliverProcessorService { await this.fetchInstanceMetadataService.fetchInstanceMetadata(_server).then(() => {}); const server = await this.federatedInstanceService.fetch(host); - /** - * RSAキーを強制するかでレベルを変える - */ - const level = job.data.forceMainKey ? - server.httpMessageSignaturesImplementationLevel === '11' ? - '10' : - '00' - : server.httpMessageSignaturesImplementationLevel; - - await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content, level, job.data.digest); + await this.apRequestService.signedPost( + job.data.user, + job.data.to, + job.data.content, + server.httpMessageSignaturesImplementationLevel, + job.data.digest, + job.data.privateKey, + ); // Update stats if (server.isNotResponding) { diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 74e0c0a66eed..12e313627023 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -8,7 +8,7 @@ import type { MiDriveFile } from '@/models/DriveFile.js'; import type { MiNote } from '@/models/Note.js'; import type { MiUser } from '@/models/User.js'; import type { MiWebhook } from '@/models/Webhook.js'; -import type { IActivity } from '@/core/activitypub/type.js'; +import type { IActivity, PrivateKey } from '@/core/activitypub/type.js'; import type { ParsedSignature } from '@misskey-dev/node-http-message-signatures'; /** @@ -39,7 +39,7 @@ export type DeliverJobData = { /** whether it is sharedInbox */ isSharedInbox: boolean; /** force to use main (rsa) key */ - forceMainKey?: boolean; + privateKey?: PrivateKey; }; export type InboxJobData = { From e4fea42436e82f3e08411bac253f8c3fe8d5f27c Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 12:30:22 +0000 Subject: [PATCH 067/134] addAllKnowingSharedInboxRecipe --- .../backend/src/core/UserKeypairService.ts | 23 ++++--- .../backend/src/core/UserSuspendService.ts | 66 +++++-------------- .../activitypub/ApDeliverManagerService.ts | 39 ++++++++++- .../src/core/activitypub/ApRequestService.ts | 3 +- 4 files changed, 65 insertions(+), 66 deletions(-) diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 26190024cdf5..6ec9a8b333bb 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -49,30 +49,29 @@ export class UserKeypairService implements OnApplicationShutdown { /** * * @param userIdOrHint user id or MiUserKeypair - * @param preferType 'main' or 'ed25519'; If 'ed25519' is specified and ed25519 keypair is not exists, it will return main keypair + * @param preferType If ed25519-like(`ed25519`, `01`, `11`) is specified, ed25519 keypair is returned if exists. Otherwise, main keypair is returned. * @returns */ @bindThis public async getLocalUserKeypairWithKeyId( - userIdOrHint: MiUser['id'] | MiUserKeypair, preferType: 'main' | 'ed25519' + userIdOrHint: MiUser['id'] | MiUserKeypair, preferType?: string, ): Promise<{ keyId: string; publicKey: string; privateKey: string; }> { const keypair = typeof userIdOrHint === 'string' ? await this.getUserKeypair(userIdOrHint) : userIdOrHint; - if (preferType === 'ed25519' && keypair.ed25519PublicKey != null && keypair.ed25519PrivateKey != null) { + if ( + preferType && ['01', '11', 'ed25519'].includes(preferType.toLowerCase()) && + keypair.ed25519PublicKey != null && keypair.ed25519PrivateKey != null + ) { return { keyId: `${this.userEntityService.genLocalUserUri(keypair.userId)}#ed25519-key`, publicKey: keypair.ed25519PublicKey, privateKey: keypair.ed25519PrivateKey, }; } - if (preferType === 'main') { - return { - keyId: `${this.userEntityService.genLocalUserUri(keypair.userId)}#main-key`, - publicKey: keypair.publicKey, - privateKey: keypair.privateKey, - }; - } - - throw new Error('invalid type'); + return { + keyId: `${this.userEntityService.genLocalUserUri(keypair.userId)}#main-key`, + publicKey: keypair.publicKey, + privateKey: keypair.privateKey, + }; } @bindThis diff --git a/packages/backend/src/core/UserSuspendService.ts b/packages/backend/src/core/UserSuspendService.ts index d594a223f4e2..af42e4405b08 100644 --- a/packages/backend/src/core/UserSuspendService.ts +++ b/packages/backend/src/core/UserSuspendService.ts @@ -3,27 +3,23 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Inject, Injectable } from '@nestjs/common'; -import { Not, IsNull } from 'typeorm'; -import type { FollowingsRepository } from '@/models/_.js'; +import { Injectable } from '@nestjs/common'; import type { MiUser } from '@/models/User.js'; -import { QueueService } from '@/core/QueueService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; -import { DI } from '@/di-symbols.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; +import { UserKeypairService } from './UserKeypairService.js'; +import { ApDeliverManagerService } from './activitypub/ApDeliverManagerService.js'; @Injectable() export class UserSuspendService { constructor( - @Inject(DI.followingsRepository) - private followingsRepository: FollowingsRepository, - private userEntityService: UserEntityService, - private queueService: QueueService, private globalEventService: GlobalEventService, private apRendererService: ApRendererService, + private userKeypairService: UserKeypairService, + private apDeliverManagerService: ApDeliverManagerService, ) { } @@ -32,28 +28,12 @@ export class UserSuspendService { this.globalEventService.publishInternalEvent('userChangeSuspendedState', { id: user.id, isSuspended: true }); if (this.userEntityService.isLocalUser(user)) { - // 知り得る全SharedInboxにDelete配信 const content = this.apRendererService.addContext(this.apRendererService.renderDelete(this.userEntityService.genLocalUserUri(user.id), user)); - - const queue: string[] = []; - - const followings = await this.followingsRepository.find({ - where: [ - { followerSharedInbox: Not(IsNull()) }, - { followeeSharedInbox: Not(IsNull()) }, - ], - select: ['followerSharedInbox', 'followeeSharedInbox'], - }); - - const inboxes = followings.map(x => x.followerSharedInbox ?? x.followeeSharedInbox); - - for (const inbox of inboxes) { - if (inbox != null && !queue.includes(inbox)) queue.push(inbox); - } - - for (const inbox of queue) { - this.queueService.deliver(user, content, inbox, true); - } + const manager = this.apDeliverManagerService.createDeliverManager(user, content); + manager.addAllKnowingSharedInboxRecipe(); + // process delivre時にはキーペアが消去されているはずなので、ここで挿入する + const keypairs = await this.userKeypairService.getLocalUserKeypairWithKeyId(user.id, 'main'); + manager.execute({ privateKey: { keyId: keypairs.keyId, privateKeyPem: keypairs.privateKey } }); } } @@ -62,28 +42,12 @@ export class UserSuspendService { this.globalEventService.publishInternalEvent('userChangeSuspendedState', { id: user.id, isSuspended: false }); if (this.userEntityService.isLocalUser(user)) { - // 知り得る全SharedInboxにUndo Delete配信 const content = this.apRendererService.addContext(this.apRendererService.renderUndo(this.apRendererService.renderDelete(this.userEntityService.genLocalUserUri(user.id), user), user)); - - const queue: string[] = []; - - const followings = await this.followingsRepository.find({ - where: [ - { followerSharedInbox: Not(IsNull()) }, - { followeeSharedInbox: Not(IsNull()) }, - ], - select: ['followerSharedInbox', 'followeeSharedInbox'], - }); - - const inboxes = followings.map(x => x.followerSharedInbox ?? x.followeeSharedInbox); - - for (const inbox of inboxes) { - if (inbox != null && !queue.includes(inbox)) queue.push(inbox); - } - - for (const inbox of queue) { - this.queueService.deliver(user as any, content, inbox, true); - } + const manager = this.apDeliverManagerService.createDeliverManager(user, content); + manager.addAllKnowingSharedInboxRecipe(); + // process delivre時にはキーペアが消去されているはずなので、ここで挿入する + const keypairs = await this.userKeypairService.getLocalUserKeypairWithKeyId(user.id, 'main'); + manager.execute({ privateKey: { keyId: keypairs.keyId, privateKeyPem: keypairs.privateKey } }); } } } diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index 7f22f167af5d..1c533ef7c3f4 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -30,12 +30,19 @@ interface IDirectRecipe extends IRecipe { to: MiRemoteUser; } +interface IAllKnowingSharedInboxRecipe extends IRecipe { + type: 'AllKnowingSharedInbox'; +} + const isFollowers = (recipe: IRecipe): recipe is IFollowersRecipe => recipe.type === 'Followers'; const isDirect = (recipe: IRecipe): recipe is IDirectRecipe => recipe.type === 'Direct'; +const isAllKnowingSharedInbox = (recipe: IRecipe): recipe is IAllKnowingSharedInboxRecipe => + recipe.type === 'AllKnowingSharedInbox'; + class DeliverManager { private actor: ThinUser; private activity: IActivity | null; @@ -96,6 +103,18 @@ class DeliverManager { this.addRecipe(recipe); } + /** + * Add recipe for all-knowing shared inbox deliver + */ + @bindThis + public addAllKnowingSharedInboxRecipe(): void { + const deliver: IAllKnowingSharedInboxRecipe = { + type: 'AllKnowingSharedInbox', + }; + + this.addRecipe(deliver); + } + /** * Add recipe * @param recipe Recipe @@ -120,16 +139,33 @@ class DeliverManager { // createdが存在するということは新規作成されたということなので、フォロワーに配信する this.logger.info(`ed25519 key pair created for user ${this.actor.id} and publishing to followers`); // リモートに配信 - const keyPair = await this.userKeypairService.getLocalUserKeypairWithKeyId(created, 'ed25519'); + const keyPair = await this.userKeypairService.getLocalUserKeypairWithKeyId(created, 'main'); await this.accountUpdateService.publishToFollowers(this.actor.id, { keyId: keyPair.keyId, privateKeyPem: keyPair.privateKey }); } } //#endregion + //#region correct inboxes by recipes // The value flags whether it is shared or not. // key: inbox URL, value: whether it is sharedInbox const inboxes = new Map(); + if (this.recipes.some(r => isAllKnowingSharedInbox(r))) { + // all-knowing shared inbox + const followings = await this.followingsRepository.find({ + where: [ + { followerSharedInbox: Not(IsNull()) }, + { followeeSharedInbox: Not(IsNull()) }, + ], + select: ['followerSharedInbox', 'followeeSharedInbox'], + }); + + for (const following of followings) { + if (following.followeeSharedInbox) inboxes.set(following.followeeSharedInbox, true); + if (following.followerSharedInbox) inboxes.set(following.followerSharedInbox, true); + } + } + // build inbox list // Process follower recipes first to avoid duplication when processing direct recipes later. if (this.recipes.some(r => isFollowers(r))) { @@ -163,6 +199,7 @@ class DeliverManager { inboxes.set(recipe.to.inbox, false); } + //#endregion // deliver await this.queueService.deliverMany(this.actor, this.activity, inboxes, opts?.privateKey); diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index a6b905370624..6c0df36d7e3e 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -87,8 +87,7 @@ export class ApRequestService { */ @bindThis private async getPrivateKey(userId: MiUser['id'], level: string): Promise { - const type = level === '00' || level === '10' ? 'ed25519' : 'main'; - const keypair = await this.userKeypairService.getLocalUserKeypairWithKeyId(userId, type); + const keypair = await this.userKeypairService.getLocalUserKeypairWithKeyId(userId, level); return { keyId: keypair.keyId, From 021801c721bb4406460a10fb40c6822b6a23e1dd Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 12:55:30 +0000 Subject: [PATCH 068/134] nanka meccha kaeta --- packages/backend/src/core/QueueService.ts | 4 +-- packages/backend/src/core/RelayService.ts | 12 +++---- .../backend/src/core/UserSuspendService.ts | 8 ++--- .../activitypub/ApDeliverManagerService.ts | 2 +- .../src/core/activitypub/ApRendererService.ts | 8 ++--- .../src/core/activitypub/ApRequestService.ts | 31 +++++++------------ packages/backend/src/core/activitypub/type.ts | 2 +- packages/backend/test/unit/ap-request.ts | 4 +-- 8 files changed, 31 insertions(+), 40 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index b73d0de4d013..afea871575a1 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -84,7 +84,7 @@ export class QueueService { digest: await genRFC3230DigestHeader(contentBody, 'SHA-256'), to, isSharedInbox, - privateKey, + privateKey: privateKey && { keyId: privateKey.keyId, privateKey: privateKey.privateKey }, }; return this.deliverQueue.add(to, data, { @@ -126,7 +126,7 @@ export class QueueService { content: contentBody, to: d[0], isSharedInbox: d[1], - privateKey, + privateKey: privateKey && { keyId: privateKey.keyId, privateKey: privateKey.privateKey }, } as DeliverJobData, opts, }))); diff --git a/packages/backend/src/core/RelayService.ts b/packages/backend/src/core/RelayService.ts index c5099a542556..f106ae1669b9 100644 --- a/packages/backend/src/core/RelayService.ts +++ b/packages/backend/src/core/RelayService.ts @@ -16,7 +16,8 @@ import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { DI } from '@/di-symbols.js'; import { deepClone } from '@/misc/clone.js'; import { bindThis } from '@/decorators.js'; -import { PrivateKey } from './activitypub/type.js'; +import { UserKeypairService } from './UserKeypairService.js'; +import type { PrivateKey } from './activitypub/type.js'; const ACTOR_USERNAME = 'relay.actor' as const; @@ -35,6 +36,7 @@ export class RelayService { private queueService: QueueService, private createSystemUserService: CreateSystemUserService, private apRendererService: ApRendererService, + private userKeypairService: UserKeypairService, ) { this.relaysCache = new MemorySingleCache(1000 * 60 * 10); } @@ -122,11 +124,9 @@ export class RelayService { const copy = deepClone(activity); if (!copy.to) copy.to = ['https://www.w3.org/ns/activitystreams#Public']; + privateKey = privateKey ?? await this.userKeypairService.getLocalUserKeypairWithKeyId(user.id); + const signed = await this.apRendererService.attachLdSignature(copy, user, privateKey); - const signed = await this.apRendererService.attachLdSignature(copy, user); - - for (const relay of relays) { - this.queueService.deliver(user, signed, relay.inbox, false, privateKey); - } + this.queueService.deliverMany(user, signed, new Map(relays.map(({ inbox }) => [inbox, false])), privateKey); } } diff --git a/packages/backend/src/core/UserSuspendService.ts b/packages/backend/src/core/UserSuspendService.ts index af42e4405b08..042e13a54064 100644 --- a/packages/backend/src/core/UserSuspendService.ts +++ b/packages/backend/src/core/UserSuspendService.ts @@ -32,8 +32,8 @@ export class UserSuspendService { const manager = this.apDeliverManagerService.createDeliverManager(user, content); manager.addAllKnowingSharedInboxRecipe(); // process delivre時にはキーペアが消去されているはずなので、ここで挿入する - const keypairs = await this.userKeypairService.getLocalUserKeypairWithKeyId(user.id, 'main'); - manager.execute({ privateKey: { keyId: keypairs.keyId, privateKeyPem: keypairs.privateKey } }); + const privateKey = await this.userKeypairService.getLocalUserKeypairWithKeyId(user.id, 'main'); + manager.execute({ privateKey }); } } @@ -46,8 +46,8 @@ export class UserSuspendService { const manager = this.apDeliverManagerService.createDeliverManager(user, content); manager.addAllKnowingSharedInboxRecipe(); // process delivre時にはキーペアが消去されているはずなので、ここで挿入する - const keypairs = await this.userKeypairService.getLocalUserKeypairWithKeyId(user.id, 'main'); - manager.execute({ privateKey: { keyId: keypairs.keyId, privateKeyPem: keypairs.privateKey } }); + const privateKey = await this.userKeypairService.getLocalUserKeypairWithKeyId(user.id, 'main'); + manager.execute({ privateKey }); } } } diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index 1c533ef7c3f4..4b0dd1403d65 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -140,7 +140,7 @@ class DeliverManager { this.logger.info(`ed25519 key pair created for user ${this.actor.id} and publishing to followers`); // リモートに配信 const keyPair = await this.userKeypairService.getLocalUserKeypairWithKeyId(created, 'main'); - await this.accountUpdateService.publishToFollowers(this.actor.id, { keyId: keyPair.keyId, privateKeyPem: keyPair.privateKey }); + await this.accountUpdateService.publishToFollowers(this.actor.id, keyPair); } } //#endregion diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index d929fa4ee1cd..e4b3074abd72 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -30,7 +30,7 @@ import { isNotNull } from '@/misc/is-not-null.js'; import { IdService } from '@/core/IdService.js'; import { LdSignatureService } from './LdSignatureService.js'; import { ApMfmService } from './ApMfmService.js'; -import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate } from './type.js'; +import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate, PrivateKey } from './type.js'; @Injectable() export class ApRendererService { @@ -657,12 +657,10 @@ export class ApRendererService { } @bindThis - public async attachLdSignature(activity: any, user: { id: MiUser['id']; host: null; }): Promise { - const keypair = await this.userKeypairService.getUserKeypair(user.id); - + public async attachLdSignature(activity: any, user: { id: MiUser['id']; host: null; }, key: PrivateKey): Promise { const ldSignature = this.ldSignatureService.use(); ldSignature.debug = false; - activity = await ldSignature.signRsaSignature2017(activity, keypair.privateKey, `${this.config.url}/users/${user.id}#main-key`); + activity = await ldSignature.signRsaSignature2017(activity, key.privateKey, key.keyId); return activity; } diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 6c0df36d7e3e..c9bcbd961af5 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -34,7 +34,11 @@ export async function createSignedPost(args: { level: string; key: PrivateKey; u const digestHeader = args.digest ?? await genRFC3230DigestHeader(args.body, 'SHA-256'); request.headers['Digest'] = digestHeader; - const result = await signAsDraftToRequest(request, args.key, ['(request-target)', 'date', 'host', 'digest']); + const result = await signAsDraftToRequest( + request, + { keyId: args.key.keyId, privateKeyPem: args.key.privateKey }, + ['(request-target)', 'date', 'host', 'digest'], + ); return { request, @@ -56,7 +60,11 @@ export async function createSignedGet(args: { level: string; key: PrivateKey; ur }; // TODO: httpMessageSignaturesImplementationLevelによって新規格で通信をするようにする - const result = await signAsDraftToRequest(request, args.key, ['(request-target)', 'date', 'host', 'accept']); + const result = await signAsDraftToRequest( + request, + { keyId: args.key.keyId, privateKeyPem: args.key.privateKey }, + ['(request-target)', 'date', 'host', 'accept'], + ); return { request, @@ -80,25 +88,10 @@ export class ApRequestService { this.logger = this.loggerService?.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる } - /** - * Get private key by user id and implementation level - * @param userId User id - * @param level Implementation level - */ - @bindThis - private async getPrivateKey(userId: MiUser['id'], level: string): Promise { - const keypair = await this.userKeypairService.getLocalUserKeypairWithKeyId(userId, level); - - return { - keyId: keypair.keyId, - privateKeyPem: keypair.privateKey, - }; - } - @bindThis public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, level: string, digest?: string, key?: PrivateKey): Promise { const body = typeof object === 'string' ? object : JSON.stringify(object); - key = key ?? await this.getPrivateKey(user.id, level); + key = key ?? await this.userKeypairService.getLocalUserKeypairWithKeyId(user.id, level); const req = await createSignedPost({ level, key, @@ -131,7 +124,7 @@ export class ApRequestService { */ @bindThis public async signedGet(url: string, user: { id: MiUser['id'] }, level: string): Promise { - const key = await this.getPrivateKey(user.id, level); + const key = await this.userKeypairService.getLocalUserKeypairWithKeyId(user.id, level); const req = await createSignedGet({ level, key, diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 54d1c52dc090..6fd5ada9fe15 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -328,6 +328,6 @@ export const isFlag = (object: IObject): object is IFlag => getApType(object) == export const isMove = (object: IObject): object is IMove => getApType(object) === 'Move'; export type PrivateKey = { - privateKeyPem: string; + privateKey: string; keyId: string; }; diff --git a/packages/backend/test/unit/ap-request.ts b/packages/backend/test/unit/ap-request.ts index 797ef73e62a5..1c0159ed9585 100644 --- a/packages/backend/test/unit/ap-request.ts +++ b/packages/backend/test/unit/ap-request.ts @@ -35,7 +35,7 @@ describe('ap-request', () => { describe.each(['00', '01'])('createSignedPost with verify', (level) => { test('pass', async () => { const keypair = await getKeyPair(level); - const key = { keyId: 'x', 'privateKeyPem': keypair.privateKey }; + const key = { keyId: 'x', 'privateKey': keypair.privateKey }; const url = 'https://example.com/inbox'; const activity = { a: 1 }; const body = JSON.stringify(activity); @@ -56,7 +56,7 @@ describe('ap-request', () => { describe.each(['00', '01'])('createSignedGet with verify', (level) => { test('pass', async () => { const keypair = await getKeyPair(level); - const key = { keyId: 'x', 'privateKeyPem': keypair.privateKey }; + const key = { keyId: 'x', 'privateKey': keypair.privateKey }; const url = 'https://example.com/outbox'; const headers = { 'User-Agent': 'UA', From 6b02efac32b4068f67196705bd076d9f3e6d8835 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 13:18:00 +0000 Subject: [PATCH 069/134] delivre --- packages/backend/src/core/UserSuspendService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/UserSuspendService.ts b/packages/backend/src/core/UserSuspendService.ts index 042e13a54064..7abeb3d46db8 100644 --- a/packages/backend/src/core/UserSuspendService.ts +++ b/packages/backend/src/core/UserSuspendService.ts @@ -31,7 +31,7 @@ export class UserSuspendService { const content = this.apRendererService.addContext(this.apRendererService.renderDelete(this.userEntityService.genLocalUserUri(user.id), user)); const manager = this.apDeliverManagerService.createDeliverManager(user, content); manager.addAllKnowingSharedInboxRecipe(); - // process delivre時にはキーペアが消去されているはずなので、ここで挿入する + // process deliver時にはキーペアが消去されているはずなので、ここで挿入する const privateKey = await this.userKeypairService.getLocalUserKeypairWithKeyId(user.id, 'main'); manager.execute({ privateKey }); } @@ -45,7 +45,7 @@ export class UserSuspendService { const content = this.apRendererService.addContext(this.apRendererService.renderUndo(this.apRendererService.renderDelete(this.userEntityService.genLocalUserUri(user.id), user), user)); const manager = this.apDeliverManagerService.createDeliverManager(user, content); manager.addAllKnowingSharedInboxRecipe(); - // process delivre時にはキーペアが消去されているはずなので、ここで挿入する + // process deliver時にはキーペアが消去されているはずなので、ここで挿入する const privateKey = await this.userKeypairService.getLocalUserKeypairWithKeyId(user.id, 'main'); manager.execute({ privateKey }); } From 0e509c440e7ede74639333fc4a36dde91732da67 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 13:58:29 +0000 Subject: [PATCH 070/134] =?UTF-8?q?=E3=82=AD=E3=83=A3=E3=83=83=E3=82=B7?= =?UTF-8?q?=E3=83=A5=E6=9C=89=E5=8A=B9=E3=83=81=E3=82=A7=E3=83=83=E3=82=AF?= =?UTF-8?q?=E3=81=AF=E3=83=AD=E3=83=83=E3=82=AF=E5=8F=96=E5=BE=97=E5=89=8D?= =?UTF-8?q?=E3=81=AB=E8=A1=8C=E3=81=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/core/FetchInstanceMetadataService.ts | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index e8d9cec80455..9d1ce3ac5dfd 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -63,7 +63,7 @@ export class FetchInstanceMetadataService { return await this.redisClient.set( `fetchInstanceMetadata:mutex:v2:${host}`, '1', 'EX', 30, // 30秒したら自動でロック解除 https://github.com/misskey-dev/misskey/issues/13506#issuecomment-1975375395 - 'GET' // 古い値を返す（なかったらnull） + 'GET', // 古い値を返す（なかったらnull） ); } @@ -77,22 +77,24 @@ export class FetchInstanceMetadataService { public async fetchInstanceMetadata(instance: MiInstance, force = false): Promise { const host = instance.host; - // finallyでunlockされてしまうのでtry内でロックチェックをしない - // （returnであってもfinallyは実行される） - if (!force && await this.tryLock(host) === '1') { - // 1が返ってきていたらロックされているという意味なので、何もしない - return; - } + if (!force) { + // キャッシュ有効チェックはロック取得前に行う + const _instance = await this.federatedInstanceService.fetch(host); + const now = Date.now(); + if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < REMOTE_SERVER_CACHE_TTL)) { + this.logger.debug(`Skip because updated recently ${_instance.infoUpdatedAt.toJSON()}`); + return; + } - try { - if (!force) { - const _instance = await this.federatedInstanceService.fetch(host); - const now = Date.now(); - if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < REMOTE_SERVER_CACHE_TTL)) { - throw new Error(`Skip because updated recently ${_instance.infoUpdatedAt.toJSON()}`); - } + // finallyでunlockされてしまうのでtry内でロックチェックをしない + // （returnであってもfinallyは実行される） + if (await this.tryLock(host) === '1') { + // 1が返ってきていたら他にロックされているという意味なので、何もしない + return; } + } + try { this.logger.info(`Fetching metadata of ${instance.host} ...`); const [info, dom, manifest] = await Promise.all([ From 834f46537d06b1b6e17ea2e36668811302749ff7 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 15:26:25 +0000 Subject: [PATCH 071/134] @misskey-dev/node-http-message-signatures@0.0.3 --- packages/backend/package.json | 2 +- pnpm-lock.yaml | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 82675fafc2c4..8b310c2d61eb 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -79,7 +79,7 @@ "@fastify/multipart": "8.1.0", "@fastify/static": "6.12.0", "@fastify/view": "8.2.0", - "@misskey-dev/node-http-message-signatures": "0.0.1", + "@misskey-dev/node-http-message-signatures": "0.0.3", "@misskey-dev/sharp-read-bmp": "1.2.0", "@misskey-dev/summaly": "5.0.3", "@nestjs/common": "10.3.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5853c00bb7ab..1ba0cfcf7449 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,8 +111,8 @@ importers: specifier: 8.2.0 version: 8.2.0 '@misskey-dev/node-http-message-signatures': - specifier: 0.0.1 - version: 0.0.1 + specifier: 0.0.3 + version: 0.0.3 '@misskey-dev/sharp-read-bmp': specifier: 1.2.0 version: 1.2.0 @@ -4754,8 +4754,9 @@ packages: eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.1.0)(eslint@8.57.0) dev: true - /@misskey-dev/node-http-message-signatures@0.0.1: - resolution: {integrity: sha512-BaMYEOSBwBtDW0NJcljE4S39gFpgNigVzbVFlVsKzu4+k7snL72mXEaAoMCHWCw7XmdY3+fKADbQoqc4SFrpLw==} + /@misskey-dev/node-http-message-signatures@0.0.3: + resolution: {integrity: sha512-IDsdj01d0Jz/D5L4ubHGOeBLEjbIpHpGIrS7p0t2Z36ahP5cvzeawkxZW2ZuqyWQI4JendJWhGnSmDrH4N3IFg==} + engines: {node: '>=18.4.0'} dependencies: '@lapo/asn1js': 1.2.4 dev: false @@ -6647,7 +6648,7 @@ packages: ts-dedent: 2.2.0 type-fest: 2.19.0 vue: 3.4.21(typescript@5.3.3) - vue-component-type-helpers: 2.0.3 + vue-component-type-helpers: 2.0.5 transitivePeerDependencies: - encoding - supports-color @@ -19446,8 +19447,8 @@ packages: resolution: {integrity: sha512-6bnLkn8O0JJyiFSIF0EfCogzeqNXpnjJ0vW/SZzNHfe6sPx30lTtTXlE5TFs2qhJlAtDFybStVNpL73cPe3OMQ==} dev: true - /vue-component-type-helpers@2.0.3: - resolution: {integrity: sha512-dVPXmrwul+lLt2ErNqwfIGTWkZoPQjlarFx5pwXnCwdJ80ZaJ2nmqtt3KcUgVaKBwBQBpJqOlJPaMxICBay+Cg==} + /vue-component-type-helpers@2.0.5: + resolution: {integrity: sha512-v9N4ufDSnd8YHcDq/vURPjxDyBVak5ZVAQ6aGNIrf7ZAj/VxRKpLZXFHEaqt9yHkWi0/TZp76Jmf8yNJxDQi4g==} dev: true /vue-demi@0.14.7(vue@3.4.21): From 689a9ce5f977b00d3a2a47cfcce086ff6fa8582a Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 15:53:24 +0000 Subject: [PATCH 072/134] PrivateKeyPem --- packages/backend/package.json | 2 +- packages/backend/src/core/AccountUpdateService.ts | 4 ++-- packages/backend/src/core/QueueService.ts | 12 ++++++------ packages/backend/src/core/RelayService.ts | 6 +++--- packages/backend/src/core/UserKeypairService.ts | 12 +++++------- packages/backend/src/core/UserSuspendService.ts | 4 ++-- .../src/core/activitypub/ApDeliverManagerService.ts | 9 +++++---- .../src/core/activitypub/ApRendererService.ts | 8 ++++---- .../backend/src/core/activitypub/ApRequestService.ts | 12 ++++++------ packages/backend/src/core/activitypub/type.ts | 5 ----- packages/backend/src/queue/types.ts | 6 +++--- pnpm-lock.yaml | 8 ++++---- 12 files changed, 41 insertions(+), 47 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 8b310c2d61eb..912768c93489 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -79,7 +79,7 @@ "@fastify/multipart": "8.1.0", "@fastify/static": "6.12.0", "@fastify/view": "8.2.0", - "@misskey-dev/node-http-message-signatures": "0.0.3", + "@misskey-dev/node-http-message-signatures": "0.0.4", "@misskey-dev/sharp-read-bmp": "1.2.0", "@misskey-dev/summaly": "5.0.3", "@nestjs/common": "10.3.3", diff --git a/packages/backend/src/core/AccountUpdateService.ts b/packages/backend/src/core/AccountUpdateService.ts index 208131ed6e4b..d4a7b145546d 100644 --- a/packages/backend/src/core/AccountUpdateService.ts +++ b/packages/backend/src/core/AccountUpdateService.ts @@ -13,7 +13,7 @@ import { RelayService } from '@/core/RelayService.js'; import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; -import type { PrivateKey } from './activitypub/type.js'; +import type { PrivateKeyWithPem } from '@misskey-dev/node-http-message-signatures'; @Injectable() export class AccountUpdateService implements OnModuleInit { @@ -40,7 +40,7 @@ export class AccountUpdateService implements OnModuleInit { * @param userId ユーザーID * @param isKeyUpdation Ed25519キーの作成など公開鍵のアップデートによる呼び出しか？ trueにするとメインキーを使うようになる */ - public async publishToFollowers(userId: MiUser['id'], deliverKey?: PrivateKey) { + public async publishToFollowers(userId: MiUser['id'], deliverKey?: PrivateKeyWithPem) { const user = await this.usersRepository.findOneBy({ id: userId }); if (user == null) throw new Error('user not found'); diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index afea871575a1..a0ec9987a756 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -5,7 +5,7 @@ import { randomUUID } from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; -import type { IActivity, PrivateKey } from '@/core/activitypub/type.js'; +import type { IActivity } from '@/core/activitypub/type.js'; import type { MiDriveFile } from '@/models/DriveFile.js'; import type { MiWebhook, webhookEventTypes } from '@/models/Webhook.js'; import type { Config } from '@/config.js'; @@ -15,7 +15,7 @@ import type { Antenna } from '@/server/api/endpoints/i/import-antennas.js'; import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, RelationshipQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js'; import type { DbJobData, DeliverJobData, RelationshipJobData, ThinUser } from '../queue/types.js'; import type * as Bull from 'bullmq'; -import { genRFC3230DigestHeader, type ParsedSignature } from '@misskey-dev/node-http-message-signatures'; +import { genRFC3230DigestHeader, type PrivateKeyWithPem, type ParsedSignature } from '@misskey-dev/node-http-message-signatures'; @Injectable() export class QueueService { @@ -70,7 +70,7 @@ export class QueueService { } @bindThis - public async deliver(user: ThinUser, content: IActivity | null, to: string | null, isSharedInbox: boolean, privateKey?: PrivateKey) { + public async deliver(user: ThinUser, content: IActivity | null, to: string | null, isSharedInbox: boolean, privateKey?: PrivateKeyWithPem) { if (content == null) return null; if (to == null) return null; @@ -84,7 +84,7 @@ export class QueueService { digest: await genRFC3230DigestHeader(contentBody, 'SHA-256'), to, isSharedInbox, - privateKey: privateKey && { keyId: privateKey.keyId, privateKey: privateKey.privateKey }, + privateKey: privateKey && { keyId: privateKey.keyId, privateKeyPem: privateKey.privateKeyPem }, }; return this.deliverQueue.add(to, data, { @@ -106,7 +106,7 @@ export class QueueService { * @returns void */ @bindThis - public async deliverMany(user: ThinUser, content: IActivity | null, inboxes: Map, privateKey?: PrivateKey) { + public async deliverMany(user: ThinUser, content: IActivity | null, inboxes: Map, privateKey?: PrivateKeyWithPem) { if (content == null) return null; const contentBody = JSON.stringify(content); @@ -126,7 +126,7 @@ export class QueueService { content: contentBody, to: d[0], isSharedInbox: d[1], - privateKey: privateKey && { keyId: privateKey.keyId, privateKey: privateKey.privateKey }, + privateKey: privateKey && { keyId: privateKey.keyId, privateKeyPem: privateKey.privateKeyPem }, } as DeliverJobData, opts, }))); diff --git a/packages/backend/src/core/RelayService.ts b/packages/backend/src/core/RelayService.ts index f106ae1669b9..5e4b6d36aea6 100644 --- a/packages/backend/src/core/RelayService.ts +++ b/packages/backend/src/core/RelayService.ts @@ -17,7 +17,7 @@ import { DI } from '@/di-symbols.js'; import { deepClone } from '@/misc/clone.js'; import { bindThis } from '@/decorators.js'; import { UserKeypairService } from './UserKeypairService.js'; -import type { PrivateKey } from './activitypub/type.js'; +import type { PrivateKeyWithPem } from '@misskey-dev/node-http-message-signatures'; const ACTOR_USERNAME = 'relay.actor' as const; @@ -114,7 +114,7 @@ export class RelayService { } @bindThis - public async deliverToRelays(user: { id: MiUser['id']; host: null; }, activity: any, privateKey?: PrivateKey): Promise { + public async deliverToRelays(user: { id: MiUser['id']; host: null; }, activity: any, privateKey?: PrivateKeyWithPem): Promise { if (activity == null) return; const relays = await this.relaysCache.fetch(() => this.relaysRepository.findBy({ @@ -124,7 +124,7 @@ export class RelayService { const copy = deepClone(activity); if (!copy.to) copy.to = ['https://www.w3.org/ns/activitystreams#Public']; - privateKey = privateKey ?? await this.userKeypairService.getLocalUserKeypairWithKeyId(user.id); + privateKey = privateKey ?? await this.userKeypairService.getLocalUserPrivateKeyPem(user.id); const signed = await this.apRendererService.attachLdSignature(copy, user, privateKey); this.queueService.deliverMany(user, signed, new Map(relays.map(({ inbox }) => [inbox, false])), privateKey); diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 6ec9a8b333bb..00f97143c85d 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -5,7 +5,7 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; -import { genEd25519KeyPair } from '@misskey-dev/node-http-message-signatures'; +import { genEd25519KeyPair, PrivateKeyWithPem } from '@misskey-dev/node-http-message-signatures'; import type { MiUser } from '@/models/User.js'; import type { UserKeypairsRepository } from '@/models/_.js'; import { RedisKVCache } from '@/misc/cache.js'; @@ -53,9 +53,9 @@ export class UserKeypairService implements OnApplicationShutdown { * @returns */ @bindThis - public async getLocalUserKeypairWithKeyId( + public async getLocalUserPrivateKeyPem( userIdOrHint: MiUser['id'] | MiUserKeypair, preferType?: string, - ): Promise<{ keyId: string; publicKey: string; privateKey: string; }> { + ): Promise { const keypair = typeof userIdOrHint === 'string' ? await this.getUserKeypair(userIdOrHint) : userIdOrHint; if ( preferType && ['01', '11', 'ed25519'].includes(preferType.toLowerCase()) && @@ -63,14 +63,12 @@ export class UserKeypairService implements OnApplicationShutdown { ) { return { keyId: `${this.userEntityService.genLocalUserUri(keypair.userId)}#ed25519-key`, - publicKey: keypair.ed25519PublicKey, - privateKey: keypair.ed25519PrivateKey, + privateKeyPem: keypair.ed25519PrivateKey, }; } return { keyId: `${this.userEntityService.genLocalUserUri(keypair.userId)}#main-key`, - publicKey: keypair.publicKey, - privateKey: keypair.privateKey, + privateKeyPem: keypair.privateKey, }; } diff --git a/packages/backend/src/core/UserSuspendService.ts b/packages/backend/src/core/UserSuspendService.ts index 7abeb3d46db8..fc5a68c72ec2 100644 --- a/packages/backend/src/core/UserSuspendService.ts +++ b/packages/backend/src/core/UserSuspendService.ts @@ -32,7 +32,7 @@ export class UserSuspendService { const manager = this.apDeliverManagerService.createDeliverManager(user, content); manager.addAllKnowingSharedInboxRecipe(); // process deliver時にはキーペアが消去されているはずなので、ここで挿入する - const privateKey = await this.userKeypairService.getLocalUserKeypairWithKeyId(user.id, 'main'); + const privateKey = await this.userKeypairService.getLocalUserPrivateKeyPem(user.id, 'main'); manager.execute({ privateKey }); } } @@ -46,7 +46,7 @@ export class UserSuspendService { const manager = this.apDeliverManagerService.createDeliverManager(user, content); manager.addAllKnowingSharedInboxRecipe(); // process deliver時にはキーペアが消去されているはずなので、ここで挿入する - const privateKey = await this.userKeypairService.getLocalUserKeypairWithKeyId(user.id, 'main'); + const privateKey = await this.userKeypairService.getLocalUserPrivateKeyPem(user.id, 'main'); manager.execute({ privateKey }); } } diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index 4b0dd1403d65..e09b548d3a8f 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -10,12 +10,13 @@ import type { FollowingsRepository } from '@/models/_.js'; import type { MiLocalUser, MiRemoteUser, MiUser } from '@/models/User.js'; import { QueueService } from '@/core/QueueService.js'; import { bindThis } from '@/decorators.js'; -import type { IActivity, PrivateKey } from '@/core/activitypub/type.js'; +import type { IActivity } from '@/core/activitypub/type.js'; import { ThinUser } from '@/queue/types.js'; import { AccountUpdateService } from '@/core/AccountUpdateService.js'; import type Logger from '@/logger.js'; import { UserKeypairService } from '../UserKeypairService.js'; import { ApLoggerService } from './ApLoggerService.js'; +import type { PrivateKeyWithPem } from '@misskey-dev/node-http-message-signatures'; interface IRecipe { type: string; @@ -128,7 +129,7 @@ class DeliverManager { * Execute delivers */ @bindThis - public async execute(opts?: { privateKey?: PrivateKey }): Promise { + public async execute(opts?: { privateKey?: PrivateKeyWithPem }): Promise { //#region MIGRATION if (!opts?.privateKey) { /** @@ -139,7 +140,7 @@ class DeliverManager { // createdが存在するということは新規作成されたということなので、フォロワーに配信する this.logger.info(`ed25519 key pair created for user ${this.actor.id} and publishing to followers`); // リモートに配信 - const keyPair = await this.userKeypairService.getLocalUserKeypairWithKeyId(created, 'main'); + const keyPair = await this.userKeypairService.getLocalUserPrivateKeyPem(created, 'main'); await this.accountUpdateService.publishToFollowers(this.actor.id, keyPair); } } @@ -230,7 +231,7 @@ export class ApDeliverManagerService { * @param forceMainKey Force to use main (rsa) key */ @bindThis - public async deliverToFollowers(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity, privateKey?: PrivateKey): Promise { + public async deliverToFollowers(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity, privateKey?: PrivateKeyWithPem): Promise { const manager = new DeliverManager( this.userKeypairService, this.followingsRepository, diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index e4b3074abd72..5d9ca74fe1bb 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -22,7 +22,6 @@ import { UserKeypairService } from '@/core/UserKeypairService.js'; import { MfmService } from '@/core/MfmService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; -import type { MiUserKeypair } from '@/models/UserKeypair.js'; import type { UsersRepository, UserProfilesRepository, NotesRepository, DriveFilesRepository, PollsRepository } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; import { CustomEmojiService } from '@/core/CustomEmojiService.js'; @@ -30,7 +29,8 @@ import { isNotNull } from '@/misc/is-not-null.js'; import { IdService } from '@/core/IdService.js'; import { LdSignatureService } from './LdSignatureService.js'; import { ApMfmService } from './ApMfmService.js'; -import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate, PrivateKey } from './type.js'; +import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate } from './type.js'; +import type { PrivateKeyWithPem } from '@misskey-dev/node-http-message-signatures'; @Injectable() export class ApRendererService { @@ -657,10 +657,10 @@ export class ApRendererService { } @bindThis - public async attachLdSignature(activity: any, user: { id: MiUser['id']; host: null; }, key: PrivateKey): Promise { + public async attachLdSignature(activity: any, user: { id: MiUser['id']; host: null; }, key: PrivateKeyWithPem): Promise { const ldSignature = this.ldSignatureService.use(); ldSignature.debug = false; - activity = await ldSignature.signRsaSignature2017(activity, key.privateKey, key.keyId); + activity = await ldSignature.signRsaSignature2017(activity, key.privateKeyPem, key.keyId); return activity; } diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index c9bcbd961af5..892e11b2d0b6 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -15,7 +15,7 @@ import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import type Logger from '@/logger.js'; import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js'; -import type { PrivateKey } from './type.js'; +import type { PrivateKeyWithPem, PrivateKey } from '@misskey-dev/node-http-message-signatures'; export async function createSignedPost(args: { level: string; key: PrivateKey; url: string; body: string; digest?: string, additionalHeaders: Record }) { const u = new URL(args.url); @@ -36,7 +36,7 @@ export async function createSignedPost(args: { level: string; key: PrivateKey; u const result = await signAsDraftToRequest( request, - { keyId: args.key.keyId, privateKeyPem: args.key.privateKey }, + args.key, ['(request-target)', 'date', 'host', 'digest'], ); @@ -62,7 +62,7 @@ export async function createSignedGet(args: { level: string; key: PrivateKey; ur // TODO: httpMessageSignaturesImplementationLevelによって新規格で通信をするようにする const result = await signAsDraftToRequest( request, - { keyId: args.key.keyId, privateKeyPem: args.key.privateKey }, + args.key, ['(request-target)', 'date', 'host', 'accept'], ); @@ -89,9 +89,9 @@ export class ApRequestService { } @bindThis - public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, level: string, digest?: string, key?: PrivateKey): Promise { + public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, level: string, digest?: string, key?: PrivateKeyWithPem): Promise { const body = typeof object === 'string' ? object : JSON.stringify(object); - key = key ?? await this.userKeypairService.getLocalUserKeypairWithKeyId(user.id, level); + key = key ?? await this.userKeypairService.getLocalUserPrivateKeyPem(user.id, level); const req = await createSignedPost({ level, key, @@ -124,7 +124,7 @@ export class ApRequestService { */ @bindThis public async signedGet(url: string, user: { id: MiUser['id'] }, level: string): Promise { - const key = await this.userKeypairService.getLocalUserKeypairWithKeyId(user.id, level); + const key = await this.userKeypairService.getLocalUserPrivateKeyPem(user.id, level); const req = await createSignedGet({ level, key, diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 6fd5ada9fe15..0b34fd766447 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -326,8 +326,3 @@ export const isAnnounce = (object: IObject): object is IAnnounce => getApType(ob export const isBlock = (object: IObject): object is IBlock => getApType(object) === 'Block'; export const isFlag = (object: IObject): object is IFlag => getApType(object) === 'Flag'; export const isMove = (object: IObject): object is IMove => getApType(object) === 'Move'; - -export type PrivateKey = { - privateKey: string; - keyId: string; -}; diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 12e313627023..85b1c4924b44 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -8,8 +8,8 @@ import type { MiDriveFile } from '@/models/DriveFile.js'; import type { MiNote } from '@/models/Note.js'; import type { MiUser } from '@/models/User.js'; import type { MiWebhook } from '@/models/Webhook.js'; -import type { IActivity, PrivateKey } from '@/core/activitypub/type.js'; -import type { ParsedSignature } from '@misskey-dev/node-http-message-signatures'; +import type { IActivity } from '@/core/activitypub/type.js'; +import type { ParsedSignature, PrivateKeyWithPem } from '@misskey-dev/node-http-message-signatures'; /** * @peertube/http-signature 時代の古いデータにも対応しておく @@ -39,7 +39,7 @@ export type DeliverJobData = { /** whether it is sharedInbox */ isSharedInbox: boolean; /** force to use main (rsa) key */ - privateKey?: PrivateKey; + privateKey?: PrivateKeyWithPem; }; export type InboxJobData = { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1ba0cfcf7449..b9f77b46d4c7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,8 +111,8 @@ importers: specifier: 8.2.0 version: 8.2.0 '@misskey-dev/node-http-message-signatures': - specifier: 0.0.3 - version: 0.0.3 + specifier: 0.0.4 + version: 0.0.4 '@misskey-dev/sharp-read-bmp': specifier: 1.2.0 version: 1.2.0 @@ -4754,8 +4754,8 @@ packages: eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.1.0)(eslint@8.57.0) dev: true - /@misskey-dev/node-http-message-signatures@0.0.3: - resolution: {integrity: sha512-IDsdj01d0Jz/D5L4ubHGOeBLEjbIpHpGIrS7p0t2Z36ahP5cvzeawkxZW2ZuqyWQI4JendJWhGnSmDrH4N3IFg==} + /@misskey-dev/node-http-message-signatures@0.0.4: + resolution: {integrity: sha512-hAQFaJZwJsg63JAY1RZ/yvcaxHbpZg8fLvNdJFucS270TZGunKYv7I16tuNQMKV7+BYLQNxiE7uL11KVZ9jfrg==} engines: {node: '>=18.4.0'} dependencies: '@lapo/asn1js': 1.2.4 From 0127f8929893ea5d337dace2502c5a5a6da71d8f Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 16:27:13 +0000 Subject: [PATCH 073/134] getLocalUserPrivateKey --- .../backend/src/core/UserKeypairService.ts | 78 ++++++++++++++++--- .../src/core/activitypub/ApRendererService.ts | 2 +- .../src/core/activitypub/ApRequestService.ts | 5 +- packages/backend/src/misc/cache.ts | 3 + 4 files changed, 74 insertions(+), 14 deletions(-) diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 00f97143c85d..74d1257bc364 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -5,19 +5,21 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; -import { genEd25519KeyPair, PrivateKeyWithPem } from '@misskey-dev/node-http-message-signatures'; +import { genEd25519KeyPair, importPrivateKey, PrivateKey, PrivateKeyWithPem } from '@misskey-dev/node-http-message-signatures'; import type { MiUser } from '@/models/User.js'; import type { UserKeypairsRepository } from '@/models/_.js'; -import { RedisKVCache } from '@/misc/cache.js'; +import { RedisKVCache, MemoryKVCache } from '@/misc/cache.js'; import type { MiUserKeypair } from '@/models/UserKeypair.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { GlobalEventService, GlobalEvents } from '@/core/GlobalEventService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import type { webcrypto } from 'node:crypto'; @Injectable() export class UserKeypairService implements OnApplicationShutdown { - private cache: RedisKVCache; + private keypairEntityCache: RedisKVCache; + private privateKeyObjectCache: MemoryKVCache; constructor( @Inject(DI.redis) @@ -30,26 +32,29 @@ export class UserKeypairService implements OnApplicationShutdown { private globalEventService: GlobalEventService, private userEntityService: UserEntityService, ) { - this.cache = new RedisKVCache(this.redisClient, 'userKeypair', { + this.keypairEntityCache = new RedisKVCache(this.redisClient, 'userKeypair', { lifetime: 1000 * 60 * 60 * 24, // 24h memoryCacheLifetime: Infinity, fetcher: (key) => this.userKeypairsRepository.findOneByOrFail({ userId: key }), toRedisConverter: (value) => JSON.stringify(value), fromRedisConverter: (value) => JSON.parse(value), }); + this.privateKeyObjectCache = new MemoryKVCache(1000 * 60 * 60 * 1); this.redisForSub.on('message', this.onMessage); } @bindThis public async getUserKeypair(userId: MiUser['id']): Promise { - return await this.cache.fetch(userId); + return await this.keypairEntityCache.fetch(userId); } /** - * + * Get private key [Only PrivateKeyWithPem for queue data etc.] * @param userIdOrHint user id or MiUserKeypair - * @param preferType If ed25519-like(`ed25519`, `01`, `11`) is specified, ed25519 keypair is returned if exists. Otherwise, main keypair is returned. + * @param preferType + * If ed25519-like(`ed25519`, `01`, `11`) is specified, ed25519 keypair is returned if exists. + * Otherwise, main keypair is returned. * @returns */ @bindThis @@ -72,9 +77,62 @@ export class UserKeypairService implements OnApplicationShutdown { }; } + /** + * Get private key [Only PrivateKey for ap request] + * Using cache due to performance reasons of `crypto.subtle.importKey` + * @param userIdOrHint user id, MiUserKeypair, or PrivateKeyWithPem + * @param preferType + * If ed25519-like(`ed25519`, `01`, `11`) is specified, ed25519 keypair is returned if exists. + * Otherwise, main keypair is returned. (ignored if userIdOrHint is PrivateKeyWithPem) + * @returns + */ + @bindThis + public async getLocalUserPrivateKey( + userIdOrHint: MiUser['id'] | MiUserKeypair | PrivateKeyWithPem, preferType?: string, + ): Promise { + if (typeof userIdOrHint === 'object' && 'privateKeyPem' in userIdOrHint) { + // userIdOrHint is PrivateKeyWithPem + return { + keyId: userIdOrHint.keyId, + privateKey: await this.privateKeyObjectCache.fetch(userIdOrHint.keyId, async () => { + return await importPrivateKey(userIdOrHint.privateKeyPem); + }), + }; + } + + const userId = typeof userIdOrHint === 'string' ? userIdOrHint : userIdOrHint.userId; + const getKeypair = () => typeof userIdOrHint === 'string' ? this.getUserKeypair(userId) : userIdOrHint; + + if (preferType && ['01', '11', 'ed25519'].includes(preferType.toLowerCase())) { + const keyId = `${this.userEntityService.genLocalUserUri(userId)}#ed25519-key`; + const fetched = await this.privateKeyObjectCache.fetchMaybe(keyId, async () => { + const keypair = await getKeypair(); + if (keypair.ed25519PublicKey != null && keypair.ed25519PrivateKey != null) { + return await importPrivateKey(keypair.ed25519PrivateKey); + } + return; + }); + if (fetched) { + return { + keyId, + privateKey: fetched, + }; + } + } + + const keyId = `${this.userEntityService.genLocalUserUri(userId)}#main-key`; + return { + keyId, + privateKey: await this.privateKeyObjectCache.fetch(keyId, async () => { + const keypair = await getKeypair(); + return await importPrivateKey(keypair.privateKey); + }), + }; + } + @bindThis public async refresh(userId: MiUser['id']): Promise { - return await this.cache.refresh(userId); + return await this.keypairEntityCache.refresh(userId); } /** @@ -85,7 +143,7 @@ export class UserKeypairService implements OnApplicationShutdown { @bindThis public async refreshAndprepareEd25519KeyPair(userId: MiUser['id']): Promise { await this.refresh(userId); - const keypair = await this.cache.fetch(userId); + const keypair = await this.keypairEntityCache.fetch(userId); if (keypair.ed25519PublicKey != null) { return; } @@ -119,7 +177,7 @@ export class UserKeypairService implements OnApplicationShutdown { } @bindThis public dispose(): void { - this.cache.dispose(); + this.keypairEntityCache.dispose(); } @bindThis diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 5d9ca74fe1bb..d191bfd0899f 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -252,7 +252,7 @@ export class ApRendererService { @bindThis public renderKey(user: MiLocalUser, publicKey: string, postfix?: string): IKey { return { - id: `${this.config.url}/users/${user.id}${postfix ?? '/publickey'}`, + id: `${this.userEntityService.genLocalUserUri(user.id)}${postfix ?? '/publickey'}`, type: 'Key', owner: this.userEntityService.genLocalUserUri(user.id), publicKeyPem: createPublicKey(publicKey).export({ diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 892e11b2d0b6..8338a8ca109a 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -91,10 +91,9 @@ export class ApRequestService { @bindThis public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, level: string, digest?: string, key?: PrivateKeyWithPem): Promise { const body = typeof object === 'string' ? object : JSON.stringify(object); - key = key ?? await this.userKeypairService.getLocalUserPrivateKeyPem(user.id, level); const req = await createSignedPost({ level, - key, + key: await this.userKeypairService.getLocalUserPrivateKey(key ?? user.id, level), url, body, additionalHeaders: { @@ -124,7 +123,7 @@ export class ApRequestService { */ @bindThis public async signedGet(url: string, user: { id: MiUser['id'] }, level: string): Promise { - const key = await this.userKeypairService.getLocalUserPrivateKeyPem(user.id, level); + const key = await this.userKeypairService.getLocalUserPrivateKey(user.id, level); const req = await createSignedGet({ level, key, diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index bba64a06eff2..f498c110bf52 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -195,6 +195,9 @@ export class MemoryKVCache { private lifetime: number; private gcIntervalHandle: NodeJS.Timeout; + /** + * @param lifetime キャッシュの生存期間 (ms) + */ constructor(lifetime: MemoryKVCache['lifetime']) { this.cache = new Map(); this.lifetime = lifetime; From 01b8d2fdb12e26be852475313d36bdfd7c111be4 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 16:44:11 +0000 Subject: [PATCH 074/134] fix test --- .../src/core/FetchInstanceMetadataService.ts | 2 +- .../src/core/activitypub/ApRequestService.ts | 5 ++-- .../test/unit/FetchInstanceMetadataService.ts | 23 +++++++++---------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 9d1ce3ac5dfd..ad68e575cfe5 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -81,7 +81,7 @@ export class FetchInstanceMetadataService { // キャッシュ有効チェックはロック取得前に行う const _instance = await this.federatedInstanceService.fetch(host); const now = Date.now(); - if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < REMOTE_SERVER_CACHE_TTL)) { + if (_instance && _instance.infoUpdatedAt != null && (now - _instance.infoUpdatedAt.getTime() < REMOTE_SERVER_CACHE_TTL)) { this.logger.debug(`Skip because updated recently ${_instance.infoUpdatedAt.toJSON()}`); return; } diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 8338a8ca109a..631430e46a3f 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -91,9 +91,10 @@ export class ApRequestService { @bindThis public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, level: string, digest?: string, key?: PrivateKeyWithPem): Promise { const body = typeof object === 'string' ? object : JSON.stringify(object); + const keyFetched = await this.userKeypairService.getLocalUserPrivateKey(key ?? user.id, level); const req = await createSignedPost({ level, - key: await this.userKeypairService.getLocalUserPrivateKey(key ?? user.id, level), + key: keyFetched, url, body, additionalHeaders: { @@ -106,7 +107,7 @@ export class ApRequestService { version: 'draft', level, url, - keyId: key.keyId, + keyId: keyFetched.keyId, }); await this.httpRequestService.send(url, { diff --git a/packages/backend/test/unit/FetchInstanceMetadataService.ts b/packages/backend/test/unit/FetchInstanceMetadataService.ts index bf8f3ab0e306..f87a6765a9cd 100644 --- a/packages/backend/test/unit/FetchInstanceMetadataService.ts +++ b/packages/backend/test/unit/FetchInstanceMetadataService.ts @@ -75,62 +75,61 @@ describe('FetchInstanceMetadataService', () => { test('Lock and update', async () => { redisClient.set = mockRedis(); const now = Date.now(); - federatedInstanceService.fetch.mockResolvedValue({ infoUpdatedAt: { getTime: () => { return now - 10 * 1000 * 60 * 60 * 24; } } } as any); + federatedInstanceService.fetch.mockResolvedValue({ infoUpdatedAt: new Date(now - 10 * 1000 * 60 * 60 * 24) } as any); httpRequestService.getJson.mockImplementation(() => { throw Error(); }); const tryLockSpy = jest.spyOn(fetchInstanceMetadataService, 'tryLock'); const unlockSpy = jest.spyOn(fetchInstanceMetadataService, 'unlock'); await fetchInstanceMetadataService.fetchInstanceMetadata({ host: 'example.com' } as any); + expect(federatedInstanceService.fetch).toHaveBeenCalledTimes(1); expect(tryLockSpy).toHaveBeenCalledTimes(1); expect(unlockSpy).toHaveBeenCalledTimes(1); - expect(federatedInstanceService.fetch).toHaveBeenCalledTimes(1); expect(httpRequestService.getJson).toHaveBeenCalled(); }); - test('Lock and don\'t update', async () => { + test('Don\'t lock and update when recently updated', async () => { redisClient.set = mockRedis(); - const now = Date.now(); - federatedInstanceService.fetch.mockResolvedValue({ infoUpdatedAt: { getTime: () => now } } as any); + federatedInstanceService.fetch.mockResolvedValue({ infoUpdatedAt: new Date() } as any); httpRequestService.getJson.mockImplementation(() => { throw Error(); }); const tryLockSpy = jest.spyOn(fetchInstanceMetadataService, 'tryLock'); const unlockSpy = jest.spyOn(fetchInstanceMetadataService, 'unlock'); await fetchInstanceMetadataService.fetchInstanceMetadata({ host: 'example.com' } as any); - expect(tryLockSpy).toHaveBeenCalledTimes(1); - expect(unlockSpy).toHaveBeenCalledTimes(1); expect(federatedInstanceService.fetch).toHaveBeenCalledTimes(1); + expect(tryLockSpy).toHaveBeenCalledTimes(0); + expect(unlockSpy).toHaveBeenCalledTimes(0); expect(httpRequestService.getJson).toHaveBeenCalledTimes(0); }); test('Do nothing when lock not acquired', async () => { redisClient.set = mockRedis(); const now = Date.now(); - federatedInstanceService.fetch.mockResolvedValue({ infoUpdatedAt: { getTime: () => now - 10 * 1000 * 60 * 60 * 24 } } as any); + federatedInstanceService.fetch.mockResolvedValue({ infoUpdatedAt: new Date(now - 10 * 1000 * 60 * 60 * 24) } as any); httpRequestService.getJson.mockImplementation(() => { throw Error(); }); await fetchInstanceMetadataService.tryLock('example.com'); const tryLockSpy = jest.spyOn(fetchInstanceMetadataService, 'tryLock'); const unlockSpy = jest.spyOn(fetchInstanceMetadataService, 'unlock'); await fetchInstanceMetadataService.fetchInstanceMetadata({ host: 'example.com' } as any); + expect(federatedInstanceService.fetch).toHaveBeenCalledTimes(1); expect(tryLockSpy).toHaveBeenCalledTimes(1); expect(unlockSpy).toHaveBeenCalledTimes(0); - expect(federatedInstanceService.fetch).toHaveBeenCalledTimes(0); expect(httpRequestService.getJson).toHaveBeenCalledTimes(0); }); - test('Do when lock not acquired but forced', async () => { + test('Do when forced', async () => { redisClient.set = mockRedis(); const now = Date.now(); - federatedInstanceService.fetch.mockResolvedValue({ infoUpdatedAt: { getTime: () => now - 10 * 1000 * 60 * 60 * 24 } } as any); + federatedInstanceService.fetch.mockResolvedValue({ infoUpdatedAt: new Date(now - 10 * 1000 * 60 * 60 * 24) } as any); httpRequestService.getJson.mockImplementation(() => { throw Error(); }); await fetchInstanceMetadataService.tryLock('example.com'); const tryLockSpy = jest.spyOn(fetchInstanceMetadataService, 'tryLock'); const unlockSpy = jest.spyOn(fetchInstanceMetadataService, 'unlock'); await fetchInstanceMetadataService.fetchInstanceMetadata({ host: 'example.com' } as any, true); + expect(federatedInstanceService.fetch).toHaveBeenCalledTimes(0); expect(tryLockSpy).toHaveBeenCalledTimes(0); expect(unlockSpy).toHaveBeenCalledTimes(1); - expect(federatedInstanceService.fetch).toHaveBeenCalledTimes(0); expect(httpRequestService.getJson).toHaveBeenCalled(); }); }); From ab29cbab41f303025e35f6070c0e6455f793d4fa Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 16:44:32 +0000 Subject: [PATCH 075/134] if --- packages/backend/test/unit/FetchInstanceMetadataService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/test/unit/FetchInstanceMetadataService.ts b/packages/backend/test/unit/FetchInstanceMetadataService.ts index f87a6765a9cd..2e66b81fcd9f 100644 --- a/packages/backend/test/unit/FetchInstanceMetadataService.ts +++ b/packages/backend/test/unit/FetchInstanceMetadataService.ts @@ -87,7 +87,7 @@ describe('FetchInstanceMetadataService', () => { expect(httpRequestService.getJson).toHaveBeenCalled(); }); - test('Don\'t lock and update when recently updated', async () => { + test('Don\'t lock and update if recently updated', async () => { redisClient.set = mockRedis(); federatedInstanceService.fetch.mockResolvedValue({ infoUpdatedAt: new Date() } as any); httpRequestService.getJson.mockImplementation(() => { throw Error(); }); From a701fed9e5c945261cb31d7eb80502d30ca8c445 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 5 Mar 2024 16:50:26 +0000 Subject: [PATCH 076/134] fix ap-request --- packages/backend/test/unit/ap-request.ts | 41 ++++++++++++++++-------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/packages/backend/test/unit/ap-request.ts b/packages/backend/test/unit/ap-request.ts index 1c0159ed9585..8406cb7ec2a5 100644 --- a/packages/backend/test/unit/ap-request.ts +++ b/packages/backend/test/unit/ap-request.ts @@ -4,7 +4,7 @@ */ import * as assert from 'assert'; -import { verifyDraftSignature, parseRequestSignature, genEd25519KeyPair, genRsaKeyPair } from '@misskey-dev/node-http-message-signatures'; +import { verifyDraftSignature, parseRequestSignature, genEd25519KeyPair, genRsaKeyPair, importPrivateKey } from '@misskey-dev/node-http-message-signatures'; import { createSignedGet, createSignedPost } from '@/core/activitypub/ApRequestService.js'; export const buildParsedSignature = (signingString: string, signature: string, algorithm: string) => { @@ -31,32 +31,47 @@ async function getKeyPair(level: string) { throw new Error('Invalid level'); } -describe('ap-request', () => { +describe('ap-request post', () => { + const url = 'https://example.com/inbox'; + const activity = { a: 1 }; + const body = JSON.stringify(activity); + const headers = { + 'User-Agent': 'UA', + }; + describe.each(['00', '01'])('createSignedPost with verify', (level) => { - test('pass', async () => { + test('pem', async () => { const keypair = await getKeyPair(level); - const key = { keyId: 'x', 'privateKey': keypair.privateKey }; - const url = 'https://example.com/inbox'; - const activity = { a: 1 }; - const body = JSON.stringify(activity); - const headers = { - 'User-Agent': 'UA', - }; + const key = { keyId: 'x', 'privateKeyPem': keypair.privateKey }; + + const req = await createSignedPost({ level, key, url, body, additionalHeaders: headers }); + + const parsed = parseRequestSignature(req.request); + expect(parsed.version).toBe('draft'); + if (!parsed) return; + const verify = await verifyDraftSignature(parsed.value, keypair.publicKey); + assert.deepStrictEqual(verify, true); + }); + test('imported', async () => { + const keypair = await getKeyPair(level); + const key = { keyId: 'x', 'privateKey': await importPrivateKey(keypair.privateKey) }; const req = await createSignedPost({ level, key, url, body, additionalHeaders: headers }); const parsed = parseRequestSignature(req.request); - expect(parsed?.version).toBe('draft'); + expect(parsed.version).toBe('draft'); if (!parsed) return; const verify = await verifyDraftSignature(parsed.value, keypair.publicKey); assert.deepStrictEqual(verify, true); }); }); +}); +describe('ap-request get', () => { describe.each(['00', '01'])('createSignedGet with verify', (level) => { test('pass', async () => { const keypair = await getKeyPair(level); - const key = { keyId: 'x', 'privateKey': keypair.privateKey }; + const key = { keyId: 'x', 'privateKeyPem': keypair.privateKey }; const url = 'https://example.com/outbox'; const headers = { 'User-Agent': 'UA', @@ -65,7 +80,7 @@ describe('ap-request', () => { const req = await createSignedGet({ level, key, url, additionalHeaders: headers }); const parsed = parseRequestSignature(req.request); - expect(parsed?.version).toBe('draft'); + expect(parsed.version).toBe('draft'); if (!parsed) return; const verify = await verifyDraftSignature(parsed.value, keypair.publicKey); assert.deepStrictEqual(verify, true); From 39fba74dd12aa0fd22c1b3f59ec053858443f2f8 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 6 Mar 2024 06:48:03 +0000 Subject: [PATCH 077/134] update node-http-message-signatures --- packages/backend/package.json | 2 +- pnpm-lock.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 912768c93489..daf00e3238f3 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -79,7 +79,7 @@ "@fastify/multipart": "8.1.0", "@fastify/static": "6.12.0", "@fastify/view": "8.2.0", - "@misskey-dev/node-http-message-signatures": "0.0.4", + "@misskey-dev/node-http-message-signatures": "0.0.5", "@misskey-dev/sharp-read-bmp": "1.2.0", "@misskey-dev/summaly": "5.0.3", "@nestjs/common": "10.3.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b9f77b46d4c7..5b98adc1c8aa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,8 +111,8 @@ importers: specifier: 8.2.0 version: 8.2.0 '@misskey-dev/node-http-message-signatures': - specifier: 0.0.4 - version: 0.0.4 + specifier: 0.0.5 + version: 0.0.5 '@misskey-dev/sharp-read-bmp': specifier: 1.2.0 version: 1.2.0 @@ -4754,8 +4754,8 @@ packages: eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.1.0)(eslint@8.57.0) dev: true - /@misskey-dev/node-http-message-signatures@0.0.4: - resolution: {integrity: sha512-hAQFaJZwJsg63JAY1RZ/yvcaxHbpZg8fLvNdJFucS270TZGunKYv7I16tuNQMKV7+BYLQNxiE7uL11KVZ9jfrg==} + /@misskey-dev/node-http-message-signatures@0.0.5: + resolution: {integrity: sha512-lULO/ekgFA3p8K9Tc8HqKf/0jUFn3pR6SWB845OL44qKrIa8wsPAbMJN3u5NLBeFvPAGXhQfHJLcWfuTwlLB6Q==} engines: {node: '>=18.4.0'} dependencies: '@lapo/asn1js': 1.2.4 From fef9ebfe06113e211d21520faf65dc6e68ca0b56 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 6 Mar 2024 12:56:03 +0000 Subject: [PATCH 078/134] fix type error --- .../backend/src/queue/processors/InboxProcessorService.ts | 4 ++++ .../src/server/api/endpoints/admin/queue/inbox-delayed.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 23e377d7907e..e790267a0497 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -53,6 +53,10 @@ export class InboxProcessorService { @bindThis public async process(job: Bull.Job): Promise { const signature = 'version' in job.data.signature ? job.data.signature.value : job.data.signature; + if (Array.isArray(signature)) { + // RFC 9401はsignatureが配列になるが、とりあえずエラーにする + throw new Error('signature is array'); + } const activity = job.data.activity; //#region Log diff --git a/packages/backend/src/server/api/endpoints/admin/queue/inbox-delayed.ts b/packages/backend/src/server/api/endpoints/admin/queue/inbox-delayed.ts index 7839d8b6f5a1..42afa084850f 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/inbox-delayed.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/inbox-delayed.ts @@ -57,7 +57,7 @@ export default class extends Endpoint { // eslint- for (const job of jobs) { const signature = 'version' in job.data.signature ? job.data.signature.value : job.data.signature; - const host = new URL(signature.keyId).host; + const host = Array.isArray(signature) ? 'TODO' : new URL(signature.keyId).host; if (res.find(x => x[0] === host)) { res.find(x => x[0] === host)![1]++; } else { From 844feb1bb398f1037084dd0fbcf18b73b8462d73 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 7 Mar 2024 10:52:38 +0000 Subject: [PATCH 079/134] update package --- packages/backend/package.json | 2 +- pnpm-lock.yaml | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index daf00e3238f3..2cf1fc55b397 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -79,7 +79,7 @@ "@fastify/multipart": "8.1.0", "@fastify/static": "6.12.0", "@fastify/view": "8.2.0", - "@misskey-dev/node-http-message-signatures": "0.0.5", + "@misskey-dev/node-http-message-signatures": "0.0.6", "@misskey-dev/sharp-read-bmp": "1.2.0", "@misskey-dev/summaly": "5.0.3", "@nestjs/common": "10.3.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5b98adc1c8aa..20aab296788b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,8 +111,8 @@ importers: specifier: 8.2.0 version: 8.2.0 '@misskey-dev/node-http-message-signatures': - specifier: 0.0.5 - version: 0.0.5 + specifier: 0.0.6 + version: 0.0.6 '@misskey-dev/sharp-read-bmp': specifier: 1.2.0 version: 1.2.0 @@ -4754,11 +4754,12 @@ packages: eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.1.0)(eslint@8.57.0) dev: true - /@misskey-dev/node-http-message-signatures@0.0.5: - resolution: {integrity: sha512-lULO/ekgFA3p8K9Tc8HqKf/0jUFn3pR6SWB845OL44qKrIa8wsPAbMJN3u5NLBeFvPAGXhQfHJLcWfuTwlLB6Q==} + /@misskey-dev/node-http-message-signatures@0.0.6: + resolution: {integrity: sha512-Yw/bPzdXs0q/W1FkCOWNFl3woDwAr3wA2eg9ujfk5GlyfdrosWOdep3Vcc9rj4s+POr+2F2EAvTWNrduBWHkNQ==} engines: {node: '>=18.4.0'} dependencies: '@lapo/asn1js': 1.2.4 + structured-headers: 1.0.1 dev: false /@misskey-dev/sharp-read-bmp@1.2.0: @@ -6648,7 +6649,7 @@ packages: ts-dedent: 2.2.0 type-fest: 2.19.0 vue: 3.4.21(typescript@5.3.3) - vue-component-type-helpers: 2.0.5 + vue-component-type-helpers: 2.0.6 transitivePeerDependencies: - encoding - supports-color @@ -18286,6 +18287,11 @@ packages: peek-readable: 5.0.0 dev: false + /structured-headers@1.0.1: + resolution: {integrity: sha512-QYBxdBtA4Tl5rFPuqmbmdrS9kbtren74RTJTcs0VSQNVV5iRhJD4QlYTLD0+81SBwUQctjEQzjTRI3WG4DzICA==} + engines: {node: '>= 14', npm: '>=6'} + dev: false + /stylehacks@6.0.3(postcss@8.4.35): resolution: {integrity: sha512-KzBqjnqktc8/I0ERCb+lGq06giF/JxDbw2r9kEVhen9noHeIDRtMWUp9r62sOk+/2bbX6sFG1GhsS7ToXG0PEg==} engines: {node: ^14 || ^16 || >=18.0} @@ -19447,8 +19453,8 @@ packages: resolution: {integrity: sha512-6bnLkn8O0JJyiFSIF0EfCogzeqNXpnjJ0vW/SZzNHfe6sPx30lTtTXlE5TFs2qhJlAtDFybStVNpL73cPe3OMQ==} dev: true - /vue-component-type-helpers@2.0.5: - resolution: {integrity: sha512-v9N4ufDSnd8YHcDq/vURPjxDyBVak5ZVAQ6aGNIrf7ZAj/VxRKpLZXFHEaqt9yHkWi0/TZp76Jmf8yNJxDQi4g==} + /vue-component-type-helpers@2.0.6: + resolution: {integrity: sha512-qdGXCtoBrwqk1BT6r2+1Wcvl583ZVkuSZ3or7Y1O2w5AvWtlvvxwjGhmz5DdPJS9xqRdDlgTJ/38ehWnEi0tFA==} dev: true /vue-demi@0.14.7(vue@3.4.21): From e543ffe368e842d4501d6d46d918c8c9203a9867 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 7 Mar 2024 11:38:23 +0000 Subject: [PATCH 080/134] fix type --- packages/backend/src/core/activitypub/ApRequestService.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 631430e46a3f..420844cc174b 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -19,7 +19,7 @@ import type { PrivateKeyWithPem, PrivateKey } from '@misskey-dev/node-http-messa export async function createSignedPost(args: { level: string; key: PrivateKey; url: string; body: string; digest?: string, additionalHeaders: Record }) { const u = new URL(args.url); - const request: RequestLike = { + const request = { url: u.href, method: 'POST', headers: { @@ -27,7 +27,7 @@ export async function createSignedPost(args: { level: string; key: PrivateKey; u 'Host': u.host, 'Content-Type': 'application/activity+json', ...args.additionalHeaders, - }, + } as Record, }; // TODO: httpMessageSignaturesImplementationLevelによって新規格で通信をするようにする @@ -48,7 +48,7 @@ export async function createSignedPost(args: { level: string; key: PrivateKey; u export async function createSignedGet(args: { level: string; key: PrivateKey; url: string; additionalHeaders: Record }) { const u = new URL(args.url); - const request: RequestLike = { + const request = { url: u.href, method: 'GET', headers: { @@ -56,7 +56,7 @@ export async function createSignedGet(args: { level: string; key: PrivateKey; ur 'Date': new Date().toUTCString(), 'Host': new URL(args.url).host, ...args.additionalHeaders, - }, + } as Record, }; // TODO: httpMessageSignaturesImplementationLevelによって新規格で通信をするようにする From 74c8f0a483d020e7b1b60fb3a44d33043b18bb0c Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 7 Mar 2024 15:21:07 +0000 Subject: [PATCH 081/134] update package --- packages/backend/package.json | 2 +- pnpm-lock.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index e2f4180d1c2f..57f718210fb7 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -79,7 +79,7 @@ "@fastify/multipart": "8.1.0", "@fastify/static": "6.12.0", "@fastify/view": "8.2.0", - "@misskey-dev/node-http-message-signatures": "0.0.6", + "@misskey-dev/node-http-message-signatures": "0.0.7", "@misskey-dev/sharp-read-bmp": "1.2.0", "@misskey-dev/summaly": "5.0.3", "@nestjs/common": "10.3.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 20aab296788b..f853e68f6a36 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,8 +111,8 @@ importers: specifier: 8.2.0 version: 8.2.0 '@misskey-dev/node-http-message-signatures': - specifier: 0.0.6 - version: 0.0.6 + specifier: 0.0.7 + version: 0.0.7 '@misskey-dev/sharp-read-bmp': specifier: 1.2.0 version: 1.2.0 @@ -4754,8 +4754,8 @@ packages: eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.1.0)(eslint@8.57.0) dev: true - /@misskey-dev/node-http-message-signatures@0.0.6: - resolution: {integrity: sha512-Yw/bPzdXs0q/W1FkCOWNFl3woDwAr3wA2eg9ujfk5GlyfdrosWOdep3Vcc9rj4s+POr+2F2EAvTWNrduBWHkNQ==} + /@misskey-dev/node-http-message-signatures@0.0.7: + resolution: {integrity: sha512-NmXxPZGHtNniKMaAhLDoA+jrnubLqYmjdbzDzQVieaCTqAT0sn0Mg+IyHbfDR4+kePUSKMBQIeM1YNJWextFtA==} engines: {node: '>=18.4.0'} dependencies: '@lapo/asn1js': 1.2.4 From 6907b6505afc7eeee0efcde9232ffa433934ea98 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 9 Mar 2024 10:24:55 +0000 Subject: [PATCH 082/134] retry no key --- .../backend/src/queue/processors/InboxProcessorService.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index e790267a0497..b1468a2a9352 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -103,7 +103,9 @@ export class InboxProcessorService { // publicKey がなくても終了 if (authUser.key == null) { - throw new Bull.UnrecoverableError('skip: failed to resolve user publicKey'); + // publicKeyがないのはpublicKeyの変更（主にmain→ed25519）に + // 対応しきれていない場合があるためリトライする + throw new Error('skip: failed to resolve user publicKey'); } // HTTP-Signatureの検証 From d0da9f32dc91546eb8aab19e6a609045693d9bf8 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 9 Mar 2024 10:25:58 +0000 Subject: [PATCH 083/134] @misskey-dev/node-http-message-signatures@0.0.8 --- packages/backend/package.json | 2 +- pnpm-lock.yaml | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 57f718210fb7..76a17cc88cdd 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -79,7 +79,7 @@ "@fastify/multipart": "8.1.0", "@fastify/static": "6.12.0", "@fastify/view": "8.2.0", - "@misskey-dev/node-http-message-signatures": "0.0.7", + "@misskey-dev/node-http-message-signatures": "0.0.8", "@misskey-dev/sharp-read-bmp": "1.2.0", "@misskey-dev/summaly": "5.0.3", "@nestjs/common": "10.3.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f853e68f6a36..99195bbfc729 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,8 +111,8 @@ importers: specifier: 8.2.0 version: 8.2.0 '@misskey-dev/node-http-message-signatures': - specifier: 0.0.7 - version: 0.0.7 + specifier: 0.0.8 + version: 0.0.8 '@misskey-dev/sharp-read-bmp': specifier: 1.2.0 version: 1.2.0 @@ -4754,11 +4754,12 @@ packages: eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.1.0)(eslint@8.57.0) dev: true - /@misskey-dev/node-http-message-signatures@0.0.7: - resolution: {integrity: sha512-NmXxPZGHtNniKMaAhLDoA+jrnubLqYmjdbzDzQVieaCTqAT0sn0Mg+IyHbfDR4+kePUSKMBQIeM1YNJWextFtA==} + /@misskey-dev/node-http-message-signatures@0.0.8: + resolution: {integrity: sha512-ymyjluFpY2xK/3UYyj+MiRfA7752BIWwSb+uFU0mFK1Yq2fGI6lAGZg3+D2lD8W/OkltnMQm1gSOXzE3CNIgtA==} engines: {node: '>=18.4.0'} dependencies: '@lapo/asn1js': 1.2.4 + rfc4648: 1.5.3 structured-headers: 1.0.1 dev: false @@ -17349,6 +17350,10 @@ packages: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + /rfc4648@1.5.3: + resolution: {integrity: sha512-MjOWxM065+WswwnmNONOT+bD1nXzY9Km6u3kzvnx8F8/HXGZdz3T6e6vZJ8Q/RIMUSp/nxqjH3GwvJDy8ijeQQ==} + dev: false + /rfdc@1.3.0: resolution: {integrity: sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==} From 75a2f1c1e870657b86a985404d92ee53cf9325dc Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 9 Mar 2024 12:27:52 +0000 Subject: [PATCH 084/134] fix type error --- packages/backend/test/unit/ap-request.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/backend/test/unit/ap-request.ts b/packages/backend/test/unit/ap-request.ts index 8406cb7ec2a5..50894c8b8152 100644 --- a/packages/backend/test/unit/ap-request.ts +++ b/packages/backend/test/unit/ap-request.ts @@ -48,8 +48,8 @@ describe('ap-request post', () => { const parsed = parseRequestSignature(req.request); expect(parsed.version).toBe('draft'); - if (!parsed) return; - const verify = await verifyDraftSignature(parsed.value, keypair.publicKey); + expect(Array.isArray(parsed.value)).toBe(false); + const verify = await verifyDraftSignature(parsed.value as any, keypair.publicKey); assert.deepStrictEqual(verify, true); }); test('imported', async () => { @@ -60,8 +60,8 @@ describe('ap-request post', () => { const parsed = parseRequestSignature(req.request); expect(parsed.version).toBe('draft'); - if (!parsed) return; - const verify = await verifyDraftSignature(parsed.value, keypair.publicKey); + expect(Array.isArray(parsed.value)).toBe(false); + const verify = await verifyDraftSignature(parsed.value as any, keypair.publicKey); assert.deepStrictEqual(verify, true); }); }); @@ -81,8 +81,8 @@ describe('ap-request get', () => { const parsed = parseRequestSignature(req.request); expect(parsed.version).toBe('draft'); - if (!parsed) return; - const verify = await verifyDraftSignature(parsed.value, keypair.publicKey); + expect(Array.isArray(parsed.value)).toBe(false); + const verify = await verifyDraftSignature(parsed.value as any, keypair.publicKey); assert.deepStrictEqual(verify, true); }); }); From 4310229ca52add406a06ef84b047ec6f539d7e9c Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 9 Mar 2024 16:39:01 +0000 Subject: [PATCH 085/134] log keyid --- packages/backend/src/queue/processors/InboxProcessorService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index b1468a2a9352..dcaa24e8b705 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -105,7 +105,7 @@ export class InboxProcessorService { if (authUser.key == null) { // publicKeyがないのはpublicKeyの変更（主にmain→ed25519）に // 対応しきれていない場合があるためリトライする - throw new Error('skip: failed to resolve user publicKey'); + throw new Error(`skip: failed to resolve user publicKey: keyId=${signature.keyId}`); } // HTTP-Signatureの検証 From d168ec7dd58d179679b9459bdcdcdf75ef841155 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 9 Mar 2024 17:22:16 +0000 Subject: [PATCH 086/134] logger --- .../core/activitypub/ApDbResolverService.ts | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 61fcc6731dc9..93f0e1bb5483 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -13,8 +13,10 @@ import { CacheService } from '@/core/CacheService.js'; import type { MiNote } from '@/models/Note.js'; import { bindThis } from '@/decorators.js'; import { MiLocalUser, MiRemoteUser } from '@/models/User.js'; +import Logger from '@/logger.js'; import { getApId } from './type.js'; import { ApPersonService } from './models/ApPersonService.js'; +import { ApLoggerService } from './ApLoggerService.js'; import type { IObject } from './type.js'; export type UriParseResult = { @@ -36,6 +38,7 @@ export type UriParseResult = { @Injectable() export class ApDbResolverService implements OnApplicationShutdown { private publicKeyByUserIdCache: MemoryKVCache; + private logger: Logger; constructor( @Inject(DI.config) @@ -52,8 +55,10 @@ export class ApDbResolverService implements OnApplicationShutdown { private cacheService: CacheService, private apPersonService: ApPersonService, + private apLoggerService: ApLoggerService, ) { this.publicKeyByUserIdCache = new MemoryKVCache(Infinity); + this.logger = this.apLoggerService.logger; } @bindThis @@ -118,9 +123,14 @@ export class ApDbResolverService implements OnApplicationShutdown { private async refreshAndfindKey(userId: MiUser['id'], keyId: string): Promise { this.refreshCacheByUserId(userId); const keys = await this.getPublicKeyByUserId(userId); - if (keys == null || !Array.isArray(keys)) return null; - - return keys.find(x => x.keyId === keyId) ?? null; + if (keys == null || !Array.isArray(keys) || keys.length === 0) { + this.logger.warn(`No key found (refreshAndfindKey) userId=${userId} keyId=${keyId} keys=${keys}`); + return null; + } + const exactKey = keys.find(x => x.keyId === keyId); + if (exactKey) return exactKey; + this.logger.warn(`No exact key found (refreshAndfindKey) userId=${userId} keyId=${keyId} keys=${keys}`); + return null; } /** @@ -138,7 +148,10 @@ export class ApDbResolverService implements OnApplicationShutdown { const keys = await this.getPublicKeyByUserId(user.id); - if (keys == null || !Array.isArray(keys)) return { user, key: null }; + if (keys == null || !Array.isArray(keys) || keys.length === 0) { + this.logger.warn(`No key found uri=${uri} userId=${user.id} keys=${keys}`); + return { user, key: null }; + } if (!keyId) { // mainっぽいのを選ぶ @@ -173,12 +186,14 @@ export class ApDbResolverService implements OnApplicationShutdown { // lastFetchedAtでの更新制限を弱めて再取得 if (user.lastFetchedAt == null || user.lastFetchedAt < new Date(Date.now() - 1000 * 60 * 12)) { + this.logger.info(`Fetching user to find public key uri=${uri} userId=${user.id} keyId=${keyId}`); const renewed = await this.apPersonService.fetchPersonWithRenewal(uri, 0); if (renewed == null || renewed.isDeleted) return null; return { user, key: await this.refreshAndfindKey(user.id, keyId) }; } + this.logger.warn(`No key found uri=${uri} userId=${user.id} keyId=${keyId}`); return { user, key: null }; } From 1690e0617e232707929d1423e31fd19eb5abc804 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 9 Mar 2024 17:25:24 +0000 Subject: [PATCH 087/134] db-resolver --- packages/backend/src/core/activitypub/ApDbResolverService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 93f0e1bb5483..677f8c2cfa90 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -58,7 +58,7 @@ export class ApDbResolverService implements OnApplicationShutdown { private apLoggerService: ApLoggerService, ) { this.publicKeyByUserIdCache = new MemoryKVCache(Infinity); - this.logger = this.apLoggerService.logger; + this.logger = this.apLoggerService.logger.createSubLogger('db-resolver'); } @bindThis From da4a44b337717b3b51671b75c7586df308c839f0 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 9 Mar 2024 17:29:23 +0000 Subject: [PATCH 088/134] JSON.stringify --- .../backend/src/core/activitypub/ApDbResolverService.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 677f8c2cfa90..8509f04c09d4 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -124,12 +124,12 @@ export class ApDbResolverService implements OnApplicationShutdown { this.refreshCacheByUserId(userId); const keys = await this.getPublicKeyByUserId(userId); if (keys == null || !Array.isArray(keys) || keys.length === 0) { - this.logger.warn(`No key found (refreshAndfindKey) userId=${userId} keyId=${keyId} keys=${keys}`); + this.logger.warn(`No key found (refreshAndfindKey) userId=${userId} keyId=${keyId} keys=${JSON.stringify(keys)}`); return null; } const exactKey = keys.find(x => x.keyId === keyId); if (exactKey) return exactKey; - this.logger.warn(`No exact key found (refreshAndfindKey) userId=${userId} keyId=${keyId} keys=${keys}`); + this.logger.warn(`No exact key found (refreshAndfindKey) userId=${userId} keyId=${keyId} keys=${JSON.stringify(keys)}`); return null; } @@ -149,7 +149,7 @@ export class ApDbResolverService implements OnApplicationShutdown { const keys = await this.getPublicKeyByUserId(user.id); if (keys == null || !Array.isArray(keys) || keys.length === 0) { - this.logger.warn(`No key found uri=${uri} userId=${user.id} keys=${keys}`); + this.logger.warn(`No key found uri=${uri} userId=${user.id} keys=${JSON.stringify(keys)}`); return { user, key: null }; } From 8104963e1dcd1ba1a3dec7f526321b11c7459920 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 9 Mar 2024 19:06:59 +0000 Subject: [PATCH 089/134] =?UTF-8?q?HTTP=20Signature=E3=81=8C=E3=81=AA?= =?UTF-8?q?=E3=81=8B=E3=81=A3=E3=81=9F=E3=82=8A=E4=BD=BF=E3=81=88=E3=81=AA?= =?UTF-8?q?=E3=81=8B=E3=81=A3=E3=81=9F=E3=82=8A=E3=81=97=E3=81=9D=E3=81=86?= =?UTF-8?q?=E3=81=AA=E5=A0=B4=E5=90=88=E3=81=ABLD=20Signature=E3=82=92?= =?UTF-8?q?=E6=B4=BB=E7=94=A8=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/core/QueueService.ts | 2 +- .../core/activitypub/ApDbResolverService.ts | 31 ++++++- .../queue/processors/InboxProcessorService.ts | 88 ++++++++++--------- packages/backend/src/queue/types.ts | 2 +- .../src/server/ActivityPubServerService.ts | 24 ++++- 5 files changed, 100 insertions(+), 47 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index a0ec9987a756..64886878670f 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -135,7 +135,7 @@ export class QueueService { } @bindThis - public inbox(activity: IActivity, signature: ParsedSignature) { + public inbox(activity: IActivity, signature: ParsedSignature | null) { const data = { activity: activity, signature, diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 8509f04c09d4..16804a240da8 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -137,14 +137,41 @@ export class ApDbResolverService implements OnApplicationShutdown { * AP Actor id => Misskey User and Key * @param uri AP Actor id * @param keyId Key id to find. If not specified, main key will be selected. + * keyIdがURLライクの場合、ハッシュを削除したkeyIdはuriと同一であることが期待される + * @returns + * 1. uriとkeyIdが一致しない場合`null` + * 2. userが見つからない場合`{ user: null, key: null }` + * 3. keyが見つからない場合`{ user, key: null }` */ @bindThis public async getAuthUserFromApId(uri: string, keyId?: string): Promise<{ user: MiRemoteUser; key: MiUserPublickey | null; - } | null> { + } | { + user: null; + key: null; + } | + null> { + if (keyId) { + try { + const actorUrl = new URL(uri); + const keyUrl = new URL(keyId); + actorUrl.hash = ''; + keyUrl.hash = ''; + if (actorUrl.href !== keyUrl.href) { + // uriとkeyId（のhashなし）が一致しない場合、actorと鍵の所有者が一致していないということである + // その場合、そもそも署名は有効といえないのでキーの検索は無意味 + this.logger.warn(`actor uri and keyId are not matched uri=${uri} keyId=${keyId}`); + return null; + } + } catch (err) { + // キーがURLっぽくない場合はエラーになるはず。そういった場合はとりあえずキー検索してみる + this.logger.warn(`maybe actor uri or keyId are not url like: uri=${uri} keyId=${keyId}`, { err }); + } + } + const user = await this.apPersonService.resolvePerson(uri, undefined, true) as MiRemoteUser; - if (user.isDeleted) return null; + if (user.isDeleted) return { user: null, key: null }; const keys = await this.getPublicKeyByUserId(user.id); diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index dcaa24e8b705..07f36f4d7a84 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -52,12 +52,15 @@ export class InboxProcessorService { @bindThis public async process(job: Bull.Job): Promise { - const signature = 'version' in job.data.signature ? job.data.signature.value : job.data.signature; + const signature = job.data.signature ? + 'version' in job.data.signature ? job.data.signature.value : job.data.signature + : null; if (Array.isArray(signature)) { // RFC 9401はsignatureが配列になるが、とりあえずエラーにする throw new Error('signature is array'); } const activity = job.data.activity; + const actorUri = getApId(activity.actor); //#region Log const info = Object.assign({}, activity); @@ -65,7 +68,7 @@ export class InboxProcessorService { this.logger.debug(JSON.stringify(info, null, 2)); //#endregion - const host = this.utilityService.toPuny(new URL(signature.keyId).hostname); + const host = this.utilityService.toPuny(new URL(activity.actor).hostname); // ブロックしてたら中断 const meta = await this.metaService.fetch(); @@ -73,19 +76,12 @@ export class InboxProcessorService { return `Blocked request: ${host}`; } - const keyIdLower = signature.keyId.toLowerCase(); - if (keyIdLower.startsWith('acct:')) { - return `Old keyId is no longer supported. ${keyIdLower}`; - } - // HTTP-Signature keyIdを元にDBから取得 - let authUser: { - user: MiRemoteUser; - key: MiUserPublickey | null; - } | null = null; + let authUser: Awaited> = null; + let httpSignatureIsValid = null as boolean | null; try { - authUser = await this.apDbResolverService.getAuthUserFromApId(getApId(activity.actor), signature.keyId); + authUser = await this.apDbResolverService.getAuthUserFromApId(actorUri, signature?.keyId); } catch (err) { // 対象が4xxならスキップ if (err instanceof StatusError) { @@ -96,45 +92,58 @@ export class InboxProcessorService { } } - // それでもわからなければ終了 - if (authUser == null) { + // authUser.userがnullならスキップ + if (authUser != null && authUser.user == null) { throw new Bull.UnrecoverableError('skip: failed to resolve user'); } - // publicKey がなくても終了 - if (authUser.key == null) { - // publicKeyがないのはpublicKeyの変更（主にmain→ed25519）に - // 対応しきれていない場合があるためリトライする - throw new Error(`skip: failed to resolve user publicKey: keyId=${signature.keyId}`); + if (signature != null && authUser != null) { + if (signature.keyId.toLowerCase().startsWith('acct:')) { + this.logger.warn(`Old keyId is no longer supported. lowerKeyId=${signature.keyId.toLowerCase()}`); + } else if (authUser.key != null) { + // keyがなかったらLD Signatureで検証するべき + // HTTP-Signatureの検証 + const errorLogger = (ms: any) => this.logger.error(ms); + httpSignatureIsValid = await verifyDraftSignature(signature, authUser.key.keyPem, errorLogger); + this.logger.debug('Inbox message validation: ', { + userId: authUser.user.id, + userAcct: Acct.toString(authUser.user), + parsedKeyId: signature.keyId, + foundKeyId: authUser.key.keyId, + httpSignatureValid: httpSignatureIsValid, + }); + } } - // HTTP-Signatureの検証 - const errorLogger = (ms: any) => this.logger.error(ms); - const httpSignatureValidated = await verifyDraftSignature(signature, authUser.key.keyPem, errorLogger); - this.logger.debug('Inbox message validation: ', { - userId: authUser.user.id, - userAcct: Acct.toString(authUser.user), - parsedKeyId: signature.keyId, - foundKeyId: authUser.key.keyId, - httpSignatureValidated, - }); - - // また、signatureのsignerは、activity.actorと一致する必要がある - if (httpSignatureValidated !== true || authUser.user.uri !== activity.actor) { + if ( + authUser == null || + httpSignatureIsValid !== true || + authUser.user.uri !== actorUri // 一応チェック + ) { // 一致しなくても、でもLD-Signatureがありそうならそっちも見る if (activity.signature?.creator) { if (activity.signature.type !== 'RsaSignature2017') { throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${activity.signature.type}`); } - authUser = await this.apDbResolverService.getAuthUserFromApId(activity.signature.creator.replace(/#.*/, '')); + if (activity.signature.creator.toLowerCase().startsWith('acct:')) { + throw new Bull.UnrecoverableError(`old key not supported ${activity.signature.creator}`); + } + + authUser = await this.apDbResolverService.getAuthUserFromApId(actorUri, activity.signature.creator); if (authUser == null) { - throw new Bull.UnrecoverableError('skip: LD-Signatureのユーザーが取得できませんでした'); + throw new Bull.UnrecoverableError(`skip: LD-Signatureのactorとcreatorが一致しませんでした uri=${actorUri} creator=${activity.signature.creator}`); + } + if (authUser.user == null) { + throw new Bull.UnrecoverableError(`skip: LD-Signatureのユーザーが取得できませんでした uri=${actorUri} creator=${activity.signature.creator}`); + } + // 一応actorチェック + if (authUser.user.uri !== actorUri) { + throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${actorUri})`); } - if (authUser.key == null) { - throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした'); + throw new Bull.UnrecoverableError(`skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした uri=${actorUri} creator=${activity.signature.creator}`); } // LD-Signature検証 @@ -144,18 +153,13 @@ export class InboxProcessorService { throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました'); } - // もう一度actorチェック - if (authUser.user.uri !== activity.actor) { - throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`); - } - // ブロックしてたら中断 const ldHost = this.utilityService.extractDbHost(authUser.user.uri); if (this.utilityService.isBlockedHost(meta.blockedHosts, ldHost)) { throw new Bull.UnrecoverableError(`Blocked request: ${ldHost}`); } } else { - throw new Bull.UnrecoverableError(`skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`); + throw new Bull.UnrecoverableError(`skip: http-signature verification failed and no LD-Signature. http_signature_keyId=${signature?.keyId}`); } } diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 85b1c4924b44..eaa836d965f1 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -44,7 +44,7 @@ export type DeliverJobData = { export type InboxJobData = { activity: IActivity; - signature: ParsedSignature | OldParsedSignature; + signature: ParsedSignature | OldParsedSignature | null; }; export type RelationshipJobData = { diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index 78b89e868464..4a24ddf53cb7 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -30,12 +30,17 @@ import { IActivity } from '@/core/activitypub/type.js'; import { isPureRenote } from '@/misc/is-pure-renote.js'; import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions, FastifyBodyParser } from 'fastify'; import type { FindOptionsWhere } from 'typeorm'; +import { LoggerService } from '@/core/LoggerService.js'; +import Logger from '@/logger.js'; const ACTIVITY_JSON = 'application/activity+json; charset=utf-8'; const LD_JSON = 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8'; @Injectable() export class ActivityPubServerService { + private logger: Logger; + private inboxLogger: Logger; + constructor( @Inject(DI.config) private config: Config, @@ -70,8 +75,11 @@ export class ActivityPubServerService { private queueService: QueueService, private userKeypairService: UserKeypairService, private queryService: QueryService, + private loggerService: LoggerService, ) { //this.createServer = this.createServer.bind(this); + this.logger = this.loggerService.getLogger('server-ap', 'gray', false); + this.inboxLogger = this.logger.createSubLogger('inbox', 'gray', false); } @bindThis @@ -100,10 +108,17 @@ export class ActivityPubServerService { @bindThis private async inbox(request: FastifyRequest, reply: FastifyReply) { + if (request.body == null) { + this.inboxLogger.warn('request body is empty'); + reply.code(400); + return; + } + let signature: ReturnType; const verifyDigest = await verifyDigestHeader(request.raw, request.rawBody || '', true); if (verifyDigest !== true) { + this.inboxLogger.warn('digest verification failed'); reply.code(401); return; } @@ -115,12 +130,19 @@ export class ActivityPubServerService { }, }); } catch (e) { + if (typeof request.body === 'object' && 'signature' in request.body) { + // LD SignatureがあればOK + this.queueService.inbox(request.body as IActivity, null); + reply.code(202); + return; + } + + this.inboxLogger.warn('signature header parsing failed and LD signature not found'); reply.code(401); return; } this.queueService.inbox(request.body as IActivity, signature); - reply.code(202); } From 154a2026eae812e2ecdceac463ab7e0b557c4d69 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 9 Mar 2024 19:15:40 +0000 Subject: [PATCH 090/134] inbox-delayed use actor if no signature --- .../src/server/api/endpoints/admin/queue/inbox-delayed.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/admin/queue/inbox-delayed.ts b/packages/backend/src/server/api/endpoints/admin/queue/inbox-delayed.ts index 42afa084850f..bfe230da8d9a 100644 --- a/packages/backend/src/server/api/endpoints/admin/queue/inbox-delayed.ts +++ b/packages/backend/src/server/api/endpoints/admin/queue/inbox-delayed.ts @@ -56,8 +56,8 @@ export default class extends Endpoint { // eslint- const res = [] as [string, number][]; for (const job of jobs) { - const signature = 'version' in job.data.signature ? job.data.signature.value : job.data.signature; - const host = Array.isArray(signature) ? 'TODO' : new URL(signature.keyId).host; + const signature = job.data.signature ? 'version' in job.data.signature ? job.data.signature.value : job.data.signature : null; + const host = signature ? Array.isArray(signature) ? 'TODO' : new URL(signature.keyId).host : new URL(job.data.activity.actor).host; if (res.find(x => x[0] === host)) { res.find(x => x[0] === host)![1]++; } else { From eb8495648eac739412bcf551451c2fa275ebb1a8 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 9 Mar 2024 20:01:07 +0000 Subject: [PATCH 091/134] =?UTF-8?q?=E3=83=A6=E3=83=BC=E3=82=B6=E3=83=BC?= =?UTF-8?q?=E3=81=A8=E3=82=AD=E3=83=BC=E3=81=AE=E5=90=8C=E4=B8=80=E6=80=A7?= =?UTF-8?q?=E3=83=81=E3=82=A7=E3=83=83=E3=82=AF=E3=81=AFhost=E3=81=AE?= =?UTF-8?q?=E4=B8=80=E8=87=B4=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/activitypub/ApDbResolverService.ts | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 16804a240da8..66f649c9bde2 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -18,6 +18,7 @@ import { getApId } from './type.js'; import { ApPersonService } from './models/ApPersonService.js'; import { ApLoggerService } from './ApLoggerService.js'; import type { IObject } from './type.js'; +import { UtilityService } from '../UtilityService.js'; export type UriParseResult = { /** wether the URI was generated by us */ @@ -56,11 +57,18 @@ export class ApDbResolverService implements OnApplicationShutdown { private cacheService: CacheService, private apPersonService: ApPersonService, private apLoggerService: ApLoggerService, + private utilityService: UtilityService, ) { this.publicKeyByUserIdCache = new MemoryKVCache(Infinity); this.logger = this.apLoggerService.logger.createSubLogger('db-resolver'); } + private punyHost(url: string): string { + const urlObj = new URL(url); + const host = `${this.utilityService.toPuny(urlObj.hostname)}${urlObj.port.length > 0 ? ':' + urlObj.port : ''}`; + return host; + } + @bindThis public parseUri(value: string | IObject): UriParseResult { const separator = '/'; @@ -137,9 +145,8 @@ export class ApDbResolverService implements OnApplicationShutdown { * AP Actor id => Misskey User and Key * @param uri AP Actor id * @param keyId Key id to find. If not specified, main key will be selected. - * keyIdがURLライクの場合、ハッシュを削除したkeyIdはuriと同一であることが期待される * @returns - * 1. uriとkeyIdが一致しない場合`null` + * 1. ユーザーとキーのホストが一致しない場合`null` * 2. userが見つからない場合`{ user: null, key: null }` * 3. keyが見つからない場合`{ user, key: null }` */ @@ -153,20 +160,19 @@ export class ApDbResolverService implements OnApplicationShutdown { } | null> { if (keyId) { - try { - const actorUrl = new URL(uri); - const keyUrl = new URL(keyId); - actorUrl.hash = ''; - keyUrl.hash = ''; - if (actorUrl.href !== keyUrl.href) { - // uriとkeyId（のhashなし）が一致しない場合、actorと鍵の所有者が一致していないということである - // その場合、そもそも署名は有効といえないのでキーの検索は無意味 - this.logger.warn(`actor uri and keyId are not matched uri=${uri} keyId=${keyId}`); - return null; - } - } catch (err) { - // キーがURLっぽくない場合はエラーになるはず。そういった場合はとりあえずキー検索してみる - this.logger.warn(`maybe actor uri or keyId are not url like: uri=${uri} keyId=${keyId}`, { err }); + if (this.punyHost(uri) !== this.punyHost(keyId)) { + /** + * keyIdはURL形式かつkeyIdのホストはuriのホストと一致するはず + * （ApPersonService.validateActorに由来） + * + * ただ、Mastodonはリプライ関連で他人のノートをHTTP Signature署名して送ってくることがある + * そのような署名は有効性に疑問があるので無視することにする + * ここではuriとkeyIdのホストが一致しない場合は無視する + * ハッシュをなくしたkeyIdとuriの同一性を比べてみてもいいが、`uri#*-key`というkeyIdを設定するのが + * 決まりごとというわけでもないため幅を持たせることにする + */ + this.logger.warn(`actor uri and keyId are not matched uri=${uri} keyId=${keyId}`); + return null; } } From 9bfa38e6017efe5659830b7ad015ea45d5c7ab96 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 9 Mar 2024 20:02:33 +0000 Subject: [PATCH 092/134] log signature parse err --- packages/backend/src/server/ActivityPubServerService.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index 4a24ddf53cb7..a63d346da643 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -129,7 +129,9 @@ export class ActivityPubServerService { draft: ['(request-target)', 'digest', 'host', 'date'], }, }); - } catch (e) { + } catch (err) { + this.inboxLogger.warn('signature header parsing failed', { err }); + if (typeof request.body === 'object' && 'signature' in request.body) { // LD SignatureがあればOK this.queueService.inbox(request.body as IActivity, null); From e2b574a97c3e9f717fd0b71a5f16f7bf92271381 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 10 Mar 2024 16:00:25 +0000 Subject: [PATCH 093/134] save array --- .../src/core/activitypub/models/ApPersonService.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index a977c9251d50..814c4646b675 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -396,18 +396,16 @@ export class ApPersonService implements OnModuleInit { })); if (person.publicKey) { - const keys = new Map([ + const publicKeys = new Map([ ...(person.additionalPublicKeys ? person.additionalPublicKeys.map(key => [key.id, key] as const) : []), [person.publicKey.id, person.publicKey], ]); - for (const key of keys.values()) { - await transactionalEntityManager.save(new MiUserPublickey({ - keyId: key.id, - userId: user.id, - keyPem: key.publicKeyPem, - })); - } + await this.userPublickeysRepository.save(Array.from(publicKeys.values(), key => ({ + keyId: key.id, + userId: user!.id, + keyPem: key.publicKeyPem, + }))); } }); } catch (e) { From 76487de5ed7d77aa1f3c4b9882530dfec7b7477f Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 10 Mar 2024 16:17:09 +0000 Subject: [PATCH 094/134] =?UTF-8?q?=E3=81=A8=E3=82=8A=E3=81=82=E3=81=88?= =?UTF-8?q?=E3=81=9Atry=E3=81=A7=E5=9B=B2=E3=81=A3=E3=81=A6=E3=81=8A?= =?UTF-8?q?=E3=81=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../activitypub/models/ApPersonService.ts | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 814c4646b675..1b01b9e2a996 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -551,23 +551,29 @@ export class ApPersonService implements OnModuleInit { // Update user await this.usersRepository.update(exist.id, updates); - const publicKeys = new Map(); - if (person.publicKey) { - (person.additionalPublicKeys ?? []).forEach(key => publicKeys.set(key.id, key)); - publicKeys.set(person.publicKey.id, person.publicKey); + try { + // Deleteアクティビティ受信時にもここが走ってsaveがuserforeign key制約エラーを吐くことがある + // とりあえずtry-catchで囲っておく + const publicKeys = new Map(); + if (person.publicKey) { + (person.additionalPublicKeys ?? []).forEach(key => publicKeys.set(key.id, key)); + publicKeys.set(person.publicKey.id, person.publicKey); + + await this.userPublickeysRepository.save(Array.from(publicKeys.values(), key => ({ + keyId: key.id, + userId: exist.id, + keyPem: key.publicKeyPem, + }))); + } - await this.userPublickeysRepository.save(Array.from(publicKeys.values(), key => ({ - keyId: key.id, + this.userPublickeysRepository.delete({ + keyId: Not(In(Array.from(publicKeys.keys()))), userId: exist.id, - keyPem: key.publicKeyPem, - }))); + }); + } catch (err) { + this.logger.error('something happened while updating remote user public keys:', { err }); } - this.userPublickeysRepository.delete({ - keyId: Not(In(Array.from(publicKeys.keys()))), - userId: exist.id, - }); - let _description: string | null = null; if (person._misskey_summary) { From d7c32cef70133179039695d2b5feb6a77856bddb Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 10 Mar 2024 16:38:53 +0000 Subject: [PATCH 095/134] =?UTF-8?q?fetchPersonWithRenewal=E3=81=A7?= =?UTF-8?q?=E3=82=A8=E3=83=A9=E3=83=BC=E3=81=8C=E8=B5=B7=E3=81=8D=E3=81=9F?= =?UTF-8?q?=E3=82=89=E5=8F=A4=E3=81=84=E3=83=87=E3=83=BC=E3=82=BF=E3=82=92?= =?UTF-8?q?=E8=BF=94=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/core/activitypub/models/ApPersonService.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 1b01b9e2a996..545a8af4ecb4 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -249,6 +249,12 @@ export class ApPersonService implements OnModuleInit { return null; } + /** + * uriからUser(Person)をフェッチします。 + * + * Misskeyに対象のPersonが登録されていればそれを返し、登録がなければnullを返します。 + * また、TTLが0でない場合、TTLを過ぎていた場合はupdatePersonを実行します。 + */ @bindThis async fetchPersonWithRenewal(uri: string, TTL = REMOTE_USER_CACHE_TTL): Promise { const exist = await this.fetchPerson(uri); @@ -257,8 +263,12 @@ export class ApPersonService implements OnModuleInit { if (this.userEntityService.isRemoteUser(exist)) { if (TTL === 0 || exist.lastFetchedAt == null || Date.now() - exist.lastFetchedAt.getTime() > TTL) { this.logger.debug('fetchPersonWithRenewal: renew', { uri, TTL, lastFetchedAt: exist.lastFetchedAt }); - await this.updatePerson(exist.uri); - return await this.fetchPerson(uri); + try { + await this.updatePerson(exist.uri); + return await this.fetchPerson(uri); + } catch (err) { + this.logger.error('error occurred while renewing user', { err }); + } } this.logger.debug('fetchPersonWithRenewal: use cache', { uri, TTL, lastFetchedAt: exist.lastFetchedAt }); } From aa5181cdfc698d2e0d39f09ddbd464708aa2d775 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 10 Mar 2024 16:44:05 +0000 Subject: [PATCH 096/134] use transactionalEntityManager --- packages/backend/src/core/activitypub/models/ApPersonService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 545a8af4ecb4..f315e49a5442 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -411,7 +411,7 @@ export class ApPersonService implements OnModuleInit { [person.publicKey.id, person.publicKey], ]); - await this.userPublickeysRepository.save(Array.from(publicKeys.values(), key => ({ + await transactionalEntityManager.save(Array.from(publicKeys.values(), key => new MiUserPublickey({ keyId: key.id, userId: user!.id, keyPem: key.publicKeyPem, From c58b4f8c24b67ebb1eeb575db4d071ccf7db72e7 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 2 May 2024 15:38:21 +0900 Subject: [PATCH 097/134] fix spdx --- packages/backend/test/unit/misc/gen-key-pair.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/backend/test/unit/misc/gen-key-pair.ts b/packages/backend/test/unit/misc/gen-key-pair.ts index 771147d299dc..9d76c68009c8 100644 --- a/packages/backend/test/unit/misc/gen-key-pair.ts +++ b/packages/backend/test/unit/misc/gen-key-pair.ts @@ -1,3 +1,8 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + import * as crypto from 'node:crypto'; import { genRSAAndEd25519KeyPair } from '@/misc/gen-key-pair.js'; From be102f262207b24c6af1bdd1652628c08ffe95e9 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 8 May 2024 22:03:14 +0900 Subject: [PATCH 098/134] @misskey-dev/node-http-message-signatures@0.0.10 --- packages/backend/package.json | 2 +- pnpm-lock.yaml | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index db8a695db758..34f76695d471 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -79,7 +79,7 @@ "@fastify/multipart": "8.2.0", "@fastify/static": "7.0.3", "@fastify/view": "9.1.0", - "@misskey-dev/node-http-message-signatures": "0.0.8", + "@misskey-dev/node-http-message-signatures": "0.0.10", "@misskey-dev/sharp-read-bmp": "1.2.0", "@misskey-dev/summaly": "5.1.0", "@napi-rs/canvas": "^0.1.52", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 27bdb521f875..e73ccc9edc49 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -120,8 +120,8 @@ importers: specifier: 9.1.0 version: 9.1.0 '@misskey-dev/node-http-message-signatures': - specifier: 0.0.8 - version: 0.0.8 + specifier: 0.0.10 + version: 0.0.10 '@misskey-dev/sharp-read-bmp': specifier: 1.2.0 version: 1.2.0 @@ -3012,9 +3012,10 @@ packages: '@kurkle/color@0.3.2': resolution: {integrity: sha512-fuscdXJ9G1qb7W8VdHi+IwRqij3lBkosAm4ydQtEmbY58OzHXqQhvlxqEkoz0yssNVn38bcpRWgA9PP+OGoisw==} - '@lapo/asn1js@1.3.0': - resolution: {integrity: sha512-guWBAktMu8vcPvNUCZ5YWmEf4W4vWocjh2VSMQnMa84Ojqar3a2tkJHHCKxKpWXyC6PMYpT0IjFXmxwBPe9w1A==} + '@lapo/asn1js@2.0.4': + resolution: {integrity: sha512-KJD3wQAZxozcraJdWp3utDU6DEZgAVBGp9INCdptUpZaXCEYkpwNb7h7wyYh5y6DxtpvIud8k0suhWJ/z2rKvw==} engines: {node: '>=12.20.0'} + hasBin: true '@levischuck/tiny-cbor@0.2.2': resolution: {integrity: sha512-f5CnPw997Y2GQ8FAvtuVVC19FX8mwNNC+1XJcIi16n/LTJifKO6QBgGLgN3YEmqtGMk17SKSuoWES3imJVxAVw==} @@ -3067,8 +3068,8 @@ packages: eslint: '>= 3' eslint-plugin-import: '>= 2' - '@misskey-dev/node-http-message-signatures@0.0.8': - resolution: {integrity: sha512-ymyjluFpY2xK/3UYyj+MiRfA7752BIWwSb+uFU0mFK1Yq2fGI6lAGZg3+D2lD8W/OkltnMQm1gSOXzE3CNIgtA==} + '@misskey-dev/node-http-message-signatures@0.0.10': + resolution: {integrity: sha512-HiAuc//tOU077KFUJhHYLAPWku9enTpOFIqQiK6l2i2mIizRvv7HhV7Y+yuav5quDOAz+WZGK/i5C9OR5fkKIg==} engines: {node: '>=18.4.0'} '@misskey-dev/sharp-read-bmp@1.2.0': @@ -13441,7 +13442,7 @@ snapshots: '@kurkle/color@0.3.2': {} - '@lapo/asn1js@1.3.0': {} + '@lapo/asn1js@2.0.4': {} '@levischuck/tiny-cbor@0.2.2': {} @@ -13535,9 +13536,9 @@ snapshots: eslint: 8.57.0 eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.7.1(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0) - '@misskey-dev/node-http-message-signatures@0.0.8': + '@misskey-dev/node-http-message-signatures@0.0.10': dependencies: - '@lapo/asn1js': 1.3.0 + '@lapo/asn1js': 2.0.4 rfc4648: 1.5.3 structured-headers: 1.0.1 From f31996eb42df2834fab105b0b4ccfcfe21069a72 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 11 Jun 2024 14:37:22 +0900 Subject: [PATCH 099/134] add comment --- .../backend/src/core/activitypub/ApInboxService.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index de3178b4827f..62287615b769 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -115,15 +115,8 @@ export class ApInboxService { result = await this.performOneActivity(actor, activity); } - // ついでにリモートユーザーの情報が古かったら更新しておく - if (actor.uri) { - if (actor.lastFetchedAt == null || Date.now() - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) { - setImmediate(() => { - this.apPersonService.updatePerson(actor.uri); - }); - } - } - return result; + // ついでにリモートユーザーの情報が古かったら更新しておく? + // → No, この関数が呼び出される前に署名検証で更新されているはず } @bindThis From 3717ff35a3bc464c32d08584e0251ddd061ccf67 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 11 Jun 2024 14:45:09 +0900 Subject: [PATCH 100/134] fix --- packages/backend/src/server/ActivityPubServerService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index 7745c2a4480f..753eaad0472b 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -78,8 +78,8 @@ export class ActivityPubServerService { private loggerService: LoggerService, ) { //this.createServer = this.createServer.bind(this); - this.logger = this.loggerService.getLogger('server-ap', 'gray', false); - this.inboxLogger = this.logger.createSubLogger('inbox', 'gray', false); + this.logger = this.loggerService.getLogger('server-ap', 'gray'); + this.inboxLogger = this.logger.createSubLogger('inbox', 'gray'); } @bindThis From 64004fdea244f039c34e9f59a103be9b9e164f77 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 11 Jun 2024 15:32:55 +0900 Subject: [PATCH 101/134] =?UTF-8?q?publicKey=E3=81=AB=E9=85=8D=E5=88=97?= =?UTF-8?q?=E3=81=8C=E5=85=A5=E3=81=A3=E3=81=A6=E3=82=82=E3=81=84=E3=81=84?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=E3=81=99=E3=82=8B=20https://github.?= =?UTF-8?q?com/misskey-dev/misskey/pull/13950?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../activitypub/models/ApPersonService.ts | 21 +++++++++++++------ packages/backend/src/core/activitypub/type.ts | 2 +- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index f315e49a5442..a712ca95cac0 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -184,15 +184,24 @@ export class ApPersonService implements OnModuleInit { throw new Error('invalid Actor: id has different host'); } - if (x.publicKey) { - if (typeof x.publicKey.id !== 'string') { - throw new Error('invalid Actor: publicKey.id is not a string'); - } - + if (x.publicKey && typeof x.publicKey.id !== 'string') { const publicKeyIdHost = this.punyHost(x.publicKey.id); if (publicKeyIdHost !== expectHost) { throw new Error('invalid Actor: publicKey.id has different host'); } + } else if (x.publicKey && Array.isArray(x.publicKey)) { + for (const publicKey of x.publicKey) { + if (typeof publicKey.id !== 'string') { + throw new Error('invalid Actor: publicKey.id is not a string'); + } + + const publicKeyIdHost = this.punyHost(publicKey.id); + if (publicKeyIdHost !== expectHost) { + throw new Error('invalid Actor: publicKey.id has different host'); + } + } + } else if (x.publicKey) { + throw new Error('invalid Actor: publicKey is not an object or an array'); } if (x.additionalPublicKeys) { @@ -408,7 +417,7 @@ export class ApPersonService implements OnModuleInit { if (person.publicKey) { const publicKeys = new Map([ ...(person.additionalPublicKeys ? person.additionalPublicKeys.map(key => [key.id, key] as const) : []), - [person.publicKey.id, person.publicKey], + ...(Array.isArray(person.publicKey) ? person.publicKey.map(key => [key.id, key] as const) : [[person.publicKey.id, person.publicKey]] as const), ]); await transactionalEntityManager.save(Array.from(publicKeys.values(), key => new MiUserPublickey({ diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 0cd941ae193e..368650a83d20 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -169,7 +169,7 @@ export interface IActor extends IObject { discoverable?: boolean; inbox: string; sharedInbox?: string; // 後方互換性のため - publicKey?: IKey; + publicKey?: IKey | IKey[]; additionalPublicKeys?: IKey[]; followers?: string | ICollection | IOrderedCollection; following?: string | ICollection | IOrderedCollection; From 133970a184fe8ab955055def3248665b7e0710cf Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 11 Jun 2024 15:42:17 +0900 Subject: [PATCH 102/134] define additionalPublicKeys --- packages/backend/src/core/activitypub/misc/contexts.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend/src/core/activitypub/misc/contexts.ts b/packages/backend/src/core/activitypub/misc/contexts.ts index feb8c42c563c..fc4e3e3bef6e 100644 --- a/packages/backend/src/core/activitypub/misc/contexts.ts +++ b/packages/backend/src/core/activitypub/misc/contexts.ts @@ -134,6 +134,7 @@ const security_v1 = { 'privateKey': { '@id': 'sec:privateKey', '@type': '@id' }, 'privateKeyPem': 'sec:privateKeyPem', 'publicKey': { '@id': 'sec:publicKey', '@type': '@id' }, + 'additionalPublicKeys': { '@id': 'sec:publicKey', '@type': '@id' }, 'publicKeyBase58': 'sec:publicKeyBase58', 'publicKeyPem': 'sec:publicKeyPem', 'publicKeyWif': 'sec:publicKeyWif', From a3d4eae99d847466fa041810c0066a284dfd32e5 Mon Sep 17 00:00:00 2001 From: tamaina Date: Tue, 11 Jun 2024 15:50:55 +0900 Subject: [PATCH 103/134] fix --- .../activitypub/models/ApPersonService.ts | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index a712ca95cac0..7ff125466033 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -184,13 +184,10 @@ export class ApPersonService implements OnModuleInit { throw new Error('invalid Actor: id has different host'); } - if (x.publicKey && typeof x.publicKey.id !== 'string') { - const publicKeyIdHost = this.punyHost(x.publicKey.id); - if (publicKeyIdHost !== expectHost) { - throw new Error('invalid Actor: publicKey.id has different host'); - } - } else if (x.publicKey && Array.isArray(x.publicKey)) { - for (const publicKey of x.publicKey) { + if (x.publicKey) { + const publicKeys = Array.isArray(x.publicKey) ? x.publicKey : [x.publicKey]; + + for (const publicKey of publicKeys) { if (typeof publicKey.id !== 'string') { throw new Error('invalid Actor: publicKey.id is not a string'); } @@ -200,8 +197,6 @@ export class ApPersonService implements OnModuleInit { throw new Error('invalid Actor: publicKey.id has different host'); } } - } else if (x.publicKey) { - throw new Error('invalid Actor: publicKey is not an object or an array'); } if (x.additionalPublicKeys) { @@ -415,10 +410,9 @@ export class ApPersonService implements OnModuleInit { })); if (person.publicKey) { - const publicKeys = new Map([ - ...(person.additionalPublicKeys ? person.additionalPublicKeys.map(key => [key.id, key] as const) : []), - ...(Array.isArray(person.publicKey) ? person.publicKey.map(key => [key.id, key] as const) : [[person.publicKey.id, person.publicKey]] as const), - ]); + const publicKeys = new Map(); + (person.additionalPublicKeys ?? []).forEach(key => publicKeys.set(key.id, key)); + (Array.isArray(person.publicKey) ? person.publicKey : [person.publicKey]).forEach(key => publicKeys.set(key.id, key)); await transactionalEntityManager.save(Array.from(publicKeys.values(), key => new MiUserPublickey({ keyId: key.id, @@ -576,7 +570,7 @@ export class ApPersonService implements OnModuleInit { const publicKeys = new Map(); if (person.publicKey) { (person.additionalPublicKeys ?? []).forEach(key => publicKeys.set(key.id, key)); - publicKeys.set(person.publicKey.id, person.publicKey); + (Array.isArray(person.publicKey) ? person.publicKey : [person.publicKey]).forEach(key => publicKeys.set(key.id, key)); await this.userPublickeysRepository.save(Array.from(publicKeys.values(), key => ({ keyId: key.id, From 7353c7397f6064e265c4233aa500b237d312c4b2 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 29 Jun 2024 09:24:14 +0900 Subject: [PATCH 104/134] merge fix --- packages/backend/src/core/activitypub/ApRequestService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 420844cc174b..9359dccc863d 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -5,7 +5,7 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; -import { genRFC3230DigestHeader, RequestLike, signAsDraftToRequest } from '@misskey-dev/node-http-message-signatures'; +import { genRFC3230DigestHeader, signAsDraftToRequest } from '@misskey-dev/node-http-message-signatures'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import type { MiUser } from '@/models/User.js'; From c80b16cdf8140d8d1fcb41e2fd190ff78eef4237 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 15:29:13 +0900 Subject: [PATCH 105/134] =?UTF-8?q?refreshAndprepareEd25519KeyPair=20?= =?UTF-8?q?=E2=86=92=20refreshAndPrepareEd25519KeyPair?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/core/UserKeypairService.ts | 2 +- .../backend/src/core/activitypub/ApDeliverManagerService.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 74d1257bc364..9edc8ecc2753 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -141,7 +141,7 @@ export class UserKeypairService implements OnApplicationShutdown { * @returns MiUserKeypair if keypair is created, void if keypair is already exists */ @bindThis - public async refreshAndprepareEd25519KeyPair(userId: MiUser['id']): Promise { + public async refreshAndPrepareEd25519KeyPair(userId: MiUser['id']): Promise { await this.refresh(userId); const keypair = await this.keypairEntityCache.fetch(userId); if (keypair.ed25519PublicKey != null) { diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index e09b548d3a8f..decb02156131 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -135,7 +135,7 @@ class DeliverManager { /** * ed25519の署名がなければ追加する */ - const created = await this.userKeypairService.refreshAndprepareEd25519KeyPair(this.actor.id); + const created = await this.userKeypairService.refreshAndPrepareEd25519KeyPair(this.actor.id); if (created) { // createdが存在するということは新規作成されたということなので、フォロワーに配信する this.logger.info(`ed25519 key pair created for user ${this.actor.id} and publishing to followers`); From 7e2c3e4439a6539aeea014af43e9f005ef806a7b Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 16:03:44 +0900 Subject: [PATCH 106/134] remove gen-key-pair.ts --- .../backend/test/unit/misc/gen-key-pair.ts | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 packages/backend/test/unit/misc/gen-key-pair.ts diff --git a/packages/backend/test/unit/misc/gen-key-pair.ts b/packages/backend/test/unit/misc/gen-key-pair.ts deleted file mode 100644 index 9d76c68009c8..000000000000 --- a/packages/backend/test/unit/misc/gen-key-pair.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * SPDX-FileCopyrightText: syuilo and misskey-project - * SPDX-License-Identifier: AGPL-3.0-only - */ - -import * as crypto from 'node:crypto'; -import { genRSAAndEd25519KeyPair } from '@/misc/gen-key-pair.js'; - -describe(genRSAAndEd25519KeyPair, () => { - test('generates key pair', async () => { - const keyPair = await genRSAAndEd25519KeyPair(); - // 毎回違うキーペアが生成されることを確認するために2回生成して比較してみる - const keyPair2 = await genRSAAndEd25519KeyPair(); - console.log(Object.entries(keyPair).map(([k, v]) => `${k}: ${v.length}`).join('\n')); - console.log(Object.entries(keyPair).map(([k, v]) => `${k}\n${v}`).join('\n')); - - expect(keyPair.publicKey).toMatch(/^-----BEGIN PUBLIC KEY-----/); - expect(keyPair.publicKey).toMatch(/-----END PUBLIC KEY-----\n$/); - expect(keyPair.publicKey).not.toBe(keyPair2.publicKey); - - const publicKeyObj = crypto.createPublicKey(keyPair.publicKey); - expect(publicKeyObj.asymmetricKeyType).toBe('rsa'); - - expect(keyPair.privateKey).toMatch(/^-----BEGIN PRIVATE KEY-----/); - expect(keyPair.privateKey).toMatch(/-----END PRIVATE KEY-----\n$/); - expect(keyPair.privateKey).not.toBe(keyPair2.privateKey); - expect(keyPair.ed25519PublicKey).toMatch(/^-----BEGIN PUBLIC KEY-----/); - expect(keyPair.ed25519PublicKey).toMatch(/-----END PUBLIC KEY-----\n$/); - expect(keyPair.ed25519PublicKey).not.toBe(keyPair2.ed25519PublicKey); - - const ed25519PublicKeyObj = crypto.createPublicKey(keyPair.ed25519PublicKey); - expect(ed25519PublicKeyObj.asymmetricKeyType).toBe('ed25519'); - - expect(keyPair.ed25519PrivateKey).toMatch(/^-----BEGIN PRIVATE KEY-----/); - expect(keyPair.ed25519PrivateKey).toMatch(/-----END PRIVATE KEY-----\n$/); - expect(keyPair.ed25519PrivateKey).not.toBe(keyPair2.ed25519PrivateKey); - //const imported = await webCrypto.subtle.importKey('spki', Buffer.from(keyPair.publicKey).buffer, { name: 'rsa-pss', hash: 'sha-256' }, false, ['verify']); - }); -}); From f2c412c18057a9300540794ccbe4dfbf6d259ed6 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 16:14:24 +0900 Subject: [PATCH 107/134] defaultMaxListeners = 512 --- packages/backend/src/boot/entry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/boot/entry.ts b/packages/backend/src/boot/entry.ts index 25375c3015f7..aef999b68363 100644 --- a/packages/backend/src/boot/entry.ts +++ b/packages/backend/src/boot/entry.ts @@ -22,7 +22,7 @@ import 'reflect-metadata'; process.title = `Misskey (${cluster.isPrimary ? 'master' : 'worker'})`; Error.stackTraceLimit = Infinity; -EventEmitter.defaultMaxListeners = 128; +EventEmitter.defaultMaxListeners = 512; const logger = new Logger('core', 'cyan'); const clusterLogger = logger.createSubLogger('cluster', 'orange'); From 57bfffedae107864ee8b305cf3e3cd6de6958ef3 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 16:17:09 +0900 Subject: [PATCH 108/134] Revert "defaultMaxListeners = 512" This reverts commit f2c412c18057a9300540794ccbe4dfbf6d259ed6. --- packages/backend/src/boot/entry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/boot/entry.ts b/packages/backend/src/boot/entry.ts index aef999b68363..25375c3015f7 100644 --- a/packages/backend/src/boot/entry.ts +++ b/packages/backend/src/boot/entry.ts @@ -22,7 +22,7 @@ import 'reflect-metadata'; process.title = `Misskey (${cluster.isPrimary ? 'master' : 'worker'})`; Error.stackTraceLimit = Infinity; -EventEmitter.defaultMaxListeners = 512; +EventEmitter.defaultMaxListeners = 128; const logger = new Logger('core', 'cyan'); const clusterLogger = logger.createSubLogger('cluster', 'orange'); From d0aada55c1ed5aa98f18731ec82f3ac5eb5a6c16 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 16:20:39 +0900 Subject: [PATCH 109/134] =?UTF-8?q?genRSAAndEd25519KeyPair=E3=81=A7?= =?UTF-8?q?=E3=81=AF=E3=82=AD=E3=83=BC=E3=82=92=E7=9B=B4=E5=88=97=E3=81=AB?= =?UTF-8?q?=E7=94=9F=E6=88=90=E3=81=99=E3=82=8B=3F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/misc/gen-key-pair.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/misc/gen-key-pair.ts b/packages/backend/src/misc/gen-key-pair.ts index 0b033ec33e2a..58e3091369c0 100644 --- a/packages/backend/src/misc/gen-key-pair.ts +++ b/packages/backend/src/misc/gen-key-pair.ts @@ -6,7 +6,8 @@ import { genEd25519KeyPair, genRsaKeyPair } from '@misskey-dev/node-http-message-signatures'; export async function genRSAAndEd25519KeyPair(rsaModulusLength = 4096) { - const [rsa, ed25519] = await Promise.all([genRsaKeyPair(rsaModulusLength), genEd25519KeyPair()]); + const rsa = await genRsaKeyPair(rsaModulusLength); + const ed25519 = await genEd25519KeyPair(); return { publicKey: rsa.publicKey, privateKey: rsa.privateKey, From 613c1273b84be74d80743afd0d203f99384296c5 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 16:30:54 +0900 Subject: [PATCH 110/134] maxConcurrency: 8 --- packages/backend/jest.config.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/jest.config.cjs b/packages/backend/jest.config.cjs index 5a4aa4e15aa4..1552fa36209a 100644 --- a/packages/backend/jest.config.cjs +++ b/packages/backend/jest.config.cjs @@ -216,5 +216,5 @@ module.exports = { logHeapUsage: true, // To debug when out-of-memory happens on CI workerIdleMemoryLimit: '1GiB', // Limit the worker to 1GB (GitHub Workflows dies at 2GB) - maxConcurrency: 32, + maxConcurrency: 8, }; From a80a7f6458ac20bad167b6ba7804c3c12db4a2a3 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 16:53:34 +0900 Subject: [PATCH 111/134] maxConcurrency: 16 --- packages/backend/jest.config.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/jest.config.cjs b/packages/backend/jest.config.cjs index 1552fa36209a..d5565c8412e3 100644 --- a/packages/backend/jest.config.cjs +++ b/packages/backend/jest.config.cjs @@ -216,5 +216,5 @@ module.exports = { logHeapUsage: true, // To debug when out-of-memory happens on CI workerIdleMemoryLimit: '1GiB', // Limit the worker to 1GB (GitHub Workflows dies at 2GB) - maxConcurrency: 8, + maxConcurrency: 16, }; From c2d084bac4857b877c7f9deded389906242485d5 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 16:58:55 +0900 Subject: [PATCH 112/134] maxConcurrency: 8 --- packages/backend/jest.config.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/jest.config.cjs b/packages/backend/jest.config.cjs index d5565c8412e3..1552fa36209a 100644 --- a/packages/backend/jest.config.cjs +++ b/packages/backend/jest.config.cjs @@ -216,5 +216,5 @@ module.exports = { logHeapUsage: true, // To debug when out-of-memory happens on CI workerIdleMemoryLimit: '1GiB', // Limit the worker to 1GB (GitHub Workflows dies at 2GB) - maxConcurrency: 16, + maxConcurrency: 8, }; From d3280fe7b365aa1f4577765623cfacbe15ae2448 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 16:59:01 +0900 Subject: [PATCH 113/134] =?UTF-8?q?Revert=20"genRSAAndEd25519KeyPair?= =?UTF-8?q?=E3=81=A7=E3=81=AF=E3=82=AD=E3=83=BC=E3=82=92=E7=9B=B4=E5=88=97?= =?UTF-8?q?=E3=81=AB=E7=94=9F=E6=88=90=E3=81=99=E3=82=8B=3F"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit d0aada55c1ed5aa98f18731ec82f3ac5eb5a6c16. --- packages/backend/src/misc/gen-key-pair.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/backend/src/misc/gen-key-pair.ts b/packages/backend/src/misc/gen-key-pair.ts index 58e3091369c0..0b033ec33e2a 100644 --- a/packages/backend/src/misc/gen-key-pair.ts +++ b/packages/backend/src/misc/gen-key-pair.ts @@ -6,8 +6,7 @@ import { genEd25519KeyPair, genRsaKeyPair } from '@misskey-dev/node-http-message-signatures'; export async function genRSAAndEd25519KeyPair(rsaModulusLength = 4096) { - const rsa = await genRsaKeyPair(rsaModulusLength); - const ed25519 = await genEd25519KeyPair(); + const [rsa, ed25519] = await Promise.all([genRsaKeyPair(rsaModulusLength), genEd25519KeyPair()]); return { publicKey: rsa.publicKey, privateKey: rsa.privateKey, From 9e0a93f110456320d6485a871f014f7cdab29b33 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 17:08:08 +0900 Subject: [PATCH 114/134] maxWorkers: '90%' --- packages/backend/jest.config.cjs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/backend/jest.config.cjs b/packages/backend/jest.config.cjs index 1552fa36209a..a4bd6ea9a262 100644 --- a/packages/backend/jest.config.cjs +++ b/packages/backend/jest.config.cjs @@ -23,7 +23,7 @@ module.exports = { collectCoverageFrom: ['src/**/*.ts', '!src/**/*.test.ts'], // The directory where Jest should output its coverage files - coverageDirectory: "coverage", + coverageDirectory: 'coverage', // An array of regexp pattern strings used to skip coverage collection // coveragePathIgnorePatterns: [ @@ -31,7 +31,7 @@ module.exports = { // ], // Indicates which provider should be used to instrument code for coverage - coverageProvider: "v8", + coverageProvider: 'v8', // A list of reporter names that Jest uses when writing coverage reports // coverageReporters: [ @@ -129,7 +129,7 @@ module.exports = { // A list of paths to directories that Jest should use to search for files in roots: [ - "" + '', ], // Allows you to use a custom runner instead of Jest's default test runner @@ -148,7 +148,7 @@ module.exports = { // snapshotSerializers: [], // The test environment that will be used for testing - testEnvironment: "node", + testEnvironment: 'node', // Options that will be passed to the testEnvironment // testEnvironmentOptions: {}, @@ -158,8 +158,8 @@ module.exports = { // The glob patterns Jest uses to detect test files testMatch: [ - "/test/unit/**/*.ts", - "/src/**/*.test.ts", + '/test/unit/**/*.ts', + '/src/**/*.test.ts', ], // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped @@ -184,7 +184,7 @@ module.exports = { // A map from regular expressions to paths to transformers transform: { - "^.+\\.(t|j)sx?$": ["@swc/jest"], + '^.+\\.(t|j)sx?$': ['@swc/jest'], }, // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation @@ -212,7 +212,7 @@ module.exports = { // Let Jest kill the test worker whenever it grows too much // (It seems there's a known memory leak issue in Node.js' vm.Script used by Jest) // https://github.com/facebook/jest/issues/11956 - maxWorkers: 1, // Make it use worker (that can be killed and restarted) + maxWorkers: '90%', // Make it use worker (that can be killed and restarted) logHeapUsage: true, // To debug when out-of-memory happens on CI workerIdleMemoryLimit: '1GiB', // Limit the worker to 1GB (GitHub Workflows dies at 2GB) From 38a5e09a3693e57bc65e27b45457a6812b0727d1 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 17:10:10 +0900 Subject: [PATCH 115/134] Revert "maxWorkers: '90%'" This reverts commit 9e0a93f110456320d6485a871f014f7cdab29b33. --- packages/backend/jest.config.cjs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/backend/jest.config.cjs b/packages/backend/jest.config.cjs index a4bd6ea9a262..1552fa36209a 100644 --- a/packages/backend/jest.config.cjs +++ b/packages/backend/jest.config.cjs @@ -23,7 +23,7 @@ module.exports = { collectCoverageFrom: ['src/**/*.ts', '!src/**/*.test.ts'], // The directory where Jest should output its coverage files - coverageDirectory: 'coverage', + coverageDirectory: "coverage", // An array of regexp pattern strings used to skip coverage collection // coveragePathIgnorePatterns: [ @@ -31,7 +31,7 @@ module.exports = { // ], // Indicates which provider should be used to instrument code for coverage - coverageProvider: 'v8', + coverageProvider: "v8", // A list of reporter names that Jest uses when writing coverage reports // coverageReporters: [ @@ -129,7 +129,7 @@ module.exports = { // A list of paths to directories that Jest should use to search for files in roots: [ - '', + "" ], // Allows you to use a custom runner instead of Jest's default test runner @@ -148,7 +148,7 @@ module.exports = { // snapshotSerializers: [], // The test environment that will be used for testing - testEnvironment: 'node', + testEnvironment: "node", // Options that will be passed to the testEnvironment // testEnvironmentOptions: {}, @@ -158,8 +158,8 @@ module.exports = { // The glob patterns Jest uses to detect test files testMatch: [ - '/test/unit/**/*.ts', - '/src/**/*.test.ts', + "/test/unit/**/*.ts", + "/src/**/*.test.ts", ], // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped @@ -184,7 +184,7 @@ module.exports = { // A map from regular expressions to paths to transformers transform: { - '^.+\\.(t|j)sx?$': ['@swc/jest'], + "^.+\\.(t|j)sx?$": ["@swc/jest"], }, // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation @@ -212,7 +212,7 @@ module.exports = { // Let Jest kill the test worker whenever it grows too much // (It seems there's a known memory leak issue in Node.js' vm.Script used by Jest) // https://github.com/facebook/jest/issues/11956 - maxWorkers: '90%', // Make it use worker (that can be killed and restarted) + maxWorkers: 1, // Make it use worker (that can be killed and restarted) logHeapUsage: true, // To debug when out-of-memory happens on CI workerIdleMemoryLimit: '1GiB', // Limit the worker to 1GB (GitHub Workflows dies at 2GB) From 44f00643010bfc96b1c603cc80579d0baed283b6 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 17:33:27 +0900 Subject: [PATCH 116/134] =?UTF-8?q?e2e/timelines.ts=E3=81=A7=E5=80=8B?= =?UTF-8?q?=E3=80=85=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E3=81=AB=E5=AF=BE?= =?UTF-8?q?=E3=81=99=E3=82=8Btimeout=E3=82=92=E5=89=8A=E9=99=A4,=20maxConc?= =?UTF-8?q?urrency:=2032?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/jest.config.cjs | 2 +- packages/backend/test/e2e/timelines.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/backend/jest.config.cjs b/packages/backend/jest.config.cjs index 1552fa36209a..5a4aa4e15aa4 100644 --- a/packages/backend/jest.config.cjs +++ b/packages/backend/jest.config.cjs @@ -216,5 +216,5 @@ module.exports = { logHeapUsage: true, // To debug when out-of-memory happens on CI workerIdleMemoryLimit: '1GiB', // Limit the worker to 1GB (GitHub Workflows dies at 2GB) - maxConcurrency: 8, + maxConcurrency: 32, }; diff --git a/packages/backend/test/e2e/timelines.ts b/packages/backend/test/e2e/timelines.ts index 540b866b2816..fce1eacf00a1 100644 --- a/packages/backend/test/e2e/timelines.ts +++ b/packages/backend/test/e2e/timelines.ts @@ -378,7 +378,7 @@ describe('Timelines', () => { assert.strictEqual(res.body.some(note => note.id === bobNote2.id), true); assert.strictEqual(res.body.some(note => note.id === carolNote1.id), false); assert.strictEqual(res.body.some(note => note.id === carolNote2.id), false); - }, 1000 * 10); + }); test.concurrent('フォローしているユーザーのチャンネル投稿が含まれない', async () => { const [alice, bob] = await Promise.all([signup(), signup()]); @@ -672,7 +672,7 @@ describe('Timelines', () => { assert.strictEqual(res.body.some(note => note.id === bobNote1.id), false); assert.strictEqual(res.body.some(note => note.id === bobNote2.id), true); - }, 1000 * 10); + }); }); describe('Social TL', () => { @@ -812,7 +812,7 @@ describe('Timelines', () => { assert.strictEqual(res.body.some(note => note.id === bobNote1.id), false); assert.strictEqual(res.body.some(note => note.id === bobNote2.id), true); - }, 1000 * 10); + }); }); describe('User List TL', () => { @@ -1025,7 +1025,7 @@ describe('Timelines', () => { assert.strictEqual(res.body.some(note => note.id === bobNote1.id), false); assert.strictEqual(res.body.some(note => note.id === bobNote2.id), true); - }, 1000 * 10); + }); test.concurrent('リスインしているユーザーの自身宛ての visibility: specified なノートが含まれる', async () => { const [alice, bob] = await Promise.all([signup(), signup()]); @@ -1184,7 +1184,7 @@ describe('Timelines', () => { assert.strictEqual(res.body.some(note => note.id === bobNote1.id), false); assert.strictEqual(res.body.some(note => note.id === bobNote2.id), true); - }, 1000 * 10); + }); test.concurrent('[withChannelNotes: true] チャンネル投稿が含まれる', async () => { const [alice, bob] = await Promise.all([signup(), signup()]); From 09b2e71e62179ff102120ce478ec7efa1ffbf111 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 17:57:45 +0900 Subject: [PATCH 117/134] better error handling of this.userPublickeysRepository.delete --- .../backend/src/core/activitypub/models/ApPersonService.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index bea9d02abdbb..c41fc713d5fc 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -589,9 +589,11 @@ export class ApPersonService implements OnModuleInit { this.userPublickeysRepository.delete({ keyId: Not(In(Array.from(publicKeys.keys()))), userId: exist.id, + }).catch(err => { + this.logger.error('something happened while deleting remote user public keys:', { userId: exist.id, err }); }); } catch (err) { - this.logger.error('something happened while updating remote user public keys:', { err }); + this.logger.error('something happened while updating remote user public keys:', { userId: exist.id, err }); } let _description: string | null = null; From 41883c451d88253705d6133a4dad2ecb19b1b26a Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 18:01:03 +0900 Subject: [PATCH 118/134] better comment --- packages/backend/src/core/AccountUpdateService.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/core/AccountUpdateService.ts b/packages/backend/src/core/AccountUpdateService.ts index d4a7b145546d..ca0864f6799a 100644 --- a/packages/backend/src/core/AccountUpdateService.ts +++ b/packages/backend/src/core/AccountUpdateService.ts @@ -36,9 +36,9 @@ export class AccountUpdateService implements OnModuleInit { @bindThis /** - * ユーザーのアップデートをフォロワーに配信する - * @param userId ユーザーID - * @param isKeyUpdation Ed25519キーの作成など公開鍵のアップデートによる呼び出しか？ trueにするとメインキーを使うようになる + * Deliver account update to followers + * @param userId user id + * @param deliverKey optional. Private key to sign the deliver. */ public async publishToFollowers(userId: MiUser['id'], deliverKey?: PrivateKeyWithPem) { const user = await this.usersRepository.findOneBy({ id: userId }); From 5afc659afab86e35542804b715af096498b1601d Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 18:15:17 +0900 Subject: [PATCH 119/134] set result to keypairEntityCache --- packages/backend/src/core/UserKeypairService.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 9edc8ecc2753..de8f7c17da7e 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -136,7 +136,8 @@ export class UserKeypairService implements OnApplicationShutdown { } /** - * + * If DB has ed25519 keypair, refresh cache and return it. + * If not, create, save and return ed25519 keypair. * @param userId user id * @returns MiUserKeypair if keypair is created, void if keypair is already exists */ @@ -154,11 +155,13 @@ export class UserKeypairService implements OnApplicationShutdown { ed25519PrivateKey: ed25519.privateKey, }); this.globalEventService.publishInternalEvent('userKeypairUpdated', { userId }); - return { + const result = { ...keypair, ed25519PublicKey: ed25519.publicKey, ed25519PrivateKey: ed25519.privateKey, }; + this.keypairEntityCache.set(userId, result); + return result; } @bindThis From ffd12d0539785ccb9d51bd7423abeb529f95d4e0 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 18:36:04 +0900 Subject: [PATCH 120/134] deliverJobConcurrency: 16, deliverJobPerSec: 1024, inboxJobConcurrency: 4 --- .config/docker_example.yml | 4 ++-- .config/example.yml | 6 +++--- .devcontainer/devcontainer.yml | 6 +++--- chart/files/default.yml | 6 +++--- packages/backend/src/core/HttpRequestService.ts | 2 +- packages/backend/src/queue/QueueProcessorService.ts | 6 +++--- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.config/docker_example.yml b/.config/docker_example.yml index d347882d1a91..985ba989c4bc 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -164,8 +164,8 @@ id: 'aidx' #clusterLimit: 1 # Job concurrency per worker -# deliverJobConcurrency: 128 -# inboxJobConcurrency: 16 +# deliverJobConcurrency: 16 +# inboxJobConcurrency: 4 # Job rate limiter # deliverJobPerSec: 128 diff --git a/.config/example.yml b/.config/example.yml index b11cbd137328..54f21aa0f9c0 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -230,14 +230,14 @@ id: 'aidx' #clusterLimit: 1 # Job concurrency per worker -#deliverJobConcurrency: 128 -#inboxJobConcurrency: 16 +#deliverJobConcurrency: 16 +#inboxJobConcurrency: 4 #relationshipJobConcurrency: 16 # What's relationshipJob?: # Follow, unfollow, block and unblock(ings) while following-imports, etc. or account migrations. # Job rate limiter -#deliverJobPerSec: 128 +#deliverJobPerSec: 1024 #inboxJobPerSec: 32 #relationshipJobPerSec: 64 diff --git a/.devcontainer/devcontainer.yml b/.devcontainer/devcontainer.yml index beefcfd0a2d5..42c81588d613 100644 --- a/.devcontainer/devcontainer.yml +++ b/.devcontainer/devcontainer.yml @@ -157,11 +157,11 @@ id: 'aidx' #clusterLimit: 1 # Job concurrency per worker -# deliverJobConcurrency: 128 -# inboxJobConcurrency: 16 +# deliverJobConcurrency: 16 +# inboxJobConcurrency: 4 # Job rate limiter -# deliverJobPerSec: 128 +# deliverJobPerSec: 1024 # inboxJobPerSec: 32 # Job attempts diff --git a/chart/files/default.yml b/chart/files/default.yml index f98b8ebfee04..76062380489f 100644 --- a/chart/files/default.yml +++ b/chart/files/default.yml @@ -178,11 +178,11 @@ id: "aidx" #clusterLimit: 1 # Job concurrency per worker -# deliverJobConcurrency: 128 -# inboxJobConcurrency: 16 +# deliverJobConcurrency: 16 +# inboxJobConcurrency: 4 # Job rate limiter -# deliverJobPerSec: 128 +# deliverJobPerSec: 1024 # inboxJobPerSec: 32 # Job attempts diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index 7f3cac7c5807..4249c158d7cc 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -70,7 +70,7 @@ export class HttpRequestService { localAddress: config.outgoingAddress, }); - const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128); + const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 16); this.httpAgent = config.proxy ? new HttpProxyAgent({ diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index 7bd74f3210f8..df56391e9e12 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -250,9 +250,9 @@ export class QueueProcessorService implements OnApplicationShutdown { }, { ...baseQueueOptions(this.config, QUEUE.DELIVER), autorun: false, - concurrency: this.config.deliverJobConcurrency ?? 128, + concurrency: this.config.deliverJobConcurrency ?? 16, limiter: { - max: this.config.deliverJobPerSec ?? 128, + max: this.config.deliverJobPerSec ?? 1024, duration: 1000, }, settings: { @@ -290,7 +290,7 @@ export class QueueProcessorService implements OnApplicationShutdown { }, { ...baseQueueOptions(this.config, QUEUE.INBOX), autorun: false, - concurrency: this.config.inboxJobConcurrency ?? 16, + concurrency: this.config.inboxJobConcurrency ?? 4, limiter: { max: this.config.inboxJobPerSec ?? 32, duration: 1000, From fe77f216c38802200f1ba4dca58d6e1a6663d49a Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 22:49:07 +0900 Subject: [PATCH 121/134] inboxJobPerSec: 64 --- .config/docker_example.yml | 2 +- .config/example.yml | 2 +- .devcontainer/devcontainer.yml | 2 +- chart/files/default.yml | 2 +- packages/backend/src/queue/QueueProcessorService.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.config/docker_example.yml b/.config/docker_example.yml index 985ba989c4bc..bd0ad2872af6 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -169,7 +169,7 @@ id: 'aidx' # Job rate limiter # deliverJobPerSec: 128 -# inboxJobPerSec: 32 +# inboxJobPerSec: 64 # Job attempts # deliverJobMaxAttempts: 12 diff --git a/.config/example.yml b/.config/example.yml index 54f21aa0f9c0..0d525f61c4ee 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -238,7 +238,7 @@ id: 'aidx' # Job rate limiter #deliverJobPerSec: 1024 -#inboxJobPerSec: 32 +#inboxJobPerSec: 64 #relationshipJobPerSec: 64 # Job attempts diff --git a/.devcontainer/devcontainer.yml b/.devcontainer/devcontainer.yml index 42c81588d613..d74d741e02c0 100644 --- a/.devcontainer/devcontainer.yml +++ b/.devcontainer/devcontainer.yml @@ -162,7 +162,7 @@ id: 'aidx' # Job rate limiter # deliverJobPerSec: 1024 -# inboxJobPerSec: 32 +# inboxJobPerSec: 64 # Job attempts # deliverJobMaxAttempts: 12 diff --git a/chart/files/default.yml b/chart/files/default.yml index 76062380489f..4017588fa075 100644 --- a/chart/files/default.yml +++ b/chart/files/default.yml @@ -183,7 +183,7 @@ id: "aidx" # Job rate limiter # deliverJobPerSec: 1024 -# inboxJobPerSec: 32 +# inboxJobPerSec: 64 # Job attempts # deliverJobMaxAttempts: 12 diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index df56391e9e12..169b22c3f52a 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -292,7 +292,7 @@ export class QueueProcessorService implements OnApplicationShutdown { autorun: false, concurrency: this.config.inboxJobConcurrency ?? 4, limiter: { - max: this.config.inboxJobPerSec ?? 32, + max: this.config.inboxJobPerSec ?? 64, duration: 1000, }, settings: { From b7349e5771ad29377e97b7a1ed3c46f3960f196e Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 23:03:30 +0900 Subject: [PATCH 122/134] delete request.headers['host']; --- packages/backend/src/core/activitypub/ApRequestService.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 9359dccc863d..1b0ae6b5aaee 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -40,6 +40,8 @@ export async function createSignedPost(args: { level: string; key: PrivateKey; u ['(request-target)', 'date', 'host', 'digest'], ); + delete request.headers['host']; + return { request, ...result, @@ -66,6 +68,8 @@ export async function createSignedGet(args: { level: string; key: PrivateKey; ur ['(request-target)', 'date', 'host', 'accept'], ); + delete request.headers['host']; + return { request, ...result, From 1f0e7a40b6a120555fd908f9f305c086af717e12 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 23:26:57 +0900 Subject: [PATCH 123/134] fix --- packages/backend/src/core/activitypub/ApRequestService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 1b0ae6b5aaee..e48c1865d91e 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -40,7 +40,7 @@ export async function createSignedPost(args: { level: string; key: PrivateKey; u ['(request-target)', 'date', 'host', 'digest'], ); - delete request.headers['host']; + delete request.headers['Host']; return { request, @@ -68,7 +68,7 @@ export async function createSignedGet(args: { level: string; key: PrivateKey; ur ['(request-target)', 'date', 'host', 'accept'], ); - delete request.headers['host']; + delete request.headers['Host']; return { request, From de677a5b1fbf6ab9d2438f94e373c836c71f240b Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 23:28:37 +0900 Subject: [PATCH 124/134] // node-fetch will generate this for us. if we keep 'Host', it won't change with redirects! --- packages/backend/src/core/activitypub/ApRequestService.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index e48c1865d91e..352a45fcfc8f 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -40,6 +40,7 @@ export async function createSignedPost(args: { level: string; key: PrivateKey; u ['(request-target)', 'date', 'host', 'digest'], ); + // node-fetch will generate this for us. if we keep 'Host', it won't change with redirects! delete request.headers['Host']; return { @@ -68,6 +69,7 @@ export async function createSignedGet(args: { level: string; key: PrivateKey; ur ['(request-target)', 'date', 'host', 'accept'], ); + // node-fetch will generate this for us. if we keep 'Host', it won't change with redirects! delete request.headers['Host']; return { From e602f2efdaad1e677b3e1e2d69824a22367ff550 Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 17 Jul 2024 23:55:39 +0900 Subject: [PATCH 125/134] move delete host --- .../backend/src/core/activitypub/ApRequestService.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 352a45fcfc8f..0cae91316b0e 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -40,9 +40,6 @@ export async function createSignedPost(args: { level: string; key: PrivateKey; u ['(request-target)', 'date', 'host', 'digest'], ); - // node-fetch will generate this for us. if we keep 'Host', it won't change with redirects! - delete request.headers['Host']; - return { request, ...result, @@ -69,9 +66,6 @@ export async function createSignedGet(args: { level: string; key: PrivateKey; ur ['(request-target)', 'date', 'host', 'accept'], ); - // node-fetch will generate this for us. if we keep 'Host', it won't change with redirects! - delete request.headers['Host']; - return { request, ...result, @@ -109,6 +103,9 @@ export class ApRequestService { digest, }); + // node-fetch will generate this for us. if we keep 'Host', it won't change with redirects! + delete req.request.headers['Host']; + this.logger.debug('create signed post', { version: 'draft', level, @@ -140,6 +137,9 @@ export class ApRequestService { }, }); + // node-fetch will generate this for us. if we keep 'Host', it won't change with redirects! + delete req.request.headers['Host']; + this.logger.debug('create signed get', { version: 'draft', level, From 72cda5ca809171982cded34b256ac1a210990536 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 18 Jul 2024 00:02:36 +0900 Subject: [PATCH 126/134] modify comment --- packages/backend/src/core/UserKeypairService.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index de8f7c17da7e..902f008660b5 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -53,13 +53,14 @@ export class UserKeypairService implements OnApplicationShutdown { * Get private key [Only PrivateKeyWithPem for queue data etc.] * @param userIdOrHint user id or MiUserKeypair * @param preferType - * If ed25519-like(`ed25519`, `01`, `11`) is specified, ed25519 keypair is returned if exists. - * Otherwise, main keypair is returned. + * If ed25519-like(`ed25519`, `01`, `11`) is specified, ed25519 keypair will be returned if exists. + * Otherwise, main keypair will be returned. * @returns */ @bindThis public async getLocalUserPrivateKeyPem( - userIdOrHint: MiUser['id'] | MiUserKeypair, preferType?: string, + userIdOrHint: MiUser['id'] | MiUserKeypair, + preferType?: string, ): Promise { const keypair = typeof userIdOrHint === 'string' ? await this.getUserKeypair(userIdOrHint) : userIdOrHint; if ( From 95918607f4ee404ddcdd1db8cde7a6d92331f965 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 18 Jul 2024 00:04:35 +0900 Subject: [PATCH 127/134] modify comment --- packages/backend/src/core/UserKeypairService.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 902f008660b5..aa90f1e209e7 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -83,13 +83,14 @@ export class UserKeypairService implements OnApplicationShutdown { * Using cache due to performance reasons of `crypto.subtle.importKey` * @param userIdOrHint user id, MiUserKeypair, or PrivateKeyWithPem * @param preferType - * If ed25519-like(`ed25519`, `01`, `11`) is specified, ed25519 keypair is returned if exists. - * Otherwise, main keypair is returned. (ignored if userIdOrHint is PrivateKeyWithPem) + * If ed25519-like(`ed25519`, `01`, `11`) is specified, ed25519 keypair will be returned if exists. + * Otherwise, main keypair will be returned. (ignored if userIdOrHint is PrivateKeyWithPem) * @returns */ @bindThis public async getLocalUserPrivateKey( - userIdOrHint: MiUser['id'] | MiUserKeypair | PrivateKeyWithPem, preferType?: string, + userIdOrHint: MiUser['id'] | MiUserKeypair | PrivateKeyWithPem, + preferType?: string, ): Promise { if (typeof userIdOrHint === 'object' && 'privateKeyPem' in userIdOrHint) { // userIdOrHint is PrivateKeyWithPem From cd19ad694c520dcdef1499254da20df2cee942b9 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 18 Jul 2024 00:09:20 +0900 Subject: [PATCH 128/134] =?UTF-8?q?fix=20correct=20=E2=86=92=20collect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/src/core/activitypub/ApDeliverManagerService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index decb02156131..db3302e6ffe0 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -146,7 +146,7 @@ class DeliverManager { } //#endregion - //#region correct inboxes by recipes + //#region collect inboxes by recipes // The value flags whether it is shared or not. // key: inbox URL, value: whether it is sharedInbox const inboxes = new Map(); From 99113d59f454e1d9107cc8dd63232e029d445b2c Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 18 Jul 2024 00:12:21 +0900 Subject: [PATCH 129/134] =?UTF-8?q?refreshAndfindKey=20=E2=86=92=20refresh?= =?UTF-8?q?AndFindKey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/core/activitypub/ApDbResolverService.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 66f649c9bde2..75c2241233e3 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -128,16 +128,16 @@ export class ApDbResolverService implements OnApplicationShutdown { } @bindThis - private async refreshAndfindKey(userId: MiUser['id'], keyId: string): Promise { + private async refreshAndFindKey(userId: MiUser['id'], keyId: string): Promise { this.refreshCacheByUserId(userId); const keys = await this.getPublicKeyByUserId(userId); if (keys == null || !Array.isArray(keys) || keys.length === 0) { - this.logger.warn(`No key found (refreshAndfindKey) userId=${userId} keyId=${keyId} keys=${JSON.stringify(keys)}`); + this.logger.warn(`No key found (refreshAndFindKey) userId=${userId} keyId=${keyId} keys=${JSON.stringify(keys)}`); return null; } const exactKey = keys.find(x => x.keyId === keyId); if (exactKey) return exactKey; - this.logger.warn(`No exact key found (refreshAndfindKey) userId=${userId} keyId=${keyId} keys=${JSON.stringify(keys)}`); + this.logger.warn(`No exact key found (refreshAndFindKey) userId=${userId} keyId=${keyId} keys=${JSON.stringify(keys)}`); return null; } @@ -213,7 +213,7 @@ export class ApDbResolverService implements OnApplicationShutdown { // まずはキャッシュを更新して再取得 const cacheRaw = this.publicKeyByUserIdCache.cache.get(user.id); if (cacheRaw && cacheRaw.date > Date.now() - 1000 * 60 * 12) { - const exactKey = await this.refreshAndfindKey(user.id, keyId); + const exactKey = await this.refreshAndFindKey(user.id, keyId); if (exactKey) return { user, key: exactKey }; } @@ -223,7 +223,7 @@ export class ApDbResolverService implements OnApplicationShutdown { const renewed = await this.apPersonService.fetchPersonWithRenewal(uri, 0); if (renewed == null || renewed.isDeleted) return null; - return { user, key: await this.refreshAndfindKey(user.id, keyId) }; + return { user, key: await this.refreshAndFindKey(user.id, keyId) }; } this.logger.warn(`No key found uri=${uri} userId=${user.id} keyId=${keyId}`); From c00b61e90b3a68248e028190f918a9b335a2743b Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 18 Jul 2024 00:24:47 +0900 Subject: [PATCH 130/134] modify comment --- .../core/activitypub/ApDbResolverService.ts | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 75c2241233e3..973394683f86 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -146,9 +146,10 @@ export class ApDbResolverService implements OnApplicationShutdown { * @param uri AP Actor id * @param keyId Key id to find. If not specified, main key will be selected. * @returns - * 1. ユーザーとキーのホストが一致しない場合`null` - * 2. userが見つからない場合`{ user: null, key: null }` - * 3. keyが見つからない場合`{ user, key: null }` + * 1. `null` if the user and key host do not match + * 2. `{ user: null, key: null }` if the user is not found + * 3. `{ user: MiRemoteUser, key: null }` if key is not found + * 4. `{ user: MiRemoteUser, key: MiUserPublickey }` if both are found */ @bindThis public async getAuthUserFromApId(uri: string, keyId?: string): Promise<{ @@ -165,11 +166,21 @@ export class ApDbResolverService implements OnApplicationShutdown { * keyIdはURL形式かつkeyIdのホストはuriのホストと一致するはず * （ApPersonService.validateActorに由来） * - * ただ、Mastodonはリプライ関連で他人のノートをHTTP Signature署名して送ってくることがある + * ただ、Mastodonはリプライ関連で他人のトゥートをHTTP Signature署名して送ってくることがある * そのような署名は有効性に疑問があるので無視することにする * ここではuriとkeyIdのホストが一致しない場合は無視する * ハッシュをなくしたkeyIdとuriの同一性を比べてみてもいいが、`uri#*-key`というkeyIdを設定するのが * 決まりごとというわけでもないため幅を持たせることにする + * + * + * The keyId should be in URL format and its host should match the host of the uri + * (derived from ApPersonService.validateActor) + * + * However, Mastodon sometimes sends toots from other users with HTTP Signature signing for reply-related purposes + * Such signatures are of questionable validity, so we choose to ignore them + * Here, we ignore cases where the hosts of uri and keyId do not match + * We could also compare the equality of keyId without the hash and uri, but since setting a keyId like `uri#*-key` + * is not a strict rule, we decide to allow for some flexibility */ this.logger.warn(`actor uri and keyId are not matched uri=${uri} keyId=${keyId}`); return null; @@ -187,7 +198,7 @@ export class ApDbResolverService implements OnApplicationShutdown { } if (!keyId) { - // mainっぽいのを選ぶ + // Choose the main-like const mainKey = keys.find(x => { try { const url = new URL(x.keyId); @@ -209,15 +220,20 @@ export class ApDbResolverService implements OnApplicationShutdown { const exactKey = keys.find(x => x.keyId === keyId); if (exactKey) return { user, key: exactKey }; - // keyIdで見つからない場合 - // まずはキャッシュを更新して再取得 + /** + * keyIdで見つからない場合、まずはキャッシュを更新して再取得 + * If not found with keyId, update cache and reacquire + */ const cacheRaw = this.publicKeyByUserIdCache.cache.get(user.id); if (cacheRaw && cacheRaw.date > Date.now() - 1000 * 60 * 12) { const exactKey = await this.refreshAndFindKey(user.id, keyId); if (exactKey) return { user, key: exactKey }; } - // lastFetchedAtでの更新制限を弱めて再取得 + /** + * lastFetchedAtでの更新制限を弱めて再取得 + * Reacquisition with weakened update limit at lastFetchedAt + */ if (user.lastFetchedAt == null || user.lastFetchedAt < new Date(Date.now() - 1000 * 60 * 12)) { this.logger.info(`Fetching user to find public key uri=${uri} userId=${user.id} keyId=${keyId}`); const renewed = await this.apPersonService.fetchPersonWithRenewal(uri, 0); From 29d9bbf05b9ef5f65004915ae2242db0da89eadf Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 18 Jul 2024 00:29:15 +0900 Subject: [PATCH 131/134] modify attachLdSignature --- packages/backend/src/core/RelayService.ts | 2 +- packages/backend/src/core/activitypub/ApRendererService.ts | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/core/RelayService.ts b/packages/backend/src/core/RelayService.ts index edf4ffc2e1d2..ad01f989029a 100644 --- a/packages/backend/src/core/RelayService.ts +++ b/packages/backend/src/core/RelayService.ts @@ -125,7 +125,7 @@ export class RelayService { const copy = deepClone(activity); if (!copy.to) copy.to = ['https://www.w3.org/ns/activitystreams#Public']; privateKey = privateKey ?? await this.userKeypairService.getLocalUserPrivateKeyPem(user.id); - const signed = await this.apRendererService.attachLdSignature(copy, user, privateKey); + const signed = await this.apRendererService.attachLdSignature(copy, privateKey); this.queueService.deliverMany(user, signed, new Map(relays.map(({ inbox }) => [inbox, false])), privateKey); } diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 34a9826fb3e3..5d7419f9346a 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -625,9 +625,7 @@ export class ApRendererService { } @bindThis - public async attachLdSignature(activity: any, user: { id: MiUser['id']; host: null; }, key: PrivateKeyWithPem): Promise { - const keypair = await this.userKeypairService.getUserKeypair(user.id); - + public async attachLdSignature(activity: any, key: PrivateKeyWithPem): Promise { const jsonLd = this.jsonLdService.use(); jsonLd.debug = false; activity = await jsonLd.signRsaSignature2017(activity, key.privateKeyPem, key.keyId); From aed28060e7d596261df723a419fdc1d49c903455 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 18 Jul 2024 00:42:47 +0900 Subject: [PATCH 132/134] getApId, InboxProcessorService --- packages/backend/src/core/activitypub/type.ts | 2 +- packages/backend/src/queue/processors/InboxProcessorService.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 368650a83d20..1d559716605e 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -55,7 +55,7 @@ export function getOneApId(value: ApObject): string { export function getApId(value: string | IObject): string { if (typeof value === 'string') return value; if (typeof value.id === 'string') return value.id; - throw new Error('cannot detemine id'); + throw new Error('cannot determine id'); } /** diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 183399c14cf7..935c623df151 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -69,7 +69,7 @@ export class InboxProcessorService { this.logger.debug(JSON.stringify(info, null, 2)); //#endregion - const host = this.utilityService.toPuny(new URL(activity.actor).hostname); + const host = this.utilityService.toPuny(new URL(actorUri).hostname); // ブロックしてたら中断 const meta = await this.metaService.fetch(); From 8c76c7b8b5ce740cb9cd37c57f55ce77602d2259 Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 18 Jul 2024 00:43:31 +0900 Subject: [PATCH 133/134] TODO --- packages/backend/src/queue/types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index a2566d0cde9a..f2466f2e3d67 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -13,6 +13,7 @@ import type { ParsedSignature, PrivateKeyWithPem } from '@misskey-dev/node-http- /** * @peertube/http-signature 時代の古いデータにも対応しておく + * TODO: 2026年ぐらいには消す */ export interface OldParsedSignature { scheme: 'Signature'; From 61d904e8f1ec1196f84a3f791725563391ecce9f Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 18 Jul 2024 01:08:14 +0900 Subject: [PATCH 134/134] [skip ci] add CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75f739f546e2..19e3cbae1509 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ - Fix: Dockerコンテナの立ち上げ時に`pnpm`のインストールで固まることがある問題 - Fix: デフォルトテーマに無効なテーマコードを入力するとUIが使用できなくなる問題を修正 - Enhance: Allow negative delay for MFM animation elements (`tada`, `jelly`, `twitch`, `shake`, `spin`, `jump`, `bounce`, `rainbow`) +- Feat: 連合に使うHTTP SignaturesがEd25519鍵に対応するように #13464 + - Ed25519署名に対応するサーバーが増えると、deliverで要求されるサーバーリソースが削減されます ### Client - Enhance: 内蔵APIドキュメントのデザイン・パフォーマンスを改善