Fix: outboxFetch(キャッシュが正しく使われない、型定義) (yojo-art#560)

CHANGELOG_YOJO.md

@@ -8,6 +8,7 @@
 - Enhance: 表示中のタグTLをお気に入り登録するボタンを追加 [#561](https://github.com/yojo-art/cherrypick/pull/561)
 
 ### Server
+- Fix: `api/ap/fetch-outbox`が正しく動作しないのを修正[#560](https://github.com/yojo-art/cherrypick/pull/560)
 - Fix: PersonのserchableByが正しく連合できていないのを修正[#556](https://github.com/yojo-art/cherrypick/pull/556)
 - Fix: SerchableByが未設定の時にプライバシーを更新できないことがある[#567](https://github.com/yojo-art/cherrypick/pull/567)
 - Enhance: `/users/${id}`に`Accept: application/ld+json`ではないリクエストが来たとき`/@${username}`にリダイレクトするように [#554](https://github.com/yojo-art/cherrypick/pull/554)

packages/backend/src/core/activitypub/ApResolverService.ts

@@ -15,7 +15,7 @@ import { UtilityService } from '@/core/UtilityService.js';
 import { bindThis } from '@/decorators.js';
 import { LoggerService } from '@/core/LoggerService.js';
 import type Logger from '@/logger.js';
-import { isCollectionOrOrderedCollection, isIOrderedCollectionPage } from './type.js';
+import { isCollectionOrOrderedCollection, isIOrderedCollectionPage, isOrderedCollection } from './type.js';
 import { ApDbResolverService } from './ApDbResolverService.js';
 import { ApRendererService } from './ApRendererService.js';
 import { ApRequestService } from './ApRequestService.js';
@@ -65,6 +65,19 @@ export class Resolver {
 		}
 	}
 
+	@bindThis
+	public async resolveOrderedCollection(value: string | IObject): Promise<IOrderedCollection> {
+		const collection = typeof value === 'string'
+			? await this.resolve(value)
+			: value;
+
+		if (isOrderedCollection(collection)) {
+			return collection;
+		} else {
+			throw new Error(`unrecognized collection type: ${collection.type}`);
+		}
+	}
+
 	@bindThis
 	public async resolveOrderedCollectionPage(value: string | IObject): Promise<IOrderedCollectionPage> {
 		const collection = typeof value === 'string'

packages/backend/src/core/activitypub/models/ApOutboxFetchService.ts

@@ -22,7 +22,7 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
 import { NoteCreateService } from '@/core/NoteCreateService.js';
 import { IdentifiableError } from '@/misc/identifiable-error.js';
 import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js';
-import { isIOrderedCollectionPage, isCreate, IOrderedCollectionPage, isNote } from '../type.js';
+import { isCreate, IOrderedCollectionPage, isNote } from '../type.js';
 import { ApAudienceService } from '../ApAudienceService.js';
 import type { OnModuleInit } from '@nestjs/common';
 import type { ApNoteService } from './ApNoteService.js';
@@ -83,55 +83,42 @@ export class ApOutboxFetchService implements OnModuleInit {
 		this.logger.info(`Fetcing the Outbox: ${outboxUrl}`);
 		const Resolver = resolver ?? this.apResolverService.createResolver();
 		const cache = await this.redisClient.get(`${outboxUrl}--next`);
-		// Resolve to (Ordered)Collection Object
-		const outbox = cache ? await Resolver.resolveOrderedCollectionPage(cache) : await Resolver.resolveCollection(outboxUrl);
+		let next: string | IOrderedCollectionPage;
 
-		if (!cache && outbox.type !== 'OrderedCollection') throw new IdentifiableError('0be2f5a1-2345-46d8-b8c3-430b111c68d3', 'outbox type is not OrderedCollection');
-		if (!cache && !outbox.first) throw new IdentifiableError('a723c2df-0250-4091-b5fc-e3a7b36c7b61', 'outbox first page not exist');
+		if (!cache) {
+			// Resolve to (Ordered)Collection Object
+			const outbox = await Resolver.resolveOrderedCollection(outboxUrl);
+			if (!outbox.first) throw new IdentifiableError('a723c2df-0250-4091-b5fc-e3a7b36c7b61', 'outbox first page not exist');
+			next = outbox.first;
+		} else next = cache;
 
-		let nextUrl = cache ? (outbox as IOrderedCollectionPage).next : outbox.first;
-		let page = 0;
 		let created = 0;
-		if (typeof(nextUrl) !== 'string') {
-			const first = (nextUrl as any);
-			if (first.partOf !== user.outbox) throw new IdentifiableError('6603433f-99db-4134-980c-48705ae57ab8', 'outbox part is invalid');
 
-			const activityes = first.orderedItems ?? first.items;
-			await this.fetchObjects(user, activityes, includeAnnounce, created);
+		for (let page = 0; page < pagelimit; page++) {
+			const collection = (typeof(next) === 'string' ? await Resolver.resolveOrderedCollectionPage(next) : next);
+			if (collection.partOf !== user.outbox) throw new IdentifiableError('6603433f-99db-4134-980c-48705ae57ab8', 'outbox part is invalid');
 
-			page = 1;
-			if (!first.next) return;
-		}
-
-		for (; page < pagelimit; page++) {
-			this.logger.info(nextUrl as string);
-			const collectionPage = (typeof(nextUrl) === 'string' ? await Resolver.resolveOrderedCollectionPage(nextUrl) : nextUrl) as IOrderedCollectionPage;
-			if (!isIOrderedCollectionPage(collectionPage)) throw new IdentifiableError('2a05bb06-f38c-4854-af6f-7fd5e87c98ee', 'Object is not collectionPage');
-			if (collectionPage.partOf !== user.outbox) throw new IdentifiableError('6603433f-99db-4134-980c-48705ae57ab8', 'outbox part is invalid');
-
-			const activityes = (collectionPage.orderedItems ?? collectionPage.items);
-			nextUrl = collectionPage.next;
-			if (!activityes) continue;
+			const activityes = (collection.orderedItems ?? collection.items);
+			if (!activityes) throw new IdentifiableError('2a05bb06-f38c-4854-af6f-7fd5e87c98ee', 'item is unavailable');
 
 			created = await this.fetchObjects(user, activityes, includeAnnounce, created);
 			if (createLimit <= created) break;//次ページ見て一件だけしか取れないのは微妙
-			if (!nextUrl) {
-				break;
-			}
+			if (!collection.next) break;
 
-			await this.redisClient.set(`${outboxUrl}--next`, `${nextUrl}`, 'EX', 60 * 15);//15min
+			next = collection.next;
+			await this.redisClient.set(`${outboxUrl}--next`, `${next}`, 'EX', 60 * 15);//15min
 		}
 		this.logger.succ(`Outbox Fetced: ${outboxUrl}`);
-		//this.logger.info(`Outbox Fetced last: ${nextUrl}`);
 	}
 
 	@bindThis
 	private async fetchObjects(user: MiRemoteUser, activityes: any[], includeAnnounce:boolean, created: number): Promise<number> {
 		for (const activity of activityes) {
 			if (createLimit < created) return created;
 			try {
-				if (includeAnnounce && activity.type === 'Announce') {
-					const object = await	this.apDbResolverService.getNoteFromApId(activity.id);
+				if (activity.actor !== user.uri) throw new IdentifiableError('bde7c204-5441-4a87-9b7e-f81e8d05788a');
+				if (activity.type === 'Announce' && includeAnnounce) {
+					const object = await this.apNoteService.fetchNote(activity.id);
 
 					if (object) continue;
 
@@ -188,18 +175,22 @@ export class ApOutboxFetchService implements OnModuleInit {
 					} finally {
 						unlock();
 					}
-				} else if (isCreate(activity) && typeof(activity.object) !== 'string' && isNote(activity.object)) {
-					const object = await	this.apDbResolverService.getNoteFromApId(activity.object);
-					if (object) continue;
+				} else if (isCreate(activity)) {
+					if (typeof(activity.object) !== 'string') {
+						if (!isNote(activity)) continue;
+					}
+					const fetch = await this.apNoteService.fetchNote(activity.object);
+					if (fetch) continue;
 					await this.apNoteService.createNote(activity.object, undefined, true);
 				}
 			} catch (err) {
-				if (err instanceof AbortError) {
-					this.logger.warn(`Aborted note: ${activity.id}`);
+				//リモートのリモートが落ちてるなどで止まるとほかが見れなくなってしまうので再スローしない
+				if (err instanceof IdentifiableError) {
+					if (err.id === 'bde7c204-5441-4a87-9b7e-f81e8d05788a') this.logger.error(`fetchErrorInvalidActor:${activity.id}`);
 				} else {
-					this.logger.warn(JSON.stringify(err));
-					this.logger.warn(JSON.stringify(activity));
-					throw err;
+					this.logger.error(`fetchError:${activity.id}`);
+					this.logger.error(`${err}`);
+					continue;
 				}
 			}
 			created ++;

packages/backend/src/core/activitypub/type.ts

@@ -112,15 +112,14 @@ export interface IOrderedCollection extends IObject {
 	type: 'OrderedCollection';
 	totalItems?: number;
 	orderedItems?: ApObject;
-	first?: IObject | string;
+	first?: IOrderedCollectionPage | string;
 	last?: IObject | string;
 }
 
 export interface IOrderedCollectionPage extends IObject {
 	type: 'OrderedCollectionPage';
 	partOf: string;
 	totalItems?: number;
-	first?: IObject | string;
 	orderedItems?: IObject[];
 	items?: IObject[];
 	prev: string;

packages/backend/src/server/api/endpoints/ap/fetch-outbox.ts

@@ -43,6 +43,16 @@ export const meta = {
 			code: 'OUTBOX_FIRST_PAGE_UNDEFINED_THIS_USER',
 			id: 'e1f29e66-86a9-4fdc-9be6-63d4587dc350',
 		},
+		invalidPart: {
+			message: 'outbox part is invalid',
+			code: 'OUTBOX_PART_IS_INVALID',
+			id: 'c3e584df-068a-4b1d-967e-54f2f30f7cba',
+		},
+		itemIsUnavailable: {
+			message: 'outbox item is unavailable',
+			code: 'OUTBOX_ITEM_IS_UNAVAILABLE',
+			id: 'a07b05af-5f66-4203-918c-ebff9e9384bf',
+		},
 	},
 } as const;
 
@@ -83,11 +93,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 						if (err.id === '3fc5a089-cab4-48db-b9f3-f220574b3c0a') throw new ApiError(meta.errors.noSuchUser);
 						if (err.id === '67070303-177c-4600-af93-b26a7ab889c6') throw new ApiError(meta.errors.isLocalUser);
 						if (err.id === 'e7a2e510-a8ce-40e9-b1e6-c007bacdc89f') throw new ApiError(meta.errors.outboxUndefined);
-						//if (err.id === 'b27090c8-8a68-4189-a445-14591c32a89c')
-						//if (err.id === '0be2f5a1-2345-46d8-b8c3-430b111c68d3')
 						if (err.id === 'a723c2df-0250-4091-b5fc-e3a7b36c7b61') throw new ApiError(meta.errors.outboxFirstPageUndefined);
-						//if (err.id === '6603433f-99db-4134-980c-48705ae57ab8')
-						//if (err.id === '2a05bb06-f38c-4854-af6f-7fd5e87c98ee')
+						if (err.id === '6603433f-99db-4134-980c-48705ae57ab8') throw new ApiError(meta.errors.invalidPart);
+						if (err.id === '2a05bb06-f38c-4854-af6f-7fd5e87c98ee') throw new ApiError(meta.errors.itemIsUnavailable);
 					}
 					throw (err);
 				}