From d2cfeb91b901dbd9cb229c0d19db3f42ae05b6c1 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Fri, 22 Nov 2024 11:08:33 +0800 Subject: [PATCH] feat: add trending subject api --- lib/openapi/index.ts | 1 + lib/trending/subject.ts | 85 +++++++++++++++++++++++++ lib/trending/type.ts | 22 +++++++ lib/types/fetcher.ts | 24 +++++++ routes/__snapshots__/index.test.ts.snap | 79 +++++++++++++++++++++++ routes/private/index.ts | 2 + routes/private/routes/trending.ts | 64 +++++++++++++++++++ tasks/trending.ts | 73 ++------------------- 8 files changed, 281 insertions(+), 69 deletions(-) create mode 100644 lib/trending/subject.ts create mode 100644 lib/trending/type.ts create mode 100644 routes/private/routes/trending.ts diff --git a/lib/openapi/index.ts b/lib/openapi/index.ts index 660e09c1..ae6c1659 100644 --- a/lib/openapi/index.ts +++ b/lib/openapi/index.ts @@ -8,6 +8,7 @@ export const Tag = { Topic: 'topic', User: 'user', Wiki: 'wiki', + Trending: 'trending', } as const; export const Security = { diff --git a/lib/trending/subject.ts b/lib/trending/subject.ts new file mode 100644 index 00000000..b650160d --- /dev/null +++ b/lib/trending/subject.ts @@ -0,0 +1,85 @@ +import { db, op } from '@app/drizzle/db.ts'; +import * as schema from '@app/drizzle/schema'; +import { logger } from '@app/lib/logger'; +import redis from '@app/lib/redis.ts'; +import { SubjectType } from '@app/lib/subject/type.ts'; +import type { TrendingItem } from '@app/lib/trending/type.ts'; +import { getTrendingPeriodDuration, TrendingPeriod } from '@app/lib/trending/type.ts'; + +function getSubjectTrendingKey(type: SubjectType, period: TrendingPeriod) { + return `trending:subjects:${type}:${period}`; +} + +export async function updateTrendingSubjects( + subjectType: SubjectType, + period = TrendingPeriod.Month, + flush = false, +) { + const trendingKey = getSubjectTrendingKey(subjectType, period); + const lockKey = `lock:${trendingKey}`; + if (flush) { + await redis.del(lockKey); + } + if (await redis.get(lockKey)) { + logger.info('Already calculating trending subjects for %s(%s)...', subjectType, period); + return; + } + await redis.set(lockKey, 1, 'EX', 3600); + logger.info('Calculating trending subjects for %s(%s)...', subjectType, period); + + const now = Date.now(); + const duration = getTrendingPeriodDuration(period); + if (!duration) { + logger.error('Invalid period: %s', period); + return; + } + const minDateline = now - duration; + let doingDateline = true; + if (subjectType === SubjectType.Book || subjectType === SubjectType.Music) { + doingDateline = false; + } + + const data = await db + .select({ subjectID: schema.chiiSubjects.id, total: op.count(schema.chiiSubjects.id) }) + .from(schema.chiiSubjectInterests) + .innerJoin( + schema.chiiSubjects, + op.eq(schema.chiiSubjects.id, schema.chiiSubjectInterests.subjectID), + ) + .where( + op.and( + op.eq(schema.chiiSubjects.typeID, subjectType), + op.ne(schema.chiiSubjects.ban, 1), + op.eq(schema.chiiSubjects.nsfw, false), + doingDateline + ? op.gt(schema.chiiSubjectInterests.doingDateline, minDateline) + : op.gt(schema.chiiSubjectInterests.updatedAt, minDateline), + ), + ) + .groupBy(schema.chiiSubjects.id) + .orderBy(op.desc(op.count(schema.chiiSubjects.id))) + .limit(1000) + .execute(); + + const ids = []; + for (const item of data) { + ids.push({ id: item.subjectID, total: item.total } as TrendingItem); + } + await redis.set(trendingKey, JSON.stringify(ids)); + await redis.del(lockKey); +} + +export async function getTrendingSubjects( + subjectType: SubjectType, + period = TrendingPeriod.Month, + limit = 20, + offset = 0, +): Promise { + const trendingKey = getSubjectTrendingKey(subjectType, period); + const data = await redis.get(trendingKey); + if (!data) { + return []; + } + const ids = JSON.parse(data) as TrendingItem[]; + return ids.slice(offset, offset + limit); +} diff --git a/lib/trending/type.ts b/lib/trending/type.ts new file mode 100644 index 00000000..7f788832 --- /dev/null +++ b/lib/trending/type.ts @@ -0,0 +1,22 @@ +export interface TrendingItem { + id: number; + total: number; +} + +export enum TrendingPeriod { + All = 'all', + Day = 'day', + Week = 'week', + Month = 'month', +} + +export function getTrendingPeriodDuration(period: TrendingPeriod): number { + const now = Date.now(); + const duration = { + all: now, + day: 86400, + week: 86400 * 7, + month: 86400 * 30, + }[period]; + return duration; +} diff --git a/lib/types/fetcher.ts b/lib/types/fetcher.ts index cbb58ac5..4b71cde7 100644 --- a/lib/types/fetcher.ts +++ b/lib/types/fetcher.ts @@ -59,6 +59,30 @@ export async function fetchSubjectByID( return null; } +export async function fetchSubjectsByIDs( + ids: number[], + allowNsfw = false, +): Promise> { + const data = await db + .select() + .from(schema.chiiSubjects) + .innerJoin(schema.chiiSubjectFields, op.eq(schema.chiiSubjects.id, schema.chiiSubjectFields.id)) + .where( + op.and( + op.inArray(schema.chiiSubjects.id, ids), + op.ne(schema.chiiSubjects.ban, 1), + allowNsfw ? undefined : op.eq(schema.chiiSubjects.nsfw, false), + ), + ) + .execute(); + const map = new Map(); + for (const d of data) { + const subject = convert.toSubject(d.chii_subjects, d.chii_subject_fields); + map.set(subject.id, subject); + } + return map; +} + export async function fetchSlimCharacterByID( id: number, allowNsfw = false, diff --git a/routes/__snapshots__/index.test.ts.snap b/routes/__snapshots__/index.test.ts.snap index f25cd326..5cf61195 100644 --- a/routes/__snapshots__/index.test.ts.snap +++ b/routes/__snapshots__/index.test.ts.snap @@ -1719,6 +1719,16 @@ exports[`should build private api spec 1`] = ` - replies - reactions type: object + TrendingSubject: + properties: + count: + type: integer + subject: + $ref: '#/components/schemas/Subject' + required: + - subject + - count + type: object User: properties: avatar: @@ -4400,6 +4410,75 @@ paths: summary: 创建条目讨论版 tags: - subject + /p1/trending/subjects: + get: + operationId: getTrendingSubjects + parameters: + - description: 条目类型 + in: query + name: type + required: true + schema: + anyOf: + - enum: + - 1 + type: number + - enum: + - 2 + type: number + - enum: + - 3 + type: number + - enum: + - 4 + type: number + - enum: + - 6 + type: number + - description: max 100 + in: query + name: limit + required: false + schema: + default: 20 + maximum: 100 + minimum: 1 + type: integer + - description: min 0 + in: query + name: offset + required: false + schema: + default: 0 + minimum: 0 + type: integer + responses: + '200': + content: + application/json: + schema: + properties: + data: + items: + $ref: '#/components/schemas/TrendingSubject' + type: array + total: + type: integer + required: + - data + - total + type: object + description: Default Response + '500': + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: 意料之外的服务器错误 + description: 意料之外的服务器错误 + summary: 获取热门条目 + tags: + - trending /p1/turnstile: get: description: 为防止滥用，Redirect URI 为白名单机制，如需添加请提交 PR。 diff --git a/routes/private/index.ts b/routes/private/index.ts index 203f8833..9b89baf7 100644 --- a/routes/private/index.ts +++ b/routes/private/index.ts @@ -13,6 +13,7 @@ import * as person from './routes/person.ts'; import * as post from './routes/post.ts'; import * as subject from './routes/subject.ts'; import * as group from './routes/topic.ts'; +import * as trending from './routes/trending.ts'; import * as user from './routes/user.ts'; import * as wiki from './routes/wiki/index.ts'; @@ -50,6 +51,7 @@ async function API(app: App) { await app.register(person.setup); await app.register(post.setup); await app.register(subject.setup); + await app.register(trending.setup); await app.register(user.setup); await app.register(wiki.setup, { prefix: '/wiki' }); } diff --git a/routes/private/routes/trending.ts b/routes/private/routes/trending.ts new file mode 100644 index 00000000..4f97b75a --- /dev/null +++ b/routes/private/routes/trending.ts @@ -0,0 +1,64 @@ +import type { Static } from '@sinclair/typebox'; +import { Type as t } from '@sinclair/typebox'; + +import { Tag } from '@app/lib/openapi/index.ts'; +import { SubjectType } from '@app/lib/subject/type.ts'; +import { getTrendingSubjects } from '@app/lib/trending/subject.ts'; +import { TrendingPeriod } from '@app/lib/trending/type'; +import * as fetcher from '@app/lib/types/fetcher.ts'; +import * as res from '@app/lib/types/res.ts'; +import type { App } from '@app/routes/type.ts'; + +export type ITrendingSubject = Static; +const TrendingSubject = t.Object( + { + subject: t.Ref(res.Subject), + count: t.Integer(), + }, + { $id: 'TrendingSubject' }, +); + +// eslint-disable-next-line @typescript-eslint/require-await +export async function setup(app: App) { + app.addSchema(TrendingSubject); + + app.get( + '/trending/subjects', + { + schema: { + summary: '获取热门条目', + operationId: 'getTrendingSubjects', + tags: [Tag.Trending], + querystring: t.Object({ + type: t.Enum(SubjectType, { description: '条目类型' }), + limit: t.Optional( + t.Integer({ default: 20, minimum: 1, maximum: 100, description: 'max 100' }), + ), + offset: t.Optional(t.Integer({ default: 0, minimum: 0, description: 'min 0' })), + }), + response: { + 200: res.Paged(t.Ref(TrendingSubject)), + }, + }, + }, + async ({ query: { type, limit = 20, offset = 0 } }) => { + const items = await getTrendingSubjects(type, TrendingPeriod.Month, limit, offset); + const subjectIDs = items.map((item) => item.id); + const subjects = await fetcher.fetchSubjectsByIDs(subjectIDs); + const data = []; + for (const item of items) { + const subject = subjects.get(item.id); + if (subject) { + data.push({ + subject, + count: item.total, + }); + } + } + return { + total: 1000, + data, + }; + }, + ); +} diff --git a/tasks/trending.ts b/tasks/trending.ts index dfa5237e..514ffe28 100644 --- a/tasks/trending.ts +++ b/tasks/trending.ts @@ -1,74 +1,9 @@ -import { db, op } from '@app/drizzle/db.ts'; -import * as schema from '@app/drizzle/schema'; -import { logger } from '@app/lib/logger'; -import redis from '@app/lib/redis.ts'; -import { SubjectType, SubjectTypeValues } from '@app/lib/subject/type.ts'; - -interface TrendingItem { - id: number; - total: number; -} - -async function updateTrendingSubjects(subjectType: SubjectType, period = 'month', flush = false) { - const trendingKey = `trending:subjects:${subjectType}:${period}`; - const lockKey = `lock:${trendingKey}`; - if (flush) { - await redis.del(lockKey); - } - if (await redis.get(lockKey)) { - logger.info('Already calculating trending subjects for %s(%s)...', subjectType, period); - return; - } - await redis.set(lockKey, 1, 'EX', 3600); - logger.info('Calculating trending subjects for %s(%s)...', subjectType, period); - - const now = Date.now(); - const duration = { - all: now, - day: 86400, - week: 86400 * 7, - month: 86400 * 30, - }[period]; - if (!duration) { - logger.error('Invalid period: %s', period); - return; - } - const minDateline = now - duration; - let doingDateline = true; - if (subjectType === SubjectType.Book || subjectType === SubjectType.Music) { - doingDateline = false; - } - - const data = await db - .select({ subjectID: schema.chiiSubjects.id, total: op.count(schema.chiiSubjects.id) }) - .from(schema.chiiSubjectInterests) - .innerJoin( - schema.chiiSubjects, - op.eq(schema.chiiSubjects.id, schema.chiiSubjectInterests.subjectID), - ) - .where( - op.and( - op.eq(schema.chiiSubjects.typeID, subjectType), - doingDateline - ? op.gt(schema.chiiSubjectInterests.doingDateline, minDateline) - : op.gt(schema.chiiSubjectInterests.updatedAt, minDateline), - ), - ) - .groupBy(schema.chiiSubjects.id) - .orderBy(op.desc(op.count(schema.chiiSubjects.id))) - .limit(1000) - .execute(); - - const ids = []; - for (const item of data) { - ids.push({ id: item.subjectID, total: item.total } as TrendingItem); - } - await redis.set(trendingKey, JSON.stringify(ids)); - await redis.del(lockKey); -} +import { SubjectTypeValues } from '@app/lib/subject/type.ts'; +import { updateTrendingSubjects } from '@app/lib/trending/subject.ts'; +import { TrendingPeriod } from '@app/lib/trending/type.ts'; export async function trendingSubjects() { for (const subjectType of SubjectTypeValues) { - await updateTrendingSubjects(subjectType, 'month'); + await updateTrendingSubjects(subjectType, TrendingPeriod.Month); } }