Skip to content

Commit

Permalink
Merge branch 'DIYgod/master' into cnki
Browse files Browse the repository at this point in the history
* DIYgod/master:
  feat(route): add route UK Parliament Commons Library research by topic (DIYgod#16941)
  feat(route): add route UK Parliament Lords Library research by topic (DIYgod#16933)
  fix(route): change url in `/tvb/news` (DIYgod#17098)
  chore(deps): bump @hono/node-server from 1.13.1 to 1.13.2 (DIYgod#17088)
  chore(deps): bump @scalar/hono-api-reference from 0.5.152 to 0.5.153 (DIYgod#17085)
  chore(deps-dev): bump @babel/preset-env from 7.25.7 to 7.25.8 (DIYgod#17087)
  feat(route): also support podcast id in xiaoyuzhou
  fix: catch Invalid Date (DIYgod#17081)
  chore(deps): bump undici from 6.19.8 to 6.20.0 (DIYgod#17058)

# Conflicts:
#	lib/routes/xiaoyuzhou/podcast.ts
  • Loading branch information
bxb100 committed Oct 13, 2024
2 parents bdbb95a + 9376572 commit fa165e6
Show file tree
Hide file tree
Showing 7 changed files with 481 additions and 536 deletions.
12 changes: 10 additions & 2 deletions lib/middleware/template.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,16 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
}

if (outputType !== 'rss') {
item.pubDate && (item.pubDate = convertDateToISO8601(item.pubDate) || '');
item.updated && (item.updated = convertDateToISO8601(item.updated) || '');
try {
item.pubDate && (item.pubDate = convertDateToISO8601(item.pubDate) || '');
} catch {
item.pubDate = '';
}
try {
item.updated && (item.updated = convertDateToISO8601(item.updated) || '');
} catch {
item.updated = '';
}
}
}
}
Expand Down
55 changes: 55 additions & 0 deletions lib/routes/parliament.uk/commonslibrary.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import { load } from 'cheerio';
import { Route } from '@/types';
import puppeteer from '@/utils/puppeteer';
import timezone from '@/utils/timezone';

export const route: Route = {
path: '/commonslibrary/type/:topic?',
categories: ['government'],
example: '/parliament.uk/commonslibrary/type/research-briefing',
parameters: { topic: 'research by topic, string, example: [research-briefing|data-dashboard]' },
features: {
requireConfig: false,
requirePuppeteer: true,
antiCrawler: false,
supportBT: false,
supportPodcast: false,
supportScihub: false,
},
name: 'Commonlibrary',
maintainers: ['AntiKnot'],
handler,
};

async function handler(ctx) {
const { topic } = ctx.req.param();
const baseUrl = 'https://commonslibrary.parliament.uk/type';
const url = `${baseUrl}/${topic}/`;
const browser = await puppeteer();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
});
await page.goto(url, {
waitUntil: 'domcontentloaded',
});

const html = await page.evaluate(() => document.documentElement.innerHTML);
await page.close();
const $ = load(html);
const items = $('article.card--horizontal')
.map((_, article) => ({
title: $(article).find('.card__text a').text().trim(),
link: $(article).find('.card__text a').attr('href'),
description: $(article).find('p').last().text().trim(),
pubDate: timezone($(article).find('.card__date time').attr('datetime')),
}))
.toArray();
browser.close();
return {
title: `parliament - lordslibrary - ${topic}`,
link: baseUrl,
item: items,
};
}
55 changes: 55 additions & 0 deletions lib/routes/parliament.uk/lordslibrary.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import { load } from 'cheerio';
import { Route } from '@/types';
import puppeteer from '@/utils/puppeteer';
import timezone from '@/utils/timezone';

export const route: Route = {
path: '/lordslibrary/type/:topic?',
categories: ['government'],
example: '/parliament.uk/lordslibrary/type/research-briefing',
parameters: { topic: 'research by topic, string, example: [research-briefing|buisness|economy]' },
features: {
requireConfig: false,
requirePuppeteer: true,
antiCrawler: false,
supportBT: false,
supportPodcast: false,
supportScihub: false,
},
name: 'House of Lords Library',
maintainers: ['AntiKnot'],
handler,
};

async function handler(ctx) {
const { topic } = ctx.req.param();
const baseUrl = 'https://lordslibrary.parliament.uk/type';
const url = `${baseUrl}/${topic}/`;
const browser = await puppeteer();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
});
await page.goto(url, {
waitUntil: 'domcontentloaded',
});

const html = await page.evaluate(() => document.documentElement.innerHTML);
await page.close();
const $ = load(html);
const items = $('article.card--horizontal')
.map((_, article) => ({
title: $(article).find('.card__text a').text().trim(),
link: $(article).find('.card__text a').attr('href'),
description: $(article).find('p').last().text().trim(),
pubDate: timezone($(article).find('.card__date time').attr('datetime')),
}))
.toArray();
browser.close();
return {
title: `parliament - lordslibrary - ${topic}`,
link: baseUrl,
item: items,
};
}
6 changes: 6 additions & 0 deletions lib/routes/parliament.uk/namespace.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import type { Namespace } from '@/types';

export const namespace: Namespace = {
name: 'UK Parliament',
url: 'parliament.uk',
};
3 changes: 2 additions & 1 deletion lib/routes/tvb/news.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ async function handler(ctx) {
const language = ctx.req.param('language') ?? 'tc';

const rootUrl = 'https://inews-api.tvb.com';
const linkRootUrl = 'https://news.tvb.com';
const apiUrl = `${rootUrl}/news/entry/category`;
const currentUrl = `${rootUrl}/${language}/${category}`;

Expand All @@ -102,7 +103,7 @@ async function handler(ctx) {

const items = response.data.content.map((item) => ({
title: item.title,
link: `${rootUrl}/${language}/${category}/${item.id}`,
link: `${linkRootUrl}/${language}/${category}/${item.id}`,
pubDate: parseDate(item.publish_datetime),
category: [...item.category.map((c) => c.title), ...item.tags],
description: art(path.join(__dirname, 'templates/description.art'), {
Expand Down
8 changes: 4 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
"*.yml": "eslint --cache --fix"
},
"dependencies": {
"@hono/node-server": "1.13.1",
"@hono/node-server": "1.13.2",
"@hono/zod-openapi": "0.16.4",
"@notionhq/client": "2.2.15",
"@opentelemetry/api": "1.9.0",
Expand All @@ -62,7 +62,7 @@
"@opentelemetry/semantic-conventions": "1.27.0",
"@postlight/parser": "2.2.3",
"@rss3/sdk": "0.0.22",
"@scalar/hono-api-reference": "0.5.152",
"@scalar/hono-api-reference": "0.5.153",
"@sentry/node": "7.119.1",
"@tonyrl/rand-user-agent": "2.0.81",
"aes-js": "3.1.2",
Expand Down Expand Up @@ -128,14 +128,14 @@
"tough-cookie": "5.0.0",
"tsx": "4.19.1",
"twitter-api-v2": "1.18.0",
"undici": "6.19.8",
"undici": "6.20.0",
"uuid": "10.0.0",
"winston": "3.15.0",
"xxhash-wasm": "1.0.2",
"zod": "3.23.8"
},
"devDependencies": {
"@babel/preset-env": "7.25.7",
"@babel/preset-env": "7.25.8",
"@babel/preset-typescript": "7.25.7",
"@eslint/eslintrc": "3.1.0",
"@eslint/js": "9.12.0",
Expand Down
Loading

0 comments on commit fa165e6

Please sign in to comment.