Merge branch 'DIYgod:master' into master

DIYgod · Dec 16, 2024 · cd87f3d · cd87f3d
2 parents 17e0a80 + 8952721
commit cd87f3d
Show file tree

Hide file tree

Showing 344 changed files with 8,172 additions and 2,380 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -16,9 +16,9 @@
                 "EditorConfig.EditorConfig",
                 "esbenp.prettier-vscode",
                 "deepscan.vscode-deepscan",
-                "rangav.vscode-thunder-client",
                 "SonarSource.sonarlint-vscode",
                 "unifiedjs.vscode-mdx",
+                "VASubasRaj.flashpost", // Thunder Client is paywalled in WSL/Codespaces/SSH > 2.30.0
                 "ZihanLi.at-helper"
             ]
         }

diff --git a/.github/workflows/docker-test-cont.yml b/.github/workflows/docker-test-cont.yml
@@ -51,7 +51,7 @@ jobs:
 
       - name: Fetch Docker image
         if: (env.TEST_CONTINUE)
-        uses: dawidd6/action-download-artifact@v6
+        uses: dawidd6/action-download-artifact@v7
         with:
           workflow: ${{ github.event.workflow_run.workflow_id }}
           run_id: ${{ github.event.workflow_run.id }}

diff --git a/.gitpod.yml b/.gitpod.yml
@@ -32,7 +32,7 @@ vscode:
     - EditorConfig.EditorConfig
     - esbenp.prettier-vscode
     - deepscan.vscode-deepscan
-    - rangav.vscode-thunder-client
     - sonarsource.sonarlint-vscode
+    # - VASubasRaj.flashpost not available on Open VSX, Thunder Client is paywalled in WSL/Codespaces/SSH > 2.30.0
     - unifiedjs.vscode-mdx
     # - ZihanLi.at-helper not available on Open VSX
diff --git a/lib/config.ts b/lib/config.ts
@@ -330,6 +330,7 @@ export type Config = {
         authenticationSecret?: string[];
         phoneOrEmail?: string[];
         authToken?: string[];
+        thirdPartyApi?: string;
     };
     uestc: {
         bbsCookie?: string;
@@ -351,6 +352,9 @@ export type Config = {
         device_id?: string;
         refresh_token?: string;
     };
+    xiaohongshu: {
+        cookie?: string;
+    };
     ximalaya: {
         token?: string;
     };
@@ -751,6 +755,7 @@ const calculateValue = () => {
             authenticationSecret: envs.TWITTER_AUTHENTICATION_SECRET?.split(','),
             phoneOrEmail: envs.TWITTER_PHONE_OR_EMAIL?.split(','),
             authToken: envs.TWITTER_AUTH_TOKEN?.split(','),
+            thirdPartyApi: envs.TWITTER_THIRD_PARTY_API,
         },
         uestc: {
             bbsCookie: envs.UESTC_BBS_COOKIE,
@@ -772,6 +777,9 @@ const calculateValue = () => {
             device_id: envs.XIAOYUZHOU_ID,
             refresh_token: envs.XIAOYUZHOU_TOKEN,
         },
+        xiaohongshu: {
+            cookie: envs.XIAOHONGSHU_COOKIE,
+        },
         ximalaya: {
             token: envs.XIMALAYA_TOKEN,
         },

diff --git a/lib/index.ts b/lib/index.ts
@@ -20,6 +20,9 @@ const server = serve({
     fetch: app.fetch,
     hostname: config.listenInaddrAny ? '::' : '127.0.0.1',
     port,
+    serverOptions: {
+        maxHeaderSize: 1024 * 32,
+    },
 });
 
 export default server;
diff --git a/lib/middleware/cache.ts b/lib/middleware/cache.ts
@@ -8,17 +8,19 @@ import { Data } from '@/types';
 
 const bypassList = new Set(['/', '/robots.txt', '/logo.png', '/favicon.ico']);
 // only give cache string, as the `!` condition tricky
-// md5 is used to shrink key size
+// XXH64 is used to shrink key size
 // plz, write these tips in comments!
 const middleware: MiddlewareHandler = async (ctx, next) => {
     if (!cacheModule.status.available || bypassList.has(ctx.req.path)) {
         await next();
         return;
     }
 
+    const requestPath = ctx.req.path;
+    const limit = ctx.req.query('limit') ? `:${ctx.req.query('limit')}` : '';
     const { h64ToString } = await xxhash();
-    const key = 'rsshub:koa-redis-cache:' + h64ToString(ctx.req.path);
-    const controlKey = 'rsshub:path-requested:' + h64ToString(ctx.req.path);
+    const key = 'rsshub:koa-redis-cache:' + h64ToString(requestPath + limit);
+    const controlKey = 'rsshub:path-requested:' + h64ToString(requestPath + limit);
 
     const isRequesting = await cacheModule.globalCache.get(controlKey);
 

diff --git a/lib/router.js b/lib/router.js
@@ -874,9 +874,6 @@ router.get('/guet/xwzx/:type?', lazyloadRouteHandler('./routes/guet/news'));
 // はてな匿名ダイアリー
 router.get('/hatena/anonymous_diary/archive', lazyloadRouteHandler('./routes/hatena/anonymous_diary/archive'));
 
-// IEEE Xplore [Sci Journal]
-router.get('/ieee/author/:aid/:sortType/:count?', lazyloadRouteHandler('./routes/ieee/author'));
-
 // PNAS [Sci Journal]
 // router.get('/pnas/:topic?', lazyloadRouteHandler('./routes/pnas/index'));
 

diff --git a/lib/routes-deprecated/ieee/author.js b/lib/routes-deprecated/ieee/author.js
diff --git a/lib/routes/36kr/hot-list.ts b/lib/routes/36kr/hot-list.ts
@@ -4,6 +4,7 @@ import got from '@/utils/got';
 import { parseDate } from '@/utils/parse-date';
 
 import { rootUrl, ProcessItem } from './utils';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 const categories = {
     24: {
@@ -26,7 +27,7 @@ const categories = {
 
 export const route: Route = {
     path: '/hot-list/:category?',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/36kr/hot-list',
     parameters: { category: '分类，默认为24小时热榜' },
     features: {
@@ -54,6 +55,10 @@ export const route: Route = {
 async function handler(ctx) {
     const category = ctx.req.param('category') ?? '24';
 
+    if (!categories[category]) {
+        throw new InvalidParameterError('This category does not exist. Please refer to the documentation for the correct usage.');
+    }
+
     const currentUrl = category === '24' ? rootUrl : `${rootUrl}/hot-list/catalog`;
 
     const response = await got({

diff --git a/lib/routes/36kr/index.ts b/lib/routes/36kr/index.ts
@@ -18,7 +18,7 @@ const shortcuts = {
 
 export const route: Route = {
     path: '/:category/:subCategory?/:keyword?',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/36kr/newsflashes',
     parameters: {
         category: '分类，必填项',

diff --git a/lib/routes/aeon/category.ts b/lib/routes/aeon/category.ts
@@ -1,13 +1,24 @@
 import { Route } from '@/types';
-import { load } from 'cheerio';
-import got from '@/utils/got';
-import { getData } from './utils';
+import ofetch from '@/utils/ofetch';
+import { getBuildId, getData } from './utils';
+import { parseDate } from '@/utils/parse-date';
 
 export const route: Route = {
     path: '/category/:category',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/aeon/category/philosophy',
-    parameters: { category: 'Category' },
+    parameters: {
+        category: {
+            description: 'Category',
+            options: [
+                { value: 'philosophy', label: 'Philosophy' },
+                { value: 'science', label: 'Science' },
+                { value: 'psychology', label: 'Psychology' },
+                { value: 'society', label: 'Society' },
+                { value: 'culture', label: 'Culture' },
+            ],
+        },
+    },
     features: {
         requireConfig: false,
         requirePuppeteer: false,
@@ -18,34 +29,40 @@ export const route: Route = {
     },
     radar: [
         {
-            source: ['aeon.aeon.co/:category'],
+            source: ['aeon.co/:category'],
         },
     ],
     name: 'Categories',
     maintainers: ['emdoe'],
     handler,
-    description: `Supported categories: Philosophy, Science, Psychology, Society, and Culture.`,
 };
 
 async function handler(ctx) {
-    const url = `https://aeon.co/${ctx.req.param('category')}`;
-    const { data: response } = await got(url);
-    const $ = load(response);
+    const category = ctx.req.param('category').toLowerCase();
+    const url = `https://aeon.co/category/${category}`;
+    const buildId = await getBuildId();
+    const response = await ofetch(`https://aeon.co/_next/data/${buildId}/${category}.json`);
 
-    const data = JSON.parse($('script#__NEXT_DATA__').text());
+    const section = response.pageProps.section;
 
-    const list = data.props.pageProps.section.articles.edges.map((item) => ({
-        title: item.node.title,
-        author: item.node.authors.map((author) => author.displayName).join(', '),
-        link: `https://aeon.co/${item.node.type.toLowerCase()}s/${item.node.slug}`,
+    const list = section.articles.edges.map(({ node }) => ({
+        title: node.title,
+        description: node.standfirstLong,
+        author: node.authors.map((author) => author.displayName).join(', '),
+        link: `https://aeon.co/${node.type}s/${node.slug}`,
+        pubDate: parseDate(node.createdAt),
+        category: [node.section.title, ...node.topics.map((topic) => topic.title)],
+        image: node.image.url,
+        type: node.type,
+        slug: node.slug,
     }));
 
-    const items = await getData(ctx, list);
+    const items = await getData(list);
 
     return {
-        title: `AEON | ${data.props.pageProps.section.title}`,
+        title: `AEON | ${section.title}`,
         link: url,
-        description: data.props.pageProps.section.metaDescription,
+        description: section.metaDescription,
         item: items,
     };
 }
diff --git a/lib/routes/aeon/namespace.ts b/lib/routes/aeon/namespace.ts
@@ -2,6 +2,6 @@ import type { Namespace } from '@/types';
 
 export const namespace: Namespace = {
     name: 'AEON',
-    url: 'aeon.aeon.co',
+    url: 'aeon.co',
     lang: 'en',
 };
diff --git a/lib/routes/aeon/templates/essay.art b/lib/routes/aeon/templates/essay.art
@@ -1,3 +1,9 @@
-<img src="{{ banner }}" alt="">
+{{ if banner.url }}
+<figure>
+    <img src="{{ banner.url }}" alt="{{ banner.alt }}">
+    {{ if banner.caption }}
+    <figcaption>{{ banner.caption }}</figcaption>
+    {{ /if }}
+{{ /if }}
 {{@ authorsBio }}
-{{@ content}}
+{{@ content }}
diff --git a/lib/routes/aeon/templates/video.art b/lib/routes/aeon/templates/video.art
@@ -1,10 +1,10 @@
 {{ set video = article.hosterId }}
 {{ if article.hoster === 'vimeo' }}
-    {{ set video = "https://player.vimeo.com/video/" + video + "?dnt=1"}}
-{{ else if article.hoster == 'youtube' }}
+    {{ set video = "https://player.vimeo.com/video/" + video + "?dnt=1" }}
+{{ else if article.hoster === 'youtube' }}
     {{ set video = "https://www.youtube-nocookie.com/embed/" + video }}
 {{ /if }}
 
 <iframe width="672" height="377" src="{{ video }}" frameborder="0" allowfullscreen></iframe>
-{{@ article.credits}}
-{{@ article.description}}
+{{@ article.credits }}
+{{@ article.description }}
diff --git a/lib/routes/aeon/type.ts b/lib/routes/aeon/type.ts
@@ -1,13 +1,22 @@
 import { Route } from '@/types';
-import { load } from 'cheerio';
-import got from '@/utils/got';
-import { getData } from './utils';
+import ofetch from '@/utils/ofetch';
+import { getBuildId, getData } from './utils';
+import { parseDate } from '@/utils/parse-date';
 
 export const route: Route = {
     path: '/:type',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/aeon/essays',
-    parameters: { type: 'Type' },
+    parameters: {
+        type: {
+            description: 'Type',
+            options: [
+                { value: 'essays', label: 'Essays' },
+                { value: 'videos', label: 'Videos' },
+                { value: 'audio', label: 'Audio' },
+            ],
+        },
+    },
     features: {
         requireConfig: false,
         requirePuppeteer: false,
@@ -18,36 +27,38 @@ export const route: Route = {
     },
     radar: [
         {
-            source: ['aeon.aeon.co/:type'],
+            source: ['aeon.co/:type'],
         },
     ],
     name: 'Types',
     maintainers: ['emdoe'],
     handler,
     description: `Supported types: Essays, Videos, and Audio.
 
-  Compared to the official one, the RSS feed generated by RSSHub not only has more fine-grained options, but also eliminates pull quotes, which can't be easily distinguished from other paragraphs by any RSS reader, but only disrupt the reading flow. This feed also provides users with a bio of the author at the top.
-
-  However, The content generated under \`audio\` does not contain links to audio files.`,
+  Compared to the official one, the RSS feed generated by RSSHub not only has more fine-grained options, but also eliminates pull quotes, which can't be easily distinguished from other paragraphs by any RSS reader, but only disrupt the reading flow. This feed also provides users with a bio of the author at the top.`,
 };
 
 async function handler(ctx) {
     const type = ctx.req.param('type');
-    const binaryType = type === 'videos' ? 'videos' : 'essays';
     const capitalizedType = type.charAt(0).toUpperCase() + type.slice(1);
 
+    const buildId = await getBuildId();
     const url = `https://aeon.co/${type}`;
-    const { data: response } = await got(url);
-    const $ = load(response);
-
-    const data = JSON.parse($('script#__NEXT_DATA__').text());
+    const response = await ofetch(`https://aeon.co/_next/data/${buildId}/${type}.json`);
 
-    const list = data.props.pageProps.articles.map((item) => ({
-        title: item.title,
-        link: `https://aeon.co/${binaryType}/${item.slug}`,
+    const list = response.pageProps.articles.map((node) => ({
+        title: node.title,
+        description: node.standfirstLong,
+        author: node.authors.map((author) => author.displayName).join(', '),
+        link: `https://aeon.co/${node.type}s/${node.slug}`,
+        pubDate: parseDate(node.createdAt),
+        category: [node.section.title, ...node.topics.map((topic) => topic.title)],
+        image: node.image.url,
+        type: node.type,
+        slug: node.slug,
     }));
 
-    const items = await getData(ctx, list);
+    const items = await getData(list);
 
     return {
         title: `AEON | ${capitalizedType}`,