Merge pull request #569 from DIYgod/master

[pull] master from diygod:master

lib/v2/gov/maintainer.js

@@ -37,7 +37,7 @@ module.exports = {
     '/moj/aac/news/:type?': ['TonyRL'],
     '/mot/:category?': ['nczitzk'],
     '/news/:uid': ['EsuRt'],
-    '/nifdc/:path+': ['nczitzk'],
+    '/nifdc/:path?': ['nczitzk'],
     '/nmpa/:path+': ['TonyRL'],
     '/nopss/:path+': ['nczitzk'],
     '/npc/:caty': ['233yeee'],

lib/v2/gov/nifdc/index.js

@@ -4,68 +4,75 @@ const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
-    const params = ctx.path === '/nifdc' ? '/nifdc/bshff/ylqxbzhgl/qxggtzh' : ctx.path;
+    const { path = 'bshff/ylqxbzhgl/qxggtzh' } = ctx.params;
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 30;
 
     const rootUrl = 'https://www.nifdc.org.cn';
-    const currentUrl = `${rootUrl}${params}/index.html`;
+    const currentUrl = new URL(`nifdc/${path}/`, rootUrl).href;
 
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
+    const { data: response } = await got(currentUrl);
 
-    const $ = cheerio.load(response.data);
+    const $ = cheerio.load(response);
 
-    let items = $('.list ul li a')
-        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 20)
+    let items = $('div.list ul li')
+        .slice(0, limit)
         .toArray()
         .map((item) => {
             item = $(item);
 
-            const link = item.attr('href');
+            const a = item.find('a');
+            const link = a.prop('href');
 
             return {
-                title: item.text(),
-                link: /^http/.test(link) ? link : new URL(link, currentUrl).href,
-                pubDate: parseDate(
-                    item
-                        .next()
-                        .text()
-                        .match(/\((.*)\)/)
-                ),
+                title: a.prop('title') || a.text(),
+                link: link.startsWith('http') ? link : new URL(link, currentUrl).href,
+                pubDate: parseDate(item.find('span').text().replace(/\(|\)/g, '')),
             };
         });
 
     items = await Promise.all(
         items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
-                if (/^https:\/\/www\.nmpa\.gov\.cn\//.test(item.link)) {
-                    const { data: html } = await got(item.link);
+                try {
+                    const { data: detailResponse } = await got(item.link);
 
-                    const content = cheerio.load(html);
+                    const content = cheerio.load(detailResponse);
 
-                    item.description = content('.text').html();
-                    item.pubDate = timezone(parseDate($('meta[name="PubDate"]').attr('content')), +8);
-                } else {
-                    const detailResponse = await got({
-                        method: 'get',
-                        url: item.link,
-                    });
+                    item.title = content('.title').text();
+                    item.description = content('div.text').append(content('div.fujian')).html();
+                    item.author = content('meta[name="ContentSource"]').prop('content');
+                    item.category = [
+                        ...new Set([content('meta[name="ColumnName"]').prop('content'), content('meta[name="ColumnType"]').prop('content'), ...(content('meta[name="ColumnKeywords"]').prop('content').split(/,|;/) ?? [])]),
+                    ].filter((c) => c);
+                    item.pubDate = timezone(parseDate(content('meta[name="PubDate"]').prop('content')), +8);
+                    item.enclosure_url = content('a.fujianClass').first().prop('href');
 
-                    const content = cheerio.load(detailResponse.data);
-
-                    item.description = content('.text').append(content('.fujian')).html();
-                    item.pubDate = timezone(parseDate(content('meta[name="PubDate"]').attr('content')), +8);
+                    if (item.enclosure_url) {
+                        item.enclosure_url = new URL(item.enclosure_url, rootUrl).href;
+                        item.enclosure_type = `application/${item.enclosure_url.split(/\./).pop()}`;
+                    }
+                } catch (e) {
+                    // no-empty
                 }
 
                 return item;
             })
         )
     );
 
+    const image = new URL($('div.logo img').prop('src'), currentUrl).href;
+    const icon = new URL($('link[rel="shortcut icon"]').prop('href'), currentUrl).href;
+
     ctx.state.data = {
+        item: items,
         title: $('title').text().replace(/----/, ' - '),
         link: currentUrl,
-        item: items,
+        description: $('meta[name="ColumnDescription"]').prop('content'),
+        language: 'zh',
+        image,
+        icon,
+        logo: icon,
+        subtitle: $('meta[ name="ColumnName"]').prop('content'),
+        author: $('meta[name="SiteName"]').prop('content'),
     };
 };

lib/v2/gov/radar.js

@@ -1054,8 +1054,8 @@ module.exports = {
             {
                 title: '通用',
                 docs: 'https://docs.rsshub.app/routes/government#guo-jia-yao-pin-jian-du-guan-li-ju-yi-liao-qi-xie-biao-zhun-guan-li-zhong-xin',
-                source: ['/*path'],
-                target: (params) => `/gov/nifdc/${params.path.replace('/index.html', '')}`,
+                source: ['/*'],
+                target: (params) => `/gov/nifdc/${params.path.replace(/\/index\.html$/, '')}`,
             },
         ],
     },

lib/v2/gov/router.js

@@ -30,7 +30,7 @@ module.exports = function (router) {
     router.get('/mofcom/article/:suffix+', require('./mofcom/article'));
     router.get('/moj/aac/news/:type?', require('./moj/aac/news'));
     router.get('/news/:uid', require('./news'));
-    router.get(/nifdc\/([\w/-]+)?/, require('./nifdc'));
+    router.get('/nifdc/:path*', require('./nifdc'));
     router.get(/\/nmpa\/([\w][\w/]+)?/, require('./nmpa/generic'));
     router.get(/nopss(\/[\w/-]+)?/, require('./nopss'));
     router.get('/npc/:caty', require('./npc/index'));

lib/v2/instagram/templates/images.art

@@ -1,5 +1,11 @@
-{{@ summary.replace(/\n/g, '<br>') }}
+{{ if summary }}
+    {{@ summary.replace(/\n/g, '<br>') }}
+    <br>
+{{ /if }}
 
 {{ each images i }}
-<img src="{{ i.url }}" height="{{ i.height }}" width="{{ i.width }}">
+    <img src="{{ i.url }}"
+        {{ if i.height }} height="{{ i.height }}" {{ /if }}
+        {{ if i.width }} width="{{ i.width }}" {{ /if }}
+    >
 {{ /each }}

lib/v2/instagram/templates/video.art

@@ -1,4 +1,7 @@
-{{@ summary.replace(/\n/g, '<br>') }}
+{{ if summary }}
+    {{@ summary.replace(/\n/g, '<br>') }}
+    <br>
+{{ /if }}
 
 <video controls preload="none" poster="{{ image }}" width="{{ video.width }}">
     <source src="{{ video.url }}" type="video/mp4">

lib/v2/instagram/web-api/index.js

@@ -1,12 +1,12 @@
 const { CookieJar } = require('tough-cookie');
 const config = require('@/config').value;
 const { renderItems } = require('../common-utils');
-const { baseUrl, COOKIE_URL, getUserInfo, getUserFeedItems, getTagsFeedItems } = require('./utils');
+const { baseUrl, COOKIE_URL, getUserInfo, getUserFeedItems, getTagsFeedItems, renderGuestItems } = require('./utils');
 
 module.exports = async (ctx) => {
-    if (!config.instagram || !config.instagram.cookie) {
-        throw Error('Instagram RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>');
-    }
+    // if (!config.instagram || !config.instagram.cookie) {
+    //     throw Error('Instagram RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>');
+    // }
     const availableCategories = ['user', 'tags'];
     const { category, key } = ctx.params;
     const { cookie } = config.instagram;
@@ -16,13 +16,18 @@ module.exports = async (ctx) => {
 
     let cookieJar = await ctx.cache.get('instagram:cookieJar');
     const cacheMiss = !cookieJar;
-    if (cacheMiss) {
-        cookieJar = new CookieJar();
-        for await (const c of cookie.split('; ')) {
-            await cookieJar.setCookie(c, COOKIE_URL);
+
+    if (cookie) {
+        if (cacheMiss) {
+            cookieJar = new CookieJar();
+            for await (const c of cookie.split('; ')) {
+                await cookieJar.setCookie(c, COOKIE_URL);
+            }
+        } else {
+            cookieJar = CookieJar.fromJSON(cookieJar);
         }
     } else {
-        cookieJar = CookieJar.fromJSON(cookieJar);
+        cookieJar = new CookieJar();
     }
 
     let feedTitle, feedLink, feedDescription, feedLogo;
@@ -39,7 +44,12 @@ module.exports = async (ctx) => {
             feedLogo = userInfo.profile_pic_url_hd ?? userInfo.hd_profile_pic_url_info?.url ?? userInfo.profile_pic_url;
             feedLink = `${baseUrl}/${username}`;
 
-            items = await getUserFeedItems(id, username, cookieJar, ctx.cache.tryGet);
+            if (cookie) {
+                items = await getUserFeedItems(id, username, cookieJar, ctx.cache.tryGet);
+            } else {
+                items = [...userInfo.edge_felix_video_timeline.edges, ...userInfo.edge_owner_to_timeline_media.edges];
+            }
+
             break;
         }
         case 'tags': {
@@ -61,7 +71,7 @@ module.exports = async (ctx) => {
         title: feedTitle,
         link: feedLink,
         description: feedDescription,
-        item: renderItems(items),
+        item: cookie ? renderItems(items) : renderGuestItems(items),
         icon: `${baseUrl}/static/images/ico/xxhdpi_launcher.png/99cf3909d459.png`,
         logo: feedLogo,
         image: feedLogo,

lib/v2/instagram/web-api/utils.js

@@ -1,5 +1,8 @@
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
 const config = require('@/config').value;
+const { art } = require('@/utils/render');
+const path = require('path');
 
 const baseUrl = 'https://www.instagram.com';
 const COOKIE_URL = 'https://instagram.com';
@@ -23,23 +26,30 @@ const getUserInfo = async (username, cookieJar, cache) => {
     let userInfoCache = await cache.get(`instagram:userInfo:${id}`);
 
     if (!userInfoCache) {
-        const response = await got(`${baseUrl}/api/v1/users/web_profile_info/`, {
-            cookieJar,
-            headers: await getHeaders(cookieJar),
-            searchParams: {
-                username,
-            },
-        });
-        if (response.url.includes('/accounts/login/')) {
-            throw Error('Invalid cookie');
-        }
-        igWwwClaim = response.headers['x-ig-set-www-claim'] || igWwwClaim;
+        try {
+            const response = await got(`${baseUrl}/api/v1/users/web_profile_info/`, {
+                cookieJar,
+                headers: await getHeaders(cookieJar),
+                searchParams: {
+                    username,
+                },
+            });
+            if (response.url.includes('/accounts/login/')) {
+                throw Error('Invalid cookie');
+            }
+            igWwwClaim = response.headers['x-ig-set-www-claim'] || igWwwClaim;
 
-        webProfileInfo = response.data.data.user;
-        id = webProfileInfo.id;
+            webProfileInfo = response.data.data.user;
+            id = webProfileInfo.id;
 
-        await cache.set(`instagram:getIdByUsername:${username}`, id, 31536000); // 1 year since it will never change
-        await cache.set(`instagram:userInfo:${id}`, webProfileInfo);
+            await cache.set(`instagram:getIdByUsername:${username}`, id, 31536000); // 1 year since it will never change
+            await cache.set(`instagram:userInfo:${id}`, webProfileInfo);
+        } catch (e) {
+            if (e.message.includes("Cookie not in this host's domain")) {
+                throw Error('Invalid cookie');
+            }
+            throw e;
+        }
     }
 
     userInfoCache = typeof userInfoCache === 'string' ? JSON.parse(userInfoCache) : userInfoCache;
@@ -90,10 +100,71 @@ const getTagsFeedItems = (tag, tab, cookieJar, tryGet) =>
         false
     );
 
+const renderGuestItems = (items) => {
+    const renderVideo = (node, summary) =>
+        art(path.join(__dirname, '../templates/video.art'), {
+            summary,
+            image: node.display_url,
+            video: {
+                url: node.video_url,
+                height: node.dimensions.height,
+                width: node.dimensions.width,
+            },
+        });
+    const renderImages = (node, summary) =>
+        art(path.join(__dirname, '../templates/images.art'), {
+            summary,
+            images: [{ url: node.display_url, height: node.dimensions.height, width: node.dimensions.width }],
+        });
+
+    return items.map(({ node }) => {
+        const type = node.__typename;
+        const summary = node.edge_media_to_caption.edges[0]?.node.text ?? '';
+
+        let description = '';
+        switch (type) {
+            // carousel, can include GraphVideo and GraphImage
+            case 'GraphSidecar':
+                description = node.edge_sidecar_to_children.edges
+                    .map(({ node }, i) => {
+                        const _type = node.__typename;
+                        switch (_type) {
+                            case 'GraphVideo':
+                                return renderVideo(node, i === 0 ? summary : '');
+                            case 'GraphImage':
+                                return renderImages(node, i === 0 ? summary : '');
+                            default:
+                                throw Error(`Instagram: Unhandled carousel type: ${_type}`);
+                        }
+                    })
+                    .join('');
+                break;
+            case 'GraphVideo':
+                description = renderVideo(node, summary);
+                break;
+            case 'GraphImage':
+                description = renderImages(node, summary);
+                break;
+            default:
+                throw Error(`Instagram: Unhandled feed type: ${type}`);
+        }
+
+        return {
+            title: summary.split('\n')[0],
+            pubDate: parseDate(node.taken_at_timestamp, 'X'),
+            author: node.owner.username,
+            link: `${baseUrl}/p/${node.shortcode}/`,
+            summary,
+            description,
+        };
+    });
+};
+
 module.exports = {
     baseUrl,
     COOKIE_URL,
     getUserInfo,
     getUserFeedItems,
     getTagsFeedItems,
+    renderGuestItems,
 };

lib/v2/twitter/web-api/twitter-api.js

@@ -9,6 +9,10 @@ const queryString = require('query-string');
 let tokenIndex = 0;
 
 const twitterGot = async (url, params) => {
+    if (!config.twitter.oauthTokens.length || !config.twitter.oauthTokenSecrets.length || config.twitter.oauthTokens.length !== config.twitter.oauthTokenSecrets.length) {
+        throw Error('Invalid twitter oauth tokens');
+    }
+
     const oauth = OAuth({
         consumer: {
             key: consumerKey,
@@ -201,8 +205,14 @@ const userByAuto = (id) => {
     return userByScreenName(id);
 };
 const getUserData = (cache, id) => cache.tryGet(`twitter-userdata-${id}`, () => userByAuto(id));
-const getUserID = async (cache, id) => (await getUserData(cache, id)).data.user_result.result.rest_id;
-const getUser = async (cache, id) => (await getUserData(cache, id)).data.user_result.result.legacy;
+const getUserID = async (cache, id) => {
+    const userData = await getUserData(cache, id);
+    return (userData.data?.user || userData.data?.user_result)?.result?.rest_id;
+};
+const getUser = async (cache, id) => {
+    const userData = await getUserData(cache, id);
+    return (userData.data?.user || userData.data?.user_result)?.result?.legacy;
+};
 
 const cacheTryGet = async (cache, _id, params, func) => {
     const id = await getUserID(cache, _id);