Merge pull request #644 from DIYgod/master

[pull] master from diygod:master

lib/middleware/parameter.js

@@ -1,7 +1,7 @@
 const entities = require('entities');
 const cheerio = require('cheerio');
 const { simplecc } = require('simplecc-wasm');
-// const got = require('@/utils/got');
+const got = require('@/utils/got');
 const config = require('@/config').value;
 const RE2 = require('re2');
 
@@ -252,17 +252,12 @@ module.exports = async (ctx, next) => {
                     const parsed_result = await ctx.cache.tryGet(`mercury-cache-${link}`, async () => {
                         // if parser failed, return default description and not report error
                         try {
-                            mercury_parser = mercury_parser || require('@postlight/mercury-parser');
+                            mercury_parser = mercury_parser || require('@postlight/parser');
 
-                            // const res = await got(link);
-                            // const $ = cheerio.load(res.data, {
-                            // xmlMode: true,
-                            // });
+                            const { data: res } = await got(link);
+                            const $ = cheerio.load(res);
                             const result = await mercury_parser.parse(link, {
-                                // html: $.html(),
-                                headers: {
-                                    'User-Agent': config.ua,
-                                },
+                                html: $.html(),
                             });
                             return result;
                         } catch (e) {

package.json

@@ -69,6 +69,7 @@
     "remark-parse": "9.0.0",
     "remark-preset-prettier": "0.5.1",
     "remark-stringify": "9.0.1",
+    "request-promise-native": "1.0.9",
     "staged-git-files": "1.3.0",
     "string-width": "4.2.3",
     "supertest": "6.2.4",
@@ -81,7 +82,7 @@
   },
   "dependencies": {
     "@koa/router": "12.0.0",
-    "@postlight/mercury-parser": "2.2.1",
+    "@postlight/parser": "2.2.2",
     "@sentry/node": "7.13.0",
     "aes-js": "3.1.2",
     "art-template": "4.13.2",
@@ -98,7 +99,7 @@
     "etag": "1.8.1",
     "fanfou-sdk": "4.2.0",
     "git-rev-sync": "3.0.2",
-    "googleapis": "107.0.0",
+    "googleapis": "108.0.0",
     "got": "11.8.5",
     "https-proxy-agent": "5.0.1",
     "iconv-lite": "0.6.3",
@@ -122,7 +123,7 @@
     "pidusage": "3.0.0",
     "plist": "3.0.6",
     "proxy-chain": "2.0.7",
-    "puppeteer": "18.0.4",
+    "puppeteer": "18.0.5",
     "puppeteer-extra": "3.3.4",
     "puppeteer-extra-plugin-stealth": "2.11.1",
     "query-string": "7.1.1",

test/middleware/parameter.js

@@ -313,12 +313,12 @@ describe('wrong_path', () => {
 });
 
 describe('fulltext_mode', () => {
-    it.skip(`fulltext`, async () => {
+    it(`fulltext`, async () => {
         const response = await request.get('/test/1?mode=fulltext');
         expect(response.status).toBe(200);
         const parsed = await parser.parseString(response.text);
         expect(parsed.items[0].content).not.toBe(undefined);
-    }, 10000);
+    }, 20000);
 });
 
 describe('complicated_description', () => {