diff --git a/client/package.json b/client/package.json index 348e14508..83e7f37b5 100644 --- a/client/package.json +++ b/client/package.json @@ -18,6 +18,7 @@ "@next/font": "13.1.6", "@tanstack/react-query": "^4.24.10", "@types/nprogress": "^0.2.0", + "@mozilla/readability": "^0.4.4", "axios": "^1.3.3", "cookie": "^0.5.0", "crypto": "^1.0.1", @@ -58,6 +59,7 @@ "request-ip": "^3.3.0", "sass": "^1.58.3", "tunnel": "^0.0.6", + "jsdom": "^22.1.0" "winston": "^3.10.0", "winston-mongodb": "^5.1.1", "zustand": "^4.3.5" @@ -77,6 +79,7 @@ "@types/react-syntax-highlighter": "^15.5.6", "@types/request-ip": "^0.0.37", "@types/tunnel": "^0.0.3", + "@types/jsdom": "^21.1.1", "eslint": "8.34.0", "eslint-config-next": "13.1.6", "typescript": "4.9.5" diff --git a/client/src/pages/api/plugins/kb/fetchContent.ts b/client/src/pages/api/plugins/kb/fetchContent.ts new file mode 100644 index 000000000..96c4e94ac --- /dev/null +++ b/client/src/pages/api/plugins/kb/fetchContent.ts @@ -0,0 +1,51 @@ +// pages/api/fetchContent.ts +import { NextApiRequest, NextApiResponse } from 'next'; +import axios from 'axios'; +import { JSDOM } from 'jsdom'; +import { Readability } from '@mozilla/readability'; +import { jsonRes } from '@/service/response'; + +const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => { + const { url } = req.body; + if (!url) { + return res.status(400).json({ error: 'URL is required' }); + } + + try { + const response = await axios.get(url, { + httpsAgent: new (require('https').Agent)({ + rejectUnauthorized: false, + }), + }); + + const dom = new JSDOM(response.data, { + url, + contentType: 'text/html', + }); + + const reader = new Readability(dom.window.document); + const article = reader.parse(); + + if (!article) { + jsonRes(res, { + code: 500, + error: '页面获取失败或页面为空' + }); + return; + } + + jsonRes(res, { + code: 200, + data: article.content + }); + + } catch (error:any) { + jsonRes(res, { + code: 500, + error: error + }); + } + +}; + +export default fetchContent; diff --git a/docs/zh/examples/ChatGLM2/image.png b/docs/zh/examples/ChatGLM2/image.png new file mode 100644 index 000000000..1fbf61365 Binary files /dev/null and b/docs/zh/examples/ChatGLM2/image.png differ