From b4dda6a41b8d2644652940040ebe946301b973e7 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Tue, 4 Mar 2025 14:45:29 +0800 Subject: [PATCH] fix: Check the url to avoid ssrf attacks (#3965) * fix: Check the url to avoid ssrf attacks * Delete docSite/content/zh-cn/docs/development/upgrading/490.md --- packages/plugins/src/fetchUrl/template.json | 158 +++++++++++++----- packages/service/common/string/cheerio.ts | 11 ++ packages/service/common/system/utils.ts | 63 +++++++ .../core/app/httpPlugin/getApiSchemaByUrl.ts | 29 ++-- 4 files changed, 208 insertions(+), 53 deletions(-) create mode 100644 packages/service/common/system/utils.ts diff --git a/packages/plugins/src/fetchUrl/template.json b/packages/plugins/src/fetchUrl/template.json index 38700ad6a..571787c17 100644 --- a/packages/plugins/src/fetchUrl/template.json +++ b/packages/plugins/src/fetchUrl/template.json @@ -16,7 +16,7 @@ "nodeId": "lmpb9v2lo2lk", "name": "插件开始", "intro": "自定义配置外部输入，使用插件时，仅暴露自定义配置的输入", - "avatar": "/imgs/workflow/input.png", + "avatar": "core/workflow/template/workflowStart", "flowNodeType": "pluginInput", "showStatus": false, "position": { @@ -26,14 +26,16 @@ "version": "481", "inputs": [ { - "renderTypeList": ["reference"], + "renderTypeList": ["input", "reference"], "selectedTypeIndex": 0, "valueType": "string", "key": "url", "label": "url", "description": "需要读取的网页链接", "required": true, - "toolDescription": "需要读取的网页链接" + "toolDescription": "需要读取的网页链接", + "list": [], + "defaultValue": "" } ], "outputs": [ @@ -50,12 +52,12 @@ "nodeId": "i7uow4wj2wdp", "name": "插件输出", "intro": "自定义配置外部输出，使用插件时，仅暴露自定义配置的输出", - "avatar": "/imgs/workflow/output.png", + "avatar": "core/workflow/template/pluginOutput", "flowNodeType": "pluginOutput", "showStatus": false, "position": { - "x": 1607.7142331269129, - "y": -150.8808596935447 + "x": 1853.935047606551, + "y": -154.13661665265613 }, "version": "481", "inputs": [ @@ -81,12 +83,12 @@ "nodeId": "ebLCxU43hHuZ", "name": "HTTP 请求", "intro": "可以发出一个 HTTP 请求，实现更为复杂的操作（联网搜索、数据库查询等）", - "avatar": "/imgs/workflow/http.png", + "avatar": "core/workflow/template/httpRequest", "flowNodeType": "httpRequest468", "showStatus": true, "position": { - "x": 1050.9890727421412, - "y": -415.2085119990912 + "x": 1054.2940501177068, + "y": -503.13661665265613 }, "version": "481", "inputs": [ @@ -96,7 +98,7 @@ "valueType": "dynamic", "label": "", "required": false, - "description": "core.module.input.description.HTTP Dynamic Input", + "description": "common:core.module.input.description.HTTP Dynamic Input", "customInputConfig": { "selectValueTypeList": [ "string", @@ -107,16 +109,19 @@ "arrayNumber", "arrayBoolean", "arrayObject", + "arrayAny", "any", "chatHistory", "datasetQuote", "dynamic", - "selectApp", - "selectDataset" + "selectDataset", + "selectApp" ], "showDescription": false, "showDefaultValue": true - } + }, + "debugLabel": "", + "toolDescription": "" }, { "key": "system_httpMethod", @@ -124,17 +129,33 @@ "valueType": "string", "label": "", "value": "POST", - "required": true + "required": true, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpTimeout", + "renderTypeList": ["custom"], + "valueType": "number", + "label": "", + "value": 30, + "min": 5, + "max": 600, + "required": true, + "debugLabel": "", + "toolDescription": "" }, { "key": "system_httpReqUrl", "renderTypeList": ["hidden"], "valueType": "string", "label": "", - "description": "core.module.input.description.Http Request Url", + "description": "common:core.module.input.description.Http Request Url", "placeholder": "https://api.ai.com/getInventory", "required": false, - "value": "fetchUrl" + "value": "fetchUrl", + "debugLabel": "", + "toolDescription": "" }, { "key": "system_httpHeader", @@ -142,9 +163,11 @@ "valueType": "any", "value": [], "label": "", - "description": "core.module.input.description.Http Request Header", - "placeholder": "core.module.input.description.Http Request Header", - "required": false + "description": "common:core.module.input.description.Http Request Header", + "placeholder": "common:core.module.input.description.Http Request Header", + "required": false, + "debugLabel": "", + "toolDescription": "" }, { "key": "system_httpParams", @@ -152,7 +175,9 @@ "valueType": "any", "value": [], "label": "", - "required": false + "required": false, + "debugLabel": "", + "toolDescription": "" }, { "key": "system_httpJsonBody", @@ -160,7 +185,29 @@ "valueType": "any", "value": "{\n \"url\": \"{{url}}\"\n}", "label": "", - "required": false + "required": false, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpFormBody", + "renderTypeList": ["hidden"], + "valueType": "any", + "value": [], + "label": "", + "required": false, + "debugLabel": "", + "toolDescription": "" + }, + { + "key": "system_httpContentType", + "renderTypeList": ["hidden"], + "valueType": "string", + "value": "json", + "label": "", + "required": false, + "debugLabel": "", + "toolDescription": "" }, { "renderTypeList": ["reference"], @@ -178,12 +225,13 @@ "arrayNumber", "arrayBoolean", "arrayObject", + "arrayAny", "any", "chatHistory", "datasetQuote", "dynamic", - "selectApp", - "selectDataset" + "selectDataset", + "selectApp" ], "showDescription": false, "showDefaultValue": true @@ -193,6 +241,23 @@ } ], "outputs": [ + { + "id": "error", + "key": "error", + "label": "workflow:request_error", + "description": "HTTP请求错误信息，成功时返回空", + "valueType": "object", + "type": "static" + }, + { + "id": "httpRawResponse", + "key": "httpRawResponse", + "required": true, + "label": "workflow:raw_response", + "description": "HTTP请求的原始响应。只能接受字符串或JSON类型响应数据。", + "valueType": "any", + "type": "static" + }, { "id": "system_addOutputParam", "key": "system_addOutputParam", @@ -220,23 +285,6 @@ "showDefaultValue": true } }, - { - "id": "error", - "key": "error", - "label": "请求错误", - "description": "HTTP请求错误信息，成功时返回空", - "valueType": "object", - "type": "static" - }, - { - "id": "httpRawResponse", - "key": "httpRawResponse", - "label": "原始响应", - "required": true, - "description": "HTTP请求的原始响应。只能接受字符串或JSON类型响应数据。", - "valueType": "any", - "type": "static" - }, { "id": "rH4tMV02robs", "valueType": "string", @@ -260,6 +308,34 @@ "sourceHandle": "ebLCxU43hHuZ-source-right", "targetHandle": "i7uow4wj2wdp-target-left" } - ] + ], + "chatConfig": { + "welcomeText": "", + "variables": [], + "questionGuide": { + "open": false, + "model": "gpt-4o-mini", + "customPrompt": "You are an AI assistant tasked with predicting the user's next question based on the conversation history. Your goal is to generate 3 potential questions that will guide the user to continue the conversation. When generating these questions, adhere to the following rules:\n\n1. Use the same language as the user's last question in the conversation history.\n2. Keep each question under 20 characters in length.\n\nAnalyze the conversation history provided to you and use it as context to generate relevant and engaging follow-up questions. Your predictions should be logical extensions of the current topic or related areas that the user might be interested in exploring further.\n\nRemember to maintain consistency in tone and style with the existing conversation while providing diverse options for the user to choose from. Your goal is to keep the conversation flowing naturally and help the user delve deeper into the subject matter or explore related topics." + }, + "ttsConfig": { + "type": "web" + }, + "whisperConfig": { + "open": false, + "autoSend": false, + "autoTTSResponse": false + }, + "chatInputGuide": { + "open": false, + "textList": [], + "customUrl": "" + }, + "instruction": "", + "autoExecute": { + "open": false, + "defaultPrompt": "" + }, + "_id": "677b59849d672185a5671b45" + } } } diff --git a/packages/service/common/string/cheerio.ts b/packages/service/common/string/cheerio.ts index 32726eb04..4e8b52d10 100644 --- a/packages/service/common/string/cheerio.ts +++ b/packages/service/common/string/cheerio.ts @@ -2,6 +2,7 @@ import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/ap import * as cheerio from 'cheerio'; import axios from 'axios'; import { htmlToMarkdown } from './utils'; +import { isInternalAddress } from '../system/utils'; export const cheerioToHtml = ({ fetchUrl, @@ -75,6 +76,16 @@ export const urlsFetch = async ({ const response = await Promise.all( urlList.map(async (url) => { + const isInternal = isInternalAddress(url); + if (isInternal) { + return { + url, + title: '', + content: 'Cannot fetch internal url', + selector: '' + }; + } + try { const fetchRes = await axios.get(url, { timeout: 30000 diff --git a/packages/service/common/system/utils.ts b/packages/service/common/system/utils.ts new file mode 100644 index 000000000..3151a4394 --- /dev/null +++ b/packages/service/common/system/utils.ts @@ -0,0 +1,63 @@ +import { SERVICE_LOCAL_HOST } from './tools'; + +export const isInternalAddress = (url: string): boolean => { + try { + const parsedUrl = new URL(url); + const hostname = parsedUrl.hostname; + const fullUrl = parsedUrl.toString(); + + // Check for localhost and common internal domains + if (hostname === SERVICE_LOCAL_HOST) { + return true; + } + + // Metadata endpoints whitelist + const metadataEndpoints = [ + // AWS + 'http://169.254.169.254/latest/meta-data/', + // Azure + 'http://169.254.169.254/metadata/instance?api-version=2021-02-01', + // GCP + 'http://metadata.google.internal/computeMetadata/v1/', + // Alibaba Cloud + 'http://100.100.100.200/latest/meta-data/', + // Tencent Cloud + 'http://metadata.tencentyun.com/latest/meta-data/', + // Huawei Cloud + 'http://169.254.169.254/latest/meta-data/' + ]; + if (metadataEndpoints.some((endpoint) => fullUrl.startsWith(endpoint))) { + return true; + } + + // For non-metadata URLs, check if it's a domain name + const ipv4Pattern = /^(\d{1,3}\.){3}\d{1,3}$/; + if (!ipv4Pattern.test(hostname)) { + return true; + } + + // ... existing IP validation code ... + const parts = hostname.split('.').map(Number); + + if (parts.length !== 4 || parts.some((part) => part < 0 || part > 255)) { + return false; + } + + // Only allow public IP ranges + return ( + parts[0] !== 0 && + parts[0] !== 10 && + parts[0] !== 127 && + !(parts[0] === 169 && parts[1] === 254) && + !(parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) && + !(parts[0] === 192 && parts[1] === 168) && + !(parts[0] >= 224 && parts[0] <= 239) && + !(parts[0] >= 240 && parts[0] <= 255) && + !(parts[0] === 100 && parts[1] >= 64 && parts[1] <= 127) && + !(parts[0] === 9 && parts[1] === 0) && + !(parts[0] === 11 && parts[1] === 0) + ); + } catch { + return false; // If URL parsing fails, reject it as potentially unsafe + } +}; diff --git a/projects/app/src/pages/api/core/app/httpPlugin/getApiSchemaByUrl.ts b/projects/app/src/pages/api/core/app/httpPlugin/getApiSchemaByUrl.ts index 3ad92c46e..48780eb6e 100644 --- a/projects/app/src/pages/api/core/app/httpPlugin/getApiSchemaByUrl.ts +++ b/projects/app/src/pages/api/core/app/httpPlugin/getApiSchemaByUrl.ts @@ -1,18 +1,23 @@ import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; import { loadOpenAPISchemaFromUrl } from '@fastgpt/global/common/string/swagger'; +import { NextAPI } from '@/service/middleware/entry'; +import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; +import { isInternalAddress } from '@fastgpt/service/common/system/utils'; -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const apiURL = req.body.url as string; +async function handler(req: NextApiRequest, res: NextApiResponse) { + const apiURL = req.body.url as string; - return jsonRes(res, { - data: await loadOpenAPISchemaFromUrl(apiURL) - }); - } catch (err) { - jsonRes(res, { - code: 500, - error: err - }); + if (!apiURL) { + return Promise.reject(CommonErrEnum.missingParams); } + + const isInternal = isInternalAddress(apiURL); + + if (isInternal) { + return Promise.reject('Invalid url'); + } + + return await loadOpenAPISchemaFromUrl(apiURL); } + +export default NextAPI(handler);