feat: mix search weight (#4170)
* feat: mix search weight * feat: svg render
This commit is contained in:
parent
a534b839d7
commit
fa0a8dd2da
@ -13,7 +13,8 @@ weight: 799
|
|||||||
2. 知识库分块阅读器。
|
2. 知识库分块阅读器。
|
||||||
3. API 知识库支持 PDF 增强解析。
|
3. API 知识库支持 PDF 增强解析。
|
||||||
4. 邀请团队成员,改为邀请链接模式。
|
4. 邀请团队成员,改为邀请链接模式。
|
||||||
5. 支持重排模型选择和权重设置,同时调整了知识库搜索权重计算方式,改成 搜索权重 + 重排权重,而不是向量检索权重+全文检索权重+重排权重。
|
5. 支持混合检索权重设置。
|
||||||
|
6. 支持重排模型选择和权重设置,同时调整了知识库搜索权重计算方式,改成 搜索权重 + 重排权重,而不是向量检索权重+全文检索权重+重排权重。
|
||||||
|
|
||||||
## ⚙️ 优化
|
## ⚙️ 优化
|
||||||
|
|
||||||
|
|||||||
1
packages/global/core/app/type.d.ts
vendored
1
packages/global/core/app/type.d.ts
vendored
@ -75,6 +75,7 @@ export type AppDatasetSearchParamsType = {
|
|||||||
searchMode: `${DatasetSearchModeEnum}`;
|
searchMode: `${DatasetSearchModeEnum}`;
|
||||||
limit?: number; // limit max tokens
|
limit?: number; // limit max tokens
|
||||||
similarity?: number;
|
similarity?: number;
|
||||||
|
embeddingWeight?: number; // embedding weight, fullText weight = 1 - embeddingWeight
|
||||||
|
|
||||||
usingReRank?: boolean;
|
usingReRank?: boolean;
|
||||||
rerankModel?: string;
|
rerankModel?: string;
|
||||||
|
|||||||
@ -108,6 +108,10 @@ export const appWorkflow2Form = ({
|
|||||||
defaultAppForm.dataset.searchMode =
|
defaultAppForm.dataset.searchMode =
|
||||||
findInputValueByKey(node.inputs, NodeInputKeyEnum.datasetSearchMode) ||
|
findInputValueByKey(node.inputs, NodeInputKeyEnum.datasetSearchMode) ||
|
||||||
DatasetSearchModeEnum.embedding;
|
DatasetSearchModeEnum.embedding;
|
||||||
|
defaultAppForm.dataset.embeddingWeight = findInputValueByKey(
|
||||||
|
node.inputs,
|
||||||
|
NodeInputKeyEnum.datasetSearchEmbeddingWeight
|
||||||
|
);
|
||||||
// Rerank
|
// Rerank
|
||||||
defaultAppForm.dataset.usingReRank = !!findInputValueByKey(
|
defaultAppForm.dataset.usingReRank = !!findInputValueByKey(
|
||||||
node.inputs,
|
node.inputs,
|
||||||
|
|||||||
@ -185,7 +185,7 @@ export enum SearchScoreTypeEnum {
|
|||||||
}
|
}
|
||||||
export const SearchScoreTypeMap = {
|
export const SearchScoreTypeMap = {
|
||||||
[SearchScoreTypeEnum.embedding]: {
|
[SearchScoreTypeEnum.embedding]: {
|
||||||
label: i18nT('common:core.dataset.search.score.embedding'),
|
label: i18nT('common:core.dataset.search.mode.embedding'),
|
||||||
desc: i18nT('common:core.dataset.search.score.embedding desc'),
|
desc: i18nT('common:core.dataset.search.score.embedding desc'),
|
||||||
showScore: true
|
showScore: true
|
||||||
},
|
},
|
||||||
|
|||||||
@ -154,9 +154,12 @@ export enum NodeInputKeyEnum {
|
|||||||
datasetSimilarity = 'similarity',
|
datasetSimilarity = 'similarity',
|
||||||
datasetMaxTokens = 'limit',
|
datasetMaxTokens = 'limit',
|
||||||
datasetSearchMode = 'searchMode',
|
datasetSearchMode = 'searchMode',
|
||||||
|
datasetSearchEmbeddingWeight = 'embeddingWeight',
|
||||||
|
|
||||||
datasetSearchUsingReRank = 'usingReRank',
|
datasetSearchUsingReRank = 'usingReRank',
|
||||||
datasetSearchRerankWeight = 'rerankWeight',
|
datasetSearchRerankWeight = 'rerankWeight',
|
||||||
datasetSearchRerankModel = 'rerankModel',
|
datasetSearchRerankModel = 'rerankModel',
|
||||||
|
|
||||||
datasetSearchUsingExtensionQuery = 'datasetSearchUsingExtensionQuery',
|
datasetSearchUsingExtensionQuery = 'datasetSearchUsingExtensionQuery',
|
||||||
datasetSearchExtensionModel = 'datasetSearchExtensionModel',
|
datasetSearchExtensionModel = 'datasetSearchExtensionModel',
|
||||||
datasetSearchExtensionBg = 'datasetSearchExtensionBg',
|
datasetSearchExtensionBg = 'datasetSearchExtensionBg',
|
||||||
|
|||||||
@ -133,6 +133,9 @@ export type DispatchNodeResponseType = {
|
|||||||
similarity?: number;
|
similarity?: number;
|
||||||
limit?: number;
|
limit?: number;
|
||||||
searchMode?: `${DatasetSearchModeEnum}`;
|
searchMode?: `${DatasetSearchModeEnum}`;
|
||||||
|
embeddingWeight?: number;
|
||||||
|
rerankModel?: string;
|
||||||
|
rerankWeight?: number;
|
||||||
searchUsingReRank?: boolean;
|
searchUsingReRank?: boolean;
|
||||||
queryExtensionResult?: {
|
queryExtensionResult?: {
|
||||||
model: string;
|
model: string;
|
||||||
|
|||||||
@ -64,6 +64,13 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
|
|||||||
valueType: WorkflowIOValueTypeEnum.string,
|
valueType: WorkflowIOValueTypeEnum.string,
|
||||||
value: DatasetSearchModeEnum.embedding
|
value: DatasetSearchModeEnum.embedding
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
key: NodeInputKeyEnum.datasetSearchEmbeddingWeight,
|
||||||
|
renderTypeList: [FlowNodeInputTypeEnum.hidden],
|
||||||
|
label: '',
|
||||||
|
valueType: WorkflowIOValueTypeEnum.number,
|
||||||
|
value: 0.5
|
||||||
|
},
|
||||||
// Rerank
|
// Rerank
|
||||||
{
|
{
|
||||||
key: NodeInputKeyEnum.datasetSearchUsingReRank,
|
key: NodeInputKeyEnum.datasetSearchUsingReRank,
|
||||||
|
|||||||
@ -40,6 +40,7 @@ export type SearchDatasetDataProps = {
|
|||||||
[NodeInputKeyEnum.datasetSimilarity]?: number; // min distance
|
[NodeInputKeyEnum.datasetSimilarity]?: number; // min distance
|
||||||
[NodeInputKeyEnum.datasetMaxTokens]: number; // max Token limit
|
[NodeInputKeyEnum.datasetMaxTokens]: number; // max Token limit
|
||||||
[NodeInputKeyEnum.datasetSearchMode]?: `${DatasetSearchModeEnum}`;
|
[NodeInputKeyEnum.datasetSearchMode]?: `${DatasetSearchModeEnum}`;
|
||||||
|
[NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
|
||||||
|
|
||||||
[NodeInputKeyEnum.datasetSearchUsingReRank]?: boolean;
|
[NodeInputKeyEnum.datasetSearchUsingReRank]?: boolean;
|
||||||
[NodeInputKeyEnum.datasetSearchRerankModel]?: RerankModelItemType;
|
[NodeInputKeyEnum.datasetSearchRerankModel]?: RerankModelItemType;
|
||||||
@ -161,6 +162,7 @@ export async function searchDatasetData(
|
|||||||
similarity = 0,
|
similarity = 0,
|
||||||
limit: maxTokens,
|
limit: maxTokens,
|
||||||
searchMode = DatasetSearchModeEnum.embedding,
|
searchMode = DatasetSearchModeEnum.embedding,
|
||||||
|
embeddingWeight = 0.5,
|
||||||
usingReRank = false,
|
usingReRank = false,
|
||||||
rerankModel,
|
rerankModel,
|
||||||
rerankWeight = 0.5,
|
rerankWeight = 0.5,
|
||||||
@ -731,16 +733,20 @@ export async function searchDatasetData(
|
|||||||
})();
|
})();
|
||||||
|
|
||||||
// embedding recall and fullText recall rrf concat
|
// embedding recall and fullText recall rrf concat
|
||||||
|
const baseK = 120;
|
||||||
|
const embK = Math.round(baseK * (1 - embeddingWeight)); // 搜索结果的 k 值
|
||||||
|
const fullTextK = Math.round(baseK * embeddingWeight); // rerank 结果的 k 值
|
||||||
|
|
||||||
const rrfSearchResult = datasetSearchResultConcat([
|
const rrfSearchResult = datasetSearchResultConcat([
|
||||||
{ k: 60, list: embeddingRecallResults },
|
{ k: embK, list: embeddingRecallResults },
|
||||||
{ k: 60, list: fullTextRecallResults }
|
{ k: fullTextK, list: fullTextRecallResults }
|
||||||
]);
|
]);
|
||||||
const rrfConcatResults = (() => {
|
const rrfConcatResults = (() => {
|
||||||
|
if (reRankResults.length === 0) return rrfSearchResult;
|
||||||
if (rerankWeight === 1) return reRankResults;
|
if (rerankWeight === 1) return reRankResults;
|
||||||
|
|
||||||
const baseK = 30;
|
const searchK = Math.round(baseK * rerankWeight); // 搜索结果的 k 值
|
||||||
const searchK = Math.round(baseK / (1 - rerankWeight)); // 搜索结果的 k 值
|
const rerankK = Math.round(baseK * (1 - rerankWeight)); // rerank 结果的 k 值
|
||||||
const rerankK = Math.round(baseK / rerankWeight); // rerank 结果的 k 值
|
|
||||||
|
|
||||||
return datasetSearchResultConcat([
|
return datasetSearchResultConcat([
|
||||||
{ k: searchK, list: rrfSearchResult },
|
{ k: searchK, list: rrfSearchResult },
|
||||||
|
|||||||
@ -22,8 +22,9 @@ type DatasetSearchProps = ModuleDispatchProps<{
|
|||||||
[NodeInputKeyEnum.datasetSelectList]: SelectedDatasetType;
|
[NodeInputKeyEnum.datasetSelectList]: SelectedDatasetType;
|
||||||
[NodeInputKeyEnum.datasetSimilarity]: number;
|
[NodeInputKeyEnum.datasetSimilarity]: number;
|
||||||
[NodeInputKeyEnum.datasetMaxTokens]: number;
|
[NodeInputKeyEnum.datasetMaxTokens]: number;
|
||||||
[NodeInputKeyEnum.datasetSearchMode]: `${DatasetSearchModeEnum}`;
|
|
||||||
[NodeInputKeyEnum.userChatInput]?: string;
|
[NodeInputKeyEnum.userChatInput]?: string;
|
||||||
|
[NodeInputKeyEnum.datasetSearchMode]: `${DatasetSearchModeEnum}`;
|
||||||
|
[NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
|
||||||
|
|
||||||
[NodeInputKeyEnum.datasetSearchUsingReRank]: boolean;
|
[NodeInputKeyEnum.datasetSearchUsingReRank]: boolean;
|
||||||
[NodeInputKeyEnum.datasetSearchRerankModel]?: string;
|
[NodeInputKeyEnum.datasetSearchRerankModel]?: string;
|
||||||
@ -57,11 +58,11 @@ export async function dispatchDatasetSearch(
|
|||||||
datasets = [],
|
datasets = [],
|
||||||
similarity,
|
similarity,
|
||||||
limit = 1500,
|
limit = 1500,
|
||||||
searchMode,
|
|
||||||
userChatInput = '',
|
userChatInput = '',
|
||||||
authTmbId = false,
|
authTmbId = false,
|
||||||
collectionFilterMatch,
|
collectionFilterMatch,
|
||||||
|
searchMode,
|
||||||
|
embeddingWeight,
|
||||||
usingReRank,
|
usingReRank,
|
||||||
rerankModel,
|
rerankModel,
|
||||||
rerankWeight,
|
rerankWeight,
|
||||||
@ -129,6 +130,7 @@ export async function dispatchDatasetSearch(
|
|||||||
limit,
|
limit,
|
||||||
datasetIds,
|
datasetIds,
|
||||||
searchMode,
|
searchMode,
|
||||||
|
embeddingWeight,
|
||||||
usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)),
|
usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)),
|
||||||
rerankModel: getRerankModel(rerankModel),
|
rerankModel: getRerankModel(rerankModel),
|
||||||
rerankWeight,
|
rerankWeight,
|
||||||
@ -228,6 +230,9 @@ export async function dispatchDatasetSearch(
|
|||||||
similarity: usingSimilarityFilter ? similarity : undefined,
|
similarity: usingSimilarityFilter ? similarity : undefined,
|
||||||
limit,
|
limit,
|
||||||
searchMode,
|
searchMode,
|
||||||
|
embeddingWeight: searchMode === DatasetSearchModeEnum.mixedRecall ? embeddingWeight : undefined,
|
||||||
|
rerankModel: usingReRank ? getRerankModel(rerankModel)?.name : undefined,
|
||||||
|
rerankWeight: usingReRank ? rerankWeight : undefined,
|
||||||
searchUsingReRank: searchUsingReRank,
|
searchUsingReRank: searchUsingReRank,
|
||||||
quoteList: searchRes,
|
quoteList: searchRes,
|
||||||
queryExtensionResult,
|
queryExtensionResult,
|
||||||
|
|||||||
@ -429,6 +429,7 @@ export const iconPaths = {
|
|||||||
'price/bg': () => import('./icons/price/bg.svg'),
|
'price/bg': () => import('./icons/price/bg.svg'),
|
||||||
'price/right': () => import('./icons/price/right.svg'),
|
'price/right': () => import('./icons/price/right.svg'),
|
||||||
save: () => import('./icons/save.svg'),
|
save: () => import('./icons/save.svg'),
|
||||||
|
sliderTag: () => import('./icons/sliderTag.svg'),
|
||||||
stop: () => import('./icons/stop.svg'),
|
stop: () => import('./icons/stop.svg'),
|
||||||
'support/account/laf': () => import('./icons/support/account/laf.svg'),
|
'support/account/laf': () => import('./icons/support/account/laf.svg'),
|
||||||
'support/account/loginoutLight': () => import('./icons/support/account/loginoutLight.svg'),
|
'support/account/loginoutLight': () => import('./icons/support/account/loginoutLight.svg'),
|
||||||
|
|||||||
3
packages/web/components/common/Icon/icons/sliderTag.svg
Normal file
3
packages/web/components/common/Icon/icons/sliderTag.svg
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 11 11" >
|
||||||
|
<path d="M5.04123 0.144501L9.47821 4.82132C9.83075 5.19292 10.0273 5.68562 10.0273 6.19784V8.65565C10.0273 9.76022 9.13185 10.6557 8.02728 10.6557H2.05518C0.950606 10.6557 0.0551758 9.76022 0.0551758 8.65565V6.19785C0.0551758 5.68562 0.251705 5.19292 0.604247 4.82132L5.04123 0.144501Z" fill="#3370FF"/>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 375 B |
@ -120,7 +120,6 @@
|
|||||||
"publish_success": "Publish Successful",
|
"publish_success": "Publish Successful",
|
||||||
"question_guide_tip": "After the conversation, 3 guiding questions will be generated for you.",
|
"question_guide_tip": "After the conversation, 3 guiding questions will be generated for you.",
|
||||||
"reasoning_response": "Output thinking",
|
"reasoning_response": "Output thinking",
|
||||||
"rerank_weight": "Rearrange weights",
|
|
||||||
"response_format": "Response format",
|
"response_format": "Response format",
|
||||||
"saved_success": "Saved successfully! \nTo use this version externally, click Save and Publish",
|
"saved_success": "Saved successfully! \nTo use this version externally, click Save and Publish",
|
||||||
"search_app": "Search apps",
|
"search_app": "Search apps",
|
||||||
|
|||||||
@ -49,6 +49,7 @@
|
|||||||
"response.child total points": "Sub-workflow point consumption",
|
"response.child total points": "Sub-workflow point consumption",
|
||||||
"response.dataset_concat_length": "Combined total",
|
"response.dataset_concat_length": "Combined total",
|
||||||
"response.node_inputs": "Node Inputs",
|
"response.node_inputs": "Node Inputs",
|
||||||
|
"response_hybrid_weight": "Embedding : Full text = {{emb}} : {{text}}",
|
||||||
"select": "Select",
|
"select": "Select",
|
||||||
"select_file": "Upload File",
|
"select_file": "Upload File",
|
||||||
"select_file_img": "Upload file / image",
|
"select_file_img": "Upload file / image",
|
||||||
|
|||||||
@ -1025,6 +1025,7 @@
|
|||||||
"question_feedback": "Work order",
|
"question_feedback": "Work order",
|
||||||
"read_quote": "View citations",
|
"read_quote": "View citations",
|
||||||
"required": "Required",
|
"required": "Required",
|
||||||
|
"rerank_weight": "Rearrange weights",
|
||||||
"resume_failed": "Resume Failed",
|
"resume_failed": "Resume Failed",
|
||||||
"select_reference_variable": "Select Reference Variable",
|
"select_reference_variable": "Select Reference Variable",
|
||||||
"share_link": "Share Link",
|
"share_link": "Share Link",
|
||||||
|
|||||||
@ -120,7 +120,6 @@
|
|||||||
"publish_success": "发布成功",
|
"publish_success": "发布成功",
|
||||||
"question_guide_tip": "对话结束后,会为你生成 3 个引导性问题。",
|
"question_guide_tip": "对话结束后,会为你生成 3 个引导性问题。",
|
||||||
"reasoning_response": "输出思考",
|
"reasoning_response": "输出思考",
|
||||||
"rerank_weight": "重排权重",
|
|
||||||
"response_format": "回复格式",
|
"response_format": "回复格式",
|
||||||
"saved_success": "保存成功!如需在外部使用该版本,请点击“保存并发布”",
|
"saved_success": "保存成功!如需在外部使用该版本,请点击“保存并发布”",
|
||||||
"search_app": "搜索应用",
|
"search_app": "搜索应用",
|
||||||
|
|||||||
@ -49,6 +49,7 @@
|
|||||||
"response.child total points": "子工作流积分消耗",
|
"response.child total points": "子工作流积分消耗",
|
||||||
"response.dataset_concat_length": "合并后总数",
|
"response.dataset_concat_length": "合并后总数",
|
||||||
"response.node_inputs": "节点输入",
|
"response.node_inputs": "节点输入",
|
||||||
|
"response_hybrid_weight": "语义检索 : 全文检索 = {{emb}} : {{text}}",
|
||||||
"select": "选择",
|
"select": "选择",
|
||||||
"select_file": "上传文件",
|
"select_file": "上传文件",
|
||||||
"select_file_img": "上传文件/图片",
|
"select_file_img": "上传文件/图片",
|
||||||
|
|||||||
@ -623,7 +623,6 @@
|
|||||||
"core.dataset.search.mode.fullTextRecall desc": "使用传统的全文检索,适合查找一些关键词和主谓语特殊的数据",
|
"core.dataset.search.mode.fullTextRecall desc": "使用传统的全文检索,适合查找一些关键词和主谓语特殊的数据",
|
||||||
"core.dataset.search.mode.mixedRecall": "混合检索",
|
"core.dataset.search.mode.mixedRecall": "混合检索",
|
||||||
"core.dataset.search.mode.mixedRecall desc": "使用向量检索与全文检索的综合结果返回,使用 RRF 算法进行排序。",
|
"core.dataset.search.mode.mixedRecall desc": "使用向量检索与全文检索的综合结果返回,使用 RRF 算法进行排序。",
|
||||||
"core.dataset.search.score.embedding": "语义检索",
|
|
||||||
"core.dataset.search.score.embedding desc": "通过计算向量之间的距离获取得分,范围为 0~1。",
|
"core.dataset.search.score.embedding desc": "通过计算向量之间的距离获取得分,范围为 0~1。",
|
||||||
"core.dataset.search.score.fullText": "全文检索",
|
"core.dataset.search.score.fullText": "全文检索",
|
||||||
"core.dataset.search.score.fullText desc": "计算相同关键词的得分,范围为 0~无穷。",
|
"core.dataset.search.score.fullText desc": "计算相同关键词的得分,范围为 0~无穷。",
|
||||||
@ -1029,6 +1028,7 @@
|
|||||||
"question_feedback": "工单咨询",
|
"question_feedback": "工单咨询",
|
||||||
"read_quote": "查看引用",
|
"read_quote": "查看引用",
|
||||||
"required": "必须",
|
"required": "必须",
|
||||||
|
"rerank_weight": "重排权重",
|
||||||
"resume_failed": "恢复失败",
|
"resume_failed": "恢复失败",
|
||||||
"select_reference_variable": "选择引用变量",
|
"select_reference_variable": "选择引用变量",
|
||||||
"share_link": "分享链接",
|
"share_link": "分享链接",
|
||||||
|
|||||||
@ -120,7 +120,6 @@
|
|||||||
"publish_success": "發布成功",
|
"publish_success": "發布成功",
|
||||||
"question_guide_tip": "對話結束後,會為你產生 3 個引導性問題。",
|
"question_guide_tip": "對話結束後,會為你產生 3 個引導性問題。",
|
||||||
"reasoning_response": "輸出思考",
|
"reasoning_response": "輸出思考",
|
||||||
"rerank_weight": "重排權重",
|
|
||||||
"response_format": "回复格式",
|
"response_format": "回复格式",
|
||||||
"saved_success": "保存成功!\n如需在外部使用該版本,請點擊“儲存並發布”",
|
"saved_success": "保存成功!\n如需在外部使用該版本,請點擊“儲存並發布”",
|
||||||
"search_app": "搜尋應用程式",
|
"search_app": "搜尋應用程式",
|
||||||
|
|||||||
@ -48,6 +48,7 @@
|
|||||||
"response.child total points": "子工作流程點數消耗",
|
"response.child total points": "子工作流程點數消耗",
|
||||||
"response.dataset_concat_length": "合併總數",
|
"response.dataset_concat_length": "合併總數",
|
||||||
"response.node_inputs": "節點輸入",
|
"response.node_inputs": "節點輸入",
|
||||||
|
"response_hybrid_weight": "語義檢索 : 全文檢索 = {{emb}} : {{text}}",
|
||||||
"select": "選取",
|
"select": "選取",
|
||||||
"select_file": "上傳檔案",
|
"select_file": "上傳檔案",
|
||||||
"select_file_img": "上傳檔案 / 圖片",
|
"select_file_img": "上傳檔案 / 圖片",
|
||||||
|
|||||||
@ -1024,6 +1024,7 @@
|
|||||||
"question_feedback": "工單諮詢",
|
"question_feedback": "工單諮詢",
|
||||||
"read_quote": "查看引用",
|
"read_quote": "查看引用",
|
||||||
"required": "必填",
|
"required": "必填",
|
||||||
|
"rerank_weight": "重排權重",
|
||||||
"resume_failed": "恢復失敗",
|
"resume_failed": "恢復失敗",
|
||||||
"select_reference_variable": "選擇引用變數",
|
"select_reference_variable": "選擇引用變數",
|
||||||
"share_link": "分享連結",
|
"share_link": "分享連結",
|
||||||
|
|||||||
@ -85,7 +85,7 @@ export default React.memo(Markdown);
|
|||||||
function Code(e: any) {
|
function Code(e: any) {
|
||||||
const { className, codeBlock, children } = e;
|
const { className, codeBlock, children } = e;
|
||||||
const match = /language-(\w+)/.exec(className || '');
|
const match = /language-(\w+)/.exec(className || '');
|
||||||
const codeType = match?.[1];
|
const codeType = match?.[1]?.toLowerCase();
|
||||||
|
|
||||||
const strChildren = String(children);
|
const strChildren = String(children);
|
||||||
|
|
||||||
@ -105,7 +105,7 @@ function Code(e: any) {
|
|||||||
if (codeType === CodeClassNameEnum.iframe) {
|
if (codeType === CodeClassNameEnum.iframe) {
|
||||||
return <IframeCodeBlock code={strChildren} />;
|
return <IframeCodeBlock code={strChildren} />;
|
||||||
}
|
}
|
||||||
if (codeType && codeType.toLowerCase() === CodeClassNameEnum.html) {
|
if (codeType === CodeClassNameEnum.html || codeType === CodeClassNameEnum.svg) {
|
||||||
return (
|
return (
|
||||||
<IframeHtmlCodeBlock className={className} codeBlock={codeBlock} match={match}>
|
<IframeHtmlCodeBlock className={className} codeBlock={codeBlock} match={match}>
|
||||||
{children}
|
{children}
|
||||||
|
|||||||
@ -8,6 +8,7 @@ export enum CodeClassNameEnum {
|
|||||||
latex = 'latex',
|
latex = 'latex',
|
||||||
iframe = 'iframe',
|
iframe = 'iframe',
|
||||||
html = 'html',
|
html = 'html',
|
||||||
|
svg = 'svg',
|
||||||
video = 'video',
|
video = 'video',
|
||||||
audio = 'audio'
|
audio = 'audio'
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,5 +1,17 @@
|
|||||||
import React, { useEffect, useMemo, useState } from 'react';
|
import React, { useEffect, useMemo, useState } from 'react';
|
||||||
import { Box, Button, Flex, HStack, ModalBody, ModalFooter, Switch } from '@chakra-ui/react';
|
import {
|
||||||
|
Box,
|
||||||
|
Button,
|
||||||
|
Flex,
|
||||||
|
HStack,
|
||||||
|
ModalBody,
|
||||||
|
ModalFooter,
|
||||||
|
Switch,
|
||||||
|
Slider,
|
||||||
|
SliderTrack,
|
||||||
|
SliderFilledTrack,
|
||||||
|
SliderThumb
|
||||||
|
} from '@chakra-ui/react';
|
||||||
import { useForm } from 'react-hook-form';
|
import { useForm } from 'react-hook-form';
|
||||||
import MyModal from '@fastgpt/web/components/common/MyModal';
|
import MyModal from '@fastgpt/web/components/common/MyModal';
|
||||||
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
|
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
|
||||||
@ -17,6 +29,7 @@ import { defaultDatasetMaxTokens } from '@fastgpt/global/core/app/constants';
|
|||||||
import InputSlider from '@fastgpt/web/components/common/MySlider/InputSlider';
|
import InputSlider from '@fastgpt/web/components/common/MySlider/InputSlider';
|
||||||
import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
|
import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
|
||||||
import { AppDatasetSearchParamsType } from '@fastgpt/global/core/app/type';
|
import { AppDatasetSearchParamsType } from '@fastgpt/global/core/app/type';
|
||||||
|
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||||
|
|
||||||
enum SearchSettingTabEnum {
|
enum SearchSettingTabEnum {
|
||||||
searchMode = 'searchMode',
|
searchMode = 'searchMode',
|
||||||
@ -28,6 +41,7 @@ const DatasetParamsModal = ({
|
|||||||
searchMode = DatasetSearchModeEnum.embedding,
|
searchMode = DatasetSearchModeEnum.embedding,
|
||||||
limit,
|
limit,
|
||||||
similarity,
|
similarity,
|
||||||
|
embeddingWeight,
|
||||||
usingReRank,
|
usingReRank,
|
||||||
rerankModel,
|
rerankModel,
|
||||||
rerankWeight,
|
rerankWeight,
|
||||||
@ -62,12 +76,13 @@ const DatasetParamsModal = ({
|
|||||||
const { register, setValue, getValues, handleSubmit, watch } =
|
const { register, setValue, getValues, handleSubmit, watch } =
|
||||||
useForm<AppDatasetSearchParamsType>({
|
useForm<AppDatasetSearchParamsType>({
|
||||||
defaultValues: {
|
defaultValues: {
|
||||||
limit,
|
|
||||||
similarity,
|
|
||||||
searchMode,
|
searchMode,
|
||||||
|
embeddingWeight: embeddingWeight || 0.5,
|
||||||
usingReRank: !!usingReRank && teamPlanStatus?.standardConstants?.permissionReRank !== false,
|
usingReRank: !!usingReRank && teamPlanStatus?.standardConstants?.permissionReRank !== false,
|
||||||
rerankModel: rerankModel || defaultModels?.rerank?.model,
|
rerankModel: rerankModel || defaultModels?.rerank?.model,
|
||||||
rerankWeight: rerankWeight || 0.5,
|
rerankWeight: rerankWeight || 0.5,
|
||||||
|
limit,
|
||||||
|
similarity,
|
||||||
datasetSearchUsingExtensionQuery,
|
datasetSearchUsingExtensionQuery,
|
||||||
datasetSearchExtensionModel: datasetSearchExtensionModel || defaultModels.llm?.model,
|
datasetSearchExtensionModel: datasetSearchExtensionModel || defaultModels.llm?.model,
|
||||||
datasetSearchExtensionBg
|
datasetSearchExtensionBg
|
||||||
@ -75,6 +90,12 @@ const DatasetParamsModal = ({
|
|||||||
});
|
});
|
||||||
|
|
||||||
const searchModeWatch = watch('searchMode');
|
const searchModeWatch = watch('searchMode');
|
||||||
|
const embeddingWeightWatch = watch('embeddingWeight');
|
||||||
|
const fullTextWeightWatch = useMemo(() => {
|
||||||
|
const val = 1 - (embeddingWeightWatch || 0.5);
|
||||||
|
return Number(val.toFixed(2));
|
||||||
|
}, [embeddingWeightWatch]);
|
||||||
|
|
||||||
const datasetSearchUsingCfrForm = watch('datasetSearchUsingExtensionQuery');
|
const datasetSearchUsingCfrForm = watch('datasetSearchUsingExtensionQuery');
|
||||||
const queryExtensionModel = watch('datasetSearchExtensionModel');
|
const queryExtensionModel = watch('datasetSearchExtensionModel');
|
||||||
|
|
||||||
@ -168,8 +189,45 @@ const DatasetParamsModal = ({
|
|||||||
{
|
{
|
||||||
title: t('common:core.dataset.search.mode.mixedRecall'),
|
title: t('common:core.dataset.search.mode.mixedRecall'),
|
||||||
desc: t('common:core.dataset.search.mode.mixedRecall desc'),
|
desc: t('common:core.dataset.search.mode.mixedRecall desc'),
|
||||||
value: DatasetSearchModeEnum.mixedRecall
|
value: DatasetSearchModeEnum.mixedRecall,
|
||||||
// children: searchModeWatch === DatasetSearchModeEnum.mixedRecall && <Box>111</Box>
|
children: searchModeWatch === DatasetSearchModeEnum.mixedRecall && (
|
||||||
|
<Box mt={3}>
|
||||||
|
<HStack justifyContent={'space-between'}>
|
||||||
|
<Flex alignItems={'center'}>
|
||||||
|
<Box fontSize={'sm'} color={'myGray.900'}>
|
||||||
|
{t('common:core.dataset.search.mode.embedding')}
|
||||||
|
</Box>
|
||||||
|
<Box fontSize={'xs'} color={'myGray.500'}>
|
||||||
|
{embeddingWeightWatch}
|
||||||
|
</Box>
|
||||||
|
</Flex>
|
||||||
|
<Flex alignItems={'center'}>
|
||||||
|
<Box fontSize={'sm'} color={'myGray.900'}>
|
||||||
|
{t('common:core.dataset.search.score.fullText')}
|
||||||
|
</Box>
|
||||||
|
<Box fontSize={'xs'} color={'myGray.500'}>
|
||||||
|
{fullTextWeightWatch}
|
||||||
|
</Box>
|
||||||
|
</Flex>
|
||||||
|
</HStack>
|
||||||
|
<Slider
|
||||||
|
defaultValue={embeddingWeightWatch}
|
||||||
|
min={0.1}
|
||||||
|
max={0.9}
|
||||||
|
step={0.01}
|
||||||
|
onChange={(e) => {
|
||||||
|
setValue('embeddingWeight', Number(e.toFixed(2)));
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<SliderTrack bg={'#F9518E'}>
|
||||||
|
<SliderFilledTrack bg={'#3370FF'} />
|
||||||
|
</SliderTrack>
|
||||||
|
<SliderThumb boxShadow={'none'} bg={'none'}>
|
||||||
|
<MyIcon transform={'translateY(10px)'} name={'sliderTag'} w={'1rem'} />
|
||||||
|
</SliderThumb>
|
||||||
|
</Slider>
|
||||||
|
</Box>
|
||||||
|
)
|
||||||
}
|
}
|
||||||
]}
|
]}
|
||||||
value={searchModeWatch}
|
value={searchModeWatch}
|
||||||
@ -201,7 +259,7 @@ const DatasetParamsModal = ({
|
|||||||
<>
|
<>
|
||||||
<HStack mt={3} justifyContent={'space-between'}>
|
<HStack mt={3} justifyContent={'space-between'}>
|
||||||
<Box fontSize={'sm'} flex={'0 0 100px'} color={'myGray.700'}>
|
<Box fontSize={'sm'} flex={'0 0 100px'} color={'myGray.700'}>
|
||||||
{t('app:rerank_weight')}
|
{t('common:rerank_weight')}
|
||||||
</Box>
|
</Box>
|
||||||
<Box flex={'1 0 0'}>
|
<Box flex={'1 0 0'}>
|
||||||
<InputSlider
|
<InputSlider
|
||||||
|
|||||||
@ -228,10 +228,23 @@ export const WholeResponseContent = ({
|
|||||||
{activeModule?.searchMode && (
|
{activeModule?.searchMode && (
|
||||||
<Row
|
<Row
|
||||||
label={t('common:core.dataset.search.search mode')}
|
label={t('common:core.dataset.search.search mode')}
|
||||||
// @ts-ignore
|
rawDom={
|
||||||
value={t(DatasetSearchModeMap[activeModule.searchMode]?.title)}
|
<Flex border={'base'} borderRadius={'md'} p={2}>
|
||||||
|
<Box>
|
||||||
|
{/* @ts-ignore */}
|
||||||
|
{t(DatasetSearchModeMap[activeModule.searchMode]?.title)}
|
||||||
|
</Box>
|
||||||
|
{activeModule.embeddingWeight && (
|
||||||
|
<>{`(${t('chat:response_hybrid_weight', {
|
||||||
|
emb: activeModule.embeddingWeight,
|
||||||
|
text: 1 - activeModule.embeddingWeight
|
||||||
|
})})`}</>
|
||||||
|
)}
|
||||||
|
</Flex>
|
||||||
|
}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
<Row
|
<Row
|
||||||
label={t('common:core.chat.response.module similarity')}
|
label={t('common:core.chat.response.module similarity')}
|
||||||
value={activeModule?.similarity}
|
value={activeModule?.similarity}
|
||||||
@ -239,7 +252,19 @@ export const WholeResponseContent = ({
|
|||||||
<Row label={t('common:core.chat.response.module limit')} value={activeModule?.limit} />
|
<Row label={t('common:core.chat.response.module limit')} value={activeModule?.limit} />
|
||||||
<Row
|
<Row
|
||||||
label={t('common:core.chat.response.search using reRank')}
|
label={t('common:core.chat.response.search using reRank')}
|
||||||
value={`${activeModule?.searchUsingReRank}`}
|
rawDom={
|
||||||
|
<Box border={'base'} borderRadius={'md'} p={2}>
|
||||||
|
{activeModule?.searchUsingReRank ? (
|
||||||
|
activeModule?.rerankModel ? (
|
||||||
|
<Box>{`${activeModule.rerankModel}: ${activeModule.rerankWeight}`}</Box>
|
||||||
|
) : (
|
||||||
|
'True'
|
||||||
|
)
|
||||||
|
) : (
|
||||||
|
`False`
|
||||||
|
)}
|
||||||
|
</Box>
|
||||||
|
}
|
||||||
/>
|
/>
|
||||||
{activeModule.queryExtensionResult && (
|
{activeModule.queryExtensionResult && (
|
||||||
<>
|
<>
|
||||||
|
|||||||
@ -63,8 +63,13 @@ export type SearchTestProps = {
|
|||||||
text: string;
|
text: string;
|
||||||
[NodeInputKeyEnum.datasetSimilarity]?: number;
|
[NodeInputKeyEnum.datasetSimilarity]?: number;
|
||||||
[NodeInputKeyEnum.datasetMaxTokens]?: number;
|
[NodeInputKeyEnum.datasetMaxTokens]?: number;
|
||||||
|
|
||||||
[NodeInputKeyEnum.datasetSearchMode]?: `${DatasetSearchModeEnum}`;
|
[NodeInputKeyEnum.datasetSearchMode]?: `${DatasetSearchModeEnum}`;
|
||||||
|
[NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
|
||||||
|
|
||||||
[NodeInputKeyEnum.datasetSearchUsingReRank]?: boolean;
|
[NodeInputKeyEnum.datasetSearchUsingReRank]?: boolean;
|
||||||
|
[NodeInputKeyEnum.datasetSearchRerankModel]?: string;
|
||||||
|
[NodeInputKeyEnum.datasetSearchRerankWeight]?: number;
|
||||||
|
|
||||||
[NodeInputKeyEnum.datasetSearchUsingExtensionQuery]?: boolean;
|
[NodeInputKeyEnum.datasetSearchUsingExtensionQuery]?: boolean;
|
||||||
[NodeInputKeyEnum.datasetSearchExtensionModel]?: string;
|
[NodeInputKeyEnum.datasetSearchExtensionModel]?: string;
|
||||||
|
|||||||
@ -24,6 +24,7 @@ const SelectDatasetParam = ({ inputs = [], nodeId }: RenderInputProps) => {
|
|||||||
|
|
||||||
const [data, setData] = useState<AppDatasetSearchParamsType>({
|
const [data, setData] = useState<AppDatasetSearchParamsType>({
|
||||||
searchMode: DatasetSearchModeEnum.embedding,
|
searchMode: DatasetSearchModeEnum.embedding,
|
||||||
|
embeddingWeight: 0.5,
|
||||||
limit: 3000,
|
limit: 3000,
|
||||||
similarity: 0.5,
|
similarity: 0.5,
|
||||||
usingReRank: false,
|
usingReRank: false,
|
||||||
|
|||||||
@ -36,9 +36,14 @@ type FormType = {
|
|||||||
inputText: string;
|
inputText: string;
|
||||||
searchParams: {
|
searchParams: {
|
||||||
searchMode: `${DatasetSearchModeEnum}`;
|
searchMode: `${DatasetSearchModeEnum}`;
|
||||||
|
embeddingWeight?: number;
|
||||||
|
|
||||||
|
usingReRank?: boolean;
|
||||||
|
rerankModel?: string;
|
||||||
|
rerankWeight?: number;
|
||||||
|
|
||||||
similarity?: number;
|
similarity?: number;
|
||||||
limit?: number;
|
limit?: number;
|
||||||
usingReRank?: boolean;
|
|
||||||
datasetSearchUsingExtensionQuery?: boolean;
|
datasetSearchUsingExtensionQuery?: boolean;
|
||||||
datasetSearchExtensionModel?: string;
|
datasetSearchExtensionModel?: string;
|
||||||
datasetSearchExtensionBg?: string;
|
datasetSearchExtensionBg?: string;
|
||||||
@ -53,7 +58,6 @@ const Test = ({ datasetId }: { datasetId: string }) => {
|
|||||||
const { pushDatasetTestItem } = useSearchTestStore();
|
const { pushDatasetTestItem } = useSearchTestStore();
|
||||||
const [inputType, setInputType] = useState<'text' | 'file'>('text');
|
const [inputType, setInputType] = useState<'text' | 'file'>('text');
|
||||||
const [datasetTestItem, setDatasetTestItem] = useState<SearchTestStoreItemType>();
|
const [datasetTestItem, setDatasetTestItem] = useState<SearchTestStoreItemType>();
|
||||||
const [refresh, setRefresh] = useState(false);
|
|
||||||
const [isFocus, setIsFocus] = useState(false);
|
const [isFocus, setIsFocus] = useState(false);
|
||||||
const { File, onOpen } = useSelectFile({
|
const { File, onOpen } = useSelectFile({
|
||||||
fileType: '.csv',
|
fileType: '.csv',
|
||||||
@ -66,7 +70,10 @@ const Test = ({ datasetId }: { datasetId: string }) => {
|
|||||||
inputText: '',
|
inputText: '',
|
||||||
searchParams: {
|
searchParams: {
|
||||||
searchMode: DatasetSearchModeEnum.embedding,
|
searchMode: DatasetSearchModeEnum.embedding,
|
||||||
|
embeddingWeight: 0.5,
|
||||||
usingReRank: false,
|
usingReRank: false,
|
||||||
|
rerankModel: defaultModels?.rerank?.model,
|
||||||
|
rerankWeight: 0.5,
|
||||||
limit: 5000,
|
limit: 5000,
|
||||||
similarity: 0,
|
similarity: 0,
|
||||||
datasetSearchUsingExtensionQuery: false,
|
datasetSearchUsingExtensionQuery: false,
|
||||||
@ -77,6 +84,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
const searchModeData = DatasetSearchModeMap[getValues(`searchParams.searchMode`)];
|
const searchModeData = DatasetSearchModeMap[getValues(`searchParams.searchMode`)];
|
||||||
|
const searchParams = getValues('searchParams');
|
||||||
|
|
||||||
const {
|
const {
|
||||||
isOpen: isOpenSelectMode,
|
isOpen: isOpenSelectMode,
|
||||||
@ -294,15 +302,14 @@ const Test = ({ datasetId }: { datasetId: string }) => {
|
|||||||
|
|
||||||
{isOpenSelectMode && (
|
{isOpenSelectMode && (
|
||||||
<DatasetParamsModal
|
<DatasetParamsModal
|
||||||
{...getValues('searchParams')}
|
{...searchParams}
|
||||||
maxTokens={20000}
|
maxTokens={20000}
|
||||||
onClose={onCloseSelectMode}
|
onClose={onCloseSelectMode}
|
||||||
onSuccess={(e) => {
|
onSuccess={(e) => {
|
||||||
setValue('searchParams', {
|
setValue('searchParams', {
|
||||||
...getValues('searchParams'),
|
...searchParams,
|
||||||
...e
|
...e
|
||||||
});
|
});
|
||||||
setRefresh((state) => !state);
|
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
|||||||
@ -16,6 +16,7 @@ import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
|
|||||||
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
|
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
|
||||||
import { useIPFrequencyLimit } from '@fastgpt/service/common/middle/reqFrequencyLimit';
|
import { useIPFrequencyLimit } from '@fastgpt/service/common/middle/reqFrequencyLimit';
|
||||||
import { ApiRequestProps } from '@fastgpt/service/type/next';
|
import { ApiRequestProps } from '@fastgpt/service/type/next';
|
||||||
|
import { getRerankModel } from '@fastgpt/service/core/ai/model';
|
||||||
|
|
||||||
async function handler(req: ApiRequestProps<SearchTestProps>): Promise<SearchTestResponse> {
|
async function handler(req: ApiRequestProps<SearchTestProps>): Promise<SearchTestResponse> {
|
||||||
const {
|
const {
|
||||||
@ -24,7 +25,11 @@ async function handler(req: ApiRequestProps<SearchTestProps>): Promise<SearchTes
|
|||||||
limit = 1500,
|
limit = 1500,
|
||||||
similarity,
|
similarity,
|
||||||
searchMode,
|
searchMode,
|
||||||
|
embeddingWeight,
|
||||||
|
|
||||||
usingReRank,
|
usingReRank,
|
||||||
|
rerankModel,
|
||||||
|
rerankWeight,
|
||||||
|
|
||||||
datasetSearchUsingExtensionQuery = false,
|
datasetSearchUsingExtensionQuery = false,
|
||||||
datasetSearchExtensionModel,
|
datasetSearchExtensionModel,
|
||||||
@ -63,7 +68,10 @@ async function handler(req: ApiRequestProps<SearchTestProps>): Promise<SearchTes
|
|||||||
similarity,
|
similarity,
|
||||||
datasetIds: [datasetId],
|
datasetIds: [datasetId],
|
||||||
searchMode,
|
searchMode,
|
||||||
usingReRank: usingReRank && (await checkTeamReRankPermission(teamId))
|
embeddingWeight,
|
||||||
|
usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)),
|
||||||
|
rerankModel: getRerankModel(rerankModel),
|
||||||
|
rerankWeight
|
||||||
};
|
};
|
||||||
const { searchRes, tokens, queryExtensionResult, deepSearchResult, ...result } = datasetDeepSearch
|
const { searchRes, tokens, queryExtensionResult, deepSearchResult, ...result } = datasetDeepSearch
|
||||||
? await deepRagSearch({
|
? await deepRagSearch({
|
||||||
|
|||||||
@ -268,6 +268,13 @@ export function form2AppWorkflow(
|
|||||||
valueType: WorkflowIOValueTypeEnum.string,
|
valueType: WorkflowIOValueTypeEnum.string,
|
||||||
value: formData.dataset.searchMode
|
value: formData.dataset.searchMode
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
key: NodeInputKeyEnum.datasetSearchEmbeddingWeight,
|
||||||
|
renderTypeList: [FlowNodeInputTypeEnum.hidden],
|
||||||
|
label: '',
|
||||||
|
valueType: WorkflowIOValueTypeEnum.number,
|
||||||
|
value: formData.dataset.embeddingWeight
|
||||||
|
},
|
||||||
{
|
{
|
||||||
key: NodeInputKeyEnum.datasetSearchUsingReRank,
|
key: NodeInputKeyEnum.datasetSearchUsingReRank,
|
||||||
renderTypeList: [FlowNodeInputTypeEnum.hidden],
|
renderTypeList: [FlowNodeInputTypeEnum.hidden],
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user