4.8.19 test (#3584)

* faet: dataset search filter

* fix: scroll page
This commit is contained in:
Archer 2025-01-14 12:03:21 +08:00 committed by archer
parent f468ba2f30
commit 6f8c6b6ad1
No known key found for this signature in database
GPG Key ID: 4446499B846D4A9E
21 changed files with 280 additions and 98 deletions

View File

@ -0,0 +1,13 @@
---
title: 'V4.8.19(进行中)'
description: 'FastGPT V4.8.19 更新说明'
icon: 'upgrade'
draft: false
toc: true
weight: 806
---
## 完整更新内容
1. 新增 - 工作流知识库检索支持按知识库权限进行过滤

View File

@ -152,6 +152,7 @@ export enum NodeInputKeyEnum {
datasetSearchExtensionModel = 'datasetSearchExtensionModel',
datasetSearchExtensionBg = 'datasetSearchExtensionBg',
collectionFilterMatch = 'collectionFilterMatch',
authTmbId = 'authTmbId',
// concat dataset
datasetQuoteList = 'system_datasetQuoteList',

View File

@ -41,6 +41,10 @@ export type ChatDispatchProps = {
teamId: string;
tmbId: string; // App tmbId
};
runningUserInfo: {
teamId: string;
tmbId: string;
};
uid: string; // Who run this workflow
chatId?: string;

View File

@ -89,6 +89,13 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
valueType: WorkflowIOValueTypeEnum.string,
value: ''
},
{
key: NodeInputKeyEnum.authTmbId,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
label: '',
valueType: WorkflowIOValueTypeEnum.boolean,
value: false
},
{
...Input_Template_UserChatInput,
toolDescription: i18nT('workflow:content_to_search')

View File

@ -0,0 +1,39 @@
import { getTmbInfoByTmbId } from '../../support/user/team/controller';
import { getResourcePermission } from '../../support/permission/controller';
import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller';
// TODO: 需要优化成批量获取权限
export const filterDatasetsByTmbId = async ({
datasetIds,
tmbId
}: {
datasetIds: string[];
tmbId: string;
}) => {
const { teamId, permission: tmbPer } = await getTmbInfoByTmbId({ tmbId });
// First get all permissions
const permissions = await Promise.all(
datasetIds.map(async (datasetId) => {
const per = await getResourcePermission({
teamId,
tmbId,
resourceId: datasetId,
resourceType: PerResourceTypeEnum.dataset
});
if (per === undefined) return false;
const datasetPer = new DatasetPermission({
per,
isOwner: tmbPer.isOwner
});
return datasetPer.hasReadPer;
})
);
// Then filter datasetIds based on permissions
return datasetIds.filter((_, index) => permissions[index]);
};

View File

@ -17,6 +17,7 @@ import { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type';
import { checkTeamReRankPermission } from '../../../../support/permission/teamLimit';
import { MongoDataset } from '../../../dataset/schema';
import { i18nT } from '../../../../../web/i18n/utils';
import { filterDatasetsByTmbId } from '../../../dataset/utils';
type DatasetSearchProps = ModuleDispatchProps<{
[NodeInputKeyEnum.datasetSelectList]: SelectedDatasetType;
@ -29,6 +30,7 @@ type DatasetSearchProps = ModuleDispatchProps<{
[NodeInputKeyEnum.datasetSearchExtensionModel]: string;
[NodeInputKeyEnum.datasetSearchExtensionBg]: string;
[NodeInputKeyEnum.collectionFilterMatch]: string;
[NodeInputKeyEnum.authTmbId]: boolean;
}>;
export type DatasetSearchResponse = DispatchNodeResultType<{
[NodeOutputKeyEnum.datasetQuoteQA]: SearchDataResponseItemType[];
@ -39,6 +41,7 @@ export async function dispatchDatasetSearch(
): Promise<DatasetSearchResponse> {
const {
runningAppInfo: { teamId },
runningUserInfo: { tmbId },
histories,
node,
params: {
@ -52,7 +55,8 @@ export async function dispatchDatasetSearch(
datasetSearchUsingExtensionQuery,
datasetSearchExtensionModel,
datasetSearchExtensionBg,
collectionFilterMatch
collectionFilterMatch,
authTmbId = false
}
} = props as DatasetSearchProps;
@ -64,18 +68,20 @@ export async function dispatchDatasetSearch(
return Promise.reject(i18nT('common:core.chat.error.Select dataset empty'));
}
const emptyResult = {
quoteQA: [],
[DispatchNodeResponseKeyEnum.nodeResponse]: {
totalPoints: 0,
query: '',
limit,
searchMode
},
nodeDispatchUsages: [],
[DispatchNodeResponseKeyEnum.toolResponses]: []
};
if (!userChatInput) {
return {
quoteQA: [],
[DispatchNodeResponseKeyEnum.nodeResponse]: {
totalPoints: 0,
query: '',
limit,
searchMode
},
nodeDispatchUsages: [],
[DispatchNodeResponseKeyEnum.toolResponses]: []
};
return emptyResult;
}
// query extension
@ -83,13 +89,24 @@ export async function dispatchDatasetSearch(
? getLLMModel(datasetSearchExtensionModel)
: undefined;
const { concatQueries, rewriteQuery, aiExtensionResult } = await datasetSearchQueryExtension({
query: userChatInput,
extensionModel,
extensionBg: datasetSearchExtensionBg,
histories: getHistories(6, histories)
});
const [{ concatQueries, rewriteQuery, aiExtensionResult }, datasetIds] = await Promise.all([
datasetSearchQueryExtension({
query: userChatInput,
extensionModel,
extensionBg: datasetSearchExtensionBg,
histories: getHistories(6, histories)
}),
authTmbId
? filterDatasetsByTmbId({
datasetIds: datasets.map((item) => item.datasetId),
tmbId
})
: Promise.resolve(datasets.map((item) => item.datasetId))
]);
if (datasetIds.length === 0) {
return emptyResult;
}
// console.log(concatQueries, rewriteQuery, aiExtensionResult);
// get vector
@ -110,7 +127,7 @@ export async function dispatchDatasetSearch(
model: vectorModel.model,
similarity,
limit,
datasetIds: datasets.map((item) => item.datasetId),
datasetIds,
searchMode,
usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)),
collectionFilterMatch

View File

@ -79,19 +79,16 @@ export const checkWebSyncLimit = async ({
*/
export async function addSourceMember<T extends { tmbId: string }>({
list,
teamId,
session
}: {
list: T[];
teamId?: string;
session?: ClientSession;
}): Promise<Array<T & { sourceMember: SourceMemberType }>> {
if (!list.length) return [];
if (!Array.isArray(list)) return [];
const tmbList = await MongoTeamMember.find(
{
_id: { $in: list.map((item) => String(item.tmbId)) },
...(teamId && { teamId })
_id: { $in: list.map((item) => String(item.tmbId)) }
},
'tmbId name avatar status',
{
@ -103,9 +100,10 @@ export async function addSourceMember<T extends { tmbId: string }>({
.map((item) => {
const tmb = tmbList.find((tmb) => String(tmb._id) === String(item.tmbId));
if (!tmb) return;
return {
...item,
...(tmb && { sourceMember: { name: tmb.name, avatar: tmb.avatar, status: tmb.status } })
sourceMember: { name: tmb.name, avatar: tmb.avatar, status: tmb.status }
};
})
.filter(Boolean) as Array<T & { sourceMember: SourceMemberType }>;

View File

@ -13,6 +13,8 @@
"append_application_reply_to_history_as_new_context": "Append the application's reply to the history as new context",
"application_call": "Application Call",
"assigned_reply": "Assigned Reply",
"auth_tmb_id": "Auth member",
"auth_tmb_id_tip": "After it is turned on, when the application is released to the outside world, the knowledge base will be filtered based on whether the user has permission to the knowledge base.\n\nIf it is not enabled, the configured knowledge base will be searched directly without permission filtering.",
"can_not_loop": "This node can't loop.",
"choose_another_application_to_call": "Select another application to call",
"classification_result": "Classification Result",

View File

@ -13,6 +13,8 @@
"append_application_reply_to_history_as_new_context": "将该应用回复内容拼接到历史记录中,作为新的上下文返回",
"application_call": "应用调用",
"assigned_reply": "指定回复",
"auth_tmb_id": "使用者鉴权",
"auth_tmb_id_tip": "开启后,对外发布该应用时,还会根据用户是否有该知识库权限进行知识库过滤。\n若未开启则直接按配置的知识库进行检索不进行权限过滤。",
"can_not_loop": "该节点不支持循环嵌套",
"choose_another_application_to_call": "选择一个其他应用进行调用",
"classification_result": "分类结果",

View File

@ -13,6 +13,8 @@
"append_application_reply_to_history_as_new_context": "將應用程式的回覆附加到歷史紀錄中,作為新的脈絡",
"application_call": "應用程式呼叫",
"assigned_reply": "指定回覆",
"auth_tmb_id": "使用者鑑權",
"auth_tmb_id_tip": "開啟後,對外發布應用程式時,也會根據使用者是否有該知識庫權限進行知識庫過濾。\n\n若未開啟則直接按配置的知識庫進行檢索不進行權限過濾。",
"can_not_loop": "這個節點不能迴圈。",
"choose_another_application_to_call": "選擇另一個應用程式來呼叫",
"classification_result": "分類結果",

View File

@ -57,3 +57,5 @@ WORKFLOW_MAX_LOOP_TIMES=50
# CHAT_LOG_INTERVAL=10000
# # 日志来源ID前缀
# CHAT_LOG_SOURCE_ID_PREFIX=fastgpt-
# 自定义跨域,不配置时,默认都允许跨域(逗号分割)
ALLOWED_ORIGINS=

View File

@ -36,14 +36,14 @@ async function handler(
.limit(pageSize)
.lean();
return (
await addSourceMember({
list: versions
})
).map((item) => ({
...item,
isPublish: !!item.isPublish
}));
return addSourceMember({
list: versions
}).then((list) =>
list.map((item) => ({
...item,
isPublish: !!item.isPublish
}))
);
})(),
MongoAppVersion.countDocuments({ appId })
]);

View File

@ -164,6 +164,10 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
runningAppInfo: {
id: appId,
teamId: app.teamId,
tmbId: app.tmbId
},
runningUserInfo: {
teamId,
tmbId
},

View File

@ -18,6 +18,7 @@ import { getGroupsByTmbId } from '@fastgpt/service/support/permission/memberGrou
import { concatPer } from '@fastgpt/service/support/permission/controller';
import { getOrgIdSetWithParentByTmbId } from '@fastgpt/service/support/permission/org/controllers';
import { addSourceMember } from '@fastgpt/service/support/user/utils';
import { getVectorModel } from '@fastgpt/service/core/ai/model';
export type GetDatasetListBody = {
parentId: ParentIdType;
@ -166,7 +167,15 @@ async function handler(req: ApiRequestProps<GetDatasetListBody>) {
})();
return {
...dataset,
_id: dataset._id,
avatar: dataset.avatar,
name: dataset.name,
intro: dataset.intro,
type: dataset.type,
vectorModel: getVectorModel(dataset.vectorModel),
inheritPermission: dataset.inheritPermission,
tmbId: dataset.tmbId,
updateTime: dataset.updateTime,
permission: Per,
privateDataset
};
@ -174,8 +183,7 @@ async function handler(req: ApiRequestProps<GetDatasetListBody>) {
.filter((app) => app.permission.hasReadPer);
return addSourceMember({
list: formatDatasets,
teamId
list: formatDatasets
});
}

View File

@ -45,7 +45,11 @@ async function handler(
requestOrigin: req.headers.origin,
mode: 'debug',
runningAppInfo: {
id: appId,
id: app._id,
teamId: app.teamId,
tmbId: app.tmbId
},
runningUserInfo: {
teamId,
tmbId
},

View File

@ -280,6 +280,10 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
teamId: String(app.teamId),
tmbId: String(app.tmbId)
},
runningUserInfo: {
teamId,
tmbId
},
uid: String(outLinkUserId || tmbId),
chatId,

View File

@ -1,5 +1,5 @@
import { FlowNodeInputItemType } from '@fastgpt/global/core/workflow/type/io.d';
import React, { useCallback, useMemo } from 'react';
import React, { useCallback } from 'react';
import { useTranslation } from 'next-i18next';
import { Box, Flex } from '@chakra-ui/react';
@ -10,14 +10,14 @@ import { useContextSelector } from 'use-context-selector';
import { WorkflowContext } from '@/pages/app/detail/components/WorkflowComponents/context';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import VariableTip from '@/components/common/Textarea/MyTextarea/VariableTip';
type Props = {
nodeId: string;
input: FlowNodeInputItemType;
RightComponent?: React.JSX.Element;
};
const InputLabel = ({ nodeId, input }: Props) => {
const InputLabel = ({ nodeId, input, RightComponent }: Props) => {
const { t } = useTranslation();
const onChangeNode = useContextSelector(WorkflowContext, (v) => v.onChangeNode);
@ -68,11 +68,11 @@ const InputLabel = ({ nodeId, input }: Props) => {
</Box>
)}
{/* Variable picker tip */}
{input.renderTypeList[input.selectedTypeIndex ?? 0] === FlowNodeInputTypeEnum.textarea && (
{/* Right Component */}
{RightComponent && (
<>
<Box flex={1} />
<VariableTip transform={'translateY(2px)'} />
<Box flex={'1'} />
{RightComponent}
</>
)}
</Flex>

View File

@ -8,71 +8,72 @@ import InputLabel from './Label';
import type { RenderInputProps } from './type';
import { useSystemStore } from '@/web/common/system/useSystemStore';
const RenderList: {
types: FlowNodeInputTypeEnum[];
Component: React.ComponentType<RenderInputProps>;
}[] = [
{
types: [FlowNodeInputTypeEnum.reference],
const RenderList: Record<
FlowNodeInputTypeEnum,
| {
Component: React.ComponentType<RenderInputProps>;
LableRightComponent?: React.ComponentType<RenderInputProps>;
}
| undefined
> = {
[FlowNodeInputTypeEnum.reference]: {
Component: dynamic(() => import('./templates/Reference'))
},
{
types: [FlowNodeInputTypeEnum.fileSelect],
[FlowNodeInputTypeEnum.fileSelect]: {
Component: dynamic(() => import('./templates/Reference'))
},
{
types: [FlowNodeInputTypeEnum.select],
[FlowNodeInputTypeEnum.select]: {
Component: dynamic(() => import('./templates/Select'))
},
{
types: [FlowNodeInputTypeEnum.numberInput],
[FlowNodeInputTypeEnum.numberInput]: {
Component: dynamic(() => import('./templates/NumberInput'))
},
{
types: [FlowNodeInputTypeEnum.switch],
[FlowNodeInputTypeEnum.switch]: {
Component: dynamic(() => import('./templates/Switch'))
},
{
types: [FlowNodeInputTypeEnum.selectApp],
[FlowNodeInputTypeEnum.selectApp]: {
Component: dynamic(() => import('./templates/SelectApp'))
},
{
types: [FlowNodeInputTypeEnum.selectLLMModel],
[FlowNodeInputTypeEnum.selectLLMModel]: {
Component: dynamic(() => import('./templates/SelectLLMModel'))
},
{
types: [FlowNodeInputTypeEnum.settingLLMModel],
[FlowNodeInputTypeEnum.settingLLMModel]: {
Component: dynamic(() => import('./templates/SettingLLMModel'))
},
{
types: [FlowNodeInputTypeEnum.selectDataset],
Component: dynamic(() => import('./templates/SelectDataset'))
[FlowNodeInputTypeEnum.selectDataset]: {
Component: dynamic(() =>
import('./templates/SelectDataset').then((mod) => mod.SelectDatasetRender)
),
LableRightComponent: dynamic(() =>
import('./templates/SelectDataset').then((mod) => mod.SwitchAuthTmb)
)
},
{
types: [FlowNodeInputTypeEnum.selectDatasetParamsModal],
[FlowNodeInputTypeEnum.selectDatasetParamsModal]: {
Component: dynamic(() => import('./templates/SelectDatasetParams'))
},
{
types: [FlowNodeInputTypeEnum.addInputParam],
[FlowNodeInputTypeEnum.addInputParam]: {
Component: dynamic(() => import('./templates/DynamicInputs/index'))
},
{
types: [FlowNodeInputTypeEnum.JSONEditor],
[FlowNodeInputTypeEnum.JSONEditor]: {
Component: dynamic(() => import('./templates/JsonEditor'))
},
{
types: [FlowNodeInputTypeEnum.settingDatasetQuotePrompt],
[FlowNodeInputTypeEnum.settingDatasetQuotePrompt]: {
Component: dynamic(() => import('./templates/SettingQuotePrompt'))
},
{
types: [FlowNodeInputTypeEnum.input],
[FlowNodeInputTypeEnum.input]: {
Component: dynamic(() => import('./templates/TextInput'))
},
{
types: [FlowNodeInputTypeEnum.textarea],
Component: dynamic(() => import('./templates/Textarea'))
}
];
[FlowNodeInputTypeEnum.textarea]: {
Component: dynamic(() => import('./templates/Textarea')),
LableRightComponent: dynamic(() =>
import('./templates/Textarea').then((mod) => mod.TextareaRightComponent)
)
},
[FlowNodeInputTypeEnum.customVariable]: undefined,
[FlowNodeInputTypeEnum.hidden]: undefined,
[FlowNodeInputTypeEnum.custom]: undefined
};
const hideLabelTypeList = [FlowNodeInputTypeEnum.addInputParam];
@ -101,7 +102,7 @@ const RenderInput = ({ flowInputList, nodeId, CustomComponent, mb = 5 }: Props)
return true;
});
}, [feConfigs?.isPlus, flowInputList]);
}, [filterProInputs]);
return (
<>
@ -110,23 +111,41 @@ const RenderInput = ({ flowInputList, nodeId, CustomComponent, mb = 5 }: Props)
const RenderComponent = (() => {
if (renderType === FlowNodeInputTypeEnum.custom && CustomComponent?.[input.key]) {
return <>{CustomComponent?.[input.key]({ ...input })}</>;
return {
Component: <>{CustomComponent?.[input.key]({ ...input })}</>
};
}
const Component = RenderList.find((item) => item.types.includes(renderType))?.Component;
const RenderItem = RenderList[renderType];
if (!Component) return null;
return <Component inputs={filterProInputs} item={input} nodeId={nodeId} />;
if (!RenderItem) return null;
return {
Component: (
<RenderItem.Component inputs={filterProInputs} item={input} nodeId={nodeId} />
),
LableRightComponent: RenderItem.LableRightComponent ? (
<RenderItem.LableRightComponent
inputs={filterProInputs}
item={input}
nodeId={nodeId}
/>
) : undefined
};
})();
return (
<Box key={input.key} _notLast={{ mb }} position={'relative'}>
{!!input.label && !hideLabelTypeList.includes(renderType) && (
<InputLabel nodeId={nodeId} input={input} />
<InputLabel
nodeId={nodeId}
input={input}
RightComponent={RenderComponent?.LableRightComponent}
/>
)}
{!!RenderComponent && (
<Box mt={2} className={'nodrag'}>
{RenderComponent}
{RenderComponent.Component}
</Box>
)}
</Box>

View File

@ -1,6 +1,6 @@
import React, { useEffect, useMemo, useState } from 'react';
import type { RenderInputProps } from '../type';
import { Box, Button, Flex, Grid, useDisclosure, useTheme } from '@chakra-ui/react';
import { Box, Button, Flex, Grid, Switch, useDisclosure, useTheme } from '@chakra-ui/react';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { SelectedDatasetType } from '@fastgpt/global/core/workflow/api';
import Avatar from '@fastgpt/web/components/common/Avatar';
@ -12,12 +12,17 @@ import dynamic from 'next/dynamic';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useContextSelector } from 'use-context-selector';
import { WorkflowContext } from '@/pages/app/detail/components/WorkflowComponents/context';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
const DatasetSelectModal = dynamic(() => import('@/components/core/app/DatasetSelectModal'));
const SelectDatasetRender = ({ inputs = [], item, nodeId }: RenderInputProps) => {
export const SelectDatasetRender = React.memo(function SelectDatasetRender({
inputs = [],
item,
nodeId
}: RenderInputProps) {
const { t } = useTranslation();
const theme = useTheme();
const onChangeNode = useContextSelector(WorkflowContext, (v) => v.onChangeNode);
const [data, setData] = useState({
@ -80,8 +85,9 @@ const SelectDatasetRender = ({ inputs = [], item, nodeId }: RenderInputProps) =>
key={item._id}
alignItems={'center'}
h={10}
border={theme.borders.base}
borderColor={'myGray.200'}
boxShadow={'sm'}
bg={'white'}
border={'base'}
px={2}
borderRadius={'md'}
>
@ -128,11 +134,49 @@ const SelectDatasetRender = ({ inputs = [], item, nodeId }: RenderInputProps) =>
onOpenDatasetSelect,
selectedDatasets,
selectedDatasetsValue,
t,
theme.borders.base
t
]);
return Render;
};
});
export default React.memo(SelectDatasetRender);
export const SwitchAuthTmb = React.memo(function SwitchAuthTmb({
inputs = [],
item,
nodeId
}: RenderInputProps) {
const { t } = useTranslation();
const onChangeNode = useContextSelector(WorkflowContext, (v) => v.onChangeNode);
const authTmbIdInput = useMemo(
() => inputs.find((v) => v.key === NodeInputKeyEnum.authTmbId),
[inputs]
);
console.log(authTmbIdInput, '--');
return authTmbIdInput ? (
<Flex alignItems={'center'}>
<Box fontSize={'sm'}>{t('workflow:auth_tmb_id')}</Box>
<QuestionTip label={t('workflow:auth_tmb_id_tip')} />
<Switch
ml={1}
size={'sm'}
isChecked={!!authTmbIdInput.value}
onChange={(e) => {
onChangeNode({
nodeId,
key: NodeInputKeyEnum.authTmbId,
type: 'updateInput',
value: {
...authTmbIdInput,
value: e.target.checked
}
});
}}
/>
</Flex>
) : null;
});
export default SelectDatasetRender;

View File

@ -9,6 +9,7 @@ import { AppContext } from '@/pages/app/detail/components/context';
import { getEditorVariables } from '../../../../../utils';
import { WorkflowNodeEdgeContext } from '../../../../../context/workflowInitContext';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import VariableTip from '@/components/common/Textarea/MyTextarea/VariableTip';
const TextareaRender = ({ item, nodeId }: RenderInputProps) => {
const { t } = useTranslation();
@ -84,3 +85,10 @@ const TextareaRender = ({ item, nodeId }: RenderInputProps) => {
};
export default React.memo(TextareaRender);
export const TextareaRightComponent = React.memo(function TextareaRightComponent({
item,
nodeId
}: RenderInputProps) {
return <VariableTip transform={'translateY(2px)'} />;
});

View File

@ -65,6 +65,10 @@ export const getScheduleTriggerApp = async () => {
teamId: String(app.teamId),
tmbId: String(app.tmbId)
},
runningUserInfo: {
teamId: String(app.teamId),
tmbId: String(app.tmbId)
},
uid: String(app.tmbId),
runtimeNodes: storeNodes2RuntimeNodes(nodes, getWorkflowEntryNodeIds(nodes)),
runtimeEdges: initWorkflowEdgeStatus(edges),