fix :Get application bound knowledge base information logical rewrite (#4057)

* fix :Get application bound knowledge base information logical rewrite

* fix :Get application bound knowledge base information logical rewrite

* fix :Get application bound knowledge base information logical rewrite

* fix :Get application bound knowledge base information logical rewrite
This commit is contained in:
gggaaallleee 2025-03-11 12:31:30 +08:00 committed by archer
parent 39a2a4a592
commit bf72defbff
No known key found for this signature in database
GPG Key ID: 4446499B846D4A9E
15 changed files with 181 additions and 121 deletions

View File

@ -1,7 +1,12 @@
import { EmbeddingModelItemType } from '../ai/model.d';
import { NodeInputKeyEnum } from './constants';
export type SelectedDatasetType = { datasetId: string }[];
export type SelectedDatasetType = {
datasetId: string;
avatar: string;
name: string;
vectorModel: EmbeddingModelItemType;
}[];
export type HttpBodyType<T = Record<string, any>> = {
// [NodeInputKeyEnum.addInputParam]: Record<string, any>;

View File

@ -0,0 +1,123 @@
import { MongoDataset } from '../dataset/schema';
import { getEmbeddingModel } from '../ai/model';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import type { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node';
export type ListByAppIdAndDatasetIdsBody = {
teamId: string;
datasetIdList: string[];
};
interface Dataset {
datasetId: string;
[key: string]: any;
}
export async function listAppDatasetDataByTeamIdAndDatasetIds({
teamId,
datasetIdList
}: ListByAppIdAndDatasetIdsBody) {
const myDatasets = await MongoDataset.find({
teamId,
_id: { $in: datasetIdList }
}).lean();
return myDatasets.map((item) => ({
datasetId: item._id,
avatar: item.avatar,
name: item.name,
vectorModel: getEmbeddingModel(item.vectorModel)
}));
}
export async function rewriteAppWorkflowToDetail(nodes: StoreNodeItemType[], teamId: string) {
const datasetIdSet = new Set<string>();
nodes.forEach((node) => {
if (node.flowNodeType !== FlowNodeTypeEnum.datasetSearchNode) return;
const input = node.inputs.find((item) => item.key === NodeInputKeyEnum.datasetSelectList);
if (!input) return;
const rawValue = input.value as undefined | { datasetId: string }[] | { datasetId: string };
const datasetIds = Array.isArray(rawValue)
? rawValue
.map((v) => v?.datasetId)
.filter((id): id is string => !!id && typeof id === 'string')
: rawValue?.datasetId
? [String(rawValue.datasetId)]
: [];
if (datasetIds.length === 0) return;
datasetIds.forEach((id) => datasetIdSet.add(id));
});
if (datasetIdSet.size === 0) return;
const uniqueDatasetIds = Array.from(datasetIdSet);
const datasetList = await listAppDatasetDataByTeamIdAndDatasetIds({
teamId,
datasetIdList: uniqueDatasetIds
});
const datasetMap = new Map(
datasetList.map((ds) => [
String(ds.datasetId),
{
...ds,
vectorModel: getEmbeddingModel(ds.vectorModel.model)
}
])
);
nodes.forEach((node) => {
if (node.flowNodeType !== FlowNodeTypeEnum.datasetSearchNode) return;
const input = node.inputs.find((item) => item.key === NodeInputKeyEnum.datasetSelectList);
if (!input) return;
const rawValue = input.value as undefined | { datasetId: string }[] | { datasetId: string };
const datasetIds = Array.isArray(rawValue)
? rawValue
.map((v) => v?.datasetId)
.filter((id): id is string => !!id && typeof id === 'string')
: rawValue?.datasetId
? [String(rawValue.datasetId)]
: [];
if (datasetIds.length === 0) return;
input.value = datasetIds
.map((id) => {
const data = datasetMap.get(String(id));
return data
? {
datasetId: data.datasetId,
avatar: data.avatar,
name: data.name,
vectorModel: data.vectorModel
}
: undefined;
})
.filter((item): item is NonNullable<typeof item> => !!item);
});
}
export async function rewriteAppWorkflowToSimple(formatNodes: StoreNodeItemType[]) {
formatNodes.forEach((node) => {
if (node.flowNodeType !== FlowNodeTypeEnum.datasetSearchNode) return;
const datasetsInput = node.inputs.find(
(input) => input.key === NodeInputKeyEnum.datasetSelectList
);
if (datasetsInput?.value) {
datasetsInput.value = datasetsInput.value.map((dataset: Dataset) => ({
datasetId: dataset.datasetId
}));
}
});
}

View File

@ -17,7 +17,6 @@ import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { useTranslation } from 'next-i18next';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import DatasetSelectContainer, { useDatasetSelect } from '@/components/core/dataset/SelectModal';
import { useLoading } from '@fastgpt/web/hooks/useLoading';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
@ -35,29 +34,24 @@ export const DatasetSelectModal = ({
}) => {
const { t } = useTranslation();
const theme = useTheme();
const { allDatasets } = useDatasetStore();
const [selectedDatasets, setSelectedDatasets] = useState<SelectedDatasetType>(
defaultSelectedDatasets.filter((dataset) => {
return allDatasets.find((item) => item._id === dataset.datasetId);
})
);
const [selectedDatasets, setSelectedDatasets] =
useState<SelectedDatasetType>(defaultSelectedDatasets);
const { toast } = useToast();
const { paths, setParentId, datasets, isFetching } = useDatasetSelect();
const { Loading } = useLoading();
const filterDatasets = useMemo(() => {
return {
selected: allDatasets.filter((item) =>
const filtered = {
selected: datasets.filter((item) =>
selectedDatasets.find((dataset) => dataset.datasetId === item._id)
),
unSelected: datasets.filter(
(item) => !selectedDatasets.find((dataset) => dataset.datasetId === item._id)
)
};
}, [datasets, allDatasets, selectedDatasets]);
const activeVectorModel = allDatasets.find(
(dataset) => dataset._id === selectedDatasets[0]?.datasetId
)?.vectorModel?.model;
return filtered;
}, [datasets, selectedDatasets]);
const activeVectorModel = defaultSelectedDatasets[0]?.vectorModel?.model;
return (
<DatasetSelectContainer
@ -150,7 +144,15 @@ export const DatasetSelectModal = ({
title: t('common:dataset.Select Dataset Tips')
});
}
setSelectedDatasets((state) => [...state, { datasetId: item._id }]);
setSelectedDatasets((state) => [
...state,
{
datasetId: item._id,
avatar: item.avatar,
name: item.name,
vectorModel: item.vectorModel
}
]);
}
}}
>
@ -200,7 +202,7 @@ export const DatasetSelectModal = ({
onClick={() => {
// filter out the dataset that is not in the kList
const filterDatasets = selectedDatasets.filter((dataset) => {
return allDatasets.find((item) => item._id === dataset.datasetId);
return datasets.find((item) => item._id === dataset.datasetId);
});
onClose();

View File

@ -11,7 +11,6 @@ import { useI18n } from '@/web/context/I18n';
import { useContextSelector } from 'use-context-selector';
import { AppContext } from '../context';
import { useChatTest } from '../useChatTest';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import ChatItemContextProvider from '@/web/core/chat/context/chatItemContext';
import ChatRecordContextProvider from '@/web/core/chat/context/chatRecordContext';
import { useChatStore } from '@/web/core/chat/context/useChatStore';
@ -23,8 +22,6 @@ const ChatTest = ({ appForm }: Props) => {
const { appT } = useI18n();
const { appDetail } = useContextSelector(AppContext, (v) => v);
// form2AppWorkflow dependent allDatasets
const { allDatasets } = useDatasetStore();
const [workflowData, setWorkflowData] = useSafeState({
nodes: appDetail.modules || [],
@ -33,10 +30,8 @@ const ChatTest = ({ appForm }: Props) => {
useEffect(() => {
const { nodes, edges } = form2AppWorkflow(appForm, t);
// console.log(form2AppWorkflow(appForm, t));
setWorkflowData({ nodes, edges });
}, [appForm, setWorkflowData, allDatasets, t]);
}, [appForm, setWorkflowData, t]);
const { ChatContainer, restartChat, loading } = useChatTest({
...workflowData,
chatConfig: appForm.chatConfig,

View File

@ -10,9 +10,9 @@ import {
HStack
} from '@chakra-ui/react';
import type { AppSimpleEditFormType } from '@fastgpt/global/core/app/type.d';
import type { DatasetSimpleItemType } from '@fastgpt/global/core/dataset/type.d';
import { useRouter } from 'next/router';
import { useTranslation } from 'next-i18next';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import dynamic from 'next/dynamic';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
@ -68,18 +68,9 @@ const EditForm = ({
const { t } = useTranslation();
const { appDetail } = useContextSelector(AppContext, (v) => v);
const { allDatasets } = useDatasetStore();
const selectDatasets = useMemo(() => appForm?.dataset?.datasets, [appForm]);
const [, startTst] = useTransition();
const selectDatasets = useMemo(
() =>
allDatasets.filter((item) =>
appForm.dataset?.datasets.find((dataset) => dataset.datasetId === item._id)
),
[allDatasets, appForm?.dataset?.datasets]
);
const {
isOpen: isOpenDatasetSelect,
onOpen: onOpenKbSelect,
@ -252,7 +243,7 @@ const EditForm = ({
)}
<Grid gridTemplateColumns={'repeat(2, minmax(0, 1fr))'} gridGap={[2, 4]}>
{selectDatasets.map((item) => (
<MyTooltip key={item._id} label={t('common:core.dataset.Read Dataset')}>
<MyTooltip key={item.datasetId} label={t('common:core.dataset.Read Dataset')}>
<Flex
overflow={'hidden'}
alignItems={'center'}
@ -266,7 +257,7 @@ const EditForm = ({
router.push({
pathname: '/dataset/detail',
query: {
datasetId: item._id
datasetId: item.datasetId
}
})
}
@ -413,8 +404,10 @@ const EditForm = ({
<DatasetSelectModal
isOpen={isOpenDatasetSelect}
defaultSelectedDatasets={selectDatasets.map((item) => ({
datasetId: item._id,
vectorModel: item.vectorModel
datasetId: item.datasetId,
vectorModel: item.vectorModel,
name: item.name,
avatar: item.avatar
}))}
onClose={onCloseKbSelect}
onChange={(e) => {

View File

@ -17,7 +17,6 @@ import { publishStatusStyle } from '../constants';
import { useSystem } from '@fastgpt/web/hooks/useSystem';
import { formatTime2YMDHMS } from '@fastgpt/global/common/string/time';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import SaveButton from '../Workflow/components/SaveButton';
import { useBoolean, useDebounceEffect, useLockFn } from 'ahooks';
import { appWorkflow2Form } from '@fastgpt/global/core/app/utils';
@ -61,7 +60,6 @@ const Header = ({
const currentTab = useContextSelector(AppContext, (v) => v.currentTab);
const { lastAppListRouteType } = useSystemStore();
const { allDatasets } = useDatasetStore();
const { data: paths = [] } = useRequest2(() => getAppFolderPath(appId), {
manual: false,
@ -159,7 +157,7 @@ const Header = ({
const val = compareSimpleAppSnapshot(savedSnapshot?.appForm, appForm);
setIsSaved(val);
},
[past, allDatasets],
[past],
{ wait: 500 }
);

View File

@ -10,7 +10,6 @@ import { Box, Flex } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import { SimpleAppSnapshotType, useSimpleAppSnapshots } from './useSnapshots';
import { useDebounceEffect, useMount } from 'ahooks';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { v1Workflow2V2 } from '@/web/core/workflow/adapt';
import { getAppConfigByDiff } from '@/web/core/app/diff';
@ -19,7 +18,6 @@ const PublishChannel = dynamic(() => import('../Publish'));
const SimpleEdit = () => {
const { t } = useTranslation();
const { loadAllDatasets } = useDatasetStore();
const { currentTab, appDetail } = useContextSelector(AppContext, (v) => v);
const { forbiddenSaveSnapshot, past, setPast, saveSnapshot } = useSimpleAppSnapshots(
@ -31,7 +29,6 @@ const SimpleEdit = () => {
// Init app form
useMount(() => {
// show selected dataset
loadAllDatasets();
if (appDetail.version !== 'v2') {
return setAppForm(

View File

@ -1,13 +1,10 @@
import React, { useEffect, useMemo, useState } from 'react';
import type { RenderInputProps } from '../type';
import { Box, Button, Flex, Grid, Switch, useDisclosure, useTheme } from '@chakra-ui/react';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { SelectedDatasetType } from '@fastgpt/global/core/workflow/api';
import Avatar from '@fastgpt/web/components/common/Avatar';
import { useQuery } from '@tanstack/react-query';
import { useTranslation } from 'next-i18next';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
import dynamic from 'next/dynamic';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useContextSelector } from 'use-context-selector';
@ -32,26 +29,17 @@ export const SelectDatasetRender = React.memo(function SelectDatasetRender({
usingReRank: false
});
const { allDatasets, loadAllDatasets } = useDatasetStore();
const {
isOpen: isOpenDatasetSelect,
onOpen: onOpenDatasetSelect,
onClose: onCloseDatasetSelect
} = useDisclosure();
const selectedDatasetsValue = useMemo(() => {
const selectedDatasets = useMemo(() => {
if (Array.isArray(item.value)) return item.value as SelectedDatasetType;
return [] as SelectedDatasetType;
}, [item.value]);
const selectedDatasets = useMemo(() => {
return allDatasets.filter((dataset) =>
selectedDatasetsValue?.find((item) => item.datasetId === dataset._id)
);
}, [allDatasets, selectedDatasetsValue]);
useQuery(['loadAllDatasets'], loadAllDatasets);
useEffect(() => {
inputs.forEach((input) => {
// @ts-ignore
@ -82,7 +70,7 @@ export const SelectDatasetRender = React.memo(function SelectDatasetRender({
</Button>
{selectedDatasets.map((item) => (
<Flex
key={item._id}
key={item.datasetId}
alignItems={'center'}
h={10}
boxShadow={'sm'}
@ -108,7 +96,12 @@ export const SelectDatasetRender = React.memo(function SelectDatasetRender({
{isOpenDatasetSelect && (
<DatasetSelectModal
isOpen={isOpenDatasetSelect}
defaultSelectedDatasets={selectedDatasetsValue}
defaultSelectedDatasets={selectedDatasets.map((item) => ({
datasetId: item.datasetId,
vectorModel: item.vectorModel,
name: item.name,
avatar: item.avatar
}))}
onChange={(e) => {
onChangeNode({
nodeId,
@ -133,7 +126,6 @@ export const SelectDatasetRender = React.memo(function SelectDatasetRender({
onCloseDatasetSelect,
onOpenDatasetSelect,
selectedDatasets,
selectedDatasetsValue,
t
]);

View File

@ -4,7 +4,7 @@ import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { checkNode } from '@/service/core/app/utils';
import { rewriteAppWorkflowToDetail } from '@fastgpt/service/core/app/utils';
/* 获取应用详情 */
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
const { appId } = req.query as { appId: string };
@ -14,6 +14,9 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
}
// 凭证校验
const { app } = await authApp({ req, authToken: true, appId, per: ReadPermissionVal });
const teamId = app.teamId;
await rewriteAppWorkflowToDetail(app.modules, teamId);
if (!app.permission.hasWritePer) {
return {

View File

@ -6,6 +6,7 @@ import { getAppLatestVersion } from '@fastgpt/service/core/app/version/controlle
import { AppChatConfigType } from '@fastgpt/global/core/app/type';
import { StoreEdgeItemType } from '@fastgpt/global/core/workflow/type/edge';
import { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node';
import { rewriteAppWorkflowToDetail } from '@fastgpt/service/core/app/utils';
export type getLatestVersionQuery = {
appId: string;
@ -30,6 +31,10 @@ async function handler(
per: WritePermissionVal
});
const teamId = app.teamId;
await rewriteAppWorkflowToDetail(app.modules, teamId);
return getAppLatestVersion(req.query.appId, app);
}

View File

@ -10,6 +10,7 @@ import { PostPublishAppProps } from '@/global/core/app/api';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
import { rewriteAppWorkflowToSimple } from '@fastgpt/service/core/app/utils';
async function handler(req: ApiRequestProps<PostPublishAppProps>, res: NextApiResponse<any>) {
const { appId } = req.query as { appId: string };
@ -22,6 +23,8 @@ async function handler(req: ApiRequestProps<PostPublishAppProps>, res: NextApiRe
isPlugin: app.type === AppTypeEnum.plugin
});
await rewriteAppWorkflowToSimple(formatNodes);
if (autoSave) {
return MongoApp.findByIdAndUpdate(appId, {
modules: formatNodes,

View File

@ -1,38 +0,0 @@
import type { NextApiRequest } from 'next';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { getEmbeddingModel } from '@fastgpt/service/core/ai/model';
import type { DatasetSimpleItemType } from '@fastgpt/global/core/dataset/type.d';
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { authUserPer } from '@fastgpt/service/support/permission/user/auth';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
/* get all dataset by teamId or tmbId */
async function handler(req: NextApiRequest): Promise<DatasetSimpleItemType[]> {
const { teamId } = await authUserPer({
req,
authToken: true,
authApiKey: true,
per: ReadPermissionVal
});
const myDatasets = await MongoDataset.find({
teamId,
type: {
$ne: DatasetTypeEnum.folder
}
})
.sort({
updateTime: -1
})
.lean();
return myDatasets.map((item) => ({
_id: item._id,
avatar: item.avatar,
name: item.name,
vectorModel: getEmbeddingModel(item.vectorModel)
}));
}
export default NextAPI(handler);

View File

@ -21,7 +21,6 @@ import { getNanoid } from '@fastgpt/global/common/string/tools';
import { StoreEdgeItemType } from '@fastgpt/global/core/workflow/type/edge';
import { EditorVariablePickerType } from '@fastgpt/web/components/common/Textarea/PromptEditor/type';
import { ToolModule } from '@fastgpt/global/core/workflow/template/system/tools';
import { useDatasetStore } from '../dataset/store/dataset';
import {
WorkflowStart,
userFilesInput
@ -54,12 +53,7 @@ export function form2AppWorkflow(
} {
const datasetNodeId = 'iKBoX2vIzETU';
const aiChatNodeId = '7BdojPlukIQw';
const allDatasets = useDatasetStore.getState().allDatasets;
const selectedDatasets = data.dataset.datasets.filter((item) =>
allDatasets.some((ds) => ds._id === item.datasetId)
);
const selectedDatasets = data.dataset.datasets;
function systemConfigTemplate(): StoreNodeItemType {
return {
nodeId: SystemConfigNode.id,

View File

@ -70,10 +70,11 @@ import { GetQuoteDataResponse } from '@/pages/api/core/dataset/data/getQuoteData
export const getDatasets = (data: GetDatasetListBody) =>
POST<DatasetListItemType[]>(`/core/dataset/list`, data);
export const getDatasetsByAppIdAndDatasetIds = (data: { appId: string; datasetIdList: string[] }) =>
POST<DatasetSimpleItemType[]>(`/core/dataset/listByAppIdAndDatasetIds`, data);
/**
* get type=dataset list
*/
export const getAllDataset = () => GET<DatasetSimpleItemType[]>(`/core/dataset/allDataset`);
export const getDatasetPaths = (parentId: ParentIdType) =>
GET<ParentTreePathItemType[]>('/core/dataset/paths', { parentId });

View File

@ -1,15 +1,10 @@
import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import type {
DatasetListItemType,
DatasetSimpleItemType
} from '@fastgpt/global/core/dataset/type.d';
import { getAllDataset, getDatasets } from '@/web/core/dataset/api';
import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import { getDatasets } from '@/web/core/dataset/api';
type State = {
allDatasets: DatasetSimpleItemType[];
loadAllDatasets: () => Promise<DatasetSimpleItemType[]>;
myDatasets: DatasetListItemType[];
loadMyDatasets: (parentId?: string) => Promise<DatasetListItemType[]>;
};
@ -18,14 +13,6 @@ export const useDatasetStore = create<State>()(
devtools(
persist(
immer((set, get) => ({
allDatasets: [],
async loadAllDatasets() {
const res = await getAllDataset();
set((state) => {
state.allDatasets = res;
});
return res;
},
myDatasets: [],
async loadMyDatasets(parentId = '') {
const res = await getDatasets({ parentId });