feat: api dataset support pdf parse;fix: chunk reader auth (#4117)

* feat: api dataset support pdf parse

* fix: chunk reader auth
This commit is contained in:
Archer 2025-03-12 12:41:19 +08:00 committed by archer
parent 30f83f848d
commit 1a3613cd2c
No known key found for this signature in database
GPG Key ID: 4446499B846D4A9E
27 changed files with 378 additions and 355 deletions

View File

@ -11,6 +11,7 @@ weight: 799
1. 商业版支持单团队模式,更好的管理内部成员。
2. 知识库分块阅读器。
3. API 知识库支持 PDF 增强解析。
## ⚙️ 优化

View File

@ -111,11 +111,13 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
const getFileContent = async ({
teamId,
tmbId,
apiFileId
apiFileId,
customPdfParse
}: {
teamId: string;
tmbId: string;
apiFileId: string;
customPdfParse?: boolean;
}) => {
const data = await request<APIFileContentResponse>(
`/v1/file/content`,
@ -133,7 +135,8 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
teamId,
tmbId,
url: previewUrl,
relatedId: apiFileId
relatedId: apiFileId,
customPdfParse
});
return rawText;
}

View File

@ -127,7 +127,8 @@ export const readApiServerFileContent = async ({
yuqueServer,
apiFileId,
teamId,
tmbId
tmbId,
customPdfParse
}: {
apiServer?: APIFileServer;
feishuServer?: FeishuServer;
@ -135,9 +136,15 @@ export const readApiServerFileContent = async ({
apiFileId: string;
teamId: string;
tmbId: string;
customPdfParse?: boolean;
}) => {
if (apiServer) {
return useApiDatasetRequest({ apiServer }).getFileContent({ teamId, tmbId, apiFileId });
return useApiDatasetRequest({ apiServer }).getFileContent({
teamId,
tmbId,
apiFileId,
customPdfParse
});
}
if (feishuServer || yuqueServer) {

View File

@ -11,19 +11,17 @@ import { useChatStore } from '@/web/core/chat/context/useChatStore';
import { getQuoteDataList } from '@/web/core/chat/api';
const QuoteList = React.memo(function QuoteList({
chatItemId,
rawSearch = [],
chatTime
chatItemDataId = '',
rawSearch = []
}: {
chatItemId?: string;
chatItemDataId?: string;
rawSearch: SearchDataResponseItemType[];
chatTime: Date;
}) {
const theme = useTheme();
const { chatId, appId, outLinkAuthData } = useChatStore();
const RawSourceBoxProps = useContextSelector(ChatBoxContext, (v) => ({
chatItemId,
chatItemDataId,
appId: v.appId,
chatId: v.chatId,
...(v.outLinkAuthData || {})
@ -34,13 +32,12 @@ const QuoteList = React.memo(function QuoteList({
(v) => v.showRouteToDatasetDetail
);
const { data } = useRequest2(
const { data: quoteList } = useRequest2(
async () =>
await getQuoteDataList({
datasetDataIdList: rawSearch.map((item) => item.id),
chatTime,
collectionIdList: [...new Set(rawSearch.map((item) => item.collectionId))],
chatItemId: chatItemId || '',
chatItemDataId,
appId,
chatId,
...outLinkAuthData
@ -53,7 +50,7 @@ const QuoteList = React.memo(function QuoteList({
const formatedDataList = useMemo(() => {
return rawSearch
.map((item) => {
const currentFilterItem = data?.quoteList.find((res) => res._id === item.id);
const currentFilterItem = quoteList?.find((res) => res._id === item.id);
return {
...item,
@ -66,7 +63,7 @@ const QuoteList = React.memo(function QuoteList({
const bScore = formatScore(b.score);
return (bScore.primaryScore?.value || 0) - (aScore.primaryScore?.value || 0);
});
}, [data?.quoteList, rawSearch]);
}, [quoteList, rawSearch]);
return (
<>

View File

@ -42,6 +42,8 @@ const ResponseTags = ({
const [quoteFolded, setQuoteFolded] = useState<boolean>(true);
const chatType = useContextSelector(ChatBoxContext, (v) => v.chatType);
const appId = useContextSelector(ChatBoxContext, (v) => v.appId);
const chatId = useContextSelector(ChatBoxContext, (v) => v.chatId);
const setQuoteData = useContextSelector(ChatItemContext, (v) => v.setQuoteData);
@ -156,17 +158,15 @@ const ResponseTags = ({
e.stopPropagation();
setQuoteData({
chatTime,
rawSearch: quoteList,
metadata: {
appId,
chatId,
chatItemDataId: dataId,
collectionId: item.collectionId,
collectionIdList: [
...new Set(quoteList.map((item) => item.collectionId))
],
sourceId: item.sourceId || '',
sourceName: item.sourceName,
datasetId: item.datasetId,
chatItemId: historyItem.dataId
datasetId: item.datasetId
}
});
}}
@ -225,15 +225,12 @@ const ResponseTags = ({
e.stopPropagation();
setQuoteData({
chatTime,
rawSearch: quoteList,
metadata: {
collectionId: '',
collectionIdList: [...new Set(quoteList.map((item) => item.collectionId))],
chatItemId: historyItem.dataId,
sourceId: '',
sourceName: '',
datasetId: ''
appId,
chatId,
chatItemDataId: dataId,
collectionIdList: [...new Set(quoteList.map((item) => item.collectionId))]
}
});
}}

View File

@ -265,13 +265,7 @@ export const WholeResponseContent = ({
{activeModule.quoteList && activeModule.quoteList.length > 0 && (
<Row
label={t('common:core.chat.response.module quoteList')}
rawDom={
<QuoteList
chatItemId={dataId}
chatTime={chatTime || new Date()}
rawSearch={activeModule.quoteList}
/>
}
rawDom={<QuoteList chatItemDataId={dataId} rawSearch={activeModule.quoteList} />}
/>
)}
</>

View File

@ -243,7 +243,7 @@ const QuoteItem = ({
color={'primary.500'}
href={`/dataset/detail?datasetId=${quoteItem.datasetId}&currentTab=dataCard&collectionId=${quoteItem.collectionId}`}
>
{t('common:core.dataset.Go Dataset')}
{t('chat:to_dataset')}
<MyIcon name={'common/rightArrowLight'} w={'10px'} />
</Link>
)}

View File

@ -23,7 +23,7 @@ const RawSourceBox = ({
collectionId,
appId,
chatId,
chatItemId,
chatItemDataId,
shareId,
outLinkUid,
teamId,
@ -40,7 +40,7 @@ const RawSourceBox = ({
collectionId,
appId,
chatId,
chatItemId,
chatItemDataId,
shareId,
outLinkUid,
teamId,

View File

@ -184,7 +184,6 @@ const DetailLogsModal = ({ appId, chatId, onClose }: Props) => {
borderRadius={'md'}
>
<ChatQuoteList
chatTime={quoteData.chatTime}
rawSearch={quoteData.rawSearch}
metadata={quoteData.metadata}
onClose={() => setQuoteData(undefined)}

View File

@ -88,7 +88,6 @@ const ChatTest = ({ appForm, setRenderEdit }: Props) => {
{quoteData && (
<Box flex={'1 0 0'} w={0} maxW={'560px'} {...cardStyles} boxShadow={'3'}>
<ChatQuoteList
chatTime={quoteData.chatTime}
rawSearch={quoteData.rawSearch}
metadata={quoteData.metadata}
onClose={() => setQuoteData(undefined)}

View File

@ -163,7 +163,6 @@ const ChatTest = ({ isOpen, nodes = [], edges = [], onClose }: Props) => {
borderRadius={'md'}
>
<ChatQuoteList
chatTime={quoteData.chatTime}
rawSearch={quoteData.rawSearch}
metadata={quoteData.metadata}
onClose={() => setQuoteData(undefined)}

View File

@ -1,4 +1,4 @@
import { Box, Button, Flex, HStack } from '@chakra-ui/react';
import { Box, Flex, HStack } from '@chakra-ui/react';
import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
import MyIcon from '@fastgpt/web/components/common/Icon';
@ -8,41 +8,34 @@ import DownloadButton from './DownloadButton';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { downloadFetch } from '@/web/common/system/utils';
import { useCallback, useEffect, useMemo, useState } from 'react';
import { getCollectionSource, getDatasetDataPermission } from '@/web/core/dataset/api';
import { useChatStore } from '@/web/core/chat/context/useChatStore';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { getDatasetDataPermission } from '@/web/core/dataset/api';
import ScoreTag from './ScoreTag';
import { formatScore } from '@/components/core/dataset/QuoteItem';
import NavButton from './NavButton';
import { useLinkedScroll } from '@fastgpt/web/hooks/useLinkedScroll';
import CollectionQuoteItem from './CollectionQuoteItem';
import { DatasetDataListItemType } from '@/global/core/dataset/type';
import { metadataType } from '@/web/core/chat/context/chatItemContext';
import { GetCollectionQuoteDataProps } from '@/web/core/chat/context/chatItemContext';
import { useUserStore } from '@/web/support/user/useUserStore';
import { getCollectionQuote } from '@/web/core/chat/api';
import MyIconButton from '@fastgpt/web/components/common/Icon/button';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { getCollectionSourceAndOpen } from '@/web/core/dataset/hooks/readCollectionSource';
import { QuoteDataItemType } from '@/service/core/chat/constants';
const CollectionReader = ({
rawSearch,
metadata,
chatTime,
onClose
}: {
rawSearch: SearchDataResponseItemType[];
metadata: metadataType;
chatTime: Date;
metadata: GetCollectionQuoteDataProps;
onClose: () => void;
}) => {
const { t } = useTranslation();
const router = useRouter();
const { toast } = useToast();
const { chatId, appId, outLinkAuthData } = useChatStore();
const { userInfo } = useUserStore();
const { collectionId, datasetId, chatItemId, sourceId, sourceName } = metadata;
const { collectionId, datasetId, chatItemDataId, sourceId, sourceName } = metadata;
const [quoteIndex, setQuoteIndex] = useState(0);
const { data: permissionData, loading: isPermissionLoading } = useRequest2(
@ -75,11 +68,10 @@ const CollectionReader = ({
refreshDeps: [collectionId],
params: {
collectionId,
chatTime,
chatItemId,
chatId,
appId,
...outLinkAuthData
chatItemDataId,
chatId: metadata.chatId,
appId: metadata.appId,
...metadata.outLinkAuthData
},
initialId: currentQuoteItem?.id,
initialIndex: currentQuoteItem?.chunkIndex,
@ -87,11 +79,14 @@ const CollectionReader = ({
});
const loading = isLoading || isPermissionLoading;
const isDeleted = !datasetDataList.find((item) => item._id === currentQuoteItem?.id);
const isDeleted = useMemo(
() => !datasetDataList.find((item) => item._id === currentQuoteItem?.id),
[datasetDataList, currentQuoteItem?.id]
);
const formatedDataList = useMemo(
() =>
datasetDataList.map((item: DatasetDataListItemType) => {
datasetDataList.map((item: QuoteDataItemType) => {
const isCurrentSelected = currentQuoteItem?.id === item._id;
const quoteIndex = filterResults.findIndex((res) => res.id === item._id);
@ -115,17 +110,12 @@ const CollectionReader = ({
filename: 'data.csv',
body: {
collectionId: collectionId,
chatTime: chatTime,
chatItemId: chatItemId
chatItemDataId
}
});
});
const handleRead = getCollectionSourceAndOpen({
appId,
chatId,
...metadata
});
const handleRead = getCollectionSourceAndOpen(metadata);
const handleNavigate = useCallback(
async (targetIndex: number) => {

View File

@ -4,75 +4,60 @@ import MyIcon from '@fastgpt/web/components/common/Icon';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { useTranslation } from 'react-i18next';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useChatStore } from '@/web/core/chat/context/useChatStore';
import QuoteItem from './QuoteItem';
import { useMemo } from 'react';
import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
import { formatScore } from '@/components/core/dataset/QuoteItem';
import { metadataType } from '@/web/core/chat/context/chatItemContext';
import { GetAllQuoteDataProps } from '@/web/core/chat/context/chatItemContext';
import { getQuoteDataList } from '@/web/core/chat/api';
const QuoteReader = ({
rawSearch,
metadata,
chatTime,
onClose
}: {
rawSearch: SearchDataResponseItemType[];
metadata: metadataType;
chatTime: Date;
metadata: GetAllQuoteDataProps;
onClose: () => void;
}) => {
const { t } = useTranslation();
const { chatId, appId, outLinkAuthData } = useChatStore();
const { data, loading } = useRequest2(
const { data: quoteList, loading } = useRequest2(
async () =>
await getQuoteDataList({
datasetDataIdList: rawSearch.map((item) => item.id),
chatTime,
collectionIdList: metadata.collectionIdList,
chatItemId: metadata.chatItemId,
appId,
chatId,
...outLinkAuthData
chatItemDataId: metadata.chatItemDataId,
appId: metadata.appId,
chatId: metadata.chatId,
...metadata.outLinkAuthData
}),
{
manual: false
}
);
const filterResults = useMemo(() => {
if (!metadata.collectionId) {
return rawSearch;
}
return rawSearch.filter(
(item) => item.collectionId === metadata.collectionId && item.sourceId === metadata.sourceId
);
}, [metadata, rawSearch]);
const formatedDataList = useMemo(() => {
return filterResults
.map((item) => {
const currentFilterItem = data?.quoteList.find((res) => res._id === item.id);
return rawSearch
.map((searchItem) => {
const dataItem = quoteList?.find((item) => item._id === searchItem.id);
return {
...item,
q: currentFilterItem?.q || '',
a: currentFilterItem?.a || '',
score: formatScore(item.score),
id: searchItem.id,
q: dataItem?.q || 'Can not find Data',
a: dataItem?.a || '',
score: formatScore(searchItem.score),
sourceName: searchItem?.sourceName || '',
icon: getSourceNameIcon({
sourceId: item.sourceId,
sourceName: item.sourceName
sourceId: searchItem.sourceId,
sourceName: searchItem.sourceName
})
};
})
.sort((a, b) => {
return (b.score.primaryScore?.value || 0) - (a.score.primaryScore?.value || 0);
});
}, [data?.quoteList, filterResults]);
}, [quoteList, rawSearch]);
return (
<Flex flexDirection={'column'} h={'full'}>
@ -86,18 +71,7 @@ const QuoteReader = ({
>
<Box flex={1} py={4}>
<Flex gap={2} mr={2} mb={1}>
<MyIcon
name={
metadata.sourceId && metadata.sourceName
? (getSourceNameIcon({
sourceId: metadata.sourceId,
sourceName: metadata.sourceName
}) as any)
: 'core/chat/quoteFill'
}
w={['1rem', '1.25rem']}
color={'primary.600'}
/>
<MyIcon name={'core/chat/quoteFill'} w={['1rem', '1.25rem']} color={'primary.600'} />
<Box
maxW={['200px', '300px']}
className={'textEllipsis'}
@ -105,9 +79,7 @@ const QuoteReader = ({
color={'myGray.900'}
fontWeight={'medium'}
>
{metadata.sourceName
? metadata.sourceName
: t('common:core.chat.Quote Amount', { amount: rawSearch.length })}
{t('common:core.chat.Quote Amount', { amount: rawSearch.length })}
</Box>
</Flex>
<Box fontSize={'mini'} color={'myGray.500'}>

View File

@ -1,39 +1,28 @@
import React from 'react';
import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import { useContextSelector } from 'use-context-selector';
import { ChatItemContext, metadataType } from '@/web/core/chat/context/chatItemContext';
import { ChatItemContext, GetQuoteProps } from '@/web/core/chat/context/chatItemContext';
import CollectionQuoteReader from './CollectionQuoteReader';
import QuoteReader from './QuoteReader';
const ChatQuoteList = ({
chatTime,
rawSearch = [],
metadata,
onClose
}: {
chatTime: Date;
rawSearch: SearchDataResponseItemType[];
metadata: metadataType;
metadata: GetQuoteProps;
onClose: () => void;
}) => {
const isShowReadRawSource = useContextSelector(ChatItemContext, (v) => v.isShowReadRawSource);
return (
<>
{metadata.collectionId && isShowReadRawSource ? (
<CollectionQuoteReader
rawSearch={rawSearch}
metadata={metadata}
chatTime={chatTime}
onClose={onClose}
/>
) : (
<QuoteReader
rawSearch={rawSearch}
metadata={metadata}
chatTime={chatTime}
onClose={onClose}
/>
{'collectionId' in metadata && isShowReadRawSource && (
<CollectionQuoteReader rawSearch={rawSearch} metadata={metadata} onClose={onClose} />
)}
{'collectionIdList' in metadata && (
<QuoteReader rawSearch={rawSearch} metadata={metadata} onClose={onClose} />
)}
</>
);

View File

@ -5,25 +5,25 @@ import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { LinkedListResponse, LinkedPaginationProps } from '@fastgpt/web/common/fetch/type';
import { FilterQuery, Types } from 'mongoose';
import { dataFieldSelector, processChatTimeFilter } from './getQuote';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { quoteDataFieldSelector, QuoteDataItemType } from '@/service/core/chat/constants';
import { processChatTimeFilter } from '@/service/core/chat/utils';
import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
export type GetCollectionQuoteProps = LinkedPaginationProps & {
chatTime: Date;
chatId: string;
chatItemDataId: string;
isInitialLoad: boolean;
collectionId: string;
chatItemId: string;
appId: string;
chatId: string;
shareId?: string;
outLinkUid?: string;
teamId?: string;
teamToken?: string;
};
export type GetCollectionQuoteRes = LinkedListResponse<DatasetDataSchemaType>;
export type GetCollectionQuoteRes = LinkedListResponse<QuoteDataItemType>;
type BaseMatchType = FilterQuery<DatasetDataSchemaType>;
@ -37,11 +37,10 @@ async function handler(
prevIndex,
nextId,
nextIndex,
chatTime,
isInitialLoad,
collectionId,
chatItemId,
chatItemDataId,
appId,
chatId,
shareId,
@ -53,61 +52,50 @@ async function handler(
const limitedPageSize = Math.min(pageSize, 30);
try {
await authDatasetCollection({
const [{ chat }, { chatItem }] = await Promise.all([
authChatCrud({
req,
authToken: true,
authApiKey: true,
collectionId: req.body.collectionId,
per: ReadPermissionVal
});
} catch (error) {
await Promise.all([
authChatCrud({
req,
authToken: true,
appId,
chatId,
shareId,
outLinkUid,
teamId,
teamToken
}),
authCollectionInChat({ appId, chatId, chatItemId, collectionId })
]);
}
appId,
chatId,
shareId,
outLinkUid,
teamId,
teamToken
}),
authCollectionInChat({ appId, chatId, chatItemDataId, collectionIds: [collectionId] })
]);
if (!chat) return Promise.reject(ChatErrEnum.unAuthChat);
const baseMatch: BaseMatchType = {
collectionId,
$or: [
{ updateTime: { $lt: new Date(chatTime) } },
{ history: { $elemMatch: { updateTime: { $lt: new Date(chatTime) } } } }
{ updateTime: { $lt: new Date(chatItem.time) } },
{ history: { $elemMatch: { updateTime: { $lt: new Date(chatItem.time) } } } }
]
};
if (initialId && initialIndex !== undefined) {
return await handleInitialLoad(
return await handleInitialLoad({
initialId,
initialIndex,
limitedPageSize,
chatTime,
chatItemId,
pageSize: limitedPageSize,
chatTime: chatItem.time,
isInitialLoad,
baseMatch
);
});
}
if ((prevId && prevIndex !== undefined) || (nextId && nextIndex !== undefined)) {
return await handlePaginatedLoad(
return await handlePaginatedLoad({
prevId,
prevIndex,
nextId,
nextIndex,
limitedPageSize,
chatTime,
chatItemId,
pageSize: limitedPageSize,
chatTime: chatItem.time,
baseMatch
);
});
}
return { list: [], hasMorePrev: false, hasMoreNext: false };
@ -115,38 +103,39 @@ async function handler(
export default NextAPI(handler);
async function handleInitialLoad(
initialId: string,
initialIndex: number,
pageSize: number,
chatTime: Date,
chatItemId: string,
isInitialLoad: boolean,
baseMatch: BaseMatchType
): Promise<GetCollectionQuoteRes> {
async function handleInitialLoad({
initialId,
initialIndex,
pageSize,
chatTime,
isInitialLoad,
baseMatch
}: {
initialId: string;
initialIndex: number;
pageSize: number;
chatTime: Date;
isInitialLoad: boolean;
baseMatch: BaseMatchType;
}): Promise<GetCollectionQuoteRes> {
const centerNode = await MongoDatasetData.findOne(
{
_id: new Types.ObjectId(initialId)
},
dataFieldSelector
quoteDataFieldSelector
).lean();
if (!centerNode) {
if (isInitialLoad) {
const list = await MongoDatasetData.find(baseMatch, dataFieldSelector)
const list = await MongoDatasetData.find(baseMatch, quoteDataFieldSelector)
.sort({ chunkIndex: 1, _id: -1 })
.limit(pageSize)
.lean();
const listRes = list.map((item, index) => ({
...item,
index: item.chunkIndex
}));
const hasMoreNext = list.length === pageSize;
return {
list: listRes,
list: processChatTimeFilter(list, chatTime),
hasMorePrev: false,
hasMoreNext
};
@ -173,28 +162,30 @@ async function handleInitialLoad(
const resultList = [...prevList, centerNode, ...nextList];
const list = processChatTimeFilter(resultList, chatTime);
return {
list: list.map((item) => ({
...item,
index: item.chunkIndex
})),
list: processChatTimeFilter(resultList, chatTime),
hasMorePrev,
hasMoreNext
};
}
async function handlePaginatedLoad(
prevId: string | undefined,
prevIndex: number | undefined,
nextId: string | undefined,
nextIndex: number | undefined,
pageSize: number,
chatTime: Date,
chatItemId: string,
baseMatch: BaseMatchType
): Promise<GetCollectionQuoteRes> {
async function handlePaginatedLoad({
prevId,
prevIndex,
nextId,
nextIndex,
pageSize,
chatTime,
baseMatch
}: {
prevId: string | undefined;
prevIndex: number | undefined;
nextId: string | undefined;
nextIndex: number | undefined;
pageSize: number;
chatTime: Date;
baseMatch: BaseMatchType;
}): Promise<GetCollectionQuoteRes> {
const { list, hasMore } =
prevId && prevIndex !== undefined
? await getPrevNodes(prevId, prevIndex, pageSize, baseMatch)
@ -203,10 +194,7 @@ async function handlePaginatedLoad(
const processedList = processChatTimeFilter(list, chatTime);
return {
list: processedList.map((item) => ({
...item,
index: item.chunkIndex
})),
list: processedList,
hasMorePrev: !!prevId && hasMore,
hasMoreNext: !!nextId && hasMore
};
@ -217,7 +205,10 @@ async function getPrevNodes(
initialIndex: number,
limit: number,
baseMatch: BaseMatchType
) {
): Promise<{
list: DatasetDataSchemaType[];
hasMore: boolean;
}> {
const match: BaseMatchType = {
...baseMatch,
$or: [
@ -226,7 +217,7 @@ async function getPrevNodes(
]
};
const list = await MongoDatasetData.find(match, dataFieldSelector)
const list = await MongoDatasetData.find(match, quoteDataFieldSelector)
.sort({ chunkIndex: -1, _id: 1 })
.limit(limit)
.lean();
@ -242,7 +233,10 @@ async function getNextNodes(
initialIndex: number,
limit: number,
baseMatch: BaseMatchType
) {
): Promise<{
list: DatasetDataSchemaType[];
hasMore: boolean;
}> {
const match: BaseMatchType = {
...baseMatch,
$or: [
@ -251,7 +245,7 @@ async function getNextNodes(
]
};
const list = await MongoDatasetData.find(match, dataFieldSelector)
const list = await MongoDatasetData.find(match, quoteDataFieldSelector)
.sort({ chunkIndex: 1, _id: -1 })
.limit(limit)
.lean();

View File

@ -1,100 +1,64 @@
import { NextAPI } from '@/service/middleware/entry';
import { authChatCrud, authCollectionInChat } from '@/service/support/permission/auth/chat';
import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { quoteDataFieldSelector, QuoteDataItemType } from '@/service/core/chat/constants';
import { processChatTimeFilter } from '@/service/core/chat/utils';
import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
export type GetQuoteDataProps = {
datasetDataIdList: string[];
chatTime: Date;
collectionIdList: string[];
chatItemId: string;
appId: string;
chatId: string;
chatItemDataId: string;
appId: string;
shareId?: string;
outLinkUid?: string;
teamId?: string;
teamToken?: string;
};
export type GetQuoteDataRes = {
quoteList: DatasetDataSchemaType[];
};
export const dataFieldSelector =
'_id datasetId collectionId q a chunkIndex history updateTime currentChatItemId prevId';
export type GetQuoteDataRes = QuoteDataItemType[];
async function handler(req: ApiRequestProps<GetQuoteDataProps>): Promise<GetQuoteDataRes> {
const {
datasetDataIdList,
chatTime,
appId,
chatId,
chatItemDataId,
shareId,
outLinkUid,
teamId,
teamToken,
collectionIdList,
chatItemId,
chatId,
appId,
shareId,
outLinkUid,
teamId,
teamToken
datasetDataIdList
} = req.body;
await authChatCrud({
req,
authToken: true,
appId,
chatId,
shareId,
outLinkUid,
teamId,
teamToken
});
await Promise.all(
collectionIdList.map(async (collectionId) => {
await authCollectionInChat({ appId, chatId, chatItemId, collectionId });
})
);
const [chat, { chatItem }] = await Promise.all([
authChatCrud({
req,
authToken: true,
appId,
chatId,
shareId,
outLinkUid,
teamId,
teamToken
}),
authCollectionInChat({ appId, chatId, chatItemDataId, collectionIds: collectionIdList })
]);
if (!chat) return Promise.reject(ChatErrEnum.unAuthChat);
const list = await MongoDatasetData.find(
{ _id: { $in: datasetDataIdList } },
dataFieldSelector
{ _id: { $in: datasetDataIdList }, collectionId: { $in: collectionIdList } },
quoteDataFieldSelector
).lean();
const quoteList = processChatTimeFilter(list, chatTime);
const quoteList = processChatTimeFilter(list, chatItem.time);
return {
quoteList
};
return quoteList;
}
export default NextAPI(handler);
export function processChatTimeFilter(list: DatasetDataSchemaType[], chatTime: Date) {
return list.map((item) => {
if (!item.history) return item;
const { history, ...rest } = item;
const formatedChatTime = new Date(chatTime);
if (item.updateTime <= formatedChatTime) {
return rest;
}
const latestHistoryIndex = history.findIndex(
(historyItem: any) => historyItem.updateTime <= formatedChatTime
);
if (latestHistoryIndex === -1) return rest;
const latestHistory = history[latestHistoryIndex];
return {
...rest,
q: latestHistory?.q || item.q,
a: latestHistory?.a || item.a,
updated: true
};
});
}

View File

@ -2,11 +2,7 @@ import type { NextApiRequest } from 'next';
import type { ApiDatasetCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller';
import {
TrainingModeEnum,
DatasetCollectionTypeEnum,
DatasetCollectionDataProcessModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
@ -16,7 +12,8 @@ import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
async function handler(req: NextApiRequest): CreateCollectionResponse {
const { name, apiFileId, ...body } = req.body as ApiDatasetCreateDatasetCollectionParams;
const { name, apiFileId, customPdfParse, ...body } =
req.body as ApiDatasetCreateDatasetCollectionParams;
const { teamId, tmbId, dataset } = await authDataset({
req,
@ -50,7 +47,8 @@ async function handler(req: NextApiRequest): CreateCollectionResponse {
yuqueServer,
apiFileId,
teamId,
tmbId
tmbId,
customPdfParse
});
const { collectionId, insertResults } = await createCollectionAndInsertData({
@ -62,11 +60,12 @@ async function handler(req: NextApiRequest): CreateCollectionResponse {
teamId,
tmbId,
type: DatasetCollectionTypeEnum.apiFile,
name: name,
name,
apiFileId,
metadata: {
relatedImgId: apiFileId
}
},
customPdfParse
}
});

View File

@ -1,9 +1,13 @@
import { NextAPI } from '@/service/middleware/entry';
import { authChatCrud, authCollectionInChat } from '@/service/support/permission/auth/chat';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { useIPFrequencyLimit } from '@fastgpt/service/common/middle/reqFrequencyLimit';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { responseWriteController } from '@fastgpt/service/common/response';
import { addLog } from '@fastgpt/service/common/system/log';
import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { ApiRequestProps } from '@fastgpt/service/type/next';
@ -11,21 +15,69 @@ import { NextApiResponse } from 'next';
export type ExportCollectionBody = {
collectionId: string;
appId?: string;
chatId?: string;
chatItemDataId?: string;
chatTime: Date;
};
} & OutLinkChatAuthProps;
async function handler(req: ApiRequestProps<ExportCollectionBody, {}>, res: NextApiResponse) {
let { collectionId, chatTime } = req.body;
const { teamId, collection } = await authDatasetCollection({
req,
authToken: true,
const {
collectionId,
per: ReadPermissionVal
});
appId,
chatId,
chatItemDataId,
shareId,
outLinkUid,
teamId,
teamToken,
chatTime
} = req.body;
const { collection, teamId: userTeamId } = await (async () => {
if (!appId || !chatId || !chatItemDataId) {
return authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId: req.body.collectionId,
per: ReadPermissionVal
});
}
/*
1. auth chat read permission
2. auth collection quote in chat
3. auth outlink open show quote
*/
const [authRes, collection] = await Promise.all([
authChatCrud({
req,
authToken: true,
appId,
chatId,
shareId,
outLinkUid,
teamId,
teamToken
}),
getCollectionWithDataset(collectionId),
authCollectionInChat({ appId, chatId, chatItemDataId, collectionIds: [collectionId] })
]);
if (!authRes.showRawSource) {
return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
}
return {
...authRes,
collection
};
})();
const where = {
teamId,
teamId: userTeamId,
datasetId: collection.datasetId,
collectionId,
...(chatTime

View File

@ -19,7 +19,7 @@ export type readCollectionSourceBody = {
appId?: string;
chatId?: string;
chatItemId?: string;
chatItemDataId?: string;
} & OutLinkChatAuthProps;
export type readCollectionSourceResponse = {
@ -30,7 +30,7 @@ export type readCollectionSourceResponse = {
async function handler(
req: ApiRequestProps<readCollectionSourceBody, readCollectionSourceQuery>
): Promise<readCollectionSourceResponse> {
const { collectionId, appId, chatId, chatItemId, shareId, outLinkUid, teamId, teamToken } =
const { collectionId, appId, chatId, chatItemDataId, shareId, outLinkUid, teamId, teamToken } =
req.body;
const {
@ -39,7 +39,7 @@ async function handler(
tmbId: uid,
authType
} = await (async () => {
if (!appId || !chatId || !chatItemId) {
if (!appId || !chatId || !chatItemDataId) {
return authDatasetCollection({
req,
authToken: true,
@ -66,7 +66,7 @@ async function handler(
teamToken
}),
getCollectionWithDataset(collectionId),
authCollectionInChat({ appId, chatId, chatItemId, collectionId })
authCollectionInChat({ appId, chatId, chatItemDataId, collectionIds: [collectionId] })
]);
if (!authRes.showRawSource) {

View File

@ -231,7 +231,6 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
{quoteData && (
<PageContainer flex={'1 0 0'} w={0} maxW={'560px'}>
<ChatQuoteList
chatTime={quoteData.chatTime}
rawSearch={quoteData.rawSearch}
metadata={quoteData.metadata}
onClose={() => setQuoteData(undefined)}

View File

@ -304,7 +304,6 @@ const OutLink = (props: Props) => {
{quoteData && (
<PageContainer flex={'1 0 0'} w={0} maxW={'560px'}>
<ChatQuoteList
chatTime={quoteData.chatTime}
rawSearch={quoteData.rawSearch}
metadata={quoteData.metadata}
onClose={() => setQuoteData(undefined)}

View File

@ -245,7 +245,6 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
{quoteData && (
<PageContainer flex={'1 0 0'} w={0} maxW={'560px'}>
<ChatQuoteList
chatTime={quoteData.chatTime}
rawSearch={quoteData.rawSearch}
metadata={quoteData.metadata}
onClose={() => setQuoteData(undefined)}

View File

@ -0,0 +1,13 @@
import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type';
export const quoteDataFieldSelector = '_id q a history updateTime chunkIndex';
export type QuoteDataItemType = {
_id: string;
q: DatasetDataSchemaType['q'];
a: DatasetDataSchemaType['a'];
history?: DatasetDataSchemaType['history'];
updateTime: DatasetDataSchemaType['updateTime'];
index: DatasetDataSchemaType['chunkIndex'];
updated?: boolean;
};

View File

@ -0,0 +1,46 @@
import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type';
import { QuoteDataItemType } from './constants';
// 获取对话时间时,引用的内容
export function processChatTimeFilter(
dataList: DatasetDataSchemaType[],
chatTime: Date
): QuoteDataItemType[] {
return dataList.map((item) => {
const defaultItem = {
_id: item._id,
q: item.q,
a: item.a,
updateTime: item.updateTime,
index: item.chunkIndex
};
if (!item.history) return defaultItem;
const history = item.history;
const formatedChatTime = new Date(chatTime);
if (item.updateTime <= formatedChatTime) {
return defaultItem;
}
const latestHistoryIndex = history.findIndex(
(historyItem) => historyItem.updateTime <= formatedChatTime
);
if (latestHistoryIndex === -1) {
return defaultItem;
}
const latestHistory = history[latestHistoryIndex];
return {
_id: item._id,
q: latestHistory.q,
a: latestHistory.a,
updateTime: latestHistory.updateTime,
index: item.chunkIndex,
updated: true
};
});
}

View File

@ -25,6 +25,14 @@ const defaultResponseShow = {
showNodeStatus: true,
showRawSource: true
};
type AuthChatCommonProps = {
appId: string;
shareId?: string;
outLinkUid?: string;
teamId?: string;
teamToken?: string;
};
export async function authChatCrud({
appId,
chatId,
@ -35,14 +43,10 @@ export async function authChatCrud({
teamId: spaceTeamId,
teamToken,
...props
}: AuthModeType & {
appId: string;
chatId?: string;
shareId?: string;
outLinkUid?: string;
teamId?: string;
teamToken?: string;
}): Promise<{
}: AuthModeType &
AuthChatCommonProps & {
chatId?: string;
}): Promise<{
teamId: string;
tmbId: string;
uid: string;
@ -188,27 +192,29 @@ export async function authChatCrud({
}
export const authCollectionInChat = async ({
collectionId,
collectionIds,
appId,
chatId,
chatItemId
chatItemDataId
}: {
collectionId: string;
collectionIds: string[];
appId: string;
chatId: string;
chatItemId: string;
}) => {
chatItemDataId: string;
}): Promise<{
chatItem: { time: Date; responseData?: ChatHistoryItemResType[] };
}> => {
try {
const chatItem = (await MongoChatItem.findOne(
{
appId,
chatId,
dataId: chatItemId
dataId: chatItemDataId
},
'responseData'
).lean()) as AIChatItemType;
'responseData time'
).lean()) as { time: Date; responseData?: ChatHistoryItemResType[] };
if (!chatItem) return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
if (!chatItem) return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
// 找 responseData 里,是否有该文档 id
const responseData = chatItem.responseData || [];
@ -224,15 +230,16 @@ export const authCollectionInChat = async ({
})
.flat() || [];
if (
flatResData.some((item) => {
if (item.quoteList) {
return item.quoteList.some((quote) => quote.collectionId === collectionId);
}
return false;
})
) {
return true;
const quoteListSet = new Set(
flatResData
.map((item) => item.quoteList?.map((quote) => String(quote.collectionId)) || [])
.flat()
);
if (collectionIds.every((id) => quoteListSet.has(id))) {
return {
chatItem
};
}
} catch (error) {}
return Promise.reject(DatasetErrEnum.unAuthDatasetFile);

View File

@ -1,11 +1,6 @@
import { GET, POST, DELETE, PUT } from '@/web/common/api/request';
import type {
ChatHistoryItemType,
ChatHistoryItemResType,
ChatSiteItemType,
ChatItemType
} from '@fastgpt/global/core/chat/type.d';
import { getResDataQuery } from '@/pages/api/core/chat/getResData';
import type { ChatHistoryItemType, ChatHistoryItemResType } from '@fastgpt/global/core/chat/type.d';
import type { getResDataQuery } from '@/pages/api/core/chat/getResData';
import type {
CloseCustomFeedbackParams,
InitChatProps,
@ -22,16 +17,16 @@ import type {
DeleteChatItemProps,
UpdateHistoryProps
} from '@/global/core/chat/api.d';
import { UpdateChatFeedbackProps } from '@fastgpt/global/core/chat/api';
import { AuthTeamTagTokenProps } from '@fastgpt/global/support/user/team/tag';
import { AppListItemType } from '@fastgpt/global/core/app/type';
import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
import type { UpdateChatFeedbackProps } from '@fastgpt/global/core/chat/api';
import type { AuthTeamTagTokenProps } from '@fastgpt/global/support/user/team/tag';
import type { AppListItemType } from '@fastgpt/global/core/app/type';
import type { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
import type {
getPaginationRecordsBody,
getPaginationRecordsResponse
} from '@/pages/api/core/chat/getPaginationRecords';
import { GetQuoteDataProps, GetQuoteDataRes } from '@/pages/api/core/chat/quote/getQuote';
import {
import type { GetQuoteDataProps, GetQuoteDataRes } from '@/pages/api/core/chat/quote/getQuote';
import type {
GetCollectionQuoteProps,
GetCollectionQuoteRes
} from '@/pages/api/core/chat/quote/getCollectionQuote';

View File

@ -9,6 +9,7 @@ import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
import { AppChatConfigType, VariableItemType } from '@fastgpt/global/core/app/type';
import { FlowNodeInputItemType } from '@fastgpt/global/core/workflow/type/io';
import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat';
type ContextProps = {
showRouteToAppDetail: boolean;
@ -32,19 +33,27 @@ type ChatBoxDataType = {
};
};
export type metadataType = {
export type GetQuoteDataBasicProps = {
appId: string;
chatId: string;
chatItemDataId: string;
outLinkAuthData?: OutLinkChatAuthProps;
};
// 获取单个集合引用
export type GetCollectionQuoteDataProps = GetQuoteDataBasicProps & {
collectionId: string;
collectionIdList: string[];
chatItemId: string;
sourceId: string;
sourceName: string;
datasetId: string;
};
export type GetAllQuoteDataProps = GetQuoteDataBasicProps & {
collectionIdList: string[];
};
export type GetQuoteProps = GetAllQuoteDataProps | GetCollectionQuoteDataProps;
export type QuoteDataType = {
rawSearch: SearchDataResponseItemType[];
metadata: metadataType;
chatTime: Date;
metadata: GetQuoteProps;
};
type ChatItemContextType = {