feat: api dataset support pdf parse;fix: chunk reader auth (#4117)
* feat: api dataset support pdf parse * fix: chunk reader auth
This commit is contained in:
parent
d4df77e637
commit
2c7bf2548b
@ -11,6 +11,7 @@ weight: 799
|
||||
|
||||
1. 商业版支持单团队模式,更好的管理内部成员。
|
||||
2. 知识库分块阅读器。
|
||||
3. API 知识库支持 PDF 增强解析。
|
||||
|
||||
## ⚙️ 优化
|
||||
|
||||
|
||||
@ -111,11 +111,13 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
|
||||
const getFileContent = async ({
|
||||
teamId,
|
||||
tmbId,
|
||||
apiFileId
|
||||
apiFileId,
|
||||
customPdfParse
|
||||
}: {
|
||||
teamId: string;
|
||||
tmbId: string;
|
||||
apiFileId: string;
|
||||
customPdfParse?: boolean;
|
||||
}) => {
|
||||
const data = await request<APIFileContentResponse>(
|
||||
`/v1/file/content`,
|
||||
@ -133,7 +135,8 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
|
||||
teamId,
|
||||
tmbId,
|
||||
url: previewUrl,
|
||||
relatedId: apiFileId
|
||||
relatedId: apiFileId,
|
||||
customPdfParse
|
||||
});
|
||||
return rawText;
|
||||
}
|
||||
|
||||
@ -127,7 +127,8 @@ export const readApiServerFileContent = async ({
|
||||
yuqueServer,
|
||||
apiFileId,
|
||||
teamId,
|
||||
tmbId
|
||||
tmbId,
|
||||
customPdfParse
|
||||
}: {
|
||||
apiServer?: APIFileServer;
|
||||
feishuServer?: FeishuServer;
|
||||
@ -135,9 +136,15 @@ export const readApiServerFileContent = async ({
|
||||
apiFileId: string;
|
||||
teamId: string;
|
||||
tmbId: string;
|
||||
customPdfParse?: boolean;
|
||||
}) => {
|
||||
if (apiServer) {
|
||||
return useApiDatasetRequest({ apiServer }).getFileContent({ teamId, tmbId, apiFileId });
|
||||
return useApiDatasetRequest({ apiServer }).getFileContent({
|
||||
teamId,
|
||||
tmbId,
|
||||
apiFileId,
|
||||
customPdfParse
|
||||
});
|
||||
}
|
||||
|
||||
if (feishuServer || yuqueServer) {
|
||||
|
||||
@ -11,19 +11,17 @@ import { useChatStore } from '@/web/core/chat/context/useChatStore';
|
||||
import { getQuoteDataList } from '@/web/core/chat/api';
|
||||
|
||||
const QuoteList = React.memo(function QuoteList({
|
||||
chatItemId,
|
||||
rawSearch = [],
|
||||
chatTime
|
||||
chatItemDataId = '',
|
||||
rawSearch = []
|
||||
}: {
|
||||
chatItemId?: string;
|
||||
chatItemDataId?: string;
|
||||
rawSearch: SearchDataResponseItemType[];
|
||||
chatTime: Date;
|
||||
}) {
|
||||
const theme = useTheme();
|
||||
const { chatId, appId, outLinkAuthData } = useChatStore();
|
||||
|
||||
const RawSourceBoxProps = useContextSelector(ChatBoxContext, (v) => ({
|
||||
chatItemId,
|
||||
chatItemDataId,
|
||||
appId: v.appId,
|
||||
chatId: v.chatId,
|
||||
...(v.outLinkAuthData || {})
|
||||
@ -34,13 +32,12 @@ const QuoteList = React.memo(function QuoteList({
|
||||
(v) => v.showRouteToDatasetDetail
|
||||
);
|
||||
|
||||
const { data } = useRequest2(
|
||||
const { data: quoteList } = useRequest2(
|
||||
async () =>
|
||||
await getQuoteDataList({
|
||||
datasetDataIdList: rawSearch.map((item) => item.id),
|
||||
chatTime,
|
||||
collectionIdList: [...new Set(rawSearch.map((item) => item.collectionId))],
|
||||
chatItemId: chatItemId || '',
|
||||
chatItemDataId,
|
||||
appId,
|
||||
chatId,
|
||||
...outLinkAuthData
|
||||
@ -53,7 +50,7 @@ const QuoteList = React.memo(function QuoteList({
|
||||
const formatedDataList = useMemo(() => {
|
||||
return rawSearch
|
||||
.map((item) => {
|
||||
const currentFilterItem = data?.quoteList.find((res) => res._id === item.id);
|
||||
const currentFilterItem = quoteList?.find((res) => res._id === item.id);
|
||||
|
||||
return {
|
||||
...item,
|
||||
@ -66,7 +63,7 @@ const QuoteList = React.memo(function QuoteList({
|
||||
const bScore = formatScore(b.score);
|
||||
return (bScore.primaryScore?.value || 0) - (aScore.primaryScore?.value || 0);
|
||||
});
|
||||
}, [data?.quoteList, rawSearch]);
|
||||
}, [quoteList, rawSearch]);
|
||||
|
||||
return (
|
||||
<>
|
||||
|
||||
@ -42,6 +42,8 @@ const ResponseTags = ({
|
||||
const [quoteFolded, setQuoteFolded] = useState<boolean>(true);
|
||||
|
||||
const chatType = useContextSelector(ChatBoxContext, (v) => v.chatType);
|
||||
const appId = useContextSelector(ChatBoxContext, (v) => v.appId);
|
||||
const chatId = useContextSelector(ChatBoxContext, (v) => v.chatId);
|
||||
|
||||
const setQuoteData = useContextSelector(ChatItemContext, (v) => v.setQuoteData);
|
||||
|
||||
@ -156,17 +158,15 @@ const ResponseTags = ({
|
||||
e.stopPropagation();
|
||||
|
||||
setQuoteData({
|
||||
chatTime,
|
||||
rawSearch: quoteList,
|
||||
metadata: {
|
||||
appId,
|
||||
chatId,
|
||||
chatItemDataId: dataId,
|
||||
collectionId: item.collectionId,
|
||||
collectionIdList: [
|
||||
...new Set(quoteList.map((item) => item.collectionId))
|
||||
],
|
||||
sourceId: item.sourceId || '',
|
||||
sourceName: item.sourceName,
|
||||
datasetId: item.datasetId,
|
||||
chatItemId: historyItem.dataId
|
||||
datasetId: item.datasetId
|
||||
}
|
||||
});
|
||||
}}
|
||||
@ -225,15 +225,12 @@ const ResponseTags = ({
|
||||
e.stopPropagation();
|
||||
|
||||
setQuoteData({
|
||||
chatTime,
|
||||
rawSearch: quoteList,
|
||||
metadata: {
|
||||
collectionId: '',
|
||||
collectionIdList: [...new Set(quoteList.map((item) => item.collectionId))],
|
||||
chatItemId: historyItem.dataId,
|
||||
sourceId: '',
|
||||
sourceName: '',
|
||||
datasetId: ''
|
||||
appId,
|
||||
chatId,
|
||||
chatItemDataId: dataId,
|
||||
collectionIdList: [...new Set(quoteList.map((item) => item.collectionId))]
|
||||
}
|
||||
});
|
||||
}}
|
||||
|
||||
@ -265,13 +265,7 @@ export const WholeResponseContent = ({
|
||||
{activeModule.quoteList && activeModule.quoteList.length > 0 && (
|
||||
<Row
|
||||
label={t('common:core.chat.response.module quoteList')}
|
||||
rawDom={
|
||||
<QuoteList
|
||||
chatItemId={dataId}
|
||||
chatTime={chatTime || new Date()}
|
||||
rawSearch={activeModule.quoteList}
|
||||
/>
|
||||
}
|
||||
rawDom={<QuoteList chatItemDataId={dataId} rawSearch={activeModule.quoteList} />}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
|
||||
@ -243,7 +243,7 @@ const QuoteItem = ({
|
||||
color={'primary.500'}
|
||||
href={`/dataset/detail?datasetId=${quoteItem.datasetId}¤tTab=dataCard&collectionId=${quoteItem.collectionId}`}
|
||||
>
|
||||
{t('common:core.dataset.Go Dataset')}
|
||||
{t('chat:to_dataset')}
|
||||
<MyIcon name={'common/rightArrowLight'} w={'10px'} />
|
||||
</Link>
|
||||
)}
|
||||
|
||||
@ -23,7 +23,7 @@ const RawSourceBox = ({
|
||||
collectionId,
|
||||
appId,
|
||||
chatId,
|
||||
chatItemId,
|
||||
chatItemDataId,
|
||||
shareId,
|
||||
outLinkUid,
|
||||
teamId,
|
||||
@ -40,7 +40,7 @@ const RawSourceBox = ({
|
||||
collectionId,
|
||||
appId,
|
||||
chatId,
|
||||
chatItemId,
|
||||
chatItemDataId,
|
||||
shareId,
|
||||
outLinkUid,
|
||||
teamId,
|
||||
|
||||
@ -184,7 +184,6 @@ const DetailLogsModal = ({ appId, chatId, onClose }: Props) => {
|
||||
borderRadius={'md'}
|
||||
>
|
||||
<ChatQuoteList
|
||||
chatTime={quoteData.chatTime}
|
||||
rawSearch={quoteData.rawSearch}
|
||||
metadata={quoteData.metadata}
|
||||
onClose={() => setQuoteData(undefined)}
|
||||
|
||||
@ -88,7 +88,6 @@ const ChatTest = ({ appForm, setRenderEdit }: Props) => {
|
||||
{quoteData && (
|
||||
<Box flex={'1 0 0'} w={0} maxW={'560px'} {...cardStyles} boxShadow={'3'}>
|
||||
<ChatQuoteList
|
||||
chatTime={quoteData.chatTime}
|
||||
rawSearch={quoteData.rawSearch}
|
||||
metadata={quoteData.metadata}
|
||||
onClose={() => setQuoteData(undefined)}
|
||||
|
||||
@ -163,7 +163,6 @@ const ChatTest = ({ isOpen, nodes = [], edges = [], onClose }: Props) => {
|
||||
borderRadius={'md'}
|
||||
>
|
||||
<ChatQuoteList
|
||||
chatTime={quoteData.chatTime}
|
||||
rawSearch={quoteData.rawSearch}
|
||||
metadata={quoteData.metadata}
|
||||
onClose={() => setQuoteData(undefined)}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { Box, Button, Flex, HStack } from '@chakra-ui/react';
|
||||
import { Box, Flex, HStack } from '@chakra-ui/react';
|
||||
import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
|
||||
import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
|
||||
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
@ -8,41 +8,34 @@ import DownloadButton from './DownloadButton';
|
||||
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
|
||||
import { downloadFetch } from '@/web/common/system/utils';
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import { getCollectionSource, getDatasetDataPermission } from '@/web/core/dataset/api';
|
||||
import { useChatStore } from '@/web/core/chat/context/useChatStore';
|
||||
import { useToast } from '@fastgpt/web/hooks/useToast';
|
||||
import { getErrText } from '@fastgpt/global/common/error/utils';
|
||||
import { getDatasetDataPermission } from '@/web/core/dataset/api';
|
||||
import ScoreTag from './ScoreTag';
|
||||
import { formatScore } from '@/components/core/dataset/QuoteItem';
|
||||
import NavButton from './NavButton';
|
||||
import { useLinkedScroll } from '@fastgpt/web/hooks/useLinkedScroll';
|
||||
import CollectionQuoteItem from './CollectionQuoteItem';
|
||||
import { DatasetDataListItemType } from '@/global/core/dataset/type';
|
||||
import { metadataType } from '@/web/core/chat/context/chatItemContext';
|
||||
import { GetCollectionQuoteDataProps } from '@/web/core/chat/context/chatItemContext';
|
||||
import { useUserStore } from '@/web/support/user/useUserStore';
|
||||
import { getCollectionQuote } from '@/web/core/chat/api';
|
||||
import MyIconButton from '@fastgpt/web/components/common/Icon/button';
|
||||
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
|
||||
import MyBox from '@fastgpt/web/components/common/MyBox';
|
||||
import { getCollectionSourceAndOpen } from '@/web/core/dataset/hooks/readCollectionSource';
|
||||
import { QuoteDataItemType } from '@/service/core/chat/constants';
|
||||
|
||||
const CollectionReader = ({
|
||||
rawSearch,
|
||||
metadata,
|
||||
chatTime,
|
||||
onClose
|
||||
}: {
|
||||
rawSearch: SearchDataResponseItemType[];
|
||||
metadata: metadataType;
|
||||
chatTime: Date;
|
||||
metadata: GetCollectionQuoteDataProps;
|
||||
onClose: () => void;
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const router = useRouter();
|
||||
const { toast } = useToast();
|
||||
const { chatId, appId, outLinkAuthData } = useChatStore();
|
||||
const { userInfo } = useUserStore();
|
||||
const { collectionId, datasetId, chatItemId, sourceId, sourceName } = metadata;
|
||||
const { collectionId, datasetId, chatItemDataId, sourceId, sourceName } = metadata;
|
||||
const [quoteIndex, setQuoteIndex] = useState(0);
|
||||
|
||||
const { data: permissionData, loading: isPermissionLoading } = useRequest2(
|
||||
@ -75,11 +68,10 @@ const CollectionReader = ({
|
||||
refreshDeps: [collectionId],
|
||||
params: {
|
||||
collectionId,
|
||||
chatTime,
|
||||
chatItemId,
|
||||
chatId,
|
||||
appId,
|
||||
...outLinkAuthData
|
||||
chatItemDataId,
|
||||
chatId: metadata.chatId,
|
||||
appId: metadata.appId,
|
||||
...metadata.outLinkAuthData
|
||||
},
|
||||
initialId: currentQuoteItem?.id,
|
||||
initialIndex: currentQuoteItem?.chunkIndex,
|
||||
@ -87,11 +79,14 @@ const CollectionReader = ({
|
||||
});
|
||||
|
||||
const loading = isLoading || isPermissionLoading;
|
||||
const isDeleted = !datasetDataList.find((item) => item._id === currentQuoteItem?.id);
|
||||
const isDeleted = useMemo(
|
||||
() => !datasetDataList.find((item) => item._id === currentQuoteItem?.id),
|
||||
[datasetDataList, currentQuoteItem?.id]
|
||||
);
|
||||
|
||||
const formatedDataList = useMemo(
|
||||
() =>
|
||||
datasetDataList.map((item: DatasetDataListItemType) => {
|
||||
datasetDataList.map((item: QuoteDataItemType) => {
|
||||
const isCurrentSelected = currentQuoteItem?.id === item._id;
|
||||
const quoteIndex = filterResults.findIndex((res) => res.id === item._id);
|
||||
|
||||
@ -115,17 +110,12 @@ const CollectionReader = ({
|
||||
filename: 'data.csv',
|
||||
body: {
|
||||
collectionId: collectionId,
|
||||
chatTime: chatTime,
|
||||
chatItemId: chatItemId
|
||||
chatItemDataId
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const handleRead = getCollectionSourceAndOpen({
|
||||
appId,
|
||||
chatId,
|
||||
...metadata
|
||||
});
|
||||
const handleRead = getCollectionSourceAndOpen(metadata);
|
||||
|
||||
const handleNavigate = useCallback(
|
||||
async (targetIndex: number) => {
|
||||
|
||||
@ -4,75 +4,60 @@ import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
import MyBox from '@fastgpt/web/components/common/MyBox';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
|
||||
import { useChatStore } from '@/web/core/chat/context/useChatStore';
|
||||
import QuoteItem from './QuoteItem';
|
||||
import { useMemo } from 'react';
|
||||
import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
|
||||
import { formatScore } from '@/components/core/dataset/QuoteItem';
|
||||
import { metadataType } from '@/web/core/chat/context/chatItemContext';
|
||||
import { GetAllQuoteDataProps } from '@/web/core/chat/context/chatItemContext';
|
||||
import { getQuoteDataList } from '@/web/core/chat/api';
|
||||
|
||||
const QuoteReader = ({
|
||||
rawSearch,
|
||||
metadata,
|
||||
chatTime,
|
||||
onClose
|
||||
}: {
|
||||
rawSearch: SearchDataResponseItemType[];
|
||||
metadata: metadataType;
|
||||
chatTime: Date;
|
||||
metadata: GetAllQuoteDataProps;
|
||||
onClose: () => void;
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
|
||||
const { chatId, appId, outLinkAuthData } = useChatStore();
|
||||
|
||||
const { data, loading } = useRequest2(
|
||||
const { data: quoteList, loading } = useRequest2(
|
||||
async () =>
|
||||
await getQuoteDataList({
|
||||
datasetDataIdList: rawSearch.map((item) => item.id),
|
||||
chatTime,
|
||||
collectionIdList: metadata.collectionIdList,
|
||||
chatItemId: metadata.chatItemId,
|
||||
appId,
|
||||
chatId,
|
||||
...outLinkAuthData
|
||||
chatItemDataId: metadata.chatItemDataId,
|
||||
appId: metadata.appId,
|
||||
chatId: metadata.chatId,
|
||||
...metadata.outLinkAuthData
|
||||
}),
|
||||
{
|
||||
manual: false
|
||||
}
|
||||
);
|
||||
|
||||
const filterResults = useMemo(() => {
|
||||
if (!metadata.collectionId) {
|
||||
return rawSearch;
|
||||
}
|
||||
|
||||
return rawSearch.filter(
|
||||
(item) => item.collectionId === metadata.collectionId && item.sourceId === metadata.sourceId
|
||||
);
|
||||
}, [metadata, rawSearch]);
|
||||
|
||||
const formatedDataList = useMemo(() => {
|
||||
return filterResults
|
||||
.map((item) => {
|
||||
const currentFilterItem = data?.quoteList.find((res) => res._id === item.id);
|
||||
return rawSearch
|
||||
.map((searchItem) => {
|
||||
const dataItem = quoteList?.find((item) => item._id === searchItem.id);
|
||||
|
||||
return {
|
||||
...item,
|
||||
q: currentFilterItem?.q || '',
|
||||
a: currentFilterItem?.a || '',
|
||||
score: formatScore(item.score),
|
||||
id: searchItem.id,
|
||||
q: dataItem?.q || 'Can not find Data',
|
||||
a: dataItem?.a || '',
|
||||
score: formatScore(searchItem.score),
|
||||
sourceName: searchItem?.sourceName || '',
|
||||
icon: getSourceNameIcon({
|
||||
sourceId: item.sourceId,
|
||||
sourceName: item.sourceName
|
||||
sourceId: searchItem.sourceId,
|
||||
sourceName: searchItem.sourceName
|
||||
})
|
||||
};
|
||||
})
|
||||
.sort((a, b) => {
|
||||
return (b.score.primaryScore?.value || 0) - (a.score.primaryScore?.value || 0);
|
||||
});
|
||||
}, [data?.quoteList, filterResults]);
|
||||
}, [quoteList, rawSearch]);
|
||||
|
||||
return (
|
||||
<Flex flexDirection={'column'} h={'full'}>
|
||||
@ -86,18 +71,7 @@ const QuoteReader = ({
|
||||
>
|
||||
<Box flex={1} py={4}>
|
||||
<Flex gap={2} mr={2} mb={1}>
|
||||
<MyIcon
|
||||
name={
|
||||
metadata.sourceId && metadata.sourceName
|
||||
? (getSourceNameIcon({
|
||||
sourceId: metadata.sourceId,
|
||||
sourceName: metadata.sourceName
|
||||
}) as any)
|
||||
: 'core/chat/quoteFill'
|
||||
}
|
||||
w={['1rem', '1.25rem']}
|
||||
color={'primary.600'}
|
||||
/>
|
||||
<MyIcon name={'core/chat/quoteFill'} w={['1rem', '1.25rem']} color={'primary.600'} />
|
||||
<Box
|
||||
maxW={['200px', '300px']}
|
||||
className={'textEllipsis'}
|
||||
@ -105,9 +79,7 @@ const QuoteReader = ({
|
||||
color={'myGray.900'}
|
||||
fontWeight={'medium'}
|
||||
>
|
||||
{metadata.sourceName
|
||||
? metadata.sourceName
|
||||
: t('common:core.chat.Quote Amount', { amount: rawSearch.length })}
|
||||
{t('common:core.chat.Quote Amount', { amount: rawSearch.length })}
|
||||
</Box>
|
||||
</Flex>
|
||||
<Box fontSize={'mini'} color={'myGray.500'}>
|
||||
|
||||
@ -1,39 +1,28 @@
|
||||
import React from 'react';
|
||||
import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { ChatItemContext, metadataType } from '@/web/core/chat/context/chatItemContext';
|
||||
import { ChatItemContext, GetQuoteProps } from '@/web/core/chat/context/chatItemContext';
|
||||
import CollectionQuoteReader from './CollectionQuoteReader';
|
||||
import QuoteReader from './QuoteReader';
|
||||
|
||||
const ChatQuoteList = ({
|
||||
chatTime,
|
||||
rawSearch = [],
|
||||
metadata,
|
||||
onClose
|
||||
}: {
|
||||
chatTime: Date;
|
||||
rawSearch: SearchDataResponseItemType[];
|
||||
metadata: metadataType;
|
||||
metadata: GetQuoteProps;
|
||||
onClose: () => void;
|
||||
}) => {
|
||||
const isShowReadRawSource = useContextSelector(ChatItemContext, (v) => v.isShowReadRawSource);
|
||||
|
||||
return (
|
||||
<>
|
||||
{metadata.collectionId && isShowReadRawSource ? (
|
||||
<CollectionQuoteReader
|
||||
rawSearch={rawSearch}
|
||||
metadata={metadata}
|
||||
chatTime={chatTime}
|
||||
onClose={onClose}
|
||||
/>
|
||||
) : (
|
||||
<QuoteReader
|
||||
rawSearch={rawSearch}
|
||||
metadata={metadata}
|
||||
chatTime={chatTime}
|
||||
onClose={onClose}
|
||||
/>
|
||||
{'collectionId' in metadata && isShowReadRawSource && (
|
||||
<CollectionQuoteReader rawSearch={rawSearch} metadata={metadata} onClose={onClose} />
|
||||
)}
|
||||
{'collectionIdList' in metadata && (
|
||||
<QuoteReader rawSearch={rawSearch} metadata={metadata} onClose={onClose} />
|
||||
)}
|
||||
</>
|
||||
);
|
||||
|
||||
@ -5,25 +5,25 @@ import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
|
||||
import { ApiRequestProps } from '@fastgpt/service/type/next';
|
||||
import { LinkedListResponse, LinkedPaginationProps } from '@fastgpt/web/common/fetch/type';
|
||||
import { FilterQuery, Types } from 'mongoose';
|
||||
import { dataFieldSelector, processChatTimeFilter } from './getQuote';
|
||||
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
|
||||
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
|
||||
import { quoteDataFieldSelector, QuoteDataItemType } from '@/service/core/chat/constants';
|
||||
import { processChatTimeFilter } from '@/service/core/chat/utils';
|
||||
import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
|
||||
|
||||
export type GetCollectionQuoteProps = LinkedPaginationProps & {
|
||||
chatTime: Date;
|
||||
chatId: string;
|
||||
chatItemDataId: string;
|
||||
|
||||
isInitialLoad: boolean;
|
||||
collectionId: string;
|
||||
chatItemId: string;
|
||||
|
||||
appId: string;
|
||||
chatId: string;
|
||||
shareId?: string;
|
||||
outLinkUid?: string;
|
||||
teamId?: string;
|
||||
teamToken?: string;
|
||||
};
|
||||
|
||||
export type GetCollectionQuoteRes = LinkedListResponse<DatasetDataSchemaType>;
|
||||
export type GetCollectionQuoteRes = LinkedListResponse<QuoteDataItemType>;
|
||||
|
||||
type BaseMatchType = FilterQuery<DatasetDataSchemaType>;
|
||||
|
||||
@ -37,11 +37,10 @@ async function handler(
|
||||
prevIndex,
|
||||
nextId,
|
||||
nextIndex,
|
||||
chatTime,
|
||||
isInitialLoad,
|
||||
|
||||
collectionId,
|
||||
chatItemId,
|
||||
chatItemDataId,
|
||||
appId,
|
||||
chatId,
|
||||
shareId,
|
||||
@ -53,61 +52,50 @@ async function handler(
|
||||
|
||||
const limitedPageSize = Math.min(pageSize, 30);
|
||||
|
||||
try {
|
||||
await authDatasetCollection({
|
||||
const [{ chat }, { chatItem }] = await Promise.all([
|
||||
authChatCrud({
|
||||
req,
|
||||
authToken: true,
|
||||
authApiKey: true,
|
||||
collectionId: req.body.collectionId,
|
||||
per: ReadPermissionVal
|
||||
});
|
||||
} catch (error) {
|
||||
await Promise.all([
|
||||
authChatCrud({
|
||||
req,
|
||||
authToken: true,
|
||||
appId,
|
||||
chatId,
|
||||
shareId,
|
||||
outLinkUid,
|
||||
teamId,
|
||||
teamToken
|
||||
}),
|
||||
authCollectionInChat({ appId, chatId, chatItemId, collectionId })
|
||||
]);
|
||||
}
|
||||
appId,
|
||||
chatId,
|
||||
shareId,
|
||||
outLinkUid,
|
||||
teamId,
|
||||
teamToken
|
||||
}),
|
||||
authCollectionInChat({ appId, chatId, chatItemDataId, collectionIds: [collectionId] })
|
||||
]);
|
||||
if (!chat) return Promise.reject(ChatErrEnum.unAuthChat);
|
||||
|
||||
const baseMatch: BaseMatchType = {
|
||||
collectionId,
|
||||
$or: [
|
||||
{ updateTime: { $lt: new Date(chatTime) } },
|
||||
{ history: { $elemMatch: { updateTime: { $lt: new Date(chatTime) } } } }
|
||||
{ updateTime: { $lt: new Date(chatItem.time) } },
|
||||
{ history: { $elemMatch: { updateTime: { $lt: new Date(chatItem.time) } } } }
|
||||
]
|
||||
};
|
||||
|
||||
if (initialId && initialIndex !== undefined) {
|
||||
return await handleInitialLoad(
|
||||
return await handleInitialLoad({
|
||||
initialId,
|
||||
initialIndex,
|
||||
limitedPageSize,
|
||||
chatTime,
|
||||
chatItemId,
|
||||
pageSize: limitedPageSize,
|
||||
chatTime: chatItem.time,
|
||||
isInitialLoad,
|
||||
baseMatch
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
if ((prevId && prevIndex !== undefined) || (nextId && nextIndex !== undefined)) {
|
||||
return await handlePaginatedLoad(
|
||||
return await handlePaginatedLoad({
|
||||
prevId,
|
||||
prevIndex,
|
||||
nextId,
|
||||
nextIndex,
|
||||
limitedPageSize,
|
||||
chatTime,
|
||||
chatItemId,
|
||||
pageSize: limitedPageSize,
|
||||
chatTime: chatItem.time,
|
||||
baseMatch
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
return { list: [], hasMorePrev: false, hasMoreNext: false };
|
||||
@ -115,38 +103,39 @@ async function handler(
|
||||
|
||||
export default NextAPI(handler);
|
||||
|
||||
async function handleInitialLoad(
|
||||
initialId: string,
|
||||
initialIndex: number,
|
||||
pageSize: number,
|
||||
chatTime: Date,
|
||||
chatItemId: string,
|
||||
isInitialLoad: boolean,
|
||||
baseMatch: BaseMatchType
|
||||
): Promise<GetCollectionQuoteRes> {
|
||||
async function handleInitialLoad({
|
||||
initialId,
|
||||
initialIndex,
|
||||
pageSize,
|
||||
chatTime,
|
||||
isInitialLoad,
|
||||
baseMatch
|
||||
}: {
|
||||
initialId: string;
|
||||
initialIndex: number;
|
||||
pageSize: number;
|
||||
chatTime: Date;
|
||||
isInitialLoad: boolean;
|
||||
baseMatch: BaseMatchType;
|
||||
}): Promise<GetCollectionQuoteRes> {
|
||||
const centerNode = await MongoDatasetData.findOne(
|
||||
{
|
||||
_id: new Types.ObjectId(initialId)
|
||||
},
|
||||
dataFieldSelector
|
||||
quoteDataFieldSelector
|
||||
).lean();
|
||||
|
||||
if (!centerNode) {
|
||||
if (isInitialLoad) {
|
||||
const list = await MongoDatasetData.find(baseMatch, dataFieldSelector)
|
||||
const list = await MongoDatasetData.find(baseMatch, quoteDataFieldSelector)
|
||||
.sort({ chunkIndex: 1, _id: -1 })
|
||||
.limit(pageSize)
|
||||
.lean();
|
||||
|
||||
const listRes = list.map((item, index) => ({
|
||||
...item,
|
||||
index: item.chunkIndex
|
||||
}));
|
||||
|
||||
const hasMoreNext = list.length === pageSize;
|
||||
|
||||
return {
|
||||
list: listRes,
|
||||
list: processChatTimeFilter(list, chatTime),
|
||||
hasMorePrev: false,
|
||||
hasMoreNext
|
||||
};
|
||||
@ -173,28 +162,30 @@ async function handleInitialLoad(
|
||||
|
||||
const resultList = [...prevList, centerNode, ...nextList];
|
||||
|
||||
const list = processChatTimeFilter(resultList, chatTime);
|
||||
|
||||
return {
|
||||
list: list.map((item) => ({
|
||||
...item,
|
||||
index: item.chunkIndex
|
||||
})),
|
||||
list: processChatTimeFilter(resultList, chatTime),
|
||||
hasMorePrev,
|
||||
hasMoreNext
|
||||
};
|
||||
}
|
||||
|
||||
async function handlePaginatedLoad(
|
||||
prevId: string | undefined,
|
||||
prevIndex: number | undefined,
|
||||
nextId: string | undefined,
|
||||
nextIndex: number | undefined,
|
||||
pageSize: number,
|
||||
chatTime: Date,
|
||||
chatItemId: string,
|
||||
baseMatch: BaseMatchType
|
||||
): Promise<GetCollectionQuoteRes> {
|
||||
async function handlePaginatedLoad({
|
||||
prevId,
|
||||
prevIndex,
|
||||
nextId,
|
||||
nextIndex,
|
||||
pageSize,
|
||||
chatTime,
|
||||
baseMatch
|
||||
}: {
|
||||
prevId: string | undefined;
|
||||
prevIndex: number | undefined;
|
||||
nextId: string | undefined;
|
||||
nextIndex: number | undefined;
|
||||
pageSize: number;
|
||||
chatTime: Date;
|
||||
baseMatch: BaseMatchType;
|
||||
}): Promise<GetCollectionQuoteRes> {
|
||||
const { list, hasMore } =
|
||||
prevId && prevIndex !== undefined
|
||||
? await getPrevNodes(prevId, prevIndex, pageSize, baseMatch)
|
||||
@ -203,10 +194,7 @@ async function handlePaginatedLoad(
|
||||
const processedList = processChatTimeFilter(list, chatTime);
|
||||
|
||||
return {
|
||||
list: processedList.map((item) => ({
|
||||
...item,
|
||||
index: item.chunkIndex
|
||||
})),
|
||||
list: processedList,
|
||||
hasMorePrev: !!prevId && hasMore,
|
||||
hasMoreNext: !!nextId && hasMore
|
||||
};
|
||||
@ -217,7 +205,10 @@ async function getPrevNodes(
|
||||
initialIndex: number,
|
||||
limit: number,
|
||||
baseMatch: BaseMatchType
|
||||
) {
|
||||
): Promise<{
|
||||
list: DatasetDataSchemaType[];
|
||||
hasMore: boolean;
|
||||
}> {
|
||||
const match: BaseMatchType = {
|
||||
...baseMatch,
|
||||
$or: [
|
||||
@ -226,7 +217,7 @@ async function getPrevNodes(
|
||||
]
|
||||
};
|
||||
|
||||
const list = await MongoDatasetData.find(match, dataFieldSelector)
|
||||
const list = await MongoDatasetData.find(match, quoteDataFieldSelector)
|
||||
.sort({ chunkIndex: -1, _id: 1 })
|
||||
.limit(limit)
|
||||
.lean();
|
||||
@ -242,7 +233,10 @@ async function getNextNodes(
|
||||
initialIndex: number,
|
||||
limit: number,
|
||||
baseMatch: BaseMatchType
|
||||
) {
|
||||
): Promise<{
|
||||
list: DatasetDataSchemaType[];
|
||||
hasMore: boolean;
|
||||
}> {
|
||||
const match: BaseMatchType = {
|
||||
...baseMatch,
|
||||
$or: [
|
||||
@ -251,7 +245,7 @@ async function getNextNodes(
|
||||
]
|
||||
};
|
||||
|
||||
const list = await MongoDatasetData.find(match, dataFieldSelector)
|
||||
const list = await MongoDatasetData.find(match, quoteDataFieldSelector)
|
||||
.sort({ chunkIndex: 1, _id: -1 })
|
||||
.limit(limit)
|
||||
.lean();
|
||||
|
||||
@ -1,100 +1,64 @@
|
||||
import { NextAPI } from '@/service/middleware/entry';
|
||||
import { authChatCrud, authCollectionInChat } from '@/service/support/permission/auth/chat';
|
||||
import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type';
|
||||
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
|
||||
import { ApiRequestProps } from '@fastgpt/service/type/next';
|
||||
import { quoteDataFieldSelector, QuoteDataItemType } from '@/service/core/chat/constants';
|
||||
import { processChatTimeFilter } from '@/service/core/chat/utils';
|
||||
import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
|
||||
|
||||
export type GetQuoteDataProps = {
|
||||
datasetDataIdList: string[];
|
||||
chatTime: Date;
|
||||
|
||||
collectionIdList: string[];
|
||||
chatItemId: string;
|
||||
appId: string;
|
||||
chatId: string;
|
||||
chatItemDataId: string;
|
||||
appId: string;
|
||||
shareId?: string;
|
||||
outLinkUid?: string;
|
||||
teamId?: string;
|
||||
teamToken?: string;
|
||||
};
|
||||
|
||||
export type GetQuoteDataRes = {
|
||||
quoteList: DatasetDataSchemaType[];
|
||||
};
|
||||
|
||||
export const dataFieldSelector =
|
||||
'_id datasetId collectionId q a chunkIndex history updateTime currentChatItemId prevId';
|
||||
export type GetQuoteDataRes = QuoteDataItemType[];
|
||||
|
||||
async function handler(req: ApiRequestProps<GetQuoteDataProps>): Promise<GetQuoteDataRes> {
|
||||
const {
|
||||
datasetDataIdList,
|
||||
chatTime,
|
||||
appId,
|
||||
chatId,
|
||||
chatItemDataId,
|
||||
|
||||
shareId,
|
||||
outLinkUid,
|
||||
teamId,
|
||||
teamToken,
|
||||
|
||||
collectionIdList,
|
||||
chatItemId,
|
||||
chatId,
|
||||
appId,
|
||||
shareId,
|
||||
outLinkUid,
|
||||
teamId,
|
||||
teamToken
|
||||
datasetDataIdList
|
||||
} = req.body;
|
||||
|
||||
await authChatCrud({
|
||||
req,
|
||||
authToken: true,
|
||||
appId,
|
||||
chatId,
|
||||
shareId,
|
||||
outLinkUid,
|
||||
teamId,
|
||||
teamToken
|
||||
});
|
||||
|
||||
await Promise.all(
|
||||
collectionIdList.map(async (collectionId) => {
|
||||
await authCollectionInChat({ appId, chatId, chatItemId, collectionId });
|
||||
})
|
||||
);
|
||||
const [chat, { chatItem }] = await Promise.all([
|
||||
authChatCrud({
|
||||
req,
|
||||
authToken: true,
|
||||
appId,
|
||||
chatId,
|
||||
shareId,
|
||||
outLinkUid,
|
||||
teamId,
|
||||
teamToken
|
||||
}),
|
||||
authCollectionInChat({ appId, chatId, chatItemDataId, collectionIds: collectionIdList })
|
||||
]);
|
||||
if (!chat) return Promise.reject(ChatErrEnum.unAuthChat);
|
||||
|
||||
const list = await MongoDatasetData.find(
|
||||
{ _id: { $in: datasetDataIdList } },
|
||||
dataFieldSelector
|
||||
{ _id: { $in: datasetDataIdList }, collectionId: { $in: collectionIdList } },
|
||||
quoteDataFieldSelector
|
||||
).lean();
|
||||
|
||||
const quoteList = processChatTimeFilter(list, chatTime);
|
||||
const quoteList = processChatTimeFilter(list, chatItem.time);
|
||||
|
||||
return {
|
||||
quoteList
|
||||
};
|
||||
return quoteList;
|
||||
}
|
||||
|
||||
export default NextAPI(handler);
|
||||
|
||||
export function processChatTimeFilter(list: DatasetDataSchemaType[], chatTime: Date) {
|
||||
return list.map((item) => {
|
||||
if (!item.history) return item;
|
||||
|
||||
const { history, ...rest } = item;
|
||||
const formatedChatTime = new Date(chatTime);
|
||||
|
||||
if (item.updateTime <= formatedChatTime) {
|
||||
return rest;
|
||||
}
|
||||
|
||||
const latestHistoryIndex = history.findIndex(
|
||||
(historyItem: any) => historyItem.updateTime <= formatedChatTime
|
||||
);
|
||||
|
||||
if (latestHistoryIndex === -1) return rest;
|
||||
|
||||
const latestHistory = history[latestHistoryIndex];
|
||||
|
||||
return {
|
||||
...rest,
|
||||
q: latestHistory?.q || item.q,
|
||||
a: latestHistory?.a || item.a,
|
||||
updated: true
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
@ -2,11 +2,7 @@ import type { NextApiRequest } from 'next';
|
||||
import type { ApiDatasetCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
|
||||
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
|
||||
import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller';
|
||||
import {
|
||||
TrainingModeEnum,
|
||||
DatasetCollectionTypeEnum,
|
||||
DatasetCollectionDataProcessModeEnum
|
||||
} from '@fastgpt/global/core/dataset/constants';
|
||||
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
|
||||
|
||||
import { NextAPI } from '@/service/middleware/entry';
|
||||
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
|
||||
@ -16,7 +12,8 @@ import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection
|
||||
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
|
||||
|
||||
async function handler(req: NextApiRequest): CreateCollectionResponse {
|
||||
const { name, apiFileId, ...body } = req.body as ApiDatasetCreateDatasetCollectionParams;
|
||||
const { name, apiFileId, customPdfParse, ...body } =
|
||||
req.body as ApiDatasetCreateDatasetCollectionParams;
|
||||
|
||||
const { teamId, tmbId, dataset } = await authDataset({
|
||||
req,
|
||||
@ -50,7 +47,8 @@ async function handler(req: NextApiRequest): CreateCollectionResponse {
|
||||
yuqueServer,
|
||||
apiFileId,
|
||||
teamId,
|
||||
tmbId
|
||||
tmbId,
|
||||
customPdfParse
|
||||
});
|
||||
|
||||
const { collectionId, insertResults } = await createCollectionAndInsertData({
|
||||
@ -62,11 +60,12 @@ async function handler(req: NextApiRequest): CreateCollectionResponse {
|
||||
teamId,
|
||||
tmbId,
|
||||
type: DatasetCollectionTypeEnum.apiFile,
|
||||
name: name,
|
||||
name,
|
||||
apiFileId,
|
||||
metadata: {
|
||||
relatedImgId: apiFileId
|
||||
}
|
||||
},
|
||||
customPdfParse
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@ -1,9 +1,13 @@
|
||||
import { NextAPI } from '@/service/middleware/entry';
|
||||
import { authChatCrud, authCollectionInChat } from '@/service/support/permission/auth/chat';
|
||||
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
|
||||
import { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat';
|
||||
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
|
||||
import { useIPFrequencyLimit } from '@fastgpt/service/common/middle/reqFrequencyLimit';
|
||||
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
|
||||
import { responseWriteController } from '@fastgpt/service/common/response';
|
||||
import { addLog } from '@fastgpt/service/common/system/log';
|
||||
import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller';
|
||||
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
|
||||
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
|
||||
import { ApiRequestProps } from '@fastgpt/service/type/next';
|
||||
@ -11,21 +15,69 @@ import { NextApiResponse } from 'next';
|
||||
|
||||
export type ExportCollectionBody = {
|
||||
collectionId: string;
|
||||
|
||||
appId?: string;
|
||||
chatId?: string;
|
||||
chatItemDataId?: string;
|
||||
chatTime: Date;
|
||||
};
|
||||
} & OutLinkChatAuthProps;
|
||||
|
||||
async function handler(req: ApiRequestProps<ExportCollectionBody, {}>, res: NextApiResponse) {
|
||||
let { collectionId, chatTime } = req.body;
|
||||
|
||||
const { teamId, collection } = await authDatasetCollection({
|
||||
req,
|
||||
authToken: true,
|
||||
const {
|
||||
collectionId,
|
||||
per: ReadPermissionVal
|
||||
});
|
||||
appId,
|
||||
chatId,
|
||||
chatItemDataId,
|
||||
shareId,
|
||||
outLinkUid,
|
||||
teamId,
|
||||
teamToken,
|
||||
chatTime
|
||||
} = req.body;
|
||||
|
||||
const { collection, teamId: userTeamId } = await (async () => {
|
||||
if (!appId || !chatId || !chatItemDataId) {
|
||||
return authDatasetCollection({
|
||||
req,
|
||||
authToken: true,
|
||||
authApiKey: true,
|
||||
collectionId: req.body.collectionId,
|
||||
per: ReadPermissionVal
|
||||
});
|
||||
}
|
||||
|
||||
/*
|
||||
1. auth chat read permission
|
||||
2. auth collection quote in chat
|
||||
3. auth outlink open show quote
|
||||
*/
|
||||
const [authRes, collection] = await Promise.all([
|
||||
authChatCrud({
|
||||
req,
|
||||
authToken: true,
|
||||
appId,
|
||||
chatId,
|
||||
shareId,
|
||||
outLinkUid,
|
||||
teamId,
|
||||
teamToken
|
||||
}),
|
||||
getCollectionWithDataset(collectionId),
|
||||
authCollectionInChat({ appId, chatId, chatItemDataId, collectionIds: [collectionId] })
|
||||
]);
|
||||
|
||||
if (!authRes.showRawSource) {
|
||||
return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
|
||||
}
|
||||
|
||||
return {
|
||||
...authRes,
|
||||
collection
|
||||
};
|
||||
})();
|
||||
|
||||
const where = {
|
||||
teamId,
|
||||
teamId: userTeamId,
|
||||
datasetId: collection.datasetId,
|
||||
collectionId,
|
||||
...(chatTime
|
||||
|
||||
@ -19,7 +19,7 @@ export type readCollectionSourceBody = {
|
||||
|
||||
appId?: string;
|
||||
chatId?: string;
|
||||
chatItemId?: string;
|
||||
chatItemDataId?: string;
|
||||
} & OutLinkChatAuthProps;
|
||||
|
||||
export type readCollectionSourceResponse = {
|
||||
@ -30,7 +30,7 @@ export type readCollectionSourceResponse = {
|
||||
async function handler(
|
||||
req: ApiRequestProps<readCollectionSourceBody, readCollectionSourceQuery>
|
||||
): Promise<readCollectionSourceResponse> {
|
||||
const { collectionId, appId, chatId, chatItemId, shareId, outLinkUid, teamId, teamToken } =
|
||||
const { collectionId, appId, chatId, chatItemDataId, shareId, outLinkUid, teamId, teamToken } =
|
||||
req.body;
|
||||
|
||||
const {
|
||||
@ -39,7 +39,7 @@ async function handler(
|
||||
tmbId: uid,
|
||||
authType
|
||||
} = await (async () => {
|
||||
if (!appId || !chatId || !chatItemId) {
|
||||
if (!appId || !chatId || !chatItemDataId) {
|
||||
return authDatasetCollection({
|
||||
req,
|
||||
authToken: true,
|
||||
@ -66,7 +66,7 @@ async function handler(
|
||||
teamToken
|
||||
}),
|
||||
getCollectionWithDataset(collectionId),
|
||||
authCollectionInChat({ appId, chatId, chatItemId, collectionId })
|
||||
authCollectionInChat({ appId, chatId, chatItemDataId, collectionIds: [collectionId] })
|
||||
]);
|
||||
|
||||
if (!authRes.showRawSource) {
|
||||
|
||||
@ -231,7 +231,6 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
|
||||
{quoteData && (
|
||||
<PageContainer flex={'1 0 0'} w={0} maxW={'560px'}>
|
||||
<ChatQuoteList
|
||||
chatTime={quoteData.chatTime}
|
||||
rawSearch={quoteData.rawSearch}
|
||||
metadata={quoteData.metadata}
|
||||
onClose={() => setQuoteData(undefined)}
|
||||
|
||||
@ -304,7 +304,6 @@ const OutLink = (props: Props) => {
|
||||
{quoteData && (
|
||||
<PageContainer flex={'1 0 0'} w={0} maxW={'560px'}>
|
||||
<ChatQuoteList
|
||||
chatTime={quoteData.chatTime}
|
||||
rawSearch={quoteData.rawSearch}
|
||||
metadata={quoteData.metadata}
|
||||
onClose={() => setQuoteData(undefined)}
|
||||
|
||||
@ -245,7 +245,6 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
|
||||
{quoteData && (
|
||||
<PageContainer flex={'1 0 0'} w={0} maxW={'560px'}>
|
||||
<ChatQuoteList
|
||||
chatTime={quoteData.chatTime}
|
||||
rawSearch={quoteData.rawSearch}
|
||||
metadata={quoteData.metadata}
|
||||
onClose={() => setQuoteData(undefined)}
|
||||
|
||||
13
projects/app/src/service/core/chat/constants.ts
Normal file
13
projects/app/src/service/core/chat/constants.ts
Normal file
@ -0,0 +1,13 @@
|
||||
import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type';
|
||||
|
||||
export const quoteDataFieldSelector = '_id q a history updateTime chunkIndex';
|
||||
|
||||
export type QuoteDataItemType = {
|
||||
_id: string;
|
||||
q: DatasetDataSchemaType['q'];
|
||||
a: DatasetDataSchemaType['a'];
|
||||
history?: DatasetDataSchemaType['history'];
|
||||
updateTime: DatasetDataSchemaType['updateTime'];
|
||||
index: DatasetDataSchemaType['chunkIndex'];
|
||||
updated?: boolean;
|
||||
};
|
||||
46
projects/app/src/service/core/chat/utils.ts
Normal file
46
projects/app/src/service/core/chat/utils.ts
Normal file
@ -0,0 +1,46 @@
|
||||
import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type';
|
||||
import { QuoteDataItemType } from './constants';
|
||||
|
||||
// 获取对话时间时,引用的内容
|
||||
export function processChatTimeFilter(
|
||||
dataList: DatasetDataSchemaType[],
|
||||
chatTime: Date
|
||||
): QuoteDataItemType[] {
|
||||
return dataList.map((item) => {
|
||||
const defaultItem = {
|
||||
_id: item._id,
|
||||
q: item.q,
|
||||
a: item.a,
|
||||
updateTime: item.updateTime,
|
||||
index: item.chunkIndex
|
||||
};
|
||||
|
||||
if (!item.history) return defaultItem;
|
||||
|
||||
const history = item.history;
|
||||
const formatedChatTime = new Date(chatTime);
|
||||
|
||||
if (item.updateTime <= formatedChatTime) {
|
||||
return defaultItem;
|
||||
}
|
||||
|
||||
const latestHistoryIndex = history.findIndex(
|
||||
(historyItem) => historyItem.updateTime <= formatedChatTime
|
||||
);
|
||||
|
||||
if (latestHistoryIndex === -1) {
|
||||
return defaultItem;
|
||||
}
|
||||
|
||||
const latestHistory = history[latestHistoryIndex];
|
||||
|
||||
return {
|
||||
_id: item._id,
|
||||
q: latestHistory.q,
|
||||
a: latestHistory.a,
|
||||
updateTime: latestHistory.updateTime,
|
||||
index: item.chunkIndex,
|
||||
updated: true
|
||||
};
|
||||
});
|
||||
}
|
||||
@ -25,6 +25,14 @@ const defaultResponseShow = {
|
||||
showNodeStatus: true,
|
||||
showRawSource: true
|
||||
};
|
||||
type AuthChatCommonProps = {
|
||||
appId: string;
|
||||
shareId?: string;
|
||||
outLinkUid?: string;
|
||||
teamId?: string;
|
||||
teamToken?: string;
|
||||
};
|
||||
|
||||
export async function authChatCrud({
|
||||
appId,
|
||||
chatId,
|
||||
@ -35,14 +43,10 @@ export async function authChatCrud({
|
||||
teamId: spaceTeamId,
|
||||
teamToken,
|
||||
...props
|
||||
}: AuthModeType & {
|
||||
appId: string;
|
||||
chatId?: string;
|
||||
shareId?: string;
|
||||
outLinkUid?: string;
|
||||
teamId?: string;
|
||||
teamToken?: string;
|
||||
}): Promise<{
|
||||
}: AuthModeType &
|
||||
AuthChatCommonProps & {
|
||||
chatId?: string;
|
||||
}): Promise<{
|
||||
teamId: string;
|
||||
tmbId: string;
|
||||
uid: string;
|
||||
@ -188,27 +192,29 @@ export async function authChatCrud({
|
||||
}
|
||||
|
||||
export const authCollectionInChat = async ({
|
||||
collectionId,
|
||||
collectionIds,
|
||||
appId,
|
||||
chatId,
|
||||
chatItemId
|
||||
chatItemDataId
|
||||
}: {
|
||||
collectionId: string;
|
||||
collectionIds: string[];
|
||||
appId: string;
|
||||
chatId: string;
|
||||
chatItemId: string;
|
||||
}) => {
|
||||
chatItemDataId: string;
|
||||
}): Promise<{
|
||||
chatItem: { time: Date; responseData?: ChatHistoryItemResType[] };
|
||||
}> => {
|
||||
try {
|
||||
const chatItem = (await MongoChatItem.findOne(
|
||||
{
|
||||
appId,
|
||||
chatId,
|
||||
dataId: chatItemId
|
||||
dataId: chatItemDataId
|
||||
},
|
||||
'responseData'
|
||||
).lean()) as AIChatItemType;
|
||||
'responseData time'
|
||||
).lean()) as { time: Date; responseData?: ChatHistoryItemResType[] };
|
||||
|
||||
if (!chatItem) return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
|
||||
if (!chatItem) return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
|
||||
|
||||
// 找 responseData 里,是否有该文档 id
|
||||
const responseData = chatItem.responseData || [];
|
||||
@ -224,15 +230,16 @@ export const authCollectionInChat = async ({
|
||||
})
|
||||
.flat() || [];
|
||||
|
||||
if (
|
||||
flatResData.some((item) => {
|
||||
if (item.quoteList) {
|
||||
return item.quoteList.some((quote) => quote.collectionId === collectionId);
|
||||
}
|
||||
return false;
|
||||
})
|
||||
) {
|
||||
return true;
|
||||
const quoteListSet = new Set(
|
||||
flatResData
|
||||
.map((item) => item.quoteList?.map((quote) => String(quote.collectionId)) || [])
|
||||
.flat()
|
||||
);
|
||||
|
||||
if (collectionIds.every((id) => quoteListSet.has(id))) {
|
||||
return {
|
||||
chatItem
|
||||
};
|
||||
}
|
||||
} catch (error) {}
|
||||
return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
|
||||
|
||||
@ -1,11 +1,6 @@
|
||||
import { GET, POST, DELETE, PUT } from '@/web/common/api/request';
|
||||
import type {
|
||||
ChatHistoryItemType,
|
||||
ChatHistoryItemResType,
|
||||
ChatSiteItemType,
|
||||
ChatItemType
|
||||
} from '@fastgpt/global/core/chat/type.d';
|
||||
import { getResDataQuery } from '@/pages/api/core/chat/getResData';
|
||||
import type { ChatHistoryItemType, ChatHistoryItemResType } from '@fastgpt/global/core/chat/type.d';
|
||||
import type { getResDataQuery } from '@/pages/api/core/chat/getResData';
|
||||
import type {
|
||||
CloseCustomFeedbackParams,
|
||||
InitChatProps,
|
||||
@ -22,16 +17,16 @@ import type {
|
||||
DeleteChatItemProps,
|
||||
UpdateHistoryProps
|
||||
} from '@/global/core/chat/api.d';
|
||||
import { UpdateChatFeedbackProps } from '@fastgpt/global/core/chat/api';
|
||||
import { AuthTeamTagTokenProps } from '@fastgpt/global/support/user/team/tag';
|
||||
import { AppListItemType } from '@fastgpt/global/core/app/type';
|
||||
import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
|
||||
import type { UpdateChatFeedbackProps } from '@fastgpt/global/core/chat/api';
|
||||
import type { AuthTeamTagTokenProps } from '@fastgpt/global/support/user/team/tag';
|
||||
import type { AppListItemType } from '@fastgpt/global/core/app/type';
|
||||
import type { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
|
||||
import type {
|
||||
getPaginationRecordsBody,
|
||||
getPaginationRecordsResponse
|
||||
} from '@/pages/api/core/chat/getPaginationRecords';
|
||||
import { GetQuoteDataProps, GetQuoteDataRes } from '@/pages/api/core/chat/quote/getQuote';
|
||||
import {
|
||||
import type { GetQuoteDataProps, GetQuoteDataRes } from '@/pages/api/core/chat/quote/getQuote';
|
||||
import type {
|
||||
GetCollectionQuoteProps,
|
||||
GetCollectionQuoteRes
|
||||
} from '@/pages/api/core/chat/quote/getCollectionQuote';
|
||||
|
||||
@ -9,6 +9,7 @@ import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
|
||||
import { AppChatConfigType, VariableItemType } from '@fastgpt/global/core/app/type';
|
||||
import { FlowNodeInputItemType } from '@fastgpt/global/core/workflow/type/io';
|
||||
import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
|
||||
import { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat';
|
||||
|
||||
type ContextProps = {
|
||||
showRouteToAppDetail: boolean;
|
||||
@ -32,19 +33,27 @@ type ChatBoxDataType = {
|
||||
};
|
||||
};
|
||||
|
||||
export type metadataType = {
|
||||
export type GetQuoteDataBasicProps = {
|
||||
appId: string;
|
||||
chatId: string;
|
||||
chatItemDataId: string;
|
||||
outLinkAuthData?: OutLinkChatAuthProps;
|
||||
};
|
||||
// 获取单个集合引用
|
||||
export type GetCollectionQuoteDataProps = GetQuoteDataBasicProps & {
|
||||
collectionId: string;
|
||||
collectionIdList: string[];
|
||||
chatItemId: string;
|
||||
sourceId: string;
|
||||
sourceName: string;
|
||||
datasetId: string;
|
||||
};
|
||||
export type GetAllQuoteDataProps = GetQuoteDataBasicProps & {
|
||||
collectionIdList: string[];
|
||||
};
|
||||
export type GetQuoteProps = GetAllQuoteDataProps | GetCollectionQuoteDataProps;
|
||||
|
||||
export type QuoteDataType = {
|
||||
rawSearch: SearchDataResponseItemType[];
|
||||
metadata: metadataType;
|
||||
chatTime: Date;
|
||||
metadata: GetQuoteProps;
|
||||
};
|
||||
|
||||
type ChatItemContextType = {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user