feat: api dataset support pdf parse;fix: chunk reader auth (#4117)

* feat: api dataset support pdf parse

* fix: chunk reader auth

docSite/content/zh-cn/docs/development/upgrading/491.md

@@ -11,6 +11,7 @@ weight: 799
 
 1. 商业版支持单团队模式，更好的管理内部成员。
 2. 知识库分块阅读器。
+3. API 知识库支持 PDF 增强解析。
 
 ## ⚙️ 优化
 

packages/service/core/dataset/apiDataset/api.ts

@@ -111,11 +111,13 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
   const getFileContent = async ({
     teamId,
     tmbId,
-    apiFileId
+    apiFileId,
+    customPdfParse
   }: {
     teamId: string;
     tmbId: string;
     apiFileId: string;
+    customPdfParse?: boolean;
   }) => {
     const data = await request<APIFileContentResponse>(
       `/v1/file/content`,
@@ -133,7 +135,8 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
         teamId,
         tmbId,
         url: previewUrl,
-        relatedId: apiFileId
+        relatedId: apiFileId,
+        customPdfParse
       });
       return rawText;
     }

packages/service/core/dataset/read.ts

@@ -127,7 +127,8 @@ export const readApiServerFileContent = async ({
   yuqueServer,
   apiFileId,
   teamId,
-  tmbId
+  tmbId,
+  customPdfParse
 }: {
   apiServer?: APIFileServer;
   feishuServer?: FeishuServer;
@@ -135,9 +136,15 @@ export const readApiServerFileContent = async ({
   apiFileId: string;
   teamId: string;
   tmbId: string;
+  customPdfParse?: boolean;
 }) => {
   if (apiServer) {
-    return useApiDatasetRequest({ apiServer }).getFileContent({ teamId, tmbId, apiFileId });
+    return useApiDatasetRequest({ apiServer }).getFileContent({
+      teamId,
+      tmbId,
+      apiFileId,
+      customPdfParse
+    });
   }
 
   if (feishuServer || yuqueServer) {

projects/app/src/components/core/chat/ChatContainer/ChatBox/components/QuoteList.tsx

@@ -11,19 +11,17 @@ import { useChatStore } from '@/web/core/chat/context/useChatStore';
 import { getQuoteDataList } from '@/web/core/chat/api';
 
 const QuoteList = React.memo(function QuoteList({
-  chatItemId,
+  chatItemDataId = '',
-  rawSearch = [],
+  rawSearch = []
-  chatTime
 }: {
-  chatItemId?: string;
+  chatItemDataId?: string;
   rawSearch: SearchDataResponseItemType[];
-  chatTime: Date;
 }) {
   const theme = useTheme();
   const { chatId, appId, outLinkAuthData } = useChatStore();
 
   const RawSourceBoxProps = useContextSelector(ChatBoxContext, (v) => ({
-    chatItemId,
+    chatItemDataId,
     appId: v.appId,
     chatId: v.chatId,
     ...(v.outLinkAuthData || {})
@@ -34,13 +32,12 @@ const QuoteList = React.memo(function QuoteList({
     (v) => v.showRouteToDatasetDetail
   );
 
-  const { data } = useRequest2(
+  const { data: quoteList } = useRequest2(
     async () =>
       await getQuoteDataList({
         datasetDataIdList: rawSearch.map((item) => item.id),
-        chatTime,
         collectionIdList: [...new Set(rawSearch.map((item) => item.collectionId))],
-        chatItemId: chatItemId || '',
+        chatItemDataId,
         appId,
         chatId,
         ...outLinkAuthData
@@ -53,7 +50,7 @@ const QuoteList = React.memo(function QuoteList({
   const formatedDataList = useMemo(() => {
     return rawSearch
       .map((item) => {
-        const currentFilterItem = data?.quoteList.find((res) => res._id === item.id);
+        const currentFilterItem = quoteList?.find((res) => res._id === item.id);
 
         return {
           ...item,
@@ -66,7 +63,7 @@ const QuoteList = React.memo(function QuoteList({
         const bScore = formatScore(b.score);
         return (bScore.primaryScore?.value || 0) - (aScore.primaryScore?.value || 0);
       });
-  }, [data?.quoteList, rawSearch]);
+  }, [quoteList, rawSearch]);
 
   return (
     <>

projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ResponseTags.tsx

@@ -42,6 +42,8 @@ const ResponseTags = ({
   const [quoteFolded, setQuoteFolded] = useState<boolean>(true);
 
   const chatType = useContextSelector(ChatBoxContext, (v) => v.chatType);
+  const appId = useContextSelector(ChatBoxContext, (v) => v.appId);
+  const chatId = useContextSelector(ChatBoxContext, (v) => v.chatId);
 
   const setQuoteData = useContextSelector(ChatItemContext, (v) => v.setQuoteData);
 
@@ -156,17 +158,15 @@ const ResponseTags = ({
                       e.stopPropagation();
 
                       setQuoteData({
-                        chatTime,
                         rawSearch: quoteList,
                         metadata: {
+                          appId,
+                          chatId,
+                          chatItemDataId: dataId,
                           collectionId: item.collectionId,
-                          collectionIdList: [
-                            ...new Set(quoteList.map((item) => item.collectionId))
-                          ],
                           sourceId: item.sourceId || '',
                           sourceName: item.sourceName,
-                          datasetId: item.datasetId,
+                          datasetId: item.datasetId
-                          chatItemId: historyItem.dataId
                         }
                       });
                     }}
@@ -225,15 +225,12 @@ const ResponseTags = ({
                   e.stopPropagation();
 
                   setQuoteData({
-                    chatTime,
                     rawSearch: quoteList,
                     metadata: {
-                      collectionId: '',
+                      appId,
-                      collectionIdList: [...new Set(quoteList.map((item) => item.collectionId))],
+                      chatId,
-                      chatItemId: historyItem.dataId,
+                      chatItemDataId: dataId,
-                      sourceId: '',
+                      collectionIdList: [...new Set(quoteList.map((item) => item.collectionId))]
-                      sourceName: '',
-                      datasetId: ''
                     }
                   });
                 }}

projects/app/src/components/core/chat/components/WholeResponseModal.tsx

@@ -265,13 +265,7 @@ export const WholeResponseContent = ({
         {activeModule.quoteList && activeModule.quoteList.length > 0 && (
           <Row
             label={t('common:core.chat.response.module quoteList')}
-            rawDom={
+            rawDom={<QuoteList chatItemDataId={dataId} rawSearch={activeModule.quoteList} />}
-              <QuoteList
-                chatItemId={dataId}
-                chatTime={chatTime || new Date()}
-                rawSearch={activeModule.quoteList}
-              />
-            }
           />
         )}
       </>

projects/app/src/components/core/dataset/QuoteItem.tsx

@@ -243,7 +243,7 @@ const QuoteItem = ({
               color={'primary.500'}
               href={`/dataset/detail?datasetId=${quoteItem.datasetId}&currentTab=dataCard&collectionId=${quoteItem.collectionId}`}
             >
-              {t('common:core.dataset.Go Dataset')}
+              {t('chat:to_dataset')}
               <MyIcon name={'common/rightArrowLight'} w={'10px'} />
             </Link>
           )}

projects/app/src/components/core/dataset/RawSourceBox.tsx

@@ -23,7 +23,7 @@ const RawSourceBox = ({
   collectionId,
   appId,
   chatId,
-  chatItemId,
+  chatItemDataId,
   shareId,
   outLinkUid,
   teamId,
@@ -40,7 +40,7 @@ const RawSourceBox = ({
     collectionId,
     appId,
     chatId,
-    chatItemId,
+    chatItemDataId,
     shareId,
     outLinkUid,
     teamId,

projects/app/src/pageComponents/app/detail/Logs/DetailLogsModal.tsx

@@ -184,7 +184,6 @@ const DetailLogsModal = ({ appId, chatId, onClose }: Props) => {
               borderRadius={'md'}
             >
               <ChatQuoteList
-                chatTime={quoteData.chatTime}
                 rawSearch={quoteData.rawSearch}
                 metadata={quoteData.metadata}
                 onClose={() => setQuoteData(undefined)}

projects/app/src/pageComponents/app/detail/SimpleApp/ChatTest.tsx

@@ -88,7 +88,6 @@ const ChatTest = ({ appForm, setRenderEdit }: Props) => {
       {quoteData && (
         <Box flex={'1 0 0'} w={0} maxW={'560px'} {...cardStyles} boxShadow={'3'}>
           <ChatQuoteList
-            chatTime={quoteData.chatTime}
             rawSearch={quoteData.rawSearch}
             metadata={quoteData.metadata}
             onClose={() => setQuoteData(undefined)}

projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/ChatTest.tsx

@@ -163,7 +163,6 @@ const ChatTest = ({ isOpen, nodes = [], edges = [], onClose }: Props) => {
               borderRadius={'md'}
             >
               <ChatQuoteList
-                chatTime={quoteData.chatTime}
                 rawSearch={quoteData.rawSearch}
                 metadata={quoteData.metadata}
                 onClose={() => setQuoteData(undefined)}

projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx

@@ -1,4 +1,4 @@
-import { Box, Button, Flex, HStack } from '@chakra-ui/react';
+import { Box, Flex, HStack } from '@chakra-ui/react';
 import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
 import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
 import MyIcon from '@fastgpt/web/components/common/Icon';
@@ -8,41 +8,34 @@ import DownloadButton from './DownloadButton';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import { downloadFetch } from '@/web/common/system/utils';
 import { useCallback, useEffect, useMemo, useState } from 'react';
-import { getCollectionSource, getDatasetDataPermission } from '@/web/core/dataset/api';
+import { getDatasetDataPermission } from '@/web/core/dataset/api';
-import { useChatStore } from '@/web/core/chat/context/useChatStore';
-import { useToast } from '@fastgpt/web/hooks/useToast';
-import { getErrText } from '@fastgpt/global/common/error/utils';
 import ScoreTag from './ScoreTag';
 import { formatScore } from '@/components/core/dataset/QuoteItem';
 import NavButton from './NavButton';
 import { useLinkedScroll } from '@fastgpt/web/hooks/useLinkedScroll';
 import CollectionQuoteItem from './CollectionQuoteItem';
-import { DatasetDataListItemType } from '@/global/core/dataset/type';
+import { GetCollectionQuoteDataProps } from '@/web/core/chat/context/chatItemContext';
-import { metadataType } from '@/web/core/chat/context/chatItemContext';
 import { useUserStore } from '@/web/support/user/useUserStore';
 import { getCollectionQuote } from '@/web/core/chat/api';
 import MyIconButton from '@fastgpt/web/components/common/Icon/button';
 import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
 import MyBox from '@fastgpt/web/components/common/MyBox';
 import { getCollectionSourceAndOpen } from '@/web/core/dataset/hooks/readCollectionSource';
+import { QuoteDataItemType } from '@/service/core/chat/constants';
 
 const CollectionReader = ({
   rawSearch,
   metadata,
-  chatTime,
   onClose
 }: {
   rawSearch: SearchDataResponseItemType[];
-  metadata: metadataType;
+  metadata: GetCollectionQuoteDataProps;
-  chatTime: Date;
   onClose: () => void;
 }) => {
   const { t } = useTranslation();
   const router = useRouter();
-  const { toast } = useToast();
-  const { chatId, appId, outLinkAuthData } = useChatStore();
   const { userInfo } = useUserStore();
-  const { collectionId, datasetId, chatItemId, sourceId, sourceName } = metadata;
+  const { collectionId, datasetId, chatItemDataId, sourceId, sourceName } = metadata;
   const [quoteIndex, setQuoteIndex] = useState(0);
 
   const { data: permissionData, loading: isPermissionLoading } = useRequest2(
@@ -75,11 +68,10 @@ const CollectionReader = ({
     refreshDeps: [collectionId],
     params: {
       collectionId,
-      chatTime,
+      chatItemDataId,
-      chatItemId,
+      chatId: metadata.chatId,
-      chatId,
+      appId: metadata.appId,
-      appId,
+      ...metadata.outLinkAuthData
-      ...outLinkAuthData
     },
     initialId: currentQuoteItem?.id,
     initialIndex: currentQuoteItem?.chunkIndex,
@@ -87,11 +79,14 @@ const CollectionReader = ({
   });
 
   const loading = isLoading || isPermissionLoading;
-  const isDeleted = !datasetDataList.find((item) => item._id === currentQuoteItem?.id);
+  const isDeleted = useMemo(
+    () => !datasetDataList.find((item) => item._id === currentQuoteItem?.id),
+    [datasetDataList, currentQuoteItem?.id]
+  );
 
   const formatedDataList = useMemo(
     () =>
-      datasetDataList.map((item: DatasetDataListItemType) => {
+      datasetDataList.map((item: QuoteDataItemType) => {
         const isCurrentSelected = currentQuoteItem?.id === item._id;
         const quoteIndex = filterResults.findIndex((res) => res.id === item._id);
 
@@ -115,17 +110,12 @@ const CollectionReader = ({
       filename: 'data.csv',
       body: {
         collectionId: collectionId,
-        chatTime: chatTime,
+        chatItemDataId
-        chatItemId: chatItemId
       }
     });
   });
 
-  const handleRead = getCollectionSourceAndOpen({
+  const handleRead = getCollectionSourceAndOpen(metadata);
-    appId,
-    chatId,
-    ...metadata
-  });
 
   const handleNavigate = useCallback(
     async (targetIndex: number) => {

projects/app/src/pageComponents/chat/ChatQuoteList/QuoteReader.tsx

@@ -4,75 +4,60 @@ import MyIcon from '@fastgpt/web/components/common/Icon';
 import MyBox from '@fastgpt/web/components/common/MyBox';
 import { useTranslation } from 'react-i18next';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
-import { useChatStore } from '@/web/core/chat/context/useChatStore';
 import QuoteItem from './QuoteItem';
 import { useMemo } from 'react';
 import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
 import { formatScore } from '@/components/core/dataset/QuoteItem';
-import { metadataType } from '@/web/core/chat/context/chatItemContext';
+import { GetAllQuoteDataProps } from '@/web/core/chat/context/chatItemContext';
 import { getQuoteDataList } from '@/web/core/chat/api';
 
 const QuoteReader = ({
   rawSearch,
   metadata,
-  chatTime,
   onClose
 }: {
   rawSearch: SearchDataResponseItemType[];
-  metadata: metadataType;
+  metadata: GetAllQuoteDataProps;
-  chatTime: Date;
   onClose: () => void;
 }) => {
   const { t } = useTranslation();
 
-  const { chatId, appId, outLinkAuthData } = useChatStore();
+  const { data: quoteList, loading } = useRequest2(
-
-  const { data, loading } = useRequest2(
     async () =>
       await getQuoteDataList({
         datasetDataIdList: rawSearch.map((item) => item.id),
-        chatTime,
         collectionIdList: metadata.collectionIdList,
-        chatItemId: metadata.chatItemId,
+        chatItemDataId: metadata.chatItemDataId,
-        appId,
+        appId: metadata.appId,
-        chatId,
+        chatId: metadata.chatId,
-        ...outLinkAuthData
+        ...metadata.outLinkAuthData
       }),
     {
       manual: false
     }
   );
 
-  const filterResults = useMemo(() => {
-    if (!metadata.collectionId) {
-      return rawSearch;
-    }
-
-    return rawSearch.filter(
-      (item) => item.collectionId === metadata.collectionId && item.sourceId === metadata.sourceId
-    );
-  }, [metadata, rawSearch]);
-
   const formatedDataList = useMemo(() => {
-    return filterResults
+    return rawSearch
-      .map((item) => {
+      .map((searchItem) => {
-        const currentFilterItem = data?.quoteList.find((res) => res._id === item.id);
+        const dataItem = quoteList?.find((item) => item._id === searchItem.id);
 
         return {
-          ...item,
+          id: searchItem.id,
-          q: currentFilterItem?.q || '',
+          q: dataItem?.q || 'Can not find Data',
-          a: currentFilterItem?.a || '',
+          a: dataItem?.a || '',
-          score: formatScore(item.score),
+          score: formatScore(searchItem.score),
+          sourceName: searchItem?.sourceName || '',
           icon: getSourceNameIcon({
-            sourceId: item.sourceId,
+            sourceId: searchItem.sourceId,
-            sourceName: item.sourceName
+            sourceName: searchItem.sourceName
           })
         };
       })
       .sort((a, b) => {
         return (b.score.primaryScore?.value || 0) - (a.score.primaryScore?.value || 0);
       });
-  }, [data?.quoteList, filterResults]);
+  }, [quoteList, rawSearch]);
 
   return (
     <Flex flexDirection={'column'} h={'full'}>
@@ -86,18 +71,7 @@ const QuoteReader = ({
       >
         <Box flex={1} py={4}>
           <Flex gap={2} mr={2} mb={1}>
-            <MyIcon
+            <MyIcon name={'core/chat/quoteFill'} w={['1rem', '1.25rem']} color={'primary.600'} />
-              name={
-                metadata.sourceId && metadata.sourceName
-                  ? (getSourceNameIcon({
-                      sourceId: metadata.sourceId,
-                      sourceName: metadata.sourceName
-                    }) as any)
-                  : 'core/chat/quoteFill'
-              }
-              w={['1rem', '1.25rem']}
-              color={'primary.600'}
-            />
             <Box
               maxW={['200px', '300px']}
               className={'textEllipsis'}
@@ -105,9 +79,7 @@ const QuoteReader = ({
               color={'myGray.900'}
               fontWeight={'medium'}
             >
-              {metadata.sourceName
+              {t('common:core.chat.Quote Amount', { amount: rawSearch.length })}
-                ? metadata.sourceName
-                : t('common:core.chat.Quote Amount', { amount: rawSearch.length })}
             </Box>
           </Flex>
           <Box fontSize={'mini'} color={'myGray.500'}>

projects/app/src/pageComponents/chat/ChatQuoteList/index.tsx

@@ -1,39 +1,28 @@
 import React from 'react';
 import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
 import { useContextSelector } from 'use-context-selector';
-import { ChatItemContext, metadataType } from '@/web/core/chat/context/chatItemContext';
+import { ChatItemContext, GetQuoteProps } from '@/web/core/chat/context/chatItemContext';
 import CollectionQuoteReader from './CollectionQuoteReader';
 import QuoteReader from './QuoteReader';
 
 const ChatQuoteList = ({
-  chatTime,
   rawSearch = [],
   metadata,
   onClose
 }: {
-  chatTime: Date;
   rawSearch: SearchDataResponseItemType[];
-  metadata: metadataType;
+  metadata: GetQuoteProps;
   onClose: () => void;
 }) => {
   const isShowReadRawSource = useContextSelector(ChatItemContext, (v) => v.isShowReadRawSource);
 
   return (
     <>
-      {metadata.collectionId && isShowReadRawSource ? (
+      {'collectionId' in metadata && isShowReadRawSource && (
-        <CollectionQuoteReader
+        <CollectionQuoteReader rawSearch={rawSearch} metadata={metadata} onClose={onClose} />
-          rawSearch={rawSearch}
+      )}
-          metadata={metadata}
+      {'collectionIdList' in metadata && (
-          chatTime={chatTime}
+        <QuoteReader rawSearch={rawSearch} metadata={metadata} onClose={onClose} />
-          onClose={onClose}
-        />
-      ) : (
-        <QuoteReader
-          rawSearch={rawSearch}
-          metadata={metadata}
-          chatTime={chatTime}
-          onClose={onClose}
-        />
       )}
     </>
   );

projects/app/src/pages/api/core/chat/quote/getCollectionQuote.ts

@@ -5,25 +5,25 @@ import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
 import { ApiRequestProps } from '@fastgpt/service/type/next';
 import { LinkedListResponse, LinkedPaginationProps } from '@fastgpt/web/common/fetch/type';
 import { FilterQuery, Types } from 'mongoose';
-import { dataFieldSelector, processChatTimeFilter } from './getQuote';
+import { quoteDataFieldSelector, QuoteDataItemType } from '@/service/core/chat/constants';
-import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
+import { processChatTimeFilter } from '@/service/core/chat/utils';
-import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
+import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
 
 export type GetCollectionQuoteProps = LinkedPaginationProps & {
-  chatTime: Date;
+  chatId: string;
+  chatItemDataId: string;
 
   isInitialLoad: boolean;
   collectionId: string;
-  chatItemId: string;
+
   appId: string;
-  chatId: string;
   shareId?: string;
   outLinkUid?: string;
   teamId?: string;
   teamToken?: string;
 };
 
-export type GetCollectionQuoteRes = LinkedListResponse<DatasetDataSchemaType>;
+export type GetCollectionQuoteRes = LinkedListResponse<QuoteDataItemType>;
 
 type BaseMatchType = FilterQuery<DatasetDataSchemaType>;
 
@@ -37,11 +37,10 @@ async function handler(
     prevIndex,
     nextId,
     nextIndex,
-    chatTime,
     isInitialLoad,
 
     collectionId,
-    chatItemId,
+    chatItemDataId,
     appId,
     chatId,
     shareId,
@@ -53,16 +52,7 @@ async function handler(
 
   const limitedPageSize = Math.min(pageSize, 30);
 
-  try {
+  const [{ chat }, { chatItem }] = await Promise.all([
-    await authDatasetCollection({
-      req,
-      authToken: true,
-      authApiKey: true,
-      collectionId: req.body.collectionId,
-      per: ReadPermissionVal
-    });
-  } catch (error) {
-    await Promise.all([
     authChatCrud({
       req,
       authToken: true,
@@ -73,41 +63,39 @@ async function handler(
       teamId,
       teamToken
     }),
-      authCollectionInChat({ appId, chatId, chatItemId, collectionId })
+    authCollectionInChat({ appId, chatId, chatItemDataId, collectionIds: [collectionId] })
   ]);
-  }
+  if (!chat) return Promise.reject(ChatErrEnum.unAuthChat);
 
   const baseMatch: BaseMatchType = {
     collectionId,
     $or: [
-      { updateTime: { $lt: new Date(chatTime) } },
+      { updateTime: { $lt: new Date(chatItem.time) } },
-      { history: { $elemMatch: { updateTime: { $lt: new Date(chatTime) } } } }
+      { history: { $elemMatch: { updateTime: { $lt: new Date(chatItem.time) } } } }
     ]
   };
 
   if (initialId && initialIndex !== undefined) {
-    return await handleInitialLoad(
+    return await handleInitialLoad({
       initialId,
       initialIndex,
-      limitedPageSize,
+      pageSize: limitedPageSize,
-      chatTime,
+      chatTime: chatItem.time,
-      chatItemId,
       isInitialLoad,
       baseMatch
-    );
+    });
   }
 
   if ((prevId && prevIndex !== undefined) || (nextId && nextIndex !== undefined)) {
-    return await handlePaginatedLoad(
+    return await handlePaginatedLoad({
       prevId,
       prevIndex,
       nextId,
       nextIndex,
-      limitedPageSize,
+      pageSize: limitedPageSize,
-      chatTime,
+      chatTime: chatItem.time,
-      chatItemId,
       baseMatch
-    );
+    });
   }
 
   return { list: [], hasMorePrev: false, hasMoreNext: false };
@@ -115,38 +103,39 @@ async function handler(
 
 export default NextAPI(handler);
 
-async function handleInitialLoad(
+async function handleInitialLoad({
-  initialId: string,
+  initialId,
-  initialIndex: number,
+  initialIndex,
-  pageSize: number,
+  pageSize,
-  chatTime: Date,
+  chatTime,
-  chatItemId: string,
+  isInitialLoad,
-  isInitialLoad: boolean,
+  baseMatch
-  baseMatch: BaseMatchType
+}: {
-): Promise<GetCollectionQuoteRes> {
+  initialId: string;
+  initialIndex: number;
+  pageSize: number;
+  chatTime: Date;
+  isInitialLoad: boolean;
+  baseMatch: BaseMatchType;
+}): Promise<GetCollectionQuoteRes> {
   const centerNode = await MongoDatasetData.findOne(
     {
       _id: new Types.ObjectId(initialId)
     },
-    dataFieldSelector
+    quoteDataFieldSelector
   ).lean();
 
   if (!centerNode) {
     if (isInitialLoad) {
-      const list = await MongoDatasetData.find(baseMatch, dataFieldSelector)
+      const list = await MongoDatasetData.find(baseMatch, quoteDataFieldSelector)
         .sort({ chunkIndex: 1, _id: -1 })
         .limit(pageSize)
         .lean();
 
-      const listRes = list.map((item, index) => ({
-        ...item,
-        index: item.chunkIndex
-      }));
-
       const hasMoreNext = list.length === pageSize;
 
       return {
-        list: listRes,
+        list: processChatTimeFilter(list, chatTime),
         hasMorePrev: false,
         hasMoreNext
       };
@@ -173,28 +162,30 @@ async function handleInitialLoad(
 
   const resultList = [...prevList, centerNode, ...nextList];
 
-  const list = processChatTimeFilter(resultList, chatTime);
-
   return {
-    list: list.map((item) => ({
+    list: processChatTimeFilter(resultList, chatTime),
-      ...item,
-      index: item.chunkIndex
-    })),
     hasMorePrev,
     hasMoreNext
   };
 }
 
-async function handlePaginatedLoad(
+async function handlePaginatedLoad({
-  prevId: string | undefined,
+  prevId,
-  prevIndex: number | undefined,
+  prevIndex,
-  nextId: string | undefined,
+  nextId,
-  nextIndex: number | undefined,
+  nextIndex,
-  pageSize: number,
+  pageSize,
-  chatTime: Date,
+  chatTime,
-  chatItemId: string,
+  baseMatch
-  baseMatch: BaseMatchType
+}: {
-): Promise<GetCollectionQuoteRes> {
+  prevId: string | undefined;
+  prevIndex: number | undefined;
+  nextId: string | undefined;
+  nextIndex: number | undefined;
+  pageSize: number;
+  chatTime: Date;
+  baseMatch: BaseMatchType;
+}): Promise<GetCollectionQuoteRes> {
   const { list, hasMore } =
     prevId && prevIndex !== undefined
       ? await getPrevNodes(prevId, prevIndex, pageSize, baseMatch)
@@ -203,10 +194,7 @@ async function handlePaginatedLoad(
   const processedList = processChatTimeFilter(list, chatTime);
 
   return {
-    list: processedList.map((item) => ({
+    list: processedList,
-      ...item,
-      index: item.chunkIndex
-    })),
     hasMorePrev: !!prevId && hasMore,
     hasMoreNext: !!nextId && hasMore
   };
@@ -217,7 +205,10 @@ async function getPrevNodes(
   initialIndex: number,
   limit: number,
   baseMatch: BaseMatchType
-) {
+): Promise<{
+  list: DatasetDataSchemaType[];
+  hasMore: boolean;
+}> {
   const match: BaseMatchType = {
     ...baseMatch,
     $or: [
@@ -226,7 +217,7 @@ async function getPrevNodes(
     ]
   };
 
-  const list = await MongoDatasetData.find(match, dataFieldSelector)
+  const list = await MongoDatasetData.find(match, quoteDataFieldSelector)
     .sort({ chunkIndex: -1, _id: 1 })
     .limit(limit)
     .lean();
@@ -242,7 +233,10 @@ async function getNextNodes(
   initialIndex: number,
   limit: number,
   baseMatch: BaseMatchType
-) {
+): Promise<{
+  list: DatasetDataSchemaType[];
+  hasMore: boolean;
+}> {
   const match: BaseMatchType = {
     ...baseMatch,
     $or: [
@@ -251,7 +245,7 @@ async function getNextNodes(
     ]
   };
 
-  const list = await MongoDatasetData.find(match, dataFieldSelector)
+  const list = await MongoDatasetData.find(match, quoteDataFieldSelector)
     .sort({ chunkIndex: 1, _id: -1 })
     .limit(limit)
     .lean();

projects/app/src/pages/api/core/chat/quote/getQuote.ts

@@ -1,46 +1,43 @@
 import { NextAPI } from '@/service/middleware/entry';
 import { authChatCrud, authCollectionInChat } from '@/service/support/permission/auth/chat';
-import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type';
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
 import { ApiRequestProps } from '@fastgpt/service/type/next';
+import { quoteDataFieldSelector, QuoteDataItemType } from '@/service/core/chat/constants';
+import { processChatTimeFilter } from '@/service/core/chat/utils';
+import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
 
 export type GetQuoteDataProps = {
   datasetDataIdList: string[];
-  chatTime: Date;
 
   collectionIdList: string[];
-  chatItemId: string;
-  appId: string;
   chatId: string;
+  chatItemDataId: string;
+  appId: string;
   shareId?: string;
   outLinkUid?: string;
   teamId?: string;
   teamToken?: string;
 };
 
-export type GetQuoteDataRes = {
+export type GetQuoteDataRes = QuoteDataItemType[];
-  quoteList: DatasetDataSchemaType[];
-};
-
-export const dataFieldSelector =
-  '_id datasetId collectionId q a chunkIndex history updateTime currentChatItemId prevId';
 
 async function handler(req: ApiRequestProps<GetQuoteDataProps>): Promise<GetQuoteDataRes> {
   const {
-    datasetDataIdList,
-    chatTime,
-
-    collectionIdList,
-    chatItemId,
-    chatId,
     appId,
+    chatId,
+    chatItemDataId,
+
     shareId,
     outLinkUid,
     teamId,
-    teamToken
+    teamToken,
+
+    collectionIdList,
+    datasetDataIdList
   } = req.body;
 
-  await authChatCrud({
+  const [chat, { chatItem }] = await Promise.all([
+    authChatCrud({
       req,
       authToken: true,
       appId,
@@ -49,52 +46,19 @@ async function handler(req: ApiRequestProps<GetQuoteDataProps>): Promise<GetQuot
       outLinkUid,
       teamId,
       teamToken
-  });
+    }),
-
+    authCollectionInChat({ appId, chatId, chatItemDataId, collectionIds: collectionIdList })
-  await Promise.all(
+  ]);
-    collectionIdList.map(async (collectionId) => {
+  if (!chat) return Promise.reject(ChatErrEnum.unAuthChat);
-      await authCollectionInChat({ appId, chatId, chatItemId, collectionId });
-    })
-  );
 
   const list = await MongoDatasetData.find(
-    { _id: { $in: datasetDataIdList } },
+    { _id: { $in: datasetDataIdList }, collectionId: { $in: collectionIdList } },
-    dataFieldSelector
+    quoteDataFieldSelector
   ).lean();
 
-  const quoteList = processChatTimeFilter(list, chatTime);
+  const quoteList = processChatTimeFilter(list, chatItem.time);
 
-  return {
+  return quoteList;
-    quoteList
-  };
 }
 
 export default NextAPI(handler);
-
-export function processChatTimeFilter(list: DatasetDataSchemaType[], chatTime: Date) {
-  return list.map((item) => {
-    if (!item.history) return item;
-
-    const { history, ...rest } = item;
-    const formatedChatTime = new Date(chatTime);
-
-    if (item.updateTime <= formatedChatTime) {
-      return rest;
-    }
-
-    const latestHistoryIndex = history.findIndex(
-      (historyItem: any) => historyItem.updateTime <= formatedChatTime
-    );
-
-    if (latestHistoryIndex === -1) return rest;
-
-    const latestHistory = history[latestHistoryIndex];
-
-    return {
-      ...rest,
-      q: latestHistory?.q || item.q,
-      a: latestHistory?.a || item.a,
-      updated: true
-    };
-  });
-}

projects/app/src/pages/api/core/dataset/collection/create/apiCollection.ts

@@ -2,11 +2,7 @@ import type { NextApiRequest } from 'next';
 import type { ApiDatasetCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
 import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
 import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller';
-import {
+import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
-  TrainingModeEnum,
-  DatasetCollectionTypeEnum,
-  DatasetCollectionDataProcessModeEnum
-} from '@fastgpt/global/core/dataset/constants';
 
 import { NextAPI } from '@/service/middleware/entry';
 import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
@@ -16,7 +12,8 @@ import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection
 import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
 
 async function handler(req: NextApiRequest): CreateCollectionResponse {
-  const { name, apiFileId, ...body } = req.body as ApiDatasetCreateDatasetCollectionParams;
+  const { name, apiFileId, customPdfParse, ...body } =
+    req.body as ApiDatasetCreateDatasetCollectionParams;
 
   const { teamId, tmbId, dataset } = await authDataset({
     req,
@@ -50,7 +47,8 @@ async function handler(req: NextApiRequest): CreateCollectionResponse {
     yuqueServer,
     apiFileId,
     teamId,
-    tmbId
+    tmbId,
+    customPdfParse
   });
 
   const { collectionId, insertResults } = await createCollectionAndInsertData({
@@ -62,11 +60,12 @@ async function handler(req: NextApiRequest): CreateCollectionResponse {
       teamId,
       tmbId,
       type: DatasetCollectionTypeEnum.apiFile,
-      name: name,
+      name,
       apiFileId,
       metadata: {
         relatedImgId: apiFileId
-      }
+      },
+      customPdfParse
     }
   });
 

projects/app/src/pages/api/core/dataset/collection/export.ts

@@ -1,9 +1,13 @@
 import { NextAPI } from '@/service/middleware/entry';
+import { authChatCrud, authCollectionInChat } from '@/service/support/permission/auth/chat';
+import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
+import { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat';
 import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
 import { useIPFrequencyLimit } from '@fastgpt/service/common/middle/reqFrequencyLimit';
 import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
 import { responseWriteController } from '@fastgpt/service/common/response';
 import { addLog } from '@fastgpt/service/common/system/log';
+import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller';
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
 import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
 import { ApiRequestProps } from '@fastgpt/service/type/next';
@@ -11,21 +15,69 @@ import { NextApiResponse } from 'next';
 
 export type ExportCollectionBody = {
   collectionId: string;
+
+  appId?: string;
+  chatId?: string;
+  chatItemDataId?: string;
   chatTime: Date;
-};
+} & OutLinkChatAuthProps;
 
 async function handler(req: ApiRequestProps<ExportCollectionBody, {}>, res: NextApiResponse) {
-  let { collectionId, chatTime } = req.body;
+  const {
+    collectionId,
+    appId,
+    chatId,
+    chatItemDataId,
+    shareId,
+    outLinkUid,
+    teamId,
+    teamToken,
+    chatTime
+  } = req.body;
 
-  const { teamId, collection } = await authDatasetCollection({
+  const { collection, teamId: userTeamId } = await (async () => {
+    if (!appId || !chatId || !chatItemDataId) {
+      return authDatasetCollection({
         req,
         authToken: true,
-    collectionId,
+        authApiKey: true,
+        collectionId: req.body.collectionId,
         per: ReadPermissionVal
       });
+    }
+
+    /* 
+      1. auth chat read permission
+      2. auth collection quote in chat
+      3. auth outlink open show quote
+    */
+    const [authRes, collection] = await Promise.all([
+      authChatCrud({
+        req,
+        authToken: true,
+        appId,
+        chatId,
+        shareId,
+        outLinkUid,
+        teamId,
+        teamToken
+      }),
+      getCollectionWithDataset(collectionId),
+      authCollectionInChat({ appId, chatId, chatItemDataId, collectionIds: [collectionId] })
+    ]);
+
+    if (!authRes.showRawSource) {
+      return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
+    }
+
+    return {
+      ...authRes,
+      collection
+    };
+  })();
 
   const where = {
-    teamId,
+    teamId: userTeamId,
     datasetId: collection.datasetId,
     collectionId,
     ...(chatTime

projects/app/src/pages/api/core/dataset/collection/read.ts

@@ -19,7 +19,7 @@ export type readCollectionSourceBody = {
 
   appId?: string;
   chatId?: string;
-  chatItemId?: string;
+  chatItemDataId?: string;
 } & OutLinkChatAuthProps;
 
 export type readCollectionSourceResponse = {
@@ -30,7 +30,7 @@ export type readCollectionSourceResponse = {
 async function handler(
   req: ApiRequestProps<readCollectionSourceBody, readCollectionSourceQuery>
 ): Promise<readCollectionSourceResponse> {
-  const { collectionId, appId, chatId, chatItemId, shareId, outLinkUid, teamId, teamToken } =
+  const { collectionId, appId, chatId, chatItemDataId, shareId, outLinkUid, teamId, teamToken } =
     req.body;
 
   const {
@@ -39,7 +39,7 @@ async function handler(
     tmbId: uid,
     authType
   } = await (async () => {
-    if (!appId || !chatId || !chatItemId) {
+    if (!appId || !chatId || !chatItemDataId) {
       return authDatasetCollection({
         req,
         authToken: true,
@@ -66,7 +66,7 @@ async function handler(
         teamToken
       }),
       getCollectionWithDataset(collectionId),
-      authCollectionInChat({ appId, chatId, chatItemId, collectionId })
+      authCollectionInChat({ appId, chatId, chatItemDataId, collectionIds: [collectionId] })
     ]);
 
     if (!authRes.showRawSource) {

projects/app/src/pages/chat/index.tsx

@@ -231,7 +231,6 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
       {quoteData && (
         <PageContainer flex={'1 0 0'} w={0} maxW={'560px'}>
           <ChatQuoteList
-            chatTime={quoteData.chatTime}
             rawSearch={quoteData.rawSearch}
             metadata={quoteData.metadata}
             onClose={() => setQuoteData(undefined)}

projects/app/src/pages/chat/share.tsx

@@ -304,7 +304,6 @@ const OutLink = (props: Props) => {
         {quoteData && (
           <PageContainer flex={'1 0 0'} w={0} maxW={'560px'}>
             <ChatQuoteList
-              chatTime={quoteData.chatTime}
               rawSearch={quoteData.rawSearch}
               metadata={quoteData.metadata}
               onClose={() => setQuoteData(undefined)}

projects/app/src/pages/chat/team.tsx

@@ -245,7 +245,6 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
       {quoteData && (
         <PageContainer flex={'1 0 0'} w={0} maxW={'560px'}>
           <ChatQuoteList
-            chatTime={quoteData.chatTime}
             rawSearch={quoteData.rawSearch}
             metadata={quoteData.metadata}
             onClose={() => setQuoteData(undefined)}

projects/app/src/service/core/chat/constants.ts

@@ -0,0 +1,13 @@
+import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type';
+
+export const quoteDataFieldSelector = '_id q a history updateTime chunkIndex';
+
+export type QuoteDataItemType = {
+  _id: string;
+  q: DatasetDataSchemaType['q'];
+  a: DatasetDataSchemaType['a'];
+  history?: DatasetDataSchemaType['history'];
+  updateTime: DatasetDataSchemaType['updateTime'];
+  index: DatasetDataSchemaType['chunkIndex'];
+  updated?: boolean;
+};

projects/app/src/service/core/chat/utils.ts

@@ -0,0 +1,46 @@
+import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type';
+import { QuoteDataItemType } from './constants';
+
+// 获取对话时间时，引用的内容
+export function processChatTimeFilter(
+  dataList: DatasetDataSchemaType[],
+  chatTime: Date
+): QuoteDataItemType[] {
+  return dataList.map((item) => {
+    const defaultItem = {
+      _id: item._id,
+      q: item.q,
+      a: item.a,
+      updateTime: item.updateTime,
+      index: item.chunkIndex
+    };
+
+    if (!item.history) return defaultItem;
+
+    const history = item.history;
+    const formatedChatTime = new Date(chatTime);
+
+    if (item.updateTime <= formatedChatTime) {
+      return defaultItem;
+    }
+
+    const latestHistoryIndex = history.findIndex(
+      (historyItem) => historyItem.updateTime <= formatedChatTime
+    );
+
+    if (latestHistoryIndex === -1) {
+      return defaultItem;
+    }
+
+    const latestHistory = history[latestHistoryIndex];
+
+    return {
+      _id: item._id,
+      q: latestHistory.q,
+      a: latestHistory.a,
+      updateTime: latestHistory.updateTime,
+      index: item.chunkIndex,
+      updated: true
+    };
+  });
+}

projects/app/src/service/support/permission/auth/chat.ts

@@ -25,6 +25,14 @@ const defaultResponseShow = {
   showNodeStatus: true,
   showRawSource: true
 };
+type AuthChatCommonProps = {
+  appId: string;
+  shareId?: string;
+  outLinkUid?: string;
+  teamId?: string;
+  teamToken?: string;
+};
+
 export async function authChatCrud({
   appId,
   chatId,
@@ -35,13 +43,9 @@ export async function authChatCrud({
   teamId: spaceTeamId,
   teamToken,
   ...props
-}: AuthModeType & {
+}: AuthModeType &
-  appId: string;
+  AuthChatCommonProps & {
     chatId?: string;
-  shareId?: string;
-  outLinkUid?: string;
-  teamId?: string;
-  teamToken?: string;
   }): Promise<{
   teamId: string;
   tmbId: string;
@@ -188,27 +192,29 @@ export async function authChatCrud({
 }
 
 export const authCollectionInChat = async ({
-  collectionId,
+  collectionIds,
   appId,
   chatId,
-  chatItemId
+  chatItemDataId
 }: {
-  collectionId: string;
+  collectionIds: string[];
   appId: string;
   chatId: string;
-  chatItemId: string;
+  chatItemDataId: string;
-}) => {
+}): Promise<{
+  chatItem: { time: Date; responseData?: ChatHistoryItemResType[] };
+}> => {
   try {
     const chatItem = (await MongoChatItem.findOne(
       {
         appId,
         chatId,
-        dataId: chatItemId
+        dataId: chatItemDataId
       },
-      'responseData'
+      'responseData time'
-    ).lean()) as AIChatItemType;
+    ).lean()) as { time: Date; responseData?: ChatHistoryItemResType[] };
 
-    if (!chatItem) return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
+    if (!chatItem) return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
 
     // 找 responseData 里，是否有该文档 id
     const responseData = chatItem.responseData || [];
@@ -224,15 +230,16 @@ export const authCollectionInChat = async ({
         })
         .flat() || [];
 
-    if (
+    const quoteListSet = new Set(
-      flatResData.some((item) => {
+      flatResData
-        if (item.quoteList) {
+        .map((item) => item.quoteList?.map((quote) => String(quote.collectionId)) || [])
-          return item.quoteList.some((quote) => quote.collectionId === collectionId);
+        .flat()
-        }
+    );
-        return false;
+
-      })
+    if (collectionIds.every((id) => quoteListSet.has(id))) {
-    ) {
+      return {
-      return true;
+        chatItem
+      };
     }
   } catch (error) {}
   return Promise.reject(DatasetErrEnum.unAuthDatasetFile);

projects/app/src/web/core/chat/api.ts

@@ -1,11 +1,6 @@
 import { GET, POST, DELETE, PUT } from '@/web/common/api/request';
-import type {
+import type { ChatHistoryItemType, ChatHistoryItemResType } from '@fastgpt/global/core/chat/type.d';
-  ChatHistoryItemType,
+import type { getResDataQuery } from '@/pages/api/core/chat/getResData';
-  ChatHistoryItemResType,
-  ChatSiteItemType,
-  ChatItemType
-} from '@fastgpt/global/core/chat/type.d';
-import { getResDataQuery } from '@/pages/api/core/chat/getResData';
 import type {
   CloseCustomFeedbackParams,
   InitChatProps,
@@ -22,16 +17,16 @@ import type {
   DeleteChatItemProps,
   UpdateHistoryProps
 } from '@/global/core/chat/api.d';
-import { UpdateChatFeedbackProps } from '@fastgpt/global/core/chat/api';
+import type { UpdateChatFeedbackProps } from '@fastgpt/global/core/chat/api';
-import { AuthTeamTagTokenProps } from '@fastgpt/global/support/user/team/tag';
+import type { AuthTeamTagTokenProps } from '@fastgpt/global/support/user/team/tag';
-import { AppListItemType } from '@fastgpt/global/core/app/type';
+import type { AppListItemType } from '@fastgpt/global/core/app/type';
-import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
+import type { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
 import type {
   getPaginationRecordsBody,
   getPaginationRecordsResponse
 } from '@/pages/api/core/chat/getPaginationRecords';
-import { GetQuoteDataProps, GetQuoteDataRes } from '@/pages/api/core/chat/quote/getQuote';
+import type { GetQuoteDataProps, GetQuoteDataRes } from '@/pages/api/core/chat/quote/getQuote';
-import {
+import type {
   GetCollectionQuoteProps,
   GetCollectionQuoteRes
 } from '@/pages/api/core/chat/quote/getCollectionQuote';

projects/app/src/web/core/chat/context/chatItemContext.tsx

@@ -9,6 +9,7 @@ import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
 import { AppChatConfigType, VariableItemType } from '@fastgpt/global/core/app/type';
 import { FlowNodeInputItemType } from '@fastgpt/global/core/workflow/type/io';
 import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
+import { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat';
 
 type ContextProps = {
   showRouteToAppDetail: boolean;
@@ -32,19 +33,27 @@ type ChatBoxDataType = {
   };
 };
 
-export type metadataType = {
+export type GetQuoteDataBasicProps = {
+  appId: string;
+  chatId: string;
+  chatItemDataId: string;
+  outLinkAuthData?: OutLinkChatAuthProps;
+};
+// 获取单个集合引用
+export type GetCollectionQuoteDataProps = GetQuoteDataBasicProps & {
   collectionId: string;
-  collectionIdList: string[];
-  chatItemId: string;
   sourceId: string;
   sourceName: string;
   datasetId: string;
 };
+export type GetAllQuoteDataProps = GetQuoteDataBasicProps & {
+  collectionIdList: string[];
+};
+export type GetQuoteProps = GetAllQuoteDataProps | GetCollectionQuoteDataProps;
 
 export type QuoteDataType = {
   rawSearch: SearchDataResponseItemType[];
-  metadata: metadataType;
+  metadata: GetQuoteProps;
-  chatTime: Date;
 };
 
 type ChatItemContextType = {