From edd8ba9e5aca4af901b9e37819d8c9a4b75ef924 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Thu, 13 Mar 2025 21:30:40 +0800 Subject: [PATCH] hiden dataset source (#4152) * hiden dataset source * perf: reader --- packages/web/hooks/useLinkedScroll.tsx | 204 ++++++------------ packages/web/i18n/en/chat.json | 1 + packages/web/i18n/en/common.json | 1 - packages/web/i18n/zh-CN/chat.json | 1 + packages/web/i18n/zh-CN/common.json | 1 - packages/web/i18n/zh-Hant/chat.json | 1 + packages/web/i18n/zh-Hant/common.json | 1 - .../ChatQuoteList/CollectionQuoteItem.tsx | 5 +- .../ChatQuoteList/CollectionQuoteReader.tsx | 175 +++++++-------- .../api/core/chat/quote/getCollectionQuote.ts | 29 +-- 10 files changed, 157 insertions(+), 262 deletions(-) diff --git a/packages/web/hooks/useLinkedScroll.tsx b/packages/web/hooks/useLinkedScroll.tsx index f48658c68..ee0903aa4 100644 --- a/packages/web/hooks/useLinkedScroll.tsx +++ b/packages/web/hooks/useLinkedScroll.tsx @@ -1,10 +1,11 @@ -import { useCallback, useEffect, useRef, useState, ReactNode } from 'react'; +import { useEffect, useRef, useState, ReactNode, useCallback } from 'react'; import { LinkedListResponse, LinkedPaginationProps } from '../common/fetch/type'; import { Box, BoxProps } from '@chakra-ui/react'; import { useTranslation } from 'next-i18next'; import { useScroll, useMemoizedFn, useDebounceEffect } from 'ahooks'; import MyBox from '../components/common/MyBox'; import { useRequest2 } from './useRequest'; +import { delay } from '../../global/common/system/utils'; const threshold = 100; @@ -14,92 +15,95 @@ export function useLinkedScroll< >( api: (data: TParams) => Promise, { - refreshDeps = [], pageSize = 15, params = {}, - initialId, - initialIndex, - canLoadData = false + currentData }: { - refreshDeps?: any[]; pageSize?: number; params?: Record; - initialId?: string; - initialIndex?: number; - canLoadData?: boolean; + currentData?: { id: string; index: number }; } ) { const { t } = useTranslation(); const [dataList, setDataList] = useState([]); const [hasMorePrev, setHasMorePrev] = useState(true); const [hasMoreNext, setHasMoreNext] = useState(true); - const [initialLoadDone, setInitialLoadDone] = useState(false); - const hasScrolledToInitial = useRef(false); const anchorRef = useRef({ top: null as { _id: string; index: number } | null, bottom: null as { _id: string; index: number } | null }); - const containerRef = useRef(null); const itemRefs = useRef<(HTMLElement | null)[]>([]); - const { runAsync: callApi, loading: isLoading } = useRequest2( - async (apiParams: TParams) => await api(apiParams), - { - onError: (error) => { - return Promise.reject(error); - } - } - ); + const scrollToItem = async (id: string, retry = 3) => { + const itemIndex = dataList.findIndex((item) => item._id === id); + if (itemIndex === -1) return; - const loadData = useCallback( - async ({ - id, - index, - isInitialLoad = false - }: { - id: string; - index: number; - isInitialLoad?: boolean; - }) => { - if (isLoading) return null; + const element = itemRefs.current[itemIndex]; + + if (!element || !containerRef.current) { + if (retry > 0) { + await delay(500); + return scrollToItem(id, retry - 1); + } + return; + } + + const elementRect = element.getBoundingClientRect(); + const containerRect = containerRef.current.getBoundingClientRect(); + + const scrollTop = containerRef.current.scrollTop + elementRect.top - containerRect.top; + + containerRef.current.scrollTo({ + top: scrollTop + }); + }; + + const { runAsync: callApi, loading: isLoading } = useRequest2(api); + + let scroolSign = useRef(false); + const { runAsync: loadInitData } = useRequest2( + async (scrollWhenFinish = true) => { + if (!currentData || isLoading) return; + + const item = dataList.find((item) => item._id === currentData.id); + if (item) { + scrollToItem(item._id); + return; + } const response = await callApi({ - initialId: id, - initialIndex: index, + initialId: currentData.id, + initialIndex: currentData.index, pageSize, - isInitialLoad, ...params } as TParams); - if (!response) return null; - setHasMorePrev(response.hasMorePrev); setHasMoreNext(response.hasMoreNext); + + scroolSign.current = scrollWhenFinish; setDataList(response.list); if (response.list.length > 0) { anchorRef.current.top = response.list[0]; anchorRef.current.bottom = response.list[response.list.length - 1]; } - - setInitialLoadDone(true); - - const scrollIndex = response.list.findIndex((item) => item._id === id); - - if (scrollIndex !== -1 && itemRefs.current?.[scrollIndex]) { - setTimeout(() => { - scrollToItem(scrollIndex); - }, 100); - } - - return response; }, - [callApi, params, dataList, hasMorePrev, hasMoreNext, isLoading] + { + refreshDeps: [currentData], + manual: false + } ); + useEffect(() => { + if (scroolSign.current && currentData) { + scroolSign.current = false; + scrollToItem(currentData.id); + } + }, [dataList]); - const loadPrevData = useCallback( + const { runAsync: loadPrevData, loading: prevLoading } = useRequest2( async (scrollRef = containerRef) => { if (!anchorRef.current.top || !hasMorePrev || isLoading) return; @@ -132,10 +136,12 @@ export function useLinkedScroll< return response; }, - [callApi, hasMorePrev, isLoading, params, pageSize] + { + refreshDeps: [hasMorePrev, isLoading, params, pageSize] + } ); - const loadNextData = useCallback( + const { runAsync: loadNextData, loading: nextLoading } = useRequest2( async (scrollRef = containerRef) => { if (!anchorRef.current.bottom || !hasMoreNext || isLoading) return; @@ -165,85 +171,17 @@ export function useLinkedScroll< return response; }, - [callApi, hasMoreNext, isLoading, params, pageSize] - ); - - const scrollToItem = useCallback( - (itemIndex: number) => { - if (itemIndex >= 0 && itemIndex < dataList.length && itemRefs.current?.[itemIndex]) { - try { - const element = itemRefs.current[itemIndex]; - if (!element) { - return false; - } - - setTimeout(() => { - if (element && containerRef.current) { - const elementRect = element.getBoundingClientRect(); - const containerRect = containerRef.current.getBoundingClientRect(); - - const relativeTop = elementRect.top - containerRect.top; - - const scrollTop = - containerRef.current.scrollTop + - relativeTop - - containerRect.height / 2 + - elementRect.height / 2; - - containerRef.current.scrollTo({ - top: scrollTop, - behavior: 'smooth' - }); - } - }, 50); - - return true; - } catch (error) { - console.error('Error scrolling to item:', error); - return false; - } - } - return false; - }, - [dataList.length] - ); - - // 初始加载 - useEffect(() => { - if (canLoadData) { - setInitialLoadDone(false); - hasScrolledToInitial.current = false; - - loadData({ - id: initialId || '', - index: initialIndex || 0, - isInitialLoad: true - }); + { + refreshDeps: [hasMoreNext, isLoading, params, pageSize] } - }, [canLoadData, ...refreshDeps]); - - // 监听初始加载完成，执行初始滚动 - useEffect(() => { - if (initialLoadDone && dataList.length > 0 && !hasScrolledToInitial.current) { - const foundIndex = dataList.findIndex((item) => item._id === initialId); - - if (foundIndex >= 0) { - hasScrolledToInitial.current = true; - setTimeout(() => { - scrollToItem(foundIndex); - }, 200); - } - } - }, [initialLoadDone, ...refreshDeps]); + ); const ScrollData = useMemoizedFn( ({ children, ScrollContainerRef, - isLoading: externalLoading, ...props }: { - isLoading?: boolean; children: ReactNode; ScrollContainerRef?: React.RefObject; } & BoxProps) => { @@ -252,17 +190,17 @@ export function useLinkedScroll< useDebounceEffect( () => { - if (!ref?.current || isLoading || !initialLoadDone) return; + if (!ref?.current || isLoading) return; const { scrollTop, scrollHeight, clientHeight } = ref.current; // 滚动到底部附近，加载更多下方数据 - if (scrollTop + clientHeight >= scrollHeight - threshold && hasMoreNext) { + if (scrollTop + clientHeight >= scrollHeight - threshold) { loadNextData(ref); } // 滚动到顶部附近，加载更多上方数据 - if (scrollTop <= threshold && hasMorePrev) { + if (scrollTop <= threshold) { loadPrevData(ref); } }, @@ -271,20 +209,14 @@ export function useLinkedScroll< ); return ( - - {hasMorePrev && isLoading && initialLoadDone && ( + + {hasMorePrev && prevLoading && ( {t('common:common.is_requesting')} )} {children} - {hasMoreNext && isLoading && initialLoadDone && ( + {hasMoreNext && nextLoading && ( {t('common:common.is_requesting')} @@ -298,7 +230,7 @@ export function useLinkedScroll< dataList, setDataList, isLoading, - loadData, + loadInitData, ScrollData, itemRefs, scrollToItem diff --git a/packages/web/i18n/en/chat.json b/packages/web/i18n/en/chat.json index b3326244f..655fb50c7 100644 --- a/packages/web/i18n/en/chat.json +++ b/packages/web/i18n/en/chat.json @@ -18,6 +18,7 @@ "contextual_preview": "Contextual Preview {{num}} Items", "csv_input_lexicon_tip": "Only CSV batch import is supported, click to download the template", "custom_input_guide_url": "Custom Lexicon URL", + "data_source": "Source Dataset: {{name}}", "dataset_quote_type error": "Knowledge base reference type is wrong, correct type: { datasetId: string }[]", "delete_all_input_guide_confirm": "Are you sure you want to clear the input guide lexicon?", "download_chunks": "Download data", diff --git a/packages/web/i18n/en/common.json b/packages/web/i18n/en/common.json index 61d24feb4..f8c05ab6b 100644 --- a/packages/web/i18n/en/common.json +++ b/packages/web/i18n/en/common.json @@ -457,7 +457,6 @@ "core.chat.quote.Read Quote": "View Quote", "core.chat.quote.afterUpdate": "After update", "core.chat.quote.beforeUpdate": "Before update", - "core.chat.quote.source": "From: {{source}}", "core.chat.response.Complete Response": "Complete Response", "core.chat.response.Extension model": "Question Optimization Model", "core.chat.response.Read complete response": "View Details", diff --git a/packages/web/i18n/zh-CN/chat.json b/packages/web/i18n/zh-CN/chat.json index 4f8d9e476..72413c36b 100644 --- a/packages/web/i18n/zh-CN/chat.json +++ b/packages/web/i18n/zh-CN/chat.json @@ -18,6 +18,7 @@ "contextual_preview": "上下文预览 {{num}} 条", "csv_input_lexicon_tip": "仅支持 CSV 批量导入，点击下载模板", "custom_input_guide_url": "自定义词库地址", + "data_source": "来源知识库: {{name}}", "dataset_quote_type error": "知识库引用类型错误，正确类型：{ datasetId: string }[]", "delete_all_input_guide_confirm": "确定要清空输入引导词库吗？", "download_chunks": "下载数据", diff --git a/packages/web/i18n/zh-CN/common.json b/packages/web/i18n/zh-CN/common.json index ff004ddb6..d703f612b 100644 --- a/packages/web/i18n/zh-CN/common.json +++ b/packages/web/i18n/zh-CN/common.json @@ -460,7 +460,6 @@ "core.chat.quote.Read Quote": "查看引用", "core.chat.quote.afterUpdate": "更新后", "core.chat.quote.beforeUpdate": "更新前", - "core.chat.quote.source": "来源：{{source}}", "core.chat.response.Complete Response": "完整响应", "core.chat.response.Extension model": "问题优化模型", "core.chat.response.Read complete response": "查看详情", diff --git a/packages/web/i18n/zh-Hant/chat.json b/packages/web/i18n/zh-Hant/chat.json index bceac9809..312e3ca92 100644 --- a/packages/web/i18n/zh-Hant/chat.json +++ b/packages/web/i18n/zh-Hant/chat.json @@ -18,6 +18,7 @@ "contextual_preview": "上下文預覽 {{num}} 筆", "csv_input_lexicon_tip": "僅支援 CSV 批次匯入，點選下載範本", "custom_input_guide_url": "自訂詞彙庫網址", + "data_source": "來源知識庫: {{name}}", "dataset_quote_type error": "知識庫引用類型錯誤，正確類型：{ datasetId: string }[]", "delete_all_input_guide_confirm": "確定要清除輸入導引詞彙庫嗎？", "download_chunks": "下載數據", diff --git a/packages/web/i18n/zh-Hant/common.json b/packages/web/i18n/zh-Hant/common.json index 0f2b533d5..65ea94516 100644 --- a/packages/web/i18n/zh-Hant/common.json +++ b/packages/web/i18n/zh-Hant/common.json @@ -456,7 +456,6 @@ "core.chat.quote.Read Quote": "檢視引用", "core.chat.quote.afterUpdate": "更新後", "core.chat.quote.beforeUpdate": "更新前", - "core.chat.quote.source": "來源：{{source}}", "core.chat.response.Complete Response": "完整回應", "core.chat.response.Extension model": "問題最佳化模型", "core.chat.response.Read complete response": "檢視詳細資料", diff --git a/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteItem.tsx b/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteItem.tsx index 4a0915b5e..454a970ba 100644 --- a/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteItem.tsx +++ b/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteItem.tsx @@ -175,10 +175,7 @@ const CollectionQuoteItem = ({ {editInputData && ( setEditInputData(undefined)} - onSuccess={() => { - console.log('onSuccess'); - refreshList(); - }} + onSuccess={refreshList} dataId={editInputData.dataId} collectionId={editInputData.collectionId} /> diff --git a/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx b/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx index 82e0ec7d3..0e9a3c6f5 100644 --- a/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx +++ b/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx @@ -7,7 +7,7 @@ import { useTranslation } from 'react-i18next'; import DownloadButton from './DownloadButton'; import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; import { downloadFetch } from '@/web/common/system/utils'; -import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useMemo, useState } from 'react'; import { getDatasetDataPermission } from '@/web/core/dataset/api'; import ScoreTag from './ScoreTag'; import { formatScore } from '@/components/core/dataset/QuoteItem'; @@ -39,51 +39,53 @@ const CollectionReader = ({ const [quoteIndex, setQuoteIndex] = useState(0); // Get dataset permission - const { data: datasetData, loading: isPermissionLoading } = useRequest2( - async () => await getDatasetDataPermission(datasetId), - { - manual: !userInfo || !datasetId, - refreshDeps: [datasetId, userInfo] - } - ); + const { data: datasetData } = useRequest2(async () => await getDatasetDataPermission(datasetId), { + manual: !userInfo || !datasetId, + refreshDeps: [datasetId, userInfo] + }); const filterResults = useMemo(() => { - const results = rawSearch.filter( - (item) => item.collectionId === metadata.collectionId && item.sourceId === metadata.sourceId - ); - - return results.sort((a, b) => (a.chunkIndex || 0) - (b.chunkIndex || 0)); - }, [metadata, rawSearch]); - - const currentQuoteItem = filterResults[quoteIndex]; + setQuoteIndex(0); + return rawSearch + .filter((item) => item.collectionId === collectionId) + .sort((a, b) => (a.chunkIndex || 0) - (b.chunkIndex || 0)); + }, [collectionId, rawSearch]); + const currentQuoteItem = useMemo(() => { + const item = filterResults[quoteIndex]; + if (item) { + return { + id: item.id, + index: item.chunkIndex, + score: item.score + }; + } + }, [filterResults, quoteIndex]); // Get quote list - const { - dataList: datasetDataList, - setDataList: setDatasetDataList, - isLoading, - loadData, - ScrollData, - itemRefs, - scrollToItem - } = useLinkedScroll(getCollectionQuote, { - refreshDeps: [collectionId], - params: { + const params = useMemo( + () => ({ collectionId, chatItemDataId, chatId: metadata.chatId, appId: metadata.appId, ...metadata.outLinkAuthData - }, - initialId: currentQuoteItem?.id, - initialIndex: currentQuoteItem?.chunkIndex, - canLoadData: !!currentQuoteItem?.id && !isPermissionLoading + }), + [metadata] + ); + const { + dataList: datasetDataList, + isLoading, + ScrollData, + itemRefs, + loadInitData + } = useLinkedScroll(getCollectionQuote, { + params, + currentData: currentQuoteItem }); - const loading = isLoading || isPermissionLoading; const isDeleted = useMemo( - () => !datasetDataList.find((item) => item._id === currentQuoteItem?.id), - [datasetDataList, currentQuoteItem?.id] + () => !isLoading && !datasetDataList.find((item) => item._id === currentQuoteItem?.id), + [datasetDataList, currentQuoteItem?.id, isLoading] ); const formatedDataList = useMemo( @@ -101,11 +103,6 @@ const CollectionReader = ({ [currentQuoteItem?.id, datasetDataList, filterResults] ); - useEffect(() => { - setQuoteIndex(0); - setDatasetDataList([]); - }, [collectionId, setDatasetDataList]); - const { runAsync: handleDownload } = useRequest2(async () => { await downloadFetch({ url: '/api/core/dataset/collection/export', @@ -119,30 +116,6 @@ const CollectionReader = ({ const handleRead = getCollectionSourceAndOpen(metadata); - const handleNavigate = useCallback( - async (targetIndex: number) => { - if (targetIndex < 0 || targetIndex >= filterResults.length) return; - const targetItemId = filterResults[targetIndex].id; - const targetItemIndex = filterResults[targetIndex].chunkIndex; - - setQuoteIndex(targetIndex); - const dataIndex = datasetDataList.findIndex((item) => item._id === targetItemId); - - if (dataIndex !== -1) { - setTimeout(() => { - scrollToItem(dataIndex); - }, 50); - } else { - try { - await loadData({ id: targetItemId, index: targetItemIndex }); - } catch (error) { - console.error('Failed to navigate:', error); - } - } - }, - [filterResults, datasetDataList, scrollToItem, loadData] - ); - return ( {/* title */} @@ -163,6 +136,16 @@ const CollectionReader = ({ fontSize={'sm'} color={'myGray.900'} fontWeight={'medium'} + {...(!!userInfo && + datasetData?.permission?.hasReadPer && { + cursor: 'pointer', + _hover: { color: 'primary.600', textDecoration: 'underline' }, + onClick: () => { + router.push( + `/dataset/detail?datasetId=${datasetId}¤tTab=dataCard&collectionId=${collectionId}` + ); + } + })} > {sourceName || t('common:common.UnKnow Source')} @@ -181,26 +164,22 @@ const CollectionReader = ({ onClick={onClose} /> - {!isPermissionLoading && ( + {datasetData?.permission?.hasReadPer && ( { - if (!!userInfo && datasetData?.permission?.hasReadPer) { - router.push( - `/dataset/detail?datasetId=${datasetId}¤tTab=dataCard&collectionId=${collectionId}` - ); - } - }} - {...(!!userInfo && datasetData?.permission?.hasReadPer + {...(!!userInfo ? { cursor: 'pointer', - _hover: { color: 'primary.600', textDecoration: 'underline' } + _hover: { color: 'primary.600', textDecoration: 'underline' }, + onClick: () => { + router.push(`/dataset/detail?datasetId=${datasetId}`); + } } : {})} > - {t('common:core.chat.quote.source', { - source: datasetData?.datasetName + {t('chat:data_source', { + name: datasetData.datasetName })} )} @@ -231,23 +210,22 @@ const CollectionReader = ({ {/* 检索分数 */} - {!loading && - (!isDeleted ? ( - - ) : ( - - - {t('chat:chat.quote.deleted')} - - ))} + {currentQuoteItem?.score ? ( + + ) : isDeleted ? ( + + + {t('chat:chat.quote.deleted')} + + ) : null} @@ -256,12 +234,12 @@ const CollectionReader = ({ handleNavigate(quoteIndex - 1)} + onClick={() => setQuoteIndex(quoteIndex - 1)} /> handleNavigate(quoteIndex + 1)} + onClick={() => setQuoteIndex(quoteIndex + 1)} /> @@ -272,8 +250,8 @@ const CollectionReader = ({ )} {/* quote list */} - {loading || datasetDataList.length > 0 ? ( - + {isLoading || datasetDataList.length > 0 ? ( + {formatedDataList.map((item, index) => ( } quoteIndex={item.quoteIndex} setQuoteIndex={setQuoteIndex} - refreshList={() => - currentQuoteItem?.id && - loadData({ id: currentQuoteItem.id, index: currentQuoteItem.chunkIndex }) - } + refreshList={() => loadInitData(false)} updated={item.updated} isCurrentSelected={item.isCurrentSelected} q={item.q} diff --git a/projects/app/src/pages/api/core/chat/quote/getCollectionQuote.ts b/projects/app/src/pages/api/core/chat/quote/getCollectionQuote.ts index 1a3f61bec..2afa42849 100644 --- a/projects/app/src/pages/api/core/chat/quote/getCollectionQuote.ts +++ b/projects/app/src/pages/api/core/chat/quote/getCollectionQuote.ts @@ -13,7 +13,6 @@ export type GetCollectionQuoteProps = LinkedPaginationProps & { chatId: string; chatItemDataId: string; - isInitialLoad: boolean; collectionId: string; appId: string; @@ -37,7 +36,6 @@ async function handler( prevIndex, nextId, nextIndex, - isInitialLoad, collectionId, chatItemDataId, @@ -84,7 +82,6 @@ async function handler( initialIndex, pageSize: limitedPageSize, chatTime: chatItem.time, - isInitialLoad, baseMatch }); } @@ -111,14 +108,12 @@ async function handleInitialLoad({ initialIndex, pageSize, chatTime, - isInitialLoad, baseMatch }: { initialId: string; initialIndex: number; pageSize: number; chatTime: Date; - isInitialLoad: boolean; baseMatch: BaseMatchType; }): Promise { const centerNode = await MongoDatasetData.findOne( @@ -129,22 +124,18 @@ async function handleInitialLoad({ ).lean(); if (!centerNode) { - if (isInitialLoad) { - const list = await MongoDatasetData.find(baseMatch, quoteDataFieldSelector) - .sort({ chunkIndex: 1, _id: -1 }) - .limit(pageSize) - .lean(); + const list = await MongoDatasetData.find(baseMatch, quoteDataFieldSelector) + .sort({ chunkIndex: 1, _id: -1 }) + .limit(pageSize) + .lean(); - const hasMoreNext = list.length === pageSize; + const hasMoreNext = list.length === pageSize; - return { - list: processChatTimeFilter(list, chatTime), - hasMorePrev: false, - hasMoreNext - }; - } - - return Promise.reject('centerNode not found'); + return { + list: processChatTimeFilter(list, chatTime), + hasMorePrev: false, + hasMoreNext + }; } const prevHalfSize = Math.floor(pageSize / 2);