gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hiden dataset source (#4152)

Browse Source 
* hiden dataset source

* perf: reader
This commit is contained in:
Archer 2025-03-13 21:30:40 +08:00 committed by archer
parent 268f0f56fb
commit edd8ba9e5a
No known key found for this signature in database
GPG Key ID: 4446499B846D4A9E
10 changed files with 157 additions and 262 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

204
packages/web/hooks/useLinkedScroll.tsx
View File

@ -1,10 +1,11 @@
import { useCallback, useEffect, useRef, useState, ReactNode } from 'react';
import { useEffect, useRef, useState, ReactNode, useCallback } from 'react';
import { LinkedListResponse, LinkedPaginationProps } from '../common/fetch/type';
import { Box, BoxProps } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import { useScroll, useMemoizedFn, useDebounceEffect } from 'ahooks';
import MyBox from '../components/common/MyBox';
import { useRequest2 } from './useRequest';
import { delay } from '../../global/common/system/utils';
const threshold = 100;
@ -14,92 +15,95 @@ export function useLinkedScroll<
>(
api: (data: TParams) => Promise<TData>,
{
refreshDeps = [],
pageSize = 15,
params = {},
initialId,
initialIndex,
canLoadData = false
currentData
}: {
refreshDeps?: any[];
pageSize?: number;
params?: Record<string, any>;
initialId?: string;
initialIndex?: number;
canLoadData?: boolean;
currentData?: { id: string; index: number };
}
) {
const { t } = useTranslation();
const [dataList, setDataList] = useState<TData['list']>([]);
const [hasMorePrev, setHasMorePrev] = useState(true);
const [hasMoreNext, setHasMoreNext] = useState(true);
const [initialLoadDone, setInitialLoadDone] = useState(false);
const hasScrolledToInitial = useRef(false);
const anchorRef = useRef({
top: null as { _id: string; index: number } | null,
bottom: null as { _id: string; index: number } | null
});
const containerRef = useRef<HTMLDivElement>(null);
const itemRefs = useRef<(HTMLElement | null)[]>([]);
const { runAsync: callApi, loading: isLoading } = useRequest2(
async (apiParams: TParams) => await api(apiParams),
{
onError: (error) => {
return Promise.reject(error);
}
}
);
const scrollToItem = async (id: string, retry = 3) => {
const itemIndex = dataList.findIndex((item) => item._id === id);
if (itemIndex === -1) return;
const loadData = useCallback(
async ({
id,
index,
isInitialLoad = false
}: {
id: string;
index: number;
isInitialLoad?: boolean;
}) => {
if (isLoading) return null;
const element = itemRefs.current[itemIndex];
if (!element || !containerRef.current) {
if (retry > 0) {
await delay(500);
return scrollToItem(id, retry - 1);
}
return;
}
const elementRect = element.getBoundingClientRect();
const containerRect = containerRef.current.getBoundingClientRect();
const scrollTop = containerRef.current.scrollTop + elementRect.top - containerRect.top;
containerRef.current.scrollTo({
top: scrollTop
});
};
const { runAsync: callApi, loading: isLoading } = useRequest2(api);
let scroolSign = useRef(false);
const { runAsync: loadInitData } = useRequest2(
async (scrollWhenFinish = true) => {
if (!currentData || isLoading) return;
const item = dataList.find((item) => item._id === currentData.id);
if (item) {
scrollToItem(item._id);
return;
}
const response = await callApi({
initialId: id,
initialIndex: index,
initialId: currentData.id,
initialIndex: currentData.index,
pageSize,
isInitialLoad,
...params
} as TParams);
if (!response) return null;
setHasMorePrev(response.hasMorePrev);
setHasMoreNext(response.hasMoreNext);
scroolSign.current = scrollWhenFinish;
setDataList(response.list);
if (response.list.length > 0) {
anchorRef.current.top = response.list[0];
anchorRef.current.bottom = response.list[response.list.length - 1];
}
setInitialLoadDone(true);
const scrollIndex = response.list.findIndex((item) => item._id === id);
if (scrollIndex !== -1 && itemRefs.current?.[scrollIndex]) {
setTimeout(() => {
scrollToItem(scrollIndex);
}, 100);
}
return response;
},
[callApi, params, dataList, hasMorePrev, hasMoreNext, isLoading]
{
refreshDeps: [currentData],
manual: false
}
);
useEffect(() => {
if (scroolSign.current && currentData) {
scroolSign.current = false;
scrollToItem(currentData.id);
}
}, [dataList]);
const loadPrevData = useCallback(
const { runAsync: loadPrevData, loading: prevLoading } = useRequest2(
async (scrollRef = containerRef) => {
if (!anchorRef.current.top || !hasMorePrev || isLoading) return;
@ -132,10 +136,12 @@ export function useLinkedScroll<
return response;
},
[callApi, hasMorePrev, isLoading, params, pageSize]
{
refreshDeps: [hasMorePrev, isLoading, params, pageSize]
}
);
const loadNextData = useCallback(
const { runAsync: loadNextData, loading: nextLoading } = useRequest2(
async (scrollRef = containerRef) => {
if (!anchorRef.current.bottom || !hasMoreNext || isLoading) return;
@ -165,85 +171,17 @@ export function useLinkedScroll<
return response;
},
[callApi, hasMoreNext, isLoading, params, pageSize]
);
const scrollToItem = useCallback(
(itemIndex: number) => {
if (itemIndex >= 0 && itemIndex < dataList.length && itemRefs.current?.[itemIndex]) {
try {
const element = itemRefs.current[itemIndex];
if (!element) {
return false;
}
setTimeout(() => {
if (element && containerRef.current) {
const elementRect = element.getBoundingClientRect();
const containerRect = containerRef.current.getBoundingClientRect();
const relativeTop = elementRect.top - containerRect.top;
const scrollTop =
containerRef.current.scrollTop +
relativeTop -
containerRect.height / 2 +
elementRect.height / 2;
containerRef.current.scrollTo({
top: scrollTop,
behavior: 'smooth'
});
}
}, 50);
return true;
} catch (error) {
console.error('Error scrolling to item:', error);
return false;
}
}
return false;
},
[dataList.length]
);
// 初始加载
useEffect(() => {
if (canLoadData) {
setInitialLoadDone(false);
hasScrolledToInitial.current = false;
loadData({
id: initialId || '',
index: initialIndex || 0,
isInitialLoad: true
});
{
refreshDeps: [hasMoreNext, isLoading, params, pageSize]
}
}, [canLoadData, ...refreshDeps]);
// 监听初始加载完成，执行初始滚动
useEffect(() => {
if (initialLoadDone && dataList.length > 0 && !hasScrolledToInitial.current) {
const foundIndex = dataList.findIndex((item) => item._id === initialId);
if (foundIndex >= 0) {
hasScrolledToInitial.current = true;
setTimeout(() => {
scrollToItem(foundIndex);
}, 200);
}
}
}, [initialLoadDone, ...refreshDeps]);
);
const ScrollData = useMemoizedFn(
({
children,
ScrollContainerRef,
isLoading: externalLoading,
...props
}: {
isLoading?: boolean;
children: ReactNode;
ScrollContainerRef?: React.RefObject<HTMLDivElement>;
} & BoxProps) => {
@ -252,17 +190,17 @@ export function useLinkedScroll<
useDebounceEffect(
() => {
if (!ref?.current || isLoading || !initialLoadDone) return;
if (!ref?.current || isLoading) return;
const { scrollTop, scrollHeight, clientHeight } = ref.current;
// 滚动到底部附近，加载更多下方数据
if (scrollTop + clientHeight >= scrollHeight - threshold && hasMoreNext) {
if (scrollTop + clientHeight >= scrollHeight - threshold) {
loadNextData(ref);
}
// 滚动到顶部附近，加载更多上方数据
if (scrollTop <= threshold && hasMorePrev) {
if (scrollTop <= threshold) {
loadPrevData(ref);
}
},
@ -271,20 +209,14 @@ export function useLinkedScroll<
);
return (
<MyBox
ref={ref}
h={'100%'}
overflow={'auto'}
isLoading={externalLoading || isLoading}
{...props}
>
{hasMorePrev && isLoading && initialLoadDone && (
<MyBox ref={ref} h={'100%'} overflow={'auto'} isLoading={isLoading} {...props}>
{hasMorePrev && prevLoading && (
<Box mt={2} fontSize={'xs'} color={'blackAlpha.500'} textAlign={'center'}>
{t('common:common.is_requesting')}
</Box>
)}
{children}
{hasMoreNext && isLoading && initialLoadDone && (
{hasMoreNext && nextLoading && (
<Box mt={2} fontSize={'xs'} color={'blackAlpha.500'} textAlign={'center'}>
{t('common:common.is_requesting')}
</Box>
@ -298,7 +230,7 @@ export function useLinkedScroll<
dataList,
setDataList,
isLoading,
loadData,
loadInitData,
ScrollData,
itemRefs,
scrollToItem

1
packages/web/i18n/en/chat.json
View File

@ -18,6 +18,7 @@
"contextual_preview": "Contextual Preview {{num}} Items",
"csv_input_lexicon_tip": "Only CSV batch import is supported, click to download the template",
"custom_input_guide_url": "Custom Lexicon URL",
"data_source": "Source Dataset: {{name}}",
"dataset_quote_type error": "Knowledge base reference type is wrong, correct type: { datasetId: string }[]",
"delete_all_input_guide_confirm": "Are you sure you want to clear the input guide lexicon?",
"download_chunks": "Download data",

1
packages/web/i18n/en/common.json
View File

@ -457,7 +457,6 @@
"core.chat.quote.Read Quote": "View Quote",
"core.chat.quote.afterUpdate": "After update",
"core.chat.quote.beforeUpdate": "Before update",
"core.chat.quote.source": "From: {{source}}",
"core.chat.response.Complete Response": "Complete Response",
"core.chat.response.Extension model": "Question Optimization Model",
"core.chat.response.Read complete response": "View Details",

1
packages/web/i18n/zh-CN/chat.json
View File

@ -18,6 +18,7 @@
"contextual_preview": "上下文预览 {{num}} 条",
"csv_input_lexicon_tip": "仅支持 CSV 批量导入，点击下载模板",
"custom_input_guide_url": "自定义词库地址",
"data_source": "来源知识库: {{name}}",
"dataset_quote_type error": "知识库引用类型错误，正确类型：{ datasetId: string }[]",
"delete_all_input_guide_confirm": "确定要清空输入引导词库吗？",
"download_chunks": "下载数据",

1
packages/web/i18n/zh-CN/common.json
View File

@ -460,7 +460,6 @@
"core.chat.quote.Read Quote": "查看引用",
"core.chat.quote.afterUpdate": "更新后",
"core.chat.quote.beforeUpdate": "更新前",
"core.chat.quote.source": "来源：{{source}}",
"core.chat.response.Complete Response": "完整响应",
"core.chat.response.Extension model": "问题优化模型",
"core.chat.response.Read complete response": "查看详情",

1
packages/web/i18n/zh-Hant/chat.json
View File

@ -18,6 +18,7 @@
"contextual_preview": "上下文預覽 {{num}} 筆",
"csv_input_lexicon_tip": "僅支援 CSV 批次匯入，點選下載範本",
"custom_input_guide_url": "自訂詞彙庫網址",
"data_source": "來源知識庫: {{name}}",
"dataset_quote_type error": "知識庫引用類型錯誤，正確類型：{ datasetId: string }[]",
"delete_all_input_guide_confirm": "確定要清除輸入導引詞彙庫嗎？",
"download_chunks": "下載數據",

1
packages/web/i18n/zh-Hant/common.json
View File

@ -456,7 +456,6 @@
"core.chat.quote.Read Quote": "檢視引用",
"core.chat.quote.afterUpdate": "更新後",
"core.chat.quote.beforeUpdate": "更新前",
"core.chat.quote.source": "來源：{{source}}",
"core.chat.response.Complete Response": "完整回應",
"core.chat.response.Extension model": "問題最佳化模型",
"core.chat.response.Read complete response": "檢視詳細資料",

5
projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteItem.tsx
View File

@ -175,10 +175,7 @@ const CollectionQuoteItem = ({
{editInputData && (
<InputDataModal
onClose={() => setEditInputData(undefined)}
onSuccess={() => {
console.log('onSuccess');
refreshList();
}}
onSuccess={refreshList}
dataId={editInputData.dataId}
collectionId={editInputData.collectionId}
/>

175
projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx
View File

@ -7,7 +7,7 @@ import { useTranslation } from 'react-i18next';
import DownloadButton from './DownloadButton';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { downloadFetch } from '@/web/common/system/utils';
import { useCallback, useEffect, useMemo, useState } from 'react';
import { useMemo, useState } from 'react';
import { getDatasetDataPermission } from '@/web/core/dataset/api';
import ScoreTag from './ScoreTag';
import { formatScore } from '@/components/core/dataset/QuoteItem';
@ -39,51 +39,53 @@ const CollectionReader = ({
const [quoteIndex, setQuoteIndex] = useState(0);
// Get dataset permission
const { data: datasetData, loading: isPermissionLoading } = useRequest2(
async () => await getDatasetDataPermission(datasetId),
{
manual: !userInfo || !datasetId,
refreshDeps: [datasetId, userInfo]
}
);
const { data: datasetData } = useRequest2(async () => await getDatasetDataPermission(datasetId), {
manual: !userInfo || !datasetId,
refreshDeps: [datasetId, userInfo]
});
const filterResults = useMemo(() => {
const results = rawSearch.filter(
(item) => item.collectionId === metadata.collectionId && item.sourceId === metadata.sourceId
);
return results.sort((a, b) => (a.chunkIndex || 0) - (b.chunkIndex || 0));
}, [metadata, rawSearch]);
const currentQuoteItem = filterResults[quoteIndex];
setQuoteIndex(0);
return rawSearch
.filter((item) => item.collectionId === collectionId)
.sort((a, b) => (a.chunkIndex || 0) - (b.chunkIndex || 0));
}, [collectionId, rawSearch]);
const currentQuoteItem = useMemo(() => {
const item = filterResults[quoteIndex];
if (item) {
return {
id: item.id,
index: item.chunkIndex,
score: item.score
};
}
}, [filterResults, quoteIndex]);
// Get quote list
const {
dataList: datasetDataList,
setDataList: setDatasetDataList,
isLoading,
loadData,
ScrollData,
itemRefs,
scrollToItem
} = useLinkedScroll(getCollectionQuote, {
refreshDeps: [collectionId],
params: {
const params = useMemo(
() => ({
collectionId,
chatItemDataId,
chatId: metadata.chatId,
appId: metadata.appId,
...metadata.outLinkAuthData
},
initialId: currentQuoteItem?.id,
initialIndex: currentQuoteItem?.chunkIndex,
canLoadData: !!currentQuoteItem?.id && !isPermissionLoading
}),
[metadata]
);
const {
dataList: datasetDataList,
isLoading,
ScrollData,
itemRefs,
loadInitData
} = useLinkedScroll(getCollectionQuote, {
params,
currentData: currentQuoteItem
});
const loading = isLoading || isPermissionLoading;
const isDeleted = useMemo(
() => !datasetDataList.find((item) => item._id === currentQuoteItem?.id),
[datasetDataList, currentQuoteItem?.id]
() => !isLoading && !datasetDataList.find((item) => item._id === currentQuoteItem?.id),
[datasetDataList, currentQuoteItem?.id, isLoading]
);
const formatedDataList = useMemo(
@ -101,11 +103,6 @@ const CollectionReader = ({
[currentQuoteItem?.id, datasetDataList, filterResults]
);
useEffect(() => {
setQuoteIndex(0);
setDatasetDataList([]);
}, [collectionId, setDatasetDataList]);
const { runAsync: handleDownload } = useRequest2(async () => {
await downloadFetch({
url: '/api/core/dataset/collection/export',
@ -119,30 +116,6 @@ const CollectionReader = ({
const handleRead = getCollectionSourceAndOpen(metadata);
const handleNavigate = useCallback(
async (targetIndex: number) => {
if (targetIndex < 0 || targetIndex >= filterResults.length) return;
const targetItemId = filterResults[targetIndex].id;
const targetItemIndex = filterResults[targetIndex].chunkIndex;
setQuoteIndex(targetIndex);
const dataIndex = datasetDataList.findIndex((item) => item._id === targetItemId);
if (dataIndex !== -1) {
setTimeout(() => {
scrollToItem(dataIndex);
}, 50);
} else {
try {
await loadData({ id: targetItemId, index: targetItemIndex });
} catch (error) {
console.error('Failed to navigate:', error);
}
}
},
[filterResults, datasetDataList, scrollToItem, loadData]
);
return (
<MyBox display={'flex'} flexDirection={'column'} h={'full'}>
{/* title */}
@ -163,6 +136,16 @@ const CollectionReader = ({
fontSize={'sm'}
color={'myGray.900'}
fontWeight={'medium'}
{...(!!userInfo &&
datasetData?.permission?.hasReadPer && {
cursor: 'pointer',
_hover: { color: 'primary.600', textDecoration: 'underline' },
onClick: () => {
router.push(
`/dataset/detail?datasetId=${datasetId}&currentTab=dataCard&collectionId=${collectionId}`
);
}
})}
>
{sourceName || t('common:common.UnKnow Source')}
</Box>
@ -181,26 +164,22 @@ const CollectionReader = ({
onClick={onClose}
/>
</HStack>
{!isPermissionLoading && (
{datasetData?.permission?.hasReadPer && (
<Box
fontSize={'mini'}
color={'myGray.500'}
onClick={() => {
if (!!userInfo && datasetData?.permission?.hasReadPer) {
router.push(
`/dataset/detail?datasetId=${datasetId}&currentTab=dataCard&collectionId=${collectionId}`
);
}
}}
{...(!!userInfo && datasetData?.permission?.hasReadPer
{...(!!userInfo
? {
cursor: 'pointer',
_hover: { color: 'primary.600', textDecoration: 'underline' }
_hover: { color: 'primary.600', textDecoration: 'underline' },
onClick: () => {
router.push(`/dataset/detail?datasetId=${datasetId}`);
}
}
: {})}
>
{t('common:core.chat.quote.source', {
source: datasetData?.datasetName
{t('chat:data_source', {
name: datasetData.datasetName
})}
</Box>
)}
@ -231,23 +210,22 @@ const CollectionReader = ({
</Flex>
{/* 检索分数 */}
{!loading &&
(!isDeleted ? (
<ScoreTag {...formatScore(currentQuoteItem?.score)} />
) : (
<Flex
borderRadius={'sm'}
py={1}
px={2}
color={'red.600'}
bg={'red.50'}
alignItems={'center'}
fontSize={'11px'}
>
<MyIcon name="common/info" w={'14px'} mr={1} color={'red.600'} />
{t('chat:chat.quote.deleted')}
</Flex>
))}
{currentQuoteItem?.score ? (
<ScoreTag {...formatScore(currentQuoteItem?.score)} />
) : isDeleted ? (
<Flex
borderRadius={'sm'}
py={1}
px={2}
color={'red.600'}
bg={'red.50'}
alignItems={'center'}
fontSize={'11px'}
>
<MyIcon name="common/info" w={'14px'} mr={1} color={'red.600'} />
{t('chat:chat.quote.deleted')}
</Flex>
) : null}
<Box flex={1} />
@ -256,12 +234,12 @@ const CollectionReader = ({
<NavButton
direction="up"
isDisabled={quoteIndex === 0}
onClick={() => handleNavigate(quoteIndex - 1)}
onClick={() => setQuoteIndex(quoteIndex - 1)}
/>
<NavButton
direction="down"
isDisabled={quoteIndex === filterResults.length - 1}
onClick={() => handleNavigate(quoteIndex + 1)}
onClick={() => setQuoteIndex(quoteIndex + 1)}
/>
</Flex>
</Flex>
@ -272,8 +250,8 @@ const CollectionReader = ({
)}
{/* quote list */}
{loading || datasetDataList.length > 0 ? (
<ScrollData flex={'1 0 0'} mt={2} px={5} py={1} isLoading={loading}>
{isLoading || datasetDataList.length > 0 ? (
<ScrollData flex={'1 0 0'} mt={2} px={5} py={1}>
<Flex flexDir={'column'}>
{formatedDataList.map((item, index) => (
<CollectionQuoteItem
@ -282,10 +260,7 @@ const CollectionReader = ({
quoteRefs={itemRefs as React.MutableRefObject<(HTMLDivElement | null)[]>}
quoteIndex={item.quoteIndex}
setQuoteIndex={setQuoteIndex}
refreshList={() =>
currentQuoteItem?.id &&
loadData({ id: currentQuoteItem.id, index: currentQuoteItem.chunkIndex })
}
refreshList={() => loadInitData(false)}
updated={item.updated}
isCurrentSelected={item.isCurrentSelected}
q={item.q}

29
projects/app/src/pages/api/core/chat/quote/getCollectionQuote.ts
View File

@ -13,7 +13,6 @@ export type GetCollectionQuoteProps = LinkedPaginationProps & {
chatId: string;
chatItemDataId: string;
isInitialLoad: boolean;
collectionId: string;
appId: string;
@ -37,7 +36,6 @@ async function handler(
prevIndex,
nextId,
nextIndex,
isInitialLoad,
collectionId,
chatItemDataId,
@ -84,7 +82,6 @@ async function handler(
initialIndex,
pageSize: limitedPageSize,
chatTime: chatItem.time,
isInitialLoad,
baseMatch
});
}
@ -111,14 +108,12 @@ async function handleInitialLoad({
initialIndex,
pageSize,
chatTime,
isInitialLoad,
baseMatch
}: {
initialId: string;
initialIndex: number;
pageSize: number;
chatTime: Date;
isInitialLoad: boolean;
baseMatch: BaseMatchType;
}): Promise<GetCollectionQuoteRes> {
const centerNode = await MongoDatasetData.findOne(
@ -129,22 +124,18 @@ async function handleInitialLoad({
).lean();
if (!centerNode) {
if (isInitialLoad) {
const list = await MongoDatasetData.find(baseMatch, quoteDataFieldSelector)
.sort({ chunkIndex: 1, _id: -1 })
.limit(pageSize)
.lean();
const list = await MongoDatasetData.find(baseMatch, quoteDataFieldSelector)
.sort({ chunkIndex: 1, _id: -1 })
.limit(pageSize)
.lean();
const hasMoreNext = list.length === pageSize;
const hasMoreNext = list.length === pageSize;
return {
list: processChatTimeFilter(list, chatTime),
hasMorePrev: false,
hasMoreNext
};
}
return Promise.reject('centerNode not found');
return {
list: processChatTimeFilter(list, chatTime),
hasMorePrev: false,
hasMoreNext
};
}
const prevHalfSize = Math.floor(pageSize / 2);