diff --git a/docSite/assets/imgs/collection-tags-1.png b/docSite/assets/imgs/collection-tags-1.png new file mode 100644 index 000000000..e8b799580 Binary files /dev/null and b/docSite/assets/imgs/collection-tags-1.png differ diff --git a/docSite/assets/imgs/collection-tags-2.png b/docSite/assets/imgs/collection-tags-2.png new file mode 100644 index 000000000..6772b43cd Binary files /dev/null and b/docSite/assets/imgs/collection-tags-2.png differ diff --git a/docSite/assets/imgs/collection-tags-3.png b/docSite/assets/imgs/collection-tags-3.png new file mode 100644 index 000000000..04784d170 Binary files /dev/null and b/docSite/assets/imgs/collection-tags-3.png differ diff --git a/docSite/content/zh-cn/docs/course/chat_input_guide.md b/docSite/content/zh-cn/docs/course/chat_input_guide.md index 0e99b2c3e..07c8436b0 100644 --- a/docSite/content/zh-cn/docs/course/chat_input_guide.md +++ b/docSite/content/zh-cn/docs/course/chat_input_guide.md @@ -4,7 +4,7 @@ description: "FastGPT 对话问题引导" icon: "code" draft: false toc: true -weight: 350 +weight: 108 --- ![](/imgs/questionGuide.png) diff --git a/docSite/content/zh-cn/docs/course/collection_tags.md b/docSite/content/zh-cn/docs/course/collection_tags.md new file mode 100644 index 000000000..212d168f2 --- /dev/null +++ b/docSite/content/zh-cn/docs/course/collection_tags.md @@ -0,0 +1,50 @@ +--- +title: "知识库集合标签" +description: "FastGPT 知识库集合标签使用说明" +icon: "developer_guide" +draft: false +toc: true +weight: 108 +--- + +知识库集合标签是 FastGPT 商业版特有功能。它允许你对知识库中的数据集合添加标签进行分类，更高效地管理知识库数据。 + +而进一步可以在问答中，搜索知识库时添加集合过滤，实现更精确的搜索。 + +| | | | +| --------------------- | --------------------- | --------------------- | +| ![](/imgs/collection-tags-1.png) | ![](/imgs/collection-tags-2.png) | ![](/imgs/collection-tags-3.png) | + +## 标签基础操作说明 + +在知识库详情页面，可以对标签进行管理，可执行的操作有 + +- 创建标签 +- 修改标签名 +- 删除标签 +- 将一个标签赋给多个数据集合 +- 给一个数据集合添加多个标签 + +也可以利用标签对数据集合进行筛选 + +## 知识库搜索-集合过滤说明 + +利用标签可以在知识库搜索时，通过填写「集合过滤」这一栏来实现更精确的搜索，具体的填写示例如下 + +```json +{ + "tags": { + "$and": ["标签 1","标签 2"], + "$or": ["有 $and 标签时，and 生效，or 不生效"] + }, + "createTime": { + "$gte": "YYYY-MM-DD HH:mm 格式即可，集合的创建时间大于该时间", + "$lte": "YYYY-MM-DD HH:mm 格式即可，集合的创建时间小于该时间,可和 $gte 共同使用" + } +} +``` + +在填写时有两个注意的点， + +- 标签值可以为 `string` 类型的标签名，也可以为 `null`，而 `null` 代表着未设置标签的数据集合 +- 标签过滤有 `$and` 和 `$or` 两种条件类型，在同时设置了 `$and` 和 `$or` 的情况下，只有 `$and` 会生效 diff --git a/packages/global/core/dataset/api.d.ts b/packages/global/core/dataset/api.d.ts index fcbd54797..00c20cb18 100644 --- a/packages/global/core/dataset/api.d.ts +++ b/packages/global/core/dataset/api.d.ts @@ -74,6 +74,23 @@ export type ExternalFileCreateDatasetCollectionParams = ApiCreateDatasetCollecti filename?: string; }; +/* ================= tag ===================== */ +export type CreateDatasetCollectionTagParams = { + datasetId: string; + tag: string; +}; +export type AddTagsToCollectionsParams = { + originCollectionIds: string[]; + collectionIds: string[]; + datasetId: string; + tag: string; +}; +export type UpdateDatasetCollectionTagParams = { + datasetId: string; + tagId: string; + tag: string; +}; + /* ================= data ===================== */ export type PgSearchRawType = { id: string; diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index 6abd3bd8b..104c67b09 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -69,6 +69,13 @@ export type DatasetCollectionSchemaType = { }; }; +export type DatasetCollectionTagsSchemaType = { + _id: string; + teamId: string; + datasetId: string; + tag: string; +}; + export type DatasetDataIndexItemType = { defaultIndex: boolean; dataId: string; // pg data id @@ -144,6 +151,17 @@ export type DatasetItemType = Omit => { const client = await this.getClient(); - const { teamId, datasetIds, vector, limit, forbidCollectionIdList, retry = 2 } = props; + const { + teamId, + datasetIds, + vector, + limit, + forbidCollectionIdList, + filterCollectionIdList, + retry = 2 + } = props; + // Forbid collection + const formatForbidCollectionIdList = (() => { + if (!filterCollectionIdList) return forbidCollectionIdList; + const list = forbidCollectionIdList + .map((id) => String(id)) + .filter((id) => !filterCollectionIdList.includes(id)); + return list; + })(); const forbidColQuery = - forbidCollectionIdList.length > 0 - ? `and (collectionId not in [${forbidCollectionIdList.map((id) => `"${String(id)}"`).join(',')}])` + formatForbidCollectionIdList.length > 0 + ? `and (collectionId not in [${formatForbidCollectionIdList.map((id) => `"${id}"`).join(',')}])` : ''; + // filter collection id + const formatFilterCollectionId = (() => { + if (!filterCollectionIdList) return; + return filterCollectionIdList + .map((id) => String(id)) + .filter((id) => !forbidCollectionIdList.includes(id)); + })(); + const collectionIdQuery = formatFilterCollectionId + ? `and (collectionId in [${formatFilterCollectionId.map((id) => `"${id}"`)}])` + : ``; + // Empty data + if (formatFilterCollectionId && formatFilterCollectionId.length === 0) { + return { results: [] }; + } + try { const { results } = await client.search({ collection_name: DatasetVectorTableName, data: vector, limit, - filter: `(teamId == "${teamId}") and (datasetId in [${datasetIds.map((id) => `"${String(id)}"`).join(',')}]) ${forbidColQuery}`, + filter: `(teamId == "${teamId}") and (datasetId in [${datasetIds.map((id) => `"${id}"`).join(',')}]) ${collectionIdQuery} ${forbidColQuery}`, output_fields: ['collectionId'] }); diff --git a/packages/service/common/vectorStore/pg/class.ts b/packages/service/common/vectorStore/pg/class.ts index a4cd2362d..ebd80f17d 100644 --- a/packages/service/common/vectorStore/pg/class.ts +++ b/packages/service/common/vectorStore/pg/class.ts @@ -119,14 +119,44 @@ export class PgVectorCtrl { } }; embRecall = async (props: EmbeddingRecallCtrlProps): Promise => { - const { teamId, datasetIds, vector, limit, forbidCollectionIdList, retry = 2 } = props; + const { + teamId, + datasetIds, + vector, + limit, + forbidCollectionIdList, + filterCollectionIdList, + retry = 2 + } = props; + // Get forbid collection + const formatForbidCollectionIdList = (() => { + if (!filterCollectionIdList) return forbidCollectionIdList; + const list = forbidCollectionIdList + .map((id) => String(id)) + .filter((id) => !filterCollectionIdList.includes(id)); + return list; + })(); const forbidCollectionSql = - forbidCollectionIdList.length > 0 - ? `AND collection_id NOT IN (${forbidCollectionIdList.map((id) => `'${String(id)}'`).join(',')})` - : 'AND collection_id IS NOT NULL'; - // const forbidDataSql = - // forbidEmbIndexIdList.length > 0 ? `AND id NOT IN (${forbidEmbIndexIdList.join(',')})` : ''; + formatForbidCollectionIdList.length > 0 + ? `AND collection_id NOT IN (${formatForbidCollectionIdList.map((id) => `'${id}'`).join(',')})` + : ''; + + // Filter by collectionId + const formatFilterCollectionId = (() => { + if (!filterCollectionIdList) return; + + return filterCollectionIdList + .map((id) => String(id)) + .filter((id) => !forbidCollectionIdList.includes(id)); + })(); + const filterCollectionIdSql = formatFilterCollectionId + ? `AND collection_id IN (${formatFilterCollectionId.map((id) => `'${id}'`).join(',')})` + : ''; + // Empty data + if (formatFilterCollectionId && formatFilterCollectionId.length === 0) { + return { results: [] }; + } try { // const explan: any = await PgClient.query( @@ -150,6 +180,7 @@ export class PgVectorCtrl { from ${DatasetVectorTableName} where team_id='${teamId}' AND dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')}) + ${filterCollectionIdSql} ${forbidCollectionSql} order by score limit ${limit}; COMMIT;` diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts index 2e1b35722..beb158180 100644 --- a/packages/service/core/dataset/collection/schema.ts +++ b/packages/service/core/dataset/collection/schema.ts @@ -106,8 +106,10 @@ try { updateTime: -1 }); - // get forbid - // DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, forbid: 1 }); + // Tag filter + DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, tags: 1 }); + // create time filter + DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, createTime: 1 }); } catch (error) { console.log(error); } diff --git a/packages/service/core/dataset/search/controller.ts b/packages/service/core/dataset/search/controller.ts index e7536da05..134b6f08b 100644 --- a/packages/service/core/dataset/search/controller.ts +++ b/packages/service/core/dataset/search/controller.ts @@ -20,6 +20,9 @@ import { hashStr } from '@fastgpt/global/common/string/tools'; import { jiebaSplit } from '../../../common/string/jieba'; import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils'; import { Types } from '../../../common/mongo'; +import json5 from 'json5'; +import { MongoDatasetCollectionTags } from '../tag/schema'; +import { readFromSecondary } from '../../../common/mongo/utils'; type SearchDatasetDataProps = { teamId: string; @@ -31,6 +34,20 @@ type SearchDatasetDataProps = { usingReRank?: boolean; reRankQuery: string; queries: string[]; + + /* + { + tags: { + $and: ["str1","str2"], + $or: ["str1","str2",null] null means no tags + }, + createTime: { + $gte: 'xx', + $lte: 'xxx' + } + } + */ + collectionFilterMatch?: string; }; export async function searchDatasetData(props: SearchDatasetDataProps) { @@ -43,7 +60,8 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { limit: maxTokens, searchMode = DatasetSearchModeEnum.embedding, usingReRank = false, - datasetIds = [] + datasetIds = [], + collectionFilterMatch } = props; /* init params */ @@ -87,14 +105,148 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { forbidCollectionIdList: collections.map((item) => String(item._id)) }; }; + /* + Collection metadata filter + 标签过滤： + 1. and 先生效 + 2. and 标签和 null 不能共存，否则返回空数组 + */ + const filterCollectionByMetadata = async (): Promise => { + if (!collectionFilterMatch || !global.feConfigs.isPlus) return; + + let tagCollectionIdList: string[] | undefined = undefined; + let createTimeCollectionIdList: string[] | undefined = undefined; + + try { + const jsonMatch = json5.parse(collectionFilterMatch); + + // Tag + let andTags = jsonMatch?.tags?.$and as (string | null)[] | undefined; + let orTags = jsonMatch?.tags?.$or as (string | null)[] | undefined; + + // get andTagIds + if (andTags && andTags.length > 0) { + // tag 去重 + andTags = Array.from(new Set(andTags)); + + if (andTags.includes(null) && andTags.some((tag) => typeof tag === 'string')) { + return []; + } + + if (andTags.every((tag) => typeof tag === 'string')) { + // Get tagId by tag string + const andTagIdList = await MongoDatasetCollectionTags.find( + { + teamId, + datasetId: { $in: datasetIds }, + tag: { $in: andTags } + }, + '_id', + { + ...readFromSecondary + } + ).lean(); + + // If you enter a tag that does not exist, none will be found + if (andTagIdList.length !== andTags.length) return []; + + // Get collectionId by tagId + const collections = await MongoDatasetCollection.find( + { + teamId, + datasetId: { $in: datasetIds }, + tags: { $all: andTagIdList.map((item) => String(item._id)) } + }, + '_id', + { + ...readFromSecondary + } + ).lean(); + tagCollectionIdList = collections.map((item) => String(item._id)); + } else if (andTags.every((tag) => tag === null)) { + const collections = await MongoDatasetCollection.find( + { + teamId, + datasetId: { $in: datasetIds }, + $or: [{ tags: { $size: 0 } }, { tags: { $exists: false } }] + }, + '_id', + { + ...readFromSecondary + } + ).lean(); + tagCollectionIdList = collections.map((item) => String(item._id)); + } + } else if (orTags && orTags.length > 0) { + // Get tagId by tag string + const orTagArray = await MongoDatasetCollectionTags.find( + { + teamId, + datasetId: { $in: datasetIds }, + tag: { $in: orTags.filter((tag) => tag !== null) } + }, + '_id', + { ...readFromSecondary } + ).lean(); + const orTagIds = orTagArray.map((item) => String(item._id)); + + // Get collections by tagId + const collections = await MongoDatasetCollection.find( + { + teamId, + datasetId: { $in: datasetIds }, + $or: [ + { tags: { $in: orTagIds } }, + ...(orTags.includes(null) ? [{ tags: { $size: 0 } }] : []) + ] + }, + '_id', + { ...readFromSecondary } + ).lean(); + + tagCollectionIdList = collections.map((item) => String(item._id)); + } + + // time + const getCreateTime = jsonMatch?.createTime?.$gte as string | undefined; + const lteCreateTime = jsonMatch?.createTime?.$lte as string | undefined; + if (getCreateTime || lteCreateTime) { + const collections = await MongoDatasetCollection.find( + { + teamId, + datasetId: { $in: datasetIds }, + createTime: { + ...(getCreateTime && { $gte: new Date(getCreateTime) }), + ...(lteCreateTime && { + $lte: new Date(lteCreateTime) + }) + } + }, + '_id' + ); + createTimeCollectionIdList = collections.map((item) => String(item._id)); + } + + // Concat tag and time + if (tagCollectionIdList && createTimeCollectionIdList) { + return tagCollectionIdList.filter((id) => createTimeCollectionIdList!.includes(id)); + } else if (tagCollectionIdList) { + return tagCollectionIdList; + } else if (createTimeCollectionIdList) { + return createTimeCollectionIdList; + } + } catch (error) {} + }; const embeddingRecall = async ({ query, limit, - forbidCollectionIdList + forbidCollectionIdList, + filterCollectionIdList }: { query: string; limit: number; forbidCollectionIdList: string[]; + filterCollectionIdList?: string[]; }) => { const { vectors, tokens } = await getVectorsByText({ model: getVectorModel(model), @@ -107,7 +259,8 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { datasetIds, vector: vectors[0], limit, - forbidCollectionIdList + forbidCollectionIdList, + filterCollectionIdList }); // get q and a @@ -165,10 +318,12 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { }; const fullTextRecall = async ({ query, - limit + limit, + filterCollectionIdList }: { query: string; limit: number; + filterCollectionIdList?: string[]; }): Promise<{ fullTextRecallResults: SearchDataResponseItemType[]; tokenLen: number; @@ -188,7 +343,14 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { $match: { teamId: new Types.ObjectId(teamId), datasetId: new Types.ObjectId(id), - $text: { $search: jiebaSplit({ text: query }) } + $text: { $search: jiebaSplit({ text: query }) }, + ...(filterCollectionIdList && filterCollectionIdList.length > 0 + ? { + collectionId: { + $in: filterCollectionIdList.map((id) => new Types.ObjectId(id)) + } + } + : {}) } }, { @@ -327,19 +489,24 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { const fullTextRecallResList: SearchDataResponseItemType[][] = []; let totalTokens = 0; - const { forbidCollectionIdList } = await getForbidData(); - + const [{ forbidCollectionIdList }, filterCollectionIdList] = await Promise.all([ + getForbidData(), + filterCollectionByMetadata() + ]); + console.log(filterCollectionIdList, '==='); await Promise.all( queries.map(async (query) => { const [{ tokens, embeddingRecallResults }, { fullTextRecallResults }] = await Promise.all([ embeddingRecall({ query, limit: embeddingLimit, - forbidCollectionIdList + forbidCollectionIdList, + filterCollectionIdList }), fullTextRecall({ query, - limit: fullTextLimit + limit: fullTextLimit, + filterCollectionIdList }) ]); totalTokens += tokens; diff --git a/packages/service/core/dataset/tag/schema.ts b/packages/service/core/dataset/tag/schema.ts new file mode 100644 index 000000000..71e0462a8 --- /dev/null +++ b/packages/service/core/dataset/tag/schema.ts @@ -0,0 +1,35 @@ +import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; +import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; +import { DatasetCollectionName } from '../schema'; +import { DatasetCollectionTagsSchemaType } from '@fastgpt/global/core/dataset/type'; +const { Schema } = connectionMongo; + +export const DatasetCollectionTagsName = 'dataset_collection_tags'; + +const DatasetCollectionTagsSchema = new Schema({ + teamId: { + type: Schema.Types.ObjectId, + ref: TeamCollectionName, + required: true + }, + datasetId: { + type: Schema.Types.ObjectId, + ref: DatasetCollectionName, + required: true + }, + tag: { + type: String, + required: true + } +}); + +try { + DatasetCollectionTagsSchema.index({ teamId: 1, datasetId: 1, tag: 1 }); +} catch (error) { + console.log(error); +} + +export const MongoDatasetCollectionTags = getMongoModel( + DatasetCollectionTagsName, + DatasetCollectionTagsSchema +); diff --git a/packages/service/core/workflow/dispatch/dataset/search.ts b/packages/service/core/workflow/dispatch/dataset/search.ts index e86402372..c5abff3c3 100644 --- a/packages/service/core/workflow/dispatch/dataset/search.ts +++ b/packages/service/core/workflow/dispatch/dataset/search.ts @@ -27,6 +27,7 @@ type DatasetSearchProps = ModuleDispatchProps<{ [NodeInputKeyEnum.datasetSearchUsingExtensionQuery]: boolean; [NodeInputKeyEnum.datasetSearchExtensionModel]: string; [NodeInputKeyEnum.datasetSearchExtensionBg]: string; + [NodeInputKeyEnum.collectionFilterMatch]: string; }>; export type DatasetSearchResponse = DispatchNodeResultType<{ [NodeOutputKeyEnum.datasetQuoteQA]: SearchDataResponseItemType[]; @@ -49,7 +50,8 @@ export async function dispatchDatasetSearch( datasetSearchUsingExtensionQuery, datasetSearchExtensionModel, - datasetSearchExtensionBg + datasetSearchExtensionBg, + collectionFilterMatch } } = props as DatasetSearchProps; @@ -99,7 +101,8 @@ export async function dispatchDatasetSearch( limit, datasetIds: datasets.map((item) => item.datasetId), searchMode, - usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)) + usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)), + collectionFilterMatch }); // count bill results diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts index b8f179daa..bb61d857a 100644 --- a/packages/web/components/common/Icon/constants.ts +++ b/packages/web/components/common/Icon/constants.ts @@ -131,6 +131,7 @@ export const iconPaths = { 'core/dataset/rerank': () => import('./icons/core/dataset/rerank.svg'), 'core/dataset/splitLight': () => import('./icons/core/dataset/splitLight.svg'), 'core/dataset/tableCollection': () => import('./icons/core/dataset/tableCollection.svg'), + 'core/dataset/tag': () => import('./icons/core/dataset/tag.svg'), 'core/dataset/websiteDataset': () => import('./icons/core/dataset/websiteDataset.svg'), 'core/modules/basicNode': () => import('./icons/core/modules/basicNode.svg'), 'core/modules/fixview': () => import('./icons/core/modules/fixview.svg'), diff --git a/packages/web/components/common/Icon/icons/core/dataset/tag.svg b/packages/web/components/common/Icon/icons/core/dataset/tag.svg new file mode 100644 index 000000000..c8bd2d35a --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/dataset/tag.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/packages/web/components/common/MyModal/index.tsx b/packages/web/components/common/MyModal/index.tsx index 10a60d15c..c01253a66 100644 --- a/packages/web/components/common/MyModal/index.tsx +++ b/packages/web/components/common/MyModal/index.tsx @@ -21,6 +21,7 @@ export interface MyModalProps extends ModalContentProps { isLoading?: boolean; isOpen: boolean; onClose?: () => void; + closeOnOverlayClick?: boolean; } const MyModal = ({ @@ -33,6 +34,7 @@ const MyModal = ({ isLoading, w = 'auto', maxW = ['90vw', '600px'], + closeOnOverlayClick = true, ...props }: MyModalProps) => { const isPc = useSystem(); @@ -44,6 +46,7 @@ const MyModal = ({ autoFocus={false} isCentered={isPc ? isCentered : true} blockScrollOnMount={false} + closeOnOverlayClick={closeOnOverlayClick} > void }) => React.ReactNode; + onCloseFunc?: () => void; + closeOnBlur?: boolean; +} + const MyPopover = ({ Trigger, placement, offset, trigger, - children -}: { - Trigger: React.ReactNode; - placement?: PlacementWithLogical; - offset?: [number, number]; - trigger?: 'hover' | 'click'; - children: (e: { onClose: () => void }) => React.ReactNode; -}) => { + hasArrow = true, + children, + onCloseFunc, + closeOnBlur = false, + ...props +}: Props) => { const firstFieldRef = React.useRef(null); const { onOpen, onClose, isOpen } = useDisclosure(); @@ -30,10 +40,13 @@ const MyPopover = ({ isOpen={isOpen} initialFocusRef={firstFieldRef} onOpen={onOpen} - onClose={onClose} + onClose={() => { + onClose(); + onCloseFunc && onCloseFunc(); + }} placement={placement} offset={offset} - closeOnBlur={false} + closeOnBlur={closeOnBlur} trigger={trigger} openDelay={100} closeDelay={100} @@ -41,8 +54,8 @@ const MyPopover = ({ lazyBehavior="keepMounted" > {Trigger} - - + + {hasArrow && } {children({ onClose })} diff --git a/packages/web/hooks/useScrollPagination.tsx b/packages/web/hooks/useScrollPagination.tsx index 73d3e79d7..c8055dfc5 100644 --- a/packages/web/hooks/useScrollPagination.tsx +++ b/packages/web/hooks/useScrollPagination.tsx @@ -7,7 +7,6 @@ import { useBoolean, useLockFn, useMemoizedFn, - useMount, useScroll, useVirtualList, useRequest @@ -50,6 +49,7 @@ export function useScrollPagination< const { toast } = useToast(); const [current, setCurrent] = useState(1); const [data, setData] = useState([]); + const [total, setTotal] = useState(0); const [isLoading, { setTrue, setFalse }] = useBoolean(false); const [list] = useVirtualList(data, { @@ -71,6 +71,7 @@ export function useScrollPagination< ...defaultParams } as TParams); + setTotal(res.total); setCurrent(num); if (num === 1) { @@ -146,6 +147,7 @@ export function useScrollPagination< return { containerRef, list, + total, data, setData, isLoading, diff --git a/packages/web/i18n/en/common.json b/packages/web/i18n/en/common.json index 228b347fa..7df02d575 100644 --- a/packages/web/i18n/en/common.json +++ b/packages/web/i18n/en/common.json @@ -1,7 +1,5 @@ { "App": "App", - "click_to_resume": "Resume", - "code_editor": "Code edit", "Export": "Export", "Folder": "Folder", "Login": "Login", @@ -13,6 +11,8 @@ "UnKnow": "Unknown", "Warning": "Warning", "add_new": "Add new", + "click_to_resume": "Resume", + "code_editor": "Code edit", "common": { "Action": "Action", "Add": "Add", @@ -91,6 +91,8 @@ "Root folder": "Root folder", "Run": "Run", "Save": "Save", + "Save Failed": "Saved failed", + "Save Success": "Saved success", "Search": "Search", "Select File Failed": "Select File Failed", "Select template": "Select template", @@ -482,7 +484,8 @@ "success": "Start syncing" } }, - "training": {} + "training": { + } }, "data": { "Auxiliary Data": "Auxiliary data", @@ -505,13 +508,13 @@ "Data not found": "Data does not exist or has been deleted", "Start Sync Failed": "Failed to start syncing", "Template does not exist": "Template does not exist", + "invalidVectorModelOrQAModel": "Invalid vector model or QA model", "unAuthDataset": "Unauthorized to operate this dataset", "unAuthDatasetCollection": "Unauthorized to operate this collection", "unAuthDatasetData": "Unauthorized to operate this data", "unAuthDatasetFile": "Unauthorized to operate this file", "unCreateCollection": "Unauthorized to operate this data", - "unLinkCollection": "Not a network link collection", - "invalidVectorModelOrQAModel": "Invalid vector model or QA model" + "unLinkCollection": "Not a network link collection" }, "externalFile": "external file repository", "file": "File", diff --git a/packages/web/i18n/en/dataset.json b/packages/web/i18n/en/dataset.json index 4ab7210a0..42fa06faf 100644 --- a/packages/web/i18n/en/dataset.json +++ b/packages/web/i18n/en/dataset.json @@ -1,26 +1,41 @@ { + "Disabled": "Disabled", + "Enable": "Enable", + "Enabled": "Enabled", + "collection": { + "Create update time": "Create/Update time", + "Training type": "Training type" + }, "collection_tags": "Tags", "common_dataset": "Common dataset", "common_dataset_desc": "Can be built by importing files, web links, or manual entry", "confirm_to_rebuild_embedding_tip": "Are you sure to switch the knowledge base index?\nSwitching index is a very heavy operation that requires re-indexing all the data in your knowledge base, which may take a long time. Please ensure that the remaining points in your account are sufficient.\n\nIn addition, you need to be careful to modify the applications that select this knowledge base to avoid mixing them with other index model knowledge bases.", - "Disabled": "Disabled", - "Enable": "Enable", - "Enabled": "Enabled", + "dataset": { + "no_collections": "no collections", + "no_tags": "no tags" + }, "external_file": "External file", "external_file_dataset_desc": "You can import files from an external file library to build a knowledge base. Files are not stored twice", "external_id": "File id", "external_read_url": "External read url", "external_read_url_tip": "You can configure the reading address of your file library. This allows users to read and authenticate. You can currently use the {{fileId}} variable to refer to the external file ID.", "external_url": "File read url", + "filename": "filename", "folder_dataset": "Folder", "rebuild_embedding_start_tip": "The task of switching index models has begun", "rebuilding_index_count": "Rebuilding count: {{count}}", + "tag": { + "Add New": "Add new", + "Add_new_tag": "Add new tag", + "Edit_tag": "Edit tag", + "add": "Add", + "cancel": "Cancel", + "delete_tag_confirm": "Confirm to delete tag", + "manage": "Manage", + "searchOrAddTag": "Search or add tags", + "tags": "Tags" + }, "the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "The knowledge base has indexes that are being trained or being rebuilt", "website_dataset": "Web site", - "website_dataset_desc": "Web site synchronization allows you to use a web page link to build a dataset", - "collection": { - "Create update time": "Create/Update time", - "Training type": "Training type" - }, - "filename": "filename" -} \ No newline at end of file + "website_dataset_desc": "Web site synchronization allows you to use a web page link to build a dataset" +} diff --git a/packages/web/i18n/zh/common.json b/packages/web/i18n/zh/common.json index 250fd2a1b..550c1039c 100644 --- a/packages/web/i18n/zh/common.json +++ b/packages/web/i18n/zh/common.json @@ -1,7 +1,5 @@ { "App": "应用", - "click_to_resume": "点击恢复", - "code_editor": "代码编辑", "Export": "导出", "Folder": "文件夹", "Login": "登录", @@ -13,6 +11,8 @@ "UnKnow": "未知", "Warning": "提示", "add_new": "新增", + "click_to_resume": "点击恢复", + "code_editor": "代码编辑", "common": { "Action": "操作", "Add": "添加", @@ -91,6 +91,8 @@ "Root folder": "根目录", "Run": "运行", "Save": "保存", + "Save Failed": "保存失败", + "Save Success": "保存成功", "Search": "搜索", "Select File Failed": "选择文件异常", "Select template": "选择模板", @@ -482,7 +484,8 @@ "success": "开始同步" } }, - "training": {} + "training": { + } }, "data": { "Auxiliary Data": "辅助数据", @@ -505,13 +508,13 @@ "Data not found": "数据不存在或已被删除", "Start Sync Failed": "开始同步失败", "Template does not exist": "模板不存在", + "invalidVectorModelOrQAModel": "VectorModel 或 QA 模型错误", "unAuthDataset": "无权操作该知识库", "unAuthDatasetCollection": "无权操作该数据集", "unAuthDatasetData": "无权操作该数据", "unAuthDatasetFile": "无权操作该文件", "unCreateCollection": "无权操作该数据", - "unLinkCollection": "不是网络链接集合", - "invalidVectorModelOrQAModel": "VectorModel 或 QA 模型错误" + "unLinkCollection": "不是网络链接集合" }, "externalFile": "外部文件库", "file": "文件", diff --git a/packages/web/i18n/zh/dataset.json b/packages/web/i18n/zh/dataset.json index 711c85775..9dd006120 100644 --- a/packages/web/i18n/zh/dataset.json +++ b/packages/web/i18n/zh/dataset.json @@ -1,26 +1,41 @@ { + "Disabled": "已禁用", + "Enable": "启用", + "Enabled": "已启用", + "collection": { + "Create update time": "创建/更新时间", + "Training type": "训练模式" + }, "collection_tags": "集合标签", "common_dataset": "通用知识库", "common_dataset_desc": "可通过导入文件、网页链接或手动录入形式构建知识库", "confirm_to_rebuild_embedding_tip": "确认为知识库切换索引？\n切换索引是一个非常重量的操作，需要对您知识库内所有数据进行重新索引，时间可能较长，请确保账号内剩余积分充足。\n\n此外，你还需要注意修改选择该知识库的应用，避免它们与其他索引模型知识库混用。", - "Disabled": "已禁用", - "Enable": "启用", - "Enabled": "已启用", + "dataset": { + "no_collections": "暂无数据集", + "no_tags": "暂无标签" + }, "external_file": "外部文件库", "external_file_dataset_desc": "可以从外部文件库导入文件构建知识库，文件不会进行二次存储", "external_id": "文件阅读 ID", "external_read_url": "外部预览地址", "external_read_url_tip": "可以配置你文件库的阅读地址。便于对用户进行阅读鉴权操作。目前可以使用 {{fileId}} 变量来指代外部文件 ID。", "external_url": "文件访问 URL", + "filename": "文件名", "folder_dataset": "文件夹", "rebuild_embedding_start_tip": "切换索引模型任务已开始", "rebuilding_index_count": "重建中索引数量：{{count}}", + "tag": { + "Add New": "新建", + "Add_new_tag": "新建标签", + "Edit_tag": "编辑标签", + "add": "创建", + "cancel": "取消选择", + "delete_tag_confirm": "确定删除标签？", + "manage": "标签管理", + "searchOrAddTag": "搜索或添加标签", + "tags": "标签" + }, "the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "知识库有训练中或正在重建的索引", "website_dataset": "Web 站点同步", - "website_dataset_desc": "Web 站点同步允许你直接使用一个网页链接构建知识库", - "collection": { - "Create update time": "创建/更新时间", - "Training type": "训练模式" - }, - "filename": "文件名" -} \ No newline at end of file + "website_dataset_desc": "Web 站点同步允许你直接使用一个网页链接构建知识库" +} diff --git a/projects/app/src/global/core/api/datasetReq.d.ts b/projects/app/src/global/core/api/datasetReq.d.ts index cb340df7b..7edc4cec0 100644 --- a/projects/app/src/global/core/api/datasetReq.d.ts +++ b/projects/app/src/global/core/api/datasetReq.d.ts @@ -10,6 +10,7 @@ import { UploadChunkItemType } from '@fastgpt/global/core/dataset/type'; import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type'; import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant'; import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.d'; +import { PaginationProps } from '@fastgpt/web/common/fetch/type'; /* ===== dataset ===== */ @@ -18,6 +19,7 @@ export type GetDatasetCollectionsProps = RequestPaging & { datasetId: string; parentId?: string; searchText?: string; + filterTags?: string[]; simple?: boolean; selectFolder?: boolean; }; diff --git a/projects/app/src/global/core/dataset/type.d.ts b/projects/app/src/global/core/dataset/type.d.ts index 57e277018..a487b7f1c 100644 --- a/projects/app/src/global/core/dataset/type.d.ts +++ b/projects/app/src/global/core/dataset/type.d.ts @@ -1,7 +1,8 @@ import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type'; import { DatasetCollectionSchemaType, - DatasetDataSchemaType + DatasetDataSchemaType, + DatasetTagType } from '@fastgpt/global/core/dataset/type.d'; import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller'; @@ -18,6 +19,7 @@ export type DatasetCollectionsListItemType = { updateTime: DatasetCollectionSchemaType['updateTime']; forbid?: DatasetCollectionSchemaType['forbid']; trainingType?: DatasetCollectionSchemaType['trainingType']; + tags?: string[]; fileId?: string; rawLink?: string; diff --git a/projects/app/src/pages/api/core/dataset/collection/list.ts b/projects/app/src/pages/api/core/dataset/collection/list.ts index 3f33534cf..1d64a3e17 100644 --- a/projects/app/src/pages/api/core/dataset/collection/list.ts +++ b/projects/app/src/pages/api/core/dataset/collection/list.ts @@ -20,6 +20,7 @@ async function handler(req: NextApiRequest): Promise; + +async function handler( + req: ApiRequestProps<{}, GetScrollCollectionsProps> +): Promise> { + let { + datasetId, + pageSize = 10, + current = 1, + parentId = null, + searchText = '', + selectFolder = false, + filterTags = [], + simple = false + } = req.query; + if (!datasetId) { + return Promise.reject(CommonErrEnum.missingParams); + } + searchText = searchText?.replace(/'/g, ''); + pageSize = Math.min(pageSize, 30); + + // auth dataset and get my role + const { teamId, permission } = await authDataset({ + req, + authToken: true, + authApiKey: true, + datasetId, + per: ReadPermissionVal + }); + + const match = { + teamId: new Types.ObjectId(teamId), + datasetId: new Types.ObjectId(datasetId), + parentId: parentId ? new Types.ObjectId(parentId) : null, + ...(selectFolder ? { type: DatasetCollectionTypeEnum.folder } : {}), + ...(searchText + ? { + name: new RegExp(searchText, 'i') + } + : {}), + ...(filterTags.length ? { tags: { $all: filterTags } } : {}) + }; + + const selectField = { + _id: 1, + parentId: 1, + tmbId: 1, + name: 1, + type: 1, + forbid: 1, + createTime: 1, + updateTime: 1, + trainingType: 1, + fileId: 1, + rawLink: 1, + tags: 1 + }; + + // not count data amount + if (simple) { + const collections = await MongoDatasetCollection.find(match) + .select(selectField) + .sort({ + updateTime: -1 + }) + .skip(pageSize * (current - 1)) + .limit(pageSize) + .lean(); + + return { + list: await Promise.all( + collections.map(async (item) => ({ + ...item, + dataAmount: 0, + trainingAmount: 0, + permission + })) + ), + total: await MongoDatasetCollection.countDocuments(match) + }; + } + + const [collections, total]: [DatasetCollectionsListItemType[], number] = await Promise.all([ + MongoDatasetCollection.aggregate([ + { + $match: match + }, + { + $sort: { updateTime: -1 } + }, + { + $skip: (current - 1) * pageSize + }, + { + $limit: pageSize + }, + // count training data + { + $lookup: { + from: DatasetTrainingCollectionName, + let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId }, + pipeline: [ + { + $match: { + $expr: { + $and: [{ $eq: ['$teamId', '$$team_id'] }, { $eq: ['$collectionId', '$$id'] }] + } + } + }, + { $count: 'count' } + ], + as: 'trainingCount' + } + }, + // count collection total data + { + $lookup: { + from: DatasetDataCollectionName, + let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId }, + pipeline: [ + { + $match: { + $expr: { + $and: [ + { $eq: ['$teamId', '$$team_id'] }, + { $eq: ['$datasetId', '$$dataset_id'] }, + { $eq: ['$collectionId', '$$id'] } + ] + } + } + }, + { $count: 'count' } + ], + as: 'dataCount' + } + }, + { + $project: { + ...selectField, + dataAmount: { + $ifNull: [{ $arrayElemAt: ['$dataCount.count', 0] }, 0] + }, + trainingAmount: { + $ifNull: [{ $arrayElemAt: ['$trainingCount.count', 0] }, 0] + } + } + } + ]), + MongoDatasetCollection.countDocuments(match) + ]); + + const data = await Promise.all( + collections.map(async (item) => ({ + ...item, + permission + })) + ); + + if (data.find((item) => item.trainingAmount > 0)) { + startTrainingQueue(); + } + + // count collections + return { + list: data, + total + }; +} + +export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/collection/update.ts b/projects/app/src/pages/api/core/dataset/collection/update.ts index 60e352b55..bf9e40472 100644 --- a/projects/app/src/pages/api/core/dataset/collection/update.ts +++ b/projects/app/src/pages/api/core/dataset/collection/update.ts @@ -10,11 +10,12 @@ export type UpdateDatasetCollectionParams = { id: string; parentId?: string; name?: string; + tags?: string[]; forbid?: boolean; }; async function handler(req: ApiRequestProps) { - const { id, parentId, name, forbid } = req.body; + const { id, parentId, name, tags, forbid } = req.body; if (!id) { return Promise.reject(CommonErrEnum.missingParams); @@ -32,6 +33,7 @@ async function handler(req: ApiRequestProps) { const updateFields: Record = { ...(parentId !== undefined && { parentId: parentId || null }), ...(name && { name, updateTime: getCollectionUpdateTime({ name }) }), + ...(tags && { tags }), ...(forbid !== undefined && { forbid }) }; diff --git a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/RenderInput/index.tsx b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/RenderInput/index.tsx index c74666f79..c3c92b644 100644 --- a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/RenderInput/index.tsx +++ b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/RenderInput/index.tsx @@ -6,6 +6,7 @@ import dynamic from 'next/dynamic'; import InputLabel from './Label'; import type { RenderInputProps } from './type'; +import { useSystemStore } from '@/web/common/system/useSystemStore'; const RenderList: { types: FlowNodeInputTypeEnum[]; @@ -74,7 +75,18 @@ type Props = { mb?: number; }; const RenderInput = ({ flowInputList, nodeId, CustomComponent, mb = 5 }: Props) => { - const copyInputs = useMemo(() => JSON.stringify(flowInputList), [flowInputList]); + const { feConfigs } = useSystemStore(); + + const copyInputs = useMemo( + () => + JSON.stringify( + flowInputList.filter((input) => { + if (input.isPro && !feConfigs?.isPlus) return false; + return true; + }) + ), + [feConfigs?.isPlus, flowInputList] + ); const filterInputs = useMemo(() => { return JSON.parse(copyInputs) as FlowNodeInputItemType[]; }, [copyInputs]); diff --git a/projects/app/src/pages/chat/components/SliderApps.tsx b/projects/app/src/pages/chat/components/SliderApps.tsx index e5622fdf8..d1a500f10 100644 --- a/projects/app/src/pages/chat/components/SliderApps.tsx +++ b/projects/app/src/pages/chat/components/SliderApps.tsx @@ -90,6 +90,7 @@ const SliderApps = ({ apps, activeAppId }: { apps: AppListItemType[]; activeAppI >; + filterTags: string[]; + setFilterTags: Dispatch>; }; export const CollectionPageContext = createContext({ @@ -52,6 +54,10 @@ export const CollectionPageContext = createContext({ searchText: '', setSearchText: function (value: SetStateAction): void { throw new Error('Function not implemented.'); + }, + filterTags: [], + setFilterTags: function (value: SetStateAction): void { + throw new Error('Function not implemented.'); } }); @@ -96,6 +102,7 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => // collection list const [searchText, setSearchText] = useState(''); + const [filterTags, setFilterTags] = useState([]); const { data: collections, Pagination, @@ -110,7 +117,8 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => params: { datasetId, parentId, - searchText + searchText, + filterTags }, defaultRequest: false }); @@ -124,6 +132,8 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => searchText, setSearchText, + filterTags, + setFilterTags, collections, Pagination, total, diff --git a/projects/app/src/pages/dataset/detail/components/CollectionCard/Header.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard/Header.tsx index bfaab5a28..039c4c879 100644 --- a/projects/app/src/pages/dataset/detail/components/CollectionCard/Header.tsx +++ b/projects/app/src/pages/dataset/detail/components/CollectionCard/Header.tsx @@ -32,13 +32,14 @@ import { useContextSelector } from 'use-context-selector'; import { CollectionPageContext } from './Context'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; import { useSystem } from '@fastgpt/web/hooks/useSystem'; +import HeaderTagPopOver from './HeaderTagPopOver'; const FileSourceSelector = dynamic(() => import('../Import/components/FileSourceSelector')); const Header = ({}: {}) => { const { t } = useTranslation(); const theme = useTheme(); - const { setLoading } = useSystemStore(); + const { setLoading, feConfigs } = useSystemStore(); const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); const router = useRouter(); @@ -154,7 +155,6 @@ const Header = ({}: {}) => { {isPc && ( { {/* diff collection button */} {datasetDetail.permission.hasWritePer && ( - <> + + {feConfigs?.isPlus && } + {datasetDetail?.type === DatasetTypeEnum.dataset && ( { ]} /> )} - + )} {/* modal */} diff --git a/projects/app/src/pages/dataset/detail/components/CollectionCard/HeaderTagPopOver.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard/HeaderTagPopOver.tsx new file mode 100644 index 000000000..bcfb8e544 --- /dev/null +++ b/projects/app/src/pages/dataset/detail/components/CollectionCard/HeaderTagPopOver.tsx @@ -0,0 +1,229 @@ +import { Box, Button, Checkbox, Flex, Input, useDisclosure } from '@chakra-ui/react'; +import MyPopover from '@fastgpt/web/components/common/MyPopover'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import MyBox from '@fastgpt/web/components/common/MyBox'; +import { postCreateDatasetCollectionTag } from '@/web/core/dataset/api'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import { useTranslation } from 'react-i18next'; +import { useCallback, useEffect, useState } from 'react'; +import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { CollectionPageContext } from './Context'; +import { debounce, isEqual } from 'lodash'; +import TagManageModal from './TagManageModal'; +import { DatasetTagType } from '@fastgpt/global/core/dataset/type'; + +const HeaderTagPopOver = () => { + const { t } = useTranslation(); + const [searchTag, setSearchTag] = useState(''); + const [checkedTags, setCheckedTags] = useState([]); + + const { datasetDetail, datasetTags, loadDatasetTags, checkedDatasetTag, setCheckedDatasetTag } = + useContextSelector(DatasetPageContext, (v) => v); + + const { mutate: onCreateCollectionTag, isLoading: isCreateCollectionTagLoading } = useRequest({ + mutationFn: async (tag: string) => { + const id = await postCreateDatasetCollectionTag({ + datasetId: datasetDetail._id, + tag + }); + return id; + }, + + onSuccess() { + setSearchTag(''); + }, + successToast: t('common:common.Create Success'), + errorToast: t('common:common.Create Failed') + }); + + const { filterTags, setFilterTags, getData } = useContextSelector( + CollectionPageContext, + (v) => v + ); + const debounceRefetch = useCallback( + debounce(() => { + getData(1); + }, 300), + [] + ); + + useEffect(() => { + loadDatasetTags({ id: datasetDetail._id, searchKey: searchTag }); + }, [searchTag]); + + const { + isOpen: isTagManageModalOpen, + onOpen: onOpenTagManageModal, + onClose: onCloseTagManageModal + } = useDisclosure(); + + const checkTags = (tag: DatasetTagType) => { + let currentCheckedTags = []; + if (checkedTags.includes(tag._id)) { + currentCheckedTags = checkedTags.filter((t) => t !== tag._id); + setCheckedTags(currentCheckedTags); + setCheckedDatasetTag(checkedDatasetTag.filter((t) => t._id !== tag._id)); + } else { + currentCheckedTags = [...checkedTags, tag._id]; + setCheckedTags([...checkedTags, tag._id]); + setCheckedDatasetTag([...checkedDatasetTag, tag]); + } + if (isEqual(currentCheckedTags, filterTags)) return; + setFilterTags(currentCheckedTags); + debounceRefetch(); + }; + + return ( + <> + + + {t('dataset:tag.tags')} + + {checkedTags.length > 0 && ( + + {`(${checkedTags.length})`} + + )} + + + + + } + > + {({ onClose }) => ( + e.stopPropagation()}> + + setSearchTag(e.target.value)} + /> + + + + {searchTag && !datasetTags.map((item) => item.tag).includes(searchTag) && ( + { + onCreateCollectionTag(searchTag); + }} + > + + + {t('dataset:tag.add') + ` "${searchTag}"`} + + + )} + + {[ + ...new Map( + [...checkedDatasetTag, ...datasetTags].map((item) => [item._id, item]) + ).values() + ].map((item) => { + const checked = checkedTags.includes(item._id); + return ( + { + e.preventDefault(); + checkTags(item); + }} + > + { + checkTags(item); + }} + size={'md'} + /> + {item.tag} + + ); + })} + + + + + + + + )} + + {isTagManageModalOpen && ( + { + onCloseTagManageModal(); + debounceRefetch(); + }} + /> + )} + + ); +}; + +export default HeaderTagPopOver; diff --git a/projects/app/src/pages/dataset/detail/components/CollectionCard/TagManageModal.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard/TagManageModal.tsx new file mode 100644 index 000000000..202a6bf69 --- /dev/null +++ b/projects/app/src/pages/dataset/detail/components/CollectionCard/TagManageModal.tsx @@ -0,0 +1,530 @@ +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { Input, Button, Flex, Box, Checkbox } from '@chakra-ui/react'; +import MyModal from '@fastgpt/web/components/common/MyModal'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import { CollectionPageContext } from './Context'; +import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils'; +import { + delDatasetCollectionTag, + getDatasetCollectionTags, + getScrollCollectionList, + getTagUsage, + postAddTagsToCollections, + postCreateDatasetCollectionTag, + updateDatasetCollectionTag +} from '@/web/core/dataset/api'; +import { useRequest, useRequest2 } from '@fastgpt/web/hooks/useRequest'; +import MyInput from '@/components/MyInput'; +import { DatasetTagType } from '@fastgpt/global/core/dataset/type'; +import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination'; +import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; +import { useQuery } from '@tanstack/react-query'; +import PopoverConfirm from '@fastgpt/web/components/common/MyPopover/PopoverConfirm'; +import MyBox from '@fastgpt/web/components/common/MyBox'; + +const TagManageModal = ({ onClose }: { onClose: () => void }) => { + const { t } = useTranslation(); + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + const loadDatasetTags = useContextSelector(DatasetPageContext, (v) => v.loadDatasetTags); + const loadAllDatasetTags = useContextSelector(DatasetPageContext, (v) => v.loadAllDatasetTags); + const { getData } = useContextSelector(CollectionPageContext, (v) => v); + + const tagInputRef = useRef(null); + const editInputRef = useRef(null); + + const [currentAddTag, setCurrentAddTag] = useState< + (DatasetTagType & { collections: string[] }) | undefined + >(undefined); + + const [newTag, setNewTag] = useState(undefined); + + const [currentEditTagContent, setCurrentEditTagContent] = useState(undefined); + const [currentEditTag, setCurrentEditTag] = useState(undefined); + + useEffect(() => { + if (newTag !== undefined && tagInputRef.current) { + tagInputRef.current?.focus(); + } + }, [newTag]); + + useEffect(() => { + if (currentEditTag !== undefined && editInputRef.current) { + editInputRef.current?.focus(); + } + }, [currentEditTag]); + + const { mutate: onCreateCollectionTag, isLoading: isCreateCollectionTagLoading } = useRequest({ + mutationFn: async (tag: string) => { + const id = await postCreateDatasetCollectionTag({ + datasetId: datasetDetail._id, + tag + }); + return id; + }, + + onSuccess() { + fetchData(1); + loadDatasetTags({ id: datasetDetail._id, searchKey: '' }); + loadAllDatasetTags({ id: datasetDetail._id }); + }, + successToast: t('common:common.Create Success'), + errorToast: t('common:common.Create Failed') + }); + + const { mutate: onDeleteCollectionTag, isLoading: isDeleteCollectionTagLoading } = useRequest({ + mutationFn: async (tag: string) => { + const id = await delDatasetCollectionTag({ + datasetId: datasetDetail._id, + id: tag + }); + return id; + }, + + onSuccess() { + fetchData(1); + loadDatasetTags({ id: datasetDetail._id, searchKey: '' }); + loadAllDatasetTags({ id: datasetDetail._id }); + }, + successToast: t('common:common.Delete Success'), + errorToast: t('common:common.Delete Failed') + }); + + const { mutate: onUpdateCollectionTag, isLoading: isUpdateCollectionTagLoading } = useRequest({ + mutationFn: async (tag: DatasetTagType) => { + const id = await updateDatasetCollectionTag({ + datasetId: datasetDetail._id, + tagId: tag._id, + tag: tag.tag + }); + return id; + }, + onSuccess() { + fetchData(1); + loadDatasetTags({ id: datasetDetail._id, searchKey: '' }); + loadAllDatasetTags({ id: datasetDetail._id }); + } + }); + + const { mutate: onSaveCollectionTag, isLoading: isSaveCollectionTagLoading } = useRequest({ + mutationFn: async ({ + tag, + originCollectionIds, + collectionIds + }: { + tag: string; + originCollectionIds: string[]; + collectionIds: string[]; + }) => { + try { + await postAddTagsToCollections({ + tag, + originCollectionIds, + collectionIds, + datasetId: datasetDetail._id + }); + } catch (error) {} + }, + + onSuccess() { + getData(1); + }, + successToast: t('common:common.Save Success'), + errorToast: t('common:common.Save Failed') + }); + + const { + list, + ScrollList, + isLoading: isRequesting, + fetchData, + total: tagsTotal + } = useScrollPagination(getDatasetCollectionTags, { + refreshDeps: [''], + debounceWait: 300, + + itemHeight: 56, + overscan: 10, + + pageSize: 10, + defaultParams: { + datasetId: datasetDetail._id, + searchText: '' + } + }); + + const { data: tagUsages } = useRequest2(() => getTagUsage(datasetDetail._id), { + manual: false + }); + + return ( + + {currentAddTag === undefined ? ( + <> + + + {`共${tagsTotal}个标签`} + + + } + > + {newTag !== undefined && ( + + setNewTag(e.target.value)} + ref={tagInputRef} + w={'200px'} + onBlur={() => { + if (newTag && !list.map((item) => item.data.tag).includes(newTag)) { + onCreateCollectionTag(newTag); + } + setNewTag(undefined); + }} + /> + + )} + {list.map((listItem) => { + const item = listItem.data; + const tagUsage = tagUsages?.find((tagUsage) => tagUsage.tagId === item._id); + const collections = tagUsage?.collections || []; + const usage = collections.length; + + return ( + + + { + setCurrentAddTag({ ...item, collections }); + }} + cursor={'pointer'} + > + {currentEditTag?._id !== item._id ? ( + + {item.tag} + + ) : ( + setCurrentEditTagContent(e.target.value)} + ref={editInputRef} + w={'200px'} + onBlur={() => { + if ( + currentEditTagContent && + !list.map((item) => item.data.tag).includes(currentEditTagContent) + ) { + onUpdateCollectionTag({ + tag: currentEditTagContent, + _id: item._id + }); + } + setCurrentEditTag(undefined); + setCurrentEditTagContent(undefined); + }} + /> + )} + {`(${usage})`} + + { + setCurrentAddTag({ ...item, collections }); + }} + cursor={'pointer'} + > + + + { + setCurrentEditTag(item); + editInputRef.current?.focus(); + }} + > + + + + + + } + onConfirm={() => onDeleteCollectionTag(item._id)} + /> + + + ); + })} + + + ) : ( + + )} + + ); +}; + +export default TagManageModal; + +const AddTagToCollections = ({ + currentAddTag, + setCurrentAddTag, + onSaveCollectionTag +}: { + currentAddTag: DatasetTagType & { collections: string[] }; + setCurrentAddTag: (tag: (DatasetTagType & { collections: string[] }) | undefined) => void; + onSaveCollectionTag: ({ + tag, + originCollectionIds, + collectionIds + }: { + tag: string; + originCollectionIds: string[]; + collectionIds: string[]; + }) => void; +}) => { + const { t } = useTranslation(); + + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + const [selectedCollections, setSelectedCollections] = useState([]); + + useEffect(() => { + setSelectedCollections(currentAddTag.collections); + }, []); + + const [searchText, setSearchText] = useState(''); + + const { + list: collectionsList, + ScrollList: ScrollListCollections, + isLoading: isCollectionLoading + } = useScrollPagination(getScrollCollectionList, { + refreshDeps: [searchText], + debounceWait: 300, + + itemHeight: 29, + overscan: 10, + + pageSize: 30, + defaultParams: { + datasetId: datasetDetail._id, + searchText + } + }); + + const formatCollections = useMemo( + () => + collectionsList.map((item) => { + const collection = item.data; + const icon = getCollectionIcon(collection.type, collection.name); + return { + id: collection._id, + tags: collection.tags, + name: collection.name, + icon + }; + }), + [collectionsList] + ); + + return ( + + + { + setCurrentAddTag(undefined); + setSearchText(''); + }} + /> + { + + + {currentAddTag.tag} + + {`(${selectedCollections.length})`} + + } + + { + setSearchText(e.target.value); + }} + /> + + + } + > + {formatCollections.map((collection) => { + return ( + { + setSelectedCollections((prev) => { + if (prev.includes(collection.id)) { + return prev.filter((id) => id !== collection.id); + } else { + return [...prev, collection.id]; + } + }); + }} + > + { + setSelectedCollections((prev) => { + if (prev.includes(collection.id)) { + return prev.filter((id) => id !== collection.id); + } else { + return [...prev, collection.id]; + } + }); + }} + isChecked={selectedCollections.includes(collection.id)} + /> + + + {collection.name} + + + ); + })} + + + ); +}; diff --git a/projects/app/src/pages/dataset/detail/components/CollectionCard/TagsPopOver.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard/TagsPopOver.tsx new file mode 100644 index 000000000..9debc0d3b --- /dev/null +++ b/projects/app/src/pages/dataset/detail/components/CollectionCard/TagsPopOver.tsx @@ -0,0 +1,287 @@ +import { Box, Checkbox, Flex, Input } from '@chakra-ui/react'; +import MyPopover from '@fastgpt/web/components/common/MyPopover'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import MyBox from '@fastgpt/web/components/common/MyBox'; +import { postCreateDatasetCollectionTag, putDatasetCollectionById } from '@/web/core/dataset/api'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import { useTranslation } from 'react-i18next'; +import { useEffect, useMemo, useRef, useState } from 'react'; +import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { useDeepCompareEffect } from 'ahooks'; +import { DatasetCollectionItemType, DatasetTagType } from '@fastgpt/global/core/dataset/type'; +import { isEqual } from 'lodash'; +import { DatasetCollectionsListItemType } from '@/global/core/dataset/type'; + +const TagsPopOver = ({ + currentCollection +}: { + currentCollection: DatasetCollectionItemType | DatasetCollectionsListItemType; +}) => { + const { t } = useTranslation(); + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + const datasetTags = useContextSelector(DatasetPageContext, (v) => v.datasetTags); + const loadDatasetTags = useContextSelector(DatasetPageContext, (v) => v.loadDatasetTags); + const allDatasetTags = useContextSelector(DatasetPageContext, (v) => v.allDatasetTags); + const loadAllDatasetTags = useContextSelector(DatasetPageContext, (v) => v.loadAllDatasetTags); + + const [collectionTags, setCollectionTags] = useState([]); + const [searchTag, setSearchTag] = useState(''); + const [checkedTags, setCheckedTags] = useState([]); + + const [showTagManage, setShowTagManage] = useState(false); + const [isFocusInput, setIsFocusInput] = useState(false); + const [isUpdateLoading, setIsUpdateLoading] = useState(false); + + useEffect(() => { + if (!currentCollection.tags) return; + setCollectionTags(currentCollection.tags); + }, [currentCollection]); + + const tagList = useMemo( + () => + (collectionTags + ?.map((tagId) => { + const tagObject = allDatasetTags.find((tag) => tag._id === tagId); + return tagObject ? { _id: tagObject._id, tag: tagObject.tag } : null; + }) + .filter((tag) => tag !== null) as { + _id: string; + tag: string; + }[]) || [], + [collectionTags, allDatasetTags] + ); + + useEffect(() => { + if (!isFocusInput) return; + loadDatasetTags({ id: datasetDetail._id, searchKey: searchTag }); + }, [datasetDetail._id, isFocusInput, loadDatasetTags, searchTag]); + + const [visibleTags, setVisibleTags] = useState(tagList); + const [overflowTags, setOverflowTags] = useState([]); + const containerRef = useRef(null); + + useDeepCompareEffect(() => { + const calculateTags = () => { + if (!containerRef.current || !tagList) return; + + const containerWidth = containerRef.current.offsetWidth; + const tagWidth = 11; + let totalWidth = 30; + let visibleCount = 0; + + for (let i = 0; i < tagList.length; i++) { + const tag = tagList[i]; + const estimatedWidth = tag.tag.length * tagWidth + 16; // 加上左右 padding 的宽度 + if (totalWidth + estimatedWidth <= containerWidth) { + totalWidth += estimatedWidth; + visibleCount++; + } else { + break; + } + } + + setVisibleTags(tagList.slice(0, visibleCount)); + setOverflowTags(tagList.slice(visibleCount)); + }; + + setTimeout(calculateTags, 100); + setCheckedTags(tagList); + + window.addEventListener('resize', calculateTags); + + return () => { + window.removeEventListener('resize', calculateTags); + }; + }, [tagList]); + + const { mutate: onCreateCollectionTag, isLoading: isCreateCollectionTagLoading } = useRequest({ + mutationFn: async (tag: string) => { + const id = await postCreateDatasetCollectionTag({ + datasetId: datasetDetail._id, + tag + }); + return id; + }, + + onSuccess() { + setSearchTag(''); + loadDatasetTags({ id: datasetDetail._id, searchKey: '' }); + loadAllDatasetTags({ id: datasetDetail._id }); + }, + successToast: t('common:common.Create Success'), + errorToast: t('common:common.Create Failed') + }); + + return ( + { + e.stopPropagation(); + if (!e.currentTarget.parentElement || !e.currentTarget.parentElement.parentElement) + return; + e.currentTarget.parentElement.parentElement.style.backgroundColor = 'white'; + }} + onMouseLeave={(e) => { + if (!e.currentTarget.parentElement || !e.currentTarget.parentElement.parentElement) + return; + e.currentTarget.parentElement.parentElement.style.backgroundColor = ''; + }} + onClick={(e) => { + e.stopPropagation(); + setShowTagManage(true); + }} + cursor={'pointer'} + > + + {visibleTags.map((item, index) => ( + + {item.tag} + + ))} + + {overflowTags.length > 0 && ( + + {`+${overflowTags.length}`} + + )} + + } + onCloseFunc={async () => { + setShowTagManage(false); + if (isEqual(checkedTags, tagList) || !showTagManage) return; + setIsUpdateLoading(true); + await putDatasetCollectionById({ + id: currentCollection._id, + tags: checkedTags.map((tag) => tag._id) + }); + setCollectionTags(checkedTags.map((tag) => tag._id)); + setIsUpdateLoading(false); + }} + display={showTagManage || overflowTags.length > 0 ? 'block' : 'none'} + > + {({}) => ( + <> + {showTagManage ? ( + e.stopPropagation()}> + + setIsFocusInput(true)} + onBlur={() => setIsFocusInput(false)} + pl={2} + h={7} + borderRadius={'4px'} + value={searchTag} + placeholder={t('dataset:tag.searchOrAddTag')} + onChange={(e) => setSearchTag(e.target.value)} + /> + + + {searchTag && !datasetTags.map((item) => item.tag).includes(searchTag) && ( + { + onCreateCollectionTag(searchTag); + }} + > + + + {t('dataset:tag.add') + ` "${searchTag}"`} + + + )} + {datasetTags?.map((item) => { + const tagsList = checkedTags.map((tag) => tag.tag); + return ( + { + e.preventDefault(); + if (tagsList.includes(item.tag)) { + setCheckedTags(checkedTags.filter((t) => t.tag !== item.tag)); + } else { + setCheckedTags([...checkedTags, item]); + } + }} + > + { + if (e.target.checked) { + setCheckedTags([...checkedTags, item]); + } else { + setCheckedTags(checkedTags.filter((t) => t._id !== item._id)); + } + }} + /> + {item.tag} + + ); + })} + + + ) : ( + + {overflowTags.map((tag, index) => ( + + {tag.tag} + + ))} + + )} + + )} + + ); +}; + +export default TagsPopOver; diff --git a/projects/app/src/pages/dataset/detail/components/CollectionCard/index.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard/index.tsx index 41e9d6eec..b4ea1fdfc 100644 --- a/projects/app/src/pages/dataset/detail/components/CollectionCard/index.tsx +++ b/projects/app/src/pages/dataset/detail/components/CollectionCard/index.tsx @@ -49,6 +49,8 @@ import { getTrainingTypeLabel } from '@fastgpt/global/core/dataset/collection/utils'; import { useFolderDrag } from '@/components/common/folder/useFolderDrag'; +import TagsPopOver from './TagsPopOver'; +import { useSystemStore } from '@/web/common/system/useSystemStore'; const Header = dynamic(() => import('./Header')); const EmptyCollectionTip = dynamic(() => import('./EmptyCollectionTip')); @@ -60,6 +62,7 @@ const CollectionCard = () => { const { t } = useTranslation(); const { datasetT } = useI18n(); const { datasetDetail, loadDatasetDetail } = useContextSelector(DatasetPageContext, (v) => v); + const { feConfigs } = useSystemStore(); const { openConfirm: openDeleteConfirm, ConfirmModal: ConfirmDeleteModal } = useConfirm({ content: t('common:dataset.Confirm to delete the file'), @@ -244,6 +247,9 @@ const CollectionCard = () => { + {feConfigs?.isPlus && !!collection.tags?.length && ( + + )} {!checkCollectionIsFolder(collection.type) ? ( diff --git a/projects/app/src/pages/dataset/detail/components/DataCard.tsx b/projects/app/src/pages/dataset/detail/components/DataCard.tsx index 1e106e988..ecacdca77 100644 --- a/projects/app/src/pages/dataset/detail/components/DataCard.tsx +++ b/projects/app/src/pages/dataset/detail/components/DataCard.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useState, useRef, useMemo } from 'react'; +import React, { useState, useRef, useMemo } from 'react'; import { Box, Card, @@ -14,8 +14,7 @@ import { DrawerOverlay, DrawerContent, useDisclosure, - HStack, - Switch + HStack } from '@chakra-ui/react'; import { getDatasetDataList, @@ -26,19 +25,16 @@ import { import { DeleteIcon } from '@chakra-ui/icons'; import { useQuery } from '@tanstack/react-query'; import { useToast } from '@fastgpt/web/hooks/useToast'; -import { debounce } from 'lodash'; import { getErrText } from '@fastgpt/global/common/error/utils'; import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; import { useTranslation } from 'next-i18next'; import { useRouter } from 'next/router'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyInput from '@/components/MyInput'; -import { useLoading } from '@fastgpt/web/hooks/useLoading'; import InputDataModal from '../components/InputDataModal'; import RawSourceBox from '@/components/core/dataset/RawSourceBox'; import type { DatasetDataListItemType } from '@/global/core/dataset/type.d'; import { TabEnum } from '..'; -import { useSystemStore } from '@/web/common/system/useSystemStore'; import { DatasetCollectionTypeMap, TrainingTypeMap } from '@fastgpt/global/core/dataset/constants'; import { formatTime2YMDHM } from '@fastgpt/global/common/string/time'; import { formatFileSize } from '@fastgpt/global/common/file/tools'; @@ -54,6 +50,8 @@ import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; import MyTag from '@fastgpt/web/components/common/Tag/index'; import MyBox from '@fastgpt/web/components/common/MyBox'; import { useSystem } from '@fastgpt/web/hooks/useSystem'; +import TagsPopOver from './CollectionCard/TagsPopOver'; +import { useSystemStore } from '@/web/common/system/useSystemStore'; const DataCard = () => { const BoxRef = useRef(null); @@ -66,6 +64,7 @@ const DataCard = () => { datasetId: string; }; const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + const { feConfigs } = useSystemStore(); const { t } = useTranslation(); const { datasetT } = useI18n(); @@ -224,28 +223,34 @@ const DataCard = () => { } /> - - {collection?._id && ( - - )} - - {t('common:core.dataset.collection.id')}:{' '} - - {collection?._id} + + + {collection?._id && ( + + )} + + {t('common:core.dataset.collection.id')}:{' '} + + {collection?._id} + + {feConfigs?.isPlus && !!collection?.tags?.length && ( + + )} {canWrite && (